ccls/src/command_line.cc

// TODO: cleanup includes
#include "cache.h"
#include "clang_complete.h"
#include "file_consumer.h"
#include "include_complete.h"
#include "indexer.h"
#include "ipc_manager.h"
#include "language_server_api.h"
#include "lex_utils.h"
#include "lru_cache.h"
#include "match.h"
#include "options.h"
#include "platform.h"
#include "project.h"
#include "query.h"
#include "query_utils.h"
#include "serializer.h"
#include "standard_includes.h"
#include "test.h"
#include "threaded_queue.h"
#include "timer.h"
#include "work_thread.h"
#include "working_files.h"

#include <doctest/doctest.h>
#include <rapidjson/istreamwrapper.h>
#include <rapidjson/ostreamwrapper.h>
#include <loguru.hpp>

#include <climits>
#include <fstream>
#include <functional>
#include <future>
#include <iostream>
#include <iterator>
#include <sstream>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>

// TODO: provide a feature like 'https://github.com/goldsborough/clang-expand',
// ie, a fully linear view of a function with inline function calls expanded.
// We can probably use vscode decorators to achieve it.

// TODO: implement ThreadPool type which monitors CPU usage / number of work
// items per second completed and scales up/down number of running threads.

namespace {

std::vector<std::string> kEmptyArgs;

// Expected client version. We show an error if this doesn't match.
const int kExpectedClientVersion = 3;

// If true stdout will be printed to stderr.
bool g_log_stdin_stdout_to_stderr = false;

// Cached completion information, so we can give fast completion results when
// the user erases a character. vscode will resend the completion request if
// that happens.
struct CodeCompleteCache {
  // NOTE: Make sure to access these variables under |WithLock|.
  optional<std::string> cached_path_;
  optional<lsPosition> cached_completion_position_;
  NonElidedVector<lsCompletionItem> cached_results_;

  std::mutex mutex_;

  void WithLock(std::function<void()> action) {
    std::lock_guard<std::mutex> lock(mutex_);
    action();
  }

  bool IsCacheValid(lsTextDocumentPositionParams position) {
    std::lock_guard<std::mutex> lock(mutex_);
    return cached_path_ == position.textDocument.uri.GetPath() &&
           cached_completion_position_ == position.position;
  }
};

// This function returns true if e2e timing should be displayed for the given
// IpcId.
bool ShouldDisplayIpcTiming(IpcId id) {
  switch (id) {
    case IpcId::TextDocumentPublishDiagnostics:
    case IpcId::CqueryPublishInactiveRegions:
    case IpcId::Unknown:
      return false;
    default:
      return true;
  }
}

void PushBack(NonElidedVector<lsLocation>* result,
              optional<lsLocation> location) {
  if (location)
    result->push_back(*location);
}

bool FindFileOrFail(QueryDatabase* db,
                    optional<lsRequestId> id,
                    const std::string& absolute_path,
                    QueryFile** out_query_file,
                    QueryFileId* out_file_id = nullptr) {
  *out_query_file = nullptr;

  auto it = db->usr_to_file.find(LowerPathIfCaseInsensitive(absolute_path));
  if (it != db->usr_to_file.end()) {
    QueryFile& file = db->files[it->second.id];
    if (file.def) {
      *out_query_file = &file;
      if (out_file_id)
        *out_file_id = QueryFileId(it->second.id);
      return true;
    }
  }

  if (out_file_id)
    *out_file_id = QueryFileId((size_t)-1);

  LOG_S(INFO) << "Unable to find file \"" << absolute_path << "\"";

  if (id) {
    Out_Error out;
    out.id = *id;
    out.error.code = lsErrorCodes::InternalError;
    out.error.message = "Unable to find file " + absolute_path;
    IpcManager::WriteStdout(IpcId::Unknown, out);
  }

  return false;
}

void EmitInactiveLines(WorkingFile* working_file,
                       const std::vector<Range>& inactive_regions) {
  Out_CquerySetInactiveRegion out;
  out.params.uri = lsDocumentUri::FromPath(working_file->filename);
  for (Range skipped : inactive_regions) {
    optional<lsRange> ls_skipped = GetLsRange(working_file, skipped);
    if (ls_skipped)
      out.params.inactiveRegions.push_back(*ls_skipped);
  }
  IpcManager::WriteStdout(IpcId::CqueryPublishInactiveRegions, out);
}

// Caches symbols for a single file for semantic highlighting to provide
// relatively stable ids. Only supports xxx files at a time.
struct SemanticHighlightSymbolCache {
  struct Entry {
    // The path this cache belongs to.
    std::string path;
    // Detailed symbol name to stable id.
    using TNameToId = std::unordered_map<std::string, int>;
    TNameToId detailed_type_name_to_stable_id;
    TNameToId detailed_func_name_to_stable_id;
    TNameToId detailed_var_name_to_stable_id;

    explicit Entry(const std::string& path) : path(path) {}

    int GetStableId(SymbolKind kind, const std::string& detailed_name) {
      TNameToId* map = nullptr;
      switch (kind) {
        case SymbolKind::Type:
          map = &detailed_type_name_to_stable_id;
          break;
        case SymbolKind::Func:
          map = &detailed_func_name_to_stable_id;
          break;
        case SymbolKind::Var:
          map = &detailed_var_name_to_stable_id;
          break;
        default:
          assert(false);
          return 0;
      }
      assert(map);
      auto it = map->find(detailed_name);
      if (it != map->end())
        return it->second;
      return (*map)[detailed_name] = map->size();
    }
  };

  constexpr static int kCacheSize = 10;
  LruCache<std::string, Entry> cache_;

  SemanticHighlightSymbolCache() : cache_(kCacheSize) {}

  std::shared_ptr<Entry> GetCacheForFile(const std::string& path) {
    return cache_.Get(path, [&]() { return std::make_shared<Entry>(path); });
  }
};

void EmitSemanticHighlighting(QueryDatabase* db,
                              SemanticHighlightSymbolCache* semantic_cache,
                              WorkingFile* working_file,
                              QueryFile* file) {
  assert(file->def);
  auto map_symbol_kind_to_symbol_type = [](SymbolKind kind) {
    switch (kind) {
      case SymbolKind::Type:
        return Out_CqueryPublishSemanticHighlighting::SymbolType::Type;
      case SymbolKind::Func:
        return Out_CqueryPublishSemanticHighlighting::SymbolType::Function;
      case SymbolKind::Var:
        return Out_CqueryPublishSemanticHighlighting::SymbolType::Variable;
      default:
        assert(false);
        return Out_CqueryPublishSemanticHighlighting::SymbolType::Variable;
    }
  };

  auto semantic_cache_for_file =
      semantic_cache->GetCacheForFile(file->def->path);

  // Group symbols together.
  std::unordered_map<SymbolIdx, Out_CqueryPublishSemanticHighlighting::Symbol>
      grouped_symbols;
  for (SymbolRef sym : file->def->all_symbols) {
    std::string detailed_name;
    bool is_type_member = false;
    // This switch statement also filters out symbols that are not highlighted.
    switch (sym.idx.kind) {
      case SymbolKind::Func: {
        QueryFunc* func = &db->funcs[sym.idx.idx];
        if (!func->def)
          continue;  // applies to for loop
        if (func->def->is_operator)
          continue;  // applies to for loop
        is_type_member = func->def->declaring_type.has_value();
        detailed_name = func->def->short_name;
        break;
      }
      case SymbolKind::Var: {
        QueryVar* var = &db->vars[sym.idx.idx];
        if (!var->def)
          continue;  // applies to for loop
        if (!var->def->is_local && !var->def->declaring_type)
          continue;  // applies to for loop
        is_type_member = var->def->declaring_type.has_value();
        detailed_name = var->def->short_name;
        break;
      }
      case SymbolKind::Type: {
        QueryType* type = &db->types[sym.idx.idx];
        if (!type->def)
          continue;  // applies to for loop
        detailed_name = type->def->detailed_name;
        break;
      }
      default:
        continue;  // applies to for loop
    }

    optional<lsRange> loc = GetLsRange(working_file, sym.loc.range);
    if (loc) {
      auto it = grouped_symbols.find(sym.idx);
      if (it != grouped_symbols.end()) {
        it->second.ranges.push_back(*loc);
      } else {
        Out_CqueryPublishSemanticHighlighting::Symbol symbol;
        symbol.stableId =
            semantic_cache_for_file->GetStableId(sym.idx.kind, detailed_name);
        symbol.type = map_symbol_kind_to_symbol_type(sym.idx.kind);
        symbol.isTypeMember = is_type_member;
        symbol.ranges.push_back(*loc);
        grouped_symbols[sym.idx] = symbol;
      }
    }
  }

  // Publish.
  Out_CqueryPublishSemanticHighlighting out;
  out.params.uri = lsDocumentUri::FromPath(working_file->filename);
  for (auto& entry : grouped_symbols)
    out.params.symbols.push_back(entry.second);
  IpcManager::WriteStdout(IpcId::CqueryPublishSemanticHighlighting, out);
}

optional<int> FindIncludeLine(const std::vector<std::string>& lines,
                              const std::string& full_include_line) {
  //
  // This returns an include line. For example,
  //
  //    #include <a>  // 0
  //    #include <c>  // 1
  //
  // Given #include <b>, this will return '1', which means that the
  // #include <b> text should be inserted at the start of line 1. Inserting
  // at the start of a line allows insertion at both the top and bottom of the
  // document.
  //
  // If the include line is already in the document this returns nullopt.
  //

  optional<int> last_include_line;
  optional<int> best_include_line;

  //  1 => include line is gt content (ie, it should go after)
  // -1 => include line is lt content (ie, it should go before)
  int last_line_compare = 1;

  for (int line = 0; line < (int)lines.size(); ++line) {
    if (!StartsWith(lines[line], "#include")) {
      last_line_compare = 1;
      continue;
    }

    last_include_line = line;

    int current_line_compare = full_include_line.compare(lines[line]);
    if (current_line_compare == 0)
      return nullopt;

    if (last_line_compare == 1 && current_line_compare == -1)
      best_include_line = line;
    last_line_compare = current_line_compare;
  }

  if (best_include_line)
    return *best_include_line;
  // If |best_include_line| didn't match that means we likely didn't find an
  // include which was lt the new one, so put it at the end of the last include
  // list.
  if (last_include_line)
    return *last_include_line + 1;
  // No includes, use top of document.
  return 0;
}

optional<QueryFileId> GetImplementationFile(QueryDatabase* db,
                                            QueryFileId file_id,
                                            QueryFile* file) {
  for (SymbolRef sym : file->def->outline) {
    switch (sym.idx.kind) {
      case SymbolKind::Func: {
        QueryFunc& func = db->funcs[sym.idx.idx];
        // Note: we ignore the definition if it is in the same file (ie,
        // possibly a header).
        if (func.def && func.def->definition_extent &&
            func.def->definition_extent->path != file_id) {
          return func.def->definition_extent->path;
        }
        break;
      }
      case SymbolKind::Var: {
        QueryVar& var = db->vars[sym.idx.idx];
        // Note: we ignore the definition if it is in the same file (ie,
        // possibly a header).
        if (var.def && var.def->definition_extent &&
            var.def->definition_extent->path != file_id) {
          return db->vars[sym.idx.idx].def->definition_extent->path;
        }
        break;
      }
      default:
        break;
    }
  }

  // No associated definition, scan the project for a file in the same
  // directory with the same base-name.
  std::string original_path = LowerPathIfCaseInsensitive(file->def->path);
  std::string target_path = original_path;
  size_t last = target_path.find_last_of('.');
  if (last != std::string::npos) {
    target_path = target_path.substr(0, last);
  }

  LOG_S(INFO) << "!! Looking for impl file that starts with " << target_path;

  for (auto& entry : db->usr_to_file) {
    Usr path = entry.first;

    // Do not consider header files for implementation files.
    // TODO: make file extensions configurable.
    if (EndsWith(path, ".h") || EndsWith(path, ".hpp"))
      continue;

    if (StartsWith(path, target_path) && path != original_path) {
      return entry.second;
    }
  }

  return nullopt;
}

void EnsureImplFile(QueryDatabase* db,
                    QueryFileId file_id,
                    optional<lsDocumentUri>& impl_uri,
                    optional<QueryFileId>& impl_file_id) {
  if (!impl_uri.has_value()) {
    QueryFile& file = db->files[file_id.id];
    assert(file.def);

    impl_file_id = GetImplementationFile(db, file_id, &file);
    if (!impl_file_id.has_value())
      impl_file_id = file_id;

    QueryFile& impl_file = db->files[impl_file_id->id];
    if (impl_file.def)
      impl_uri = lsDocumentUri::FromPath(impl_file.def->path);
    else
      impl_uri = lsDocumentUri::FromPath(file.def->path);
  }
}

optional<lsTextEdit> BuildAutoImplementForFunction(QueryDatabase* db,
                                                   WorkingFiles* working_files,
                                                   WorkingFile* working_file,
                                                   int default_line,
                                                   QueryFileId decl_file_id,
                                                   QueryFileId impl_file_id,
                                                   QueryFunc& func) {
  assert(func.def);
  for (const QueryLocation& decl : func.declarations) {
    if (decl.path != decl_file_id)
      continue;

    optional<lsRange> ls_decl = GetLsRange(working_file, decl.range);
    if (!ls_decl)
      continue;

    optional<std::string> type_name;
    optional<lsPosition> same_file_insert_end;
    if (func.def->declaring_type) {
      QueryType& declaring_type = db->types[func.def->declaring_type->id];
      if (declaring_type.def) {
        type_name = declaring_type.def->short_name;
        optional<lsRange> ls_type_def_extent = GetLsRange(
            working_file, declaring_type.def->definition_extent->range);
        if (ls_type_def_extent) {
          same_file_insert_end = ls_type_def_extent->end;
          same_file_insert_end->character += 1;  // move past semicolon.
        }
      }
    }

    std::string insert_text;
    int newlines_after_name = 0;
    LexFunctionDeclaration(working_file->buffer_content, ls_decl->start,
                           type_name, &insert_text, &newlines_after_name);

    if (!same_file_insert_end) {
      same_file_insert_end = ls_decl->end;
      same_file_insert_end->line += newlines_after_name;
      same_file_insert_end->character = 1000;
    }

    lsTextEdit edit;

    if (decl_file_id == impl_file_id) {
      edit.range.start = *same_file_insert_end;
      edit.range.end = *same_file_insert_end;
      edit.newText = "\n\n" + insert_text;
    } else {
      lsPosition best_pos;
      best_pos.line = default_line;
      int best_dist = INT_MAX;

      QueryFile& file = db->files[impl_file_id.id];
      assert(file.def);
      for (SymbolRef sym : file.def->outline) {
        switch (sym.idx.kind) {
          case SymbolKind::Func: {
            QueryFunc& sym_func = db->funcs[sym.idx.idx];
            if (!sym_func.def || !sym_func.def->definition_extent)
              break;

            for (QueryLocation& func_decl : sym_func.declarations) {
              if (func_decl.path == decl_file_id) {
                int dist = func_decl.range.start.line - decl.range.start.line;
                if (abs(dist) < abs(best_dist)) {
                  optional<lsLocation> def_loc = GetLsLocation(
                      db, working_files, *sym_func.def->definition_extent);
                  if (!def_loc)
                    continue;

                  best_dist = dist;

                  if (dist > 0)
                    best_pos = def_loc->range.start;
                  else
                    best_pos = def_loc->range.end;
                }
              }
            }

            break;
          }
          case SymbolKind::Var: {
            // TODO: handle vars.
            break;
          }
          case SymbolKind::Invalid:
          case SymbolKind::File:
          case SymbolKind::Type:
            LOG_S(WARNING) << "Unexpected SymbolKind "
                           << static_cast<int>(sym.idx.kind);
            break;
        }
      }

      edit.range.start = best_pos;
      edit.range.end = best_pos;
      if (best_dist < 0)
        edit.newText = "\n\n" + insert_text;
      else
        edit.newText = insert_text + "\n\n";
    }

    return edit;
  }

  return nullopt;
}

void EmitDiagnostics(WorkingFiles* working_files,
                     std::string path,
                     NonElidedVector<lsDiagnostic> diagnostics) {
  // Emit diagnostics.
  Out_TextDocumentPublishDiagnostics out;
  out.params.uri = lsDocumentUri::FromPath(path);
  out.params.diagnostics = diagnostics;
  IpcManager::WriteStdout(IpcId::TextDocumentPublishDiagnostics, out);

  // Cache diagnostics so we can show fixits.
  working_files->DoActionOnFile(path, [&](WorkingFile* working_file) {
    if (working_file)
      working_file->diagnostics_ = diagnostics;
  });
}

// Pre-filters completion responses before sending to vscode. This results in a
// significantly snappier completion experience as vscode is easily overloaded
// when given 1000+ completion items.
void FilterCompletionResponse(Out_TextDocumentComplete* complete_response,
                              const std::string& complete_text) {
// Used to inject more completions.
#if false
  const size_t kNumIterations = 250;
  size_t size = complete_response->result.items.size();
  complete_response->result.items.reserve(size * (kNumIterations + 1));
  for (size_t iteration = 0; iteration < kNumIterations; ++iteration) {
    for (size_t i = 0; i < size; ++i) {
      auto item = complete_response->result.items[i];
      item.label += "#" + std::to_string(iteration);
      complete_response->result.items.push_back(item);
    }
  }
#endif

  const size_t kMaxResultSize = 100u;
  if (complete_response->result.items.size() > kMaxResultSize) {
    if (complete_text.empty()) {
      complete_response->result.items.resize(kMaxResultSize);
    } else {
      NonElidedVector<lsCompletionItem> filtered_result;
      filtered_result.reserve(kMaxResultSize);

      std::unordered_set<std::string> inserted;
      inserted.reserve(kMaxResultSize);

      // Find literal matches first.
      for (const lsCompletionItem& item : complete_response->result.items) {
        if (item.label.find(complete_text) != std::string::npos) {
          // Don't insert the same completion entry.
          if (!inserted.insert(item.InsertedContent()).second)
            continue;

          filtered_result.push_back(item);
          if (filtered_result.size() >= kMaxResultSize)
            break;
        }
      }

      // Find fuzzy matches if we haven't found all of the literal matches.
      if (filtered_result.size() < kMaxResultSize) {
        for (const lsCompletionItem& item : complete_response->result.items) {
          if (SubstringMatch(complete_text, item.label)) {
            // Don't insert the same completion entry.
            if (!inserted.insert(item.InsertedContent()).second)
              continue;

            filtered_result.push_back(item);
            if (filtered_result.size() >= kMaxResultSize)
              break;
          }
        }
      }

      complete_response->result.items = filtered_result;
    }

    // Assuming the client does not support out-of-order completion (ie, ao
    // matches against oa), our filtering is guaranteed to contain any
    // potential matches, so the completion is only incomplete if we have the
    // max number of emitted matches.
    if (complete_response->result.items.size() >= kMaxResultSize) {
      LOG_S(INFO) << "Marking completion results as incomplete";
      complete_response->result.isIncomplete = true;
    }
  }
}

struct Index_Request {
  std::string path;
  // TODO: make |args| a string that is parsed lazily.
  std::vector<std::string> args;
  bool is_interactive;
  optional<std::string> contents;  // Preloaded contents. Useful for tests.

  Index_Request(const std::string& path,
                const std::vector<std::string>& args,
                bool is_interactive,
                optional<std::string> contents)
      : path(path),
        args(args),
        is_interactive(is_interactive),
        contents(contents) {}
};

struct Index_DoIdMap {
  std::unique_ptr<IndexFile> current;
  std::unique_ptr<IndexFile> previous;

  PerformanceImportFile perf;
  bool is_interactive = false;
  bool write_to_disk = false;
  bool load_previous = false;

  Index_DoIdMap(std::unique_ptr<IndexFile> current,
                PerformanceImportFile perf,
                bool is_interactive,
                bool write_to_disk)
      : current(std::move(current)),
        perf(perf),
        is_interactive(is_interactive),
        write_to_disk(write_to_disk) {
    assert(this->current);
  }
};

struct Index_OnIdMapped {
  struct File {
    std::unique_ptr<IndexFile> file;
    std::unique_ptr<IdMap> ids;

    File(std::unique_ptr<IndexFile> file, std::unique_ptr<IdMap> ids)
        : file(std::move(file)), ids(std::move(ids)) {}
  };

  std::unique_ptr<File> previous;
  std::unique_ptr<File> current;

  PerformanceImportFile perf;
  bool is_interactive;
  bool write_to_disk;

  Index_OnIdMapped(PerformanceImportFile perf,
                   bool is_interactive,
                   bool write_to_disk)
      : perf(perf),
        is_interactive(is_interactive),
        write_to_disk(write_to_disk) {}
};

struct Index_OnIndexed {
  IndexUpdate update;
  PerformanceImportFile perf;

  Index_OnIndexed(IndexUpdate& update, PerformanceImportFile perf)
      : update(update), perf(perf) {}
};

struct QueueManager {
  using Index_RequestQueue = ThreadedQueue<Index_Request>;
  using Index_DoIdMapQueue = ThreadedQueue<Index_DoIdMap>;
  using Index_OnIdMappedQueue = ThreadedQueue<Index_OnIdMapped>;
  using Index_OnIndexedQueue = ThreadedQueue<Index_OnIndexed>;

  Index_RequestQueue index_request;
  Index_DoIdMapQueue do_id_map;
  Index_DoIdMapQueue load_previous_index;
  Index_OnIdMappedQueue on_id_mapped;
  Index_OnIndexedQueue on_indexed;

  QueueManager(MultiQueueWaiter* waiter)
      : index_request(waiter),
        do_id_map(waiter),
        load_previous_index(waiter),
        on_id_mapped(waiter),
        on_indexed(waiter) {}

  bool HasWork() {
    return !index_request.IsEmpty() || !do_id_map.IsEmpty() ||
           !load_previous_index.IsEmpty() || !on_id_mapped.IsEmpty() ||
           !on_indexed.IsEmpty();
  }
};

void RegisterMessageTypes() {
  MessageRegistry::instance()->Register<Ipc_CancelRequest>();
  MessageRegistry::instance()->Register<Ipc_InitializeRequest>();
  MessageRegistry::instance()->Register<Ipc_InitializedNotification>();
  MessageRegistry::instance()->Register<Ipc_Exit>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDidOpen>();
  MessageRegistry::instance()->Register<Ipc_CqueryTextDocumentDidView>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDidChange>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDidClose>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDidSave>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentRename>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentComplete>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentSignatureHelp>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDefinition>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDocumentHighlight>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentHover>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentReferences>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDocumentSymbol>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentDocumentLink>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentCodeAction>();
  MessageRegistry::instance()->Register<Ipc_TextDocumentCodeLens>();
  MessageRegistry::instance()->Register<Ipc_CodeLensResolve>();
  MessageRegistry::instance()->Register<Ipc_WorkspaceSymbol>();
  MessageRegistry::instance()->Register<Ipc_CqueryFreshenIndex>();
  MessageRegistry::instance()->Register<Ipc_CqueryTypeHierarchyTree>();
  MessageRegistry::instance()->Register<Ipc_CqueryCallTreeInitial>();
  MessageRegistry::instance()->Register<Ipc_CqueryCallTreeExpand>();
  MessageRegistry::instance()->Register<Ipc_CqueryVars>();
  MessageRegistry::instance()->Register<Ipc_CqueryCallers>();
  MessageRegistry::instance()->Register<Ipc_CqueryBase>();
  MessageRegistry::instance()->Register<Ipc_CqueryDerived>();
  MessageRegistry::instance()->Register<Ipc_CqueryIndexFile>();
  MessageRegistry::instance()->Register<Ipc_CqueryQueryDbWaitForIdleIndexer>();
  MessageRegistry::instance()->Register<Ipc_CqueryExitWhenIdle>();
}

// Manages files inside of the indexing pipeline so we don't have the same file
// being imported multiple times.
//
// NOTE: This is not thread safe and should only be used on the querydb thread.
struct ImportManager {
  // Try to mark the given dependency as imported. A dependency can only ever be
  // imported once.
  bool TryMarkDependencyImported(const std::string& path) {
    std::lock_guard<std::mutex> lock(depdency_mutex_);
    return depdency_imported_.insert(path).second;
  }

  // Try to import the given file into querydb. We should only ever be
  // importing a file into querydb once per file. Returns true if the file
  // can be imported.
  bool StartQueryDbImport(const std::string& path) {
    return querydb_processing_.insert(path).second;
  }

  // The file has been fully imported and can be imported again later on.
  void DoneQueryDbImport(const std::string& path) {
    querydb_processing_.erase(path);
  }

  // Returns true if there any any files currently being imported.
  bool HasActiveQuerydbImports() { return !querydb_processing_.empty(); }

  std::unordered_set<std::string> querydb_processing_;

  // TODO: use std::shared_mutex so we can have multiple readers.
  std::mutex depdency_mutex_;
  std::unordered_set<std::string> depdency_imported_;
};

// Manages loading caches from file paths for the indexer process.
struct CacheLoader {
  explicit CacheLoader(Config* config) : config_(config) {}

  IndexFile* TryLoad(const std::string& path) {
    auto it = caches.find(path);
    if (it != caches.end())
      return it->second.get();

    std::unique_ptr<IndexFile> cache = LoadCachedIndex(config_, path);
    if (!cache)
      return nullptr;

    caches[path] = std::move(cache);
    return caches[path].get();
  }

  // Takes the existing cache or loads the cache at |path|. May return nullptr
  // if the cache does not exist.
  std::unique_ptr<IndexFile> TryTakeOrLoad(const std::string& path) {
    auto it = caches.find(path);
    if (it != caches.end()) {
      auto result = std::move(it->second);
      caches.erase(it);
      return result;
    }

    return LoadCachedIndex(config_, path);
  }

  // Takes the existing cache or loads the cache at |path|. Asserts the cache
  // exists.
  std::unique_ptr<IndexFile> TakeOrLoad(const std::string& path) {
    auto result = TryTakeOrLoad(path);
    assert(result);
    return result;
  }

  std::unordered_map<std::string, std::unique_ptr<IndexFile>> caches;
  Config* config_;
};

// Caches timestamps of cc files so we can avoid a filesystem reads. This is
// important for import perf, as during dependency checking the same files are
// checked over and over again if they are common headers.
struct TimestampManager {
  optional<int64_t> GetLastCachedModificationTime(CacheLoader* cache_loader,
                                                  const std::string& path) {
    {
      std::lock_guard<std::mutex> guard(mutex_);
      auto it = timestamps_.find(path);
      if (it != timestamps_.end())
        return it->second;
    }
    IndexFile* file = cache_loader->TryLoad(path);
    if (!file)
      return nullopt;

    UpdateCachedModificationTime(path, file->last_modification_time);
    return file->last_modification_time;
  }

  void UpdateCachedModificationTime(const std::string& path,
                                    int64_t timestamp) {
    std::lock_guard<std::mutex> guard(mutex_);
    timestamps_[path] = timestamp;
  }

  // TODO: use std::shared_mutex so we can have multiple readers.
  std::mutex mutex_;
  std::unordered_map<std::string, int64_t> timestamps_;
};

struct IndexManager {
  std::unordered_set<std::string> files_being_indexed_;
  std::mutex mutex_;

  // Marks a file as being indexed. Returns true if the file is not already
  // being indexed.
  bool MarkIndex(const std::string& path) {
    std::lock_guard<std::mutex> lock(mutex_);

    return files_being_indexed_.insert(path).second;
  }

  // Unmarks a file as being indexed, so it can get indexed again in the
  // future.
  void ClearIndex(const std::string& path) {
    std::lock_guard<std::mutex> lock(mutex_);

    auto it = files_being_indexed_.find(path);
    assert(it != files_being_indexed_.end());
    files_being_indexed_.erase(it);
  }
};

// Send indexing progress to client if reporting is enabled.
void EmitProgress(Config* config, QueueManager* queue) {
  if (config->enableProgressReports) {
    Out_Progress out;
    out.params.indexRequestCount = queue->index_request.Size();
    out.params.doIdMapCount = queue->do_id_map.Size();
    out.params.loadPreviousIndexCount = queue->load_previous_index.Size();
    out.params.onIdMappedCount = queue->on_id_mapped.Size();
    out.params.onIndexedCount = queue->on_indexed.Size();

    IpcManager::WriteStdout(IpcId::Unknown, out);
  }
}

}  // namespace

////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// IMPORT PIPELINE /////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

enum class FileParseQuery { NeedsParse, DoesNotNeedParse, NoSuchFile };

std::vector<Index_DoIdMap> DoParseFile(
    Config* config,
    WorkingFiles* working_files,
    ClangIndex* index,
    FileConsumer::SharedState* file_consumer_shared,
    TimestampManager* timestamp_manager,
    ImportManager* import_manager,
    CacheLoader* cache_loader,
    bool is_interactive,
    const std::string& path,
    const std::vector<std::string>& args,
    const optional<FileContents>& contents) {
  std::vector<Index_DoIdMap> result;

  // Always run this block, even if we are interactive, so we can check
  // dependencies and reset files in |file_consumer_shared|.
  IndexFile* previous_index = cache_loader->TryLoad(path);
  if (previous_index) {
    // If none of the dependencies have changed and the index is not
    // interactive (ie, requested by a file save), skip parsing and just load
    // from cache.

    // Checks if |path| needs to be reparsed. This will modify cached state
    // such that calling this function twice with the same path may return true
    // the first time but will return false the second.
    auto file_needs_parse = [&](const std::string& path, bool is_dependency) {
      // If the file is a dependency but another file as already imported it,
      // don't bother.
      if (!is_interactive && is_dependency &&
          !import_manager->TryMarkDependencyImported(path)) {
        return FileParseQuery::DoesNotNeedParse;
      }

      optional<int64_t> modification_timestamp = GetLastModificationTime(path);

      // Cannot find file.
      if (!modification_timestamp)
        return FileParseQuery::NoSuchFile;

      optional<int64_t> last_cached_modification =
          timestamp_manager->GetLastCachedModificationTime(cache_loader, path);

      // File has been changed.
      if (!last_cached_modification ||
          modification_timestamp != *last_cached_modification) {
        file_consumer_shared->Reset(path);
        timestamp_manager->UpdateCachedModificationTime(
            path, *modification_timestamp);
        return FileParseQuery::NeedsParse;
      }

      // File has not changed, do not parse it.
      return FileParseQuery::DoesNotNeedParse;
    };

    // Check timestamps and update |file_consumer_shared|.
    FileParseQuery path_state = file_needs_parse(path, false /*is_dependency*/);

    // Target file does not exist on disk, do not emit any indexes.
    // TODO: Dependencies should be reassigned to other files. We can do this by
    // updating the "primary_file" if it doesn't exist. Might not actually be a
    // problem in practice.
    if (path_state == FileParseQuery::NoSuchFile)
      return result;

    bool needs_reparse =
        is_interactive || path_state == FileParseQuery::NeedsParse;

    for (const std::string& dependency : previous_index->dependencies) {
      assert(!dependency.empty());

      // note: Use != as there are multiple failure results for FileParseQuery.
      if (file_needs_parse(dependency, true /*is_dependency*/) !=
          FileParseQuery::DoesNotNeedParse) {
        LOG_S(INFO) << "Timestamp has changed for " << dependency << " (via "
                    << previous_index->path << ")";
        needs_reparse = true;
        // SUBTLE: Do not break here, as |file_consumer_shared| is updated
        // inside of |file_needs_parse|.
      }
    }

    // No timestamps changed - load directly from cache.
    if (!needs_reparse) {
      LOG_S(INFO) << "Skipping parse; no timestamp change for " << path;

      // TODO/FIXME: real perf
      PerformanceImportFile perf;
      result.push_back(Index_DoIdMap(cache_loader->TakeOrLoad(path), perf,
                                     is_interactive, false /*write_to_disk*/));
      for (const std::string& dependency : previous_index->dependencies) {
        // Only load a dependency if it is not already loaded.
        //
        // This is important for perf in large projects where there are lots of
        // dependencies shared between many files.
        if (!file_consumer_shared->Mark(dependency))
          continue;

        LOG_S(INFO) << "Emitting index result for " << dependency << " (via "
                    << previous_index->path << ")";

        std::unique_ptr<IndexFile> dependency_index =
            cache_loader->TryTakeOrLoad(dependency);

        // |dependency_index| may be null if there is no cache for it but
        // another file has already started importing it.
        if (!dependency_index)
          continue;

        result.push_back(Index_DoIdMap(std::move(dependency_index), perf,
                                       is_interactive,
                                       false /*write_to_disk*/));
      }
      return result;
    }
  }

  LOG_S(INFO) << "Parsing " << path;

  // Load file contents for all dependencies into memory. If the dependencies
  // for the file changed we may not end up using all of the files we
  // preloaded. If a new dependency was added the indexer will grab the file
  // contents as soon as possible.
  //
  // We do this to minimize the race between indexing a file and capturing the
  // file contents.
  //
  // TODO: We might be able to optimize perf by only copying for files in
  //       working_files. We can pass that same set of files to the indexer as
  //       well. We then default to a fast file-copy if not in working set.
  bool loaded_primary = false;
  std::vector<FileContents> file_contents;
  if (contents) {
    loaded_primary = loaded_primary || contents->path == path;
    file_contents.push_back(*contents);
  }
  for (const auto& it : cache_loader->caches) {
    const std::unique_ptr<IndexFile>& index = it.second;
    assert(index);
    optional<std::string> index_content = ReadContent(index->path);
    if (!index_content) {
      LOG_S(ERROR) << "Failed to preload index content for " << index->path;
      continue;
    }
    file_contents.push_back(FileContents(index->path, *index_content));

    loaded_primary = loaded_primary || index->path == path;
  }
  if (!loaded_primary) {
    optional<std::string> content = ReadContent(path);
    if (!content) {
      LOG_S(ERROR) << "Skipping index (file cannot be found): " << path;
      return result;
    }
    file_contents.push_back(FileContents(path, *content));
  }

  PerformanceImportFile perf;
  std::vector<std::unique_ptr<IndexFile>> indexes = Parse(
      config, file_consumer_shared, path, args, file_contents, &perf, index);
  for (std::unique_ptr<IndexFile>& new_index : indexes) {
    Timer time;

    // Only emit diagnostics for non-interactive sessions, which makes it easier
    // to identify indexing problems. For interactive sessions, diagnostics are
    // handled by code completion.
    if (!is_interactive)
      EmitDiagnostics(working_files, new_index->path, new_index->diagnostics_);

    // When main thread does IdMap request it will request the previous index if
    // needed.
    LOG_S(INFO) << "Emitting index result for " << new_index->path;
    result.push_back(Index_DoIdMap(std::move(new_index), perf, is_interactive,
                                   true /*write_to_disk*/));
  }

  return result;
}

// Index a file using an already-parsed translation unit from code completion.
// Since most of the time for indexing a file comes from parsing, we can do
// real-time indexing.
// TODO: add option to disable this.
void IndexWithTuFromCodeCompletion(
    QueueManager* queue,
    FileConsumer::SharedState* file_consumer_shared,
    ClangTranslationUnit* tu,
    const std::vector<CXUnsavedFile>& file_contents,
    const std::string& path,
    const std::vector<std::string>& args) {
  file_consumer_shared->Reset(path);

  PerformanceImportFile perf;
  ClangIndex index;
  std::vector<std::unique_ptr<IndexFile>> indexes = ParseWithTu(
      file_consumer_shared, &perf, tu, &index, path, args, file_contents);

  std::vector<Index_DoIdMap> result;
  for (std::unique_ptr<IndexFile>& new_index : indexes) {
    Timer time;

    // When main thread does IdMap request it will request the previous index if
    // needed.
    LOG_S(INFO) << "Emitting index result for " << new_index->path;
    result.push_back(Index_DoIdMap(std::move(new_index), perf,
                                   true /*is_interactive*/,
                                   true /*write_to_disk*/));
  }

  LOG_IF_S(WARNING, result.size() > 1)
      << "Code completion index update generated more than one index";

  queue->do_id_map.EnqueueAll(std::move(result));
}

std::vector<Index_DoIdMap> ParseFile(
    Config* config,
    WorkingFiles* working_files,
    ClangIndex* index,
    FileConsumer::SharedState* file_consumer_shared,
    TimestampManager* timestamp_manager,
    ImportManager* import_manager,
    bool is_interactive,
    const Project::Entry& entry,
    const optional<std::string>& contents) {
  optional<FileContents> file_contents;
  if (contents)
    file_contents = FileContents(entry.filename, *contents);

  CacheLoader cache_loader(config);

  // Try to determine the original import file by loading the file from cache.
  // This lets the user request an index on a header file, which clang will
  // complain about if indexed by itself.
  IndexFile* entry_cache = cache_loader.TryLoad(entry.filename);
  std::string tu_path = entry_cache ? entry_cache->import_file : entry.filename;
  return DoParseFile(config, working_files, index, file_consumer_shared,
                     timestamp_manager, import_manager, &cache_loader,
                     is_interactive, tu_path, entry.args, file_contents);
}

bool IndexMain_DoParse(Config* config,
                       WorkingFiles* working_files,
                       QueueManager* queue,
                       FileConsumer::SharedState* file_consumer_shared,
                       TimestampManager* timestamp_manager,
                       ImportManager* import_manager,
                       ClangIndex* index) {
  optional<Index_Request> request = queue->index_request.TryDequeue();
  if (!request)
    return false;

  Project::Entry entry;
  entry.filename = request->path;
  entry.args = request->args;
  std::vector<Index_DoIdMap> responses = ParseFile(
      config, working_files, index, file_consumer_shared, timestamp_manager,
      import_manager, request->is_interactive, entry, request->contents);

  // Don't bother sending an IdMap request if there are no responses.
  if (responses.empty())
    return false;

  // EnqueueAll will clear |responses|.
  queue->do_id_map.EnqueueAll(std::move(responses));
  return true;
}

bool IndexMain_DoCreateIndexUpdate(Config* config,
                                   QueueManager* queue,
                                   TimestampManager* timestamp_manager) {
  optional<Index_OnIdMapped> response = queue->on_id_mapped.TryDequeue();
  if (!response)
    return false;

  Timer time;

  IdMap* previous_id_map = nullptr;
  IndexFile* previous_index = nullptr;
  if (response->previous) {
    previous_id_map = response->previous->ids.get();
    previous_index = response->previous->file.get();
  }

  // Build delta update.
  IndexUpdate update =
      IndexUpdate::CreateDelta(previous_id_map, response->current->ids.get(),
                               previous_index, response->current->file.get());
  response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
  LOG_S(INFO) << "Built index update for " << response->current->file->path
              << " (is_delta=" << !!response->previous << ")";

  // Write current index to disk if requested.
  if (response->write_to_disk) {
    LOG_S(INFO) << "Writing cached index to disk for "
                << response->current->file->path;
    time.Reset();
    WriteToCache(config, *response->current->file);
    response->perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
    timestamp_manager->UpdateCachedModificationTime(
        response->current->file->path,
        response->current->file->last_modification_time);
  }

#if false
#define PRINT_SECTION(name)                                                    \
  if (response->perf.name) {                                                   \
    total += response->perf.name;                                              \
    output << " " << #name << ": " << FormatMicroseconds(response->perf.name); \
  }
  std::stringstream output;
  long long total = 0;
  output << "[perf]";
  PRINT_SECTION(index_parse);
  PRINT_SECTION(index_build);
  PRINT_SECTION(index_save_to_disk);
  PRINT_SECTION(index_load_cached);
  PRINT_SECTION(querydb_id_map);
  PRINT_SECTION(index_make_delta);
  output << "\n       total: " << FormatMicroseconds(total);
  output << " path: " << response->current_index->path;
  LOG_S(INFO) << output.rdbuf();
#undef PRINT_SECTION

  if (response->is_interactive)
    LOG_S(INFO) << "Applying IndexUpdate" << std::endl << update.ToString();
#endif

  Index_OnIndexed reply(update, response->perf);
  queue->on_indexed.Enqueue(std::move(reply));

  return true;
}

bool IndexMain_LoadPreviousIndex(Config* config, QueueManager* queue) {
  optional<Index_DoIdMap> response = queue->load_previous_index.TryDequeue();
  if (!response)
    return false;

  response->previous = LoadCachedIndex(config, response->current->path);
  LOG_IF_S(ERROR, !response->previous)
      << "Unable to load previous index for already imported index "
      << response->current->path;

  queue->do_id_map.Enqueue(std::move(*response));
  return true;
}

bool IndexMergeIndexUpdates(QueueManager* queue) {
  optional<Index_OnIndexed> root = queue->on_indexed.TryDequeue();
  if (!root)
    return false;

  bool did_merge = false;
  while (true) {
    optional<Index_OnIndexed> to_join = queue->on_indexed.TryDequeue();
    if (!to_join) {
      queue->on_indexed.Enqueue(std::move(*root));
      return did_merge;
    }

    did_merge = true;
    Timer time;
    root->update.Merge(to_join->update);
    // time.ResetAndPrint("Joined querydb updates for files: " +
    // StringJoinMap(root->update.files_def_update,
    //[](const QueryFile::DefUpdate& update) {
    // return update.path;
    //}));
  }
}

WorkThread::Result IndexMain(Config* config,
                             FileConsumer::SharedState* file_consumer_shared,
                             TimestampManager* timestamp_manager,
                             ImportManager* import_manager,
                             Project* project,
                             WorkingFiles* working_files,
                             MultiQueueWaiter* waiter,
                             QueueManager* queue) {
  EmitProgress(config, queue);

  // TODO: dispose of index after it is not used for a while.
  ClangIndex index;

  // TODO: process all off IndexMain_DoIndex before calling
  // IndexMain_DoCreateIndexUpdate for
  //       better icache behavior. We need to have some threads spinning on
  //       both though
  //       otherwise memory usage will get bad.

  // We need to make sure to run both IndexMain_DoParse and
  // IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any
  // work. Running both also lets the user query the partially constructed
  // index.
  bool did_parse =
      IndexMain_DoParse(config, working_files, queue, file_consumer_shared,
                        timestamp_manager, import_manager, &index);

  bool did_create_update =
      IndexMain_DoCreateIndexUpdate(config, queue, timestamp_manager);

  bool did_load_previous = IndexMain_LoadPreviousIndex(config, queue);

  // Nothing to index and no index updates to create, so join some already
  // created index updates to reduce work on querydb thread.
  bool did_merge = false;
  if (!did_parse && !did_create_update && !did_load_previous)
    did_merge = IndexMergeIndexUpdates(queue);

  // We didn't do any work, so wait for a notification.
  if (!did_parse && !did_create_update && !did_merge && !did_load_previous) {
    waiter->Wait({&queue->index_request, &queue->on_id_mapped,
                  &queue->load_previous_index, &queue->on_indexed});
  }

  return queue->HasWork() ? WorkThread::Result::MoreWork
                          : WorkThread::Result::NoWork;
}

bool QueryDb_ImportMain(Config* config,
                        QueryDatabase* db,
                        ImportManager* import_manager,
                        QueueManager* queue,
                        SemanticHighlightSymbolCache* semantic_cache,
                        WorkingFiles* working_files) {
  EmitProgress(config, queue);

  bool did_work = false;

  while (true) {
    optional<Index_DoIdMap> request = queue->do_id_map.TryDequeue();
    if (!request)
      break;
    did_work = true;

    assert(request->current);

    // If the request does not have previous state and we have already imported
    // it, load the previous state from disk and rerun IdMap logic later. Do not
    // do this if we have already attempted in the past.
    if (!request->load_previous && !request->previous &&
        db->usr_to_file.find(LowerPathIfCaseInsensitive(
            request->current->path)) != db->usr_to_file.end()) {
      assert(!request->load_previous);
      request->load_previous = true;
      queue->load_previous_index.Enqueue(std::move(*request));
      continue;
    }

    // Check if the file is already being imported into querydb. If it is, drop
    // the request.
    //
    // Note, we must do this *after* we have checked for the previous index,
    // otherwise we will never actually generate the IdMap.
    if (!import_manager->StartQueryDbImport(request->current->path)) {
      LOG_S(INFO) << "Dropping index as it is already being imported for "
                  << request->current->path;
      continue;
    }

    Index_OnIdMapped response(request->perf, request->is_interactive,
                              request->write_to_disk);
    Timer time;

    auto make_map = [db](std::unique_ptr<IndexFile> file)
        -> std::unique_ptr<Index_OnIdMapped::File> {
      if (!file)
        return nullptr;

      auto id_map = MakeUnique<IdMap>(db, file->id_cache);
      return MakeUnique<Index_OnIdMapped::File>(std::move(file),
                                                std::move(id_map));
    };
    response.current = make_map(std::move(request->current));
    response.previous = make_map(std::move(request->previous));
    response.perf.querydb_id_map = time.ElapsedMicrosecondsAndReset();

    queue->on_id_mapped.Enqueue(std::move(response));
  }

  while (true) {
    optional<Index_OnIndexed> response = queue->on_indexed.TryDequeue();
    if (!response)
      break;

    did_work = true;

    Timer time;

    for (auto& updated_file : response->update.files_def_update) {
      // TODO: We're reading a file on querydb thread. This is slow!! If this
      // a real problem in practice we can load the file in a previous stage.
      // It should be fine though because we only do it if the user has the
      // file open.
      WorkingFile* working_file =
          working_files->GetFileByFilename(updated_file.path);
      if (working_file) {
        optional<std::string> cached_file_contents =
            LoadCachedFileContents(config, updated_file.path);
        if (cached_file_contents)
          working_file->SetIndexContent(*cached_file_contents);
        else
          working_file->SetIndexContent(working_file->buffer_content);
        time.ResetAndPrint(
            "Update WorkingFile index contents (via disk load) for " +
            updated_file.path);

        // Update inactive region.
        EmitInactiveLines(working_file, updated_file.inactive_regions);
      }
    }

    time.Reset();
    db->ApplyIndexUpdate(&response->update);
    time.ResetAndPrint("Applying index update for " +
                       StringJoinMap(response->update.files_def_update,
                                     [](const QueryFile::DefUpdate& value) {
                                       return value.path;
                                     }));

    // Update semantic highlighting.
    for (auto& updated_file : response->update.files_def_update) {
      WorkingFile* working_file =
          working_files->GetFileByFilename(updated_file.path);
      if (working_file) {
        QueryFileId file_id =
            db->usr_to_file[LowerPathIfCaseInsensitive(working_file->filename)];
        QueryFile* file = &db->files[file_id.id];
        EmitSemanticHighlighting(db, semantic_cache, working_file, file);
      }
    }

    // Mark the files as being done in querydb stage after we apply the index
    // update.
    for (auto& updated_file : response->update.files_def_update)
      import_manager->DoneQueryDbImport(updated_file.path);
  }

  return did_work;
}

////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// QUERYDB MAIN ////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

bool QueryDbMainLoop(Config* config,
                     QueryDatabase* db,
                     bool* exit_when_idle,
                     MultiQueueWaiter* waiter,
                     QueueManager* queue,
                     Project* project,
                     FileConsumer::SharedState* file_consumer_shared,
                     ImportManager* import_manager,
                     TimestampManager* timestamp_manager,
                     SemanticHighlightSymbolCache* semantic_cache,
                     WorkingFiles* working_files,
                     ClangCompleteManager* clang_complete,
                     IncludeComplete* include_complete,
                     CodeCompleteCache* global_code_complete_cache,
                     CodeCompleteCache* non_global_code_complete_cache,
                     CodeCompleteCache* signature_cache) {
  IpcManager* ipc = IpcManager::instance();

  bool did_work = false;

  std::vector<std::unique_ptr<BaseIpcMessage>> messages =
      ipc->for_querydb.DequeueAll();
  for (auto& message : messages) {
    did_work = true;

    switch (message->method_id) {
      case IpcId::Initialize: {
        auto request = message->As<Ipc_InitializeRequest>();

        // Log initialization parameters.
        rapidjson::StringBuffer output;
        Writer writer(output);
        Reflect(writer, request->params.initializationOptions);
        LOG_S(INFO) << "Init parameters: " << output.GetString();

        if (request->params.rootUri) {
          std::string project_path = request->params.rootUri->GetPath();
          LOG_S(INFO) << "[querydb] Initialize in directory " << project_path
                      << " with uri " << request->params.rootUri->raw_uri;

          if (!request->params.initializationOptions) {
            LOG_S(FATAL) << "Initialization parameters (particularily "
                            "cacheDirectory) are required";
            exit(1);
          }

          *config = *request->params.initializationOptions;

          // Check client version.
          if (config->clientVersion != kExpectedClientVersion &&
              config->clientVersion != -1 /*disable check*/) {
            Out_ShowLogMessage out;
            out.display_type = Out_ShowLogMessage::DisplayType::Show;
            out.params.type = lsMessageType::Error;
            out.params.message =
                "cquery client (v" + std::to_string(config->clientVersion) +
                ") and server (v" + std::to_string(kExpectedClientVersion) +
                ") version mismatch. Please update ";
            if (config->clientVersion > kExpectedClientVersion)
              out.params.message += "the cquery binary.";
            else
              out.params.message +=
                  "your extension client (VSIX file). Make sure to uninstall "
                  "the cquery extension and restart vscode before "
                  "reinstalling.";
            out.Write(std::cout);
          }

          // Make sure cache directory is valid.
          if (config->cacheDirectory.empty()) {
            LOG_S(FATAL) << "Exiting; no cache directory";
            exit(1);
          }

          config->cacheDirectory = NormalizePath(config->cacheDirectory);
          EnsureEndsInSlash(config->cacheDirectory);

          // Ensure there is a resource directory.
          if (config->resourceDirectory.empty()) {
            config->resourceDirectory = GetWorkingDirectory();
#if defined(_WIN32)
            config->resourceDirectory +=
                std::string("../../clang_resource_dir/");
#else
            config->resourceDirectory += std::string("../clang_resource_dir/");
#endif
          }
          config->resourceDirectory = NormalizePath(config->resourceDirectory);
          LOG_S(INFO) << "Using -resource-dir=" << config->resourceDirectory;

          // Send initialization before starting indexers, so we don't send a
          // status update too early.
          // TODO: query request->params.capabilities.textDocument and support
          // only things the client supports.

          Out_InitializeResponse out;
          out.id = request->id;

          // out.result.capabilities.textDocumentSync =
          // lsTextDocumentSyncOptions();
          // out.result.capabilities.textDocumentSync->openClose = true;
          // out.result.capabilities.textDocumentSync->change =
          // lsTextDocumentSyncKind::Full;
          // out.result.capabilities.textDocumentSync->willSave = true;
          // out.result.capabilities.textDocumentSync->willSaveWaitUntil =
          // true;
          out.result.capabilities.textDocumentSync =
              lsTextDocumentSyncKind::Incremental;

          out.result.capabilities.renameProvider = true;

          out.result.capabilities.completionProvider = lsCompletionOptions();
          out.result.capabilities.completionProvider->resolveProvider = false;
          // vscode doesn't support trigger character sequences, so we use ':'
          // for
          // '::' and '>' for '->'. See
          // https://github.com/Microsoft/language-server-protocol/issues/138.
          out.result.capabilities.completionProvider->triggerCharacters = {
              ".", ":", ">", "#"};

          out.result.capabilities.signatureHelpProvider =
              lsSignatureHelpOptions();
          // NOTE: If updating signature help tokens make sure to also update
          // WorkingFile::FindClosestCallNameInBuffer.
          out.result.capabilities.signatureHelpProvider->triggerCharacters = {
              "(", ","};

          out.result.capabilities.codeLensProvider = lsCodeLensOptions();
          out.result.capabilities.codeLensProvider->resolveProvider = false;

          out.result.capabilities.definitionProvider = true;
          out.result.capabilities.documentHighlightProvider = true;
          out.result.capabilities.hoverProvider = true;
          out.result.capabilities.referencesProvider = true;

          out.result.capabilities.codeActionProvider = true;

          out.result.capabilities.documentSymbolProvider = true;
          out.result.capabilities.workspaceSymbolProvider = true;

          out.result.capabilities.documentLinkProvider =
              lsDocumentLinkOptions();
          out.result.capabilities.documentLinkProvider->resolveProvider = false;

          IpcManager::WriteStdout(IpcId::Initialize, out);

          // Set project root.
          config->projectRoot =
              NormalizePath(request->params.rootUri->GetPath());
          EnsureEndsInSlash(config->projectRoot);
          MakeDirectoryRecursive(config->cacheDirectory +
                                 EscapeFileName(config->projectRoot));

          // Start indexer threads.
          if (config->indexerCount == 0) {
            // If the user has not specified how many indexers to run, try to
            // guess an appropriate value. Default to 80% utilization.
            const float kDefaultTargetUtilization = 0.8;
            config->indexerCount =
                std::thread::hardware_concurrency() * kDefaultTargetUtilization;
            if (config->indexerCount <= 0)
              config->indexerCount = 1;
          }
          LOG_S(INFO) << "Starting " << config->indexerCount << " indexers";
          for (int i = 0; i < config->indexerCount; ++i) {
            WorkThread::StartThread("indexer" + std::to_string(i), [=]() {
              return IndexMain(config, file_consumer_shared, timestamp_manager,
                               import_manager, project, working_files, waiter,
                               queue);
            });
          }

          Timer time;

          // Open up / load the project.
          project->Load(config->extraClangArguments,
                        config->compilationDatabaseDirectory, project_path,
                        config->resourceDirectory);
          time.ResetAndPrint("[perf] Loaded compilation entries (" +
                             std::to_string(project->entries.size()) +
                             " files)");

          // Start scanning include directories before dispatching project
          // files, because that takes a long time.
          include_complete->Rescan();

          time.Reset();
          project->ForAllFilteredFiles(
              config, [&](int i, const Project::Entry& entry) {
                bool is_interactive =
                    working_files->GetFileByFilename(entry.filename) != nullptr;
                queue->index_request.Enqueue(Index_Request(
                    entry.filename, entry.args, is_interactive, nullopt));
              });

          // We need to support multiple concurrent index processes.
          time.ResetAndPrint("[perf] Dispatched initial index requests");
        }

        break;
      }

      case IpcId::Exit: {
        LOG_S(INFO) << "Exiting; got IpcId::Exit";
        exit(0);
        break;
      }

      case IpcId::CqueryFreshenIndex: {
        LOG_S(INFO) << "Freshening " << project->entries.size() << " files";

        // TODO: think about this flow and test it more.

        // Unmark all files whose timestamp has changed.
        CacheLoader cache_loader(config);
        for (const auto& file : db->files) {
          if (!file.def)
            continue;

          optional<int64_t> modification_timestamp =
              GetLastModificationTime(file.def->path);
          if (!modification_timestamp)
            continue;

          optional<int64_t> cached_modification =
              timestamp_manager->GetLastCachedModificationTime(&cache_loader,
                                                               file.def->path);
          if (modification_timestamp != cached_modification)
            file_consumer_shared->Reset(file.def->path);
        }

        // Send index requests for every file.
        project->ForAllFilteredFiles(
            config, [&](int i, const Project::Entry& entry) {
              LOG_S(INFO) << "[" << i << "/" << (project->entries.size() - 1)
                          << "] Dispatching index request for file "
                          << entry.filename;
              bool is_interactive =
                  working_files->GetFileByFilename(entry.filename) != nullptr;
              queue->index_request.Enqueue(Index_Request(
                  entry.filename, entry.args, is_interactive, nullopt));
            });
        break;
      }

      case IpcId::CqueryTypeHierarchyTree: {
        auto msg = message->As<Ipc_CqueryTypeHierarchyTree>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_CqueryTypeHierarchyTree out;
        out.id = msg->id;

        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          if (ref.idx.kind == SymbolKind::Type) {
            out.result = BuildInheritanceHierarchyForType(
                db, working_files, QueryTypeId(ref.idx.idx));
            break;
          }
          if (ref.idx.kind == SymbolKind::Func) {
            out.result = BuildInheritanceHierarchyForFunc(
                db, working_files, QueryFuncId(ref.idx.idx));
            break;
          }
        }

        IpcManager::WriteStdout(IpcId::CqueryTypeHierarchyTree, out);
        break;
      }

      case IpcId::CqueryCallTreeInitial: {
        auto msg = message->As<Ipc_CqueryCallTreeInitial>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_CqueryCallTree out;
        out.id = msg->id;

        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          if (ref.idx.kind == SymbolKind::Func) {
            out.result = BuildInitialCallTree(db, working_files,
                                              QueryFuncId(ref.idx.idx));
            break;
          }
        }

        IpcManager::WriteStdout(IpcId::CqueryCallTreeInitial, out);
        break;
      }

      case IpcId::CqueryCallTreeExpand: {
        auto msg = message->As<Ipc_CqueryCallTreeExpand>();

        Out_CqueryCallTree out;
        out.id = msg->id;

        auto func_id = db->usr_to_func.find(msg->params.usr);
        if (func_id != db->usr_to_func.end())
          out.result = BuildExpandCallTree(db, working_files, func_id->second);

        IpcManager::WriteStdout(IpcId::CqueryCallTreeExpand, out);
        break;
      }

      case IpcId::CqueryVars: {
        auto msg = message->As<Ipc_CqueryVars>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_LocationList out;
        out.id = msg->id;
        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          if (ref.idx.kind == SymbolKind::Type) {
            QueryType& type = db->types[ref.idx.idx];
            std::vector<QueryLocation> locations =
                ToQueryLocation(db, type.instances);
            out.result = GetLsLocations(db, working_files, locations);
          }
        }
        IpcManager::WriteStdout(IpcId::CqueryVars, out);
        break;
      }

      case IpcId::CqueryCallers: {
        auto msg = message->As<Ipc_CqueryCallers>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_LocationList out;
        out.id = msg->id;
        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          if (ref.idx.kind == SymbolKind::Func) {
            QueryFunc& func = db->funcs[ref.idx.idx];
            std::vector<QueryLocation> locations =
                ToQueryLocation(db, func.callers);
            for (QueryFuncRef func_ref :
                 GetCallersForAllBaseFunctions(db, func))
              locations.push_back(func_ref.loc);
            for (QueryFuncRef func_ref :
                 GetCallersForAllDerivedFunctions(db, func))
              locations.push_back(func_ref.loc);

            out.result = GetLsLocations(db, working_files, locations);
          }
        }
        IpcManager::WriteStdout(IpcId::CqueryCallers, out);
        break;
      }

      case IpcId::CqueryBase: {
        auto msg = message->As<Ipc_CqueryBase>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_LocationList out;
        out.id = msg->id;
        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          if (ref.idx.kind == SymbolKind::Type) {
            QueryType& type = db->types[ref.idx.idx];
            if (!type.def)
              continue;
            std::vector<QueryLocation> locations =
                ToQueryLocation(db, type.def->parents);
            out.result = GetLsLocations(db, working_files, locations);
          } else if (ref.idx.kind == SymbolKind::Func) {
            QueryFunc& func = db->funcs[ref.idx.idx];
            optional<QueryLocation> location =
                GetBaseDefinitionOrDeclarationSpelling(db, func);
            if (!location)
              continue;
            optional<lsLocation> ls_loc =
                GetLsLocation(db, working_files, *location);
            if (!ls_loc)
              continue;
            out.result.push_back(*ls_loc);
          }
        }
        IpcManager::WriteStdout(IpcId::CqueryBase, out);
        break;
      }

      case IpcId::CqueryDerived: {
        auto msg = message->As<Ipc_CqueryDerived>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_LocationList out;
        out.id = msg->id;
        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          if (ref.idx.kind == SymbolKind::Type) {
            QueryType& type = db->types[ref.idx.idx];
            std::vector<QueryLocation> locations =
                ToQueryLocation(db, type.derived);
            out.result = GetLsLocations(db, working_files, locations);
          } else if (ref.idx.kind == SymbolKind::Func) {
            QueryFunc& func = db->funcs[ref.idx.idx];
            std::vector<QueryLocation> locations =
                ToQueryLocation(db, func.derived);
            out.result = GetLsLocations(db, working_files, locations);
          }
        }
        IpcManager::WriteStdout(IpcId::CqueryDerived, out);
        break;
      }

      case IpcId::TextDocumentDidOpen: {
        // NOTE: This function blocks code lens. If it starts taking a long time
        // we will need to find a way to unblock the code lens request.

        Timer time;
        auto msg = message->As<Ipc_TextDocumentDidOpen>();
        std::string path = msg->params.textDocument.uri.GetPath();
        WorkingFile* working_file = working_files->OnOpen(msg->params);
        optional<std::string> cached_file_contents =
            LoadCachedFileContents(config, path);
        if (cached_file_contents)
          working_file->SetIndexContent(*cached_file_contents);
        else
          working_file->SetIndexContent(working_file->buffer_content);

        QueryFile* file = nullptr;
        FindFileOrFail(db, nullopt, path, &file);
        if (file && file->def) {
          EmitInactiveLines(working_file, file->def->inactive_regions);
          EmitSemanticHighlighting(db, semantic_cache, working_file, file);
        }

        time.ResetAndPrint(
            "[querydb] Loading cached index file for DidOpen (blocks "
            "CodeLens)");

        include_complete->AddFile(working_file->filename);
        clang_complete->NotifyView(path);

        // Submit new index request.
        const Project::Entry& entry =
            project->FindCompilationEntryForFile(path);
        queue->index_request.PriorityEnqueue(Index_Request(
            entry.filename, entry.args, true /*is_interactive*/, nullopt));

        break;
      }

      case IpcId::CqueryTextDocumentDidView: {
        auto msg = message->As<Ipc_CqueryTextDocumentDidView>();
        std::string path = msg->params.textDocumentUri.GetPath();

        WorkingFile* working_file = working_files->GetFileByFilename(path);
        if (!working_file)
          break;
        QueryFile* file = nullptr;
        if (!FindFileOrFail(db, nullopt, path, &file))
          break;

        clang_complete->NotifyView(path);
        if (file->def) {
          EmitInactiveLines(working_file, file->def->inactive_regions);
          EmitSemanticHighlighting(db, semantic_cache, working_file, file);
        }
        break;
      }

      case IpcId::TextDocumentDidChange: {
        auto msg = message->As<Ipc_TextDocumentDidChange>();
        std::string path = msg->params.textDocument.uri.GetPath();
        working_files->OnChange(msg->params);
        clang_complete->NotifyEdit(path);
        clang_complete->DiagnosticsUpdate(
            msg->params.textDocument.AsTextDocumentIdentifier());
        break;
      }

      case IpcId::TextDocumentDidClose: {
        auto msg = message->As<Ipc_TextDocumentDidClose>();
        std::string path = msg->params.textDocument.uri.GetPath();

        // Clear any diagnostics for the file.
        Out_TextDocumentPublishDiagnostics out;
        out.params.uri = msg->params.textDocument.uri;
        IpcManager::WriteStdout(IpcId::TextDocumentPublishDiagnostics, out);

        // Remove internal state.
        working_files->OnClose(msg->params);
        clang_complete->NotifyClose(path);

        break;
      }

      case IpcId::TextDocumentDidSave: {
        auto msg = message->As<Ipc_TextDocumentDidSave>();

        std::string path = msg->params.textDocument.uri.GetPath();
        // Send out an index request, and copy the current buffer state so we
        // can update the cached index contents when the index is done.
        //
        // We also do not index if there is already an index request.
        //
        // TODO: Cancel outgoing index request. Might be tricky to make
        //       efficient since we have to cancel.
        //    - we could have an |atomic<int> active_cancellations| variable
        //      that all of the indexers check before accepting an index. if
        //      zero we don't slow down fast-path. if non-zero we acquire
        //      mutex and check to see if we should skip the current request.
        //      if so, ignore that index response.
        // TODO: send as priority request
        Project::Entry entry = project->FindCompilationEntryForFile(path);
        queue->index_request.Enqueue(Index_Request(
            entry.filename, entry.args, true /*is_interactive*/, nullopt));

        clang_complete->NotifySave(path);

        break;
      }

      case IpcId::TextDocumentRename: {
        auto msg = message->As<Ipc_TextDocumentRename>();

        QueryFileId file_id;
        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file, &file_id))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_TextDocumentRename out;
        out.id = msg->id;

        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          // Found symbol. Return references to rename.
          std::vector<QueryLocation> uses = GetUsesOfSymbol(db, ref.idx);
          out.result =
              BuildWorkspaceEdit(db, working_files, uses, msg->params.newName);
          break;
        }

        IpcManager::WriteStdout(IpcId::TextDocumentRename, out);
        break;
      }

      case IpcId::TextDocumentCompletion: {
        auto msg = std::shared_ptr<Ipc_TextDocumentComplete>(
            static_cast<Ipc_TextDocumentComplete*>(message.release()));

        std::string path = msg->params.textDocument.uri.GetPath();
        WorkingFile* file = working_files->GetFileByFilename(path);

        // It shouldn't be possible, but sometimes vscode will send queries out
        // of order, ie, we get completion request before buffer content update.
        std::string buffer_line;
        if (msg->params.position.line >= 0 &&
            msg->params.position.line < file->all_buffer_lines.size())
          buffer_line = file->all_buffer_lines[msg->params.position.line];

        if (ShouldRunIncludeCompletion(buffer_line)) {
          Out_TextDocumentComplete out;
          out.id = msg->id;

          {
            std::unique_lock<std::mutex> lock(
                include_complete->completion_items_mutex, std::defer_lock);
            if (include_complete->is_scanning)
              lock.lock();
            out.result.items.assign(include_complete->completion_items.begin(),
                                    include_complete->completion_items.end());
            if (lock)
              lock.unlock();

            // Update textEdit params.
            for (lsCompletionItem& item : out.result.items) {
              item.textEdit->range.start.line = msg->params.position.line;
              item.textEdit->range.start.character = 0;
              item.textEdit->range.end.line = msg->params.position.line;
              item.textEdit->range.end.character = (int)buffer_line.size();
            }
          }

          FilterCompletionResponse(&out, buffer_line);
          IpcManager::WriteStdout(IpcId::TextDocumentCompletion, out);
        } else {
          bool is_global_completion = false;
          std::string existing_completion;
          if (file) {
            msg->params.position = file->FindStableCompletionSource(
                msg->params.position, &is_global_completion,
                &existing_completion);
          }

          ClangCompleteManager::OnComplete callback = std::bind(
              [global_code_complete_cache, non_global_code_complete_cache,
               is_global_completion, existing_completion,
               msg](const NonElidedVector<lsCompletionItem>& results,
                    bool is_cached_result) {
                Out_TextDocumentComplete out;
                out.id = msg->id;
                out.result.items = results;

                // Emit completion results.
                FilterCompletionResponse(&out, existing_completion);
                IpcManager::WriteStdout(IpcId::TextDocumentCompletion, out);

                // Cache completion results.
                if (!is_cached_result) {
                  std::string path = msg->params.textDocument.uri.GetPath();
                  if (is_global_completion) {
                    global_code_complete_cache->WithLock([&]() {
                      global_code_complete_cache->cached_path_ = path;
                      global_code_complete_cache->cached_results_ = results;
                    });
                  } else {
                    non_global_code_complete_cache->WithLock([&]() {
                      non_global_code_complete_cache->cached_path_ = path;
                      non_global_code_complete_cache
                          ->cached_completion_position_ = msg->params.position;
                      non_global_code_complete_cache->cached_results_ = results;
                    });
                  }
                }
              },
              std::placeholders::_1, std::placeholders::_2);

          bool is_cache_match = false;
          global_code_complete_cache->WithLock([&]() {
            is_cache_match =
                is_global_completion &&
                global_code_complete_cache->cached_path_ == path &&
                !global_code_complete_cache->cached_results_.empty();
          });
          if (is_cache_match) {
            ClangCompleteManager::OnComplete freshen_global =
                [global_code_complete_cache](
                    NonElidedVector<lsCompletionItem> results,
                    bool is_cached_result) {
                  assert(!is_cached_result);

                  // note: path is updated in the normal completion handler.
                  global_code_complete_cache->WithLock([&]() {
                    global_code_complete_cache->cached_results_ = results;
                  });
                };

            global_code_complete_cache->WithLock([&]() {
              callback(global_code_complete_cache->cached_results_,
                       true /*is_cached_result*/);
            });
            clang_complete->CodeComplete(msg->params, freshen_global);
          } else if (non_global_code_complete_cache->IsCacheValid(
                         msg->params)) {
            non_global_code_complete_cache->WithLock([&]() {
              callback(non_global_code_complete_cache->cached_results_,
                       true /*is_cached_result*/);
            });
          } else {
            clang_complete->CodeComplete(msg->params, callback);
          }
        }

        break;
      }

      case IpcId::TextDocumentSignatureHelp: {
        auto msg = message->As<Ipc_TextDocumentSignatureHelp>();
        lsTextDocumentPositionParams& params = msg->params;
        WorkingFile* file =
            working_files->GetFileByFilename(params.textDocument.uri.GetPath());
        std::string search;
        int active_param = 0;
        if (file) {
          lsPosition completion_position;
          search = file->FindClosestCallNameInBuffer(
              params.position, &active_param, &completion_position);
          params.position = completion_position;
        }
        if (search.empty())
          break;

        ClangCompleteManager::OnComplete callback = std::bind(
            [signature_cache](BaseIpcMessage* message, std::string search,
                              int active_param,
                              const NonElidedVector<lsCompletionItem>& results,
                              bool is_cached_result) {
              auto msg = message->As<Ipc_TextDocumentSignatureHelp>();
              auto ipc = IpcManager::instance();

              Out_TextDocumentSignatureHelp out;
              out.id = msg->id;

              for (auto& result : results) {
                if (result.label != search)
                  continue;

                lsSignatureInformation signature;
                signature.label = result.detail;
                for (auto& parameter : result.parameters_) {
                  lsParameterInformation ls_param;
                  ls_param.label = parameter;
                  signature.parameters.push_back(ls_param);
                }
                out.result.signatures.push_back(signature);
              }

              // Guess the signature the user wants based on available parameter
              // count.
              out.result.activeSignature = 0;
              for (size_t i = 0; i < out.result.signatures.size(); ++i) {
                if (active_param < out.result.signatures.size()) {
                  out.result.activeSignature = (int)i;
                  break;
                }
              }

              // Set signature to what we parsed from the working file.
              out.result.activeParameter = active_param;

              Timer timer;
              IpcManager::WriteStdout(IpcId::TextDocumentSignatureHelp, out);

              if (!is_cached_result) {
                signature_cache->WithLock([&]() {
                  signature_cache->cached_path_ =
                      msg->params.textDocument.uri.GetPath();
                  signature_cache->cached_completion_position_ =
                      msg->params.position;
                  signature_cache->cached_results_ = results;
                });
              }

              delete message;
            },
            message.release(), search, active_param, std::placeholders::_1,
            std::placeholders::_2);

        if (signature_cache->IsCacheValid(params)) {
          signature_cache->WithLock([&]() {
            callback(signature_cache->cached_results_,
                     true /*is_cached_result*/);
          });
        } else {
          clang_complete->CodeComplete(params, std::move(callback));
        }

        break;
      }

      case IpcId::TextDocumentDefinition: {
        auto msg = message->As<Ipc_TextDocumentDefinition>();

        QueryFileId file_id;
        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file, &file_id))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_TextDocumentDefinition out;
        out.id = msg->id;

        int target_line = msg->params.position.line + 1;
        int target_column = msg->params.position.character + 1;

        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          // Found symbol. Return definition.

          // Special cases which are handled:
          //  - symbol has declaration but no definition (ie, pure virtual)
          //  - start at spelling but end at extent for better mouse tooltip
          //  - goto declaration while in definition of recursive type

          optional<QueryLocation> def_loc =
              GetDefinitionSpellingOfSymbol(db, ref.idx);

          // We use spelling start and extent end because this causes vscode to
          // highlight the entire definition when previewing / hoving with the
          // mouse.
          optional<QueryLocation> def_extent =
              GetDefinitionExtentOfSymbol(db, ref.idx);
          if (def_loc && def_extent)
            def_loc->range.end = def_extent->range.end;

          // If the cursor is currently at or in the definition we should goto
          // the declaration if possible. We also want to use declarations if
          // we're pointing to, ie, a pure virtual function which has no
          // definition.
          if (!def_loc ||
              (def_loc->path == file_id &&
               def_loc->range.Contains(target_line, target_column))) {
            // Goto declaration.

            std::vector<QueryLocation> declarations =
                GetDeclarationsOfSymbolForGotoDefinition(db, ref.idx);
            for (auto declaration : declarations) {
              optional<lsLocation> ls_declaration =
                  GetLsLocation(db, working_files, declaration);
              if (ls_declaration)
                out.result.push_back(*ls_declaration);
            }
            // We found some declarations. Break so we don't add the definition
            // location.
            if (!out.result.empty())
              break;
          }

          if (def_loc) {
            PushBack(&out.result, GetLsLocation(db, working_files, *def_loc));
          }

          if (!out.result.empty())
            break;
        }

        // No symbols - check for includes.
        if (out.result.empty()) {
          for (const IndexInclude& include : file->def->includes) {
            if (include.line == target_line) {
              lsLocation result;
              result.uri = lsDocumentUri::FromPath(include.resolved_path);
              out.result.push_back(result);
              break;
            }
          }
        }

        IpcManager::WriteStdout(IpcId::TextDocumentDefinition, out);
        break;
      }

      case IpcId::TextDocumentDocumentHighlight: {
        auto msg = message->As<Ipc_TextDocumentDocumentHighlight>();

        QueryFileId file_id;
        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file, &file_id))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_TextDocumentDocumentHighlight out;
        out.id = msg->id;

        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          // Found symbol. Return references to highlight.
          std::vector<QueryLocation> uses = GetUsesOfSymbol(db, ref.idx);
          out.result.reserve(uses.size());
          for (const QueryLocation& use : uses) {
            if (use.path != file_id)
              continue;

            optional<lsLocation> ls_location =
                GetLsLocation(db, working_files, use);
            if (!ls_location)
              continue;

            lsDocumentHighlight highlight;
            highlight.kind = lsDocumentHighlightKind::Text;
            highlight.range = ls_location->range;
            out.result.push_back(highlight);
          }
          break;
        }

        IpcManager::WriteStdout(IpcId::TextDocumentDocumentHighlight, out);
        break;
      }

      case IpcId::TextDocumentHover: {
        auto msg = message->As<Ipc_TextDocumentHover>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_TextDocumentHover out;
        out.id = msg->id;

        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          // Found symbol. Return hover.
          optional<lsRange> ls_range = GetLsRange(
              working_files->GetFileByFilename(file->def->path), ref.loc.range);
          if (!ls_range)
            continue;

          out.result.contents.value = GetHoverForSymbol(db, ref.idx);
          out.result.contents.language = file->def->language;

          out.result.range = *ls_range;
          break;
        }

        IpcManager::WriteStdout(IpcId::TextDocumentHover, out);
        break;
      }

      case IpcId::TextDocumentReferences: {
        auto msg = message->As<Ipc_TextDocumentReferences>();

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        WorkingFile* working_file =
            working_files->GetFileByFilename(file->def->path);

        Out_TextDocumentReferences out;
        out.id = msg->id;

        for (const SymbolRef& ref :
             FindSymbolsAtLocation(working_file, file, msg->params.position)) {
          optional<QueryLocation> excluded_declaration;
          if (!msg->params.context.includeDeclaration) {
            LOG_S(INFO) << "Excluding declaration in references";
            excluded_declaration = GetDefinitionSpellingOfSymbol(db, ref.idx);
          }

          // Found symbol. Return references.
          std::vector<QueryLocation> uses = GetUsesOfSymbol(db, ref.idx);
          out.result.reserve(uses.size());
          for (const QueryLocation& use : uses) {
            if (excluded_declaration.has_value() &&
                use == *excluded_declaration)
              continue;

            optional<lsLocation> ls_location =
                GetLsLocation(db, working_files, use);
            if (ls_location)
              out.result.push_back(*ls_location);
          }
          break;
        }

        IpcManager::WriteStdout(IpcId::TextDocumentReferences, out);
        break;
      }

      case IpcId::TextDocumentDocumentSymbol: {
        auto msg = message->As<Ipc_TextDocumentDocumentSymbol>();

        Out_TextDocumentDocumentSymbol out;
        out.id = msg->id;

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        for (SymbolRef ref : file->def->outline) {
          optional<lsSymbolInformation> info =
              GetSymbolInfo(db, working_files, ref.idx);
          if (!info)
            continue;

          optional<lsLocation> location =
              GetLsLocation(db, working_files, ref.loc);
          if (!location)
            continue;
          info->location = *location;
          out.result.push_back(*info);
        }

        IpcManager::WriteStdout(IpcId::TextDocumentDocumentSymbol, out);
        break;
      }

      case IpcId::TextDocumentDocumentLink: {
        auto msg = message->As<Ipc_TextDocumentDocumentLink>();

        Out_TextDocumentDocumentLink out;
        out.id = msg->id;

        if (config->showDocumentLinksOnIncludes) {
          QueryFile* file;
          if (!FindFileOrFail(db, msg->id,
                              msg->params.textDocument.uri.GetPath(), &file))
            break;

          WorkingFile* working_file = working_files->GetFileByFilename(
              msg->params.textDocument.uri.GetPath());
          if (!working_file) {
            LOG_S(INFO) << "Unable to find working file "
                        << msg->params.textDocument.uri.GetPath();
            break;
          }
          for (const IndexInclude& include : file->def->includes) {
            optional<int> buffer_line;
            optional<std::string> buffer_line_content =
                working_file->GetBufferLineContentFromIndexLine(include.line,
                                                                &buffer_line);
            if (!buffer_line || !buffer_line_content)
              continue;

            // Subtract 1 from line because querydb stores 1-based lines but
            // vscode expects 0-based lines.
            optional<lsRange> between_quotes =
                ExtractQuotedRange(*buffer_line - 1, *buffer_line_content);
            if (!between_quotes)
              continue;

            lsDocumentLink link;
            link.target = lsDocumentUri::FromPath(include.resolved_path);
            link.range = *between_quotes;
            out.result.push_back(link);
          }
        }

        IpcManager::WriteStdout(IpcId::TextDocumentDocumentLink, out);
        break;
      }

      case IpcId::TextDocumentCodeAction: {
        // NOTE: This code snippet will generate some FixIts for testing:
        //
        //    struct origin { int x, int y };
        //    void foo() {
        //      point origin = {
        //        x: 0.0,
        //        y: 0.0
        //      };
        //    }
        //
        auto msg = message->As<Ipc_TextDocumentCodeAction>();

        QueryFileId file_id;
        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file, &file_id))
          break;

        WorkingFile* working_file = working_files->GetFileByFilename(
            msg->params.textDocument.uri.GetPath());
        if (!working_file) {
          // TODO: send error response.
          LOG_S(INFO)
              << "[error] textDocument/codeAction could not find working file";
          break;
        }

        Out_TextDocumentCodeAction out;
        out.id = msg->id;

        // TODO: auto-insert namespace?

        int default_line = (int)working_file->all_buffer_lines.size();

        // Make sure to call EnsureImplFile before using these. We lazy load
        // them because computing the values could involve an entire project
        // scan.
        optional<lsDocumentUri> impl_uri;
        optional<QueryFileId> impl_file_id;

        std::vector<SymbolRef> syms =
            FindSymbolsAtLocation(working_file, file, msg->params.range.start);
        for (SymbolRef sym : syms) {
          switch (sym.idx.kind) {
            case SymbolKind::Type: {
              QueryType& type = db->types[sym.idx.idx];
              if (!type.def)
                break;

              int num_edits = 0;

              // Get implementation file.
              Out_TextDocumentCodeAction::Command command;

              for (QueryFuncId func_id : type.def->funcs) {
                QueryFunc& func_def = db->funcs[func_id.id];
                if (!func_def.def || func_def.def->definition_extent)
                  continue;

                EnsureImplFile(db, file_id, impl_uri /*out*/,
                               impl_file_id /*out*/);
                optional<lsTextEdit> edit = BuildAutoImplementForFunction(
                    db, working_files, working_file, default_line, file_id,
                    *impl_file_id, func_def);
                if (!edit)
                  continue;

                ++num_edits;

                // Merge edits together if they are on the same line.
                // TODO: be smarter about newline merging? ie, don't end up
                //       with foo()\n\n\n\nfoo(), we want foo()\n\nfoo()\n\n
                //
                if (!command.arguments.edits.empty() &&
                    command.arguments.edits[command.arguments.edits.size() - 1]
                            .range.end.line == edit->range.start.line) {
                  command.arguments.edits[command.arguments.edits.size() - 1]
                      .newText += edit->newText;
                } else {
                  command.arguments.edits.push_back(*edit);
                }
              }
              if (command.arguments.edits.empty())
                break;

              // If we're inserting at the end of the document, put a newline
              // before the insertion.
              if (command.arguments.edits[0].range.start.line >= default_line)
                command.arguments.edits[0].newText.insert(0, "\n");

              command.arguments.textDocumentUri = *impl_uri;
              command.title = "Auto-Implement " + std::to_string(num_edits) +
                              " methods on " + type.def->short_name;
              command.command = "cquery._autoImplement";
              out.result.push_back(command);
              break;
            }

            case SymbolKind::Func: {
              QueryFunc& func = db->funcs[sym.idx.idx];
              if (!func.def || func.def->definition_extent)
                break;

              EnsureImplFile(db, file_id, impl_uri /*out*/,
                             impl_file_id /*out*/);

              // Get implementation file.
              Out_TextDocumentCodeAction::Command command;
              command.title = "Auto-Implement " + func.def->short_name;
              command.command = "cquery._autoImplement";
              command.arguments.textDocumentUri = *impl_uri;
              optional<lsTextEdit> edit = BuildAutoImplementForFunction(
                  db, working_files, working_file, default_line, file_id,
                  *impl_file_id, func);
              if (!edit)
                break;

              // If we're inserting at the end of the document, put a newline
              // before the insertion.
              if (edit->range.start.line >= default_line)
                edit->newText.insert(0, "\n");
              command.arguments.edits.push_back(*edit);
              out.result.push_back(command);
              break;
            }
            default:
              break;
          }

          // Only show one auto-impl section.
          if (!out.result.empty())
            break;
        }

        std::vector<lsDiagnostic> diagnostics;
        working_files->DoAction(
            [&]() { diagnostics = working_file->diagnostics_; });
        for (lsDiagnostic& diag : diagnostics) {
          if (diag.range.start.line != msg->params.range.start.line)
            continue;

          // For error diagnostics, provide an action to resolve an include.
          // TODO: find a way to index diagnostic contents so line numbers
          // don't get mismatched when actively editing a file.
          std::string include_query =
              LexWordAroundPos(diag.range.start, working_file->buffer_content);
          if (diag.severity == lsDiagnosticSeverity::Error &&
              !include_query.empty()) {
            const size_t kMaxResults = 20;

            std::unordered_set<std::string> include_absolute_paths;

            // Find include candidate strings.
            for (int i = 0; i < db->detailed_names.size(); ++i) {
              if (include_absolute_paths.size() > kMaxResults)
                break;
              if (db->detailed_names[i].find(include_query) ==
                  std::string::npos)
                continue;

              optional<QueryFileId> decl_file_id =
                  GetDeclarationFileForSymbol(db, db->symbols[i]);
              if (!decl_file_id)
                continue;

              QueryFile& decl_file = db->files[decl_file_id->id];
              if (!decl_file.def)
                continue;

              include_absolute_paths.insert(decl_file.def->path);
            }

            // Build include strings.
            std::unordered_set<std::string> include_insert_strings;
            include_insert_strings.reserve(include_absolute_paths.size());

            for (const std::string& path : include_absolute_paths) {
              optional<lsCompletionItem> item =
                  include_complete->FindCompletionItemForAbsolutePath(path);
              if (!item)
                continue;
              if (item->textEdit)
                include_insert_strings.insert(item->textEdit->newText);
              else if (!item->insertText.empty())
                include_insert_strings.insert(item->insertText);
              else
                assert(false &&
                       "unable to determine insert string for include "
                       "completion item");
            }

            // Build code action.
            if (!include_insert_strings.empty()) {
              Out_TextDocumentCodeAction::Command command;

              // Build edits.
              for (const std::string& include_insert_string :
                   include_insert_strings) {
                lsTextEdit edit;
                optional<int> include_line = FindIncludeLine(
                    working_file->all_buffer_lines, include_insert_string);
                if (!include_line)
                  continue;

                edit.range.start.line = *include_line;
                edit.range.end.line = *include_line;
                edit.newText = include_insert_string + "\n";
                command.arguments.edits.push_back(edit);
              }

              // Setup metadata and send to client.
              if (include_insert_strings.size() == 1)
                command.title = "Insert " + *include_insert_strings.begin();
              else
                command.title = "Pick one of " +
                                std::to_string(command.arguments.edits.size()) +
                                " includes to insert";
              command.command = "cquery._insertInclude";
              command.arguments.textDocumentUri = msg->params.textDocument.uri;
              out.result.push_back(command);
            }
          }

          // clang does not provide accurate enough column reporting for
          // diagnostics to do good column filtering, so report all
          // diagnostics on the line.
          if (!diag.fixits_.empty()) {
            Out_TextDocumentCodeAction::Command command;
            command.title = "FixIt: " + diag.message;
            command.command = "cquery._applyFixIt";
            command.arguments.textDocumentUri = msg->params.textDocument.uri;
            command.arguments.edits = diag.fixits_;
            out.result.push_back(command);
          }
        }

        IpcManager::WriteStdout(IpcId::TextDocumentCodeAction, out);
        break;
      }

      case IpcId::TextDocumentCodeLens: {
        auto msg = message->As<Ipc_TextDocumentCodeLens>();

        Out_TextDocumentCodeLens out;
        out.id = msg->id;

        lsDocumentUri file_as_uri = msg->params.textDocument.uri;
        std::string path = file_as_uri.GetPath();

        clang_complete->NotifyView(path);

        QueryFile* file;
        if (!FindFileOrFail(db, msg->id, msg->params.textDocument.uri.GetPath(),
                            &file))
          break;

        CommonCodeLensParams common;
        common.result = &out.result;
        common.db = db;
        common.working_files = working_files;
        common.working_file = working_files->GetFileByFilename(file->def->path);

        for (SymbolRef ref : file->def->outline) {
          // NOTE: We OffsetColumn so that the code lens always show up in a
          // predictable order. Otherwise, the client may randomize it.

          SymbolIdx symbol = ref.idx;
          switch (symbol.kind) {
            case SymbolKind::Type: {
              QueryType& type = db->types[symbol.idx];
              if (!type.def)
                continue;
              AddCodeLens("ref", "refs", &common, ref.loc.OffsetStartColumn(0),
                          type.uses, type.def->definition_spelling,
                          true /*force_display*/);
              AddCodeLens("derived", "derived", &common,
                          ref.loc.OffsetStartColumn(1),
                          ToQueryLocation(db, type.derived), nullopt,
                          false /*force_display*/);
              AddCodeLens("var", "vars", &common, ref.loc.OffsetStartColumn(2),
                          ToQueryLocation(db, type.instances), nullopt,
                          false /*force_display*/);
              break;
            }
            case SymbolKind::Func: {
              QueryFunc& func = db->funcs[symbol.idx];
              if (!func.def)
                continue;

              int16_t offset = 0;

              std::vector<QueryFuncRef> base_callers =
                  GetCallersForAllBaseFunctions(db, func);
              std::vector<QueryFuncRef> derived_callers =
                  GetCallersForAllDerivedFunctions(db, func);
              if (base_callers.empty() && derived_callers.empty()) {
                AddCodeLens("call", "calls", &common,
                            ref.loc.OffsetStartColumn(offset++),
                            ToQueryLocation(db, func.callers), nullopt,
                            true /*force_display*/);
              } else {
                AddCodeLens("direct call", "direct calls", &common,
                            ref.loc.OffsetStartColumn(offset++),
                            ToQueryLocation(db, func.callers), nullopt,
                            false /*force_display*/);
                if (!base_callers.empty())
                  AddCodeLens("base call", "base calls", &common,
                              ref.loc.OffsetStartColumn(offset++),
                              ToQueryLocation(db, base_callers), nullopt,
                              false /*force_display*/);
                if (!derived_callers.empty())
                  AddCodeLens("derived call", "derived calls", &common,
                              ref.loc.OffsetStartColumn(offset++),
                              ToQueryLocation(db, derived_callers), nullopt,
                              false /*force_display*/);
              }

              AddCodeLens("derived", "derived", &common,
                          ref.loc.OffsetStartColumn(offset++),
                          ToQueryLocation(db, func.derived), nullopt,
                          false /*force_display*/);

              // "Base"
              optional<QueryLocation> base_loc =
                  GetBaseDefinitionOrDeclarationSpelling(db, func);
              if (base_loc) {
                optional<lsLocation> ls_base =
                    GetLsLocation(db, working_files, *base_loc);
                if (ls_base) {
                  optional<lsRange> range =
                      GetLsRange(common.working_file, ref.loc.range);
                  if (range) {
                    TCodeLens code_lens;
                    code_lens.range = *range;
                    code_lens.range.start.character += offset++;
                    code_lens.command = lsCommand<lsCodeLensCommandArguments>();
                    code_lens.command->title = "Base";
                    code_lens.command->command = "cquery.goto";
                    code_lens.command->arguments.uri = ls_base->uri;
                    code_lens.command->arguments.position =
                        ls_base->range.start;
                    out.result.push_back(code_lens);
                  }
                }
              }

              break;
            }
            case SymbolKind::Var: {
              QueryVar& var = db->vars[symbol.idx];
              if (!var.def)
                continue;

              if (var.def->is_local && !config->codeLensOnLocalVariables)
                continue;

              bool force_display = true;
              // Do not show 0 refs on macro with no uses, as it is most likely
              // a header guard.
              if (var.def->is_macro)
                force_display = false;

              AddCodeLens("ref", "refs", &common, ref.loc.OffsetStartColumn(0),
                          var.uses, var.def->definition_spelling,
                          force_display);
              break;
            }
            case SymbolKind::File:
            case SymbolKind::Invalid: {
              assert(false && "unexpected");
              break;
            }
          };
        }

        IpcManager::WriteStdout(IpcId::TextDocumentCodeLens, out);

        break;
      }

      case IpcId::WorkspaceSymbol: {
        // TODO: implement fuzzy search, see
        // https://github.com/junegunn/fzf/blob/master/src/matcher.go for
        // inspiration
        auto msg = message->As<Ipc_WorkspaceSymbol>();

        Out_WorkspaceSymbol out;
        out.id = msg->id;

        LOG_S(INFO) << "[querydb] Considering " << db->detailed_names.size()
                    << " candidates for query " << msg->params.query;

        std::string query = msg->params.query;

        std::unordered_set<std::string> inserted_results;
        inserted_results.reserve(config->maxWorkspaceSearchResults);

        for (int i = 0; i < db->detailed_names.size(); ++i) {
          if (db->detailed_names[i].find(query) != std::string::npos) {
            // Do not show the same entry twice.
            if (!inserted_results.insert(db->detailed_names[i]).second)
              continue;

            InsertSymbolIntoResult(db, working_files, db->symbols[i],
                                   &out.result);
            if (out.result.size() >= config->maxWorkspaceSearchResults)
              break;
          }
        }

        if (out.result.size() < config->maxWorkspaceSearchResults) {
          for (int i = 0; i < db->detailed_names.size(); ++i) {
            if (SubstringMatch(query, db->detailed_names[i])) {
              // Do not show the same entry twice.
              if (!inserted_results.insert(db->detailed_names[i]).second)
                continue;

              InsertSymbolIntoResult(db, working_files, db->symbols[i],
                                     &out.result);
              if (out.result.size() >= config->maxWorkspaceSearchResults)
                break;
            }
          }
        }

        LOG_S(INFO) << "[querydb] Found " << out.result.size()
                    << " results for query " << query;
        IpcManager::WriteStdout(IpcId::WorkspaceSymbol, out);
        break;
      }

      case IpcId::CqueryIndexFile: {
        auto msg = message->As<Ipc_CqueryIndexFile>();
        queue->index_request.Enqueue(
            Index_Request(NormalizePath(msg->params.path), msg->params.args,
                          msg->params.is_interactive, msg->params.contents));
        break;
      }

      case IpcId::CqueryQueryDbWaitForIdleIndexer: {
        LOG_S(INFO) << "Waiting for idle";
        int idle_count = 0;
        while (true) {
          bool has_work = false;
          has_work |= import_manager->HasActiveQuerydbImports();
          has_work |= queue->HasWork();
          has_work |= QueryDb_ImportMain(config, db, import_manager, queue,
                                         semantic_cache, working_files);
          if (!has_work)
            ++idle_count;
          else
            idle_count = 0;

          // There are race conditions between each of the three checks above,
          // so we retry a bunch of times to try to avoid any.
          if (idle_count > 10)
            break;
        }
        LOG_S(INFO) << "Done waiting for idle";
        break;
      }

      case IpcId::CqueryExitWhenIdle: {
        *exit_when_idle = true;
        WorkThread::request_exit_on_idle = true;
        break;
      }

      default: {
        LOG_S(FATAL) << "Exiting; unhandled IPC message "
                     << IpcIdToString(message->method_id);
        exit(1);
      }
    }
  }

  // TODO: consider rate-limiting and checking for IPC messages so we don't
  // block requests / we can serve partial requests.

  if (QueryDb_ImportMain(config, db, import_manager, queue, semantic_cache,
                         working_files)) {
    did_work = true;
  }

  return did_work;
}

void RunQueryDbThread(const std::string& bin_name,
                      Config* config,
                      MultiQueueWaiter* waiter,
                      QueueManager* queue) {
  bool exit_when_idle = false;
  Project project;
  SemanticHighlightSymbolCache semantic_cache;
  WorkingFiles working_files;
  FileConsumer::SharedState file_consumer_shared;

  ClangCompleteManager clang_complete(
      config, &project, &working_files,
      std::bind(&EmitDiagnostics, &working_files, std::placeholders::_1,
                std::placeholders::_2),
      std::bind(&IndexWithTuFromCodeCompletion, queue, &file_consumer_shared,
                std::placeholders::_1, std::placeholders::_2,
                std::placeholders::_3, std::placeholders::_4));

  IncludeComplete include_complete(config, &project);
  auto global_code_complete_cache = MakeUnique<CodeCompleteCache>();
  auto non_global_code_complete_cache = MakeUnique<CodeCompleteCache>();
  auto signature_cache = MakeUnique<CodeCompleteCache>();
  ImportManager import_manager;
  TimestampManager timestamp_manager;

  // Run query db main loop.
  SetCurrentThreadName("querydb");
  QueryDatabase db;
  while (true) {
    bool did_work = QueryDbMainLoop(
        config, &db, &exit_when_idle, waiter, queue, &project,
        &file_consumer_shared, &import_manager, &timestamp_manager,
        &semantic_cache, &working_files, &clang_complete, &include_complete,
        global_code_complete_cache.get(), non_global_code_complete_cache.get(),
        signature_cache.get());

    // No more work left and exit request. Exit.
    if (!did_work && exit_when_idle && WorkThread::num_active_threads == 0) {
      LOG_S(INFO) << "Exiting; exit_when_idle is set and there is no more work";
      exit(0);
    }

    // Cleanup and free any unused memory.
    FreeUnusedMemory();

    if (!did_work) {
      waiter->Wait({&IpcManager::instance()->for_querydb, &queue->do_id_map,
                    &queue->on_indexed});
    }
  }
}

////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// STDIN MAIN //////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// Separate thread whose only job is to read from stdin and
// dispatch read commands to the actual indexer program. This
// cannot be done on the main thread because reading from std::cin
// blocks.
//
// |ipc| is connected to a server.
void LaunchStdinLoop(Config* config,
                     std::unordered_map<IpcId, Timer>* request_times) {
  WorkThread::StartThread("stdin", [request_times]() {
    IpcManager* ipc = IpcManager::instance();

    std::unique_ptr<BaseIpcMessage> message =
        MessageRegistry::instance()->ReadMessageFromStdin(
            g_log_stdin_stdout_to_stderr);

    // Message parsing can fail if we don't recognize the method.
    if (!message)
      return WorkThread::Result::MoreWork;

    (*request_times)[message->method_id] = Timer();

    switch (message->method_id) {
      case IpcId::Initialized: {
        // TODO: don't send output until we get this notification
        break;
      }

      case IpcId::CancelRequest: {
        // TODO: support cancellation
        break;
      }

      case IpcId::Exit: {
        LOG_S(INFO) << "Exiting";
        exit(0);
        break;
      }

      case IpcId::CqueryExitWhenIdle: {
        // querydb needs to know to exit when idle. We return out of the stdin
        // loop to exit the thread. If we keep parsing input stdin is likely
        // closed so cquery will exit.
        LOG_S(INFO) << "cquery will exit when all threads are idle";
        ipc->for_querydb.Enqueue(std::move(message));
        return WorkThread::Result::ExitThread;
      }

      case IpcId::Initialize:
      case IpcId::TextDocumentDidOpen:
      case IpcId::CqueryTextDocumentDidView:
      case IpcId::TextDocumentDidChange:
      case IpcId::TextDocumentDidClose:
      case IpcId::TextDocumentDidSave:
      case IpcId::TextDocumentRename:
      case IpcId::TextDocumentCompletion:
      case IpcId::TextDocumentSignatureHelp:
      case IpcId::TextDocumentDefinition:
      case IpcId::TextDocumentDocumentHighlight:
      case IpcId::TextDocumentHover:
      case IpcId::TextDocumentReferences:
      case IpcId::TextDocumentDocumentSymbol:
      case IpcId::TextDocumentDocumentLink:
      case IpcId::TextDocumentCodeAction:
      case IpcId::TextDocumentCodeLens:
      case IpcId::WorkspaceSymbol:
      case IpcId::CqueryFreshenIndex:
      case IpcId::CqueryTypeHierarchyTree:
      case IpcId::CqueryCallTreeInitial:
      case IpcId::CqueryCallTreeExpand:
      case IpcId::CqueryVars:
      case IpcId::CqueryCallers:
      case IpcId::CqueryBase:
      case IpcId::CqueryDerived:
      case IpcId::CqueryIndexFile:
      case IpcId::CqueryQueryDbWaitForIdleIndexer: {
        ipc->for_querydb.Enqueue(std::move(message));
        break;
      }

      default: {
        LOG_S(ERROR) << "Unhandled IPC message "
                     << IpcIdToString(message->method_id);
        exit(1);
      }
    }

    return WorkThread::Result::MoreWork;
  });
}

void LaunchStdoutThread(std::unordered_map<IpcId, Timer>* request_times,
                        MultiQueueWaiter* waiter,
                        QueueManager* queue) {
  WorkThread::StartThread("stdout", [=]() {
    IpcManager* ipc = IpcManager::instance();

    std::vector<IpcManager::StdoutMessage> messages =
        ipc->for_stdout.DequeueAll();
    if (messages.empty()) {
      waiter->Wait({&ipc->for_stdout});
      return queue->HasWork() ? WorkThread::Result::MoreWork
                              : WorkThread::Result::NoWork;
    }

    for (auto& message : messages) {
      if (ShouldDisplayIpcTiming(message.id)) {
        Timer time = (*request_times)[message.id];
        time.ResetAndPrint("[e2e] Running " +
                           std::string(IpcIdToString(message.id)));
      }

      if (g_log_stdin_stdout_to_stderr) {
        std::ostringstream sstream;
        sstream << "[COUT] |";
        sstream << message.content;
        sstream << "|\n";
        std::cerr << sstream.str();
        std::cerr.flush();
      }

      std::cout << message.content;
      std::cout.flush();
    }

    return WorkThread::Result::MoreWork;
  });
}

void LanguageServerMain(const std::string& bin_name,
                        Config* config,
                        MultiQueueWaiter* waiter) {
  QueueManager queue(waiter);
  std::unordered_map<IpcId, Timer> request_times;

  std::cin.tie(NULL);
  LaunchStdinLoop(config, &request_times);

  // We run a dedicated thread for writing to stdout because there can be an
  // unknown number of delays when output information.
  LaunchStdoutThread(&request_times, waiter, &queue);

  // Start querydb which takes over this thread. The querydb will launch
  // indexer threads as needed.
  RunQueryDbThread(bin_name, config, waiter, &queue);
}

////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
// MAIN ////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
int main(int argc, char** argv) {
  loguru::init(argc, argv);
  loguru::add_file("cquery_diagnostics.log", loguru::Truncate,
                   loguru::Verbosity_MAX);
  loguru::g_flush_interval_ms = 0;
  loguru::g_stderr_verbosity = 1;

  MultiQueueWaiter waiter;
  IpcManager::CreateInstance(&waiter);

  // bool loop = true;
  // while (loop)
  //  std::this_thread::sleep_for(std::chrono::milliseconds(10));
  // std::this_thread::sleep_for(std::chrono::seconds(10));

  PlatformInit();
  IndexInit();

  RegisterMessageTypes();

  std::unordered_map<std::string, std::string> options =
      ParseOptions(argc, argv);

  bool print_help = true;

  if (HasOption(options, "--clang-sanity-check")) {
    print_help = false;
    ClangSanityCheck();
  }

  if (HasOption(options, "--log-stdin-stdout-to-stderr"))
    g_log_stdin_stdout_to_stderr = true;

  if (HasOption(options, "--test-unit")) {
    print_help = false;
    doctest::Context context;
    context.applyCommandLine(argc, argv);
    int res = context.run();
    if (context.shouldExit())
      return res;
  }

  if (HasOption(options, "--test-index")) {
    print_help = false;
    RunIndexTests();
#if defined(_WIN32)
    std::cerr << std::endl << "[Enter] to exit" << std::endl;
    std::cin.get();
#endif
  }

  if (HasOption(options, "--language-server")) {
    print_help = false;
    // std::cerr << "Running language server" << std::endl;
    auto config = MakeUnique<Config>();
    LanguageServerMain(argv[0], config.get(), &waiter);
    return 0;
  }

  if (print_help) {
    std::cout << R"help(cquery help:

  cquery is a low-latency C++ language server.

  General:
    --help        Print this help information.
    --language-server
                  Run as a language server. This implements the language
                  server spec over STDIN and STDOUT.
    --test-unit   Run unit tests.
    --test-index  Run index tests.
    --log-stdin-stdout-to-stderr
                  Print stdin and stdout messages to stderr. This is a aid for
                  developing new language clients, as it makes it easier to
                  figure out how the client is interacting with cquery.
    --clang-sanity-check
                  Run a simple index test. Verifies basic clang functionality.
                  Needs to be executed from the cquery root checkout directory.

  Configuration:
    When opening up a directory, cquery will look for a compile_commands.json
    file emitted by your preferred build system. If not present, cquery will
    use a recursive directory listing instead. Command line flags can be
    provided by adding a file named `.cquery` in the top-level directory. Each
    line in that file is a separate argument.

    There are also a number of configuration options available when
    initializing the language server - your editor should have tooling to
    describe those options. See |Config| in this source code for a detailed
    list of all currently supported options.
)help";
  }

  return 0;
}

TEST_SUITE("LexFunctionDeclaration") {
  TEST_CASE("simple") {
    std::string buffer_content = " void Foo(); ";
    lsPosition declaration = CharPos(buffer_content, 'F');
    std::string insert_text;
    int newlines_after_name = 0;

    LexFunctionDeclaration(buffer_content, declaration, nullopt, &insert_text,
                           &newlines_after_name);
    REQUIRE(insert_text == "void Foo() {\n}");
    REQUIRE(newlines_after_name == 0);

    LexFunctionDeclaration(buffer_content, declaration, std::string("Type"),
                           &insert_text, &newlines_after_name);
    REQUIRE(insert_text == "void Type::Foo() {\n}");
    REQUIRE(newlines_after_name == 0);
  }

  TEST_CASE("ctor") {
    std::string buffer_content = " Foo(); ";
    lsPosition declaration = CharPos(buffer_content, 'F');
    std::string insert_text;
    int newlines_after_name = 0;

    LexFunctionDeclaration(buffer_content, declaration, std::string("Foo"),
                           &insert_text, &newlines_after_name);
    REQUIRE(insert_text == "Foo::Foo() {\n}");
    REQUIRE(newlines_after_name == 0);
  }

  TEST_CASE("dtor") {
    std::string buffer_content = " ~Foo(); ";
    lsPosition declaration = CharPos(buffer_content, '~');
    std::string insert_text;
    int newlines_after_name = 0;

    LexFunctionDeclaration(buffer_content, declaration, std::string("Foo"),
                           &insert_text, &newlines_after_name);
    REQUIRE(insert_text == "Foo::~Foo() {\n}");
    REQUIRE(newlines_after_name == 0);
  }

  TEST_CASE("complex return type") {
    std::string buffer_content = " std::vector<int> Foo(); ";
    lsPosition declaration = CharPos(buffer_content, 'F');
    std::string insert_text;
    int newlines_after_name = 0;

    LexFunctionDeclaration(buffer_content, declaration, nullopt, &insert_text,
                           &newlines_after_name);
    REQUIRE(insert_text == "std::vector<int> Foo() {\n}");
    REQUIRE(newlines_after_name == 0);

    LexFunctionDeclaration(buffer_content, declaration, std::string("Type"),
                           &insert_text, &newlines_after_name);
    REQUIRE(insert_text == "std::vector<int> Type::Foo() {\n}");
    REQUIRE(newlines_after_name == 0);
  }

  TEST_CASE("extra complex return type") {
    std::string buffer_content = " std::function < int() > \n Foo(); ";
    lsPosition declaration = CharPos(buffer_content, 'F');
    std::string insert_text;
    int newlines_after_name = 0;

    LexFunctionDeclaration(buffer_content, declaration, nullopt, &insert_text,
                           &newlines_after_name);
    REQUIRE(insert_text == "std::function < int() > \n Foo() {\n}");
    REQUIRE(newlines_after_name == 0);

    LexFunctionDeclaration(buffer_content, declaration, std::string("Type"),
                           &insert_text, &newlines_after_name);
    REQUIRE(insert_text == "std::function < int() > \n Type::Foo() {\n}");
    REQUIRE(newlines_after_name == 0);
  }

  TEST_CASE("parameters") {
    std::string buffer_content = "void Foo(int a,\n\n    int b); ";
    lsPosition declaration = CharPos(buffer_content, 'F');
    std::string insert_text;
    int newlines_after_name = 0;

    LexFunctionDeclaration(buffer_content, declaration, nullopt, &insert_text,
                           &newlines_after_name);
    REQUIRE(insert_text == "void Foo(int a,\n\n    int b) {\n}");
    REQUIRE(newlines_after_name == 2);

    LexFunctionDeclaration(buffer_content, declaration, std::string("Type"),
                           &insert_text, &newlines_after_name);
    REQUIRE(insert_text == "void Type::Foo(int a,\n\n    int b) {\n}");
    REQUIRE(newlines_after_name == 2);
  }
}

TEST_SUITE("LexWordAroundPos") {
  TEST_CASE("edges") {
    std::string content = "Foobar";
    REQUIRE(LexWordAroundPos(CharPos(content, 'F'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'o'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'b'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'a'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'r'), content) == "Foobar");
  }

  TEST_CASE("simple") {
    std::string content = "  Foobar  ";
    REQUIRE(LexWordAroundPos(CharPos(content, 'F'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'o'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'b'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'a'), content) == "Foobar");
    REQUIRE(LexWordAroundPos(CharPos(content, 'r'), content) == "Foobar");
  }

  TEST_CASE("underscores and numbers") {
    std::string content = "  _my_t5ype7  ";
    REQUIRE(LexWordAroundPos(CharPos(content, '_'), content) == "_my_t5ype7");
    REQUIRE(LexWordAroundPos(CharPos(content, '5'), content) == "_my_t5ype7");
    REQUIRE(LexWordAroundPos(CharPos(content, 'e'), content) == "_my_t5ype7");
    REQUIRE(LexWordAroundPos(CharPos(content, '7'), content) == "_my_t5ype7");
  }

  TEST_CASE("dot, dash, colon are skipped") {
    std::string content = "1. 2- 3:";
    REQUIRE(LexWordAroundPos(CharPos(content, '1'), content) == "1");
    REQUIRE(LexWordAroundPos(CharPos(content, '2'), content) == "2");
    REQUIRE(LexWordAroundPos(CharPos(content, '3'), content) == "3");
  }
}

TEST_SUITE("FindIncludeLine") {
  TEST_CASE("in document") {
    std::vector<std::string> lines = {
        "#include <bbb>",  // 0
        "#include <ddd>"   // 1
    };

    REQUIRE(FindIncludeLine(lines, "#include <bbb>") == nullopt);
  }

  TEST_CASE("insert before") {
    std::vector<std::string> lines = {
        "#include <bbb>",  // 0
        "#include <ddd>"   // 1
    };

    REQUIRE(FindIncludeLine(lines, "#include <aaa>") == 0);
  }

  TEST_CASE("insert middle") {
    std::vector<std::string> lines = {
        "#include <bbb>",  // 0
        "#include <ddd>"   // 1
    };

    REQUIRE(FindIncludeLine(lines, "#include <ccc>") == 1);
  }

  TEST_CASE("insert after") {
    std::vector<std::string> lines = {
        "#include <bbb>",  // 0
        "#include <ddd>",  // 1
        "",                // 2
    };

    REQUIRE(FindIncludeLine(lines, "#include <eee>") == 2);
  }

  TEST_CASE("ignore header") {
    std::vector<std::string> lines = {
        "// FOOBAR",       // 0
        "// FOOBAR",       // 1
        "// FOOBAR",       // 2
        "// FOOBAR",       // 3
        "",                // 4
        "#include <bbb>",  // 5
        "#include <ddd>",  // 6
        "",                // 7
    };

    REQUIRE(FindIncludeLine(lines, "#include <a>") == 5);
    REQUIRE(FindIncludeLine(lines, "#include <c>") == 6);
    REQUIRE(FindIncludeLine(lines, "#include <e>") == 7);
  }
}