Introduce MessageHandler abstraction. Mainly just code reorg.

Only the initialize request uses it so far, but this will enable pulling
quite a bit of code out of command_line.cc.

src/cache_loader.cc

@@ -0,0 +1,36 @@
+#include "cache_loader.h"
+
+#include "cache.h"
+#include "indexer.h"
+
+CacheLoader::CacheLoader(Config* config) : config_(config) {}
+
+IndexFile* CacheLoader::TryLoad(const std::string& path) {
+  auto it = caches.find(path);
+  if (it != caches.end())
+    return it->second.get();
+
+  std::unique_ptr<IndexFile> cache = LoadCachedIndex(config_, path);
+  if (!cache)
+    return nullptr;
+
+  caches[path] = std::move(cache);
+  return caches[path].get();
+}
+
+std::unique_ptr<IndexFile> CacheLoader::TryTakeOrLoad(const std::string& path) {
+  auto it = caches.find(path);
+  if (it != caches.end()) {
+    auto result = std::move(it->second);
+    caches.erase(it);
+    return result;
+  }
+
+  return LoadCachedIndex(config_, path);
+}
+
+std::unique_ptr<IndexFile> CacheLoader::TakeOrLoad(const std::string& path) {
+  auto result = TryTakeOrLoad(path);
+  assert(result);
+  return result;
+}

src/cache_loader.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "config.h"
+
+#include <unordered_map>
+
+// Manages loading caches from file paths for the indexer process.
+struct CacheLoader {
+  explicit CacheLoader(Config* config);
+
+  IndexFile* TryLoad(const std::string& path);
+
+  // Takes the existing cache or loads the cache at |path|. May return nullptr
+  // if the cache does not exist.
+  std::unique_ptr<IndexFile> TryTakeOrLoad(const std::string& path);
+
+  // Takes the existing cache or loads the cache at |path|. Asserts the cache
+  // exists.
+  std::unique_ptr<IndexFile> TakeOrLoad(const std::string& path);
+
+  std::unordered_map<std::string, std::unique_ptr<IndexFile>> caches;
+  Config* config_;
+};

src/clang_complete.h

@@ -1,3 +1,5 @@
+#pragma once
+
 #include "atomic_object.h"
 #include "clang_index.h"
 #include "clang_translation_unit.h"

src/code_complete_cache.cc

@@ -0,0 +1,12 @@
+#include "code_complete_cache.h"
+
+void CodeCompleteCache::WithLock(std::function<void()> action) {
+  std::lock_guard<std::mutex> lock(mutex_);
+  action();
+}
+
+bool CodeCompleteCache::IsCacheValid(lsTextDocumentPositionParams position) {
+  std::lock_guard<std::mutex> lock(mutex_);
+  return cached_path_ == position.textDocument.uri.GetPath() &&
+         cached_completion_position_ == position.position;
+}

src/code_complete_cache.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "language_server_api.h"
+
+#include <optional.h>
+
+#include <mutex>
+
+using namespace std::experimental;
+using std::experimental::nullopt;
+
+// Cached completion information, so we can give fast completion results when
+// the user erases a character. vscode will resend the completion request if
+// that happens.
+struct CodeCompleteCache {
+  // NOTE: Make sure to access these variables under |WithLock|.
+  optional<std::string> cached_path_;
+  optional<lsPosition> cached_completion_position_;
+  NonElidedVector<lsCompletionItem> cached_results_;
+
+  std::mutex mutex_;
+
+  void WithLock(std::function<void()> action);
+  bool IsCacheValid(lsTextDocumentPositionParams position);
+};

src/command_line.cc

@@ -1,5 +1,6 @@
 // TODO: cleanup includes
 #include "cache.h"
+#include "cache_loader.h"
 #include "clang_complete.h"
 #include "file_consumer.h"
 #include "include_complete.h"
@@ -9,6 +10,7 @@
 #include "lex_utils.h"
 #include "lru_cache.h"
 #include "match.h"
+#include "message_handler.h"
 #include "options.h"
 #include "platform.h"
 #include "project.h"
@@ -19,6 +21,7 @@
 #include "test.h"
 #include "threaded_queue.h"
 #include "timer.h"
+#include "timestamp_manager.h"
 #include "work_thread.h"
 #include "working_files.h"
 
@@ -50,35 +53,9 @@ namespace {
 
 std::vector<std::string> kEmptyArgs;
 
-// Expected client version. We show an error if this doesn't match.
-const int kExpectedClientVersion = 3;
-
 // If true stdout will be printed to stderr.
 bool g_log_stdin_stdout_to_stderr = false;
 
-// Cached completion information, so we can give fast completion results when
-// the user erases a character. vscode will resend the completion request if
-// that happens.
-struct CodeCompleteCache {
-  // NOTE: Make sure to access these variables under |WithLock|.
-  optional<std::string> cached_path_;
-  optional<lsPosition> cached_completion_position_;
-  NonElidedVector<lsCompletionItem> cached_results_;
-
-  std::mutex mutex_;
-
-  void WithLock(std::function<void()> action) {
-    std::lock_guard<std::mutex> lock(mutex_);
-    action();
-  }
-
-  bool IsCacheValid(lsTextDocumentPositionParams position) {
-    std::lock_guard<std::mutex> lock(mutex_);
-    return cached_path_ == position.textDocument.uri.GetPath() &&
-           cached_completion_position_ == position.position;
-  }
-};
-
 // This function returns true if e2e timing should be displayed for the given
 // IpcId.
 bool ShouldDisplayIpcTiming(IpcId id) {
@@ -604,103 +581,8 @@ void FilterCompletionResponse(Out_TextDocumentComplete* complete_response,
   }
 }
 
-struct Index_Request {
-  std::string path;
-  // TODO: make |args| a string that is parsed lazily.
-  std::vector<std::string> args;
-  bool is_interactive;
-  optional<std::string> contents;  // Preloaded contents. Useful for tests.
-
-  Index_Request(const std::string& path,
-                const std::vector<std::string>& args,
-                bool is_interactive,
-                optional<std::string> contents)
-      : path(path),
-        args(args),
-        is_interactive(is_interactive),
-        contents(contents) {}
-};
-
-struct Index_DoIdMap {
-  std::unique_ptr<IndexFile> current;
-  std::unique_ptr<IndexFile> previous;
-
-  PerformanceImportFile perf;
-  bool is_interactive = false;
-  bool write_to_disk = false;
-  bool load_previous = false;
-
-  Index_DoIdMap(std::unique_ptr<IndexFile> current,
-                PerformanceImportFile perf,
-                bool is_interactive,
-                bool write_to_disk)
-      : current(std::move(current)),
-        perf(perf),
-        is_interactive(is_interactive),
-        write_to_disk(write_to_disk) {
-    assert(this->current);
-  }
-};
-
-struct Index_OnIdMapped {
-  struct File {
-    std::unique_ptr<IndexFile> file;
-    std::unique_ptr<IdMap> ids;
-
-    File(std::unique_ptr<IndexFile> file, std::unique_ptr<IdMap> ids)
-        : file(std::move(file)), ids(std::move(ids)) {}
-  };
-
-  std::unique_ptr<File> previous;
-  std::unique_ptr<File> current;
-
-  PerformanceImportFile perf;
-  bool is_interactive;
-  bool write_to_disk;
-
-  Index_OnIdMapped(PerformanceImportFile perf,
-                   bool is_interactive,
-                   bool write_to_disk)
-      : perf(perf),
-        is_interactive(is_interactive),
-        write_to_disk(write_to_disk) {}
-};
-
-struct Index_OnIndexed {
-  IndexUpdate update;
-  PerformanceImportFile perf;
-
-  Index_OnIndexed(IndexUpdate& update, PerformanceImportFile perf)
-      : update(update), perf(perf) {}
-};
-
-struct QueueManager {
-  using Index_RequestQueue = ThreadedQueue<Index_Request>;
-  using Index_DoIdMapQueue = ThreadedQueue<Index_DoIdMap>;
-  using Index_OnIdMappedQueue = ThreadedQueue<Index_OnIdMapped>;
-  using Index_OnIndexedQueue = ThreadedQueue<Index_OnIndexed>;
-
-  Index_RequestQueue index_request;
-  Index_DoIdMapQueue do_id_map;
-  Index_DoIdMapQueue load_previous_index;
-  Index_OnIdMappedQueue on_id_mapped;
-  Index_OnIndexedQueue on_indexed;
-
-  QueueManager(MultiQueueWaiter* waiter)
-      : index_request(waiter),
-        do_id_map(waiter),
-        load_previous_index(waiter),
-        on_id_mapped(waiter),
-        on_indexed(waiter) {}
-
-  bool HasWork() {
-    return !index_request.IsEmpty() || !do_id_map.IsEmpty() ||
-           !load_previous_index.IsEmpty() || !on_id_mapped.IsEmpty() ||
-           !on_indexed.IsEmpty();
-  }
-};
-
 void RegisterMessageTypes() {
+  // TODO: use automatic registration similar to MessageHandler.
   MessageRegistry::instance()->Register<Ipc_CancelRequest>();
   MessageRegistry::instance()->Register<Ipc_InitializeRequest>();
   MessageRegistry::instance()->Register<Ipc_InitializedNotification>();
@@ -736,136 +618,6 @@ void RegisterMessageTypes() {
   MessageRegistry::instance()->Register<Ipc_CqueryExitWhenIdle>();
 }
 
-// Manages files inside of the indexing pipeline so we don't have the same file
-// being imported multiple times.
-//
-// NOTE: This is not thread safe and should only be used on the querydb thread.
-struct ImportManager {
-  // Try to mark the given dependency as imported. A dependency can only ever be
-  // imported once.
-  bool TryMarkDependencyImported(const std::string& path) {
-    std::lock_guard<std::mutex> lock(depdency_mutex_);
-    return depdency_imported_.insert(path).second;
-  }
-
-  // Try to import the given file into querydb. We should only ever be
-  // importing a file into querydb once per file. Returns true if the file
-  // can be imported.
-  bool StartQueryDbImport(const std::string& path) {
-    return querydb_processing_.insert(path).second;
-  }
-
-  // The file has been fully imported and can be imported again later on.
-  void DoneQueryDbImport(const std::string& path) {
-    querydb_processing_.erase(path);
-  }
-
-  // Returns true if there any any files currently being imported.
-  bool HasActiveQuerydbImports() { return !querydb_processing_.empty(); }
-
-  std::unordered_set<std::string> querydb_processing_;
-
-  // TODO: use std::shared_mutex so we can have multiple readers.
-  std::mutex depdency_mutex_;
-  std::unordered_set<std::string> depdency_imported_;
-};
-
-// Manages loading caches from file paths for the indexer process.
-struct CacheLoader {
-  explicit CacheLoader(Config* config) : config_(config) {}
-
-  IndexFile* TryLoad(const std::string& path) {
-    auto it = caches.find(path);
-    if (it != caches.end())
-      return it->second.get();
-
-    std::unique_ptr<IndexFile> cache = LoadCachedIndex(config_, path);
-    if (!cache)
-      return nullptr;
-
-    caches[path] = std::move(cache);
-    return caches[path].get();
-  }
-
-  // Takes the existing cache or loads the cache at |path|. May return nullptr
-  // if the cache does not exist.
-  std::unique_ptr<IndexFile> TryTakeOrLoad(const std::string& path) {
-    auto it = caches.find(path);
-    if (it != caches.end()) {
-      auto result = std::move(it->second);
-      caches.erase(it);
-      return result;
-    }
-
-    return LoadCachedIndex(config_, path);
-  }
-
-  // Takes the existing cache or loads the cache at |path|. Asserts the cache
-  // exists.
-  std::unique_ptr<IndexFile> TakeOrLoad(const std::string& path) {
-    auto result = TryTakeOrLoad(path);
-    assert(result);
-    return result;
-  }
-
-  std::unordered_map<std::string, std::unique_ptr<IndexFile>> caches;
-  Config* config_;
-};
-
-// Caches timestamps of cc files so we can avoid a filesystem reads. This is
-// important for import perf, as during dependency checking the same files are
-// checked over and over again if they are common headers.
-struct TimestampManager {
-  optional<int64_t> GetLastCachedModificationTime(CacheLoader* cache_loader,
-                                                  const std::string& path) {
-    {
-      std::lock_guard<std::mutex> guard(mutex_);
-      auto it = timestamps_.find(path);
-      if (it != timestamps_.end())
-        return it->second;
-    }
-    IndexFile* file = cache_loader->TryLoad(path);
-    if (!file)
-      return nullopt;
-
-    UpdateCachedModificationTime(path, file->last_modification_time);
-    return file->last_modification_time;
-  }
-
-  void UpdateCachedModificationTime(const std::string& path,
-                                    int64_t timestamp) {
-    std::lock_guard<std::mutex> guard(mutex_);
-    timestamps_[path] = timestamp;
-  }
-
-  // TODO: use std::shared_mutex so we can have multiple readers.
-  std::mutex mutex_;
-  std::unordered_map<std::string, int64_t> timestamps_;
-};
-
-struct IndexManager {
-  std::unordered_set<std::string> files_being_indexed_;
-  std::mutex mutex_;
-
-  // Marks a file as being indexed. Returns true if the file is not already
-  // being indexed.
-  bool MarkIndex(const std::string& path) {
-    std::lock_guard<std::mutex> lock(mutex_);
-
-    return files_being_indexed_.insert(path).second;
-  }
-
-  // Unmarks a file as being indexed, so it can get indexed again in the
-  // future.
-  void ClearIndex(const std::string& path) {
-    std::lock_guard<std::mutex> lock(mutex_);
-
-    auto it = files_being_indexed_.find(path);
-    assert(it != files_being_indexed_.end());
-    files_being_indexed_.erase(it);
-  }
-};
-
 // Send indexing progress to client if reporting is enabled.
 void EmitProgress(Config* config, QueueManager* queue) {
   if (config->enableProgressReports) {
@@ -1487,183 +1239,17 @@ bool QueryDbMainLoop(Config* config,
   for (auto& message : messages) {
     did_work = true;
 
-    switch (message->method_id) {
-      case IpcId::Initialize: {
-        auto request = message->As<Ipc_InitializeRequest>();
-
-        // Log initialization parameters.
-        rapidjson::StringBuffer output;
-        Writer writer(output);
-        Reflect(writer, request->params.initializationOptions);
-        LOG_S(INFO) << "Init parameters: " << output.GetString();
-
-        if (request->params.rootUri) {
-          std::string project_path = request->params.rootUri->GetPath();
-          LOG_S(INFO) << "[querydb] Initialize in directory " << project_path
-                      << " with uri " << request->params.rootUri->raw_uri;
-
-          if (!request->params.initializationOptions) {
-            LOG_S(FATAL) << "Initialization parameters (particularily "
-                            "cacheDirectory) are required";
-            exit(1);
-          }
-
-          *config = *request->params.initializationOptions;
-
-          // Check client version.
-          if (config->clientVersion.has_value() &&
-              *config->clientVersion != kExpectedClientVersion) {
-            Out_ShowLogMessage out;
-            out.display_type = Out_ShowLogMessage::DisplayType::Show;
-            out.params.type = lsMessageType::Error;
-            out.params.message =
-                "cquery client (v" + std::to_string(*config->clientVersion) +
-                ") and server (v" + std::to_string(kExpectedClientVersion) +
-                ") version mismatch. Please update ";
-            if (config->clientVersion > kExpectedClientVersion)
-              out.params.message += "the cquery binary.";
-            else
-              out.params.message +=
-                  "your extension client (VSIX file). Make sure to uninstall "
-                  "the cquery extension and restart vscode before "
-                  "reinstalling.";
-            out.Write(std::cout);
-          }
-
-          // Make sure cache directory is valid.
-          if (config->cacheDirectory.empty()) {
-            LOG_S(FATAL) << "Exiting; no cache directory";
-            exit(1);
-          }
-
-          config->cacheDirectory = NormalizePath(config->cacheDirectory);
-          EnsureEndsInSlash(config->cacheDirectory);
-
-          // Ensure there is a resource directory.
-          if (config->resourceDirectory.empty()) {
-            config->resourceDirectory = GetWorkingDirectory();
-#if defined(_WIN32)
-            config->resourceDirectory +=
-                std::string("../../clang_resource_dir/");
-#else
-            config->resourceDirectory += std::string("../clang_resource_dir/");
-#endif
-          }
-          config->resourceDirectory = NormalizePath(config->resourceDirectory);
-          LOG_S(INFO) << "Using -resource-dir=" << config->resourceDirectory;
-
-          // Send initialization before starting indexers, so we don't send a
-          // status update too early.
-          // TODO: query request->params.capabilities.textDocument and support
-          // only things the client supports.
-
-          Out_InitializeResponse out;
-          out.id = request->id;
-
-          // out.result.capabilities.textDocumentSync =
-          // lsTextDocumentSyncOptions();
-          // out.result.capabilities.textDocumentSync->openClose = true;
-          // out.result.capabilities.textDocumentSync->change =
-          // lsTextDocumentSyncKind::Full;
-          // out.result.capabilities.textDocumentSync->willSave = true;
-          // out.result.capabilities.textDocumentSync->willSaveWaitUntil =
-          // true;
-          out.result.capabilities.textDocumentSync =
-              lsTextDocumentSyncKind::Incremental;
-
-          out.result.capabilities.renameProvider = true;
-
-          out.result.capabilities.completionProvider = lsCompletionOptions();
-          out.result.capabilities.completionProvider->resolveProvider = false;
-          // vscode doesn't support trigger character sequences, so we use ':'
-          // for
-          // '::' and '>' for '->'. See
-          // https://github.com/Microsoft/language-server-protocol/issues/138.
-          out.result.capabilities.completionProvider->triggerCharacters = {
-              ".", ":", ">", "#"};
-
-          out.result.capabilities.signatureHelpProvider =
-              lsSignatureHelpOptions();
-          // NOTE: If updating signature help tokens make sure to also update
-          // WorkingFile::FindClosestCallNameInBuffer.
-          out.result.capabilities.signatureHelpProvider->triggerCharacters = {
-              "(", ","};
-
-          out.result.capabilities.codeLensProvider = lsCodeLensOptions();
-          out.result.capabilities.codeLensProvider->resolveProvider = false;
-
-          out.result.capabilities.definitionProvider = true;
-          out.result.capabilities.documentHighlightProvider = true;
-          out.result.capabilities.hoverProvider = true;
-          out.result.capabilities.referencesProvider = true;
-
-          out.result.capabilities.codeActionProvider = true;
-
-          out.result.capabilities.documentSymbolProvider = true;
-          out.result.capabilities.workspaceSymbolProvider = true;
-
-          out.result.capabilities.documentLinkProvider =
-              lsDocumentLinkOptions();
-          out.result.capabilities.documentLinkProvider->resolveProvider = false;
-
-          IpcManager::WriteStdout(IpcId::Initialize, out);
-
-          // Set project root.
-          config->projectRoot =
-              NormalizePath(request->params.rootUri->GetPath());
-          EnsureEndsInSlash(config->projectRoot);
-          MakeDirectoryRecursive(config->cacheDirectory +
-                                 EscapeFileName(config->projectRoot));
-
-          // Start indexer threads.
-          if (config->indexerCount == 0) {
-            // If the user has not specified how many indexers to run, try to
-            // guess an appropriate value. Default to 80% utilization.
-            const float kDefaultTargetUtilization = 0.8;
-            config->indexerCount =
-                std::thread::hardware_concurrency() * kDefaultTargetUtilization;
-            if (config->indexerCount <= 0)
-              config->indexerCount = 1;
-          }
-          LOG_S(INFO) << "Starting " << config->indexerCount << " indexers";
-          for (int i = 0; i < config->indexerCount; ++i) {
-            WorkThread::StartThread("indexer" + std::to_string(i), [=]() {
-              return IndexMain(config, file_consumer_shared, timestamp_manager,
-                               import_manager, project, working_files, waiter,
-                               queue);
-            });
-          }
-
-          Timer time;
-
-          // Open up / load the project.
-          project->Load(config->extraClangArguments,
-                        config->compilationDatabaseDirectory, project_path,
-                        config->resourceDirectory);
-          time.ResetAndPrint("[perf] Loaded compilation entries (" +
-                             std::to_string(project->entries.size()) +
-                             " files)");
-
-          // Start scanning include directories before dispatching project
-          // files, because that takes a long time.
-          include_complete->Rescan();
-
-          time.Reset();
-          project->ForAllFilteredFiles(
-              config, [&](int i, const Project::Entry& entry) {
-                bool is_interactive =
-                    working_files->GetFileByFilename(entry.filename) != nullptr;
-                queue->index_request.Enqueue(Index_Request(
-                    entry.filename, entry.args, is_interactive, nullopt));
-              });
-
-          // We need to support multiple concurrent index processes.
-          time.ResetAndPrint("[perf] Dispatched initial index requests");
-        }
-
+    for (MessageHandler* handler : *MessageHandler::message_handlers) {
+      if (handler->GetId() == message->method_id) {
+        handler->Run(std::move(message));
         break;
       }
+    }
+    if (!message)
+      continue;
+    // FIXME: assert(!message), ie, verify that a handler was run.
 
+    switch (message->method_id) {
       case IpcId::Exit: {
         LOG_S(INFO) << "Exiting; got IpcId::Exit";
         exit(0);
@@ -3036,10 +2622,30 @@ void RunQueryDbThread(const std::string& bin_name,
   auto signature_cache = MakeUnique<CodeCompleteCache>();
   ImportManager import_manager;
   TimestampManager timestamp_manager;
+  QueryDatabase db;
+
+  // Setup shared references.
+  for (MessageHandler* handler : *MessageHandler::message_handlers) {
+    handler->config = config;
+    handler->db = &db;
+    handler->exit_when_idle = &exit_when_idle;
+    handler->waiter = waiter;
+    handler->queue = queue;
+    handler->project = &project;
+    handler->file_consumer_shared = &file_consumer_shared;
+    handler->import_manager = &import_manager;
+    handler->timestamp_manager = &timestamp_manager;
+    handler->working_files = &working_files;
+    handler->clang_complete = &clang_complete;
+    handler->include_complete = &include_complete;
+    handler->global_code_complete_cache = global_code_complete_cache.get();
+    handler->non_global_code_complete_cache =
+        non_global_code_complete_cache.get();
+    handler->signature_cache = signature_cache.get();
+  }
 
   // Run query db main loop.
   SetCurrentThreadName("querydb");
-  QueryDatabase db;
   while (true) {
     bool did_work = QueryDbMainLoop(
         config, &db, &exit_when_idle, waiter, queue, &project,

src/config.h

@@ -107,3 +107,6 @@ MAKE_REFLECT_STRUCT(Config,
 
                     clientVersion,
                     enableSnippetInsertion);
+
+// Expected client version. We show an error if this doesn't match.
+constexpr const int kExpectedClientVersion = 3;

src/entry_points.h

@@ -0,0 +1,22 @@
+#pragma once
+
+#include "config.h"
+#include "file_consumer.h"
+#include "import_manager.h"
+#include "ipc_manager.h"
+#include "project.h"
+#include "threaded_queue.h"
+#include "timestamp_manager.h"
+#include "work_thread.h"
+#include "working_files.h"
+
+// Contains declarations for some of the thread-main functions.
+
+WorkThread::Result IndexMain(Config* config,
+                             FileConsumer::SharedState* file_consumer_shared,
+                             TimestampManager* timestamp_manager,
+                             ImportManager* import_manager,
+                             Project* project,
+                             WorkingFiles* working_files,
+                             MultiQueueWaiter* waiter,
+                             QueueManager* queue);

src/import_manager.cc

@@ -0,0 +1,18 @@
+#include "import_manager.h"
+
+bool ImportManager::TryMarkDependencyImported(const std::string& path) {
+  std::lock_guard<std::mutex> lock(depdency_mutex_);
+  return depdency_imported_.insert(path).second;
+}
+
+bool ImportManager::StartQueryDbImport(const std::string& path) {
+  return querydb_processing_.insert(path).second;
+}
+
+void ImportManager::DoneQueryDbImport(const std::string& path) {
+  querydb_processing_.erase(path);
+}
+
+bool ImportManager::HasActiveQuerydbImports() {
+  return !querydb_processing_.empty();
+}

src/import_manager.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <mutex>
+#include <string>
+#include <unordered_set>
+
+// Manages files inside of the indexing pipeline so we don't have the same file
+// being imported multiple times.
+//
+// NOTE: This is not thread safe and should only be used on the querydb thread.
+struct ImportManager {
+  // Try to mark the given dependency as imported. A dependency can only ever be
+  // imported once.
+  bool TryMarkDependencyImported(const std::string& path);
+
+  // Try to import the given file into querydb. We should only ever be
+  // importing a file into querydb once per file. Returns true if the file
+  // can be imported.
+  bool StartQueryDbImport(const std::string& path);
+
+  // The file has been fully imported and can be imported again later on.
+  void DoneQueryDbImport(const std::string& path);
+
+  // Returns true if there any any files currently being imported.
+  bool HasActiveQuerydbImports();
+
+  std::unordered_set<std::string> querydb_processing_;
+
+  // TODO: use std::shared_mutex so we can have multiple readers.
+  std::mutex depdency_mutex_;
+  std::unordered_set<std::string> depdency_imported_;
+};

src/ipc_manager.cc

@@ -1,6 +1,7 @@
 #include "ipc_manager.h"
 
 #include "language_server_api.h"
+#include "query.h"
 
 #include <sstream>
 
@@ -29,3 +30,51 @@ void IpcManager::WriteStdout(IpcId id, lsBaseOutMessage& response) {
 
 IpcManager::IpcManager(MultiQueueWaiter* waiter)
     : for_stdout(waiter), for_querydb(waiter) {}
+
+Index_Request::Index_Request(const std::string& path,
+                             const std::vector<std::string>& args,
+                             bool is_interactive,
+                             optional<std::string> contents)
+    : path(path),
+      args(args),
+      is_interactive(is_interactive),
+      contents(contents) {}
+
+Index_DoIdMap::Index_DoIdMap(std::unique_ptr<IndexFile> current,
+                             PerformanceImportFile perf,
+                             bool is_interactive,
+                             bool write_to_disk)
+    : current(std::move(current)),
+      perf(perf),
+      is_interactive(is_interactive),
+      write_to_disk(write_to_disk) {
+  assert(this->current);
+}
+
+Index_OnIdMapped::File::File(std::unique_ptr<IndexFile> file,
+                             std::unique_ptr<IdMap> ids)
+    : file(std::move(file)), ids(std::move(ids)) {}
+
+Index_OnIdMapped::Index_OnIdMapped(PerformanceImportFile perf,
+                                   bool is_interactive,
+                                   bool write_to_disk)
+    : perf(perf),
+      is_interactive(is_interactive),
+      write_to_disk(write_to_disk) {}
+
+Index_OnIndexed::Index_OnIndexed(IndexUpdate& update,
+                                 PerformanceImportFile perf)
+    : update(update), perf(perf) {}
+
+QueueManager::QueueManager(MultiQueueWaiter* waiter)
+    : index_request(waiter),
+      do_id_map(waiter),
+      load_previous_index(waiter),
+      on_id_mapped(waiter),
+      on_indexed(waiter) {}
+
+bool QueueManager::HasWork() {
+  return !index_request.IsEmpty() || !do_id_map.IsEmpty() ||
+         !load_previous_index.IsEmpty() || !on_id_mapped.IsEmpty() ||
+         !on_indexed.IsEmpty();
+}

src/ipc_manager.h

@@ -1,10 +1,14 @@
 #pragma once
 
 #include "ipc.h"
+#include "performance.h"
+#include "query.h"
 #include "threaded_queue.h"
 
 #include <memory>
 
+// TODO/FIXME: Merge IpcManager and QueueManager.
+
 struct lsBaseOutMessage;
 
 struct IpcManager {
@@ -26,3 +30,75 @@ struct IpcManager {
 
   static IpcManager* instance_;
 };
+
+struct Index_Request {
+  std::string path;
+  // TODO: make |args| a string that is parsed lazily.
+  std::vector<std::string> args;
+  bool is_interactive;
+  optional<std::string> contents;  // Preloaded contents. Useful for tests.
+
+  Index_Request(const std::string& path,
+                const std::vector<std::string>& args,
+                bool is_interactive,
+                optional<std::string> contents);
+};
+
+struct Index_DoIdMap {
+  std::unique_ptr<IndexFile> current;
+  std::unique_ptr<IndexFile> previous;
+
+  PerformanceImportFile perf;
+  bool is_interactive = false;
+  bool write_to_disk = false;
+  bool load_previous = false;
+
+  Index_DoIdMap(std::unique_ptr<IndexFile> current,
+                PerformanceImportFile perf,
+                bool is_interactive,
+                bool write_to_disk);
+};
+
+struct Index_OnIdMapped {
+  struct File {
+    std::unique_ptr<IndexFile> file;
+    std::unique_ptr<IdMap> ids;
+
+    File(std::unique_ptr<IndexFile> file, std::unique_ptr<IdMap> ids);
+  };
+
+  std::unique_ptr<File> previous;
+  std::unique_ptr<File> current;
+
+  PerformanceImportFile perf;
+  bool is_interactive;
+  bool write_to_disk;
+
+  Index_OnIdMapped(PerformanceImportFile perf,
+                   bool is_interactive,
+                   bool write_to_disk);
+};
+
+struct Index_OnIndexed {
+  IndexUpdate update;
+  PerformanceImportFile perf;
+
+  Index_OnIndexed(IndexUpdate& update, PerformanceImportFile perf);
+};
+
+struct QueueManager {
+  using Index_RequestQueue = ThreadedQueue<Index_Request>;
+  using Index_DoIdMapQueue = ThreadedQueue<Index_DoIdMap>;
+  using Index_OnIdMappedQueue = ThreadedQueue<Index_OnIdMapped>;
+  using Index_OnIndexedQueue = ThreadedQueue<Index_OnIndexed>;
+
+  Index_RequestQueue index_request;
+  Index_DoIdMapQueue do_id_map;
+  Index_DoIdMapQueue load_previous_index;
+  Index_OnIdMappedQueue on_id_mapped;
+  Index_OnIndexedQueue on_indexed;
+
+  QueueManager(MultiQueueWaiter* waiter);
+
+  bool HasWork();
+};

src/message_handler.cc

@@ -0,0 +1,12 @@
+#include "message_handler.h"
+
+MessageHandler::MessageHandler() {
+  // Dynamically allocate |message_handlers|, otherwise there will be static
+  // initialization order races.
+  if (!message_handlers)
+    message_handlers = new std::vector<MessageHandler*>();
+  message_handlers->push_back(this);
+}
+
+// static
+std::vector<MessageHandler*>* MessageHandler::message_handlers = nullptr;

src/message_handler.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "cache_loader.h"
+#include "clang_complete.h"
+#include "code_complete_cache.h"
+#include "config.h"
+#include "import_manager.h"
+#include "include_complete.h"
+#include "ipc_manager.h"
+#include "project.h"
+#include "query.h"
+#include "threaded_queue.h"
+#include "timestamp_manager.h"
+#include "working_files.h"
+
+// Usage:
+//
+//  struct FooHandler : MessageHandler {
+//    // ...
+//  };
+//  REGISTER_MESSAGE_HANDLER(FooHandler);
+//
+// Then there will be a global FooHandler instance in
+// |MessageHandler::message_handlers|.
+
+#define REGISTER_MESSAGE_HANDLER(type) \
+  static type type##message_handler_instance_;
+
+struct MessageHandler {
+  Config* config = nullptr;
+  QueryDatabase* db = nullptr;
+  bool* exit_when_idle = nullptr;
+  MultiQueueWaiter* waiter = nullptr;
+  QueueManager* queue = nullptr;
+  Project* project = nullptr;
+  FileConsumer::SharedState* file_consumer_shared = nullptr;
+  ImportManager* import_manager = nullptr;
+  TimestampManager* timestamp_manager = nullptr;
+  WorkingFiles* working_files = nullptr;
+  ClangCompleteManager* clang_complete = nullptr;
+  IncludeComplete* include_complete = nullptr;
+  CodeCompleteCache* global_code_complete_cache = nullptr;
+  CodeCompleteCache* non_global_code_complete_cache = nullptr;
+  CodeCompleteCache* signature_cache = nullptr;
+
+  virtual IpcId GetId() const = 0;
+  virtual void Run(std::unique_ptr<BaseIpcMessage> message) = 0;
+
+  static std::vector<MessageHandler*>* message_handlers;
+
+ protected:
+  MessageHandler();
+};
+
+template <typename TMessage>
+struct BaseMessageHandler : MessageHandler {
+  virtual void Run(TMessage* message) = 0;
+
+  // MessageHandler:
+  IpcId GetId() const override { return TMessage::kIpcId; }
+  void Run(std::unique_ptr<BaseIpcMessage> message) override {
+    Run(message->As<TMessage>());
+  }
+};

src/messages/initialize.cc

@@ -0,0 +1,176 @@
+#include "entry_points.h"
+#include "message_handler.h"
+#include "platform.h"
+#include "timer.h"
+
+#include <loguru.hpp>
+
+struct InitializeHandler : BaseMessageHandler<Ipc_InitializeRequest> {
+  void Run(Ipc_InitializeRequest* request) override {
+    // Log initialization parameters.
+    rapidjson::StringBuffer output;
+    Writer writer(output);
+    Reflect(writer, request->params.initializationOptions);
+    LOG_S(INFO) << "Init parameters: " << output.GetString();
+
+    if (request->params.rootUri) {
+      std::string project_path = request->params.rootUri->GetPath();
+      LOG_S(INFO) << "[querydb] Initialize in directory " << project_path
+                  << " with uri " << request->params.rootUri->raw_uri;
+
+      if (!request->params.initializationOptions) {
+        LOG_S(FATAL) << "Initialization parameters (particularily "
+                        "cacheDirectory) are required";
+        exit(1);
+      }
+
+      *config = *request->params.initializationOptions;
+
+      // Check client version.
+      if (config->clientVersion.has_value() &&
+          *config->clientVersion != kExpectedClientVersion) {
+        Out_ShowLogMessage out;
+        out.display_type = Out_ShowLogMessage::DisplayType::Show;
+        out.params.type = lsMessageType::Error;
+        out.params.message =
+            "cquery client (v" + std::to_string(*config->clientVersion) +
+            ") and server (v" + std::to_string(kExpectedClientVersion) +
+            ") version mismatch. Please update ";
+        if (config->clientVersion > kExpectedClientVersion)
+          out.params.message += "the cquery binary.";
+        else
+          out.params.message +=
+              "your extension client (VSIX file). Make sure to uninstall "
+              "the cquery extension and restart vscode before "
+              "reinstalling.";
+        out.Write(std::cout);
+      }
+
+      // Make sure cache directory is valid.
+      if (config->cacheDirectory.empty()) {
+        LOG_S(FATAL) << "Exiting; no cache directory";
+        exit(1);
+      }
+
+      config->cacheDirectory = NormalizePath(config->cacheDirectory);
+      EnsureEndsInSlash(config->cacheDirectory);
+
+      // Ensure there is a resource directory.
+      if (config->resourceDirectory.empty()) {
+        config->resourceDirectory = GetWorkingDirectory();
+#if defined(_WIN32)
+        config->resourceDirectory += std::string("../../clang_resource_dir/");
+#else
+        config->resourceDirectory += std::string("../clang_resource_dir/");
+#endif
+      }
+      config->resourceDirectory = NormalizePath(config->resourceDirectory);
+      LOG_S(INFO) << "Using -resource-dir=" << config->resourceDirectory;
+
+      // Send initialization before starting indexers, so we don't send a
+      // status update too early.
+      // TODO: query request->params.capabilities.textDocument and support
+      // only things the client supports.
+
+      Out_InitializeResponse out;
+      out.id = request->id;
+
+      // out.result.capabilities.textDocumentSync =
+      // lsTextDocumentSyncOptions();
+      // out.result.capabilities.textDocumentSync->openClose = true;
+      // out.result.capabilities.textDocumentSync->change =
+      // lsTextDocumentSyncKind::Full;
+      // out.result.capabilities.textDocumentSync->willSave = true;
+      // out.result.capabilities.textDocumentSync->willSaveWaitUntil =
+      // true;
+      out.result.capabilities.textDocumentSync =
+          lsTextDocumentSyncKind::Incremental;
+
+      out.result.capabilities.renameProvider = true;
+
+      out.result.capabilities.completionProvider = lsCompletionOptions();
+      out.result.capabilities.completionProvider->resolveProvider = false;
+      // vscode doesn't support trigger character sequences, so we use ':'
+      // for
+      // '::' and '>' for '->'. See
+      // https://github.com/Microsoft/language-server-protocol/issues/138.
+      out.result.capabilities.completionProvider->triggerCharacters = {
+          ".", ":", ">", "#"};
+
+      out.result.capabilities.signatureHelpProvider = lsSignatureHelpOptions();
+      // NOTE: If updating signature help tokens make sure to also update
+      // WorkingFile::FindClosestCallNameInBuffer.
+      out.result.capabilities.signatureHelpProvider->triggerCharacters = {"(",
+                                                                          ","};
+
+      out.result.capabilities.codeLensProvider = lsCodeLensOptions();
+      out.result.capabilities.codeLensProvider->resolveProvider = false;
+
+      out.result.capabilities.definitionProvider = true;
+      out.result.capabilities.documentHighlightProvider = true;
+      out.result.capabilities.hoverProvider = true;
+      out.result.capabilities.referencesProvider = true;
+
+      out.result.capabilities.codeActionProvider = true;
+
+      out.result.capabilities.documentSymbolProvider = true;
+      out.result.capabilities.workspaceSymbolProvider = true;
+
+      out.result.capabilities.documentLinkProvider = lsDocumentLinkOptions();
+      out.result.capabilities.documentLinkProvider->resolveProvider = false;
+
+      IpcManager::WriteStdout(IpcId::Initialize, out);
+
+      // Set project root.
+      config->projectRoot = NormalizePath(request->params.rootUri->GetPath());
+      EnsureEndsInSlash(config->projectRoot);
+      MakeDirectoryRecursive(config->cacheDirectory +
+                             EscapeFileName(config->projectRoot));
+
+      // Start indexer threads.
+      if (config->indexerCount == 0) {
+        // If the user has not specified how many indexers to run, try to
+        // guess an appropriate value. Default to 80% utilization.
+        const float kDefaultTargetUtilization = 0.8;
+        config->indexerCount =
+            std::thread::hardware_concurrency() * kDefaultTargetUtilization;
+        if (config->indexerCount <= 0)
+          config->indexerCount = 1;
+      }
+      LOG_S(INFO) << "Starting " << config->indexerCount << " indexers";
+      for (int i = 0; i < config->indexerCount; ++i) {
+        WorkThread::StartThread("indexer" + std::to_string(i), [=]() {
+          return IndexMain(config, file_consumer_shared, timestamp_manager,
+                           import_manager, project, working_files, waiter,
+                           queue);
+        });
+      }
+
+      Timer time;
+
+      // Open up / load the project.
+      project->Load(config->extraClangArguments,
+                    config->compilationDatabaseDirectory, project_path,
+                    config->resourceDirectory);
+      time.ResetAndPrint("[perf] Loaded compilation entries (" +
+                         std::to_string(project->entries.size()) + " files)");
+
+      // Start scanning include directories before dispatching project
+      // files, because that takes a long time.
+      include_complete->Rescan();
+
+      time.Reset();
+      project->ForAllFilteredFiles(
+          config, [&](int i, const Project::Entry& entry) {
+            bool is_interactive =
+                working_files->GetFileByFilename(entry.filename) != nullptr;
+            queue->index_request.Enqueue(Index_Request(
+                entry.filename, entry.args, is_interactive, nullopt));
+          });
+
+      // We need to support multiple concurrent index processes.
+      time.ResetAndPrint("[perf] Dispatched initial index requests");
+    }
+  }
+};
+REGISTER_MESSAGE_HANDLER(InitializeHandler);

src/timestamp_manager.cc

@@ -0,0 +1,26 @@
+#include "timestamp_manager.h"
+
+#include "indexer.h"
+
+optional<int64_t> TimestampManager::GetLastCachedModificationTime(
+    CacheLoader* cache_loader,
+    const std::string& path) {
+  {
+    std::lock_guard<std::mutex> guard(mutex_);
+    auto it = timestamps_.find(path);
+    if (it != timestamps_.end())
+      return it->second;
+  }
+  IndexFile* file = cache_loader->TryLoad(path);
+  if (!file)
+    return nullopt;
+
+  UpdateCachedModificationTime(path, file->last_modification_time);
+  return file->last_modification_time;
+}
+
+void TimestampManager::UpdateCachedModificationTime(const std::string& path,
+                                                    int64_t timestamp) {
+  std::lock_guard<std::mutex> guard(mutex_);
+  timestamps_[path] = timestamp;
+}

src/timestamp_manager.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "cache_loader.h"
+
+#include <optional.h>
+
+#include <mutex>
+#include <unordered_map>
+
+using namespace std::experimental;
+using std::experimental::nullopt;
+
+// Caches timestamps of cc files so we can avoid a filesystem reads. This is
+// important for import perf, as during dependency checking the same files are
+// checked over and over again if they are common headers.
+struct TimestampManager {
+  optional<int64_t> GetLastCachedModificationTime(CacheLoader* cache_loader,
+                                                  const std::string& path);
+
+  void UpdateCachedModificationTime(const std::string& path, int64_t timestamp);
+
+  // TODO: use std::shared_mutex so we can have multiple readers.
+  std::mutex mutex_;
+  std::unordered_map<std::string, int64_t> timestamps_;
+};