Redesign import_pipeline.cc and mitigate race (duplicate Query*::uses for initial indexing)

CMakeLists.txt

@@ -244,7 +244,6 @@ target_sources(ccls PRIVATE
                src/messages/ccls_derived.cc
                src/messages/ccls_file_info.cc
                src/messages/ccls_freshen_index.cc
-               src/messages/ccls_index_file.cc
                src/messages/ccls_inheritance_hierarchy.cc
                src/messages/ccls_member_hierarchy.cc
                src/messages/ccls_random.cc

src/cache_manager.cc

@@ -11,38 +11,7 @@
 #include <unordered_map>
 
 namespace {
-
+std::string GetCachePath(const std::string& source_file) {
-// Manages loading caches from file paths for the indexer process.
-struct RealCacheManager : ICacheManager {
-  explicit RealCacheManager() {}
-  ~RealCacheManager() override = default;
-
-  void WriteToCache(IndexFile& file) override {
-    std::string cache_path = GetCachePath(file.path);
-    WriteToFile(cache_path, file.file_contents);
-
-    std::string indexed_content = Serialize(g_config->cacheFormat, file);
-    WriteToFile(AppendSerializationFormat(cache_path), indexed_content);
-  }
-
-  std::optional<std::string> LoadCachedFileContents(
-      const std::string& path) override {
-    return ReadContent(GetCachePath(path));
-  }
-
-  std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path) override {
-    std::string cache_path = GetCachePath(path);
-    std::optional<std::string> file_content = ReadContent(cache_path);
-    std::optional<std::string> serialized_indexed_content =
-        ReadContent(AppendSerializationFormat(cache_path));
-    if (!file_content || !serialized_indexed_content)
-      return nullptr;
-
-    return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content,
-                       *file_content, IndexFile::kMajorVersion);
-  }
-
-  std::string GetCachePath(const std::string& source_file) {
   assert(!g_config->cacheDirectory.empty());
   std::string cache_file;
   size_t len = g_config->projectRoot.size();
@@ -55,85 +24,41 @@ struct RealCacheManager : ICacheManager {
   }
 
   return g_config->cacheDirectory + cache_file;
-  }
+}
 
-  std::string AppendSerializationFormat(const std::string& base) {
+std::string AppendSerializationFormat(const std::string& base) {
   switch (g_config->cacheFormat) {
     case SerializeFormat::Binary:
       return base + ".blob";
     case SerializeFormat::Json:
       return base + ".json";
   }
-  }
+}
-};
-
-struct FakeCacheManager : ICacheManager {
-  explicit FakeCacheManager(const std::vector<FakeCacheEntry>& entries)
-      : entries_(entries) {}
-
-  void WriteToCache(IndexFile& file) override { assert(false); }
-
-  std::optional<std::string> LoadCachedFileContents(
-      const std::string& path) override {
-    for (const FakeCacheEntry& entry : entries_) {
-      if (entry.path == path) {
-        return entry.content;
-      }
-    }
-
-    return std::nullopt;
-  }
-
-  std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path) override {
-    for (const FakeCacheEntry& entry : entries_) {
-      if (entry.path == path) {
-        return Deserialize(SerializeFormat::Json, path, entry.json, "<empty>",
-                           std::nullopt);
-      }
-    }
-
-    return nullptr;
-  }
-
-  std::vector<FakeCacheEntry> entries_;
-};
-
-}  // namespace
-
-// static
-std::shared_ptr<ICacheManager> ICacheManager::Make() {
-  return std::make_shared<RealCacheManager>();
 }
 
-// static
+// Manages loading caches from file paths for the indexer process.
-std::shared_ptr<ICacheManager> ICacheManager::MakeFake(
+void ICacheManager::WriteToCache(IndexFile& file) {
-    const std::vector<FakeCacheEntry>& entries) {
+  std::string cache_path = GetCachePath(file.path);
-  return std::make_shared<FakeCacheManager>(entries);
+  WriteToFile(cache_path, file.file_contents);
+
+  std::string indexed_content = Serialize(g_config->cacheFormat, file);
+  WriteToFile(AppendSerializationFormat(cache_path), indexed_content);
 }
 
-ICacheManager::~ICacheManager() = default;
+std::optional<std::string> ICacheManager::LoadCachedFileContents(
-
-IndexFile* ICacheManager::TryLoad(const std::string& path) {
-  auto it = caches_.find(path);
-  if (it != caches_.end())
-    return it->second.get();
-
-  std::unique_ptr<IndexFile> cache = RawCacheLoad(path);
-  if (!cache)
-    return nullptr;
-
-  caches_[path] = std::move(cache);
-  return caches_[path].get();
-}
-
-std::unique_ptr<IndexFile> ICacheManager::TryTakeOrLoad(
     const std::string& path) {
-  auto it = caches_.find(path);
+  return ReadContent(GetCachePath(path));
-  if (it != caches_.end()) {
+}
-    auto result = std::move(it->second);
+
-    caches_.erase(it);
+std::unique_ptr<IndexFile> ICacheManager::RawCacheLoad(
-    return result;
+    const std::string& path) {
-  }
+  std::string cache_path = GetCachePath(path);
-
+  std::optional<std::string> file_content = ReadContent(cache_path);
-  return RawCacheLoad(path);
+  std::optional<std::string> serialized_indexed_content =
+      ReadContent(AppendSerializationFormat(cache_path));
+  if (!file_content || !serialized_indexed_content)
+    return nullptr;
+
+  return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content,
+                     *file_content, IndexFile::kMajorVersion);
 }

src/cache_manager.h

@@ -7,34 +7,12 @@
 #include <unordered_map>
 #include <vector>
 
-struct Config;
 struct IndexFile;
 
 struct ICacheManager {
-  struct FakeCacheEntry {
+  void WriteToCache(IndexFile& file);
-    std::string path;
-    std::string content;
-    std::string json;
-  };
 
-  static std::shared_ptr<ICacheManager> Make();
+  std::optional<std::string> LoadCachedFileContents(const std::string& path);
-  static std::shared_ptr<ICacheManager> MakeFake(
-      const std::vector<FakeCacheEntry>& entries);
-
-  virtual ~ICacheManager();
-
-  // Tries to load a cache for |path|, returning null if there is none. The
-  // cache loader still owns the cache.
-  IndexFile* TryLoad(const std::string& path);
-
-  // Takes the existing cache or loads the cache at |path|. May return null if
-  // the cache does not exist.
-  std::unique_ptr<IndexFile> TryTakeOrLoad(const std::string& path);
-
-  virtual void WriteToCache(IndexFile& file) = 0;
-
-  virtual std::optional<std::string> LoadCachedFileContents(
-      const std::string& path) = 0;
 
   template <typename Fn>
   void IterateLoadedCaches(Fn fn) {
@@ -42,7 +20,7 @@ struct ICacheManager {
       fn(cache.second.get());
   }
 
- protected:
+  std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path);
-  virtual std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path) = 0;
+
   std::unordered_map<std::string, std::unique_ptr<IndexFile>> caches_;
 };

src/clang_complete.cc

@@ -647,12 +647,10 @@ void DiagnosticQueryMain(ClangCompleteManager* completion_manager) {
 ClangCompleteManager::ClangCompleteManager(Project* project,
                                            WorkingFiles* working_files,
                                            OnDiagnostic on_diagnostic,
-                                           OnIndex on_index,
                                            OnDropped on_dropped)
     : project_(project),
       working_files_(working_files),
       on_diagnostic_(on_diagnostic),
-      on_index_(on_index),
       on_dropped_(on_dropped),
       preloaded_sessions_(kMaxPreloadedSessions),
       completion_sessions_(kMaxCompletionSessions) {

src/clang_complete.h

@@ -82,7 +82,6 @@ struct ClangCompleteManager {
   ClangCompleteManager(Project* project,
                        WorkingFiles* working_files,
                        OnDiagnostic on_diagnostic,
-                       OnIndex on_index,
                        OnDropped on_dropped);
 
   // Start a code completion at the given location. |on_complete| will run when
@@ -127,7 +126,6 @@ struct ClangCompleteManager {
   Project* project_;
   WorkingFiles* working_files_;
   OnDiagnostic on_diagnostic_;
-  OnIndex on_index_;
   OnDropped on_dropped_;
 
   using LruSessionCache = LruCache<std::string, CompletionSession>;

src/clang_indexer.cc

@@ -9,11 +9,12 @@
 
 #include <loguru.hpp>
 
+#include <assert.h>
 #include <inttypes.h>
 #include <limits.h>
 #include <algorithm>
-#include <cassert>
 #include <chrono>
+#include <unordered_set>
 
 #if CINDEX_VERSION >= 47
 #define CINDEX_HAVE_PRETTY 1
@@ -279,7 +280,7 @@ struct IndexParam {
   std::unordered_set<CXFile> seen_cx_files;
   std::vector<std::string> seen_files;
   std::unordered_map<std::string, FileContents> file_contents;
-  std::unordered_map<std::string, int64_t> file_modification_times;
+  std::unordered_map<std::string, int64_t> file2write_time;
 
   // Only use this when strictly needed (ie, primary translation unit is
   // needed). Most logic should get the IndexFile instance via
@@ -373,11 +374,11 @@ IndexFile* ConsumeFile(IndexParam* param, CXFile file) {
       param->seen_files.push_back(file_name);
 
       // Set modification time.
-      std::optional<int64_t> modification_time = LastWriteTime(file_name);
+      std::optional<int64_t> write_time = LastWriteTime(file_name);
-      LOG_IF_S(ERROR, !modification_time)
+      LOG_IF_S(ERROR, !write_time) << "failed to fetch write time for "
-          << "Failed fetching modification time for " << file_name;
+                                   << file_name;
-      if (modification_time)
+      if (write_time)
-        param->file_modification_times[file_name] = *modification_time;
+        param->file2write_time[file_name] = *write_time;
     }
   }
 
@@ -2010,7 +2011,7 @@ void OnIndexReference(CXClientData client_data, const CXIdxEntityRefInfo* ref) {
 }
 
 std::vector<std::unique_ptr<IndexFile>> Parse(
-    FileConsumerSharedState* file_consumer_shared,
+    VFS* vfs,
     std::string file,
     const std::vector<std::string>& args,
     const std::vector<FileContents>& file_contents,
@@ -2041,12 +2042,11 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
 
   perf->index_parse = timer.ElapsedMicrosecondsAndReset();
 
-  return ParseWithTu(file_consumer_shared, perf, tu.get(), index, file,
+  return ParseWithTu(vfs, perf, tu.get(), index, file, args, unsaved_files);
-                     args, unsaved_files);
 }
 
 std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
-    FileConsumerSharedState* file_consumer_shared,
+    VFS* vfs,
     PerformanceImportFile* perf,
     ClangTranslationUnit* tu,
     ClangIndex* index,
@@ -2067,7 +2067,7 @@ std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
   callback.indexDeclaration = &OnIndexDeclaration;
   callback.indexEntityReference = &OnIndexReference;
 
-  FileConsumer file_consumer(file_consumer_shared, file);
+  FileConsumer file_consumer(vfs, file);
   IndexParam param(tu, &file_consumer);
   for (const CXUnsavedFile& contents : file_contents) {
     param.file_contents[contents.Filename] = FileContents(
@@ -2140,15 +2140,13 @@ std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
     }
 
     // Update file contents and modification time.
-    entry->last_modification_time = param.file_modification_times[entry->path];
+    entry->last_write_time = param.file2write_time[entry->path];
 
     // Update dependencies for the file. Do not include the file in its own
     // dependency set.
-    entry->dependencies = param.seen_files;
+    for (const std::string& path : param.seen_files)
-    entry->dependencies.erase(
+      if (path != entry->path && path != entry->import_file)
-        std::remove(entry->dependencies.begin(), entry->dependencies.end(),
+        entry->dependencies[path] = param.file2write_time[path];
-                    entry->path),
-        entry->dependencies.end());
   }
 
   return result;
@@ -2229,10 +2227,8 @@ struct TestIndexer : IIndexer {
     return result;
   }
 
-  ~TestIndexer() override = default;
-
   std::vector<std::unique_ptr<IndexFile>> Index(
-      FileConsumerSharedState* file_consumer_shared,
+      VFS* vfs,
       std::string file,
       const std::vector<std::string>& args,
       const std::vector<FileContents>& file_contents,

src/command_line.cc

@@ -110,9 +110,8 @@ See more on https://github.com/MaskRay/ccls/wiki
 bool QueryDbMainLoop(QueryDatabase* db,
                      MultiQueueWaiter* waiter,
                      Project* project,
-                     FileConsumerSharedState* file_consumer_shared,
+                     VFS* vfs,
                      ImportPipelineStatus* status,
-                     TimestampManager* timestamp_manager,
                      SemanticHighlightSymbolCache* semantic_cache,
                      WorkingFiles* working_files,
                      ClangCompleteManager* clang_complete,
@@ -154,7 +153,7 @@ void RunQueryDbThread(const std::string& bin_name,
   Project project;
   SemanticHighlightSymbolCache semantic_cache;
   WorkingFiles working_files;
-  FileConsumerSharedState file_consumer_shared;
+  VFS vfs;
   DiagnosticsEngine diag_engine;
 
   ClangCompleteManager clang_complete(
@@ -162,11 +161,6 @@ void RunQueryDbThread(const std::string& bin_name,
       [&](std::string path, std::vector<lsDiagnostic> diagnostics) {
         diag_engine.Publish(&working_files, path, diagnostics);
       },
-      [&](ClangTranslationUnit* tu, const std::vector<CXUnsavedFile>& unsaved,
-          const std::string& path, const std::vector<std::string>& args) {
-        IndexWithTuFromCodeCompletion(&file_consumer_shared, tu, unsaved, path,
-                                      args);
-      },
       [](lsRequestId id) {
         if (id.Valid()) {
           Out_Error out;
@@ -184,7 +178,6 @@ void RunQueryDbThread(const std::string& bin_name,
   auto non_global_code_complete_cache = std::make_unique<CodeCompleteCache>();
   auto signature_cache = std::make_unique<CodeCompleteCache>();
   ImportPipelineStatus import_pipeline_status;
-  TimestampManager timestamp_manager;
   QueryDatabase db;
 
   // Setup shared references.
@@ -193,9 +186,8 @@ void RunQueryDbThread(const std::string& bin_name,
     handler->waiter = indexer_waiter;
     handler->project = &project;
     handler->diag_engine = &diag_engine;
-    handler->file_consumer_shared = &file_consumer_shared;
+    handler->vfs = &vfs;
     handler->import_pipeline_status = &import_pipeline_status;
-    handler->timestamp_manager = &timestamp_manager;
     handler->semantic_cache = &semantic_cache;
     handler->working_files = &working_files;
     handler->clang_complete = &clang_complete;
@@ -210,8 +202,8 @@ void RunQueryDbThread(const std::string& bin_name,
   SetThreadName("querydb");
   while (true) {
     bool did_work = QueryDbMainLoop(
-        &db, querydb_waiter, &project, &file_consumer_shared,
+        &db, querydb_waiter, &project, &vfs,
-        &import_pipeline_status, &timestamp_manager,
+        &import_pipeline_status,
         &semantic_cache, &working_files, &clang_complete, &include_complete,
         global_code_complete_cache.get(), non_global_code_complete_cache.get(),
         signature_cache.get());

src/config.cc

@@ -1,3 +1,4 @@
 #include "config.h"
 
 std::unique_ptr<Config> g_config;
+thread_local int g_thread_id;

src/config.h

@@ -42,44 +42,22 @@ struct Config {
   // It is not schema-aware and you need to re-index whenever a struct
   // member has changed.
   SerializeFormat cacheFormat = SerializeFormat::Binary;
-  // Value to use for clang -resource-dir if not present in
-  // compile_commands.json.
-  //
-  // ccls includes a resource directory, this should not need to be configured
-  // unless you're using an esoteric configuration. Consider reporting a bug and
-  // fixing upstream instead of configuring this.
-  //
-  // Example value: "/path/to/lib/clang/5.0.1/"
-  std::string resourceDirectory;
 
+  struct Clang {
     // Additional arguments to pass to clang.
-  std::vector<std::string> extraClangArguments;
+    std::vector<std::string> extraArgs;
 
-  // If true, ccls will send progress reports while indexing
+    // Value to use for clang -resource-dir if not specified.
-  // How often should ccls send progress report messages?
-  //  -1: never
-  //   0: as often as possible
-  //   xxx: at most every xxx milliseconds
     //
-  // Empty progress reports (ie, idle) are delivered as often as they are
+    // This option defaults to clang -print-resource-dir and should not be
-  // available and may exceed this value.
+    // specified unless you are using an esoteric configuration.
-  //
+    std::string resourceDir;
-  // This does not guarantee a progress report will be delivered every
+  } clang;
-  // interval; it could take significantly longer if ccls is completely idle.
-  int progressReportFrequencyMs = 500;
-
-  // If true, document links are reported for #include directives.
-  bool showDocumentLinksOnIncludes = true;
-
-  // Version of the client. If undefined the version check is skipped. Used to
-  // inform users their vscode client is too old and needs to be updated.
-  std::optional<int> clientVersion;
 
   struct ClientCapability {
     // TextDocumentClientCapabilities.completion.completionItem.snippetSupport
     bool snippetSupport = false;
-  };
+  } client;
-  ClientCapability client;
 
   struct CodeLens {
     // Enables code lens on parameter and function variables.
@@ -228,6 +206,7 @@ struct Config {
     int maxNum = 2000;
   } xref;
 };
+MAKE_REFLECT_STRUCT(Config::Clang, extraArgs, resourceDir);
 MAKE_REFLECT_STRUCT(Config::ClientCapability, snippetSupport);
 MAKE_REFLECT_STRUCT(Config::CodeLens, localVariables);
 MAKE_REFLECT_STRUCT(Config::Completion,
@@ -260,16 +239,8 @@ MAKE_REFLECT_STRUCT(Config,
                     compilationDatabaseDirectory,
                     cacheDirectory,
                     cacheFormat,
-                    resourceDirectory,
-
-                    extraClangArguments,
-
-                    progressReportFrequencyMs,
-
-                    showDocumentLinksOnIncludes,
-
-                    clientVersion,
 
+                    clang,
                     client,
                     codeLens,
                     completion,
@@ -280,3 +251,4 @@ MAKE_REFLECT_STRUCT(Config,
                     xref);
 
 extern std::unique_ptr<Config> g_config;
+thread_local extern int g_thread_id;

src/file_consumer.cc

@@ -55,21 +55,48 @@ std::optional<std::string> FileContents::ContentsInRange(Range range) const {
   return std::nullopt;
 }
 
-bool FileConsumerSharedState::Mark(const std::string& file) {
+VFS::State VFS::Get(const std::string& file) {
   std::lock_guard<std::mutex> lock(mutex);
-  return used_files.insert(file).second;
+  auto it = state.find(file);
+  if (it != state.end())
+    return it->second;
+  return {0, 0, 0};
 }
 
-void FileConsumerSharedState::Reset(const std::string& file) {
+bool VFS::Mark(const std::string& file, int owner, int stage) {
   std::lock_guard<std::mutex> lock(mutex);
-  auto it = used_files.find(file);
+  State& st = state[file];
-  if (it != used_files.end())
+  if (st.stage < stage) {
-    used_files.erase(it);
+    st.owner = owner;
+    st.stage = stage;
+    return true;
+  } else
+    return false;
 }
 
-FileConsumer::FileConsumer(FileConsumerSharedState* shared_state,
+bool VFS::Stamp(const std::string& file, int64_t ts) {
-                           const std::string& parse_file)
+  std::lock_guard<std::mutex> lock(mutex);
-    : shared_(shared_state), parse_file_(parse_file) {}
+  State& st = state[file];
+  if (st.timestamp < ts) {
+    st.timestamp = ts;
+    return true;
+  } else
+    return false;
+}
+
+void VFS::ResetLocked(const std::string& file) {
+  State& st = state[file];
+  if (st.owner == g_thread_id)
+    st.stage = 0;
+}
+
+void VFS::Reset(const std::string& file) {
+  std::lock_guard<std::mutex> lock(mutex);
+  ResetLocked(file);
+}
+
+FileConsumer::FileConsumer(VFS* vfs, const std::string& parse_file)
+    : vfs_(vfs), parse_file_(parse_file), thread_id_(g_thread_id) {}
 
 IndexFile* FileConsumer::TryConsumeFile(
     CXFile file,
@@ -96,12 +123,9 @@ IndexFile* FileConsumer::TryConsumeFile(
 
   std::string file_name = FileName(file);
 
-  // No result in local; we need to query global.
-  bool did_insert = shared_->Mark(file_name);
-
   // We did not take the file from global. Cache that we failed so we don't try
   // again and return nullptr.
-  if (!did_insert) {
+  if (!vfs_->Mark(file_name, thread_id_, 2)) {
     local_[file_id] = nullptr;
     return nullptr;
   }

src/file_consumer.h

@@ -8,7 +8,6 @@
 #include <functional>
 #include <mutex>
 #include <unordered_map>
-#include <unordered_set>
 #include <vector>
 
 struct IndexFile;
@@ -30,13 +29,19 @@ struct FileContents {
   std::vector<int> line_offsets_;
 };
 
-struct FileConsumerSharedState {
+struct VFS {
-  mutable std::unordered_set<std::string> used_files;
+  struct State {
+    int64_t timestamp;
+    int owner;
+    int stage;
+  };
+  mutable std::unordered_map<std::string, State> state;
   mutable std::mutex mutex;
 
-  // Mark the file as used. Returns true if the file was not previously used.
+  State Get(const std::string& file);
-  bool Mark(const std::string& file);
+  bool Mark(const std::string& file, int owner, int stage);
-  // Reset the used state (ie, mark the file as unused).
+  bool Stamp(const std::string& file, int64_t ts);
+  void ResetLocked(const std::string& file);
   void Reset(const std::string& file);
 };
 
@@ -48,8 +53,7 @@ struct FileConsumerSharedState {
 // The indexer does this because header files do not have their own translation
 // units but we still want to index them.
 struct FileConsumer {
-  FileConsumer(FileConsumerSharedState* shared_state,
+  FileConsumer(VFS* vfs, const std::string& parse_file);
-               const std::string& parse_file);
 
   // Returns true if this instance owns given |file|. This will also attempt to
   // take ownership over |file|.
@@ -69,6 +73,7 @@ struct FileConsumer {
 
  private:
   std::unordered_map<CXFileUniqueID, std::unique_ptr<IndexFile>> local_;
-  FileConsumerSharedState* shared_;
+  VFS* vfs_;
   std::string parse_file_;
+  int thread_id_;
 };

src/import_pipeline.cc

@@ -18,115 +18,25 @@
 
 namespace {
 
-struct Out_Progress : public lsOutMessage<Out_Progress> {
-  struct Params {
-    int indexRequestCount = 0;
-    int loadPreviousIndexCount = 0;
-    int onIdMappedCount = 0;
-    int onIndexedCount = 0;
-    int activeThreads = 0;
-  };
-  std::string method = "$ccls/progress";
-  Params params;
-};
-MAKE_REFLECT_STRUCT(Out_Progress::Params,
-                    indexRequestCount,
-                    loadPreviousIndexCount,
-                    onIdMappedCount,
-                    onIndexedCount,
-                    activeThreads);
-MAKE_REFLECT_STRUCT(Out_Progress, jsonrpc, method, params);
-
-long long GetCurrentTimeInMilliseconds() {
-  auto time_since_epoch = Timer::Clock::now().time_since_epoch();
-  long long elapsed_milliseconds =
-      std::chrono::duration_cast<std::chrono::milliseconds>(time_since_epoch)
-          .count();
-  return elapsed_milliseconds;
-}
-
-struct ActiveThread {
-  ActiveThread(ImportPipelineStatus* status)
-      : status_(status) {
-    if (g_config && g_config->progressReportFrequencyMs < 0)
-      return;
-
-    ++status_->num_active_threads;
-  }
-  ~ActiveThread() {
-    if (g_config && g_config->progressReportFrequencyMs < 0)
-      return;
-
-    --status_->num_active_threads;
-    EmitProgress();
-  }
-
-  // Send indexing progress to client if reporting is enabled.
-  void EmitProgress() {
-    auto* queue = QueueManager::instance();
-    Out_Progress out;
-    out.params.indexRequestCount = queue->index_request.Size();
-    out.params.onIdMappedCount = queue->on_id_mapped.Size();
-    out.params.onIndexedCount = queue->on_indexed.Size();
-    out.params.activeThreads = status_->num_active_threads;
-
-    // Ignore this progress update if the last update was too recent.
-    if (g_config && g_config->progressReportFrequencyMs != 0) {
-      // Make sure we output a status update if queue lengths are zero.
-      bool all_zero = out.params.indexRequestCount == 0 &&
-                      out.params.loadPreviousIndexCount == 0 &&
-                      out.params.onIdMappedCount == 0 &&
-                      out.params.onIndexedCount == 0 &&
-                      out.params.activeThreads == 0;
-      if (!all_zero &&
-          GetCurrentTimeInMilliseconds() < status_->next_progress_output)
-        return;
-      status_->next_progress_output =
-          GetCurrentTimeInMilliseconds() + g_config->progressReportFrequencyMs;
-    }
-
-    QueueManager::WriteStdout(kMethodType_Unknown, out);
-  }
-
-  ImportPipelineStatus* status_;
-};
-
-enum class ShouldParse { Yes, No, NoSuchFile };
-
 // Checks if |path| needs to be reparsed. This will modify cached state
 // such that calling this function twice with the same path may return true
 // the first time but will return false the second.
 //
 // |from|: The file which generated the parse request for this file.
-ShouldParse FileNeedsParse(
+bool FileNeedsParse(int64_t write_time,
+                    VFS* vfs,
                     bool is_interactive,
-    TimestampManager* timestamp_manager,
-    const std::shared_ptr<ICacheManager>& cache_manager,
                     IndexFile* opt_previous_index,
                     const std::string& path,
                     const std::vector<std::string>& args,
                     const std::optional<std::string>& from) {
-  auto unwrap_opt = [](const std::optional<std::string>& opt) -> std::string {
+  {
-    if (opt)
+    std::lock_guard<std::mutex> lock(vfs->mutex);
-      return " (via " + *opt + ")";
+    if (vfs->state[path].timestamp < write_time) {
-    return "";
+      LOG_S(INFO) << "timestamp changed for " << path
-  };
+                  << (from ? " (via " + *from + ")" : std::string());
-
+      return true;
-  std::optional<int64_t> modification_timestamp = LastWriteTime(path);
+    }
-
-  // Cannot find file.
-  if (!modification_timestamp)
-    return ShouldParse::NoSuchFile;
-
-  std::optional<int64_t> last_cached_modification =
-      timestamp_manager->GetLastCachedModificationTime(cache_manager.get(),
-                                                       path);
-
-  // File has been changed.
-  if (!last_cached_modification ||
-      modification_timestamp != *last_cached_modification) {
-    LOG_S(INFO) << "Timestamp has changed for " << path << unwrap_opt(from);
-    return ShouldParse::Yes;
   }
 
   // Command-line arguments changed.
@@ -141,183 +51,96 @@ ShouldParse FileNeedsParse(
              (is_file(prev_args[i]) && is_file(args[i]));
     }
     if (!same) {
-      LOG_S(INFO) << "Arguments have changed for " << path << unwrap_opt(from);
+      LOG_S(INFO) << "args changed for " << path << (from ? " (via " + *from + ")" : std::string());
-      return ShouldParse::Yes;
+      return true;
     }
   }
 
-  // File has not changed, do not parse it.
+  return false;
-  return ShouldParse::No;
 };
 
-enum CacheLoadResult { Parse, DoNotParse };
+bool Indexer_Parse(DiagnosticsEngine* diag_engine,
-CacheLoadResult TryLoadFromCache(
-    FileConsumerSharedState* file_consumer_shared,
-    TimestampManager* timestamp_manager,
-    const std::shared_ptr<ICacheManager>& cache_manager,
-    bool is_interactive,
-    const Project::Entry& entry,
-    const std::string& path_to_index) {
-  // Always run this block, even if we are interactive, so we can check
-  // dependencies and reset files in |file_consumer_shared|.
-  IndexFile* previous_index = cache_manager->TryLoad(path_to_index);
-  if (!previous_index)
-    return CacheLoadResult::Parse;
-
-  // If none of the dependencies have changed and the index is not
-  // interactive (ie, requested by a file save), skip parsing and just load
-  // from cache.
-
-  // Check timestamps and update |file_consumer_shared|.
-  ShouldParse path_state =
-      FileNeedsParse(is_interactive, timestamp_manager, cache_manager,
-                     previous_index, path_to_index, entry.args, std::nullopt);
-  if (path_state == ShouldParse::Yes)
-    file_consumer_shared->Reset(path_to_index);
-
-  // Target file does not exist on disk, do not emit any indexes.
-  // TODO: Dependencies should be reassigned to other files. We can do this by
-  // updating the "primary_file" if it doesn't exist. Might not actually be a
-  // problem in practice.
-  if (path_state == ShouldParse::NoSuchFile)
-    return CacheLoadResult::DoNotParse;
-
-  bool needs_reparse = is_interactive || path_state == ShouldParse::Yes;
-
-  for (const std::string& dependency : previous_index->dependencies) {
-    assert(!dependency.empty());
-
-    if (FileNeedsParse(is_interactive, timestamp_manager, cache_manager,
-                       previous_index, dependency, entry.args,
-                       previous_index->path) == ShouldParse::Yes) {
-      needs_reparse = true;
-
-      // Do not break here, as we need to update |file_consumer_shared| for
-      // every dependency that needs to be reparsed.
-      file_consumer_shared->Reset(dependency);
-    }
-  }
-
-  // FIXME: should we still load from cache?
-  if (needs_reparse)
-    return CacheLoadResult::Parse;
-
-  // No timestamps changed - load directly from cache.
-  LOG_S(INFO) << "load index for " << path_to_index;
-
-  // TODO/FIXME: real perf
-  PerformanceImportFile perf;
-
-  std::vector<Index_OnIdMapped> result;
-  result.push_back(Index_OnIdMapped(
-      cache_manager, nullptr, cache_manager->TryTakeOrLoad(path_to_index), perf,
-      is_interactive, false /*write_to_disk*/));
-  for (const std::string& dependency : previous_index->dependencies) {
-    // Only load a dependency if it is not already loaded.
-    //
-    // This is important for perf in large projects where there are lots of
-    // dependencies shared between many files.
-    if (!file_consumer_shared->Mark(dependency))
-      continue;
-
-    LOG_S(INFO) << "emit index for " << dependency << " via "
-                << previous_index->path;
-
-    // |dependency_index| may be null if there is no cache for it but
-    // another file has already started importing it.
-    if (std::unique_ptr<IndexFile> dependency_index =
-            cache_manager->TryTakeOrLoad(dependency)) {
-      result.push_back(
-          Index_OnIdMapped(cache_manager, nullptr, std::move(dependency_index),
-                           perf, is_interactive, false /*write_to_disk*/));
-    }
-  }
-
-  QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result));
-  return CacheLoadResult::DoNotParse;
-}
-
-std::vector<FileContents> PreloadFileContents(
-    const std::shared_ptr<ICacheManager>& cache_manager,
-    const Project::Entry& entry,
-    const std::string& entry_contents,
-    const std::string& path_to_index) {
-  // Load file contents for all dependencies into memory. If the dependencies
-  // for the file changed we may not end up using all of the files we
-  // preloaded. If a new dependency was added the indexer will grab the file
-  // contents as soon as possible.
-  //
-  // We do this to minimize the race between indexing a file and capturing the
-  // file contents.
-  //
-  // TODO: We might be able to optimize perf by only copying for files in
-  //       working_files. We can pass that same set of files to the indexer as
-  //       well. We then default to a fast file-copy if not in working set.
-
-  // index->file_contents comes from cache, so we need to check if that cache is
-  // still valid. if so, we can use it, otherwise we need to load from disk.
-  auto get_latest_content = [](const std::string& path, int64_t cached_time,
-                               const std::string& cached) -> std::string {
-    std::optional<int64_t> mod_time = LastWriteTime(path);
-    if (!mod_time)
-      return "";
-
-    if (*mod_time == cached_time)
-      return cached;
-
-    std::optional<std::string> fresh_content = ReadContent(path);
-    if (!fresh_content) {
-      LOG_S(ERROR) << "Failed to load content for " << path;
-      return "";
-    }
-    return *fresh_content;
-  };
-
-  std::vector<FileContents> file_contents;
-  file_contents.push_back(FileContents(entry.filename, entry_contents));
-  cache_manager->IterateLoadedCaches([&](IndexFile* index) {
-    if (index->path == entry.filename)
-      return;
-    file_contents.push_back(FileContents(
-        index->path,
-        get_latest_content(index->path, index->last_modification_time,
-                           index->file_contents)));
-  });
-
-  return file_contents;
-}
-
-void ParseFile(DiagnosticsEngine* diag_engine,
                    WorkingFiles* working_files,
-               FileConsumerSharedState* file_consumer_shared,
+                   Project* project,
-               TimestampManager* timestamp_manager,
+                   VFS* vfs,
-               IIndexer* indexer,
+                   IIndexer* indexer) {
-               const Index_Request& request,
+  auto* queue = QueueManager::instance();
-               const Project::Entry& entry) {
+  std::optional<Index_Request> opt_request = queue->index_request.TryPopFront();
-  // If the file is inferred, we may not actually be able to parse that file
+  if (!opt_request)
-  // directly (ie, a header file, which are not listed in the project). If this
+    return false;
-  // file is inferred, then try to use the file which originally imported it.
+  auto& request = *opt_request;
-  std::string path_to_index = entry.filename;
+  ICacheManager cache;
-  if (entry.is_inferred) {
+
-    IndexFile* entry_cache = request.cache_manager->TryLoad(entry.filename);
+  Project::Entry entry;
-    if (entry_cache)
+  {
-      path_to_index = entry_cache->import_file;
+    std::lock_guard<std::mutex> lock(project->mutex_);
+    auto it = project->absolute_path_to_entry_index_.find(request.path);
+    if (it != project->absolute_path_to_entry_index_.end())
+      entry = project->entries[it->second];
+    else {
+      entry.filename = request.path;
+      entry.args = request.args;
     }
+  }
+  std::string path_to_index = entry.filename;
+  std::unique_ptr<IndexFile> prev;
 
   // Try to load the file from cache.
-  if (TryLoadFromCache(file_consumer_shared, timestamp_manager,
+  std::optional<int64_t> write_time = LastWriteTime(path_to_index);
-                       request.cache_manager, request.is_interactive, entry,
+  if (!write_time)
-                       path_to_index) == CacheLoadResult::DoNotParse)
+    return true;
-    return;
+  // FIXME Don't drop
+  if (!vfs->Mark(path_to_index, g_thread_id, 1))
+    return true;
+
+  int reparse; // request.is_interactive;
+  prev = cache.RawCacheLoad(path_to_index);
+  if (!prev)
+    reparse = 2;
+  else {
+    reparse = vfs->Stamp(path_to_index, prev->last_write_time);
+    if (FileNeedsParse(*write_time, vfs, request.is_interactive, &*prev,
+                       path_to_index, entry.args, std::nullopt))
+      reparse = 2;
+    for (const auto& dep : prev->dependencies)
+      if (auto write_time1 = LastWriteTime(dep.first)) {
+        if (dep.second < *write_time1) {
+          reparse = 2;
+          std::lock_guard<std::mutex> lock(vfs->mutex);
+          vfs->state[dep.first].stage = 0;
+        }
+      } else
+        reparse = 2;
+  }
+
+  if (reparse < 2) {
+    PerformanceImportFile perf;
+    auto dependencies = prev->dependencies;
+    if (reparse) {
+      IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
+      queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf),
+        request.is_interactive);
+    }
+    for (const auto& dep : dependencies)
+      if (vfs->Mark(dep.first, 0, 2)) {
+        prev = cache.RawCacheLoad(dep.first);
+        IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
+        queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf),
+          request.is_interactive);
+      }
+
+    std::lock_guard<std::mutex> lock(vfs->mutex);
+    VFS::State& state = vfs->state[path_to_index];
+    if (state.owner == g_thread_id)
+      state.stage = 0;
+    return true;
+  }
 
   LOG_S(INFO) << "parse " << path_to_index;
-  std::vector<FileContents> file_contents = PreloadFileContents(
-      request.cache_manager, entry, request.contents, path_to_index);
 
   std::vector<Index_OnIdMapped> result;
   PerformanceImportFile perf;
-  auto indexes = indexer->Index(file_consumer_shared, path_to_index, entry.args,
+  auto indexes = indexer->Index(vfs, path_to_index, entry.args, {}, &perf);
-                                file_contents, &perf);
 
   if (indexes.empty()) {
     if (g_config->index.enabled && request.id.Valid()) {
@@ -327,160 +150,52 @@ void ParseFile(DiagnosticsEngine* diag_engine,
       out.error.message = "Failed to index " + path_to_index;
       QueueManager::WriteStdout(kMethodType_Unknown, out);
     }
-    return;
+    vfs->Reset(path_to_index);
+    return true;
   }
 
-  for (std::unique_ptr<IndexFile>& new_index : indexes) {
+  for (std::unique_ptr<IndexFile>& curr : indexes) {
-    Timer time;
-
     // Only emit diagnostics for non-interactive sessions, which makes it easier
     // to identify indexing problems. For interactive sessions, diagnostics are
     // handled by code completion.
     if (!request.is_interactive)
-      diag_engine->Publish(working_files, new_index->path,
+      diag_engine->Publish(working_files, curr->path, curr->diagnostics_);
-                           new_index->diagnostics_);
 
-    // When main thread does IdMap request it will request the previous index if
+    std::string path = curr->path;
-    // needed.
+    if (!(vfs->Stamp(path, curr->last_write_time) || path == path_to_index))
-    LOG_S(INFO) << "emit index for " << new_index->path;
+      continue;
-    result.push_back(
+    LOG_S(INFO) << "emit index for " << path;
-        Index_OnIdMapped(request.cache_manager,
+    prev = cache.RawCacheLoad(path);
-                         request.cache_manager->TryTakeOrLoad(path_to_index),
-                         std::move(new_index), perf, request.is_interactive,
-                         true /*write_to_disk*/));
-  }
-
-  QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result),
-                                                    request.is_interactive);
-}
-
-bool IndexMain_DoParse(
-    DiagnosticsEngine* diag_engine,
-    WorkingFiles* working_files,
-    FileConsumerSharedState* file_consumer_shared,
-    TimestampManager* timestamp_manager,
-    IIndexer* indexer) {
-  auto* queue = QueueManager::instance();
-  std::optional<Index_Request> request = queue->index_request.TryPopFront();
-  if (!request)
-    return false;
-
-  Project::Entry entry;
-  entry.filename = request->path;
-  entry.args = request->args;
-  ParseFile(diag_engine, working_files, file_consumer_shared, timestamp_manager,
-            indexer, request.value(), entry);
-  return true;
-}
-
-bool IndexMain_DoCreateIndexUpdate(TimestampManager* timestamp_manager) {
-  auto* queue = QueueManager::instance();
-
-  bool did_work = false;
-  for (int i = 100; i--; ) {
-    std::optional<Index_OnIdMapped> response = queue->on_id_mapped.TryPopFront();
-    if (!response)
-      return did_work;
-
-    did_work = true;
-
-    Timer time;
 
     // Write current index to disk if requested.
-    std::string path = response->current->path;
-    if (response->write_to_disk) {
     LOG_S(INFO) << "store index for " << path;
-      time.Reset();
+    Timer time;
-      response->cache_manager->WriteToCache(*response->current);
+    cache.WriteToCache(*curr);
-      response->perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
+    perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
-      timestamp_manager->UpdateCachedModificationTime(
+
-          path, response->current->last_modification_time);
+    vfs->Reset(path_to_index);
+    if (entry.id >= 0) {
+      std::lock_guard<std::mutex> lock(project->mutex_);
+      for (auto& dep : curr->dependencies)
+        project->absolute_path_to_entry_index_[dep.first] = entry.id;
     }
 
     // Build delta update.
-    IndexUpdate update = IndexUpdate::CreateDelta(response->previous.get(),
+    IndexUpdate update = IndexUpdate::CreateDelta(prev.get(), curr.get());
-                                                  response->current.get());
+    perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
-    response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
+    LOG_S(INFO) << "built index for " << path << " (is_delta=" << !!prev << ")";
-    LOG_S(INFO) << "built index for " << path
-                << " (is_delta=" << !!response->previous << ")";
 
-    Index_OnIndexed reply(std::move(update), response->perf);
+    Index_OnIndexed reply(std::move(update), perf);
-    queue->on_indexed.PushBack(std::move(reply), response->is_interactive);
+    queue->on_indexed.PushBack(std::move(reply), request.is_interactive);
   }
 
-  return did_work;
+  return true;
 }
 
 }  // namespace
 
-std::optional<int64_t> TimestampManager::GetLastCachedModificationTime(
-    ICacheManager* cache_manager,
-    const std::string& path) {
-  {
-    std::lock_guard<std::mutex> guard(mutex_);
-    auto it = timestamps_.find(path);
-    if (it != timestamps_.end())
-      return it->second;
-  }
-  IndexFile* file = cache_manager->TryLoad(path);
-  if (!file)
-    return std::nullopt;
-
-  UpdateCachedModificationTime(path, file->last_modification_time);
-  return file->last_modification_time;
-}
-
-void TimestampManager::UpdateCachedModificationTime(const std::string& path,
-                                                    int64_t timestamp) {
-  std::lock_guard<std::mutex> guard(mutex_);
-  timestamps_[path] = timestamp;
-}
-
-ImportPipelineStatus::ImportPipelineStatus()
-    : num_active_threads(0), next_progress_output(0) {}
-
-// Index a file using an already-parsed translation unit from code completion.
-// Since most of the time for indexing a file comes from parsing, we can do
-// real-time indexing.
-// TODO: add option to disable this.
-void IndexWithTuFromCodeCompletion(
-    FileConsumerSharedState* file_consumer_shared,
-    ClangTranslationUnit* tu,
-    const std::vector<CXUnsavedFile>& file_contents,
-    const std::string& path,
-    const std::vector<std::string>& args) {
-  file_consumer_shared->Reset(path);
-
-  PerformanceImportFile perf;
-  ClangIndex index;
-  auto indexes = ParseWithTu(file_consumer_shared, &perf, tu, &index, path,
-                             args, file_contents);
-  if (indexes.empty())
-    return;
-
-  std::vector<Index_OnIdMapped> result;
-  for (std::unique_ptr<IndexFile>& new_index : indexes) {
-    Timer time;
-
-    std::shared_ptr<ICacheManager> cache_manager;
-    assert(false && "FIXME cache_manager");
-    // When main thread does IdMap request it will request the previous index if
-    // needed.
-    LOG_S(INFO) << "Emitting index for " << new_index->path;
-    result.push_back(Index_OnIdMapped(
-        cache_manager, cache_manager->TryTakeOrLoad(path), std::move(new_index),
-        perf, true /*is_interactive*/, true /*write_to_disk*/));
-  }
-
-  LOG_IF_S(WARNING, result.size() > 1)
-      << "Code completion index update generated more than one index";
-
-  QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result));
-}
-
 void Indexer_Main(DiagnosticsEngine* diag_engine,
-                  FileConsumerSharedState* file_consumer_shared,
+                  VFS* vfs,
-                  TimestampManager* timestamp_manager,
                   ImportPipelineStatus* status,
                   Project* project,
                   WorkingFiles* working_files,
@@ -489,35 +204,9 @@ void Indexer_Main(DiagnosticsEngine* diag_engine,
   // Build one index per-indexer, as building the index acquires a global lock.
   auto indexer = std::make_unique<ClangIndexer>();
 
-  while (true) {
+  while (true)
-    bool did_work = false;
+    if (!Indexer_Parse(diag_engine, working_files, project, vfs, indexer.get()))
-
+      waiter->Wait(&queue->index_request);
-    {
-      ActiveThread active_thread(status);
-
-      // TODO: process all off IndexMain_DoIndex before calling
-      // IndexMain_DoCreateIndexUpdate for better icache behavior. We need to
-      // have some threads spinning on both though otherwise memory usage will
-      // get bad.
-
-      // We need to make sure to run both IndexMain_DoParse and
-      // IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any
-      // work. Running both also lets the user query the partially constructed
-      // index.
-      did_work = IndexMain_DoParse(diag_engine, working_files,
-                                   file_consumer_shared, timestamp_manager,
-                                   indexer.get()) ||
-                 did_work;
-
-      did_work = IndexMain_DoCreateIndexUpdate(timestamp_manager) || did_work;
-    }
-
-    // We didn't do any work, so wait for a notification.
-    if (!did_work) {
-      waiter->Wait(&queue->on_indexed, &queue->index_request,
-                   &queue->on_id_mapped);
-    }
-  }
 }
 
 namespace {
@@ -559,9 +248,6 @@ bool QueryDb_ImportMain(QueryDatabase* db,
                         SemanticHighlightSymbolCache* semantic_cache,
                         WorkingFiles* working_files) {
   auto* queue = QueueManager::instance();
-
-  ActiveThread active_thread(status);
-
   bool did_work = false;
 
   for (int i = 80; i--; ) {

src/import_pipeline.h

@@ -1,19 +1,10 @@
 #pragma once
 
-// FIXME: do not include clang-c outside of clang_ files.
-#include <clang-c/Index.h>
-
 #include <atomic>
-#include <memory>
-#include <optional>
-#include <string>
-#include <mutex>
-#include <unordered_map>
-#include <vector>
 
 struct ClangTranslationUnit;
 class DiagnosticsEngine;
-struct FileConsumerSharedState;
+struct VFS;
 struct ICacheManager;
 struct MultiQueueWaiter;
 struct Project;
@@ -21,37 +12,13 @@ struct QueryDatabase;
 struct SemanticHighlightSymbolCache;
 struct WorkingFiles;
 
-// Caches timestamps of cc files so we can avoid a filesystem reads. This is
-// important for import perf, as during dependency checking the same files are
-// checked over and over again if they are common headers.
-struct TimestampManager {
-  std::optional<int64_t> GetLastCachedModificationTime(ICacheManager* cache_manager,
-                                                  const std::string& path);
-
-  void UpdateCachedModificationTime(const std::string& path, int64_t timestamp);
-
-  // TODO: use std::shared_mutex so we can have multiple readers.
-  std::mutex mutex_;
-  std::unordered_map<std::string, int64_t> timestamps_;
-};
-
 struct ImportPipelineStatus {
-  std::atomic<int> num_active_threads;
+  std::atomic<int> num_active_threads = {0};
-  std::atomic<long long> next_progress_output;
+  std::atomic<long long> next_progress_output = {0};
-
-  ImportPipelineStatus();
 };
 
-void IndexWithTuFromCodeCompletion(
-    FileConsumerSharedState* file_consumer_shared,
-    ClangTranslationUnit* tu,
-    const std::vector<CXUnsavedFile>& file_contents,
-    const std::string& path,
-    const std::vector<std::string>& args);
-
 void Indexer_Main(DiagnosticsEngine* diag_engine,
-                  FileConsumerSharedState* file_consumer_shared,
+                  VFS* vfs,
-                  TimestampManager* timestamp_manager,
                   ImportPipelineStatus* status,
                   Project* project,
                   WorkingFiles* working_files,

src/indexer.h

@@ -285,7 +285,7 @@ struct IndexFile {
 
   std::string path;
   std::vector<std::string> args;
-  int64_t last_modification_time = 0;
+  int64_t last_write_time = 0;
   LanguageId language = LanguageId::Unknown;
 
   // The path to the translation unit cc file which caused the creation of this
@@ -298,7 +298,7 @@ struct IndexFile {
   std::vector<Range> skipped_by_preprocessor;
 
   std::vector<IndexInclude> includes;
-  std::vector<std::string> dependencies;
+  std::unordered_map<std::string, int64_t> dependencies;
   std::unordered_map<Usr, IndexFunc> usr2func;
   std::unordered_map<Usr, IndexType> usr2type;
   std::unordered_map<Usr, IndexVar> usr2var;
@@ -334,14 +334,14 @@ struct NamespaceHelper {
 // |dependencies| are the existing dependencies of |import_file| if this is a
 // reparse.
 std::vector<std::unique_ptr<IndexFile>> Parse(
-    FileConsumerSharedState* file_consumer_shared,
+    VFS* vfs,
     std::string file,
     const std::vector<std::string>& args,
     const std::vector<FileContents>& file_contents,
     PerformanceImportFile* perf,
     ClangIndex* index);
 std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
-    FileConsumerSharedState* file_consumer_shared,
+    VFS* vfs,
     PerformanceImportFile* perf,
     ClangTranslationUnit* tu,
     ClangIndex* index,
@@ -367,7 +367,7 @@ struct IIndexer {
 
   virtual ~IIndexer() = default;
   virtual std::vector<std::unique_ptr<IndexFile>> Index(
-      FileConsumerSharedState* file_consumer_shared,
+      VFS* vfs,
       std::string file,
       const std::vector<std::string>& args,
       const std::vector<FileContents>& file_contents,
@@ -376,12 +376,12 @@ struct IIndexer {
 
 struct ClangIndexer : IIndexer {
   std::vector<std::unique_ptr<IndexFile>> Index(
-      FileConsumerSharedState* file_consumer_shared,
+      VFS* vfs,
       std::string file,
       const std::vector<std::string>& args,
       const std::vector<FileContents>& file_contents,
       PerformanceImportFile* perf) override {
-    return Parse(file_consumer_shared, file, args, file_contents, perf, &index);
+    return Parse(vfs, file, args, file_contents, perf, &index);
   }
 
   // Note: constructing this acquires a global lock

src/message_handler.cc

@@ -126,7 +126,7 @@ MessageHandler::MessageHandler() {
 std::vector<MessageHandler*>* MessageHandler::message_handlers = nullptr;
 
 bool FindFileOrFail(QueryDatabase* db,
-                    const Project* project,
+                    Project* project,
                     std::optional<lsRequestId> id,
                     const std::string& absolute_path,
                     QueryFile** out_query_file,
@@ -147,8 +147,12 @@ bool FindFileOrFail(QueryDatabase* db,
   if (out_file_id)
     *out_file_id = -1;
 
-  bool indexing = project->absolute_path_to_entry_index_.find(absolute_path) !=
+  bool indexing;
+  {
+    std::lock_guard<std::mutex> lock(project->mutex_);
+    indexing = project->absolute_path_to_entry_index_.find(absolute_path) !=
       project->absolute_path_to_entry_index_.end();
+  }
   if (indexing)
     LOG_S(INFO) << "\"" << absolute_path << "\" is being indexed.";
   else

src/message_handler.h

@@ -15,14 +15,13 @@ struct ClangCompleteManager;
 struct CodeCompleteCache;
 struct Config;
 class DiagnosticsEngine;
-struct FileConsumerSharedState;
+struct VFS;
 struct ImportManager;
 struct ImportPipelineStatus;
 struct IncludeComplete;
 struct MultiQueueWaiter;
 struct Project;
 struct QueryDatabase;
-struct TimestampManager;
 struct WorkingFile;
 struct WorkingFiles;
 
@@ -107,10 +106,9 @@ struct MessageHandler {
   MultiQueueWaiter* waiter = nullptr;
   Project* project = nullptr;
   DiagnosticsEngine* diag_engine = nullptr;
-  FileConsumerSharedState* file_consumer_shared = nullptr;
+  VFS* vfs = nullptr;
   ImportManager* import_manager = nullptr;
   ImportPipelineStatus* import_pipeline_status = nullptr;
-  TimestampManager* timestamp_manager = nullptr;
   SemanticHighlightSymbolCache* semantic_cache = nullptr;
   WorkingFiles* working_files = nullptr;
   ClangCompleteManager* clang_complete = nullptr;
@@ -139,7 +137,7 @@ struct BaseMessageHandler : MessageHandler {
 };
 
 bool FindFileOrFail(QueryDatabase* db,
-                    const Project* project,
+                    Project* project,
                     std::optional<lsRequestId> id,
                     const std::string& absolute_path,
                     QueryFile** out_query_file,

src/messages/ccls_call_hierarchy.cc

@@ -4,6 +4,8 @@
 
 #include <loguru.hpp>
 
+#include <unordered_set>
+
 namespace {
 
 MethodType kMethodType = "$ccls/callHierarchy";

src/messages/ccls_freshen_index.cc

@@ -34,14 +34,8 @@ REGISTER_IN_MESSAGE(In_CclsFreshenIndex);
 struct Handler_CclsFreshenIndex : BaseMessageHandler<In_CclsFreshenIndex> {
   MethodType GetMethodType() const override { return kMethodType; }
   void Run(In_CclsFreshenIndex* request) override {
-    LOG_S(INFO) << "Freshening " << project->entries.size() << " files";
-
-    // TODO: think about this flow and test it more.
     GroupMatch matcher(request->params.whitelist, request->params.blacklist);
 
-    // Unmark all files whose timestamp has changed.
-    std::shared_ptr<ICacheManager> cache_manager = ICacheManager::Make();
-
     std::queue<const QueryFile*> q;
     // |need_index| stores every filename ever enqueued.
     std::unordered_set<std::string> need_index;
@@ -65,15 +59,15 @@ struct Handler_CclsFreshenIndex : BaseMessageHandler<In_CclsFreshenIndex> {
       q.pop();
       need_index.insert(file->def->path);
 
-      std::optional<int64_t> modification_timestamp =
+      std::optional<int64_t> write_time = LastWriteTime(file->def->path);
-          LastWriteTime(file->def->path);
+      if (!write_time)
-      if (!modification_timestamp)
         continue;
-      std::optional<int64_t> cached_modification =
+      {
-          timestamp_manager->GetLastCachedModificationTime(cache_manager.get(),
+        std::lock_guard<std::mutex> lock(vfs->mutex);
-                                                           file->def->path);
+        VFS::State& st = vfs->state[file->def->path];
-      if (modification_timestamp != cached_modification)
+        if (st.timestamp < write_time)
-        file_consumer_shared->Reset(file->def->path);
+          st.stage = 0;
+      }
 
       if (request->params.dependencies)
         for (const std::string& path : graph[file->def->path]) {
@@ -85,10 +79,8 @@ struct Handler_CclsFreshenIndex : BaseMessageHandler<In_CclsFreshenIndex> {
         }
     }
 
-    Timer time;
     // Send index requests for every file.
     project->Index(QueueManager::instance(), working_files, lsRequestId());
-    time.ResetAndPrint("[perf] Dispatched $ccls/freshenIndex index requests");
   }
 };
 REGISTER_MESSAGE_HANDLER(Handler_CclsFreshenIndex);

src/messages/ccls_index_file.cc

@@ -1,40 +0,0 @@
-#include "cache_manager.h"
-#include "message_handler.h"
-#include "platform.h"
-#include "queue_manager.h"
-
-#include <loguru/loguru.hpp>
-
-namespace {
-MethodType kMethodType = "$ccls/indexFile";
-
-struct In_CclsIndexFile : public NotificationInMessage {
-  MethodType GetMethodType() const override { return kMethodType; }
-  struct Params {
-    std::string path;
-    std::vector<std::string> args;
-    bool is_interactive = false;
-    std::string contents;
-  };
-  Params params;
-};
-MAKE_REFLECT_STRUCT(In_CclsIndexFile::Params,
-                    path,
-                    args,
-                    is_interactive,
-                    contents);
-MAKE_REFLECT_STRUCT(In_CclsIndexFile, params);
-REGISTER_IN_MESSAGE(In_CclsIndexFile);
-
-struct Handler_CclsIndexFile : BaseMessageHandler<In_CclsIndexFile> {
-  MethodType GetMethodType() const override { return kMethodType; }
-  void Run(In_CclsIndexFile* request) override {
-    LOG_S(INFO) << "Indexing file " << request->params.path;
-    QueueManager::instance()->index_request.PushBack(
-        Index_Request(NormalizePath(request->params.path), request->params.args,
-                      request->params.is_interactive, request->params.contents,
-                      ICacheManager::Make()));
-  }
-};
-REGISTER_MESSAGE_HANDLER(Handler_CclsIndexFile);
-}  // namespace

src/messages/ccls_member_hierarchy.cc

@@ -2,6 +2,8 @@
 #include "query_utils.h"
 #include "queue_manager.h"
 
+#include <unordered_set>
+
 namespace {
 MethodType kMethodType = "$ccls/memberHierarchy";
 

src/messages/initialize.cc

@@ -473,9 +473,9 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
       }
 
       // Ensure there is a resource directory.
-      if (config->resourceDirectory.empty())
+      if (config->clang.resourceDir.empty())
-        config->resourceDirectory = GetDefaultResourceDirectory();
+        config->clang.resourceDir = GetDefaultResourceDirectory();
-      LOG_S(INFO) << "Using -resource-dir=" << config->resourceDirectory;
+      LOG_S(INFO) << "Using -resource-dir=" << config->clang.resourceDir;
 
       // Send initialization before starting indexers, so we don't send a
       // status update too early.
@@ -519,9 +519,9 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
       LOG_S(INFO) << "Starting " << g_config->index.threads << " indexers";
       for (int i = 0; i < g_config->index.threads; i++) {
         std::thread([=]() {
+          g_thread_id = i + 1;
           SetThreadName("indexer" + std::to_string(i));
-          Indexer_Main(diag_engine, file_consumer_shared, timestamp_manager,
+          Indexer_Main(diag_engine, vfs, import_pipeline_status, project,
-                       import_pipeline_status, project,
                        working_files, waiter);
         }).detach();
       }

src/messages/text_document_did_change.cc

@@ -33,7 +33,7 @@ struct Handler_TextDocumentDidChange
         Project::Entry entry = project->FindCompilationEntryForFile(path);
         QueueManager::instance()->index_request.PushBack(
             Index_Request(entry.filename, entry.args, true /*is_interactive*/,
-                          *content, ICacheManager::Make()),
+                          *content),
             true);
       }
     }

src/messages/text_document_did_open.cc

@@ -38,13 +38,12 @@ struct Handler_TextDocumentDidOpen
     // NOTE: This function blocks code lens. If it starts taking a long time
     // we will need to find a way to unblock the code lens request.
     const auto& params = request->params;
-    Timer time;
     std::string path = params.textDocument.uri.GetPath();
 
-    std::shared_ptr<ICacheManager> cache_manager = ICacheManager::Make();
+    ICacheManager cache;
     WorkingFile* working_file = working_files->OnOpen(params.textDocument);
     std::optional<std::string> cached_file_contents =
-        cache_manager->LoadCachedFileContents(path);
+        cache.LoadCachedFileContents(path);
     if (cached_file_contents)
       working_file->SetIndexContent(*cached_file_contents);
 
@@ -55,10 +54,6 @@ struct Handler_TextDocumentDidOpen
       EmitSemanticHighlighting(db, semantic_cache, working_file, file);
     }
 
-    time.ResetAndPrint(
-        "[querydb] Loading cached index file for DidOpen (blocks "
-        "CodeLens)");
-
     include_complete->AddFile(working_file->filename);
     clang_complete->NotifyView(path);
     if (params.args.size())
@@ -68,9 +63,9 @@ struct Handler_TextDocumentDidOpen
     if (SourceFileLanguage(path) != LanguageId::Unknown) {
       Project::Entry entry = project->FindCompilationEntryForFile(path);
       QueueManager::instance()->index_request.PushBack(
-        Index_Request(
+          Index_Request(entry.filename,
-          entry.filename, params.args.size() ? params.args : entry.args,
+                        params.args.size() ? params.args : entry.args,
-          true /*is_interactive*/, params.textDocument.text, cache_manager),
+                        true /*is_interactive*/, params.textDocument.text),
           true /* priority */);
 
       clang_complete->FlushSession(entry.filename);

src/messages/text_document_did_save.cc

@@ -55,7 +55,7 @@ struct Handler_TextDocumentDidSave
         Project::Entry entry = project->FindCompilationEntryForFile(path);
         QueueManager::instance()->index_request.PushBack(
             Index_Request(entry.filename, entry.args, true /*is_interactive*/,
-                          *content, ICacheManager::Make()),
+                          *content),
             true);
       }
     }

src/messages/workspace_did_change_watched_files.cc

@@ -41,10 +41,14 @@ struct Handler_WorkspaceDidChangeWatchedFiles
   void Run(In_WorkspaceDidChangeWatchedFiles* request) override {
     for (lsFileEvent& event : request->params.changes) {
       std::string path = event.uri.GetPath();
+      Project::Entry entry;
+      {
+        std::lock_guard<std::mutex> lock(project->mutex_);
         auto it = project->absolute_path_to_entry_index_.find(path);
         if (it == project->absolute_path_to_entry_index_.end())
           continue;
-      const Project::Entry& entry = project->entries[it->second];
+        entry = project->entries[it->second];
+      }
       bool is_interactive =
           working_files->GetFileByFilename(entry.filename) != nullptr;
       switch (event.type) {
@@ -55,8 +59,7 @@ struct Handler_WorkspaceDidChangeWatchedFiles
             LOG_S(ERROR) << "Unable to read file content after saving " << path;
           else {
             QueueManager::instance()->index_request.PushBack(
-                Index_Request(path, entry.args, is_interactive, *content,
+                Index_Request(path, entry.args, is_interactive, *content));
-                              ICacheManager::Make()));
             if (is_interactive)
               clang_complete->NotifySave(path);
           }
@@ -64,8 +67,7 @@ struct Handler_WorkspaceDidChangeWatchedFiles
         }
         case lsFileChangeType::Deleted:
           QueueManager::instance()->index_request.PushBack(
-              Index_Request(path, entry.args, is_interactive, std::string(),
+              Index_Request(path, entry.args, is_interactive, std::string()));
-                            ICacheManager::Make()));
           break;
       }
     }

src/port.h

@@ -10,6 +10,10 @@
 #define ATTRIBUTE_UNUSED
 #endif
 
+#ifdef __clang__
+#define GUARDED_BY(x)  __attribute__((guarded_by(x)))
+#endif
+
 // TODO GCC
 #if __has_builtin(__builtin_unreachable)
 #define CCLS_BUILTIN_UNREACHABLE __builtin_unreachable()

src/project.cc

@@ -225,7 +225,7 @@ Project::Entry GetCompilationEntryFromCompileCommandEntry(
   // Add -resource-dir so clang can correctly resolve system includes like
   // <cstddef>
   if (!AnyStartsWith(result.args, "-resource-dir"))
-    result.args.push_back("-resource-dir=" + g_config->resourceDirectory);
+    result.args.push_back("-resource-dir=" + g_config->clang.resourceDir);
 
   // There could be a clang version mismatch between what the project uses and
   // what ccls uses. Make sure we do not emit warnings for mismatched options.
@@ -441,7 +441,7 @@ int ComputeGuessScore(std::string_view a, std::string_view b) {
 void Project::Load(const std::string& root_directory) {
   // Load data.
   ProjectConfig project;
-  project.extra_flags = g_config->extraClangArguments;
+  project.extra_flags = g_config->clang.extraArgs;
   project.project_dir = root_directory;
   entries = LoadCompilationEntriesFromDirectory(
       &project, g_config->compilationDatabaseDirectory);
@@ -461,14 +461,18 @@ void Project::Load(const std::string& root_directory) {
   }
 
   // Setup project entries.
+  std::lock_guard<std::mutex> lock(mutex_);
   absolute_path_to_entry_index_.reserve(entries.size());
-  for (size_t i = 0; i < entries.size(); ++i)
+  for (size_t i = 0; i < entries.size(); ++i) {
+    entries[i].id = i;
     absolute_path_to_entry_index_[entries[i].filename] = i;
+  }
 }
 
 void Project::SetFlagsForFile(
     const std::vector<std::string>& flags,
     const std::string& path) {
+  std::lock_guard<std::mutex> lock(mutex_);
   auto it = absolute_path_to_entry_index_.find(path);
   if (it != absolute_path_to_entry_index_.end()) {
     // The entry already exists in the project, just set the flags.
@@ -485,9 +489,12 @@ void Project::SetFlagsForFile(
 
 Project::Entry Project::FindCompilationEntryForFile(
     const std::string& filename) {
+  {
+    std::lock_guard<std::mutex> lock(mutex_);
     auto it = absolute_path_to_entry_index_.find(filename);
     if (it != absolute_path_to_entry_index_.end())
       return entries[it->second];
+  }
 
   // We couldn't find the file. Try to infer it.
   // TODO: Cache inferred file in a separate array (using a lock or similar)
@@ -554,8 +561,7 @@ void Project::Index(QueueManager* queue,
     }
     bool is_interactive = wfiles->GetFileByFilename(entry.filename) != nullptr;
     queue->index_request.PushBack(Index_Request(entry.filename, entry.args,
-                                                is_interactive, *content,
+                                                is_interactive, *content, id));
-                                                ICacheManager::Make(), id));
   });
 }
 
@@ -564,7 +570,7 @@ TEST_SUITE("Project") {
                   std::vector<std::string> raw,
                   std::vector<std::string> expected) {
     g_config = std::make_unique<Config>();
-    g_config->resourceDirectory = "/w/resource_dir/";
+    g_config->clang.resourceDir = "/w/resource_dir/";
     ProjectConfig project;
     project.project_dir = "/w/c/s/";
 

src/project.h

@@ -19,6 +19,7 @@ struct Project {
     std::vector<std::string> args;
     // If true, this entry is inferred and was not read from disk.
     bool is_inferred = false;
+    int id = -1;
   };
 
   // Include directories for "" headers
@@ -27,7 +28,8 @@ struct Project {
   std::vector<std::string> angle_include_directories;
 
   std::vector<Entry> entries;
-  std::unordered_map<std::string, int> absolute_path_to_entry_index_;
+  std::mutex mutex_;
+  std::unordered_map<std::string, int> absolute_path_to_entry_index_ GUARDED_BY(mutex_);
 
   // Loads a project for the given |directory|.
   //

src/query.cc

@@ -76,7 +76,9 @@ QueryFile::DefUpdate BuildFileDefUpdate(const IndexFile& indexed) {
   def.args = std::move(indexed.args);
   def.includes = std::move(indexed.includes);
   def.inactive_regions = std::move(indexed.skipped_by_preprocessor);
-  def.dependencies = std::move(indexed.dependencies);
+  def.dependencies.reserve(indexed.dependencies.size());
+  for (auto& dep : indexed.dependencies)
+    def.dependencies.push_back(dep.first);
   def.language = indexed.language;
 
   auto add_all_symbols = [&](Use use, Usr usr, SymbolKind kind) {

src/queue_manager.cc

@@ -11,13 +11,11 @@ Index_Request::Index_Request(
     const std::vector<std::string>& args,
     bool is_interactive,
     const std::string& contents,
-    const std::shared_ptr<ICacheManager>& cache_manager,
     lsRequestId id)
     : path(path),
       args(args),
       is_interactive(is_interactive),
       contents(contents),
-      cache_manager(cache_manager),
       id(id) {}
 
 Index_OnIndexed::Index_OnIndexed(IndexUpdate&& update,
@@ -51,10 +49,4 @@ QueueManager::QueueManager(MultiQueueWaiter* querydb_waiter,
     : for_stdout(stdout_waiter),
       for_querydb(querydb_waiter),
       on_indexed(querydb_waiter),
-      index_request(indexer_waiter),
+      index_request(indexer_waiter) {}
-      on_id_mapped(indexer_waiter) {}
-
-bool QueueManager::HasWork() {
-  return !index_request.IsEmpty() || !on_id_mapped.IsEmpty() ||
-         !on_indexed.IsEmpty();
-}

src/queue_manager.h

@@ -21,14 +21,12 @@ struct Index_Request {
   std::vector<std::string> args;
   bool is_interactive;
   std::string contents;  // Preloaded contents.
-  std::shared_ptr<ICacheManager> cache_manager;
   lsRequestId id;
 
   Index_Request(const std::string& path,
                 const std::vector<std::string>& args,
                 bool is_interactive,
                 const std::string& contents,
-                const std::shared_ptr<ICacheManager>& cache_manager,
                 lsRequestId id = {});
 };
 
@@ -72,8 +70,6 @@ class QueueManager {
                    MultiQueueWaiter* stdout_waiter);
   static void WriteStdout(MethodType method, lsBaseOutMessage& response);
 
-  bool HasWork();
-
   // Messages received by "stdout" thread.
   ThreadedQueue<Stdout_Request> for_stdout;
 
@@ -83,7 +79,6 @@ class QueueManager {
 
   // Runs on indexer threads.
   ThreadedQueue<Index_Request> index_request;
-  ThreadedQueue<Index_OnIdMapped> on_id_mapped;
 
  private:
   explicit QueueManager(MultiQueueWaiter* querydb_waiter,

src/serializer.cc

@@ -283,14 +283,13 @@ template <typename TVisitor>
 void Reflect(TVisitor& visitor, IndexFile& value) {
   REFLECT_MEMBER_START();
   if (!gTestOutputMode) {
-    REFLECT_MEMBER(last_modification_time);
+    REFLECT_MEMBER(last_write_time);
     REFLECT_MEMBER(language);
     REFLECT_MEMBER(import_file);
     REFLECT_MEMBER(args);
+    REFLECT_MEMBER(dependencies);
   }
   REFLECT_MEMBER(includes);
-  if (!gTestOutputMode)
-    REFLECT_MEMBER(dependencies);
   REFLECT_MEMBER(skipped_by_preprocessor);
   REFLECT_MEMBER(usr2func);
   REFLECT_MEMBER(usr2type);
@@ -314,6 +313,27 @@ void Reflect(Writer& visitor, SerializeFormat& value) {
   }
 }
 
+void Reflect(Reader& visitor, std::unordered_map<std::string, int64_t>& map) {
+  visitor.IterArray([&](Reader& entry) {
+                      std::string name;
+    Reflect(entry, name);
+    if (visitor.Format() == SerializeFormat::Binary)
+      Reflect(entry, map[name]);
+    else
+      map[name] = 0;
+  });
+}
+void Reflect(Writer& visitor, std::unordered_map<std::string, int64_t>& map) {
+  visitor.StartArray(map.size());
+  for (auto& it : map) {
+    std::string key = it.first;
+    Reflect(visitor, key);
+    if (visitor.Format() == SerializeFormat::Binary)
+      Reflect(visitor, it.second);
+  }
+  visitor.EndArray();
+}
+
 std::string Serialize(SerializeFormat format, IndexFile& file) {
   switch (format) {
     case SerializeFormat::Binary: {

src/serializer.h

@@ -262,12 +262,12 @@ void Reflect(Writer& visitor, std::vector<T>& values) {
 
 // std::unordered_map
 template <typename V>
-void Reflect(Reader& visitor, std::unordered_map<uint64_t, V>& values) {
+void Reflect(Reader& visitor, std::unordered_map<uint64_t, V>& map) {
   visitor.IterArray([&](Reader& entry) {
     V val;
     Reflect(entry, val);
     auto usr = val.usr;
-    values[usr] = std::move(val);
+    map[usr] = std::move(val);
   });
 }
 template <typename V>
@@ -281,6 +281,10 @@ void Reflect(Writer& visitor, std::unordered_map<uint64_t, V>& map) {
   visitor.EndArray();
 }
 
+// Used by IndexFile::dependencies. Timestamps are emitted for Binary.
+void Reflect(Reader& visitor, std::unordered_map<std::string, int64_t>& map);
+void Reflect(Writer& visitor, std::unordered_map<std::string, int64_t>& map);
+
 // ReflectMember
 
 template <typename T>

src/test.cc

@@ -292,9 +292,9 @@ bool RunIndexTests(const std::string& filter_path, bool enable_update) {
 
         // Run test.
         g_config = std::make_unique<Config>();
-        FileConsumerSharedState file_consumer_shared;
+        VFS vfs;
         PerformanceImportFile perf;
-        auto dbs = Parse(&file_consumer_shared, path, flags, {}, &perf, &index);
+        auto dbs = Parse(&vfs, path, flags, {}, &perf, &index);
 
         for (const auto& entry : all_expected_output) {
           const std::string& expected_path = entry.first;