Add cache.{hierarchicalPath,retainInMemory}

cache.hierarchicalPath: store cache files as $directory/a/b/c.cc.blob to
work around NAME_MAX limitation.

cache.retainInMemory: after this number of loads, keep a copy of file
index in memory. If set to 1, it avoids cache corruption if the index
file is changed after the initial load, which may happen if several
language clients open the same project and share the same cache
directory.

Also rename cacheDirectory cacheFormat to cache.{directory,format}

src/config.hh

@@ -40,19 +40,41 @@ struct Config {
   std::string compilationDatabaseCommand;
   // Directory containing compile_commands.json.
   std::string compilationDatabaseDirectory;
+
+  struct Cache {
     // Cache directory for indexed files, either absolute or relative to the
     // project root.
-  // If empty, cache will be stored in memory.
+    //
-  std::string cacheDirectory = ".ccls-cache";
+    // If empty, retainInMemory will be set to 1 and cache will be stored in
+    // memory.
+    std::string directory = ".ccls-cache";
+
     // Cache serialization format.
     //
-  // "json" generates `cacheDirectory/.../xxx.json` files which can be pretty
+    // "json" generates $directory/.../xxx.json files which can be pretty
     // printed with jq.
     //
     // "binary" uses a compact binary serialization format.
-  // It is not schema-aware and you need to re-index whenever an internal struct
+    // It is not schema-aware and you need to re-index whenever an internal
-  // member has changed.
+    // struct member has changed.
-  SerializeFormat cacheFormat = SerializeFormat::Binary;
+    SerializeFormat format = SerializeFormat::Binary;
+
+    // If false, store cache files as $directory/@a@b/c.cc.blob
+    //
+    // If true, $directory/a/b/c.cc.blob. If cache.directory is absolute, make
+    // sure different projects use different cache.directory as they would have
+    // conflicting cache files for system headers.
+    bool hierarchicalPath = false;
+
+    // After this number of loads, keep a copy of file index in memory (which
+    // increases memory usage). During incremental updates, the index subtracted
+    // will come from the in-memory copy, instead of the on-disk file.
+    //
+    // The initial load or a save action is counted as one load.
+    // 0: never retain; 1: retain after initial load; 2: retain after 2 loads
+    // (initial load+first save)
+    int retainInMemory = 2;
+  } cache;
 
   struct ServerCap {
     struct DocumentOnTypeFormattingOptions {
@@ -292,6 +314,8 @@ struct Config {
     int maxNum = 2000;
   } xref;
 };
+REFLECT_STRUCT(Config::Cache, directory, format, hierarchicalPath,
+               retainInMemory);
 REFLECT_STRUCT(Config::ServerCap::DocumentOnTypeFormattingOptions,
                firstTriggerCharacter, moreTriggerCharacter);
 REFLECT_STRUCT(Config::ServerCap::Workspace::WorkspaceFolders, supported,
@@ -321,9 +345,9 @@ REFLECT_STRUCT(Config::Session, maxNum);
 REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort);
 REFLECT_STRUCT(Config::Xref, maxNum);
 REFLECT_STRUCT(Config, compilationDatabaseCommand, compilationDatabaseDirectory,
-               cacheDirectory, cacheFormat, capabilities, clang, client,
+               cache, capabilities, clang, client, codeLens, completion,
-               codeLens, completion, diagnostics, highlight, index, request,
+               diagnostics, highlight, index, request, session, workspaceSymbol,
-               session, workspaceSymbol, xref);
+               xref);
 
 extern Config *g_config;
 

src/messages/initialize.cc

@@ -293,12 +293,12 @@ void Initialize(MessageHandler *m, InitializeParam &param, ReplyOnce &reply) {
     Reflect(json_writer, *g_config);
     LOG_S(INFO) << "initializationOptions: " << output.GetString();
 
-    if (g_config->cacheDirectory.size()) {
+    if (g_config->cache.directory.size()) {
-      SmallString<256> Path(g_config->cacheDirectory);
+      SmallString<256> Path(g_config->cache.directory);
       sys::fs::make_absolute(project_path, Path);
       // Use upper case for the Driver letter on Windows.
-      g_config->cacheDirectory = NormalizePath(Path.str());
+      g_config->cache.directory = NormalizePath(Path.str());
-      EnsureEndsInSlash(g_config->cacheDirectory);
+      EnsureEndsInSlash(g_config->cache.directory);
     }
   }
 
@@ -345,13 +345,16 @@ void Initialize(MessageHandler *m, InitializeParam &param, ReplyOnce &reply) {
   }
   if (param.workspaceFolders.empty())
     g_config->workspaceFolders.push_back(project_path);
-  if (g_config->cacheDirectory.size())
+
+  if (g_config->cache.directory.empty())
+    g_config->cache.retainInMemory = 1;
+  else if (!g_config->cache.hierarchicalPath)
     for (const std::string &folder : g_config->workspaceFolders) {
       // Create two cache directories for files inside and outside of the
       // project.
       std::string escaped = EscapeFileName(folder.substr(0, folder.size() - 1));
-      sys::fs::create_directories(g_config->cacheDirectory + escaped);
+      sys::fs::create_directories(g_config->cache.directory + escaped);
-      sys::fs::create_directories(g_config->cacheDirectory + '@' + escaped);
+      sys::fs::create_directories(g_config->cache.directory + '@' + escaped);
     }
 
   idx::Init();

src/messages/textDocument_did.cc

@@ -35,6 +35,7 @@ void MessageHandler::textDocument_didClose(TextDocumentParam &param) {
   std::string path = param.textDocument.uri.GetPath();
   wfiles->OnClose(path);
   manager->OnClose(path);
+  pipeline::RemoveCache(path);
 }
 
 void MessageHandler::textDocument_didOpen(DidOpenTextDocumentParam &param) {

src/messages/workspace.cc

@@ -46,8 +46,8 @@ void MessageHandler::workspace_didChangeWatchedFiles(
     DidChangeWatchedFilesParam &param) {
   for (auto &event : param.changes) {
     std::string path = event.uri.GetPath();
-    if ((g_config->cacheDirectory.size() &&
+    if ((g_config->cache.directory.size() &&
-         StringRef(path).startswith(g_config->cacheDirectory)) ||
+         StringRef(path).startswith(g_config->cache.directory)) ||
         lookupExtension(path).first == LanguageId::Unknown)
       return;
     for (std::string cur = path; cur.size(); cur = sys::path::parent_path(cur))

src/pipeline.cc

@@ -57,7 +57,7 @@ void VFS::Clear() {
   state.clear();
 }
 
-bool VFS::Loaded(const std::string &path) {
+int VFS::Loaded(const std::string &path) {
   std::lock_guard lock(mutex);
   return state[path].loaded;
 }
@@ -137,7 +137,7 @@ bool CacheInvalid(VFS *vfs, IndexFile *prev, const std::string &path,
 };
 
 std::string AppendSerializationFormat(const std::string &base) {
-  switch (g_config->cacheFormat) {
+  switch (g_config->cache.format) {
   case SerializeFormat::Binary:
     return base + ".blob";
   case SerializeFormat::Json:
@@ -145,27 +145,35 @@ std::string AppendSerializationFormat(const std::string &base) {
   }
 }
 
-std::string GetCachePath(const std::string &source_file) {
+std::string GetCachePath(const std::string &src) {
-  for (auto &root : g_config->workspaceFolders)
+  if (g_config->cache.hierarchicalPath) {
-    if (StringRef(source_file).startswith(root)) {
+    std::string ret = src[0] == '/' ? src.substr(1) : src;
-      auto len = root.size();
+#ifdef _WIN32
-      return g_config->cacheDirectory +
+    std::replace(ret.begin(), ret.end(), ':', '@');
-             EscapeFileName(root.substr(0, len - 1)) + '/' +
+#endif
-             EscapeFileName(source_file.substr(len));
+    return g_config->cache.directory + ret;
   }
-  return g_config->cacheDirectory + '@' +
+  for (auto &root : g_config->workspaceFolders)
+    if (StringRef(src).startswith(root)) {
+      auto len = root.size();
+      return g_config->cache.directory +
+             EscapeFileName(root.substr(0, len - 1)) + '/' +
+             EscapeFileName(src.substr(len));
+    }
+  return g_config->cache.directory + '@' +
          EscapeFileName(g_config->fallbackFolder.substr(
              0, g_config->fallbackFolder.size() - 1)) +
-         '/' + EscapeFileName(source_file);
+         '/' + EscapeFileName(src);
 }
 
 std::unique_ptr<IndexFile> RawCacheLoad(const std::string &path) {
-  if (g_config->cacheDirectory.empty()) {
+  if (g_config->cache.retainInMemory) {
     std::shared_lock lock(g_index_mutex);
     auto it = g_index.find(path);
-    if (it == g_index.end())
+    if (it != g_index.end())
-      return nullptr;
       return std::make_unique<IndexFile>(it->second.index);
+    if (g_config->cache.directory.empty())
+      return nullptr;
   }
 
   std::string cache_path = GetCachePath(path);
@@ -175,7 +183,7 @@ std::unique_ptr<IndexFile> RawCacheLoad(const std::string &path) {
   if (!file_content || !serialized_indexed_content)
     return nullptr;
 
-  return ccls::Deserialize(g_config->cacheFormat, path,
+  return ccls::Deserialize(g_config->cache.format, path,
                            *serialized_indexed_content, *file_content,
                            IndexFile::kMajorVersion);
 }
@@ -287,7 +295,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
         request.mode != IndexMode::NonInteractive);
       {
         std::lock_guard lock1(vfs->mutex);
-        vfs->state[path_to_index].loaded = true;
+        vfs->state[path_to_index].loaded++;
       }
       lock.unlock();
 
@@ -304,7 +312,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
           VFS::State &st = vfs->state[path];
           if (st.loaded)
             continue;
-          st.loaded = true;
+          st.loaded++;
           st.timestamp = prev->mtime;
         }
         IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
@@ -367,31 +375,37 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
                            << " (delta: " << !!prev << ")";
     {
       std::lock_guard lock(GetFileMutex(path));
-      if (vfs->Loaded(path))
+      int loaded = vfs->Loaded(path), retain = g_config->cache.retainInMemory;
+      if (loaded)
         prev = RawCacheLoad(path);
       else
         prev.reset();
-      if (g_config->cacheDirectory.empty()) {
+      if (retain > 0 && retain <= loaded + 1) {
         std::lock_guard lock(g_index_mutex);
         auto it = g_index.insert_or_assign(
           path, InMemoryIndexFile{curr->file_contents, *curr});
         std::string().swap(it.first->second.index.file_contents);
-      } else {
+      }
+      if (g_config->cache.directory.size()) {
         std::string cache_path = GetCachePath(path);
         if (deleted) {
           (void)sys::fs::remove(cache_path);
           (void)sys::fs::remove(AppendSerializationFormat(cache_path));
         } else {
+          if (g_config->cache.hierarchicalPath)
+            sys::fs::create_directories(
+                sys::path::parent_path(cache_path, sys::path::Style::posix),
+                true);
           WriteToFile(cache_path, curr->file_contents);
           WriteToFile(AppendSerializationFormat(cache_path),
-                      Serialize(g_config->cacheFormat, *curr));
+                      Serialize(g_config->cache.format, *curr));
         }
       }
       on_indexed->PushBack(IndexUpdate::CreateDelta(prev.get(), curr.get()),
                            request.mode != IndexMode::NonInteractive);
       {
         std::lock_guard lock1(vfs->mutex);
-        vfs->state[path].loaded = true;
+        vfs->state[path].loaded++;
       }
       if (entry.id >= 0) {
         std::lock_guard lock(project->mtx);
@@ -718,8 +732,15 @@ void Index(const std::string &path, const std::vector<const char *> &args,
                           mode != IndexMode::NonInteractive);
 }
 
+void RemoveCache(const std::string &path) {
+  if (g_config->cache.directory.size()) {
+    std::lock_guard lock(g_index_mutex);
+    g_index.erase(path);
+  }
+}
+
 std::optional<std::string> LoadIndexedContent(const std::string &path) {
-  if (g_config->cacheDirectory.empty()) {
+  if (g_config->cache.directory.empty()) {
     std::shared_lock lock(g_index_mutex);
     auto it = g_index.find(path);
     if (it == g_index.end())

src/pipeline.hh

@@ -19,13 +19,13 @@ struct VFS {
   struct State {
     int64_t timestamp;
     int step;
-    bool loaded;
+    int loaded;
   };
   std::unordered_map<std::string, State> state;
   std::mutex mutex;
 
   void Clear();
-  bool Loaded(const std::string &path);
+  int Loaded(const std::string &path);
   bool Stamp(const std::string &path, int64_t ts, int step);
 };
 
@@ -52,7 +52,7 @@ void Standalone(const std::string &root);
 
 void Index(const std::string &path, const std::vector<const char *> &args,
            IndexMode mode, bool must_exist, RequestId id = {});
-
+void RemoveCache(const std::string &path);
 std::optional<std::string> LoadIndexedContent(const std::string& path);
 
 void NotifyOrRequest(const char *method, bool request,

src/utils.cc

@@ -124,9 +124,10 @@ void EnsureEndsInSlash(std::string &path) {
 
 std::string EscapeFileName(std::string path) {
   bool slash = path.size() && path.back() == '/';
-  for (char &c : path)
+#ifdef _WIN32
-    if (c == '\\' || c == '/' || c == ':')
+  std::replace(path.begin(), path.end(), ':', '@');
-      c = '@';
+#endif
+  std::replace(path.begin(), path.end(), '/', '@');
   if (slash)
     path += '/';
   return path;