From 2fc2d1b16ca239f4d56ced02cbdb885ef345dce1 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 21 Feb 2019 23:46:20 +0800 Subject: [PATCH] Add cache.{hierarchicalPath,retainInMemory} cache.hierarchicalPath: store cache files as $directory/a/b/c.cc.blob to work around NAME_MAX limitation. cache.retainInMemory: the number of loads before keeping a copy of file index in memory. During incremental updates, the removed file index will be taken from the in-memory copy. If set to 1, it avoids cache corruption if the index file is changed after the initial load, which may happen if several language clients open the same project and share the same cache directory. Also rename cacheDirectory cacheFormat to cache.{directory,format} --- src/config.hh | 56 +++++++++++++++++++++++++++----------- src/messages/initialize.cc | 17 +++++++----- src/messages/workspace.cc | 4 +-- src/pipeline.cc | 55 +++++++++++++++++++++++-------------- src/pipeline.hh | 4 +-- src/utils.cc | 7 +++-- 6 files changed, 93 insertions(+), 50 deletions(-) diff --git a/src/config.hh b/src/config.hh index 3e9aa793..0daeec82 100644 --- a/src/config.hh +++ b/src/config.hh @@ -40,19 +40,41 @@ struct Config { std::string compilationDatabaseCommand; // Directory containing compile_commands.json. std::string compilationDatabaseDirectory; - // Cache directory for indexed files, either absolute or relative to the - // project root. - // If empty, cache will be stored in memory. - std::string cacheDirectory = ".ccls-cache"; - // Cache serialization format. - // - // "json" generates `cacheDirectory/.../xxx.json` files which can be pretty - // printed with jq. - // - // "binary" uses a compact binary serialization format. - // It is not schema-aware and you need to re-index whenever an internal struct - // member has changed. - SerializeFormat cacheFormat = SerializeFormat::Binary; + + struct Cache { + // Cache directory for indexed files, either absolute or relative to the + // project root. + // + // If empty, retainInMemory will be set to 1 and cache will be stored in + // memory. + std::string directory = ".ccls-cache"; + + // Cache serialization format. + // + // "json" generates $directory/.../xxx.json files which can be pretty + // printed with jq. + // + // "binary" uses a compact binary serialization format. + // It is not schema-aware and you need to re-index whenever an internal + // struct member has changed. + SerializeFormat format = SerializeFormat::Binary; + + // If false, store cache files as $directory/@a@b/c.cc.blob + // + // If true, $directory/a/b/c.cc.blob. If cache.directory is absolute, make + // sure different projects use different cache.directory as they would have + // conflicting cache files for system headers. + bool hierarchicalPath = false; + + // After this number of loads, keep a copy of file index in memory (which + // increases memory usage). During incremental updates, the removed file + // index will be taken from the in-memory copy, instead of the on-disk file. + // + // Every index action is counted: the initial load, a save action. + // 0: never retain; 1: retain after initial load; 2: retain after 2 loads + // (initial load+first save) + int retainInMemory = 1; + } cache; struct ServerCap { struct DocumentOnTypeFormattingOptions { @@ -292,6 +314,8 @@ struct Config { int maxNum = 2000; } xref; }; +REFLECT_STRUCT(Config::Cache, directory, format, hierarchicalPath, + retainInMemory); REFLECT_STRUCT(Config::ServerCap::DocumentOnTypeFormattingOptions, firstTriggerCharacter, moreTriggerCharacter); REFLECT_STRUCT(Config::ServerCap::Workspace::WorkspaceFolders, supported, @@ -321,9 +345,9 @@ REFLECT_STRUCT(Config::Session, maxNum); REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort); REFLECT_STRUCT(Config::Xref, maxNum); REFLECT_STRUCT(Config, compilationDatabaseCommand, compilationDatabaseDirectory, - cacheDirectory, cacheFormat, capabilities, clang, client, - codeLens, completion, diagnostics, highlight, index, request, - session, workspaceSymbol, xref); + cache, capabilities, clang, client, codeLens, completion, + diagnostics, highlight, index, request, session, workspaceSymbol, + xref); extern Config *g_config; diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc index 707798d0..437895db 100644 --- a/src/messages/initialize.cc +++ b/src/messages/initialize.cc @@ -293,12 +293,12 @@ void Initialize(MessageHandler *m, InitializeParam ¶m, ReplyOnce &reply) { Reflect(json_writer, *g_config); LOG_S(INFO) << "initializationOptions: " << output.GetString(); - if (g_config->cacheDirectory.size()) { - SmallString<256> Path(g_config->cacheDirectory); + if (g_config->cache.directory.size()) { + SmallString<256> Path(g_config->cache.directory); sys::fs::make_absolute(project_path, Path); // Use upper case for the Driver letter on Windows. - g_config->cacheDirectory = NormalizePath(Path.str()); - EnsureEndsInSlash(g_config->cacheDirectory); + g_config->cache.directory = NormalizePath(Path.str()); + EnsureEndsInSlash(g_config->cache.directory); } } @@ -345,13 +345,16 @@ void Initialize(MessageHandler *m, InitializeParam ¶m, ReplyOnce &reply) { } if (param.workspaceFolders.empty()) g_config->workspaceFolders.push_back(project_path); - if (g_config->cacheDirectory.size()) + + if (g_config->cache.directory.empty()) + g_config->cache.retainInMemory = 1; + else if (!g_config->cache.hierarchicalPath) for (const std::string &folder : g_config->workspaceFolders) { // Create two cache directories for files inside and outside of the // project. std::string escaped = EscapeFileName(folder.substr(0, folder.size() - 1)); - sys::fs::create_directories(g_config->cacheDirectory + escaped); - sys::fs::create_directories(g_config->cacheDirectory + '@' + escaped); + sys::fs::create_directories(g_config->cache.directory + escaped); + sys::fs::create_directories(g_config->cache.directory + '@' + escaped); } idx::Init(); diff --git a/src/messages/workspace.cc b/src/messages/workspace.cc index 5435d98f..67e3e8a7 100644 --- a/src/messages/workspace.cc +++ b/src/messages/workspace.cc @@ -46,8 +46,8 @@ void MessageHandler::workspace_didChangeWatchedFiles( DidChangeWatchedFilesParam ¶m) { for (auto &event : param.changes) { std::string path = event.uri.GetPath(); - if ((g_config->cacheDirectory.size() && - StringRef(path).startswith(g_config->cacheDirectory)) || + if ((g_config->cache.directory.size() && + StringRef(path).startswith(g_config->cache.directory)) || lookupExtension(path).first == LanguageId::Unknown) return; for (std::string cur = path; cur.size(); cur = sys::path::parent_path(cur)) diff --git a/src/pipeline.cc b/src/pipeline.cc index fa8a3fa8..4d7a1515 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -57,7 +57,7 @@ void VFS::Clear() { state.clear(); } -bool VFS::Loaded(const std::string &path) { +int VFS::Loaded(const std::string &path) { std::lock_guard lock(mutex); return state[path].loaded; } @@ -137,7 +137,7 @@ bool CacheInvalid(VFS *vfs, IndexFile *prev, const std::string &path, }; std::string AppendSerializationFormat(const std::string &base) { - switch (g_config->cacheFormat) { + switch (g_config->cache.format) { case SerializeFormat::Binary: return base + ".blob"; case SerializeFormat::Json: @@ -145,27 +145,36 @@ std::string AppendSerializationFormat(const std::string &base) { } } -std::string GetCachePath(const std::string &source_file) { +std::string GetCachePath(const std::string &src) { + if (g_config->cache.hierarchicalPath) { + std::string ret = + g_config->cache.directory + (src[0] == '/' ? src.substr(1) : src); +#ifdef _WIN32 + std::replace(ret.begin(), ret.end(), ':', '@'); +#endif + return ret; + } for (auto &root : g_config->workspaceFolders) - if (StringRef(source_file).startswith(root)) { + if (StringRef(src).startswith(root)) { auto len = root.size(); - return g_config->cacheDirectory + + return g_config->cache.directory + EscapeFileName(root.substr(0, len - 1)) + '/' + - EscapeFileName(source_file.substr(len)); + EscapeFileName(src.substr(len)); } - return g_config->cacheDirectory + '@' + + return g_config->cache.directory + '@' + EscapeFileName(g_config->fallbackFolder.substr( 0, g_config->fallbackFolder.size() - 1)) + - '/' + EscapeFileName(source_file); + '/' + EscapeFileName(src); } std::unique_ptr RawCacheLoad(const std::string &path) { - if (g_config->cacheDirectory.empty()) { + if (g_config->cache.retainInMemory) { std::shared_lock lock(g_index_mutex); auto it = g_index.find(path); - if (it == g_index.end()) + if (it != g_index.end()) + return std::make_unique(it->second.index); + if (g_config->cache.directory.empty()) return nullptr; - return std::make_unique(it->second.index); } std::string cache_path = GetCachePath(path); @@ -175,7 +184,7 @@ std::unique_ptr RawCacheLoad(const std::string &path) { if (!file_content || !serialized_indexed_content) return nullptr; - return ccls::Deserialize(g_config->cacheFormat, path, + return ccls::Deserialize(g_config->cache.format, path, *serialized_indexed_content, *file_content, IndexFile::kMajorVersion); } @@ -287,7 +296,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, request.mode != IndexMode::NonInteractive); { std::lock_guard lock1(vfs->mutex); - vfs->state[path_to_index].loaded = true; + vfs->state[path_to_index].loaded++; } lock.unlock(); @@ -304,7 +313,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, VFS::State &st = vfs->state[path]; if (st.loaded) continue; - st.loaded = true; + st.loaded++; st.timestamp = prev->mtime; } IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); @@ -367,31 +376,37 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, << " (delta: " << !!prev << ")"; { std::lock_guard lock(GetFileMutex(path)); - if (vfs->Loaded(path)) + int loaded = vfs->Loaded(path), retain = g_config->cache.retainInMemory; + if (loaded) prev = RawCacheLoad(path); else prev.reset(); - if (g_config->cacheDirectory.empty()) { + if (retain > 0 && retain <= loaded + 1) { std::lock_guard lock(g_index_mutex); auto it = g_index.insert_or_assign( path, InMemoryIndexFile{curr->file_contents, *curr}); std::string().swap(it.first->second.index.file_contents); - } else { + } + if (g_config->cache.directory.size()) { std::string cache_path = GetCachePath(path); if (deleted) { (void)sys::fs::remove(cache_path); (void)sys::fs::remove(AppendSerializationFormat(cache_path)); } else { + if (g_config->cache.hierarchicalPath) + sys::fs::create_directories( + sys::path::parent_path(cache_path, sys::path::Style::posix), + true); WriteToFile(cache_path, curr->file_contents); WriteToFile(AppendSerializationFormat(cache_path), - Serialize(g_config->cacheFormat, *curr)); + Serialize(g_config->cache.format, *curr)); } } on_indexed->PushBack(IndexUpdate::CreateDelta(prev.get(), curr.get()), request.mode != IndexMode::NonInteractive); { std::lock_guard lock1(vfs->mutex); - vfs->state[path].loaded = true; + vfs->state[path].loaded++; } if (entry.id >= 0) { std::shared_lock lock(project->mtx); @@ -719,7 +734,7 @@ void Index(const std::string &path, const std::vector &args, } std::optional LoadIndexedContent(const std::string &path) { - if (g_config->cacheDirectory.empty()) { + if (g_config->cache.directory.empty()) { std::shared_lock lock(g_index_mutex); auto it = g_index.find(path); if (it == g_index.end()) diff --git a/src/pipeline.hh b/src/pipeline.hh index c61c6dc2..72964fb9 100644 --- a/src/pipeline.hh +++ b/src/pipeline.hh @@ -19,13 +19,13 @@ struct VFS { struct State { int64_t timestamp; int step; - bool loaded; + int loaded; }; std::unordered_map state; std::mutex mutex; void Clear(); - bool Loaded(const std::string &path); + int Loaded(const std::string &path); bool Stamp(const std::string &path, int64_t ts, int step); }; diff --git a/src/utils.cc b/src/utils.cc index 48e9c0d0..f37ac58e 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -124,9 +124,10 @@ void EnsureEndsInSlash(std::string &path) { std::string EscapeFileName(std::string path) { bool slash = path.size() && path.back() == '/'; - for (char &c : path) - if (c == '\\' || c == '/' || c == ':') - c = '@'; +#ifdef _WIN32 + std::replace(path.begin(), path.end(), ':', '@'); +#endif + std::replace(path.begin(), path.end(), '/', '@'); if (slash) path += '/'; return path;