From 49e042e070b8ab87fe3935270550c62dab744ea7 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 5 May 2018 15:29:17 -0700 Subject: [PATCH] Redesign import_pipeline.cc and mitigate race (duplicate Query*::uses for initial indexing) --- CMakeLists.txt | 1 - src/cache_manager.cc | 163 ++---- src/cache_manager.h | 30 +- src/clang_complete.cc | 2 - src/clang_complete.h | 2 - src/clang_indexer.cc | 38 +- src/command_line.cc | 18 +- src/config.cc | 1 + src/config.h | 54 +- src/file_consumer.cc | 50 +- src/file_consumer.h | 23 +- src/import_pipeline.cc | 548 ++++-------------- src/import_pipeline.h | 41 +- src/indexer.h | 14 +- src/message_handler.cc | 10 +- src/message_handler.h | 8 +- src/messages/ccls_call_hierarchy.cc | 2 + src/messages/ccls_freshen_index.cc | 24 +- src/messages/ccls_index_file.cc | 40 -- src/messages/ccls_member_hierarchy.cc | 2 + src/messages/initialize.cc | 10 +- src/messages/text_document_did_change.cc | 2 +- src/messages/text_document_did_open.cc | 17 +- src/messages/text_document_did_save.cc | 2 +- .../workspace_did_change_watched_files.cc | 18 +- src/port.h | 4 + src/project.cc | 24 +- src/project.h | 4 +- src/query.cc | 4 +- src/queue_manager.cc | 10 +- src/queue_manager.h | 5 - src/serializer.cc | 26 +- src/serializer.h | 8 +- src/test.cc | 4 +- 34 files changed, 365 insertions(+), 844 deletions(-) delete mode 100644 src/messages/ccls_index_file.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index ae4500b2..13659132 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -244,7 +244,6 @@ target_sources(ccls PRIVATE src/messages/ccls_derived.cc src/messages/ccls_file_info.cc src/messages/ccls_freshen_index.cc - src/messages/ccls_index_file.cc src/messages/ccls_inheritance_hierarchy.cc src/messages/ccls_member_hierarchy.cc src/messages/ccls_random.cc diff --git a/src/cache_manager.cc b/src/cache_manager.cc index 2c67b63a..79a159a4 100644 --- a/src/cache_manager.cc +++ b/src/cache_manager.cc @@ -11,129 +11,54 @@ #include namespace { +std::string GetCachePath(const std::string& source_file) { + assert(!g_config->cacheDirectory.empty()); + std::string cache_file; + size_t len = g_config->projectRoot.size(); + if (StartsWith(source_file, g_config->projectRoot)) { + cache_file = EscapeFileName(g_config->projectRoot) + + EscapeFileName(source_file.substr(len)); + } else { + cache_file = '@' + EscapeFileName(g_config->projectRoot) + + EscapeFileName(source_file); + } + + return g_config->cacheDirectory + cache_file; +} + +std::string AppendSerializationFormat(const std::string& base) { + switch (g_config->cacheFormat) { + case SerializeFormat::Binary: + return base + ".blob"; + case SerializeFormat::Json: + return base + ".json"; + } +} +} // Manages loading caches from file paths for the indexer process. -struct RealCacheManager : ICacheManager { - explicit RealCacheManager() {} - ~RealCacheManager() override = default; +void ICacheManager::WriteToCache(IndexFile& file) { + std::string cache_path = GetCachePath(file.path); + WriteToFile(cache_path, file.file_contents); - void WriteToCache(IndexFile& file) override { - std::string cache_path = GetCachePath(file.path); - WriteToFile(cache_path, file.file_contents); - - std::string indexed_content = Serialize(g_config->cacheFormat, file); - WriteToFile(AppendSerializationFormat(cache_path), indexed_content); - } - - std::optional LoadCachedFileContents( - const std::string& path) override { - return ReadContent(GetCachePath(path)); - } - - std::unique_ptr RawCacheLoad(const std::string& path) override { - std::string cache_path = GetCachePath(path); - std::optional file_content = ReadContent(cache_path); - std::optional serialized_indexed_content = - ReadContent(AppendSerializationFormat(cache_path)); - if (!file_content || !serialized_indexed_content) - return nullptr; - - return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content, - *file_content, IndexFile::kMajorVersion); - } - - std::string GetCachePath(const std::string& source_file) { - assert(!g_config->cacheDirectory.empty()); - std::string cache_file; - size_t len = g_config->projectRoot.size(); - if (StartsWith(source_file, g_config->projectRoot)) { - cache_file = EscapeFileName(g_config->projectRoot) + - EscapeFileName(source_file.substr(len)); - } else { - cache_file = '@' + EscapeFileName(g_config->projectRoot) + - EscapeFileName(source_file); - } - - return g_config->cacheDirectory + cache_file; - } - - std::string AppendSerializationFormat(const std::string& base) { - switch (g_config->cacheFormat) { - case SerializeFormat::Binary: - return base + ".blob"; - case SerializeFormat::Json: - return base + ".json"; - } - } -}; - -struct FakeCacheManager : ICacheManager { - explicit FakeCacheManager(const std::vector& entries) - : entries_(entries) {} - - void WriteToCache(IndexFile& file) override { assert(false); } - - std::optional LoadCachedFileContents( - const std::string& path) override { - for (const FakeCacheEntry& entry : entries_) { - if (entry.path == path) { - return entry.content; - } - } - - return std::nullopt; - } - - std::unique_ptr RawCacheLoad(const std::string& path) override { - for (const FakeCacheEntry& entry : entries_) { - if (entry.path == path) { - return Deserialize(SerializeFormat::Json, path, entry.json, "", - std::nullopt); - } - } - - return nullptr; - } - - std::vector entries_; -}; - -} // namespace - -// static -std::shared_ptr ICacheManager::Make() { - return std::make_shared(); + std::string indexed_content = Serialize(g_config->cacheFormat, file); + WriteToFile(AppendSerializationFormat(cache_path), indexed_content); } -// static -std::shared_ptr ICacheManager::MakeFake( - const std::vector& entries) { - return std::make_shared(entries); -} - -ICacheManager::~ICacheManager() = default; - -IndexFile* ICacheManager::TryLoad(const std::string& path) { - auto it = caches_.find(path); - if (it != caches_.end()) - return it->second.get(); - - std::unique_ptr cache = RawCacheLoad(path); - if (!cache) - return nullptr; - - caches_[path] = std::move(cache); - return caches_[path].get(); -} - -std::unique_ptr ICacheManager::TryTakeOrLoad( +std::optional ICacheManager::LoadCachedFileContents( const std::string& path) { - auto it = caches_.find(path); - if (it != caches_.end()) { - auto result = std::move(it->second); - caches_.erase(it); - return result; - } - - return RawCacheLoad(path); + return ReadContent(GetCachePath(path)); +} + +std::unique_ptr ICacheManager::RawCacheLoad( + const std::string& path) { + std::string cache_path = GetCachePath(path); + std::optional file_content = ReadContent(cache_path); + std::optional serialized_indexed_content = + ReadContent(AppendSerializationFormat(cache_path)); + if (!file_content || !serialized_indexed_content) + return nullptr; + + return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content, + *file_content, IndexFile::kMajorVersion); } diff --git a/src/cache_manager.h b/src/cache_manager.h index a410d608..daffdd48 100644 --- a/src/cache_manager.h +++ b/src/cache_manager.h @@ -7,34 +7,12 @@ #include #include -struct Config; struct IndexFile; struct ICacheManager { - struct FakeCacheEntry { - std::string path; - std::string content; - std::string json; - }; + void WriteToCache(IndexFile& file); - static std::shared_ptr Make(); - static std::shared_ptr MakeFake( - const std::vector& entries); - - virtual ~ICacheManager(); - - // Tries to load a cache for |path|, returning null if there is none. The - // cache loader still owns the cache. - IndexFile* TryLoad(const std::string& path); - - // Takes the existing cache or loads the cache at |path|. May return null if - // the cache does not exist. - std::unique_ptr TryTakeOrLoad(const std::string& path); - - virtual void WriteToCache(IndexFile& file) = 0; - - virtual std::optional LoadCachedFileContents( - const std::string& path) = 0; + std::optional LoadCachedFileContents(const std::string& path); template void IterateLoadedCaches(Fn fn) { @@ -42,7 +20,7 @@ struct ICacheManager { fn(cache.second.get()); } - protected: - virtual std::unique_ptr RawCacheLoad(const std::string& path) = 0; + std::unique_ptr RawCacheLoad(const std::string& path); + std::unordered_map> caches_; }; diff --git a/src/clang_complete.cc b/src/clang_complete.cc index 25eec8e7..02961826 100644 --- a/src/clang_complete.cc +++ b/src/clang_complete.cc @@ -647,12 +647,10 @@ void DiagnosticQueryMain(ClangCompleteManager* completion_manager) { ClangCompleteManager::ClangCompleteManager(Project* project, WorkingFiles* working_files, OnDiagnostic on_diagnostic, - OnIndex on_index, OnDropped on_dropped) : project_(project), working_files_(working_files), on_diagnostic_(on_diagnostic), - on_index_(on_index), on_dropped_(on_dropped), preloaded_sessions_(kMaxPreloadedSessions), completion_sessions_(kMaxCompletionSessions) { diff --git a/src/clang_complete.h b/src/clang_complete.h index 7eace626..ee0cd6a7 100644 --- a/src/clang_complete.h +++ b/src/clang_complete.h @@ -82,7 +82,6 @@ struct ClangCompleteManager { ClangCompleteManager(Project* project, WorkingFiles* working_files, OnDiagnostic on_diagnostic, - OnIndex on_index, OnDropped on_dropped); // Start a code completion at the given location. |on_complete| will run when @@ -127,7 +126,6 @@ struct ClangCompleteManager { Project* project_; WorkingFiles* working_files_; OnDiagnostic on_diagnostic_; - OnIndex on_index_; OnDropped on_dropped_; using LruSessionCache = LruCache; diff --git a/src/clang_indexer.cc b/src/clang_indexer.cc index a5df8596..355cda29 100644 --- a/src/clang_indexer.cc +++ b/src/clang_indexer.cc @@ -9,11 +9,12 @@ #include +#include #include #include #include -#include #include +#include #if CINDEX_VERSION >= 47 #define CINDEX_HAVE_PRETTY 1 @@ -279,7 +280,7 @@ struct IndexParam { std::unordered_set seen_cx_files; std::vector seen_files; std::unordered_map file_contents; - std::unordered_map file_modification_times; + std::unordered_map file2write_time; // Only use this when strictly needed (ie, primary translation unit is // needed). Most logic should get the IndexFile instance via @@ -373,11 +374,11 @@ IndexFile* ConsumeFile(IndexParam* param, CXFile file) { param->seen_files.push_back(file_name); // Set modification time. - std::optional modification_time = LastWriteTime(file_name); - LOG_IF_S(ERROR, !modification_time) - << "Failed fetching modification time for " << file_name; - if (modification_time) - param->file_modification_times[file_name] = *modification_time; + std::optional write_time = LastWriteTime(file_name); + LOG_IF_S(ERROR, !write_time) << "failed to fetch write time for " + << file_name; + if (write_time) + param->file2write_time[file_name] = *write_time; } } @@ -2010,7 +2011,7 @@ void OnIndexReference(CXClientData client_data, const CXIdxEntityRefInfo* ref) { } std::vector> Parse( - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, std::string file, const std::vector& args, const std::vector& file_contents, @@ -2041,12 +2042,11 @@ std::vector> Parse( perf->index_parse = timer.ElapsedMicrosecondsAndReset(); - return ParseWithTu(file_consumer_shared, perf, tu.get(), index, file, - args, unsaved_files); + return ParseWithTu(vfs, perf, tu.get(), index, file, args, unsaved_files); } std::vector> ParseWithTu( - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, PerformanceImportFile* perf, ClangTranslationUnit* tu, ClangIndex* index, @@ -2067,7 +2067,7 @@ std::vector> ParseWithTu( callback.indexDeclaration = &OnIndexDeclaration; callback.indexEntityReference = &OnIndexReference; - FileConsumer file_consumer(file_consumer_shared, file); + FileConsumer file_consumer(vfs, file); IndexParam param(tu, &file_consumer); for (const CXUnsavedFile& contents : file_contents) { param.file_contents[contents.Filename] = FileContents( @@ -2140,15 +2140,13 @@ std::vector> ParseWithTu( } // Update file contents and modification time. - entry->last_modification_time = param.file_modification_times[entry->path]; + entry->last_write_time = param.file2write_time[entry->path]; // Update dependencies for the file. Do not include the file in its own // dependency set. - entry->dependencies = param.seen_files; - entry->dependencies.erase( - std::remove(entry->dependencies.begin(), entry->dependencies.end(), - entry->path), - entry->dependencies.end()); + for (const std::string& path : param.seen_files) + if (path != entry->path && path != entry->import_file) + entry->dependencies[path] = param.file2write_time[path]; } return result; @@ -2229,10 +2227,8 @@ struct TestIndexer : IIndexer { return result; } - ~TestIndexer() override = default; - std::vector> Index( - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, std::string file, const std::vector& args, const std::vector& file_contents, diff --git a/src/command_line.cc b/src/command_line.cc index 5109dcf7..70599b01 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -110,9 +110,8 @@ See more on https://github.com/MaskRay/ccls/wiki bool QueryDbMainLoop(QueryDatabase* db, MultiQueueWaiter* waiter, Project* project, - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, ImportPipelineStatus* status, - TimestampManager* timestamp_manager, SemanticHighlightSymbolCache* semantic_cache, WorkingFiles* working_files, ClangCompleteManager* clang_complete, @@ -154,7 +153,7 @@ void RunQueryDbThread(const std::string& bin_name, Project project; SemanticHighlightSymbolCache semantic_cache; WorkingFiles working_files; - FileConsumerSharedState file_consumer_shared; + VFS vfs; DiagnosticsEngine diag_engine; ClangCompleteManager clang_complete( @@ -162,11 +161,6 @@ void RunQueryDbThread(const std::string& bin_name, [&](std::string path, std::vector diagnostics) { diag_engine.Publish(&working_files, path, diagnostics); }, - [&](ClangTranslationUnit* tu, const std::vector& unsaved, - const std::string& path, const std::vector& args) { - IndexWithTuFromCodeCompletion(&file_consumer_shared, tu, unsaved, path, - args); - }, [](lsRequestId id) { if (id.Valid()) { Out_Error out; @@ -184,7 +178,6 @@ void RunQueryDbThread(const std::string& bin_name, auto non_global_code_complete_cache = std::make_unique(); auto signature_cache = std::make_unique(); ImportPipelineStatus import_pipeline_status; - TimestampManager timestamp_manager; QueryDatabase db; // Setup shared references. @@ -193,9 +186,8 @@ void RunQueryDbThread(const std::string& bin_name, handler->waiter = indexer_waiter; handler->project = &project; handler->diag_engine = &diag_engine; - handler->file_consumer_shared = &file_consumer_shared; + handler->vfs = &vfs; handler->import_pipeline_status = &import_pipeline_status; - handler->timestamp_manager = ×tamp_manager; handler->semantic_cache = &semantic_cache; handler->working_files = &working_files; handler->clang_complete = &clang_complete; @@ -210,8 +202,8 @@ void RunQueryDbThread(const std::string& bin_name, SetThreadName("querydb"); while (true) { bool did_work = QueryDbMainLoop( - &db, querydb_waiter, &project, &file_consumer_shared, - &import_pipeline_status, ×tamp_manager, + &db, querydb_waiter, &project, &vfs, + &import_pipeline_status, &semantic_cache, &working_files, &clang_complete, &include_complete, global_code_complete_cache.get(), non_global_code_complete_cache.get(), signature_cache.get()); diff --git a/src/config.cc b/src/config.cc index 68cdecc4..102d05a6 100644 --- a/src/config.cc +++ b/src/config.cc @@ -1,3 +1,4 @@ #include "config.h" std::unique_ptr g_config; +thread_local int g_thread_id; diff --git a/src/config.h b/src/config.h index c0d3774a..ba90abbc 100644 --- a/src/config.h +++ b/src/config.h @@ -42,44 +42,22 @@ struct Config { // It is not schema-aware and you need to re-index whenever a struct // member has changed. SerializeFormat cacheFormat = SerializeFormat::Binary; - // Value to use for clang -resource-dir if not present in - // compile_commands.json. - // - // ccls includes a resource directory, this should not need to be configured - // unless you're using an esoteric configuration. Consider reporting a bug and - // fixing upstream instead of configuring this. - // - // Example value: "/path/to/lib/clang/5.0.1/" - std::string resourceDirectory; - // Additional arguments to pass to clang. - std::vector extraClangArguments; + struct Clang { + // Additional arguments to pass to clang. + std::vector extraArgs; - // If true, ccls will send progress reports while indexing - // How often should ccls send progress report messages? - // -1: never - // 0: as often as possible - // xxx: at most every xxx milliseconds - // - // Empty progress reports (ie, idle) are delivered as often as they are - // available and may exceed this value. - // - // This does not guarantee a progress report will be delivered every - // interval; it could take significantly longer if ccls is completely idle. - int progressReportFrequencyMs = 500; - - // If true, document links are reported for #include directives. - bool showDocumentLinksOnIncludes = true; - - // Version of the client. If undefined the version check is skipped. Used to - // inform users their vscode client is too old and needs to be updated. - std::optional clientVersion; + // Value to use for clang -resource-dir if not specified. + // + // This option defaults to clang -print-resource-dir and should not be + // specified unless you are using an esoteric configuration. + std::string resourceDir; + } clang; struct ClientCapability { // TextDocumentClientCapabilities.completion.completionItem.snippetSupport bool snippetSupport = false; - }; - ClientCapability client; + } client; struct CodeLens { // Enables code lens on parameter and function variables. @@ -228,6 +206,7 @@ struct Config { int maxNum = 2000; } xref; }; +MAKE_REFLECT_STRUCT(Config::Clang, extraArgs, resourceDir); MAKE_REFLECT_STRUCT(Config::ClientCapability, snippetSupport); MAKE_REFLECT_STRUCT(Config::CodeLens, localVariables); MAKE_REFLECT_STRUCT(Config::Completion, @@ -260,16 +239,8 @@ MAKE_REFLECT_STRUCT(Config, compilationDatabaseDirectory, cacheDirectory, cacheFormat, - resourceDirectory, - - extraClangArguments, - - progressReportFrequencyMs, - - showDocumentLinksOnIncludes, - - clientVersion, + clang, client, codeLens, completion, @@ -280,3 +251,4 @@ MAKE_REFLECT_STRUCT(Config, xref); extern std::unique_ptr g_config; +thread_local extern int g_thread_id; diff --git a/src/file_consumer.cc b/src/file_consumer.cc index 6ba69cfb..5db6afca 100644 --- a/src/file_consumer.cc +++ b/src/file_consumer.cc @@ -55,21 +55,48 @@ std::optional FileContents::ContentsInRange(Range range) const { return std::nullopt; } -bool FileConsumerSharedState::Mark(const std::string& file) { +VFS::State VFS::Get(const std::string& file) { std::lock_guard lock(mutex); - return used_files.insert(file).second; + auto it = state.find(file); + if (it != state.end()) + return it->second; + return {0, 0, 0}; } -void FileConsumerSharedState::Reset(const std::string& file) { +bool VFS::Mark(const std::string& file, int owner, int stage) { std::lock_guard lock(mutex); - auto it = used_files.find(file); - if (it != used_files.end()) - used_files.erase(it); + State& st = state[file]; + if (st.stage < stage) { + st.owner = owner; + st.stage = stage; + return true; + } else + return false; } -FileConsumer::FileConsumer(FileConsumerSharedState* shared_state, - const std::string& parse_file) - : shared_(shared_state), parse_file_(parse_file) {} +bool VFS::Stamp(const std::string& file, int64_t ts) { + std::lock_guard lock(mutex); + State& st = state[file]; + if (st.timestamp < ts) { + st.timestamp = ts; + return true; + } else + return false; +} + +void VFS::ResetLocked(const std::string& file) { + State& st = state[file]; + if (st.owner == g_thread_id) + st.stage = 0; +} + +void VFS::Reset(const std::string& file) { + std::lock_guard lock(mutex); + ResetLocked(file); +} + +FileConsumer::FileConsumer(VFS* vfs, const std::string& parse_file) + : vfs_(vfs), parse_file_(parse_file), thread_id_(g_thread_id) {} IndexFile* FileConsumer::TryConsumeFile( CXFile file, @@ -96,12 +123,9 @@ IndexFile* FileConsumer::TryConsumeFile( std::string file_name = FileName(file); - // No result in local; we need to query global. - bool did_insert = shared_->Mark(file_name); - // We did not take the file from global. Cache that we failed so we don't try // again and return nullptr. - if (!did_insert) { + if (!vfs_->Mark(file_name, thread_id_, 2)) { local_[file_id] = nullptr; return nullptr; } diff --git a/src/file_consumer.h b/src/file_consumer.h index 4f8216a4..eb006a32 100644 --- a/src/file_consumer.h +++ b/src/file_consumer.h @@ -8,7 +8,6 @@ #include #include #include -#include #include struct IndexFile; @@ -30,13 +29,19 @@ struct FileContents { std::vector line_offsets_; }; -struct FileConsumerSharedState { - mutable std::unordered_set used_files; +struct VFS { + struct State { + int64_t timestamp; + int owner; + int stage; + }; + mutable std::unordered_map state; mutable std::mutex mutex; - // Mark the file as used. Returns true if the file was not previously used. - bool Mark(const std::string& file); - // Reset the used state (ie, mark the file as unused). + State Get(const std::string& file); + bool Mark(const std::string& file, int owner, int stage); + bool Stamp(const std::string& file, int64_t ts); + void ResetLocked(const std::string& file); void Reset(const std::string& file); }; @@ -48,8 +53,7 @@ struct FileConsumerSharedState { // The indexer does this because header files do not have their own translation // units but we still want to index them. struct FileConsumer { - FileConsumer(FileConsumerSharedState* shared_state, - const std::string& parse_file); + FileConsumer(VFS* vfs, const std::string& parse_file); // Returns true if this instance owns given |file|. This will also attempt to // take ownership over |file|. @@ -69,6 +73,7 @@ struct FileConsumer { private: std::unordered_map> local_; - FileConsumerSharedState* shared_; + VFS* vfs_; std::string parse_file_; + int thread_id_; }; diff --git a/src/import_pipeline.cc b/src/import_pipeline.cc index 210dc72b..14a341d2 100644 --- a/src/import_pipeline.cc +++ b/src/import_pipeline.cc @@ -18,115 +18,25 @@ namespace { -struct Out_Progress : public lsOutMessage { - struct Params { - int indexRequestCount = 0; - int loadPreviousIndexCount = 0; - int onIdMappedCount = 0; - int onIndexedCount = 0; - int activeThreads = 0; - }; - std::string method = "$ccls/progress"; - Params params; -}; -MAKE_REFLECT_STRUCT(Out_Progress::Params, - indexRequestCount, - loadPreviousIndexCount, - onIdMappedCount, - onIndexedCount, - activeThreads); -MAKE_REFLECT_STRUCT(Out_Progress, jsonrpc, method, params); - -long long GetCurrentTimeInMilliseconds() { - auto time_since_epoch = Timer::Clock::now().time_since_epoch(); - long long elapsed_milliseconds = - std::chrono::duration_cast(time_since_epoch) - .count(); - return elapsed_milliseconds; -} - -struct ActiveThread { - ActiveThread(ImportPipelineStatus* status) - : status_(status) { - if (g_config && g_config->progressReportFrequencyMs < 0) - return; - - ++status_->num_active_threads; - } - ~ActiveThread() { - if (g_config && g_config->progressReportFrequencyMs < 0) - return; - - --status_->num_active_threads; - EmitProgress(); - } - - // Send indexing progress to client if reporting is enabled. - void EmitProgress() { - auto* queue = QueueManager::instance(); - Out_Progress out; - out.params.indexRequestCount = queue->index_request.Size(); - out.params.onIdMappedCount = queue->on_id_mapped.Size(); - out.params.onIndexedCount = queue->on_indexed.Size(); - out.params.activeThreads = status_->num_active_threads; - - // Ignore this progress update if the last update was too recent. - if (g_config && g_config->progressReportFrequencyMs != 0) { - // Make sure we output a status update if queue lengths are zero. - bool all_zero = out.params.indexRequestCount == 0 && - out.params.loadPreviousIndexCount == 0 && - out.params.onIdMappedCount == 0 && - out.params.onIndexedCount == 0 && - out.params.activeThreads == 0; - if (!all_zero && - GetCurrentTimeInMilliseconds() < status_->next_progress_output) - return; - status_->next_progress_output = - GetCurrentTimeInMilliseconds() + g_config->progressReportFrequencyMs; - } - - QueueManager::WriteStdout(kMethodType_Unknown, out); - } - - ImportPipelineStatus* status_; -}; - -enum class ShouldParse { Yes, No, NoSuchFile }; - // Checks if |path| needs to be reparsed. This will modify cached state // such that calling this function twice with the same path may return true // the first time but will return false the second. // // |from|: The file which generated the parse request for this file. -ShouldParse FileNeedsParse( - bool is_interactive, - TimestampManager* timestamp_manager, - const std::shared_ptr& cache_manager, - IndexFile* opt_previous_index, - const std::string& path, - const std::vector& args, - const std::optional& from) { - auto unwrap_opt = [](const std::optional& opt) -> std::string { - if (opt) - return " (via " + *opt + ")"; - return ""; - }; - - std::optional modification_timestamp = LastWriteTime(path); - - // Cannot find file. - if (!modification_timestamp) - return ShouldParse::NoSuchFile; - - std::optional last_cached_modification = - timestamp_manager->GetLastCachedModificationTime(cache_manager.get(), - path); - - // File has been changed. - if (!last_cached_modification || - modification_timestamp != *last_cached_modification) { - LOG_S(INFO) << "Timestamp has changed for " << path << unwrap_opt(from); - return ShouldParse::Yes; +bool FileNeedsParse(int64_t write_time, + VFS* vfs, + bool is_interactive, + IndexFile* opt_previous_index, + const std::string& path, + const std::vector& args, + const std::optional& from) { + { + std::lock_guard lock(vfs->mutex); + if (vfs->state[path].timestamp < write_time) { + LOG_S(INFO) << "timestamp changed for " << path + << (from ? " (via " + *from + ")" : std::string()); + return true; + } } // Command-line arguments changed. @@ -141,183 +51,96 @@ ShouldParse FileNeedsParse( (is_file(prev_args[i]) && is_file(args[i])); } if (!same) { - LOG_S(INFO) << "Arguments have changed for " << path << unwrap_opt(from); - return ShouldParse::Yes; + LOG_S(INFO) << "args changed for " << path << (from ? " (via " + *from + ")" : std::string()); + return true; } } - // File has not changed, do not parse it. - return ShouldParse::No; + return false; }; -enum CacheLoadResult { Parse, DoNotParse }; -CacheLoadResult TryLoadFromCache( - FileConsumerSharedState* file_consumer_shared, - TimestampManager* timestamp_manager, - const std::shared_ptr& cache_manager, - bool is_interactive, - const Project::Entry& entry, - const std::string& path_to_index) { - // Always run this block, even if we are interactive, so we can check - // dependencies and reset files in |file_consumer_shared|. - IndexFile* previous_index = cache_manager->TryLoad(path_to_index); - if (!previous_index) - return CacheLoadResult::Parse; +bool Indexer_Parse(DiagnosticsEngine* diag_engine, + WorkingFiles* working_files, + Project* project, + VFS* vfs, + IIndexer* indexer) { + auto* queue = QueueManager::instance(); + std::optional opt_request = queue->index_request.TryPopFront(); + if (!opt_request) + return false; + auto& request = *opt_request; + ICacheManager cache; - // If none of the dependencies have changed and the index is not - // interactive (ie, requested by a file save), skip parsing and just load - // from cache. - - // Check timestamps and update |file_consumer_shared|. - ShouldParse path_state = - FileNeedsParse(is_interactive, timestamp_manager, cache_manager, - previous_index, path_to_index, entry.args, std::nullopt); - if (path_state == ShouldParse::Yes) - file_consumer_shared->Reset(path_to_index); - - // Target file does not exist on disk, do not emit any indexes. - // TODO: Dependencies should be reassigned to other files. We can do this by - // updating the "primary_file" if it doesn't exist. Might not actually be a - // problem in practice. - if (path_state == ShouldParse::NoSuchFile) - return CacheLoadResult::DoNotParse; - - bool needs_reparse = is_interactive || path_state == ShouldParse::Yes; - - for (const std::string& dependency : previous_index->dependencies) { - assert(!dependency.empty()); - - if (FileNeedsParse(is_interactive, timestamp_manager, cache_manager, - previous_index, dependency, entry.args, - previous_index->path) == ShouldParse::Yes) { - needs_reparse = true; - - // Do not break here, as we need to update |file_consumer_shared| for - // every dependency that needs to be reparsed. - file_consumer_shared->Reset(dependency); + Project::Entry entry; + { + std::lock_guard lock(project->mutex_); + auto it = project->absolute_path_to_entry_index_.find(request.path); + if (it != project->absolute_path_to_entry_index_.end()) + entry = project->entries[it->second]; + else { + entry.filename = request.path; + entry.args = request.args; } } - - // FIXME: should we still load from cache? - if (needs_reparse) - return CacheLoadResult::Parse; - - // No timestamps changed - load directly from cache. - LOG_S(INFO) << "load index for " << path_to_index; - - // TODO/FIXME: real perf - PerformanceImportFile perf; - - std::vector result; - result.push_back(Index_OnIdMapped( - cache_manager, nullptr, cache_manager->TryTakeOrLoad(path_to_index), perf, - is_interactive, false /*write_to_disk*/)); - for (const std::string& dependency : previous_index->dependencies) { - // Only load a dependency if it is not already loaded. - // - // This is important for perf in large projects where there are lots of - // dependencies shared between many files. - if (!file_consumer_shared->Mark(dependency)) - continue; - - LOG_S(INFO) << "emit index for " << dependency << " via " - << previous_index->path; - - // |dependency_index| may be null if there is no cache for it but - // another file has already started importing it. - if (std::unique_ptr dependency_index = - cache_manager->TryTakeOrLoad(dependency)) { - result.push_back( - Index_OnIdMapped(cache_manager, nullptr, std::move(dependency_index), - perf, is_interactive, false /*write_to_disk*/)); - } - } - - QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result)); - return CacheLoadResult::DoNotParse; -} - -std::vector PreloadFileContents( - const std::shared_ptr& cache_manager, - const Project::Entry& entry, - const std::string& entry_contents, - const std::string& path_to_index) { - // Load file contents for all dependencies into memory. If the dependencies - // for the file changed we may not end up using all of the files we - // preloaded. If a new dependency was added the indexer will grab the file - // contents as soon as possible. - // - // We do this to minimize the race between indexing a file and capturing the - // file contents. - // - // TODO: We might be able to optimize perf by only copying for files in - // working_files. We can pass that same set of files to the indexer as - // well. We then default to a fast file-copy if not in working set. - - // index->file_contents comes from cache, so we need to check if that cache is - // still valid. if so, we can use it, otherwise we need to load from disk. - auto get_latest_content = [](const std::string& path, int64_t cached_time, - const std::string& cached) -> std::string { - std::optional mod_time = LastWriteTime(path); - if (!mod_time) - return ""; - - if (*mod_time == cached_time) - return cached; - - std::optional fresh_content = ReadContent(path); - if (!fresh_content) { - LOG_S(ERROR) << "Failed to load content for " << path; - return ""; - } - return *fresh_content; - }; - - std::vector file_contents; - file_contents.push_back(FileContents(entry.filename, entry_contents)); - cache_manager->IterateLoadedCaches([&](IndexFile* index) { - if (index->path == entry.filename) - return; - file_contents.push_back(FileContents( - index->path, - get_latest_content(index->path, index->last_modification_time, - index->file_contents))); - }); - - return file_contents; -} - -void ParseFile(DiagnosticsEngine* diag_engine, - WorkingFiles* working_files, - FileConsumerSharedState* file_consumer_shared, - TimestampManager* timestamp_manager, - IIndexer* indexer, - const Index_Request& request, - const Project::Entry& entry) { - // If the file is inferred, we may not actually be able to parse that file - // directly (ie, a header file, which are not listed in the project). If this - // file is inferred, then try to use the file which originally imported it. std::string path_to_index = entry.filename; - if (entry.is_inferred) { - IndexFile* entry_cache = request.cache_manager->TryLoad(entry.filename); - if (entry_cache) - path_to_index = entry_cache->import_file; - } + std::unique_ptr prev; // Try to load the file from cache. - if (TryLoadFromCache(file_consumer_shared, timestamp_manager, - request.cache_manager, request.is_interactive, entry, - path_to_index) == CacheLoadResult::DoNotParse) - return; + std::optional write_time = LastWriteTime(path_to_index); + if (!write_time) + return true; + // FIXME Don't drop + if (!vfs->Mark(path_to_index, g_thread_id, 1)) + return true; + + int reparse; // request.is_interactive; + prev = cache.RawCacheLoad(path_to_index); + if (!prev) + reparse = 2; + else { + reparse = vfs->Stamp(path_to_index, prev->last_write_time); + if (FileNeedsParse(*write_time, vfs, request.is_interactive, &*prev, + path_to_index, entry.args, std::nullopt)) + reparse = 2; + for (const auto& dep : prev->dependencies) + if (auto write_time1 = LastWriteTime(dep.first)) { + if (dep.second < *write_time1) { + reparse = 2; + std::lock_guard lock(vfs->mutex); + vfs->state[dep.first].stage = 0; + } + } else + reparse = 2; + } + + if (reparse < 2) { + PerformanceImportFile perf; + auto dependencies = prev->dependencies; + if (reparse) { + IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); + queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf), + request.is_interactive); + } + for (const auto& dep : dependencies) + if (vfs->Mark(dep.first, 0, 2)) { + prev = cache.RawCacheLoad(dep.first); + IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); + queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf), + request.is_interactive); + } + + std::lock_guard lock(vfs->mutex); + VFS::State& state = vfs->state[path_to_index]; + if (state.owner == g_thread_id) + state.stage = 0; + return true; + } LOG_S(INFO) << "parse " << path_to_index; - std::vector file_contents = PreloadFileContents( - request.cache_manager, entry, request.contents, path_to_index); std::vector result; PerformanceImportFile perf; - auto indexes = indexer->Index(file_consumer_shared, path_to_index, entry.args, - file_contents, &perf); + auto indexes = indexer->Index(vfs, path_to_index, entry.args, {}, &perf); if (indexes.empty()) { if (g_config->index.enabled && request.id.Valid()) { @@ -327,160 +150,52 @@ void ParseFile(DiagnosticsEngine* diag_engine, out.error.message = "Failed to index " + path_to_index; QueueManager::WriteStdout(kMethodType_Unknown, out); } - return; + vfs->Reset(path_to_index); + return true; } - for (std::unique_ptr& new_index : indexes) { - Timer time; - + for (std::unique_ptr& curr : indexes) { // Only emit diagnostics for non-interactive sessions, which makes it easier // to identify indexing problems. For interactive sessions, diagnostics are // handled by code completion. if (!request.is_interactive) - diag_engine->Publish(working_files, new_index->path, - new_index->diagnostics_); + diag_engine->Publish(working_files, curr->path, curr->diagnostics_); - // When main thread does IdMap request it will request the previous index if - // needed. - LOG_S(INFO) << "emit index for " << new_index->path; - result.push_back( - Index_OnIdMapped(request.cache_manager, - request.cache_manager->TryTakeOrLoad(path_to_index), - std::move(new_index), perf, request.is_interactive, - true /*write_to_disk*/)); - } - - QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result), - request.is_interactive); -} - -bool IndexMain_DoParse( - DiagnosticsEngine* diag_engine, - WorkingFiles* working_files, - FileConsumerSharedState* file_consumer_shared, - TimestampManager* timestamp_manager, - IIndexer* indexer) { - auto* queue = QueueManager::instance(); - std::optional request = queue->index_request.TryPopFront(); - if (!request) - return false; - - Project::Entry entry; - entry.filename = request->path; - entry.args = request->args; - ParseFile(diag_engine, working_files, file_consumer_shared, timestamp_manager, - indexer, request.value(), entry); - return true; -} - -bool IndexMain_DoCreateIndexUpdate(TimestampManager* timestamp_manager) { - auto* queue = QueueManager::instance(); - - bool did_work = false; - for (int i = 100; i--; ) { - std::optional response = queue->on_id_mapped.TryPopFront(); - if (!response) - return did_work; - - did_work = true; - - Timer time; + std::string path = curr->path; + if (!(vfs->Stamp(path, curr->last_write_time) || path == path_to_index)) + continue; + LOG_S(INFO) << "emit index for " << path; + prev = cache.RawCacheLoad(path); // Write current index to disk if requested. - std::string path = response->current->path; - if (response->write_to_disk) { - LOG_S(INFO) << "store index for " << path; - time.Reset(); - response->cache_manager->WriteToCache(*response->current); - response->perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset(); - timestamp_manager->UpdateCachedModificationTime( - path, response->current->last_modification_time); + LOG_S(INFO) << "store index for " << path; + Timer time; + cache.WriteToCache(*curr); + perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset(); + + vfs->Reset(path_to_index); + if (entry.id >= 0) { + std::lock_guard lock(project->mutex_); + for (auto& dep : curr->dependencies) + project->absolute_path_to_entry_index_[dep.first] = entry.id; } // Build delta update. - IndexUpdate update = IndexUpdate::CreateDelta(response->previous.get(), - response->current.get()); - response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset(); - LOG_S(INFO) << "built index for " << path - << " (is_delta=" << !!response->previous << ")"; + IndexUpdate update = IndexUpdate::CreateDelta(prev.get(), curr.get()); + perf.index_make_delta = time.ElapsedMicrosecondsAndReset(); + LOG_S(INFO) << "built index for " << path << " (is_delta=" << !!prev << ")"; - Index_OnIndexed reply(std::move(update), response->perf); - queue->on_indexed.PushBack(std::move(reply), response->is_interactive); + Index_OnIndexed reply(std::move(update), perf); + queue->on_indexed.PushBack(std::move(reply), request.is_interactive); } - return did_work; + return true; } } // namespace -std::optional TimestampManager::GetLastCachedModificationTime( - ICacheManager* cache_manager, - const std::string& path) { - { - std::lock_guard guard(mutex_); - auto it = timestamps_.find(path); - if (it != timestamps_.end()) - return it->second; - } - IndexFile* file = cache_manager->TryLoad(path); - if (!file) - return std::nullopt; - - UpdateCachedModificationTime(path, file->last_modification_time); - return file->last_modification_time; -} - -void TimestampManager::UpdateCachedModificationTime(const std::string& path, - int64_t timestamp) { - std::lock_guard guard(mutex_); - timestamps_[path] = timestamp; -} - -ImportPipelineStatus::ImportPipelineStatus() - : num_active_threads(0), next_progress_output(0) {} - -// Index a file using an already-parsed translation unit from code completion. -// Since most of the time for indexing a file comes from parsing, we can do -// real-time indexing. -// TODO: add option to disable this. -void IndexWithTuFromCodeCompletion( - FileConsumerSharedState* file_consumer_shared, - ClangTranslationUnit* tu, - const std::vector& file_contents, - const std::string& path, - const std::vector& args) { - file_consumer_shared->Reset(path); - - PerformanceImportFile perf; - ClangIndex index; - auto indexes = ParseWithTu(file_consumer_shared, &perf, tu, &index, path, - args, file_contents); - if (indexes.empty()) - return; - - std::vector result; - for (std::unique_ptr& new_index : indexes) { - Timer time; - - std::shared_ptr cache_manager; - assert(false && "FIXME cache_manager"); - // When main thread does IdMap request it will request the previous index if - // needed. - LOG_S(INFO) << "Emitting index for " << new_index->path; - result.push_back(Index_OnIdMapped( - cache_manager, cache_manager->TryTakeOrLoad(path), std::move(new_index), - perf, true /*is_interactive*/, true /*write_to_disk*/)); - } - - LOG_IF_S(WARNING, result.size() > 1) - << "Code completion index update generated more than one index"; - - QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result)); -} - void Indexer_Main(DiagnosticsEngine* diag_engine, - FileConsumerSharedState* file_consumer_shared, - TimestampManager* timestamp_manager, + VFS* vfs, ImportPipelineStatus* status, Project* project, WorkingFiles* working_files, @@ -489,35 +204,9 @@ void Indexer_Main(DiagnosticsEngine* diag_engine, // Build one index per-indexer, as building the index acquires a global lock. auto indexer = std::make_unique(); - while (true) { - bool did_work = false; - - { - ActiveThread active_thread(status); - - // TODO: process all off IndexMain_DoIndex before calling - // IndexMain_DoCreateIndexUpdate for better icache behavior. We need to - // have some threads spinning on both though otherwise memory usage will - // get bad. - - // We need to make sure to run both IndexMain_DoParse and - // IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any - // work. Running both also lets the user query the partially constructed - // index. - did_work = IndexMain_DoParse(diag_engine, working_files, - file_consumer_shared, timestamp_manager, - indexer.get()) || - did_work; - - did_work = IndexMain_DoCreateIndexUpdate(timestamp_manager) || did_work; - } - - // We didn't do any work, so wait for a notification. - if (!did_work) { - waiter->Wait(&queue->on_indexed, &queue->index_request, - &queue->on_id_mapped); - } - } + while (true) + if (!Indexer_Parse(diag_engine, working_files, project, vfs, indexer.get())) + waiter->Wait(&queue->index_request); } namespace { @@ -559,9 +248,6 @@ bool QueryDb_ImportMain(QueryDatabase* db, SemanticHighlightSymbolCache* semantic_cache, WorkingFiles* working_files) { auto* queue = QueueManager::instance(); - - ActiveThread active_thread(status); - bool did_work = false; for (int i = 80; i--; ) { diff --git a/src/import_pipeline.h b/src/import_pipeline.h index 9b74dd13..0ba25eea 100644 --- a/src/import_pipeline.h +++ b/src/import_pipeline.h @@ -1,19 +1,10 @@ #pragma once -// FIXME: do not include clang-c outside of clang_ files. -#include - #include -#include -#include -#include -#include -#include -#include struct ClangTranslationUnit; class DiagnosticsEngine; -struct FileConsumerSharedState; +struct VFS; struct ICacheManager; struct MultiQueueWaiter; struct Project; @@ -21,37 +12,13 @@ struct QueryDatabase; struct SemanticHighlightSymbolCache; struct WorkingFiles; -// Caches timestamps of cc files so we can avoid a filesystem reads. This is -// important for import perf, as during dependency checking the same files are -// checked over and over again if they are common headers. -struct TimestampManager { - std::optional GetLastCachedModificationTime(ICacheManager* cache_manager, - const std::string& path); - - void UpdateCachedModificationTime(const std::string& path, int64_t timestamp); - - // TODO: use std::shared_mutex so we can have multiple readers. - std::mutex mutex_; - std::unordered_map timestamps_; -}; - struct ImportPipelineStatus { - std::atomic num_active_threads; - std::atomic next_progress_output; - - ImportPipelineStatus(); + std::atomic num_active_threads = {0}; + std::atomic next_progress_output = {0}; }; -void IndexWithTuFromCodeCompletion( - FileConsumerSharedState* file_consumer_shared, - ClangTranslationUnit* tu, - const std::vector& file_contents, - const std::string& path, - const std::vector& args); - void Indexer_Main(DiagnosticsEngine* diag_engine, - FileConsumerSharedState* file_consumer_shared, - TimestampManager* timestamp_manager, + VFS* vfs, ImportPipelineStatus* status, Project* project, WorkingFiles* working_files, diff --git a/src/indexer.h b/src/indexer.h index d332e1fa..fce01508 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -285,7 +285,7 @@ struct IndexFile { std::string path; std::vector args; - int64_t last_modification_time = 0; + int64_t last_write_time = 0; LanguageId language = LanguageId::Unknown; // The path to the translation unit cc file which caused the creation of this @@ -298,7 +298,7 @@ struct IndexFile { std::vector skipped_by_preprocessor; std::vector includes; - std::vector dependencies; + std::unordered_map dependencies; std::unordered_map usr2func; std::unordered_map usr2type; std::unordered_map usr2var; @@ -334,14 +334,14 @@ struct NamespaceHelper { // |dependencies| are the existing dependencies of |import_file| if this is a // reparse. std::vector> Parse( - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, std::string file, const std::vector& args, const std::vector& file_contents, PerformanceImportFile* perf, ClangIndex* index); std::vector> ParseWithTu( - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, PerformanceImportFile* perf, ClangTranslationUnit* tu, ClangIndex* index, @@ -367,7 +367,7 @@ struct IIndexer { virtual ~IIndexer() = default; virtual std::vector> Index( - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, std::string file, const std::vector& args, const std::vector& file_contents, @@ -376,12 +376,12 @@ struct IIndexer { struct ClangIndexer : IIndexer { std::vector> Index( - FileConsumerSharedState* file_consumer_shared, + VFS* vfs, std::string file, const std::vector& args, const std::vector& file_contents, PerformanceImportFile* perf) override { - return Parse(file_consumer_shared, file, args, file_contents, perf, &index); + return Parse(vfs, file, args, file_contents, perf, &index); } // Note: constructing this acquires a global lock diff --git a/src/message_handler.cc b/src/message_handler.cc index 1a2a3c87..a16f59bb 100644 --- a/src/message_handler.cc +++ b/src/message_handler.cc @@ -126,7 +126,7 @@ MessageHandler::MessageHandler() { std::vector* MessageHandler::message_handlers = nullptr; bool FindFileOrFail(QueryDatabase* db, - const Project* project, + Project* project, std::optional id, const std::string& absolute_path, QueryFile** out_query_file, @@ -147,8 +147,12 @@ bool FindFileOrFail(QueryDatabase* db, if (out_file_id) *out_file_id = -1; - bool indexing = project->absolute_path_to_entry_index_.find(absolute_path) != - project->absolute_path_to_entry_index_.end(); + bool indexing; + { + std::lock_guard lock(project->mutex_); + indexing = project->absolute_path_to_entry_index_.find(absolute_path) != + project->absolute_path_to_entry_index_.end(); + } if (indexing) LOG_S(INFO) << "\"" << absolute_path << "\" is being indexed."; else diff --git a/src/message_handler.h b/src/message_handler.h index 8f35dbb0..2afcbc43 100644 --- a/src/message_handler.h +++ b/src/message_handler.h @@ -15,14 +15,13 @@ struct ClangCompleteManager; struct CodeCompleteCache; struct Config; class DiagnosticsEngine; -struct FileConsumerSharedState; +struct VFS; struct ImportManager; struct ImportPipelineStatus; struct IncludeComplete; struct MultiQueueWaiter; struct Project; struct QueryDatabase; -struct TimestampManager; struct WorkingFile; struct WorkingFiles; @@ -107,10 +106,9 @@ struct MessageHandler { MultiQueueWaiter* waiter = nullptr; Project* project = nullptr; DiagnosticsEngine* diag_engine = nullptr; - FileConsumerSharedState* file_consumer_shared = nullptr; + VFS* vfs = nullptr; ImportManager* import_manager = nullptr; ImportPipelineStatus* import_pipeline_status = nullptr; - TimestampManager* timestamp_manager = nullptr; SemanticHighlightSymbolCache* semantic_cache = nullptr; WorkingFiles* working_files = nullptr; ClangCompleteManager* clang_complete = nullptr; @@ -139,7 +137,7 @@ struct BaseMessageHandler : MessageHandler { }; bool FindFileOrFail(QueryDatabase* db, - const Project* project, + Project* project, std::optional id, const std::string& absolute_path, QueryFile** out_query_file, diff --git a/src/messages/ccls_call_hierarchy.cc b/src/messages/ccls_call_hierarchy.cc index 3b3794c1..ffcca708 100644 --- a/src/messages/ccls_call_hierarchy.cc +++ b/src/messages/ccls_call_hierarchy.cc @@ -4,6 +4,8 @@ #include +#include + namespace { MethodType kMethodType = "$ccls/callHierarchy"; diff --git a/src/messages/ccls_freshen_index.cc b/src/messages/ccls_freshen_index.cc index 1989578d..f9e047fa 100644 --- a/src/messages/ccls_freshen_index.cc +++ b/src/messages/ccls_freshen_index.cc @@ -34,14 +34,8 @@ REGISTER_IN_MESSAGE(In_CclsFreshenIndex); struct Handler_CclsFreshenIndex : BaseMessageHandler { MethodType GetMethodType() const override { return kMethodType; } void Run(In_CclsFreshenIndex* request) override { - LOG_S(INFO) << "Freshening " << project->entries.size() << " files"; - - // TODO: think about this flow and test it more. GroupMatch matcher(request->params.whitelist, request->params.blacklist); - // Unmark all files whose timestamp has changed. - std::shared_ptr cache_manager = ICacheManager::Make(); - std::queue q; // |need_index| stores every filename ever enqueued. std::unordered_set need_index; @@ -65,15 +59,15 @@ struct Handler_CclsFreshenIndex : BaseMessageHandler { q.pop(); need_index.insert(file->def->path); - std::optional modification_timestamp = - LastWriteTime(file->def->path); - if (!modification_timestamp) + std::optional write_time = LastWriteTime(file->def->path); + if (!write_time) continue; - std::optional cached_modification = - timestamp_manager->GetLastCachedModificationTime(cache_manager.get(), - file->def->path); - if (modification_timestamp != cached_modification) - file_consumer_shared->Reset(file->def->path); + { + std::lock_guard lock(vfs->mutex); + VFS::State& st = vfs->state[file->def->path]; + if (st.timestamp < write_time) + st.stage = 0; + } if (request->params.dependencies) for (const std::string& path : graph[file->def->path]) { @@ -85,10 +79,8 @@ struct Handler_CclsFreshenIndex : BaseMessageHandler { } } - Timer time; // Send index requests for every file. project->Index(QueueManager::instance(), working_files, lsRequestId()); - time.ResetAndPrint("[perf] Dispatched $ccls/freshenIndex index requests"); } }; REGISTER_MESSAGE_HANDLER(Handler_CclsFreshenIndex); diff --git a/src/messages/ccls_index_file.cc b/src/messages/ccls_index_file.cc deleted file mode 100644 index 0e4ecd86..00000000 --- a/src/messages/ccls_index_file.cc +++ /dev/null @@ -1,40 +0,0 @@ -#include "cache_manager.h" -#include "message_handler.h" -#include "platform.h" -#include "queue_manager.h" - -#include - -namespace { -MethodType kMethodType = "$ccls/indexFile"; - -struct In_CclsIndexFile : public NotificationInMessage { - MethodType GetMethodType() const override { return kMethodType; } - struct Params { - std::string path; - std::vector args; - bool is_interactive = false; - std::string contents; - }; - Params params; -}; -MAKE_REFLECT_STRUCT(In_CclsIndexFile::Params, - path, - args, - is_interactive, - contents); -MAKE_REFLECT_STRUCT(In_CclsIndexFile, params); -REGISTER_IN_MESSAGE(In_CclsIndexFile); - -struct Handler_CclsIndexFile : BaseMessageHandler { - MethodType GetMethodType() const override { return kMethodType; } - void Run(In_CclsIndexFile* request) override { - LOG_S(INFO) << "Indexing file " << request->params.path; - QueueManager::instance()->index_request.PushBack( - Index_Request(NormalizePath(request->params.path), request->params.args, - request->params.is_interactive, request->params.contents, - ICacheManager::Make())); - } -}; -REGISTER_MESSAGE_HANDLER(Handler_CclsIndexFile); -} // namespace diff --git a/src/messages/ccls_member_hierarchy.cc b/src/messages/ccls_member_hierarchy.cc index c8daab6b..8654ed01 100644 --- a/src/messages/ccls_member_hierarchy.cc +++ b/src/messages/ccls_member_hierarchy.cc @@ -2,6 +2,8 @@ #include "query_utils.h" #include "queue_manager.h" +#include + namespace { MethodType kMethodType = "$ccls/memberHierarchy"; diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc index 7c8687f7..9cbcba60 100644 --- a/src/messages/initialize.cc +++ b/src/messages/initialize.cc @@ -473,9 +473,9 @@ struct Handler_Initialize : BaseMessageHandler { } // Ensure there is a resource directory. - if (config->resourceDirectory.empty()) - config->resourceDirectory = GetDefaultResourceDirectory(); - LOG_S(INFO) << "Using -resource-dir=" << config->resourceDirectory; + if (config->clang.resourceDir.empty()) + config->clang.resourceDir = GetDefaultResourceDirectory(); + LOG_S(INFO) << "Using -resource-dir=" << config->clang.resourceDir; // Send initialization before starting indexers, so we don't send a // status update too early. @@ -519,9 +519,9 @@ struct Handler_Initialize : BaseMessageHandler { LOG_S(INFO) << "Starting " << g_config->index.threads << " indexers"; for (int i = 0; i < g_config->index.threads; i++) { std::thread([=]() { + g_thread_id = i + 1; SetThreadName("indexer" + std::to_string(i)); - Indexer_Main(diag_engine, file_consumer_shared, timestamp_manager, - import_pipeline_status, project, + Indexer_Main(diag_engine, vfs, import_pipeline_status, project, working_files, waiter); }).detach(); } diff --git a/src/messages/text_document_did_change.cc b/src/messages/text_document_did_change.cc index bc976cbd..10a646ca 100644 --- a/src/messages/text_document_did_change.cc +++ b/src/messages/text_document_did_change.cc @@ -33,7 +33,7 @@ struct Handler_TextDocumentDidChange Project::Entry entry = project->FindCompilationEntryForFile(path); QueueManager::instance()->index_request.PushBack( Index_Request(entry.filename, entry.args, true /*is_interactive*/, - *content, ICacheManager::Make()), + *content), true); } } diff --git a/src/messages/text_document_did_open.cc b/src/messages/text_document_did_open.cc index 20d8e256..c3ae0b07 100644 --- a/src/messages/text_document_did_open.cc +++ b/src/messages/text_document_did_open.cc @@ -38,13 +38,12 @@ struct Handler_TextDocumentDidOpen // NOTE: This function blocks code lens. If it starts taking a long time // we will need to find a way to unblock the code lens request. const auto& params = request->params; - Timer time; std::string path = params.textDocument.uri.GetPath(); - std::shared_ptr cache_manager = ICacheManager::Make(); + ICacheManager cache; WorkingFile* working_file = working_files->OnOpen(params.textDocument); std::optional cached_file_contents = - cache_manager->LoadCachedFileContents(path); + cache.LoadCachedFileContents(path); if (cached_file_contents) working_file->SetIndexContent(*cached_file_contents); @@ -55,10 +54,6 @@ struct Handler_TextDocumentDidOpen EmitSemanticHighlighting(db, semantic_cache, working_file, file); } - time.ResetAndPrint( - "[querydb] Loading cached index file for DidOpen (blocks " - "CodeLens)"); - include_complete->AddFile(working_file->filename); clang_complete->NotifyView(path); if (params.args.size()) @@ -68,10 +63,10 @@ struct Handler_TextDocumentDidOpen if (SourceFileLanguage(path) != LanguageId::Unknown) { Project::Entry entry = project->FindCompilationEntryForFile(path); QueueManager::instance()->index_request.PushBack( - Index_Request( - entry.filename, params.args.size() ? params.args : entry.args, - true /*is_interactive*/, params.textDocument.text, cache_manager), - true /* priority */); + Index_Request(entry.filename, + params.args.size() ? params.args : entry.args, + true /*is_interactive*/, params.textDocument.text), + true /* priority */); clang_complete->FlushSession(entry.filename); LOG_S(INFO) << "Flushed clang complete sessions for " << entry.filename; diff --git a/src/messages/text_document_did_save.cc b/src/messages/text_document_did_save.cc index 29ceffa8..c6891c2c 100644 --- a/src/messages/text_document_did_save.cc +++ b/src/messages/text_document_did_save.cc @@ -55,7 +55,7 @@ struct Handler_TextDocumentDidSave Project::Entry entry = project->FindCompilationEntryForFile(path); QueueManager::instance()->index_request.PushBack( Index_Request(entry.filename, entry.args, true /*is_interactive*/, - *content, ICacheManager::Make()), + *content), true); } } diff --git a/src/messages/workspace_did_change_watched_files.cc b/src/messages/workspace_did_change_watched_files.cc index 19b4ad13..6dbac73c 100644 --- a/src/messages/workspace_did_change_watched_files.cc +++ b/src/messages/workspace_did_change_watched_files.cc @@ -41,10 +41,14 @@ struct Handler_WorkspaceDidChangeWatchedFiles void Run(In_WorkspaceDidChangeWatchedFiles* request) override { for (lsFileEvent& event : request->params.changes) { std::string path = event.uri.GetPath(); - auto it = project->absolute_path_to_entry_index_.find(path); - if (it == project->absolute_path_to_entry_index_.end()) - continue; - const Project::Entry& entry = project->entries[it->second]; + Project::Entry entry; + { + std::lock_guard lock(project->mutex_); + auto it = project->absolute_path_to_entry_index_.find(path); + if (it == project->absolute_path_to_entry_index_.end()) + continue; + entry = project->entries[it->second]; + } bool is_interactive = working_files->GetFileByFilename(entry.filename) != nullptr; switch (event.type) { @@ -55,8 +59,7 @@ struct Handler_WorkspaceDidChangeWatchedFiles LOG_S(ERROR) << "Unable to read file content after saving " << path; else { QueueManager::instance()->index_request.PushBack( - Index_Request(path, entry.args, is_interactive, *content, - ICacheManager::Make())); + Index_Request(path, entry.args, is_interactive, *content)); if (is_interactive) clang_complete->NotifySave(path); } @@ -64,8 +67,7 @@ struct Handler_WorkspaceDidChangeWatchedFiles } case lsFileChangeType::Deleted: QueueManager::instance()->index_request.PushBack( - Index_Request(path, entry.args, is_interactive, std::string(), - ICacheManager::Make())); + Index_Request(path, entry.args, is_interactive, std::string())); break; } } diff --git a/src/port.h b/src/port.h index ef19d772..d50e597b 100644 --- a/src/port.h +++ b/src/port.h @@ -10,6 +10,10 @@ #define ATTRIBUTE_UNUSED #endif +#ifdef __clang__ +#define GUARDED_BY(x) __attribute__((guarded_by(x))) +#endif + // TODO GCC #if __has_builtin(__builtin_unreachable) #define CCLS_BUILTIN_UNREACHABLE __builtin_unreachable() diff --git a/src/project.cc b/src/project.cc index d45cd063..d30e57e1 100644 --- a/src/project.cc +++ b/src/project.cc @@ -225,7 +225,7 @@ Project::Entry GetCompilationEntryFromCompileCommandEntry( // Add -resource-dir so clang can correctly resolve system includes like // if (!AnyStartsWith(result.args, "-resource-dir")) - result.args.push_back("-resource-dir=" + g_config->resourceDirectory); + result.args.push_back("-resource-dir=" + g_config->clang.resourceDir); // There could be a clang version mismatch between what the project uses and // what ccls uses. Make sure we do not emit warnings for mismatched options. @@ -441,7 +441,7 @@ int ComputeGuessScore(std::string_view a, std::string_view b) { void Project::Load(const std::string& root_directory) { // Load data. ProjectConfig project; - project.extra_flags = g_config->extraClangArguments; + project.extra_flags = g_config->clang.extraArgs; project.project_dir = root_directory; entries = LoadCompilationEntriesFromDirectory( &project, g_config->compilationDatabaseDirectory); @@ -461,14 +461,18 @@ void Project::Load(const std::string& root_directory) { } // Setup project entries. + std::lock_guard lock(mutex_); absolute_path_to_entry_index_.reserve(entries.size()); - for (size_t i = 0; i < entries.size(); ++i) + for (size_t i = 0; i < entries.size(); ++i) { + entries[i].id = i; absolute_path_to_entry_index_[entries[i].filename] = i; + } } void Project::SetFlagsForFile( const std::vector& flags, const std::string& path) { + std::lock_guard lock(mutex_); auto it = absolute_path_to_entry_index_.find(path); if (it != absolute_path_to_entry_index_.end()) { // The entry already exists in the project, just set the flags. @@ -485,9 +489,12 @@ void Project::SetFlagsForFile( Project::Entry Project::FindCompilationEntryForFile( const std::string& filename) { - auto it = absolute_path_to_entry_index_.find(filename); - if (it != absolute_path_to_entry_index_.end()) - return entries[it->second]; + { + std::lock_guard lock(mutex_); + auto it = absolute_path_to_entry_index_.find(filename); + if (it != absolute_path_to_entry_index_.end()) + return entries[it->second]; + } // We couldn't find the file. Try to infer it. // TODO: Cache inferred file in a separate array (using a lock or similar) @@ -554,8 +561,7 @@ void Project::Index(QueueManager* queue, } bool is_interactive = wfiles->GetFileByFilename(entry.filename) != nullptr; queue->index_request.PushBack(Index_Request(entry.filename, entry.args, - is_interactive, *content, - ICacheManager::Make(), id)); + is_interactive, *content, id)); }); } @@ -564,7 +570,7 @@ TEST_SUITE("Project") { std::vector raw, std::vector expected) { g_config = std::make_unique(); - g_config->resourceDirectory = "/w/resource_dir/"; + g_config->clang.resourceDir = "/w/resource_dir/"; ProjectConfig project; project.project_dir = "/w/c/s/"; diff --git a/src/project.h b/src/project.h index 0767e945..4b5220b9 100644 --- a/src/project.h +++ b/src/project.h @@ -19,6 +19,7 @@ struct Project { std::vector args; // If true, this entry is inferred and was not read from disk. bool is_inferred = false; + int id = -1; }; // Include directories for "" headers @@ -27,7 +28,8 @@ struct Project { std::vector angle_include_directories; std::vector entries; - std::unordered_map absolute_path_to_entry_index_; + std::mutex mutex_; + std::unordered_map absolute_path_to_entry_index_ GUARDED_BY(mutex_); // Loads a project for the given |directory|. // diff --git a/src/query.cc b/src/query.cc index 761ca5e1..49b7a893 100644 --- a/src/query.cc +++ b/src/query.cc @@ -76,7 +76,9 @@ QueryFile::DefUpdate BuildFileDefUpdate(const IndexFile& indexed) { def.args = std::move(indexed.args); def.includes = std::move(indexed.includes); def.inactive_regions = std::move(indexed.skipped_by_preprocessor); - def.dependencies = std::move(indexed.dependencies); + def.dependencies.reserve(indexed.dependencies.size()); + for (auto& dep : indexed.dependencies) + def.dependencies.push_back(dep.first); def.language = indexed.language; auto add_all_symbols = [&](Use use, Usr usr, SymbolKind kind) { diff --git a/src/queue_manager.cc b/src/queue_manager.cc index b4ec6ad1..87cb794a 100644 --- a/src/queue_manager.cc +++ b/src/queue_manager.cc @@ -11,13 +11,11 @@ Index_Request::Index_Request( const std::vector& args, bool is_interactive, const std::string& contents, - const std::shared_ptr& cache_manager, lsRequestId id) : path(path), args(args), is_interactive(is_interactive), contents(contents), - cache_manager(cache_manager), id(id) {} Index_OnIndexed::Index_OnIndexed(IndexUpdate&& update, @@ -51,10 +49,4 @@ QueueManager::QueueManager(MultiQueueWaiter* querydb_waiter, : for_stdout(stdout_waiter), for_querydb(querydb_waiter), on_indexed(querydb_waiter), - index_request(indexer_waiter), - on_id_mapped(indexer_waiter) {} - -bool QueueManager::HasWork() { - return !index_request.IsEmpty() || !on_id_mapped.IsEmpty() || - !on_indexed.IsEmpty(); -} + index_request(indexer_waiter) {} diff --git a/src/queue_manager.h b/src/queue_manager.h index 10c9d121..63657fea 100644 --- a/src/queue_manager.h +++ b/src/queue_manager.h @@ -21,14 +21,12 @@ struct Index_Request { std::vector args; bool is_interactive; std::string contents; // Preloaded contents. - std::shared_ptr cache_manager; lsRequestId id; Index_Request(const std::string& path, const std::vector& args, bool is_interactive, const std::string& contents, - const std::shared_ptr& cache_manager, lsRequestId id = {}); }; @@ -72,8 +70,6 @@ class QueueManager { MultiQueueWaiter* stdout_waiter); static void WriteStdout(MethodType method, lsBaseOutMessage& response); - bool HasWork(); - // Messages received by "stdout" thread. ThreadedQueue for_stdout; @@ -83,7 +79,6 @@ class QueueManager { // Runs on indexer threads. ThreadedQueue index_request; - ThreadedQueue on_id_mapped; private: explicit QueueManager(MultiQueueWaiter* querydb_waiter, diff --git a/src/serializer.cc b/src/serializer.cc index ea5b0023..fd07a045 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -283,14 +283,13 @@ template void Reflect(TVisitor& visitor, IndexFile& value) { REFLECT_MEMBER_START(); if (!gTestOutputMode) { - REFLECT_MEMBER(last_modification_time); + REFLECT_MEMBER(last_write_time); REFLECT_MEMBER(language); REFLECT_MEMBER(import_file); REFLECT_MEMBER(args); + REFLECT_MEMBER(dependencies); } REFLECT_MEMBER(includes); - if (!gTestOutputMode) - REFLECT_MEMBER(dependencies); REFLECT_MEMBER(skipped_by_preprocessor); REFLECT_MEMBER(usr2func); REFLECT_MEMBER(usr2type); @@ -314,6 +313,27 @@ void Reflect(Writer& visitor, SerializeFormat& value) { } } +void Reflect(Reader& visitor, std::unordered_map& map) { + visitor.IterArray([&](Reader& entry) { + std::string name; + Reflect(entry, name); + if (visitor.Format() == SerializeFormat::Binary) + Reflect(entry, map[name]); + else + map[name] = 0; + }); +} +void Reflect(Writer& visitor, std::unordered_map& map) { + visitor.StartArray(map.size()); + for (auto& it : map) { + std::string key = it.first; + Reflect(visitor, key); + if (visitor.Format() == SerializeFormat::Binary) + Reflect(visitor, it.second); + } + visitor.EndArray(); +} + std::string Serialize(SerializeFormat format, IndexFile& file) { switch (format) { case SerializeFormat::Binary: { diff --git a/src/serializer.h b/src/serializer.h index a7d353ff..69b524dd 100644 --- a/src/serializer.h +++ b/src/serializer.h @@ -262,12 +262,12 @@ void Reflect(Writer& visitor, std::vector& values) { // std::unordered_map template -void Reflect(Reader& visitor, std::unordered_map& values) { +void Reflect(Reader& visitor, std::unordered_map& map) { visitor.IterArray([&](Reader& entry) { V val; Reflect(entry, val); auto usr = val.usr; - values[usr] = std::move(val); + map[usr] = std::move(val); }); } template @@ -281,6 +281,10 @@ void Reflect(Writer& visitor, std::unordered_map& map) { visitor.EndArray(); } +// Used by IndexFile::dependencies. Timestamps are emitted for Binary. +void Reflect(Reader& visitor, std::unordered_map& map); +void Reflect(Writer& visitor, std::unordered_map& map); + // ReflectMember template diff --git a/src/test.cc b/src/test.cc index 7bf1d928..74d85bf8 100644 --- a/src/test.cc +++ b/src/test.cc @@ -292,9 +292,9 @@ bool RunIndexTests(const std::string& filter_path, bool enable_update) { // Run test. g_config = std::make_unique(); - FileConsumerSharedState file_consumer_shared; + VFS vfs; PerformanceImportFile perf; - auto dbs = Parse(&file_consumer_shared, path, flags, {}, &perf, &index); + auto dbs = Parse(&vfs, path, flags, {}, &perf, &index); for (const auto& entry : all_expected_output) { const std::string& expected_path = entry.first;