diff --git a/src/command_line.cc b/src/command_line.cc index 35feb59c..c40ee245 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -35,7 +35,7 @@ namespace { const bool kUseMultipleProcesses = false; // TODO: initialization options not passed properly when set to true. - +std::vector kEmptyArgs; @@ -783,11 +783,11 @@ struct Index_DoIndex { enum class Type { ImportAndUpdate, ImportOnly, - Update + Parse, }; std::string path; - std::vector args; + optional> args; Type type; Index_DoIndex(Type type) : type(type) {} @@ -871,7 +871,82 @@ void RegisterMessageTypes() { +void ImportCachedIndex(IndexerConfig* config, + Index_DoIndexQueue* queue_do_index, + Index_DoIdMapQueue* queue_do_id_map, + const std::string path, + int64_t* last_modification_time) { + *last_modification_time = 0; + Timer time; + + std::unique_ptr cache = LoadCachedFile(config, path); + time.ResetAndPrint("Reading cached index from disk " + path); + if (!cache) + return; + + // Import all dependencies. + for (auto& dependency_path : cache->dependencies) { + std::cerr << "- Dispatching dependency import " << dependency_path << std::endl; + Index_DoIndex dep_index_request(Index_DoIndex::Type::ImportOnly); + dep_index_request.path = dependency_path; + queue_do_index->PriorityEnqueue(std::move(dep_index_request)); + } + + *last_modification_time = cache->last_modification_time; + Index_DoIdMap response(nullptr, std::move(cache)); + queue_do_id_map->Enqueue(std::move(response)); +} + +void ParseFile(IndexerConfig* config, + FileConsumer::SharedState* file_consumer_shared, + Index_DoIdMapQueue* queue_do_id_map, + const std::string& path, + const optional>& args) { + Timer time; + + // Parse request and send a response. + std::unique_ptr cached_path_index = LoadCachedFile(config, path); + + // Skip index if file modification time didn't change. + if (cached_path_index && GetLastModificationTime(path) == cached_path_index->last_modification_time) { + time.ResetAndPrint("Skipping index update on " + path + " since file modification time has not changed"); + return; + } + + std::vector> indexes = Parse( + config, file_consumer_shared, + path, cached_path_index ? cached_path_index->import_file : path, + args ? *args : cached_path_index ? cached_path_index->args : kEmptyArgs); + time.ResetAndPrint("Parsing/indexing " + path); + + for (std::unique_ptr& new_index : indexes) { + std::cerr << "Got index for " << new_index->path << std::endl; + + // Load the cached index. + std::unique_ptr cached_index; + if (new_index->path == path) + cached_index = std::move(cached_path_index); + else + cached_index = LoadCachedFile(config, new_index->path); + time.ResetAndPrint("Loading cached index"); + + // Update dependencies on |new_index|, since they won't get reparsed if we + // have parsed them once before. + if (cached_index) + AddRange(&new_index->dependencies, cached_index->dependencies); + + // Cache the newly indexed file. This replaces the existing cache. + // TODO: Run this as another import pipeline stage. + WriteToCache(config, new_index->path, *new_index); + time.ResetAndPrint("Cache index update to disk"); + + // Dispatch IdMap creation request, which will happen on querydb thread. + Index_DoIdMap response(std::move(cached_index), std::move(new_index)); + queue_do_id_map->Enqueue(std::move(response)); + } + +} bool IndexMain_DoIndex(IndexerConfig* config, FileConsumer::SharedState* file_consumer_shared, @@ -884,87 +959,35 @@ bool IndexMain_DoIndex(IndexerConfig* config, Timer time; - // If the index update is an import, then we will load the previous index - // into memory if we have a previous index. After that, we dispatch an - // update request to get the latest version. - if (index_request->type == Index_DoIndex::Type::ImportAndUpdate || - index_request->type == Index_DoIndex::Type::ImportOnly) { + switch (index_request->type) { + case Index_DoIndex::Type::ImportOnly: { + int64_t cache_modification_time; + ImportCachedIndex(config, queue_do_index, queue_do_id_map, index_request->path, &cache_modification_time); + break; + } - std::unique_ptr old_index = LoadCachedFile(config, index_request->path); - time.ResetAndPrint("Reading cached index from disk " + index_request->path); + case Index_DoIndex::Type::ImportAndUpdate: { + int64_t cache_modification_time; + ImportCachedIndex(config, queue_do_index, queue_do_id_map, index_request->path, &cache_modification_time); - // If import fails just do a standard update. - if (old_index) { - for (auto& dependency_path : old_index->dependencies) { - // TODO: These requests should go to the front of the queue. - std::cerr << "- Dispatching dependency import " << dependency_path << std::endl; - Index_DoIndex dep_index_request(Index_DoIndex::Type::ImportOnly); - dep_index_request.path = dependency_path; - dep_index_request.args = index_request->args; - queue_do_index->PriorityEnqueue(std::move(dep_index_request)); - } - - project->UpdateFileState(index_request->path, old_index->import_file, old_index->last_modification_time); - - Index_DoIdMap response(nullptr, std::move(old_index)); - queue_do_id_map->Enqueue(std::move(response)); - - // If we need a reparse, send the document to the back of the queue so it - // gets processed. - if (index_request->type == Index_DoIndex::Type::ImportAndUpdate) { - index_request->type = Index_DoIndex::Type::Update; + // If the file has been updated, we need to reparse it. + if (GetLastModificationTime(index_request->path) > cache_modification_time) { + // Instead of parsing the file immediate, we push the request to the + // back of the queue so we will finish all of the Import requests + // before starting to run libclang. This gives the user a + // partially-correct index potentially much sooner. + index_request->type = Index_DoIndex::Type::Parse; queue_do_index->Enqueue(std::move(*index_request)); } - return true; + break; } - } - // Parse request and send a response. - std::string import_file = index_request->path; - std::vector import_dependencies; - - // Skip index if file modification time didn't change. - optional entry = project->FindCompilationEntryForFile(index_request->path); - if (entry && entry->last_modification_time) { - import_file = entry->import_file; - - int64_t modification_time = GetLastModificationTime(index_request->path); - if (modification_time == *entry->last_modification_time) { - time.ResetAndPrint("Skipping index update on " + index_request->path + " since file modification time has not changed"); - return true; + case Index_DoIndex::Type::Parse: { + ParseFile(config, file_consumer_shared, queue_do_id_map, index_request->path, index_request->args); + break; } } - std::vector> indexes = Parse( - config, file_consumer_shared, - index_request->path, import_file, - index_request->args); - time.ResetAndPrint("Parsing/indexing " + index_request->path); - - for (auto& current_index : indexes) { - std::cerr << "Got index for " << current_index->path << std::endl; - - project->UpdateFileState(current_index->path, current_index->import_file, current_index->last_modification_time); - - std::unique_ptr old_index = LoadCachedFile(config, current_index->path); - time.ResetAndPrint("Loading cached index"); - - if (old_index) - AddRange(¤t_index->dependencies, old_index->dependencies); - - // TODO: Cache to disk on a separate thread. Maybe we do the cache after we - // have imported the index (so the import pipeline has five stages instead - // of the current 4). - - // Cache file so we can diff it later. - WriteToCache(config, current_index->path, *current_index); - time.ResetAndPrint("Cache index update to disk"); - - // Send response to create id map. - Index_DoIdMap response(std::move(old_index), std::move(current_index)); - queue_do_id_map->Enqueue(std::move(response)); - } - return true; } @@ -1213,11 +1236,9 @@ void QueryDbMainLoop( } // Send an index update request. - Index_DoIndex request(Index_DoIndex::Type::Update); - optional entry = project->FindCompilationEntryForFile(msg->params.textDocument.uri.GetPath()); + Index_DoIndex request(Index_DoIndex::Type::Parse); request.path = msg->params.textDocument.uri.GetPath(); - if (entry) - request.args = entry->args; + request.args = project->FindArgsForFile(msg->params.textDocument.uri.GetPath()); queue_do_index->Enqueue(std::move(request)); break; } diff --git a/src/indexer.cc b/src/indexer.cc index 12a721de..19d13026 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -1392,6 +1392,7 @@ std::vector> Parse( entry->last_modification_time = GetLastModificationTime(entry->path); entry->import_file = import_file; + entry->args = args; } // TODO: Fix interesting checks. diff --git a/src/indexer.h b/src/indexer.h index 22404ccb..235b606b 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -469,6 +469,7 @@ struct IndexedFile { int version = 0; std::string path; + std::vector args; int64_t last_modification_time = 0; // The path to the translation unit cc file which caused the creation of this diff --git a/src/project.cc b/src/project.cc index a5652557..ed7d353f 100644 --- a/src/project.cc +++ b/src/project.cc @@ -250,43 +250,15 @@ void Project::Load(const std::string& directory) { } optional Project::FindCompilationEntryForFile(const std::string& filename) { - // TODO: There might be a lot of thread contention here. - std::lock_guard lock(entries_modification_mutex_); - auto it = absolute_path_to_entry_index_.find(filename); if (it != absolute_path_to_entry_index_.end()) return entries[it->second]; return nullopt; } -void Project::UpdateFileState(const std::string& filename, const std::string& import_file, uint64_t modification_time) { - { - // TODO: There might be a lot of thread contention here. - std::lock_guard lock(entries_modification_mutex_); - auto it = absolute_path_to_entry_index_.find(filename); - if (it != absolute_path_to_entry_index_.end()) { - auto& entry = entries[it->second]; - entry.import_file = import_file; - entry.last_modification_time = modification_time; - return; - } - } - - { - optional import_entry = FindCompilationEntryForFile(import_file); - - Project::Entry entry; - entry.filename = filename; - if (import_entry) { - entry.args = import_entry->args; - } - - entry.import_file = import_file; - entry.last_modification_time = modification_time; - - // TODO: There might be a lot of thread contention here. - std::lock_guard lock(entries_modification_mutex_); - absolute_path_to_entry_index_[filename] = entries.size(); - entries.push_back(entry); - } -} +optional> Project::FindArgsForFile(const std::string& filename) { + auto entry = FindCompilationEntryForFile(filename); + if (!entry) + return nullopt; + return entry->args; +} \ No newline at end of file diff --git a/src/project.h b/src/project.h index 3727ed76..b64021a9 100644 --- a/src/project.h +++ b/src/project.h @@ -14,14 +14,10 @@ struct Project { struct Entry { std::string filename; std::vector args; - - std::string import_file; - optional last_modification_time; }; std::vector entries; spp::sparse_hash_map absolute_path_to_entry_index_; - std::mutex entries_modification_mutex_; // Loads a project for the given |directory|. // @@ -34,7 +30,7 @@ struct Project { // Lookup the CompilationEntry for |filename|. optional FindCompilationEntryForFile(const std::string& filename); - // Update the modification time for the given filename. This is thread-safe. - void UpdateFileState(const std::string& filename, const std::string& import_file, uint64_t modification_time); + // Helper that uses FindCompilationEntryForFile. + optional> FindArgsForFile(const std::string& filename); }; diff --git a/src/serializer.cc b/src/serializer.cc index c990e291..354eb4df 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -230,6 +230,7 @@ void Reflect(TVisitor& visitor, IndexedFile& value) { REFLECT_MEMBER(version); REFLECT_MEMBER(last_modification_time); REFLECT_MEMBER(import_file); + REFLECT_MEMBER(args); } REFLECT_MEMBER(dependencies); REFLECT_MEMBER(types);