From 9338bcfd0e56722c1e1585ee2f2a2957afb22677 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Wed, 19 Apr 2017 22:46:10 -0700 Subject: [PATCH] Improvements to loading project. - Don't reindex the file if the modification time has not changed. - Import file dependencies before importing other files, which might be a full-on index operation. --- src/command_line.cc | 21 ++++++++++++++++++--- src/project.cc | 30 ++++++++++++++++++++++++++---- src/project.h | 10 +++++++++- src/threaded_queue.h | 30 +++++++++++++++++++++++++++--- 4 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src/command_line.cc b/src/command_line.cc index 28f6d39a..c254da56 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -867,6 +867,7 @@ void RegisterMessageTypes() { bool IndexMain_DoIndex(IndexerConfig* config, FileConsumer::SharedState* file_consumer_shared, + Project* project, Index_DoIndexQueue* queue_do_index, Index_DoIdMapQueue* queue_do_id_map) { optional index_request = queue_do_index->TryDequeue(); @@ -892,8 +893,10 @@ bool IndexMain_DoIndex(IndexerConfig* config, Index_DoIndex dep_index_request(Index_DoIndex::Type::ImportOnly); dep_index_request.path = dependency_path; dep_index_request.args = index_request->args; - queue_do_index->Enqueue(std::move(dep_index_request)); + queue_do_index->PriorityEnqueue(std::move(dep_index_request)); } + + project->UpdateModificationTime(index_request->path, old_index->last_modification_time); Index_DoIdMap response(nullptr, std::move(old_index)); queue_do_id_map->Enqueue(std::move(response)); @@ -909,6 +912,17 @@ bool IndexMain_DoIndex(IndexerConfig* config, } // Parse request and send a response. + + // Skip index if file modification time didn't change. + optional entry = project->FindCompilationEntryForFile(index_request->path); + if (entry && entry->last_modification_time) { + int64_t modification_time = GetLastModificationTime(index_request->path); + if (modification_time == *entry->last_modification_time) { + time.ResetAndPrint("Skipping index update on " + index_request->path + " since file modification time has not changed"); + return true; + } + } + std::vector> indexes = Parse(config, file_consumer_shared, index_request->path, index_request->args); time.ResetAndPrint("Parsing/indexing " + index_request->path); @@ -973,6 +987,7 @@ void IndexJoinIndexUpdates(Index_OnIndexedQueue* queue_on_indexed) { void IndexMain( IndexerConfig* config, FileConsumer::SharedState* file_consumer_shared, + Project* project, Index_DoIndexQueue* queue_do_index, Index_DoIdMapQueue* queue_do_id_map, Index_OnIdMappedQueue* queue_on_id_mapped, @@ -990,7 +1005,7 @@ void IndexMain( // IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any // work. Running both also lets the user query the partially constructed // index. - bool did_index = IndexMain_DoIndex(config, file_consumer_shared, queue_do_index, queue_do_id_map); + bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, queue_do_index, queue_do_id_map); bool did_create_update = IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed); if (!did_index && !did_create_update) { @@ -1632,7 +1647,7 @@ void QueryDbMain(IndexerConfig* config) { std::cerr << "[querydb] Starting " << config->indexerCount << " indexers" << std::endl; for (int i = 0; i < config->indexerCount; ++i) { new std::thread([&]() { - IndexMain(config, &file_consumer_shared, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed); + IndexMain(config, &file_consumer_shared, &project, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed); }); } diff --git a/src/project.cc b/src/project.cc index 883dac05..9420bdcf 100644 --- a/src/project.cc +++ b/src/project.cc @@ -243,13 +243,35 @@ std::vector LoadCompilationEntriesFromDirectory(const std::strin void Project::Load(const std::string& directory) { entries = LoadCompilationEntriesFromDirectory(directory); + + absolute_path_to_entry_index_.resize(entries.size()); + for (int i = 0; i < entries.size(); ++i) + absolute_path_to_entry_index_[entries[i].filename] = i; } optional Project::FindCompilationEntryForFile(const std::string& filename) { - for (auto& entry : entries) { - if (filename == entry.filename) - return entry; - } + std::lock_guard lock(entries_modification_mutex_); + auto it = absolute_path_to_entry_index_.find(filename); + if (it != absolute_path_to_entry_index_.end()) + return entries[it->second]; return nullopt; } + +void Project::UpdateModificationTime(const std::string& filename, uint64_t modification_time) { + // TODO: There might be a lot of thread contention here. + std::lock_guard lock(entries_modification_mutex_); + + auto it = absolute_path_to_entry_index_.find(filename); + if (it != absolute_path_to_entry_index_.end()) { + entries[it->second].last_modification_time = modification_time; + } + else { + Project::Entry entry; + entry.filename = filename; + entry.last_modification_time = modification_time; + + absolute_path_to_entry_index_[filename] = entries.size(); + entries.push_back(entry); + } +} diff --git a/src/project.h b/src/project.h index 8495d653..b71d520d 100644 --- a/src/project.h +++ b/src/project.h @@ -1,6 +1,9 @@ #pragma once #include +#include + +#include #include #include @@ -11,10 +14,12 @@ struct Project { struct Entry { std::string filename; std::vector args; - // optional last_modification_time; + optional last_modification_time; }; std::vector entries; + spp::sparse_hash_map absolute_path_to_entry_index_; + std::mutex entries_modification_mutex_; // Loads a project for the given |directory|. // @@ -26,5 +31,8 @@ struct Project { // Lookup the CompilationEntry for |filename|. optional FindCompilationEntryForFile(const std::string& filename); + + // Update the modification time for the given filename. This is thread-safe. + void UpdateModificationTime(const std::string& filename, uint64_t modification_time); }; diff --git a/src/threaded_queue.h b/src/threaded_queue.h index ca0985ea..1a103e42 100644 --- a/src/threaded_queue.h +++ b/src/threaded_queue.h @@ -14,6 +14,13 @@ template class ThreadedQueue { public: + // Add an element to the front of the queue. + void PriorityEnqueue(T&& t) { + std::lock_guard lock(mutex_); + priority_.push(std::move(t)); + cv_.notify_one(); + } + // Add an element to the queue. void Enqueue(T&& t) { std::lock_guard lock(mutex_); @@ -26,7 +33,11 @@ public: std::lock_guard lock(mutex_); std::vector result; - result.reserve(queue_.size()); + result.reserve(priority_.size() + queue_.size()); + while (!priority_.empty()) { + result.emplace_back(std::move(priority_.front())); + priority_.pop(); + } while (!queue_.empty()) { result.emplace_back(std::move(queue_.front())); queue_.pop(); @@ -38,11 +49,17 @@ public: // If the queue is empty, wait untill an element is avaiable. T Dequeue() { std::unique_lock lock(mutex_); - while (queue_.empty()) { + while (priority_.empty() && queue_.empty()) { // release lock as long as the wait and reaquire it afterwards. cv_.wait(lock); } + if (!priority_.empty()) { + auto val = std::move(priority_.front()); + priority_.pop(); + return val; + } + auto val = std::move(queue_.front()); queue_.pop(); return val; @@ -52,15 +69,22 @@ public: // value if the queue is empty. optional TryDequeue() { std::lock_guard lock(mutex_); - if (queue_.empty()) + if (priority_.empty() && queue_.empty()) return nullopt; + if (!priority_.empty()) { + auto val = std::move(priority_.front()); + priority_.pop(); + return val; + } + auto val = std::move(queue_.front()); queue_.pop(); return std::move(val); } private: + std::queue priority_; std::queue queue_; mutable std::mutex mutex_; std::condition_variable cv_;