Improvements to loading project.

- Don't reindex the file if the modification time has not changed.
- Import file dependencies before importing other files, which might be a full-on index operation.

src/command_line.cc

@@ -867,6 +867,7 @@ void RegisterMessageTypes() {
 
 bool IndexMain_DoIndex(IndexerConfig* config,
                        FileConsumer::SharedState* file_consumer_shared,
+                       Project* project,
                        Index_DoIndexQueue* queue_do_index,
                        Index_DoIdMapQueue* queue_do_id_map) {
   optional<Index_DoIndex> index_request = queue_do_index->TryDequeue();
@@ -892,8 +893,10 @@ bool IndexMain_DoIndex(IndexerConfig* config,
         Index_DoIndex dep_index_request(Index_DoIndex::Type::ImportOnly);
         dep_index_request.path = dependency_path;
         dep_index_request.args = index_request->args;
-        queue_do_index->Enqueue(std::move(dep_index_request));
+        queue_do_index->PriorityEnqueue(std::move(dep_index_request));
       }
+      
+      project->UpdateModificationTime(index_request->path, old_index->last_modification_time);
 
       Index_DoIdMap response(nullptr, std::move(old_index));
       queue_do_id_map->Enqueue(std::move(response));
@@ -909,6 +912,17 @@ bool IndexMain_DoIndex(IndexerConfig* config,
   }
 
   // Parse request and send a response.
+
+  // Skip index if file modification time didn't change.
+  optional<Project::Entry> entry = project->FindCompilationEntryForFile(index_request->path);
+  if (entry && entry->last_modification_time) {
+    int64_t modification_time = GetLastModificationTime(index_request->path);
+    if (modification_time == *entry->last_modification_time) {
+      time.ResetAndPrint("Skipping index update on " + index_request->path + " since file modification time has not changed");
+      return true;
+    }
+  }
+
   std::vector<std::unique_ptr<IndexedFile>> indexes = Parse(config, file_consumer_shared, index_request->path, index_request->args);
   time.ResetAndPrint("Parsing/indexing " + index_request->path);
 
@@ -973,6 +987,7 @@ void IndexJoinIndexUpdates(Index_OnIndexedQueue* queue_on_indexed) {
 void IndexMain(
   IndexerConfig* config,
   FileConsumer::SharedState* file_consumer_shared,
+  Project* project,
   Index_DoIndexQueue* queue_do_index,
   Index_DoIdMapQueue* queue_do_id_map,
   Index_OnIdMappedQueue* queue_on_id_mapped,
@@ -990,7 +1005,7 @@ void IndexMain(
     // IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any
     // work. Running both also lets the user query the partially constructed
     // index.
-    bool did_index = IndexMain_DoIndex(config, file_consumer_shared, queue_do_index, queue_do_id_map);
+    bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, queue_do_index, queue_do_id_map);
     bool did_create_update = IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed);
     if (!did_index && !did_create_update) {
 
@@ -1632,7 +1647,7 @@ void QueryDbMain(IndexerConfig* config) {
   std::cerr << "[querydb] Starting " << config->indexerCount << " indexers" << std::endl;
   for (int i = 0; i < config->indexerCount; ++i) {
     new std::thread([&]() {
-      IndexMain(config, &file_consumer_shared, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed);
+      IndexMain(config, &file_consumer_shared, &project, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed);
     });
   }
 

src/project.cc

@@ -243,13 +243,35 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(const std::strin
 
 void Project::Load(const std::string& directory) {
   entries = LoadCompilationEntriesFromDirectory(directory);
+
+  absolute_path_to_entry_index_.resize(entries.size());
+  for (int i = 0; i < entries.size(); ++i)
+    absolute_path_to_entry_index_[entries[i].filename] = i;
 }
 
 optional<Project::Entry> Project::FindCompilationEntryForFile(const std::string& filename) {
-  for (auto& entry : entries) {
-    if (filename == entry.filename)
-      return entry;
-  }
+  std::lock_guard<std::mutex> lock(entries_modification_mutex_);
 
+  auto it = absolute_path_to_entry_index_.find(filename);
+  if (it != absolute_path_to_entry_index_.end())
+    return entries[it->second];
   return nullopt;
 }
+
+void Project::UpdateModificationTime(const std::string& filename, uint64_t modification_time) {
+  // TODO: There might be a lot of thread contention here.
+  std::lock_guard<std::mutex> lock(entries_modification_mutex_);
+
+  auto it = absolute_path_to_entry_index_.find(filename);
+  if (it != absolute_path_to_entry_index_.end()) {
+    entries[it->second].last_modification_time = modification_time;
+  }
+  else {
+    Project::Entry entry;
+    entry.filename = filename;
+    entry.last_modification_time = modification_time;
+
+    absolute_path_to_entry_index_[filename] = entries.size();
+    entries.push_back(entry);
+  }
+}

src/project.h

@@ -1,6 +1,9 @@
 #pragma once
 
 #include <optional.h>
+#include <sparsepp/spp.h>
+
+#include <mutex>
 #include <string>
 #include <vector>
 
@@ -11,10 +14,12 @@ struct Project {
   struct Entry {
     std::string filename;
     std::vector<std::string> args;
-    // optional<uint64_t> last_modification_time;
+    optional<uint64_t> last_modification_time;
   };
 
   std::vector<Entry> entries;
+  spp::sparse_hash_map<std::string, int> absolute_path_to_entry_index_;
+  std::mutex entries_modification_mutex_;
 
   // Loads a project for the given |directory|.
   //
@@ -26,5 +31,8 @@ struct Project {
 
   // Lookup the CompilationEntry for |filename|.
   optional<Entry> FindCompilationEntryForFile(const std::string& filename);
+
+  // Update the modification time for the given filename. This is thread-safe.
+  void UpdateModificationTime(const std::string& filename, uint64_t modification_time);
 };
 

src/threaded_queue.h

@@ -14,6 +14,13 @@
 template <class T>
 class ThreadedQueue {
 public:
+  // Add an element to the front of the queue.
+  void PriorityEnqueue(T&& t) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    priority_.push(std::move(t));
+    cv_.notify_one();
+  }
+
   // Add an element to the queue.
   void Enqueue(T&& t) {
     std::lock_guard<std::mutex> lock(mutex_);
@@ -26,7 +33,11 @@ public:
     std::lock_guard<std::mutex> lock(mutex_);
 
     std::vector<T> result;
-    result.reserve(queue_.size());
+    result.reserve(priority_.size() + queue_.size());
+    while (!priority_.empty()) {
+      result.emplace_back(std::move(priority_.front()));
+      priority_.pop();
+    }
     while (!queue_.empty()) {
       result.emplace_back(std::move(queue_.front()));
       queue_.pop();
@@ -38,11 +49,17 @@ public:
   // If the queue is empty, wait untill an element is avaiable.
   T Dequeue() {
     std::unique_lock<std::mutex> lock(mutex_);
-    while (queue_.empty()) {
+    while (priority_.empty() && queue_.empty()) {
       // release lock as long as the wait and reaquire it afterwards.
       cv_.wait(lock);
     }
     
+    if (!priority_.empty()) {
+      auto val = std::move(priority_.front());
+      priority_.pop();
+      return val;
+    }
+
     auto val = std::move(queue_.front());
     queue_.pop();
     return val;
@@ -52,15 +69,22 @@ public:
   // value if the queue is empty.
   optional<T> TryDequeue() {
     std::lock_guard<std::mutex> lock(mutex_);
-    if (queue_.empty())
+    if (priority_.empty() && queue_.empty())
       return nullopt;
 
+    if (!priority_.empty()) {
+      auto val = std::move(priority_.front());
+      priority_.pop();
+      return val;
+    }
+
     auto val = std::move(queue_.front());
     queue_.pop();
     return std::move(val);
   }
 
  private:
+  std::queue<T> priority_;
   std::queue<T> queue_;
   mutable std::mutex mutex_;
   std::condition_variable cv_;