Update WorkingFile indexed content correctly.

We copy the file contents we indexed over to the index cache folder. Then we load those file contents into the WorkingFile instance as needed.

This means code lens should never get out of sync, as the index buffer cache will always be correct.

src/cache.cc

@@ -1,27 +1,29 @@
 #include "cache.h"
 
 #include "indexer.h"
+#include "platform.h"
+#include "language_server_api.h"
 
 #include <algorithm>
 
 namespace {
 
-std::string GetCachedFileName(const std::string& cache_directory, std::string source_file) {
+std::string GetCachedBaseFileName(const std::string& cache_directory, std::string source_file) {
   assert(!cache_directory.empty());
   std::replace(source_file.begin(), source_file.end(), '\\', '_');
   std::replace(source_file.begin(), source_file.end(), '/', '_');
   std::replace(source_file.begin(), source_file.end(), ':', '_');
   std::replace(source_file.begin(), source_file.end(), '.', '_');
-  return cache_directory + source_file + ".json";
+  return cache_directory + source_file;
 }
 
 }  // namespace
 
-std::unique_ptr<IndexedFile> LoadCachedFile(IndexerConfig* config, const std::string& filename) {
+std::unique_ptr<IndexedFile> LoadCachedIndex(IndexerConfig* config, const std::string& filename) {
   if (!config->enableCacheRead)
     return nullptr;
 
-  optional<std::string> file_content = ReadContent(GetCachedFileName(config->cacheDirectory, filename));
+  optional<std::string> file_content = ReadContent(GetCachedBaseFileName(config->cacheDirectory, filename) + ".json");
   if (!file_content)
     return nullptr;
 
@@ -32,14 +34,24 @@ std::unique_ptr<IndexedFile> LoadCachedFile(IndexerConfig* config, const std::st
   return nullptr;
 }
 
+optional<std::string> LoadCachedFileContents(IndexerConfig* config, const std::string& filename) {
+  if (!config->enableCacheRead)
+    return nullptr;
+
+  return ReadContent(GetCachedBaseFileName(config->cacheDirectory, filename) + ".txt");
+}
+
 void WriteToCache(IndexerConfig* config, const std::string& filename, IndexedFile& file) {
   if (!config->enableCacheWrite)
     return;
 
-  std::string indexed_content = Serialize(file);
+  std::string cache_basename = GetCachedBaseFileName(config->cacheDirectory, filename);
 
+  CopyFileTo(cache_basename + ".txt", filename);
+
+  std::string indexed_content = Serialize(file);
   std::ofstream cache;
-  cache.open(GetCachedFileName(config->cacheDirectory, filename));
+  cache.open(cache_basename + ".json");
   assert(cache.good());
   cache << indexed_content;
   cache.close();

src/cache.h

@@ -1,12 +1,17 @@
 #pragma once
 
-#include "language_server_api.h"
+#include <optional.h>
-
 #include <memory>
 #include <string>
 
+using std::experimental::optional;
+using std::experimental::nullopt;
+
+struct IndexerConfig;
 struct IndexedFile;
 
-std::unique_ptr<IndexedFile> LoadCachedFile(IndexerConfig* config, const std::string& filename);
+std::unique_ptr<IndexedFile> LoadCachedIndex(IndexerConfig* config, const std::string& filename);
+
+optional<std::string> LoadCachedFileContents(IndexerConfig* config, const std::string& filename);
 
 void WriteToCache(IndexerConfig* config, const std::string& filename, IndexedFile& file);

src/command_line.cc

@@ -893,7 +893,7 @@ void ImportCachedIndex(IndexerConfig* config,
 
   Timer time;
 
-  std::unique_ptr<IndexedFile> cache = LoadCachedFile(config, path);
+  std::unique_ptr<IndexedFile> cache = LoadCachedIndex(config, path);
   time.ResetAndPrint("Reading cached index from disk " + path);
   if (!cache)
     return;
@@ -914,7 +914,7 @@ void ParseFile(IndexerConfig* config,
   Timer time;
 
   // Parse request and send a response.
-  std::unique_ptr<IndexedFile> cached_path_index = LoadCachedFile(config, path);
+  std::unique_ptr<IndexedFile> cached_path_index = LoadCachedIndex(config, path);
 
   if (cached_path_index) {
     // Give the user dependencies if requested.
@@ -946,7 +946,7 @@ void ParseFile(IndexerConfig* config,
     if (new_index->path == path)
       cached_index = std::move(cached_path_index);
     else
-      cached_index = LoadCachedFile(config, new_index->path);
+      cached_index = LoadCachedIndex(config, new_index->path);
     time.ResetAndPrint("Loading cached index");
 
     // Update dependencies on |new_index|, since they won't get reparsed if we
@@ -1026,10 +1026,10 @@ bool IndexMain_DoCreateIndexUpdate(
   Timer time;
   IndexUpdate update = IndexUpdate::CreateDelta(response->previous_id_map.get(), response->current_id_map.get(),
     response->previous_index.get(), response->current_index.get());
-  time.ResetAndPrint("Creating delta IndexUpdate");
+  time.ResetAndPrint("[indexer] Creating delta IndexUpdate");
   Index_OnIndexed reply(update);
   queue_on_indexed->Enqueue(std::move(reply));
-  time.ResetAndPrint("Sending update to server");
+  time.ResetAndPrint("[indexer] Sending update to server");
 
   return true;
 }
@@ -1048,7 +1048,7 @@ void IndexJoinIndexUpdates(Index_OnIndexedQueue* queue_on_indexed) {
 
     Timer time;
     root->update.Merge(to_join->update);
-    time.ResetAndPrint("Indexer joining two querydb updates");
+    time.ResetAndPrint("[indexer] Joining two querydb updates");
   }
 }
 
@@ -1199,7 +1199,13 @@ void QueryDbMainLoop(
     case IpcId::TextDocumentDidOpen: {
       auto msg = static_cast<Ipc_TextDocumentDidOpen*>(message.get());
       //std::cerr << "Opening " << msg->params.textDocument.uri.GetPath() << std::endl;
-      working_files->OnOpen(msg->params);
+      WorkingFile* working_file = working_files->OnOpen(msg->params);
+      optional<std::string> cached_file_contents = LoadCachedFileContents(config, msg->params.textDocument.uri.GetPath());
+      if (cached_file_contents)
+        working_file->SetIndexContent(*cached_file_contents);
+      else
+        working_file->SetIndexContent(working_file->buffer_content);
+
       break;
     }
     case IpcId::TextDocumentDidChange: {
@@ -1219,15 +1225,14 @@ void QueryDbMainLoop(
       auto msg = static_cast<Ipc_TextDocumentDidSave*>(message.get());
 
       std::string path = msg->params.textDocument.uri.GetPath();
-
-      // TODO: Update working file indexed content when we actually apply the
-      // index update.
-      WorkingFile* working_file = working_files->GetFileByFilename(path);
-      if (working_file)
-        working_file->SetIndexContent(working_file->buffer_content);
-
       // Send an index update request.
       queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::Parse, path, project->FindArgsForFile(path)));
+
+      // Copy current buffer content so it can be applied when index request is done.
+      WorkingFile* working_file = working_files->GetFileByFilename(path);
+      if (working_file)
+        working_file->pending_new_index_content = working_file->buffer_content;
+
       break;
     }
 
@@ -1579,13 +1584,13 @@ void QueryDbMainLoop(
       response.id = msg->id;
 
 
-      std::cerr << "- Considering " << db->detailed_names.size()
+      std::cerr << "[querydb] Considering " << db->detailed_names.size()
         << " candidates for query " << msg->params.query << std::endl;
 
       std::string query = msg->params.query;
       for (int i = 0; i < db->detailed_names.size(); ++i) {
         if (response.result.size() >= config->maxWorkspaceSearchResults) {
-          std::cerr << "Query exceeded maximum number of responses (" << config->maxWorkspaceSearchResults << "), output may not contain all results" << std::endl;
+          std::cerr << "[querydb] - Query exceeded maximum number of responses (" << config->maxWorkspaceSearchResults << "), output may not contain all results" << std::endl;
           break;
         }
 
@@ -1610,7 +1615,7 @@ void QueryDbMainLoop(
         }
       }
 
-      std::cerr << "- Found " << response.result.size() << " results for query " << query << std::endl;
+      std::cerr << "[querydb] - Found " << response.result.size() << " results for query " << query << std::endl;
       ipc->SendOutMessageToClient(response);
       break;
     }
@@ -1634,6 +1639,7 @@ void QueryDbMainLoop(
 
     Index_OnIdMapped response;
     Timer time;
+
     if (request->previous) {
       response.previous_id_map = MakeUnique<IdMap>(db, request->previous->id_cache);
       response.previous_index = std::move(request->previous);
@@ -1641,7 +1647,7 @@ void QueryDbMainLoop(
 
     assert(request->current);
     response.current_id_map = MakeUnique<IdMap>(db, request->current->id_cache);
-    time.ResetAndPrint("Create IdMap " + request->current->path);
+    time.ResetAndPrint("[querydb] Create IdMap " + request->current->path);
     response.current_index = std::move(request->current);
 
     queue_on_id_mapped->Enqueue(std::move(response));
@@ -1653,8 +1659,31 @@ void QueryDbMainLoop(
       break;
 
     Timer time;
+
+    for (auto& updated_file : response->update.files_def_update) {
+      // TODO: We're reading a file on querydb thread. This is slow!! If it is a
+      // problem in practice we need to create a file reader queue, dispatch the
+      // read to it, get a response, and apply the new index then.
+      WorkingFile* working_file = working_files->GetFileByFilename(updated_file.path);
+      if (working_file) {
+        if (working_file->pending_new_index_content) {
+          working_file->SetIndexContent(*working_file->pending_new_index_content);
+          working_file->pending_new_index_content = nullopt;
+          time.ResetAndPrint("[querydb] Update WorkingFile index contents (via in-memory buffer) for " + updated_file.path);
+        }
+        else {
+          optional<std::string> cached_file_contents = LoadCachedFileContents(config, updated_file.path);
+          if (cached_file_contents)
+            working_file->SetIndexContent(*cached_file_contents);
+          else
+            working_file->SetIndexContent(working_file->buffer_content);
+          time.ResetAndPrint("[querydb] Update WorkingFile index contents (via disk load) for " + updated_file.path);
+        }
+      }
+    }
+
     db->ApplyIndexUpdate(&response->update);
-    time.ResetAndPrint("Applying index update");
+    time.ResetAndPrint("[querydb] Applying index update");
   }
 }
 

src/platform.h

@@ -39,5 +39,7 @@ void SetCurrentThreadName(const std::string& thread_name);
 
 int64_t GetLastModificationTime(const std::string& absolute_path);
 
+void CopyFileTo(const std::string& destination, const std::string& source);
+
 // Returns any clang arguments that are specific to the current platform.
 std::vector<std::string> GetPlatformClangArguments();

src/platform_linux.cc

@@ -23,6 +23,9 @@
 #include <sys/types.h> // required for stat.h
 #include <sys/stat.h>
 
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
 
 #include <fcntl.h>    /* For O_* constants */
 #include <sys/stat.h> /* For mode constants */
@@ -182,6 +185,52 @@ int64_t GetLastModificationTime(const std::string& absolute_path) {
   return buf.st_mtime;
 }
 
+// See http://stackoverflow.com/q/13198627
+void CopyFileTo(const std::string& dest, const std::string& source) {
+  char buf[4096];
+  ssize_t nread;
+  int saved_errno;
+
+  int fd_from = open(source.c_str(), O_RDONLY);
+  if (fd_from < 0)
+    return;
+
+  int fd_to = open(dest.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0666);
+  if (fd_to < 0)
+    goto out_error;
+
+  while (nread = read(fd_from, buf, sizeof buf), nread > 0) {
+    char *out_ptr = buf;
+    ssize_t nwritten;
+
+    do {
+      nwritten = write(fd_to, out_ptr, nread);
+
+      if (nwritten >= 0) {
+        nread -= nwritten;
+        out_ptr += nwritten;
+      }
+      else if (errno != EINTR)
+        goto out_error;
+    } while (nread > 0);
+  }
+
+  if (nread == 0) {
+    if (close(fd_to) < 0) {
+      fd_to = -1;
+      goto out_error;
+    }
+    close(fd_from);
+
+    return;
+  }
+
+out_error:
+  close(fd_from);
+  if (fd_to >= 0)
+    close(fd_to);
+}
+
 std::vector<std::string> GetPlatformClangArguments() {
   // TODO: use install config variable for path?
   return {

src/platform_win.cc

@@ -202,6 +202,13 @@ int64_t GetLastModificationTime(const std::string& absolute_path) {
   return buf.st_mtime;
 }
 
+void CopyFileTo(const std::string& destination, const std::string& source) {
+  CopyFile(
+    source.c_str(),
+    destination.c_str(),
+    false /*failIfExists*/);
+}
+
 std::vector<std::string> GetPlatformClangArguments() {
   return {
     "-fms-compatibility",

src/working_files.cc

@@ -24,8 +24,8 @@ int GetOffsetForPosition(lsPosition position, const std::string& content) {
 WorkingFile::WorkingFile(const std::string& filename, const std::string& buffer_content)
     : filename(filename), buffer_content(buffer_content) {
   OnBufferContentUpdated();
-  // TODO: use cached index file contents
+
-  SetIndexContent(buffer_content);
+  // SetIndexContent gets called when the file is opened.
 }
 
 void WorkingFile::SetIndexContent(const std::string& index_content) {
@@ -155,21 +155,20 @@ WorkingFile* WorkingFiles::GetFileByFilename(const std::string& filename) {
   return nullptr;
 }
 
-void WorkingFiles::OnOpen(const Ipc_TextDocumentDidOpen::Params& open) {
+WorkingFile* WorkingFiles::OnOpen(const Ipc_TextDocumentDidOpen::Params& open) {
   std::string filename = open.textDocument.uri.GetPath();
   std::string content = open.textDocument.text;
 
   // The file may already be open.
   if (WorkingFile* file = GetFileByFilename(filename)) {
     file->version = open.textDocument.version;
-    // TODO: Load saved indexed content and not the initial buffer content.
-    file->SetIndexContent(content);
     file->buffer_content = content;
     file->OnBufferContentUpdated();
-    return;
+    return file;
   }
 
   files.push_back(MakeUnique<WorkingFile>(filename, content));
+  return files[files.size() - 1].get();
 }
 
 void WorkingFiles::OnChange(const Ipc_TextDocumentDidChange::Params& change) {

src/working_files.h

@@ -27,6 +27,12 @@ struct WorkingFile {
   // Note: The items in the value entry are 1-based liness.
   std::unordered_map<std::string, std::vector<int>> all_buffer_lines_lookup;
 
+  // The buffer content when the file was saved. This will be applied with
+  // SetIndexContent when the index has been updated and applied.
+  //
+  // This is an optimization that lets us avoid a disk read on the querydb
+  // thread when actively editing and saving a file.
+  optional<std::string> pending_new_index_content;
 
   WorkingFile(const std::string& filename, const std::string& buffer_content);
 
@@ -48,7 +54,7 @@ struct WorkingFile {
 struct WorkingFiles {
   // Find the file with the given filename.
   WorkingFile* GetFileByFilename(const std::string& filename);
-  void OnOpen(const Ipc_TextDocumentDidOpen::Params& open);
+  WorkingFile* OnOpen(const Ipc_TextDocumentDidOpen::Params& open);
   void OnChange(const Ipc_TextDocumentDidChange::Params& change);
   void OnClose(const Ipc_TextDocumentDidClose::Params& close);