WIP on updating index; locations still broken

src/cache.cc

@@ -15,9 +15,6 @@ std::string GetCachedFileName(std::string source_file) {
 }
 
 std::unique_ptr<IndexedFile> LoadCachedFile(std::string filename) {
-  // TODO FIXME FIXME FIXME
-  return nullptr;
-
   std::string cache_file = GetCachedFileName(filename);
 
   std::ifstream cache;

src/command_line.cc

@@ -377,6 +377,7 @@ std::unique_ptr<IpcMessageQueue> BuildIpcMessageQueue(const std::string& name, s
   RegisterId<Ipc_TextDocumentDidOpen>(ipc.get());
   RegisterId<Ipc_TextDocumentDidChange>(ipc.get());
   RegisterId<Ipc_TextDocumentDidClose>(ipc.get());
+  RegisterId<Ipc_TextDocumentDidSave>(ipc.get());
   RegisterId<Ipc_TextDocumentComplete>(ipc.get());
   RegisterId<Ipc_TextDocumentDefinition>(ipc.get());
   RegisterId<Ipc_TextDocumentDocumentSymbol>(ipc.get());
@@ -397,6 +398,7 @@ void RegisterMessageTypes() {
   MessageRegistry::instance()->Register<Ipc_TextDocumentDidOpen>();
   MessageRegistry::instance()->Register<Ipc_TextDocumentDidChange>();
   MessageRegistry::instance()->Register<Ipc_TextDocumentDidClose>();
+  MessageRegistry::instance()->Register<Ipc_TextDocumentDidSave>();
   MessageRegistry::instance()->Register<Ipc_TextDocumentComplete>();
   MessageRegistry::instance()->Register<Ipc_TextDocumentDefinition>();
   MessageRegistry::instance()->Register<Ipc_TextDocumentDocumentSymbol>();
@@ -414,6 +416,17 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer,
 
   Timer time;
 
+  // TODO TODO TODO TODO TODO
+  // TODO TODO TODO TODO TODO
+  // TODO TODO TODO TODO TODO
+  // TODO TODO TODO TODO TODO
+  // We're not loading cached header files on restore. We should store the
+  // list of headers associated with a cc file in the cache and then load
+  // them here.
+  // TODO TODO TODO TODO TODO
+  // TODO TODO TODO TODO TODO
+  // TODO TODO TODO TODO TODO
+  // TODO TODO TODO TODO TODO
 
 
 
@@ -423,7 +436,7 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer,
   if (index_request->type == Index_DoIndex::Type::Import) {
     index_request->type = Index_DoIndex::Type::Update;
     std::unique_ptr<IndexedFile> old_index = LoadCachedFile(index_request->path);
-    time.ResetAndPrint("Loading cached index");
+    time.ResetAndPrint("Loading cached index " + index_request->path);
 
     // If import fails just do a standard update.
     if (old_index) {
@@ -437,7 +450,7 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer,
 
   // Parse request and send a response.
   std::vector<std::unique_ptr<IndexedFile>> indexes = Parse(file_consumer, index_request->path, index_request->args);
-  time.ResetAndPrint("Parsing/indexing");
+  time.ResetAndPrint("Parsing/indexing " + index_request->path);
 
   for (auto& current_index : indexes) {
     std::cerr << "Got index for " << current_index->path << std::endl;
@@ -450,7 +463,7 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer,
     // of the current 4).
 
     // Cache file so we can diff it later.
-    WriteToCache(index_request->path, *current_index);
+    WriteToCache(current_index->path, *current_index);
     time.ResetAndPrint("Cache index update to disk");
 
     // Send response to create id map.
@@ -602,28 +615,26 @@ void QueryDbMainLoop(
     case IpcId::TextDocumentDidChange: {
       auto msg = static_cast<Ipc_TextDocumentDidChange*>(message.get());
       working_files->OnChange(msg->params);
-
-      
-      // Send an index update request.
-      // TODO: we should only do this when we save. Figure out a way to handle code lens (dynamic offsets?)
-
-#if false
-      Index_DoIndex request(Index_DoIndex::Type::Update);
-      //WorkingFile* changed = working_files->GetFileByFilename(msg->params.textDocument.uri.GetPath());
-      optional<CompilationEntry> entry = project->FindCompilationEntryForFile(msg->params.textDocument.uri.GetPath());
-      request.path = msg->params.textDocument.uri.GetPath();
-      if (entry)
-        request.args = entry->args;
-      queue_do_index->Enqueue(std::move(request));
-#endif
-
       //std::cerr << "Changing " << msg->params.textDocument.uri.GetPath() << std::endl;
       break;
     }
     case IpcId::TextDocumentDidClose: {
       auto msg = static_cast<Ipc_TextDocumentDidClose*>(message.get());
       std::cerr << "Closing " << msg->params.textDocument.uri.GetPath() << std::endl;
-      //working_files->OnClose(msg->params);
+      working_files->OnClose(msg->params);
+      break;
+    }
+
+    case IpcId::TextDocumentDidSave: {
+      auto msg = static_cast<Ipc_TextDocumentDidSave*>(message.get());
+
+      // Send an index update request.
+      Index_DoIndex request(Index_DoIndex::Type::Update);
+      optional<CompilationEntry> entry = project->FindCompilationEntryForFile(msg->params.textDocument.uri.GetPath());
+      request.path = msg->params.textDocument.uri.GetPath();
+      if (entry)
+        request.args = entry->args;
+      queue_do_index->Enqueue(std::move(request));
       break;
     }
 
@@ -848,7 +859,7 @@ void QueryDbMainLoop(
 }
 
 void QueryDbMain() {
-  std::cerr << "Running QueryDb" << std::endl;
+  //std::cerr << "Running QueryDb" << std::endl;
 
   // Create queues.
   std::unique_ptr<IpcMessageQueue> ipc = BuildIpcMessageQueue(kIpcLanguageClientName, kQueueSizeBytes);
@@ -921,7 +932,7 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) {
       //response.result.capabilities.textDocumentSync->change = lsTextDocumentSyncKind::Full;
       //response.result.capabilities.textDocumentSync->willSave = true;
       //response.result.capabilities.textDocumentSync->willSaveWaitUntil = true;
-      response.result.capabilities.textDocumentSync = lsTextDocumentSyncKind::Incremental; // TODO: use incremental at some point
+      response.result.capabilities.textDocumentSync = lsTextDocumentSyncKind::Incremental;
 
       response.result.capabilities.completionProvider = lsCompletionOptions();
       response.result.capabilities.completionProvider->resolveProvider = false;
@@ -936,7 +947,7 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) {
 
       response.result.capabilities.workspaceSymbolProvider = true;
 
-      response.Write(std::cerr);
+      //response.Write(std::cerr);
       response.Write(std::cout);
       break;
     }
@@ -953,12 +964,13 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) {
 
     case IpcId::TextDocumentDidOpen:
     case IpcId::TextDocumentDidChange:
-    case IpcId::TextDocumentDidClose: {
+    case IpcId::TextDocumentDidClose:
+    case IpcId::TextDocumentDidSave:
     case IpcId::TextDocumentCompletion:
     case IpcId::TextDocumentDefinition:
     case IpcId::TextDocumentDocumentSymbol:
     case IpcId::TextDocumentCodeLens:
-    case IpcId::WorkspaceSymbol:
+    case IpcId::WorkspaceSymbol: {
       //std::cerr << "Sending message " << (int)message->method_id << std::endl;
       ipc->SendMessage(&ipc->for_server, message->method_id, *message.get());
       break;
@@ -1043,7 +1055,7 @@ int main(int argc, char** argv) {
   //bool loop = true;
   //while (loop)
   //  std::this_thread::sleep_for(std::chrono::milliseconds(10));
-  //std::this_thread::sleep_for(std::chrono::seconds(3));
+  std::this_thread::sleep_for(std::chrono::seconds(3));
 
   PlatformInit();
   RegisterMessageTypes();
@@ -1092,17 +1104,17 @@ int main(int argc, char** argv) {
     exit(0);
   }
   else if (HasOption(options, "--language-server")) {
-    std::cerr << "Running language server" << std::endl;
+    //std::cerr << "Running language server" << std::endl;
     LanguageServerMain(argv[0]);
     return 0;
   }
   else if (HasOption(options, "--querydb")) {
-    std::cerr << "Running querydb" << std::endl;
+    //std::cerr << "Running querydb" << std::endl;
     QueryDbMain();
     return 0;
   }
   else {
-    std::cerr << "Running language server" << std::endl;
+    //std::cerr << "Running language server" << std::endl;
     LanguageServerMain(argv[0]);
     return 0;
   }

src/file_consumer.cc

@@ -5,15 +5,6 @@
 
 FileConsumer::FileConsumer(SharedState* shared_state) : shared_(shared_state) {}
 
-std::vector<std::unique_ptr<IndexedFile>> FileConsumer::TakeLocalState() {
-  std::vector<std::unique_ptr<IndexedFile>> result;
-  for (auto& entry : local_) {
-    if (entry.second)
-      result.push_back(std::move(entry.second));
-  }
-  return result;
-}
-
 IndexedFile* FileConsumer::TryConsumeFile(const std::string& file) {
   // Try to find cached local result.
   auto it = local_.find(file);
@@ -29,3 +20,18 @@ IndexedFile* FileConsumer::TryConsumeFile(const std::string& file) {
   local_[file] = did_insert ? MakeUnique<IndexedFile>(file) : nullptr;
   return local_[file].get();
 }
+
+void FileConsumer::ForceLocal(const std::string& file) {
+  auto it = local_.find(file);
+  if (it == local_.end())
+    local_[file] = MakeUnique<IndexedFile>(file);
+}
+
+std::vector<std::unique_ptr<IndexedFile>> FileConsumer::TakeLocalState() {
+  std::vector<std::unique_ptr<IndexedFile>> result;
+  for (auto& entry : local_) {
+    if (entry.second)
+      result.push_back(std::move(entry.second));
+  }
+  return result;
+}

src/file_consumer.h

@@ -27,6 +27,9 @@ struct FileConsumer {
   // Returns IndexedFile for the file or nullptr.
   IndexedFile* TryConsumeFile(const std::string& file);
 
+  // Forcibly create a local file, even if it has already been parsed.
+  void ForceLocal(const std::string& file);
+
   // Returns and passes ownership of all local state.
   std::vector<std::unique_ptr<IndexedFile>> TakeLocalState();
 

src/indexer.cc

@@ -1254,6 +1254,8 @@ void indexEntityReference(CXClientData client_data,
 }
 
 std::vector<std::unique_ptr<IndexedFile>> Parse(FileConsumer* file_consumer, std::string filename, std::vector<std::string> args, bool dump_ast) {
+  //return {};
+
   clang_enableStackTraces();
   clang_toggleCrashRecovery(1);
 
@@ -1288,6 +1290,8 @@ std::vector<std::unique_ptr<IndexedFile>> Parse(FileConsumer* file_consumer, std
   NamespaceHelper ns;
   IndexParam param(file_consumer, &ns);
 
+  file_consumer->ForceLocal(filename);
+
   std::cerr << "!! [START] Indexing " << filename << std::endl;
   clang_indexTranslationUnit(index_action, &param, callbacks, sizeof(callbacks),
                              CXIndexOpt_IndexFunctionLocalSymbols | CXIndexOpt_SkipParsedBodiesInSession | CXIndexOpt_IndexImplicitTemplateInstantiations,

src/ipc.cc

@@ -16,6 +16,8 @@ const char* IpcIdToString(IpcId id) {
     return "textDocument/didChange";
   case IpcId::TextDocumentDidClose:
     return "textDocument/didClose";
+  case IpcId::TextDocumentDidSave:
+    return "textDocument/didSave";
   case IpcId::TextDocumentCompletion:
     return "textDocument/completion";
   case IpcId::TextDocumentDefinition:

src/ipc.h

@@ -13,6 +13,7 @@ enum class IpcId : int {
   TextDocumentDidOpen,
   TextDocumentDidChange,
   TextDocumentDidClose,
+  TextDocumentDidSave,
   TextDocumentCompletion,
   TextDocumentDefinition,
   TextDocumentDocumentSymbol,

src/language_server_api.h

@@ -1075,7 +1075,21 @@ MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidClose::Params, textDocument);
 MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidClose, params);
 
 
+struct Ipc_TextDocumentDidSave : public IpcMessage<Ipc_TextDocumentDidSave> {
+  struct Params {
+    // The document that was saved.
+    lsTextDocumentIdentifier textDocument;
     
+    // Optional the content when saved. Depends on the includeText value
+    // when the save notifcation was requested.
+    // std::string text;
+  };
+
+  const static IpcId kIpcId = IpcId::TextDocumentDidSave;
+  Params params;
+};
+MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidSave::Params, textDocument);
+MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidSave, params);
 
 
 

src/platform_win.cc

@@ -41,7 +41,7 @@ struct PlatformMutexWin : public PlatformMutex {
   HANDLE raw_mutex = INVALID_HANDLE_VALUE;
 
   PlatformMutexWin(const std::string& name) {
-    std::cerr << "[win] Creating mutex with name " << name << std::endl;
+    //std::cerr << "[win] Creating mutex with name " << name << std::endl;
     raw_mutex = CreateMutex(nullptr, false /*initial_owner*/, name.c_str());
     CheckForError({ ERROR_ALREADY_EXISTS });
   }
@@ -72,8 +72,8 @@ struct PlatformSharedMemoryWin : public PlatformSharedMemory {
   HANDLE shmem_;
 
   PlatformSharedMemoryWin(const std::string& name, size_t capacity) {
-    std::cerr << "[win] Creating shared memory with name " << name
+    //std::cerr << "[win] Creating shared memory with name " << name
-      << " and capacity " << capacity << std::endl;
+    //  << " and capacity " << capacity << std::endl;
     this->name = name;
 
     shmem_ = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0,

src/project.cc

@@ -13,12 +13,16 @@ std::vector<CompilationEntry> LoadFromDirectoryListing(const std::string& projec
   std::vector<CompilationEntry> result;
 
   std::vector<std::string> args;
+  std::cerr << "Using arguments: ";
   for (const std::string& line : ReadLines(project_directory + "/clang_args")) {
     if (line.empty() || StartsWith(line, "#"))
       continue;
-    std::cerr << "Adding argument " << line << std::endl;
+    if (!args.empty())
+      std::cerr << ", ";
+    std::cerr << line;
     args.push_back(line);
   }
+  std::cerr << std::endl;
 
 
   std::vector<std::string> files = GetFilesInFolder(project_directory, true /*recursive*/, true /*add_folder_to_path*/);

src/query.cc

@@ -299,9 +299,10 @@ void CompareGroups(
 
 // TODO: consider having separate lookup maps so they are smaller (maybe
 // lookups will go faster).
+// TODO: Figure out where the invalid SymbolKinds are coming from.
 QueryFileId GetQueryFileIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
   auto it = query_db->usr_to_symbol.find(usr);
-  if (it != query_db->usr_to_symbol.end()) {
+  if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) {
     assert(it->second.kind == SymbolKind::File);
     return QueryFileId(it->second.idx);
   }
@@ -314,7 +315,7 @@ QueryFileId GetQueryFileIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
 
 QueryTypeId GetQueryTypeIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
   auto it = query_db->usr_to_symbol.find(usr);
-  if (it != query_db->usr_to_symbol.end()) {
+  if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) {
     assert(it->second.kind == SymbolKind::Type);
     return QueryTypeId(it->second.idx);
   }
@@ -327,7 +328,7 @@ QueryTypeId GetQueryTypeIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
 
 QueryFuncId GetQueryFuncIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
   auto it = query_db->usr_to_symbol.find(usr);
-  if (it != query_db->usr_to_symbol.end()) {
+  if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) {
     assert(it->second.kind == SymbolKind::Func);
     return QueryFuncId(it->second.idx);
   }
@@ -340,7 +341,7 @@ QueryFuncId GetQueryFuncIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
 
 QueryVarId GetQueryVarIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
   auto it = query_db->usr_to_symbol.find(usr);
-  if (it != query_db->usr_to_symbol.end()) {
+  if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) {
     assert(it->second.kind == SymbolKind::Var);
     return QueryVarId(it->second.idx);
   }
@@ -491,8 +492,8 @@ IndexUpdate::IndexUpdate(const IdMap& previous_id_map, const IdMap& current_id_m
                       previous, current, \
                       &removed, &added); \
     if (did_add) {\
-      std::cerr << "Adding mergeable update on " << current_def->def.short_name << " (" << current_def->def.usr << ") for field " << #index_name << std::endl; \
+      /*std::cerr << "Adding mergeable update on " << current_def->def.short_name << " (" << current_def->def.usr << ") for field " << #index_name << std::endl;*/ \
-      query_name.push_back(MergeableUpdate<type>(current_def->def.usr, removed, added)); \
+      query_name.push_back(MergeableUpdate<type>(current_def->def.usr, added, removed)); \
     } \
   }
   // File

src/query.h

@@ -67,7 +67,7 @@ struct SymbolIdx {
   SymbolKind kind;
   size_t idx;
 
-  SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use.
+  explicit SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use.
   SymbolIdx(SymbolKind kind, uint64_t idx) : kind(kind), idx(idx) {}
 
   bool operator==(const SymbolIdx& that) const {

src/serializer.cc

@@ -241,5 +241,21 @@ optional<IndexedFile> Deserialize(std::string path, std::string serialized) {
   IndexedFile file(path);
   Reflect(reader, file);
   
+  // Restore non-serialized state.
+  file.path = path;
+  file.id_cache.primary_file = file.path;
+  for (const auto& type : file.types) {
+    file.id_cache.type_id_to_usr[type.id] = type.def.usr;
+    file.id_cache.usr_to_type_id[type.def.usr] = type.id;
+  }
+  for (const auto& func : file.funcs) {
+    file.id_cache.func_id_to_usr[func.id] = func.def.usr;
+    file.id_cache.usr_to_func_id[func.def.usr] = func.id;
+  }
+  for (const auto& var : file.vars) {
+    file.id_cache.var_id_to_usr[var.id] = var.def.usr;
+    file.id_cache.usr_to_var_id[var.def.usr] = var.id;
+  }
+
   return file;
 }