From 527439b7c2115c681f022c326de12fb56af13a49 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Sun, 9 Apr 2017 17:08:54 -0700 Subject: [PATCH] WIP on updating index; locations still broken --- src/cache.cc | 3 -- src/command_line.cc | 68 +++++++++++++++++++++++---------------- src/file_consumer.cc | 24 ++++++++------ src/file_consumer.h | 3 ++ src/indexer.cc | 4 +++ src/ipc.cc | 2 ++ src/ipc.h | 1 + src/language_server_api.h | 14 ++++++++ src/platform_win.cc | 6 ++-- src/project.cc | 6 +++- src/query.cc | 13 ++++---- src/query.h | 2 +- src/serializer.cc | 16 +++++++++ 13 files changed, 111 insertions(+), 51 deletions(-) diff --git a/src/cache.cc b/src/cache.cc index f33ae966..249ba44d 100644 --- a/src/cache.cc +++ b/src/cache.cc @@ -15,9 +15,6 @@ std::string GetCachedFileName(std::string source_file) { } std::unique_ptr LoadCachedFile(std::string filename) { - // TODO FIXME FIXME FIXME - return nullptr; - std::string cache_file = GetCachedFileName(filename); std::ifstream cache; diff --git a/src/command_line.cc b/src/command_line.cc index e8f8ac96..2ec3577e 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -377,6 +377,7 @@ std::unique_ptr BuildIpcMessageQueue(const std::string& name, s RegisterId(ipc.get()); RegisterId(ipc.get()); RegisterId(ipc.get()); + RegisterId(ipc.get()); RegisterId(ipc.get()); RegisterId(ipc.get()); RegisterId(ipc.get()); @@ -397,6 +398,7 @@ void RegisterMessageTypes() { MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); + MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); @@ -414,6 +416,17 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer, Timer time; + // TODO TODO TODO TODO TODO + // TODO TODO TODO TODO TODO + // TODO TODO TODO TODO TODO + // TODO TODO TODO TODO TODO + // We're not loading cached header files on restore. We should store the + // list of headers associated with a cc file in the cache and then load + // them here. + // TODO TODO TODO TODO TODO + // TODO TODO TODO TODO TODO + // TODO TODO TODO TODO TODO + // TODO TODO TODO TODO TODO @@ -423,7 +436,7 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer, if (index_request->type == Index_DoIndex::Type::Import) { index_request->type = Index_DoIndex::Type::Update; std::unique_ptr old_index = LoadCachedFile(index_request->path); - time.ResetAndPrint("Loading cached index"); + time.ResetAndPrint("Loading cached index " + index_request->path); // If import fails just do a standard update. if (old_index) { @@ -437,7 +450,7 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer, // Parse request and send a response. std::vector> indexes = Parse(file_consumer, index_request->path, index_request->args); - time.ResetAndPrint("Parsing/indexing"); + time.ResetAndPrint("Parsing/indexing " + index_request->path); for (auto& current_index : indexes) { std::cerr << "Got index for " << current_index->path << std::endl; @@ -450,7 +463,7 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer, // of the current 4). // Cache file so we can diff it later. - WriteToCache(index_request->path, *current_index); + WriteToCache(current_index->path, *current_index); time.ResetAndPrint("Cache index update to disk"); // Send response to create id map. @@ -602,28 +615,26 @@ void QueryDbMainLoop( case IpcId::TextDocumentDidChange: { auto msg = static_cast(message.get()); working_files->OnChange(msg->params); - - - // Send an index update request. - // TODO: we should only do this when we save. Figure out a way to handle code lens (dynamic offsets?) - -#if false - Index_DoIndex request(Index_DoIndex::Type::Update); - //WorkingFile* changed = working_files->GetFileByFilename(msg->params.textDocument.uri.GetPath()); - optional entry = project->FindCompilationEntryForFile(msg->params.textDocument.uri.GetPath()); - request.path = msg->params.textDocument.uri.GetPath(); - if (entry) - request.args = entry->args; - queue_do_index->Enqueue(std::move(request)); -#endif - //std::cerr << "Changing " << msg->params.textDocument.uri.GetPath() << std::endl; break; } case IpcId::TextDocumentDidClose: { auto msg = static_cast(message.get()); std::cerr << "Closing " << msg->params.textDocument.uri.GetPath() << std::endl; - //working_files->OnClose(msg->params); + working_files->OnClose(msg->params); + break; + } + + case IpcId::TextDocumentDidSave: { + auto msg = static_cast(message.get()); + + // Send an index update request. + Index_DoIndex request(Index_DoIndex::Type::Update); + optional entry = project->FindCompilationEntryForFile(msg->params.textDocument.uri.GetPath()); + request.path = msg->params.textDocument.uri.GetPath(); + if (entry) + request.args = entry->args; + queue_do_index->Enqueue(std::move(request)); break; } @@ -848,7 +859,7 @@ void QueryDbMainLoop( } void QueryDbMain() { - std::cerr << "Running QueryDb" << std::endl; + //std::cerr << "Running QueryDb" << std::endl; // Create queues. std::unique_ptr ipc = BuildIpcMessageQueue(kIpcLanguageClientName, kQueueSizeBytes); @@ -921,7 +932,7 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) { //response.result.capabilities.textDocumentSync->change = lsTextDocumentSyncKind::Full; //response.result.capabilities.textDocumentSync->willSave = true; //response.result.capabilities.textDocumentSync->willSaveWaitUntil = true; - response.result.capabilities.textDocumentSync = lsTextDocumentSyncKind::Incremental; // TODO: use incremental at some point + response.result.capabilities.textDocumentSync = lsTextDocumentSyncKind::Incremental; response.result.capabilities.completionProvider = lsCompletionOptions(); response.result.capabilities.completionProvider->resolveProvider = false; @@ -936,7 +947,7 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) { response.result.capabilities.workspaceSymbolProvider = true; - response.Write(std::cerr); + //response.Write(std::cerr); response.Write(std::cout); break; } @@ -953,12 +964,13 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) { case IpcId::TextDocumentDidOpen: case IpcId::TextDocumentDidChange: - case IpcId::TextDocumentDidClose: { + case IpcId::TextDocumentDidClose: + case IpcId::TextDocumentDidSave: case IpcId::TextDocumentCompletion: case IpcId::TextDocumentDefinition: case IpcId::TextDocumentDocumentSymbol: case IpcId::TextDocumentCodeLens: - case IpcId::WorkspaceSymbol: + case IpcId::WorkspaceSymbol: { //std::cerr << "Sending message " << (int)message->method_id << std::endl; ipc->SendMessage(&ipc->for_server, message->method_id, *message.get()); break; @@ -1043,7 +1055,7 @@ int main(int argc, char** argv) { //bool loop = true; //while (loop) // std::this_thread::sleep_for(std::chrono::milliseconds(10)); - //std::this_thread::sleep_for(std::chrono::seconds(3)); + std::this_thread::sleep_for(std::chrono::seconds(3)); PlatformInit(); RegisterMessageTypes(); @@ -1092,17 +1104,17 @@ int main(int argc, char** argv) { exit(0); } else if (HasOption(options, "--language-server")) { - std::cerr << "Running language server" << std::endl; + //std::cerr << "Running language server" << std::endl; LanguageServerMain(argv[0]); return 0; } else if (HasOption(options, "--querydb")) { - std::cerr << "Running querydb" << std::endl; + //std::cerr << "Running querydb" << std::endl; QueryDbMain(); return 0; } else { - std::cerr << "Running language server" << std::endl; + //std::cerr << "Running language server" << std::endl; LanguageServerMain(argv[0]); return 0; } diff --git a/src/file_consumer.cc b/src/file_consumer.cc index b470ef75..f558eb40 100644 --- a/src/file_consumer.cc +++ b/src/file_consumer.cc @@ -5,15 +5,6 @@ FileConsumer::FileConsumer(SharedState* shared_state) : shared_(shared_state) {} -std::vector> FileConsumer::TakeLocalState() { - std::vector> result; - for (auto& entry : local_) { - if (entry.second) - result.push_back(std::move(entry.second)); - } - return result; -} - IndexedFile* FileConsumer::TryConsumeFile(const std::string& file) { // Try to find cached local result. auto it = local_.find(file); @@ -28,4 +19,19 @@ IndexedFile* FileConsumer::TryConsumeFile(const std::string& file) { } local_[file] = did_insert ? MakeUnique(file) : nullptr; return local_[file].get(); +} + +void FileConsumer::ForceLocal(const std::string& file) { + auto it = local_.find(file); + if (it == local_.end()) + local_[file] = MakeUnique(file); +} + +std::vector> FileConsumer::TakeLocalState() { + std::vector> result; + for (auto& entry : local_) { + if (entry.second) + result.push_back(std::move(entry.second)); + } + return result; } \ No newline at end of file diff --git a/src/file_consumer.h b/src/file_consumer.h index f277fa1e..12f2508e 100644 --- a/src/file_consumer.h +++ b/src/file_consumer.h @@ -27,6 +27,9 @@ struct FileConsumer { // Returns IndexedFile for the file or nullptr. IndexedFile* TryConsumeFile(const std::string& file); + // Forcibly create a local file, even if it has already been parsed. + void ForceLocal(const std::string& file); + // Returns and passes ownership of all local state. std::vector> TakeLocalState(); diff --git a/src/indexer.cc b/src/indexer.cc index 39b60bb7..67cb4975 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -1254,6 +1254,8 @@ void indexEntityReference(CXClientData client_data, } std::vector> Parse(FileConsumer* file_consumer, std::string filename, std::vector args, bool dump_ast) { + //return {}; + clang_enableStackTraces(); clang_toggleCrashRecovery(1); @@ -1288,6 +1290,8 @@ std::vector> Parse(FileConsumer* file_consumer, std NamespaceHelper ns; IndexParam param(file_consumer, &ns); + file_consumer->ForceLocal(filename); + std::cerr << "!! [START] Indexing " << filename << std::endl; clang_indexTranslationUnit(index_action, ¶m, callbacks, sizeof(callbacks), CXIndexOpt_IndexFunctionLocalSymbols | CXIndexOpt_SkipParsedBodiesInSession | CXIndexOpt_IndexImplicitTemplateInstantiations, diff --git a/src/ipc.cc b/src/ipc.cc index 1d72d8d8..30de97a5 100644 --- a/src/ipc.cc +++ b/src/ipc.cc @@ -16,6 +16,8 @@ const char* IpcIdToString(IpcId id) { return "textDocument/didChange"; case IpcId::TextDocumentDidClose: return "textDocument/didClose"; + case IpcId::TextDocumentDidSave: + return "textDocument/didSave"; case IpcId::TextDocumentCompletion: return "textDocument/completion"; case IpcId::TextDocumentDefinition: diff --git a/src/ipc.h b/src/ipc.h index 838a098c..de4639d0 100644 --- a/src/ipc.h +++ b/src/ipc.h @@ -13,6 +13,7 @@ enum class IpcId : int { TextDocumentDidOpen, TextDocumentDidChange, TextDocumentDidClose, + TextDocumentDidSave, TextDocumentCompletion, TextDocumentDefinition, TextDocumentDocumentSymbol, diff --git a/src/language_server_api.h b/src/language_server_api.h index 7f5e06c9..7e8877ef 100644 --- a/src/language_server_api.h +++ b/src/language_server_api.h @@ -1075,7 +1075,21 @@ MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidClose::Params, textDocument); MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidClose, params); +struct Ipc_TextDocumentDidSave : public IpcMessage { + struct Params { + // The document that was saved. + lsTextDocumentIdentifier textDocument; + + // Optional the content when saved. Depends on the includeText value + // when the save notifcation was requested. + // std::string text; + }; + const static IpcId kIpcId = IpcId::TextDocumentDidSave; + Params params; +}; +MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidSave::Params, textDocument); +MAKE_REFLECT_STRUCT(Ipc_TextDocumentDidSave, params); diff --git a/src/platform_win.cc b/src/platform_win.cc index e8b6cc91..652b0704 100644 --- a/src/platform_win.cc +++ b/src/platform_win.cc @@ -41,7 +41,7 @@ struct PlatformMutexWin : public PlatformMutex { HANDLE raw_mutex = INVALID_HANDLE_VALUE; PlatformMutexWin(const std::string& name) { - std::cerr << "[win] Creating mutex with name " << name << std::endl; + //std::cerr << "[win] Creating mutex with name " << name << std::endl; raw_mutex = CreateMutex(nullptr, false /*initial_owner*/, name.c_str()); CheckForError({ ERROR_ALREADY_EXISTS }); } @@ -72,8 +72,8 @@ struct PlatformSharedMemoryWin : public PlatformSharedMemory { HANDLE shmem_; PlatformSharedMemoryWin(const std::string& name, size_t capacity) { - std::cerr << "[win] Creating shared memory with name " << name - << " and capacity " << capacity << std::endl; + //std::cerr << "[win] Creating shared memory with name " << name + // << " and capacity " << capacity << std::endl; this->name = name; shmem_ = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, diff --git a/src/project.cc b/src/project.cc index e66e9045..c63cc601 100644 --- a/src/project.cc +++ b/src/project.cc @@ -13,12 +13,16 @@ std::vector LoadFromDirectoryListing(const std::string& projec std::vector result; std::vector args; + std::cerr << "Using arguments: "; for (const std::string& line : ReadLines(project_directory + "/clang_args")) { if (line.empty() || StartsWith(line, "#")) continue; - std::cerr << "Adding argument " << line << std::endl; + if (!args.empty()) + std::cerr << ", "; + std::cerr << line; args.push_back(line); } + std::cerr << std::endl; std::vector files = GetFilesInFolder(project_directory, true /*recursive*/, true /*add_folder_to_path*/); diff --git a/src/query.cc b/src/query.cc index b904da88..f2e3d593 100644 --- a/src/query.cc +++ b/src/query.cc @@ -299,9 +299,10 @@ void CompareGroups( // TODO: consider having separate lookup maps so they are smaller (maybe // lookups will go faster). +// TODO: Figure out where the invalid SymbolKinds are coming from. QueryFileId GetQueryFileIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { auto it = query_db->usr_to_symbol.find(usr); - if (it != query_db->usr_to_symbol.end()) { + if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) { assert(it->second.kind == SymbolKind::File); return QueryFileId(it->second.idx); } @@ -314,7 +315,7 @@ QueryFileId GetQueryFileIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { QueryTypeId GetQueryTypeIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { auto it = query_db->usr_to_symbol.find(usr); - if (it != query_db->usr_to_symbol.end()) { + if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) { assert(it->second.kind == SymbolKind::Type); return QueryTypeId(it->second.idx); } @@ -327,7 +328,7 @@ QueryTypeId GetQueryTypeIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { QueryFuncId GetQueryFuncIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { auto it = query_db->usr_to_symbol.find(usr); - if (it != query_db->usr_to_symbol.end()) { + if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) { assert(it->second.kind == SymbolKind::Func); return QueryFuncId(it->second.idx); } @@ -340,7 +341,7 @@ QueryFuncId GetQueryFuncIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { QueryVarId GetQueryVarIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { auto it = query_db->usr_to_symbol.find(usr); - if (it != query_db->usr_to_symbol.end()) { + if (it != query_db->usr_to_symbol.end() && it->second.kind != SymbolKind::Invalid) { assert(it->second.kind == SymbolKind::Var); return QueryVarId(it->second.idx); } @@ -491,8 +492,8 @@ IndexUpdate::IndexUpdate(const IdMap& previous_id_map, const IdMap& current_id_m previous, current, \ &removed, &added); \ if (did_add) {\ - std::cerr << "Adding mergeable update on " << current_def->def.short_name << " (" << current_def->def.usr << ") for field " << #index_name << std::endl; \ - query_name.push_back(MergeableUpdate(current_def->def.usr, removed, added)); \ + /*std::cerr << "Adding mergeable update on " << current_def->def.short_name << " (" << current_def->def.usr << ") for field " << #index_name << std::endl;*/ \ + query_name.push_back(MergeableUpdate(current_def->def.usr, added, removed)); \ } \ } // File diff --git a/src/query.h b/src/query.h index caa996f6..bf4f5277 100644 --- a/src/query.h +++ b/src/query.h @@ -67,7 +67,7 @@ struct SymbolIdx { SymbolKind kind; size_t idx; - SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use. + explicit SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use. SymbolIdx(SymbolKind kind, uint64_t idx) : kind(kind), idx(idx) {} bool operator==(const SymbolIdx& that) const { diff --git a/src/serializer.cc b/src/serializer.cc index 82d84f13..dd5ea96f 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -240,6 +240,22 @@ optional Deserialize(std::string path, std::string serialized) { IndexedFile file(path); Reflect(reader, file); + + // Restore non-serialized state. + file.path = path; + file.id_cache.primary_file = file.path; + for (const auto& type : file.types) { + file.id_cache.type_id_to_usr[type.id] = type.def.usr; + file.id_cache.usr_to_type_id[type.def.usr] = type.id; + } + for (const auto& func : file.funcs) { + file.id_cache.func_id_to_usr[func.id] = func.def.usr; + file.id_cache.usr_to_func_id[func.def.usr] = func.id; + } + for (const auto& var : file.vars) { + file.id_cache.var_id_to_usr[var.id] = var.def.usr; + file.id_cache.usr_to_var_id[var.def.usr] = var.id; + } return file; }