From c060e5178b817ded1768c535d391380b278d1746 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Sat, 25 Mar 2017 12:18:25 -0700 Subject: [PATCH] ipc cleanup --- clang_args | 5 +- command_line.cc | 1240 ++++++++++++++------------------ indexer.cpp | 13 +- ipc.cc | 436 ----------- ipc.h | 155 ---- language_server_api.h | 55 +- libclangmm/TranslationUnit.cc | 22 +- old.cc | 175 ----- query.cc | 12 +- src/threaded_queue.h | 51 ++ src/timer.cc | 14 + src/timer.h | 19 + src/typed_bidi_message_queue.h | 7 +- task.cc | 224 ------ 14 files changed, 693 insertions(+), 1735 deletions(-) delete mode 100644 ipc.cc delete mode 100644 ipc.h delete mode 100644 old.cc create mode 100644 src/threaded_queue.h create mode 100644 src/timer.cc create mode 100644 src/timer.h delete mode 100644 task.cc diff --git a/clang_args b/clang_args index 1f034de8..7b36ff31 100644 --- a/clang_args +++ b/clang_args @@ -1,2 +1,5 @@ -std=c++11 --Ithird_party/rapidjson/include \ No newline at end of file +-Ithird_party/rapidjson/include +-IC:/Program Files/LLVM/include +-fms-compatibility +-fdelayed-template-parsing \ No newline at end of file diff --git a/command_line.cc b/command_line.cc index 59dd55f6..93a5f0b8 100644 --- a/command_line.cc +++ b/command_line.cc @@ -6,15 +6,15 @@ #include "compilation_database_loader.h" #include "indexer.h" -#include "ipc.h" #include "query.h" #include "language_server_api.h" #include "test.h" +#include "src/timer.h" +#include "src/threaded_queue.h" #include "src/typed_bidi_message_queue.h" #include "third_party/tiny-process-library/process.hpp" - #include "third_party/doctest/doctest/doctest.h" #include @@ -25,13 +25,33 @@ #include #endif -const char* kIpcIndexerName = "indexer"; const char* kIpcLanguageClientName = "language_client"; const int kNumIndexers = 8 - 1; +const int kQueueSizeBytes = 1024 * 1024 * 32; + +struct IndexTranslationUnitRequest { + std::string path; + std::vector args; +}; + +struct IndexTranslationUnitResponse { + IndexUpdate update; + explicit IndexTranslationUnitResponse(IndexUpdate& update) : update(update) {} +}; + +// TODO: Rename TypedBidiMessageQueue to IpcTransport? +using IpcMessageQueue = TypedBidiMessageQueue; +using IndexRequestQueue = ThreadedQueue; +using IndexResponseQueue = ThreadedQueue; + +template +void SendMessage(IpcMessageQueue& t, MessageQueue* destination, TMessage& message) { + t.SendMessage(destination, TMessage::kMethod, message); +} std::unordered_map ParseOptions(int argc, - char** argv) { + char** argv) { std::unordered_map output; std::string previous_arg; @@ -42,13 +62,14 @@ std::unordered_map ParseOptions(int argc, if (arg[0] != '-') { if (previous_arg.size() == 0) { std::cerr << "Invalid arguments; switches must start with -" - << std::endl; + << std::endl; exit(1); } output[previous_arg] = arg; previous_arg = ""; - } else { + } + else { output[arg] = ""; previous_arg = arg; } @@ -58,7 +79,7 @@ std::unordered_map ParseOptions(int argc, } bool HasOption(const std::unordered_map& options, - const std::string& option) { + const std::string& option) { return options.find(option) != options.end(); } @@ -86,7 +107,7 @@ std::unique_ptr ParseMessage() { // bad input that is not a message. if (content_length < 0) { std::cerr << "parsing command failed (no Content-Length header)" - << std::endl; + << std::endl; return nullptr; } @@ -118,34 +139,24 @@ std::string Join(const std::vector& elements, std::string sep) { } template -struct BaseIpcMessage : public IpcMessage { - BaseIpcMessage() : IpcMessage(T::kIpcId) {} - - // IpcMessage: - void Serialize(Writer& writer) override { - T& value = *static_cast(this); - Reflect(writer, value); - } - void Deserialize(Reader& reader) override { - T& value = *static_cast(this); - Reflect(reader, value); - } +struct BaseIpcMessage : public InMessage { + BaseIpcMessage() : InMessage(T::kMethod) {} }; struct IpcMessage_Quit : public BaseIpcMessage { - static constexpr IpcId kIpcId = IpcId::Quit; + static constexpr lsMethodId kMethod = lsMethodId::Quit; }; template void Reflect(TVisitor& visitor, IpcMessage_Quit& value) {} struct IpcMessage_IsAlive : public BaseIpcMessage { - static constexpr IpcId kIpcId = IpcId::IsAlive; + static constexpr lsMethodId kMethod = lsMethodId::IsAlive; }; template void Reflect(TVisitor& visitor, IpcMessage_IsAlive& value) {} struct IpcMessage_OpenProject : public BaseIpcMessage { - static constexpr IpcId kIpcId = IpcId::OpenProject; + static constexpr lsMethodId kMethod = lsMethodId::OpenProject; std::string project_path; }; template @@ -153,52 +164,71 @@ void Reflect(TVisitor& visitor, IpcMessage_OpenProject& value) { Reflect(visitor, value.project_path); } -struct IpcMessage_IndexTranslationUnitRequest - : public BaseIpcMessage { - static constexpr IpcId kIpcId = IpcId::IndexTranslationUnitRequest; - std::string path; - std::vector args; +struct IpcMessage_Cout : public BaseIpcMessage { + static constexpr lsMethodId kMethod = lsMethodId::Cout; + std::string content; + + IpcMessage_Cout() {} + IpcMessage_Cout(OutMessage& message) { + std::ostringstream out; + message.Send(out); + content = out.str(); + } }; template -void Reflect(TVisitor& visitor, IpcMessage_IndexTranslationUnitRequest& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER(path); - REFLECT_MEMBER(args); - REFLECT_MEMBER_END(); +void Reflect(TVisitor& visitor, IpcMessage_Cout& value) { + Reflect(visitor, value.content); } -struct IpcMessage_IndexTranslationUnitResponse - : public BaseIpcMessage { - static constexpr IpcId kIpcId = IpcId::IndexTranslationUnitResponse; - IndexUpdate update; - - IpcMessage_IndexTranslationUnitResponse() {} - explicit IpcMessage_IndexTranslationUnitResponse(IndexUpdate& update) - : update(update) {} -}; -template -void Reflect(TVisitor& visitor, - IpcMessage_IndexTranslationUnitResponse& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER(update); - REFLECT_MEMBER_END(); +void SendOutMessageToClient(IpcMessageQueue* queue, OutMessage& response) { + IpcMessage_Cout out(response); + queue->SendMessage(&queue->for_client, IpcMessage_Cout::kMethod, out); } -struct IpcMessage_LanguageServerRequest - : public BaseIpcMessage { - static constexpr IpcId kIpcId = IpcId::LanguageServerRequest; - // TODO: provide a way to get the request state. - lsMethodId method_id; -}; -template -void Reflect(TVisitor& visitor, IpcMessage_LanguageServerRequest& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER(method_id); - REFLECT_MEMBER_END(); + + +template +void RegisterId(IpcMessageQueue* t) { + t->RegisterId(T::kMethod, + [](Writer& visitor, lsBaseMessage& message) { + T& m = static_cast(message); + Reflect(visitor, m); + }, [](Reader& visitor) { + auto m = MakeUnique(); + Reflect(visitor, *m); + return m; + }); } +std::unique_ptr BuildIpcMessageQueue(const std::string& name, size_t buffer_size) { + auto ipc = MakeUnique(name, buffer_size); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + RegisterId(ipc.get()); + return ipc; +} + +void RegisterMessageTypes() { + MessageRegistry::instance()->Register(); + MessageRegistry::instance()->Register(); + MessageRegistry::instance()->Register(); + MessageRegistry::instance()->Register(); + MessageRegistry::instance()->Register(); + MessageRegistry::instance()->Register(); + MessageRegistry::instance()->Register(); +} + +#if false struct IpcMessage_DocumentSymbolsRequest - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::DocumentSymbolsRequest; RequestId request_id; std::string document; @@ -212,7 +242,7 @@ void Reflect(TVisitor& visitor, IpcMessage_DocumentSymbolsRequest& value) { } struct IpcMessage_DocumentSymbolsResponse - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::DocumentSymbolsResponse; RequestId request_id; std::vector symbols; @@ -226,7 +256,7 @@ void Reflect(TVisitor& visitor, IpcMessage_DocumentSymbolsResponse& value) { } struct IpcMessage_WorkspaceSymbolsRequest - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::WorkspaceSymbolsRequest; RequestId request_id; std::string query; @@ -240,7 +270,7 @@ void Reflect(TVisitor& visitor, IpcMessage_WorkspaceSymbolsRequest& value) { } struct IpcMessage_DocumentCodeLensRequest - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::DocumentCodeLensRequest; RequestId request_id; std::string document; @@ -254,7 +284,7 @@ void Reflect(TVisitor& visitor, IpcMessage_DocumentCodeLensRequest& value) { } struct IpcMessage_DocumentCodeLensResponse - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::DocumentCodeLensResponse; RequestId request_id; std::vector code_lens; @@ -268,7 +298,7 @@ void Reflect(TVisitor& visitor, IpcMessage_DocumentCodeLensResponse& value) { } struct IpcMessage_CodeLensResolveRequest - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::CodeLensResolveRequest; RequestId request_id; TCodeLens code_lens; @@ -282,7 +312,7 @@ void Reflect(TVisitor& visitor, IpcMessage_CodeLensResolveRequest& value) { } struct IpcMessage_CodeLensResolveResponse - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::CodeLensResolveResponse; RequestId request_id; TCodeLens code_lens; @@ -296,7 +326,7 @@ void Reflect(TVisitor& visitor, IpcMessage_CodeLensResolveResponse& value) { } struct IpcMessage_WorkspaceSymbolsResponse - : public BaseIpcMessage { + : public BaseIpcMessage { static constexpr IpcId kIpcId = IpcId::WorkspaceSymbolsResponse; RequestId request_id; std::vector symbols; @@ -308,68 +338,37 @@ void Reflect(TVisitor& visitor, IpcMessage_WorkspaceSymbolsResponse& value) { REFLECT_MEMBER(symbols); REFLECT_MEMBER_END(); } +#endif -struct Timer { - using Clock = std::chrono::high_resolution_clock; - std::chrono::time_point start_; - - Timer() { Reset(); } - - void Reset() { start_ = Clock::now(); } - - long long ElapsedMilliseconds() { - std::chrono::time_point end = Clock::now(); - return std::chrono::duration_cast(end - start_) - .count(); - } -}; - -void IndexMainLoop(IpcClient* client) { - std::vector> messages = client->TakeMessages(); - for (auto& message : messages) { - // std::cerr << "Processing message " << static_cast(message->ipc_id) - // << std::endl; - - switch (message->ipc_id) { - case IpcId::Quit: { - exit(0); - break; - } - - case IpcId::IndexTranslationUnitRequest: { - IpcMessage_IndexTranslationUnitRequest* msg = - static_cast(message.get()); - - std::cerr << "Parsing file " << msg->path << " with args " - << Join(msg->args, ", ") << std::endl; - - Timer time; - IndexedFile file = Parse(msg->path, msg->args); - std::cerr << "Parsing/indexing took " << time.ElapsedMilliseconds() - << "ms" << std::endl; - - time.Reset(); - IndexUpdate update(file); - auto response = IpcMessage_IndexTranslationUnitResponse(update); - std::cerr << "Creating index update took " << time.ElapsedMilliseconds() - << "ms" << std::endl; - - time.Reset(); - client->SendToServer(&response); - std::cerr << "Sending to server took " << time.ElapsedMilliseconds() - << "ms" << std::endl; - - break; - } - } - } -} - -void IndexMain(int id) { - IpcClient client_ipc(kIpcIndexerName, id); +void IndexMain(IndexRequestQueue* requests, IndexResponseQueue* responses) { while (true) { - IndexMainLoop(&client_ipc); - std::this_thread::sleep_for(std::chrono::milliseconds(10)); + // Try to get a request. If there isn't one, sleep for a little while. + optional request = requests->TryDequeue(); + if (!request) { + // TODO: use CV to wakeup? + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + continue; + } + + // Parse request and send a response. + std::cerr << "Parsing file " << request->path << " with args " + << Join(request->args, ", ") << std::endl; + + Timer time; + IndexedFile file = Parse(request->path, request->args); + std::cerr << "Parsing/indexing took " << time.ElapsedMilliseconds() + << "ms" << std::endl; + + time.Reset(); + IndexUpdate update(file); + IndexTranslationUnitResponse response(update); + std::cerr << "Creating index update took " << time.ElapsedMilliseconds() + << "ms" << std::endl; + + time.Reset(); + responses->Enqueue(response); + std::cerr << "Sending to server took " << time.ElapsedMilliseconds() + << "ms" << std::endl; } } @@ -390,22 +389,22 @@ QueryableFile* FindFile(QueryableDatabase* db, const std::string& filename) { lsLocation GetLsLocation(const QueryableLocation& location) { return lsLocation( - lsDocumentUri::FromPath(location.path), - lsRange(lsPosition(location.line - 1, location.column - 1))); + lsDocumentUri::FromPath(location.path), + lsRange(lsPosition(location.line - 1, location.column - 1))); } void AddCodeLens(std::vector* result, - QueryableLocation loc, - const std::vector& uses, - bool only_interesting, - const char* singular, - const char* plural) { + QueryableLocation loc, + const std::vector& uses, + bool only_interesting, + const char* singular, + const char* plural) { TCodeLens code_lens; code_lens.range.start.line = - loc.line - 1; // TODO: cleanup indexer to negate by 1. + loc.line - 1; // TODO: cleanup indexer to negate by 1. code_lens.range.start.character = - loc.column - 1; // TODO: cleanup indexer to negate by 1. - // TODO: store range information. + loc.column - 1; // TODO: cleanup indexer to negate by 1. + // TODO: store range information. code_lens.range.end.line = code_lens.range.start.line; code_lens.range.end.character = code_lens.range.start.character; @@ -422,7 +421,7 @@ void AddCodeLens(std::vector* result, unique_uses.insert(GetLsLocation(use)); } code_lens.command->arguments.locations.assign(unique_uses.begin(), - unique_uses.end()); + unique_uses.end()); // User visible label int num_usages = unique_uses.size(); @@ -437,11 +436,11 @@ void AddCodeLens(std::vector* result, } void AddCodeLens(std::vector* result, - QueryableLocation loc, - const std::vector& uses, - bool only_interesting, - const char* singular, - const char* plural) { + QueryableLocation loc, + const std::vector& uses, + bool only_interesting, + const char* singular, + const char* plural) { std::vector uses0; uses0.reserve(uses.size()); for (const UsrRef& use : uses) @@ -450,354 +449,344 @@ void AddCodeLens(std::vector* result, } void AddCodeLens(std::vector* result, - QueryableDatabase* db, - QueryableLocation loc, - const std::vector& usrs, - bool only_interesting, - const char* singular, - const char* plural) { + QueryableDatabase* db, + QueryableLocation loc, + const std::vector& usrs, + bool only_interesting, + const char* singular, + const char* plural) { std::vector uses0; uses0.reserve(usrs.size()); for (const Usr& usr : usrs) { SymbolIdx symbol = db->usr_to_symbol[usr]; switch (symbol.kind) { - case SymbolKind::Type: { - QueryableTypeDef* def = &db->types[symbol.idx]; - if (def->def.definition) - uses0.push_back(def->def.definition.value()); - break; - } - case SymbolKind::Func: { - QueryableFuncDef* def = &db->funcs[symbol.idx]; - if (def->def.definition) - uses0.push_back(def->def.definition.value()); - break; - } - case SymbolKind::Var: { - QueryableVarDef* def = &db->vars[symbol.idx]; - if (def->def.definition) - uses0.push_back(def->def.definition.value()); - break; - } - case SymbolKind::File: - case SymbolKind::Invalid: { - assert(false && "unexpected"); - break; - } + case SymbolKind::Type: { + QueryableTypeDef* def = &db->types[symbol.idx]; + if (def->def.definition) + uses0.push_back(def->def.definition.value()); + break; + } + case SymbolKind::Func: { + QueryableFuncDef* def = &db->funcs[symbol.idx]; + if (def->def.definition) + uses0.push_back(def->def.definition.value()); + break; + } + case SymbolKind::Var: { + QueryableVarDef* def = &db->vars[symbol.idx]; + if (def->def.definition) + uses0.push_back(def->def.definition.value()); + break; + } + case SymbolKind::File: + case SymbolKind::Invalid: { + assert(false && "unexpected"); + break; + } } } AddCodeLens(result, loc, uses0, only_interesting, singular, plural); } -void QueryDbMainLoop(IpcServer* language_client, - IpcServer* indexers, - QueryableDatabase* db) { - std::vector> messages = - language_client->TakeMessages(); +void QueryDbMainLoop( + QueryableDatabase* db, + IpcMessageQueue* language_client, + IndexRequestQueue* index_requests, + IndexResponseQueue* index_responses) { + std::vector> messages = language_client->GetMessages(&language_client->for_server); for (auto& message : messages) { // std::cerr << "Processing message " << static_cast(message->ipc_id) // << std::endl; - switch (message->ipc_id) { - case IpcId::Quit: { - break; + switch (message->method_id) { + case lsMethodId::Quit: { + std::cerr << "Got quit message (exiting)" << std::endl; + exit(0); + break; + } + + case lsMethodId::IsAlive: { + IpcMessage_IsAlive response; + language_client->SendMessage(&language_client->for_client, response.method_id, response); + break; + } + + case lsMethodId::OpenProject: { + IpcMessage_OpenProject* msg = + static_cast(message.get()); + std::string path = msg->project_path; + + std::vector entries = + LoadCompilationEntriesFromDirectory(path); + for (int i = 0; i < entries.size(); ++i) { + const CompilationEntry& entry = entries[i]; + std::string filepath = path + "/" + entry.filename; + std::cerr << "[" << i << "/" << (entries.size() - 1) + << "] Dispatching index request for file " << filepath + << std::endl; + + IndexTranslationUnitRequest request; + request.path = filepath; + request.args = entry.args; + index_requests->Enqueue(request); } + std::cerr << "Done" << std::endl; + break; + } - case IpcId::IsAlive: { - IpcMessage_IsAlive response; - language_client->SendToClient(0, &response); // todo: make non-blocking - break; - } + case lsMethodId::TextDocumentDocumentSymbol: { + auto msg = static_cast(message.get()); - case IpcId::OpenProject: { - IpcMessage_OpenProject* msg = - static_cast(message.get()); - std::string path = msg->project_path; + Out_DocumentSymbolResponse response; + response.id = msg->id; - std::vector entries = - LoadCompilationEntriesFromDirectory(path); - for (int i = 0; i < entries.size(); ++i) { - const CompilationEntry& entry = entries[i]; - std::string filepath = path + "/" + entry.filename; - std::cerr << "[" << i << "/" << (entries.size() - 1) - << "] Dispatching index request for file " << filepath - << std::endl; + QueryableFile* file = FindFile(db, msg->params.textDocument.uri.GetPath()); + if (file) { + for (UsrRef ref : file->outline) { + SymbolIdx symbol = db->usr_to_symbol[ref.usr]; - // TODO: indexers should steal work and load balance. - IpcMessage_IndexTranslationUnitRequest request; - request.path = filepath; - request.args = entry.args; - indexers->SendToClient(i % indexers->num_clients(), &request); - // IndexedFile file = Parse(filepath, entry.args); - // IndexUpdate update(file); - // db->ApplyIndexUpdate(&update); - } - std::cerr << "Done" << std::endl; - break; - } + lsSymbolInformation info; + info.location.range.start.line = + ref.loc.line - 1; // TODO: cleanup indexer to negate by 1. + info.location.range.start.character = + ref.loc.column - 1; // TODO: cleanup indexer to negate by 1. + // TODO: store range information. + info.location.range.end.line = info.location.range.start.line; + info.location.range.end.character = + info.location.range.start.character; - case IpcId::DocumentSymbolsRequest: { - auto msg = - static_cast(message.get()); - - IpcMessage_DocumentSymbolsResponse response; - response.request_id = msg->request_id; - - QueryableFile* file = FindFile(db, msg->document); - if (file) { - for (UsrRef ref : file->outline) { - SymbolIdx symbol = db->usr_to_symbol[ref.usr]; - - lsSymbolInformation info; - info.location.range.start.line = - ref.loc.line - 1; // TODO: cleanup indexer to negate by 1. - info.location.range.start.character = - ref.loc.column - 1; // TODO: cleanup indexer to negate by 1. - // TODO: store range information. - info.location.range.end.line = info.location.range.start.line; - info.location.range.end.character = - info.location.range.start.character; - - // TODO: cleanup namespace/naming so there is only one SymbolKind. - switch (symbol.kind) { - case SymbolKind::Type: { - QueryableTypeDef& def = db->types[symbol.idx]; - info.name = def.def.qualified_name; - info.kind = lsSymbolKind::Class; - break; - } - case SymbolKind::Func: { - QueryableFuncDef& def = db->funcs[symbol.idx]; - info.name = def.def.qualified_name; - if (def.def.declaring_type.has_value()) { - info.kind = lsSymbolKind::Method; - Usr declaring = def.def.declaring_type.value(); - info.containerName = - db->types[db->usr_to_symbol[declaring].idx] - .def.qualified_name; - } else { - info.kind = lsSymbolKind::Function; - } - break; - } - case SymbolKind::Var: { - QueryableVarDef& def = db->vars[symbol.idx]; - info.name = def.def.qualified_name; - info.kind = lsSymbolKind::Variable; - break; - } - case SymbolKind::File: - case SymbolKind::Invalid: { - assert(false && "unexpected"); - break; - } - }; - - response.symbols.push_back(info); + // TODO: cleanup namespace/naming so there is only one SymbolKind. + switch (symbol.kind) { + case SymbolKind::Type: { + QueryableTypeDef& def = db->types[symbol.idx]; + info.name = def.def.qualified_name; + info.kind = lsSymbolKind::Class; + break; } - } - - language_client->SendToClient(0, &response); - break; - } - - case IpcId::DocumentCodeLensRequest: { - auto msg = - static_cast(message.get()); - - IpcMessage_DocumentCodeLensResponse response; - response.request_id = msg->request_id; - - lsDocumentUri file_as_uri; - file_as_uri.SetPath(msg->document); - - QueryableFile* file = FindFile(db, msg->document); - if (file) { - for (UsrRef ref : file->outline) { - SymbolIdx symbol = db->usr_to_symbol[ref.usr]; - switch (symbol.kind) { - case SymbolKind::Type: { - QueryableTypeDef& def = db->types[symbol.idx]; - AddCodeLens(&response.code_lens, ref.loc, def.uses, - true /*only_interesting*/, "reference", - "references"); - AddCodeLens(&response.code_lens, db, ref.loc, def.derived, - false /*only_interesting*/, "derived", "derived"); - break; - } - case SymbolKind::Func: { - QueryableFuncDef& def = db->funcs[symbol.idx]; - AddCodeLens(&response.code_lens, ref.loc, def.uses, - false /*only_interesting*/, "reference", - "references"); - AddCodeLens(&response.code_lens, ref.loc, def.callers, - false /*only_interesting*/, "caller", "callers"); - AddCodeLens(&response.code_lens, ref.loc, def.def.callees, - false /*only_interesting*/, "callee", "callees"); - AddCodeLens(&response.code_lens, db, ref.loc, def.derived, - false /*only_interesting*/, "derived", "derived"); - break; - } - case SymbolKind::Var: { - QueryableVarDef& def = db->vars[symbol.idx]; - AddCodeLens(&response.code_lens, ref.loc, def.uses, - false /*only_interesting*/, "reference", - "references"); - break; - } - case SymbolKind::File: - case SymbolKind::Invalid: { - assert(false && "unexpected"); - break; - } - }; + case SymbolKind::Func: { + QueryableFuncDef& def = db->funcs[symbol.idx]; + info.name = def.def.qualified_name; + if (def.def.declaring_type.has_value()) { + info.kind = lsSymbolKind::Method; + Usr declaring = def.def.declaring_type.value(); + info.containerName = + db->types[db->usr_to_symbol[declaring].idx] + .def.qualified_name; + } + else { + info.kind = lsSymbolKind::Function; + } + break; } - } - - language_client->SendToClient(0, &response); - break; - } - - case IpcId::WorkspaceSymbolsRequest: { - auto msg = - static_cast(message.get()); - - IpcMessage_WorkspaceSymbolsResponse response; - response.request_id = msg->request_id; - - std::cerr << "- Considering " << db->qualified_names.size() - << " candidates " << std::endl; - - for (int i = 0; i < db->qualified_names.size(); ++i) { - const std::string& name = db->qualified_names[i]; - // std::cerr << "- Considering " << name << std::endl; - - if (name.find(msg->query) != std::string::npos) { - lsSymbolInformation info; - info.name = name; - - SymbolIdx symbol = db->symbols[i]; - - // TODO: dedup this code w/ above (ie, add ctor to convert symbol to - // SymbolInformation) - switch (symbol.kind) { - // TODO: file - case SymbolKind::Type: { - QueryableTypeDef& def = db->types[symbol.idx]; - info.name = def.def.qualified_name; - info.kind = lsSymbolKind::Class; - - if (def.def.definition.has_value()) { - info.location.uri.SetPath(def.def.definition->path); - info.location.range.start.line = def.def.definition->line - 1; - info.location.range.start.character = - def.def.definition->column - 1; - } - break; - } - case SymbolKind::Func: { - QueryableFuncDef& def = db->funcs[symbol.idx]; - info.name = def.def.qualified_name; - if (def.def.declaring_type.has_value()) { - info.kind = lsSymbolKind::Method; - Usr declaring = def.def.declaring_type.value(); - info.containerName = - db->types[db->usr_to_symbol[declaring].idx] - .def.qualified_name; - } else { - info.kind = lsSymbolKind::Function; - } - - if (def.def.definition.has_value()) { - info.location.uri.SetPath(def.def.definition->path); - info.location.range.start.line = def.def.definition->line - 1; - info.location.range.start.character = - def.def.definition->column - 1; - } - break; - } - case SymbolKind::Var: { - QueryableVarDef& def = db->vars[symbol.idx]; - info.name = def.def.qualified_name; - info.kind = lsSymbolKind::Variable; - - if (def.def.definition.has_value()) { - info.location.uri.SetPath(def.def.definition->path); - info.location.range.start.line = def.def.definition->line - 1; - info.location.range.start.character = - def.def.definition->column - 1; - } - break; - } - case SymbolKind::Invalid: { - assert(false && "unexpected"); - break; - } - }; - - // TODO: store range information. - info.location.range.end.line = info.location.range.start.line; - info.location.range.end.character = - info.location.range.start.character; - - response.symbols.push_back(info); + case SymbolKind::Var: { + QueryableVarDef& def = db->vars[symbol.idx]; + info.name = def.def.qualified_name; + info.kind = lsSymbolKind::Variable; + break; } + case SymbolKind::File: + case SymbolKind::Invalid: { + assert(false && "unexpected"); + break; + } + }; + + response.result.push_back(info); } - - language_client->SendToClient(0, &response); - break; } - default: { - std::cerr << "Unhandled IPC message with kind " - << static_cast(message->ipc_id) << std::endl; - exit(1); + SendOutMessageToClient(language_client, response); + break; + } + + case lsMethodId::TextDocumentCodeLens: { + auto msg = static_cast(message.get()); + + Out_DocumentCodeLensResponse response; + response.id = msg->id; + + lsDocumentUri file_as_uri = msg->params.textDocument.uri; + + QueryableFile* file = FindFile(db, file_as_uri.GetPath()); + if (file) { + for (UsrRef ref : file->outline) { + SymbolIdx symbol = db->usr_to_symbol[ref.usr]; + switch (symbol.kind) { + case SymbolKind::Type: { + QueryableTypeDef& def = db->types[symbol.idx]; + AddCodeLens(&response.result, ref.loc, def.uses, + true /*only_interesting*/, "reference", + "references"); + AddCodeLens(&response.result, db, ref.loc, def.derived, + false /*only_interesting*/, "derived", "derived"); + break; + } + case SymbolKind::Func: { + QueryableFuncDef& def = db->funcs[symbol.idx]; + AddCodeLens(&response.result, ref.loc, def.uses, + false /*only_interesting*/, "reference", + "references"); + AddCodeLens(&response.result, ref.loc, def.callers, + false /*only_interesting*/, "caller", "callers"); + AddCodeLens(&response.result, ref.loc, def.def.callees, + false /*only_interesting*/, "callee", "callees"); + AddCodeLens(&response.result, db, ref.loc, def.derived, + false /*only_interesting*/, "derived", "derived"); + break; + } + case SymbolKind::Var: { + QueryableVarDef& def = db->vars[symbol.idx]; + AddCodeLens(&response.result, ref.loc, def.uses, + false /*only_interesting*/, "reference", + "references"); + break; + } + case SymbolKind::File: + case SymbolKind::Invalid: { + assert(false && "unexpected"); + break; + } + }; + } } + + SendOutMessageToClient(language_client, response); + break; + } + + case lsMethodId::WorkspaceSymbol: { + auto msg = static_cast(message.get()); + + Out_WorkspaceSymbolResponse response; + response.id = msg->id; + + + std::cerr << "- Considering " << db->qualified_names.size() + << " candidates " << std::endl; + + std::string query = msg->params.query; + for (int i = 0; i < db->qualified_names.size(); ++i) { + const std::string& name = db->qualified_names[i]; + // std::cerr << "- Considering " << name << std::endl; + + if (name.find(query) != std::string::npos) { + lsSymbolInformation info; + info.name = name; + + SymbolIdx symbol = db->symbols[i]; + + // TODO: dedup this code w/ above (ie, add ctor to convert symbol to + // SymbolInformation) + switch (symbol.kind) { + // TODO: file + case SymbolKind::Type: { + QueryableTypeDef& def = db->types[symbol.idx]; + info.name = def.def.qualified_name; + info.kind = lsSymbolKind::Class; + + if (def.def.definition.has_value()) { + info.location.uri.SetPath(def.def.definition->path); + info.location.range.start.line = def.def.definition->line - 1; + info.location.range.start.character = + def.def.definition->column - 1; + } + break; + } + case SymbolKind::Func: { + QueryableFuncDef& def = db->funcs[symbol.idx]; + info.name = def.def.qualified_name; + if (def.def.declaring_type.has_value()) { + info.kind = lsSymbolKind::Method; + Usr declaring = def.def.declaring_type.value(); + info.containerName = + db->types[db->usr_to_symbol[declaring].idx] + .def.qualified_name; + } + else { + info.kind = lsSymbolKind::Function; + } + + if (def.def.definition.has_value()) { + info.location.uri.SetPath(def.def.definition->path); + info.location.range.start.line = def.def.definition->line - 1; + info.location.range.start.character = + def.def.definition->column - 1; + } + break; + } + case SymbolKind::Var: { + QueryableVarDef& def = db->vars[symbol.idx]; + info.name = def.def.qualified_name; + info.kind = lsSymbolKind::Variable; + + if (def.def.definition.has_value()) { + info.location.uri.SetPath(def.def.definition->path); + info.location.range.start.line = def.def.definition->line - 1; + info.location.range.start.character = + def.def.definition->column - 1; + } + break; + } + case SymbolKind::Invalid: { + assert(false && "unexpected"); + break; + } + }; + + // TODO: store range information. + info.location.range.end.line = info.location.range.start.line; + info.location.range.end.character = + info.location.range.start.character; + + response.result.push_back(info); + } + } + + SendOutMessageToClient(language_client, response); + break; + } + + default: { + std::cerr << "Unhandled IPC message with kind " + << static_cast(message->method_id) << std::endl; + exit(1); + } } } - messages = indexers->TakeMessages(); - for (auto& message : messages) { - // std::cerr << "Processing message " << static_cast(message->ipc_id) - // << std::endl; - switch (message->ipc_id) { - case IpcId::IndexTranslationUnitResponse: { - IpcMessage_IndexTranslationUnitResponse* msg = - static_cast( - message.get()); - Timer time; - db->ApplyIndexUpdate(&msg->update); - std::cerr << "Applying index update took " << time.ElapsedMilliseconds() - << "ms" << std::endl; - break; - } + // TODO: consider rate-limiting and checking for IPC messages so we don't block + // requests / we can serve partial requests. + while (true) { + optional response = index_responses->TryDequeue(); + if (!response) + break; - default: { - std::cerr << "Unhandled IPC message with kind " - << static_cast(message->ipc_id) << std::endl; - exit(1); - } - } + Timer time; + db->ApplyIndexUpdate(&response->update); + std::cerr << "Applying index update took " << time.ElapsedMilliseconds() + << "ms" << std::endl; } } void QueryDbMain() { std::cerr << "Running QueryDb" << std::endl; - IpcServer language_client(kIpcLanguageClientName, 1 /*num_clients*/); - IpcServer indexers(kIpcIndexerName, kNumIndexers); - QueryableDatabase db; - std::cerr << "!! starting processes" << std::endl; - // Start indexer processes. + // Create queues. + std::unique_ptr ipc = BuildIpcMessageQueue(kIpcLanguageClientName, kQueueSizeBytes); + IndexRequestQueue index_request_queue; + IndexResponseQueue index_response_queue; + + // Start indexer threads. for (int i = 0; i < kNumIndexers; ++i) { - // new Process(process_name + " --indexer " + std::to_string(i + 1)); - new std::thread([i]() { IndexMain(i); }); + new std::thread([&]() { + IndexMain(&index_request_queue, &index_response_queue); + }); } - std::cerr << "!! done processes" << std::endl; - // Pump query db main loop. + // Run query db main loop. + QueryableDatabase db; while (true) { - QueryDbMainLoop(&language_client, &indexers, &db); + QueryDbMainLoop(&db, ipc.get(), &index_request_queue, &index_response_queue); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } @@ -810,7 +799,7 @@ void QueryDbMain() { // blocks. // // |ipc| is connected to a server. -void LanguageServerStdinLoop(IpcClient* server) { +void LanguageServerStdinLoop(IpcMessageQueue* ipc) { while (true) { std::unique_ptr message = ParseMessage(); @@ -819,320 +808,181 @@ void LanguageServerStdinLoop(IpcClient* server) { continue; std::cerr << "[info]: Got message of type " - << MethodIdToString(message->method_id) << std::endl; + << MethodIdToString(message->method_id) << std::endl; switch (message->method_id) { - case lsMethodId::Initialize: { - auto request = static_cast(message.get()); - if (request->params.rootUri) { - std::string project_path = request->params.rootUri->GetPath(); - std::cerr << "Initialize in directory " << project_path - << " with uri " << request->params.rootUri->raw_uri - << std::endl; - IpcMessage_OpenProject open_project; - open_project.project_path = project_path; - server->SendToServer(&open_project); - } - - auto response = Out_InitializeResponse(); - response.id = message->id.value(); - response.result.capabilities.documentSymbolProvider = true; - // response.result.capabilities.referencesProvider = true; - response.result.capabilities.codeLensProvider = lsCodeLensOptions(); - response.result.capabilities.codeLensProvider->resolveProvider = false; - response.result.capabilities.workspaceSymbolProvider = true; - response.Send(); - break; + // TODO: For simplicitly lets just proxy the initialize request like + // all other requests so that stdin loop thread becomes super simple. + case lsMethodId::Initialize: { + auto request = static_cast(message.get()); + if (request->params.rootUri) { + std::string project_path = request->params.rootUri->GetPath(); + std::cerr << "Initialize in directory " << project_path + << " with uri " << request->params.rootUri->raw_uri + << std::endl; + IpcMessage_OpenProject open_project; + open_project.project_path = project_path; + ipc->SendMessage(&ipc->for_server, IpcMessage_OpenProject::kMethod, open_project); } - case lsMethodId::TextDocumentDocumentSymbol: { - // TODO: response should take id as input. - // TODO: message should not have top-level id. - auto request = static_cast(message.get()); + auto response = Out_InitializeResponse(); + response.id = request->id; + response.result.capabilities.documentSymbolProvider = true; + // response.result.capabilities.referencesProvider = true; + response.result.capabilities.codeLensProvider = lsCodeLensOptions(); + response.result.capabilities.codeLensProvider->resolveProvider = false; + response.result.capabilities.workspaceSymbolProvider = true; + response.Send(std::cout); + break; + } - IpcMessage_DocumentSymbolsRequest ipc_request; - ipc_request.request_id = request->id.value(); - ipc_request.document = request->params.textDocument.uri.GetPath(); - std::cerr << "Request textDocument=" << ipc_request.document - << std::endl; - server->SendToServer(&ipc_request); - break; - } - - case lsMethodId::TextDocumentCodeLens: { - auto request = static_cast(message.get()); - - IpcMessage_DocumentCodeLensRequest ipc_request; - ipc_request.request_id = request->id.value(); - ipc_request.document = request->params.textDocument.uri.GetPath(); - std::cerr << "Request codeLens on textDocument=" << ipc_request.document - << std::endl; - server->SendToServer(&ipc_request); - break; - } - - case lsMethodId::WorkspaceSymbol: { - auto request = static_cast(message.get()); - IpcMessage_WorkspaceSymbolsRequest ipc_request; - ipc_request.request_id = request->id.value(); - ipc_request.query = request->params.query; - std::cerr << "Request query=" << ipc_request.query << std::endl; - server->SendToServer(&ipc_request); - break; - } + case lsMethodId::TextDocumentDocumentSymbol: + case lsMethodId::TextDocumentCodeLens: + case lsMethodId::WorkspaceSymbol: { + ipc->SendMessage(&ipc->for_server, message->method_id, *message.get()); + break; + } } } } -void LanguageServerMainLoop(IpcClient* ipc) { - std::vector> messages = ipc->TakeMessages(); +void LanguageServerMainLoop(IpcMessageQueue* ipc) { + std::vector> messages = ipc->GetMessages(&ipc->for_client); for (auto& message : messages) { - switch (message->ipc_id) { - case IpcId::Quit: { - exit(0); - break; - } + switch (message->method_id) { + case lsMethodId::Quit: { + std::cerr << "Got quit message (exiting)" << std::endl; + exit(0); + break; + } - case IpcId::DocumentSymbolsResponse: { - auto msg = - static_cast(message.get()); + case lsMethodId::Cout: { + auto msg = static_cast(message.get()); + std::cout << msg->content; + std::cout.flush(); + break; + } - auto response = Out_DocumentSymbolResponse(); - response.id = msg->request_id; - response.result = msg->symbols; - response.Send(); - std::cerr << "Sent symbol response to client (" - << response.result.size() << " symbols)" << std::endl; - break; - } - - case IpcId::DocumentCodeLensResponse: { - auto msg = - static_cast(message.get()); - - auto response = Out_DocumentCodeLensResponse(); - response.id = msg->request_id; - response.result = msg->code_lens; - response.Send(); - std::cerr << "Sent code lens response to client (" - << response.result.size() << " symbols)" << std::endl; - break; - } - - case IpcId::WorkspaceSymbolsResponse: { - auto msg = - static_cast(message.get()); - - auto response = Out_WorkspaceSymbolResponse(); - response.id = msg->request_id; - response.result = msg->symbols; - response.Send(); - std::cerr << "Send symbol response to client (" - << response.result.size() << " symbols)" << std::endl; - break; - } - - default: { - std::cerr << "Unhandled IPC message with kind " - << static_cast(message->ipc_id) << std::endl; - exit(1); - } + default: { + std::cerr << "Unhandled IPC message with kind " + << static_cast(message->method_id) << std::endl; + exit(1); + } } } } -void LanguageServerMain(std::string process_name) { - IpcClient client_ipc(kIpcLanguageClientName, 0); - - // Discard any left-over messages from previous runs. - client_ipc.TakeMessages(); - +bool IsQueryDbProcessRunning(IpcMessageQueue* ipc) { // Emit an alive check. Sleep so the server has time to respond. IpcMessage_IsAlive check_alive; - client_ipc.SendToServer(&check_alive); + SendMessage(*ipc, &ipc->for_server, check_alive); // TODO: Tune this value or make it configurable. std::this_thread::sleep_for(std::chrono::milliseconds(100)); // Check if we got an IsAlive message back. - std::vector> messages = client_ipc.TakeMessages(); - bool has_server = false; + std::vector> messages = ipc->GetMessages(&ipc->for_client); for (auto& message : messages) { - if (IpcId::IsAlive == message->ipc_id) { - has_server = true; - break; - } + if (lsMethodId::IsAlive == message->method_id) + return true; } -// No server is running. Start it. -#if false + return false; +} + +void LanguageServerMain(std::string process_name) { + std::unique_ptr ipc = BuildIpcMessageQueue(kIpcLanguageClientName, kQueueSizeBytes); + + // Discard any left-over messages from previous runs. + ipc->GetMessages(&ipc->for_client); + + bool has_server = IsQueryDbProcessRunning(ipc.get()); + + // No server is running. Start it in-process. If the user wants to run the + // server out of process they have to start it themselves. if (!has_server) { - if (process_name.empty()) - return; - - Process p(process_name + " --querydb", "", - /*stdout*/[](const char* bytes, size_t n) { - for (int i = 0; i < n; ++i) - std::cerr << bytes[i]; - }, - /*stderr*/[](const char* bytes, size_t n) { - for (int i = 0; i < n; ++i) - std::cerr << bytes[i]; - }, - /*open_stdin*/false); - std::this_thread::sleep_for(std::chrono::seconds(1)); - // Pass empty process name so we only try to start the querydb once. - LanguageServerMain(""); - return; - } -#endif - - // for debugging attach - // std::this_thread::sleep_for(std::chrono::seconds(4)); - - if (!has_server) { - // No server. Run it in-process. - new std::thread([&]() { QueryDbMain(); }); + new std::thread(&QueryDbMain); } // Run language client. - std::thread stdio_reader(&LanguageServerStdinLoop, &client_ipc); + new std::thread(&LanguageServerStdinLoop, ipc.get()); while (true) { - LanguageServerMainLoop(&client_ipc); + LanguageServerMainLoop(ipc.get()); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } } void PreMain() { -// We need to write to stdout in binary mode because in Windows, writing -// \n will implicitly write \r\n. Language server API will ignore a -// \r\r\n split request. + // We need to write to stdout in binary mode because in Windows, writing + // \n will implicitly write \r\n. Language server API will ignore a + // \r\r\n split request. #ifdef _WIN32 _setmode(_fileno(stdout), O_BINARY); _setmode(_fileno(stdin), O_BINARY); #endif - - IpcRegistry::instance()->Register(IpcMessage_Quit::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_IsAlive::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_OpenProject::kIpcId); - - IpcRegistry::instance()->Register( - IpcMessage_IndexTranslationUnitRequest::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_IndexTranslationUnitResponse::kIpcId); - - IpcRegistry::instance()->Register( - IpcMessage_DocumentSymbolsRequest::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_DocumentSymbolsResponse::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_DocumentCodeLensRequest::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_DocumentCodeLensResponse::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_CodeLensResolveRequest::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_CodeLensResolveResponse::kIpcId); - - IpcRegistry::instance()->Register( - IpcMessage_WorkspaceSymbolsRequest::kIpcId); - IpcRegistry::instance()->Register( - IpcMessage_WorkspaceSymbolsResponse::kIpcId); -} - -template -void RegisterId(TypedBidiMessageQueue& t) { - t.RegisterId(T::kMethod, - [](Writer& visitor, lsBaseMessage& message) { - T& m = static_cast(message); - Reflect(visitor, m); - }, [](Reader& visitor) { - auto m = MakeUnique(); - Reflect(visitor, *m); - return m; - }); } int main(int argc, char** argv) { - // TODO: real queue size - const int kQueueSize = 128; - TypedBidiMessageQueue t("foo", kQueueSize); - RegisterId(t); - RegisterId(t); - RegisterId(t); - RegisterId(t); - RegisterId(t); - RegisterId(t); - - /* - // TODO: We can make this entire function a template. - t.RegisterId(In_DocumentSymbolRequest::kMethod, - [](Writer& visitor, lsBaseMessage& message) { - In_DocumentSymbolRequest& m = static_cast(message); - Reflect(visitor, m); - }, [](Reader& visitor) { - auto m = MakeUnique(); - Reflect(visitor, *m); - return m; - }); - */ - - //struct In_DocumentSymbolRequest : public InRequestMessage { - // const static lsMethodId kMethod = lsMethodId::TextDocumentDocumentSymbol; - - //MyMessageType mm; - //t.SendMessage(&t.for_client, lsMethodId::Initialize, &mm); - //t.GetMessages(&t.for_client); - bool loop = false; while (loop) std::this_thread::sleep_for(std::chrono::milliseconds(10)); PreMain(); + RegisterMessageTypes(); // if (argc == 1) { // QueryDbMain(); // return 0; //} - if (argc == 1) { + + + std::unordered_map options = + ParseOptions(argc, argv); + + if (argc == 1 || HasOption(options, "--test")) { doctest::Context context; context.applyCommandLine(argc, argv); int res = context.run(); if (context.shouldExit()) - return res; + return res; //RunTests(); return 0; } + else if (options.find("--help") != options.end()) { + std::cout << R"help(clang-querydb help: - std::unordered_map options = - ParseOptions(argc, argv); + clang-querydb is a low-latency C++ language server. - if (HasOption(options, "--language-server")) { + General: + --help Print this help information. + --language-server + Run as a language server. The language server will look for + an existing querydb process, otherwise it will run querydb + in-process. This implements the language server spec. + --querydb Run the querydb. The querydb stores the program index and + serves index request tasks. + --test Run tests. Does nothing if test support is not compiled in. + + Configuration: + When opening up a directory, clang-querydb will look for a + compile_commands.json file emitted by your preferred build system. If not + present, clang-querydb will use a recursive directory listing instead. + Command line flags can be provided by adding a "clang_args" file in the + top-level directory. Each line in that file is a separate argument. +)help"; + exit(0); + } + else if (HasOption(options, "--language-server")) { std::cerr << "Running language server" << std::endl; LanguageServerMain(argv[0]); return 0; } - /* TODO: out of process querydb -- maybe? else if (HasOption(options, "--querydb")) { std::cerr << "Running querydb" << std::endl; - QueryableDatabase db; - IpcServer ipc(kIpcServername); - while (true) { - QueryDbMainLoop(&ipc, &db); - // TODO: use a condition variable. - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } + QueryDbMain(); return 0; } - */ - else if (HasOption(options, "--indexer")) { - int index = atoi(options["--indexer"].c_str()); - if (index == 0) - std::cerr << "--indexer expects an indexer id > 0" << std::endl; - IndexMain(index); - } else { + else { std::cerr << "Running language server" << std::endl; LanguageServerMain(argv[0]); return 0; diff --git a/indexer.cpp b/indexer.cpp index 6414a3b0..0a2a8155 100644 --- a/indexer.cpp +++ b/indexer.cpp @@ -1,5 +1,6 @@ #include "indexer.h" +#include #include #include "serializer.h" @@ -1204,18 +1205,6 @@ void emptyIndexEntityReference(CXClientData client_data, IndexedFile Parse(std::string filename, std::vector args, bool dump_ast) { - // TODO!! - // TODO!! - // TODO!! - // TODO!!: Strip useless defs from IndexedFile before returning - // TODO!!: Strip useless defs from IndexedFile before returning - // TODO!!: Strip useless defs from IndexedFile before returning - // TODO!!: Strip useless defs from IndexedFile before returning - // TODO!!: Strip useless defs from IndexedFile before returning - // TODO!!: Strip useless defs from IndexedFile before returning - // TODO!! - // TODO!! - // TODO!! clang_toggleCrashRecovery(1); args.push_back("-std=c++11"); diff --git a/ipc.cc b/ipc.cc deleted file mode 100644 index 1adecab7..00000000 --- a/ipc.cc +++ /dev/null @@ -1,436 +0,0 @@ -#include "ipc.h" -#include "serializer.h" -#include "utils.h" - -#include "third_party/doctest/doctest/doctest.h" - -namespace { -// The absolute smallest partial payload we should send. This must be >0, ie, 1 is the -// minimum. Keep a reasonably high value so we don't send needlessly send tiny payloads. -const int kMinimumPartialPayloadSize = 128; - -const int kBufferSize = 1024 * 1024 * 32; // number of chars/bytes (32mb) in the message buffer. - -// JSON-encoded message that is passed across shared memory. -// -// Messages are funky objects. They contain potentially variable amounts of -// data and are passed between processes. This means that they need to be -// fully relocatable, ie, it is possible to memmove them in memory to a -// completely different address. -struct JsonMessage { - IpcId ipc_id; - int partial_message_id; - bool has_more_chunks; - size_t payload_size; - void* payload() { - return reinterpret_cast(this) + sizeof(JsonMessage); - } - - void Setup(IpcId ipc_id, int partial_message_id, bool has_more_chunks, size_t payload_size, const char* payload) { - this->ipc_id = ipc_id; - this->partial_message_id = partial_message_id; - this->has_more_chunks = has_more_chunks; - this->payload_size = payload_size; - - char* payload_dest = reinterpret_cast(this) + sizeof(JsonMessage); - memcpy(payload_dest, payload, payload_size); - } -}; - -std::string NameToServerName(const std::string& name) { - return name + "server"; -} - -std::string NameToClientName(const std::string& name, int client_id) { - return name + "client" + std::to_string(client_id); -} -} - -IpcRegistry* IpcRegistry::instance_ = nullptr; - -std::unique_ptr IpcRegistry::Allocate(IpcId id) { - assert(allocators_); - auto it = allocators_->find(id); - assert(it != allocators_->end() && "No registered allocator for id"); - return std::unique_ptr(it->second()); -} - -struct IpcDirectionalChannel::MessageBuffer { - MessageBuffer(void* buffer, size_t buffer_size, bool initialize) { - real_buffer = buffer; - real_buffer_size = buffer_size; - if (initialize) - new(real_buffer) Metadata(); - } - - // Pointer to the start of the actual buffer and the - // amount of storage actually available. - void* real_buffer; - size_t real_buffer_size; - - template - T* Offset(size_t offset) const { - return reinterpret_cast(static_cast(real_buffer) + offset); - } - - struct Metadata { - // The number of bytes that are currently used in the buffer minus the - // size of this Metadata struct. - size_t bytes_used = 0; - int next_partial_message_id = 0; - int num_outstanding_partial_messages = 0; - }; - - Metadata* metadata() const { - return Offset(0); - } - - size_t bytes_available() const { - return real_buffer_size - sizeof(Metadata) - metadata()->bytes_used; - } - - JsonMessage* message_at_offset(size_t offset) const { - return Offset(sizeof(Metadata) + offset); - } - - // First json message. - JsonMessage* first_message() const { - return message_at_offset(0); - } - - // First free, writable json message. Make sure to increase *bytes_used() - // by any written size. - JsonMessage* free_message() const { - return message_at_offset(metadata()->bytes_used); - } - - struct Iterator { - void* buffer; - size_t remaining_bytes; - - Iterator(void* buffer, size_t remaining_bytes) : buffer(buffer), remaining_bytes(remaining_bytes) {} - - JsonMessage* get() const { - assert(buffer); - return reinterpret_cast(buffer); - } - - JsonMessage* operator*() const { - return get(); - } - - JsonMessage* operator->() const { - return get(); - } - - void operator++() { - size_t next_message_offset = sizeof(JsonMessage) + get()->payload_size; - if (next_message_offset >= remaining_bytes) { - assert(next_message_offset == remaining_bytes); - buffer = nullptr; - remaining_bytes = 0; - return; - } - - buffer = (char*)buffer + next_message_offset; - remaining_bytes -= next_message_offset; - } - - bool operator==(const Iterator& other) const { - return buffer == other.buffer && remaining_bytes == other.remaining_bytes; - } - bool operator!=(const Iterator& other) const { - return !(*this == other); - } - }; - - Iterator begin() const { - if (metadata()->bytes_used == 0) - return end(); - - return Iterator(first_message(), metadata()->bytes_used); - } - Iterator end() const { - return Iterator(nullptr, 0); - } -}; - -struct IpcDirectionalChannel::ResizableBuffer { - void* memory; - size_t size; - size_t capacity; - - ResizableBuffer() { - memory = malloc(128); - size = 0; - capacity = 128; - } - - ~ResizableBuffer() { - free(memory); - size = 0; - capacity = 0; - } - - void Append(void* content, size_t content_size) { - assert(capacity); - - // Grow memory if needed. - if ((size + content_size) >= capacity) { - size_t new_capacity = capacity * 2; - while (new_capacity < size + content_size) - new_capacity *= 2; - void* new_memory = malloc(new_capacity); - assert(size < capacity); - memcpy(new_memory, memory, size); - free(memory); - memory = new_memory; - capacity = new_capacity; - } - - // Append new content into memory. - memcpy((char*)memory + size, content, content_size); - size += content_size; - } - - void Reset() { - size = 0; - } -}; - -IpcDirectionalChannel::ResizableBuffer* IpcDirectionalChannel::CreateOrFindResizableBuffer(int id) { - auto it = resizable_buffers.find(id); - if (it != resizable_buffers.end()) - return it->second.get(); - return (resizable_buffers[id] = MakeUnique()).get(); -} - -void IpcDirectionalChannel::RemoveResizableBuffer(int id) { - resizable_buffers.erase(id); -} - -IpcDirectionalChannel::IpcDirectionalChannel(const std::string& name, bool initialize_shared_memory) { - shared = CreatePlatformSharedMemory(name + "memory", kBufferSize); - mutex = CreatePlatformMutex(name + "mutex"); - local = std::unique_ptr(new char[kBufferSize]); - - // TODO: connecting a client will allocate reset shared state on the - // buffer. We need to store if we "initialized". - shared_buffer = MakeUnique(shared->data, kBufferSize, initialize_shared_memory); - local_buffer = MakeUnique(local.get(), kBufferSize, true /*initialize*/); -} - -IpcDirectionalChannel::~IpcDirectionalChannel() {} - -enum class DispatchResult { - RunAgain, - Break -}; - -// Run |action| an arbitrary number of times. -void IpcDispatch(PlatformMutex* mutex, std::function action) { - bool first = true; - int log_iteration_count = 0; - int log_count = 0; - while (true) { - if (!first) { - if (log_iteration_count > 1000) { - log_iteration_count = 0; - std::cerr << "[info]: shmem full, waiting (" << log_count++ << ")" << std::endl; // TODO: remove - } - ++log_iteration_count; - // TODO: See if we can figure out a way to use condition variables cross-process. - std::this_thread::sleep_for(std::chrono::microseconds(0)); - } - first = false; - - std::unique_ptr lock = CreatePlatformScopedMutexLock(mutex); - if (action() == DispatchResult::RunAgain) - continue; - break; - } -} - -void IpcDirectionalChannel::PushMessage(IpcMessage* message) { - assert(message->ipc_id != IpcId::Invalid); - assert(kBufferSize > sizeof(JsonMessage) + kMinimumPartialPayloadSize); - - rapidjson::StringBuffer output; - rapidjson::PrettyWriter writer(output); - writer.SetFormatOptions( - rapidjson::PrettyFormatOptions::kFormatSingleLineArray); - writer.SetIndent(' ', 2); - message->Serialize(writer); - - //std::cerr << "Sending message with id " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl; - - - size_t payload_size = output.GetSize(); - const char* payload = output.GetString(); - if (payload_size == 0) - return; - - int partial_message_id = 0; - - std::cerr << "Starting dispatch of payload with size " << payload_size << std::endl; - int count = 0; - - IpcDispatch(mutex.get(), [&]() { - assert(payload_size > 0); - - // We cannot find the entire payload in the buffer. We - // have to send chunks of it over time. - if ((sizeof(JsonMessage) + payload_size) > shared_buffer->bytes_available()) { - if ((sizeof(JsonMessage) + kMinimumPartialPayloadSize) > shared_buffer->bytes_available()) - return DispatchResult::RunAgain; - - if (partial_message_id == 0) - partial_message_id = ++shared_buffer->metadata()->next_partial_message_id; // note: pre-increment so we 1 as initial value - - size_t sent_payload_size = shared_buffer->bytes_available() - sizeof(JsonMessage); - shared_buffer->free_message()->Setup(message->ipc_id, partial_message_id, true /*has_more_chunks*/, sent_payload_size, payload); - shared_buffer->metadata()->bytes_used += sizeof(JsonMessage) + sent_payload_size; - //shared_buffer->free_message()->ipc_id = IpcId::Invalid; // Note: free_message() may be past writable memory. - - if (count++ > 50) { - std::cerr << "x50 Sending partial message with payload_size=" << sent_payload_size << std::endl; - count = 0; - } - - // Prepare for next time. - payload_size -= sent_payload_size; - payload += sent_payload_size; - return DispatchResult::RunAgain; - } - // The entire payload fits. Send it all now. - else { - // Include partial message id, as there could have been previous parts of this payload. - shared_buffer->free_message()->Setup(message->ipc_id, partial_message_id, false /*has_more_chunks*/, payload_size, payload); - shared_buffer->metadata()->bytes_used += sizeof(JsonMessage) + payload_size; - shared_buffer->free_message()->ipc_id = IpcId::Invalid; - //std::cerr << "Sending full message with payload_size=" << payload_size << std::endl; - - return DispatchResult::Break; - } - }); -} - -void AddIpcMessageFromJsonMessage(std::vector>& result, IpcId ipc_id, void* payload, size_t payload_size) { - rapidjson::Document document; - document.Parse(reinterpret_cast(payload), payload_size); - bool has_error = document.HasParseError(); - auto error = document.GetParseError(); - - std::unique_ptr base_message = IpcRegistry::instance()->Allocate(ipc_id); - base_message->Deserialize(document); - result.emplace_back(std::move(base_message)); -} - -std::vector> IpcDirectionalChannel::TakeMessages() { - std::vector> result; - - do { - // Move data from shared memory into a local buffer. Do this - // before parsing the blocks so that other processes can begin - // posting data as soon as possible. - { - std::unique_ptr lock = CreatePlatformScopedMutexLock(mutex.get()); - assert(shared_buffer->metadata()->bytes_used <= kBufferSize); - memcpy(local.get(), shared->data, sizeof(MessageBuffer::Metadata) + shared_buffer->metadata()->bytes_used); - shared_buffer->metadata()->bytes_used = 0; - shared_buffer->free_message()->ipc_id = IpcId::Invalid; - } - - // Parse blocks from shared memory. - for (JsonMessage* message : *local_buffer) { - //std::cerr << "Got message with payload_size=" << message->payload_size << std::endl; - - if (message->partial_message_id != 0) { - auto* buf = CreateOrFindResizableBuffer(message->partial_message_id); - buf->Append(message->payload(), message->payload_size); - if (!message->has_more_chunks) { - AddIpcMessageFromJsonMessage(result, message->ipc_id, buf->memory, buf->size); - RemoveResizableBuffer(message->partial_message_id); - } - } - else { - assert(!message->has_more_chunks); - AddIpcMessageFromJsonMessage(result, message->ipc_id, message->payload(), message->payload_size); - } - } - local_buffer->metadata()->bytes_used = 0; - - // Let other threads run. We still want to run as fast as possible, though. - std::this_thread::sleep_for(std::chrono::microseconds(0)); - } while (resizable_buffers.size() > 0); - - return result; -} - - - -IpcServer::IpcServer(const std::string& name, int num_clients) - : server_(NameToServerName(name), true /*initialize_shared_memory*/) { - - for (int i = 0; i < num_clients; ++i) { - clients_.push_back(MakeUnique(NameToClientName(name, i), true /*initialize_shared_memory*/)); - } -} - -void IpcServer::SendToClient(int client_id, IpcMessage* message) { - clients_[client_id]->PushMessage(message); -} - -std::vector> IpcServer::TakeMessages() { - return server_.TakeMessages(); -} - -IpcClient::IpcClient(const std::string& name, int client_id) - : server_(NameToServerName(name), false /*initialize_shared_memory*/), - client_(NameToClientName(name, client_id), false /*initialize_shared_memory*/) {} - -void IpcClient::SendToServer(IpcMessage* message) { - server_.PushMessage(message); -} - -std::vector> IpcClient::TakeMessages() { - return client_.TakeMessages(); -} - - - - - -template -struct TestIpcMessage : IpcMessage { - T data; - - TestIpcMessage() : IpcMessage(IpcId::Test) {} - ~TestIpcMessage() override {} - - // IpcMessage: - void Serialize(Writer& writer) override { - Reflect(writer, data); - } - void Deserialize(Reader& reader) override { - Reflect(reader, data); - } -}; - -#if false -TEST_CASE("foo") { - IpcRegistry::instance()->Register>(IpcId::Test); - - IpcDirectionalChannel channel0("indexertestmemory", true /*initialize_shared_memory*/); - IpcDirectionalChannel channel1("indexertestmemory", false /*initialize_shared_memory*/); - - TestIpcMessage m; - m.data = "hey there"; - - channel0.PushMessage(&m); - std::vector> messages = channel1.TakeMessages(); - REQUIRE(messages.size() == 1); - REQUIRE(messages[0]->ipc_id == m.ipc_id); - REQUIRE(reinterpret_cast*>(messages[0].get())->data == m.data); -} -#endif \ No newline at end of file diff --git a/ipc.h b/ipc.h deleted file mode 100644 index 66d88d30..00000000 --- a/ipc.h +++ /dev/null @@ -1,155 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include -#include - -#include "src/platform.h" -#include "serializer.h" -#include "utils.h" - -// TODO: We need to add support for payloads larger than the maximum shared memory buffer size. - - -enum class IpcId : int { - // Invalid request id. - Invalid = 0, - - Quit = 1, - IsAlive, - OpenProject, - - IndexTranslationUnitRequest, - IndexTranslationUnitResponse, - - // This is a language server request. The actual request method - // id is embedded within the request state. - LanguageServerRequest, - // TODO: remove - DocumentSymbolsRequest, - DocumentSymbolsResponse, - DocumentCodeLensRequest, - DocumentCodeLensResponse, - CodeLensResolveRequest, - CodeLensResolveResponse, - WorkspaceSymbolsRequest, - WorkspaceSymbolsResponse, - - Test -}; - -namespace std { - template <> - struct hash { - size_t operator()(const IpcId& k) const { - return hash()(static_cast(k)); - } - }; -} - -struct IpcMessage { - IpcMessage(IpcId ipc_id) : ipc_id(ipc_id) {} - virtual ~IpcMessage() {} - - const IpcId ipc_id; - - virtual void Serialize(Writer& writer) = 0; - virtual void Deserialize(Reader& reader) = 0; -}; - -struct IpcRegistry { - using Allocator = std::function; - - // Use unique_ptrs so we can initialize on first use - // (static init order might not be right). - std::unique_ptr> allocators_; - - template - void Register(IpcId id); - - std::unique_ptr Allocate(IpcId id); - - static IpcRegistry* instance() { - if (!instance_) - instance_ = new IpcRegistry(); - return instance_; - } - static IpcRegistry* instance_; -}; - -template -void IpcRegistry::Register(IpcId id) { - if (!allocators_) - allocators_ = MakeUnique>(); - - assert(allocators_->find(id) == allocators_->end() && - "There is already an IPC message with the given id"); - - (*allocators_)[id] = [id]() { - return new T(); - }; -} - - - - - - -struct IpcDirectionalChannel { - // NOTE: We keep all pointers in terms of char* so pointer arithmetic is - // always relative to bytes. - - explicit IpcDirectionalChannel(const std::string& name, bool initialize_shared_memory); - ~IpcDirectionalChannel(); - - void PushMessage(IpcMessage* message); - std::vector> TakeMessages(); - - struct MessageBuffer; - struct ResizableBuffer; - - ResizableBuffer* CreateOrFindResizableBuffer(int id); - void RemoveResizableBuffer(int id); - std::unordered_map> resizable_buffers; - - // Pointer to process shared memory and process shared mutex. - std::unique_ptr shared; - std::unique_ptr mutex; - - // Pointer to process-local memory. - std::unique_ptr local; - - std::unique_ptr shared_buffer; - std::unique_ptr local_buffer; -}; - -struct IpcServer { - IpcServer(const std::string& name, int num_clients); - - void SendToClient(int client_id, IpcMessage* message); - std::vector> TakeMessages(); - - int num_clients() const { return clients_.size(); } - -private: - IpcDirectionalChannel server_; // Local / us. - std::vector> clients_; -}; - -struct IpcClient { - IpcClient(const std::string& name, int client_id); - - void SendToServer(IpcMessage* message); - std::vector> TakeMessages(); - - IpcDirectionalChannel* client() { return &client_; } - -private: - IpcDirectionalChannel server_; - IpcDirectionalChannel client_; -}; diff --git a/language_server_api.h b/language_server_api.h index d0fce2f4..017ab0d9 100644 --- a/language_server_api.h +++ b/language_server_api.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -28,6 +29,12 @@ enum class lsMethodId : int { TextDocumentCodeLens, CodeLensResolve, WorkspaceSymbol, + + // Internal implementation detail. + Quit, + IsAlive, + OpenProject, + Cout }; MAKE_ENUM_HASHABLE(lsMethodId); @@ -140,14 +147,16 @@ struct MessageRegistry { static MessageRegistry* instance_; static MessageRegistry* instance(); - using Allocator = std::function(optional id, Reader& params)>; + using Allocator = std::function(Reader& visitor)>; std::unordered_map allocators; template void Register() { std::string method_name = MethodIdToString(T::kMethod); - allocators[method_name] = [](optional id, Reader& params) { - return MakeUnique(id, params); + allocators[method_name] = [](Reader& visitor) { + auto result = MakeUnique(); + Reflect(visitor, *result); + return result; }; } @@ -156,9 +165,6 @@ struct MessageRegistry { if (jsonrpc != "2.0") exit(1); - optional id; - ReflectMember(visitor, "id", id); - std::string method; ReflectMember(visitor, "method", method); @@ -168,20 +174,7 @@ struct MessageRegistry { } Allocator& allocator = allocators[method]; - - - // We run the allocator with actual params object or a null - // params object if there are no params. Unifying the two ifs is - // tricky because the second allocator param is a reference. - if (visitor.FindMember("params") != visitor.MemberEnd()) { - Reader& params = visitor["params"]; - return allocator(id, params); - } - else { - Reader params; - params.SetNull(); - return allocator(id, params); - } + return allocator(visitor); } }; @@ -197,8 +190,6 @@ struct lsBaseMessage {}; struct InMessage : public lsBaseMessage { const lsMethodId method_id; - optional id; - InMessage(lsMethodId method_id) : method_id(method_id) {} }; @@ -216,7 +207,7 @@ struct OutMessage : public lsBaseMessage { virtual void WriteMessageBody(Writer& writer) = 0; // Send the message to the language client by writing it to stdout. - void Send() { + void Send(std::ostream& out) { rapidjson::StringBuffer output; Writer writer(output); writer.StartObject(); @@ -225,10 +216,10 @@ struct OutMessage : public lsBaseMessage { WriteMessageBody(writer); writer.EndObject(); - std::cout << "Content-Length: " << output.GetSize(); - std::cout << (char)13 << char(10) << char(13) << char(10); - std::cout << output.GetString(); - std::cout.flush(); + out << "Content-Length: " << output.GetSize(); + out << (char)13 << char(10) << char(13) << char(10); + out << output.GetString(); + out.flush(); } }; @@ -346,6 +337,8 @@ struct OutNotificationMessage : public OutMessage { struct In_CancelRequest : public InNotificationMessage { static const lsMethodId kMethod = lsMethodId::CancelRequest; + RequestId id; + In_CancelRequest() : InNotificationMessage(kMethod) {} }; @@ -1282,6 +1275,8 @@ void Reflect(TVisitor& visitor, lsInitializeResult& value) { struct In_InitializeRequest : public InRequestMessage { const static lsMethodId kMethod = lsMethodId::Initialize; + + RequestId id; lsInitializeParams params; In_InitializeRequest() : InRequestMessage(kMethod) {} @@ -1306,6 +1301,8 @@ struct Out_InitializeResponse : public OutResponseMessage { struct In_InitializedNotification : public InNotificationMessage { const static lsMethodId kMethod = lsMethodId::Initialized; + + RequestId id; In_InitializedNotification() : InNotificationMessage(kMethod) {} }; @@ -1375,6 +1372,7 @@ void Reflect(TVisitor& visitor, lsDocumentSymbolParams& value) { struct In_DocumentSymbolRequest : public InRequestMessage { const static lsMethodId kMethod = lsMethodId::TextDocumentDocumentSymbol; + RequestId id; lsDocumentSymbolParams params; In_DocumentSymbolRequest() : InRequestMessage(kMethod) {} @@ -1448,6 +1446,7 @@ using TCodeLens = lsCodeLens; struct In_DocumentCodeLensRequest : public InRequestMessage { const static lsMethodId kMethod = lsMethodId::TextDocumentCodeLens; + RequestId id; lsDocumentCodeLensParams params; In_DocumentCodeLensRequest() : InRequestMessage(kMethod) {} @@ -1473,6 +1472,7 @@ struct Out_DocumentCodeLensResponse : public OutResponseMessage { struct In_DocumentCodeLensResolveRequest : public InRequestMessage { const static lsMethodId kMethod = lsMethodId::CodeLensResolve; + RequestId id; TCodeLens params; In_DocumentCodeLensResolveRequest() : InRequestMessage(kMethod) {} @@ -1516,6 +1516,7 @@ void Reflect(TVisitor& visitor, lsWorkspaceSymbolParams& value) { struct In_WorkspaceSymbolRequest : public InRequestMessage { const static lsMethodId kMethod = lsMethodId::WorkspaceSymbol; + RequestId id; lsWorkspaceSymbolParams params; In_WorkspaceSymbolRequest() : InRequestMessage(kMethod) {} diff --git a/libclangmm/TranslationUnit.cc b/libclangmm/TranslationUnit.cc index 8716bed6..80e25858 100644 --- a/libclangmm/TranslationUnit.cc +++ b/libclangmm/TranslationUnit.cc @@ -4,10 +4,11 @@ #include #include #include +#include namespace clang { - /* +/* TranslationUnit::TranslationUnit(Index &index, const std::string &file_path, const std::vector &command_line_args, const std::string &buffer, unsigned flags) { @@ -34,12 +35,27 @@ TranslationUnit::TranslationUnit(Index &index, const std::string &file_path, std::vector args; for (const std::string& a : command_line_args) { //if (a.size() >= 2 && a[0] == '-' && a[1] == 'D') - args.push_back(a.c_str()); + args.push_back(a.c_str()); } CXErrorCode error_code = clang_parseTranslationUnit2( index.cx_index, file_path.c_str(), args.data(), args.size(), nullptr, 0, flags, &cx_tu); - assert(!error_code); + switch (error_code) { + case CXError_Success: + break; + case CXError_Failure: + std::cerr << "libclang generic failure for " << file_path << std::endl; + break; + case CXError_Crashed: + std::cerr << "libclang crashed for " << file_path << std::endl; + break; + case CXError_InvalidArguments: + std::cerr << "libclang had invalid arguments for " << file_path << std::endl; + break; + case CXError_ASTReadError: + std::cerr << "libclang had ast read error for " << file_path << std::endl; + break; + } } TranslationUnit::~TranslationUnit() { diff --git a/old.cc b/old.cc deleted file mode 100644 index 1d3d8d8e..00000000 --- a/old.cc +++ /dev/null @@ -1,175 +0,0 @@ -#if false - - -/* - -// Connects to a running --project-directory instance. Forks -// and creates it if not running. -// -// Implements language server spec. -indexer.exe --language-server - -// Holds the runtime db that the --language-server instance -// runs queries against. -indexer.exe --project-directory /work2/chrome/src - -// Created from the --project-directory (server) instance -indexer.exe --index-file /work2/chrome/src/chrome/foo.cc - -// Configuration data is read from a JSON file. -{ -"max_threads": 40, -"cache_directory": "/work/indexer_cache/" - -} -*/ - - -bool ParsePreferredSymbolLocation(const std::string& content, PreferredSymbolLocation* obj) { -#define PARSE_AS(name, string) \ - if (content == #string) { \ - *obj = name; \ - return true; \ - } - - PARSE_AS(PreferredSymbolLocation::Declaration, "declaration"); - PARSE_AS(PreferredSymbolLocation::Definition, "definition"); - - return false; -#undef PARSE_AS -} - -bool ParseCommand(const std::string& content, Command* obj) { -#define PARSE_AS(name, string) \ - if (content == #string) { \ - *obj = name; \ - return true; \ - } - - PARSE_AS(Command::Callees, "callees"); - PARSE_AS(Command::Callers, "callers"); - PARSE_AS(Command::FindAllUsages, "find-all-usages"); - PARSE_AS(Command::FindInterestingUsages, "find-interesting-usages"); - PARSE_AS(Command::GotoReferenced, "goto-referenced"); - PARSE_AS(Command::Hierarchy, "hierarchy"); - PARSE_AS(Command::Outline, "outline"); - PARSE_AS(Command::Search, "search"); - - return false; -#undef PARSE_AS -} - - - -int main(int argc, char** argv) { - if (argc == 1 || options.find("--help") != options.end()) { - std::cout << R"help(clang-indexer help: - - General: - --help Print this help information. - --help-commands - Print all available query commands. - --project Path to compile_commands.json. Needed for the server, and - optionally by clients if there are multiple servers running. - --print-config - Emit all configuration data this executable is using. - - - Server: - --server If present, this binary will run in server mode. The binary - will not return until killed or an exit is requested. The - server computes and caches an index of the entire program - which is then queried by short-lived client processes. A - client is created by running this binary with a --command - flag. - --cache-dir Directory to cache the index and other useful information. If - a previous cache is present, the database will try to reuse - it. If this flag is not present, the database will be - in-memory only. - --threads Number of threads to use for indexing and querying tasks. - This value is optional; a good estimate is computed by - default. - - - Client: - --command Execute a query command against the index. See - --command-help for a listing of valid commands and a - description of what they do. Presence of this flag indicates - that the indexer is in client mode; this flag is mutually - exclusive with --server. - --location Location of the query. Some commands require only a file, - other require a line and column as well. Format is - filename[:line:column]. For example, "foobar.cc" and - "foobar.cc:1:10" are valid inputs. - --preferred-symbol-location - When looking up symbols, try to return either the - 'declaration' or the 'definition'. Defaults to 'definition'. -)help"; - exit(0); - } - - if (HasOption(options, "--help-commands")) { - std::cout << R"(Available commands: - - callees: - callers: - Emit all functions (with location) that this function calls ("callees") or - that call this function ("callers"). Requires a location. - - find-all-usages: - Emit every usage of the given symbol. This is intended to support a rename - refactoring. This output contains many uninteresting usages of symbols; - prefer find-interesting-usges. Requires a location. - - find-interesting-usages: - Emit only usages of the given symbol which are semantically interesting. - Requires a location. - - goto-referenced: - Find an associated reference (either definition or declaration) for the - given symbol. Requires a location. - - hierarchy: - List the type hierarchy (ie, inherited and derived members) for the given - method or type. Requires a location. - - outline: - Emit a file outline, listing all of the symbols in the file. - - search: - Search for a symbol by name. -)"; - exit(0); - } - - if (HasOption(options, "--project")) { - std::vector entries = LoadCompilationEntriesFromDirectory(options["--project"]); - - - for (const CompilationEntry& entry : entries) { - std::cout << "Parsing " << entry.filename << std::endl; - QueryableDatabase db; - IndexedFile file = Parse(entry.filename, entry.args); - - IndexUpdate update(file); - db.ApplyIndexUpdate(&update); - //std::cout << db.ToString() << std::endl << std::endl; - } - - std::cin.get(); - exit(0); - } - - if (HasOption(options, "--command")) { - Command command; - if (!ParseCommand(options["--command"], &command)) - Fail("Unknown command \"" + options["--command"] + "\"; see --help-commands"); - - - } - - std::cout << "Invalid arguments. Try --help."; - exit(1); - return 0; -} -#endif \ No newline at end of file diff --git a/query.cc b/query.cc index a0811c28..32dda1f4 100644 --- a/query.cc +++ b/query.cc @@ -343,12 +343,18 @@ void CompareGroups( IndexUpdate::IndexUpdate(IndexedFile& file) { files_added.push_back(QueryableFile(file)); - for (const IndexedTypeDef& def : file.types) + for (const IndexedTypeDef& def : file.types) { + if (def.is_bad_def) continue; types_added.push_back(QueryableTypeDef(file.id_cache, def)); - for (const IndexedFuncDef& def : file.funcs) + } + for (const IndexedFuncDef& def : file.funcs) { + if (def.is_bad_def) continue; funcs_added.push_back(QueryableFuncDef(file.id_cache, def)); - for (const IndexedVarDef& def : file.vars) + } + for (const IndexedVarDef& def : file.vars) { + if (def.is_bad_def) continue; vars_added.push_back(QueryableVarDef(file.id_cache, def)); + } } IndexUpdate::IndexUpdate(IndexedFile& previous_file, IndexedFile& current_file) { diff --git a/src/threaded_queue.h b/src/threaded_queue.h new file mode 100644 index 00000000..845df3e1 --- /dev/null +++ b/src/threaded_queue.h @@ -0,0 +1,51 @@ +#pragma once + +// TODO: cleanup includes. +#include +#include +#include +#include + +#include "../optional.h" + +// A threadsafe-queue. http://stackoverflow.com/a/16075550 +template +class ThreadedQueue { +public: + // Add an element to the queue. + void Enqueue(T t) { + std::lock_guard lock(mutex_); + queue_.push(t); + cv_.notify_one(); + } + + // Get the "front"-element. + // If the queue is empty, wait till a element is avaiable. + T Dequeue() { + std::unique_lock lock(mutex_); + while (queue_.empty()) { + // release lock as long as the wait and reaquire it afterwards. + cv_.wait(lock); + } + T val = queue_.front(); + queue_.pop(); + return val; + } + + // Get the "front"-element. + // Returns empty if the queue is empty. + optional TryDequeue() { + std::unique_lock lock(mutex_); + if (queue_.empty()) + return nullopt; + + T val = queue_.front(); + queue_.pop(); + return val; + } + +private: + std::queue queue_; + mutable std::mutex mutex_; + std::condition_variable cv_; +}; diff --git a/src/timer.cc b/src/timer.cc new file mode 100644 index 00000000..6ef17120 --- /dev/null +++ b/src/timer.cc @@ -0,0 +1,14 @@ +#include "timer.h" + +Timer::Timer() { + Reset(); +} + +void Timer::Reset() { + start = Clock::now(); +} + +long long Timer::ElapsedMilliseconds() { + std::chrono::time_point end = Clock::now(); + return std::chrono::duration_cast(end - start).count(); +} diff --git a/src/timer.h b/src/timer.h new file mode 100644 index 00000000..8bf5d01b --- /dev/null +++ b/src/timer.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +struct Timer { + using Clock = std::chrono::high_resolution_clock; + + // Creates a new timer. A timer is always running. + Timer(); + + // Restart/reset the timer. + void Reset(); + + // Return the number of milliseconds since the timer was last reset. + long long ElapsedMilliseconds(); + + // Raw start time. + std::chrono::time_point start; +}; diff --git a/src/typed_bidi_message_queue.h b/src/typed_bidi_message_queue.h index 466833c0..e6626e01 100644 --- a/src/typed_bidi_message_queue.h +++ b/src/typed_bidi_message_queue.h @@ -35,17 +35,16 @@ struct TypedBidiMessageQueue { deserializers_[id] = deserializer; } - void SendMessage(MessageQueue* destination, TId id, TMessage* message) { + void SendMessage(MessageQueue* destination, TId id, TMessage& message) { // Create writer. rapidjson::StringBuffer output; rapidjson::PrettyWriter writer(output); writer.SetIndent(' ', 0); // Serialize the message. - assert(serializers_.find(message->id) != serializers_.end() && - "No registered serializer"); + assert(serializers_.find(id) != serializers_.end() && "No registered serializer"); const Serializer& serializer = serializers_.find(id)->second; - serializer(writer, *message); + serializer(writer, message); // Send message. void* payload = malloc(sizeof(MessageHeader) + output.GetSize()); diff --git a/task.cc b/task.cc deleted file mode 100644 index 1094119f..00000000 --- a/task.cc +++ /dev/null @@ -1,224 +0,0 @@ -#include -#include -#include -#include -#include - -#include "compilation_database_loader.h" -#include "indexer.h" -#include "query.h" -#include "optional.h" -#include "utils.h" -//#include "third_party/tiny-process-library/process.hpp" - -#include -#include -#include -#include - -using std::experimental::optional; -using std::experimental::nullopt; - -// A threadsafe-queue. http://stackoverflow.com/a/16075550 -template -class SafeQueue { -public: - // Add an element to the queue. - void enqueue(T t) { - std::lock_guard lock(mutex_); - queue_.push(t); - cv_.notify_one(); - } - - // Get the "front"-element. - // If the queue is empty, wait till a element is avaiable. - T dequeue() { - std::unique_lock lock(mutex_); - while (queue_.empty()) { - // release lock as long as the wait and reaquire it afterwards. - cv_.wait(lock); - } - T val = queue_.front(); - queue_.pop(); - return val; - } - - // Get the "front"-element. - // Returns empty if the queue is empty. - optional try_dequeue() { - std::unique_lock lock(mutex_); - if (queue_.empty()) - return nullopt; - - T val = queue_.front(); - queue_.pop(); - return val; - } - -private: - std::queue queue_; - mutable std::mutex mutex_; - std::condition_variable cv_; -}; - -struct Task { - int priority = 0; - bool writes_to_index = false; - - enum class Kind { - CreateIndex, - IndexImport, - Exit - }; - Kind kind; - - struct CreateIndexState { - CompilationEntry data; - }; - struct IndexImportState { - std::string path; - }; - struct ExitState {}; - - // TODO: Move into a union? - CreateIndexState create_index; - IndexImportState index_import; - ExitState exit; - - static Task MakeExit() { - Task task; - task.kind = Kind::Exit; - return task; - } - - static Task MakeCreateIndexTask(CompilationEntry compilation_entry) { - Task task; - task.kind = Kind::CreateIndex; - task.create_index.data = compilation_entry; - return task; - } - - static Task MakeIndexImportTask(std::string filename) { - Task task; - task.kind = Kind::IndexImport; - task.index_import.path = filename; - return task; - } - - // TODO: Create index task. - // Task running in a separate process, parsing a file into something we can - // import. - - // TODO: Index import task. - // Completed parse task that wants to import content into the global database. - // Runs in main process, primary thread. Stops all other threads. - - // TODO: Index fresh task. - // Completed parse task that wants to update content previously imported into - // the global database. Runs in main process, primary thread. Stops all other - // threads. - // - // Note that this task just contains a set of operations to apply to the global - // database. The operations come from a diff based on the previously indexed - // state in comparison to the newly indexed state. - // - // TODO: We may be able to run multiple freshen and import tasks in parallel if - // we restrict what ranges of the db they may change. - - // TODO: QueryTask - // Task running a query against the global database. Run in main process, - // separate thread. - //Command query; - //Location location; - //std::string argument; -}; - -struct Config { - // Cache directory. Always ends with / - std::string cache_directory; -}; - -// NOTE: When something enters a value into master db, it will have to have a -// ref count, since multiple parsings could enter it (unless we require -// that it be defined in that declaration unit!) -struct TaskManager { - SafeQueue queued_tasks; - - // Available threads. - std::vector threads; - - TaskManager(int num_threads, Config* config); -}; - -void PostTaskToIndexer(TaskManager* tm, Task task) { - tm->queued_tasks.enqueue(task); -} - -void RunIndexTask(Config* config, TaskManager* tm, CompilationEntry entry) { - IndexedFile file = Parse(entry.filename, entry.args); - - std::string cleaned_file_path = entry.directory + "/" + entry.filename; - std::replace(cleaned_file_path.begin(), cleaned_file_path.end(), '/', '_'); - std::replace(cleaned_file_path.begin(), cleaned_file_path.end(), '\\', '_'); - std::string filename = config->cache_directory + cleaned_file_path; - WriteToFile(filename, file.ToString()); - - PostTaskToIndexer(tm, Task::MakeIndexImportTask(filename)); -} - -void LoadProject(Config* config, TaskManager* tm, std::vector entries) { - for (CompilationEntry entry : entries) { - tm->queued_tasks.enqueue(Task::MakeCreateIndexTask(entry)); - } -} - -static void ThreadMain(int id, Config* config, TaskManager* tm) { - while (true) { - Task task = tm->queued_tasks.dequeue(); - switch (task.kind) { - case Task::Kind::CreateIndex: - RunIndexTask(config, tm, task.create_index.data); - break; - case Task::Kind::IndexImport: - assert(false); - break; - case Task::Kind::Exit: - std::cerr << id << ": Exiting" << std::endl; - return; - } - - std::cerr << id << ": waking" << std::endl; - } - -} - -TaskManager::TaskManager(int num_threads, Config* config) { - for (int i = 0; i < num_threads; ++i) { - threads.push_back(std::thread(&ThreadMain, i, config, this)); - } -} - -void Pump(TaskManager* tm) { - //tm->threads[0]. -} - -int main252525225(int argc, char** argv) { - Config config; - TaskManager tm(5, &config); - LoadProject(&config, &tm, LoadCompilationEntriesFromDirectory("full_tests/simple_cross_reference")); - - // TODO: looks like we will have to write shared memory support. - - // TODO: We signal thread to pick data, thread signals data pick is done. - // Repeat until we encounter a writer, wait for all threads to signal - // they are done. - // TODO: Let's use a thread safe queue/vector/etc instead. - //for (int i = 0; i < 10; ++i) - // tm.queued_tasks.enqueue(Task::MakeExit()); - - for (std::thread& thread : tm.threads) - thread.join(); - - std::cin.get(); - return 0; -}