From 72776d4c1123030c36d50c40ab434cc449c05467 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Tue, 14 Mar 2017 21:59:05 -0700 Subject: [PATCH] good --- command_line.cc | 310 ++++++++++++++++--------- full_tests/simple_cross_reference/c.cc | 9 + ipc.cc | 32 ++- ipc.h | 22 +- platform.h | 1 + platform_win.cc | 2 + query.cc | 72 ++++-- query.h | 1 + 8 files changed, 296 insertions(+), 153 deletions(-) create mode 100644 full_tests/simple_cross_reference/c.cc diff --git a/command_line.cc b/command_line.cc index d3a74119..fa5e9c47 100644 --- a/command_line.cc +++ b/command_line.cc @@ -21,6 +21,11 @@ #include #endif +const char* kIpcIndexerName = "indexer"; +const char* kIpcLanguageClientName = "language_client"; + +const int kNumIndexers = 8 - 1; + std::unordered_map ParseOptions(int argc, char** argv) { std::unordered_map output; @@ -204,6 +209,7 @@ struct IpcMessage_IndexTranslationUnitResponse : public BaseIpcMessage @@ -310,10 +316,10 @@ struct Timer { }; -void IndexMainLoop(IpcClient* ipc) { - std::vector> messages = ipc->TakeMessages(); +void IndexMainLoop(IpcClient* client) { + std::vector> messages = client->TakeMessages(); for (auto& message : messages) { - std::cerr << "Processing message " << static_cast(message->ipc_id) << std::endl; + //std::cerr << "Processing message " << static_cast(message->ipc_id) << std::endl; switch (message->ipc_id) { case IpcId::Quit: { @@ -335,7 +341,7 @@ void IndexMainLoop(IpcClient* ipc) { std::cerr << "Creating index update took " << time.ElapsedMilliseconds() << "ms" << std::endl; time.Reset(); - ipc->SendToServer(&response); + client->SendToServer(&response); std::cerr << "Sending to server took " << time.ElapsedMilliseconds() << "ms" << std::endl; break; @@ -345,10 +351,7 @@ void IndexMainLoop(IpcClient* ipc) { } void IndexMain(int id) { - return; - - IpcClient client_ipc("indexer", id); - + IpcClient client_ipc(kIpcIndexerName, id); while (true) { IndexMainLoop(&client_ipc); std::this_thread::sleep_for(std::chrono::milliseconds(10)); @@ -367,11 +370,41 @@ void IndexMain(int id) { -void QueryDbMainLoop(IpcServer* client, std::vector>& indexers, QueryableDatabase* db) { - std::vector> messages = client->TakeMessages(); + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +void QueryDbMainLoop(IpcServer* language_client, IpcServer* indexers, QueryableDatabase* db) { + std::vector> messages = language_client->TakeMessages(); for (auto& message : messages) { - std::cerr << "Processing message " << static_cast(message->ipc_id) << std::endl; + //std::cerr << "Processing message " << static_cast(message->ipc_id) << std::endl; switch (message->ipc_id) { case IpcId::Quit: { @@ -380,7 +413,7 @@ void QueryDbMainLoop(IpcServer* client, std::vector>& case IpcId::IsAlive: { IpcMessage_IsAlive response; - client->SendToClient(&response); // todo: make non-blocking + language_client->SendToClient(0, &response); // todo: make non-blocking break; } @@ -395,11 +428,11 @@ void QueryDbMainLoop(IpcServer* client, std::vector>& std::string filepath = path + "/" + entry.filename; std::cerr << "[" << i << "/" << (entries.size() - 1) << "] Dispatching index request for file " << filepath << std::endl; - // TODO: indexers should steal work. load balance. + // TODO: indexers should steal work and load balance. IpcMessage_IndexTranslationUnitRequest request; request.path = filepath; request.args = entry.args; - indexers[i % indexers.size()]->SendToClient(&request); + indexers->SendToClient(i % indexers->num_clients(), &request); //IndexedFile file = Parse(filepath, entry.args); //IndexUpdate update(file); //db->ApplyIndexUpdate(&update); @@ -408,14 +441,6 @@ void QueryDbMainLoop(IpcServer* client, std::vector>& break; } - case IpcId::IndexTranslationUnitResponse: { - IpcMessage_IndexTranslationUnitResponse* msg = static_cast(message.get()); - Timer time; - db->ApplyIndexUpdate(&msg->update); - std::cerr << "Applying index update took " << time.ElapsedMilliseconds() << "ms" << std::endl; - break; - } - case IpcId::DocumentSymbolsRequest: { auto msg = static_cast(message.get()); @@ -487,7 +512,7 @@ void QueryDbMainLoop(IpcServer* client, std::vector>& - client->SendToClient(&response); + language_client->SendToClient(0, &response); break; } @@ -577,7 +602,27 @@ void QueryDbMainLoop(IpcServer* client, std::vector>& } - client->SendToClient(&response); + language_client->SendToClient(0, &response); + break; + } + + default: { + std::cerr << "Unhandled IPC message with kind " << static_cast(message->ipc_id) << std::endl; + exit(1); + } + } + } + + messages = indexers->TakeMessages(); + for (auto& message : messages) { + //std::cerr << "Processing message " << static_cast(message->ipc_id) << std::endl; + switch (message->ipc_id) { + + case IpcId::IndexTranslationUnitResponse: { + IpcMessage_IndexTranslationUnitResponse* msg = static_cast(message.get()); + Timer time; + db->ApplyIndexUpdate(&msg->update); + std::cerr << "Applying index update took " << time.ElapsedMilliseconds() << "ms" << std::endl; break; } @@ -588,6 +633,54 @@ void QueryDbMainLoop(IpcServer* client, std::vector>& } } } + +void QueryDbMain() { + std::cerr << "Running QueryDb" << std::endl; + IpcServer language_client(kIpcLanguageClientName, 1 /*num_clients*/); + IpcServer indexers(kIpcIndexerName, kNumIndexers); + QueryableDatabase db; + + std::cerr << "!! starting processes" << std::endl; + // Start indexer processes. + for (int i = 0; i < kNumIndexers; ++i) { + //new Process(process_name + " --indexer " + std::to_string(i + 1)); + new std::thread([i]() { + IndexMain(i); + }); + } + std::cerr << "!! done processes" << std::endl; + + // Pump query db main loop. + while (true) { + QueryDbMainLoop(&language_client, &indexers, &db); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } +} + + + + + + + + + + + + + + + + + + + + + + + + + @@ -722,10 +815,8 @@ void LanguageServerMainLoop(IpcClient* ipc) { } } -const int kNumIndexers = 8 - 1; - void LanguageServerMain(std::string process_name) { - IpcClient client_ipc("languageserver", 0); + IpcClient client_ipc(kIpcLanguageClientName, 0); // Discard any left-over messages from previous runs. client_ipc.TakeMessages(); @@ -767,109 +858,77 @@ void LanguageServerMain(std::string process_name) { // Pass empty process name so we only try to start the querydb once. LanguageServerMain(""); return; - } +} #endif // for debugging attach //std::this_thread::sleep_for(std::chrono::seconds(4)); - std::thread stdio_reader(&LanguageServerStdinLoop, &client_ipc); if (!has_server) { - std::vector> server_indexers; - IpcServer server_ipc("languageserver", 0); - QueryableDatabase db; - // No server. Run it in-process. new std::thread([&]() { - - std::cerr << "!! starting processes" << std::endl; - // Start indexer processes. - // TODO: make sure to handle this when running querydb out of process. - for (int i = 0; i < kNumIndexers; ++i) { - server_indexers.emplace_back(MakeUnique("indexer", i + 1)); - //new Process(process_name + " --indexer " + std::to_string(i + 1)); - new std::thread([i]() { - IndexMain(i + 1); - }); - } - std::cerr << "!! done processes" << std::endl; - - - - while (true) { - QueryDbMainLoop(&server_ipc, server_indexers, &db); - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } + QueryDbMain(); }); } // Run language client. + std::thread stdio_reader(&LanguageServerStdinLoop, &client_ipc); while (true) { LanguageServerMainLoop(&client_ipc); std::this_thread::sleep_for(std::chrono::milliseconds(10)); } -} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -int main(int argc, char** argv) { - bool loop = false; - while (loop) - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - - if (argc == 1) { - RunTests(); - return 0; } + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +void PreMain() { // We need to write to stdout in binary mode because in Windows, writing // \n will implicitly write \r\n. Language server API will ignore a // \r\r\n split request. @@ -878,21 +937,40 @@ int main(int argc, char** argv) { _setmode(_fileno(stdin), O_BINARY); #endif - IpcRegistry::instance()->Register(IpcId::Quit); - IpcRegistry::instance()->Register(IpcId::IsAlive); - IpcRegistry::instance()->Register(IpcId::OpenProject); + IpcRegistry::instance()->Register(IpcMessage_Quit::kIpcId); + IpcRegistry::instance()->Register(IpcMessage_IsAlive::kIpcId); + IpcRegistry::instance()->Register(IpcMessage_OpenProject::kIpcId); - IpcRegistry::instance()->Register(IpcId::DocumentSymbolsRequest); - IpcRegistry::instance()->Register(IpcId::DocumentSymbolsResponse); - IpcRegistry::instance()->Register(IpcId::WorkspaceSymbolsRequest); - IpcRegistry::instance()->Register(IpcId::WorkspaceSymbolsResponse); + IpcRegistry::instance()->Register(IpcMessage_IndexTranslationUnitRequest::kIpcId); + IpcRegistry::instance()->Register(IpcMessage_IndexTranslationUnitResponse::kIpcId); + + IpcRegistry::instance()->Register(IpcMessage_DocumentSymbolsRequest::kIpcId); + IpcRegistry::instance()->Register(IpcMessage_DocumentSymbolsResponse::kIpcId); + IpcRegistry::instance()->Register(IpcMessage_WorkspaceSymbolsRequest::kIpcId); + IpcRegistry::instance()->Register(IpcMessage_WorkspaceSymbolsResponse::kIpcId); MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); MessageRegistry::instance()->Register(); +} +int main(int argc, char** argv) { + bool loop = false; + while (loop) + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + + PreMain(); + + if (argc == 1) { + QueryDbMain(); + return 0; + } + if (argc == 1) { + RunTests(); + return 0; + } @@ -909,7 +987,7 @@ int main(int argc, char** argv) { else if (HasOption(options, "--querydb")) { std::cerr << "Running querydb" << std::endl; QueryableDatabase db; - IpcServer ipc("languageserver"); + IpcServer ipc(kIpcServername); while (true) { QueryDbMainLoop(&ipc, &db); // TODO: use a condition variable. diff --git a/full_tests/simple_cross_reference/c.cc b/full_tests/simple_cross_reference/c.cc new file mode 100644 index 00000000..d1dc600f --- /dev/null +++ b/full_tests/simple_cross_reference/c.cc @@ -0,0 +1,9 @@ +#include "a.h" + +#if RANDOM_DEFINE +static void LocalC() { + Common(); +} +#endif // RANDOM_DEFINE + +static void LocalD() {} \ No newline at end of file diff --git a/ipc.cc b/ipc.cc index 30e4e3d1..55e3b31b 100644 --- a/ipc.cc +++ b/ipc.cc @@ -45,14 +45,18 @@ namespace { IpcRegistry* IpcRegistry::instance_ = nullptr; std::unique_ptr IpcRegistry::Allocate(IpcId id) { - return std::unique_ptr((*allocators)[id]()); + assert(allocators_); + auto it = allocators_->find(id); + assert(it != allocators_->end() && "No registered allocator for id"); + return std::unique_ptr(it->second()); } struct IpcDirectionalChannel::MessageBuffer { - MessageBuffer(void* buffer, size_t buffer_size) { + MessageBuffer(void* buffer, size_t buffer_size, bool initialize) { real_buffer = buffer; real_buffer_size = buffer_size; - new(real_buffer) Metadata(); + if (initialize) + new(real_buffer) Metadata(); } // Pointer to the start of the actual buffer and the @@ -201,15 +205,15 @@ void IpcDirectionalChannel::RemoveResizableBuffer(int id) { resizable_buffers.erase(id); } -IpcDirectionalChannel::IpcDirectionalChannel(const std::string& name) { +IpcDirectionalChannel::IpcDirectionalChannel(const std::string& name, bool initialize_shared_memory) { shared = CreatePlatformSharedMemory(name + "memory"); mutex = CreatePlatformMutex(name + "mutex"); local = std::unique_ptr(new char[shmem_size]); // TODO: connecting a client will allocate reset shared state on the // buffer. We need to store if we "initialized". - shared_buffer = MakeUnique(shared->shared, shmem_size); - local_buffer = MakeUnique(local.get(), shmem_size); + shared_buffer = MakeUnique(shared->shared, shmem_size, initialize_shared_memory); + local_buffer = MakeUnique(local.get(), shmem_size, true /*initialize*/); } IpcDirectionalChannel::~IpcDirectionalChannel() {} @@ -361,11 +365,16 @@ std::vector> IpcDirectionalChannel::TakeMessages() { -IpcServer::IpcServer(const std::string& name, int client_id) - : server_(NameToServerName(name)), client_(NameToClientName(name, client_id)) {} +IpcServer::IpcServer(const std::string& name, int num_clients) + : server_(NameToServerName(name), true /*initialize_shared_memory*/) { -void IpcServer::SendToClient(IpcMessage* message) { - client_.PushMessage(message); + for (int i = 0; i < num_clients; ++i) { + clients_.push_back(MakeUnique(NameToClientName(name, i), true /*initialize_shared_memory*/)); + } +} + +void IpcServer::SendToClient(int client_id, IpcMessage* message) { + clients_[client_id]->PushMessage(message); } std::vector> IpcServer::TakeMessages() { @@ -373,7 +382,8 @@ std::vector> IpcServer::TakeMessages() { } IpcClient::IpcClient(const std::string& name, int client_id) - : server_(NameToServerName(name)), client_(NameToClientName(name, client_id)) {} + : server_(NameToServerName(name), false /*initialize_shared_memory*/), + client_(NameToClientName(name, client_id), false /*initialize_shared_memory*/) {} void IpcClient::SendToServer(IpcMessage* message) { server_.PushMessage(message); diff --git a/ipc.h b/ipc.h index e6c2fe14..289e953e 100644 --- a/ipc.h +++ b/ipc.h @@ -61,7 +61,7 @@ struct IpcRegistry { // Use unique_ptrs so we can initialize on first use // (static init order might not be right). - std::unique_ptr> allocators; + std::unique_ptr> allocators_; template void Register(IpcId id); @@ -78,13 +78,13 @@ struct IpcRegistry { template void IpcRegistry::Register(IpcId id) { - if (!allocators) - allocators = MakeUnique>(); + if (!allocators_) + allocators_ = MakeUnique>(); - assert(allocators->find(id) == allocators->end() && + assert(allocators_->find(id) == allocators_->end() && "There is already an IPC message with the given id"); - (*allocators)[id] = [id]() { + (*allocators_)[id] = [id]() { return new T(); }; } @@ -98,7 +98,7 @@ struct IpcDirectionalChannel { // NOTE: We keep all pointers in terms of char* so pointer arithmetic is // always relative to bytes. - explicit IpcDirectionalChannel(const std::string& name); + explicit IpcDirectionalChannel(const std::string& name, bool initialize_shared_memory); ~IpcDirectionalChannel(); void PushMessage(IpcMessage* message); @@ -123,14 +123,16 @@ struct IpcDirectionalChannel { }; struct IpcServer { - IpcServer(const std::string& name, int client_id); + IpcServer(const std::string& name, int num_clients); - void SendToClient(IpcMessage* message); + void SendToClient(int client_id, IpcMessage* message); std::vector> TakeMessages(); + int num_clients() const { return clients_.size(); } + private: - IpcDirectionalChannel server_; - IpcDirectionalChannel client_; + IpcDirectionalChannel server_; // Local / us. + std::vector> clients_; }; struct IpcClient { diff --git a/platform.h b/platform.h index a0ad82c7..03cd70c6 100644 --- a/platform.h +++ b/platform.h @@ -12,6 +12,7 @@ struct PlatformScopedMutexLock { struct PlatformSharedMemory { virtual ~PlatformSharedMemory() {} void* shared; + std::string name; }; const int shmem_size = 1024 * 1024 * 32; // number of chars/bytes (32mb) diff --git a/platform_win.cc b/platform_win.cc index fbc01a65..8e0d9a8b 100644 --- a/platform_win.cc +++ b/platform_win.cc @@ -37,6 +37,8 @@ struct PlatformSharedMemoryWin : public PlatformSharedMemory { HANDLE shmem_; PlatformSharedMemoryWin(const std::string& name) { + this->name = name; + shmem_ = CreateFileMapping( INVALID_HANDLE_VALUE, NULL, diff --git a/query.cc b/query.cc index 9d4af271..a0811c28 100644 --- a/query.cc +++ b/query.cc @@ -512,37 +512,77 @@ void QueryableDatabase::RemoveUsrs(const std::vector& to_remove) { void QueryableDatabase::Import(const std::vector& defs) { for (auto& def : defs) { - qualified_names.push_back(def.file_id); - symbols.push_back(SymbolIdx(SymbolKind::File, files.size())); - usr_to_symbol[def.file_id] = SymbolIdx(SymbolKind::File, files.size()); - files.push_back(def); + auto it = usr_to_symbol.find(def.file_id); + if (it == usr_to_symbol.end()) { + qualified_names.push_back(def.file_id); + symbols.push_back(SymbolIdx(SymbolKind::File, files.size())); + usr_to_symbol[def.file_id] = SymbolIdx(SymbolKind::File, files.size()); + + files.push_back(def); + } + else { + QueryableFile& existing = files[it->second.idx]; + // Replace the entire file. We don't ever want to merge files. + existing = def; + } } } void QueryableDatabase::Import(const std::vector& defs) { for (auto& def : defs) { - qualified_names.push_back(def.def.qualified_name); - symbols.push_back(SymbolIdx(SymbolKind::Type, types.size())); - usr_to_symbol[def.def.usr] = SymbolIdx(SymbolKind::Type, types.size()); - types.push_back(def); + auto it = usr_to_symbol.find(def.def.usr); + if (it == usr_to_symbol.end()) { + qualified_names.push_back(def.def.qualified_name); + symbols.push_back(SymbolIdx(SymbolKind::Type, types.size())); + usr_to_symbol[def.def.usr] = SymbolIdx(SymbolKind::Type, types.size()); + types.push_back(def); + } + else { + QueryableTypeDef& existing = types[it->second.idx]; + if (def.def.definition) + existing.def = def.def; + AddRange(&existing.derived, def.derived); + AddRange(&existing.uses, def.uses); + } } } void QueryableDatabase::Import(const std::vector& defs) { for (auto& def : defs) { - qualified_names.push_back(def.def.qualified_name); - symbols.push_back(SymbolIdx(SymbolKind::Func, funcs.size())); - usr_to_symbol[def.def.usr] = SymbolIdx(SymbolKind::Func, funcs.size()); - funcs.push_back(def); + auto it = usr_to_symbol.find(def.def.usr); + if (it == usr_to_symbol.end()) { + qualified_names.push_back(def.def.qualified_name); + symbols.push_back(SymbolIdx(SymbolKind::Func, funcs.size())); + usr_to_symbol[def.def.usr] = SymbolIdx(SymbolKind::Func, funcs.size()); + funcs.push_back(def); + } + else { + QueryableFuncDef& existing = funcs[it->second.idx]; + if (def.def.definition) + existing.def = def.def; + AddRange(&existing.callers, def.callers); + AddRange(&existing.declarations, def.declarations); + AddRange(&existing.derived, def.derived); + AddRange(&existing.uses, def.uses); + } } } void QueryableDatabase::Import(const std::vector& defs) { for (auto& def : defs) { - qualified_names.push_back(def.def.qualified_name); - symbols.push_back(SymbolIdx(SymbolKind::Var, vars.size())); - usr_to_symbol[def.def.usr] = SymbolIdx(SymbolKind::Var, vars.size()); - vars.push_back(def); + auto it = usr_to_symbol.find(def.def.usr); + if (it == usr_to_symbol.end()) { + qualified_names.push_back(def.def.qualified_name); + symbols.push_back(SymbolIdx(SymbolKind::Var, vars.size())); + usr_to_symbol[def.def.usr] = SymbolIdx(SymbolKind::Var, vars.size()); + vars.push_back(def); + } + else { + QueryableVarDef& existing = vars[it->second.idx]; + if (def.def.definition) + existing.def = def.def; + AddRange(&existing.uses, def.uses); + } } } diff --git a/query.h b/query.h index 9e19dd48..3e985b24 100644 --- a/query.h +++ b/query.h @@ -271,6 +271,7 @@ struct IndexUpdate { std::vector vars_def_changed; std::vector vars_uses; + IndexUpdate() {} // Creates a new IndexUpdate that will import |file|. explicit IndexUpdate(IndexedFile& file);