diff --git a/src/command_line.cc b/src/command_line.cc index 0767ddf8..cff9530e 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -67,10 +67,10 @@ struct Index_OnIndexed { // TODO: Rename TypedBidiMessageQueue to IpcTransport? using IpcMessageQueue = TypedBidiMessageQueue; -using Index_DoIndexQueue = ThreadedQueue>; -using Index_DoIdMapQueue = ThreadedQueue>; -using Index_OnIdMappedQueue = ThreadedQueue>; -using Index_OnIndexedQueue = ThreadedQueue>; +using Index_DoIndexQueue = ThreadedQueue; +using Index_DoIdMapQueue = ThreadedQueue; +using Index_OnIdMappedQueue = ThreadedQueue; +using Index_OnIndexedQueue = ThreadedQueue; template void SendMessage(IpcMessageQueue& t, MessageQueue* destination, TMessage& message) { @@ -290,62 +290,74 @@ void WriteToCache(std::string filename, IndexedFile& file) { bool IndexMain_DoIndex(Index_DoIndexQueue* queue_do_index, - Index_DoIdMapQueue* queue_do_id_map) { - optional> opt_index_request = queue_do_index->TryDequeue(); - if (!opt_index_request) + Index_DoIdMapQueue* queue_do_id_map) { + optional index_request = queue_do_index->TryDequeue(); + if (!index_request) return false; - std::unique_ptr index_request = std::move(opt_index_request.value()); Timer time; - std::unique_ptr old_index = LoadCachedFile(index_request->path); - time.ResetAndPrint("Loading cached index"); + // If the index update is an import, then we will load the previous index // into memory if we have a previous index. After that, we dispatch an // update request to get the latest version. - if (old_index && index_request->type == Index_DoIndex::Type::Import) { - auto response = MakeUnique(nullptr /*previous*/, std::move(old_index) /*current*/); - queue_do_id_map->Enqueue(std::move(response)); - + if (index_request->type == Index_DoIndex::Type::Import) { index_request->type = Index_DoIndex::Type::Update; - queue_do_index->Enqueue(std::move(index_request)); + std::unique_ptr old_index = LoadCachedFile(index_request->path); + time.ResetAndPrint("Loading cached index"); + + // If import fails just do a standard update. + if (old_index) { + Index_DoIdMap response(nullptr, std::move(old_index)); + queue_do_id_map->Enqueue(std::move(response)); + + queue_do_index->Enqueue(std::move(*index_request)); + return true; + } } - else { - // Parse request and send a response. - std::cerr << "Parsing file " << index_request->path << " with args " - << Join(index_request->args, ", ") << std::endl; - // TODO: parse should return unique_ptr. Then we can eliminate copy below. Make sure to not - // reuse moved pointer in WriteToCache if we do so. - IndexedFile current_index = Parse(index_request->path, index_request->args); + // Parse request and send a response. + std::cerr << "Parsing file " << index_request->path << " with args " + << Join(index_request->args, ", ") << std::endl; - time.ResetAndPrint("Parsing/indexing"); + // TODO: parse should return unique_ptr. Then we can eliminate copy below. Make sure to not + // reuse moved pointer in WriteToCache if we do so. + std::vector> indexes = Parse(index_request->path, index_request->args); + time.ResetAndPrint("Parsing/indexing"); - auto response = MakeUnique(std::move(old_index) /*previous*/, MakeUnique(current_index) /*current*/); - queue_do_id_map->Enqueue(std::move(response)); + for (auto& current_index : indexes) { + std::unique_ptr old_index = LoadCachedFile(current_index->path); + time.ResetAndPrint("Loading cached index"); + + // TODO: Cache to disk on a separate thread. Maybe we do the cache after we + // have imported the index (so the import pipeline has five stages instead + // of the current 4). // Cache file so we can diff it later. - WriteToCache(index_request->path, current_index); + WriteToCache(index_request->path, *current_index); time.ResetAndPrint("Cache index update to disk"); + + // Send response to create id map. + Index_DoIdMap response(std::move(old_index), std::move(current_index)); + queue_do_id_map->Enqueue(std::move(response)); } return true; } bool IndexMain_DoCreateIndexUpdate(Index_OnIdMappedQueue* queue_on_id_mapped, - Index_OnIndexedQueue* queue_on_indexed) { - optional> opt_response = queue_on_id_mapped->TryDequeue(); - if (!opt_response) + Index_OnIndexedQueue* queue_on_indexed) { + optional response = queue_on_id_mapped->TryDequeue(); + if (!response) return false; - std::unique_ptr response = std::move(opt_response.value()); Timer time; IndexUpdate update = IndexUpdate::CreateDelta(response->previous_id_map.get(), response->current_id_map.get(), - response->previous_index.get(), response->current_index.get()); + response->previous_index.get(), response->current_index.get()); time.ResetAndPrint("Creating delta IndexUpdate"); - auto reply = MakeUnique(update); + Index_OnIndexed reply(update); queue_on_indexed->Enqueue(std::move(reply)); time.ResetAndPrint("Sending update to server"); @@ -353,16 +365,16 @@ bool IndexMain_DoCreateIndexUpdate(Index_OnIdMappedQueue* queue_on_id_mapped, } void IndexMain(Index_DoIndexQueue* queue_do_index, - Index_DoIdMapQueue* queue_do_id_map, - Index_OnIdMappedQueue* queue_on_id_mapped, - Index_OnIndexedQueue* queue_on_indexed) { + Index_DoIdMapQueue* queue_do_id_map, + Index_OnIdMappedQueue* queue_on_id_mapped, + Index_OnIndexedQueue* queue_on_indexed) { while (true) { // TODO: process all off IndexMain_DoIndex before calling IndexMain_DoCreateIndexUpdate for // better icache behavior. We need to have some threads spinning on both though // otherwise memory usage will get bad. if (!IndexMain_DoIndex(queue_do_index, queue_do_id_map) && - !IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed)) { + !IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed)) { // TODO: use CV to wakeup? std::this_thread::sleep_for(std::chrono::milliseconds(500)); } @@ -370,15 +382,9 @@ void IndexMain(Index_DoIndexQueue* queue_do_index, } QueryableFile* FindFile(QueryableDatabase* db, const std::string& filename) { - // std::cerr << "Wanted file " << msg->document << std::endl; - // TODO: hashmap lookup. - for (auto& file : db->files) { - // std::cerr << " - Have file " << file.file_id << std::endl; - if (file.def.usr == filename) { - //std::cerr << "Found file " << filename << std::endl; - return &file; - } - } + auto it = db->usr_to_symbol.find(filename); + if (it != db->usr_to_symbol.end()) + return &db->files[it->second.idx]; std::cerr << "Unable to find file " << filename << std::endl; return nullptr; @@ -386,8 +392,8 @@ QueryableFile* FindFile(QueryableDatabase* db, const std::string& filename) { lsRange GetLsRange(const Range& location) { return lsRange( - lsPosition(location.start.line - 1, location.start.column - 1), - lsPosition(location.end.line - 1, location.end.column - 1)); + lsPosition(location.start.line - 1, location.start.column - 1), + lsPosition(location.end.line - 1, location.end.column - 1)); } lsDocumentUri GetLsDocumentUri(QueryableDatabase* db, QueryFileId file_id) { @@ -635,9 +641,9 @@ void QueryDbMainLoop( << "] Dispatching index request for file " << filepath << std::endl; - auto request = MakeUnique(Index_DoIndex::Type::Import); - request->path = filepath; - request->args = entry.args; + Index_DoIndex request(Index_DoIndex::Type::Import); + request.path = filepath; + request.args = entry.args; queue_do_index->Enqueue(std::move(request)); } std::cerr << "Done" << std::endl; @@ -697,7 +703,7 @@ void QueryDbMainLoop( for (const SymbolRef& ref : file->def.all_symbols) { if (ref.loc.range.start.line >= target_line && ref.loc.range.end.line <= target_line && - ref.loc.range.start.column <= target_column && ref.loc.range.end.column >= target_column) { + ref.loc.range.start.column <= target_column && ref.loc.range.end.column >= target_column) { optional location = GetDefinitionSpellingOfSymbol(db, ref.idx); if (location) response.result.push_back(GetLsLocation(db, location.value())); @@ -869,7 +875,7 @@ void QueryDbMainLoop( info.location.uri.SetPath(def.def.usr); break; } - // TODO: file + // TODO: file case SymbolKind::Type: { QueryableTypeDef& def = db->types[symbol.idx]; info.name = def.def.qualified_name; @@ -932,29 +938,30 @@ void QueryDbMainLoop( while (true) { - optional> opt_request = queue_do_id_map->TryDequeue(); - if (!opt_request) + optional request = queue_do_id_map->TryDequeue(); + if (!request) break; - std::unique_ptr request = std::move(opt_request.value()); - auto response = MakeUnique(); + + Index_OnIdMapped response; Timer time; if (request->previous) { - response->previous_id_map = MakeUnique(db, request->previous->id_cache); - response->previous_index = std::move(request->previous); + response.previous_id_map = MakeUnique(db, request->previous->id_cache); + response.previous_index = std::move(request->previous); } - response->current_id_map = MakeUnique(db, request->current->id_cache); - response->current_index = std::move(request->current); + + assert(request->current); + response.current_id_map = MakeUnique(db, request->current->id_cache); + response.current_index = std::move(request->current); time.ResetAndPrint("Create IdMap"); queue_on_id_mapped->Enqueue(std::move(response)); } while (true) { - optional> opt_response = queue_on_indexed->TryDequeue(); - if (!opt_response) + optional response = queue_on_indexed->TryDequeue(); + if (!response) break; - std::unique_ptr response = std::move(opt_response.value()); Timer time; db->ApplyIndexUpdate(&response->update); diff --git a/src/file_consumer.cc b/src/file_consumer.cc new file mode 100644 index 00000000..ad5a9fec --- /dev/null +++ b/src/file_consumer.cc @@ -0,0 +1,24 @@ +#include "file_consumer.h" + +FileConsumer::FileConsumer(SharedState* shared_state) : shared_(shared_state) {} + +void FileConsumer::ClearOwnership() { + for (auto& entry : local_) + entry.second = Ownership::DoesNotOwn; +} + +bool FileConsumer::DoesOwnFile(const std::string& file) { + // Try to find cached local result. + auto it = local_.find(file); + if (it != local_.end()) + return it->second == Ownership::Owns; + + // No result in local; we need to query global. + bool did_insert = false; + { + std::lock_guard lock(shared_->muetx); + did_insert = shared_->files.insert(file).second; + } + local_[file] = did_insert ? Ownership::Owns : Ownership::DoesNotOwn; + return did_insert; +} \ No newline at end of file diff --git a/src/file_consumer.h b/src/file_consumer.h new file mode 100644 index 00000000..01500a53 --- /dev/null +++ b/src/file_consumer.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +// FileConsumer is used by the indexer. When it encouters a file, it tries to +// take ownership over it. If the indexer has ownership over a file, it will +// produce an index, otherwise, it will emit nothing for that declarations +// and references coming from that file. +// +// The indexer does this because header files do not have their own translation +// units but we still want to index them. +struct FileConsumer { + struct SharedState { + mutable std::unordered_set files; + mutable std::mutex muetx; + }; + + FileConsumer(SharedState* shared_state); + + // Returns true if this instance owns given |file|. This will also attempt to + // take ownership over |file|. + bool DoesOwnFile(const std::string& file); + + // Clear all ownership state. + void ClearOwnership(); + + private: + enum class Ownership { + Owns, + DoesNotOwn + }; + + std::unordered_map local_; + SharedState* shared_; +}; \ No newline at end of file diff --git a/src/indexer.cc b/src/indexer.cc index aca763f7..c4289338 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -1263,12 +1263,7 @@ void indexEntityReference(CXClientData client_data, } } -IndexedFile Parse(std::string filename, - std::vector args, - bool dump_ast) { - // TODO: We are currently emitting too much information for things not in the main file. If we're - // not in the main file, we should only emit references. - +std::vector> Parse(std::string filename, std::vector args, bool dump_ast) { clang_enableStackTraces(); clang_toggleCrashRecovery(1); @@ -1300,9 +1295,9 @@ IndexedFile Parse(std::string filename, */ }; - IndexedFile db(filename); + auto db = MakeUnique(filename); NamespaceHelper ns; - IndexParam param(&db, &ns); + IndexParam param(db.get(), &ns); std::cerr << "!! [START] Indexing " << filename << std::endl; clang_indexTranslationUnit(index_action, ¶m, callbacks, sizeof(callbacks), @@ -1311,5 +1306,7 @@ IndexedFile Parse(std::string filename, std::cerr << "!! [END] Indexing " << filename << std::endl; clang_IndexAction_dispose(index_action); - return db; + std::vector> result; + result.emplace_back(std::move(db)); + return std::move(result); } diff --git a/src/indexer.h b/src/indexer.h index e1555509..24659a6c 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -448,6 +448,4 @@ struct IndexedFile { std::string ToString(); }; -IndexedFile Parse(std::string filename, - std::vector args, - bool dump_ast = false); +std::vector> Parse(std::string filename, std::vector args, bool dump_ast = false); diff --git a/src/project.cc b/src/project.cc index 4a5642c9..38a3ab0f 100644 --- a/src/project.cc +++ b/src/project.cc @@ -23,10 +23,7 @@ std::vector LoadFromDirectoryListing(const std::string& projec std::vector files = GetFilesInFolder(project_directory, true /*recursive*/, true /*add_folder_to_path*/); for (const std::string& file : files) { - if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") || - EndsWith(file, ".c") || EndsWith(file, ".h") || - EndsWith(file, ".hpp")) { - + if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") || EndsWith(file, ".c") || EndsWith(file, ".h")) { CompilationEntry entry; entry.filename = NormalizePath(file); entry.args = args; diff --git a/src/test.cc b/src/test.cc index daf3f4d8..835bb841 100644 --- a/src/test.cc +++ b/src/test.cc @@ -72,20 +72,20 @@ void DiffDocuments(std::string path, rapidjson::Document& expected, rapidjson::D if (actual_output.size() > len) { std::cout << "Additional output in actual:" << std::endl; - for (int i = len; i < actual_output.size(); ++i) + for (size_t i = len; i < actual_output.size(); ++i) std::cout << " " << actual_output[i] << std::endl; } if (expected_output.size() > len) { std::cout << "Additional output in expected:" << std::endl; - for (int i = len; i < expected_output.size(); ++i) + for (size_t i = len; i < expected_output.size(); ++i) std::cout << " " << expected_output[i] << std::endl; } } -void VerifySerializeToFrom(IndexedFile& file) { - std::string expected = file.ToString(); - std::string actual = Deserialize("--.cc", Serialize(file)).value().ToString(); +void VerifySerializeToFrom(IndexedFile* file) { + std::string expected = file->ToString(); + std::string actual = Deserialize("--.cc", Serialize(*file)).value().ToString(); if (expected != actual) { std::cerr << "Serialization failure" << std::endl;; assert(false); @@ -123,7 +123,7 @@ void RunTests() { // Run test. std::cout << "[START] " << path << std::endl; - IndexedFile db = Parse(path, { + std::vector> dbs = Parse(path, { "-xc++", "-std=c++11", "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/", @@ -131,8 +131,23 @@ void RunTests() { "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/rapidjson/include", "-IC:/Users/jacob/Desktop/superindex/indexer/src" }, false /*dump_ast*/); - VerifySerializeToFrom(db); - std::string actual_output = db.ToString(); + + // TODO: Supporting tests for more than just primary indexed file. + + // Find primary file. + std::unique_ptr db; + for (auto& i : dbs) { + if (i->path == path) { + db = std::move(i); + break; + } + } + + // TODO: Always pass IndexedFile by pointer, ie, search and remove all IndexedFile& refs. + // TODO: Rename IndexedFile to IndexFile + + VerifySerializeToFrom(db.get()); + std::string actual_output = db->ToString(); rapidjson::Document actual; actual.Parse(actual_output.c_str());