From 32940e2c88d552d1536b97552d87c57914f49a52 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Wed, 24 May 2017 19:04:19 -0700 Subject: [PATCH] Experiment with caching CXIndex across multiple sessions --- src/code_completion.cc | 3 +- src/command_line.cc | 25 ++++-- src/indexer.cc | 10 ++- src/indexer.h | 2 + src/libclangmm/TranslationUnit.cc | 4 +- src/libclangmm/TranslationUnit.h | 2 +- src/test.cc | 145 ++++++++++++++---------------- src/utils.cc | 7 ++ src/utils.h | 2 + 9 files changed, 110 insertions(+), 90 deletions(-) diff --git a/src/code_completion.cc b/src/code_completion.cc index 61f9bf98..acdc0e0a 100644 --- a/src/code_completion.cc +++ b/src/code_completion.cc @@ -10,6 +10,7 @@ namespace { unsigned Flags() { + // TODO: use clang_defaultEditingTranslationUnitOptions()? return CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing | @@ -249,7 +250,7 @@ void EnsureDocumentParsed(CompletionSession* session, std::cerr << "[complete] Creating completion session with arguments " << StringJoin(args) << std::endl; *index = MakeUnique(0 /*excludeDeclarationsFromPCH*/, 0 /*displayDiagnostics*/); - *tu = MakeUnique(*index->get(), session->file.filename, args, unsaved, Flags()); + *tu = MakeUnique(index->get(), session->file.filename, args, unsaved, Flags()); std::cerr << "[complete] Done creating active; did_fail=" << (*tu)->did_fail << std::endl; } diff --git a/src/command_line.cc b/src/command_line.cc index e2c4516b..62adf529 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -1142,6 +1142,7 @@ bool ImportCachedIndex(Config* config, void ParseFile(Config* config, WorkingFiles* working_files, FileConsumer::SharedState* file_consumer_shared, + clang::Index* index, Index_DoIdMapQueue* queue_do_id_map, const Project::Entry& entry, const optional& indexed_content, @@ -1158,7 +1159,7 @@ void ParseFile(Config* config, config, file_consumer_shared, tu_path, tu_args, entry.filename, indexed_content, - &perf); + &perf, index); for (std::unique_ptr& new_index : indexes) { Timer time; @@ -1271,6 +1272,7 @@ bool IndexMain_DoIndex(Config* config, FileConsumer::SharedState* file_consumer_shared, Project* project, WorkingFiles* working_files, + clang::Index* index, Index_DoIndexQueue* queue_do_index, Index_DoIdMapQueue* queue_do_id_map) { optional index_request = queue_do_index->TryDequeue(); @@ -1301,7 +1303,7 @@ bool IndexMain_DoIndex(Config* config, case Index_DoIndex::Type::Parse: { // index_request->path can be a cc/tu or a dependency path. file_consumer_shared->Reset(index_request->entry.filename); - ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive); + ParseFile(config, working_files, file_consumer_shared, index, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive); break; } @@ -1311,7 +1313,7 @@ bool IndexMain_DoIndex(Config* config, bool needs_reparse = ResetStaleFiles(config, file_consumer_shared, index_request->entry.filename); if (needs_reparse) - ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive); + ParseFile(config, working_files, file_consumer_shared, index, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive); break; } } @@ -1327,6 +1329,7 @@ bool IndexMain_DoCreateIndexUpdate( return false; Timer time; + // TODO/FIXME: Running call tree on IndexUpdate::CreateDelta crashes cquery. IndexUpdate update = IndexUpdate::CreateDelta(response->previous_id_map.get(), response->current_id_map.get(), response->previous_index.get(), response->current_index.get()); response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset(); @@ -1398,6 +1401,9 @@ void IndexMain( Index_OnIndexedQueue* queue_on_indexed) { SetCurrentThreadName("indexer"); + // TODO: dispose of index after it is not used for a while. + clang::Index index(1, 0); + while (true) { // TODO: process all off IndexMain_DoIndex before calling IndexMain_DoCreateIndexUpdate for // better icache behavior. We need to have some threads spinning on both though @@ -1407,7 +1413,7 @@ void IndexMain( // IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any // work. Running both also lets the user query the partially constructed // index. - bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, working_files, queue_do_index, queue_do_id_map); + bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, working_files, &index, queue_do_index, queue_do_id_map); bool did_create_update = IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed); bool did_merge = false; @@ -2949,7 +2955,16 @@ int main(int argc, char** argv) { if (context.shouldExit()) return res; - RunTests(); + for (int i = 0; i < 50; ++i) + RunTests(); + + for (int i = 0; i < 20; ++i) { + std::this_thread::sleep_for(std::chrono::seconds(5)); + std::cerr << "[POST] " << GetProcessMemoryUsedInMb() << std::endl; + } + + std::cerr << std::endl << "[Enter] to exit" << std::endl; + std::cin.get(); return 0; } else if (HasOption(options, "--language-server")) { diff --git a/src/indexer.cc b/src/indexer.cc index e76729f2..18395a65 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -1536,6 +1536,7 @@ std::vector> Parse( const std::string& file_contents_path, const optional& file_contents, PerformanceImportFile* perf, + clang::Index* index, bool dump_ast) { if (!config->enableIndexing) @@ -1545,8 +1546,9 @@ std::vector> Parse( Timer timer; - clang::Index index(0 /*excludeDeclarationsFromPCH*/, - 0 /*displayDiagnostics*/); + //clang::Index index(0 /*excludeDeclarationsFromPCH*/, + // 0 /*displayDiagnostics*/); + std::vector unsaved_files; if (file_contents) { CXUnsavedFile unsaved; @@ -1555,7 +1557,7 @@ std::vector> Parse( unsaved.Length = (unsigned long)file_contents->size(); unsaved_files.push_back(unsaved); } - clang::TranslationUnit tu(index, file, args, unsaved_files, CXTranslationUnit_KeepGoing | CXTranslationUnit_DetailedPreprocessingRecord); + clang::TranslationUnit tu(index, file, args, unsaved_files, CXTranslationUnit_KeepGoing | CXTranslationUnit_DetailedPreprocessingRecord | CXTranslationUnit_Incomplete); perf->index_parse = timer.ElapsedMicrosecondsAndReset(); @@ -1576,7 +1578,7 @@ std::vector> Parse( param.primary_file = ConsumeFile(¶m, cx_file); //std::cerr << "!! [START] Indexing " << file << std::endl; - CXIndexAction index_action = clang_IndexAction_create(index.cx_index); + CXIndexAction index_action = clang_IndexAction_create(index->cx_index); clang_indexTranslationUnit(index_action, ¶m, callbacks, sizeof(callbacks), CXIndexOpt_IndexFunctionLocalSymbols | CXIndexOpt_SkipParsedBodiesInSession | CXIndexOpt_IndexImplicitTemplateInstantiations, tu.cx_tu); diff --git a/src/indexer.h b/src/indexer.h index 1080eb34..73f1a823 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -5,6 +5,7 @@ #include "serializer.h" #include "utils.h" #include "language_server_api.h" +#include "libclangmm/Index.h" #include "libclangmm/Utility.h" #include "performance.h" @@ -554,5 +555,6 @@ std::vector> Parse( const std::string& file_contents_path, const optional& file_contents, PerformanceImportFile* perf, + clang::Index* index, bool dump_ast = false); void IndexInit(); diff --git a/src/libclangmm/TranslationUnit.cc b/src/libclangmm/TranslationUnit.cc index 5d055d8a..92571de4 100644 --- a/src/libclangmm/TranslationUnit.cc +++ b/src/libclangmm/TranslationUnit.cc @@ -10,7 +10,7 @@ namespace clang { -TranslationUnit::TranslationUnit(Index& index, +TranslationUnit::TranslationUnit(Index* index, const std::string& filepath, const std::vector& arguments, std::vector unsaved_files, @@ -27,7 +27,7 @@ TranslationUnit::TranslationUnit(Index& index, //CXErrorCode error_code = clang_parseTranslationUnit2FullArgv( CXErrorCode error_code = clang_parseTranslationUnit2( - index.cx_index, filepath.c_str(), args.data(), (int)args.size(), + index->cx_index, filepath.c_str(), args.data(), (int)args.size(), unsaved_files.data(), (unsigned)unsaved_files.size(), flags, &cx_tu); switch (error_code) { diff --git a/src/libclangmm/TranslationUnit.h b/src/libclangmm/TranslationUnit.h index 1c89bc1a..267c0ad5 100644 --- a/src/libclangmm/TranslationUnit.h +++ b/src/libclangmm/TranslationUnit.h @@ -12,7 +12,7 @@ namespace clang { class TranslationUnit { public: - TranslationUnit(Index& index, + TranslationUnit(Index* index, const std::string& filepath, const std::vector& arguments, std::vector unsaved_files, diff --git a/src/test.cc b/src/test.cc index 6d051b69..5f9a79fd 100644 --- a/src/test.cc +++ b/src/test.cc @@ -98,29 +98,31 @@ void RunTests() { // TODO: Assert that we need to be on clang >= 3.9.1 bool update_all = false; + clang::Index index(1, 0); for (std::string path : GetFilesInFolder("tests", true /*recursive*/, true /*add_folder_to_path*/)) { - //if (path != "tests/templates/specialized_func_definition.cc") continue; - //if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue; - //if (path != "tests/multi_file/funky_enum.cc") continue; - //if (path != "tests/multi_file/simple_impl.cc") continue; - //if (path != "tests/usage/func_called_implicit_ctor.cc") continue; - //if (path != "tests/templates/implicit_variable_instantiation.cc") continue; - //if (path != "tests/_empty_test.cc") continue; + float memory_before = GetProcessMemoryUsedInMb(); + float memory_after = -1.; - //if (path != "tests/templates/template_class_type_usage_folded_into_one.cc") continue; - //path = "C:/Users/jacob/Desktop/superindex/indexer/" + path; + { + //if (path != "tests/templates/specialized_func_definition.cc") continue; + //if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue; + //if (path != "tests/multi_file/funky_enum.cc") continue; + //if (path != "tests/multi_file/simple_impl.cc") continue; + //if (path != "tests/usage/func_called_implicit_ctor.cc") continue; + //if (path != "tests/templates/implicit_variable_instantiation.cc") continue; + //if (path != "tests/multi_file/bad_type_remove.cc") continue; - // Parse expected output from the test, parse it into JSON document. - std::unordered_map all_expected_output = ParseTestExpectation(path); + //if (path != "tests/templates/template_class_type_usage_folded_into_one.cc") continue; + //path = "C:/Users/jacob/Desktop/superindex/indexer/" + path; - Config config; - FileConsumer::SharedState file_consumer_shared; + Config config; + FileConsumer::SharedState file_consumer_shared; - // Run test. - std::cout << "[START] " << path << std::endl; - PerformanceImportFile perf; - std::vector> dbs = Parse( + // Run test. + //std::cout << "[START] " << path << std::endl; + PerformanceImportFile perf; + std::vector> dbs = Parse( &config, &file_consumer_shared, path, { @@ -133,82 +135,71 @@ void RunTests() { }, "", nullopt, &perf, + &index, false /*dump_ast*/); #if false - for (auto& db : dbs) { - assert(db); - if (!db) { - std::cerr << "no db!!!" << std::endl; - continue; - } + // Parse expected output from the test, parse it into JSON document. + std::unordered_map all_expected_output = ParseTestExpectation(path); + for (auto& entry : all_expected_output) { + const std::string& expected_path = entry.first; + const std::string& expected_output = entry.second; - for (auto& func : db->funcs) { - if (!func.HasInterestingState()) - continue; + // Get output from index operation. + IndexFile* db = FindDbForPathEnding(expected_path, dbs); + std::string actual_output = "{}"; + if (db) { + VerifySerializeToFrom(db); + actual_output = db->ToString(); + } + // Compare output via rapidjson::Document to ignore any formatting + // differences. + rapidjson::Document actual; + actual.Parse(actual_output.c_str()); + rapidjson::Document expected; + expected.Parse(expected_output.c_str()); - if (func.uses.size() != - (func.callers.size() + func.declarations.size() + (func.def.definition_spelling.has_value() ? 1 : 0))) { + if (actual == expected) { + std::cout << "[PASSED] " << path << std::endl; + } + else { + DiffDocuments(path, expected_path, expected, actual); + std::cout << std::endl; + std::cout << std::endl; + std::cout << "[Enter to continue - type u to update test, a to update all]"; + char c = 'u'; + if (!update_all) { + c = (char)std::cin.get(); + std::cin.get(); + } - std::cout << "func.def.usr = " << func.def.usr << std::endl; - std::cout << "func.uses.size() = " << func.uses.size() << std::endl; - std::cout << "func.callers.size() = " << func.callers.size() << std::endl; - std::cout << "func.declarations.size() = " << func.declarations.size() << std::endl; - std::cout << "func.definition_spelling.has_value() = " << func.def.definition_spelling.has_value() << std::endl; + if (c == 'a') + update_all = true; - std::cerr << "err" << std::endl; + if (update_all || c == 'u') { + UpdateTestExpectation(path, expected_output, ToString(actual) + "\n"); + } } } - } #endif - for (auto& entry : all_expected_output) { - const std::string& expected_path = entry.first; - const std::string& expected_output = entry.second; - - // Get output from index operation. - IndexFile* db = FindDbForPathEnding(expected_path, dbs); - std::string actual_output = "{}"; - if (db) { - VerifySerializeToFrom(db); - actual_output = db->ToString(); - } - - // Compare output via rapidjson::Document to ignore any formatting - // differences. - rapidjson::Document actual; - actual.Parse(actual_output.c_str()); - rapidjson::Document expected; - expected.Parse(expected_output.c_str()); - - if (actual == expected) { - std::cout << "[PASSED] " << path << std::endl; - } - else { - DiffDocuments(path, expected_path, expected, actual); - std::cout << std::endl; - std::cout << std::endl; - std::cout << "[Enter to continue - type u to update test, a to update all]"; - char c = 'u'; - if (!update_all) { - c = (char)std::cin.get(); - std::cin.get(); - } - - if (c == 'a') - update_all = true; - - if (update_all || c == 'u') { - UpdateTestExpectation(path, expected_output, ToString(actual) + "\n"); - } - - } + memory_after = GetProcessMemoryUsedInMb(); } + + float memory_cleanup = GetProcessMemoryUsedInMb(); + std::cerr << "[memory] before=" << memory_before << "mb, after=" << memory_after << "mb, cleanup=" << memory_cleanup << "mb" << std::endl; } - std::cin.get(); + std::cerr << "[final presleep] " << GetProcessMemoryUsedInMb() << "mb" << std::endl; + //std::this_thread::sleep_for(std::chrono::seconds(10)); + //std::cerr << "[final postsleep] " << GetProcessMemoryUsedInMb() << "mb" << std::endl; + std::cerr << std::endl; + std::cerr << std::endl; + std::cerr << std::endl; + std::cerr << std::endl; + std::cerr << std::endl; } // TODO: ctor/dtor, copy ctor diff --git a/src/utils.cc b/src/utils.cc index 55a014d3..2bee20e0 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -2,6 +2,7 @@ #include "platform.h" +#include #include #include @@ -315,3 +316,9 @@ void WriteToFile(const std::string& filename, const std::string& content) { std::ofstream file(filename); file << content; } + +float GetProcessMemoryUsedInMb() { + const float kBytesToMb = 1000000; + uint64_t memory_after = spp::GetProcessMemoryUsed(); + return memory_after / kBytesToMb; +} \ No newline at end of file diff --git a/src/utils.h b/src/utils.h index 929fa8eb..b24bee94 100644 --- a/src/utils.h +++ b/src/utils.h @@ -122,3 +122,5 @@ inline void hash_combine(std::size_t& seed, const T& v, Rest... rest) { }\ };\ } + +float GetProcessMemoryUsedInMb(); \ No newline at end of file