Zengtudor/ccls
1
0
Fork 0
mirror of https://github.com/MaskRay/ccls.git synced 2025-01-19 12:05:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Experiment with caching CXIndex across multiple sessions

Browse Source
This commit is contained in:
Jacob Dufault 2017-05-24 19:04:19 -07:00
parent d47869ad0f
commit 32940e2c88
9 changed files with 110 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/code_completion.cc
View File

@ -10,6 +10,7 @@
namespace {
unsigned Flags() {
// TODO: use clang_defaultEditingTranslationUnitOptions()?
return
CXTranslationUnit_Incomplete |
CXTranslationUnit_KeepGoing |
@ -249,7 +250,7 @@ void EnsureDocumentParsed(CompletionSession* session,
std::cerr << "[complete] Creating completion session with arguments " << StringJoin(args) << std::endl;
*index = MakeUnique<clang::Index>(0 /*excludeDeclarationsFromPCH*/, 0 /*displayDiagnostics*/);
*tu = MakeUnique<clang::TranslationUnit>(*index->get(), session->file.filename, args, unsaved, Flags());
*tu = MakeUnique<clang::TranslationUnit>(index->get(), session->file.filename, args, unsaved, Flags());
std::cerr << "[complete] Done creating active; did_fail=" << (*tu)->did_fail << std::endl;
}

25
src/command_line.cc
View File

@ -1142,6 +1142,7 @@ bool ImportCachedIndex(Config* config,
void ParseFile(Config* config,
WorkingFiles* working_files,
FileConsumer::SharedState* file_consumer_shared,
clang::Index* index,
Index_DoIdMapQueue* queue_do_id_map,
const Project::Entry& entry,
const optional<std::string>& indexed_content,
@ -1158,7 +1159,7 @@ void ParseFile(Config* config,
config, file_consumer_shared,
tu_path, tu_args,
entry.filename, indexed_content,
&perf);
&perf, index);
for (std::unique_ptr<IndexFile>& new_index : indexes) {
Timer time;
@ -1271,6 +1272,7 @@ bool IndexMain_DoIndex(Config* config,
FileConsumer::SharedState* file_consumer_shared,
Project* project,
WorkingFiles* working_files,
clang::Index* index,
Index_DoIndexQueue* queue_do_index,
Index_DoIdMapQueue* queue_do_id_map) {
optional<Index_DoIndex> index_request = queue_do_index->TryDequeue();
@ -1301,7 +1303,7 @@ bool IndexMain_DoIndex(Config* config,
case Index_DoIndex::Type::Parse: {
// index_request->path can be a cc/tu or a dependency path.
file_consumer_shared->Reset(index_request->entry.filename);
ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
ParseFile(config, working_files, file_consumer_shared, index, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
break;
}
@ -1311,7 +1313,7 @@ bool IndexMain_DoIndex(Config* config,
bool needs_reparse = ResetStaleFiles(config, file_consumer_shared, index_request->entry.filename);
if (needs_reparse)
ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
ParseFile(config, working_files, file_consumer_shared, index, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
break;
}
}
@ -1327,6 +1329,7 @@ bool IndexMain_DoCreateIndexUpdate(
return false;
Timer time;
// TODO/FIXME: Running call tree on IndexUpdate::CreateDelta crashes cquery.
IndexUpdate update = IndexUpdate::CreateDelta(response->previous_id_map.get(), response->current_id_map.get(),
response->previous_index.get(), response->current_index.get());
response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
@ -1398,6 +1401,9 @@ void IndexMain(
Index_OnIndexedQueue* queue_on_indexed) {
SetCurrentThreadName("indexer");
// TODO: dispose of index after it is not used for a while.
clang::Index index(1, 0);
while (true) {
// TODO: process all off IndexMain_DoIndex before calling IndexMain_DoCreateIndexUpdate for
// better icache behavior. We need to have some threads spinning on both though
@ -1407,7 +1413,7 @@ void IndexMain(
// IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any
// work. Running both also lets the user query the partially constructed
// index.
bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, working_files, queue_do_index, queue_do_id_map);
bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, working_files, &index, queue_do_index, queue_do_id_map);
bool did_create_update = IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed);
bool did_merge = false;
@ -2949,7 +2955,16 @@ int main(int argc, char** argv) {
if (context.shouldExit())
return res;
RunTests();
for (int i = 0; i < 50; ++i)
RunTests();
for (int i = 0; i < 20; ++i) {
std::this_thread::sleep_for(std::chrono::seconds(5));
std::cerr << "[POST] " << GetProcessMemoryUsedInMb() << std::endl;
}
std::cerr << std::endl << "[Enter] to exit" << std::endl;
std::cin.get();
return 0;
}
else if (HasOption(options, "--language-server")) {

10
src/indexer.cc
View File

@ -1536,6 +1536,7 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
const std::string& file_contents_path,
const optional<std::string>& file_contents,
PerformanceImportFile* perf,
clang::Index* index,
bool dump_ast) {
if (!config->enableIndexing)
@ -1545,8 +1546,9 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
Timer timer;
clang::Index index(0 /*excludeDeclarationsFromPCH*/,
0 /*displayDiagnostics*/);
//clang::Index index(0 /*excludeDeclarationsFromPCH*/,
// 0 /*displayDiagnostics*/);
std::vector<CXUnsavedFile> unsaved_files;
if (file_contents) {
CXUnsavedFile unsaved;
@ -1555,7 +1557,7 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
unsaved.Length = (unsigned long)file_contents->size();
unsaved_files.push_back(unsaved);
}
clang::TranslationUnit tu(index, file, args, unsaved_files, CXTranslationUnit_KeepGoing | CXTranslationUnit_DetailedPreprocessingRecord);
clang::TranslationUnit tu(index, file, args, unsaved_files, CXTranslationUnit_KeepGoing | CXTranslationUnit_DetailedPreprocessingRecord | CXTranslationUnit_Incomplete);
perf->index_parse = timer.ElapsedMicrosecondsAndReset();
@ -1576,7 +1578,7 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
param.primary_file = ConsumeFile(&param, cx_file);
//std::cerr << "!! [START] Indexing " << file << std::endl;
CXIndexAction index_action = clang_IndexAction_create(index.cx_index);
CXIndexAction index_action = clang_IndexAction_create(index->cx_index);
clang_indexTranslationUnit(index_action, &param, callbacks, sizeof(callbacks),
CXIndexOpt_IndexFunctionLocalSymbols | CXIndexOpt_SkipParsedBodiesInSession | CXIndexOpt_IndexImplicitTemplateInstantiations,
tu.cx_tu);

2
src/indexer.h
View File

@ -5,6 +5,7 @@
#include "serializer.h"
#include "utils.h"
#include "language_server_api.h"
#include "libclangmm/Index.h"
#include "libclangmm/Utility.h"
#include "performance.h"
@ -554,5 +555,6 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
const std::string& file_contents_path,
const optional<std::string>& file_contents,
PerformanceImportFile* perf,
clang::Index* index,
bool dump_ast = false);
void IndexInit();

4
src/libclangmm/TranslationUnit.cc
View File

@ -10,7 +10,7 @@
namespace clang {
TranslationUnit::TranslationUnit(Index& index,
TranslationUnit::TranslationUnit(Index* index,
const std::string& filepath,
const std::vector<std::string>& arguments,
std::vector<CXUnsavedFile> unsaved_files,
@ -27,7 +27,7 @@ TranslationUnit::TranslationUnit(Index& index,
//CXErrorCode error_code = clang_parseTranslationUnit2FullArgv(
CXErrorCode error_code = clang_parseTranslationUnit2(
index.cx_index, filepath.c_str(), args.data(), (int)args.size(),
index->cx_index, filepath.c_str(), args.data(), (int)args.size(),
unsaved_files.data(), (unsigned)unsaved_files.size(), flags, &cx_tu);
switch (error_code) {

2
src/libclangmm/TranslationUnit.h
View File

@ -12,7 +12,7 @@
namespace clang {
class TranslationUnit {
public:
TranslationUnit(Index& index,
TranslationUnit(Index* index,
const std::string& filepath,
const std::vector<std::string>& arguments,
std::vector<CXUnsavedFile> unsaved_files,

145
src/test.cc
View File

@ -98,29 +98,31 @@ void RunTests() {
// TODO: Assert that we need to be on clang >= 3.9.1
bool update_all = false;
clang::Index index(1, 0);
for (std::string path : GetFilesInFolder("tests", true /*recursive*/, true /*add_folder_to_path*/)) {
//if (path != "tests/templates/specialized_func_definition.cc") continue;
//if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue;
//if (path != "tests/multi_file/funky_enum.cc") continue;
//if (path != "tests/multi_file/simple_impl.cc") continue;
//if (path != "tests/usage/func_called_implicit_ctor.cc") continue;
//if (path != "tests/templates/implicit_variable_instantiation.cc") continue;
//if (path != "tests/_empty_test.cc") continue;
float memory_before = GetProcessMemoryUsedInMb();
float memory_after = -1.;
//if (path != "tests/templates/template_class_type_usage_folded_into_one.cc") continue;
//path = "C:/Users/jacob/Desktop/superindex/indexer/" + path;
{
//if (path != "tests/templates/specialized_func_definition.cc") continue;
//if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue;
//if (path != "tests/multi_file/funky_enum.cc") continue;
//if (path != "tests/multi_file/simple_impl.cc") continue;
//if (path != "tests/usage/func_called_implicit_ctor.cc") continue;
//if (path != "tests/templates/implicit_variable_instantiation.cc") continue;
//if (path != "tests/multi_file/bad_type_remove.cc") continue;
// Parse expected output from the test, parse it into JSON document.
std::unordered_map<std::string, std::string> all_expected_output = ParseTestExpectation(path);
//if (path != "tests/templates/template_class_type_usage_folded_into_one.cc") continue;
//path = "C:/Users/jacob/Desktop/superindex/indexer/" + path;
Config config;
FileConsumer::SharedState file_consumer_shared;
Config config;
FileConsumer::SharedState file_consumer_shared;
// Run test.
std::cout << "[START] " << path << std::endl;
PerformanceImportFile perf;
std::vector<std::unique_ptr<IndexFile>> dbs = Parse(
// Run test.
//std::cout << "[START] " << path << std::endl;
PerformanceImportFile perf;
std::vector<std::unique_ptr<IndexFile>> dbs = Parse(
&config, &file_consumer_shared,
path,
{
@ -133,82 +135,71 @@ void RunTests() {
},
"", nullopt,
&perf,
&index,
false /*dump_ast*/);
#if false
for (auto& db : dbs) {
assert(db);
if (!db) {
std::cerr << "no db!!!" << std::endl;
continue;
}
// Parse expected output from the test, parse it into JSON document.
std::unordered_map<std::string, std::string> all_expected_output = ParseTestExpectation(path);
for (auto& entry : all_expected_output) {
const std::string& expected_path = entry.first;
const std::string& expected_output = entry.second;
for (auto& func : db->funcs) {
if (!func.HasInterestingState())
continue;
// Get output from index operation.
IndexFile* db = FindDbForPathEnding(expected_path, dbs);
std::string actual_output = "{}";
if (db) {
VerifySerializeToFrom(db);
actual_output = db->ToString();
}
// Compare output via rapidjson::Document to ignore any formatting
// differences.
rapidjson::Document actual;
actual.Parse(actual_output.c_str());
rapidjson::Document expected;
expected.Parse(expected_output.c_str());
if (func.uses.size() !=
(func.callers.size() + func.declarations.size() + (func.def.definition_spelling.has_value() ? 1 : 0))) {
if (actual == expected) {
std::cout << "[PASSED] " << path << std::endl;
}
else {
DiffDocuments(path, expected_path, expected, actual);
std::cout << std::endl;
std::cout << std::endl;
std::cout << "[Enter to continue - type u to update test, a to update all]";
char c = 'u';
if (!update_all) {
c = (char)std::cin.get();
std::cin.get();
}
std::cout << "func.def.usr = " << func.def.usr << std::endl;
std::cout << "func.uses.size() = " << func.uses.size() << std::endl;
std::cout << "func.callers.size() = " << func.callers.size() << std::endl;
std::cout << "func.declarations.size() = " << func.declarations.size() << std::endl;
std::cout << "func.definition_spelling.has_value() = " << func.def.definition_spelling.has_value() << std::endl;
if (c == 'a')
update_all = true;
std::cerr << "err" << std::endl;
if (update_all || c == 'u') {
UpdateTestExpectation(path, expected_output, ToString(actual) + "\n");
}
}
}
}
#endif
for (auto& entry : all_expected_output) {
const std::string& expected_path = entry.first;
const std::string& expected_output = entry.second;
// Get output from index operation.
IndexFile* db = FindDbForPathEnding(expected_path, dbs);
std::string actual_output = "{}";
if (db) {
VerifySerializeToFrom(db);
actual_output = db->ToString();
}
// Compare output via rapidjson::Document to ignore any formatting
// differences.
rapidjson::Document actual;
actual.Parse(actual_output.c_str());
rapidjson::Document expected;
expected.Parse(expected_output.c_str());
if (actual == expected) {
std::cout << "[PASSED] " << path << std::endl;
}
else {
DiffDocuments(path, expected_path, expected, actual);
std::cout << std::endl;
std::cout << std::endl;
std::cout << "[Enter to continue - type u to update test, a to update all]";
char c = 'u';
if (!update_all) {
c = (char)std::cin.get();
std::cin.get();
}
if (c == 'a')
update_all = true;
if (update_all || c == 'u') {
UpdateTestExpectation(path, expected_output, ToString(actual) + "\n");
}
}
memory_after = GetProcessMemoryUsedInMb();
}
float memory_cleanup = GetProcessMemoryUsedInMb();
std::cerr << "[memory] before=" << memory_before << "mb, after=" << memory_after << "mb, cleanup=" << memory_cleanup << "mb" << std::endl;
}
std::cin.get();
std::cerr << "[final presleep] " << GetProcessMemoryUsedInMb() << "mb" << std::endl;
//std::this_thread::sleep_for(std::chrono::seconds(10));
//std::cerr << "[final postsleep] " << GetProcessMemoryUsedInMb() << "mb" << std::endl;
std::cerr << std::endl;
std::cerr << std::endl;
std::cerr << std::endl;
std::cerr << std::endl;
std::cerr << std::endl;
}
// TODO: ctor/dtor, copy ctor

7
src/utils.cc
View File

@ -2,6 +2,7 @@
#include "platform.h"
#include <sparsepp/spp_memory.h>
#include <tinydir.h>
#include <algorithm>
@ -315,3 +316,9 @@ void WriteToFile(const std::string& filename, const std::string& content) {
std::ofstream file(filename);
file << content;
}
float GetProcessMemoryUsedInMb() {
const float kBytesToMb = 1000000;
uint64_t memory_after = spp::GetProcessMemoryUsed();
return memory_after / kBytesToMb;
}

2
src/utils.h
View File

@ -122,3 +122,5 @@ inline void hash_combine(std::size_t& seed, const T& v, Rest... rest) {
}\
};\
}
float GetProcessMemoryUsedInMb();