From d5e5d96a9e1b6ad69ed12d6c24373e4374218d48 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Wed, 12 Apr 2017 23:01:42 -0700 Subject: [PATCH] - Slightly more robust file parsing - Make memory ownership slightly clearer in indexer --- src/command_line.cc | 11 +++++------ src/indexer.cc | 7 ++++--- src/indexer.h | 2 +- src/language_server_api.cc | 11 +++++++++++ src/test.cc | 3 +-- src/utils.cc | 20 ++++++++++++++++++++ src/utils.h | 1 + 7 files changed, 43 insertions(+), 12 deletions(-) diff --git a/src/command_line.cc b/src/command_line.cc index a3bf3239..e7273dd7 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -58,6 +58,7 @@ QueryableFile* FindFile(QueryableDatabase* db, const std::string& filename, Quer } QueryableFile* FindFile(QueryableDatabase* db, const std::string& filename) { + // TODO: consider calling NormalizePath here. It might add too much latency though. auto it = db->usr_to_symbol.find(filename); if (it != db->usr_to_symbol.end()) return &db->files[it->second.idx]; @@ -538,7 +539,7 @@ void RegisterMessageTypes() { MessageRegistry::instance()->Register(); } -bool IndexMain_DoIndex(FileConsumer* file_consumer, +bool IndexMain_DoIndex(FileConsumer::SharedState* file_consumer_shared, Index_DoIndexQueue* queue_do_index, Index_DoIdMapQueue* queue_do_id_map) { optional index_request = queue_do_index->TryDequeue(); @@ -589,7 +590,7 @@ bool IndexMain_DoIndex(FileConsumer* file_consumer, } // Parse request and send a response. - std::vector> indexes = Parse(file_consumer, index_request->path, index_request->args); + std::vector> indexes = Parse(file_consumer_shared, index_request->path, index_request->args); time.ResetAndPrint("Parsing/indexing " + index_request->path); for (auto& current_index : indexes) { @@ -657,8 +658,6 @@ void IndexMain( Index_OnIdMappedQueue* queue_on_id_mapped, Index_OnIndexedQueue* queue_on_indexed) { - FileConsumer file_consumer(file_consumer_shared); - while (true) { // TODO: process all off IndexMain_DoIndex before calling IndexMain_DoCreateIndexUpdate for // better icache behavior. We need to have some threads spinning on both though @@ -666,7 +665,7 @@ void IndexMain( int count = 0; - if (!IndexMain_DoIndex(&file_consumer, queue_do_index, queue_do_id_map) && + if (!IndexMain_DoIndex(file_consumer_shared, queue_do_index, queue_do_id_map) && !IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed)) { //if (count++ > 2) { @@ -850,7 +849,7 @@ void QueryDbMainLoop( // - goto declaration while in definition of recursive type optional def_loc = GetDefinitionSpellingOfSymbol(db, ref.idx); - + // We use spelling start and extent end because this causes vscode // to highlight the entire definition when previewing / hoving with // the mouse. diff --git a/src/indexer.cc b/src/indexer.cc index 7350294f..c4bfb155 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -1394,7 +1394,7 @@ void indexEntityReference(CXClientData client_data, -std::vector> Parse(FileConsumer* file_consumer, std::string filename, std::vector args, bool dump_ast) { +std::vector> Parse(FileConsumer::SharedState* file_consumer_shared, std::string filename, std::vector args, bool dump_ast) { filename = NormalizePath(filename); //return {}; @@ -1415,10 +1415,11 @@ std::vector> Parse(FileConsumer* file_consumer, std &indexEntityReference} }; - IndexParam param(file_consumer); + FileConsumer file_consumer(file_consumer_shared); + IndexParam param(&file_consumer); CXFile file = clang_getFile(tu.cx_tu, filename.c_str()); - param.primary_file = file_consumer->ForceLocal(file); + param.primary_file = file_consumer.ForceLocal(file); std::cerr << "!! [START] Indexing " << filename << std::endl; CXIndexAction index_action = clang_IndexAction_create(index.cx_index); diff --git a/src/indexer.h b/src/indexer.h index 9978371e..63c2a962 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -444,4 +444,4 @@ struct IndexedFile { std::string ToString(); }; -std::vector> Parse(FileConsumer* file_consumer, std::string filename, std::vector args, bool dump_ast = false); +std::vector> Parse(FileConsumer::SharedState* file_consumer_shared, std::string filename, std::vector args, bool dump_ast = false); diff --git a/src/language_server_api.cc b/src/language_server_api.cc index 9c6dbfe8..157170a4 100644 --- a/src/language_server_api.cc +++ b/src/language_server_api.cc @@ -126,6 +126,10 @@ void lsDocumentUri::SetPath(const std::string& path) { raw_uri.replace(raw_uri.begin() + index, raw_uri.begin() + index + 1, "%3A"); } + raw_uri = ReplaceAll(raw_uri, " ", "%20"); + raw_uri = ReplaceAll(raw_uri, "(", "%28"); + raw_uri = ReplaceAll(raw_uri, ")", "%29"); + // TODO: proper fix #if defined(_WIN32) raw_uri = "file:///" + raw_uri; @@ -136,9 +140,16 @@ void lsDocumentUri::SetPath(const std::string& path) { } std::string lsDocumentUri::GetPath() const { + // c:/Program%20Files%20%28x86%29/Microsoft%20Visual%20Studio%2014.0/VC/include/vcruntime. + // C:/Program Files (x86) + // TODO: make this not a hack. std::string result = raw_uri; + result = ReplaceAll(result, "%20", " "); + result = ReplaceAll(result, "%28", "("); + result = ReplaceAll(result, "%29", ")"); + size_t index = result.find("%3A"); if (index != -1) { result.replace(result.begin() + index, result.begin() + index + 3, ":"); diff --git a/src/test.cc b/src/test.cc index 7fd069b4..5bc9f2a1 100644 --- a/src/test.cc +++ b/src/test.cc @@ -130,11 +130,10 @@ void RunTests() { std::unordered_map all_expected_output = ParseTestExpectation(path); FileConsumer::SharedState file_consumer_shared; - FileConsumer file_consumer(&file_consumer_shared); // Run test. std::cout << "[START] " << path << std::endl; - std::vector> dbs = Parse(&file_consumer, path, { + std::vector> dbs = Parse(&file_consumer_shared, path, { "-xc++", "-std=c++11", "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/", diff --git a/src/utils.cc b/src/utils.cc index fa505087..953e5dfa 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -20,6 +20,26 @@ bool StartsWith(const std::string& value, const std::string& start) { return std::equal(start.begin(), start.end(), value.begin()); } +// See http://stackoverflow.com/a/29752943 +std::string ReplaceAll(const std::string& source, const std::string& from, const std::string& to) { + std::string result; + result.reserve(source.length()); // avoids a few memory allocations + + std::string::size_type last_pos = 0; + std::string::size_type find_pos; + + while (std::string::npos != (find_pos = source.find(from, last_pos))) { + result.append(source, last_pos, find_pos - last_pos); + result += to; + last_pos = find_pos + from.length(); + } + + // Care for the rest after last occurrence + result += source.substr(last_pos); + + return result; +} + static std::vector GetFilesInFolderHelper(std::string folder, bool recursive, std::string output_prefix) { std::vector result; diff --git a/src/utils.h b/src/utils.h index da4bf00d..da37f89f 100644 --- a/src/utils.h +++ b/src/utils.h @@ -10,6 +10,7 @@ // Returns true if |value| starts/ends with |start| or |ending|. bool StartsWith(const std::string& value, const std::string& start); bool EndsWith(const std::string& value, const std::string& ending); +std::string ReplaceAll(const std::string& source, const std::string& from, const std::string& to); // Finds all files in the given folder. This is recursive. std::vector GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path);