From 864ff122d87588c8513c07971ff8e28521190972 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Sun, 21 May 2017 12:51:15 -0700 Subject: [PATCH] Scan include directories for include completion candidates. --- src/cache.cc | 8 +- src/cache.h | 8 +- src/clang_args | 2 - src/code_completion.cc | 2 +- src/code_completion.h | 6 +- src/command_line.cc | 195 +++++++++++-------------------------- src/config.h | 67 +++++++++++++ src/fuzzy.h | 12 --- src/include_completion.cc | 151 ++++++++++++++++++++++++++++ src/include_completion.h | 36 +++++++ src/indexer.cc | 2 +- src/indexer.h | 2 +- src/language_server_api.h | 58 +---------- src/{fuzzy.cc => match.cc} | 32 +++++- src/match.h | 25 +++++ src/project.cc | 63 +++--------- src/project.h | 4 +- src/test.cc | 2 +- src/utils.cc | 46 ++++++--- src/utils.h | 14 +-- 20 files changed, 446 insertions(+), 289 deletions(-) create mode 100644 src/config.h delete mode 100644 src/fuzzy.h create mode 100644 src/include_completion.cc create mode 100644 src/include_completion.h rename src/{fuzzy.cc => match.cc} (53%) create mode 100644 src/match.h diff --git a/src/cache.cc b/src/cache.cc index a5cb8230..5b12de42 100644 --- a/src/cache.cc +++ b/src/cache.cc @@ -20,8 +20,8 @@ std::string GetCachedBaseFileName(const std::string& cache_directory, } // namespace -std::unique_ptr LoadCachedIndex(IndexerConfig* config, - const std::string& filename) { +std::unique_ptr LoadCachedIndex(Config* config, + const std::string& filename) { if (!config->enableCacheRead) return nullptr; @@ -37,7 +37,7 @@ std::unique_ptr LoadCachedIndex(IndexerConfig* config, return nullptr; } -optional LoadCachedFileContents(IndexerConfig* config, +optional LoadCachedFileContents(Config* config, const std::string& filename) { if (!config->enableCacheRead) return nullopt; @@ -45,7 +45,7 @@ optional LoadCachedFileContents(IndexerConfig* config, return ReadContent(GetCachedBaseFileName(config->cacheDirectory, filename)); } -void WriteToCache(IndexerConfig* config, +void WriteToCache(Config* config, const std::string& filename, IndexFile& file, const optional& indexed_file_content) { diff --git a/src/cache.h b/src/cache.h index 89db0ef5..95aeb4ed 100644 --- a/src/cache.h +++ b/src/cache.h @@ -7,16 +7,16 @@ using std::experimental::optional; using std::experimental::nullopt; -struct IndexerConfig; +struct Config; struct IndexFile; -std::unique_ptr LoadCachedIndex(IndexerConfig* config, +std::unique_ptr LoadCachedIndex(Config* config, const std::string& filename); -optional LoadCachedFileContents(IndexerConfig* config, +optional LoadCachedFileContents(Config* config, const std::string& filename); -void WriteToCache(IndexerConfig* config, +void WriteToCache(Config* config, const std::string& filename, IndexFile& file, const optional& indexed_file_contents); \ No newline at end of file diff --git a/src/clang_args b/src/clang_args index dbe13b5b..a6841fc1 100644 --- a/src/clang_args +++ b/src/clang_args @@ -9,8 +9,6 @@ -IC:/Users/jacob/Desktop/superindex/indexer/third_party/rapidjson/include -IC:/Users/jacob/Desktop/superindex/indexer/third_party/sparsepp -IC:/Program Files/LLVM/include --IC:/Program Files (x86)/Windows Kits/10/Include/10.0.14393.0 --IC:/Program Files (x86)/Microsoft Visual Studio 14.0/VC/include # OSX diff --git a/src/code_completion.cc b/src/code_completion.cc index d703e50e..6637c543 100644 --- a/src/code_completion.cc +++ b/src/code_completion.cc @@ -369,7 +369,7 @@ CompletionSession::CompletionSession(const Project::Entry& file, WorkingFiles* w CompletionSession::~CompletionSession() {} -CompletionManager::CompletionManager(IndexerConfig* config, Project* project, WorkingFiles* working_files) +CompletionManager::CompletionManager(Config* config, Project* project, WorkingFiles* working_files) : config(config), project(project), working_files(working_files) { new std::thread([&]() { SetCurrentThreadName("completequery"); diff --git a/src/code_completion.h b/src/code_completion.h index e87a7e45..9dbf9119 100644 --- a/src/code_completion.h +++ b/src/code_completion.h @@ -10,6 +10,8 @@ #include #include +// TODO: rename this file to clang_completion.h/cc + struct CompletionSession { Project::Entry file; WorkingFiles* working_files; @@ -35,7 +37,7 @@ struct CompletionSession { struct CompletionManager { std::vector> sessions; - IndexerConfig* config; + Config* config; Project* project; WorkingFiles* working_files; @@ -51,7 +53,7 @@ struct CompletionManager { // Request that the given path be reparsed. AtomicObject reparse_request; - CompletionManager(IndexerConfig* config, Project* project, WorkingFiles* working_files); + CompletionManager(Config* config, Project* project, WorkingFiles* working_files); // Start a code completion at the given location. |on_complete| will run when // completion results are available. |on_complete| may run on any thread. diff --git a/src/command_line.cc b/src/command_line.cc index 24b07f06..58713d50 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -2,7 +2,8 @@ #include "cache.h" #include "code_completion.h" #include "file_consumer.h" -#include "fuzzy.h" +#include "match.h" +#include "include_completion.h" #include "indexer.h" #include "query.h" #include "language_server_api.h" @@ -179,63 +180,7 @@ bool ShouldDisplayIpcTiming(IpcId id) { -std::string BaseName(const std::string& path) { - int i = path.size() - 1; - while (i > 0) { - char c = path[i - 1]; - if (c == '/' || c == '\\') - break; - --i; - } - return path.substr(i); -} -int TrimCommonPathPrefix(const std::string& result, const std::string& trimmer) { - size_t i = 0; - while (i < result.size() && i < trimmer.size()) { - char a = result[i]; - char b = trimmer[i]; -#if defined(_WIN32) - a = tolower(a); - b = tolower(b); -#endif - if (a != b) - break; - ++i; - } - - if (i == trimmer.size()) - return i; - return 0; -} - - -// Returns true iff angle brackets should be used. -bool TrimPath(Project* project, const std::string& project_root, std::string& insert_path) { - int start = 0; - bool angle = false; - - int len = TrimCommonPathPrefix(insert_path, project_root); - if (len > start) - start = len; - - for (auto& include_dir : project->quote_include_directories) { - len = TrimCommonPathPrefix(insert_path, include_dir); - if (len > start) - start = len; - } - - for (auto& include_dir : project->angle_include_directories) { - len = TrimCommonPathPrefix(insert_path, include_dir); - if (len > start) { - start = len; - angle = true; - } - } - - insert_path = insert_path.substr(start); - return angle; -} bool ShouldRunIncludeCompletion(const std::string& line) { size_t start = 0; @@ -1094,7 +1039,7 @@ void RegisterMessageTypes() { -bool ImportCachedIndex(IndexerConfig* config, +bool ImportCachedIndex(Config* config, FileConsumer::SharedState* file_consumer_shared, Index_DoIdMapQueue* queue_do_id_map, const std::string& tu_path, @@ -1135,7 +1080,7 @@ bool ImportCachedIndex(IndexerConfig* config, return needs_reparse; } -void ParseFile(IndexerConfig* config, +void ParseFile(Config* config, WorkingFiles* working_files, FileConsumer::SharedState* file_consumer_shared, Index_DoIdMapQueue* queue_do_id_map, @@ -1231,7 +1176,7 @@ void ParseFile(IndexerConfig* config, } -bool ResetStaleFiles(IndexerConfig* config, +bool ResetStaleFiles(Config* config, FileConsumer::SharedState* file_consumer_shared, const std::string& tu_path) { Timer time; @@ -1263,7 +1208,7 @@ bool ResetStaleFiles(IndexerConfig* config, return needs_reparse; } -bool IndexMain_DoIndex(IndexerConfig* config, +bool IndexMain_DoIndex(Config* config, FileConsumer::SharedState* file_consumer_shared, Project* project, WorkingFiles* working_files, @@ -1327,6 +1272,7 @@ bool IndexMain_DoCreateIndexUpdate( response->previous_index.get(), response->current_index.get()); response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset(); +#if false #define PRINT_SECTION(name) \ if (response->perf.name) {\ total += response->perf.name; \ @@ -1351,6 +1297,7 @@ bool IndexMain_DoCreateIndexUpdate( if (response->is_interactive) std::cerr << "Applying IndexUpdate" << std::endl << update.ToString() << std::endl; +#endif Index_OnIndexed reply(update, response->indexed_content, response->perf); queue_on_indexed->Enqueue(std::move(reply)); @@ -1372,16 +1319,16 @@ bool IndexMergeIndexUpdates(Index_OnIndexedQueue* queue_on_indexed) { } did_merge = true; - Timer time; + //Timer time; root->update.Merge(to_join->update); for (auto&& entry : to_join->indexed_content) root->indexed_content.emplace(entry); - time.ResetAndPrint("[indexer] Joining two querydb updates"); + //time.ResetAndPrint("[indexer] Joining two querydb updates"); } } void IndexMain( - IndexerConfig* config, + Config* config, FileConsumer::SharedState* file_consumer_shared, Project* project, WorkingFiles* working_files, @@ -1470,7 +1417,7 @@ void IndexMain( bool QueryDbMainLoop( - IndexerConfig* config, + Config* config, QueryDatabase* db, MultiQueueWaiter* waiter, Index_DoIndexQueue* queue_do_index, @@ -1481,6 +1428,7 @@ bool QueryDbMainLoop( FileConsumer::SharedState* file_consumer_shared, WorkingFiles* working_files, CompletionManager* completion_manager, + IncludeCompletion* include_completion, CodeCompleteCache* code_complete_cache, CodeCompleteCache* signature_cache) { IpcManager* ipc = IpcManager::instance(); @@ -1530,18 +1478,16 @@ bool QueryDbMainLoop( // Make sure cache directory is valid. if (config->cacheDirectory.empty()) { - std::cerr << "No cache directory" << std::endl; + std::cerr << "[fatal] No cache directory" << std::endl; exit(1); } config->cacheDirectory = NormalizePath(config->cacheDirectory); - if (config->cacheDirectory[config->cacheDirectory.size() - 1] != '/') - config->cacheDirectory += '/'; + EnsureEndsInSlash(config->cacheDirectory); MakeDirectoryRecursive(config->cacheDirectory); // Set project root. config->projectRoot = NormalizePath(request->params.rootUri->GetPath()); - if (config->projectRoot.empty() || config->projectRoot[config->projectRoot.size() - 1] != '/') - config->projectRoot += '/'; + EnsureEndsInSlash(config->projectRoot); // Start indexer threads. int indexer_count = std::max(std::thread::hardware_concurrency(), 2) - 1; @@ -1554,17 +1500,23 @@ bool QueryDbMainLoop( }); } + Timer time; + // Open up / load the project. project->Load(config->extraClangArguments, project_path); - std::cerr << "Loaded compilation entries (" << project->entries.size() << " files)" << std::endl; + time.ResetAndPrint("[perf] Loaded compilation entries (" + std::to_string(project->entries.size()) + " files)"); + // Start scanning include directories before dispatching project files, because that takes a long time. + include_completion->Rescan(); + + time.Reset(); project->ForAllFilteredFiles(config, [&](int i, const Project::Entry& entry) { - std::cerr << "[" << i << "/" << (project->entries.size() - 1) - << "] Dispatching index request for file " << entry.filename - << std::endl; - + //std::cerr << "[" << i << "/" << (project->entries.size() - 1) + // << "] Dispatching index request for file " << entry.filename + // << std::endl; queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::ImportThenParse, entry, nullopt, false /*is_interactive*/)); }); + time.ResetAndPrint("[perf] Dispatched initial index requests"); } // TODO: query request->params.capabilities.textDocument and support only things @@ -1768,6 +1720,8 @@ bool QueryDbMainLoop( time.ResetAndPrint("[querydb] Loading cached index file for DidOpen (blocks CodeLens)"); + include_completion->AddFile(working_file->filename); + break; } @@ -1847,66 +1801,26 @@ bool QueryDbMainLoop( Out_TextDocumentComplete complete_response; complete_response.id = msg->id; complete_response.result.isIncomplete = false; + + { + std::unique_lock lock(include_completion->completion_items_mutex, std::defer_lock); + if (include_completion->is_scanning) + lock.lock(); + complete_response.result.items.assign( + include_completion->completion_items.begin(), + include_completion->completion_items.end()); + if (lock) + lock.unlock(); - for (const char* stl_header : kStandardLibraryIncludes) { - lsCompletionItem item; - item.label = "#include <" + std::string(stl_header) + ">"; - item.insertTextFormat = lsInsertTextFormat::PlainText; - item.kind = lsCompletionItemKind::Module; - - // Replace the entire existing content. - item.textEdit = lsTextEdit(); - item.textEdit->range.start.line = params.position.line; - item.textEdit->range.start.character = 0; - item.textEdit->range.end.line = params.position.line; - item.textEdit->range.end.character = buffer_line.size(); - item.textEdit->newText = item.label; - - complete_response.result.items.push_back(item); - } - - for (optional& file : db->files) { - if (!file) - continue; - - // TODO: codify list of file extensions somewhere - if (EndsWith(file->def.path, ".c") || - EndsWith(file->def.path, ".cc") || - EndsWith(file->def.path, ".cpp")) - continue; - - lsCompletionItem item; - - // Standard library headers are handled differently. - std::string base_name = BaseName(file->def.path); - if (std::find(std::begin(kStandardLibraryIncludes), std::end(kStandardLibraryIncludes), base_name) != std::end(kStandardLibraryIncludes)) - continue; - - // Trim the path so it is relative to an include directory. - std::string insert_path = file->def.path; - std::string start_quote = "\""; - std::string end_quote = "\""; - if (TrimPath(project, config->projectRoot, insert_path)) { - start_quote = "<"; - end_quote = ">"; + // Update textEdit params. + for (lsCompletionItem& item : complete_response.result.items) { + item.textEdit->range.start.line = params.position.line; + item.textEdit->range.start.character = 0; + item.textEdit->range.end.line = params.position.line; + item.textEdit->range.end.character = (int)buffer_line.size(); } - - item.label = "#include " + start_quote + insert_path + end_quote; - item.insertTextFormat = lsInsertTextFormat::PlainText; - item.kind = lsCompletionItemKind::File; - - // Replace the entire existing content. - item.textEdit = lsTextEdit(); - item.textEdit->range.start.line = params.position.line; - item.textEdit->range.start.character = 0; - item.textEdit->range.end.line = params.position.line; - item.textEdit->range.end.character = buffer_line.size(); - item.textEdit->newText = item.label; - - complete_response.result.items.push_back(item); } - complete_response.Write(std::cerr); ipc->SendOutMessageToClient(IpcId::TextDocumentCompletion, complete_response); } else { @@ -2445,6 +2359,7 @@ bool QueryDbMainLoop( } case IpcId::WorkspaceSymbol: { + // TODO: implement fuzzy search, see https://github.com/junegunn/fzf/blob/master/src/matcher.go for inspiration auto msg = static_cast(message.get()); Out_WorkspaceSymbol response; @@ -2559,7 +2474,7 @@ bool QueryDbMainLoop( return did_work; } -void QueryDbMain(IndexerConfig* config, MultiQueueWaiter* waiter) { +void QueryDbMain(Config* config, MultiQueueWaiter* waiter) { // Create queues. Index_DoIndexQueue queue_do_index(waiter); Index_DoIdMapQueue queue_do_id_map(waiter); @@ -2569,6 +2484,7 @@ void QueryDbMain(IndexerConfig* config, MultiQueueWaiter* waiter) { Project project; WorkingFiles working_files; CompletionManager completion_manager(config, &project, &working_files); + IncludeCompletion include_completion(config, &project); CodeCompleteCache code_complete_cache; CodeCompleteCache signature_cache; FileConsumer::SharedState file_consumer_shared; @@ -2577,7 +2493,10 @@ void QueryDbMain(IndexerConfig* config, MultiQueueWaiter* waiter) { SetCurrentThreadName("querydb"); QueryDatabase db; while (true) { - bool did_work = QueryDbMainLoop(config, &db, waiter, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed, &project, &file_consumer_shared, &working_files, &completion_manager, &code_complete_cache, &signature_cache); + bool did_work = QueryDbMainLoop( + config, &db, waiter, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed, + &project, &file_consumer_shared, &working_files, + &completion_manager, &include_completion, &code_complete_cache, &signature_cache); if (!did_work) { IpcManager* ipc = IpcManager::instance(); waiter->Wait({ @@ -2659,7 +2578,7 @@ void QueryDbMain(IndexerConfig* config, MultiQueueWaiter* waiter) { // blocks. // // |ipc| is connected to a server. -void LanguageServerStdinLoop(IndexerConfig* config, std::unordered_map* request_times) { +void LanguageServerStdinLoop(Config* config, std::unordered_map* request_times) { IpcManager* ipc = IpcManager::instance(); SetCurrentThreadName("stdin"); @@ -2801,7 +2720,7 @@ void StdoutMain(std::unordered_map* request_times, MultiQueueWaite } } -void LanguageServerMain(IndexerConfig* config, MultiQueueWaiter* waiter) { +void LanguageServerMain(Config* config, MultiQueueWaiter* waiter) { std::unordered_map request_times; // Start stdin reader. Reading from stdin is a blocking operation so this @@ -2899,7 +2818,7 @@ int main(int argc, char** argv) { } else if (HasOption(options, "--language-server")) { //std::cerr << "Running language server" << std::endl; - IndexerConfig config; + Config config; LanguageServerMain(&config, &waiter); return 0; } @@ -2924,8 +2843,8 @@ int main(int argc, char** argv) { There are also a number of configuration options available when initializing the language server - your editor should have tooling to - describe those options. See |IndexerConfig| in this source code for a - detailed list of all currently supported options. + describe those options. See |Config| in this source code for a detailed + list of all currently supported options. )help"; return 0; } diff --git a/src/config.h b/src/config.h new file mode 100644 index 00000000..9e72a669 --- /dev/null +++ b/src/config.h @@ -0,0 +1,67 @@ +#pragma once + +#include "serializer.h" + +#include + +struct Config { + // Root directory of the project. **Not serialized** + std::string projectRoot; + + std::string cacheDirectory; + std::vector indexWhitelist; + std::vector indexBlacklist; + std::vector extraClangArguments; + + // Maximum workspace search results. + int maxWorkspaceSearchResults = 1000; + + // Force a certain number of indexer threads. If less than 1 a default value + // should be used. + int indexerCount = 0; + // If false, the indexer will be disabled. + bool enableIndexing = true; + // If false, indexed files will not be written to disk. + bool enableCacheWrite = true; + // If false, the index will not be loaded from a previous run. + bool enableCacheRead = true; + + // If true, document links are reported for #include directives. + bool showDocumentLinksOnIncludes = true; + + // Whitelist that file paths will be tested against. If a file path does not + // end in one of these values, it will not be considered for auto-completion. + // An example value is { ".h", ".hpp" } + // + // This is significantly faster than using a regex. + std::vector includeCompletionWhitelistLiteralEnding; + // Regex patterns to match include completion candidates against. They + // receive the absolute file path. + // + // For example, to hide all files in a /CACHE/ folder, use ".*/CACHE/.*" + std::vector includeCompletionWhitelist; + std::vector includeCompletionBlacklist; + + // Enables code lens on parameter and function variables. + bool codeLensOnLocalVariables = true; + + // Version of the client. + int clientVersion = 0; +}; +MAKE_REFLECT_STRUCT(Config, + cacheDirectory, + indexWhitelist, indexBlacklist, + extraClangArguments, + + maxWorkspaceSearchResults, + indexerCount, + enableIndexing, enableCacheWrite, enableCacheRead, + + includeCompletionWhitelistLiteralEnding, + includeCompletionWhitelist, includeCompletionBlacklist, + + showDocumentLinksOnIncludes, + + codeLensOnLocalVariables, + + clientVersion); \ No newline at end of file diff --git a/src/fuzzy.h b/src/fuzzy.h deleted file mode 100644 index 8d1d1870..00000000 --- a/src/fuzzy.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include - -struct Matcher { - Matcher(const std::string& search); - - bool IsMatch(const std::string& value) const; - - std::string regex_string; - std::regex regex; -}; diff --git a/src/include_completion.cc b/src/include_completion.cc new file mode 100644 index 00000000..72adaf89 --- /dev/null +++ b/src/include_completion.cc @@ -0,0 +1,151 @@ +#pragma once + +#include "include_completion.h" + +#include "match.h" +#include "platform.h" +#include "project.h" +#include "standard_includes.h" +#include "timer.h" + +namespace { + +size_t TrimCommonPathPrefix(const std::string& result, const std::string& trimmer) { + size_t i = 0; + while (i < result.size() && i < trimmer.size()) { + char a = result[i]; + char b = trimmer[i]; +#if defined(_WIN32) + a = tolower(a); + b = tolower(b); +#endif + if (a != b) + break; + ++i; + } + + if (i == trimmer.size()) + return i; + return 0; +} + +// Returns true iff angle brackets should be used. +bool TrimPath(Project* project, const std::string& project_root, std::string& insert_path) { + size_t start = 0; + bool angle = false; + + size_t len = TrimCommonPathPrefix(insert_path, project_root); + if (len > start) + start = len; + + for (auto& include_dir : project->quote_include_directories) { + len = TrimCommonPathPrefix(insert_path, include_dir); + if (len > start) + start = len; + } + + for (auto& include_dir : project->angle_include_directories) { + len = TrimCommonPathPrefix(insert_path, include_dir); + if (len > start) { + start = len; + angle = true; + } + } + + insert_path = insert_path.substr(start); + return angle; +} + +lsCompletionItem BuildCompletionItem(std::string path, bool use_angle_brackets, bool is_stl) { + lsCompletionItem item; + if (use_angle_brackets) + item.label = "#include <" + std::string(path) + ">"; + else + item.label = "#include \"" + std::string(path) + "\""; + + item.insertTextFormat = lsInsertTextFormat::PlainText; + if (is_stl) + item.kind = lsCompletionItemKind::Module; + else + item.kind = lsCompletionItemKind::File; + + // Replace the entire existing content. + // NOTE: When submitting completion items, textEdit->range must be updated. + item.textEdit = lsTextEdit(); + item.textEdit->newText = item.label; + + return item; +} + +} // namespace + +IncludeCompletion::IncludeCompletion(Config* config, Project* project) + : config_(config), project_(project), is_scanning(false) {} + +void IncludeCompletion::Rescan() { + if (is_scanning) + return; + + completion_items.clear(); + + if (!match_ && (!config_->includeCompletionWhitelist.empty() || !config_->includeCompletionBlacklist.empty())) + match_ = MakeUnique(config_->includeCompletionWhitelist, config_->includeCompletionBlacklist); + + is_scanning = true; + new std::thread([this]() { + SetCurrentThreadName("include_scanner"); + Timer timer; + + InsertStlIncludes(); + InsertIncludesFromDirectory(config_->projectRoot, false /*use_angle_brackets*/); + for (const std::string& dir : project_->quote_include_directories) + InsertIncludesFromDirectory(dir, false /*use_angle_brackets*/); + for (const std::string& dir : project_->angle_include_directories) + InsertIncludesFromDirectory(dir, true /*use_angle_brackets*/); + + timer.ResetAndPrint("[perf] Scanning for includes"); + is_scanning = false; + }); +} + +void IncludeCompletion::AddFile(std::string path) { + if (!EndsWithAny(path, config_->includeCompletionWhitelistLiteralEnding)) + return; + if (match_ && !match_->IsMatch(path)) + return; + + bool use_angle_brackets = TrimPath(project_, config_->projectRoot, path); + lsCompletionItem item = BuildCompletionItem(path, use_angle_brackets, false /*is_stl*/); + + if (is_scanning) { + std::lock_guard lock(completion_items_mutex); + completion_items.insert(item); + } + else { + completion_items.insert(item); + } +} + +void IncludeCompletion::InsertIncludesFromDirectory( + const std::string& directory, + bool use_angle_brackets) { + std::vector result; + GetFilesInFolder(directory, true /*recursive*/, false /*add_folder_to_path*/, [&](const std::string& path) { + if (!EndsWithAny(path, config_->includeCompletionWhitelistLiteralEnding)) + return; + if (match_ && !match_->IsMatch(directory + path)) + return; + + result.push_back(BuildCompletionItem(path, use_angle_brackets, false /*is_stl*/)); + }); + + std::lock_guard lock(completion_items_mutex); + completion_items.insert(result.begin(), result.end()); +} + +void IncludeCompletion::InsertStlIncludes() { + std::lock_guard lock(completion_items_mutex); + for (const char* stl_header : kStandardLibraryIncludes) { + completion_items.insert(BuildCompletionItem(stl_header, true /*use_angle_brackets*/, true /*is_stl*/)); + } +} diff --git a/src/include_completion.h b/src/include_completion.h new file mode 100644 index 00000000..9eb45f1e --- /dev/null +++ b/src/include_completion.h @@ -0,0 +1,36 @@ +#pragma once + +#include "language_server_api.h" + +#include +#include +#include + +struct GroupMatch; +struct Project; + +struct IncludeCompletion { + IncludeCompletion(Config* config, Project* project); + + // Starts scanning directories. Clears existing cache. + void Rescan(); + + // Ensures the one-off file is inside |completion_items|. + void AddFile(std::string path); + + // Scans the given directory and inserts all includes from this. This is a + // blocking function and should be run off the querydb thread. + void InsertIncludesFromDirectory(const std::string& directory, bool use_angle_brackets); + void InsertStlIncludes(); + + // Guards |completion_items| when |is_scanning| is true. + std::mutex completion_items_mutex; + std::atomic is_scanning; + std::unordered_set completion_items; + + // Cached references + Config* config_; + Project* project_; + std::unique_ptr match_; +}; + diff --git a/src/indexer.cc b/src/indexer.cc index 1b8fccf5..26cc366d 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -1471,7 +1471,7 @@ void indexEntityReference(CXClientData client_data, std::vector> Parse( - IndexerConfig* config, FileConsumer::SharedState* file_consumer_shared, + Config* config, FileConsumer::SharedState* file_consumer_shared, std::string file, std::vector args, const std::string& file_contents_path, diff --git a/src/indexer.h b/src/indexer.h index 312d83e4..25d01c60 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -550,7 +550,7 @@ struct IndexFile { // |desired_index_file| is the (h or cc) file which has actually changed. // |dependencies| are the existing dependencies of |import_file| if this is a reparse. std::vector> Parse( - IndexerConfig* config, FileConsumer::SharedState* file_consumer_shared, + Config* config, FileConsumer::SharedState* file_consumer_shared, std::string file, std::vector args, const std::string& file_contents_path, diff --git a/src/language_server_api.h b/src/language_server_api.h index 8fb41822..19e9e7b1 100644 --- a/src/language_server_api.h +++ b/src/language_server_api.h @@ -1,5 +1,6 @@ #pragma once +#include "config.h" #include "ipc.h" #include "serializer.h" #include "utils.h" @@ -50,57 +51,6 @@ void Reflect(Reader& visitor, lsRequestId& id); - - - - - -struct IndexerConfig { - // Root directory of the project. **Not serialized** - std::string projectRoot; - - std::string cacheDirectory; - NonElidedVector whitelist; - NonElidedVector blacklist; - std::vector extraClangArguments; - - // Maximum workspace search results. - int maxWorkspaceSearchResults = 1000; - - // Force a certain number of indexer threads. If less than 1 a default value - // should be used. - int indexerCount = 0; - // If false, the indexer will be disabled. - bool enableIndexing = true; - // If false, indexed files will not be written to disk. - bool enableCacheWrite = true; - // If false, the index will not be loaded from a previous run. - bool enableCacheRead = true; - - // If true, document links are reported for #include directives. - bool showDocumentLinksOnIncludes = true; - - // Enables code lens on parameter and function variables. - bool codeLensOnLocalVariables = true; - - // Version of the client. - int clientVersion = 0; -}; -MAKE_REFLECT_STRUCT(IndexerConfig, - cacheDirectory, - whitelist, blacklist, - extraClangArguments, - - maxWorkspaceSearchResults, - indexerCount, - enableIndexing, enableCacheWrite, enableCacheRead, - - showDocumentLinksOnIncludes, - - codeLensOnLocalVariables, - - clientVersion); - @@ -510,6 +460,8 @@ struct lsCompletionItem { // An data entry field that is preserved on a completion item between // a completion and a completion resolve request. // data ? : any + + inline bool operator==(const lsCompletionItem& other) const { return label == other.label; } }; MAKE_REFLECT_STRUCT(lsCompletionItem, label, @@ -520,7 +472,7 @@ MAKE_REFLECT_STRUCT(lsCompletionItem, insertText, insertTextFormat, textEdit); - +MAKE_HASHABLE(lsCompletionItem, t.label); struct lsTextDocumentItem { // The text document's URI. @@ -909,7 +861,7 @@ struct lsInitializeParams { optional rootUri; // User provided initialization options. - optional initializationOptions; + optional initializationOptions; // The capabilities provided by the client (editor or tool) lsClientCapabilities capabilities; diff --git a/src/fuzzy.cc b/src/match.cc similarity index 53% rename from src/fuzzy.cc rename to src/match.cc index 04dc022c..066e1bb8 100644 --- a/src/fuzzy.cc +++ b/src/match.cc @@ -1,4 +1,4 @@ -#include "fuzzy.h" +#include "match.h" #include @@ -28,6 +28,36 @@ bool Matcher::IsMatch(const std::string& value) const { return std::regex_match(value, regex, std::regex_constants::match_any); } +GroupMatch::GroupMatch( + const std::vector& whitelist, + const std::vector& blacklist) { + for (const std::string& entry : whitelist) + this->whitelist.push_back(Matcher(entry)); + for (const std::string& entry : blacklist) + this->blacklist.push_back(Matcher(entry)); +} + +bool GroupMatch::IsMatch(const std::string& value, std::string* match_failure_reason) const { + for (const Matcher& m : whitelist) { + if (!m.IsMatch(value)) { + if (match_failure_reason) + *match_failure_reason = "whitelist \"" + m.regex_string + "\""; + return false; + } + } + + for (const Matcher& m : blacklist) { + if (m.IsMatch(value)) { + if (match_failure_reason) + *match_failure_reason = "blacklist \"" + m.regex_string + "\""; + return false; + } + } + + return true; +} + + TEST_SUITE("Matcher"); TEST_CASE("sanity") { diff --git a/src/match.h b/src/match.h new file mode 100644 index 00000000..235e1e5a --- /dev/null +++ b/src/match.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include + +struct Matcher { + Matcher(const std::string& search); + + bool IsMatch(const std::string& value) const; + + std::string regex_string; + std::regex regex; +}; + +// Check multiple |Matcher| instances at the same time. +struct GroupMatch { + GroupMatch(const std::vector& whitelist, + const std::vector& blacklist); + + bool IsMatch(const std::string& value, std::string* match_failure_reason = nullptr) const; + + std::vector whitelist; + std::vector blacklist; +}; \ No newline at end of file diff --git a/src/project.cc b/src/project.cc index 75ca5827..d565eeba 100644 --- a/src/project.cc +++ b/src/project.cc @@ -1,6 +1,6 @@ #include "project.h" -#include "fuzzy.h" +#include "match.h" #include "libclangmm/Utility.h" #include "platform.h" #include "serializer.h" @@ -10,6 +10,8 @@ #include #include +#include +#include #include struct CompileCommandsEntry { @@ -187,9 +189,9 @@ Project::Entry GetCompilationEntryFromCompileCommandEntry( // Clang does not have good hueristics for determining source language. We // default to C++11 if the user has not specified. - if (!StartsWithAny(result.args, "-x")) + if (!AnyStartsWith(result.args, "-x")) result.args.push_back("-xc++"); - if (!StartsWithAny(result.args, "-std=")) + if (!AnyStartsWith(result.args, "-std=")) result.args.push_back("-std=c++11"); return result; @@ -321,11 +323,6 @@ int ComputeGuessScore(const std::string& a, const std::string& b) { return score; } -void EnsureEndsInSlash(std::string& path) { - if (path.empty() || path[path.size() - 1] != '/') - path += '/'; -} - } // namespace void Project::Load(const std::vector& extra_flags, const std::string& directory) { @@ -376,50 +373,18 @@ Project::Entry Project::FindCompilationEntryForFile(const std::string& filename) return result; } -void Project::ForAllFilteredFiles(IndexerConfig* config, std::function action) { - std::vector whitelist; - std::cerr << "Using whitelist" << std::endl; - for (const std::string& entry : config->whitelist) { - std::cerr << " - " << entry << std::endl; - whitelist.push_back(Matcher(entry)); - } - - std::vector blacklist; - std::cerr << "Using blacklist" << std::endl; - for (const std::string& entry : config->blacklist) { - std::cerr << " - " << entry << std::endl; - blacklist.push_back(Matcher(entry)); - } - - +void Project::ForAllFilteredFiles(Config* config, std::function action) { + GroupMatch matcher(config->indexWhitelist, config->indexBlacklist); for (int i = 0; i < entries.size(); ++i) { const Project::Entry& entry = entries[i]; - std::string filepath = entry.filename; - - const Matcher* is_bad = nullptr; - for (const Matcher& m : whitelist) { - if (!m.IsMatch(filepath)) { - is_bad = &m; - break; - } + std::string failure_reason; + if (matcher.IsMatch(entry.filename, &failure_reason)) + action(i, entries[i]); + else { + std::stringstream output; + output << '[' << (i + 1) << '/' << entries.size() << "] Failed " << failure_reason << "; skipping " << entry.filename << std::endl; + std::cerr << output.str(); } - if (is_bad) { - std::cerr << "[" << i << "/" << (entries.size() - 1) << "] Failed whitelist check \"" << is_bad->regex_string << "\"; skipping " << filepath << std::endl; - continue; - } - - for (const Matcher& m : blacklist) { - if (m.IsMatch(filepath)) { - is_bad = &m; - break; - } - } - if (is_bad) { - std::cerr << "[" << i << "/" << (entries.size() - 1) << "] Failed blacklist check \"" << is_bad->regex_string << "\"; skipping " << filepath << std::endl; - continue; - } - - action(i, entries[i]); } } diff --git a/src/project.h b/src/project.h index 4a9306b4..3d89f724 100644 --- a/src/project.h +++ b/src/project.h @@ -1,6 +1,6 @@ #pragma once -#include "language_server_api.h" +#include "config.h" #include #include @@ -41,6 +41,6 @@ struct Project { // will infer one based on existing project structure. Entry FindCompilationEntryForFile(const std::string& filename); - void ForAllFilteredFiles(IndexerConfig* config, std::function action); + void ForAllFilteredFiles(Config* config, std::function action); }; diff --git a/src/test.cc b/src/test.cc index 345e6af3..f8a45543 100644 --- a/src/test.cc +++ b/src/test.cc @@ -114,7 +114,7 @@ void RunTests() { // Parse expected output from the test, parse it into JSON document. std::unordered_map all_expected_output = ParseTestExpectation(path); - IndexerConfig config; + Config config; FileConsumer::SharedState file_consumer_shared; // Run test. diff --git a/src/utils.cc b/src/utils.cc index 79a4b7c2..c2ae783a 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -46,6 +46,18 @@ bool StartsWith(const std::string& value, const std::string& start) { return std::equal(start.begin(), start.end(), value.begin()); } +bool AnyStartsWith(const std::vector& values, const std::string& start) { + return std::any_of(std::begin(values), std::end(values), [&start](const std::string& value) { + return StartsWith(value, start); + }); +} + +bool EndsWithAny(const std::string& value, const std::vector& endings) { + return std::any_of(std::begin(endings), std::end(endings), [&value](const std::string& ending) { + return EndsWith(value, ending); + }); +} + // See http://stackoverflow.com/a/29752943 std::string ReplaceAll(const std::string& source, const std::string& from, const std::string& to) { std::string result; @@ -83,9 +95,8 @@ std::vector SplitString(const std::string& str, const std::string& return strings; } -static std::vector GetFilesInFolderHelper(std::string folder, bool recursive, std::string output_prefix) { - std::vector result; - +static void GetFilesInFolderHelper( + std::string folder, bool recursive, std::string output_prefix, const std::function& handler) { tinydir_dir dir; if (tinydir_open(&dir, folder.c_str()) == -1) { perror("Error opening file"); @@ -105,12 +116,11 @@ static std::vector GetFilesInFolderHelper(std::string folder, bool if (recursive) { // Note that we must always ignore the '.' and '..' directories, otherwise // this will loop infinitely. The above check handles that for us. - for (std::string nested_file : GetFilesInFolderHelper(file.path, true /*recursive*/, output_prefix + file.name + "/")) - result.push_back(nested_file); + GetFilesInFolderHelper(file.path, true /*recursive*/, output_prefix + file.name + "/", handler); } } else { - result.push_back(output_prefix + file.name); + handler(output_prefix + file.name); } } @@ -122,17 +132,29 @@ static std::vector GetFilesInFolderHelper(std::string folder, bool bail: tinydir_close(&dir); - return result; } std::vector GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path) { - assert(folder.size() > 0); - if (folder[folder.size() - 1] != '/') - folder += '/'; - - return GetFilesInFolderHelper(folder, recursive, add_folder_to_path ? folder : ""); + EnsureEndsInSlash(folder); + std::vector result; + GetFilesInFolderHelper(folder, recursive, add_folder_to_path ? folder : "", [&result](const std::string& path) { + result.push_back(path); + }); + return result; } + +void GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path, const std::function& handler) { + EnsureEndsInSlash(folder); + GetFilesInFolderHelper(folder, recursive, add_folder_to_path ? folder : "", handler); +} + +void EnsureEndsInSlash(std::string& path) { + if (path.empty() || path[path.size() - 1] != '/') + path += '/'; +} + + // http://stackoverflow.com/a/6089413 std::istream& SafeGetline(std::istream& is, std::string& t) { t.clear(); diff --git a/src/utils.h b/src/utils.h index c276558b..70150991 100644 --- a/src/utils.h +++ b/src/utils.h @@ -16,16 +16,13 @@ using std::experimental::nullopt; // Returns true if |value| starts/ends with |start| or |ending|. bool StartsWith(const std::string& value, const std::string& start); bool EndsWith(const std::string& value, const std::string& ending); +bool AnyStartsWith(const std::vector& values, const std::string& start); +bool EndsWithAny(const std::string& value, const std::vector& endings); + std::string ReplaceAll(const std::string& source, const std::string& from, const std::string& to); std::vector SplitString(const std::string& str, const std::string& delimiter); -inline bool StartsWithAny(const std::vector& values, const std::string& start) { - return std::any_of(std::begin(values), std::end(values), [&start](const std::string& value) { - return StartsWith(value, start); - }); -} - template std::string StringJoin(const TValues& values) { std::string result; @@ -41,6 +38,11 @@ std::string StringJoin(const TValues& values) { // Finds all files in the given folder. This is recursive. std::vector GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path); +void GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path, const std::function& handler); + +// Ensures that |path| ends in a slash. +void EnsureEndsInSlash(std::string& path); + optional ReadContent(const std::string& filename); std::vector ReadLines(std::string filename); std::vector ToLines(const std::string& content, bool trim_whitespace);