From 569a7624a65ba5df50bd6f06f3417b2783728c01 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Wed, 19 Sep 2018 09:31:45 -0700 Subject: [PATCH] intern args --- index_tests/types/anonymous_struct.cc | 4 +- src/clang_tu.cc | 9 ++-- src/clang_tu.h | 2 +- src/indexer.cc | 2 +- src/indexer.h | 4 +- src/messages/textDocument_didOpen.cc | 11 +++-- src/pipeline.cc | 17 +++---- src/pipeline.hh | 2 +- src/project.cc | 68 +++++++++++++-------------- src/project.h | 6 +-- src/query.h | 2 +- src/serializer.cc | 9 +++- src/test.cc | 11 +++-- 13 files changed, 76 insertions(+), 71 deletions(-) diff --git a/index_tests/types/anonymous_struct.cc b/index_tests/types/anonymous_struct.cc index b3dbecc3..9d88a8bf 100644 --- a/index_tests/types/anonymous_struct.cc +++ b/index_tests/types/anonymous_struct.cc @@ -26,9 +26,9 @@ OUTPUT: "uses": [] }, { "usr": 1428566502523368801, - "detailed_name": "struct {}", + "detailed_name": "anon struct", "qual_name_offset": 0, - "short_name": "", + "short_name": "anon struct", "kind": 23, "declarations": [], "spell": "2:3-2:9|17937907487590875128|2|1026|-1", diff --git a/src/clang_tu.cc b/src/clang_tu.cc index e32547b8..2986544e 100644 --- a/src/clang_tu.cc +++ b/src/clang_tu.cc @@ -54,17 +54,14 @@ Range FromTokenRange(const SourceManager &SM, const LangOptions &LangOpts, } std::unique_ptr -BuildCompilerInvocation(const std::vector &args, +BuildCompilerInvocation(std::vector args, IntrusiveRefCntPtr VFS) { std::string save = "-resource-dir=" + g_config->clang.resourceDir; - std::vector cargs; - for (auto &arg : args) - cargs.push_back(arg.c_str()); - cargs.push_back(save.c_str()); + args.push_back(save.c_str()); IntrusiveRefCntPtr Diags( CompilerInstance::createDiagnostics(new DiagnosticOptions)); std::unique_ptr CI = - createInvocationFromCommandLine(cargs, Diags, VFS); + createInvocationFromCommandLine(args, Diags, VFS); if (CI) { CI->getDiagnosticOpts().IgnoreWarnings = true; CI->getFrontendOpts().DisableFree = false; diff --git a/src/clang_tu.h b/src/clang_tu.h index d7363ae4..8e104e18 100644 --- a/src/clang_tu.h +++ b/src/clang_tu.h @@ -24,5 +24,5 @@ Range FromTokenRange(const clang::SourceManager &SM, llvm::sys::fs::UniqueID *UniqueID = nullptr); std::unique_ptr -BuildCompilerInvocation(const std::vector &args, +BuildCompilerInvocation(std::vector args, llvm::IntrusiveRefCntPtr VFS); diff --git a/src/indexer.cc b/src/indexer.cc index 34e28744..61209256 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -1221,7 +1221,7 @@ void Init() { std::vector> Index(CompletionManager *completion, WorkingFiles *wfiles, VFS *vfs, const std::string &opt_wdir, const std::string &file, - const std::vector &args, + const std::vector &args, const std::vector> &remapped) { if (!g_config->index.enabled) return {}; diff --git a/src/indexer.h b/src/indexer.h index f919327f..f2f74070 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -235,7 +235,7 @@ struct IndexFile { llvm::sys::fs::UniqueID UniqueID; std::string path; - std::vector args; + std::vector args; // This is unfortunately time_t as used by clang::FileEntry int64_t mtime = 0; LanguageId language = LanguageId::C; @@ -281,6 +281,6 @@ void Init(); std::vector> Index(CompletionManager *complete, WorkingFiles *wfiles, VFS *vfs, const std::string &opt_wdir, const std::string &file, - const std::vector &args, + const std::vector &args, const std::vector> &remapped); } diff --git a/src/messages/textDocument_didOpen.cc b/src/messages/textDocument_didOpen.cc index 721b2371..6bbb9114 100644 --- a/src/messages/textDocument_didOpen.cc +++ b/src/messages/textDocument_didOpen.cc @@ -52,14 +52,15 @@ struct Handler_TextDocumentDidOpen } include_complete->AddFile(working_file->filename); - if (params.args.size()) - project->SetFlagsForFile(params.args, path); + std::vector args; + for (const std::string &arg : params.args) + args.push_back(Intern(arg)); + if (args.size()) + project->SetArgsForFile(args, path); // Submit new index request if it is not a header file. if (SourceFileLanguage(path) != LanguageId::Unknown) { - pipeline::Index( - path, params.args.size() ? params.args : std::vector{}, - IndexMode::Normal); + pipeline::Index(path, args, IndexMode::Normal); clang_complete->FlushSession(path); } diff --git a/src/pipeline.cc b/src/pipeline.cc index 45c0f6bd..0c5ed435 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -69,7 +69,7 @@ namespace { struct Index_Request { std::string path; - std::vector args; + std::vector args; IndexMode mode; lsRequestId id; int64_t ts = tick++; @@ -96,7 +96,7 @@ std::shared_mutex g_index_mutex; std::unordered_map g_index; bool CacheInvalid(VFS *vfs, IndexFile *prev, const std::string &path, - const std::vector &args, + const std::vector &args, const std::optional &from) { { std::lock_guard lock(vfs->mutex); @@ -107,13 +107,14 @@ bool CacheInvalid(VFS *vfs, IndexFile *prev, const std::string &path, } } - if (prev->args != args) { + bool changed = prev->args.size() != args.size(); + for (size_t i = 0; !changed && i < args.size(); i++) + if (strcmp(prev->args[i], args[i])) + changed = true; + if (changed) LOG_S(INFO) << "args changed for " << path << (from ? " (via " + *from + ")" : std::string()); - return true; - } - - return false; + return changed; }; std::string AppendSerializationFormat(const std::string &base) { @@ -525,7 +526,7 @@ void MainLoop() { } } -void Index(const std::string &path, const std::vector &args, +void Index(const std::string &path, const std::vector &args, IndexMode mode, lsRequestId id) { index_request->PushBack({path, args, mode, id}, mode != IndexMode::NonInteractive); } diff --git a/src/pipeline.hh b/src/pipeline.hh index 7ff74533..422bc4a9 100644 --- a/src/pipeline.hh +++ b/src/pipeline.hh @@ -46,7 +46,7 @@ void Indexer_Main(CompletionManager *completion, VFS *vfs, Project *project, WorkingFiles *wfiles); void MainLoop(); -void Index(const std::string &path, const std::vector &args, +void Index(const std::string &path, const std::vector &args, IndexMode mode, lsRequestId id = {}); std::optional LastWriteTime(const std::string &path); diff --git a/src/project.cc b/src/project.cc index dac67002..90363338 100644 --- a/src/project.cc +++ b/src/project.cc @@ -43,7 +43,6 @@ enum class ProjectMode { CompileCommandsJson, DotCcls, ExternalCommand }; struct ProjectConfig { std::unordered_set quote_dirs; std::unordered_set angle_dirs; - std::vector extra_flags; std::string project_dir; ProjectMode mode = ProjectMode::CompileCommandsJson; }; @@ -73,17 +72,17 @@ struct ProjectProcessor { const std::string base_name = sys::path::filename(entry.filename); // Expand %c %cpp %clang - std::vector args; - args.reserve(entry.args.size() + config->extra_flags.size() + 3); + std::vector args; + args.reserve(entry.args.size() + g_config->clang.extraArgs.size() + 1); const LanguageId lang = SourceFileLanguage(entry.filename); - for (const std::string &arg : entry.args) { - if (arg.compare(0, 3, "%c ") == 0) { + for (const char *arg : entry.args) { + if (strncmp(arg, "%c ", 3) == 0) { if (lang == LanguageId::C) - args.push_back(arg.substr(3)); - } else if (arg.compare(0, 5, "%cpp ") == 0) { + args.push_back(arg + 3); + } else if (strncmp(arg, "%cpp ", 5) == 0) { if (lang == LanguageId::Cpp) - args.push_back(arg.substr(5)); - } else if (arg == "%clang") { + args.push_back(arg + 5); + } else if (strcmp(arg, "%clang") == 0) { args.push_back(lang == LanguageId::Cpp ? "clang++" : "clang"); } else if (!llvm::is_contained(g_config->clang.excludeArgs, arg)) { args.push_back(arg); @@ -91,8 +90,8 @@ struct ProjectProcessor { } if (args.empty()) return; - args.insert(args.end(), config->extra_flags.begin(), - config->extra_flags.end()); + for (const std::string &arg : g_config->clang.extraArgs) + args.push_back(Intern(arg)); size_t hash = std::hash{}(entry.directory); for (auto &arg : args) { @@ -105,7 +104,7 @@ struct ProjectProcessor { } hash_combine(hash, std::hash{}(arg)); } - args.push_back("-working-directory=" + entry.directory); + args.push_back(Intern("-working-directory=" + entry.directory)); if (!command_set.insert(hash).second) { entry.args = std::move(args); @@ -129,13 +128,9 @@ struct ProjectProcessor { } Driver.setCheckInputsExist(false); - std::vector cargs; - cargs.reserve(args.size() + 1); - for (auto &arg : args) - cargs.push_back(arg.c_str()); - cargs.push_back("-fsyntax-only"); + args.push_back("-fsyntax-only"); - std::unique_ptr C(Driver.BuildCompilation(cargs)); + std::unique_ptr C(Driver.BuildCompilation(args)); const driver::JobList &Jobs = C->getJobs(); if (Jobs.size() != 1) return; @@ -168,15 +163,16 @@ struct ProjectProcessor { } }; -std::vector +std::vector ReadCompilerArgumentsFromFile(const std::string &path) { auto MBOrErr = MemoryBuffer::getFile(path); if (!MBOrErr) return {}; - std::vector args; + std::vector args; for (line_iterator I(*MBOrErr.get(), true, '#'), E; I != E; ++I) { - args.push_back(*I); - DoPathMapping(args.back()); + std::string line = *I; + DoPathMapping(line); + args.push_back(Intern(line)); } return args; } @@ -186,12 +182,12 @@ std::vector LoadFromDirectoryListing(ProjectConfig *config) { config->mode = ProjectMode::DotCcls; SmallString<256> Path; sys::path::append(Path, config->project_dir, ".ccls"); - LOG_IF_S(WARNING, !sys::fs::exists(Path) && config->extra_flags.empty()) + LOG_IF_S(WARNING, !sys::fs::exists(Path) && g_config->clang.extraArgs.empty()) << "ccls has no clang arguments. Use either " "compile_commands.json or .ccls, See ccls README for " "more information."; - std::unordered_map> folder_args; + std::unordered_map> folder_args; std::vector files; GetFilesInFolder(config->project_dir, true /*recursive*/, @@ -234,7 +230,7 @@ std::vector LoadFromDirectoryListing(ProjectConfig *config) { e.args = GetCompilerArgumentForFile(file); if (e.args.empty()) e.args.push_back("%clang"); // Add a Dummy. - e.args.push_back(e.filename); + e.args.push_back(Intern(e.filename)); proc.Process(e); result.push_back(e); } @@ -312,9 +308,12 @@ LoadEntriesFromDirectory(ProjectConfig *project, entry.filename = NormalizePath(ResolveIfRelative(entry.directory, Cmd.Filename)); DoPathMapping(entry.filename); - entry.args = std::move(Cmd.CommandLine); - for (std::string &arg : entry.args) + std::vector args = std::move(Cmd.CommandLine); + entry.args.reserve(args.size()); + for (std::string &arg : args) { DoPathMapping(arg); + entry.args.push_back(Intern(arg)); + } proc.Process(entry); if (Seen.insert(entry.filename).second) result.push_back(entry); @@ -341,7 +340,6 @@ int ComputeGuessScore(std::string_view a, std::string_view b) { void Project::Load(const std::string &root_directory) { ProjectConfig project; - project.extra_flags = g_config->clang.extraArgs; project.project_dir = root_directory; entries = LoadEntriesFromDirectory(&project, g_config->compilationDatabaseDirectory); @@ -369,19 +367,19 @@ void Project::Load(const std::string &root_directory) { } } -void Project::SetFlagsForFile(const std::vector &flags, - const std::string &path) { +void Project::SetArgsForFile(const std::vector &args, + const std::string &path) { std::lock_guard lock(mutex_); auto it = path_to_entry_index.find(path); if (it != path_to_entry_index.end()) { // The entry already exists in the project, just set the flags. - this->entries[it->second].args = flags; + this->entries[it->second].args = args; } else { // Entry wasn't found, so we create a new one. Entry entry; entry.is_inferred = false; entry.filename = path; - entry.args = flags; + entry.args = args; this->entries.emplace_back(entry); } } @@ -412,7 +410,7 @@ Project::FindCompilationEntryForFile(const std::string &filename) { result.filename = filename; if (!best_entry) { result.args.push_back("%clang"); - result.args.push_back(filename); + result.args.push_back(Intern(filename)); } else { result.args = best_entry->args; @@ -420,11 +418,11 @@ Project::FindCompilationEntryForFile(const std::string &filename) { // that path to the new filename. std::string best_entry_base_name = sys::path::filename(best_entry->filename); - for (std::string &arg : result.args) { + for (const char *&arg : result.args) { try { if (arg == best_entry->filename || sys::path::filename(arg) == best_entry_base_name) - arg = filename; + arg = Intern(filename); } catch (...) { } } diff --git a/src/project.h b/src/project.h index 33098164..b992e011 100644 --- a/src/project.h +++ b/src/project.h @@ -18,7 +18,7 @@ struct Project { struct Entry { std::string directory; std::string filename; - std::vector args; + std::vector args; // If true, this entry is inferred and was not read from disk. bool is_inferred = false; int id = -1; @@ -52,8 +52,8 @@ struct Project { // If the client has overridden the flags, or specified them for a file // that is not in the compilation_database.json make sure those changes // are permanent. - void SetFlagsForFile(const std::vector &flags, - const std::string &path); + void SetArgsForFile(const std::vector &args, + const std::string &path); // Run |action| on every file in the project. void diff --git a/src/query.h b/src/query.h index 6f844656..a9112447 100644 --- a/src/query.h +++ b/src/query.h @@ -28,7 +28,7 @@ template <> struct DenseMapInfo { struct QueryFile { struct Def { std::string path; - std::vector args; + std::vector args; LanguageId language; // Includes in the file. std::vector includes; diff --git a/src/serializer.cc b/src/serializer.cc index 7061015f..c64d2461 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -452,8 +452,13 @@ Deserialize(SerializeFormat format, const std::string &path, file->path = path; if (g_config->clang.pathMappings.size()) { DoPathMapping(file->import_file); - for (std::string &arg : file->args) - DoPathMapping(arg); + std::vector args; + for (const char *arg : file->args) { + std::string s(arg); + DoPathMapping(s); + args.push_back(Intern(s)); + } + file->args = std::move(args); for (auto &[_, path] : file->lid2path) DoPathMapping(path); for (auto &include : file->includes) { diff --git a/src/test.cc b/src/test.cc index 4d734601..f54b00aa 100644 --- a/src/test.cc +++ b/src/test.cc @@ -247,6 +247,9 @@ bool RunIndexTests(const std::string &filter_path, bool enable_update) { bool update_all = false; // FIXME: show diagnostics in STL/headers when running tests. At the moment // this can be done by constructing ClangIndex index(1, 1); + CompletionManager completion( + nullptr, nullptr, [&](std::string, std::vector) {}, + [](lsRequestId id) {}); GetFilesInFolder( "index_tests", true /*recursive*/, true /*add_folder_to_path*/, [&](const std::string &path) { @@ -290,11 +293,11 @@ bool RunIndexTests(const std::string &filter_path, bool enable_update) { // Run test. g_config = new Config; VFS vfs; - CompletionManager completion( - nullptr, nullptr, [&](std::string, std::vector) {}, - [](lsRequestId id) {}); WorkingFiles wfiles; - auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, flags, {}); + std::vector cargs; + for (auto &arg : flags) + cargs.push_back(arg.c_str()); + auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs, {}); for (const auto &entry : all_expected_output) { const std::string &expected_path = entry.first;