From 4b2f53aa5d47c9ca6b4e9d1035e68738686afdf8 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 16 Sep 2018 11:07:15 -0700 Subject: [PATCH] Add clang.pathMappings to reuse cache files with differect source paths Use `>` as the separator. --- src/clang_tu.cc | 3 ++ src/config.cc | 13 ++++++++ src/config.h | 19 ++++++++++- src/messages/initialize.cc | 9 ++--- src/pipeline.cc | 7 ++-- src/project.cc | 19 ++++------- src/serializer.cc | 67 +++++++++++++++++++++++++++----------- src/serializers/json.h | 2 ++ 8 files changed, 100 insertions(+), 39 deletions(-) diff --git a/src/clang_tu.cc b/src/clang_tu.cc index 928be23c..e32547b8 100644 --- a/src/clang_tu.cc +++ b/src/clang_tu.cc @@ -4,6 +4,7 @@ #include "clang_tu.h" #include "clang_utils.h" +#include "config.h" #include using namespace clang; @@ -55,9 +56,11 @@ Range FromTokenRange(const SourceManager &SM, const LangOptions &LangOpts, std::unique_ptr BuildCompilerInvocation(const std::vector &args, IntrusiveRefCntPtr VFS) { + std::string save = "-resource-dir=" + g_config->clang.resourceDir; std::vector cargs; for (auto &arg : args) cargs.push_back(arg.c_str()); + cargs.push_back(save.c_str()); IntrusiveRefCntPtr Diags( CompilerInstance::createDiagnostics(new DiagnosticOptions)); std::unique_ptr CI = diff --git a/src/config.cc b/src/config.cc index 193e41be..d4992bf5 100644 --- a/src/config.cc +++ b/src/config.cc @@ -5,3 +5,16 @@ Config *g_config; thread_local int g_thread_id; + +namespace ccls { +void DoPathMapping(std::string &arg) { + for (const std::string &mapping : g_config->clang.pathMappings) { + auto sep = mapping.find('>'); + if (sep != std::string::npos) { + auto p = arg.find(mapping.substr(0, sep)); + if (p != std::string::npos) + arg.replace(p, sep, mapping.substr(sep + 1)); + } + } +} +} diff --git a/src/config.h b/src/config.h index 1b1fa8a3..6c7f154f 100644 --- a/src/config.h +++ b/src/config.h @@ -49,6 +49,18 @@ struct Config { // Additional arguments to pass to clang. std::vector extraArgs; + // Translate absolute paths in compile_commands.json entries, .ccls options + // and cache files. This allows to reuse cache files built otherwhere if the + // source paths are different. + // + // This is a list of colon-separated strings, e.g. ["/container:/host"] + // + // An entry of "clang -I /container/include /container/a.cc" will be + // translated to "clang -I /host/include /host/a.cc". This is simple string + // replacement, so "clang /prefix/container/a.cc" will become "clang + // /prefix/host/a.cc". + std::vector pathMappings; + // Value to use for clang -resource-dir if not specified. // // This option defaults to clang -print-resource-dir and should not be @@ -223,7 +235,8 @@ struct Config { int maxNum = 2000; } xref; }; -MAKE_REFLECT_STRUCT(Config::Clang, excludeArgs, extraArgs, resourceDir); +MAKE_REFLECT_STRUCT(Config::Clang, excludeArgs, extraArgs, pathMappings, + resourceDir); MAKE_REFLECT_STRUCT(Config::ClientCapability, snippetSupport); MAKE_REFLECT_STRUCT(Config::CodeLens, localVariables); MAKE_REFLECT_STRUCT(Config::Completion, caseSensitivity, detailedLabel, @@ -246,3 +259,7 @@ MAKE_REFLECT_STRUCT(Config, compilationDatabaseCommand, extern Config *g_config; thread_local extern int g_thread_id; + +namespace ccls { +void DoPathMapping(std::string &arg); +} diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc index 32059de9..2576e397 100644 --- a/src/messages/initialize.cc +++ b/src/messages/initialize.cc @@ -441,6 +441,7 @@ struct Handler_Initialize : BaseMessageHandler { // Ensure there is a resource directory. if (g_config->clang.resourceDir.empty()) g_config->clang.resourceDir = GetDefaultResourceDirectory(); + DoPathMapping(g_config->clang.resourceDir); LOG_S(INFO) << "Using -resource-dir=" << g_config->clang.resourceDir; // Send initialization before starting indexers, so we don't send a @@ -459,10 +460,10 @@ struct Handler_Initialize : BaseMessageHandler { if (g_config->cacheDirectory.size()) { // Create two cache directories for files inside and outside of the // project. - sys::fs::create_directories(g_config->cacheDirectory + - EscapeFileName(g_config->projectRoot)); - sys::fs::create_directories(g_config->cacheDirectory + '@' + - EscapeFileName(g_config->projectRoot)); + auto len = g_config->projectRoot.size(); + std::string escaped = EscapeFileName(g_config->projectRoot.substr(0, len - 1)); + sys::fs::create_directories(g_config->cacheDirectory + escaped); + sys::fs::create_directories(g_config->cacheDirectory + '@' + escaped); } diag_pub->Init(); diff --git a/src/pipeline.cc b/src/pipeline.cc index f8726247..54179b31 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -124,12 +124,13 @@ std::string AppendSerializationFormat(const std::string &base) { std::string GetCachePath(const std::string &source_file) { std::string cache_file; - size_t len = g_config->projectRoot.size(); + auto len = g_config->projectRoot.size(); if (StartsWith(source_file, g_config->projectRoot)) { - cache_file = EscapeFileName(g_config->projectRoot) + + cache_file = EscapeFileName(g_config->projectRoot.substr(0, len - 1)) + '/' + EscapeFileName(source_file.substr(len)); } else { - cache_file = '@' + EscapeFileName(g_config->projectRoot) + + cache_file = '@' + + EscapeFileName(g_config->projectRoot.substr(0, len - 1)) + '/' + EscapeFileName(source_file); } diff --git a/src/project.cc b/src/project.cc index a14431be..895ab92e 100644 --- a/src/project.cc +++ b/src/project.cc @@ -105,17 +105,6 @@ struct ProjectProcessor { } hash_combine(hash, std::hash{}(arg)); } - for (size_t i = 1; i < args.size(); i++) - // This is most likely the file path we will be passing to clang. The - // path needs to be absolute, otherwise clang_codeCompleteAt is extremely - // slow. See - // https://github.com/cquery-project/cquery/commit/af63df09d57d765ce12d40007bf56302a0446678. - if (args[i][0] != '-' && EndsWith(args[i], base_name)) { - args[i] = ResolveIfRelative(entry.directory, args[i]); - continue; - } - - args.push_back("-resource-dir=" + g_config->clang.resourceDir); args.push_back("-working-directory=" + entry.directory); if (!command_set.insert(hash).second) { @@ -185,8 +174,10 @@ ReadCompilerArgumentsFromFile(const std::string &path) { if (!MBOrErr) return {}; std::vector args; - for (line_iterator I(*MBOrErr.get(), true, '#'), E; I != E; ++I) + for (line_iterator I(*MBOrErr.get(), true, '#'), E; I != E; ++I) { args.push_back(*I); + DoPathMapping(args.back()); + } return args; } @@ -317,9 +308,13 @@ LoadEntriesFromDirectory(ProjectConfig *project, for (tooling::CompileCommand &Cmd : CDB->getAllCompileCommands()) { Project::Entry entry; entry.directory = NormalizePath(Cmd.Directory); + DoPathMapping(entry.directory); entry.filename = NormalizePath(ResolveIfRelative(entry.directory, Cmd.Filename)); + DoPathMapping(entry.filename); entry.args = std::move(Cmd.CommandLine); + for (std::string &arg : entry.args) + DoPathMapping(arg); proc.Process(entry); if (Seen.insert(entry.filename).second) result.push_back(entry); diff --git a/src/serializer.cc b/src/serializer.cc index 0fe244ea..c6da2ae0 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -147,26 +147,39 @@ void Reflect(Writer &visitor, std::unordered_map &map) { visitor.EndArray(); } -// Used by IndexFile::dependencies. Timestamps are emitted for Binary. -void Reflect(Reader &visitor, StringMap &map) { - visitor.IterArray([&](Reader &entry) { - std::string name; - Reflect(entry, name); - if (visitor.Format() == SerializeFormat::Binary) - Reflect(entry, map[name]); - else - map[name] = 0; - }); -} -void Reflect(Writer &visitor, StringMap &map) { - visitor.StartArray(map.size()); - for (auto &it : map) { - std::string key = it.first(); - Reflect(visitor, key); - if (visitor.Format() == SerializeFormat::Binary) - Reflect(visitor, it.second); +// Used by IndexFile::dependencies. +void Reflect(Reader &vis, StringMap &v) { + std::string name; + if (vis.Format() == SerializeFormat::Json) { + auto &vis1 = static_cast(vis); + for (auto it = vis1.m().MemberBegin(); it != vis1.m().MemberEnd(); ++it) + v[it->name.GetString()] = it->value.GetInt64(); + } else { + vis.IterArray([&](Reader &entry) { + Reflect(entry, name); + Reflect(entry, v[name]); + }); + } +} +void Reflect(Writer &vis, StringMap &v) { + if (vis.Format() == SerializeFormat::Json) { + auto &vis1 = static_cast(vis); + vis.StartObject(); + for (auto &it : v) { + std::string key = it.first(); + vis1.m().Key(key.c_str()); + vis1.m().Int64(it.second); + } + vis.EndObject(); + } else { + vis.StartArray(v.size()); + for (auto &it : v) { + std::string key = it.first(); + Reflect(vis, key); + Reflect(vis, it.second); + } + vis.EndArray(); } - visitor.EndArray(); } // TODO: Move this to indexer.cc @@ -436,6 +449,22 @@ Deserialize(SerializeFormat format, const std::string &path, // Restore non-serialized state. file->path = path; + if (g_config->clang.pathMappings.size()) { + DoPathMapping(file->import_file); + for (std::string &arg : file->args) + DoPathMapping(arg); + for (auto &[_, path] : file->lid2path) + DoPathMapping(path); + for (auto &include : file->includes) + DoPathMapping(include.resolved_path); + StringMap dependencies; + for (auto &it : file->dependencies) { + std::string path = it.first().str(); + DoPathMapping(path); + dependencies[path] = it.second; + } + file->dependencies = std::move(dependencies); + } return file; } } // namespace ccls diff --git a/src/serializers/json.h b/src/serializers/json.h index 5b5db006..636f29fc 100644 --- a/src/serializers/json.h +++ b/src/serializers/json.h @@ -15,6 +15,7 @@ class JsonReader : public Reader { public: JsonReader(rapidjson::GenericValue> *m) : m_(m) {} SerializeFormat Format() const override { return SerializeFormat::Json; } + rapidjson::GenericValue> &m() { return *m_; } bool IsBool() override { return m_->IsBool(); } bool IsNull() override { return m_->IsNull(); } @@ -83,6 +84,7 @@ class JsonWriter : public Writer { public: JsonWriter(rapidjson::Writer *m) : m_(m) {} SerializeFormat Format() const override { return SerializeFormat::Json; } + rapidjson::Writer &m() { return *m_; } void Null() override { m_->Null(); } void Bool(bool x) override { m_->Bool(x); }