Add clang.pathMappings to reuse cache files with differect source paths

Use `>` as the separator.

src/clang_tu.cc

@@ -4,6 +4,7 @@
 #include "clang_tu.h"
 
 #include "clang_utils.h"
+#include "config.h"
 
 #include <clang/Lex/Lexer.h>
 using namespace clang;
@@ -55,9 +56,11 @@ Range FromTokenRange(const SourceManager &SM, const LangOptions &LangOpts,
 std::unique_ptr<CompilerInvocation>
 BuildCompilerInvocation(const std::vector<std::string> &args,
                         IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+  std::string save = "-resource-dir=" + g_config->clang.resourceDir;
   std::vector<const char *> cargs;
   for (auto &arg : args)
     cargs.push_back(arg.c_str());
+  cargs.push_back(save.c_str());
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
       CompilerInstance::createDiagnostics(new DiagnosticOptions));
   std::unique_ptr<CompilerInvocation> CI =

src/config.cc

@@ -5,3 +5,16 @@
 
 Config *g_config;
 thread_local int g_thread_id;
+
+namespace ccls {
+void DoPathMapping(std::string &arg) {
+  for (const std::string &mapping : g_config->clang.pathMappings) {
+    auto sep = mapping.find('>');
+    if (sep != std::string::npos) {
+      auto p = arg.find(mapping.substr(0, sep));
+      if (p != std::string::npos)
+        arg.replace(p, sep, mapping.substr(sep + 1));
+    }
+  }
+}
+}

src/config.h

@@ -49,6 +49,18 @@ struct Config {
     // Additional arguments to pass to clang.
     std::vector<std::string> extraArgs;
 
+    // Translate absolute paths in compile_commands.json entries, .ccls options
+    // and cache files. This allows to reuse cache files built otherwhere if the
+    // source paths are different.
+    //
+    // This is a list of colon-separated strings, e.g. ["/container:/host"]
+    //
+    // An entry of "clang -I /container/include /container/a.cc" will be
+    // translated to "clang -I /host/include /host/a.cc". This is simple string
+    // replacement, so "clang /prefix/container/a.cc" will become "clang
+    // /prefix/host/a.cc".
+    std::vector<std::string> pathMappings;
+
     // Value to use for clang -resource-dir if not specified.
     //
     // This option defaults to clang -print-resource-dir and should not be
@@ -223,7 +235,8 @@ struct Config {
     int maxNum = 2000;
   } xref;
 };
-MAKE_REFLECT_STRUCT(Config::Clang, excludeArgs, extraArgs, resourceDir);
+MAKE_REFLECT_STRUCT(Config::Clang, excludeArgs, extraArgs, pathMappings,
+                    resourceDir);
 MAKE_REFLECT_STRUCT(Config::ClientCapability, snippetSupport);
 MAKE_REFLECT_STRUCT(Config::CodeLens, localVariables);
 MAKE_REFLECT_STRUCT(Config::Completion, caseSensitivity, detailedLabel,
@@ -246,3 +259,7 @@ MAKE_REFLECT_STRUCT(Config, compilationDatabaseCommand,
 
 extern Config *g_config;
 thread_local extern int g_thread_id;
+
+namespace ccls {
+void DoPathMapping(std::string &arg);
+}

src/messages/initialize.cc

@@ -441,6 +441,7 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
     // Ensure there is a resource directory.
     if (g_config->clang.resourceDir.empty())
       g_config->clang.resourceDir = GetDefaultResourceDirectory();
+    DoPathMapping(g_config->clang.resourceDir);
     LOG_S(INFO) << "Using -resource-dir=" << g_config->clang.resourceDir;
 
     // Send initialization before starting indexers, so we don't send a
@@ -459,10 +460,10 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
     if (g_config->cacheDirectory.size()) {
       // Create two cache directories for files inside and outside of the
       // project.
-      sys::fs::create_directories(g_config->cacheDirectory +
+      auto len = g_config->projectRoot.size();
-        EscapeFileName(g_config->projectRoot));
+      std::string escaped = EscapeFileName(g_config->projectRoot.substr(0, len - 1));
-      sys::fs::create_directories(g_config->cacheDirectory + '@' +
+      sys::fs::create_directories(g_config->cacheDirectory + escaped);
-                                  EscapeFileName(g_config->projectRoot));
+      sys::fs::create_directories(g_config->cacheDirectory + '@' + escaped);
     }
 
     diag_pub->Init();

src/pipeline.cc

@@ -124,12 +124,13 @@ std::string AppendSerializationFormat(const std::string &base) {
 
 std::string GetCachePath(const std::string &source_file) {
   std::string cache_file;
-  size_t len = g_config->projectRoot.size();
+  auto len = g_config->projectRoot.size();
   if (StartsWith(source_file, g_config->projectRoot)) {
-    cache_file = EscapeFileName(g_config->projectRoot) +
+    cache_file = EscapeFileName(g_config->projectRoot.substr(0, len - 1)) + '/' +
                  EscapeFileName(source_file.substr(len));
   } else {
-    cache_file = '@' + EscapeFileName(g_config->projectRoot) +
+    cache_file = '@' +
+                 EscapeFileName(g_config->projectRoot.substr(0, len - 1)) + '/' +
                  EscapeFileName(source_file);
   }
 

src/project.cc

@@ -105,17 +105,6 @@ struct ProjectProcessor {
       }
       hash_combine(hash, std::hash<std::string>{}(arg));
     }
-    for (size_t i = 1; i < args.size(); i++)
-      // This is most likely the file path we will be passing to clang. The
-      // path needs to be absolute, otherwise clang_codeCompleteAt is extremely
-      // slow. See
-      // https://github.com/cquery-project/cquery/commit/af63df09d57d765ce12d40007bf56302a0446678.
-      if (args[i][0] != '-' && EndsWith(args[i], base_name)) {
-        args[i] = ResolveIfRelative(entry.directory, args[i]);
-        continue;
-      }
-
-    args.push_back("-resource-dir=" + g_config->clang.resourceDir);
     args.push_back("-working-directory=" + entry.directory);
 
     if (!command_set.insert(hash).second) {
@@ -185,8 +174,10 @@ ReadCompilerArgumentsFromFile(const std::string &path) {
   if (!MBOrErr)
     return {};
   std::vector<std::string> args;
-  for (line_iterator I(*MBOrErr.get(), true, '#'), E; I != E; ++I)
+  for (line_iterator I(*MBOrErr.get(), true, '#'), E; I != E; ++I) {
     args.push_back(*I);
+    DoPathMapping(args.back());
+  }
   return args;
 }
 
@@ -317,9 +308,13 @@ LoadEntriesFromDirectory(ProjectConfig *project,
   for (tooling::CompileCommand &Cmd : CDB->getAllCompileCommands()) {
     Project::Entry entry;
     entry.directory = NormalizePath(Cmd.Directory);
+    DoPathMapping(entry.directory);
     entry.filename =
         NormalizePath(ResolveIfRelative(entry.directory, Cmd.Filename));
+    DoPathMapping(entry.filename);
     entry.args = std::move(Cmd.CommandLine);
+    for (std::string &arg : entry.args)
+      DoPathMapping(arg);
     proc.Process(entry);
     if (Seen.insert(entry.filename).second)
       result.push_back(entry);

src/serializer.cc

@@ -147,26 +147,39 @@ void Reflect(Writer &visitor, std::unordered_map<Usr, V> &map) {
   visitor.EndArray();
 }
 
-// Used by IndexFile::dependencies. Timestamps are emitted for Binary.
+// Used by IndexFile::dependencies.
-void Reflect(Reader &visitor, StringMap<int64_t> &map) {
+void Reflect(Reader &vis, StringMap<int64_t> &v) {
-  visitor.IterArray([&](Reader &entry) {
   std::string name;
+  if (vis.Format() == SerializeFormat::Json) {
+    auto &vis1 = static_cast<JsonReader&>(vis);
+    for (auto it = vis1.m().MemberBegin(); it != vis1.m().MemberEnd(); ++it)
+      v[it->name.GetString()] = it->value.GetInt64();
+  } else {
+    vis.IterArray([&](Reader &entry) {
       Reflect(entry, name);
-    if (visitor.Format() == SerializeFormat::Binary)
+      Reflect(entry, v[name]);
-      Reflect(entry, map[name]);
-    else
-      map[name] = 0;
     });
   }
-void Reflect(Writer &visitor, StringMap<int64_t> &map) {
-  visitor.StartArray(map.size());
-  for (auto &it : map) {
-    std::string key = it.first();
-    Reflect(visitor, key);
-    if (visitor.Format() == SerializeFormat::Binary)
-      Reflect(visitor, it.second);
 }
-  visitor.EndArray();
+void Reflect(Writer &vis, StringMap<int64_t> &v) {
+  if (vis.Format() == SerializeFormat::Json) {
+    auto &vis1 = static_cast<JsonWriter&>(vis);
+    vis.StartObject();
+    for (auto &it : v) {
+      std::string key = it.first();
+      vis1.m().Key(key.c_str());
+      vis1.m().Int64(it.second);
+    }
+    vis.EndObject();
+  } else {
+    vis.StartArray(v.size());
+    for (auto &it : v) {
+      std::string key = it.first();
+      Reflect(vis, key);
+      Reflect(vis, it.second);
+    }
+    vis.EndArray();
+  }
 }
 
 // TODO: Move this to indexer.cc
@@ -436,6 +449,22 @@ Deserialize(SerializeFormat format, const std::string &path,
 
   // Restore non-serialized state.
   file->path = path;
+  if (g_config->clang.pathMappings.size()) {
+    DoPathMapping(file->import_file);
+    for (std::string &arg : file->args)
+      DoPathMapping(arg);
+    for (auto &[_, path] : file->lid2path)
+      DoPathMapping(path);
+    for (auto &include : file->includes)
+      DoPathMapping(include.resolved_path);
+    StringMap<int64_t> dependencies;
+    for (auto &it : file->dependencies) {
+      std::string path = it.first().str();
+      DoPathMapping(path);
+      dependencies[path] = it.second;
+    }
+    file->dependencies = std::move(dependencies);
+  }
   return file;
 }
 } // namespace ccls

src/serializers/json.h

@@ -15,6 +15,7 @@ class JsonReader : public Reader {
 public:
   JsonReader(rapidjson::GenericValue<rapidjson::UTF8<>> *m) : m_(m) {}
   SerializeFormat Format() const override { return SerializeFormat::Json; }
+  rapidjson::GenericValue<rapidjson::UTF8<>> &m() { return *m_; }
 
   bool IsBool() override { return m_->IsBool(); }
   bool IsNull() override { return m_->IsNull(); }
@@ -83,6 +84,7 @@ class JsonWriter : public Writer {
 public:
   JsonWriter(rapidjson::Writer<rapidjson::StringBuffer> *m) : m_(m) {}
   SerializeFormat Format() const override { return SerializeFormat::Json; }
+  rapidjson::Writer<rapidjson::StringBuffer> &m() { return *m_; }
 
   void Null() override { m_->Null(); }
   void Bool(bool x) override { m_->Bool(x); }