Reload .cquery compile_commands.json upon workspace/didChangeConfiguration. fix #378

And backport https://github.com/waf-project/waf/pull/2127

src/messages/cquery_freshen_index.cc

@@ -4,6 +4,7 @@
 #include "platform.h"
 #include "project.h"
 #include "queue_manager.h"
+#include "timer.h"
 #include "timestamp_manager.h"
 #include "working_files.h"
 
@@ -82,25 +83,11 @@ struct CqueryFreshenIndexHandler : BaseMessageHandler<Ipc_CqueryFreshenIndex> {
         }
     }
 
-    auto* queue = QueueManager::instance();
+    Timer time;
-
     // Send index requests for every file.
-    project->ForAllFilteredFiles(
+    project->Index(config, QueueManager::instance(), working_files,
-        config, [&](int i, const Project::Entry& entry) {
+                   std::monostate());
-          if (!need_index.count(entry.filename))
+    time.ResetAndPrint("[perf] Dispatched $cquery/freshenIndex index requests");
-            return;
-          optional<std::string> content = ReadContent(entry.filename);
-          if (!content) {
-            LOG_S(ERROR) << "When freshening index, cannot read file "
-                         << entry.filename;
-            return;
-          }
-          bool is_interactive =
-              working_files->GetFileByFilename(entry.filename) != nullptr;
-          queue->index_request.PushBack(
-              Index_Request(entry.filename, entry.args, is_interactive,
-                            *content, ICacheManager::Make(config)));
-        });
   }
 };
 REGISTER_MESSAGE_HANDLER(CqueryFreshenIndexHandler);

src/messages/initialize.cc

@@ -487,7 +487,8 @@ struct InitializeHandler : BaseMessageHandler<Ipc_InitializeRequest> {
     LOG_S(INFO) << "Init parameters: " << output.GetString();
 
     if (request->params.rootUri) {
-      std::string project_path = request->params.rootUri->GetPath();
+      std::string project_path =
+          NormalizePath(request->params.rootUri->GetPath());
       LOG_S(INFO) << "[querydb] Initialize in directory " << project_path
                   << " with uri " << request->params.rootUri->raw_uri;
 
@@ -575,8 +576,8 @@ struct InitializeHandler : BaseMessageHandler<Ipc_InitializeRequest> {
       QueueManager::WriteStdout(IpcId::Initialize, out);
 
       // Set project root.
-      config->projectRoot = NormalizePath(request->params.rootUri->GetPath());
+      EnsureEndsInSlash(project_path);
-      EnsureEndsInSlash(config->projectRoot);
+      config->projectRoot = project_path;
       // Create two cache directories for files inside and outside of the
       // project.
       MakeDirectoryRecursive(config->cacheDirectory +
@@ -589,9 +590,7 @@ struct InitializeHandler : BaseMessageHandler<Ipc_InitializeRequest> {
       semantic_cache->Init(config);
 
       // Open up / load the project.
-      project->Load(config, config->extraClangArguments,
+      project->Load(config, project_path);
-                    config->compilationDatabaseDirectory, project_path,
-                    config->resourceDirectory);
       time.ResetAndPrint("[perf] Loaded compilation entries (" +
                          std::to_string(project->entries.size()) + " files)");
 
@@ -620,23 +619,9 @@ struct InitializeHandler : BaseMessageHandler<Ipc_InitializeRequest> {
       // files, because that takes a long time.
       include_complete->Rescan();
 
-      auto* queue = QueueManager::instance();
       time.Reset();
-      project->ForAllFilteredFiles(
+      project->Index(config, QueueManager::instance(), working_files,
-          config, [&](int i, const Project::Entry& entry) {
+                     request->id);
-            optional<std::string> content = ReadContent(entry.filename);
-            if (!content) {
-              LOG_S(ERROR) << "When loading project, canont read file "
-                           << entry.filename;
-              return;
-            }
-            bool is_interactive =
-                working_files->GetFileByFilename(entry.filename) != nullptr;
-            queue->index_request.PushBack(Index_Request(
-                entry.filename, entry.args, is_interactive, *content,
-                ICacheManager::Make(config), request->id));
-          });
-
       // We need to support multiple concurrent index processes.
       time.ResetAndPrint("[perf] Dispatched initial index requests");
     }

src/messages/workspace_did_change_configuration.cc

@@ -2,6 +2,7 @@
 #include "message_handler.h"
 #include "project.h"
 #include "queue_manager.h"
+#include "timer.h"
 #include "working_files.h"
 
 namespace {
@@ -21,6 +22,16 @@ REGISTER_IPC_MESSAGE(Ipc_WorkspaceDidChangeConfiguration);
 struct WorkspaceDidChangeConfigurationHandler
     : BaseMessageHandler<Ipc_WorkspaceDidChangeConfiguration> {
   void Run(Ipc_WorkspaceDidChangeConfiguration* request) override {
+    Timer time;
+    project->Load(config, config->projectRoot);
+    time.ResetAndPrint("[perf] Loaded compilation entries (" +
+      std::to_string(project->entries.size()) + " files)");
+
+    time.Reset();
+    project->Index(config, QueueManager::instance(), working_files,
+      std::monostate());
+    time.ResetAndPrint(
+        "[perf] Dispatched workspace/didChangeConfiguration index requests");
   }
 };
 REGISTER_MESSAGE_HANDLER(WorkspaceDidChangeConfigurationHandler);

src/project.cc

@@ -1,12 +1,15 @@
 #include "project.h"
 
+#include "cache_manager.h"
 #include "clang_utils.h"
 #include "language.h"
 #include "match.h"
 #include "platform.h"
+#include "queue_manager.h"
 #include "serializers/json.h"
 #include "timer.h"
 #include "utils.h"
+#include "working_files.h"
 
 #include <clang-c/CXCompilationDatabase.h>
 #include <doctest/doctest.h>
@@ -364,22 +367,22 @@ std::vector<Project::Entry> LoadFromDirectoryListing(Config* init_opts,
 }
 
 std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(
-    Config* init_opts,
+    Config* config,
-    ProjectConfig* config,
+    ProjectConfig* project,
     const std::string& opt_compilation_db_dir) {
   // If there is a .cquery file always load using directory listing.
-  if (FileExists(config->project_dir + "/.cquery"))
+  if (FileExists(project->project_dir + ".cquery"))
-    return LoadFromDirectoryListing(init_opts, config);
+    return LoadFromDirectoryListing(config, project);
 
   // If |compilationDatabaseCommand| is specified, execute it to get the compdb.
   std::string comp_db_dir;
-  if (init_opts->compilationDatabaseCommand.empty()) {
+  if (config->compilationDatabaseCommand.empty()) {
-    config->mode = ProjectMode::CompileCommandsJson;
+    project->mode = ProjectMode::CompileCommandsJson;
     // Try to load compile_commands.json, but fallback to a project listing.
-    comp_db_dir = opt_compilation_db_dir.empty() ? config->project_dir
+    comp_db_dir = opt_compilation_db_dir.empty() ? project->project_dir
                                                  : opt_compilation_db_dir;
   } else {
-    config->mode = ProjectMode::ExternalCommand;
+    project->mode = ProjectMode::ExternalCommand;
 #ifdef _WIN32
     // TODO
 #else
@@ -390,10 +393,10 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(
     rapidjson::StringBuffer input;
     rapidjson::Writer<rapidjson::StringBuffer> writer(input);
     JsonWriter json_writer(&writer);
-    Reflect(json_writer, *init_opts);
+    Reflect(json_writer, *config);
     std::string contents = GetExternalCommandOutput(
-        std::vector<std::string>{init_opts->compilationDatabaseCommand,
+        std::vector<std::string>{config->compilationDatabaseCommand,
-                                 config->project_dir},
+                                 project->project_dir},
         input.GetString());
     std::ofstream(comp_db_dir + "/compile_commands.json") << contents;
 #endif
@@ -403,7 +406,7 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(
   CXCompilationDatabase_Error cx_db_load_error;
   CXCompilationDatabase cx_db = clang_CompilationDatabase_fromDirectory(
       comp_db_dir.c_str(), &cx_db_load_error);
-  if (!init_opts->compilationDatabaseCommand.empty()) {
+  if (!config->compilationDatabaseCommand.empty()) {
 #ifdef _WIN32
     // TODO
 #else
@@ -415,7 +418,7 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(
   if (cx_db_load_error == CXCompilationDatabase_CanNotLoadDatabase) {
     LOG_S(INFO) << "Unable to load compile_commands.json located at \""
                 << comp_db_dir << "\"; using directory listing instead.";
-    return LoadFromDirectoryListing(init_opts, config);
+    return LoadFromDirectoryListing(config, project);
   }
 
   Timer clang_time;
@@ -461,7 +464,7 @@ std::vector<Project::Entry> LoadCompilationEntriesFromDirectory(
     entry.file = NormalizePathWithTestOptOut(absolute_filename);
 
     result.push_back(
-        GetCompilationEntryFromCompileCommandEntry(init_opts, config, entry));
+        GetCompilationEntryFromCompileCommandEntry(config, project, entry));
     our_time.Pause();
   }
 
@@ -515,24 +518,20 @@ int ComputeGuessScore(const std::string& a, const std::string& b) {
 
 }  // namespace
 
-void Project::Load(Config* init_opts,
+void Project::Load(Config* config, const std::string& root_directory) {
-                   const std::vector<std::string>& extra_flags,
-                   const std::string& opt_compilation_db_dir,
-                   const std::string& root_directory,
-                   const std::string& resource_directory) {
   // Load data.
-  ProjectConfig config;
+  ProjectConfig project;
-  config.extra_flags = extra_flags;
+  project.extra_flags = config->extraClangArguments;
-  config.project_dir = root_directory;
+  project.project_dir = root_directory;
-  config.resource_dir = resource_directory;
+  project.resource_dir = config->resourceDirectory;
-  entries = LoadCompilationEntriesFromDirectory(init_opts, &config,
+  entries = LoadCompilationEntriesFromDirectory(
-                                                opt_compilation_db_dir);
+      config, &project, config->compilationDatabaseDirectory);
 
   // Cleanup / postprocess include directories.
-  quote_include_directories.assign(config.quote_dirs.begin(),
+  quote_include_directories.assign(project.quote_dirs.begin(),
-                                   config.quote_dirs.end());
+                                   project.quote_dirs.end());
-  angle_include_directories.assign(config.angle_dirs.begin(),
+  angle_include_directories.assign(project.angle_dirs.begin(),
-                                   config.angle_dirs.end());
+                                   project.angle_dirs.end());
   for (std::string& path : quote_include_directories) {
     EnsureEndsInSlash(path);
     LOG_S(INFO) << "quote_include_dir: " << path;
@@ -607,23 +606,41 @@ void Project::ForAllFilteredFiles(
   }
 }
 
+void Project::Index(Config* config,
+                    QueueManager* queue,
+                    WorkingFiles* wfiles,
+                    lsRequestId id) {
+  ForAllFilteredFiles(config, [&](int i, const Project::Entry& entry) {
+    optional<std::string> content = ReadContent(entry.filename);
+    if (!content) {
+      LOG_S(ERROR) << "When loading project, canont read file "
+                   << entry.filename;
+      return;
+    }
+    bool is_interactive = wfiles->GetFileByFilename(entry.filename) != nullptr;
+    queue->index_request.PushBack(
+        Index_Request(entry.filename, entry.args, is_interactive, *content,
+                      ICacheManager::Make(config), id));
+  });
+}
+
 TEST_SUITE("Project") {
   void CheckFlags(const std::string& directory, const std::string& file,
                   std::vector<std::string> raw,
                   std::vector<std::string> expected) {
     g_disable_normalize_path_for_test = true;
 
-    Config init_opts;
+    Config config;
-    ProjectConfig config;
+    ProjectConfig project;
-    config.project_dir = "/w/c/s/";
+    project.project_dir = "/w/c/s/";
-    config.resource_dir = "/w/resource_dir/";
+    project.resource_dir = "/w/resource_dir/";
 
     CompileCommandsEntry entry;
     entry.directory = directory;
     entry.args = raw;
     entry.file = file;
     Project::Entry result =
-        GetCompilationEntryFromCompileCommandEntry(&init_opts, &config, entry);
+        GetCompilationEntryFromCompileCommandEntry(&config, &project, entry);
 
     if (result.args != expected) {
       std::cout << "Raw:      " << StringJoin(raw) << std::endl;

src/project.h

@@ -4,12 +4,18 @@
 
 #include <optional.h>
 #include <sparsepp/spp.h>
+#include <variant.h>
 
 #include <functional>
 #include <mutex>
 #include <string>
 #include <vector>
 
+// FIXME ipc.h
+using lsRequestId = std::variant<std::monostate, int64_t, std::string>;
+class QueueManager;
+struct WorkingFiles;
+
 struct Project {
   struct Entry {
     std::string filename;
@@ -28,17 +34,15 @@ struct Project {
 
   // Loads a project for the given |directory|.
   //
-  // If |opt_compilation_db_dir| is not empty, the compile_commands.json
+  // If |config->compilationDatabaseDirectory| is not empty, look for .cquery or
-  // file in it will be used to discover all files and args. If it's empty and
+  // compile_commands.json in it, otherwise they are retrieved in
-  // |root_directory| contains a compile_commands.json file, that one will be
+  // |root_directory|.
-  // used instead. Otherwise, a recursive directory listing of all *.cpp, *.cc,
+  // For .cquery, recursive directory listing is used and files with known
-  // *.h, and *.hpp files will be used. clang arguments can be specified in a
+  // suffixes are indexed. .cquery files can exist in subdirectories and they will affect
-  // .cquery file located inside of |root_directory|.
+  // flags in their subtrees (relative paths are relative to the project root,
-  void Load(Config* init_opts,
+  // not subdirectories).
-            const std::vector<std::string>& extra_flags,
+  // For compile_commands.json, its entries are indexed.
-            const std::string& opt_compilation_db_dir,
+  void Load(Config* config, const std::string& root_directory);
-            const std::string& root_directory,
-            const std::string& resource_directory);
 
   // Lookup the CompilationEntry for |filename|. If no entry was found this
   // will infer one based on existing project structure.
@@ -48,4 +52,9 @@ struct Project {
   void ForAllFilteredFiles(
       Config* config,
       std::function<void(int i, const Entry& entry)> action);
+
+  void Index(Config* config,
+             QueueManager* queue,
+             WorkingFiles* wfiles,
+             lsRequestId id);
 };