Real-time indexing as you type.

src/clang_complete.cc

@@ -49,7 +49,8 @@ unsigned Flags() {
   return CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing |
          CXTranslationUnit_CacheCompletionResults |
          CXTranslationUnit_PrecompiledPreamble |
-         CXTranslationUnit_IncludeBriefCommentsInCodeCompletion
+         CXTranslationUnit_IncludeBriefCommentsInCodeCompletion |
+         CXTranslationUnit_DetailedPreprocessingRecord
 #if !defined(_WIN32)
          // For whatever reason, CreatePreambleOnFirstParse causes clang to
          // become very crashy on windows.
@@ -460,6 +461,11 @@ void CompletionQueryMain(ClangCompleteManager* completion_manager) {
       }
       completion_manager->on_diagnostic_(session->file.filename,
                                          ls_diagnostics);
+
+      timer.Reset();
+      completion_manager->on_index_(session->tu.get(), unsaved,
+                                    session->file.filename, session->file.args);
+      timer.ResetAndPrint("[complete] Reindex file");
     }
 
     continue;
@@ -511,11 +517,13 @@ ClangCompleteManager::ParseRequest::ParseRequest(const std::string& path)
 ClangCompleteManager::ClangCompleteManager(Config* config,
                                            Project* project,
                                            WorkingFiles* working_files,
-                                           OnDiagnostic on_diagnostic)
+                                           OnDiagnostic on_diagnostic,
+                                           OnIndex on_index)
     : config_(config),
       project_(project),
       working_files_(working_files),
       on_diagnostic_(on_diagnostic),
+      on_index_(on_index),
       view_sessions_(kMaxViewSessions),
       edit_sessions_(kMaxEditSessions) {
   new std::thread([&]() {

src/clang_complete.h

@@ -52,6 +52,10 @@ struct ClangCompleteManager {
   using OnDiagnostic =
       std::function<void(std::string path,
                          NonElidedVector<lsDiagnostic> diagnostics)>;
+  using OnIndex = std::function<void(clang::TranslationUnit* tu,
+                                     const std::vector<CXUnsavedFile>& unsaved,
+                                     const std::string& path,
+                                     const std::vector<std::string>& args)>;
   using OnComplete =
       std::function<void(const NonElidedVector<lsCompletionItem>& results,
                          bool is_cached_result)>;
@@ -72,7 +76,8 @@ struct ClangCompleteManager {
   ClangCompleteManager(Config* config,
                        Project* project,
                        WorkingFiles* working_files,
-                       OnDiagnostic on_diagnostic);
+                       OnDiagnostic on_diagnostic,
+                       OnIndex on_index);
   ~ClangCompleteManager();
 
   // Start a code completion at the given location. |on_complete| will run when
@@ -103,6 +108,7 @@ struct ClangCompleteManager {
   Project* project_;
   WorkingFiles* working_files_;
   OnDiagnostic on_diagnostic_;
+  OnIndex on_index_;
 
   // Sessions which have never had a real text-edit applied, but are preloaded
   // to give a fast initial experience.

src/command_line.cc

@@ -892,6 +892,42 @@ std::vector<Index_DoIdMap> DoParseFile(
   return result;
 }
 
+// Index a file using an already-parsed translation unit from code completion.
+// Since most of the time for indexing a file comes from parsing, we can do
+// real-time indexing.
+// TODO: add option to disable this.
+void IndexWithTuFromCodeCompletion(
+    QueueManager* queue,
+    FileConsumer::SharedState* file_consumer_shared,
+    clang::TranslationUnit* tu,
+    const std::vector<CXUnsavedFile>& file_contents,
+    const std::string& path,
+    const std::vector<std::string>& args) {
+  file_consumer_shared->Reset(path);
+
+  PerformanceImportFile perf;
+  clang::Index index(0, 0);
+  std::vector<std::unique_ptr<IndexFile>> indexes = ParseWithTu(
+      file_consumer_shared, &perf, tu, &index, path, args, file_contents);
+
+  std::vector<Index_DoIdMap> result;
+  for (std::unique_ptr<IndexFile>& new_index : indexes) {
+    Timer time;
+
+    // When main thread does IdMap request it will request the previous index if
+    // needed.
+    LOG_S(INFO) << "Emitting index result for " << new_index->path;
+    result.push_back(Index_DoIdMap(std::move(new_index), perf,
+                                   true /*is_interactive*/,
+                                   true /*write_to_disk*/));
+  }
+
+  LOG_IF_S(WARNING, result.size() > 1)
+      << "Code completion index update generated more than one index";
+
+  queue->do_id_map.EnqueueAll(std::move(result));
+}
+
 std::vector<Index_DoIdMap> ParseFile(
     Config* config,
     WorkingFiles* working_files,
@@ -1042,11 +1078,11 @@ bool IndexMergeIndexUpdates(QueueManager* queue) {
     did_merge = true;
     Timer time;
     root->update.Merge(to_join->update);
-    //time.ResetAndPrint("Joined querydb updates for files: " +
+    // time.ResetAndPrint("Joined querydb updates for files: " +
-                       //StringJoinMap(root->update.files_def_update,
+    // StringJoinMap(root->update.files_def_update,
-                                     //[](const QueryFile::DefUpdate& update) {
+    //[](const QueryFile::DefUpdate& update) {
-                                       //return update.path;
+    // return update.path;
-                                     //}));
+    //}));
   }
 }
 
@@ -1375,8 +1411,8 @@ bool QueryDbMainLoop(Config* config,
             lsSignatureHelpOptions();
         // NOTE: If updating signature help tokens make sure to also update
         // WorkingFile::FindClosestCallNameInBuffer.
-        response.result.capabilities.signatureHelpProvider->triggerCharacters =
+        response.result.capabilities.signatureHelpProvider
-            {"(", ","};
+            ->triggerCharacters = {"(", ","};
 
         response.result.capabilities.codeLensProvider = lsCodeLensOptions();
         response.result.capabilities.codeLensProvider->resolveProvider = false;
@@ -2749,15 +2785,20 @@ void RunQueryDbThread(const std::string& bin_name,
   bool exit_when_idle = false;
   Project project;
   WorkingFiles working_files;
+  FileConsumer::SharedState file_consumer_shared;
+
   ClangCompleteManager clang_complete(
       config, &project, &working_files,
       std::bind(&EmitDiagnostics, &working_files, std::placeholders::_1,
-                std::placeholders::_2));
+                std::placeholders::_2),
+      std::bind(&IndexWithTuFromCodeCompletion, queue, &file_consumer_shared,
+                std::placeholders::_1, std::placeholders::_2,
+                std::placeholders::_3, std::placeholders::_4));
+
   IncludeComplete include_complete(config, &project);
   auto global_code_complete_cache = MakeUnique<CodeCompleteCache>();
   auto non_global_code_complete_cache = MakeUnique<CodeCompleteCache>();
   auto signature_cache = MakeUnique<CodeCompleteCache>();
-  FileConsumer::SharedState file_consumer_shared;
   ImportManager import_manager;
   TimestampManager timestamp_manager;
 

src/indexer.cc

@@ -1598,8 +1598,8 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
     Config* config,
     FileConsumer::SharedState* file_consumer_shared,
     std::string file,
-    std::vector<std::string> args,
+    const std::vector<std::string>& args,
-    std::vector<FileContents> file_contents,
+    const std::vector<FileContents>& file_contents,
     PerformanceImportFile* perf,
     clang::Index* index,
     bool dump_ast) {
@@ -1633,18 +1633,33 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
   if (dump_ast)
     Dump(tu.document_cursor());
 
+  return ParseWithTu(file_consumer_shared, perf, &tu, index, file, args,
+                     unsaved_files);
+}
+
+std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
+    FileConsumer::SharedState* file_consumer_shared,
+    PerformanceImportFile* perf,
+    clang::TranslationUnit* tu,
+    clang::Index* index,
+    const std::string& file,
+    const std::vector<std::string>& args,
+    const std::vector<CXUnsavedFile>& file_contents) {
+  Timer timer;
+
   IndexerCallbacks callbacks[] = {{&abortQuery, &diagnostic, &enteredMainFile,
                                    &ppIncludedFile, &importedASTFile,
                                    &startedTranslationUnit, &indexDeclaration,
                                    &indexEntityReference}};
 
   FileConsumer file_consumer(file_consumer_shared, file);
-  IndexParam param(&tu, &file_consumer);
+  IndexParam param(tu, &file_consumer);
-  for (const FileContents& contents : file_contents) {
+  for (const CXUnsavedFile& contents : file_contents) {
-    param.file_contents[contents.path] = contents.content;
+    param.file_contents[contents.Filename] =
+        std::string(contents.Contents, contents.Length);
   }
 
-  CXFile cx_file = clang_getFile(tu.cx_tu, file.c_str());
+  CXFile cx_file = clang_getFile(tu->cx_tu, file.c_str());
   param.primary_file = ConsumeFile(&param, cx_file);
 
   // std::cerr << "!! [START] Indexing " << file << std::endl;
@@ -1653,13 +1668,13 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
                              CXIndexOpt_IndexFunctionLocalSymbols |
                                  CXIndexOpt_SkipParsedBodiesInSession |
                                  CXIndexOpt_IndexImplicitTemplateInstantiations,
-                             tu.cx_tu);
+                             tu->cx_tu);
 
   clang_IndexAction_dispose(index_action);
   // std::cerr << "!! [END] Indexing " << file << std::endl;
 
-  tu.document_cursor().VisitChildren(&VisitMacroDefinitionAndExpansions,
+  tu->document_cursor().VisitChildren(&VisitMacroDefinitionAndExpansions,
-                                     &param);
+                                      &param);
 
   perf->index_build = timer.ElapsedMicrosecondsAndReset();
 
@@ -1679,9 +1694,9 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
         entry->dependencies.begin(), entry->dependencies.end(), entry->path));
 
     // Make sure we are using correct file contents.
-    for (const FileContents& contents : file_contents) {
+    for (const CXUnsavedFile& contents : file_contents) {
-      if (entry->path == contents.path)
+      if (entry->path == contents.Filename)
-        entry->file_contents_ = contents.content;
+        entry->file_contents_ = std::string(contents.Contents, contents.Length);
     }
   }
 

src/indexer.h

@@ -3,20 +3,19 @@
 #include "file_consumer.h"
 #include "language_server_api.h"
 #include "libclangmm/Index.h"
+#include "libclangmm/TranslationUnit.h"
 #include "libclangmm/Utility.h"
 #include "performance.h"
 #include "position.h"
 #include "serializer.h"
 #include "utils.h"
 
-
 #include <optional.h>
 #include <rapidjson/document.h>
 #include <rapidjson/prettywriter.h>
 #include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 
-
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -25,7 +24,6 @@
 #include <unordered_map>
 #include <vector>
 
-
 struct IndexType;
 struct IndexFunc;
 struct IndexVar;
@@ -524,9 +522,18 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
     Config* config,
     FileConsumer::SharedState* file_consumer_shared,
     std::string file,
-    std::vector<std::string> args,
+    const std::vector<std::string>& args,
-    std::vector<FileContents> file_contents,
+    const std::vector<FileContents>& file_contents,
     PerformanceImportFile* perf,
     clang::Index* index,
     bool dump_ast = false);
+std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
+    FileConsumer::SharedState* file_consumer_shared,
+    PerformanceImportFile* perf,
+    clang::TranslationUnit* tu,
+    clang::Index* index,
+    const std::string& file,
+    const std::vector<std::string>& args,
+    const std::vector<CXUnsavedFile>& file_contents);
+
 void IndexInit();