Experiment with caching CXIndex across multiple sessions

src/code_completion.cc

@@ -10,6 +10,7 @@
 
 namespace {
 unsigned Flags() {
+  // TODO: use clang_defaultEditingTranslationUnitOptions()?
   return
     CXTranslationUnit_Incomplete |
     CXTranslationUnit_KeepGoing |
@@ -249,7 +250,7 @@ void EnsureDocumentParsed(CompletionSession* session,
 
   std::cerr << "[complete] Creating completion session with arguments " << StringJoin(args) << std::endl;
   *index = MakeUnique<clang::Index>(0 /*excludeDeclarationsFromPCH*/, 0 /*displayDiagnostics*/);
-  *tu = MakeUnique<clang::TranslationUnit>(*index->get(), session->file.filename, args, unsaved, Flags());
+  *tu = MakeUnique<clang::TranslationUnit>(index->get(), session->file.filename, args, unsaved, Flags());
   std::cerr << "[complete] Done creating active; did_fail=" << (*tu)->did_fail << std::endl;
 }
 

src/command_line.cc

@@ -1142,6 +1142,7 @@ bool ImportCachedIndex(Config* config,
 void ParseFile(Config* config,
                WorkingFiles* working_files,
                FileConsumer::SharedState* file_consumer_shared,
+               clang::Index* index,
                Index_DoIdMapQueue* queue_do_id_map,
                const Project::Entry& entry,
                const optional<std::string>& indexed_content,
@@ -1158,7 +1159,7 @@ void ParseFile(Config* config,
     config, file_consumer_shared,
     tu_path, tu_args,
     entry.filename, indexed_content,
-    &perf);
+    &perf, index);
 
   for (std::unique_ptr<IndexFile>& new_index : indexes) {
     Timer time;
@@ -1271,6 +1272,7 @@ bool IndexMain_DoIndex(Config* config,
                        FileConsumer::SharedState* file_consumer_shared,
                        Project* project,
                        WorkingFiles* working_files,
+                       clang::Index* index,
                        Index_DoIndexQueue* queue_do_index,
                        Index_DoIdMapQueue* queue_do_id_map) {
   optional<Index_DoIndex> index_request = queue_do_index->TryDequeue();
@@ -1301,7 +1303,7 @@ bool IndexMain_DoIndex(Config* config,
     case Index_DoIndex::Type::Parse: {
       // index_request->path can be a cc/tu or a dependency path.
       file_consumer_shared->Reset(index_request->entry.filename);
-      ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
+      ParseFile(config, working_files, file_consumer_shared, index, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
       break;
     }
 
@@ -1311,7 +1313,7 @@ bool IndexMain_DoIndex(Config* config,
 
       bool needs_reparse = ResetStaleFiles(config, file_consumer_shared, index_request->entry.filename);
       if (needs_reparse)
-        ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
+        ParseFile(config, working_files, file_consumer_shared, index, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
       break;
     }
   }
@@ -1327,6 +1329,7 @@ bool IndexMain_DoCreateIndexUpdate(
     return false;
 
   Timer time;
+  // TODO/FIXME: Running call tree on IndexUpdate::CreateDelta crashes cquery.
   IndexUpdate update = IndexUpdate::CreateDelta(response->previous_id_map.get(), response->current_id_map.get(),
     response->previous_index.get(), response->current_index.get());
   response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
@@ -1398,6 +1401,9 @@ void IndexMain(
     Index_OnIndexedQueue* queue_on_indexed) {
 
   SetCurrentThreadName("indexer");
+  // TODO: dispose of index after it is not used for a while.
+  clang::Index index(1, 0);
+
   while (true) {
     // TODO: process all off IndexMain_DoIndex before calling IndexMain_DoCreateIndexUpdate for
     //       better icache behavior. We need to have some threads spinning on both though
@@ -1407,7 +1413,7 @@ void IndexMain(
     // IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any
     // work. Running both also lets the user query the partially constructed
     // index.
-    bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, working_files, queue_do_index, queue_do_id_map);
+    bool did_index = IndexMain_DoIndex(config, file_consumer_shared, project, working_files, &index, queue_do_index, queue_do_id_map);
     bool did_create_update = IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed);
     bool did_merge = false;
 
@@ -2949,7 +2955,16 @@ int main(int argc, char** argv) {
     if (context.shouldExit())
       return res;
 
-    RunTests();
+    for (int i = 0; i < 50; ++i)
+      RunTests();
+
+    for (int i = 0; i < 20; ++i) {
+      std::this_thread::sleep_for(std::chrono::seconds(5));
+      std::cerr << "[POST] " << GetProcessMemoryUsedInMb() << std::endl;
+    }
+
+    std::cerr << std::endl << "[Enter] to exit" << std::endl;
+    std::cin.get();
     return 0;
   }
   else if (HasOption(options, "--language-server")) {

src/indexer.cc

@@ -1536,6 +1536,7 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
     const std::string& file_contents_path,
     const optional<std::string>& file_contents,
     PerformanceImportFile* perf,
+    clang::Index* index,
     bool dump_ast) {
 
   if (!config->enableIndexing)
@@ -1545,8 +1546,9 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
 
   Timer timer;
 
-  clang::Index index(0 /*excludeDeclarationsFromPCH*/,
+  //clang::Index index(0 /*excludeDeclarationsFromPCH*/,
-                     0 /*displayDiagnostics*/);
+  //                   0 /*displayDiagnostics*/);
+
   std::vector<CXUnsavedFile> unsaved_files;
   if (file_contents) {
     CXUnsavedFile unsaved;
@@ -1555,7 +1557,7 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
     unsaved.Length = (unsigned long)file_contents->size();
     unsaved_files.push_back(unsaved);
   }
-  clang::TranslationUnit tu(index, file, args, unsaved_files, CXTranslationUnit_KeepGoing | CXTranslationUnit_DetailedPreprocessingRecord);
+  clang::TranslationUnit tu(index, file, args, unsaved_files, CXTranslationUnit_KeepGoing | CXTranslationUnit_DetailedPreprocessingRecord | CXTranslationUnit_Incomplete);
 
   perf->index_parse = timer.ElapsedMicrosecondsAndReset();
 
@@ -1576,7 +1578,7 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
   param.primary_file = ConsumeFile(&param, cx_file);
 
   //std::cerr << "!! [START] Indexing " << file << std::endl;
-  CXIndexAction index_action = clang_IndexAction_create(index.cx_index);
+  CXIndexAction index_action = clang_IndexAction_create(index->cx_index);
   clang_indexTranslationUnit(index_action, &param, callbacks, sizeof(callbacks),
                              CXIndexOpt_IndexFunctionLocalSymbols | CXIndexOpt_SkipParsedBodiesInSession | CXIndexOpt_IndexImplicitTemplateInstantiations,
                              tu.cx_tu);

src/indexer.h

@@ -5,6 +5,7 @@
 #include "serializer.h"
 #include "utils.h"
 #include "language_server_api.h"
+#include "libclangmm/Index.h"
 #include "libclangmm/Utility.h"
 #include "performance.h"
 
@@ -554,5 +555,6 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
     const std::string& file_contents_path,
     const optional<std::string>& file_contents,
     PerformanceImportFile* perf,
+    clang::Index* index,
     bool dump_ast = false);
 void IndexInit();

src/libclangmm/TranslationUnit.cc

@@ -10,7 +10,7 @@
 
 namespace clang {
 
-TranslationUnit::TranslationUnit(Index& index,
+TranslationUnit::TranslationUnit(Index* index,
                                  const std::string& filepath,
                                  const std::vector<std::string>& arguments,
                                  std::vector<CXUnsavedFile> unsaved_files,
@@ -27,7 +27,7 @@ TranslationUnit::TranslationUnit(Index& index,
 
   //CXErrorCode error_code = clang_parseTranslationUnit2FullArgv(
   CXErrorCode error_code = clang_parseTranslationUnit2(
-      index.cx_index, filepath.c_str(), args.data(), (int)args.size(),
+      index->cx_index, filepath.c_str(), args.data(), (int)args.size(),
       unsaved_files.data(), (unsigned)unsaved_files.size(), flags, &cx_tu);
 
   switch (error_code) {

src/libclangmm/TranslationUnit.h

@@ -12,7 +12,7 @@
 namespace clang {
 class TranslationUnit {
  public:
-  TranslationUnit(Index& index,
+  TranslationUnit(Index* index,
                   const std::string& filepath,
                   const std::vector<std::string>& arguments,
                   std::vector<CXUnsavedFile> unsaved_files,

src/test.cc

@@ -98,29 +98,31 @@ void RunTests() {
 
   // TODO: Assert that we need to be on clang >= 3.9.1
   bool update_all = false;
+  clang::Index index(1, 0);
 
   for (std::string path : GetFilesInFolder("tests", true /*recursive*/, true /*add_folder_to_path*/)) {
-    //if (path != "tests/templates/specialized_func_definition.cc") continue;
+    float memory_before = GetProcessMemoryUsedInMb();
-    //if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue;
+    float memory_after = -1.;
-    //if (path != "tests/multi_file/funky_enum.cc") continue;
-    //if (path != "tests/multi_file/simple_impl.cc") continue;
-    //if (path != "tests/usage/func_called_implicit_ctor.cc") continue;
-    //if (path != "tests/templates/implicit_variable_instantiation.cc") continue;
-    //if (path != "tests/_empty_test.cc") continue;
 
-    //if (path != "tests/templates/template_class_type_usage_folded_into_one.cc") continue;
+    {
-    //path = "C:/Users/jacob/Desktop/superindex/indexer/" + path;
+      //if (path != "tests/templates/specialized_func_definition.cc") continue;
+      //if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue;
+      //if (path != "tests/multi_file/funky_enum.cc") continue;
+      //if (path != "tests/multi_file/simple_impl.cc") continue;
+      //if (path != "tests/usage/func_called_implicit_ctor.cc") continue;
+      //if (path != "tests/templates/implicit_variable_instantiation.cc") continue;
+      //if (path != "tests/multi_file/bad_type_remove.cc") continue;
 
-    // Parse expected output from the test, parse it into JSON document.
+      //if (path != "tests/templates/template_class_type_usage_folded_into_one.cc") continue;
-    std::unordered_map<std::string, std::string> all_expected_output = ParseTestExpectation(path);
+      //path = "C:/Users/jacob/Desktop/superindex/indexer/" + path;
 
-    Config config;
+      Config config;
-    FileConsumer::SharedState file_consumer_shared;
+      FileConsumer::SharedState file_consumer_shared;
 
-    // Run test.
+      // Run test.
-    std::cout << "[START] " << path << std::endl;
+      //std::cout << "[START] " << path << std::endl;
-    PerformanceImportFile perf;
+      PerformanceImportFile perf;
-    std::vector<std::unique_ptr<IndexFile>> dbs = Parse(
+      std::vector<std::unique_ptr<IndexFile>> dbs = Parse(
         &config, &file_consumer_shared,
         path,
         {
@@ -133,82 +135,71 @@ void RunTests() {
         },
         "", nullopt,
         &perf,
+        &index,
         false /*dump_ast*/);
 
 #if false
-    for (auto& db : dbs) {
+      // Parse expected output from the test, parse it into JSON document.
-      assert(db);
+      std::unordered_map<std::string, std::string> all_expected_output = ParseTestExpectation(path);
-      if (!db) {
+      for (auto& entry : all_expected_output) {
-        std::cerr << "no db!!!" << std::endl;
+        const std::string& expected_path = entry.first;
-        continue;
+        const std::string& expected_output = entry.second;
-      }
 
-      for (auto& func : db->funcs) {
+        // Get output from index operation.
-        if (!func.HasInterestingState())
+        IndexFile* db = FindDbForPathEnding(expected_path, dbs);
-          continue;
+        std::string actual_output = "{}";
+        if (db) {
+          VerifySerializeToFrom(db);
+          actual_output = db->ToString();
+        }
 
+        // Compare output via rapidjson::Document to ignore any formatting
+        // differences.
+        rapidjson::Document actual;
+        actual.Parse(actual_output.c_str());
+        rapidjson::Document expected;
+        expected.Parse(expected_output.c_str());
 
-        if (func.uses.size() !=
+        if (actual == expected) {
-          (func.callers.size() + func.declarations.size() + (func.def.definition_spelling.has_value() ? 1 : 0))) {
+          std::cout << "[PASSED] " << path << std::endl;
+        }
+        else {
+          DiffDocuments(path, expected_path, expected, actual);
+          std::cout << std::endl;
+          std::cout << std::endl;
+          std::cout << "[Enter to continue - type u to update test, a to update all]";
+          char c = 'u';
+          if (!update_all) {
+            c = (char)std::cin.get();
+            std::cin.get();
+          }
 
-          std::cout << "func.def.usr = " << func.def.usr << std::endl;
+          if (c == 'a')
-          std::cout << "func.uses.size() = " << func.uses.size() << std::endl;
+            update_all = true;
-          std::cout << "func.callers.size() = " << func.callers.size() << std::endl;
-          std::cout << "func.declarations.size() = " << func.declarations.size() << std::endl;
-          std::cout << "func.definition_spelling.has_value() = " << func.def.definition_spelling.has_value() << std::endl;
 
-          std::cerr << "err" << std::endl;
+          if (update_all || c == 'u') {
+            UpdateTestExpectation(path, expected_output, ToString(actual) + "\n");
+          }
 
         }
       }
-    }
 #endif
 
-    for (auto& entry : all_expected_output) {
+      memory_after = GetProcessMemoryUsedInMb();
-      const std::string& expected_path = entry.first;
-      const std::string& expected_output = entry.second;
-
-      // Get output from index operation.
-      IndexFile* db = FindDbForPathEnding(expected_path, dbs);
-      std::string actual_output = "{}";
-      if (db) {
-        VerifySerializeToFrom(db);
-        actual_output = db->ToString();
-      }
-
-      // Compare output via rapidjson::Document to ignore any formatting
-      // differences.
-      rapidjson::Document actual;
-      actual.Parse(actual_output.c_str());
-      rapidjson::Document expected;
-      expected.Parse(expected_output.c_str());
-
-      if (actual == expected) {
-        std::cout << "[PASSED] " << path << std::endl;
-      }
-      else {
-        DiffDocuments(path, expected_path, expected, actual);
-        std::cout << std::endl;
-        std::cout << std::endl;
-        std::cout << "[Enter to continue - type u to update test, a to update all]";
-        char c = 'u';
-        if (!update_all) {
-          c = (char)std::cin.get();
-          std::cin.get();
-        }
-
-        if (c == 'a')
-          update_all = true;
-
-        if (update_all || c == 'u') {
-          UpdateTestExpectation(path, expected_output, ToString(actual) + "\n");
-        }
-
-      }
     }
+
+    float memory_cleanup = GetProcessMemoryUsedInMb();
+    std::cerr << "[memory] before=" << memory_before << "mb, after=" << memory_after << "mb, cleanup=" << memory_cleanup << "mb" << std::endl;
   }
 
-  std::cin.get();
+  std::cerr << "[final presleep] " << GetProcessMemoryUsedInMb() << "mb" << std::endl;
+  //std::this_thread::sleep_for(std::chrono::seconds(10));
+  //std::cerr << "[final postsleep] " << GetProcessMemoryUsedInMb() << "mb" << std::endl;
+  std::cerr << std::endl;
+  std::cerr << std::endl;
+  std::cerr << std::endl;
+  std::cerr << std::endl;
+  std::cerr << std::endl;
 }
 
 // TODO: ctor/dtor, copy ctor

src/utils.cc

@@ -2,6 +2,7 @@
 
 #include "platform.h"
 
+#include <sparsepp/spp_memory.h>
 #include <tinydir.h>
 
 #include <algorithm>
@@ -315,3 +316,9 @@ void WriteToFile(const std::string& filename, const std::string& content) {
   std::ofstream file(filename);
   file << content;
 }
+
+float GetProcessMemoryUsedInMb() {
+  const float kBytesToMb = 1000000;
+  uint64_t memory_after = spp::GetProcessMemoryUsed();
+  return memory_after / kBytesToMb;
+}

src/utils.h

@@ -122,3 +122,5 @@ inline void hash_combine(std::size_t& seed, const T& v, Rest... rest) {
             }\
         };\
     }
+
+float GetProcessMemoryUsedInMb();