Log index updates before applying them

src/command_line.cc

@@ -776,8 +776,8 @@ struct Index_DoIndex {
     Freshen,
   };
 
-  Index_DoIndex(Type type, const Project::Entry& entry, optional<std::string> content, bool show_diagnostics)
+  Index_DoIndex(Type type, const Project::Entry& entry, optional<std::string> content, bool is_interactive)
-    : type(type), entry(entry), content(content), show_diagnostics(show_diagnostics) {}
+    : type(type), entry(entry), content(content), is_interactive(is_interactive) {}
 
   // Type of index operation.
   Type type;
@@ -785,8 +785,11 @@ struct Index_DoIndex {
   Project::Entry entry;
   // File contents that should be indexed.
   optional<std::string> content;
-  // If diagnostics should be reported.
+  // If this index request is in response to an interactive user session, for
-  bool show_diagnostics = false;
+  // example, the user saving a file they are actively editing. We report
+  // additional information for interactive indexes such as the IndexUpdate
+  // delta as well as the diagnostics.
+  bool is_interactive;
 };
 
 struct Index_DoIdMap {
@@ -794,22 +797,27 @@ struct Index_DoIdMap {
   std::unique_ptr<IndexFile> current;
   optional<std::string> indexed_content;
   PerformanceImportFile perf;
+  bool is_interactive;
 
   explicit Index_DoIdMap(std::unique_ptr<IndexFile> current,
                          optional<std::string> indexed_content,
-                         PerformanceImportFile perf)
+                         PerformanceImportFile perf,
+                         bool is_interactive)
     : current(std::move(current)),
       indexed_content(indexed_content),
-      perf(perf) {}
+      perf(perf),
+      is_interactive(is_interactive) {}
 
   explicit Index_DoIdMap(std::unique_ptr<IndexFile> previous,
                          std::unique_ptr<IndexFile> current,
                          optional<std::string> indexed_content,
-                         PerformanceImportFile perf)
+                         PerformanceImportFile perf,
+                         bool is_interactive)
     : previous(std::move(previous)),
       current(std::move(current)),
       indexed_content(indexed_content),
-      perf(perf) {}
+      perf(perf),
+      is_interactive(is_interactive) {}
 };
 
 struct Index_OnIdMapped {
@@ -819,11 +827,14 @@ struct Index_OnIdMapped {
   std::unique_ptr<IdMap> current_id_map;
   optional<std::string> indexed_content;
   PerformanceImportFile perf;
+  bool is_interactive;
 
   Index_OnIdMapped(const optional<std::string>& indexed_content,
-                   PerformanceImportFile perf)
+                   PerformanceImportFile perf,
+                   bool is_interactive)
     : indexed_content(indexed_content),
-      perf(perf) {}
+      perf(perf),
+      is_interactive(is_interactive) {}
 };
 
 struct Index_OnIndexed {
@@ -1011,7 +1022,7 @@ bool ImportCachedIndex(IndexerConfig* config,
     else
       needs_reparse = true;
     if (cache)
-      queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), nullopt, perf));
+      queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), nullopt, perf, false /*is_interactive*/));
   }
 
   // Import primary file.
@@ -1019,7 +1030,7 @@ bool ImportCachedIndex(IndexerConfig* config,
     file_consumer_shared->Mark(tu_path);
   else
     needs_reparse = true;
-  queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), indexed_content, tu_perf));
+  queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), indexed_content, tu_perf, false /*is_interactive*/));
 
   return needs_reparse;
 }
@@ -1030,7 +1041,7 @@ void ParseFile(IndexerConfig* config,
                Index_DoIdMapQueue* queue_do_id_map,
                const Project::Entry& entry,
                const optional<std::string>& indexed_content,
-               bool report_diagnostics) {
+               bool is_interactive) {
 
   std::unique_ptr<IndexFile> cache_for_args = LoadCachedIndex(config, entry.filename);
 
@@ -1062,7 +1073,7 @@ void ParseFile(IndexerConfig* config,
 
     // Publish diagnostics. We guard behind a |report_diagnostics| flag to
     // avoid heavy lock contention in working_files->GetFileByFilename().
-    if (report_diagnostics) {
+    if (is_interactive) {
       WorkingFile* file = working_files->GetFileByFilename(new_index->path);
       if ((file && file->has_diagnostics) || !new_index->diagnostics.empty()) {
         if (file)
@@ -1098,7 +1109,7 @@ void ParseFile(IndexerConfig* config,
     perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
 
     // Dispatch IdMap creation request, which will happen on querydb thread.
-    Index_DoIdMap response(std::move(cached_index), std::move(new_index), content, perf);
+    Index_DoIdMap response(std::move(cached_index), std::move(new_index), content, perf, is_interactive);
     queue_do_id_map->Enqueue(std::move(response));
   }
 
@@ -1170,7 +1181,7 @@ bool IndexMain_DoIndex(IndexerConfig* config,
     case Index_DoIndex::Type::Parse: {
       // index_request->path can be a cc/tu or a dependency path.
       file_consumer_shared->Reset(index_request->entry.filename);
-      ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->show_diagnostics);
+      ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
       break;
     }
 
@@ -1180,7 +1191,7 @@ bool IndexMain_DoIndex(IndexerConfig* config,
 
       bool needs_reparse = ResetStaleFiles(config, file_consumer_shared, index_request->entry.filename);
       if (needs_reparse)
-        ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->show_diagnostics);
+        ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive);
       break;
     }
   }
@@ -1222,6 +1233,9 @@ bool IndexMain_DoCreateIndexUpdate(
   std::cerr << output.rdbuf();
 #undef PRINT_SECTION
 
+  if (response->is_interactive)
+    std::cerr << "Applying IndexUpdate" << std::endl << update.ToString() << std::endl;
+
   Index_OnIndexed reply(update, response->indexed_content, response->perf);
   queue_on_indexed->Enqueue(std::move(reply));
 
@@ -1413,7 +1427,7 @@ bool QueryDbMainLoop(
               << "] Dispatching index request for file " << entry.filename
               << std::endl;
 
-            queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::ImportThenParse, entry, nullopt, false /*show_diagnostics*/));
+            queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::ImportThenParse, entry, nullopt, false /*is_interactive*/));
           });
         }
 
@@ -1469,7 +1483,7 @@ bool QueryDbMainLoop(
           std::cerr << "[" << i << "/" << (project->entries.size() - 1)
             << "] Dispatching index request for file " << entry.filename
             << std::endl;
-          queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::Freshen, entry, nullopt, false /*show_diagnostics*/));
+          queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::Freshen, entry, nullopt, false /*is_interactive*/));
         });
         break;
       }
@@ -1641,7 +1655,7 @@ bool QueryDbMainLoop(
         //      if so, ignore that index response.
         WorkingFile* working_file = working_files->GetFileByFilename(path);
         if (working_file)
-          queue_do_index->PriorityEnqueue(Index_DoIndex(Index_DoIndex::Type::Parse, project->FindCompilationEntryForFile(path), working_file->buffer_content, true /*show_diagnostics*/));
+          queue_do_index->PriorityEnqueue(Index_DoIndex(Index_DoIndex::Type::Parse, project->FindCompilationEntryForFile(path), working_file->buffer_content, true /*is_interactive*/));
         completion_manager->UpdateActiveSession(path);
 
         break;
@@ -1945,7 +1959,6 @@ bool QueryDbMainLoop(
           break;
         }
 
-        std::cerr << "[querydb] File outline size is " << file->def.outline.size() << std::endl;
         for (SymbolRef ref : file->def.outline) {
           optional<lsSymbolInformation> info = GetSymbolInfo(db, working_files, ref.idx);
           if (!info)
@@ -1981,8 +1994,6 @@ bool QueryDbMainLoop(
         common.working_files = working_files;
         common.working_file = working_files->GetFileByFilename(file->def.path);
 
-        Timer time;
-
         for (SymbolRef ref : file->def.outline) {
           // NOTE: We OffsetColumn so that the code lens always show up in a
           // predictable order. Otherwise, the client may randomize it.
@@ -2059,7 +2070,6 @@ bool QueryDbMainLoop(
           };
         }
 
-        time.ResetAndPrint("[querydb] Building code lens for " + file->def.path);
         ipc->SendOutMessageToClient(IpcId::TextDocumentCodeLens, response);
         break;
       }
@@ -2125,7 +2135,7 @@ bool QueryDbMainLoop(
 
     did_work = true;
 
-    Index_OnIdMapped response(request->indexed_content, request->perf);
+    Index_OnIdMapped response(request->indexed_content, request->perf, request->is_interactive);
     Timer time;
     
     if (request->previous) {

src/indexer.h

@@ -60,6 +60,15 @@ bool operator!=(const Id<T>& a, const Id<T>& b) {
   return !(a == b);
 }
 
+template<typename T>
+void Reflect(Reader& visitor, Id<T>& id) {
+  id.id = visitor.GetUint64();
+}
+template<typename T>
+void Reflect(Writer& visitor, Id<T>& value) {
+  visitor.Uint64(value.id);
+}
+
 using IndexTypeId = Id<IndexType>;
 using IndexFuncId = Id<IndexFunc>;
 using IndexVarId = Id<IndexVar>;
@@ -104,6 +113,27 @@ bool operator!=(const Ref<T>& a, const Ref<T>& b) {
   return !(a == b);
 }
 
+template<typename T>
+void Reflect(Reader& visitor, Ref<T>& value) {
+  const char* str_value = visitor.GetString();
+  uint64_t id = atol(str_value);
+  const char* loc_string = strchr(str_value, '@') + 1;
+
+  value.id_ = Id<T>(id);
+  value.loc = Range(loc_string);
+}
+template<typename T>
+void Reflect(Writer& visitor, Ref<T>& value) {
+  if (value.id_.id == -1) {
+    std::string s = "-1@" + value.loc.ToString();
+    visitor.String(s.c_str());
+  }
+  else {
+    std::string s = std::to_string(value.id_.id) + "@" + value.loc.ToString();
+    visitor.String(s.c_str());
+  }
+}
+
 using IndexFuncRef = Ref<IndexFunc>;
 
 // TODO: skip as much forward-processing as possible when |is_system_def| is
@@ -188,7 +218,8 @@ void Reflect(TVisitor& visitor,
   REFLECT_MEMBER(usr);
   REFLECT_MEMBER(short_name);
   REFLECT_MEMBER(detailed_name);
-  REFLECT_MEMBER(definition);
+  REFLECT_MEMBER(definition_spelling);
+  REFLECT_MEMBER(definition_extent);
   REFLECT_MEMBER(alias_of);
   REFLECT_MEMBER(parents);
   REFLECT_MEMBER(types);
@@ -304,7 +335,8 @@ void Reflect(
   REFLECT_MEMBER(usr);
   REFLECT_MEMBER(short_name);
   REFLECT_MEMBER(detailed_name);
-  REFLECT_MEMBER(definition);
+  REFLECT_MEMBER(definition_spelling);
+  REFLECT_MEMBER(definition_extent);
   REFLECT_MEMBER(declaring_type);
   REFLECT_MEMBER(base);
   REFLECT_MEMBER(locals);

src/position.cc

@@ -140,3 +140,22 @@ bool Range::operator<(const Range& that) const {
     return true;
   return start == that.start && end < that.end;
 }
+
+// Position
+void Reflect(Reader& visitor, Position& value) {
+  value = Position(visitor.GetString());
+}
+void Reflect(Writer& visitor, Position& value) {
+  std::string output = value.ToString();
+  visitor.String(output.c_str(), (rapidjson::SizeType)output.size());
+}
+
+
+// Range
+void Reflect(Reader& visitor, Range& value) {
+  value = Range(visitor.GetString());
+}
+void Reflect(Writer& visitor, Range& value) {
+  std::string output = value.ToString();
+  visitor.String(output.c_str(), (rapidjson::SizeType)output.size());
+}

src/position.h

@@ -4,6 +4,8 @@
 #include <cstdint>
 #include <string>
 
+#include "serializer.h"
+
 struct Position {
   int16_t line = -1;
   int16_t column = -1;
@@ -40,3 +42,9 @@ struct Range {
   bool operator!=(const Range& that) const;
   bool operator<(const Range& that) const;
 };
+
+// Reflection
+void Reflect(Reader& visitor, Position& value);
+void Reflect(Writer& visitor, Position& value);
+void Reflect(Reader& visitor, Range& value);
+void Reflect(Writer& visitor, Range& value);

src/query.cc

@@ -550,7 +550,13 @@ void IndexUpdate::Merge(const IndexUpdate& update) {
 #undef INDEX_UPDATE_MERGE
 }
 
-
+std::string IndexUpdate::ToString() {
+  rapidjson::StringBuffer output;
+  Writer writer(output);
+  IndexUpdate& update = *this;
+  Reflect(writer, update);
+  return output.GetString();
+}
 
 
 

src/query.h

@@ -29,13 +29,11 @@ struct IdMap;
 
 
 
-// TODO: in types, store refs separately from irefs. Then we can drop
-// 'interesting' from location when that is cleaned up.
-
 struct QueryLocation {
   QueryFileId path;
   Range range;
 
+  QueryLocation() {} // Do not use, needed for reflect.
   QueryLocation(QueryFileId path, Range range)
     : path(path), range(range) {}
 
@@ -58,13 +56,16 @@ struct QueryLocation {
     return path == o.path && range < o.range;
   }
 };
+MAKE_REFLECT_STRUCT(QueryLocation, path, range);
 
 enum class SymbolKind { Invalid, File, Type, Func, Var };
+MAKE_REFLECT_TYPE_PROXY(SymbolKind, int);
+
 struct SymbolIdx {
   SymbolKind kind;
   size_t idx;
 
-  explicit SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use.
+  SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use.
   SymbolIdx(SymbolKind kind, uint64_t idx) : kind(kind), idx(idx) {}
 
   bool operator==(const SymbolIdx& that) const {
@@ -77,11 +78,13 @@ struct SymbolIdx {
     return kind == that.kind && idx < that.idx;
   }
 };
+MAKE_REFLECT_STRUCT(SymbolIdx, kind, idx);
 
 struct SymbolRef {
   SymbolIdx idx;
   QueryLocation loc;
 
+  SymbolRef() {} // Do not use, needed for reflect.
   SymbolRef(SymbolIdx idx, QueryLocation loc) : idx(idx), loc(loc) {}
 
   bool operator==(const SymbolRef& that) const {
@@ -94,6 +97,7 @@ struct SymbolRef {
     return idx == that.idx && loc.range.start < that.loc.range.start;
   }
 };
+MAKE_REFLECT_STRUCT(SymbolRef, idx, loc);
 
 struct QueryFuncRef {
   QueryFuncId id() const {
@@ -107,6 +111,7 @@ struct QueryFuncRef {
   QueryFuncId id_;
   QueryLocation loc;
 
+  QueryFuncRef() {} // Do not use, needed for reflect.
   QueryFuncRef(QueryFuncId id, QueryLocation loc) : id_(id), loc(loc) {}
 
   bool operator==(const QueryFuncRef& that) const {
@@ -119,6 +124,7 @@ struct QueryFuncRef {
     return id_ == that.id_ && loc.range.start < that.loc.range.start;
   }
 };
+MAKE_REFLECT_STRUCT(QueryFuncRef, id_, loc);
 
 // There are two sources of reindex updates: the (single) definition of a
 // symbol has changed, or one of many users of the symbol has changed.
@@ -143,6 +149,14 @@ struct MergeableUpdate {
   MergeableUpdate(TId id, const std::vector<TValue>& to_add, const std::vector<TValue>& to_remove)
     : id(id), to_add(to_add), to_remove(to_remove) {}
 };
+template<typename TVisitor, typename TId, typename TValue>
+void Reflect(TVisitor& visitor, MergeableUpdate<TId, TValue>& value) {
+  REFLECT_MEMBER_START();
+  REFLECT_MEMBER(id);
+  REFLECT_MEMBER(to_add);
+  REFLECT_MEMBER(to_remove);
+  REFLECT_MEMBER_END();
+}
 
 struct QueryFile {
   struct Def {
@@ -160,6 +174,7 @@ struct QueryFile {
 
   QueryFile(const std::string& path) { def.path = path; }
 };
+MAKE_REFLECT_STRUCT(QueryFile::Def, path, outline, all_symbols);
 
 struct QueryType {
   using DefUpdate = TypeDefDefinitionData<QueryTypeId, QueryFuncId, QueryVarId, QueryLocation>;
@@ -211,6 +226,9 @@ struct IndexUpdate {
   // work can be parallelized.
   void Merge(const IndexUpdate& update);
 
+  // Dump the update to a string.
+  std::string ToString();
+
   // File updates.
   std::vector<Usr> files_removed;
   std::vector<QueryFile::DefUpdate> files_def_update;
@@ -240,7 +258,11 @@ struct IndexUpdate {
   // will be applied.
   IndexUpdate(const IdMap& previous_id_map, const IdMap& current_id_map, IndexFile& previous, IndexFile& current);
 };
-
+// NOTICE: We're not reflecting on files_removed or files_def_update, it is too much output when logging
+MAKE_REFLECT_STRUCT(IndexUpdate,
+  types_removed, types_def_update, types_derived, types_instances, types_uses,
+  funcs_removed, funcs_def_update, funcs_declarations, funcs_derived, funcs_callers,
+  vars_removed, vars_def_update, vars_uses);
 
 // The query database is heavily optimized for fast queries. It is stored
 // in-memory.

src/serializer.cc

@@ -6,11 +6,18 @@ namespace {
 bool gTestOutputMode = false;
 }  // namespace
 
-// int
+// int16_t
-void Reflect(Reader& visitor, int& value) {
+void Reflect(Reader& visitor, int16_t& value) {
   value = visitor.GetInt();
 }
-void Reflect(Writer& visitor, int& value) {
+void Reflect(Writer& visitor, int16_t& value) {
+  visitor.Int(value);
+}
+// int32_t
+void Reflect(Reader& visitor, int32_t& value) {
+  value = visitor.GetInt();
+}
+void Reflect(Writer& visitor, int32_t& value) {
   visitor.Int(value);
 }
 // int64_t
@@ -20,6 +27,13 @@ void Reflect(Reader& visitor, int64_t& value) {
 void Reflect(Writer& visitor, int64_t& value) {
   visitor.Int64(value);
 }
+// uint64_t
+void Reflect(Reader& visitor, uint64_t& value) {
+  value = visitor.GetUint64();
+}
+void Reflect(Writer& visitor, uint64_t& value) {
+  visitor.Uint64(value);
+}
 // bool
 void Reflect(Reader& visitor, bool& value) {
   value = visitor.GetBool();
@@ -45,55 +59,6 @@ void ReflectMember(Writer& visitor, const char* name, std::string& value) {
 }
 
 
-// Position
-void Reflect(Reader& visitor, Position& value) {
-  value = Position(visitor.GetString());
-}
-void Reflect(Writer& visitor, Position& value) {
-  std::string output = value.ToString();
-  visitor.String(output.c_str(), (rapidjson::SizeType)output.size());
-}
-
-
-// Range
-void Reflect(Reader& visitor, Range& value) {
-  value = Range(visitor.GetString());
-}
-void Reflect(Writer& visitor, Range& value) {
-  std::string output = value.ToString();
-  visitor.String(output.c_str(), (rapidjson::SizeType)output.size());
-}
-
-// Id<T>
-template<typename T>
-void Reflect(Reader& visitor, Id<T>& id) {
-  id.id = visitor.GetUint64();
-}
-template<typename T>
-void Reflect(Writer& visitor, Id<T>& value) {
-  visitor.Uint64(value.id);
-}
-
-
-// Ref<IndexFunc>
-void Reflect(Reader& visitor, Ref<IndexFunc>& value) {
-  const char* str_value = visitor.GetString();
-  uint64_t id = atol(str_value);
-  const char* loc_string = strchr(str_value, '@') + 1;
-
-  value.id_ = Id<IndexFunc>(id);
-  value.loc = Range(loc_string);
-}
-void Reflect(Writer& visitor, Ref<IndexFunc>& value) {
-  if (value.id_.id == -1) {
-    std::string s = "-1@" + value.loc.ToString();
-    visitor.String(s.c_str());
-  }
-  else {
-    std::string s = std::to_string(value.id_.id) + "@" + value.loc.ToString();
-    visitor.String(s.c_str());
-  }
-}
 
 
 
@@ -101,9 +66,10 @@ void Reflect(Writer& visitor, Ref<IndexFunc>& value) {
 
 
 
-// TODO: Move this to indexer.cpp
+
-// TODO: Rename indexer.cpp to indexer.cc
+
-// TODO: Do not serialize a USR if it has no usages/etc outside of USR info.
+
+// TODO: Move this to indexer.cc
 
 // IndexType
 bool ReflectMemberStart(Reader& reader, IndexType& value) {

src/serializer.h

@@ -98,13 +98,18 @@ void ReflectMemberEnd(TVisitor& visitor, T& value) {
 */
 
 
-
+// int16_t
-// int
+void Reflect(Reader& visitor, int16_t& value);
-void Reflect(Reader& visitor, int& value);
+void Reflect(Writer& visitor, int16_t& value);
-void Reflect(Writer& visitor, int& value);
+// int32_t
+void Reflect(Reader& visitor, int32_t& value);
+void Reflect(Writer& visitor, int32_t& value);
 // int64_t
 void Reflect(Reader& visitor, int64_t& value);
 void Reflect(Writer& visitor, int64_t& value);
+// uint64_t
+void Reflect(Reader& visitor, uint64_t& value);
+void Reflect(Writer& visitor, uint64_t& value);
 // bool
 void Reflect(Reader& visitor, bool& value);
 void Reflect(Writer& visitor, bool& value);