From 00143bef8e232fde872b272be85a669d9cd6a88b Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Fri, 19 May 2017 00:02:01 -0700 Subject: [PATCH] Log index updates before applying them --- src/command_line.cc | 60 ++++++++++++++++++++--------------- src/indexer.h | 36 +++++++++++++++++++-- src/position.cc | 19 ++++++++++++ src/position.h | 10 +++++- src/query.cc | 8 ++++- src/query.h | 32 ++++++++++++++++--- src/serializer.cc | 76 +++++++++++++-------------------------------- src/serializer.h | 13 +++++--- 8 files changed, 161 insertions(+), 93 deletions(-) diff --git a/src/command_line.cc b/src/command_line.cc index 25f2e7c7..df3ca187 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -776,8 +776,8 @@ struct Index_DoIndex { Freshen, }; - Index_DoIndex(Type type, const Project::Entry& entry, optional content, bool show_diagnostics) - : type(type), entry(entry), content(content), show_diagnostics(show_diagnostics) {} + Index_DoIndex(Type type, const Project::Entry& entry, optional content, bool is_interactive) + : type(type), entry(entry), content(content), is_interactive(is_interactive) {} // Type of index operation. Type type; @@ -785,8 +785,11 @@ struct Index_DoIndex { Project::Entry entry; // File contents that should be indexed. optional content; - // If diagnostics should be reported. - bool show_diagnostics = false; + // If this index request is in response to an interactive user session, for + // example, the user saving a file they are actively editing. We report + // additional information for interactive indexes such as the IndexUpdate + // delta as well as the diagnostics. + bool is_interactive; }; struct Index_DoIdMap { @@ -794,22 +797,27 @@ struct Index_DoIdMap { std::unique_ptr current; optional indexed_content; PerformanceImportFile perf; + bool is_interactive; explicit Index_DoIdMap(std::unique_ptr current, optional indexed_content, - PerformanceImportFile perf) + PerformanceImportFile perf, + bool is_interactive) : current(std::move(current)), indexed_content(indexed_content), - perf(perf) {} + perf(perf), + is_interactive(is_interactive) {} explicit Index_DoIdMap(std::unique_ptr previous, std::unique_ptr current, optional indexed_content, - PerformanceImportFile perf) + PerformanceImportFile perf, + bool is_interactive) : previous(std::move(previous)), current(std::move(current)), indexed_content(indexed_content), - perf(perf) {} + perf(perf), + is_interactive(is_interactive) {} }; struct Index_OnIdMapped { @@ -819,11 +827,14 @@ struct Index_OnIdMapped { std::unique_ptr current_id_map; optional indexed_content; PerformanceImportFile perf; + bool is_interactive; Index_OnIdMapped(const optional& indexed_content, - PerformanceImportFile perf) + PerformanceImportFile perf, + bool is_interactive) : indexed_content(indexed_content), - perf(perf) {} + perf(perf), + is_interactive(is_interactive) {} }; struct Index_OnIndexed { @@ -1011,7 +1022,7 @@ bool ImportCachedIndex(IndexerConfig* config, else needs_reparse = true; if (cache) - queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), nullopt, perf)); + queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), nullopt, perf, false /*is_interactive*/)); } // Import primary file. @@ -1019,7 +1030,7 @@ bool ImportCachedIndex(IndexerConfig* config, file_consumer_shared->Mark(tu_path); else needs_reparse = true; - queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), indexed_content, tu_perf)); + queue_do_id_map->Enqueue(Index_DoIdMap(std::move(cache), indexed_content, tu_perf, false /*is_interactive*/)); return needs_reparse; } @@ -1030,7 +1041,7 @@ void ParseFile(IndexerConfig* config, Index_DoIdMapQueue* queue_do_id_map, const Project::Entry& entry, const optional& indexed_content, - bool report_diagnostics) { + bool is_interactive) { std::unique_ptr cache_for_args = LoadCachedIndex(config, entry.filename); @@ -1062,7 +1073,7 @@ void ParseFile(IndexerConfig* config, // Publish diagnostics. We guard behind a |report_diagnostics| flag to // avoid heavy lock contention in working_files->GetFileByFilename(). - if (report_diagnostics) { + if (is_interactive) { WorkingFile* file = working_files->GetFileByFilename(new_index->path); if ((file && file->has_diagnostics) || !new_index->diagnostics.empty()) { if (file) @@ -1098,7 +1109,7 @@ void ParseFile(IndexerConfig* config, perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset(); // Dispatch IdMap creation request, which will happen on querydb thread. - Index_DoIdMap response(std::move(cached_index), std::move(new_index), content, perf); + Index_DoIdMap response(std::move(cached_index), std::move(new_index), content, perf, is_interactive); queue_do_id_map->Enqueue(std::move(response)); } @@ -1170,7 +1181,7 @@ bool IndexMain_DoIndex(IndexerConfig* config, case Index_DoIndex::Type::Parse: { // index_request->path can be a cc/tu or a dependency path. file_consumer_shared->Reset(index_request->entry.filename); - ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->show_diagnostics); + ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive); break; } @@ -1180,7 +1191,7 @@ bool IndexMain_DoIndex(IndexerConfig* config, bool needs_reparse = ResetStaleFiles(config, file_consumer_shared, index_request->entry.filename); if (needs_reparse) - ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->show_diagnostics); + ParseFile(config, working_files, file_consumer_shared, queue_do_id_map, index_request->entry, index_request->content, index_request->is_interactive); break; } } @@ -1222,6 +1233,9 @@ bool IndexMain_DoCreateIndexUpdate( std::cerr << output.rdbuf(); #undef PRINT_SECTION + if (response->is_interactive) + std::cerr << "Applying IndexUpdate" << std::endl << update.ToString() << std::endl; + Index_OnIndexed reply(update, response->indexed_content, response->perf); queue_on_indexed->Enqueue(std::move(reply)); @@ -1413,7 +1427,7 @@ bool QueryDbMainLoop( << "] Dispatching index request for file " << entry.filename << std::endl; - queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::ImportThenParse, entry, nullopt, false /*show_diagnostics*/)); + queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::ImportThenParse, entry, nullopt, false /*is_interactive*/)); }); } @@ -1469,7 +1483,7 @@ bool QueryDbMainLoop( std::cerr << "[" << i << "/" << (project->entries.size() - 1) << "] Dispatching index request for file " << entry.filename << std::endl; - queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::Freshen, entry, nullopt, false /*show_diagnostics*/)); + queue_do_index->Enqueue(Index_DoIndex(Index_DoIndex::Type::Freshen, entry, nullopt, false /*is_interactive*/)); }); break; } @@ -1641,7 +1655,7 @@ bool QueryDbMainLoop( // if so, ignore that index response. WorkingFile* working_file = working_files->GetFileByFilename(path); if (working_file) - queue_do_index->PriorityEnqueue(Index_DoIndex(Index_DoIndex::Type::Parse, project->FindCompilationEntryForFile(path), working_file->buffer_content, true /*show_diagnostics*/)); + queue_do_index->PriorityEnqueue(Index_DoIndex(Index_DoIndex::Type::Parse, project->FindCompilationEntryForFile(path), working_file->buffer_content, true /*is_interactive*/)); completion_manager->UpdateActiveSession(path); break; @@ -1945,7 +1959,6 @@ bool QueryDbMainLoop( break; } - std::cerr << "[querydb] File outline size is " << file->def.outline.size() << std::endl; for (SymbolRef ref : file->def.outline) { optional info = GetSymbolInfo(db, working_files, ref.idx); if (!info) @@ -1981,8 +1994,6 @@ bool QueryDbMainLoop( common.working_files = working_files; common.working_file = working_files->GetFileByFilename(file->def.path); - Timer time; - for (SymbolRef ref : file->def.outline) { // NOTE: We OffsetColumn so that the code lens always show up in a // predictable order. Otherwise, the client may randomize it. @@ -2059,7 +2070,6 @@ bool QueryDbMainLoop( }; } - time.ResetAndPrint("[querydb] Building code lens for " + file->def.path); ipc->SendOutMessageToClient(IpcId::TextDocumentCodeLens, response); break; } @@ -2125,7 +2135,7 @@ bool QueryDbMainLoop( did_work = true; - Index_OnIdMapped response(request->indexed_content, request->perf); + Index_OnIdMapped response(request->indexed_content, request->perf, request->is_interactive); Timer time; if (request->previous) { diff --git a/src/indexer.h b/src/indexer.h index 07fee29b..a2fd39e5 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -60,6 +60,15 @@ bool operator!=(const Id& a, const Id& b) { return !(a == b); } +template +void Reflect(Reader& visitor, Id& id) { + id.id = visitor.GetUint64(); +} +template +void Reflect(Writer& visitor, Id& value) { + visitor.Uint64(value.id); +} + using IndexTypeId = Id; using IndexFuncId = Id; using IndexVarId = Id; @@ -104,6 +113,27 @@ bool operator!=(const Ref& a, const Ref& b) { return !(a == b); } +template +void Reflect(Reader& visitor, Ref& value) { + const char* str_value = visitor.GetString(); + uint64_t id = atol(str_value); + const char* loc_string = strchr(str_value, '@') + 1; + + value.id_ = Id(id); + value.loc = Range(loc_string); +} +template +void Reflect(Writer& visitor, Ref& value) { + if (value.id_.id == -1) { + std::string s = "-1@" + value.loc.ToString(); + visitor.String(s.c_str()); + } + else { + std::string s = std::to_string(value.id_.id) + "@" + value.loc.ToString(); + visitor.String(s.c_str()); + } +} + using IndexFuncRef = Ref; // TODO: skip as much forward-processing as possible when |is_system_def| is @@ -188,7 +218,8 @@ void Reflect(TVisitor& visitor, REFLECT_MEMBER(usr); REFLECT_MEMBER(short_name); REFLECT_MEMBER(detailed_name); - REFLECT_MEMBER(definition); + REFLECT_MEMBER(definition_spelling); + REFLECT_MEMBER(definition_extent); REFLECT_MEMBER(alias_of); REFLECT_MEMBER(parents); REFLECT_MEMBER(types); @@ -304,7 +335,8 @@ void Reflect( REFLECT_MEMBER(usr); REFLECT_MEMBER(short_name); REFLECT_MEMBER(detailed_name); - REFLECT_MEMBER(definition); + REFLECT_MEMBER(definition_spelling); + REFLECT_MEMBER(definition_extent); REFLECT_MEMBER(declaring_type); REFLECT_MEMBER(base); REFLECT_MEMBER(locals); diff --git a/src/position.cc b/src/position.cc index beaf126c..568c397b 100644 --- a/src/position.cc +++ b/src/position.cc @@ -139,4 +139,23 @@ bool Range::operator<(const Range& that) const { if (start < that.start) return true; return start == that.start && end < that.end; +} + +// Position +void Reflect(Reader& visitor, Position& value) { + value = Position(visitor.GetString()); +} +void Reflect(Writer& visitor, Position& value) { + std::string output = value.ToString(); + visitor.String(output.c_str(), (rapidjson::SizeType)output.size()); +} + + +// Range +void Reflect(Reader& visitor, Range& value) { + value = Range(visitor.GetString()); +} +void Reflect(Writer& visitor, Range& value) { + std::string output = value.ToString(); + visitor.String(output.c_str(), (rapidjson::SizeType)output.size()); } \ No newline at end of file diff --git a/src/position.h b/src/position.h index d933a84b..b1ccc9b2 100644 --- a/src/position.h +++ b/src/position.h @@ -4,6 +4,8 @@ #include #include +#include "serializer.h" + struct Position { int16_t line = -1; int16_t column = -1; @@ -39,4 +41,10 @@ struct Range { bool operator==(const Range& that) const; bool operator!=(const Range& that) const; bool operator<(const Range& that) const; -}; \ No newline at end of file +}; + +// Reflection +void Reflect(Reader& visitor, Position& value); +void Reflect(Writer& visitor, Position& value); +void Reflect(Reader& visitor, Range& value); +void Reflect(Writer& visitor, Range& value); diff --git a/src/query.cc b/src/query.cc index e409f648..564d3c52 100644 --- a/src/query.cc +++ b/src/query.cc @@ -550,7 +550,13 @@ void IndexUpdate::Merge(const IndexUpdate& update) { #undef INDEX_UPDATE_MERGE } - +std::string IndexUpdate::ToString() { + rapidjson::StringBuffer output; + Writer writer(output); + IndexUpdate& update = *this; + Reflect(writer, update); + return output.GetString(); +} diff --git a/src/query.h b/src/query.h index e9d3f97d..79bbd108 100644 --- a/src/query.h +++ b/src/query.h @@ -29,13 +29,11 @@ struct IdMap; -// TODO: in types, store refs separately from irefs. Then we can drop -// 'interesting' from location when that is cleaned up. - struct QueryLocation { QueryFileId path; Range range; + QueryLocation() {} // Do not use, needed for reflect. QueryLocation(QueryFileId path, Range range) : path(path), range(range) {} @@ -58,13 +56,16 @@ struct QueryLocation { return path == o.path && range < o.range; } }; +MAKE_REFLECT_STRUCT(QueryLocation, path, range); enum class SymbolKind { Invalid, File, Type, Func, Var }; +MAKE_REFLECT_TYPE_PROXY(SymbolKind, int); + struct SymbolIdx { SymbolKind kind; size_t idx; - explicit SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use. + SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use. SymbolIdx(SymbolKind kind, uint64_t idx) : kind(kind), idx(idx) {} bool operator==(const SymbolIdx& that) const { @@ -77,11 +78,13 @@ struct SymbolIdx { return kind == that.kind && idx < that.idx; } }; +MAKE_REFLECT_STRUCT(SymbolIdx, kind, idx); struct SymbolRef { SymbolIdx idx; QueryLocation loc; + SymbolRef() {} // Do not use, needed for reflect. SymbolRef(SymbolIdx idx, QueryLocation loc) : idx(idx), loc(loc) {} bool operator==(const SymbolRef& that) const { @@ -94,6 +97,7 @@ struct SymbolRef { return idx == that.idx && loc.range.start < that.loc.range.start; } }; +MAKE_REFLECT_STRUCT(SymbolRef, idx, loc); struct QueryFuncRef { QueryFuncId id() const { @@ -107,6 +111,7 @@ struct QueryFuncRef { QueryFuncId id_; QueryLocation loc; + QueryFuncRef() {} // Do not use, needed for reflect. QueryFuncRef(QueryFuncId id, QueryLocation loc) : id_(id), loc(loc) {} bool operator==(const QueryFuncRef& that) const { @@ -119,6 +124,7 @@ struct QueryFuncRef { return id_ == that.id_ && loc.range.start < that.loc.range.start; } }; +MAKE_REFLECT_STRUCT(QueryFuncRef, id_, loc); // There are two sources of reindex updates: the (single) definition of a // symbol has changed, or one of many users of the symbol has changed. @@ -143,6 +149,14 @@ struct MergeableUpdate { MergeableUpdate(TId id, const std::vector& to_add, const std::vector& to_remove) : id(id), to_add(to_add), to_remove(to_remove) {} }; +template +void Reflect(TVisitor& visitor, MergeableUpdate& value) { + REFLECT_MEMBER_START(); + REFLECT_MEMBER(id); + REFLECT_MEMBER(to_add); + REFLECT_MEMBER(to_remove); + REFLECT_MEMBER_END(); +} struct QueryFile { struct Def { @@ -160,6 +174,7 @@ struct QueryFile { QueryFile(const std::string& path) { def.path = path; } }; +MAKE_REFLECT_STRUCT(QueryFile::Def, path, outline, all_symbols); struct QueryType { using DefUpdate = TypeDefDefinitionData; @@ -211,6 +226,9 @@ struct IndexUpdate { // work can be parallelized. void Merge(const IndexUpdate& update); + // Dump the update to a string. + std::string ToString(); + // File updates. std::vector files_removed; std::vector files_def_update; @@ -240,7 +258,11 @@ struct IndexUpdate { // will be applied. IndexUpdate(const IdMap& previous_id_map, const IdMap& current_id_map, IndexFile& previous, IndexFile& current); }; - +// NOTICE: We're not reflecting on files_removed or files_def_update, it is too much output when logging +MAKE_REFLECT_STRUCT(IndexUpdate, + types_removed, types_def_update, types_derived, types_instances, types_uses, + funcs_removed, funcs_def_update, funcs_declarations, funcs_derived, funcs_callers, + vars_removed, vars_def_update, vars_uses); // The query database is heavily optimized for fast queries. It is stored // in-memory. diff --git a/src/serializer.cc b/src/serializer.cc index 43c4a23b..11738448 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -6,11 +6,18 @@ namespace { bool gTestOutputMode = false; } // namespace -// int -void Reflect(Reader& visitor, int& value) { +// int16_t +void Reflect(Reader& visitor, int16_t& value) { value = visitor.GetInt(); } -void Reflect(Writer& visitor, int& value) { +void Reflect(Writer& visitor, int16_t& value) { + visitor.Int(value); +} +// int32_t +void Reflect(Reader& visitor, int32_t& value) { + value = visitor.GetInt(); +} +void Reflect(Writer& visitor, int32_t& value) { visitor.Int(value); } // int64_t @@ -20,6 +27,13 @@ void Reflect(Reader& visitor, int64_t& value) { void Reflect(Writer& visitor, int64_t& value) { visitor.Int64(value); } +// uint64_t +void Reflect(Reader& visitor, uint64_t& value) { + value = visitor.GetUint64(); +} +void Reflect(Writer& visitor, uint64_t& value) { + visitor.Uint64(value); +} // bool void Reflect(Reader& visitor, bool& value) { value = visitor.GetBool(); @@ -45,55 +59,6 @@ void ReflectMember(Writer& visitor, const char* name, std::string& value) { } -// Position -void Reflect(Reader& visitor, Position& value) { - value = Position(visitor.GetString()); -} -void Reflect(Writer& visitor, Position& value) { - std::string output = value.ToString(); - visitor.String(output.c_str(), (rapidjson::SizeType)output.size()); -} - - -// Range -void Reflect(Reader& visitor, Range& value) { - value = Range(visitor.GetString()); -} -void Reflect(Writer& visitor, Range& value) { - std::string output = value.ToString(); - visitor.String(output.c_str(), (rapidjson::SizeType)output.size()); -} - -// Id -template -void Reflect(Reader& visitor, Id& id) { - id.id = visitor.GetUint64(); -} -template -void Reflect(Writer& visitor, Id& value) { - visitor.Uint64(value.id); -} - - -// Ref -void Reflect(Reader& visitor, Ref& value) { - const char* str_value = visitor.GetString(); - uint64_t id = atol(str_value); - const char* loc_string = strchr(str_value, '@') + 1; - - value.id_ = Id(id); - value.loc = Range(loc_string); -} -void Reflect(Writer& visitor, Ref& value) { - if (value.id_.id == -1) { - std::string s = "-1@" + value.loc.ToString(); - visitor.String(s.c_str()); - } - else { - std::string s = std::to_string(value.id_.id) + "@" + value.loc.ToString(); - visitor.String(s.c_str()); - } -} @@ -101,9 +66,10 @@ void Reflect(Writer& visitor, Ref& value) { -// TODO: Move this to indexer.cpp -// TODO: Rename indexer.cpp to indexer.cc -// TODO: Do not serialize a USR if it has no usages/etc outside of USR info. + + + +// TODO: Move this to indexer.cc // IndexType bool ReflectMemberStart(Reader& reader, IndexType& value) { diff --git a/src/serializer.h b/src/serializer.h index 9455a1ca..513d937c 100644 --- a/src/serializer.h +++ b/src/serializer.h @@ -98,13 +98,18 @@ void ReflectMemberEnd(TVisitor& visitor, T& value) { */ - -// int -void Reflect(Reader& visitor, int& value); -void Reflect(Writer& visitor, int& value); +// int16_t +void Reflect(Reader& visitor, int16_t& value); +void Reflect(Writer& visitor, int16_t& value); +// int32_t +void Reflect(Reader& visitor, int32_t& value); +void Reflect(Writer& visitor, int32_t& value); // int64_t void Reflect(Reader& visitor, int64_t& value); void Reflect(Writer& visitor, int64_t& value); +// uint64_t +void Reflect(Reader& visitor, uint64_t& value); +void Reflect(Writer& visitor, uint64_t& value); // bool void Reflect(Reader& visitor, bool& value); void Reflect(Writer& visitor, bool& value);