#pragma once #include "indexer.h" #include "serializer.h" using Usr = std::string; // TODO: Switch over to QueryableLocation. Figure out if there is // a good way to get the indexer using it. I don't think so // since we may discover more files while indexing a file. // // We could also reuse planned USR caching system for file // paths. struct QueryableLocation { Usr path; int line; int column; bool interesting; QueryableLocation() : path(""), line(-1), column(-1), interesting(false) {} QueryableLocation(Usr path, int line, int column, bool interesting) : path(path), line(line), column(column), interesting(interesting) {} QueryableLocation OffsetColumn(int offset) const { return QueryableLocation(path, line, column + offset, interesting); } bool operator==(const QueryableLocation& other) const { // Note: We ignore |is_interesting|. return path == other.path && line == other.line && column == other.column; } bool operator!=(const QueryableLocation& other) const { return !(*this == other); } bool operator<(const QueryableLocation& o) const { return path < o.path && line < o.line && column < o.column && interesting < o.interesting; } }; MAKE_REFLECT_STRUCT(QueryableLocation, path, line, column, interesting); struct UsrRef { Usr usr; QueryableLocation loc; UsrRef() {} UsrRef(Usr usr, QueryableLocation loc) : usr(usr), loc(loc) {} bool operator==(const UsrRef& other) const { return usr == other.usr && loc == other.loc; } bool operator!=(const UsrRef& other) const { return !(*this == other); } bool operator<(const UsrRef& other) const { return usr < other.usr && loc < other.loc; } }; MAKE_REFLECT_STRUCT(UsrRef, usr, loc); // There are two sources of reindex updates: the (single) definition of a // symbol has changed, or one of many users of the symbol has changed. // // For simplicitly, if the single definition has changed, we update all of the // associated single-owner definition data. See |Update*DefId|. // // If one of the many symbol users submits an update, we store the update such // that it can be merged with other updates before actually being applied to // the main database. See |MergeableUpdate|. template struct MergeableUpdate { // The type/func/var which is getting new usages. Usr usr; // Entries to add and remove. std::vector to_add; std::vector to_remove; MergeableUpdate() {} // For reflection MergeableUpdate(Usr usr, const std::vector& to_add, const std::vector& to_remove) : usr(usr), to_add(to_add), to_remove(to_remove) {} }; template void Reflect(TVisitor& visitor, MergeableUpdate& value) { REFLECT_MEMBER_START(); REFLECT_MEMBER(usr); REFLECT_MEMBER(to_add); REFLECT_MEMBER(to_remove); REFLECT_MEMBER_END(); } struct QueryableFile { using OutlineUpdate = MergeableUpdate; Usr file_id; // Outline of the file (ie, all symbols). std::vector outline; QueryableFile() {} // For serialization. QueryableFile(const IndexedFile& indexed); }; MAKE_REFLECT_STRUCT(QueryableFile, file_id, outline); struct QueryableTypeDef { using DefUpdate = TypeDefDefinitionData; using DerivedUpdate = MergeableUpdate; using InstantiationsUpdate = MergeableUpdate; using UsesUpdate = MergeableUpdate; DefUpdate def; std::vector derived; std::vector instantiations; std::vector uses; QueryableTypeDef() : def("") {} // For serialization. QueryableTypeDef(IdCache& id_cache, const IndexedTypeDef& indexed); }; MAKE_REFLECT_STRUCT(QueryableTypeDef, def, derived, instantiations, uses); struct QueryableFuncDef { using DefUpdate = FuncDefDefinitionData; using DeclarationsUpdate = MergeableUpdate; using DerivedUpdate = MergeableUpdate; using CallersUpdate = MergeableUpdate; using UsesUpdate = MergeableUpdate; DefUpdate def; std::vector declarations; std::vector derived; std::vector callers; std::vector uses; QueryableFuncDef() : def("") {} // For serialization. QueryableFuncDef(IdCache& id_cache, const IndexedFuncDef& indexed); }; MAKE_REFLECT_STRUCT(QueryableFuncDef, def, declarations, derived, callers, uses); struct QueryableVarDef { using DefUpdate = VarDefDefinitionData; using UsesUpdate = MergeableUpdate; DefUpdate def; std::vector uses; QueryableVarDef() : def("") {} // For serialization. QueryableVarDef(IdCache& id_cache, const IndexedVarDef& indexed); }; MAKE_REFLECT_STRUCT(QueryableVarDef, def, uses); enum class SymbolKind { Invalid, File, Type, Func, Var }; struct SymbolIdx { SymbolKind kind; uint64_t idx; SymbolIdx() : kind(SymbolKind::Invalid), idx(-1) {} // Default ctor needed by stdlib. Do not use. SymbolIdx(SymbolKind kind, uint64_t idx) : kind(kind), idx(idx) {} }; // TODO: We need to control Usr, std::vector allocation to make sure it happens on shmem. That or we // make IndexUpdate a POD type. // TODO: Instead of all of that work above, we pipe the IndexUpdate across processes as JSON. // We need to verify we need multiple processes first. Maybe libclang can run in a single process... // TODO: Compute IndexUpdates in main process, off the blocking thread. Use separate process for running // libclang. Solves memory worries. // TODO: Instead of passing to/from json, we can probably bass the IndexedFile type almost directly as // a raw memory dump - the type has almost zero pointers inside of it. We could do a little bit of fixup // so that passing from a separate process to the main db is really fast (no need to go through JSON). /* namespace foo2 { using Usr = size_t; struct UsrTable { size_t allocated; size_t used; const char* usrs[]; }; } */ struct IndexUpdate { // File updates. std::vector files_removed; std::vector files_added; std::vector files_outline; // Type updates. std::vector types_removed; std::vector types_added; std::vector types_def_changed; std::vector types_derived; std::vector types_instantiations; std::vector types_uses; // Function updates. std::vector funcs_removed; std::vector funcs_added; std::vector funcs_def_changed; std::vector funcs_declarations; std::vector funcs_derived; std::vector funcs_callers; std::vector funcs_uses; // Variable updates. std::vector vars_removed; std::vector vars_added; std::vector vars_def_changed; std::vector vars_uses; IndexUpdate() {} // Creates a new IndexUpdate that will import |file|. explicit IndexUpdate(IndexedFile& file); // Creates an index update assuming that |previous| is already // in the index, so only the delta between |previous| and |current| // will be applied. IndexUpdate(IndexedFile& previous, IndexedFile& current); // Merges the contents of |update| into this IndexUpdate instance. void Merge(const IndexUpdate& update); }; MAKE_REFLECT_STRUCT(IndexUpdate, files_removed, files_added, files_outline, types_removed, types_added, types_def_changed, types_derived, types_uses, funcs_removed, funcs_added, funcs_def_changed, funcs_declarations, funcs_derived, funcs_callers, funcs_uses, vars_removed, vars_added, vars_def_changed, vars_uses); // The query database is heavily optimized for fast queries. It is stored // in-memory. struct QueryableDatabase { // Indicies between lookup vectors are related to symbols, ie, index 5 in // |qualified_names| matches index 5 in |symbols|. std::vector qualified_names; std::vector symbols; // Raw data storage. std::vector files; // File path is stored as a Usr. std::vector types; std::vector funcs; std::vector vars; // Lookup symbol based on a usr. std::unordered_map usr_to_symbol; // Insert the contents of |update| into |db|. void ApplyIndexUpdate(IndexUpdate* update); void RemoveUsrs(const std::vector& to_remove); void Import(const std::vector& defs); void Import(const std::vector& defs); void Import(const std::vector& defs); void Import(const std::vector& defs); void Update(const std::vector& updates); void Update(const std::vector& updates); void Update(const std::vector& updates); }; // TODO: For supporting vscode, lets' // - have our normal daemon system // - have frontend --language-server which accepts JSON RPC language server in stdin and emits language server // JSON in stdout. vscode extension will run the executable this way. it will connect to daemon as normal. // this means that vscode instance can be killed without actually killing core indexer process. // $ indexer --language-server // - maybe? have simple front end which lets user run // $ indexer --action references --location foo.cc:20:5 // // // https://github.com/Microsoft/vscode-languageserver-node/blob/master/client/src/main.ts