From 8e703af84e9019c381fb10cbaf4b266f2b79aefd Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 3 May 2018 21:20:10 -0700 Subject: [PATCH] . --- src/clang_translation_unit.cc | 2 +- src/clang_utils.cc | 3 +- src/import_pipeline.cc | 4 +- src/indexer.h | 57 +----- src/query.cc | 322 ++++++++-------------------------- src/query.h | 88 +++------- src/utils.cc | 2 +- 7 files changed, 101 insertions(+), 377 deletions(-) diff --git a/src/clang_translation_unit.cc b/src/clang_translation_unit.cc index 6062e417..ffca1b0b 100644 --- a/src/clang_translation_unit.cc +++ b/src/clang_translation_unit.cc @@ -88,7 +88,7 @@ std::unique_ptr ClangTranslationUnit::Create( auto make_msg = [&]() { return "Please try running the following, identify which flag causes the " "issue, and report a bug. ccls will then filter the flag for you " - " automatically:\n$ " + + " automatically:\n " + StringJoin(args, " ") + " -fsyntax-only"; }; diff --git a/src/clang_utils.cc b/src/clang_utils.cc index 07685050..5a25106a 100644 --- a/src/clang_utils.cc +++ b/src/clang_utils.cc @@ -112,7 +112,8 @@ std::string FileName(CXFile file) { ret = ToString(clang_File_tryGetRealPathName(file)); #endif if (ret.empty()) - ret = ToString(clang_getFileName(file)); + // clang_getFileName return values may contain .. + ret = NormalizePath(ToString(clang_getFileName(file))); // Resolve /usr/include/c++/7.3.0 symlink. if (!StartsWith(ret, g_config->projectRoot)) ret = fs::canonical(ret); diff --git a/src/import_pipeline.cc b/src/import_pipeline.cc index 0d1f2cd9..a299f38c 100644 --- a/src/import_pipeline.cc +++ b/src/import_pipeline.cc @@ -355,7 +355,7 @@ void ParseFile(DiagnosticsEngine* diag_engine, return; } - LOG_S(INFO) << "Parsing " << path_to_index; + LOG_S(INFO) << "parse " << path_to_index; std::vector file_contents = PreloadFileContents( request.cache_manager, entry, request.contents, path_to_index); @@ -387,7 +387,7 @@ void ParseFile(DiagnosticsEngine* diag_engine, // When main thread does IdMap request it will request the previous index if // needed. - LOG_S(INFO) << "Emitting index result for " << new_index->path; + LOG_S(INFO) << "emit index for " << new_index->path; result.push_back( Index_OnIdMapped(request.cache_manager, request.cache_manager->TryTakeOrLoad(path_to_index), diff --git a/src/indexer.h b/src/indexer.h index 1d2b1aea..c675c749 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -14,12 +14,11 @@ #include "symbol.h" #include "utils.h" -#include #include #include -#include #include #include +#include #include struct IndexFile; @@ -30,48 +29,6 @@ struct QueryFile; using RawId = uint32_t; -template -struct Id { - RawId id; - - // Invalid id. - Id() : id(-1) {} - explicit Id(RawId id) : id(id) {} - // Id -> Id or Id -> Id is allowed implicitly. - template || std::is_same_v, - bool> = false> - Id(Id o) : id(o.id) {} - template < - typename U, - typename std::enable_if_t || std::is_same_v), - bool> = false> - explicit Id(Id o) : id(o.id) {} - - // Needed for google::dense_hash_map. - explicit operator RawId() const { return id; } - - bool Valid() const { return id != RawId(-1); } - - bool operator==(const Id& o) const { return id == o.id; } - bool operator!=(const Id& o) const { return id != o.id; } - bool operator<(const Id& o) const { return id < o.id; } -}; - -namespace std { -template -struct hash> { - size_t operator()(const Id& k) const { return hash()(k.id); } -}; -} // namespace std - -template -void Reflect(TVisitor& visitor, Id& id) { - Reflect(visitor, id.id); -} - -using IndexFileId = Id; - struct SymbolIdx { Usr usr; SymbolKind kind; @@ -189,8 +146,6 @@ struct IndexFunc : NameMixin { std::vector declarations; std::vector uses; std::vector derived; - - bool operator<(const IndexFunc& other) const { return usr < other.usr; } }; struct TypeDef : NameMixin { @@ -261,14 +216,8 @@ struct IndexType { std::vector uses; std::vector derived; std::vector instances; - - // Every usage, useful for things like renames. - // NOTE: Do not insert directly! Use AddUsage instead. - - bool operator<(const IndexType& other) const { return usr < other.usr; } }; - struct VarDef : NameMixin { // General metadata. std::string detailed_name; @@ -321,8 +270,6 @@ struct IndexVar { Def def; std::vector declarations; std::vector uses; - - bool operator<(const IndexVar& other) const { return usr < other.usr; } }; struct IndexInclude { @@ -434,8 +381,6 @@ struct IIndexer { }; struct ClangIndexer : IIndexer { - ~ClangIndexer() override = default; - std::vector> Index( FileConsumerSharedState* file_consumer_shared, std::string file, diff --git a/src/query.cc b/src/query.cc index cc705890..87fc67eb 100644 --- a/src/query.cc +++ b/src/query.cc @@ -16,55 +16,10 @@ #include #include -// TODO: Make all copy constructors explicit. - // Used by |HANDLE_MERGEABLE| so only |range| is needed. MAKE_HASHABLE(Range, t.start, t.end); MAKE_HASHABLE(Use, t.range); -template -void Reflect(TVisitor& visitor, MergeableUpdate& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER(usr); - REFLECT_MEMBER(to_remove); - REFLECT_MEMBER(to_add); - REFLECT_MEMBER_END(); -} - -template -void Reflect(TVisitor& visitor, WithUsr& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER(usr); - REFLECT_MEMBER(value); - REFLECT_MEMBER_END(); -} - -template -void Reflect(TVisitor& visitor, WithFileContent& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER(value); - REFLECT_MEMBER(file_content); - REFLECT_MEMBER_END(); -} - -// NOTICE: We're not reflecting on files_removed or files_def_update, it is too -// much output when logging -MAKE_REFLECT_STRUCT(IndexUpdate, - types_removed, - types_def_update, - types_derived, - types_instances, - types_uses, - funcs_removed, - funcs_def_update, - funcs_declarations, - funcs_derived, - funcs_uses, - vars_removed, - vars_def_update, - vars_declarations, - vars_uses); - namespace { void AssignFileId(int file_id, SymbolRef& ref) { @@ -117,57 +72,6 @@ void RemoveRange(std::vector& dest, const std::vector& to_remove) { } } -// Compares |previous| and |current|, adding all elements that are in |previous| -// but not |current| to |removed|, and all elements that are in |current| but -// not |previous| to |added|. -// -// Returns true iff |removed| or |added| are non-empty. -template -bool ComputeDifferenceForUpdate(std::vector previous, - std::vector current, - std::vector* removed, - std::vector* added) { - // We need to sort to use std::set_difference. - std::sort(previous.begin(), previous.end()); - std::sort(current.begin(), current.end()); - - auto it0 = previous.begin(), it1 = current.begin(); - while (it0 != previous.end() && it1 != current.end()) { - // Elements in |previous| that are not in |current|. - if (*it0 < *it1) - removed->push_back(std::move(*it0++)); - // Elements in |current| that are not in |previous|. - else if (*it1 < *it0) - added->push_back(std::move(*it1++)); - else - ++it0, ++it1; - } - while (it0 != previous.end()) - removed->push_back(std::move(*it0++)); - while (it1 != current.end()) - added->push_back(std::move(*it1++)); - - return !removed->empty() || !added->empty(); -} - -template -void CompareGroups(std::unordered_map& prev, - std::unordered_map& curr, - std::function on_remove, - std::function on_add, - std::function on_found) { - for (auto& it : prev) { - auto it1 = curr.find(it.first); - if (it1 != curr.end()) - on_found(it.second, it1->second); - else - on_remove(it.second); - } - for (auto& it : curr) - if (!prev.count(it.first)) - on_add(it.second); -} - QueryFile::DefUpdate BuildFileDefUpdate(const IndexFile& indexed) { QueryFile::Def def; def.path = indexed.path; @@ -291,144 +195,60 @@ IndexUpdate IndexUpdate::CreateDelta(IndexFile* previous, if (!previous) previous = ∅ -// |query_name| is the name of the variable on the query type. -// |index_name| is the name of the variable on the index type. -// |type| is the type of the variable. -#define PROCESS_DIFF(type_id, query_name, index_name, type) \ - { \ - /* Check for changes. */ \ - std::vector removed, added; \ - bool did_add = ComputeDifferenceForUpdate( \ - prev.index_name, curr.index_name, &removed, &added); \ - if (did_add) { \ - r.query_name.push_back(MergeableUpdate( \ - curr.usr, std::move(removed), std::move(added))); \ - } \ - } - // File r.files_def_update = BuildFileDefUpdate(*current); - // **NOTE** We only remove entries if they were defined in the previous index. - // For example, if a type is included from another file it will be defined - // simply so we can attribute the usage/reference to it. If the reference goes - // away we don't want to remove the type/func/var usage. + for (auto& it : previous->usr2func) { + auto& func = it.second; + if (func.def.spell) + r.funcs_removed.push_back(func.usr); + r.funcs_declarations[func.usr].first = func.declarations; + r.funcs_uses[func.usr].first = func.uses; + r.funcs_derived[func.usr].first = func.derived; + } + for (auto& it : current->usr2func) { + auto& func = it.second; + if (func.def.detailed_name.size()) + r.funcs_def_update.emplace_back(it.first, func.def); + r.funcs_declarations[func.usr].second = func.declarations; + r.funcs_uses[func.usr].second = func.uses; + r.funcs_derived[func.usr].second = func.derived; + } - // Functions - CompareGroups( - previous->usr2func, current->usr2func, - /*onRemoved:*/ - [&r](IndexFunc& func) { - if (func.def.spell) - r.funcs_removed.push_back(func.usr); - if (func.declarations.size()) - r.funcs_declarations.push_back(UseUpdate{func.usr, func.declarations, {}}); - if (func.uses.size()) - r.funcs_uses.push_back(UseUpdate{func.usr, func.uses, {}}); - if (func.derived.size()) - r.funcs_derived.push_back(UsrUpdate{func.usr, func.derived, {}}); - }, - /*onAdded:*/ - [&r](IndexFunc& func) { - if (func.def.detailed_name.size()) - r.funcs_def_update.emplace_back(func.usr, func.def); - if (func.declarations.size()) - r.funcs_declarations.push_back(UseUpdate{func.usr, {}, func.declarations}); - if (func.uses.size()) - r.funcs_uses.push_back(UseUpdate{func.usr, {}, func.uses}); - if (func.derived.size()) - r.funcs_derived.push_back(UsrUpdate{func.usr, {}, func.derived}); - }, - /*onFound:*/ - [&r](IndexFunc& prev, IndexFunc& curr) { - if (curr.def.detailed_name.size() && !(prev.def == curr.def)) - r.funcs_def_update.emplace_back(curr.usr, curr.def); + for (auto& it : previous->usr2type) { + auto& type = it.second; + if (type.def.spell) + r.types_removed.push_back(type.usr); + r.types_declarations[type.usr].first = type.declarations; + r.types_uses[type.usr].first = type.uses; + r.types_derived[type.usr].first = type.derived; + r.types_instances[type.usr].first = type.instances; + }; + for (auto& it : current->usr2type) { + auto& type = it.second; + if (type.def.detailed_name.size()) + r.types_def_update.emplace_back(it.first, type.def); + r.types_declarations[type.usr].second = type.declarations; + r.types_uses[type.usr].second = type.uses; + r.types_derived[type.usr].second = type.derived; + r.types_instances[type.usr].second = type.instances; + }; - PROCESS_DIFF(QueryFuncId, funcs_declarations, declarations, Use); - PROCESS_DIFF(QueryFuncId, funcs_uses, uses, Use); - PROCESS_DIFF(QueryFuncId, funcs_derived, derived, Usr); - }); - - // Types - CompareGroups( - previous->usr2type, current->usr2type, - /*onRemoved:*/ - [&r](IndexType& type) { - if (type.def.spell) - r.types_removed.push_back(type.usr); - if (type.declarations.size()) - r.types_declarations.push_back(UseUpdate{type.usr, type.declarations, {}}); - if (type.uses.size()) - r.types_uses.push_back(UseUpdate{type.usr, type.uses, {}}); - if (type.derived.size()) - r.types_derived.push_back(UsrUpdate{type.usr, type.derived, {}}); - if (type.instances.size()) - r.types_instances.push_back(UsrUpdate{type.usr, type.instances, {}}); - }, - /*onAdded:*/ - [&r](IndexType& type) { - if (type.def.detailed_name.size()) - r.types_def_update.emplace_back(type.usr, type.def); - if (type.declarations.size()) - r.types_declarations.push_back(UseUpdate{type.usr, {}, type.declarations}); - if (type.uses.size()) - r.types_uses.push_back(UseUpdate{type.usr, {}, type.uses}); - if (type.derived.size()) - r.types_derived.push_back(UsrUpdate{type.usr, {}, type.derived}); - if (type.instances.size()) - r.types_instances.push_back(UsrUpdate{type.usr, {}, type.instances}); - }, - /*onFound:*/ - [&r](IndexType& prev, IndexType& curr) { - if (curr.def.detailed_name.size() && !(prev.def == curr.def)) - r.types_def_update.emplace_back(curr.usr, curr.def); - - PROCESS_DIFF(QueryTypeId, types_declarations, declarations, Use); - PROCESS_DIFF(QueryTypeId, types_uses, uses, Use); - PROCESS_DIFF(QueryTypeId, types_derived, derived, Usr); - PROCESS_DIFF(QueryTypeId, types_instances, instances, Usr); - }); - - // Variables - CompareGroups( - previous->usr2var, current->usr2var, - /*onRemoved:*/ - [&r](IndexVar& var) { - if (var.def.spell) - r.vars_removed.push_back(var.usr); - if (!var.declarations.empty()) - r.vars_declarations.push_back(UseUpdate{var.usr, var.declarations, {}}); - if (!var.uses.empty()) - r.vars_uses.push_back(UseUpdate{var.usr, var.uses, {}}); - }, - /*onAdded:*/ - [&r](IndexVar& var) { - if (var.def.detailed_name.size()) - r.vars_def_update.emplace_back(var.usr, var.def); - if (var.declarations.size()) - r.vars_declarations.push_back(UseUpdate{var.usr, {}, var.declarations}); - if (var.uses.size()) - r.vars_uses.push_back(UseUpdate{var.usr, {}, var.uses}); - }, - /*onFound:*/ - [&r](IndexVar& prev, IndexVar& curr) { - if (curr.def.detailed_name.size() && !(prev.def == curr.def)) - r.vars_def_update.emplace_back(curr.usr, curr.def); - - PROCESS_DIFF(QueryVarId, vars_declarations, declarations, Use); - PROCESS_DIFF(QueryVarId, vars_uses, uses, Use); - }); + for (auto& it : previous->usr2var) { + auto& var = it.second; + if (var.def.spell) + r.vars_removed.push_back(var.usr); + r.vars_declarations[var.usr].first = var.declarations; + r.vars_uses[var.usr].first = var.uses; + } + for (auto& it : current->usr2var) { + auto& var = it.second; + if (var.def.detailed_name.size()) + r.vars_def_update.emplace_back(it.first, var.def); + r.vars_declarations[var.usr].second = var.declarations; + r.vars_uses[var.usr].second = var.uses; + } return r; -#undef PROCESS_DIFF -} - -std::string IndexUpdate::ToString() { - rapidjson::StringBuffer output; - rapidjson::Writer writer(output); - JsonWriter json_writer(&writer); - IndexUpdate& update = *this; - Reflect(json_writer, update); - return output.GetString(); } // ------------------------ @@ -490,11 +310,11 @@ void QueryDatabase::ApplyIndexUpdate(IndexUpdate* u) { // MergeableUpdate def => QueryType // def->def_var_name => std::vector #define HANDLE_MERGEABLE(update_var_name, def_var_name, storage_name) \ - for (auto merge_update : u->update_var_name) { \ - auto& entity = storage_name[merge_update.usr]; \ - AssignFileId(u->file_id, merge_update.to_add); \ - AddRange(u->file_id, entity.def_var_name, merge_update.to_add); \ - RemoveRange(entity.def_var_name, merge_update.to_remove); \ + for (auto& it : u->update_var_name) { \ + auto& entity = storage_name[it.first]; \ + RemoveRange(entity.def_var_name, it.second.first); \ + AssignFileId(u->file_id, it.second.second); \ + AddRange(u->file_id, entity.def_var_name, it.second.second); \ } if (u->files_removed) @@ -534,46 +354,52 @@ int QueryDatabase::Update(QueryFile::DefUpdate&& u) { return existing.id; } -void QueryDatabase::Update(int file_id, std::vector&& updates) { - for (auto& u : updates) { - auto& def = u.value; +void QueryDatabase::Update(int file_id, + std::vector>&& us) { + for (auto& u : us) { + auto& def = u.second; assert(!def.detailed_name.empty()); AssignFileId(file_id, def.spell); AssignFileId(file_id, def.extent); AssignFileId(file_id, def.callees); - QueryFunc& existing = Func(u.usr); + QueryFunc& existing = Func(u.first); + existing.usr = u.first; if (!TryReplaceDef(existing.def, std::move(def))) { existing.def.push_front(std::move(def)); - UpdateSymbols(&existing.symbol_idx, SymbolKind::Type, existing.usr); + UpdateSymbols(&existing.symbol_idx, SymbolKind::Func, u.first); } } } -void QueryDatabase::Update(int file_id, std::vector&& updates) { - for (auto& u : updates) { - auto& def = u.value; +void QueryDatabase::Update(int file_id, + std::vector>&& us) { + for (auto& u : us) { + auto& def = u.second; assert(!def.detailed_name.empty()); AssignFileId(file_id, def.spell); AssignFileId(file_id, def.extent); - QueryType& existing = Type(u.usr); + QueryType& existing = Type(u.first); + existing.usr = u.first; if (!TryReplaceDef(existing.def, std::move(def))) { existing.def.push_front(std::move(def)); - UpdateSymbols(&existing.symbol_idx, SymbolKind::Type, existing.usr); + UpdateSymbols(&existing.symbol_idx, SymbolKind::Type, u.first); } } } -void QueryDatabase::Update(int file_id, std::vector&& updates) { - for (auto& u : updates) { - auto& def = u.value; +void QueryDatabase::Update(int file_id, + std::vector>&& us) { + for (auto& u : us) { + auto& def = u.second; assert(!def.detailed_name.empty()); AssignFileId(file_id, def.spell); AssignFileId(file_id, def.extent); - QueryVar& existing = Var(u.usr); + QueryVar& existing = Var(u.first); + existing.usr = u.first; if (!TryReplaceDef(existing.def, std::move(def))) { existing.def.push_front(std::move(def)); if (!existing.def.front().is_local()) - UpdateSymbols(&existing.symbol_idx, SymbolKind::Var, existing.usr); + UpdateSymbols(&existing.symbol_idx, SymbolKind::Var, u.first); } } } diff --git a/src/query.h b/src/query.h index 18c8ffb6..6cbd9074 100644 --- a/src/query.h +++ b/src/query.h @@ -13,50 +13,6 @@ struct QueryFunc; struct QueryVar; struct QueryDatabase; -using QueryFileId = Id; -using QueryTypeId = Id; -using QueryFuncId = Id; -using QueryVarId = Id; - -struct IdMap; - -// There are two sources of reindex updates: the (single) definition of a -// symbol has changed, or one of many users of the symbol has changed. -// -// For simplicitly, if the single definition has changed, we update all of the -// associated single-owner definition data. See |Update*DefId|. -// -// If one of the many symbol users submits an update, we store the update such -// that it can be merged with other updates before actually being applied to -// the main database. See |MergeableUpdate|. - -template -struct MergeableUpdate { - // The type/func/var which is getting new usages. - Usr usr; - // Entries to add and remove. - std::vector to_remove; - std::vector to_add; - - MergeableUpdate(Usr usr, - std::vector&& to_remove, - std::vector&& to_add) - : usr(usr), to_remove(std::move(to_remove)), to_add(std::move(to_add)) {} - MergeableUpdate(Usr usr, - const std::vector& to_remove, - const std::vector& to_add) - : usr(usr), to_remove(to_remove), to_add(to_add) {} -}; - -template -struct WithUsr { - Usr usr; - T value; - - WithUsr(Usr usr, const T& value) : usr(usr), value(value) {} - WithUsr(Usr usr, T&& value) : usr(usr), value(value) {} -}; - template struct WithFileContent { T value; @@ -94,7 +50,6 @@ struct QueryFile { template struct QueryEntity { using Def = QDef; - using DefUpdate = WithUsr; Def* AnyDef() { Def* ret = nullptr; for (auto& i : static_cast(this)->def) { @@ -107,8 +62,10 @@ struct QueryEntity { const Def* AnyDef() const { return const_cast(this)->AnyDef(); } }; -using UsrUpdate = MergeableUpdate; -using UseUpdate = MergeableUpdate; +using UseUpdate = + std::unordered_map, std::vector>>; +using UsrUpdate = + std::unordered_map, std::vector>>; struct QueryFunc : QueryEntity { Usr usr; @@ -147,9 +104,6 @@ struct IndexUpdate { // work can be parallelized. void Merge(IndexUpdate&& update); - // Dump the update to a string. - std::string ToString(); - int file_id; // File updates. @@ -158,24 +112,24 @@ struct IndexUpdate { // Function updates. std::vector funcs_removed; - std::vector funcs_def_update; - std::vector funcs_declarations; - std::vector funcs_uses; - std::vector funcs_derived; + std::vector> funcs_def_update; + UseUpdate funcs_declarations; + UseUpdate funcs_uses; + UsrUpdate funcs_derived; // Type updates. std::vector types_removed; - std::vector types_def_update; - std::vector types_declarations; - std::vector types_uses; - std::vector types_derived; - std::vector types_instances; + std::vector> types_def_update; + UseUpdate types_declarations; + UseUpdate types_uses; + UsrUpdate types_derived; + UsrUpdate types_instances; // Variable updates. std::vector vars_removed; - std::vector vars_def_update; - std::vector vars_declarations; - std::vector vars_uses; + std::vector> vars_def_update; + UseUpdate vars_declarations; + UseUpdate vars_uses; }; // The query database is heavily optimized for fast queries. It is stored @@ -196,12 +150,10 @@ struct QueryDatabase { // Insert the contents of |update| into |db|. void ApplyIndexUpdate(IndexUpdate* update); int Update(QueryFile::DefUpdate&& u); - void Update(int file_id, std::vector&& updates); - void Update(int file_id, std::vector&& updates); - void Update(int file_id, std::vector&& updates); - void UpdateSymbols(int* symbol_idx, - SymbolKind kind, - Usr usr); + void Update(int file_id, std::vector>&& us); + void Update(int file_id, std::vector>&& us); + void Update(int file_id, std::vector>&& us); + void UpdateSymbols(int* symbol_idx, SymbolKind kind, Usr usr); std::string_view GetSymbolName(int symbol_idx, bool qualified); QueryFile& GetFile(SymbolIdx ref) { return files[ref.usr]; } diff --git a/src/utils.cc b/src/utils.cc index c9d05a8d..cf63245f 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -123,7 +123,7 @@ std::string EscapeFileName(std::string path) { } std::optional ReadContent(const std::string& filename) { - LOG_S(INFO) << "Reading " << filename; + LOG_S(INFO) << "read " << filename; char buf[4096]; std::string ret; FILE* f = fopen(filename.c_str(), "rb");