From 18aa28bdea347d0af19504a08e71cc42d1fcdfeb Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Sun, 26 Feb 2017 18:03:14 -0800 Subject: [PATCH] building --- .gitignore | 3 ++ indexer.cpp | 41 ++++++++++---------- indexer.h | 105 ++++++++++++++++++++++++---------------------------- query.cc | 46 +++++++---------------- query.h | 72 ++++++++++++++++++++++++----------- wscript | 1 + 6 files changed, 137 insertions(+), 131 deletions(-) diff --git a/.gitignore b/.gitignore index a857ae44..52e66609 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ .vs Debug x64 +build +waf-* +.lock-waf* *.swp *.sln *.vcxproj diff --git a/indexer.cpp b/indexer.cpp index 3f0652fb..3b00a661 100644 --- a/indexer.cpp +++ b/indexer.cpp @@ -2,8 +2,8 @@ #include "serializer.h" -IndexedFile::IndexedFile(IdCache* id_cache, FileDb* file_db) - : id_cache(id_cache), file_db(file_db) { +IndexedFile::IndexedFile(IdCache* id_cache) + : id_cache(id_cache) { // TODO: Reconsider if we should still be reusing the same id_cache. // Preallocate any existing resolved ids. @@ -322,7 +322,7 @@ void VisitDeclForTypeUsageVisitorHandler(clang::Cursor cursor, VisitDeclForTypeU if (param->is_interesting) { IndexedTypeDef* ref_type_def = db->Resolve(ref_type_id); - Location loc = db->file_db->Resolve(cursor, true /*interesting*/); + Location loc = db->id_cache->Resolve(cursor, true /*interesting*/); ref_type_def->AddUsage(loc); } } @@ -445,7 +445,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) { var_def->def.qualified_name = ns->QualifiedName(decl->semanticContainer, var_def->def.short_name); //} - Location decl_loc = db->file_db->Resolve(decl->loc, false /*interesting*/); + Location decl_loc = db->id_cache->Resolve(decl->loc, false /*interesting*/); if (decl->isDefinition) var_def->def.definition = decl_loc; else @@ -491,7 +491,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) { func_def->def.qualified_name = ns->QualifiedName(decl->semanticContainer, func_def->def.short_name); //} - Location decl_loc = db->file_db->Resolve(decl->loc, false /*interesting*/); + Location decl_loc = db->id_cache->Resolve(decl->loc, false /*interesting*/); if (decl->isDefinition) func_def->def.definition = decl_loc; else @@ -608,7 +608,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) { type_def->def.short_name = decl->entityInfo->name; type_def->def.qualified_name = ns->QualifiedName(decl->semanticContainer, type_def->def.short_name); - Location decl_loc = db->file_db->Resolve(decl->loc, true /*interesting*/); + Location decl_loc = db->id_cache->Resolve(decl->loc, true /*interesting*/); type_def->def.definition = decl_loc.WithInteresting(false); type_def->AddUsage(decl_loc); break; @@ -642,7 +642,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) { // } assert(decl->isDefinition); - Location decl_loc = db->file_db->Resolve(decl->loc, true /*interesting*/); + Location decl_loc = db->id_cache->Resolve(decl->loc, true /*interesting*/); type_def->def.definition = decl_loc.WithInteresting(false); type_def->AddUsage(decl_loc); @@ -671,7 +671,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) { } default: - std::cout << "!! Unhandled indexDeclaration: " << clang::Cursor(decl->cursor).ToString() << " at " << db->file_db->Resolve(decl->loc, false /*interesting*/).ToString() << std::endl; + std::cout << "!! Unhandled indexDeclaration: " << clang::Cursor(decl->cursor).ToString() << " at " << db->id_cache->Resolve(decl->loc, false /*interesting*/).ToString() << std::endl; std::cout << " entityInfo->kind = " << decl->entityInfo->kind << std::endl; std::cout << " entityInfo->USR = " << decl->entityInfo->USR << std::endl; if (decl->declAsContainer) @@ -707,7 +707,7 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re { VarId var_id = db->ToVarId(ref->referencedEntity->cursor); IndexedVarDef* var_def = db->Resolve(var_id); - var_def->uses.push_back(db->file_db->Resolve(ref->loc, false /*interesting*/)); + var_def->uses.push_back(db->id_cache->Resolve(ref->loc, false /*interesting*/)); break; } @@ -729,7 +729,7 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re // Don't report duplicate usages. // TODO: search full history? - Location loc = db->file_db->Resolve(ref->loc, false /*interesting*/); + Location loc = db->id_cache->Resolve(ref->loc, false /*interesting*/); if (param->last_func_usage_location == loc) break; param->last_func_usage_location = loc; @@ -757,8 +757,8 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re if (ref->referencedEntity->kind == CXIdxEntity_CXXConstructor || ref->referencedEntity->kind == CXIdxEntity_CXXDestructor) { - Location parent_loc = db->file_db->Resolve(ref->parentEntity->cursor, true /*interesting*/); - Location our_loc = db->file_db->Resolve(ref->loc, true /*is_interesting*/); + Location parent_loc = db->id_cache->Resolve(ref->parentEntity->cursor, true /*interesting*/); + Location our_loc = db->id_cache->Resolve(ref->loc, true /*is_interesting*/); if (!parent_loc.IsEqualTo(our_loc)) { IndexedFuncDef* called_def = db->Resolve(called_id); assert(called_def->def.declaring_type.has_value()); @@ -778,12 +778,12 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re { TypeId referenced_id = db->ToTypeId(ref->referencedEntity->USR); IndexedTypeDef* referenced_def = db->Resolve(referenced_id); - + // We will not get a declaration visit for forward declared types. Try to mark them as non-bad // defs here so we will output usages/etc. if (referenced_def->is_bad_def) { bool is_system_def = clang_Location_isInSystemHeader(clang_getCursorLocation(ref->referencedEntity->cursor)); - Location loc = db->file_db->Resolve(ref->referencedEntity->cursor, false /*interesting*/); + Location loc = db->id_cache->Resolve(ref->referencedEntity->cursor, false /*interesting*/); if (!is_system_def && loc.raw_file_id != -1) referenced_def->is_bad_def = false; } @@ -802,16 +802,16 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re // Foo f; // } // - referenced_def->AddUsage(db->file_db->Resolve(ref->loc, false /*interesting*/)); + referenced_def->AddUsage(db->id_cache->Resolve(ref->loc, false /*interesting*/)); break; } default: - std::cout << "!! Unhandled indexEntityReference: " << cursor.ToString() << " at " << db->file_db->Resolve(ref->loc, false /*interesting*/).ToString() << std::endl; + std::cout << "!! Unhandled indexEntityReference: " << cursor.ToString() << " at " << db->id_cache->Resolve(ref->loc, false /*interesting*/).ToString() << std::endl; std::cout << " ref->referencedEntity->kind = " << ref->referencedEntity->kind << std::endl; if (ref->parentEntity) std::cout << " ref->parentEntity->kind = " << ref->parentEntity->kind << std::endl; - std::cout << " ref->loc = " << db->file_db->Resolve(ref->loc, false /*interesting*/).ToString() << std::endl; + std::cout << " ref->loc = " << db->id_cache->Resolve(ref->loc, false /*interesting*/).ToString() << std::endl; std::cout << " ref->kind = " << ref->kind << std::endl; if (ref->parentEntity) std::cout << " parentEntity = " << clang::Cursor(ref->parentEntity->cursor).ToString() << std::endl; @@ -826,7 +826,7 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re static bool DUMP_AST = true; -IndexedFile Parse(IdCache* id_cache, FileDb* file_db, std::string filename, std::vector args) { +IndexedFile Parse(IdCache* id_cache, std::string filename, std::vector args) { clang::Index index(0 /*excludeDeclarationsFromPCH*/, 0 /*displayDiagnostics*/); clang::TranslationUnit tu(index, filename, args); @@ -849,7 +849,7 @@ IndexedFile Parse(IdCache* id_cache, FileDb* file_db, std::string filename, std: */ }; - IndexedFile db(id_cache, file_db); + IndexedFile db(id_cache); NamespaceHelper ns; IndexParam param(&db, &ns); clang_indexTranslationUnit(index_action, ¶m, callbacks, sizeof(callbacks), @@ -991,8 +991,7 @@ int main55555(int argc, char** argv) { // Run test. std::cout << "[START] " << path << std::endl; IdCache id_cache(1); - FileDb file_db(1); - IndexedFile db = Parse(&id_cache, &file_db, path, {}); + IndexedFile db = Parse(&id_cache, path, {}); std::string actual_output = db.ToString(); //WriteToFile("output.json", actual_output); diff --git a/indexer.h b/indexer.h index cc98a4b5..655c852c 100644 --- a/indexer.h +++ b/indexer.h @@ -209,55 +209,6 @@ Location WithInteresting(bool interesting) { END_BITFIELD_TYPE() #endif -struct FileDb { - GroupId group; - std::unordered_map file_path_to_file_id; - std::unordered_map file_id_to_file_path; - - FileDb(GroupId group) : group(group) { - // Reserve id 0 for unfound. - file_path_to_file_id[""] = FileId(group, 0); - file_id_to_file_path[FileId(group, 0)] = ""; - } - - Location Resolve(const CXSourceLocation& cx_loc, bool interesting) { - CXFile file; - unsigned int line, column, offset; - clang_getSpellingLocation(cx_loc, &file, &line, &column, &offset); - - FileId file_id(-1, -1); - if (file != nullptr) { - std::string path = clang::ToString(clang_getFileName(file)); - - auto it = file_path_to_file_id.find(path); - if (it != file_path_to_file_id.end()) { - file_id = it->second; - } - else { - file_id = FileId(group, file_path_to_file_id.size()); - file_path_to_file_id[path] = file_id; - file_id_to_file_path[file_id] = path; - } - } - - return Location(interesting, file_id, line, column); - } - - Location Resolve(const CXIdxLoc& cx_idx_loc, bool interesting) { - CXSourceLocation cx_loc = clang_indexLoc_getCXSourceLocation(cx_idx_loc); - return Resolve(cx_loc, interesting); - } - - Location Resolve(const CXCursor& cx_cursor, bool interesting) { - return Resolve(clang_getCursorLocation(cx_cursor), interesting); - } - - Location Resolve(const clang::Cursor& cursor, bool interesting) { - return Resolve(cursor.cx_cursor, interesting); - } -}; - - template struct Ref { Id id; @@ -295,7 +246,7 @@ using VarRef = Ref; // TODO: Either eliminate the defs created as a by-product of cross-referencing, // or do not emit things we don't have definitions for. -template +template struct TypeDefDefinitionData { // General metadata. TypeId id; @@ -376,7 +327,7 @@ namespace std { }; } -template +template struct FuncDefDefinitionData { // General metadata. FuncId id; @@ -459,7 +410,7 @@ namespace std { }; } -template +template struct VarDefDefinitionData { // General metadata. VarId id; @@ -526,25 +477,67 @@ struct IdCache { // NOTE: Every Id is resolved to a file_id of 0. The correct file_id needs // to get fixed up when inserting into the real db. GroupId group; + std::unordered_map file_path_to_file_id; std::unordered_map usr_to_type_id; std::unordered_map usr_to_func_id; std::unordered_map usr_to_var_id; + std::unordered_map file_id_to_file_path; std::unordered_map type_id_to_usr; std::unordered_map func_id_to_usr; std::unordered_map var_id_to_usr; - IdCache(GroupId group) : group(group) {} + IdCache(GroupId group) : group(group) { + // Reserve id 0 for unfound. + file_path_to_file_id[""] = FileId(group, 0); + file_id_to_file_path[FileId(group, 0)] = ""; + } + + Location Resolve(const CXSourceLocation& cx_loc, bool interesting) { + CXFile file; + unsigned int line, column, offset; + clang_getSpellingLocation(cx_loc, &file, &line, &column, &offset); + + FileId file_id(-1, -1); + if (file != nullptr) { + std::string path = clang::ToString(clang_getFileName(file)); + + auto it = file_path_to_file_id.find(path); + if (it != file_path_to_file_id.end()) { + file_id = it->second; + } + else { + file_id = FileId(group, file_path_to_file_id.size()); + file_path_to_file_id[path] = file_id; + file_id_to_file_path[file_id] = path; + } + } + + return Location(interesting, file_id, line, column); + } + + Location Resolve(const CXIdxLoc& cx_idx_loc, bool interesting) { + CXSourceLocation cx_loc = clang_indexLoc_getCXSourceLocation(cx_idx_loc); + return Resolve(cx_loc, interesting); + } + + Location Resolve(const CXCursor& cx_cursor, bool interesting) { + return Resolve(clang_getCursorLocation(cx_cursor), interesting); + } + + Location Resolve(const clang::Cursor& cursor, bool interesting) { + return Resolve(cursor.cx_cursor, interesting); + } + }; struct IndexedFile { - FileDb* file_db; IdCache* id_cache; std::vector types; std::vector funcs; std::vector vars; - IndexedFile(IdCache* id_cache, FileDb* file_db); + IndexedFile(IdCache* id_cache); TypeId ToTypeId(const std::string& usr); FuncId ToFuncId(const std::string& usr); @@ -562,4 +555,4 @@ struct IndexedFile { -IndexedFile Parse(IdCache* id_cache, FileDb* file_db, std::string filename, std::vector args); \ No newline at end of file +IndexedFile Parse(IdCache* id_cache, std::string filename, std::vector args); \ No newline at end of file diff --git a/query.cc b/query.cc index 58f92d24..b11a2377 100644 --- a/query.cc +++ b/query.cc @@ -290,23 +290,6 @@ struct IdMap { -// TODO: Switch over to QueryableLocation. Figure out if there is -// a good way to get the indexer using it. I don't think so -// since we may discover more files while indexing a file. -// -// We could also reuse planned USR caching system for file -// paths. -struct CachedFileDb { - using Id = int64_t; - std::vector file_names; -}; - -struct QueryableLocation { - CachedFileDb::Id id; - int line; - int column; - bool is_interesting; -}; @@ -337,7 +320,9 @@ Usr MapIdToUsr(IdCache& id_cache, FuncId& id) { Usr MapIdToUsr(IdCache& id_cache, VarId& id) { return id_cache.var_id_to_usr[id]; } -Location MapIdToUsr(IdCache& id_cache, Location& ids); // FIXME: We will need additional data to map locations. +QueryableLocation MapIdToUsr(IdCache& id_cache, Location& id) { + return QueryableLocation(id_cache.file_id_to_file_path[id.file_id()], id.line, id.column, id.interesting); +} std::vector MapIdToUsr(IdCache& id_cache, std::vector& ids) { return Transform(ids, [&](TypeId id) { return id_cache.type_id_to_usr[id]; }); @@ -351,12 +336,16 @@ std::vector MapIdToUsr(IdCache& id_cache, std::vector& ids) { std::vector MapIdToUsr(IdCache& id_cache, std::vector& ids) { return Transform(ids, [&](FuncRef ref) { UsrRef result; - result.loc = ref.loc; // FIXME: Patch proper location. Fix when fixing MapIdToUsr(Location). I'm thinking we will have a GlobalLocation type. + result.loc = MapIdToUsr(id_cache, ref.loc); result.usr = id_cache.func_id_to_usr[ref.id]; return result; }); } -std::vector MapIdToUsr(IdCache& id_cache, std::vector& ids); // FIXME: We will need additional data to map locations. +std::vector MapIdToUsr(IdCache& id_cache, std::vector& ids) { + return Transform(ids, [&](Location id) { + return QueryableLocation(id_cache.file_id_to_file_path[id.file_id()], id.line, id.column, id.interesting); + }); +} QueryableTypeDef::DefUpdate MapIdToUsr(IdCache& id_cache, TypeDefDefinitionData<>& def) { QueryableTypeDef::DefUpdate result(def.usr, def.usr); if (result.definition) @@ -788,6 +777,7 @@ struct QueryableDatabase { std::vector symbols; // Raw data storage. + std::vector files; // File path is stored as a Usr. std::vector types; std::vector funcs; std::vector vars; @@ -795,13 +785,6 @@ struct QueryableDatabase { // Lookup symbol based on a usr. std::unordered_map usr_to_symbol; - // |files| is indexed by FileId. Retrieve a FileId from a path using - // |file_db|. - FileDb file_db; - std::vector files; - - QueryableDatabase(GroupId group); - // Insert the contents of |update| into |db|. void ApplyIndexUpdate(IndexUpdate* update); @@ -909,8 +892,6 @@ void ApplyUpdates(std::unordered_map* id_map, std::vector* defs, } } -QueryableDatabase::QueryableDatabase(GroupId group) : file_db(group) {} - void QueryableDatabase::ApplyIndexUpdate(IndexUpdate* update) { #define JOIN(a, b) a##b #define HANDLE_MERGEABLE(update_var_name, def_var_name, storage_name) \ @@ -956,13 +937,12 @@ void QueryableDatabase::ApplyIndexUpdate(IndexUpdate* update) { int main(int argc, char** argv) { // TODO: Unify UserToIdResolver and FileDb IdCache id_cache(1); - FileDb file_db(1); - IndexedFile indexed_file_a = Parse(&id_cache, &file_db, "full_tests/index_delta/a_v0.cc", {}); + IndexedFile indexed_file_a = Parse(&id_cache, "full_tests/index_delta/a_v0.cc", {}); std::cout << indexed_file_a.ToString() << std::endl; std::cout << std::endl; - IndexedFile indexed_file_b = Parse(&id_cache, &file_db, "full_tests/index_delta/a_v1.cc", {}); + IndexedFile indexed_file_b = Parse(&id_cache, "full_tests/index_delta/a_v1.cc", {}); std::cout << indexed_file_b.ToString() << std::endl; // TODO: We don't need to do ID remapping when computting a diff. Well, we need to do it for the IndexUpdate. IndexUpdate import(indexed_file_a); @@ -970,7 +950,7 @@ int main(int argc, char** argv) { dest_ids.Import(indexed_file_b.file_db, indexed_file_b.id_cache); IndexUpdate update = ComputeDiff(indexed_file_a, indexed_file_b); */ - QueryableDatabase db(5); + QueryableDatabase db; db.ApplyIndexUpdate(&import); //db.ApplyIndexUpdate(&update); diff --git a/query.h b/query.h index 7f30edeb..f3610f65 100644 --- a/query.h +++ b/query.h @@ -23,15 +23,45 @@ enum class PreferredSymbolLocation { }; using Usr = std::string; + +// TODO: Switch over to QueryableLocation. Figure out if there is +// a good way to get the indexer using it. I don't think so +// since we may discover more files while indexing a file. +// +// We could also reuse planned USR caching system for file +// paths. +struct QueryableLocation { + Usr path; + int line; + int column; + bool interesting; + + QueryableLocation() + : path(""), line(-1), column(-1), interesting(false) {} + QueryableLocation(Usr path, int line, int column, bool interesting) + : path(path), line(line), column(column), interesting(interesting) {} + + bool operator==(const QueryableLocation& other) const { + // Note: We ignore |is_interesting|. + return + path == other.path && + line == other.line && + column == other.column; + } +}; + + struct UsrRef { Usr usr; - Location loc; + QueryableLocation loc; bool operator==(const UsrRef& other) const { return usr == other.usr && loc == other.loc; } }; + + // There are two sources of reindex updates: the (single) definition of a // symbol has changed, or one of many users of the symbol has changed. // @@ -52,45 +82,45 @@ struct MergeableUpdate { }; struct QueryableTypeDef { - TypeDefDefinitionData def; - std::vector derived; - std::vector uses; - - using DefUpdate = TypeDefDefinitionData; + using DefUpdate = TypeDefDefinitionData; using DerivedUpdate = MergeableUpdate; - using UsesUpdate = MergeableUpdate; + using UsesUpdate = MergeableUpdate; + + DefUpdate def; + std::vector derived; + std::vector uses; QueryableTypeDef(IdCache& id_cache, IndexedTypeDef& indexed); }; struct QueryableFuncDef { - FuncDefDefinitionData def; - std::vector declarations; - std::vector derived; - std::vector callers; - std::vector uses; - - using DefUpdate = FuncDefDefinitionData; - using DeclarationsUpdate = MergeableUpdate; + using DefUpdate = FuncDefDefinitionData; + using DeclarationsUpdate = MergeableUpdate; using DerivedUpdate = MergeableUpdate; using CallersUpdate = MergeableUpdate; - using UsesUpdate = MergeableUpdate; + using UsesUpdate = MergeableUpdate; + + DefUpdate def; + std::vector declarations; + std::vector derived; + std::vector callers; + std::vector uses; QueryableFuncDef(IdCache& id_cache, IndexedFuncDef& indexed); }; struct QueryableVarDef { - VarDefDefinitionData def; - std::vector uses; + using DefUpdate = VarDefDefinitionData; + using UsesUpdate = MergeableUpdate; - using DefUpdate = VarDefDefinitionData; - using UsesUpdate = MergeableUpdate; + DefUpdate def; + std::vector uses; QueryableVarDef(IdCache& id_cache, IndexedVarDef& indexed); }; -enum class SymbolKind { Invalid, Type, Func, Var }; +enum class SymbolKind { Invalid, File, Type, Func, Var }; struct SymbolIdx { SymbolKind kind; uint64_t idx; diff --git a/wscript b/wscript index 058be2b2..d23ab560 100644 --- a/wscript +++ b/wscript @@ -17,6 +17,7 @@ def configure(conf): # https://github.com/Andersbakken/rtags/blob/master/scripts/getclang.sh def download_clang(): #'http://releases.llvm.org/3.9.0/clang+llvm-3.9.0-x86_64-apple-darwin.tar.xz' + pass def build(bld): cc_files = bld.path.ant_glob(['**/*.cpp', '**/*.cc'],