From 56f0b3a90a67c8e2432eaeb48a75d9ed56c61fb9 Mon Sep 17 00:00:00 2001 From: Jacob Dufault Date: Sat, 8 Apr 2017 15:54:36 -0700 Subject: [PATCH] Auto-index header files --- ...e_commands.json => compile_commands2.json} | 0 src/command_line.cc | 30 +- src/file_consumer.cc | 23 +- src/file_consumer.h | 19 +- src/indexer.cc | 69 ++-- src/indexer.h | 3 +- src/project.cc | 2 +- src/query.cc | 3 +- src/test.cc | 103 ++--- src/utils.cc | 44 ++- src/utils.h | 7 +- tests/multi_file/header.h | 109 +----- tests/multi_file/impl.cc | 365 ++++++------------ tests/multi_file/simple_header.h | 3 + tests/multi_file/simple_impl.cc | 38 ++ .../usage/func_called_from_macro_argument.cc | 4 +- 16 files changed, 361 insertions(+), 461 deletions(-) rename full_tests/simple_cross_reference/{compile_commands.json => compile_commands2.json} (100%) create mode 100644 tests/multi_file/simple_header.h create mode 100644 tests/multi_file/simple_impl.cc diff --git a/full_tests/simple_cross_reference/compile_commands.json b/full_tests/simple_cross_reference/compile_commands2.json similarity index 100% rename from full_tests/simple_cross_reference/compile_commands.json rename to full_tests/simple_cross_reference/compile_commands2.json diff --git a/src/command_line.cc b/src/command_line.cc index cff9530e..bf913c7d 100644 --- a/src/command_line.cc +++ b/src/command_line.cc @@ -1,5 +1,6 @@ // TODO: cleanup includes #include "code_completion.h" +#include "file_consumer.h" #include "indexer.h" #include "query.h" #include "language_server_api.h" @@ -289,8 +290,9 @@ void WriteToCache(std::string filename, IndexedFile& file) { } -bool IndexMain_DoIndex(Index_DoIndexQueue* queue_do_index, - Index_DoIdMapQueue* queue_do_id_map) { +bool IndexMain_DoIndex(FileConsumer* file_consumer, + Index_DoIndexQueue* queue_do_index, + Index_DoIdMapQueue* queue_do_id_map) { optional index_request = queue_do_index->TryDequeue(); if (!index_request) return false; @@ -319,15 +321,12 @@ bool IndexMain_DoIndex(Index_DoIndexQueue* queue_do_index, } // Parse request and send a response. - std::cerr << "Parsing file " << index_request->path << " with args " - << Join(index_request->args, ", ") << std::endl; - - // TODO: parse should return unique_ptr. Then we can eliminate copy below. Make sure to not - // reuse moved pointer in WriteToCache if we do so. - std::vector> indexes = Parse(index_request->path, index_request->args); + std::vector> indexes = Parse(file_consumer, index_request->path, index_request->args); time.ResetAndPrint("Parsing/indexing"); for (auto& current_index : indexes) { + std::cerr << "Got index for " << current_index->path << std::endl; + std::unique_ptr old_index = LoadCachedFile(current_index->path); time.ResetAndPrint("Loading cached index"); @@ -364,16 +363,21 @@ bool IndexMain_DoCreateIndexUpdate(Index_OnIdMappedQueue* queue_on_id_mapped, return true; } -void IndexMain(Index_DoIndexQueue* queue_do_index, +void IndexMain( + FileConsumer::SharedState* file_consumer_shared, + Index_DoIndexQueue* queue_do_index, Index_DoIdMapQueue* queue_do_id_map, Index_OnIdMappedQueue* queue_on_id_mapped, Index_OnIndexedQueue* queue_on_indexed) { + + FileConsumer file_consumer(file_consumer_shared); + while (true) { // TODO: process all off IndexMain_DoIndex before calling IndexMain_DoCreateIndexUpdate for // better icache behavior. We need to have some threads spinning on both though // otherwise memory usage will get bad. - if (!IndexMain_DoIndex(queue_do_index, queue_do_id_map) && + if (!IndexMain_DoIndex(&file_consumer, queue_do_index, queue_do_id_map) && !IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed)) { // TODO: use CV to wakeup? std::this_thread::sleep_for(std::chrono::milliseconds(500)); @@ -646,7 +650,6 @@ void QueryDbMainLoop( request.args = entry.args; queue_do_index->Enqueue(std::move(request)); } - std::cerr << "Done" << std::endl; break; } @@ -952,8 +955,8 @@ void QueryDbMainLoop( assert(request->current); response.current_id_map = MakeUnique(db, request->current->id_cache); + time.ResetAndPrint("Create IdMap " + request->current->path); response.current_index = std::move(request->current); - time.ResetAndPrint("Create IdMap"); queue_on_id_mapped->Enqueue(std::move(response)); } @@ -982,11 +985,12 @@ void QueryDbMain() { Project project; WorkingFiles working_files; CompletionManager completion_manager(&project, &working_files); + FileConsumer::SharedState file_consumer_shared; // Start indexer threads. for (int i = 0; i < kNumIndexers; ++i) { new std::thread([&]() { - IndexMain(&queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed); + IndexMain(&file_consumer_shared, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed); }); } diff --git a/src/file_consumer.cc b/src/file_consumer.cc index ad5a9fec..b470ef75 100644 --- a/src/file_consumer.cc +++ b/src/file_consumer.cc @@ -1,24 +1,31 @@ #include "file_consumer.h" +#include "indexer.h" +#include "utils.h" + FileConsumer::FileConsumer(SharedState* shared_state) : shared_(shared_state) {} -void FileConsumer::ClearOwnership() { - for (auto& entry : local_) - entry.second = Ownership::DoesNotOwn; +std::vector> FileConsumer::TakeLocalState() { + std::vector> result; + for (auto& entry : local_) { + if (entry.second) + result.push_back(std::move(entry.second)); + } + return result; } -bool FileConsumer::DoesOwnFile(const std::string& file) { +IndexedFile* FileConsumer::TryConsumeFile(const std::string& file) { // Try to find cached local result. auto it = local_.find(file); if (it != local_.end()) - return it->second == Ownership::Owns; + return it->second.get(); // No result in local; we need to query global. bool did_insert = false; { - std::lock_guard lock(shared_->muetx); + std::lock_guard lock(shared_->mutex); did_insert = shared_->files.insert(file).second; } - local_[file] = did_insert ? Ownership::Owns : Ownership::DoesNotOwn; - return did_insert; + local_[file] = did_insert ? MakeUnique(file) : nullptr; + return local_[file].get(); } \ No newline at end of file diff --git a/src/file_consumer.h b/src/file_consumer.h index 01500a53..f277fa1e 100644 --- a/src/file_consumer.h +++ b/src/file_consumer.h @@ -4,6 +4,8 @@ #include #include +struct IndexedFile; + // FileConsumer is used by the indexer. When it encouters a file, it tries to // take ownership over it. If the indexer has ownership over a file, it will // produce an index, otherwise, it will emit nothing for that declarations @@ -14,24 +16,21 @@ struct FileConsumer { struct SharedState { mutable std::unordered_set files; - mutable std::mutex muetx; + mutable std::mutex mutex; }; FileConsumer(SharedState* shared_state); // Returns true if this instance owns given |file|. This will also attempt to // take ownership over |file|. - bool DoesOwnFile(const std::string& file); + // + // Returns IndexedFile for the file or nullptr. + IndexedFile* TryConsumeFile(const std::string& file); - // Clear all ownership state. - void ClearOwnership(); + // Returns and passes ownership of all local state. + std::vector> TakeLocalState(); private: - enum class Ownership { - Owns, - DoesNotOwn - }; - - std::unordered_map local_; + std::unordered_map> local_; SharedState* shared_; }; \ No newline at end of file diff --git a/src/indexer.cc b/src/indexer.cc index c4289338..8f38734d 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -3,6 +3,7 @@ #include #include +#include "platform.h" #include "serializer.h" IndexedFile::IndexedFile(const std::string& path) : id_cache(path), path(path) { @@ -131,8 +132,8 @@ Range IdCache::ForceResolveSpelling(const CXCursor& cx_cursor, bool interesting) optional IdCache::ResolveSpelling(const CXCursor& cx_cursor, bool interesting) { CXSourceLocation cx_loc = clang_getCursorLocation(cx_cursor); - if (!clang_Location_isFromMainFile(cx_loc)) - return nullopt; + //if (!clang_Location_isFromMainFile(cx_loc)) + // return nullopt; return ForceResolveSpelling(cx_cursor, interesting); } @@ -147,8 +148,8 @@ Range IdCache::ForceResolveExtent(const CXCursor& cx_cursor, bool interesting) { optional IdCache::ResolveExtent(const CXCursor& cx_cursor, bool interesting) { CXSourceLocation cx_loc = clang_getCursorLocation(cx_cursor); - if (!clang_Location_isFromMainFile(cx_loc)) - return nullopt; + //if (!clang_Location_isFromMainFile(cx_loc)) + // return nullopt; return ForceResolveExtent(cx_cursor, interesting); } @@ -205,14 +206,14 @@ struct NamespaceHelper { }; struct IndexParam { - IndexedFile* db; + FileConsumer* file_consumer; NamespaceHelper* ns; // Record last func usage we reported, as clang will record the reference // twice. We don't want to double report. Range last_func_usage_location; - IndexParam(IndexedFile* db, NamespaceHelper* ns) : db(db), ns(ns) {} + IndexParam(FileConsumer* file_consumer, NamespaceHelper* ns) : file_consumer(file_consumer), ns(ns) {} }; int abortQuery(CXClientData client_data, void* reserved) { @@ -240,7 +241,7 @@ void diagnostic(CXClientData client_data, // Fetch path, print. if (file != nullptr) { std::string path = clang::ToString(clang_getFileName(file)); - std::cerr << path << ':'; + std::cerr << NormalizePath(path) << ':'; } std::cerr << line << ':' << column << ": " << spelling << std::endl; @@ -672,17 +673,22 @@ bool AreEqualLocations(CXIdxLoc loc, CXCursor cursor) { // INDEX SPELLING void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) { - // TODO: we can minimize processing for cursors which return false for - // clang_Location_isFromMainFile (ie, only add usages) - bool is_system_def = - clang_Location_isInSystemHeader(clang_getCursorLocation(decl->cursor)); - if (is_system_def) + // TODO: allow user to configure if they want STL index. + if (clang_Location_isInSystemHeader(clang_indexLoc_getCXSourceLocation(decl->loc))) return; assert(AreEqualLocations(decl->loc, decl->cursor)); + // TODO: Use clang_getFileUniqueID + CXFile file; + clang_getSpellingLocation(clang_indexLoc_getCXSourceLocation(decl->loc), &file, nullptr, nullptr, nullptr); + std::string filename = clang::ToString(clang_getFileName(file)); IndexParam* param = static_cast(client_data); - IndexedFile* db = param->db; + IndexedFile* db = param->file_consumer->TryConsumeFile(filename); + if (!db) + return; + + NamespaceHelper* ns = param->ns; @@ -1070,14 +1076,26 @@ bool IsFunctionCallContext(CXCursorKind kind) { void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* ref) { + // Don't index references from or to system headers. + if (clang_Location_isInSystemHeader(clang_indexLoc_getCXSourceLocation(ref->loc)) || + clang_Location_isInSystemHeader(clang_getCursorLocation(ref->referencedEntity->cursor))) + return; + //assert(AreEqualLocations(ref->loc, ref->cursor)); // if (clang_Location_isInSystemHeader(clang_getCursorLocation(ref->cursor)) || // clang_Location_isInSystemHeader( // clang_getCursorLocation(ref->referencedEntity->cursor))) // return; + + // TODO: Use clang_getFileUniqueID + CXFile file; + clang_getSpellingLocation(clang_indexLoc_getCXSourceLocation(ref->loc), &file, nullptr, nullptr, nullptr); + std::string filename = clang::ToString(clang_getFileName(file)); IndexParam* param = static_cast(client_data); - IndexedFile* db = param->db; + IndexedFile* db = param->file_consumer->TryConsumeFile(filename); + if (!db) + return; // ref->cursor mainFile=0 // ref->loc mainFile=1 @@ -1089,11 +1107,9 @@ void indexEntityReference(CXClientData client_data, //std::cerr << "mainFile: " << mainFile << ", loc: " << loc_spelling.ToString() << std::endl; // Don't index references that are not from the main file. - if (!clang_Location_isFromMainFile(clang_getCursorLocation(ref->cursor))) - return; - // Don't index references to system headers. - if (clang_Location_isInSystemHeader(clang_getCursorLocation(ref->referencedEntity->cursor))) - return; + //if (!clang_Location_isFromMainFile(clang_getCursorLocation(ref->cursor))) + // return; + clang::Cursor cursor(ref->cursor); @@ -1263,7 +1279,7 @@ void indexEntityReference(CXClientData client_data, } } -std::vector> Parse(std::string filename, std::vector args, bool dump_ast) { +std::vector> Parse(FileConsumer* file_consumer, std::string filename, std::vector args, bool dump_ast) { clang_enableStackTraces(); clang_toggleCrashRecovery(1); @@ -1295,9 +1311,8 @@ std::vector> Parse(std::string filename, std::vecto */ }; - auto db = MakeUnique(filename); NamespaceHelper ns; - IndexParam param(db.get(), &ns); + IndexParam param(file_consumer, &ns); std::cerr << "!! [START] Indexing " << filename << std::endl; clang_indexTranslationUnit(index_action, ¶m, callbacks, sizeof(callbacks), @@ -1306,7 +1321,11 @@ std::vector> Parse(std::string filename, std::vecto std::cerr << "!! [END] Indexing " << filename << std::endl; clang_IndexAction_dispose(index_action); - std::vector> result; - result.emplace_back(std::move(db)); - return std::move(result); + auto result = param.file_consumer->TakeLocalState(); + for (auto& entry : result) { + // TODO: only store the path on one of these. + entry->path = NormalizePath(entry->path); + entry->id_cache.primary_file = entry->path; + } + return result; } diff --git a/src/indexer.h b/src/indexer.h index 24659a6c..02fa8a9a 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -1,5 +1,6 @@ #pragma once +#include "file_consumer.h" #include "position.h" #include "serializer.h" #include "utils.h" @@ -448,4 +449,4 @@ struct IndexedFile { std::string ToString(); }; -std::vector> Parse(std::string filename, std::vector args, bool dump_ast = false); +std::vector> Parse(FileConsumer* file_consumer, std::string filename, std::vector args, bool dump_ast = false); diff --git a/src/project.cc b/src/project.cc index 38a3ab0f..e66e9045 100644 --- a/src/project.cc +++ b/src/project.cc @@ -23,7 +23,7 @@ std::vector LoadFromDirectoryListing(const std::string& projec std::vector files = GetFilesInFolder(project_directory, true /*recursive*/, true /*add_folder_to_path*/); for (const std::string& file : files) { - if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") || EndsWith(file, ".c") || EndsWith(file, ".h")) { + if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") || EndsWith(file, ".c")) { CompilationEntry entry; entry.filename = NormalizePath(file); entry.args = args; diff --git a/src/query.cc b/src/query.cc index 4805ddcb..4c944f8e 100644 --- a/src/query.cc +++ b/src/query.cc @@ -284,7 +284,6 @@ void CompareGroups( // TODO: consider having separate lookup maps so they are smaller (maybe // lookups will go faster). - QueryFileId GetQueryFileIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { auto it = query_db->usr_to_symbol.find(usr); if (it != query_db->usr_to_symbol.end()) { @@ -337,7 +336,6 @@ QueryVarId GetQueryVarIdFromUsr(QueryableDatabase* query_db, const Usr& usr) { return QueryVarId(idx); } - IdMap::IdMap(QueryableDatabase* query_db, const IdCache& local_ids) : local_ids(local_ids) { primary_file = GetQueryFileIdFromUsr(query_db, local_ids.primary_file); @@ -624,6 +622,7 @@ void QueryableDatabase::RemoveUsrs(const std::vector& to_remove) { void QueryableDatabase::ImportOrUpdate(const std::vector& updates) { for (auto& def : updates) { + std::cerr << "Importing def for " << def.usr << std::endl; auto it = usr_to_symbol.find(def.usr); assert(it != usr_to_symbol.end()); diff --git a/src/test.cc b/src/test.cc index 835bb841..f76494cb 100644 --- a/src/test.cc +++ b/src/test.cc @@ -40,17 +40,16 @@ std::vector SplitString(const std::string& str, const std::string& } -void DiffDocuments(std::string path, rapidjson::Document& expected, rapidjson::Document& actual) { +void DiffDocuments(std::string path, std::string path_section, rapidjson::Document& expected, rapidjson::Document& actual) { std::string joined_actual_output = ToString(actual); std::vector actual_output = SplitString(joined_actual_output, "\n"); std::string joined_expected_output = ToString(expected); std::vector expected_output = SplitString(joined_expected_output, "\n"); - - std::cout << "[FAILED] " << path << std::endl; - std::cout << "Expected output for " << path << ":" << std::endl; + std::cout << "[FAILED] " << path << " (section " << path_section << ")" << std::endl; + std::cout << "Expected output for " << path << " (section " << path_section << "):" << std::endl; std::cout << joined_expected_output << std::endl; - std::cout << "Actual output for " << path << ":" << std::endl; + std::cout << "Actual output for " << path << " (section " << path_section << "):" << std::endl; std::cout << joined_actual_output << std::endl; std::cout << std::endl; @@ -92,17 +91,29 @@ void VerifySerializeToFrom(IndexedFile* file) { } } +std::string FindExpectedOutputForFilename(std::string filename, const std::unordered_map& expected) { + for (const auto& entry : expected) { + if (EndsWith(entry.first, filename)) + return entry.second; + } + + std::cerr << "Couldn't find expected output for " << filename << std::endl; + std::cin.get(); + std::cin.get(); + return "{}"; +} + +IndexedFile* FindDbForPathEnding(const std::string& path, const std::vector>& dbs) { + for (auto& db : dbs) { + if (EndsWith(db->path, path)) + return db.get(); + } + return nullptr; +} + void RunTests() { // TODO: Assert that we need to be on clang >= 3.9.1 - /* - ParsingDatabase db = Parse("tests/vars/function_local.cc"); - std::cout << std::endl << "== Database ==" << std::endl; - std::cout << db.ToString(); - std::cin.get(); - return 0; - */ - for (std::string path : GetFilesInFolder("tests", true /*recursive*/, true /*add_folder_to_path*/)) { //if (path != "tests/templates/specialized_func_definition.cc") continue; //if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue; @@ -116,14 +127,14 @@ void RunTests() { //path = "C:/Users/jacob/Desktop/superindex/indexer/" + path; // Parse expected output from the test, parse it into JSON document. - std::string expected_output; - ParseTestExpectation(path, &expected_output); - rapidjson::Document expected; - expected.Parse(expected_output.c_str()); + std::unordered_map all_expected_output = ParseTestExpectation(path); + + FileConsumer::SharedState file_consumer_shared; + FileConsumer file_consumer(&file_consumer_shared); // Run test. std::cout << "[START] " << path << std::endl; - std::vector> dbs = Parse(path, { + std::vector> dbs = Parse(&file_consumer, path, { "-xc++", "-std=c++11", "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/", @@ -132,37 +143,36 @@ void RunTests() { "-IC:/Users/jacob/Desktop/superindex/indexer/src" }, false /*dump_ast*/); - // TODO: Supporting tests for more than just primary indexed file. + for (auto& entry : all_expected_output) { + const std::string& expected_path = entry.first; + const std::string& expected_output = entry.second; - // Find primary file. - std::unique_ptr db; - for (auto& i : dbs) { - if (i->path == path) { - db = std::move(i); - break; + // Get output from index operation. + IndexedFile* db = FindDbForPathEnding(expected_path, dbs); + std::string actual_output = "{}"; + if (db) { + VerifySerializeToFrom(db); + actual_output = db->ToString(); } - } - // TODO: Always pass IndexedFile by pointer, ie, search and remove all IndexedFile& refs. - // TODO: Rename IndexedFile to IndexFile + // Compare output via rapidjson::Document to ignore any formatting + // differences. + rapidjson::Document actual; + actual.Parse(actual_output.c_str()); + rapidjson::Document expected; + expected.Parse(expected_output.c_str()); - VerifySerializeToFrom(db.get()); - std::string actual_output = db->ToString(); - - rapidjson::Document actual; - actual.Parse(actual_output.c_str()); - - if (actual == expected) { - std::cout << "[PASSED] " << path << std::endl; - } - else { - DiffDocuments(path, expected, actual); - std::cout << std::endl; - std::cout << std::endl; - std::cout << "[Enter to continue next test]"; - std::cin.get(); - std::cin.get(); - //break; + if (actual == expected) { + std::cout << "[PASSED] " << path << std::endl; + } + else { + DiffDocuments(path, expected_path, expected, actual); + std::cout << std::endl; + std::cout << std::endl; + std::cout << "[Enter to continue]"; + std::cin.get(); + std::cin.get(); + } } } @@ -170,3 +180,6 @@ void RunTests() { } // TODO: ctor/dtor, copy ctor +// TODO: Always pass IndexedFile by pointer, ie, search and remove all IndexedFile& refs. +// TODO: Rename IndexedFile to IndexFile + diff --git a/src/utils.cc b/src/utils.cc index 891cb5e6..fa505087 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -112,19 +113,54 @@ std::vector ReadLines(std::string filename) { return result; } -void ParseTestExpectation(std::string filename, std::string* expected_output) { +std::unordered_map ParseTestExpectation(std::string filename) { bool in_output = false; +#if false + #include "bar.h" + + void foo(); + + /* + // if no name is given assume to be this file name + // no output section means we don't check that index. + OUTPUT: bar.cc + {} + + OUTPUT: bar.h + {} + */ +#endif + + std::unordered_map result; + + std::string active_output_filename; + std::string active_output_contents; + for (std::string line : ReadLines(filename)) { if (line == "*/") break; - if (in_output) - *expected_output += line + "\n"; + if (StartsWith(line, "OUTPUT:")) { + if (in_output) { + result[active_output_filename] = active_output_contents; + } + + if (line.size() > 7) + active_output_filename = line.substr(8); + else + active_output_filename = filename; + active_output_contents = ""; - if (line == "OUTPUT:") in_output = true; + } + else if (in_output) + active_output_contents += line + "\n"; } + + if (in_output) + result[active_output_filename] = active_output_contents; + return result; } diff --git a/src/utils.h b/src/utils.h index 4e12caac..13aba67a 100644 --- a/src/utils.h +++ b/src/utils.h @@ -1,9 +1,10 @@ #pragma once #include -#include -#include #include +#include +#include > +#include // Returns true if |value| starts/ends with |start| or |ending|. bool StartsWith(const std::string& value, const std::string& start); @@ -12,7 +13,7 @@ bool EndsWith(const std::string& value, const std::string& ending); // Finds all files in the given folder. This is recursive. std::vector GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path); std::vector ReadLines(std::string filename); -void ParseTestExpectation(std::string filename, std::string* expected_output); +std::unordered_map ParseTestExpectation(std::string filename); void Fail(const std::string& message); diff --git a/tests/multi_file/header.h b/tests/multi_file/header.h index 0392794b..4d840980 100644 --- a/tests/multi_file/header.h +++ b/tests/multi_file/header.h @@ -22,111 +22,4 @@ struct Foo2 {}; enum Foo3 { A, B, C }; int Foo4; -static int Foo5; - -/* -OUTPUT: -{ - "types": [{ - "id": 0, - "usr": "c:@S@Base", - "short_name": "Base", - "qualified_name": "Base", - "definition_spelling": "10:8-10:12", - "definition_extent": "10:1-10:15", - "derived": [1], - "uses": ["*10:8-10:12", "*12:26-12:30"] - }, { - "id": 1, - "usr": "c:@S@SameFileDerived", - "short_name": "SameFileDerived", - "qualified_name": "SameFileDerived", - "definition_spelling": "12:8-12:23", - "definition_extent": "12:1-12:33", - "parents": [0], - "uses": ["*12:8-12:23", "*14:14-14:29"] - }, { - "id": 2, - "usr": "c:@Foo0", - "short_name": "Foo0", - "qualified_name": "Foo0", - "definition_spelling": "14:7-14:11", - "definition_extent": "14:1-14:29", - "alias_of": 1, - "uses": ["*14:7-14:11"] - }, { - "id": 3, - "usr": "c:@ST>1#T@Foo2", - "short_name": "Foo2", - "qualified_name": "Foo2", - "definition_spelling": "20:8-20:12", - "definition_extent": "20:1-20:15", - "uses": ["*20:8-20:12"] - }, { - "id": 4, - "usr": "c:@E@Foo3", - "short_name": "Foo3", - "qualified_name": "Foo3", - "definition_spelling": "22:6-22:10", - "definition_extent": "22:1-22:22", - "vars": [0, 1, 2], - "uses": ["*22:6-22:10"] - }], - "funcs": [{ - "id": 0, - "usr": "c:@FT@>1#TFoo1#v#", - "short_name": "Foo1", - "qualified_name": "Foo1", - "definition_spelling": "17:6-17:10", - "definition_extent": "17:1-17:12", - "uses": ["17:6-17:10"] - }], - "vars": [{ - "id": 0, - "usr": "c:@E@Foo3@A", - "short_name": "A", - "qualified_name": "Foo3::A", - "definition_spelling": "22:13-22:14", - "definition_extent": "22:13-22:14", - "variable_type": 4, - "declaring_type": 4, - "uses": ["22:13-22:14"] - }, { - "id": 1, - "usr": "c:@E@Foo3@B", - "short_name": "B", - "qualified_name": "Foo3::B", - "definition_spelling": "22:16-22:17", - "definition_extent": "22:16-22:17", - "variable_type": 4, - "declaring_type": 4, - "uses": ["22:16-22:17"] - }, { - "id": 2, - "usr": "c:@E@Foo3@C", - "short_name": "C", - "qualified_name": "Foo3::C", - "definition_spelling": "22:19-22:20", - "definition_extent": "22:19-22:20", - "variable_type": 4, - "declaring_type": 4, - "uses": ["22:19-22:20"] - }, { - "id": 3, - "usr": "c:@Foo4", - "short_name": "Foo4", - "qualified_name": "Foo4", - "definition_spelling": "24:5-24:9", - "definition_extent": "24:1-24:9", - "uses": ["24:5-24:9"] - }, { - "id": 4, - "usr": "c:header.h@Foo5", - "short_name": "Foo5", - "qualified_name": "Foo5", - "definition_spelling": "25:12-25:16", - "definition_extent": "25:1-25:16", - "uses": ["25:12-25:16"] - }] -} -*/ +static int Foo5; \ No newline at end of file diff --git a/tests/multi_file/impl.cc b/tests/multi_file/impl.cc index 15dee3c4..1f292c9f 100644 --- a/tests/multi_file/impl.cc +++ b/tests/multi_file/impl.cc @@ -1,246 +1,131 @@ -#include "serializer.h" -#include "indexer.h" - -#if false -// int -void Reflect(Reader& visitor, int& value) { - value = visitor.GetInt(); -} -void Reflect(Writer& visitor, int& value) { - visitor.Int(value); -} -// bool -void Reflect(Reader& visitor, bool& value) { - value = visitor.GetBool(); -} -void Reflect(Writer& visitor, bool& value) { - visitor.Bool(value); -} -// std::string -void Reflect(Reader& visitor, std::string& value) { - value = visitor.GetString(); -} -void Reflect(Writer& visitor, std::string& value) { - visitor.String(value.c_str(), value.size()); -} - - -// ReflectMember -void ReflectMember(Writer& visitor, const char* name, std::string& value) { - if (value.empty()) - return; - visitor.Key(name); - Reflect(visitor, value); -} - - -// Location -void Reflect(Reader& visitor, Location& value) { - value = Location(visitor.GetString()); -} -void Reflect(Writer& visitor, Location& value) { - // We only ever want to emit id=1 files. - assert(value.raw_file_id == 1); - - std::string output = value.ToString(); - visitor.String(output.c_str(), output.size()); -} - - -// Id -template -void Reflect(Reader& visitor, Id& id) { - id.id = visitor.GetUint64(); -} -template -void Reflect(Writer& visitor, Id& value) { - visitor.Uint64(value.id); -} - - -// Ref -void Reflect(Reader& visitor, Ref& value) { - const char* str_value = visitor.GetString(); - uint64_t id = atoi(str_value); - const char* loc_string = strchr(str_value, '@') + 1; - - value.id = Id(id); - value.loc = Location(loc_string); -} -void Reflect(Writer& visitor, Ref& value) { - std::string s = std::to_string(value.id.id) + "@" + value.loc.ToString(); - visitor.String(s.c_str()); -} - - - - - - - -// TODO: Move this to indexer.cpp -// TODO: Rename indexer.cpp to indexer.cc -// TODO: Do not serialize a USR if it has no usages/etc outside of USR info. - -// IndexedTypeDef -bool ReflectMemberStart(Reader& reader, IndexedTypeDef& value) { - //value.is_bad_def = false; - return true; -} -bool ReflectMemberStart(Writer& writer, IndexedTypeDef& value) { - //if (value.is_bad_def) - // return false; - DefaultReflectMemberStart(writer); - return true; -} -template -void Reflect(TVisitor& visitor, IndexedTypeDef& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER2("id", value.id); - REFLECT_MEMBER2("usr", value.def.usr); - REFLECT_MEMBER2("short_name", value.def.short_name); - REFLECT_MEMBER2("qualified_name", value.def.qualified_name); - REFLECT_MEMBER2("definition", value.def.definition); - REFLECT_MEMBER2("alias_of", value.def.alias_of); - REFLECT_MEMBER2("parents", value.def.parents); - REFLECT_MEMBER2("derived", value.derived); - REFLECT_MEMBER2("types", value.def.types); - REFLECT_MEMBER2("funcs", value.def.funcs); - REFLECT_MEMBER2("vars", value.def.vars); - REFLECT_MEMBER2("uses", value.uses); - REFLECT_MEMBER_END(); -} - - -// IndexedFuncDef -bool ReflectMemberStart(Reader& reader, IndexedFuncDef& value) { - //value.is_bad_def = false; - return true; -} -bool ReflectMemberStart(Writer& writer, IndexedFuncDef& value) { - //if (value.is_bad_def) - // return false; - DefaultReflectMemberStart(writer); - return true; -} -template -void Reflect(TVisitor& visitor, IndexedFuncDef& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER2("id", value.id); - REFLECT_MEMBER2("usr", value.def.usr); - REFLECT_MEMBER2("short_name", value.def.short_name); - REFLECT_MEMBER2("qualified_name", value.def.qualified_name); - REFLECT_MEMBER2("declarations", value.declarations); - REFLECT_MEMBER2("definition", value.def.definition); - REFLECT_MEMBER2("declaring_type", value.def.declaring_type); - REFLECT_MEMBER2("base", value.def.base); - REFLECT_MEMBER2("derived", value.derived); - REFLECT_MEMBER2("locals", value.def.locals); - REFLECT_MEMBER2("callers", value.callers); - REFLECT_MEMBER2("callees", value.def.callees); - REFLECT_MEMBER2("uses", value.uses); - REFLECT_MEMBER_END(); -} - - -// IndexedVarDef -bool ReflectMemberStart(Reader& reader, IndexedVarDef& value) { - //value.is_bad_def = false; - return true; -} -bool ReflectMemberStart(Writer& writer, IndexedVarDef& value) { - //if (value.is_bad_def) - // return false; - DefaultReflectMemberStart(writer); - return true; -} -template -void Reflect(TVisitor& visitor, IndexedVarDef& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER2("id", value.id); - REFLECT_MEMBER2("usr", value.def.usr); - REFLECT_MEMBER2("short_name", value.def.short_name); - REFLECT_MEMBER2("qualified_name", value.def.qualified_name); - REFLECT_MEMBER2("declaration", value.def.declaration); - REFLECT_MEMBER2("definition", value.def.definition); - REFLECT_MEMBER2("variable_type", value.def.variable_type); - REFLECT_MEMBER2("declaring_type", value.def.declaring_type); - REFLECT_MEMBER2("uses", value.uses); - REFLECT_MEMBER_END(); -} - - -// IndexedFile -bool ReflectMemberStart(Writer& visitor, IndexedFile& value) { - auto it = value.id_cache.usr_to_type_id.find(""); - if (it != value.id_cache.usr_to_type_id.end()) { - value.Resolve(it->second)->def.short_name = ""; - assert(value.Resolve(it->second)->uses.size() == 0); - } - - DefaultReflectMemberStart(visitor); - return true; -} -template -void Reflect(TVisitor& visitor, IndexedFile& value) { - REFLECT_MEMBER_START(); - REFLECT_MEMBER(types); - REFLECT_MEMBER(funcs); - REFLECT_MEMBER(vars); - REFLECT_MEMBER_END(); -} - - - - - - - -std::string Serialize(IndexedFile& file) { - rapidjson::StringBuffer output; - rapidjson::PrettyWriter writer(output); - //Writer writer(output); - writer.SetFormatOptions( - rapidjson::PrettyFormatOptions::kFormatSingleLineArray); - writer.SetIndent(' ', 2); - - Reflect(writer, file); - - return output.GetString(); -} - -optional Deserialize(std::string path, std::string serialized) { - rapidjson::Document reader; - reader.Parse(serialized.c_str()); - if (reader.HasParseError()) - return nullopt; - - IndexedFile file(path); - Reflect(reader, file); - - return file; -} -#endif -#if false #include "header.h" -#include "serializer.h" - -struct SeparateFileDerived : Base {}; - -void f() { - rapidjson::StringBuffer output; - rapidjson::PrettyWriter writer(output); - //Writer writer(output); - writer.SetFormatOptions( - rapidjson::PrettyFormatOptions::kFormatSingleLineArray); - writer.SetIndent(' ', 2); - - Foo2 a; +void Impl() { + Foo1(); } -#endif /* -OUTPUT: -{} +OUTPUT: header.h +{ + "types": [{ + "id": 0, + "usr": "c:@S@Base", + "short_name": "Base", + "qualified_name": "Base", + "definition_spelling": "10:8-10:12", + "definition_extent": "10:1-10:15", + "derived": [1], + "uses": ["*10:8-10:12", "*12:26-12:30"] + }, { + "id": 1, + "usr": "c:@S@SameFileDerived", + "short_name": "SameFileDerived", + "qualified_name": "SameFileDerived", + "definition_spelling": "12:8-12:23", + "definition_extent": "12:1-12:33", + "parents": [0], + "uses": ["*12:8-12:23", "*14:14-14:29"] + }, { + "id": 2, + "usr": "c:@Foo0", + "short_name": "Foo0", + "qualified_name": "Foo0", + "definition_spelling": "14:7-14:11", + "definition_extent": "14:1-14:29", + "alias_of": 1, + "uses": ["*14:7-14:11"] + }, { + "id": 3, + "usr": "c:@ST>1#T@Foo2", + "short_name": "Foo2", + "qualified_name": "Foo2", + "definition_spelling": "20:8-20:12", + "definition_extent": "20:1-20:15", + "uses": ["*20:8-20:12"] + }, { + "id": 4, + "usr": "c:@E@Foo3", + "short_name": "Foo3", + "qualified_name": "Foo3", + "definition_spelling": "22:6-22:10", + "definition_extent": "22:1-22:22", + "vars": [0, 1, 2], + "uses": ["*22:6-22:10"] + }], + "funcs": [{ + "id": 0, + "usr": "c:@FT@>1#TFoo1#v#", + "short_name": "Foo1", + "qualified_name": "Foo1", + "definition_spelling": "17:6-17:10", + "definition_extent": "17:1-17:15", + "uses": ["17:6-17:10"] + }], + "vars": [{ + "id": 0, + "usr": "c:@E@Foo3@A", + "short_name": "A", + "qualified_name": "Foo3::A", + "definition_spelling": "22:13-22:14", + "definition_extent": "22:13-22:14", + "variable_type": 4, + "declaring_type": 4, + "uses": ["22:13-22:14"] + }, { + "id": 1, + "usr": "c:@E@Foo3@B", + "short_name": "B", + "qualified_name": "Foo3::B", + "definition_spelling": "22:16-22:17", + "definition_extent": "22:16-22:17", + "variable_type": 4, + "declaring_type": 4, + "uses": ["22:16-22:17"] + }, { + "id": 2, + "usr": "c:@E@Foo3@C", + "short_name": "C", + "qualified_name": "Foo3::C", + "definition_spelling": "22:19-22:20", + "definition_extent": "22:19-22:20", + "variable_type": 4, + "declaring_type": 4, + "uses": ["22:19-22:20"] + }, { + "id": 3, + "usr": "c:@Foo4", + "short_name": "Foo4", + "qualified_name": "Foo4", + "definition_spelling": "24:5-24:9", + "definition_extent": "24:1-24:9", + "uses": ["24:5-24:9"] + }, { + "id": 4, + "usr": "c:header.h@Foo5", + "short_name": "Foo5", + "qualified_name": "Foo5", + "definition_spelling": "25:12-25:16", + "definition_extent": "25:1-25:16", + "uses": ["25:12-25:16"] + }] +} + +OUTPUT: impl.cc +{ + "funcs": [{ + "id": 0, + "usr": "c:@F@Impl#", + "short_name": "Impl", + "qualified_name": "Impl", + "definition_spelling": "3:6-3:10", + "definition_extent": "3:1-5:2", + "callees": ["1@4:3-4:7"], + "uses": ["3:6-3:10"] + }, { + "id": 1, + "usr": "c:@FT@>1#TFoo1#v#", + "callers": ["0@4:3-4:7"], + "uses": ["4:3-4:7"] + }] +} */ diff --git a/tests/multi_file/simple_header.h b/tests/multi_file/simple_header.h new file mode 100644 index 00000000..3bdf008a --- /dev/null +++ b/tests/multi_file/simple_header.h @@ -0,0 +1,3 @@ +#pragma once + +void header(); \ No newline at end of file diff --git a/tests/multi_file/simple_impl.cc b/tests/multi_file/simple_impl.cc new file mode 100644 index 00000000..3fd48721 --- /dev/null +++ b/tests/multi_file/simple_impl.cc @@ -0,0 +1,38 @@ +#include "simple_header.h" + +void impl() { + header(); +} + +/* +OUTPUT: simple_header.h +{ + "funcs": [{ + "id": 0, + "usr": "c:@F@header#", + "short_name": "header", + "qualified_name": "header", + "declarations": ["3:6-3:12"], + "uses": ["3:6-3:12"] + }] +} + +OUTPUT: simple_impl.cc +{ + "funcs": [{ + "id": 0, + "usr": "c:@F@impl#", + "short_name": "impl", + "qualified_name": "impl", + "definition_spelling": "3:6-3:10", + "definition_extent": "3:1-5:2", + "callees": ["1@4:3-4:9"], + "uses": ["3:6-3:10"] + }, { + "id": 1, + "usr": "c:@F@header#", + "callers": ["0@4:3-4:9"], + "uses": ["4:3-4:9"] + }] +} +*/ diff --git a/tests/usage/func_called_from_macro_argument.cc b/tests/usage/func_called_from_macro_argument.cc index 655a9aa5..28ae1454 100644 --- a/tests/usage/func_called_from_macro_argument.cc +++ b/tests/usage/func_called_from_macro_argument.cc @@ -16,7 +16,8 @@ OUTPUT: "short_name": "called", "qualified_name": "called", "declarations": ["3:6-3:12"], - "uses": ["3:6-3:12"] + "callers": ["1@6:14-6:20"], + "uses": ["3:6-3:12", "6:14-6:20"] }, { "id": 1, "usr": "c:@F@caller#", @@ -24,6 +25,7 @@ OUTPUT: "qualified_name": "caller", "definition_spelling": "5:6-5:12", "definition_extent": "5:1-7:2", + "callees": ["0@6:14-6:20"], "uses": ["5:6-5:12"] }] }