Auto-index header files

src/command_line.cc

@@ -1,5 +1,6 @@
 // TODO: cleanup includes
 #include "code_completion.h"
+#include "file_consumer.h"
 #include "indexer.h"
 #include "query.h"
 #include "language_server_api.h"
@@ -289,8 +290,9 @@ void WriteToCache(std::string filename, IndexedFile& file) {
 }
 
 
-bool IndexMain_DoIndex(Index_DoIndexQueue* queue_do_index,
+bool IndexMain_DoIndex(FileConsumer* file_consumer,
-  Index_DoIdMapQueue* queue_do_id_map) {
+                       Index_DoIndexQueue* queue_do_index,
+                       Index_DoIdMapQueue* queue_do_id_map) {
   optional<Index_DoIndex> index_request = queue_do_index->TryDequeue();
   if (!index_request)
     return false;
@@ -319,15 +321,12 @@ bool IndexMain_DoIndex(Index_DoIndexQueue* queue_do_index,
   }
 
   // Parse request and send a response.
-  std::cerr << "Parsing file " << index_request->path << " with args "
+  std::vector<std::unique_ptr<IndexedFile>> indexes = Parse(file_consumer, index_request->path, index_request->args);
-    << Join(index_request->args, ", ") << std::endl;
-
-  // TODO: parse should return unique_ptr. Then we can eliminate copy below. Make sure to not
-  // reuse moved pointer in WriteToCache if we do so.
-  std::vector<std::unique_ptr<IndexedFile>> indexes = Parse(index_request->path, index_request->args);
   time.ResetAndPrint("Parsing/indexing");
 
   for (auto& current_index : indexes) {
+    std::cerr << "Got index for " << current_index->path << std::endl;
+
     std::unique_ptr<IndexedFile> old_index = LoadCachedFile(current_index->path);
     time.ResetAndPrint("Loading cached index");
 
@@ -364,16 +363,21 @@ bool IndexMain_DoCreateIndexUpdate(Index_OnIdMappedQueue* queue_on_id_mapped,
   return true;
 }
 
-void IndexMain(Index_DoIndexQueue* queue_do_index,
+void IndexMain(
+  FileConsumer::SharedState* file_consumer_shared,
+  Index_DoIndexQueue* queue_do_index,
   Index_DoIdMapQueue* queue_do_id_map,
   Index_OnIdMappedQueue* queue_on_id_mapped,
   Index_OnIndexedQueue* queue_on_indexed) {
+
+  FileConsumer file_consumer(file_consumer_shared);
+
   while (true) {
     // TODO: process all off IndexMain_DoIndex before calling IndexMain_DoCreateIndexUpdate for
     //       better icache behavior. We need to have some threads spinning on both though
     //       otherwise memory usage will get bad.
 
-    if (!IndexMain_DoIndex(queue_do_index, queue_do_id_map) &&
+    if (!IndexMain_DoIndex(&file_consumer, queue_do_index, queue_do_id_map) &&
       !IndexMain_DoCreateIndexUpdate(queue_on_id_mapped, queue_on_indexed)) {
       // TODO: use CV to wakeup?
       std::this_thread::sleep_for(std::chrono::milliseconds(500));
@@ -646,7 +650,6 @@ void QueryDbMainLoop(
         request.args = entry.args;
         queue_do_index->Enqueue(std::move(request));
       }
-      std::cerr << "Done" << std::endl;
       break;
     }
 
@@ -952,8 +955,8 @@ void QueryDbMainLoop(
 
     assert(request->current);
     response.current_id_map = MakeUnique<IdMap>(db, request->current->id_cache);
+    time.ResetAndPrint("Create IdMap " + request->current->path);
     response.current_index = std::move(request->current);
-    time.ResetAndPrint("Create IdMap");
 
     queue_on_id_mapped->Enqueue(std::move(response));
   }
@@ -982,11 +985,12 @@ void QueryDbMain() {
   Project project;
   WorkingFiles working_files;
   CompletionManager completion_manager(&project, &working_files);
+  FileConsumer::SharedState file_consumer_shared;
 
   // Start indexer threads.
   for (int i = 0; i < kNumIndexers; ++i) {
     new std::thread([&]() {
-      IndexMain(&queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed);
+      IndexMain(&file_consumer_shared, &queue_do_index, &queue_do_id_map, &queue_on_id_mapped, &queue_on_indexed);
     });
   }
 

src/file_consumer.cc

@@ -1,24 +1,31 @@
 #include "file_consumer.h"
 
+#include "indexer.h"
+#include "utils.h"
+
 FileConsumer::FileConsumer(SharedState* shared_state) : shared_(shared_state) {}
 
-void FileConsumer::ClearOwnership() {
+std::vector<std::unique_ptr<IndexedFile>> FileConsumer::TakeLocalState() {
-  for (auto& entry : local_)
+  std::vector<std::unique_ptr<IndexedFile>> result;
-    entry.second = Ownership::DoesNotOwn;
+  for (auto& entry : local_) {
+    if (entry.second)
+      result.push_back(std::move(entry.second));
+  }
+  return result;
 }
 
-bool FileConsumer::DoesOwnFile(const std::string& file) {
+IndexedFile* FileConsumer::TryConsumeFile(const std::string& file) {
   // Try to find cached local result.
   auto it = local_.find(file);
   if (it != local_.end())
-    return it->second == Ownership::Owns;
+    return it->second.get();
 
   // No result in local; we need to query global.
   bool did_insert = false;
   {
-    std::lock_guard<std::mutex> lock(shared_->muetx);
+    std::lock_guard<std::mutex> lock(shared_->mutex);
     did_insert = shared_->files.insert(file).second;
   }
-  local_[file] = did_insert ? Ownership::Owns : Ownership::DoesNotOwn;
+  local_[file] = did_insert ? MakeUnique<IndexedFile>(file) : nullptr;
-  return did_insert;
+  return local_[file].get();
 }

src/file_consumer.h

@@ -4,6 +4,8 @@
 #include <unordered_set>
 #include <unordered_map>
 
+struct IndexedFile;
+
 // FileConsumer is used by the indexer. When it encouters a file, it tries to
 // take ownership over it. If the indexer has ownership over a file, it will
 // produce an index, otherwise, it will emit nothing for that declarations
@@ -14,24 +16,21 @@
 struct FileConsumer {
   struct SharedState {
     mutable std::unordered_set<std::string> files;
-    mutable std::mutex muetx;
+    mutable std::mutex mutex;
   };
 
   FileConsumer(SharedState* shared_state);
 
   // Returns true if this instance owns given |file|. This will also attempt to
   // take ownership over |file|.
-  bool DoesOwnFile(const std::string& file);
+  //
+  // Returns IndexedFile for the file or nullptr.
+  IndexedFile* TryConsumeFile(const std::string& file);
 
-  // Clear all ownership state.
+  // Returns and passes ownership of all local state.
-  void ClearOwnership();
+  std::vector<std::unique_ptr<IndexedFile>> TakeLocalState();
 
  private:
-  enum class Ownership {
+  std::unordered_map<std::string, std::unique_ptr<IndexedFile>> local_;
-    Owns,
-    DoesNotOwn
-  };
-
-  std::unordered_map<std::string, Ownership> local_;
   SharedState* shared_;
 };

src/indexer.cc

@@ -3,6 +3,7 @@
 #include <algorithm>
 #include <chrono>
 
+#include "platform.h"
 #include "serializer.h"
 
 IndexedFile::IndexedFile(const std::string& path) : id_cache(path), path(path) {
@@ -131,8 +132,8 @@ Range IdCache::ForceResolveSpelling(const CXCursor& cx_cursor, bool interesting)
 
 optional<Range> IdCache::ResolveSpelling(const CXCursor& cx_cursor, bool interesting) {
   CXSourceLocation cx_loc = clang_getCursorLocation(cx_cursor);
-  if (!clang_Location_isFromMainFile(cx_loc))
+  //if (!clang_Location_isFromMainFile(cx_loc))
-    return nullopt;
+  //  return nullopt;
   return ForceResolveSpelling(cx_cursor, interesting);
 }
 
@@ -147,8 +148,8 @@ Range IdCache::ForceResolveExtent(const CXCursor& cx_cursor, bool interesting) {
 
 optional<Range> IdCache::ResolveExtent(const CXCursor& cx_cursor, bool interesting) {
   CXSourceLocation cx_loc = clang_getCursorLocation(cx_cursor);
-  if (!clang_Location_isFromMainFile(cx_loc))
+  //if (!clang_Location_isFromMainFile(cx_loc))
-    return nullopt;
+  //  return nullopt;
   return ForceResolveExtent(cx_cursor, interesting);
 }
 
@@ -205,14 +206,14 @@ struct NamespaceHelper {
 };
 
 struct IndexParam {
-  IndexedFile* db;
+  FileConsumer* file_consumer;
   NamespaceHelper* ns;
 
   // Record last func usage we reported, as clang will record the reference
   // twice. We don't want to double report.
   Range last_func_usage_location;
 
-  IndexParam(IndexedFile* db, NamespaceHelper* ns) : db(db), ns(ns) {}
+  IndexParam(FileConsumer* file_consumer, NamespaceHelper* ns) : file_consumer(file_consumer), ns(ns) {}
 };
 
 int abortQuery(CXClientData client_data, void* reserved) {
@@ -240,7 +241,7 @@ void diagnostic(CXClientData client_data,
     // Fetch path, print.
     if (file != nullptr) {
       std::string path = clang::ToString(clang_getFileName(file));
-      std::cerr << path << ':';
+      std::cerr << NormalizePath(path) << ':';
     }
     std::cerr << line << ':' << column << ": " << spelling << std::endl;
 
@@ -672,17 +673,22 @@ bool AreEqualLocations(CXIdxLoc loc, CXCursor cursor) {
 // INDEX SPELLING
 
 void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
-  // TODO: we can minimize processing for cursors which return false for
+  // TODO: allow user to configure if they want STL index.
-  // clang_Location_isFromMainFile (ie, only add usages)
+  if (clang_Location_isInSystemHeader(clang_indexLoc_getCXSourceLocation(decl->loc)))
-  bool is_system_def =
-      clang_Location_isInSystemHeader(clang_getCursorLocation(decl->cursor));
-  if (is_system_def)
     return;
 
   assert(AreEqualLocations(decl->loc, decl->cursor));
 
+  // TODO: Use clang_getFileUniqueID
+  CXFile file;
+  clang_getSpellingLocation(clang_indexLoc_getCXSourceLocation(decl->loc), &file, nullptr, nullptr, nullptr);
+  std::string filename = clang::ToString(clang_getFileName(file));
   IndexParam* param = static_cast<IndexParam*>(client_data);
-  IndexedFile* db = param->db;
+  IndexedFile* db = param->file_consumer->TryConsumeFile(filename);
+  if (!db)
+    return;
+
+
   NamespaceHelper* ns = param->ns;
 
 
@@ -1070,14 +1076,26 @@ bool IsFunctionCallContext(CXCursorKind kind) {
 
 void indexEntityReference(CXClientData client_data,
                           const CXIdxEntityRefInfo* ref) {
+  // Don't index references from or to system headers.
+  if (clang_Location_isInSystemHeader(clang_indexLoc_getCXSourceLocation(ref->loc)) ||
+      clang_Location_isInSystemHeader(clang_getCursorLocation(ref->referencedEntity->cursor)))
+    return;
+
   //assert(AreEqualLocations(ref->loc, ref->cursor));
 
 //  if (clang_Location_isInSystemHeader(clang_getCursorLocation(ref->cursor)) ||
 //      clang_Location_isInSystemHeader(
 //          clang_getCursorLocation(ref->referencedEntity->cursor)))
 //    return;
+
+  // TODO: Use clang_getFileUniqueID
+  CXFile file;
+  clang_getSpellingLocation(clang_indexLoc_getCXSourceLocation(ref->loc), &file, nullptr, nullptr, nullptr);
+  std::string filename = clang::ToString(clang_getFileName(file));
   IndexParam* param = static_cast<IndexParam*>(client_data);
-  IndexedFile* db = param->db;
+  IndexedFile* db = param->file_consumer->TryConsumeFile(filename);
+  if (!db)
+    return;
 
   // ref->cursor mainFile=0
   // ref->loc mainFile=1
@@ -1089,11 +1107,9 @@ void indexEntityReference(CXClientData client_data,
   //std::cerr << "mainFile: " << mainFile << ", loc: " << loc_spelling.ToString() << std::endl;
 
   // Don't index references that are not from the main file.
-  if (!clang_Location_isFromMainFile(clang_getCursorLocation(ref->cursor)))
+  //if (!clang_Location_isFromMainFile(clang_getCursorLocation(ref->cursor)))
-    return;
+  //  return;
-  // Don't index references to system headers.
+
-  if (clang_Location_isInSystemHeader(clang_getCursorLocation(ref->referencedEntity->cursor)))
-    return;
 
 
   clang::Cursor cursor(ref->cursor);
@@ -1263,7 +1279,7 @@ void indexEntityReference(CXClientData client_data,
   }
 }
 
-std::vector<std::unique_ptr<IndexedFile>> Parse(std::string filename, std::vector<std::string> args, bool dump_ast) {
+std::vector<std::unique_ptr<IndexedFile>> Parse(FileConsumer* file_consumer, std::string filename, std::vector<std::string> args, bool dump_ast) {
   clang_enableStackTraces();
   clang_toggleCrashRecovery(1);
 
@@ -1295,9 +1311,8 @@ std::vector<std::unique_ptr<IndexedFile>> Parse(std::string filename, std::vecto
       */
   };
 
-  auto db = MakeUnique<IndexedFile>(filename);
   NamespaceHelper ns;
-  IndexParam param(db.get(), &ns);
+  IndexParam param(file_consumer, &ns);
 
   std::cerr << "!! [START] Indexing " << filename << std::endl;
   clang_indexTranslationUnit(index_action, &param, callbacks, sizeof(callbacks),
@@ -1306,7 +1321,11 @@ std::vector<std::unique_ptr<IndexedFile>> Parse(std::string filename, std::vecto
   std::cerr << "!! [END] Indexing " << filename << std::endl;
   clang_IndexAction_dispose(index_action);
 
-  std::vector<std::unique_ptr<IndexedFile>> result;
+  auto result = param.file_consumer->TakeLocalState();
-  result.emplace_back(std::move(db));
+  for (auto& entry : result) {
-  return std::move(result);
+    // TODO: only store the path on one of these.
+    entry->path = NormalizePath(entry->path);
+    entry->id_cache.primary_file = entry->path;
+  }
+  return result;
 }

src/indexer.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "file_consumer.h"
 #include "position.h"
 #include "serializer.h"
 #include "utils.h"
@@ -448,4 +449,4 @@ struct IndexedFile {
   std::string ToString();
 };
 
-std::vector<std::unique_ptr<IndexedFile>> Parse(std::string filename, std::vector<std::string> args, bool dump_ast = false);
+std::vector<std::unique_ptr<IndexedFile>> Parse(FileConsumer* file_consumer, std::string filename, std::vector<std::string> args, bool dump_ast = false);

src/project.cc

@@ -23,7 +23,7 @@ std::vector<CompilationEntry> LoadFromDirectoryListing(const std::string& projec
 
   std::vector<std::string> files = GetFilesInFolder(project_directory, true /*recursive*/, true /*add_folder_to_path*/);
   for (const std::string& file : files) {
-    if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") || EndsWith(file, ".c") || EndsWith(file, ".h")) {
+    if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") || EndsWith(file, ".c")) {
       CompilationEntry entry;
       entry.filename = NormalizePath(file);
       entry.args = args;

src/query.cc

@@ -284,7 +284,6 @@ void CompareGroups(
 
 // TODO: consider having separate lookup maps so they are smaller (maybe
 // lookups will go faster).
-
 QueryFileId GetQueryFileIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
   auto it = query_db->usr_to_symbol.find(usr);
   if (it != query_db->usr_to_symbol.end()) {
@@ -337,7 +336,6 @@ QueryVarId GetQueryVarIdFromUsr(QueryableDatabase* query_db, const Usr& usr) {
   return QueryVarId(idx);
 }
 
-
 IdMap::IdMap(QueryableDatabase* query_db, const IdCache& local_ids)
   : local_ids(local_ids) {
   primary_file = GetQueryFileIdFromUsr(query_db, local_ids.primary_file);
@@ -624,6 +622,7 @@ void QueryableDatabase::RemoveUsrs(const std::vector<Usr>& to_remove) {
 
 void QueryableDatabase::ImportOrUpdate(const std::vector<QueryableFile::DefUpdate>& updates) {
   for (auto& def : updates) {
+    std::cerr << "Importing def for " << def.usr << std::endl;
     auto it = usr_to_symbol.find(def.usr);
     assert(it != usr_to_symbol.end());
 

src/test.cc

@@ -40,17 +40,16 @@ std::vector<std::string> SplitString(const std::string& str, const std::string&
 }
 
 
-void DiffDocuments(std::string path, rapidjson::Document& expected, rapidjson::Document& actual) {
+void DiffDocuments(std::string path, std::string path_section, rapidjson::Document& expected, rapidjson::Document& actual) {
   std::string joined_actual_output = ToString(actual);
   std::vector<std::string> actual_output = SplitString(joined_actual_output, "\n");
   std::string joined_expected_output = ToString(expected);
   std::vector<std::string> expected_output = SplitString(joined_expected_output, "\n");
 
-
+  std::cout << "[FAILED] " << path << " (section " << path_section << ")" << std::endl;
-  std::cout << "[FAILED] " << path << std::endl;
+  std::cout << "Expected output for " << path << " (section " << path_section << "):" << std::endl;
-  std::cout << "Expected output for " << path << ":" << std::endl;
   std::cout << joined_expected_output << std::endl;
-  std::cout << "Actual output for " << path << ":" << std::endl;
+  std::cout << "Actual output for " << path << " (section " << path_section << "):" << std::endl;
   std::cout << joined_actual_output << std::endl;
   std::cout << std::endl;
 
@@ -92,17 +91,29 @@ void VerifySerializeToFrom(IndexedFile* file) {
   }
 }
 
+std::string FindExpectedOutputForFilename(std::string filename, const std::unordered_map<std::string, std::string>& expected) {
+  for (const auto& entry : expected) {
+    if (EndsWith(entry.first, filename))
+      return entry.second;
+  }
+
+  std::cerr << "Couldn't find expected output for " << filename << std::endl;
+  std::cin.get();
+  std::cin.get();
+  return "{}";
+}
+
+IndexedFile* FindDbForPathEnding(const std::string& path, const std::vector<std::unique_ptr<IndexedFile>>& dbs) {
+  for (auto& db : dbs) {
+    if (EndsWith(db->path, path))
+      return db.get();
+  }
+  return nullptr;
+}
+
 void RunTests() {
   // TODO: Assert that we need to be on clang >= 3.9.1
 
-  /*
-  ParsingDatabase db = Parse("tests/vars/function_local.cc");
-  std::cout << std::endl << "== Database ==" << std::endl;
-  std::cout << db.ToString();
-  std::cin.get();
-  return 0;
-  */
-
   for (std::string path : GetFilesInFolder("tests", true /*recursive*/, true /*add_folder_to_path*/)) {
     //if (path != "tests/templates/specialized_func_definition.cc") continue;
     //if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue;
@@ -116,14 +127,14 @@ void RunTests() {
     //path = "C:/Users/jacob/Desktop/superindex/indexer/" + path;
 
     // Parse expected output from the test, parse it into JSON document.
-    std::string expected_output;
+    std::unordered_map<std::string, std::string> all_expected_output = ParseTestExpectation(path);
-    ParseTestExpectation(path, &expected_output);
+
-    rapidjson::Document expected;
+    FileConsumer::SharedState file_consumer_shared;
-    expected.Parse(expected_output.c_str());
+    FileConsumer file_consumer(&file_consumer_shared);
 
     // Run test.
     std::cout << "[START] " << path << std::endl;
-    std::vector<std::unique_ptr<IndexedFile>> dbs = Parse(path, {
+    std::vector<std::unique_ptr<IndexedFile>> dbs = Parse(&file_consumer, path, {
         "-xc++",
         "-std=c++11",
         "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/",
@@ -132,37 +143,36 @@ void RunTests() {
         "-IC:/Users/jacob/Desktop/superindex/indexer/src"
       }, false /*dump_ast*/);
 
-    // TODO: Supporting tests for more than just primary indexed file.
+    for (auto& entry : all_expected_output) {
+      const std::string& expected_path = entry.first;
+      const std::string& expected_output = entry.second;
 
-    // Find primary file.
+      // Get output from index operation.
-    std::unique_ptr<IndexedFile> db;
+      IndexedFile* db = FindDbForPathEnding(expected_path, dbs);
-    for (auto& i : dbs) {
+      std::string actual_output = "{}";
-      if (i->path == path) {
+      if (db) {
-        db = std::move(i);
+        VerifySerializeToFrom(db);
-        break;
+        actual_output = db->ToString();
       }
-    }
 
-    // TODO: Always pass IndexedFile by pointer, ie, search and remove all IndexedFile& refs.
+      // Compare output via rapidjson::Document to ignore any formatting
-    // TODO: Rename IndexedFile to IndexFile
+      // differences.
+      rapidjson::Document actual;
+      actual.Parse(actual_output.c_str());
+      rapidjson::Document expected;
+      expected.Parse(expected_output.c_str());
 
-    VerifySerializeToFrom(db.get());
+      if (actual == expected) {
-    std::string actual_output = db->ToString();
+        std::cout << "[PASSED] " << path << std::endl;
-
+      }
-    rapidjson::Document actual;
+      else {
-    actual.Parse(actual_output.c_str());
+        DiffDocuments(path, expected_path, expected, actual);
-
+        std::cout << std::endl;
-    if (actual == expected) {
+        std::cout << std::endl;
-      std::cout << "[PASSED] " << path << std::endl;
+        std::cout << "[Enter to continue]";
-    }
+        std::cin.get();
-    else {
+        std::cin.get();
-      DiffDocuments(path, expected, actual);
+      }
-      std::cout << std::endl;
-      std::cout << std::endl;
-      std::cout << "[Enter to continue next test]";
-      std::cin.get();
-      std::cin.get();
-      //break;
     }
   }
 
@@ -170,3 +180,6 @@ void RunTests() {
 }
 
 // TODO: ctor/dtor, copy ctor
+// TODO: Always pass IndexedFile by pointer, ie, search and remove all IndexedFile& refs.
+// TODO: Rename IndexedFile to IndexFile
+

src/utils.cc

@@ -3,6 +3,7 @@
 #include <cassert>
 #include <iostream>
 #include <fstream>
+#include <unordered_map>
 
 #include <tinydir.h>
 
@@ -112,19 +113,54 @@ std::vector<std::string> ReadLines(std::string filename) {
   return result;
 }
 
-void ParseTestExpectation(std::string filename, std::string* expected_output) {
+std::unordered_map<std::string, std::string> ParseTestExpectation(std::string filename) {
   bool in_output = false;
 
+#if false
+  #include "bar.h"
+
+  void foo();
+
+  /*
+  // if no name is given assume to be this file name
+  // no output section means we don't check that index.
+  OUTPUT: bar.cc
+  {}
+
+  OUTPUT: bar.h
+  {}
+  */
+#endif
+
+  std::unordered_map<std::string, std::string> result;
+
+  std::string active_output_filename;
+  std::string active_output_contents;
+
   for (std::string line : ReadLines(filename)) {
     if (line == "*/")
       break;
 
-    if (in_output)
+    if (StartsWith(line, "OUTPUT:")) {
-      *expected_output += line + "\n";
+      if (in_output) {
+        result[active_output_filename] = active_output_contents;
+      }
+
+      if (line.size() > 7)
+        active_output_filename = line.substr(8);
+      else
+        active_output_filename = filename;
+      active_output_contents = "";
 
-    if (line == "OUTPUT:")
       in_output = true;
+    }
+    else if (in_output)
+      active_output_contents += line + "\n";
   }
+
+  if (in_output)
+    result[active_output_filename] = active_output_contents;
+  return result;
 }
 
 

src/utils.h

@@ -1,9 +1,10 @@
 #pragma once
 
 #include <functional>
-#include <string>
-#include <vector>
 #include <memory>
+#include <string>
+#include <unordered_map>>
+#include <vector>
 
 // Returns true if |value| starts/ends with |start| or |ending|.
 bool StartsWith(const std::string& value, const std::string& start);
@@ -12,7 +13,7 @@ bool EndsWith(const std::string& value, const std::string& ending);
 // Finds all files in the given folder. This is recursive.
 std::vector<std::string> GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path);
 std::vector<std::string> ReadLines(std::string filename);
-void ParseTestExpectation(std::string filename, std::string* expected_output);
+std::unordered_map<std::string, std::string> ParseTestExpectation(std::string filename);
 
 void Fail(const std::string& message);
 

tests/multi_file/header.h

@@ -22,111 +22,4 @@ struct Foo2 {};
 enum Foo3 { A, B, C };
 
 int Foo4;
-static int Foo5;
+static int Foo5;
-
-/*
-OUTPUT:
-{
-  "types": [{
-      "id": 0,
-      "usr": "c:@S@Base",
-      "short_name": "Base",
-      "qualified_name": "Base",
-      "definition_spelling": "10:8-10:12",
-      "definition_extent": "10:1-10:15",
-      "derived": [1],
-      "uses": ["*10:8-10:12", "*12:26-12:30"]
-    }, {
-      "id": 1,
-      "usr": "c:@S@SameFileDerived",
-      "short_name": "SameFileDerived",
-      "qualified_name": "SameFileDerived",
-      "definition_spelling": "12:8-12:23",
-      "definition_extent": "12:1-12:33",
-      "parents": [0],
-      "uses": ["*12:8-12:23", "*14:14-14:29"]
-    }, {
-      "id": 2,
-      "usr": "c:@Foo0",
-      "short_name": "Foo0",
-      "qualified_name": "Foo0",
-      "definition_spelling": "14:7-14:11",
-      "definition_extent": "14:1-14:29",
-      "alias_of": 1,
-      "uses": ["*14:7-14:11"]
-    }, {
-      "id": 3,
-      "usr": "c:@ST>1#T@Foo2",
-      "short_name": "Foo2",
-      "qualified_name": "Foo2",
-      "definition_spelling": "20:8-20:12",
-      "definition_extent": "20:1-20:15",
-      "uses": ["*20:8-20:12"]
-    }, {
-      "id": 4,
-      "usr": "c:@E@Foo3",
-      "short_name": "Foo3",
-      "qualified_name": "Foo3",
-      "definition_spelling": "22:6-22:10",
-      "definition_extent": "22:1-22:22",
-      "vars": [0, 1, 2],
-      "uses": ["*22:6-22:10"]
-    }],
-  "funcs": [{
-      "id": 0,
-      "usr": "c:@FT@>1#TFoo1#v#",
-      "short_name": "Foo1",
-      "qualified_name": "Foo1",
-      "definition_spelling": "17:6-17:10",
-      "definition_extent": "17:1-17:12",
-      "uses": ["17:6-17:10"]
-    }],
-  "vars": [{
-      "id": 0,
-      "usr": "c:@E@Foo3@A",
-      "short_name": "A",
-      "qualified_name": "Foo3::A",
-      "definition_spelling": "22:13-22:14",
-      "definition_extent": "22:13-22:14",
-      "variable_type": 4,
-      "declaring_type": 4,
-      "uses": ["22:13-22:14"]
-    }, {
-      "id": 1,
-      "usr": "c:@E@Foo3@B",
-      "short_name": "B",
-      "qualified_name": "Foo3::B",
-      "definition_spelling": "22:16-22:17",
-      "definition_extent": "22:16-22:17",
-      "variable_type": 4,
-      "declaring_type": 4,
-      "uses": ["22:16-22:17"]
-    }, {
-      "id": 2,
-      "usr": "c:@E@Foo3@C",
-      "short_name": "C",
-      "qualified_name": "Foo3::C",
-      "definition_spelling": "22:19-22:20",
-      "definition_extent": "22:19-22:20",
-      "variable_type": 4,
-      "declaring_type": 4,
-      "uses": ["22:19-22:20"]
-    }, {
-      "id": 3,
-      "usr": "c:@Foo4",
-      "short_name": "Foo4",
-      "qualified_name": "Foo4",
-      "definition_spelling": "24:5-24:9",
-      "definition_extent": "24:1-24:9",
-      "uses": ["24:5-24:9"]
-    }, {
-      "id": 4,
-      "usr": "c:header.h@Foo5",
-      "short_name": "Foo5",
-      "qualified_name": "Foo5",
-      "definition_spelling": "25:12-25:16",
-      "definition_extent": "25:1-25:16",
-      "uses": ["25:12-25:16"]
-    }]
-}
-*/

tests/multi_file/impl.cc

@@ -1,246 +1,131 @@
-#include "serializer.h"
-#include "indexer.h"
-
-#if false
-// int
-void Reflect(Reader& visitor, int& value) {
-  value = visitor.GetInt();
-}
-void Reflect(Writer& visitor, int& value) {
-  visitor.Int(value);
-}
-// bool
-void Reflect(Reader& visitor, bool& value) {
-  value = visitor.GetBool();
-}
-void Reflect(Writer& visitor, bool& value) {
-  visitor.Bool(value);
-}
-// std::string
-void Reflect(Reader& visitor, std::string& value) {
-  value = visitor.GetString();
-}
-void Reflect(Writer& visitor, std::string& value) {
-  visitor.String(value.c_str(), value.size());
-}
-
-
-// ReflectMember
-void ReflectMember(Writer& visitor, const char* name, std::string& value) {
-  if (value.empty())
-    return;
-  visitor.Key(name);
-  Reflect(visitor, value);
-}
-
-
-// Location
-void Reflect(Reader& visitor, Location& value) {
-  value = Location(visitor.GetString());
-}
-void Reflect(Writer& visitor, Location& value) {
-  // We only ever want to emit id=1 files.
-  assert(value.raw_file_id == 1);
-
-  std::string output = value.ToString();
-  visitor.String(output.c_str(), output.size());
-}
-
-
-// Id<T>
-template<typename T>
-void Reflect(Reader& visitor, Id<T>& id) {
-  id.id = visitor.GetUint64();
-}
-template<typename T>
-void Reflect(Writer& visitor, Id<T>& value) {
-  visitor.Uint64(value.id);
-}
-
-
-// Ref<IndexedFuncDef>
-void Reflect(Reader& visitor, Ref<IndexedFuncDef>& value) {
-  const char* str_value = visitor.GetString();
-  uint64_t id = atoi(str_value);
-  const char* loc_string = strchr(str_value, '@') + 1;
-
-  value.id = Id<IndexedFuncDef>(id);
-  value.loc = Location(loc_string);
-}
-void Reflect(Writer& visitor, Ref<IndexedFuncDef>& value) {
-  std::string s = std::to_string(value.id.id) + "@" + value.loc.ToString();
-  visitor.String(s.c_str());
-}
-
-
-
-
-
-
-
-// TODO: Move this to indexer.cpp
-// TODO: Rename indexer.cpp to indexer.cc
-// TODO: Do not serialize a USR if it has no usages/etc outside of USR info.
-
-// IndexedTypeDef
-bool ReflectMemberStart(Reader& reader, IndexedTypeDef& value) {
-  //value.is_bad_def = false;
-  return true;
-}
-bool ReflectMemberStart(Writer& writer, IndexedTypeDef& value) {
-  //if (value.is_bad_def)
-  //  return false;
-  DefaultReflectMemberStart(writer);
-  return true;
-}
-template<typename TVisitor>
-void Reflect(TVisitor& visitor, IndexedTypeDef& value) {
-  REFLECT_MEMBER_START();
-  REFLECT_MEMBER2("id", value.id);
-  REFLECT_MEMBER2("usr", value.def.usr);
-  REFLECT_MEMBER2("short_name", value.def.short_name);
-  REFLECT_MEMBER2("qualified_name", value.def.qualified_name);
-  REFLECT_MEMBER2("definition", value.def.definition);
-  REFLECT_MEMBER2("alias_of", value.def.alias_of);
-  REFLECT_MEMBER2("parents", value.def.parents);
-  REFLECT_MEMBER2("derived", value.derived);
-  REFLECT_MEMBER2("types", value.def.types);
-  REFLECT_MEMBER2("funcs", value.def.funcs);
-  REFLECT_MEMBER2("vars", value.def.vars);
-  REFLECT_MEMBER2("uses", value.uses);
-  REFLECT_MEMBER_END();
-}
-
-
-// IndexedFuncDef
-bool ReflectMemberStart(Reader& reader, IndexedFuncDef& value) {
-  //value.is_bad_def = false;
-  return true;
-}
-bool ReflectMemberStart(Writer& writer, IndexedFuncDef& value) {
-  //if (value.is_bad_def)
-  //  return false;
-  DefaultReflectMemberStart(writer);
-  return true;
-}
-template<typename TVisitor>
-void Reflect(TVisitor& visitor, IndexedFuncDef& value) {
-  REFLECT_MEMBER_START();
-  REFLECT_MEMBER2("id", value.id);
-  REFLECT_MEMBER2("usr", value.def.usr);
-  REFLECT_MEMBER2("short_name", value.def.short_name);
-  REFLECT_MEMBER2("qualified_name", value.def.qualified_name);
-  REFLECT_MEMBER2("declarations", value.declarations);
-  REFLECT_MEMBER2("definition", value.def.definition);
-  REFLECT_MEMBER2("declaring_type", value.def.declaring_type);
-  REFLECT_MEMBER2("base", value.def.base);
-  REFLECT_MEMBER2("derived", value.derived);
-  REFLECT_MEMBER2("locals", value.def.locals);
-  REFLECT_MEMBER2("callers", value.callers);
-  REFLECT_MEMBER2("callees", value.def.callees);
-  REFLECT_MEMBER2("uses", value.uses);
-  REFLECT_MEMBER_END();
-}
-
-
-// IndexedVarDef
-bool ReflectMemberStart(Reader& reader, IndexedVarDef& value) {
-  //value.is_bad_def = false;
-  return true;
-}
-bool ReflectMemberStart(Writer& writer, IndexedVarDef& value) {
-  //if (value.is_bad_def)
-  //  return false;
-  DefaultReflectMemberStart(writer);
-  return true;
-}
-template<typename TVisitor>
-void Reflect(TVisitor& visitor, IndexedVarDef& value) {
-  REFLECT_MEMBER_START();
-  REFLECT_MEMBER2("id", value.id);
-  REFLECT_MEMBER2("usr", value.def.usr);
-  REFLECT_MEMBER2("short_name", value.def.short_name);
-  REFLECT_MEMBER2("qualified_name", value.def.qualified_name);
-  REFLECT_MEMBER2("declaration", value.def.declaration);
-  REFLECT_MEMBER2("definition", value.def.definition);
-  REFLECT_MEMBER2("variable_type", value.def.variable_type);
-  REFLECT_MEMBER2("declaring_type", value.def.declaring_type);
-  REFLECT_MEMBER2("uses", value.uses);
-  REFLECT_MEMBER_END();
-}
-
-
-// IndexedFile
-bool ReflectMemberStart(Writer& visitor, IndexedFile& value) {
-  auto it = value.id_cache.usr_to_type_id.find("");
-  if (it != value.id_cache.usr_to_type_id.end()) {
-    value.Resolve(it->second)->def.short_name = "<fundamental>";
-    assert(value.Resolve(it->second)->uses.size() == 0);
-  }
-
-  DefaultReflectMemberStart(visitor);
-  return true;
-}
-template<typename TVisitor>
-void Reflect(TVisitor& visitor, IndexedFile& value) {
-  REFLECT_MEMBER_START();
-  REFLECT_MEMBER(types);
-  REFLECT_MEMBER(funcs);
-  REFLECT_MEMBER(vars);
-  REFLECT_MEMBER_END();
-}
-
-
-
-
-
-
-
-std::string Serialize(IndexedFile& file) {
-  rapidjson::StringBuffer output;
-  rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(output);
-  //Writer writer(output);
-  writer.SetFormatOptions(
-    rapidjson::PrettyFormatOptions::kFormatSingleLineArray);
-  writer.SetIndent(' ', 2);
-
-  Reflect(writer, file);
-
-  return output.GetString();
-}
-
-optional<IndexedFile> Deserialize(std::string path, std::string serialized) {
-  rapidjson::Document reader;
-  reader.Parse(serialized.c_str());
-  if (reader.HasParseError())
-    return nullopt;
-
-  IndexedFile file(path);
-  Reflect(reader, file);
-
-  return file;
-}
-#endif
-#if false
 #include "header.h"
-#include "serializer.h"
 
-
+void Impl() {
-struct SeparateFileDerived : Base {};
+  Foo1<int>();
-
-void f() {
-  rapidjson::StringBuffer output;
-  rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(output);
-  //Writer writer(output);
-  writer.SetFormatOptions(
-    rapidjson::PrettyFormatOptions::kFormatSingleLineArray);
-  writer.SetIndent(' ', 2);
-
-  Foo2<int> a;
 }
-#endif
 
 /*
-OUTPUT:
+OUTPUT: header.h
-{}
+{
+  "types": [{
+      "id": 0,
+      "usr": "c:@S@Base",
+      "short_name": "Base",
+      "qualified_name": "Base",
+      "definition_spelling": "10:8-10:12",
+      "definition_extent": "10:1-10:15",
+      "derived": [1],
+      "uses": ["*10:8-10:12", "*12:26-12:30"]
+    }, {
+      "id": 1,
+      "usr": "c:@S@SameFileDerived",
+      "short_name": "SameFileDerived",
+      "qualified_name": "SameFileDerived",
+      "definition_spelling": "12:8-12:23",
+      "definition_extent": "12:1-12:33",
+      "parents": [0],
+      "uses": ["*12:8-12:23", "*14:14-14:29"]
+    }, {
+      "id": 2,
+      "usr": "c:@Foo0",
+      "short_name": "Foo0",
+      "qualified_name": "Foo0",
+      "definition_spelling": "14:7-14:11",
+      "definition_extent": "14:1-14:29",
+      "alias_of": 1,
+      "uses": ["*14:7-14:11"]
+    }, {
+      "id": 3,
+      "usr": "c:@ST>1#T@Foo2",
+      "short_name": "Foo2",
+      "qualified_name": "Foo2",
+      "definition_spelling": "20:8-20:12",
+      "definition_extent": "20:1-20:15",
+      "uses": ["*20:8-20:12"]
+    }, {
+      "id": 4,
+      "usr": "c:@E@Foo3",
+      "short_name": "Foo3",
+      "qualified_name": "Foo3",
+      "definition_spelling": "22:6-22:10",
+      "definition_extent": "22:1-22:22",
+      "vars": [0, 1, 2],
+      "uses": ["*22:6-22:10"]
+    }],
+  "funcs": [{
+      "id": 0,
+      "usr": "c:@FT@>1#TFoo1#v#",
+      "short_name": "Foo1",
+      "qualified_name": "Foo1",
+      "definition_spelling": "17:6-17:10",
+      "definition_extent": "17:1-17:15",
+      "uses": ["17:6-17:10"]
+    }],
+  "vars": [{
+      "id": 0,
+      "usr": "c:@E@Foo3@A",
+      "short_name": "A",
+      "qualified_name": "Foo3::A",
+      "definition_spelling": "22:13-22:14",
+      "definition_extent": "22:13-22:14",
+      "variable_type": 4,
+      "declaring_type": 4,
+      "uses": ["22:13-22:14"]
+    }, {
+      "id": 1,
+      "usr": "c:@E@Foo3@B",
+      "short_name": "B",
+      "qualified_name": "Foo3::B",
+      "definition_spelling": "22:16-22:17",
+      "definition_extent": "22:16-22:17",
+      "variable_type": 4,
+      "declaring_type": 4,
+      "uses": ["22:16-22:17"]
+    }, {
+      "id": 2,
+      "usr": "c:@E@Foo3@C",
+      "short_name": "C",
+      "qualified_name": "Foo3::C",
+      "definition_spelling": "22:19-22:20",
+      "definition_extent": "22:19-22:20",
+      "variable_type": 4,
+      "declaring_type": 4,
+      "uses": ["22:19-22:20"]
+    }, {
+      "id": 3,
+      "usr": "c:@Foo4",
+      "short_name": "Foo4",
+      "qualified_name": "Foo4",
+      "definition_spelling": "24:5-24:9",
+      "definition_extent": "24:1-24:9",
+      "uses": ["24:5-24:9"]
+    }, {
+      "id": 4,
+      "usr": "c:header.h@Foo5",
+      "short_name": "Foo5",
+      "qualified_name": "Foo5",
+      "definition_spelling": "25:12-25:16",
+      "definition_extent": "25:1-25:16",
+      "uses": ["25:12-25:16"]
+    }]
+}
+
+OUTPUT: impl.cc
+{
+  "funcs": [{
+      "id": 0,
+      "usr": "c:@F@Impl#",
+      "short_name": "Impl",
+      "qualified_name": "Impl",
+      "definition_spelling": "3:6-3:10",
+      "definition_extent": "3:1-5:2",
+      "callees": ["1@4:3-4:7"],
+      "uses": ["3:6-3:10"]
+    }, {
+      "id": 1,
+      "usr": "c:@FT@>1#TFoo1#v#",
+      "callers": ["0@4:3-4:7"],
+      "uses": ["4:3-4:7"]
+    }]
+}
 */

tests/multi_file/simple_header.h

@@ -0,0 +1,3 @@
+#pragma once
+
+void header();

tests/multi_file/simple_impl.cc

@@ -0,0 +1,38 @@
+#include "simple_header.h"
+
+void impl() {
+  header();
+}
+
+/*
+OUTPUT: simple_header.h
+{
+  "funcs": [{
+      "id": 0,
+      "usr": "c:@F@header#",
+      "short_name": "header",
+      "qualified_name": "header",
+      "declarations": ["3:6-3:12"],
+      "uses": ["3:6-3:12"]
+    }]
+}
+
+OUTPUT: simple_impl.cc
+{
+  "funcs": [{
+      "id": 0,
+      "usr": "c:@F@impl#",
+      "short_name": "impl",
+      "qualified_name": "impl",
+      "definition_spelling": "3:6-3:10",
+      "definition_extent": "3:1-5:2",
+      "callees": ["1@4:3-4:9"],
+      "uses": ["3:6-3:10"]
+    }, {
+      "id": 1,
+      "usr": "c:@F@header#",
+      "callers": ["0@4:3-4:9"],
+      "uses": ["4:3-4:9"]
+    }]
+}
+*/

tests/usage/func_called_from_macro_argument.cc

@@ -16,7 +16,8 @@ OUTPUT:
       "short_name": "called",
       "qualified_name": "called",
       "declarations": ["3:6-3:12"],
-      "uses": ["3:6-3:12"]
+      "callers": ["1@6:14-6:20"],
+      "uses": ["3:6-3:12", "6:14-6:20"]
     }, {
       "id": 1,
       "usr": "c:@F@caller#",
@@ -24,6 +25,7 @@ OUTPUT:
       "qualified_name": "caller",
       "definition_spelling": "5:6-5:12",
       "definition_extent": "5:1-7:2",
+      "callees": ["0@6:14-6:20"],
       "uses": ["5:6-5:12"]
     }]
 }