wip

src/command_line.cc

@@ -30,8 +30,16 @@ const int kQueueSizeBytes = 1024 * 8;
 }
 
 struct IndexTranslationUnitRequest {
+  enum class Type {
+    Import,
+    Update
+  };
+
   std::string path;
   std::vector<std::string> args;
+  Type type;
+
+  IndexTranslationUnitRequest(Type type) : type(type) {}
 };
 
 struct IndexTranslationUnitResponse {
@@ -152,23 +160,39 @@ void RegisterMessageTypes() {
   MessageRegistry::instance()->Register<Ipc_WorkspaceSymbol>();
 }
 
-void WriteToCache(std::string filename, IndexedFile& file) {
+std::string GetCachedFileName(std::string source_file) {
   // TODO/FIXME
   const char* kCacheDirectory = "C:/Users/jacob/Desktop/superindex/indexer/CACHE/";
+  std::replace(source_file.begin(), source_file.end(), '\\', '_');
+  std::replace(source_file.begin(), source_file.end(), '/', '_');
+  std::replace(source_file.begin(), source_file.end(), ':', '_');
+  std::replace(source_file.begin(), source_file.end(), '.', '_');
+  return kCacheDirectory + source_file + ".json";
+}
 
+optional<IndexedFile> LoadCachedFile(std::string filename) {
+  // TODO FIXME FIXME FIXME
+  return nullopt;
 
-  std::replace(filename.begin(), filename.end(), '\\', '_');
+  std::string cache_file = GetCachedFileName(filename);
-  std::replace(filename.begin(), filename.end(), '/', '_');
-  std::replace(filename.begin(), filename.end(), ':', '_');
-  std::replace(filename.begin(), filename.end(), '.', '_');
-  std::string cache_file = kCacheDirectory + filename + ".json";
 
+  std::ifstream cache;
+  cache.open(GetCachedFileName(filename));
+  if (!cache.good())
+    return nullopt;
+
+  std::string file_content = std::string(
+    std::istreambuf_iterator<char>(cache),
+    std::istreambuf_iterator<char>());
+
+  return Deserialize(filename, file_content);
+}
+
+void WriteToCache(std::string filename, IndexedFile& file) {
   std::string indexed_content = Serialize(file);
 
-
-  std::cerr << "Caching to " << cache_file << std::endl;
   std::ofstream cache;
-  cache.open(cache_file);
+  cache.open(GetCachedFileName(filename));
   assert(cache.good());
   cache << indexed_content;
   cache.close();
@@ -184,24 +208,60 @@ void IndexMain(IndexRequestQueue* requests, IndexResponseQueue* responses) {
       continue;
     }
 
+
+    Timer time;
+
+    // If the index update is an import, then we will load the previous index
+    // into memory if we have a previous index. After that, we dispatch an
+    // update request to get the latest version.
+    if (request->type == IndexTranslationUnitRequest::Type::Import) {
+      request->type = IndexTranslationUnitRequest::Type::Update;
+
+      // TODO: we're not serializing out the files cache. We only ever want to import references
+      // from the primary file though, so that should be ok. We need to cleanup indexer output.
+      optional<IndexedFile> old_index = LoadCachedFile(request->path);
+      if (old_index.has_value()) {
+        IndexUpdate update(old_index.value());
+        IndexTranslationUnitResponse response(update);
+        responses->Enqueue(response);
+        time.ResetAndPrint("Loading cached index");
+        requests->Enqueue(request.value());
+        continue;
+      }
+    }
+
+    assert(request->type == IndexTranslationUnitRequest::Type::Update);
+
     // Parse request and send a response.
     std::cerr << "Parsing file " << request->path << " with args "
       << Join(request->args, ", ") << std::endl;
 
-    Timer time;
+    IndexedFile new_index = Parse(request->path, request->args);
-    IndexedFile file = Parse(request->path, request->args);
     time.ResetAndPrint("Parsing/indexing");
 
-    // TODO: Check cache for existing index; compute diff if there is one.
+    // If we have a cached index, that means it is already imported, which
-    IndexUpdate update(file);
+    // means we want to apply a delta update.
-    IndexTranslationUnitResponse response(update);
+    optional<IndexedFile> old_index = LoadCachedFile(request->path);
-    time.ResetAndPrint("Creating index update/response");
+    time.ResetAndPrint("Loading previous index");
-
+    if (old_index) {
-    responses->Enqueue(response);
+      // Apply delta update.
-    time.ResetAndPrint("Sending update to server");
+      IndexUpdate update(old_index.value(), new_index);
+      IndexTranslationUnitResponse response(update);
+      time.ResetAndPrint("Creating delta index update/response");
+      responses->Enqueue(response);
+      time.ResetAndPrint("Sending update to server");
+    }
+    else {
+      // Apply full update.
+      IndexUpdate update(new_index);
+      IndexTranslationUnitResponse response(update);
+      time.ResetAndPrint("Creating index update/response");
+      responses->Enqueue(response);
+      time.ResetAndPrint("Sending update to server");
+    }
 
     // Cache file so we can diff it later.
-    WriteToCache(request->path, file);
+    WriteToCache(request->path, new_index);
     time.ResetAndPrint("Cache index update to disk");
   }
 }
@@ -230,6 +290,7 @@ lsLocation GetLsLocation(const QueryableLocation& location) {
 void AddCodeLens(std::vector<TCodeLens>* result,
   QueryableLocation loc,
   const std::vector<QueryableLocation>& uses,
+  bool exclude_loc,
   bool only_interesting,
   const char* singular,
   const char* plural) {
@@ -248,6 +309,8 @@ void AddCodeLens(std::vector<TCodeLens>* result,
   // Add unique uses.
   std::unordered_set<lsLocation> unique_uses;
   for (const QueryableLocation& use : uses) {
+    if (exclude_loc && use == loc)
+      continue;
     if (only_interesting && !use.interesting)
       continue;
     unique_uses.insert(GetLsLocation(use));
@@ -263,13 +326,14 @@ void AddCodeLens(std::vector<TCodeLens>* result,
   else
     code_lens.command->title += plural;
 
-  if (unique_uses.size() > 0)
+  if (exclude_loc || unique_uses.size() > 0)
     result->push_back(code_lens);
 }
 
 void AddCodeLens(std::vector<TCodeLens>* result,
   QueryableLocation loc,
   const std::vector<UsrRef>& uses,
+  bool exclude_loc,
   bool only_interesting,
   const char* singular,
   const char* plural) {
@@ -277,13 +341,14 @@ void AddCodeLens(std::vector<TCodeLens>* result,
   uses0.reserve(uses.size());
   for (const UsrRef& use : uses)
     uses0.push_back(use.loc);
-  AddCodeLens(result, loc, uses0, only_interesting, singular, plural);
+  AddCodeLens(result, loc, uses0, exclude_loc, only_interesting, singular, plural);
 }
 
 void AddCodeLens(std::vector<TCodeLens>* result,
   QueryableDatabase* db,
   QueryableLocation loc,
   const std::vector<Usr>& usrs,
+  bool exclude_loc,
   bool only_interesting,
   const char* singular,
   const char* plural) {
@@ -317,7 +382,7 @@ void AddCodeLens(std::vector<TCodeLens>* result,
     }
     }
   }
-  AddCodeLens(result, loc, uses0, only_interesting, singular, plural);
+  AddCodeLens(result, loc, uses0, exclude_loc, only_interesting, singular, plural);
 }
 
 void QueryDbMainLoop(
@@ -362,7 +427,7 @@ void QueryDbMainLoop(
           << "] Dispatching index request for file " << filepath
           << std::endl;
 
-        IndexTranslationUnitRequest request;
+        IndexTranslationUnitRequest request(IndexTranslationUnitRequest::Type::Import);
         request.path = filepath;
         request.args = entry.args;
         index_requests->Enqueue(request);
@@ -488,34 +553,39 @@ void QueryDbMainLoop(
       }
 
       for (UsrRef ref : file->outline) {
+        // NOTE: We OffsetColumn so that the code lens always show up in a
+        // predictable order. Otherwise, the client may randomize it.
+
         SymbolIdx symbol = db->usr_to_symbol[ref.usr];
         switch (symbol.kind) {
         case SymbolKind::Type: {
           QueryableTypeDef& def = db->types[symbol.idx];
-          AddCodeLens(&response.result, ref.loc, def.uses,
+          AddCodeLens(&response.result, ref.loc.OffsetColumn(0), def.uses,
-            true /*only_interesting*/, "reference",
+            false /*exclude_loc*/, true /*only_interesting*/, "reference",
             "references");
-          AddCodeLens(&response.result, db, ref.loc, def.derived,
+          AddCodeLens(&response.result, db, ref.loc.OffsetColumn(1), def.derived,
-            false /*only_interesting*/, "derived", "derived");
+            false /*exclude_loc*/, false /*only_interesting*/, "derived", "derived");
+          AddCodeLens(&response.result, db, ref.loc.OffsetColumn(2), def.instantiations,
+            false /*exclude_loc*/, false /*only_interesting*/, "instantiation", "instantiations");
           break;
         }
         case SymbolKind::Func: {
           QueryableFuncDef& def = db->funcs[symbol.idx];
-          AddCodeLens(&response.result, ref.loc, def.uses,
+          //AddCodeLens(&response.result, ref.loc.OffsetColumn(0), def.uses,
-            false /*only_interesting*/, "reference",
+          //  false /*exclude_loc*/, false /*only_interesting*/, "reference",
-            "references");
+          //  "references");
-          AddCodeLens(&response.result, ref.loc, def.callers,
+          AddCodeLens(&response.result, ref.loc.OffsetColumn(1), def.callers,
-            false /*only_interesting*/, "caller", "callers");
+            false /*exclude_loc*/, false /*only_interesting*/, "caller", "callers");
-          AddCodeLens(&response.result, ref.loc, def.def.callees,
+          AddCodeLens(&response.result, ref.loc.OffsetColumn(2), def.def.callees,
-            false /*only_interesting*/, "callee", "callees");
+            false /*exclude_loc*/, false /*only_interesting*/, "callee", "callees");
-          AddCodeLens(&response.result, db, ref.loc, def.derived,
+          AddCodeLens(&response.result, db, ref.loc.OffsetColumn(3), def.derived,
-            false /*only_interesting*/, "derived", "derived");
+            false /*exclude_loc*/, false /*only_interesting*/, "derived", "derived");
           break;
         }
         case SymbolKind::Var: {
           QueryableVarDef& def = db->vars[symbol.idx];
-          AddCodeLens(&response.result, ref.loc, def.uses,
+          AddCodeLens(&response.result, ref.loc.OffsetColumn(0), def.uses,
-            false /*only_interesting*/, "reference",
+            true /*exclude_loc*/, false /*only_interesting*/, "reference",
             "references");
           break;
         }
@@ -830,9 +900,10 @@ void LanguageServerMain(std::string process_name) {
 }
 
 int main(int argc, char** argv) {
-  bool loop = false;
+  //bool loop = true;
-  while (loop)
+  //while (loop)
-    std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  //  std::this_thread::sleep_for(std::chrono::milliseconds(10));
+  std::this_thread::sleep_for(std::chrono::seconds(3));
 
   PlatformInit();
   RegisterMessageTypes();

src/indexer.cpp

@@ -5,7 +5,7 @@
 
 #include "serializer.h"
 
-IndexedFile::IndexedFile(const std::string& path) : path(path) {
+IndexedFile::IndexedFile(const std::string& path) : id_cache(path), path(path) {
   // TODO: Reconsider if we should still be reusing the same id_cache.
   // Preallocate any existing resolved ids.
   for (const auto& entry : id_cache.usr_to_type_id)
@@ -119,13 +119,13 @@ std::string Location::ToPrettyString(IdCache* id_cache) {
   return result;
 }
 
-IdCache::IdCache() {
+IdCache::IdCache(const std::string& primary_file) : primary_file(primary_file) {
   // Reserve id 0 for unfound.
   file_path_to_file_id[""] = FileId(0);
   file_id_to_file_path[FileId(0)] = "";
 }
 
-Location IdCache::Resolve(const CXSourceLocation& cx_loc, bool interesting) {
+Location IdCache::ForceResolve(const CXSourceLocation& cx_loc, bool interesting) {
   CXFile file;
   unsigned int line, column, offset;
   clang_getSpellingLocation(cx_loc, &file, &line, &column, &offset);
@@ -137,7 +137,8 @@ Location IdCache::Resolve(const CXSourceLocation& cx_loc, bool interesting) {
     auto it = file_path_to_file_id.find(path);
     if (it != file_path_to_file_id.end()) {
       file_id = it->second;
-    } else {
+    }
+    else {
       file_id = FileId(file_path_to_file_id.size());
       file_path_to_file_id[path] = file_id;
       file_id_to_file_path[file_id] = path;
@@ -147,16 +148,32 @@ Location IdCache::Resolve(const CXSourceLocation& cx_loc, bool interesting) {
   return Location(interesting, file_id, line, column);
 }
 
-Location IdCache::Resolve(const CXIdxLoc& cx_idx_loc, bool interesting) {
+Location IdCache::ForceResolve(const CXIdxLoc& cx_idx_loc, bool interesting) {
+  CXSourceLocation cx_loc = clang_indexLoc_getCXSourceLocation(cx_idx_loc);
+  return ForceResolve(cx_loc, interesting);
+}
+
+Location IdCache::ForceResolve(const CXCursor& cx_cursor, bool interesting) {
+  return ForceResolve(clang_getCursorLocation(cx_cursor), interesting);
+}
+
+
+optional<Location> IdCache::Resolve(const CXSourceLocation& cx_loc, bool interesting) {
+  if (!clang_Location_isFromMainFile(cx_loc))
+    return nullopt;
+  return ForceResolve(cx_loc, interesting);
+}
+
+optional<Location> IdCache::Resolve(const CXIdxLoc& cx_idx_loc, bool interesting) {
   CXSourceLocation cx_loc = clang_indexLoc_getCXSourceLocation(cx_idx_loc);
   return Resolve(cx_loc, interesting);
 }
 
-Location IdCache::Resolve(const CXCursor& cx_cursor, bool interesting) {
+optional<Location> IdCache::Resolve(const CXCursor& cx_cursor, bool interesting) {
   return Resolve(clang_getCursorLocation(cx_cursor), interesting);
 }
 
-Location IdCache::Resolve(const clang::Cursor& cursor, bool interesting) {
+optional<Location> IdCache::Resolve(const clang::Cursor& cursor, bool interesting) {
   return Resolve(cursor.cx_cursor, interesting);
 }
 
@@ -236,7 +253,7 @@ void diagnostic(CXClientData client_data,
 
     std::string spelling =
         clang::ToString(clang_getDiagnosticSpelling(diagnostic));
-    Location location = param->db->id_cache.Resolve(
+    Location location = param->db->id_cache.ForceResolve(
         clang_getDiagnosticLocation(diagnostic), false /*interesting*/);
 
     std::cerr << location.ToPrettyString(&param->db->id_cache) << ": "
@@ -386,8 +403,11 @@ void VisitDeclForTypeUsageVisitorHandler(clang::Cursor cursor,
 
   if (param->is_interesting) {
     IndexedTypeDef* ref_type_def = db->Resolve(ref_type_id);
-    Location loc = db->id_cache.Resolve(cursor, true /*interesting*/);
+    // TODO: Should we even be visiting this if the file is not from the main
-    AddUsage(ref_type_def->uses, loc);
+    // def? Try adding assert on |loc| later.
+    optional<Location> loc = db->id_cache.Resolve(cursor, true /*interesting*/);
+    if (loc)
+      AddUsage(ref_type_def->uses, loc.value());
   }
 }
 
@@ -451,13 +471,10 @@ optional<TypeId> ResolveToDeclarationType(IndexedFile* db,
 }
 
 // Add usages to any seen TypeRef or TemplateRef under the given |decl_cursor|.
-// This
+// This returns the first seen TypeRef or TemplateRef value, which can be
-// returns the first seen TypeRef or TemplateRef value, which can be useful if
+// useful if trying to figure out ie, what a using statement refers to. If
-// trying
+// trying to generally resolve a cursor to a type, use
-// to figure out ie, what a using statement refers to. If trying to generally
+// ResolveToDeclarationType, which works in more scenarios.
-// resolve
-// a cursor to a type, use ResolveToDeclarationType, which works in more
-// scenarios.
 optional<TypeId> AddDeclTypeUsages(
     IndexedFile* db,
     clang::Cursor decl_cursor,
@@ -638,12 +655,14 @@ clang::VisiterResult AddDeclInitializerUsagesVisitor(clang::Cursor cursor,
       if (ref_usr == "")
         break;
 
-      VarId ref_id = db->ToVarId(ref_usr);
+      optional<Location> loc = db->id_cache.Resolve(cursor, false /*interesting*/);
-      IndexedVarDef* ref_def = db->Resolve(ref_id);
-      Location loc = db->id_cache.Resolve(cursor, false /*interesting*/);
       // std::cerr << "Adding usage to id=" << ref_id.id << " usr=" << ref_usr
       // << " at " << loc.ToString() << std::endl;
-      AddUsage(ref_def->uses, loc);
+      if (loc) {
+        VarId ref_id = db->ToVarId(ref_usr);
+        IndexedVarDef* ref_def = db->Resolve(ref_id);
+        AddUsage(ref_def->uses, loc.value());
+      }
       break;
   }
 
@@ -680,6 +699,11 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
     case CXIdxEntity_Field:
     case CXIdxEntity_Variable:
     case CXIdxEntity_CXXStaticVariable: {
+      optional<Location> decl_loc =
+        db->id_cache.Resolve(decl->loc, false /*interesting*/);
+      if (!decl_loc)
+        break;
+
       clang::Cursor decl_cursor = decl->cursor;
 
       // Do not index implicit template instantiations.
@@ -692,8 +716,6 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       VarId var_id = db->ToVarId(decl->entityInfo->USR);
       IndexedVarDef* var_def = db->Resolve(var_id);
 
-      var_def->is_bad_def = is_system_def;
-
       // TODO: Eventually run with this if. Right now I want to iron out bugs
       // this may shadow.
       // TODO: Verify this gets called multiple times
@@ -703,13 +725,11 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
           ns->QualifiedName(decl->semanticContainer, var_def->def.short_name);
       //}
 
-      Location decl_loc =
-          db->id_cache.Resolve(decl->loc, false /*interesting*/);
       if (decl->isDefinition)
-        var_def->def.definition = decl_loc;
+        var_def->def.definition = decl_loc.value();
       else
-        var_def->def.declaration = decl_loc;
+        var_def->def.declaration = decl_loc.value();
-      AddUsage(var_def->uses, decl_loc);
+      AddUsage(var_def->uses, decl_loc.value());
 
       // std::cerr << std::endl << "Visiting declaration" << std::endl;
       // Dump(decl_cursor);
@@ -725,13 +745,25 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
           db, decl_cursor,
           decl_cursor.get_kind() != CXCursor_ParmDecl /*is_interesting*/,
           decl->semanticContainer, decl->lexicalContainer);
-      optional<TypeId> var_type = ResolveToDeclarationType(db, decl_cursor);
-      if (var_type.has_value())
-        var_def->def.variable_type = var_type.value();
 
+      // We don't need to assign declaring type multiple times if this variable
+      // has already been seen.
+      if (!decl->isRedeclaration) {
+        optional<TypeId> var_type = ResolveToDeclarationType(db, decl_cursor);
+        if (var_type.has_value()) {
+          // Don't treat enum definition variables as instantiations.
+          bool is_enum_member = decl->semanticContainer && decl->semanticContainer->cursor.kind == CXCursor_EnumDecl;
+          if (!is_enum_member)
+            db->Resolve(var_type.value())->instantiations.push_back(var_id.id);
+
+          var_def->def.variable_type = var_type.value();
+        }
+      }
+
+      // TODO: Refactor handlers so more things are under 'if (!decl->isRedeclaration)'
       if (decl->isDefinition && IsTypeDefinition(decl->semanticContainer)) {
         TypeId declaring_type_id =
-            db->ToTypeId(decl->semanticContainer->cursor);
+          db->ToTypeId(decl->semanticContainer->cursor);
         IndexedTypeDef* declaring_type_def = db->Resolve(declaring_type_id);
         var_def->def.declaring_type = declaring_type_id;
         declaring_type_def->def.vars.push_back(var_id);
@@ -746,6 +778,11 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
     case CXIdxEntity_CXXInstanceMethod:
     case CXIdxEntity_CXXStaticMethod:
     case CXIdxEntity_CXXConversionFunction: {
+      optional<Location> decl_loc =
+        db->id_cache.Resolve(decl->loc, false /*interesting*/);
+      if (!decl_loc)
+        break;
+
       clang::Cursor decl_cursor = decl->cursor;
       clang::Cursor resolved =
           decl_cursor.template_specialization_to_template_definition();
@@ -753,10 +790,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       FuncId func_id = db->ToFuncId(resolved.cx_cursor);
       IndexedFuncDef* func_def = db->Resolve(func_id);
 
-      Location decl_loc =
+      AddUsage(func_def->uses, decl_loc.value());
-          db->id_cache.Resolve(decl->loc, false /*interesting*/);
-
-      AddUsage(func_def->uses, decl_loc);
       // We don't actually need to know the return type, but we need to mark it
       // as an interesting usage.
       AddDeclTypeUsages(db, decl_cursor, true /*is_interesting*/,
@@ -766,9 +800,9 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       // hacking the 'declarations' field by
       // adding a definition when we really don't have one.
       if (decl->isDefinition && !func_def->def.definition.has_value())
-        func_def->def.definition = decl_loc;
+        func_def->def.definition = decl_loc.value();
       else
-        func_def->declarations.push_back(decl_loc);
+        func_def->declarations.push_back(decl_loc.value());
 
       // If decl_cursor != resolved, then decl_cursor is a template
       // specialization. We
@@ -776,8 +810,6 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       // that
       // scenario.
       if (decl_cursor == resolved) {
-        func_def->is_bad_def = is_system_def;
-
         // TODO: Eventually run with this if. Right now I want to iron out bugs
         // this may shadow.
         // if (!decl->isRedeclaration) {
@@ -805,7 +837,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
           // Mark a type reference at the ctor/dtor location.
           // TODO: Should it be interesting?
           if (is_ctor_or_dtor) {
-            Location type_usage_loc = decl_loc;
+            Location type_usage_loc = decl_loc.value();
             AddUsage(declaring_type_def->uses, type_usage_loc);
           }
 
@@ -895,6 +927,10 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
 
     case CXIdxEntity_Typedef:
     case CXIdxEntity_CXXTypeAlias: {
+      optional<Location> decl_loc = db->id_cache.Resolve(decl->loc, true /*interesting*/);
+      if (!decl_loc)
+        break;
+
       // Note we want to fetch the first TypeRef. Running
       // ResolveCursorType(decl->cursor) would return
       // the type of the typedef/using, not the type of the referenced type.
@@ -905,8 +941,6 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       TypeId type_id = db->ToTypeId(decl->entityInfo->USR);
       IndexedTypeDef* type_def = db->Resolve(type_id);
 
-      type_def->is_bad_def = is_system_def;
-
       if (alias_of)
         type_def->def.alias_of = alias_of.value();
 
@@ -914,9 +948,8 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       type_def->def.qualified_name =
           ns->QualifiedName(decl->semanticContainer, type_def->def.short_name);
 
-      Location decl_loc = db->id_cache.Resolve(decl->loc, true /*interesting*/);
+      type_def->def.definition = decl_loc.value().WithInteresting(false);
-      type_def->def.definition = decl_loc.WithInteresting(false);
+      AddUsage(type_def->uses, decl_loc.value());
-      AddUsage(type_def->uses, decl_loc);
       break;
     }
 
@@ -924,11 +957,13 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
     case CXIdxEntity_Union:
     case CXIdxEntity_Struct:
     case CXIdxEntity_CXXClass: {
+      optional<Location> decl_loc = db->id_cache.Resolve(decl->loc, true /*interesting*/);
+      if (!decl_loc)
+        break;
+
       TypeId type_id = db->ToTypeId(decl->entityInfo->USR);
       IndexedTypeDef* type_def = db->Resolve(type_id);
 
-      type_def->is_bad_def = is_system_def;
-
       // TODO: Eventually run with this if. Right now I want to iron out bugs
       // this may shadow.
       // TODO: For type section, verify if this ever runs for non definitions?
@@ -951,9 +986,8 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       // }
 
       assert(decl->isDefinition);
-      Location decl_loc = db->id_cache.Resolve(decl->loc, true /*interesting*/);
+      type_def->def.definition = decl_loc.value().WithInteresting(false);
-      type_def->def.definition = decl_loc.WithInteresting(false);
+      AddUsage(type_def->uses, decl_loc.value());
-      AddUsage(type_def->uses, decl_loc);
 
       // type_def->alias_of
       // type_def->funcs
@@ -972,9 +1006,8 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
                             decl->semanticContainer, decl->lexicalContainer);
           optional<TypeId> parent_type_id =
               ResolveToDeclarationType(db, base_class->cursor);
-          IndexedTypeDef* type_def =
+          // type_def ptr could be invalidated by ResolveToDeclarationType.
-              db->Resolve(type_id);  // type_def ptr could be invalidated by
+          IndexedTypeDef* type_def = db->Resolve(type_id);
-                                     // ResolveDeclToType.
           if (parent_type_id) {
             IndexedTypeDef* parent_type_def =
                 db->Resolve(parent_type_id.value());
@@ -990,7 +1023,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       std::cerr
           << "!! Unhandled indexDeclaration:     "
           << clang::Cursor(decl->cursor).ToString() << " at "
-          << db->id_cache.Resolve(decl->loc, false /*interesting*/).ToString()
+          << db->id_cache.ForceResolve(decl->loc, false /*interesting*/).ToString()
           << std::endl;
       std::cerr << "     entityInfo->kind  = " << decl->entityInfo->kind
                 << std::endl;
@@ -1024,9 +1057,16 @@ bool IsFunction(CXCursorKind kind) {
 
 void indexEntityReference(CXClientData client_data,
                           const CXIdxEntityRefInfo* ref) {
-  if (clang_Location_isInSystemHeader(clang_getCursorLocation(ref->cursor)) ||
+//  if (clang_Location_isInSystemHeader(clang_getCursorLocation(ref->cursor)) ||
-      clang_Location_isInSystemHeader(
+//      clang_Location_isInSystemHeader(
-          clang_getCursorLocation(ref->referencedEntity->cursor)))
+//          clang_getCursorLocation(ref->referencedEntity->cursor)))
+//    return;
+
+  // Don't index references that are not from the main file.
+  if (!clang_Location_isFromMainFile(clang_getCursorLocation(ref->cursor)))
+    return;
+  // Don't index references to system headers.
+  if (clang_Location_isInSystemHeader(clang_getCursorLocation(ref->referencedEntity->cursor)))
     return;
 
   IndexParam* param = static_cast<IndexParam*>(client_data);
@@ -1047,13 +1087,16 @@ void indexEntityReference(CXClientData client_data,
     case CXIdxEntity_CXXStaticVariable:
     case CXIdxEntity_Variable:
     case CXIdxEntity_Field: {
+      optional<Location> loc = db->id_cache.Resolve(ref->loc, false /*interesting*/);
+      if (!loc)
+        break;
+
       clang::Cursor referenced = ref->referencedEntity->cursor;
       referenced = referenced.template_specialization_to_template_definition();
 
       VarId var_id = db->ToVarId(referenced.get_usr());
       IndexedVarDef* var_def = db->Resolve(var_id);
-      Location loc = db->id_cache.Resolve(ref->loc, false /*interesting*/);
+      AddUsage(var_def->uses, loc.value());
-      AddUsage(var_def->uses, loc);
       break;
     }
 
@@ -1074,10 +1117,14 @@ void indexEntityReference(CXClientData client_data,
 
       // Don't report duplicate usages.
       // TODO: search full history?
-      Location loc = db->id_cache.Resolve(ref->loc, false /*interesting*/);
+      optional<Location> loc = db->id_cache.Resolve(ref->loc, false /*interesting*/);
-      if (param->last_func_usage_location == loc)
+      if (!loc)
         break;
-      param->last_func_usage_location = loc;
+
+      // TODO: cleanup/remove this
+      if (param->last_func_usage_location == loc.value())
+        break;
+      param->last_func_usage_location = loc.value();
 
       // Note: be careful, calling db->ToFuncId invalidates the FuncDef* ptrs.
       FuncId called_id = db->ToFuncId(ref->referencedEntity->USR);
@@ -1086,12 +1133,12 @@ void indexEntityReference(CXClientData client_data,
         IndexedFuncDef* caller_def = db->Resolve(caller_id);
         IndexedFuncDef* called_def = db->Resolve(called_id);
 
-        caller_def->def.callees.push_back(FuncRef(called_id, loc));
+        caller_def->def.callees.push_back(FuncRef(called_id, loc.value()));
-        called_def->callers.push_back(FuncRef(caller_id, loc));
+        called_def->callers.push_back(FuncRef(caller_id, loc.value()));
-        AddUsage(called_def->uses, loc);
+        AddUsage(called_def->uses, loc.value());
       } else {
         IndexedFuncDef* called_def = db->Resolve(called_id);
-        AddUsage(called_def->uses, loc);
+        AddUsage(called_def->uses, loc.value());
       }
 
       // For constructor/destructor, also add a usage against the type. Clang
@@ -1101,11 +1148,9 @@ void indexEntityReference(CXClientData client_data,
       clang::Cursor ref_cursor = ref->cursor;
       if (ref->referencedEntity->kind == CXIdxEntity_CXXConstructor ||
           ref->referencedEntity->kind == CXIdxEntity_CXXDestructor) {
-        Location parent_loc = db->id_cache.Resolve(ref->parentEntity->cursor,
+        Location parent_loc = db->id_cache.ForceResolve(ref->parentEntity->cursor,
                                                    true /*interesting*/);
-        Location our_loc =
+        if (!parent_loc.IsEqualTo(loc.value())) {
-            db->id_cache.Resolve(ref->loc, true /*is_interesting*/);
-        if (!parent_loc.IsEqualTo(our_loc)) {
           IndexedFuncDef* called_def = db->Resolve(called_id);
           // I suspect it is possible for the declaring type to be null
           // when the class is invalid.
@@ -1113,7 +1158,7 @@ void indexEntityReference(CXClientData client_data,
             // assert(called_def->def.declaring_type.has_value());
             IndexedTypeDef* type_def =
                 db->Resolve(called_def->def.declaring_type.value());
-            AddUsage(type_def->uses, our_loc);
+            AddUsage(type_def->uses, loc.value().WithInteresting(true));
           }
         }
       }
@@ -1126,23 +1171,16 @@ void indexEntityReference(CXClientData client_data,
     case CXIdxEntity_Union:
     case CXIdxEntity_Struct:
     case CXIdxEntity_CXXClass: {
+      optional<Location> loc = db->id_cache.Resolve(ref->loc, false /*interesting*/);
+      if (!loc)
+        break;
+
       clang::Cursor referenced = ref->referencedEntity->cursor;
       referenced = referenced.template_specialization_to_template_definition();
       TypeId referenced_id = db->ToTypeId(referenced.get_usr());
 
       IndexedTypeDef* referenced_def = db->Resolve(referenced_id);
 
-      // We will not get a declaration visit for forward declared types. Try to
-      // mark them as non-bad
-      // defs here so we will output usages/etc.
-      if (referenced_def->is_bad_def) {
-        bool is_system_def = clang_Location_isInSystemHeader(
-            clang_getCursorLocation(ref->referencedEntity->cursor));
-        Location loc = db->id_cache.Resolve(ref->referencedEntity->cursor,
-                                            false /*interesting*/);
-        if (!is_system_def && loc.raw_file_id != -1)
-          referenced_def->is_bad_def = false;
-      }
       //
       // The following will generate two TypeRefs to Foo, both located at the
       // same spot (line 3, column 3). One of the parents will be set to
@@ -1159,8 +1197,7 @@ void indexEntityReference(CXClientData client_data,
       //    Foo f;
       //  }
       //
-      AddUsage(referenced_def->uses,
+      AddUsage(referenced_def->uses, loc.value());
-               db->id_cache.Resolve(ref->loc, false /*interesting*/));
       break;
     }
 
@@ -1168,7 +1205,7 @@ void indexEntityReference(CXClientData client_data,
       std::cerr
           << "!! Unhandled indexEntityReference: " << cursor.ToString()
           << " at "
-          << db->id_cache.Resolve(ref->loc, false /*interesting*/).ToString()
+          << db->id_cache.ForceResolve(ref->loc, false /*interesting*/).ToString()
           << std::endl;
       std::cerr << "     ref->referencedEntity->kind = "
                 << ref->referencedEntity->kind << std::endl;
@@ -1177,7 +1214,7 @@ void indexEntityReference(CXClientData client_data,
                   << ref->parentEntity->kind << std::endl;
       std::cerr
           << "     ref->loc          = "
-          << db->id_cache.Resolve(ref->loc, false /*interesting*/).ToString()
+          << db->id_cache.ForceResolve(ref->loc, false /*interesting*/).ToString()
           << std::endl;
       std::cerr << "     ref->kind         = " << ref->kind << std::endl;
       if (ref->parentEntity)
@@ -1239,8 +1276,7 @@ IndexedFile Parse(std::string filename,
 
   std::cerr << "!! [START] Indexing " << filename << std::endl;
   clang_indexTranslationUnit(index_action, &param, callbacks, sizeof(callbacks),
-                             CXIndexOpt_IndexFunctionLocalSymbols |
+                             CXIndexOpt_IndexFunctionLocalSymbols | CXIndexOpt_SkipParsedBodiesInSession | CXIndexOpt_IndexImplicitTemplateInstantiations,
-                                 CXIndexOpt_SkipParsedBodiesInSession,
                              tu.cx_tu);
   std::cerr << "!! [END] Indexing " << filename << std::endl;
   clang_IndexAction_dispose(index_action);

src/indexer.h

@@ -323,16 +323,27 @@ struct IndexedTypeDef {
   // Immediate derived types.
   std::vector<TypeId> derived;
 
+  // Declared variables of this type.
+  // TODO: this needs a lot more work and lots of tests.
+  // TODO: add instantiation on ctor / dtor, do not add instantiation if type is ptr
+  std::vector<VarId> instantiations;
+
   // Every usage, useful for things like renames.
   // NOTE: Do not insert directly! Use AddUsage instead.
   std::vector<Location> uses;
 
-  bool is_bad_def = true;
-
   IndexedTypeDef() : def("") {}  // For serialization
 
   IndexedTypeDef(TypeId id, const std::string& usr);
 
+  bool HasInterestingState() const {
+    return
+      def.definition ||
+      !derived.empty() ||
+      !instantiations.empty() ||
+      !uses.empty();
+  }
+
   bool operator<(const IndexedTypeDef& other) const {
     return def.usr < other.def.usr;
   }
@@ -428,13 +439,21 @@ struct IndexedFuncDef {
   // All usages. For interesting usages, see callees.
   std::vector<Location> uses;
 
-  bool is_bad_def = true;
-
   IndexedFuncDef() {}  // For reflection.
   IndexedFuncDef(FuncId id, const std::string& usr) : def(usr), id(id) {
     // assert(usr.size() > 0);
   }
 
+  bool HasInterestingState() const {
+    return
+      def.definition ||
+      !def.callees.empty() ||
+      !declarations.empty() ||
+      !derived.empty() ||
+      !callers.empty() ||
+      !uses.empty();
+  }
+
   bool operator<(const IndexedFuncDef& other) const {
     return def.usr < other.def.usr;
   }
@@ -503,14 +522,18 @@ struct IndexedVarDef {
   // Usages.
   std::vector<Location> uses;
 
-  bool is_bad_def = true;
-
   IndexedVarDef() : def("") {}  // For serialization
 
   IndexedVarDef(VarId id, const std::string& usr) : def(usr), id(id) {
     // assert(usr.size() > 0);
   }
 
+  bool HasInterestingState() const {
+    return
+      def.definition ||
+      !uses.empty();
+  }
+
   bool operator<(const IndexedVarDef& other) const {
     return def.usr < other.def.usr;
   }
@@ -518,6 +541,7 @@ struct IndexedVarDef {
 MAKE_HASHABLE(IndexedVarDef, t.def.usr);
 
 struct IdCache {
+  std::string primary_file;
   std::unordered_map<std::string, FileId> file_path_to_file_id;
   std::unordered_map<std::string, TypeId> usr_to_type_id;
   std::unordered_map<std::string, FuncId> usr_to_func_id;
@@ -527,11 +551,14 @@ struct IdCache {
   std::unordered_map<FuncId, std::string> func_id_to_usr;
   std::unordered_map<VarId, std::string> var_id_to_usr;
 
-  IdCache();
+  IdCache(const std::string& primary_file);
-  Location Resolve(const CXSourceLocation& cx_loc, bool interesting);
+  Location ForceResolve(const CXSourceLocation& cx_loc, bool interesting);
-  Location Resolve(const CXIdxLoc& cx_idx_loc, bool interesting);
+  Location ForceResolve(const CXIdxLoc& cx_idx_loc, bool interesting);
-  Location Resolve(const CXCursor& cx_cursor, bool interesting);
+  Location ForceResolve(const CXCursor& cx_cursor, bool interesting);
-  Location Resolve(const clang::Cursor& cursor, bool interesting);
+  optional<Location> Resolve(const CXSourceLocation& cx_loc, bool interesting);
+  optional<Location> Resolve(const CXIdxLoc& cx_idx_loc, bool interesting);
+  optional<Location> Resolve(const CXCursor& cx_cursor, bool interesting);
+  optional<Location> Resolve(const clang::Cursor& cursor, bool interesting);
 };
 
 struct IndexedFile {

src/libclangmm/TranslationUnit.cc

@@ -24,6 +24,11 @@ TranslationUnit::TranslationUnit(
   for (const auto& arg : platform_args)
     args.push_back(arg.c_str());
 
+  std::cerr << "Parsing " << filepath << " with args ";
+  for (const auto& arg : args)
+    std::cerr << arg << " ";
+  std::cerr << std::endl;
+
   CXErrorCode error_code = clang_parseTranslationUnit2(
     index.cx_index,
     filepath.c_str(),

src/query.cc

@@ -46,8 +46,10 @@ Usr MapIdToUsr(const IdCache& id_cache, const VarId& id) {
   return id_cache.var_id_to_usr.find(id)->second;
 }
 QueryableLocation MapIdToUsr(const IdCache& id_cache, const Location& id) {
-  assert(id_cache.file_id_to_file_path.find(id.file_id()) != id_cache.file_id_to_file_path.end());
+  assert(id.raw_file_id == 1);
-  return QueryableLocation(id_cache.file_id_to_file_path.find(id.file_id())->second, id.line, id.column, id.interesting);
+  return QueryableLocation(id_cache.primary_file, id.line, id.column, id.interesting);
+  //assert(id_cache.file_id_to_file_path.find(id.file_id()) != id_cache.file_id_to_file_path.end());
+  //return QueryableLocation(id_cache.file_id_to_file_path.find(id.file_id())->second, id.line, id.column, id.interesting);
 }
 
 std::vector<Usr> MapIdToUsr(const IdCache& id_cache, const std::vector<TypeId>& ids) {
@@ -173,6 +175,7 @@ QueryableFile::QueryableFile(const IndexedFile& indexed)
 QueryableTypeDef::QueryableTypeDef(IdCache& id_cache, const IndexedTypeDef& indexed)
   : def(MapIdToUsr(id_cache, indexed.def)) {
   derived = MapIdToUsr(id_cache, indexed.derived);
+  instantiations = MapIdToUsr(id_cache, indexed.instantiations);
   uses = MapIdToUsr(id_cache, indexed.uses);
 }
 
@@ -314,12 +317,12 @@ void CompareGroups(
   while (prev_it != previous_data.end() && curr_it != current_data.end()) {
     // same id
     if (prev_it->def.usr == curr_it->def.usr) {
-      if (!prev_it->is_bad_def && !curr_it->is_bad_def)
+      //if (!prev_it->is_bad_def && !curr_it->is_bad_def)
         on_found(&*prev_it, &*curr_it);
-      else if (prev_it->is_bad_def)
+      //else if (prev_it->is_bad_def)
-        on_added(&*curr_it);
+      //  on_added(&*curr_it);
-      else if (curr_it->is_bad_def)
+      //else if (curr_it->is_bad_def)
-        on_removed(&*curr_it);
+      //  on_removed(&*curr_it);
 
       ++prev_it;
       ++curr_it;
@@ -368,17 +371,16 @@ void CompareGroups(
 
 
 IndexUpdate::IndexUpdate(IndexedFile& file) {
+  // TODO: Do not add empty data (ie, def has nothing but USR)
+
   files_added.push_back(QueryableFile(file));
   for (const IndexedTypeDef& def : file.types) {
-    if (def.is_bad_def) continue;
     types_added.push_back(QueryableTypeDef(file.id_cache, def));
   }
   for (const IndexedFuncDef& def : file.funcs) {
-    if (def.is_bad_def) continue;
     funcs_added.push_back(QueryableFuncDef(file.id_cache, def));
   }
   for (const IndexedVarDef& def : file.vars) {
-    if (def.is_bad_def) continue;
     vars_added.push_back(QueryableVarDef(file.id_cache, def));
   }
 }
@@ -491,6 +493,7 @@ void IndexUpdate::Merge(const IndexUpdate& update) {
   INDEX_UPDATE_MERGE(types_added);
   INDEX_UPDATE_MERGE(types_def_changed);
   INDEX_UPDATE_MERGE(types_derived);
+  INDEX_UPDATE_MERGE(types_instantiations);
   INDEX_UPDATE_MERGE(types_uses);
 
   INDEX_UPDATE_MERGE(funcs_removed);

src/query.h

@@ -22,6 +22,10 @@ struct QueryableLocation {
   QueryableLocation(Usr path, int line, int column, bool interesting)
     : path(path), line(line), column(column), interesting(interesting) {}
 
+  QueryableLocation OffsetColumn(int offset) const {
+    return QueryableLocation(path, line, column + offset, interesting);
+  }
+
   bool operator==(const QueryableLocation& other) const {
     // Note: We ignore |is_interesting|.
     return
@@ -104,16 +108,18 @@ MAKE_REFLECT_STRUCT(QueryableFile, file_id, outline);
 struct QueryableTypeDef {
   using DefUpdate = TypeDefDefinitionData<Usr, Usr, Usr, QueryableLocation>;
   using DerivedUpdate = MergeableUpdate<Usr>;
+  using InstantiationsUpdate = MergeableUpdate<Usr>;
   using UsesUpdate = MergeableUpdate<QueryableLocation>;
 
   DefUpdate def;
   std::vector<Usr> derived;
+  std::vector<Usr> instantiations;
   std::vector<QueryableLocation> uses;
 
   QueryableTypeDef() : def("") {} // For serialization.
   QueryableTypeDef(IdCache& id_cache, const IndexedTypeDef& indexed);
 };
-MAKE_REFLECT_STRUCT(QueryableTypeDef, def, derived, uses);
+MAKE_REFLECT_STRUCT(QueryableTypeDef, def, derived, instantiations, uses);
 
 struct QueryableFuncDef {
   using DefUpdate = FuncDefDefinitionData<Usr, Usr, Usr, UsrRef, QueryableLocation>;
@@ -186,6 +192,7 @@ struct IndexUpdate {
   std::vector<QueryableTypeDef> types_added;
   std::vector<QueryableTypeDef::DefUpdate> types_def_changed;
   std::vector<QueryableTypeDef::DerivedUpdate> types_derived;
+  std::vector<QueryableTypeDef::InstantiationsUpdate> types_instantiations;
   std::vector<QueryableTypeDef::UsesUpdate> types_uses;
 
   // Function updates.

src/serializer.cc

@@ -40,6 +40,9 @@ void Reflect(Reader& visitor, Location& value) {
   value = Location(visitor.GetString());
 }
 void Reflect(Writer& visitor, Location& value) {
+  // We only ever want to emit id=1 files.
+  assert(value.raw_file_id == 1);
+
   std::string output = value.ToString();
   visitor.String(output.c_str(), output.size());
 }
@@ -76,15 +79,22 @@ void Reflect(Writer& visitor, Ref<IndexedFuncDef>& value) {
 
 
 
+// TODO: Move this to indexer.cpp
+// TODO: Rename indexer.cpp to indexer.cc
+// TODO: Do not serialize a USR if it has no usages/etc outside of USR info.
 
 // IndexedTypeDef
 bool ReflectMemberStart(Reader& reader, IndexedTypeDef& value) {
-  value.is_bad_def = false;
+  //value.is_bad_def = false;
   return true;
 }
 bool ReflectMemberStart(Writer& writer, IndexedTypeDef& value) {
-  if (value.is_bad_def)
+  if (!value.HasInterestingState())
-    return false;
+    std::cerr << "bad";
+  assert(value.HasInterestingState());
+
+  //if (value.is_bad_def)
+  //  return false;
   DefaultReflectMemberStart(writer);
   return true;
 }
@@ -102,6 +112,7 @@ void Reflect(TVisitor& visitor, IndexedTypeDef& value) {
   REFLECT_MEMBER2("types", value.def.types);
   REFLECT_MEMBER2("funcs", value.def.funcs);
   REFLECT_MEMBER2("vars", value.def.vars);
+  REFLECT_MEMBER2("instantiations", value.instantiations);
   REFLECT_MEMBER2("uses", value.uses);
   REFLECT_MEMBER_END();
 }
@@ -109,12 +120,16 @@ void Reflect(TVisitor& visitor, IndexedTypeDef& value) {
 
 // IndexedFuncDef
 bool ReflectMemberStart(Reader& reader, IndexedFuncDef& value) {
-  value.is_bad_def = false;
+  //value.is_bad_def = false;
   return true;
 }
 bool ReflectMemberStart(Writer& writer, IndexedFuncDef& value) {
-  if (value.is_bad_def)
+  if (!value.HasInterestingState())
-    return false;
+    std::cerr << "bad";
+  assert(value.HasInterestingState());
+  
+  //if (value.is_bad_def)
+  //  return false;
   DefaultReflectMemberStart(writer);
   return true;
 }
@@ -140,12 +155,16 @@ void Reflect(TVisitor& visitor, IndexedFuncDef& value) {
 
 // IndexedVarDef
 bool ReflectMemberStart(Reader& reader, IndexedVarDef& value) {
-  value.is_bad_def = false;
+  //value.is_bad_def = false;
   return true;
 }
 bool ReflectMemberStart(Writer& writer, IndexedVarDef& value) {
-  if (value.is_bad_def)
+  if (!value.HasInterestingState())
-    return false;
+    std::cerr << "bad";
+  assert(value.HasInterestingState());
+
+  //if (value.is_bad_def)
+  //  return false;
   DefaultReflectMemberStart(writer);
   return true;
 }
@@ -204,9 +223,11 @@ std::string Serialize(IndexedFile& file) {
   return output.GetString();
 }
 
-IndexedFile Deserialize(std::string path, std::string serialized) {
+optional<IndexedFile> Deserialize(std::string path, std::string serialized) {
   rapidjson::Document reader;
   reader.Parse(serialized.c_str());
+  if (reader.HasParseError())
+    return nullopt;
 
   IndexedFile file(path);
   Reflect(reader, file);

src/serializer.h

@@ -204,4 +204,4 @@ void ReflectMember(Reader& visitor, const char* name, T& value) {
 }
 
 std::string Serialize(IndexedFile& file);
-IndexedFile Deserialize(std::string path, std::string serialized);
+optional<IndexedFile> Deserialize(std::string path, std::string serialized);

src/test.cc

@@ -22,7 +22,7 @@ std::string ToString(const rapidjson::Document& document) {
   return buffer.GetString();
 }
 
-std::vector<std::string> split_string(const std::string& str, const std::string& delimiter) {
+std::vector<std::string> SplitString(const std::string& str, const std::string& delimiter) {
   // http://stackoverflow.com/a/13172514
   std::vector<std::string> strings;
 
@@ -40,22 +40,30 @@ std::vector<std::string> split_string(const std::string& str, const std::string&
 }
 
 
-void DiffDocuments(rapidjson::Document& expected, rapidjson::Document& actual) {
+void DiffDocuments(std::string path, rapidjson::Document& expected, rapidjson::Document& actual) {
-  std::vector<std::string> actual_output;
+  std::string joined_actual_output = ToString(actual);
-  {
+  std::vector<std::string> actual_output = SplitString(joined_actual_output, "\n");
-    std::string buffer = ToString(actual);
+  std::string joined_expected_output = ToString(expected);
-    actual_output = split_string(buffer, "\n");
+  std::vector<std::string> expected_output = SplitString(joined_expected_output, "\n");
-  }
 
-  std::vector<std::string> expected_output;
+
-  {
+  std::cout << "[FAILED] " << path << std::endl;
-    std::string buffer = ToString(expected);
+  std::cout << "Expected output for " << path << ":" << std::endl;
-    expected_output = split_string(buffer, "\n");
+  std::cout << joined_expected_output << std::endl;
-  }
+  std::cout << "Actual output for " << path << ":" << std::endl;
+  std::cout << joined_actual_output << std::endl;
+  std::cout << std::endl;
+
+  int max_diff = 5;
 
   int len = std::min(actual_output.size(), expected_output.size());
   for (int i = 0; i < len; ++i) {
     if (actual_output[i] != expected_output[i]) {
+      if (--max_diff < 0) {
+        std::cout << "(... more lines may differ ...)" << std::endl;
+        break;
+      }
+
       std::cout << "Line " << i << " differs:" << std::endl;
       std::cout << "  expected: " << expected_output[i] << std::endl;
       std::cout << "  actual:   " << actual_output[i] << std::endl;
@@ -78,7 +86,7 @@ void DiffDocuments(rapidjson::Document& expected, rapidjson::Document& actual) {
 void VerifySerializeToFrom(IndexedFile& file) {
   return; // TODO: reenable
   std::string expected = file.ToString();
-  std::string actual = Deserialize("foo.cc", Serialize(file)).ToString();
+  std::string actual = Deserialize("foo.cc", Serialize(file)).value().ToString();
   if (expected != actual) {
     std::cerr << "Serialization failure" << std::endl;;
     assert(false);
@@ -99,9 +107,11 @@ void RunTests() {
   for (std::string path : GetFilesInFolder("tests", true /*recursive*/, true /*add_folder_to_path*/)) {
     //if (path != "tests/templates/specialized_func_definition.cc") continue;
     //if (path != "tests/templates/namespace_template_class_template_func_usage_folded_into_one.cc") continue;
-    //if (path != "tests/foo2.cc") continue;
+    //if (path != "tests/multi_file/header.h") continue;
-    //if (path != "tests/namespaces/namespace_reference.cc") continue;
+    //if (path != "tests/multi_file/impl.cc") continue;
+    //if (path != "tests/inheritance/class_inherit_templated_parent.cc") continue;
     //if (path != "tests/templates/implicit_variable_instantiation.cc") continue;
+    //if (path != "tests/_empty_test.cc") continue;
 
     //if (path != "tests/templates/template_class_type_usage_folded_into_one.cc") continue;
     //path = "C:/Users/jacob/Desktop/superindex/indexer/" + path;
@@ -114,7 +124,14 @@ void RunTests() {
 
     // Run test.
     std::cout << "[START] " << path << std::endl;
-    IndexedFile db = Parse(path, {}, false /*dump_ast*/);
+    IndexedFile db = Parse(path, {
+        "-xc++",
+        "-std=c++11",
+        "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/",
+        "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/doctest/",
+        "-IC:/Users/jacob/Desktop/superindex/indexer/third_party/rapidjson/include",
+        "-IC:/Users/jacob/Desktop/superindex/indexer/src"
+      }, false /*dump_ast*/);
     VerifySerializeToFrom(db);
     std::string actual_output = db.ToString();
 
@@ -125,14 +142,7 @@ void RunTests() {
       std::cout << "[PASSED] " << path << std::endl;
     }
     else {
-      std::cout << "[FAILED] " << path << std::endl;
+      DiffDocuments(path, expected, actual);
-      std::cout << "Expected output for " << path << ":" << std::endl;
-      std::cout << expected_output;
-      std::cout << "Actual output for " << path << ":" << std::endl;
-      std::cout << actual_output;
-      std::cout << std::endl;
-      std::cout << std::endl;
-      DiffDocuments(expected, actual);
       break;
     }
   }

tests/constructors/constructor.cc

@@ -18,6 +18,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:7",
       "funcs": [0],
+      "instantiations": [0, 1],
       "uses": ["*1:1:7", "1:3:3", "*1:7:3", "*1:8:3", "*1:8:17"]
     }],
   "funcs": [{

tests/constructors/destructor.cc

@@ -23,6 +23,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:7",
       "funcs": [0, 1],
+      "instantiations": [0],
       "uses": ["*1:1:7", "1:3:3", "1:4:3", "*1:8:3"]
     }],
   "funcs": [{

tests/enums/enum_usage.cc

@@ -15,6 +15,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:12",
       "vars": [0, 1],
+      "instantiations": [2],
       "uses": ["*1:1:12", "*1:6:1", "1:6:9"]
     }],
   "vars": [{

tests/foobar.cc

@@ -31,6 +31,7 @@ OUTPUT:
       "short_name": "Foo",
       "qualified_name": "Foo",
       "definition": "1:5:8",
+      "instantiations": [1],
       "uses": ["*1:5:8", "*1:9:1", "*1:10:1"]
     }, {
       "id": 3,
@@ -38,6 +39,7 @@ OUTPUT:
       "short_name": "Inner",
       "qualified_name": "Foo::Inner",
       "definition": "1:6:10",
+      "instantiations": [0],
       "uses": ["*1:6:10", "*1:9:9"]
     }],
   "vars": [{

tests/inheritance/class_inherit_templated_parent.cc

@@ -49,6 +49,10 @@ OUTPUT:
       "parents": [1],
       "derived": [5],
       "uses": ["*1:11:7", "*1:13:56"]
+    }, {
+      "id": 4,
+      "usr": "c:class_inherit_templated_parent.cc@154",
+      "uses": ["*1:11:24"]
     }, {
       "id": 5,
       "usr": "c:@S@Derived",

tests/multi_file/header.h

@@ -0,0 +1,121 @@
+#pragma once
+
+#include "../../third_party/doctest/doctest/doctest.h"
+#include "../../third_party/macro_map.h"
+#include "../../third_party/optional.h"
+#include <rapidjson/rapidjson.h>
+#include <rapidjson/document.h>
+#include <rapidjson/prettywriter.h>
+
+struct Base {};
+
+struct SameFileDerived : Base {};
+
+using Foo0 = SameFileDerived;
+
+template <typename T>
+void Foo1() {}
+
+template <typename T>
+struct Foo2 {};
+
+enum Foo3 { A, B, C };
+
+int Foo4;
+static int Foo5;
+
+/*
+OUTPUT:
+{
+  "types": [{
+      "id": 0,
+      "usr": "c:@S@Base",
+      "short_name": "Base",
+      "qualified_name": "Base",
+      "definition": "1:10:8",
+      "derived": [1],
+      "uses": ["*1:10:8", "*1:12:26"]
+    }, {
+      "id": 1,
+      "usr": "c:@S@SameFileDerived",
+      "short_name": "SameFileDerived",
+      "qualified_name": "SameFileDerived",
+      "definition": "1:12:8",
+      "parents": [0],
+      "uses": ["*1:12:8", "*1:14:14"]
+    }, {
+      "id": 2,
+      "usr": "c:@Foo0",
+      "short_name": "Foo0",
+      "qualified_name": "Foo0",
+      "definition": "1:14:7",
+      "alias_of": 1,
+      "uses": ["*1:14:7"]
+    }, {
+      "id": 3,
+      "usr": "c:@ST>1#T@Foo2",
+      "short_name": "Foo2",
+      "qualified_name": "Foo2",
+      "definition": "1:20:8",
+      "uses": ["*1:20:8"]
+    }, {
+      "id": 4,
+      "usr": "c:@E@Foo3",
+      "short_name": "Foo3",
+      "qualified_name": "Foo3",
+      "definition": "1:22:6",
+      "vars": [0, 1, 2],
+      "uses": ["*1:22:6"]
+    }],
+  "funcs": [{
+      "id": 0,
+      "usr": "c:@FT@>1#TFoo1#v#",
+      "short_name": "Foo1",
+      "qualified_name": "Foo1",
+      "definition": "1:17:6",
+      "uses": ["1:17:6"]
+    }],
+  "vars": [{
+      "id": 0,
+      "usr": "c:@E@Foo3@A",
+      "short_name": "A",
+      "qualified_name": "Foo3::A",
+      "definition": "1:22:13",
+      "variable_type": 4,
+      "declaring_type": 4,
+      "uses": ["1:22:13"]
+    }, {
+      "id": 1,
+      "usr": "c:@E@Foo3@B",
+      "short_name": "B",
+      "qualified_name": "Foo3::B",
+      "definition": "1:22:16",
+      "variable_type": 4,
+      "declaring_type": 4,
+      "uses": ["1:22:16"]
+    }, {
+      "id": 2,
+      "usr": "c:@E@Foo3@C",
+      "short_name": "C",
+      "qualified_name": "Foo3::C",
+      "definition": "1:22:19",
+      "variable_type": 4,
+      "declaring_type": 4,
+      "uses": ["1:22:19"]
+    }, {
+      "id": 3,
+      "usr": "c:@Foo4",
+      "short_name": "Foo4",
+      "qualified_name": "Foo4",
+      "definition": "1:24:5",
+      "uses": ["1:24:5"]
+    }, {
+      "id": 4,
+      "usr": "c:header.h@Foo5",
+      "short_name": "Foo5",
+      "qualified_name": "Foo5",
+      "definition": "1:25:12",
+      "uses": ["1:25:12"]
+    }]
+}
+*/

tests/outline/outline.cc

@@ -17,6 +17,11 @@ OUTPUT:
       "definition": "1:3:8",
       "vars": [0, 1, 2],
       "uses": ["*1:3:8"]
+    }, {
+      "id": 1,
+      "usr": "c:@N@std@ST>2#T#T@vector",
+      "instantiations": [2],
+      "uses": ["*1:6:8"]
     }],
   "vars": [{
       "id": 0,

tests/outline/outline2.cc

@@ -22,6 +22,16 @@ OUTPUT:
       "definition": "1:6:8",
       "vars": [0, 1, 2],
       "uses": ["*1:6:8", "*1:12:13"]
+    }, {
+      "id": 1,
+      "usr": "c:@N@std@T@string",
+      "instantiations": [0, 1],
+      "uses": ["*1:7:8", "*1:8:8", "*1:9:20"]
+    }, {
+      "id": 2,
+      "usr": "c:@N@std@ST>2#T#T@vector",
+      "instantiations": [2],
+      "uses": ["*1:9:8", "*1:12:6"]
     }],
   "funcs": [{
       "id": 0,

tests/templates/implicit_variable_instantiation.cc

@@ -23,6 +23,7 @@ OUTPUT:
       "short_name": "VarType",
       "qualified_name": "ns::VarType",
       "definition": "1:2:8",
+      "instantiations": [0],
       "uses": ["*1:2:8", "*1:6:22", "*1:6:44", "*1:10:18"]
     }, {
       "id": 1,

tests/templates/namespace_template_type_usage_folded_into_one.cc

@@ -15,6 +15,7 @@ OUTPUT:
       "short_name": "Foo",
       "qualified_name": "ns::Foo",
       "definition": "1:3:9",
+      "instantiations": [0, 1],
       "uses": ["*1:3:9", "*1:5:3", "*1:6:3"]
     }],
   "vars": [{

tests/templates/template_class_type_usage_folded_into_one.cc

@@ -57,6 +57,7 @@ OUTPUT:
       "short_name": "Inner",
       "qualified_name": "Foo::Inner",
       "definition": "1:6:10",
+      "instantiations": [0, 1],
       "uses": ["*1:6:10", "*1:9:9", "*1:10:9"]
     }],
   "vars": [{

tests/templates/template_type_usage_folded_into_one.cc

@@ -13,6 +13,7 @@ OUTPUT:
       "short_name": "Foo",
       "qualified_name": "Foo",
       "definition": "1:2:7",
+      "instantiations": [0, 1],
       "uses": ["*1:2:7", "*1:4:1", "*1:5:1"]
     }],
   "vars": [{

tests/templates/template_var_usage_folded_into_one.cc

@@ -44,6 +44,10 @@ OUTPUT:
       "qualified_name": "B",
       "definition": "1:2:6",
       "uses": ["*1:2:6", "1:8:13"]
+    }, {
+      "id": 2,
+      "usr": "c:template_var_usage_folded_into_one.cc@35",
+      "uses": ["*1:5:1"]
     }],
   "vars": [{
       "id": 0,

tests/unions/union_usage.cc

@@ -10,6 +10,8 @@ void act(Foo*) {
 }
 
 /*
+// TODO: instantiations on Foo should include parameter?
+
 OUTPUT:
 {
   "types": [{
@@ -19,6 +21,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:7",
       "vars": [0, 1],
+      "instantiations": [2],
       "uses": ["*1:1:7", "*1:6:1", "*1:8:10"]
     }],
   "funcs": [{

tests/usage/func_usage_call_method.cc

@@ -17,6 +17,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:8",
       "funcs": [0],
+      "instantiations": [0],
       "uses": ["*1:1:8", "*1:6:3"]
     }],
   "funcs": [{

tests/usage/func_usage_forward_decl_method.cc

@@ -16,6 +16,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:8",
       "funcs": [0],
+      "instantiations": [0],
       "uses": ["*1:1:8", "*1:6:3"]
     }],
   "funcs": [{

tests/usage/type_usage_as_template_parameter.cc

@@ -16,6 +16,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@ST>1#T@unique_ptr",
+      "instantiations": [0, 1, 2],
       "uses": ["1:2:7", "*1:6:8", "*1:7:8", "*1:9:1", "*1:10:3"]
     }, {
       "id": 1,

tests/usage/type_usage_as_template_parameter_complex.cc

@@ -84,6 +84,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@ST>2#T#T@unique_ptr",
+      "instantiations": [0, 1],
       "uses": ["1:2:7", "*1:15:8", "*1:15:19", "*1:33:1", "*1:33:12", "*1:33:52", "*1:54:3", "*1:54:14", "*1:65:3", "*1:79:1"]
     }, {
       "id": 1,

tests/usage/type_usage_as_template_parameter_simple.cc

@@ -14,6 +14,7 @@ OUTPUT:
       "short_name": "unique_ptr",
       "qualified_name": "unique_ptr",
       "definition": "1:2:7",
+      "instantiations": [0],
       "uses": ["*1:2:7", "*1:6:8"]
     }, {
       "id": 1,

tests/usage/type_usage_declare_extern.cc

@@ -10,6 +10,7 @@ OUTPUT:
       "short_name": "T",
       "qualified_name": "T",
       "definition": "1:1:8",
+      "instantiations": [0],
       "uses": ["*1:1:8", "*1:3:8"]
     }],
   "vars": [{

tests/usage/type_usage_declare_field.cc

@@ -12,6 +12,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@S@ForwardType",
+      "instantiations": [0],
       "uses": ["1:1:8", "*1:5:3"]
     }, {
       "id": 1,
@@ -19,6 +20,7 @@ OUTPUT:
       "short_name": "ImplementedType",
       "qualified_name": "ImplementedType",
       "definition": "1:2:8",
+      "instantiations": [1],
       "uses": ["*1:2:8", "*1:6:3"]
     }, {
       "id": 2,

tests/usage/type_usage_declare_local.cc

@@ -12,6 +12,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@S@ForwardType",
+      "instantiations": [0],
       "uses": ["1:1:8", "*1:5:3"]
     }, {
       "id": 1,
@@ -19,6 +20,7 @@ OUTPUT:
       "short_name": "ImplementedType",
       "qualified_name": "ImplementedType",
       "definition": "1:2:8",
+      "instantiations": [1],
       "uses": ["*1:2:8", "*1:6:3"]
     }],
   "funcs": [{

tests/usage/type_usage_declare_param.cc

@@ -9,6 +9,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@S@ForwardType",
+      "instantiations": [0],
       "uses": ["1:1:8", "*1:4:10"]
     }, {
       "id": 1,
@@ -16,6 +17,7 @@ OUTPUT:
       "short_name": "ImplementedType",
       "qualified_name": "ImplementedType",
       "definition": "1:2:8",
+      "instantiations": [1],
       "uses": ["*1:2:8", "*1:4:26"]
     }],
   "funcs": [{

tests/usage/type_usage_declare_param_prototype.cc

@@ -14,6 +14,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@S@Foo",
+      "instantiations": [0],
       "uses": ["1:1:8", "1:3:10", "1:3:18", "*1:4:10", "*1:4:18"]
     }],
   "funcs": [{

tests/usage/type_usage_declare_qualifiers.cc

@@ -15,6 +15,7 @@ OUTPUT:
       "short_name": "Type",
       "qualified_name": "Type",
       "definition": "1:1:8",
+      "instantiations": [0, 1, 2, 3, 4, 5],
       "uses": ["*1:1:8", "*1:3:10", "*1:3:26", "*1:4:3", "*1:5:3", "*1:6:9", "*1:7:9"]
     }],
   "funcs": [{

tests/usage/type_usage_declare_static.cc

@@ -6,6 +6,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@S@Type",
+      "instantiations": [0],
       "uses": ["1:1:8", "*1:2:8"]
     }],
   "vars": [{

tests/usage/type_usage_various.cc

@@ -19,6 +19,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:7",
       "funcs": [0],
+      "instantiations": [0, 1],
       "uses": ["*1:1:7", "*1:2:3", "*1:5:1", "1:5:6", "*1:6:3", "*1:10:8"]
     }],
   "funcs": [{

tests/usage/var_usage_class_member.cc

@@ -27,6 +27,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:7",
       "vars": [0, 1],
+      "instantiations": [2],
       "uses": ["*1:1:7", "*1:11:3"]
     }],
   "funcs": [{

tests/usage/var_usage_cstyle_cast.cc

@@ -16,6 +16,7 @@ OUTPUT:
       "short_name": "VarType",
       "qualified_name": "VarType",
       "definition": "1:1:6",
+      "instantiations": [0],
       "uses": ["*1:1:6", "*1:4:20", "*1:4:42", "*1:7:7"]
     }, {
       "id": 1,

tests/vars/class_member.cc

@@ -11,6 +11,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:7",
       "vars": [0],
+      "instantiations": [0],
       "uses": ["*1:1:7", "*1:2:3"]
     }],
   "vars": [{

tests/vars/class_static_member.cc

@@ -13,6 +13,7 @@ OUTPUT:
       "qualified_name": "Foo",
       "definition": "1:1:7",
       "vars": [0],
+      "instantiations": [0],
       "uses": ["*1:1:7", "*1:2:10", "*1:4:1", "1:4:6"]
     }],
   "vars": [{

tests/vars/function_local.cc

@@ -10,6 +10,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@S@Foo",
+      "instantiations": [0],
       "uses": ["1:1:8", "*1:4:3"]
     }],
   "funcs": [{

tests/vars/function_param.cc

@@ -8,6 +8,7 @@ OUTPUT:
   "types": [{
       "id": 0,
       "usr": "c:@S@Foo",
+      "instantiations": [0, 1],
       "uses": ["1:1:8", "*1:3:10", "*1:3:19"]
     }],
   "funcs": [{