wip template usage

libclangmm/SourceLocation.cc

@@ -32,12 +32,12 @@ SourceLocation::SourceLocation(const CXIdxLoc& cx_location)
   : SourceLocation(clang_indexLoc_getCXSourceLocation(cx_location)) {
 }
 
-bool SourceLocation::operator==(const SourceLocation& o) {
+bool operator==(const SourceLocation& a, const SourceLocation& b) {
-  return path == o.path && line == o.line && column == o.column;
+  return a.path == b.path && a.line == b.line && a.column == b.column;
 }
 
-bool SourceLocation::operator!=(const SourceLocation& o) {
+bool operator!=(const SourceLocation& a, const SourceLocation& b) {
-  return !(*this == o);
+  return !(a == b);
 }
 
 std::string SourceLocation::ToString() const {

libclangmm/SourceLocation.h

@@ -25,9 +25,6 @@ public:
   SourceLocation(const CXSourceLocation& cx_location);
   SourceLocation(const CXIdxLoc& cx_location);
 
-  bool operator==(const SourceLocation& o);
-  bool operator!=(const SourceLocation& o);
-
   std::string path;
   unsigned line = 0;
   unsigned column = 0;
@@ -36,6 +33,9 @@ public:
   std::string ToString() const;
 };
 
+bool operator==(const SourceLocation& a, const SourceLocation& b);
+bool operator!=(const SourceLocation& a, const SourceLocation& b);
+
 }  // namespace clang
 
 #endif  // SOURCELOCATION_H_

main.cpp

@@ -1193,6 +1193,25 @@ std::optional<clang::Cursor> FindChildOfKind(clang::Cursor cursor, CXCursorKind
 
 
 
+clang::VisiterResult FindTypeVisitor(clang::Cursor cursor, clang::Cursor parent, std::optional<clang::Cursor>* result) {
+  switch (cursor.get_kind()) {
+  case CXCursor_TypeRef:
+  case CXCursor_TemplateRef:
+    *result = cursor;
+    return clang::VisiterResult::Break;
+  }
+
+  return clang::VisiterResult::Recurse;
+}
+
+std::optional<clang::Cursor> FindType(clang::Cursor cursor) {
+  std::optional<clang::Cursor> result;
+  cursor.VisitChildren(&FindTypeVisitor, &result);
+  return result;
+}
+
+
+
 
 
 
@@ -1254,6 +1273,14 @@ std::string GetNamespacePrefx(const CXIdxDeclInfo* decl) {
 }
 */
 
+bool HasUsage(const std::vector<clang::SourceLocation>& usages, const clang::SourceLocation& usage) {
+  for (int i = usages.size() - 1; i >= 0; --i) {
+    if (usages[i] == usage)
+      return true;
+  }
+
+  return false;
+}
 
 // TODO: Let's switch over to the indexer api. It can index
 //        the int x = get_value() bit...
@@ -1266,14 +1293,6 @@ std::string GetNamespacePrefx(const CXIdxDeclInfo* decl) {
 //        * it doesn't seem like we get any template specialization logic
 //        * we get two decls to the same template... resolved by checking parent? maybe this will break. not sure.
 
-// Insert a reference to |type_id| using the location of the first TypeRef under |cursor|.
-void InsertInterestingTypeReference(ParsingDatabase* db, TypeId type_id, clang::Cursor cursor) {
-  std::optional<clang::Cursor> child = FindChildOfKind(cursor, CXCursor_TypeRef);
-  assert(child.has_value()); // If this assert ever fails just use |cursor| loc.
-
-  TypeDef* def = db->Resolve(type_id);
-  def->interesting_uses.push_back(child.value().get_source_location());
-}
 
 bool IsTypeDefinition(const CXIdxContainerInfo* container) {
   if (!container)
@@ -1288,6 +1307,42 @@ bool IsTypeDefinition(const CXIdxContainerInfo* container) {
   }
 }
 
+std::optional<TypeId> ResolveToType(ParsingDatabase* db, clang::Type type) {
+  clang::Type var_type = type.strip_qualifiers();
+  std::string usr = var_type.get_usr();
+
+  if (usr == "")
+    return std::nullopt;
+
+  // TODO: Add a check and don't resolve template specializations that exist in source code.
+  // Resolve template specialization so that we always point to the non-specialized type.
+  clang::Cursor decl = clang_getTypeDeclaration(var_type.cx_type);
+  clang::Cursor unresolved_decl = clang_getSpecializedCursorTemplate(decl.cx_cursor);
+  std::string template_usr = clang::Cursor(unresolved_decl).get_usr();
+  if (template_usr != "")
+    usr = template_usr;
+
+  return db->ToTypeId(usr);
+}
+
+clang::SourceLocation FindLocationOfTypeSpecifier(clang::Cursor cursor) {
+  std::cout << "FindLocationOfTypeSpecifier " << std::endl;
+  Dump(cursor);
+
+  std::optional<clang::Cursor> child = FindType(cursor);
+  assert(child.has_value()); // If this assert ever fails just use |cursor| loc or figure out what type ref we are missing.
+  return child.value().get_source_location();
+}
+
+void AddInterestingUsageToType(ParsingDatabase* db, TypeId type_id, clang::SourceLocation location) {
+  TypeDef* type_def = db->Resolve(type_id);
+  type_def->interesting_uses.push_back(location);
+}
+
+
+
+
+
 void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
   IndexParam* param = static_cast<IndexParam*>(client_data);
   ParsingDatabase* db = param->db;
@@ -1304,6 +1359,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
   case CXIdxEntity_Variable:
   case CXIdxEntity_CXXStaticVariable:
   {
+    clang::Cursor decl_cursor = decl->cursor;
     VarId var_id = db->ToVarId(decl->entityInfo->USR);
     VarDef* var_def = db->Resolve(var_id);
 
@@ -1321,19 +1377,19 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
 
     var_def->all_uses.push_back(decl->loc);
 
-    // Declaring variable type information.
-    std::string var_type_usr = clang::Cursor(decl->cursor).get_type().strip_qualifiers().get_usr();
-    if (var_type_usr != "") {
-      TypeId var_type_id = db->ToTypeId(var_type_usr);
-      var_def->variable_type = var_type_id;
 
+    // Declaring variable type information.
+    std::optional<TypeId> var_type_id = ResolveToType(db, decl_cursor.get_type());
+    if (var_type_id) {
+      var_def->variable_type = var_type_id.value();
       // Insert an interesting type usage for variable declarations. Parameters
       // are handled when a function is declared because clang doesn't provide
       // parameter declarations for unnamed parameters.
-      if (decl->cursor.kind != CXCursor_ParmDecl)
+      if (decl_cursor.get_kind() != CXCursor_ParmDecl)
-        InsertInterestingTypeReference(db, var_type_id, decl->cursor);
+        AddInterestingUsageToType(db, var_type_id.value(), FindLocationOfTypeSpecifier(decl_cursor));
     }
 
+
     if (decl->isDefinition && IsTypeDefinition(decl->semanticContainer)) {
       TypeId declaring_type_id = db->ToTypeId(decl->semanticContainer->cursor);
       TypeDef* declaring_type_def = db->Resolve(declaring_type_id);
@@ -1355,6 +1411,7 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
   case CXIdxEntity_CXXInstanceMethod:
   case CXIdxEntity_CXXStaticMethod:
   {
+    clang::Cursor decl_cursor = decl->cursor;
     FuncId func_id = db->ToFuncId(decl->entityInfo->USR);
     FuncDef* func_def = db->Resolve(func_id);
 
@@ -1383,26 +1440,27 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
       declaring_type_def->funcs.push_back(func_id);
     }
 
-    std::string return_type_usr = clang::Cursor(decl->cursor).get_type().get_return_type().strip_qualifiers().get_usr();
+    std::optional<TypeId> ret_type_id = ResolveToType(db, decl_cursor.get_type().get_return_type());
-    if (return_type_usr != "")
+    if (ret_type_id)
-      InsertInterestingTypeReference(db, db->ToTypeId(return_type_usr), decl->cursor);
+      AddInterestingUsageToType(db, ret_type_id.value(), FindLocationOfTypeSpecifier(decl_cursor));
-
-
 
     if (decl->isDefinition || is_pure_virtual) {
       // Mark type usage for parameters as interesting. We handle this here
       // instead of inside var declaration because clang will not emit a var
       // declaration for an unnamed parameter, but we still want to mark the
       // usage as interesting.
-      // TODO: Do a similar thing for function decl parameter usages.
+      // TODO: Do a similar thing for function decl parameter usages. Mark
+      //       prototype params as interesting type usages but also relate mark
+      //       them as as usages on the primary variable - requires USR to be
+      //       the same. We can work around it by declaring which variables a
+      //       parameter has declared and update the USR in the definition.
       clang::Cursor cursor = decl->cursor;
       for (clang::Cursor arg : cursor.get_arguments()) {
         switch (arg.get_kind()) {
         case CXCursor_ParmDecl:
-          std::string param_type_usr = arg.get_type().strip_qualifiers().get_usr();
+          std::optional<TypeId> arg_type_id = ResolveToType(db, arg.get_type());
-          if (param_type_usr != "") {
+          if (arg_type_id)
-            InsertInterestingTypeReference(db, db->ToTypeId(param_type_usr), arg);
+            AddInterestingUsageToType(db, arg_type_id.value(), FindLocationOfTypeSpecifier(arg));
-          }
           break;
         }
       }
@@ -1615,17 +1673,8 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re
     //param->last_type_usage_location = loc;
 
     // TODO: initializer list can many type refs...
-    bool do_break = false;
+    if (!HasUsage(referenced_def->all_uses, loc))
-    for (int i = referenced_def->all_uses.size() - 1; i >= 0; --i) {
+      referenced_def->all_uses.push_back(loc);
-      if (referenced_def->all_uses[i] == loc) {
-        do_break = true;
-        break;
-      }
-    }
-    if (do_break)
-      break;
-
-    referenced_def->all_uses.push_back(loc);
 
     /*
     //
@@ -1815,8 +1864,10 @@ int main(int argc, char** argv) {
   for (std::string path : GetFilesInFolder("tests")) {
     //if (path != "tests/declaration_vs_definition/method.cc") continue;
     //if (path == "tests/usage/type_usage_declare_extern.cc") continue;
-    //if (path != "tests/constructors/destructor.cc") continue;
+    //if (path == "tests/constructors/constructor.cc") continue;
-    //if (path != "tests/usage/usage_inside_of_call.cc") continue;
+    //if (path == "tests/constructors/destructor.cc") continue;
+    //if (path == "tests/usage/func_usage_call_method.cc") continue;
+    if (path != "tests/usage/type_usage_as_template_parameter_simple.cc") continue;
     //if (path != "tests/usage/type_usage_typedef_and_using.cc") continue;
     //if (path != "tests/usage/type_usage_declare_local.cc") continue;
     //if (path != "tests/usage/func_usage_addr_method.cc") continue;

query_db.cc

@@ -0,0 +1,78 @@
+#include <cstdint>
+#include <optional>
+#include <unordered_map>
+
+using FileId = uint64_t;
+
+struct FileDatabase {
+  std::unordered_map<std::string, FileId> filename_to_file_id;
+  std::unordered_map<FileId, std::string> file_id_to_filename;
+};
+
+struct SymbolIdx {
+  std::optional<uint64_t> type_idx;
+  std::optional<uint64_t> func_idx;
+  std::optional<uint64_t> var_idx;
+};
+
+struct File {
+  // Symbols declared in the file.
+  std::vector<SymbolIdx> declared_symbols;
+  // Symbols which have definitions in the file.
+  std::vector<SymbolIdx> defined_symbols;
+};
+
+struct TypeDef {};
+struct FuncDef {};
+struct VarDef {};
+
+struct QueryableEntry {
+  const char* const str;
+};
+
+// The query database is heavily optimized for fast queries. It is stored
+// in-memory.
+struct QueryDatabase {
+  // Indicies between lookup vectors are related to symbols, ie, index 5 in
+  // |qualified_names| matches index 5 in |symbols|.
+  std::vector<QueryableEntry> qualified_names;
+  std::vector<SymbolIdx> symbols;
+
+  // Raw data storage.
+  std::vector<TypeDef> types;
+  std::vector<FuncDef> funcs;
+  std::vector<VarDef> vars;
+
+  // |files| is indexed by FileId. Retrieve a FileId from a path using
+  // |file_locator|.
+  FileDatabase file_locator;
+  std::vector<File> files;
+};
+
+
+
+
+// Task running in a separate process, parsing a file into something we can
+// import.
+struct ParseTask {};
+// Completed parse task that wants to import content into the global database.
+// Runs in main process, primary thread. Stops all other threads.
+struct IndexImportTask {};
+// Completed parse task that wants to update content previously imported into
+// the global database. Runs in main process, primary thread. Stops all other
+// threads.
+//
+// Note that this task just contains a set of operations to apply to the global
+// database. The operations come from a diff based on the previously indexed
+// state in comparison to the newly indexed state.
+//
+// TODO: We may be able to run multiple freshen and import tasks in parallel if
+//       we restrict what ranges of the db they may change.
+struct IndexFreshenTask {};
+// Task running a query against the global database. Run in main process,
+// separate thread.
+struct QueryTask {};
+
+struct TaskManager {
+
+};

tests/usage/type_usage_as_template_parameter.cc

@@ -1,102 +1,63 @@
-// TODO: Reenable
-#if false
 template<typename T>
-class unique_ptr {
+class unique_ptr;
-public:
-  T value;
-};
 
-struct Foo {
+struct S {};
-  int x;
-};
 
-void foo() {
+static unique_ptr<bool> f0;
-  unique_ptr<Foo> f0;
+static unique_ptr<S> f1;
-  unique_ptr<Foo> f1;
 
-  f0.value.x += 5;
+unique_ptr<S>* return_type() {
+  unique_ptr<S>* local;
+  return nullptr;
 }
-#endif
 /*
-// TODO: Figure out how we want to handle template specializations. For example,
+OUTPUT:
-//       when we use unique_ptr<int>.value, do we want to generalize that to a
-//       usage on unique_ptr<T>.value, or just on unique_ptr<int>.value?
-
-OUT2PUT:
 {
   "types": [{
       "id": 0,
       "usr": "c:@ST>1#T@unique_ptr",
-      "short_name": "unique_ptr",
+      "all_uses": ["tests/usage/type_usage_as_template_parameter.cc:2:7", "tests/usage/type_usage_as_template_parameter.cc:6:8", "tests/usage/type_usage_as_template_parameter.cc:7:8", "tests/usage/type_usage_as_template_parameter.cc:9:1", "tests/usage/type_usage_as_template_parameter.cc:10:3"],
-      "qualified_name": "unique_ptr",
+      "interesting_uses": ["tests/usage/type_usage_as_template_parameter.cc:6:8", "tests/usage/type_usage_as_template_parameter.cc:7:8", "tests/usage/type_usage_as_template_parameter.cc:9:1", "tests/usage/type_usage_as_template_parameter.cc:10:3"]
-      "definition": "tests/usage/type_usage_as_template_parameter.cc:2:7",
-      "vars": [0]
     }, {
       "id": 1,
-      "usr": "c:@S@Foo",
+      "usr": "c:@S@S",
-      "short_name": "Foo",
+      "short_name": "S",
-      "qualified_name": "Foo",
+      "qualified_name": "S",
-      "definition": "tests/usage/type_usage_as_template_parameter.cc:7:8",
+      "definition": "tests/usage/type_usage_as_template_parameter.cc:4:8",
-      "vars": [1],
+      "all_uses": ["tests/usage/type_usage_as_template_parameter.cc:4:8", "tests/usage/type_usage_as_template_parameter.cc:7:19", "tests/usage/type_usage_as_template_parameter.cc:9:12", "tests/usage/type_usage_as_template_parameter.cc:10:14"]
-      "uses": ["tests/usage/type_usage_as_template_parameter.cc:12:14", "tests/usage/type_usage_as_template_parameter.cc:13:14"]
-    }, {
-      "id": 2,
-      "usr": "c:@S@unique_ptr>#$@S@Foo",
-      "uses": ["tests/usage/type_usage_as_template_parameter.cc:12:19", "tests/usage/type_usage_as_template_parameter.cc:13:19"]
     }],
   "functions": [{
       "id": 0,
-      "usr": "c:@F@foo#",
+      "usr": "c:@F@return_type#",
-      "short_name": "foo",
+      "short_name": "return_type",
-      "qualified_name": "foo",
+      "qualified_name": "return_type",
-      "definition": "tests/usage/type_usage_as_template_parameter.cc:11:6"
+      "definition": "tests/usage/type_usage_as_template_parameter.cc:9:16",
+      "all_uses": ["tests/usage/type_usage_as_template_parameter.cc:9:16"]
     }],
   "variables": [{
       "id": 0,
-      "usr": "c:@ST>1#T@unique_ptr@FI@value",
+      "usr": "c:type_usage_as_template_parameter.cc@f0",
-      "short_name": "value",
-      "qualified_name": "unique_ptr::value",
-      "declaration": "tests/usage/type_usage_as_template_parameter.cc:4:5",
-      "initializations": ["tests/usage/type_usage_as_template_parameter.cc:4:5"],
-      "declaring_type": 0
-    }, {
-      "id": 1,
-      "usr": "c:@S@Foo@FI@x",
-      "short_name": "x",
-      "qualified_name": "Foo::x",
-      "declaration": "tests/usage/type_usage_as_template_parameter.cc:8:7",
-      "initializations": ["tests/usage/type_usage_as_template_parameter.cc:8:7"],
-      "declaring_type": 1,
-      "uses": ["tests/usage/type_usage_as_template_parameter.cc:15:12"]
-    }, {
-      "id": 2,
-      "usr": "c:type_usage_as_template_parameter.cc@115@F@foo#@f0",
       "short_name": "f0",
       "qualified_name": "f0",
-      "declaration": "tests/usage/type_usage_as_template_parameter.cc:12:19",
+      "definition": "tests/usage/type_usage_as_template_parameter.cc:6:25",
-      "initializations": ["tests/usage/type_usage_as_template_parameter.cc:12:19"],
+      "variable_type": 0,
-      "variable_type": 2,
+      "all_uses": ["tests/usage/type_usage_as_template_parameter.cc:6:25"]
-      "uses": ["tests/usage/type_usage_as_template_parameter.cc:15:3"]
     }, {
-      "id": 3,
+      "id": 1,
-      "usr": "c:type_usage_as_template_parameter.cc@138@F@foo#@f1",
+      "usr": "c:type_usage_as_template_parameter.cc@f1",
       "short_name": "f1",
       "qualified_name": "f1",
-      "declaration": "tests/usage/type_usage_as_template_parameter.cc:13:19",
+      "definition": "tests/usage/type_usage_as_template_parameter.cc:7:22",
-      "initializations": ["tests/usage/type_usage_as_template_parameter.cc:13:19"],
+      "variable_type": 0,
-      "variable_type": 2
+      "all_uses": ["tests/usage/type_usage_as_template_parameter.cc:7:22"]
     }, {
-      "id": 4,
+      "id": 2,
-      "usr": "c:@S@unique_ptr>#$@S@Foo@FI@value",
+      "usr": "c:type_usage_as_template_parameter.cc@150@F@return_type#@local",
-      "uses": ["tests/usage/type_usage_as_template_parameter.cc:15:6"]
+      "short_name": "local",
+      "qualified_name": "local",
+      "definition": "tests/usage/type_usage_as_template_parameter.cc:10:18",
+      "variable_type": 0,
+      "all_uses": ["tests/usage/type_usage_as_template_parameter.cc:10:18"]
     }]
 }
-
-OUTPUT:
-{
-  "types": [],
-  "functions": [],
-  "variables": []
-}
-
 */

tests/usage/type_usage_as_template_parameter_simple.cc

@@ -0,0 +1,34 @@
+template<typename T>
+class unique_ptr;
+
+struct S;
+
+extern unique_ptr<S> f;
+
+/*
+// TODO: There should be an interesting usage on S as well.
+OUTPUT:
+{
+  "types": [{
+      "id": 0,
+      "usr": "c:@ST>1#T@unique_ptr",
+      "all_uses": ["tests/usage/type_usage_as_template_parameter_simple.cc:2:7", "tests/usage/type_usage_as_template_parameter_simple.cc:6:8"],
+      "interesting_uses": ["tests/usage/type_usage_as_template_parameter_simple.cc:6:8"]
+    }, {
+      "id": 1,
+      "usr": "c:@S@S",
+      "all_uses": ["tests/usage/type_usage_as_template_parameter_simple.cc:4:8", "tests/usage/type_usage_as_template_parameter_simple.cc:6:19"]
+    }],
+  "functions": [],
+  "variables": [{
+      "id": 0,
+      "usr": "c:@f",
+      "short_name": "f",
+      "qualified_name": "f",
+      "declaration": "tests/usage/type_usage_as_template_parameter_simple.cc:6:22",
+      "variable_type": 0,
+      "all_uses": ["tests/usage/type_usage_as_template_parameter_simple.cc:6:22"]
+    }]
+}
+
+*/