add IndexedFile deserialization

indexer.h

@@ -88,8 +88,29 @@ struct Location {
     return FileId(raw_file_id);
   }
 
-  explicit Location(const char* encoded) {
+  explicit Location(const char* encoded) : Location() {
-    // TODO
+    int len = strlen(encoded);
+    assert(len >= 0);
+
+    if (*encoded == '*') {
+      interesting = true;
+      ++encoded;
+    }
+
+    assert(encoded);
+    raw_file_id = atoi(encoded);
+    while (*encoded && *encoded != ':')
+      ++encoded;
+    if (*encoded == ':') ++encoded;
+
+    assert(encoded);
+    line = atoi(encoded);
+    while (*encoded && *encoded != ':')
+      ++encoded;
+    if (*encoded == ':') ++encoded;
+
+    assert(encoded);
+    column = atoi(encoded);
   }
 
   std::string ToString() {

query.h

@@ -163,6 +163,9 @@ struct SymbolIdx {
 //       We need to verify we need multiple processes first. Maybe libclang can run in a single process...
 // TODO: Compute IndexUpdates in main process, off the blocking thread. Use separate process for running
 //       libclang. Solves memory worries.
+// TODO: Instead of passing to/from json, we can probably bass the IndexedFile type almost directly as
+// a raw memory dump - the type has almost zero pointers inside of it. We could do a little bit of fixup
+// so that passing from a separate process to the main db is really fast (no need to go through JSON).
 namespace foo2 {
   using Usr = size_t;
   struct UsrTable {

serializer.cc

@@ -93,7 +93,7 @@ void Serialize(Writer& writer, IndexedFile* file) {
     assert(file->Resolve(it->second)->uses.size() == 0);
   }
 
-#define SERIALIZE(name, value) Serialize(writer, name, def.value)
+#define SERIALIZE(json_name, member_name) Serialize(writer, json_name, def.##member_name)
 
   writer.StartObject();
 
@@ -171,36 +171,135 @@ void Serialize(Writer& writer, IndexedFile* file) {
 #undef WRITE
 }
 
-void Deserialize(std::string& output, rapidjson::GenericValue<rapidjson::UTF8<>>& value) {
+void Deserialize(rapidjson::GenericValue<rapidjson::UTF8<>>& document, const char* name, std::string& output) {
-  output = value.GetString();
+  auto it = document.FindMember(name);
+  if (it != document.MemberEnd())
+    output = document[name].GetString();
 }
 
-void Deserialize(optional<Location>& output, rapidjson::GenericValue<rapidjson::UTF8<>>& value) {
+void Deserialize(rapidjson::GenericValue<rapidjson::UTF8<>>& document, const char* name, optional<Location>& output) {
-  if (!value.IsNull())
+  auto it = document.FindMember(name);
-    output = Location(value.GetString()); // TODO: Location parsing not implemented in Location type.
+  if (it != document.MemberEnd())
+    output = Location(it->value.GetString()); // TODO: Location parsing not implemented in Location type.
+}
+
+void Deserialize(rapidjson::GenericValue<rapidjson::UTF8<>>& document, const char* name, std::vector<Location>& output) {
+  auto it = document.FindMember(name);
+  if (it != document.MemberEnd()) {
+    for (auto& array_value : it->value.GetArray())
+      output.push_back(Location(array_value.GetString()));
+  }
+}
+
+template<typename T>
+void Deserialize(rapidjson::GenericValue<rapidjson::UTF8<>>& document, const char* name, optional<Id<T>>& output) {
+  auto it = document.FindMember(name);
+  if (it != document.MemberEnd())
+    output = Id<T>(it->value.GetUint64());
+}
+
+template<typename T>
+void Deserialize(rapidjson::GenericValue<rapidjson::UTF8<>>& document, const char* name, std::vector<Id<T>>& output) {
+  auto it = document.FindMember(name);
+  if (it != document.MemberEnd()) {
+    for (auto& array_value : it->value.GetArray())
+      output.push_back(Id<T>(array_value.GetUint64()));
+  }
+}
+
+template<typename T>
+void Deserialize(rapidjson::GenericValue<rapidjson::UTF8<>>& document, const char* name, std::vector<Ref<T>>& output) {
+  auto it = document.FindMember(name);
+  if (it != document.MemberEnd()) {
+    for (auto& array_value : it->value.GetArray()) {
+      const char* str_value = array_value.GetString();
+      uint64_t id = atoi(str_value);
+      const char* loc_string = strchr(str_value, '@') + 1;
+      output.push_back(Ref<T>(Id<T>(id), Location(loc_string)));
+    }
+  }
 }
 
 void Deserialize(Reader& reader, IndexedFile* file) {
+#define DESERIALIZE(json_name, member_name) Deserialize(entry, json_name, def.##member_name)
+
   auto& types = reader["types"].GetArray();
-  for (auto& type : types) {
+  for (auto& entry : types) {
-    TypeId id = TypeId(type["id"].GetInt64());
+    TypeId id(entry["id"].GetInt64());
-    std::string usr = type["usr"].GetString();
+    std::string usr = entry["usr"].GetString();
 
     IndexedTypeDef def(id, usr);
-    Deserialize(def.def.short_name, type["short_name"]);
+    def.is_bad_def = false;
-    Deserialize(def.def.qualified_name, type["qualified_name"]);
+    DESERIALIZE("short_name", def.short_name);
-    Deserialize(def.def.definition, type["definition"]); // TODO: What happens if entry is not present?
+    DESERIALIZE("qualified_name", def.qualified_name);
-    //SERIALIZE("short_name", def.short_name);
+    DESERIALIZE("definition", def.definition);
-    //SERIALIZE("qualified_name", def.qualified_name);
+    DESERIALIZE("alias_of", def.alias_of);
-    //SERIALIZE("definition", def.definition);
+    DESERIALIZE("parents", def.parents);
-    //SERIALIZE("alias_of", def.alias_of);
+    DESERIALIZE("derived", derived);
-    //SERIALIZE("parents", def.parents);
+    DESERIALIZE("types", def.types);
-    //SERIALIZE("derived", derived);
+    DESERIALIZE("funcs", def.funcs);
-    //SERIALIZE("types", def.types);
+    DESERIALIZE("vars", def.vars);
-    //SERIALIZE("funcs", def.funcs);
+    DESERIALIZE("uses", uses);
-    //SERIALIZE("vars", def.vars);
-    //SERIALIZE("uses", uses);
-
     file->types.push_back(def);
   }
+
+  auto& functions = reader["functions"].GetArray();
+  for (auto& entry : functions) {
+    FuncId id(entry["id"].GetInt64());
+    std::string usr = entry["usr"].GetString();
+
+    IndexedFuncDef def(id, usr);
+    def.is_bad_def = false;
+    DESERIALIZE("short_name", def.short_name);
+    DESERIALIZE("qualified_name", def.qualified_name);
+    DESERIALIZE("declarations", declarations);
+    DESERIALIZE("definition", def.definition);
+    DESERIALIZE("declaring_type", def.declaring_type);
+    DESERIALIZE("base", def.base);
+    DESERIALIZE("derived", derived);
+    DESERIALIZE("locals", def.locals);
+    DESERIALIZE("callers", callers);
+    DESERIALIZE("callees", def.callees);
+    DESERIALIZE("uses", uses);
+    file->funcs.push_back(def);
+  }
+
+  auto& vars = reader["variables"].GetArray();
+  for (auto& entry : vars) {
+    VarId id(entry["id"].GetInt64());
+    std::string usr = entry["usr"].GetString();
+
+    IndexedVarDef def(id, usr);
+    def.is_bad_def = false;
+    DESERIALIZE("short_name", def.short_name);
+    DESERIALIZE("qualified_name", def.qualified_name);
+    DESERIALIZE("declaration", def.declaration);
+    DESERIALIZE("definition", def.definition);
+    DESERIALIZE("variable_type", def.variable_type);
+    DESERIALIZE("declaring_type", def.declaring_type);
+    DESERIALIZE("uses", uses);
+    file->vars.push_back(def);
+  }
+#undef DESERIALIZE
+}
+
+std::string Serialize(IndexedFile* file) {
+  rapidjson::StringBuffer output;
+  rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(output);
+  writer.SetFormatOptions(
+    rapidjson::PrettyFormatOptions::kFormatSingleLineArray);
+  writer.SetIndent(' ', 2);
+
+  Serialize(writer, file);
+
+  return output.GetString();
+}
+
+IndexedFile Deserialize(std::string path, std::string serialized) {
+  rapidjson::Document document;
+  document.Parse(serialized.c_str());
+
+  IndexedFile file(path);
+  Deserialize(document, &file);
+  return file;
 }

serializer.h

@@ -7,3 +7,6 @@ using Reader = rapidjson::Document;
 
 void Serialize(Writer& writer, IndexedFile* file);
 void Deserialize(Reader& reader, IndexedFile* file);
+
+std::string Serialize(IndexedFile* file);
+IndexedFile Deserialize(std::string path, std::string serialized);

task.cc

@@ -131,7 +131,7 @@ void Pump(TaskManager* tm) {
   //tm->threads[0].
 }
 
-int main(int argc, char** argv) {
+int main5555555555(int argc, char** argv) {
   TaskManager tm(5);
 
   // TODO: looks like we will have to write shared memory support.

test.cc

@@ -1,25 +1,11 @@
 #include "indexer.h"
-
+#include "serializer.h"
-template<typename T>
-bool AreEqual(const std::vector<T>& a, const std::vector<T>& b) {
-  if (a.size() != b.size())
-    return false;
-
-  for (int i = 0; i < a.size(); ++i) {
-    if (a[i] != b[i])
-      return false;
-  }
-
-  return true;
-}
 
 void Write(const std::vector<std::string>& strs) {
-  for (const std::string& str : strs) {
+  for (const std::string& str : strs)
     std::cout << str << std::endl;
-  }
 }
 
-
 std::string ToString(const rapidjson::Document& document) {
   rapidjson::StringBuffer buffer;
   rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
@@ -90,7 +76,16 @@ void WriteToFile(const std::string& filename, const std::string& content) {
   file << content;
 }
 
-int main2222222(int argc, char** argv) {
+void VerifySerializeToFrom(IndexedFile* file) {
+  std::string expected = file->ToString();
+  std::string actual = Deserialize("foo.cc", Serialize(file)).ToString();
+  if (expected != actual) {
+    std::cerr << "Serialization failure" << std::endl;;
+    assert(false);
+  }
+}
+
+int main(int argc, char** argv) {
   // TODO: Assert that we need to be on clang >= 3.9.1
 
   /*
@@ -103,21 +98,6 @@ int main2222222(int argc, char** argv) {
 
   for (std::string path : GetFilesInFolder("tests")) {
     //if (path != "tests/usage/type_usage_declare_field.cc") continue;
-    //if (path != "tests/enums/enum_class_decl.cc") continue;
-    //if (path != "tests/constructors/constructor.cc") continue;
-    //if (path == "tests/constructors/destructor.cc") continue;
-    //if (path == "tests/usage/func_usage_call_method.cc") continue;
-    //if (path != "tests/usage/type_usage_as_template_parameter.cc") continue;
-    //if (path != "tests/usage/type_usage_as_template_parameter_complex.cc") continue;
-    //if (path != "tests/usage/type_usage_as_template_parameter_simple.cc") continue;
-    //if (path != "tests/usage/type_usage_typedef_and_using.cc") continue;
-    //if (path != "tests/usage/type_usage_declare_local.cc") continue;
-    //if (path == "tests/usage/type_usage_typedef_and_using_template.cc") continue;
-    //if (path != "tests/usage/func_usage_addr_method.cc") continue;
-    //if (path != "tests/usage/type_usage_typedef_and_using.cc") continue;
-    //if (path != "tests/usage/usage_inside_of_call.cc") continue;
-    //if (path != "tests/foobar.cc") continue;
-    //if (path != "tests/types/anonymous_struct.cc") continue;
 
     // Parse expected output from the test, parse it into JSON document.
     std::string expected_output;
@@ -128,11 +108,9 @@ int main2222222(int argc, char** argv) {
     // Run test.
     std::cout << "[START] " << path << std::endl;
     IndexedFile db = Parse(path, {}, true /*dump_ast*/);
+    VerifySerializeToFrom(&db);
     std::string actual_output = db.ToString();
 
-    //WriteToFile("output.json", actual_output);
-    //break;
-
     rapidjson::Document actual;
     actual.Parse(actual_output.c_str());