diff --git a/indexer.h b/indexer.h index 2b13d84b..268cea6c 100644 --- a/indexer.h +++ b/indexer.h @@ -88,8 +88,29 @@ struct Location { return FileId(raw_file_id); } - explicit Location(const char* encoded) { - // TODO + explicit Location(const char* encoded) : Location() { + int len = strlen(encoded); + assert(len >= 0); + + if (*encoded == '*') { + interesting = true; + ++encoded; + } + + assert(encoded); + raw_file_id = atoi(encoded); + while (*encoded && *encoded != ':') + ++encoded; + if (*encoded == ':') ++encoded; + + assert(encoded); + line = atoi(encoded); + while (*encoded && *encoded != ':') + ++encoded; + if (*encoded == ':') ++encoded; + + assert(encoded); + column = atoi(encoded); } std::string ToString() { diff --git a/query.h b/query.h index 7cfc5e99..773ee0bf 100644 --- a/query.h +++ b/query.h @@ -163,6 +163,9 @@ struct SymbolIdx { // We need to verify we need multiple processes first. Maybe libclang can run in a single process... // TODO: Compute IndexUpdates in main process, off the blocking thread. Use separate process for running // libclang. Solves memory worries. +// TODO: Instead of passing to/from json, we can probably bass the IndexedFile type almost directly as +// a raw memory dump - the type has almost zero pointers inside of it. We could do a little bit of fixup +// so that passing from a separate process to the main db is really fast (no need to go through JSON). namespace foo2 { using Usr = size_t; struct UsrTable { diff --git a/serializer.cc b/serializer.cc index d91850a7..b00ee87b 100644 --- a/serializer.cc +++ b/serializer.cc @@ -93,7 +93,7 @@ void Serialize(Writer& writer, IndexedFile* file) { assert(file->Resolve(it->second)->uses.size() == 0); } -#define SERIALIZE(name, value) Serialize(writer, name, def.value) +#define SERIALIZE(json_name, member_name) Serialize(writer, json_name, def.##member_name) writer.StartObject(); @@ -171,36 +171,135 @@ void Serialize(Writer& writer, IndexedFile* file) { #undef WRITE } -void Deserialize(std::string& output, rapidjson::GenericValue>& value) { - output = value.GetString(); +void Deserialize(rapidjson::GenericValue>& document, const char* name, std::string& output) { + auto it = document.FindMember(name); + if (it != document.MemberEnd()) + output = document[name].GetString(); } -void Deserialize(optional& output, rapidjson::GenericValue>& value) { - if (!value.IsNull()) - output = Location(value.GetString()); // TODO: Location parsing not implemented in Location type. +void Deserialize(rapidjson::GenericValue>& document, const char* name, optional& output) { + auto it = document.FindMember(name); + if (it != document.MemberEnd()) + output = Location(it->value.GetString()); // TODO: Location parsing not implemented in Location type. +} + +void Deserialize(rapidjson::GenericValue>& document, const char* name, std::vector& output) { + auto it = document.FindMember(name); + if (it != document.MemberEnd()) { + for (auto& array_value : it->value.GetArray()) + output.push_back(Location(array_value.GetString())); + } +} + +template +void Deserialize(rapidjson::GenericValue>& document, const char* name, optional>& output) { + auto it = document.FindMember(name); + if (it != document.MemberEnd()) + output = Id(it->value.GetUint64()); +} + +template +void Deserialize(rapidjson::GenericValue>& document, const char* name, std::vector>& output) { + auto it = document.FindMember(name); + if (it != document.MemberEnd()) { + for (auto& array_value : it->value.GetArray()) + output.push_back(Id(array_value.GetUint64())); + } +} + +template +void Deserialize(rapidjson::GenericValue>& document, const char* name, std::vector>& output) { + auto it = document.FindMember(name); + if (it != document.MemberEnd()) { + for (auto& array_value : it->value.GetArray()) { + const char* str_value = array_value.GetString(); + uint64_t id = atoi(str_value); + const char* loc_string = strchr(str_value, '@') + 1; + output.push_back(Ref(Id(id), Location(loc_string))); + } + } } void Deserialize(Reader& reader, IndexedFile* file) { +#define DESERIALIZE(json_name, member_name) Deserialize(entry, json_name, def.##member_name) + auto& types = reader["types"].GetArray(); - for (auto& type : types) { - TypeId id = TypeId(type["id"].GetInt64()); - std::string usr = type["usr"].GetString(); + for (auto& entry : types) { + TypeId id(entry["id"].GetInt64()); + std::string usr = entry["usr"].GetString(); IndexedTypeDef def(id, usr); - Deserialize(def.def.short_name, type["short_name"]); - Deserialize(def.def.qualified_name, type["qualified_name"]); - Deserialize(def.def.definition, type["definition"]); // TODO: What happens if entry is not present? - //SERIALIZE("short_name", def.short_name); - //SERIALIZE("qualified_name", def.qualified_name); - //SERIALIZE("definition", def.definition); - //SERIALIZE("alias_of", def.alias_of); - //SERIALIZE("parents", def.parents); - //SERIALIZE("derived", derived); - //SERIALIZE("types", def.types); - //SERIALIZE("funcs", def.funcs); - //SERIALIZE("vars", def.vars); - //SERIALIZE("uses", uses); - + def.is_bad_def = false; + DESERIALIZE("short_name", def.short_name); + DESERIALIZE("qualified_name", def.qualified_name); + DESERIALIZE("definition", def.definition); + DESERIALIZE("alias_of", def.alias_of); + DESERIALIZE("parents", def.parents); + DESERIALIZE("derived", derived); + DESERIALIZE("types", def.types); + DESERIALIZE("funcs", def.funcs); + DESERIALIZE("vars", def.vars); + DESERIALIZE("uses", uses); file->types.push_back(def); } -} \ No newline at end of file + + auto& functions = reader["functions"].GetArray(); + for (auto& entry : functions) { + FuncId id(entry["id"].GetInt64()); + std::string usr = entry["usr"].GetString(); + + IndexedFuncDef def(id, usr); + def.is_bad_def = false; + DESERIALIZE("short_name", def.short_name); + DESERIALIZE("qualified_name", def.qualified_name); + DESERIALIZE("declarations", declarations); + DESERIALIZE("definition", def.definition); + DESERIALIZE("declaring_type", def.declaring_type); + DESERIALIZE("base", def.base); + DESERIALIZE("derived", derived); + DESERIALIZE("locals", def.locals); + DESERIALIZE("callers", callers); + DESERIALIZE("callees", def.callees); + DESERIALIZE("uses", uses); + file->funcs.push_back(def); + } + + auto& vars = reader["variables"].GetArray(); + for (auto& entry : vars) { + VarId id(entry["id"].GetInt64()); + std::string usr = entry["usr"].GetString(); + + IndexedVarDef def(id, usr); + def.is_bad_def = false; + DESERIALIZE("short_name", def.short_name); + DESERIALIZE("qualified_name", def.qualified_name); + DESERIALIZE("declaration", def.declaration); + DESERIALIZE("definition", def.definition); + DESERIALIZE("variable_type", def.variable_type); + DESERIALIZE("declaring_type", def.declaring_type); + DESERIALIZE("uses", uses); + file->vars.push_back(def); + } +#undef DESERIALIZE +} + +std::string Serialize(IndexedFile* file) { + rapidjson::StringBuffer output; + rapidjson::PrettyWriter writer(output); + writer.SetFormatOptions( + rapidjson::PrettyFormatOptions::kFormatSingleLineArray); + writer.SetIndent(' ', 2); + + Serialize(writer, file); + + return output.GetString(); +} + +IndexedFile Deserialize(std::string path, std::string serialized) { + rapidjson::Document document; + document.Parse(serialized.c_str()); + + IndexedFile file(path); + Deserialize(document, &file); + return file; +} diff --git a/serializer.h b/serializer.h index 8d091fe4..0d99c8e3 100644 --- a/serializer.h +++ b/serializer.h @@ -6,4 +6,7 @@ using Writer = rapidjson::PrettyWriter; using Reader = rapidjson::Document; void Serialize(Writer& writer, IndexedFile* file); -void Deserialize(Reader& reader, IndexedFile* file); \ No newline at end of file +void Deserialize(Reader& reader, IndexedFile* file); + +std::string Serialize(IndexedFile* file); +IndexedFile Deserialize(std::string path, std::string serialized); diff --git a/task.cc b/task.cc index 6c8ae429..a6d17614 100644 --- a/task.cc +++ b/task.cc @@ -131,7 +131,7 @@ void Pump(TaskManager* tm) { //tm->threads[0]. } -int main(int argc, char** argv) { +int main5555555555(int argc, char** argv) { TaskManager tm(5); // TODO: looks like we will have to write shared memory support. diff --git a/test.cc b/test.cc index f111827e..e340af86 100644 --- a/test.cc +++ b/test.cc @@ -1,25 +1,11 @@ #include "indexer.h" - -template -bool AreEqual(const std::vector& a, const std::vector& b) { - if (a.size() != b.size()) - return false; - - for (int i = 0; i < a.size(); ++i) { - if (a[i] != b[i]) - return false; - } - - return true; -} +#include "serializer.h" void Write(const std::vector& strs) { - for (const std::string& str : strs) { + for (const std::string& str : strs) std::cout << str << std::endl; - } } - std::string ToString(const rapidjson::Document& document) { rapidjson::StringBuffer buffer; rapidjson::PrettyWriter writer(buffer); @@ -90,7 +76,16 @@ void WriteToFile(const std::string& filename, const std::string& content) { file << content; } -int main2222222(int argc, char** argv) { +void VerifySerializeToFrom(IndexedFile* file) { + std::string expected = file->ToString(); + std::string actual = Deserialize("foo.cc", Serialize(file)).ToString(); + if (expected != actual) { + std::cerr << "Serialization failure" << std::endl;; + assert(false); + } +} + +int main(int argc, char** argv) { // TODO: Assert that we need to be on clang >= 3.9.1 /* @@ -103,21 +98,6 @@ int main2222222(int argc, char** argv) { for (std::string path : GetFilesInFolder("tests")) { //if (path != "tests/usage/type_usage_declare_field.cc") continue; - //if (path != "tests/enums/enum_class_decl.cc") continue; - //if (path != "tests/constructors/constructor.cc") continue; - //if (path == "tests/constructors/destructor.cc") continue; - //if (path == "tests/usage/func_usage_call_method.cc") continue; - //if (path != "tests/usage/type_usage_as_template_parameter.cc") continue; - //if (path != "tests/usage/type_usage_as_template_parameter_complex.cc") continue; - //if (path != "tests/usage/type_usage_as_template_parameter_simple.cc") continue; - //if (path != "tests/usage/type_usage_typedef_and_using.cc") continue; - //if (path != "tests/usage/type_usage_declare_local.cc") continue; - //if (path == "tests/usage/type_usage_typedef_and_using_template.cc") continue; - //if (path != "tests/usage/func_usage_addr_method.cc") continue; - //if (path != "tests/usage/type_usage_typedef_and_using.cc") continue; - //if (path != "tests/usage/usage_inside_of_call.cc") continue; - //if (path != "tests/foobar.cc") continue; - //if (path != "tests/types/anonymous_struct.cc") continue; // Parse expected output from the test, parse it into JSON document. std::string expected_output; @@ -128,11 +108,9 @@ int main2222222(int argc, char** argv) { // Run test. std::cout << "[START] " << path << std::endl; IndexedFile db = Parse(path, {}, true /*dump_ast*/); + VerifySerializeToFrom(&db); std::string actual_output = db.ToString(); - //WriteToFile("output.json", actual_output); - //break; - rapidjson::Document actual; actual.Parse(actual_output.c_str());