From 5809d0c4367effc385107a951a91d2860abc7b68 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sat, 6 Jan 2018 18:01:32 -0800 Subject: [PATCH] Make Serialize accept argument `SerializeFormat format` Rename `const char* GetString()` to GetCString and add `std::string GetString()` WIP on serializers/msgpack.h --- src/cache_manager.cc | 2 +- src/indexer.cc | 2 +- src/indexer.h | 2 +- src/position.cc | 4 +-- src/serializer.cc | 76 ++++++++++++++++++++++++--------------- src/serializer.h | 5 +-- src/serializers/json.h | 2 +- src/serializers/msgpack.h | 75 +++++++++++++++++++++++++------------- src/test.cc | 6 ++-- 9 files changed, 110 insertions(+), 64 deletions(-) diff --git a/src/cache_manager.cc b/src/cache_manager.cc index 5799d016..8eb2fa6e 100644 --- a/src/cache_manager.cc +++ b/src/cache_manager.cc @@ -31,7 +31,7 @@ struct RealCacheManager : ICacheManager { WriteToFile(cache_path, file.file_contents_); } - std::string indexed_content = Serialize(file); + std::string indexed_content = Serialize(config_->cacheFormat, file); WriteToFile(AppendSerializationFormat(cache_path), indexed_content); } diff --git a/src/indexer.cc b/src/indexer.cc index 0cfb2f19..ff967332 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -479,7 +479,7 @@ IndexVar* IndexFile::Resolve(IndexVarId id) { } std::string IndexFile::ToString() { - return Serialize(*this); + return Serialize(SerializeFormat::Json, *this); } IndexType::IndexType(IndexTypeId id, const std::string& usr) diff --git a/src/indexer.h b/src/indexer.h index 3856b555..d1478f71 100644 --- a/src/indexer.h +++ b/src/indexer.h @@ -154,7 +154,7 @@ inline bool operator!=(const IndexFuncRef& a, const IndexFuncRef& b) { } inline void Reflect(Reader& visitor, IndexFuncRef& value) { - const char* str_value = visitor.GetString(); + const char* str_value = visitor.GetCString(); if (str_value[0] == '~') { value.is_implicit = true; ++str_value; diff --git a/src/position.cc b/src/position.cc index 3421f34a..71099336 100644 --- a/src/position.cc +++ b/src/position.cc @@ -149,7 +149,7 @@ void Reflect(Reader& visitor, Position& value) { if (!visitor.IsString()) value = Position(); else - value = Position(visitor.GetString()); + value = Position(visitor.GetCString()); } void Reflect(Writer& visitor, Position& value) { std::string output = value.ToString(); @@ -161,7 +161,7 @@ void Reflect(Reader& visitor, Range& value) { if (!visitor.IsString()) value = Range(); else - value = Range(visitor.GetString()); + value = Range(visitor.GetCString()); } void Reflect(Writer& visitor, Range& value) { std::string output = value.ToString(); diff --git a/src/serializer.cc b/src/serializer.cc index d00cd7a9..f8239863 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -1,6 +1,5 @@ #include "serializer.h" -// TODO Move Json* to serializers/json.cc #include "serializers/json.h" #include "serializers/msgpack.h" @@ -192,44 +191,65 @@ void Reflect(TVisitor& visitor, IndexFile& value) { REFLECT_MEMBER_END(); } -std::string Serialize(IndexFile& file) { - rapidjson::StringBuffer output; - rapidjson::PrettyWriter writer(output); - writer.SetFormatOptions( - rapidjson::PrettyFormatOptions::kFormatSingleLineArray); - writer.SetIndent(' ', 2); +std::string Serialize(SerializeFormat format, IndexFile& file) { - JsonWriter json_writer(&writer); - Reflect(json_writer, file); - - return output.GetString(); + switch (format) { + case SerializeFormat::Json: { + rapidjson::StringBuffer output; + rapidjson::PrettyWriter writer(output); + writer.SetFormatOptions( + rapidjson::PrettyFormatOptions::kFormatSingleLineArray); + writer.SetIndent(' ', 2); + JsonWriter json_writer(&writer); + Reflect(json_writer, file); + return output.GetString(); + } + case SerializeFormat::MessagePack: { + msgpack::sbuffer buf; + msgpack::packer pk(&buf); + MessagePackWriter msgpack_writer(&pk); + Reflect(msgpack_writer, file); + return std::string(buf.data(), buf.size()); + } + } + return ""; } std::unique_ptr Deserialize(SerializeFormat format, std::string path, std::string serialized, optional expected_version) { - rapidjson::Document reader; - reader.Parse(serialized.c_str()); - if (reader.HasParseError()) - return nullptr; + std::unique_ptr file; + switch (format) { + case SerializeFormat::Json: { + rapidjson::Document reader; + reader.Parse(serialized.c_str()); + if (reader.HasParseError()) + return nullptr; - // Do not deserialize a document with a bad version. Doing so could cause a - // crash because the file format may have changed. - if (expected_version) { - auto actual_version = reader.FindMember("version"); - if (actual_version == reader.MemberEnd() || - actual_version->value.GetInt() != expected_version) { - return nullptr; + // Do not deserialize a document with a bad version. Doing so could cause a + // crash because the file format may have changed. + if (expected_version) { + auto actual_version = reader.FindMember("version"); + if (actual_version == reader.MemberEnd() || + actual_version->value.GetInt() != expected_version) { + return nullptr; + } + } + + file = MakeUnique(path); + JsonReader json_reader{&reader}; + Reflect(json_reader, *file); + break; + } + + case SerializeFormat::MessagePack: { + msgpack::object_handle oh = msgpack::unpack(serialized.data(), serialized.size()); + (void)oh; + break; } } - // TODO msgpack - (void)format; - auto file = MakeUnique(path); - JsonReader json_reader{&reader}; - Reflect(json_reader, *file); - // Restore non-serialized state. file->path = path; file->id_cache.primary_file = file->path; diff --git a/src/serializer.h b/src/serializer.h index a5945109..7858c4d6 100644 --- a/src/serializer.h +++ b/src/serializer.h @@ -27,7 +27,8 @@ class Reader { virtual int GetInt() = 0; virtual int64_t GetInt64() = 0; virtual uint64_t GetUint64() = 0; - virtual const char* GetString() = 0; + virtual const char* GetCString() = 0; + virtual std::string GetString() { return GetCString(); } virtual bool HasMember(const char* x) = 0; virtual std::unique_ptr operator[](const char* x) = 0; @@ -245,7 +246,7 @@ void ReflectMember(Reader& visitor, const char* name, T& value) { visitor.DoMember(name, [&](Reader& child) { Reflect(child, value); }); } -std::string Serialize(IndexFile& file); +std::string Serialize(SerializeFormat format, IndexFile& file); std::unique_ptr Deserialize(SerializeFormat format, std::string path, std::string serialized, diff --git a/src/serializers/json.h b/src/serializers/json.h index ef12b502..b6e5b005 100644 --- a/src/serializers/json.h +++ b/src/serializers/json.h @@ -24,7 +24,7 @@ class JsonReader : public Reader { int GetInt() override { return m_->GetInt(); } int64_t GetInt64() override { return m_->GetInt64(); } uint64_t GetUint64() override { return m_->GetUint64(); } - const char* GetString() override { return m_->GetString(); } + const char* GetCString() override { return m_->GetString(); } bool HasMember(const char* x) override { return m_->HasMember(x); } std::unique_ptr operator[](const char* x) override { diff --git a/src/serializers/msgpack.h b/src/serializers/msgpack.h index 364b3fd1..5c604740 100644 --- a/src/serializers/msgpack.h +++ b/src/serializers/msgpack.h @@ -5,37 +5,61 @@ #include class MessagePackReader : public Reader { - msgpack::object_handle* m_; + msgpack::unpacker* m_; + msgpack::object_handle oh_; + + void next() { m_->next(oh_); } public: - MessagePackReader(msgpack::object_handle* m) : m_(m) {} + MessagePackReader(msgpack::unpacker* m) : m_(m) { next(); } + SerializeFormat Format() const override { return SerializeFormat::MessagePack; } - //bool GetBool() override { return m_->GetBool(); } - //int GetInt() override { return m_->GetInt(); } - //int64_t GetInt64() override { return m_->GetInt64(); } - //uint64_t GetUint64() override { return m_->GetUint64(); } - //const char* GetString() override { return m_->GetString(); } + bool IsNull() override { return oh_.get().is_nil(); } + bool IsArray() override { return oh_.get().type == msgpack::type::ARRAY; } + bool IsInt() override { + return oh_.get().type == msgpack::type::POSITIVE_INTEGER || + oh_.get().type == msgpack::type::NEGATIVE_INTEGER; + } + bool IsString() override { return oh_.get().type == msgpack::type::STR; } - //bool HasMember(const char* x) override { return m_->HasMember(x); } - //std::unique_ptr operator[](const char* x) override { - // auto& sub = (*m_)[x]; - // return std::unique_ptr(new JsonReader(&sub)); - //} + bool GetBool() override { + auto ret = oh_.get().as(); + next(); + return ret; + } + int GetInt() override { + auto ret = oh_.get().as(); + next(); + return ret; + } + int64_t GetInt64() override { + auto ret = oh_.get().as(); + next(); + return ret; + } + uint64_t GetUint64() override { + auto ret = oh_.get().as(); + next(); + return ret; + } + const char* GetCString() override { + auto ret = oh_.get().as(); + next(); + return ret; + } - //void IterArray(std::function fn) override { - // for (auto& entry : m_->GetArray()) { - // MessagePackReader sub(&entry); - // fn(sub); - // } - //} + bool HasMember(const char* x) override { return true; } + std::unique_ptr operator[](const char* x) override { + return {}; + } - //void DoMember(const char* name, std::function fn) override { - // auto it = m_->FindMember(name); - // if (it != m_->MemberEnd()) { - // MessagePackReader sub(&it->value); - // fn(sub); - // } - //} + void IterArray(std::function fn) override { + } + + void DoMember(const char* name, std::function fn) override { + const char* key = GetCString(); + fn(*this); + } }; class MessagePackWriter : public Writer { @@ -43,6 +67,7 @@ class MessagePackWriter : public Writer { public: MessagePackWriter(msgpack::packer* m) : m_(m) {} + SerializeFormat Format() const override { return SerializeFormat::MessagePack; } void Null() override { m_->pack_nil(); } void Bool(bool x) override { m_->pack(x); } diff --git a/src/test.cc b/src/test.cc index 23a5d5a8..a4a544d1 100644 --- a/src/test.cc +++ b/src/test.cc @@ -75,9 +75,9 @@ void DiffDocuments(std::string path, void VerifySerializeToFrom(IndexFile* file) { std::string expected = file->ToString(); - std::unique_ptr result = - Deserialize(SerializeFormat::Json, "--.cc", Serialize(*file), - nullopt /*expected_version*/); + std::unique_ptr result = Deserialize( + SerializeFormat::Json, "--.cc", Serialize(SerializeFormat::Json, *file), + nullopt /*expected_version*/); std::string actual = result->ToString(); if (expected != actual) { std::cerr << "Serialization failure" << std::endl;