ccls/src/serializer.cc
Fangrui Song 5935db10fd Fix some clang-tidy warnings
Incorporated some fixes by Daniel Chabrowski (#467)
2019-10-10 19:24:44 -07:00

544 lines
20 KiB
C++

// Copyright 2017-2020 ccls Authors
// SPDX-License-Identifier: Apache-2.0
#include "serializer.hh"
#include "filesystem.hh"
#include "indexer.hh"
#include "log.hh"
#include "message_handler.hh"
#include <rapidjson/document.h>
#include <rapidjson/prettywriter.h>
#include <llvm/ADT/CachedHashString.h>
#include <llvm/ADT/DenseSet.h>
#include <llvm/ADT/STLExtras.h>
#include <llvm/Support/Allocator.h>
#include <mutex>
#include <stdexcept>
using namespace llvm;
bool gTestOutputMode = false;
namespace ccls {
void JsonReader::iterArray(llvm::function_ref<void()> fn) {
if (!m->IsArray())
throw std::invalid_argument("array");
// Use "0" to indicate any element for now.
path_.push_back("0");
for (auto &entry : m->GetArray()) {
auto saved = m;
m = &entry;
fn();
m = saved;
}
path_.pop_back();
}
void JsonReader::member(const char *name, llvm::function_ref<void()> fn) {
path_.push_back(name);
auto it = m->FindMember(name);
if (it != m->MemberEnd()) {
auto saved = m;
m = &it->value;
fn();
m = saved;
}
path_.pop_back();
}
bool JsonReader::isNull() { return m->IsNull(); }
std::string JsonReader::getString() { return m->GetString(); }
std::string JsonReader::getPath() const {
std::string ret;
for (auto &t : path_)
if (t[0] == '0') {
ret += '[';
ret += t;
ret += ']';
} else {
ret += '/';
ret += t;
}
return ret;
}
void JsonWriter::startArray() { m->StartArray(); }
void JsonWriter::endArray() { m->EndArray(); }
void JsonWriter::startObject() { m->StartObject(); }
void JsonWriter::endObject() { m->EndObject(); }
void JsonWriter::key(const char *name) { m->Key(name); }
void JsonWriter::null_() { m->Null(); }
void JsonWriter::int64(int64_t v) { m->Int64(v); }
void JsonWriter::string(const char *s) { m->String(s); }
void JsonWriter::string(const char *s, size_t len) { m->String(s, len); }
// clang-format off
void reflect(JsonReader &vis, bool &v ) { if (!vis.m->IsBool()) throw std::invalid_argument("bool"); v = vis.m->GetBool(); }
void reflect(JsonReader &vis, unsigned char &v ) { if (!vis.m->IsInt()) throw std::invalid_argument("uint8_t"); v = (uint8_t)vis.m->GetInt(); }
void reflect(JsonReader &vis, short &v ) { if (!vis.m->IsInt()) throw std::invalid_argument("short"); v = (short)vis.m->GetInt(); }
void reflect(JsonReader &vis, unsigned short &v ) { if (!vis.m->IsInt()) throw std::invalid_argument("unsigned short"); v = (unsigned short)vis.m->GetInt(); }
void reflect(JsonReader &vis, int &v ) { if (!vis.m->IsInt()) throw std::invalid_argument("int"); v = vis.m->GetInt(); }
void reflect(JsonReader &vis, unsigned &v ) { if (!vis.m->IsUint64()) throw std::invalid_argument("unsigned"); v = (unsigned)vis.m->GetUint64(); }
void reflect(JsonReader &vis, long &v ) { if (!vis.m->IsInt64()) throw std::invalid_argument("long"); v = (long)vis.m->GetInt64(); }
void reflect(JsonReader &vis, unsigned long &v ) { if (!vis.m->IsUint64()) throw std::invalid_argument("unsigned long"); v = (unsigned long)vis.m->GetUint64(); }
void reflect(JsonReader &vis, long long &v ) { if (!vis.m->IsInt64()) throw std::invalid_argument("long long"); v = vis.m->GetInt64(); }
void reflect(JsonReader &vis, unsigned long long &v) { if (!vis.m->IsUint64()) throw std::invalid_argument("unsigned long long"); v = vis.m->GetUint64(); }
void reflect(JsonReader &vis, double &v ) { if (!vis.m->IsDouble()) throw std::invalid_argument("double"); v = vis.m->GetDouble(); }
void reflect(JsonReader &vis, const char *&v ) { if (!vis.m->IsString()) throw std::invalid_argument("string"); v = intern(vis.getString()); }
void reflect(JsonReader &vis, std::string &v ) { if (!vis.m->IsString()) throw std::invalid_argument("string"); v = vis.getString(); }
void reflect(JsonWriter &vis, bool &v ) { vis.m->Bool(v); }
void reflect(JsonWriter &vis, unsigned char &v ) { vis.m->Int(v); }
void reflect(JsonWriter &vis, short &v ) { vis.m->Int(v); }
void reflect(JsonWriter &vis, unsigned short &v ) { vis.m->Int(v); }
void reflect(JsonWriter &vis, int &v ) { vis.m->Int(v); }
void reflect(JsonWriter &vis, unsigned &v ) { vis.m->Uint64(v); }
void reflect(JsonWriter &vis, long &v ) { vis.m->Int64(v); }
void reflect(JsonWriter &vis, unsigned long &v ) { vis.m->Uint64(v); }
void reflect(JsonWriter &vis, long long &v ) { vis.m->Int64(v); }
void reflect(JsonWriter &vis, unsigned long long &v) { vis.m->Uint64(v); }
void reflect(JsonWriter &vis, double &v ) { vis.m->Double(v); }
void reflect(JsonWriter &vis, const char *&v ) { vis.string(v); }
void reflect(JsonWriter &vis, std::string &v ) { vis.string(v.c_str(), v.size()); }
void reflect(BinaryReader &vis, bool &v ) { v = vis.get<bool>(); }
void reflect(BinaryReader &vis, unsigned char &v ) { v = vis.get<unsigned char>(); }
void reflect(BinaryReader &vis, short &v ) { v = (short)vis.varInt(); }
void reflect(BinaryReader &vis, unsigned short &v ) { v = (unsigned short)vis.varUInt(); }
void reflect(BinaryReader &vis, int &v ) { v = (int)vis.varInt(); }
void reflect(BinaryReader &vis, unsigned &v ) { v = (unsigned)vis.varUInt(); }
void reflect(BinaryReader &vis, long &v ) { v = (long)vis.varInt(); }
void reflect(BinaryReader &vis, unsigned long &v ) { v = (unsigned long)vis.varUInt(); }
void reflect(BinaryReader &vis, long long &v ) { v = vis.varInt(); }
void reflect(BinaryReader &vis, unsigned long long &v) { v = vis.varUInt(); }
void reflect(BinaryReader &vis, double &v ) { v = vis.get<double>(); }
void reflect(BinaryReader &vis, const char *&v ) { v = intern(vis.getString()); }
void reflect(BinaryReader &vis, std::string &v ) { v = vis.getString(); }
void reflect(BinaryWriter &vis, bool &v ) { vis.pack(v); }
void reflect(BinaryWriter &vis, unsigned char &v ) { vis.pack(v); }
void reflect(BinaryWriter &vis, short &v ) { vis.varInt(v); }
void reflect(BinaryWriter &vis, unsigned short &v ) { vis.varUInt(v); }
void reflect(BinaryWriter &vis, int &v ) { vis.varInt(v); }
void reflect(BinaryWriter &vis, unsigned &v ) { vis.varUInt(v); }
void reflect(BinaryWriter &vis, long &v ) { vis.varInt(v); }
void reflect(BinaryWriter &vis, unsigned long &v ) { vis.varUInt(v); }
void reflect(BinaryWriter &vis, long long &v ) { vis.varInt(v); }
void reflect(BinaryWriter &vis, unsigned long long &v) { vis.varUInt(v); }
void reflect(BinaryWriter &vis, double &v ) { vis.pack(v); }
void reflect(BinaryWriter &vis, const char *&v ) { vis.string(v); }
void reflect(BinaryWriter &vis, std::string &v ) { vis.string(v.c_str(), v.size()); }
// clang-format on
void reflect(JsonWriter &vis, std::string_view &data) {
if (data.empty())
vis.string("");
else
vis.string(&data[0], (rapidjson::SizeType)data.size());
}
void reflect(JsonReader &vis, JsonNull &v) {}
void reflect(JsonWriter &vis, JsonNull &v) { vis.m->Null(); }
template <typename V>
void reflect(JsonReader &vis, std::unordered_map<Usr, V> &v) {
vis.iterArray([&]() {
V val;
reflect(vis, val);
v[val.usr] = std::move(val);
});
}
template <typename V>
void reflect(JsonWriter &vis, std::unordered_map<Usr, V> &v) {
// Determinism
std::vector<std::pair<uint64_t, V>> xs(v.begin(), v.end());
std::sort(xs.begin(), xs.end(),
[](const auto &a, const auto &b) { return a.first < b.first; });
vis.startArray();
for (auto &it : xs)
reflect(vis, it.second);
vis.endArray();
}
template <typename V>
void reflect(BinaryReader &vis, std::unordered_map<Usr, V> &v) {
for (auto n = vis.varUInt(); n; n--) {
V val;
reflect(vis, val);
v[val.usr] = std::move(val);
}
}
template <typename V>
void reflect(BinaryWriter &vis, std::unordered_map<Usr, V> &v) {
vis.varUInt(v.size());
for (auto &it : v)
reflect(vis, it.second);
}
// Used by IndexFile::dependencies.
void reflect(JsonReader &vis, DenseMap<CachedHashStringRef, int64_t> &v) {
std::string name;
for (auto it = vis.m->MemberBegin(); it != vis.m->MemberEnd(); ++it)
v[internH(it->name.GetString())] = it->value.GetInt64();
}
void reflect(JsonWriter &vis, DenseMap<CachedHashStringRef, int64_t> &v) {
vis.startObject();
for (auto &it : v) {
vis.m->Key(it.first.val().data()); // llvm 8 -> data()
vis.m->Int64(it.second);
}
vis.endObject();
}
void reflect(BinaryReader &vis, DenseMap<CachedHashStringRef, int64_t> &v) {
std::string name;
for (auto n = vis.varUInt(); n; n--) {
reflect(vis, name);
reflect(vis, v[internH(name)]);
}
}
void reflect(BinaryWriter &vis, DenseMap<CachedHashStringRef, int64_t> &v) {
std::string key;
vis.varUInt(v.size());
for (auto &it : v) {
key = it.first.val().str();
reflect(vis, key);
reflect(vis, it.second);
}
}
template <typename Vis> void reflect(Vis &vis, IndexInclude &v) {
reflectMemberStart(vis);
REFLECT_MEMBER(line);
REFLECT_MEMBER(resolved_path);
reflectMemberEnd(vis);
}
void reflect(JsonWriter &vis, IndexInclude &v) {
reflectMemberStart(vis);
REFLECT_MEMBER(line);
if (gTestOutputMode) {
std::string basename(llvm::sys::path::filename(v.resolved_path));
if (v.resolved_path[0] != '&')
basename = "&" + basename;
REFLECT_MEMBER2("resolved_path", basename);
} else {
REFLECT_MEMBER(resolved_path);
}
reflectMemberEnd(vis);
}
template <typename Def>
void reflectHoverAndComments(JsonReader &vis, Def &def) {
reflectMember(vis, "hover", def.hover);
reflectMember(vis, "comments", def.comments);
}
template <typename Def>
void reflectHoverAndComments(JsonWriter &vis, Def &def) {
// Don't emit empty hover and comments in JSON test mode.
if (!gTestOutputMode || def.hover[0])
reflectMember(vis, "hover", def.hover);
if (!gTestOutputMode || def.comments[0])
reflectMember(vis, "comments", def.comments);
}
template <typename Def>
void reflectHoverAndComments(BinaryReader &vis, Def &def) {
reflect(vis, def.hover);
reflect(vis, def.comments);
}
template <typename Def>
void reflectHoverAndComments(BinaryWriter &vis, Def &def) {
reflect(vis, def.hover);
reflect(vis, def.comments);
}
template <typename Def> void reflectShortName(JsonReader &vis, Def &def) {
if (gTestOutputMode) {
std::string short_name;
reflectMember(vis, "short_name", short_name);
def.short_name_offset =
std::string_view(def.detailed_name).find(short_name);
assert(def.short_name_offset != std::string::npos);
def.short_name_size = short_name.size();
} else {
reflectMember(vis, "short_name_offset", def.short_name_offset);
reflectMember(vis, "short_name_size", def.short_name_size);
}
}
template <typename Def> void reflectShortName(JsonWriter &vis, Def &def) {
if (gTestOutputMode) {
std::string_view short_name(def.detailed_name + def.short_name_offset,
def.short_name_size);
reflectMember(vis, "short_name", short_name);
} else {
reflectMember(vis, "short_name_offset", def.short_name_offset);
reflectMember(vis, "short_name_size", def.short_name_size);
}
}
template <typename Def> void reflectShortName(BinaryReader &vis, Def &def) {
reflect(vis, def.short_name_offset);
reflect(vis, def.short_name_size);
}
template <typename Def> void reflectShortName(BinaryWriter &vis, Def &def) {
reflect(vis, def.short_name_offset);
reflect(vis, def.short_name_size);
}
template <typename TVisitor> void reflect1(TVisitor &vis, IndexFunc &v) {
reflectMemberStart(vis);
REFLECT_MEMBER2("usr", v.usr);
REFLECT_MEMBER2("detailed_name", v.def.detailed_name);
REFLECT_MEMBER2("qual_name_offset", v.def.qual_name_offset);
reflectShortName(vis, v.def);
REFLECT_MEMBER2("spell", v.def.spell);
reflectHoverAndComments(vis, v.def);
REFLECT_MEMBER2("bases", v.def.bases);
REFLECT_MEMBER2("vars", v.def.vars);
REFLECT_MEMBER2("callees", v.def.callees);
REFLECT_MEMBER2("kind", v.def.kind);
REFLECT_MEMBER2("parent_kind", v.def.parent_kind);
REFLECT_MEMBER2("storage", v.def.storage);
REFLECT_MEMBER2("declarations", v.declarations);
REFLECT_MEMBER2("derived", v.derived);
REFLECT_MEMBER2("uses", v.uses);
reflectMemberEnd(vis);
}
void reflect(JsonReader &vis, IndexFunc &v) { reflect1(vis, v); }
void reflect(JsonWriter &vis, IndexFunc &v) { reflect1(vis, v); }
void reflect(BinaryReader &vis, IndexFunc &v) { reflect1(vis, v); }
void reflect(BinaryWriter &vis, IndexFunc &v) { reflect1(vis, v); }
template <typename Vis> void reflect1(Vis &vis, IndexType &v) {
reflectMemberStart(vis);
REFLECT_MEMBER2("usr", v.usr);
REFLECT_MEMBER2("detailed_name", v.def.detailed_name);
REFLECT_MEMBER2("qual_name_offset", v.def.qual_name_offset);
reflectShortName(vis, v.def);
reflectHoverAndComments(vis, v.def);
REFLECT_MEMBER2("spell", v.def.spell);
REFLECT_MEMBER2("bases", v.def.bases);
REFLECT_MEMBER2("funcs", v.def.funcs);
REFLECT_MEMBER2("types", v.def.types);
REFLECT_MEMBER2("vars", v.def.vars);
REFLECT_MEMBER2("alias_of", v.def.alias_of);
REFLECT_MEMBER2("kind", v.def.kind);
REFLECT_MEMBER2("parent_kind", v.def.parent_kind);
REFLECT_MEMBER2("declarations", v.declarations);
REFLECT_MEMBER2("derived", v.derived);
REFLECT_MEMBER2("instances", v.instances);
REFLECT_MEMBER2("uses", v.uses);
reflectMemberEnd(vis);
}
void reflect(JsonReader &vis, IndexType &v) { reflect1(vis, v); }
void reflect(JsonWriter &vis, IndexType &v) { reflect1(vis, v); }
void reflect(BinaryReader &vis, IndexType &v) { reflect1(vis, v); }
void reflect(BinaryWriter &vis, IndexType &v) { reflect1(vis, v); }
template <typename TVisitor> void reflect1(TVisitor &vis, IndexVar &v) {
reflectMemberStart(vis);
REFLECT_MEMBER2("usr", v.usr);
REFLECT_MEMBER2("detailed_name", v.def.detailed_name);
REFLECT_MEMBER2("qual_name_offset", v.def.qual_name_offset);
reflectShortName(vis, v.def);
reflectHoverAndComments(vis, v.def);
REFLECT_MEMBER2("spell", v.def.spell);
REFLECT_MEMBER2("type", v.def.type);
REFLECT_MEMBER2("kind", v.def.kind);
REFLECT_MEMBER2("parent_kind", v.def.parent_kind);
REFLECT_MEMBER2("storage", v.def.storage);
REFLECT_MEMBER2("declarations", v.declarations);
REFLECT_MEMBER2("uses", v.uses);
reflectMemberEnd(vis);
}
void reflect(JsonReader &vis, IndexVar &v) { reflect1(vis, v); }
void reflect(JsonWriter &vis, IndexVar &v) { reflect1(vis, v); }
void reflect(BinaryReader &vis, IndexVar &v) { reflect1(vis, v); }
void reflect(BinaryWriter &vis, IndexVar &v) { reflect1(vis, v); }
// IndexFile
template <typename TVisitor> void reflect1(TVisitor &vis, IndexFile &v) {
reflectMemberStart(vis);
if (!gTestOutputMode) {
REFLECT_MEMBER(mtime);
REFLECT_MEMBER(language);
REFLECT_MEMBER(no_linkage);
REFLECT_MEMBER(lid2path);
REFLECT_MEMBER(import_file);
REFLECT_MEMBER(args);
REFLECT_MEMBER(dependencies);
}
REFLECT_MEMBER(includes);
REFLECT_MEMBER(skipped_ranges);
REFLECT_MEMBER(usr2func);
REFLECT_MEMBER(usr2type);
REFLECT_MEMBER(usr2var);
reflectMemberEnd(vis);
}
void reflectFile(JsonReader &vis, IndexFile &v) { reflect1(vis, v); }
void reflectFile(JsonWriter &vis, IndexFile &v) { reflect1(vis, v); }
void reflectFile(BinaryReader &vis, IndexFile &v) { reflect1(vis, v); }
void reflectFile(BinaryWriter &vis, IndexFile &v) { reflect1(vis, v); }
void reflect(JsonReader &vis, SerializeFormat &v) {
v = vis.getString()[0] == 'j' ? SerializeFormat::Json
: SerializeFormat::Binary;
}
void reflect(JsonWriter &vis, SerializeFormat &v) {
switch (v) {
case SerializeFormat::Binary:
vis.string("binary");
break;
case SerializeFormat::Json:
vis.string("json");
break;
}
}
void reflectMemberStart(JsonReader &vis) {
if (!vis.m->IsObject())
throw std::invalid_argument("object");
}
static BumpPtrAllocator alloc;
static DenseSet<CachedHashStringRef> strings;
static std::mutex allocMutex;
CachedHashStringRef internH(StringRef s) {
if (s.empty())
s = "";
CachedHashString hs(s);
std::lock_guard lock(allocMutex);
auto r = strings.insert(hs);
if (r.second) {
char *p = alloc.Allocate<char>(s.size() + 1);
memcpy(p, s.data(), s.size());
p[s.size()] = '\0';
*r.first = CachedHashStringRef(StringRef(p, s.size()), hs.hash());
}
return *r.first;
}
const char *intern(StringRef s) { return internH(s).val().data(); }
std::string serialize(SerializeFormat format, IndexFile &file) {
switch (format) {
case SerializeFormat::Binary: {
BinaryWriter writer;
int major = IndexFile::kMajorVersion;
int minor = IndexFile::kMinorVersion;
reflect(writer, major);
reflect(writer, minor);
reflectFile(writer, file);
return writer.take();
}
case SerializeFormat::Json: {
rapidjson::StringBuffer output;
rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(output);
writer.SetFormatOptions(
rapidjson::PrettyFormatOptions::kFormatSingleLineArray);
writer.SetIndent(' ', 2);
JsonWriter json_writer(&writer);
if (!gTestOutputMode) {
std::string version = std::to_string(IndexFile::kMajorVersion);
for (char c : version)
output.Put(c);
output.Put('\n');
}
reflectFile(json_writer, file);
return output.GetString();
}
}
return "";
}
std::unique_ptr<IndexFile>
deserialize(SerializeFormat format, const std::string &path,
const std::string &serialized_index_content,
const std::string &file_content,
std::optional<int> expected_version) {
if (serialized_index_content.empty())
return nullptr;
std::unique_ptr<IndexFile> file;
switch (format) {
case SerializeFormat::Binary: {
try {
int major, minor;
if (serialized_index_content.size() < 8)
throw std::invalid_argument("Invalid");
BinaryReader reader(serialized_index_content);
reflect(reader, major);
reflect(reader, minor);
if (major != IndexFile::kMajorVersion ||
minor != IndexFile::kMinorVersion)
throw std::invalid_argument("Invalid version");
file = std::make_unique<IndexFile>(path, file_content, false);
reflectFile(reader, *file);
} catch (std::invalid_argument &e) {
LOG_S(INFO) << "failed to deserialize '" << path << "': " << e.what();
return nullptr;
}
break;
}
case SerializeFormat::Json: {
rapidjson::Document reader;
if (gTestOutputMode || !expected_version) {
reader.Parse(serialized_index_content.c_str());
} else {
const char *p = strchr(serialized_index_content.c_str(), '\n');
if (!p)
return nullptr;
if (atoi(serialized_index_content.c_str()) != *expected_version)
return nullptr;
reader.Parse(p + 1);
}
if (reader.HasParseError())
return nullptr;
file = std::make_unique<IndexFile>(path, file_content, false);
JsonReader json_reader{&reader};
try {
reflectFile(json_reader, *file);
} catch (std::invalid_argument &e) {
LOG_S(INFO) << "'" << path << "': failed to deserialize "
<< json_reader.getPath() << "." << e.what();
return nullptr;
}
break;
}
}
// Restore non-serialized state.
file->path = path;
if (g_config->clang.pathMappings.size()) {
doPathMapping(file->import_file);
std::vector<const char *> args;
for (const char *arg : file->args) {
std::string s(arg);
doPathMapping(s);
args.push_back(intern(s));
}
file->args = std::move(args);
for (auto &[_, path] : file->lid2path)
doPathMapping(path);
for (auto &include : file->includes) {
std::string p(include.resolved_path);
doPathMapping(p);
include.resolved_path = intern(p);
}
decltype(file->dependencies) dependencies;
for (auto &it : file->dependencies) {
std::string path = it.first.val().str();
doPathMapping(path);
dependencies[internH(path)] = it.second;
}
file->dependencies = std::move(dependencies);
}
return file;
}
} // namespace ccls