From 0846c12e895124e93b5f969babfb3988c548670f Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 24 Mar 2019 23:42:00 -0700 Subject: [PATCH] Add initialization option index.initialNoLinkage: false By default, the background indexer doesn't handle names of no linkage. They are indexed when their files are opened. This saves memory and makes cache files smaller. --- src/config.hh | 18 ++++++++------ src/indexer.cc | 49 ++++++++++++++++++++------------------- src/indexer.hh | 6 +++-- src/messages/workspace.cc | 2 +- src/pipeline.cc | 29 ++++++++++++++--------- src/pipeline.hh | 2 +- src/project.cc | 6 ++--- src/query.cc | 2 +- src/serializer.cc | 5 ++-- src/test.cc | 3 ++- 10 files changed, 69 insertions(+), 53 deletions(-) diff --git a/src/config.hh b/src/config.hh index ccddb2ab..9f005001 100644 --- a/src/config.hh +++ b/src/config.hh @@ -255,9 +255,12 @@ struct Config { // - https://github.com/autozimu/LanguageClient-neovim/issues/224 int comments = 2; - // By default, all project entries will be indexed on initialization. Use - // these two options to exclude some. They can still be indexed after you - // open them. + // If false, names of no linkage are not indexed in the background. They are + // indexed after the files are opened. + bool initialNoLinkage = false; + + // Use the two options to exclude files that should not be indexed in the + // background. std::vector initialBlacklist; std::vector initialWhitelist; @@ -344,10 +347,11 @@ REFLECT_STRUCT(Config::Diagnostics, blacklist, onChange, onOpen, onSave, spellChecking, whitelist) REFLECT_STRUCT(Config::Highlight, largeFileSize, lsRanges, blacklist, whitelist) REFLECT_STRUCT(Config::Index::Name, suppressUnwrittenScope); -REFLECT_STRUCT(Config::Index, blacklist, comments, initialBlacklist, - initialWhitelist, maxInitializerLines, multiVersion, - multiVersionBlacklist, multiVersionWhitelist, name, onChange, - parametersInDeclarations, threads, trackDependency, whitelist); +REFLECT_STRUCT(Config::Index, blacklist, comments, initialNoLinkage, + initialBlacklist, initialWhitelist, maxInitializerLines, + multiVersion, multiVersionBlacklist, multiVersionWhitelist, name, + onChange, parametersInDeclarations, threads, trackDependency, + whitelist); REFLECT_STRUCT(Config::Request, timeout); REFLECT_STRUCT(Config::Session, maxNum); REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort); diff --git a/src/indexer.cc b/src/indexer.cc index f4e9be8b..82674998 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -62,7 +62,8 @@ struct IndexParam { VFS &vfs; ASTContext *Ctx; - IndexParam(VFS &vfs) : vfs(vfs) {} + bool no_linkage; + IndexParam(VFS &vfs, bool no_linkage) : vfs(vfs), no_linkage(no_linkage) {} void SeenFile(const FileEntry &File) { // If this is the first time we have seen the file (ignoring if we are @@ -78,9 +79,10 @@ struct IndexParam { if (std::optional content = ReadContent(path)) it->second.content = *content; - if (!vfs.Stamp(path, it->second.mtime, 1)) + if (!vfs.Stamp(path, it->second.mtime, no_linkage ? 3 : 1)) return; - it->second.db = std::make_unique(path, it->second.content); + it->second.db = + std::make_unique(path, it->second.content, no_linkage); } } @@ -693,6 +695,12 @@ public: bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles, ArrayRef Relations, SourceLocation Loc, ASTNodeInfo ASTNode) override { + if (!param.no_linkage) { + if (auto *ND = dyn_cast(D); ND && ND->hasLinkage()) + ; + else + return true; + } SourceManager &SM = Ctx->getSourceManager(); const LangOptions &Lang = Ctx->getLangOpts(); FileID LocFID; @@ -1166,11 +1174,12 @@ public: }; } // namespace -const int IndexFile::kMajorVersion = 20; +const int IndexFile::kMajorVersion = 21; const int IndexFile::kMinorVersion = 0; -IndexFile::IndexFile(const std::string &path, const std::string &contents) - : path(path), file_contents(contents) {} +IndexFile::IndexFile(const std::string &path, const std::string &contents, + bool no_linkage) + : path(path), no_linkage(no_linkage), file_contents(contents) {} IndexFunc &IndexFile::ToFunc(Usr usr) { auto [it, inserted] = usr2func.try_emplace(usr); @@ -1217,7 +1226,7 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs, const std::string &opt_wdir, const std::string &main, const std::vector &args, const std::vector> &remapped, - bool &ok) { + bool no_linkage, bool &ok) { ok = true; auto PCH = std::make_shared(); llvm::IntrusiveRefCntPtr FS = llvm::vfs::getRealFileSystem(); @@ -1231,17 +1240,6 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs, // code completion. if (g_config->index.comments > 1) CI->getLangOpts()->CommentOpts.ParseAllComments = true; - { - // FileSystemOptions& FSOpts = CI->getFileSystemOpts(); - // if (FSOpts.WorkingDir.empty()) - // FSOpts.WorkingDir = opt_wdir; - // HeaderSearchOptions &HSOpts = CI->getHeaderSearchOpts(); - // llvm::errs() << HSOpts.ResourceDir << "\n"; - // // lib/clang/7.0.0 is incorrect - // if (HSOpts.ResourceDir.compare(0, 3, "lib") == 0 && - // HSOpts.UseBuiltinIncludes) - // HSOpts.ResourceDir = g_config->clang.resourceDir; - } std::string buf = wfiles->GetContent(main); std::vector> Bufs; if (buf.size()) @@ -1260,19 +1258,22 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs, if (!Clang->hasTarget()) return {}; - IndexParam param(*vfs); + IndexParam param(*vfs, no_linkage); auto DataConsumer = std::make_shared(param); index::IndexingOptions IndexOpts; IndexOpts.SystemSymbolFilter = index::IndexingOptions::SystemSymbolFilterKind::All; - IndexOpts.IndexFunctionLocals = true; - IndexOpts.IndexImplicitInstantiation = true; + if (no_linkage) { + IndexOpts.IndexFunctionLocals = true; + IndexOpts.IndexImplicitInstantiation = true; #if LLVM_VERSION_MAJOR >= 9 - IndexOpts.IndexParametersInDeclarations = - g_config->index.parametersInDeclarations; - IndexOpts.IndexTemplateParameters = true; + + IndexOpts.IndexParametersInDeclarations = + g_config->index.parametersInDeclarations; + IndexOpts.IndexTemplateParameters = true; #endif + } std::unique_ptr Action = createIndexingAction( DataConsumer, IndexOpts, std::make_unique(param)); diff --git a/src/indexer.hh b/src/indexer.hh index 62401ab3..9796ea75 100644 --- a/src/indexer.hh +++ b/src/indexer.hh @@ -304,6 +304,7 @@ struct IndexFile { // This is unfortunately time_t as used by clang::FileEntry int64_t mtime = 0; LanguageId language = LanguageId::C; + bool no_linkage; // uid2lid_and_path is used to generate lid2path, but not serialized. std::unordered_map> @@ -328,7 +329,8 @@ struct IndexFile { // File contents at the time of index. Not serialized. std::string file_contents; - IndexFile(const std::string &path, const std::string &contents); + IndexFile(const std::string &path, const std::string &contents, + bool no_linkage); IndexFunc &ToFunc(Usr usr); IndexType &ToType(Usr usr); @@ -348,7 +350,7 @@ Index(SemaManager *complete, WorkingFiles *wfiles, VFS *vfs, const std::string &opt_wdir, const std::string &file, const std::vector &args, const std::vector> &remapped, - bool &ok); + bool all_linkages, bool &ok); } // namespace idx } // namespace ccls diff --git a/src/messages/workspace.cc b/src/messages/workspace.cc index 60ff01ae..86ce0b25 100644 --- a/src/messages/workspace.cc +++ b/src/messages/workspace.cc @@ -55,7 +55,7 @@ void MessageHandler::workspace_didChangeWatchedFiles( return; IndexMode mode = - wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::NonInteractive; + wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::Background; switch (event.type) { case FileChangeType::Created: case FileChangeType::Changed: { diff --git a/src/pipeline.cc b/src/pipeline.cc index be01b9d8..332f1dc2 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -228,18 +228,20 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, std::string path_to_index = entry.filename; std::unique_ptr prev; - bool deleted = false; + bool deleted = false, no_linkage = g_config->index.initialNoLinkage || + request.mode != IndexMode::Background; int reparse = 0; std::optional write_time = LastWriteTime(path_to_index); if (!write_time) { deleted = true; } else { - reparse = vfs->Stamp(path_to_index, *write_time, 0); + if (vfs->Stamp(path_to_index, *write_time, no_linkage ? 2 : 0)) + reparse = no_linkage ? 2 : 1; if (request.path != path_to_index) { std::optional mtime1 = LastWriteTime(request.path); if (!mtime1) deleted = true; - else if (vfs->Stamp(request.path, *mtime1, 0)) + else if (vfs->Stamp(request.path, *mtime1, no_linkage ? 2 : 0)) reparse = 2; } } @@ -292,10 +294,13 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, auto dependencies = prev->dependencies; IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); on_indexed->PushBack(std::move(update), - request.mode != IndexMode::NonInteractive); + request.mode != IndexMode::Background); { std::lock_guard lock1(vfs->mutex); - vfs->state[path_to_index].loaded++; + VFS::State &st = vfs->state[path_to_index]; + st.loaded++; + if (prev->no_linkage) + st.step = 2; } lock.unlock(); @@ -314,10 +319,12 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, continue; st.loaded++; st.timestamp = prev->mtime; + if (prev->no_linkage) + st.step = 3; } IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); on_indexed->PushBack(std::move(update), - request.mode != IndexMode::NonInteractive); + request.mode != IndexMode::Background); if (entry.id >= 0) { std::lock_guard lock2(project->mtx); project->root2folder[entry.root].path2entry_index[path] = entry.id; @@ -338,9 +345,9 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, std::vector> indexes; if (deleted) { - indexes.push_back(std::make_unique(request.path, "")); + indexes.push_back(std::make_unique(request.path, "", false)); if (request.path != path_to_index) - indexes.push_back(std::make_unique(path_to_index, "")); + indexes.push_back(std::make_unique(path_to_index, "", false)); } else { std::vector> remapped; if (g_config->index.onChange) { @@ -350,7 +357,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, } bool ok; indexes = idx::Index(completion, wfiles, vfs, entry.directory, - path_to_index, entry.args, remapped, ok); + path_to_index, entry.args, remapped, no_linkage, ok); if (!ok) { if (request.id.Valid()) { @@ -402,7 +409,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, } } on_indexed->PushBack(IndexUpdate::CreateDelta(prev.get(), curr.get()), - request.mode != IndexMode::NonInteractive); + request.mode != IndexMode::Background); { std::lock_guard lock1(vfs->mutex); vfs->state[path].loaded++; @@ -742,7 +749,7 @@ void Index(const std::string &path, const std::vector &args, IndexMode mode, bool must_exist, RequestId id) { pending_index_requests++; index_request->PushBack({path, args, mode, must_exist, id}, - mode != IndexMode::NonInteractive); + mode != IndexMode::Background); } void RemoveCache(const std::string &path) { diff --git a/src/pipeline.hh b/src/pipeline.hh index be881c06..2d0db15b 100644 --- a/src/pipeline.hh +++ b/src/pipeline.hh @@ -30,7 +30,7 @@ struct VFS { }; enum class IndexMode { - NonInteractive, + Background, OnChange, Normal, }; diff --git a/src/project.cc b/src/project.cc index 25bfa34c..10359515 100644 --- a/src/project.cc +++ b/src/project.cc @@ -576,7 +576,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) { bool interactive = wfiles->GetFile(entry.filename) != nullptr; pipeline::Index(entry.filename, entry.args, interactive ? IndexMode::Normal - : IndexMode::NonInteractive, + : IndexMode::Background, false, id); } else { LOG_V(1) << "[" << i << "/" << folder.entries.size() << "]: " << reason @@ -590,7 +590,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) { pipeline::loaded_ts = pipeline::tick; // Dummy request to indicate that project is loaded and // trigger refreshing semantic highlight for all working files. - pipeline::Index("", {}, IndexMode::NonInteractive, false); + pipeline::Index("", {}, IndexMode::Background, false); } void Project::IndexRelated(const std::string &path) { @@ -604,7 +604,7 @@ void Project::IndexRelated(const std::string &path) { std::string reason; if (sys::path::stem(entry.filename) == stem && entry.filename != path && match.Matches(entry.filename, &reason)) - pipeline::Index(entry.filename, entry.args, IndexMode::NonInteractive, + pipeline::Index(entry.filename, entry.args, IndexMode::Background, true); } break; diff --git a/src/query.cc b/src/query.cc index 6adfb424..ca2d228f 100644 --- a/src/query.cc +++ b/src/query.cc @@ -128,7 +128,7 @@ QueryType::Def Convert(const IndexType::Def &o) { IndexUpdate IndexUpdate::CreateDelta(IndexFile *previous, IndexFile *current) { IndexUpdate r; - static IndexFile empty(current->path, ""); + static IndexFile empty(current->path, "", false); if (previous) r.prev_lid2path = std::move(previous->lid2path); else diff --git a/src/serializer.cc b/src/serializer.cc index 0dec4710..a98f53cc 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -374,6 +374,7 @@ template void Reflect1(TVisitor &vis, IndexFile &v) { if (!gTestOutputMode) { REFLECT_MEMBER(mtime); REFLECT_MEMBER(language); + REFLECT_MEMBER(no_linkage); REFLECT_MEMBER(lid2path); REFLECT_MEMBER(import_file); REFLECT_MEMBER(args); @@ -482,7 +483,7 @@ Deserialize(SerializeFormat format, const std::string &path, if (major != IndexFile::kMajorVersion || minor != IndexFile::kMinorVersion) throw std::invalid_argument("Invalid version"); - file = std::make_unique(path, file_content); + file = std::make_unique(path, file_content, false); ReflectFile(reader, *file); } catch (std::invalid_argument &e) { LOG_S(INFO) << "failed to deserialize '" << path << "': " << e.what(); @@ -505,7 +506,7 @@ Deserialize(SerializeFormat format, const std::string &path, if (reader.HasParseError()) return nullptr; - file = std::make_unique(path, file_content); + file = std::make_unique(path, file_content, false); JsonReader json_reader{&reader}; try { ReflectFile(json_reader, *file); diff --git a/src/test.cc b/src/test.cc index 7faf948c..3215fc2b 100644 --- a/src/test.cc +++ b/src/test.cc @@ -327,7 +327,8 @@ bool RunIndexTests(const std::string &filter_path, bool enable_update) { for (auto &arg : flags) cargs.push_back(arg.c_str()); bool ok; - auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs, {}, ok); + auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs, + {}, true, ok); for (const auto &entry : all_expected_output) { const std::string &expected_path = entry.first;