From 17eaca8a024782fb1c3001dade8e80c9ac81fc3e Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 24 Mar 2019 23:42:00 -0700 Subject: [PATCH] Add initialization option index.initialNoLinkage: false By default, the background indexer doesn't handle names of no linkage. They are indexed when their files are opened. This saves memory and makes cache files smaller. --- src/config.hh | 18 +++++++++++------- src/indexer.cc | 38 +++++++++++++++++++++++++------------- src/indexer.hh | 6 ++++-- src/messages/workspace.cc | 2 +- src/pipeline.cc | 34 +++++++++++++++++++++------------- src/pipeline.hh | 2 +- src/project.cc | 6 +++--- src/query.cc | 2 +- src/serializer.cc | 5 +++-- src/test.cc | 3 ++- 10 files changed, 72 insertions(+), 44 deletions(-) diff --git a/src/config.hh b/src/config.hh index 085217f1..e27ca902 100644 --- a/src/config.hh +++ b/src/config.hh @@ -243,9 +243,12 @@ struct Config { // - https://github.com/autozimu/LanguageClient-neovim/issues/224 int comments = 2; - // By default, all project entries will be indexed on initialization. Use - // these two options to exclude some. They can still be indexed after you - // open them. + // If false, names of no linkage are not indexed in the background. They are + // indexed after the files are opened. + bool initialNoLinkage = false; + + // Use the two options to exclude files that should not be indexed in the + // background. std::vector initialBlacklist; std::vector initialWhitelist; @@ -332,10 +335,11 @@ REFLECT_STRUCT(Config::Diagnostics, blacklist, onChange, onOpen, onSave, spellChecking, whitelist) REFLECT_STRUCT(Config::Highlight, largeFileSize, lsRanges, blacklist, whitelist) REFLECT_STRUCT(Config::Index::Name, suppressUnwrittenScope); -REFLECT_STRUCT(Config::Index, blacklist, comments, initialBlacklist, - initialWhitelist, maxInitializerLines, multiVersion, - multiVersionBlacklist, multiVersionWhitelist, name, onChange, - parametersInDeclarations, threads, trackDependency, whitelist); +REFLECT_STRUCT(Config::Index, blacklist, comments, initialNoLinkage, + initialBlacklist, initialWhitelist, maxInitializerLines, + multiVersion, multiVersionBlacklist, multiVersionWhitelist, name, + onChange, parametersInDeclarations, threads, trackDependency, + whitelist); REFLECT_STRUCT(Config::Request, timeout); REFLECT_STRUCT(Config::Session, maxNum); REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort); diff --git a/src/indexer.cc b/src/indexer.cc index 70a5bfce..bdeead98 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -50,7 +50,8 @@ struct IndexParam { VFS &vfs; ASTContext *Ctx; - IndexParam(VFS &vfs) : vfs(vfs) {} + bool no_linkage; + IndexParam(VFS &vfs, bool no_linkage) : vfs(vfs), no_linkage(no_linkage) {} void SeenFile(const FileEntry &File) { // If this is the first time we have seen the file (ignoring if we are @@ -66,9 +67,10 @@ struct IndexParam { if (std::optional content = ReadContent(path)) it->second.content = *content; - if (!vfs.Stamp(path, it->second.mtime, 1)) + if (!vfs.Stamp(path, it->second.mtime, no_linkage ? 3 : 1)) return; - it->second.db = std::make_unique(path, it->second.content); + it->second.db = + std::make_unique(path, it->second.content, no_linkage); } } @@ -679,6 +681,12 @@ public: bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles, ArrayRef Relations, SourceLocation Loc, ASTNodeInfo ASTNode) override { + if (!param.no_linkage) { + if (auto *ND = dyn_cast(D); ND && ND->hasLinkage()) + ; + else + return true; + } SourceManager &SM = Ctx->getSourceManager(); const LangOptions &Lang = Ctx->getLangOpts(); FileID LocFID; @@ -1153,11 +1161,12 @@ public: }; } // namespace -const int IndexFile::kMajorVersion = 20; +const int IndexFile::kMajorVersion = 21; const int IndexFile::kMinorVersion = 0; -IndexFile::IndexFile(const std::string &path, const std::string &contents) - : path(path), file_contents(contents) {} +IndexFile::IndexFile(const std::string &path, const std::string &contents, + bool no_linkage) + : path(path), no_linkage(no_linkage), file_contents(contents) {} IndexFunc &IndexFile::ToFunc(Usr usr) { auto [it, inserted] = usr2func.try_emplace(usr); @@ -1204,7 +1213,7 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs, const std::string &opt_wdir, const std::string &main, const std::vector &args, const std::vector> &remapped, - bool &ok) { + bool no_linkage, bool &ok) { ok = true; auto PCH = std::make_shared(); llvm::IntrusiveRefCntPtr FS = llvm::vfs::getRealFileSystem(); @@ -1237,19 +1246,22 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs, if (!Clang->hasTarget()) return {}; - IndexParam param(*vfs); + IndexParam param(*vfs, no_linkage); auto DataConsumer = std::make_shared(param); index::IndexingOptions IndexOpts; IndexOpts.SystemSymbolFilter = index::IndexingOptions::SystemSymbolFilterKind::All; - IndexOpts.IndexFunctionLocals = true; - IndexOpts.IndexImplicitInstantiation = true; + if (no_linkage) { + IndexOpts.IndexFunctionLocals = true; + IndexOpts.IndexImplicitInstantiation = true; #if LLVM_VERSION_MAJOR >= 9 - IndexOpts.IndexParametersInDeclarations = - g_config->index.parametersInDeclarations; - IndexOpts.IndexTemplateParameters = true; + + IndexOpts.IndexParametersInDeclarations = + g_config->index.parametersInDeclarations; + IndexOpts.IndexTemplateParameters = true; #endif + } std::unique_ptr Action = createIndexingAction( DataConsumer, IndexOpts, std::make_unique(param)); diff --git a/src/indexer.hh b/src/indexer.hh index 83c74005..ae5f4558 100644 --- a/src/indexer.hh +++ b/src/indexer.hh @@ -292,6 +292,7 @@ struct IndexFile { // This is unfortunately time_t as used by clang::FileEntry int64_t mtime = 0; LanguageId language = LanguageId::C; + bool no_linkage; // uid2lid_and_path is used to generate lid2path, but not serialized. std::unordered_map> @@ -316,7 +317,8 @@ struct IndexFile { // File contents at the time of index. Not serialized. std::string file_contents; - IndexFile(const std::string &path, const std::string &contents); + IndexFile(const std::string &path, const std::string &contents, + bool no_linkage); IndexFunc &ToFunc(Usr usr); IndexType &ToType(Usr usr); @@ -336,7 +338,7 @@ Index(SemaManager *complete, WorkingFiles *wfiles, VFS *vfs, const std::string &opt_wdir, const std::string &file, const std::vector &args, const std::vector> &remapped, - bool &ok); + bool all_linkages, bool &ok); } // namespace idx } // namespace ccls diff --git a/src/messages/workspace.cc b/src/messages/workspace.cc index 9468478c..0a17d592 100644 --- a/src/messages/workspace.cc +++ b/src/messages/workspace.cc @@ -43,7 +43,7 @@ void MessageHandler::workspace_didChangeWatchedFiles( return; IndexMode mode = - wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::NonInteractive; + wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::Background; switch (event.type) { case FileChangeType::Created: case FileChangeType::Changed: { diff --git a/src/pipeline.cc b/src/pipeline.cc index b74e1ff5..9bbddf14 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -216,18 +216,20 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, std::string path_to_index = entry.filename; std::unique_ptr prev; - bool deleted = false; + bool deleted = false, no_linkage = g_config->index.initialNoLinkage || + request.mode != IndexMode::Background; int reparse = 0; std::optional write_time = LastWriteTime(path_to_index); if (!write_time) { deleted = true; } else { - reparse = vfs->Stamp(path_to_index, *write_time, 0); + if (vfs->Stamp(path_to_index, *write_time, no_linkage ? 2 : 0)) + reparse = 1; if (request.path != path_to_index) { std::optional mtime1 = LastWriteTime(request.path); if (!mtime1) deleted = true; - else if (vfs->Stamp(request.path, *mtime1, 0)) + else if (vfs->Stamp(request.path, *mtime1, no_linkage ? 2 : 0)) reparse = 2; } } @@ -250,8 +252,9 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, do { std::unique_lock lock(GetFileMutex(path_to_index)); prev = RawCacheLoad(path_to_index); - if (!prev || CacheInvalid(vfs, prev.get(), path_to_index, entry.args, - std::nullopt)) + if (!prev || prev->no_linkage < no_linkage || + CacheInvalid(vfs, prev.get(), path_to_index, entry.args, + std::nullopt)) break; if (track) for (const auto &dep : prev->dependencies) { @@ -280,10 +283,13 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, auto dependencies = prev->dependencies; IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); on_indexed->PushBack(std::move(update), - request.mode != IndexMode::NonInteractive); + request.mode != IndexMode::Background); { std::lock_guard lock1(vfs->mutex); - vfs->state[path_to_index].loaded++; + VFS::State &st = vfs->state[path_to_index]; + st.loaded++; + if (prev->no_linkage) + st.step = 2; } lock.unlock(); @@ -302,10 +308,12 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, continue; st.loaded++; st.timestamp = prev->mtime; + if (prev->no_linkage) + st.step = 3; } IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); on_indexed->PushBack(std::move(update), - request.mode != IndexMode::NonInteractive); + request.mode != IndexMode::Background); if (entry.id >= 0) { std::lock_guard lock2(project->mtx); project->root2folder[entry.root].path2entry_index[path] = entry.id; @@ -326,9 +334,9 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, std::vector> indexes; if (deleted) { - indexes.push_back(std::make_unique(request.path, "")); + indexes.push_back(std::make_unique(request.path, "", false)); if (request.path != path_to_index) - indexes.push_back(std::make_unique(path_to_index, "")); + indexes.push_back(std::make_unique(path_to_index, "", false)); } else { std::vector> remapped; if (g_config->index.onChange) { @@ -338,7 +346,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, } bool ok; indexes = idx::Index(completion, wfiles, vfs, entry.directory, - path_to_index, entry.args, remapped, ok); + path_to_index, entry.args, remapped, no_linkage, ok); if (!ok) { if (request.id.Valid()) { @@ -390,7 +398,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles, } } on_indexed->PushBack(IndexUpdate::CreateDelta(prev.get(), curr.get()), - request.mode != IndexMode::NonInteractive); + request.mode != IndexMode::Background); { std::lock_guard lock1(vfs->mutex); vfs->state[path].loaded++; @@ -730,7 +738,7 @@ void Index(const std::string &path, const std::vector &args, IndexMode mode, bool must_exist, RequestId id) { pending_index_requests++; index_request->PushBack({path, args, mode, must_exist, id}, - mode != IndexMode::NonInteractive); + mode != IndexMode::Background); } void RemoveCache(const std::string &path) { diff --git a/src/pipeline.hh b/src/pipeline.hh index 5dd35c08..7c6c64e9 100644 --- a/src/pipeline.hh +++ b/src/pipeline.hh @@ -33,7 +33,7 @@ struct VFS { }; enum class IndexMode { - NonInteractive, + Background, OnChange, Normal, }; diff --git a/src/project.cc b/src/project.cc index 61417257..32637f65 100644 --- a/src/project.cc +++ b/src/project.cc @@ -564,7 +564,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) { bool interactive = wfiles->GetFile(entry.filename) != nullptr; pipeline::Index(entry.filename, entry.args, interactive ? IndexMode::Normal - : IndexMode::NonInteractive, + : IndexMode::Background, false, id); } else { LOG_V(1) << "[" << i << "/" << folder.entries.size() << "]: " << reason @@ -578,7 +578,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) { pipeline::loaded_ts = pipeline::tick; // Dummy request to indicate that project is loaded and // trigger refreshing semantic highlight for all working files. - pipeline::Index("", {}, IndexMode::NonInteractive, false); + pipeline::Index("", {}, IndexMode::Background, false); } void Project::IndexRelated(const std::string &path) { @@ -592,7 +592,7 @@ void Project::IndexRelated(const std::string &path) { std::string reason; if (sys::path::stem(entry.filename) == stem && entry.filename != path && match.Matches(entry.filename, &reason)) - pipeline::Index(entry.filename, entry.args, IndexMode::NonInteractive, + pipeline::Index(entry.filename, entry.args, IndexMode::Background, true); } break; diff --git a/src/query.cc b/src/query.cc index 44bfc1ad..d51244ba 100644 --- a/src/query.cc +++ b/src/query.cc @@ -116,7 +116,7 @@ QueryType::Def Convert(const IndexType::Def &o) { IndexUpdate IndexUpdate::CreateDelta(IndexFile *previous, IndexFile *current) { IndexUpdate r; - static IndexFile empty(current->path, ""); + static IndexFile empty(current->path, "", false); if (previous) r.prev_lid2path = std::move(previous->lid2path); else diff --git a/src/serializer.cc b/src/serializer.cc index aba9342c..034a552f 100644 --- a/src/serializer.cc +++ b/src/serializer.cc @@ -362,6 +362,7 @@ template void Reflect1(TVisitor &vis, IndexFile &v) { if (!gTestOutputMode) { REFLECT_MEMBER(mtime); REFLECT_MEMBER(language); + REFLECT_MEMBER(no_linkage); REFLECT_MEMBER(lid2path); REFLECT_MEMBER(import_file); REFLECT_MEMBER(args); @@ -470,7 +471,7 @@ Deserialize(SerializeFormat format, const std::string &path, if (major != IndexFile::kMajorVersion || minor != IndexFile::kMinorVersion) throw std::invalid_argument("Invalid version"); - file = std::make_unique(path, file_content); + file = std::make_unique(path, file_content, false); ReflectFile(reader, *file); } catch (std::invalid_argument &e) { LOG_S(INFO) << "failed to deserialize '" << path << "': " << e.what(); @@ -493,7 +494,7 @@ Deserialize(SerializeFormat format, const std::string &path, if (reader.HasParseError()) return nullptr; - file = std::make_unique(path, file_content); + file = std::make_unique(path, file_content, false); JsonReader json_reader{&reader}; try { ReflectFile(json_reader, *file); diff --git a/src/test.cc b/src/test.cc index 261da2c2..d245aab4 100644 --- a/src/test.cc +++ b/src/test.cc @@ -315,7 +315,8 @@ bool RunIndexTests(const std::string &filter_path, bool enable_update) { for (auto &arg : flags) cargs.push_back(arg.c_str()); bool ok; - auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs, {}, ok); + auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs, + {}, true, ok); for (const auto &entry : all_expected_output) { const std::string &expected_path = entry.first;