Add initialization option index.initialNoLinkage: false

By default, the background indexer doesn't handle names of no linkage.
They are indexed when their files are opened. This saves memory and
makes cache files smaller.

src/config.hh

@@ -255,9 +255,12 @@ struct Config {
     // - https://github.com/autozimu/LanguageClient-neovim/issues/224
     int comments = 2;
 
-    // By default, all project entries will be indexed on initialization. Use
+    // If false, names of no linkage are not indexed in the background. They are
-    // these two options to exclude some. They can still be indexed after you
+    // indexed after the files are opened.
-    // open them.
+    bool initialNoLinkage = false;
+
+    // Use the two options to exclude files that should not be indexed in the
+    // background.
     std::vector<std::string> initialBlacklist;
     std::vector<std::string> initialWhitelist;
 
@@ -344,10 +347,11 @@ REFLECT_STRUCT(Config::Diagnostics, blacklist, onChange, onOpen, onSave,
                spellChecking, whitelist)
 REFLECT_STRUCT(Config::Highlight, largeFileSize, lsRanges, blacklist, whitelist)
 REFLECT_STRUCT(Config::Index::Name, suppressUnwrittenScope);
-REFLECT_STRUCT(Config::Index, blacklist, comments, initialBlacklist,
+REFLECT_STRUCT(Config::Index, blacklist, comments, initialNoLinkage,
-               initialWhitelist, maxInitializerLines, multiVersion,
+               initialBlacklist, initialWhitelist, maxInitializerLines,
-               multiVersionBlacklist, multiVersionWhitelist, name, onChange,
+               multiVersion, multiVersionBlacklist, multiVersionWhitelist, name,
-               parametersInDeclarations, threads, trackDependency, whitelist);
+               onChange, parametersInDeclarations, threads, trackDependency,
+               whitelist);
 REFLECT_STRUCT(Config::Request, timeout);
 REFLECT_STRUCT(Config::Session, maxNum);
 REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort);

src/indexer.cc

@@ -62,7 +62,8 @@ struct IndexParam {
 
   VFS &vfs;
   ASTContext *Ctx;
-  IndexParam(VFS &vfs) : vfs(vfs) {}
+  bool no_linkage;
+  IndexParam(VFS &vfs, bool no_linkage) : vfs(vfs), no_linkage(no_linkage) {}
 
   void SeenFile(const FileEntry &File) {
     // If this is the first time we have seen the file (ignoring if we are
@@ -78,9 +79,10 @@ struct IndexParam {
       if (std::optional<std::string> content = ReadContent(path))
         it->second.content = *content;
 
-      if (!vfs.Stamp(path, it->second.mtime, 1))
+      if (!vfs.Stamp(path, it->second.mtime, no_linkage ? 3 : 1))
         return;
-      it->second.db = std::make_unique<IndexFile>(path, it->second.content);
+      it->second.db =
+          std::make_unique<IndexFile>(path, it->second.content, no_linkage);
     }
   }
 
@@ -693,6 +695,12 @@ public:
   bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles,
                            ArrayRef<index::SymbolRelation> Relations,
                            SourceLocation Loc, ASTNodeInfo ASTNode) override {
+    if (!param.no_linkage) {
+      if (auto *ND = dyn_cast<NamedDecl>(D); ND && ND->hasLinkage())
+        ;
+      else
+        return true;
+    }
     SourceManager &SM = Ctx->getSourceManager();
     const LangOptions &Lang = Ctx->getLangOpts();
     FileID LocFID;
@@ -1166,11 +1174,12 @@ public:
 };
 } // namespace
 
-const int IndexFile::kMajorVersion = 20;
+const int IndexFile::kMajorVersion = 21;
 const int IndexFile::kMinorVersion = 0;
 
-IndexFile::IndexFile(const std::string &path, const std::string &contents)
+IndexFile::IndexFile(const std::string &path, const std::string &contents,
-    : path(path), file_contents(contents) {}
+                     bool no_linkage)
+    : path(path), no_linkage(no_linkage), file_contents(contents) {}
 
 IndexFunc &IndexFile::ToFunc(Usr usr) {
   auto [it, inserted] = usr2func.try_emplace(usr);
@@ -1217,7 +1226,7 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
       const std::string &opt_wdir, const std::string &main,
       const std::vector<const char *> &args,
       const std::vector<std::pair<std::string, std::string>> &remapped,
-      bool &ok) {
+      bool no_linkage, bool &ok) {
   ok = true;
   auto PCH = std::make_shared<PCHContainerOperations>();
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = llvm::vfs::getRealFileSystem();
@@ -1231,17 +1240,6 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
   // code completion.
   if (g_config->index.comments > 1)
     CI->getLangOpts()->CommentOpts.ParseAllComments = true;
-  {
-    // FileSystemOptions& FSOpts = CI->getFileSystemOpts();
-    // if (FSOpts.WorkingDir.empty())
-    //   FSOpts.WorkingDir = opt_wdir;
-    // HeaderSearchOptions &HSOpts = CI->getHeaderSearchOpts();
-    // llvm::errs() << HSOpts.ResourceDir << "\n";
-    // // lib/clang/7.0.0 is incorrect
-    // if (HSOpts.ResourceDir.compare(0, 3, "lib") == 0 &&
-    //     HSOpts.UseBuiltinIncludes)
-    //   HSOpts.ResourceDir = g_config->clang.resourceDir;
-  }
   std::string buf = wfiles->GetContent(main);
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Bufs;
   if (buf.size())
@@ -1260,19 +1258,22 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
   if (!Clang->hasTarget())
     return {};
 
-  IndexParam param(*vfs);
+  IndexParam param(*vfs, no_linkage);
   auto DataConsumer = std::make_shared<IndexDataConsumer>(param);
 
   index::IndexingOptions IndexOpts;
   IndexOpts.SystemSymbolFilter =
       index::IndexingOptions::SystemSymbolFilterKind::All;
-  IndexOpts.IndexFunctionLocals = true;
+  if (no_linkage) {
-  IndexOpts.IndexImplicitInstantiation = true;
+    IndexOpts.IndexFunctionLocals = true;
+    IndexOpts.IndexImplicitInstantiation = true;
 #if LLVM_VERSION_MAJOR >= 9
-  IndexOpts.IndexParametersInDeclarations =
+
-      g_config->index.parametersInDeclarations;
+    IndexOpts.IndexParametersInDeclarations =
-  IndexOpts.IndexTemplateParameters = true;
+        g_config->index.parametersInDeclarations;
+    IndexOpts.IndexTemplateParameters = true;
 #endif
+  }
 
   std::unique_ptr<FrontendAction> Action = createIndexingAction(
       DataConsumer, IndexOpts, std::make_unique<IndexFrontendAction>(param));

src/indexer.hh

@@ -304,6 +304,7 @@ struct IndexFile {
   // This is unfortunately time_t as used by clang::FileEntry
   int64_t mtime = 0;
   LanguageId language = LanguageId::C;
+  bool no_linkage;
 
   // uid2lid_and_path is used to generate lid2path, but not serialized.
   std::unordered_map<llvm::sys::fs::UniqueID, std::pair<int, std::string>>
@@ -328,7 +329,8 @@ struct IndexFile {
   // File contents at the time of index. Not serialized.
   std::string file_contents;
 
-  IndexFile(const std::string &path, const std::string &contents);
+  IndexFile(const std::string &path, const std::string &contents,
+            bool no_linkage);
 
   IndexFunc &ToFunc(Usr usr);
   IndexType &ToType(Usr usr);
@@ -348,7 +350,7 @@ Index(SemaManager *complete, WorkingFiles *wfiles, VFS *vfs,
       const std::string &opt_wdir, const std::string &file,
       const std::vector<const char *> &args,
       const std::vector<std::pair<std::string, std::string>> &remapped,
-      bool &ok);
+      bool all_linkages, bool &ok);
 } // namespace idx
 } // namespace ccls
 

src/messages/workspace.cc

@@ -55,7 +55,7 @@ void MessageHandler::workspace_didChangeWatchedFiles(
         return;
 
     IndexMode mode =
-        wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::NonInteractive;
+        wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::Background;
     switch (event.type) {
     case FileChangeType::Created:
     case FileChangeType::Changed: {

src/pipeline.cc

@@ -228,18 +228,20 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
   std::string path_to_index = entry.filename;
   std::unique_ptr<IndexFile> prev;
 
-  bool deleted = false;
+  bool deleted = false, no_linkage = g_config->index.initialNoLinkage ||
+                                     request.mode != IndexMode::Background;
   int reparse = 0;
   std::optional<int64_t> write_time = LastWriteTime(path_to_index);
   if (!write_time) {
     deleted = true;
   } else {
-    reparse = vfs->Stamp(path_to_index, *write_time, 0);
+    if (vfs->Stamp(path_to_index, *write_time, no_linkage ? 2 : 0))
+      reparse = no_linkage ? 2 : 1;
     if (request.path != path_to_index) {
       std::optional<int64_t> mtime1 = LastWriteTime(request.path);
       if (!mtime1)
         deleted = true;
-      else if (vfs->Stamp(request.path, *mtime1, 0))
+      else if (vfs->Stamp(request.path, *mtime1, no_linkage ? 2 : 0))
         reparse = 2;
     }
   }
@@ -292,10 +294,13 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
       auto dependencies = prev->dependencies;
       IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
       on_indexed->PushBack(std::move(update),
-        request.mode != IndexMode::NonInteractive);
+                           request.mode != IndexMode::Background);
       {
         std::lock_guard lock1(vfs->mutex);
-        vfs->state[path_to_index].loaded++;
+        VFS::State &st = vfs->state[path_to_index];
+        st.loaded++;
+        if (prev->no_linkage)
+          st.step = 2;
       }
       lock.unlock();
 
@@ -314,10 +319,12 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
             continue;
           st.loaded++;
           st.timestamp = prev->mtime;
+          if (prev->no_linkage)
+            st.step = 3;
         }
         IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
         on_indexed->PushBack(std::move(update),
-          request.mode != IndexMode::NonInteractive);
+                             request.mode != IndexMode::Background);
         if (entry.id >= 0) {
           std::lock_guard lock2(project->mtx);
           project->root2folder[entry.root].path2entry_index[path] = entry.id;
@@ -338,9 +345,9 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
 
   std::vector<std::unique_ptr<IndexFile>> indexes;
   if (deleted) {
-    indexes.push_back(std::make_unique<IndexFile>(request.path, ""));
+    indexes.push_back(std::make_unique<IndexFile>(request.path, "", false));
     if (request.path != path_to_index)
-      indexes.push_back(std::make_unique<IndexFile>(path_to_index, ""));
+      indexes.push_back(std::make_unique<IndexFile>(path_to_index, "", false));
   } else {
     std::vector<std::pair<std::string, std::string>> remapped;
     if (g_config->index.onChange) {
@@ -350,7 +357,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
     }
     bool ok;
     indexes = idx::Index(completion, wfiles, vfs, entry.directory,
-                         path_to_index, entry.args, remapped, ok);
+                         path_to_index, entry.args, remapped, no_linkage, ok);
 
     if (!ok) {
       if (request.id.Valid()) {
@@ -402,7 +409,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
         }
       }
       on_indexed->PushBack(IndexUpdate::CreateDelta(prev.get(), curr.get()),
-                           request.mode != IndexMode::NonInteractive);
+                           request.mode != IndexMode::Background);
       {
         std::lock_guard lock1(vfs->mutex);
         vfs->state[path].loaded++;
@@ -742,7 +749,7 @@ void Index(const std::string &path, const std::vector<const char *> &args,
            IndexMode mode, bool must_exist, RequestId id) {
   pending_index_requests++;
   index_request->PushBack({path, args, mode, must_exist, id},
-                          mode != IndexMode::NonInteractive);
+                          mode != IndexMode::Background);
 }
 
 void RemoveCache(const std::string &path) {

src/pipeline.hh

@@ -30,7 +30,7 @@ struct VFS {
 };
 
 enum class IndexMode {
-  NonInteractive,
+  Background,
   OnChange,
   Normal,
 };

src/project.cc

@@ -576,7 +576,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) {
           bool interactive = wfiles->GetFile(entry.filename) != nullptr;
           pipeline::Index(entry.filename, entry.args,
                           interactive ? IndexMode::Normal
-                                      : IndexMode::NonInteractive,
+                                      : IndexMode::Background,
                           false, id);
         } else {
           LOG_V(1) << "[" << i << "/" << folder.entries.size() << "]: " << reason
@@ -590,7 +590,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) {
   pipeline::loaded_ts = pipeline::tick;
   // Dummy request to indicate that project is loaded and
   // trigger refreshing semantic highlight for all working files.
-  pipeline::Index("", {}, IndexMode::NonInteractive, false);
+  pipeline::Index("", {}, IndexMode::Background, false);
 }
 
 void Project::IndexRelated(const std::string &path) {
@@ -604,7 +604,7 @@ void Project::IndexRelated(const std::string &path) {
         std::string reason;
         if (sys::path::stem(entry.filename) == stem && entry.filename != path &&
             match.Matches(entry.filename, &reason))
-          pipeline::Index(entry.filename, entry.args, IndexMode::NonInteractive,
+          pipeline::Index(entry.filename, entry.args, IndexMode::Background,
                           true);
       }
       break;

src/query.cc

@@ -128,7 +128,7 @@ QueryType::Def Convert(const IndexType::Def &o) {
 
 IndexUpdate IndexUpdate::CreateDelta(IndexFile *previous, IndexFile *current) {
   IndexUpdate r;
-  static IndexFile empty(current->path, "<empty>");
+  static IndexFile empty(current->path, "<empty>", false);
   if (previous)
     r.prev_lid2path = std::move(previous->lid2path);
   else

src/serializer.cc

@@ -374,6 +374,7 @@ template <typename TVisitor> void Reflect1(TVisitor &vis, IndexFile &v) {
   if (!gTestOutputMode) {
     REFLECT_MEMBER(mtime);
     REFLECT_MEMBER(language);
+    REFLECT_MEMBER(no_linkage);
     REFLECT_MEMBER(lid2path);
     REFLECT_MEMBER(import_file);
     REFLECT_MEMBER(args);
@@ -482,7 +483,7 @@ Deserialize(SerializeFormat format, const std::string &path,
       if (major != IndexFile::kMajorVersion ||
           minor != IndexFile::kMinorVersion)
         throw std::invalid_argument("Invalid version");
-      file = std::make_unique<IndexFile>(path, file_content);
+      file = std::make_unique<IndexFile>(path, file_content, false);
       ReflectFile(reader, *file);
     } catch (std::invalid_argument &e) {
       LOG_S(INFO) << "failed to deserialize '" << path << "': " << e.what();
@@ -505,7 +506,7 @@ Deserialize(SerializeFormat format, const std::string &path,
     if (reader.HasParseError())
       return nullptr;
 
-    file = std::make_unique<IndexFile>(path, file_content);
+    file = std::make_unique<IndexFile>(path, file_content, false);
     JsonReader json_reader{&reader};
     try {
       ReflectFile(json_reader, *file);

src/test.cc

@@ -327,7 +327,8 @@ bool RunIndexTests(const std::string &filter_path, bool enable_update) {
         for (auto &arg : flags)
           cargs.push_back(arg.c_str());
         bool ok;
-        auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs, {}, ok);
+        auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs,
+                                    {}, true, ok);
 
         for (const auto &entry : all_expected_output) {
           const std::string &expected_path = entry.first;