Add initialization option index.initialNoLinkage: false

By default, the background indexer doesn't handle names of no linkage.
They are indexed when their files are opened. This saves memory and
makes cache files smaller.

src/config.hh

@@ -255,9 +255,12 @@ struct Config {
     // - https://github.com/autozimu/LanguageClient-neovim/issues/224
     int comments = 2;
 
-    // By default, all project entries will be indexed on initialization. Use
-    // these two options to exclude some. They can still be indexed after you
-    // open them.
+    // If false, names of no linkage are not indexed in the background. They are
+    // indexed after the files are opened.
+    bool initialNoLinkage = false;
+
+    // Use the two options to exclude files that should not be indexed in the
+    // background.
     std::vector<std::string> initialBlacklist;
     std::vector<std::string> initialWhitelist;
 
@@ -344,10 +347,11 @@ REFLECT_STRUCT(Config::Diagnostics, blacklist, onChange, onOpen, onSave,
                spellChecking, whitelist)
 REFLECT_STRUCT(Config::Highlight, largeFileSize, lsRanges, blacklist, whitelist)
 REFLECT_STRUCT(Config::Index::Name, suppressUnwrittenScope);
-REFLECT_STRUCT(Config::Index, blacklist, comments, initialBlacklist,
-               initialWhitelist, maxInitializerLines, multiVersion,
-               multiVersionBlacklist, multiVersionWhitelist, name, onChange,
-               parametersInDeclarations, threads, trackDependency, whitelist);
+REFLECT_STRUCT(Config::Index, blacklist, comments, initialNoLinkage,
+               initialBlacklist, initialWhitelist, maxInitializerLines,
+               multiVersion, multiVersionBlacklist, multiVersionWhitelist, name,
+               onChange, parametersInDeclarations, threads, trackDependency,
+               whitelist);
 REFLECT_STRUCT(Config::Request, timeout);
 REFLECT_STRUCT(Config::Session, maxNum);
 REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort);

src/indexer.cc

@@ -62,7 +62,8 @@ struct IndexParam {
 
   VFS &vfs;
   ASTContext *Ctx;
-  IndexParam(VFS &vfs) : vfs(vfs) {}
+  bool no_linkage;
+  IndexParam(VFS &vfs, bool no_linkage) : vfs(vfs), no_linkage(no_linkage) {}
 
   void SeenFile(const FileEntry &File) {
     // If this is the first time we have seen the file (ignoring if we are
@@ -78,7 +79,7 @@ struct IndexParam {
       if (std::optional<std::string> content = ReadContent(path))
         it->second.content = *content;
 
-      if (!vfs.Stamp(path, it->second.mtime, 1))
+      if (!vfs.Stamp(path, it->second.mtime, no_linkage ? 3 : 1))
         return;
       it->second.db = std::make_unique<IndexFile>(path, it->second.content);
     }
@@ -693,6 +694,12 @@ public:
   bool handleDeclOccurence(const Decl *D, index::SymbolRoleSet Roles,
                            ArrayRef<index::SymbolRelation> Relations,
                            SourceLocation Loc, ASTNodeInfo ASTNode) override {
+    if (!param.no_linkage) {
+      if (auto *ND = dyn_cast<NamedDecl>(D); ND && ND->hasLinkage())
+        ;
+      else
+        return true;
+    }
     SourceManager &SM = Ctx->getSourceManager();
     const LangOptions &Lang = Ctx->getLangOpts();
     FileID LocFID;
@@ -1217,7 +1224,7 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
       const std::string &opt_wdir, const std::string &main,
       const std::vector<const char *> &args,
       const std::vector<std::pair<std::string, std::string>> &remapped,
-      bool &ok) {
+      bool no_linkage, bool &ok) {
   ok = true;
   auto PCH = std::make_shared<PCHContainerOperations>();
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS = llvm::vfs::getRealFileSystem();
@@ -1231,17 +1238,6 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
   // code completion.
   if (g_config->index.comments > 1)
     CI->getLangOpts()->CommentOpts.ParseAllComments = true;
-  {
-    // FileSystemOptions& FSOpts = CI->getFileSystemOpts();
-    // if (FSOpts.WorkingDir.empty())
-    //   FSOpts.WorkingDir = opt_wdir;
-    // HeaderSearchOptions &HSOpts = CI->getHeaderSearchOpts();
-    // llvm::errs() << HSOpts.ResourceDir << "\n";
-    // // lib/clang/7.0.0 is incorrect
-    // if (HSOpts.ResourceDir.compare(0, 3, "lib") == 0 &&
-    //     HSOpts.UseBuiltinIncludes)
-    //   HSOpts.ResourceDir = g_config->clang.resourceDir;
-  }
   std::string buf = wfiles->GetContent(main);
   std::vector<std::unique_ptr<llvm::MemoryBuffer>> Bufs;
   if (buf.size())
@@ -1260,19 +1256,21 @@ Index(SemaManager *manager, WorkingFiles *wfiles, VFS *vfs,
   if (!Clang->hasTarget())
     return {};
 
-  IndexParam param(*vfs);
+  IndexParam param(*vfs, no_linkage);
   auto DataConsumer = std::make_shared<IndexDataConsumer>(param);
 
   index::IndexingOptions IndexOpts;
   IndexOpts.SystemSymbolFilter =
       index::IndexingOptions::SystemSymbolFilterKind::All;
-  IndexOpts.IndexFunctionLocals = true;
-  IndexOpts.IndexImplicitInstantiation = true;
+  if (no_linkage) {
+    IndexOpts.IndexFunctionLocals = true;
+    IndexOpts.IndexImplicitInstantiation = true;
 #if LLVM_VERSION_MAJOR >= 9
-  IndexOpts.IndexParametersInDeclarations =
-      g_config->index.parametersInDeclarations;
-  IndexOpts.IndexTemplateParameters = true;
+    IndexOpts.IndexParametersInDeclarations =
+        g_config->index.parametersInDeclarations;
+    IndexOpts.IndexTemplateParameters = true;
 #endif
+  }
 
   std::unique_ptr<FrontendAction> Action = createIndexingAction(
       DataConsumer, IndexOpts, std::make_unique<IndexFrontendAction>(param));

src/indexer.hh

@@ -348,7 +348,7 @@ Index(SemaManager *complete, WorkingFiles *wfiles, VFS *vfs,
       const std::string &opt_wdir, const std::string &file,
       const std::vector<const char *> &args,
       const std::vector<std::pair<std::string, std::string>> &remapped,
-      bool &ok);
+      bool all_linkages, bool &ok);
 } // namespace idx
 } // namespace ccls
 

src/messages/workspace.cc

@@ -55,7 +55,7 @@ void MessageHandler::workspace_didChangeWatchedFiles(
         return;
 
     IndexMode mode =
-        wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::NonInteractive;
+        wfiles->GetFile(path) ? IndexMode::Normal : IndexMode::Background;
     switch (event.type) {
     case FileChangeType::Created:
     case FileChangeType::Changed: {

src/pipeline.cc

@@ -229,18 +229,20 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
   std::string path_to_index = entry.filename;
   std::unique_ptr<IndexFile> prev;
 
-  bool deleted = false;
+  bool deleted = false, no_linkage = g_config->index.initialNoLinkage ||
+                                     request.mode != IndexMode::Background;
   int reparse = 0;
   std::optional<int64_t> write_time = LastWriteTime(path_to_index);
   if (!write_time) {
     deleted = true;
   } else {
-    reparse = vfs->Stamp(path_to_index, *write_time, 0);
+    if (vfs->Stamp(path_to_index, *write_time, no_linkage ? 2 : 0))
+      reparse = no_linkage ? 2 : 1;
     if (request.path != path_to_index) {
       std::optional<int64_t> mtime1 = LastWriteTime(request.path);
       if (!mtime1)
         deleted = true;
-      else if (vfs->Stamp(request.path, *mtime1, 0))
+      else if (vfs->Stamp(request.path, *mtime1, no_linkage ? 2 : 0))
         reparse = 2;
     }
   }
@@ -293,7 +295,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
       auto dependencies = prev->dependencies;
       IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
       on_indexed->PushBack(std::move(update),
-        request.mode != IndexMode::NonInteractive);
+                           request.mode != IndexMode::Background);
       {
         std::lock_guard lock1(vfs->mutex);
         vfs->state[path_to_index].loaded++;
@@ -318,7 +320,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
         }
         IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
         on_indexed->PushBack(std::move(update),
-          request.mode != IndexMode::NonInteractive);
+                             request.mode != IndexMode::Background);
         if (entry.id >= 0) {
           std::lock_guard lock2(project->mtx);
           project->root2folder[entry.root].path2entry_index[path] = entry.id;
@@ -351,7 +353,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
     }
     bool ok;
     indexes = idx::Index(completion, wfiles, vfs, entry.directory,
-                         path_to_index, entry.args, remapped, ok);
+                         path_to_index, entry.args, remapped, no_linkage, ok);
 
     if (!ok) {
       if (request.id.Valid()) {
@@ -403,7 +405,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
         }
       }
       on_indexed->PushBack(IndexUpdate::CreateDelta(prev.get(), curr.get()),
-                           request.mode != IndexMode::NonInteractive);
+                           request.mode != IndexMode::Background);
       {
         std::lock_guard lock1(vfs->mutex);
         vfs->state[path].loaded++;
@@ -743,7 +745,7 @@ void Index(const std::string &path, const std::vector<const char *> &args,
            IndexMode mode, bool must_exist, RequestId id) {
   pending_index_requests++;
   index_request->PushBack({path, args, mode, must_exist, id},
-                          mode != IndexMode::NonInteractive);
+                          mode != IndexMode::Background);
 }
 
 void RemoveCache(const std::string &path) {

src/pipeline.hh

@@ -30,7 +30,7 @@ struct VFS {
 };
 
 enum class IndexMode {
-  NonInteractive,
+  Background,
   OnChange,
   Normal,
 };

src/project.cc

@@ -576,7 +576,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) {
           bool interactive = wfiles->GetFile(entry.filename) != nullptr;
           pipeline::Index(entry.filename, entry.args,
                           interactive ? IndexMode::Normal
-                                      : IndexMode::NonInteractive,
+                                      : IndexMode::Background,
                           false, id);
         } else {
           LOG_V(1) << "[" << i << "/" << folder.entries.size() << "]: " << reason
@@ -590,7 +590,7 @@ void Project::Index(WorkingFiles *wfiles, RequestId id) {
   pipeline::loaded_ts = pipeline::tick;
   // Dummy request to indicate that project is loaded and
   // trigger refreshing semantic highlight for all working files.
-  pipeline::Index("", {}, IndexMode::NonInteractive, false);
+  pipeline::Index("", {}, IndexMode::Background, false);
 }
 
 void Project::IndexRelated(const std::string &path) {
@@ -604,7 +604,7 @@ void Project::IndexRelated(const std::string &path) {
         std::string reason;
         if (sys::path::stem(entry.filename) == stem && entry.filename != path &&
             match.Matches(entry.filename, &reason))
-          pipeline::Index(entry.filename, entry.args, IndexMode::NonInteractive,
+          pipeline::Index(entry.filename, entry.args, IndexMode::Background,
                           true);
       }
       break;

src/test.cc

@@ -327,7 +327,8 @@ bool RunIndexTests(const std::string &filter_path, bool enable_update) {
         for (auto &arg : flags)
           cargs.push_back(arg.c_str());
         bool ok;
-        auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs, {}, ok);
+        auto dbs = ccls::idx::Index(&completion, &wfiles, &vfs, "", path, cargs,
+                                    {}, true, ok);
 
         for (const auto &entry : all_expected_output) {
           const std::string &expected_path = entry.first;