Use StoreInMemory Preamble for CodeComplete

src/clang_complete.cc

@@ -22,6 +22,7 @@ limitations under the License.
 
 #include <clang/Frontend/CompilerInstance.h>
 #include <clang/Frontend/FrontendDiagnostic.h>
+#include <clang/Lex/PreprocessorOptions.h>
 #include <clang/Sema/CodeCompleteConsumer.h>
 #include <llvm/ADT/Twine.h>
 #include <llvm/Config/llvm-config.h>
@@ -320,12 +321,12 @@ public:
                                   unsigned NumResults) override {
     ls_items.reserve(NumResults);
     for (unsigned i = 0; i != NumResults; i++) {
+      if (Results[i].Availability == CXAvailability_NotAccessible ||
+          Results[i].Availability == CXAvailability_NotAvailable)
+        continue;
       CodeCompletionString *CCS = Results[i].CreateCodeCompletionString(
           S, Context, getAllocator(), getCodeCompletionTUInfo(),
           includeBriefComments());
-      if (CCS->getAvailability() == CXAvailability_NotAvailable)
-        continue;
-
       lsCompletionItem ls_item;
       ls_item.kind = GetCompletionKind(Results[i].CursorKind);
       if (const char *brief = CCS->getBriefComment())
@@ -380,7 +381,7 @@ void TryEnsureDocumentParsed(ClangCompleteManager *manager,
     return;
 
   const auto &args = session->file.args;
-  WorkingFiles::Snapshot snapshot = session->working_files->AsSnapshot(
+  WorkingFiles::Snapshot snapshot = session->wfiles->AsSnapshot(
       {StripFileType(session->file.filename)});
 
   LOG_S(INFO) << "create " << (diagnostic ? "diagnostic" : "completion")
@@ -389,6 +390,23 @@ void TryEnsureDocumentParsed(ClangCompleteManager *manager,
                                      diagnostic);
 }
 
+std::unique_ptr<CompilerInvocation>
+buildCompilerInvocation(const std::vector<std::string> &args,
+                        IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+  std::vector<const char *> cargs;
+  for (auto &arg : args)
+    cargs.push_back(arg.c_str());
+  IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
+      CompilerInstance::createDiagnostics(new DiagnosticOptions));
+  std::unique_ptr<CompilerInvocation> CI =
+      createInvocationFromCommandLine(cargs, Diags, VFS);
+  if (CI) {
+    CI->getLangOpts()->CommentOpts.ParseAllComments = true;
+    CI->getLangOpts()->SpellChecking = false;
+  }
+  return CI;
+}
+
 void CompletionPreloadMain(ClangCompleteManager *completion_manager) {
   while (true) {
     // Fetching the completion request blocks until we have a request.
@@ -403,22 +421,14 @@ void CompletionPreloadMain(ClangCompleteManager *completion_manager) {
     if (!session)
       continue;
 
-    // Note: we only preload completion. We emit diagnostics for the
+    const auto &args = session->file.args;
-    // completion preload though.
+    WorkingFiles::Snapshot snapshot = session->wfiles->AsSnapshot(
-    CompletionSession::Tu *tu = &session->completion;
+      {StripFileType(session->file.filename)});
 
-    // If we've parsed it more recently than the request time, don't bother
+    LOG_S(INFO) << "create completion session for " << session->file.filename;
-    // reparsing.
+    if (std::unique_ptr<CompilerInvocation> CI =
-    if (tu->last_parsed_at && *tu->last_parsed_at > request.request_time)
+            buildCompilerInvocation(args, session->FS))
-      continue;
+      session->BuildPreamble(*CI);
-
-    std::unique_ptr<ClangTranslationUnit> parsing;
-    TryEnsureDocumentParsed(completion_manager, session, &parsing, false);
-
-    // Activate new translation unit.
-    std::lock_guard<std::mutex> lock(tu->lock);
-    tu->last_parsed_at = std::chrono::high_resolution_clock::now();
-    tu->tu = std::move(parsing);
   }
 }
 
@@ -441,48 +451,70 @@ void CompletionQueryMain(ClangCompleteManager *completion_manager) {
         completion_manager->TryGetSession(path, true /*mark_as_completion*/,
                                           true /*create_if_needed*/);
 
-    std::lock_guard<std::mutex> lock(session->completion.lock);
+    std::unique_ptr<CompilerInvocation> CI =
-    TryEnsureDocumentParsed(completion_manager, session,
+        buildCompilerInvocation(session->file.args, session->FS);
-                            &session->completion.tu, false);
+    if (!CI)
-
+      continue;
-    // It is possible we failed to create the document despite
+    clang::CodeCompleteOptions CCOpts;
-    // |TryEnsureDocumentParsed|.
-    if (ClangTranslationUnit *tu = session->completion.tu.get()) {
-      WorkingFiles::Snapshot snapshot =
-          completion_manager->working_files_->AsSnapshot({StripFileType(path)});
-      IntrusiveRefCntPtr<FileManager> FileMgr(&tu->Unit->getFileManager());
-      IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts(new DiagnosticOptions());
-      IntrusiveRefCntPtr<DiagnosticsEngine> Diag(new DiagnosticsEngine(
-          IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs), &*DiagOpts));
-      // StoreDiags Diags;
-      // IntrusiveRefCntPtr<DiagnosticsEngine> DiagE =
-      //     CompilerInstance::createDiagnostics(DiagOpts.get(), &Diags, false);
-
-      IntrusiveRefCntPtr<SourceManager> SrcMgr(
-          new SourceManager(*Diag, *FileMgr));
-      std::vector<ASTUnit::RemappedFile> Remapped = GetRemapped(snapshot);
-      SmallVector<StoredDiagnostic, 8> Diagnostics;
-      SmallVector<const llvm::MemoryBuffer *, 1> TemporaryBuffers;
-
-      CodeCompleteOptions Opts;
-      LangOptions LangOpts;
-      Opts.IncludeBriefComments = true;
 #if LLVM_VERSION_MAJOR >= 7
-      Opts.LoadExternal = true;
+    CCOpts.IncludeFixIts = true;
-      Opts.IncludeFixIts = true;
 #endif
-      CaptureCompletionResults capture(Opts);
+    CCOpts.IncludeCodePatterns = true;
-      tu->Unit->CodeComplete(session->file.filename, request->position.line + 1,
+    auto &FOpts = CI->getFrontendOpts();
-                             request->position.character + 1, Remapped,
+    FOpts.DisableFree = false;
-                             /*IncludeMacros=*/true,
+    FOpts.CodeCompleteOpts = CCOpts;
-                             /*IncludeCodePatterns=*/false,
+    FOpts.CodeCompletionAt.FileName = session->file.filename;
-                             /*IncludeBriefComments=*/g_config->index.comments,
+    FOpts.CodeCompletionAt.Line = request->position.line + 1;
-                             capture, tu->PCHCO, *Diag, LangOpts, *SrcMgr,
+    FOpts.CodeCompletionAt.Column = request->position.character + 1;
-                             *FileMgr, Diagnostics, TemporaryBuffers);
+
-      request->on_complete(capture.ls_items, false /*is_cached_result*/);
+    WorkingFiles::Snapshot snapshot =
-      // completion_manager->on_diagnostic_(session->file.filename,
+      completion_manager->working_files_->AsSnapshot({StripFileType(path)});
-      // Diags.take());
+
+    std::vector<std::unique_ptr<llvm::MemoryBuffer>> Bufs;
+    for (auto &file : snapshot.files) {
+      Bufs.push_back(llvm::MemoryBuffer::getMemBuffer(file.content));
+      if (file.filename == session->file.filename) {
+        if (auto Preamble = session->GetPreamble()) {
+#if LLVM_VERSION_MAJOR >= 7
+          Preamble->Preamble.OverridePreamble(*CI, session->FS,
+                                              Bufs.back().get());
+#else
+          Preamble->Preamble.AddImplicitPreamble(*CI, session->FS,
+                                                 Bufs.back().get());
+#endif
+        }
+        else {
+          CI->getPreprocessorOpts().addRemappedFile(
+            CI->getFrontendOpts().Inputs[0].getFile(), Bufs.back().get());
+        }
+      } else {
+        CI->getPreprocessorOpts().addRemappedFile(file.filename,
+                                                  Bufs.back().get());
+      }
     }
+
+    IgnoringDiagConsumer DC;
+    auto Clang = std::make_unique<CompilerInstance>(session->PCH);
+    Clang->setInvocation(std::move(CI));
+    Clang->createDiagnostics(&DC, false);
+    auto Consumer = new CaptureCompletionResults(CCOpts);
+    Clang->setCodeCompletionConsumer(Consumer);
+    Clang->setVirtualFileSystem(session->FS);
+    Clang->setTarget(TargetInfo::CreateTargetInfo(
+        Clang->getDiagnostics(), Clang->getInvocation().TargetOpts));
+    if (!Clang->hasTarget())
+      continue;
+
+    SyntaxOnlyAction Action;
+    if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
+      continue;
+    if (!Action.Execute())
+      continue;
+    Action.EndSourceFile();
+    for (auto &Buf : Bufs)
+      Buf.release();
+
+    request->on_complete(Consumer->ls_items, false /*is_cached_result*/);
   }
 }
 
@@ -573,6 +605,37 @@ void DiagnosticQueryMain(ClangCompleteManager *manager) {
 
 } // namespace
 
+std::shared_ptr<PreambleData> CompletionSession::GetPreamble() {
+  std::lock_guard<std::mutex> lock(completion.lock);
+  return completion.preamble;
+}
+
+void CompletionSession::BuildPreamble(CompilerInvocation &CI) {
+  std::shared_ptr<PreambleData> OldP = GetPreamble();
+  std::string contents = wfiles->GetContent(file.filename);
+  std::unique_ptr<llvm::MemoryBuffer> Buf =
+      llvm::MemoryBuffer::getMemBuffer(contents);
+  auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), Buf.get(), 0);
+  if (OldP && OldP->Preamble.CanReuse(CI, Buf.get(), Bounds, FS.get()))
+    return;
+  CI.getFrontendOpts().SkipFunctionBodies = true;
+#if LLVM_VERSION_MAJOR >= 7
+  CI.getPreprocessorOpts().WriteCommentListToPCH = false;
+#endif
+
+  DiagnosticConsumer DC;
+  IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDE =
+      CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
+                                          &DC, false);
+  PreambleCallbacks PP;
+  if (auto NewPreamble = PrecompiledPreamble::Build(
+          CI, Buf.get(), Bounds, *PreambleDE, FS, PCH, true, PP)) {
+    std::lock_guard<std::mutex> lock(completion.lock);
+    completion.preamble =
+        std::make_shared<PreambleData>(std::move(*NewPreamble));
+  }
+}
+
 ClangCompleteManager::ClangCompleteManager(Project *project,
                                            WorkingFiles *working_files,
                                            OnDiagnostic on_diagnostic,
@@ -580,7 +643,8 @@ ClangCompleteManager::ClangCompleteManager(Project *project,
     : project_(project), working_files_(working_files),
       on_diagnostic_(on_diagnostic), on_dropped_(on_dropped),
       preloaded_sessions_(kMaxPreloadedSessions),
-      completion_sessions_(kMaxCompletionSessions) {
+      completion_sessions_(kMaxCompletionSessions),
+      PCH(std::make_shared<PCHContainerOperations>()) {
   std::thread([&]() {
     set_thread_name("comp-query");
     CompletionQueryMain(this);
@@ -682,7 +746,7 @@ bool ClangCompleteManager::EnsureCompletionOrCreatePreloadSession(
 
   // No CompletionSession, create new one.
   auto session = std::make_shared<CompletionSession>(
-      project_->FindCompilationEntryForFile(filename), working_files_);
+      project_->FindCompilationEntryForFile(filename), working_files_, PCH);
   preloaded_sessions_.Insert(session->file.filename, session);
   return true;
 }
@@ -713,7 +777,7 @@ ClangCompleteManager::TryGetSession(const std::string &filename,
       completion_sessions_.TryGet(filename);
   if (!completion_session && create_if_needed) {
     completion_session = std::make_shared<CompletionSession>(
-        project_->FindCompilationEntryForFile(filename), working_files_);
+        project_->FindCompilationEntryForFile(filename), working_files_, PCH);
     completion_sessions_.Insert(filename, completion_session);
   }
 

src/clang_complete.h

@@ -23,11 +23,19 @@ limitations under the License.
 #include "threaded_queue.h"
 #include "working_files.h"
 
+#include <clang/Frontend/CompilerInstance.h>
+#include <clang/Frontend/FrontendActions.h>
+
 #include <functional>
 #include <memory>
 #include <mutex>
 #include <string>
 
+struct PreambleData {
+  PreambleData(clang::PrecompiledPreamble P) : Preamble(std::move(P)) {}
+  clang::PrecompiledPreamble Preamble;
+};
+
 struct CompletionSession
     : public std::enable_shared_from_this<CompletionSession> {
   // Translation unit for clang.
@@ -40,14 +48,28 @@ struct CompletionSession
     std::unique_ptr<ClangTranslationUnit> tu;
   };
 
-  Project::Entry file;
+  struct CU {
-  WorkingFiles *working_files;
+    std::mutex lock;
+    std::shared_ptr<PreambleData> preamble;
+  };
 
-  Tu completion;
+  Project::Entry file;
+  WorkingFiles *wfiles;
+
+  // TODO share
+  llvm::IntrusiveRefCntPtr<clang::vfs::FileSystem> FS =
+      clang::vfs::getRealFileSystem();
+  std::shared_ptr<clang::PCHContainerOperations> PCH;
+
+  CU completion;
   Tu diagnostics;
 
-  CompletionSession(const Project::Entry &file, WorkingFiles *wfiles)
+  CompletionSession(const Project::Entry &file, WorkingFiles *wfiles,
-      : file(file), working_files(wfiles) {}
+                    std::shared_ptr<clang::PCHContainerOperations> PCH)
+      : file(file), wfiles(wfiles), PCH(PCH) {}
+
+  std::shared_ptr<PreambleData> GetPreamble();
+  void BuildPreamble(clang::CompilerInvocation &CI);
 };
 
 struct ClangCompleteManager {
@@ -145,6 +167,8 @@ struct ClangCompleteManager {
   // Parse requests. The path may already be parsed, in which case it should be
   // reparsed.
   ThreadedQueue<PreloadRequest> preload_requests_;
+
+  std::shared_ptr<clang::PCHContainerOperations> PCH;
 };
 
 // Cached completion information, so we can give fast completion results when

src/clang_tu.cc

@@ -104,13 +104,12 @@ std::unique_ptr<ClangTranslationUnit> ClangTranslationUnit::Create(
         /*ResourceFilePath=*/g_config->clang.resourceDir,
         /*OnlyLocalDecls=*/false,
         /*CaptureDiagnostics=*/diagnostic, Remapped,
-        /*RemappedFilesKeepOriginalName=*/true, 1,
+        /*RemappedFilesKeepOriginalName=*/true, 0,
         diagnostic ? TU_Complete : TU_Prefix,
         /*CacheCodeCompletionResults=*/true, g_config->index.comments,
         /*AllowPCHWithCompilerErrors=*/true,
 #if LLVM_VERSION_MAJOR >= 7
-        diagnostic ? SkipFunctionBodiesScope::None
+        SkipFunctionBodiesScope::None,
-                   : SkipFunctionBodiesScope::PreambleAndMainFile,
 #else
         !diagnostic,
 #endif

src/working_files.cc

@@ -442,6 +442,14 @@ WorkingFiles::GetFileByFilenameNoLock(const std::string &filename) {
   return nullptr;
 }
 
+std::string WorkingFiles::GetContent(const std::string &filename) {
+  std::lock_guard<std::mutex> lock(files_mutex);
+  for (auto &file : files)
+    if (file->filename == filename)
+      return file->buffer_content;
+  return "";
+}
+
 void WorkingFiles::DoAction(const std::function<void()> &action) {
   std::lock_guard<std::mutex> lock(files_mutex);
   action();

src/working_files.h

@@ -108,6 +108,7 @@ struct WorkingFiles {
   // Find the file with the given filename.
   WorkingFile *GetFileByFilename(const std::string &filename);
   WorkingFile *GetFileByFilenameNoLock(const std::string &filename);
+  std::string GetContent(const std::string &filename);
 
   // Run |action| under the lock.
   void DoAction(const std::function<void()> &action);