From f808dd8f8acacfef92c1a58fddd61ffcc2e69a52 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 27 Aug 2018 22:42:40 -0700 Subject: [PATCH] Use StoreInMemory Preamble for CodeComplete --- src/clang_complete.cc | 186 ++++++++++++++++++++++++++++-------------- src/clang_complete.h | 34 ++++++-- src/clang_tu.cc | 5 +- src/working_files.cc | 8 ++ src/working_files.h | 1 + 5 files changed, 165 insertions(+), 69 deletions(-) diff --git a/src/clang_complete.cc b/src/clang_complete.cc index c24f9c7a..e41c9fb9 100644 --- a/src/clang_complete.cc +++ b/src/clang_complete.cc @@ -22,6 +22,7 @@ limitations under the License. #include #include +#include #include #include #include @@ -320,12 +321,12 @@ public: unsigned NumResults) override { ls_items.reserve(NumResults); for (unsigned i = 0; i != NumResults; i++) { + if (Results[i].Availability == CXAvailability_NotAccessible || + Results[i].Availability == CXAvailability_NotAvailable) + continue; CodeCompletionString *CCS = Results[i].CreateCodeCompletionString( S, Context, getAllocator(), getCodeCompletionTUInfo(), includeBriefComments()); - if (CCS->getAvailability() == CXAvailability_NotAvailable) - continue; - lsCompletionItem ls_item; ls_item.kind = GetCompletionKind(Results[i].CursorKind); if (const char *brief = CCS->getBriefComment()) @@ -380,7 +381,7 @@ void TryEnsureDocumentParsed(ClangCompleteManager *manager, return; const auto &args = session->file.args; - WorkingFiles::Snapshot snapshot = session->working_files->AsSnapshot( + WorkingFiles::Snapshot snapshot = session->wfiles->AsSnapshot( {StripFileType(session->file.filename)}); LOG_S(INFO) << "create " << (diagnostic ? "diagnostic" : "completion") @@ -389,6 +390,23 @@ void TryEnsureDocumentParsed(ClangCompleteManager *manager, diagnostic); } +std::unique_ptr +buildCompilerInvocation(const std::vector &args, + IntrusiveRefCntPtr VFS) { + std::vector cargs; + for (auto &arg : args) + cargs.push_back(arg.c_str()); + IntrusiveRefCntPtr Diags( + CompilerInstance::createDiagnostics(new DiagnosticOptions)); + std::unique_ptr CI = + createInvocationFromCommandLine(cargs, Diags, VFS); + if (CI) { + CI->getLangOpts()->CommentOpts.ParseAllComments = true; + CI->getLangOpts()->SpellChecking = false; + } + return CI; +} + void CompletionPreloadMain(ClangCompleteManager *completion_manager) { while (true) { // Fetching the completion request blocks until we have a request. @@ -403,22 +421,14 @@ void CompletionPreloadMain(ClangCompleteManager *completion_manager) { if (!session) continue; - // Note: we only preload completion. We emit diagnostics for the - // completion preload though. - CompletionSession::Tu *tu = &session->completion; + const auto &args = session->file.args; + WorkingFiles::Snapshot snapshot = session->wfiles->AsSnapshot( + {StripFileType(session->file.filename)}); - // If we've parsed it more recently than the request time, don't bother - // reparsing. - if (tu->last_parsed_at && *tu->last_parsed_at > request.request_time) - continue; - - std::unique_ptr parsing; - TryEnsureDocumentParsed(completion_manager, session, &parsing, false); - - // Activate new translation unit. - std::lock_guard lock(tu->lock); - tu->last_parsed_at = std::chrono::high_resolution_clock::now(); - tu->tu = std::move(parsing); + LOG_S(INFO) << "create completion session for " << session->file.filename; + if (std::unique_ptr CI = + buildCompilerInvocation(args, session->FS)) + session->BuildPreamble(*CI); } } @@ -441,48 +451,70 @@ void CompletionQueryMain(ClangCompleteManager *completion_manager) { completion_manager->TryGetSession(path, true /*mark_as_completion*/, true /*create_if_needed*/); - std::lock_guard lock(session->completion.lock); - TryEnsureDocumentParsed(completion_manager, session, - &session->completion.tu, false); - - // It is possible we failed to create the document despite - // |TryEnsureDocumentParsed|. - if (ClangTranslationUnit *tu = session->completion.tu.get()) { - WorkingFiles::Snapshot snapshot = - completion_manager->working_files_->AsSnapshot({StripFileType(path)}); - IntrusiveRefCntPtr FileMgr(&tu->Unit->getFileManager()); - IntrusiveRefCntPtr DiagOpts(new DiagnosticOptions()); - IntrusiveRefCntPtr Diag(new DiagnosticsEngine( - IntrusiveRefCntPtr(new DiagnosticIDs), &*DiagOpts)); - // StoreDiags Diags; - // IntrusiveRefCntPtr DiagE = - // CompilerInstance::createDiagnostics(DiagOpts.get(), &Diags, false); - - IntrusiveRefCntPtr SrcMgr( - new SourceManager(*Diag, *FileMgr)); - std::vector Remapped = GetRemapped(snapshot); - SmallVector Diagnostics; - SmallVector TemporaryBuffers; - - CodeCompleteOptions Opts; - LangOptions LangOpts; - Opts.IncludeBriefComments = true; + std::unique_ptr CI = + buildCompilerInvocation(session->file.args, session->FS); + if (!CI) + continue; + clang::CodeCompleteOptions CCOpts; #if LLVM_VERSION_MAJOR >= 7 - Opts.LoadExternal = true; - Opts.IncludeFixIts = true; + CCOpts.IncludeFixIts = true; #endif - CaptureCompletionResults capture(Opts); - tu->Unit->CodeComplete(session->file.filename, request->position.line + 1, - request->position.character + 1, Remapped, - /*IncludeMacros=*/true, - /*IncludeCodePatterns=*/false, - /*IncludeBriefComments=*/g_config->index.comments, - capture, tu->PCHCO, *Diag, LangOpts, *SrcMgr, - *FileMgr, Diagnostics, TemporaryBuffers); - request->on_complete(capture.ls_items, false /*is_cached_result*/); - // completion_manager->on_diagnostic_(session->file.filename, - // Diags.take()); + CCOpts.IncludeCodePatterns = true; + auto &FOpts = CI->getFrontendOpts(); + FOpts.DisableFree = false; + FOpts.CodeCompleteOpts = CCOpts; + FOpts.CodeCompletionAt.FileName = session->file.filename; + FOpts.CodeCompletionAt.Line = request->position.line + 1; + FOpts.CodeCompletionAt.Column = request->position.character + 1; + + WorkingFiles::Snapshot snapshot = + completion_manager->working_files_->AsSnapshot({StripFileType(path)}); + + std::vector> Bufs; + for (auto &file : snapshot.files) { + Bufs.push_back(llvm::MemoryBuffer::getMemBuffer(file.content)); + if (file.filename == session->file.filename) { + if (auto Preamble = session->GetPreamble()) { +#if LLVM_VERSION_MAJOR >= 7 + Preamble->Preamble.OverridePreamble(*CI, session->FS, + Bufs.back().get()); +#else + Preamble->Preamble.AddImplicitPreamble(*CI, session->FS, + Bufs.back().get()); +#endif + } + else { + CI->getPreprocessorOpts().addRemappedFile( + CI->getFrontendOpts().Inputs[0].getFile(), Bufs.back().get()); + } + } else { + CI->getPreprocessorOpts().addRemappedFile(file.filename, + Bufs.back().get()); + } } + + IgnoringDiagConsumer DC; + auto Clang = std::make_unique(session->PCH); + Clang->setInvocation(std::move(CI)); + Clang->createDiagnostics(&DC, false); + auto Consumer = new CaptureCompletionResults(CCOpts); + Clang->setCodeCompletionConsumer(Consumer); + Clang->setVirtualFileSystem(session->FS); + Clang->setTarget(TargetInfo::CreateTargetInfo( + Clang->getDiagnostics(), Clang->getInvocation().TargetOpts)); + if (!Clang->hasTarget()) + continue; + + SyntaxOnlyAction Action; + if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0])) + continue; + if (!Action.Execute()) + continue; + Action.EndSourceFile(); + for (auto &Buf : Bufs) + Buf.release(); + + request->on_complete(Consumer->ls_items, false /*is_cached_result*/); } } @@ -573,6 +605,37 @@ void DiagnosticQueryMain(ClangCompleteManager *manager) { } // namespace +std::shared_ptr CompletionSession::GetPreamble() { + std::lock_guard lock(completion.lock); + return completion.preamble; +} + +void CompletionSession::BuildPreamble(CompilerInvocation &CI) { + std::shared_ptr OldP = GetPreamble(); + std::string contents = wfiles->GetContent(file.filename); + std::unique_ptr Buf = + llvm::MemoryBuffer::getMemBuffer(contents); + auto Bounds = ComputePreambleBounds(*CI.getLangOpts(), Buf.get(), 0); + if (OldP && OldP->Preamble.CanReuse(CI, Buf.get(), Bounds, FS.get())) + return; + CI.getFrontendOpts().SkipFunctionBodies = true; +#if LLVM_VERSION_MAJOR >= 7 + CI.getPreprocessorOpts().WriteCommentListToPCH = false; +#endif + + DiagnosticConsumer DC; + IntrusiveRefCntPtr PreambleDE = + CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(), + &DC, false); + PreambleCallbacks PP; + if (auto NewPreamble = PrecompiledPreamble::Build( + CI, Buf.get(), Bounds, *PreambleDE, FS, PCH, true, PP)) { + std::lock_guard lock(completion.lock); + completion.preamble = + std::make_shared(std::move(*NewPreamble)); + } +} + ClangCompleteManager::ClangCompleteManager(Project *project, WorkingFiles *working_files, OnDiagnostic on_diagnostic, @@ -580,7 +643,8 @@ ClangCompleteManager::ClangCompleteManager(Project *project, : project_(project), working_files_(working_files), on_diagnostic_(on_diagnostic), on_dropped_(on_dropped), preloaded_sessions_(kMaxPreloadedSessions), - completion_sessions_(kMaxCompletionSessions) { + completion_sessions_(kMaxCompletionSessions), + PCH(std::make_shared()) { std::thread([&]() { set_thread_name("comp-query"); CompletionQueryMain(this); @@ -682,7 +746,7 @@ bool ClangCompleteManager::EnsureCompletionOrCreatePreloadSession( // No CompletionSession, create new one. auto session = std::make_shared( - project_->FindCompilationEntryForFile(filename), working_files_); + project_->FindCompilationEntryForFile(filename), working_files_, PCH); preloaded_sessions_.Insert(session->file.filename, session); return true; } @@ -713,7 +777,7 @@ ClangCompleteManager::TryGetSession(const std::string &filename, completion_sessions_.TryGet(filename); if (!completion_session && create_if_needed) { completion_session = std::make_shared( - project_->FindCompilationEntryForFile(filename), working_files_); + project_->FindCompilationEntryForFile(filename), working_files_, PCH); completion_sessions_.Insert(filename, completion_session); } diff --git a/src/clang_complete.h b/src/clang_complete.h index d3f03492..5fa0ba39 100644 --- a/src/clang_complete.h +++ b/src/clang_complete.h @@ -23,11 +23,19 @@ limitations under the License. #include "threaded_queue.h" #include "working_files.h" +#include +#include + #include #include #include #include +struct PreambleData { + PreambleData(clang::PrecompiledPreamble P) : Preamble(std::move(P)) {} + clang::PrecompiledPreamble Preamble; +}; + struct CompletionSession : public std::enable_shared_from_this { // Translation unit for clang. @@ -40,14 +48,28 @@ struct CompletionSession std::unique_ptr tu; }; - Project::Entry file; - WorkingFiles *working_files; + struct CU { + std::mutex lock; + std::shared_ptr preamble; + }; - Tu completion; + Project::Entry file; + WorkingFiles *wfiles; + + // TODO share + llvm::IntrusiveRefCntPtr FS = + clang::vfs::getRealFileSystem(); + std::shared_ptr PCH; + + CU completion; Tu diagnostics; - CompletionSession(const Project::Entry &file, WorkingFiles *wfiles) - : file(file), working_files(wfiles) {} + CompletionSession(const Project::Entry &file, WorkingFiles *wfiles, + std::shared_ptr PCH) + : file(file), wfiles(wfiles), PCH(PCH) {} + + std::shared_ptr GetPreamble(); + void BuildPreamble(clang::CompilerInvocation &CI); }; struct ClangCompleteManager { @@ -145,6 +167,8 @@ struct ClangCompleteManager { // Parse requests. The path may already be parsed, in which case it should be // reparsed. ThreadedQueue preload_requests_; + + std::shared_ptr PCH; }; // Cached completion information, so we can give fast completion results when diff --git a/src/clang_tu.cc b/src/clang_tu.cc index 74130296..35f969a4 100644 --- a/src/clang_tu.cc +++ b/src/clang_tu.cc @@ -104,13 +104,12 @@ std::unique_ptr ClangTranslationUnit::Create( /*ResourceFilePath=*/g_config->clang.resourceDir, /*OnlyLocalDecls=*/false, /*CaptureDiagnostics=*/diagnostic, Remapped, - /*RemappedFilesKeepOriginalName=*/true, 1, + /*RemappedFilesKeepOriginalName=*/true, 0, diagnostic ? TU_Complete : TU_Prefix, /*CacheCodeCompletionResults=*/true, g_config->index.comments, /*AllowPCHWithCompilerErrors=*/true, #if LLVM_VERSION_MAJOR >= 7 - diagnostic ? SkipFunctionBodiesScope::None - : SkipFunctionBodiesScope::PreambleAndMainFile, + SkipFunctionBodiesScope::None, #else !diagnostic, #endif diff --git a/src/working_files.cc b/src/working_files.cc index 4f6ca27a..74b9f30d 100644 --- a/src/working_files.cc +++ b/src/working_files.cc @@ -442,6 +442,14 @@ WorkingFiles::GetFileByFilenameNoLock(const std::string &filename) { return nullptr; } +std::string WorkingFiles::GetContent(const std::string &filename) { + std::lock_guard lock(files_mutex); + for (auto &file : files) + if (file->filename == filename) + return file->buffer_content; + return ""; +} + void WorkingFiles::DoAction(const std::function &action) { std::lock_guard lock(files_mutex); action(); diff --git a/src/working_files.h b/src/working_files.h index b0ed01c7..dcd8fa84 100644 --- a/src/working_files.h +++ b/src/working_files.h @@ -108,6 +108,7 @@ struct WorkingFiles { // Find the file with the given filename. WorkingFile *GetFileByFilename(const std::string &filename); WorkingFile *GetFileByFilenameNoLock(const std::string &filename); + std::string GetContent(const std::string &filename); // Run |action| under the lock. void DoAction(const std::function &action);