Zengtudor/ccls
1
0
Fork 0
mirror of https://github.com/MaskRay/ccls.git synced 2024-11-22 07:35:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Simplify pipeline

Browse Source
This commit is contained in:
Fangrui Song 2018-05-27 17:50:02 -07:00
parent cf0d16fa0c
commit a962061698
61 changed files with 479 additions and 722 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
CMakeLists.txt
View File

@ -189,11 +189,8 @@ file(GLOB SOURCES src/*.cc src/*.h src/serializers/*.cc src/serializers/*.h
target_sources(ccls PRIVATE third_party/siphash.cc)
target_sources(ccls PRIVATE
src/cache_manager.cc
src/clang_complete.cc
src/clang_cursor.cc
src/clang_indexer.cc
src/clang_translation_unit.cc
src/clang_tu.cc
src/clang_utils.cc
src/config.cc
src/diagnostics_engine.cc
@ -202,6 +199,7 @@ target_sources(ccls PRIVATE
src/fuzzy_match.cc
src/main.cc
src/include_complete.cc
src/indexer.cc
src/method.cc
src/language.cc
src/lex_utils.cc
@ -212,12 +210,10 @@ target_sources(ccls PRIVATE
src/pipeline.cc
src/platform_posix.cc
src/platform_win.cc
src/port.cc
src/position.cc
src/project.cc
src/query_utils.cc
src/query.cc
src/queue_manager.cc
src/serializer.cc
src/test.cc
src/third_party_impl.cc

62
src/cache_manager.cc
View File

@ -1,62 +0,0 @@
#include "cache_manager.h"
#include "config.h"
#include "indexer.h"
#include "lsp.h"
#include "platform.h"
#include <algorithm>
#include <unordered_map>
namespace {
std::string GetCachePath(const std::string& source_file) {
assert(!g_config->cacheDirectory.empty());
std::string cache_file;
size_t len = g_config->projectRoot.size();
if (StartsWith(source_file, g_config->projectRoot)) {
cache_file = EscapeFileName(g_config->projectRoot) +
EscapeFileName(source_file.substr(len));
} else {
cache_file = '@' + EscapeFileName(g_config->projectRoot) +
EscapeFileName(source_file);
}
return g_config->cacheDirectory + cache_file;
}
std::string AppendSerializationFormat(const std::string& base) {
switch (g_config->cacheFormat) {
case SerializeFormat::Binary:
return base + ".blob";
case SerializeFormat::Json:
return base + ".json";
}
}
}
// Manages loading caches from file paths for the indexer process.
void ICacheManager::WriteToCache(IndexFile& file) {
std::string cache_path = GetCachePath(file.path);
WriteToFile(cache_path, file.file_contents);
std::string indexed_content = Serialize(g_config->cacheFormat, file);
WriteToFile(AppendSerializationFormat(cache_path), indexed_content);
}
std::optional<std::string> ICacheManager::LoadCachedFileContents(
const std::string& path) {
return ReadContent(GetCachePath(path));
}
std::unique_ptr<IndexFile> ICacheManager::RawCacheLoad(
const std::string& path) {
std::string cache_path = GetCachePath(path);
std::optional<std::string> file_content = ReadContent(cache_path);
std::optional<std::string> serialized_indexed_content =
ReadContent(AppendSerializationFormat(cache_path));
if (!file_content || !serialized_indexed_content)
return nullptr;
return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content,
*file_content, IndexFile::kMajorVersion);
}

26
src/cache_manager.h
View File

@ -1,26 +0,0 @@
#pragma once
#include <optional>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
struct IndexFile;
struct ICacheManager {
void WriteToCache(IndexFile& file);
std::optional<std::string> LoadCachedFileContents(const std::string& path);
template <typename Fn>
void IterateLoadedCaches(Fn fn) {
for (const auto& cache : caches_)
fn(cache.second.get());
}
std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path);
std::unordered_map<std::string, std::unique_ptr<IndexFile>> caches_;
};

31
src/clang_complete.cc
View File

@ -21,14 +21,8 @@ unsigned Flags() {
CXTranslationUnit_CacheCompletionResults |
CXTranslationUnit_PrecompiledPreamble |
CXTranslationUnit_IncludeBriefCommentsInCodeCompletion |
CXTranslationUnit_DetailedPreprocessingRecord
#if !defined(_WIN32)
// For whatever reason, CreatePreambleOnFirstParse causes clang to
// become very crashy on windows.
// TODO: do more investigation, submit fixes to clang.
| CXTranslationUnit_CreatePreambleOnFirstParse
#endif
;
CXTranslationUnit_DetailedPreprocessingRecord |
CXTranslationUnit_CreatePreambleOnFirstParse;
}
std::string StripFileType(const std::string& path) {
@ -57,23 +51,6 @@ unsigned GetCompletionPriority(const CXCompletionString& str,
return priority;
}
/*
bool IsCallKind(CXCursorKind kind) {
switch (kind) {
case CXCursor_ObjCInstanceMethodDecl:
case CXCursor_CXXMethod:
case CXCursor_FunctionTemplate:
case CXCursor_FunctionDecl:
case CXCursor_Constructor:
case CXCursor_Destructor:
case CXCursor_ConversionFunction:
return true;
default:
return false;
}
}
*/
lsCompletionItemKind GetCompletionKind(CXCursorKind cursor_kind) {
switch (cursor_kind) {
case CXCursor_UnexposedDecl:
@ -383,8 +360,8 @@ void TryEnsureDocumentParsed(ClangCompleteManager* manager,
std::vector<std::string> args = session->file.args;
// -fspell-checking enables FixIts for, ie, misspelled types.
if (!AnyStartsWith(args, "-fno-spell-checking") &&
!AnyStartsWith(args, "-fspell-checking")) {
if (!std::count(args.begin(), args.end(), "-fno-spell-checking") &&
!std::count(args.begin(), args.end(), "-fspell-checking")) {
args.push_back("-fspell-checking");
}

3
src/clang_complete.h
View File

@ -1,7 +1,6 @@
#pragma once
#include "clang_cursor.h"
#include "clang_translation_unit.h"
#include "clang_tu.h"
#include "lru_cache.h"
#include "lsp_completion.h"
#include "lsp_diagnostic.h"

155
src/clang_translation_unit.cc
View File

@ -1,155 +0,0 @@
#include "clang_translation_unit.h"
#include "clang_utils.h"
#include "log.hh"
#include "platform.h"
#include "utils.h"
namespace {
void EmitDiagnostics(std::string path,
std::vector<const char*> args,
CXTranslationUnit tu) {
std::string output = "Fatal errors while trying to parse " + path + "\n";
output +=
"Args: " +
StringJoinMap(args, [](const char* arg) { return std::string(arg); }) +
"\n";
size_t num_diagnostics = clang_getNumDiagnostics(tu);
for (unsigned i = 0; i < num_diagnostics; ++i) {
output += " - ";
CXDiagnostic diagnostic = clang_getDiagnostic(tu, i);
// Location.
CXFile file;
unsigned int line, column;
clang_getSpellingLocation(clang_getDiagnosticLocation(diagnostic), &file,
&line, &column, nullptr);
std::string path = FileName(file);
output += path + ":" + std::to_string(line - 1) + ":" +
std::to_string(column) + " ";
// Severity
switch (clang_getDiagnosticSeverity(diagnostic)) {
case CXDiagnostic_Ignored:
case CXDiagnostic_Note:
output += "[info]";
break;
case CXDiagnostic_Warning:
output += "[warning]";
break;
case CXDiagnostic_Error:
output += "[error]";
break;
case CXDiagnostic_Fatal:
output += "[fatal]";
break;
}
// Content.
output += " " + ToString(clang_getDiagnosticSpelling(diagnostic));
clang_disposeDiagnostic(diagnostic);
output += "\n";
}
LOG_S(WARNING) << output;
}
} // namespace
// static
std::unique_ptr<ClangTranslationUnit> ClangTranslationUnit::Create(
ClangIndex* index,
const std::string& filepath,
const std::vector<std::string>& arguments,
std::vector<CXUnsavedFile>& unsaved_files,
unsigned flags) {
std::vector<const char*> args;
for (auto& arg : arguments)
args.push_back(arg.c_str());
CXTranslationUnit cx_tu;
CXErrorCode error_code;
{
error_code = clang_parseTranslationUnit2FullArgv(
index->cx_index, nullptr, args.data(), (int)args.size(),
unsaved_files.data(), (unsigned)unsaved_files.size(), flags, &cx_tu);
}
if (error_code != CXError_Success && cx_tu)
EmitDiagnostics(filepath, args, cx_tu);
// We sometimes dump the command to logs and ask the user to run it. Include
// -fsyntax-only so they don't do a full compile.
auto make_msg = [&]() {
return "Please try running the following, identify which flag causes the "
"issue, and report a bug. ccls will then filter the flag for you "
" automatically:\n " +
StringJoin(args, " ") + " -fsyntax-only";
};
switch (error_code) {
case CXError_Success:
return std::make_unique<ClangTranslationUnit>(cx_tu);
case CXError_Failure:
LOG_S(ERROR) << "libclang generic failure for " << filepath << ". "
<< make_msg();
return nullptr;
case CXError_Crashed:
LOG_S(ERROR) << "libclang crashed for " << filepath << ". " << make_msg();
return nullptr;
case CXError_InvalidArguments:
LOG_S(ERROR) << "libclang had invalid arguments for " << filepath << ". "
<< make_msg();
return nullptr;
case CXError_ASTReadError:
LOG_S(ERROR) << "libclang had ast read error for " << filepath << ". "
<< make_msg();
return nullptr;
}
return nullptr;
}
// static
std::unique_ptr<ClangTranslationUnit> ClangTranslationUnit::Reparse(
std::unique_ptr<ClangTranslationUnit> tu,
std::vector<CXUnsavedFile>& unsaved) {
int error_code;
{
error_code = clang_reparseTranslationUnit(
tu->cx_tu, (unsigned)unsaved.size(), unsaved.data(),
clang_defaultReparseOptions(tu->cx_tu));
}
if (error_code != CXError_Success && tu->cx_tu)
EmitDiagnostics("<unknown>", {}, tu->cx_tu);
switch (error_code) {
case CXError_Success:
return tu;
case CXError_Failure:
LOG_S(ERROR) << "libclang reparse generic failure";
return nullptr;
case CXError_Crashed:
LOG_S(ERROR) << "libclang reparse crashed";
return nullptr;
case CXError_InvalidArguments:
LOG_S(ERROR) << "libclang reparse had invalid arguments";
return nullptr;
case CXError_ASTReadError:
LOG_S(ERROR) << "libclang reparse had ast read error";
return nullptr;
}
return nullptr;
}
ClangTranslationUnit::ClangTranslationUnit(CXTranslationUnit tu) : cx_tu(tu) {}
ClangTranslationUnit::~ClangTranslationUnit() {
clang_disposeTranslationUnit(cx_tu);
}

30
src/clang_translation_unit.h
View File

@ -1,30 +0,0 @@
#pragma once
#include "clang_cursor.h"
#include <clang-c/Index.h>
#include <memory>
#include <string>
#include <vector>
// RAII wrapper around CXTranslationUnit which also makes it much more
// challenging to use a CXTranslationUnit instance that is not correctly
// initialized.
struct ClangTranslationUnit {
static std::unique_ptr<ClangTranslationUnit> Create(
ClangIndex* index,
const std::string& filepath,
const std::vector<std::string>& arguments,
std::vector<CXUnsavedFile>& unsaved_files,
unsigned flags);
static std::unique_ptr<ClangTranslationUnit> Reparse(
std::unique_ptr<ClangTranslationUnit> tu,
std::vector<CXUnsavedFile>& unsaved);
explicit ClangTranslationUnit(CXTranslationUnit tu);
~ClangTranslationUnit();
CXTranslationUnit cx_tu;
};

154
src/clang_cursor.cc → src/clang_tu.cc
View File

@ -1,12 +1,70 @@
#include "clang_cursor.h"
#include "clang_tu.h"
#include "clang_utils.h"
#include "log.hh"
#include "platform.h"
#include "utils.h"
#include <assert.h>
#include <string.h>
#include <algorithm>
#include <mutex>
namespace {
void EmitDiagnostics(std::string path,
std::vector<const char*> args,
CXTranslationUnit tu) {
std::string output = "Fatal errors while trying to parse " + path + "\n";
output +=
"Args: " +
StringJoinMap(args, [](const char* arg) { return std::string(arg); }) +
"\n";
size_t num_diagnostics = clang_getNumDiagnostics(tu);
for (unsigned i = 0; i < num_diagnostics; ++i) {
output += " - ";
CXDiagnostic diagnostic = clang_getDiagnostic(tu, i);
// Location.
CXFile file;
unsigned int line, column;
clang_getSpellingLocation(clang_getDiagnosticLocation(diagnostic), &file,
&line, &column, nullptr);
std::string path = FileName(file);
output += path + ":" + std::to_string(line - 1) + ":" +
std::to_string(column) + " ";
// Severity
switch (clang_getDiagnosticSeverity(diagnostic)) {
case CXDiagnostic_Ignored:
case CXDiagnostic_Note:
output += "[info]";
break;
case CXDiagnostic_Warning:
output += "[warning]";
break;
case CXDiagnostic_Error:
output += "[error]";
break;
case CXDiagnostic_Fatal:
output += "[fatal]";
break;
}
// Content.
output += " " + ToString(clang_getDiagnosticSpelling(diagnostic));
clang_disposeDiagnostic(diagnostic);
output += "\n";
}
LOG_S(WARNING) << output;
}
} // namespace
Range ResolveCXSourceRange(const CXSourceRange& range, CXFile* cx_file) {
CXSourceLocation start = clang_getRangeStart(range);
CXSourceLocation end = clang_getRangeEnd(range);
@ -284,3 +342,97 @@ ClangIndex::ClangIndex(int exclude_declarations_from_pch,
ClangIndex::~ClangIndex() {
clang_disposeIndex(cx_index);
}
// static
std::unique_ptr<ClangTranslationUnit> ClangTranslationUnit::Create(
ClangIndex* index,
const std::string& filepath,
const std::vector<std::string>& arguments,
std::vector<CXUnsavedFile>& unsaved_files,
unsigned flags) {
std::vector<const char*> args;
for (auto& arg : arguments)
args.push_back(arg.c_str());
CXTranslationUnit cx_tu;
CXErrorCode error_code;
{
error_code = clang_parseTranslationUnit2FullArgv(
index->cx_index, nullptr, args.data(), (int)args.size(),
unsaved_files.data(), (unsigned)unsaved_files.size(), flags, &cx_tu);
}
if (error_code != CXError_Success && cx_tu)
EmitDiagnostics(filepath, args, cx_tu);
// We sometimes dump the command to logs and ask the user to run it. Include
// -fsyntax-only so they don't do a full compile.
auto make_msg = [&]() {
return "Please try running the following, identify which flag causes the "
"issue, and report a bug. ccls will then filter the flag for you "
" automatically:\n " +
StringJoin(args, " ") + " -fsyntax-only";
};
switch (error_code) {
case CXError_Success:
return std::make_unique<ClangTranslationUnit>(cx_tu);
case CXError_Failure:
LOG_S(ERROR) << "libclang generic failure for " << filepath << ". "
<< make_msg();
return nullptr;
case CXError_Crashed:
LOG_S(ERROR) << "libclang crashed for " << filepath << ". " << make_msg();
return nullptr;
case CXError_InvalidArguments:
LOG_S(ERROR) << "libclang had invalid arguments for " << filepath << ". "
<< make_msg();
return nullptr;
case CXError_ASTReadError:
LOG_S(ERROR) << "libclang had ast read error for " << filepath << ". "
<< make_msg();
return nullptr;
}
return nullptr;
}
// static
std::unique_ptr<ClangTranslationUnit> ClangTranslationUnit::Reparse(
std::unique_ptr<ClangTranslationUnit> tu,
std::vector<CXUnsavedFile>& unsaved) {
int error_code;
{
error_code = clang_reparseTranslationUnit(
tu->cx_tu, (unsigned)unsaved.size(), unsaved.data(),
clang_defaultReparseOptions(tu->cx_tu));
}
if (error_code != CXError_Success && tu->cx_tu)
EmitDiagnostics("<unknown>", {}, tu->cx_tu);
switch (error_code) {
case CXError_Success:
return tu;
case CXError_Failure:
LOG_S(ERROR) << "libclang reparse generic failure";
return nullptr;
case CXError_Crashed:
LOG_S(ERROR) << "libclang reparse crashed";
return nullptr;
case CXError_InvalidArguments:
LOG_S(ERROR) << "libclang reparse had invalid arguments";
return nullptr;
case CXError_ASTReadError:
LOG_S(ERROR) << "libclang reparse had ast read error";
return nullptr;
}
return nullptr;
}
ClangTranslationUnit::ClangTranslationUnit(CXTranslationUnit tu) : cx_tu(tu) {}
ClangTranslationUnit::~ClangTranslationUnit() {
clang_disposeTranslationUnit(cx_tu);
}

25
src/clang_cursor.h → src/clang_tu.h
View File

@ -1,12 +1,10 @@
#pragma once
#include "nt_string.h"
#include "position.h"
#include <clang-c/Index.h>
#include <optional>
#include <array>
#include <memory>
#include <string>
#include <vector>
@ -119,3 +117,24 @@ class ClangIndex {
~ClangIndex();
CXIndex cx_index;
};
// RAII wrapper around CXTranslationUnit which also makes it much more
// challenging to use a CXTranslationUnit instance that is not correctly
// initialized.
struct ClangTranslationUnit {
static std::unique_ptr<ClangTranslationUnit> Create(
ClangIndex* index,
const std::string& filepath,
const std::vector<std::string>& arguments,
std::vector<CXUnsavedFile>& unsaved_files,
unsigned flags);
static std::unique_ptr<ClangTranslationUnit> Reparse(
std::unique_ptr<ClangTranslationUnit> tu,
std::vector<CXUnsavedFile>& unsaved);
explicit ClangTranslationUnit(CXTranslationUnit tu);
~ClangTranslationUnit();
CXTranslationUnit cx_tu;
};

5
src/diagnostics_engine.cc
View File

@ -1,6 +1,7 @@
#include "diagnostics_engine.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include <chrono>
@ -30,6 +31,6 @@ void DiagnosticsEngine::Publish(WorkingFiles* working_files,
Out_TextDocumentPublishDiagnostics out;
out.params.uri = lsDocumentUri::FromPath(path);
out.params.diagnostics = diagnostics;
QueueManager::WriteStdout(kMethodType_TextDocumentPublishDiagnostics, out);
pipeline::WriteStdout(kMethodType_TextDocumentPublishDiagnostics, out);
}
}

2
src/include_complete.cc
View File

@ -107,7 +107,7 @@ void IncludeComplete::Rescan() {
is_scanning = true;
std::thread([this]() {
set_thread_name("scan_includes");
set_thread_name("include");
Timer timer;
for (const std::string& dir : project_->quote_include_directories)

2
src/clang_indexer.cc → src/indexer.cc
View File

@ -1,7 +1,5 @@
#include "indexer.h"
#include "clang_cursor.h"
#include "clang_utils.h"
#include "log.hh"
#include "platform.h"
#include "serializer.h"

3
src/indexer.h
View File

@ -1,7 +1,6 @@
#pragma once
#include "clang_cursor.h"
#include "clang_translation_unit.h"
#include "clang_tu.h"
#include "clang_utils.h"
#include "file_consumer.h"
#include "language.h"

10
src/main.cc
View File

@ -5,6 +5,7 @@
#include "serializers/json.h"
#include "test.h"
#include "working_files.h"
using namespace ccls;
#include <llvm/Support/CommandLine.h>
#include <llvm/Support/Process.h>
@ -56,8 +57,7 @@ int main(int argc, char** argv) {
return 0;
}
MultiQueueWaiter querydb_waiter, indexer_waiter, stdout_waiter;
QueueManager::Init(&querydb_waiter, &indexer_waiter, &stdout_waiter);
pipeline::Init();
#ifdef _WIN32
// We need to write to stdout in binary mode because in Windows, writing
@ -132,11 +132,11 @@ int main(int argc, char** argv) {
std::unordered_map<MethodType, Timer> request_times;
// The thread that reads from stdin and dispatchs commands to the main thread.
LaunchStdinLoop(&request_times);
pipeline::LaunchStdin(&request_times);
// The thread that writes responses from the main thread to stdout.
LaunchStdoutThread(&request_times, &stdout_waiter);
pipeline::LaunchStdout(&request_times);
// Main thread which also spawns indexer threads upon the "initialize" request.
MainLoop(&querydb_waiter, &indexer_waiter);
pipeline::MainLoop();
}
return 0;

5
src/match.cc
View File

@ -1,7 +1,8 @@
#include "match.h"
#include "lsp.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include <doctest/doctest.h>
@ -32,7 +33,7 @@ std::optional<Matcher> Matcher::Create(const std::string& search) {
out.params.type = lsMessageType::Error;
out.params.message = "ccls: Parsing EMCAScript regex \"" + search +
"\" failed; " + e.what();
QueueManager::WriteStdout(kMethodType_Unknown, out);
pipeline::WriteStdout(kMethodType_Unknown, out);
return std::nullopt;
}
}

9
src/message_handler.cc
View File

@ -4,7 +4,8 @@
#include "log.hh"
#include "project.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include <algorithm>
@ -167,7 +168,7 @@ bool FindFileOrFail(QueryDatabase* db,
out.error.code = lsErrorCodes::InternalError;
out.error.message = "Unable to find file " + absolute_path;
}
QueueManager::WriteStdout(kMethodType_Unknown, out);
pipeline::WriteStdout(kMethodType_Unknown, out);
}
return false;
@ -182,7 +183,7 @@ void EmitInactiveLines(WorkingFile* working_file,
if (ls_skipped)
out.params.inactiveRegions.push_back(*ls_skipped);
}
QueueManager::WriteStdout(kMethodType_CclsPublishInactiveRegions, out);
pipeline::WriteStdout(kMethodType_CclsPublishInactiveRegions, out);
}
void EmitSemanticHighlighting(QueryDatabase* db,
@ -343,5 +344,5 @@ void EmitSemanticHighlighting(QueryDatabase* db,
for (auto& entry : grouped_symbols)
if (entry.second.ranges.size())
out.params.symbols.push_back(entry.second);
QueueManager::WriteStdout(kMethodType_CclsPublishSemanticHighlighting, out);
pipeline::WriteStdout(kMethodType_CclsPublishSemanticHighlighting, out);
}

5
src/messages/ccls_base.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
namespace {
@ -43,7 +44,7 @@ struct Handler_CclsBase : BaseMessageHandler<In_CclsBase> {
break;
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsBase);

6
src/messages/ccls_call_hierarchy.cc
View File

@ -1,6 +1,8 @@
#include "message_handler.h"
#include "pipeline.hh"
using namespace ccls;
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
#include <unordered_set>
@ -225,7 +227,7 @@ struct Handler_CclsCallHierarchy
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsCallHierarchy);

5
src/messages/ccls_callers.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
namespace {
MethodType kMethodType = "$ccls/callers";
@ -39,7 +40,7 @@ struct Handler_CclsCallers : BaseMessageHandler<In_CclsCallers> {
break;
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsCallers);

5
src/messages/ccls_file_info.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
MAKE_REFLECT_STRUCT(QueryFile::Def,
path,
@ -49,7 +50,7 @@ struct Handler_CclsFileInfo : BaseMessageHandler<In_CclsFileInfo> {
out.result.language = file->def->language;
out.result.includes = file->def->includes;
out.result.inactive_regions = file->def->inactive_regions;
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsFileInfo);

7
src/messages/ccls_freshen_index.cc
View File

@ -1,12 +1,11 @@
#include "cache_manager.h"
#include "match.h"
#include "message_handler.h"
#include "pipeline.hh"
#include "platform.h"
#include "project.h"
#include "queue_manager.h"
#include "pipeline.hh"
#include "timer.h"
#include "working_files.h"
using namespace ccls;
#include <queue>
#include <unordered_set>
@ -79,7 +78,7 @@ struct Handler_CclsFreshenIndex : BaseMessageHandler<In_CclsFreshenIndex> {
}
// Send index requests for every file.
project->Index(QueueManager::instance(), working_files, lsRequestId());
project->Index(working_files, lsRequestId());
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsFreshenIndex);

13
src/messages/ccls_inheritance_hierarchy.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include <unordered_set>
@ -95,7 +96,7 @@ bool ExpandHelper(MessageHandler* m,
if (derived) {
if (levels > 0) {
for (auto usr : entity.derived) {
if (seen.insert(usr).second)
if (!seen.insert(usr).second)
continue;
Out_CclsInheritanceHierarchy::Entry entry1;
entry1.id = std::to_string(usr);
@ -110,7 +111,7 @@ bool ExpandHelper(MessageHandler* m,
} else {
if (levels > 0) {
for (auto usr : def->bases) {
if (seen.insert(usr).second)
if (!seen.insert(usr).second)
continue;
Out_CclsInheritanceHierarchy::Entry entry1;
entry1.id = std::to_string(usr);
@ -180,17 +181,15 @@ struct Handler_CclsInheritanceHierarchy
WorkingFile* wfile =
working_files->GetFileByFilename(file->def->path);
for (SymbolRef sym :
FindSymbolsAtLocation(wfile, file, params.position)) {
for (SymbolRef sym : FindSymbolsAtLocation(wfile, file, params.position))
if (sym.kind == SymbolKind::Func || sym.kind == SymbolKind::Type) {
out.result = BuildInitial(sym, params.derived, params.qualified,
params.levels);
break;
}
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsInheritanceHierarchy);

5
src/messages/ccls_member_hierarchy.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
#include <unordered_set>
@ -264,7 +265,7 @@ struct Handler_CclsMemberHierarchy
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsMemberHierarchy);

5
src/messages/ccls_random.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include <math.h>
#include <stdlib.h>
@ -126,7 +127,7 @@ struct Handler_CclsRandom : BaseMessageHandler<In_CclsRandom> {
break;
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsRandom);

5
src/messages/ccls_vars.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "$ccls/vars";
@ -48,7 +49,7 @@ struct Handler_CclsVars : BaseMessageHandler<In_CclsVars> {
break;
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsVars);

9
src/messages/initialize.cc
View File

@ -1,4 +1,3 @@
#include "cache_manager.h"
#include "diagnostics_engine.h"
#include "filesystem.hh"
#include "include_complete.h"
@ -7,10 +6,10 @@
#include "pipeline.hh"
#include "platform.h"
#include "project.h"
#include "queue_manager.h"
#include "serializers/json.h"
#include "timer.h"
#include "working_files.h"
using namespace ccls;
#include <llvm/ADT/Twine.h>
#include <llvm/Support/Threading.h>
@ -482,7 +481,7 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
Out_InitializeResponse out;
out.id = request->id;
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
// Set project root.
EnsureEndsInSlash(project_path);
@ -515,7 +514,7 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
g_thread_id = i + 1;
std::string name = "indexer" + std::to_string(i);
set_thread_name(name.c_str());
Indexer_Main(diag_engine, vfs, project, working_files, waiter);
pipeline::Indexer_Main(diag_engine, vfs, project, working_files);
}).detach();
}
@ -524,7 +523,7 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
include_complete->Rescan();
time.Reset();
project->Index(QueueManager::instance(), working_files, request->id);
project->Index(working_files, request->id);
// We need to support multiple concurrent index processes.
time.ResetAndPrint("[perf] Dispatched initial index requests");
}

5
src/messages/shutdown.cc
View File

@ -1,5 +1,6 @@
#include "message_handler.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "shutdown";
@ -21,7 +22,7 @@ struct Handler_Shutdown : BaseMessageHandler<In_Shutdown> {
void Run(In_Shutdown* request) override {
Out_Shutdown out;
out.id = request->id;
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_Shutdown);

5
src/messages/text_document_code_lens.cc
View File

@ -2,7 +2,8 @@
#include "lsp_code_action.h"
#include "message_handler.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/codeLens";
@ -225,7 +226,7 @@ struct Handler_TextDocumentCodeLens
};
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentCodeLens);

9
src/messages/text_document_completion.cc
View File

@ -2,9 +2,10 @@
#include "fuzzy_match.h"
#include "include_complete.h"
#include "message_handler.h"
#include "queue_manager.h"
#include "pipeline.hh"
#include "timer.h"
#include "working_files.h"
using namespace ccls;
#include "lex_utils.h"
@ -270,7 +271,7 @@ struct Handler_TextDocumentCompletion : MessageHandler {
auto write_empty_result = [request]() {
Out_TextDocumentComplete out;
out.id = request->id;
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
};
std::string path = request->params.textDocument.uri.GetPath();
@ -377,7 +378,7 @@ struct Handler_TextDocumentCompletion : MessageHandler {
item.textEdit->range.end.character = (int)buffer_line.size();
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
} else {
ClangCompleteManager::OnComplete callback = std::bind(
[this, request, is_global_completion, existing_completion,
@ -389,7 +390,7 @@ struct Handler_TextDocumentCompletion : MessageHandler {
// Emit completion results.
FilterAndSortCompletionResponse(&out, existing_completion, has_open_paren);
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
// Cache completion results.
if (!is_cached_result) {

5
src/messages/text_document_definition.cc
View File

@ -1,7 +1,8 @@
#include "lex_utils.h"
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
#include <ctype.h>
#include <limits.h>
@ -173,7 +174,7 @@ struct Handler_TextDocumentDefinition
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentDefinition);

8
src/messages/text_document_did_change.cc
View File

@ -1,9 +1,9 @@
#include "cache_manager.h"
#include "clang_complete.h"
#include "message_handler.h"
#include "project.h"
#include "working_files.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/didChange";
@ -25,9 +25,7 @@ struct Handler_TextDocumentDidChange
working_files->OnChange(request->params);
if (g_config->index.onDidChange) {
Project::Entry entry = project->FindCompilationEntryForFile(path);
QueueManager::instance()->index_request.PushBack(
Index_Request(entry.filename, entry.args, true /*is_interactive*/),
true);
pipeline::Index(entry.filename, entry.args, true);
}
clang_complete->NotifyEdit(path);
clang_complete->DiagnosticsUpdate(

5
src/messages/text_document_did_close.cc
View File

@ -1,7 +1,8 @@
#include "clang_complete.h"
#include "message_handler.h"
#include "queue_manager.h"
#include "pipeline.hh"
#include "working_files.h"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/didClose";
@ -27,7 +28,7 @@ struct Handler_TextDocumentDidClose
// Clear any diagnostics for the file.
Out_TextDocumentPublishDiagnostics out;
out.params.uri = request->params.textDocument.uri;
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
// Remove internal state.
working_files->OnClose(request->params.textDocument);

17
src/messages/text_document_did_open.cc
View File

@ -1,9 +1,9 @@
#include "cache_manager.h"
#include "clang_complete.h"
#include "include_complete.h"
#include "message_handler.h"
#include "project.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include "timer.h"
#include "working_files.h"
@ -38,11 +38,9 @@ struct Handler_TextDocumentDidOpen
const auto& params = request->params;
std::string path = params.textDocument.uri.GetPath();
ICacheManager cache;
WorkingFile* working_file = working_files->OnOpen(params.textDocument);
std::optional<std::string> cached_file_contents =
cache.LoadCachedFileContents(path);
if (cached_file_contents)
if (std::optional<std::string> cached_file_contents =
pipeline::LoadCachedFileContents(path))
working_file->SetIndexContent(*cached_file_contents);
QueryFile* file = nullptr;
@ -60,11 +58,8 @@ struct Handler_TextDocumentDidOpen
// Submit new index request if it is not a header file.
if (SourceFileLanguage(path) != LanguageId::Unknown) {
Project::Entry entry = project->FindCompilationEntryForFile(path);
QueueManager::instance()->index_request.PushBack(
Index_Request(entry.filename,
params.args.size() ? params.args : entry.args,
true /*is_interactive*/),
true /* priority */);
pipeline::Index(entry.filename,
params.args.size() ? params.args : entry.args, true);
clang_complete->FlushSession(entry.filename);
}

11
src/messages/text_document_did_save.cc
View File

@ -1,8 +1,8 @@
#include "cache_manager.h"
#include "clang_complete.h"
#include "message_handler.h"
#include "project.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/didSave";
@ -29,7 +29,8 @@ struct Handler_TextDocumentDidSave
MethodType GetMethodType() const override { return kMethodType; }
void Run(In_TextDocumentDidSave* request) override {
std::string path = request->params.textDocument.uri.GetPath();
const auto& params = request->params;
std::string path = params.textDocument.uri.GetPath();
// Send out an index request, and copy the current buffer state so we
// can update the cached index contents when the index is done.
@ -47,9 +48,7 @@ struct Handler_TextDocumentDidSave
// TODO: send as priority request
if (!g_config->index.onDidChange) {
Project::Entry entry = project->FindCompilationEntryForFile(path);
QueueManager::instance()->index_request.PushBack(
Index_Request(entry.filename, entry.args, true /*is_interactive*/),
true);
pipeline::Index(entry.filename, entry.args, true);
}
clang_complete->NotifySave(path);

5
src/messages/text_document_document_highlight.cc
View File

@ -1,7 +1,8 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
#include "symbol.h"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/documentHighlight";
@ -61,7 +62,7 @@ struct Handler_TextDocumentDocumentHighlight
break;
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentDocumentHighlight);

5
src/messages/text_document_document_symbol.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/documentSymbol";
@ -64,7 +65,7 @@ struct Handler_TextDocumentDocumentSymbol
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentDocumentSymbol);

5
src/messages/text_document_hover.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/hover";
@ -99,7 +100,7 @@ struct Handler_TextDocumentHover : BaseMessageHandler<In_TextDocumentHover> {
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentHover);

5
src/messages/text_document_implementation.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/implementation";
@ -41,7 +42,7 @@ struct Handler_TextDocumentImplementation
break;
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentImplementation);

5
src/messages/text_document_range_formatting.cc
View File

@ -1,7 +1,8 @@
#include "clang_format.h"
#include "lex_utils.h"
#include "message_handler.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include "working_files.h"
#include <loguru.hpp>
@ -64,7 +65,7 @@ struct Handler_TextDocumentRangeFormatting
response.result = {};
#endif
QueueManager::WriteStdout(kMethodType, response);
pipeline::WriteStdout(kMethodType, response);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentRangeFormatting);

5
src/messages/text_document_references.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
#include <unordered_set>
@ -135,7 +136,7 @@ struct Handler_TextDocumentReferences
if ((int)out.result.size() >= g_config->xref.maxNum)
out.result.resize(g_config->xref.maxNum);
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentReferences);

5
src/messages/text_document_rename.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/rename";
@ -102,7 +103,7 @@ struct Handler_TextDocumentRename : BaseMessageHandler<In_TextDocumentRename> {
break;
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentRename);

5
src/messages/text_document_signature_help.cc
View File

@ -1,6 +1,7 @@
#include "clang_complete.h"
#include "message_handler.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include "timer.h"
#include <stdint.h>
@ -142,7 +143,7 @@ struct Handler_TextDocumentSignatureHelp : MessageHandler {
out.result.activeParameter = active_param;
Timer timer;
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
if (!is_cached_result) {
signature_cache->WithLock([&]() {

5
src/messages/text_document_type_definition.cc
View File

@ -1,6 +1,7 @@
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
namespace {
MethodType kMethodType = "textDocument/typeDefinition";
@ -60,7 +61,7 @@ struct Handler_TextDocumentTypeDefinition
}
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_TextDocumentTypeDefinition);

6
src/messages/workspace_did_change_configuration.cc
View File

@ -1,8 +1,8 @@
#include "cache_manager.h"
#include "clang_complete.h"
#include "message_handler.h"
#include "project.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
#include "timer.h"
#include "working_files.h"
@ -31,7 +31,7 @@ struct Handler_WorkspaceDidChangeConfiguration
std::to_string(project->entries.size()) + " files)");
time.Reset();
project->Index(QueueManager::instance(), working_files, lsRequestId());
project->Index(working_files, lsRequestId());
time.ResetAndPrint(
"[perf] Dispatched workspace/didChangeConfiguration index requests");

10
src/messages/workspace_did_change_watched_files.cc
View File

@ -1,9 +1,9 @@
#include "cache_manager.h"
#include "clang_complete.h"
#include "message_handler.h"
#include "project.h"
#include "queue_manager.h"
#include "pipeline.hh"
#include "working_files.h"
using namespace ccls;
namespace {
MethodType kMethodType = "workspace/didChangeWatchedFiles";
@ -52,15 +52,13 @@ struct Handler_WorkspaceDidChangeWatchedFiles
switch (event.type) {
case lsFileChangeType::Created:
case lsFileChangeType::Changed: {
QueueManager::instance()->index_request.PushBack(
Index_Request(path, entry.args, is_interactive));
pipeline::Index(path, entry.args, is_interactive);
if (is_interactive)
clang_complete->NotifySave(path);
break;
}
case lsFileChangeType::Deleted:
QueueManager::instance()->index_request.PushBack(
Index_Request(path, entry.args, is_interactive));
pipeline::Index(path, entry.args, is_interactive);
break;
}
}

5
src/messages/workspace_execute_command.cc
View File

@ -1,7 +1,8 @@
#include "lsp_code_action.h"
#include "message_handler.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
using namespace ccls;
namespace {
MethodType kMethodType = "workspace/executeCommand";
@ -34,7 +35,7 @@ struct Handler_WorkspaceExecuteCommand
out.result = params.arguments.locations;
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_WorkspaceExecuteCommand);

5
src/messages/workspace_symbol.cc
View File

@ -1,8 +1,9 @@
#include "fuzzy_match.h"
#include "lex_utils.h"
#include "message_handler.h"
#include "pipeline.hh"
#include "query_utils.h"
#include "queue_manager.h"
using namespace ccls;
#include <ctype.h>
#include <limits.h>
@ -124,7 +125,7 @@ struct Handler_WorkspaceSymbol : BaseMessageHandler<In_WorkspaceSymbol> {
out.result.push_back(std::get<0>(entry));
}
QueueManager::WriteStdout(kMethodType, out);
pipeline::WriteStdout(kMethodType, out);
}
};
REGISTER_MESSAGE_HANDLER(Handler_WorkspaceSymbol);

183
src/pipeline.cc
View File

@ -1,6 +1,5 @@
#include "pipeline.hh"
#include "cache_manager.h"
#include "clang_complete.h"
#include "config.h"
#include "diagnostics_engine.h"
@ -11,18 +10,43 @@
#include "platform.h"
#include "project.h"
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
#include "timer.h"
#include <llvm/ADT/Twine.h>
#include <llvm/Support/Threading.h>
using namespace llvm;
#include <chrono>
#include <thread>
struct Index_Request {
std::string path;
std::vector<std::string> args;
bool is_interactive;
lsRequestId id;
};
struct Index_OnIndexed {
IndexUpdate update;
PerformanceImportFile perf;
};
struct Stdout_Request {
MethodType method;
std::string content;
};
namespace ccls::pipeline {
namespace {
MultiQueueWaiter* main_waiter;
MultiQueueWaiter* indexer_waiter;
MultiQueueWaiter* stdout_waiter;
ThreadedQueue<std::unique_ptr<InMessage>>* on_request;
ThreadedQueue<Index_Request>* index_request;
ThreadedQueue<Index_OnIndexed>* on_indexed;
ThreadedQueue<Stdout_Request>* for_stdout;
// Checks if |path| needs to be reparsed. This will modify cached state
// such that calling this function twice with the same path may return true
// the first time but will return false the second.
@ -64,24 +88,57 @@ bool FileNeedsParse(int64_t write_time,
return false;
};
std::string AppendSerializationFormat(const std::string& base) {
switch (g_config->cacheFormat) {
case SerializeFormat::Binary:
return base + ".blob";
case SerializeFormat::Json:
return base + ".json";
}
}
std::string GetCachePath(const std::string& source_file) {
std::string cache_file;
size_t len = g_config->projectRoot.size();
if (StartsWith(source_file, g_config->projectRoot)) {
cache_file = EscapeFileName(g_config->projectRoot) +
EscapeFileName(source_file.substr(len));
} else {
cache_file = '@' + EscapeFileName(g_config->projectRoot) +
EscapeFileName(source_file);
}
return g_config->cacheDirectory + cache_file;
}
std::unique_ptr<IndexFile> RawCacheLoad(
const std::string& path) {
std::string cache_path = GetCachePath(path);
std::optional<std::string> file_content = ReadContent(cache_path);
std::optional<std::string> serialized_indexed_content =
ReadContent(AppendSerializationFormat(cache_path));
if (!file_content || !serialized_indexed_content)
return nullptr;
return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content,
*file_content, IndexFile::kMajorVersion);
}
bool Indexer_Parse(DiagnosticsEngine* diag_engine,
WorkingFiles* working_files,
Project* project,
VFS* vfs,
ClangIndexer* indexer) {
auto* queue = QueueManager::instance();
std::optional<Index_Request> opt_request = queue->index_request.TryPopFront();
std::optional<Index_Request> opt_request = index_request->TryPopFront();
if (!opt_request)
return false;
auto& request = *opt_request;
ICacheManager cache;
// Dummy one to trigger refresh semantic highlight.
if (request.path.empty()) {
IndexUpdate dummy;
dummy.refresh = true;
queue->on_indexed.PushBack(
Index_OnIndexed(std::move(dummy), PerformanceImportFile()), false);
on_indexed->PushBack({std::move(dummy), PerformanceImportFile()}, false);
return false;
}
@ -108,7 +165,7 @@ bool Indexer_Parse(DiagnosticsEngine* diag_engine,
return true;
int reparse; // request.is_interactive;
prev = cache.RawCacheLoad(path_to_index);
prev = RawCacheLoad(path_to_index);
if (!prev)
reparse = 2;
else {
@ -132,15 +189,13 @@ bool Indexer_Parse(DiagnosticsEngine* diag_engine,
auto dependencies = prev->dependencies;
if (reparse) {
IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf),
request.is_interactive);
on_indexed->PushBack({std::move(update), perf}, request.is_interactive);
}
for (const auto& dep : dependencies)
if (vfs->Mark(dep.first().str(), 0, 2)) {
prev = cache.RawCacheLoad(dep.first().str());
prev = RawCacheLoad(dep.first().str());
IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf),
request.is_interactive);
on_indexed->PushBack({std::move(update), perf}, request.is_interactive);
}
std::lock_guard<std::mutex> lock(vfs->mutex);
@ -152,7 +207,6 @@ bool Indexer_Parse(DiagnosticsEngine* diag_engine,
LOG_S(INFO) << "parse " << path_to_index;
std::vector<Index_OnIdMapped> result;
PerformanceImportFile perf;
auto indexes = indexer->Index(vfs, path_to_index, entry.args, {}, &perf);
@ -162,7 +216,7 @@ bool Indexer_Parse(DiagnosticsEngine* diag_engine,
out.id = request.id;
out.error.code = lsErrorCodes::InternalError;
out.error.message = "Failed to index " + path_to_index;
QueueManager::WriteStdout(kMethodType_Unknown, out);
pipeline::WriteStdout(kMethodType_Unknown, out);
}
vfs->Reset(path_to_index);
return true;
@ -179,12 +233,17 @@ bool Indexer_Parse(DiagnosticsEngine* diag_engine,
if (!(vfs->Stamp(path, curr->last_write_time) || path == path_to_index))
continue;
LOG_S(INFO) << "emit index for " << path;
prev = cache.RawCacheLoad(path);
prev = RawCacheLoad(path);
// Write current index to disk if requested.
LOG_S(INFO) << "store index for " << path;
Timer time;
cache.WriteToCache(*curr);
{
std::string cache_path = GetCachePath(path);
WriteToFile(cache_path, curr->file_contents);
WriteToFile(AppendSerializationFormat(cache_path),
Serialize(g_config->cacheFormat, *curr));
}
perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
vfs->Reset(path_to_index);
@ -199,8 +258,7 @@ bool Indexer_Parse(DiagnosticsEngine* diag_engine,
perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
LOG_S(INFO) << "built index for " << path << " (is_delta=" << !!prev << ")";
Index_OnIndexed reply(std::move(update), perf);
queue->on_indexed.PushBack(std::move(reply), request.is_interactive);
on_indexed->PushBack({std::move(update), perf}, request.is_interactive);
}
return true;
@ -217,26 +275,34 @@ bool ShouldDisplayMethodTiming(MethodType type) {
} // namespace
void Init() {
main_waiter = new MultiQueueWaiter;
on_request = new ThreadedQueue<std::unique_ptr<InMessage>>(main_waiter);
on_indexed = new ThreadedQueue<Index_OnIndexed>(main_waiter);
indexer_waiter = new MultiQueueWaiter;
index_request = new ThreadedQueue<Index_Request>(indexer_waiter);
stdout_waiter = new MultiQueueWaiter;
for_stdout = new ThreadedQueue<Stdout_Request>(stdout_waiter);
}
void Indexer_Main(DiagnosticsEngine* diag_engine,
VFS* vfs,
Project* project,
WorkingFiles* working_files,
MultiQueueWaiter* waiter) {
auto* queue = QueueManager::instance();
WorkingFiles* working_files) {
// Build one index per-indexer, as building the index acquires a global lock.
ClangIndexer indexer;
while (true)
if (!Indexer_Parse(diag_engine, working_files, project, vfs, &indexer))
waiter->Wait(&queue->index_request);
indexer_waiter->Wait(index_request);
}
void QueryDb_OnIndexed(QueueManager* queue,
QueryDatabase* db,
void Main_OnIndexed(QueryDatabase* db,
SemanticHighlightSymbolCache* semantic_cache,
WorkingFiles* working_files,
Index_OnIndexed* response) {
if (response->update.refresh) {
LOG_S(INFO) << "Loaded project. Refresh semantic highlight for all working file.";
std::lock_guard<std::mutex> lock(working_files->files_mutex);
@ -257,9 +323,8 @@ void QueryDb_OnIndexed(QueueManager* queue,
if (response->update.files_def_update) {
auto& update = *response->update.files_def_update;
time.ResetAndPrint("apply index for " + update.value.path);
WorkingFile* working_file =
working_files->GetFileByFilename(update.value.path);
if (working_file) {
if (WorkingFile* working_file =
working_files->GetFileByFilename(update.value.path)) {
// Update indexed content.
working_file->SetIndexContent(update.file_content);
@ -275,10 +340,9 @@ void QueryDb_OnIndexed(QueueManager* queue,
}
}
void LaunchStdinLoop(std::unordered_map<MethodType, Timer>* request_times) {
void LaunchStdin(std::unordered_map<MethodType, Timer>* request_times) {
std::thread([request_times]() {
set_thread_name("stdin");
auto* queue = QueueManager::instance();
while (true) {
std::unique_ptr<InMessage> message;
std::optional<std::string> err =
@ -295,7 +359,7 @@ void LaunchStdinLoop(std::unordered_map<MethodType, Timer>* request_times) {
out.id = id;
out.error.code = lsErrorCodes::InvalidParams;
out.error.message = std::move(*err);
queue->WriteStdout(kMethodType_Unknown, out);
WriteStdout(kMethodType_Unknown, out);
}
}
continue;
@ -305,7 +369,7 @@ void LaunchStdinLoop(std::unordered_map<MethodType, Timer>* request_times) {
MethodType method_type = message->GetMethodType();
(*request_times)[method_type] = Timer();
queue->for_querydb.PushBack(std::move(message));
on_request->PushBack(std::move(message));
// If the message was to exit then querydb will take care of the actual
// exit. Stop reading from stdin since it might be detached.
@ -315,16 +379,14 @@ void LaunchStdinLoop(std::unordered_map<MethodType, Timer>* request_times) {
}).detach();
}
void LaunchStdoutThread(std::unordered_map<MethodType, Timer>* request_times,
MultiQueueWaiter* waiter) {
void LaunchStdout(std::unordered_map<MethodType, Timer>* request_times) {
std::thread([=]() {
set_thread_name("stdout");
auto* queue = QueueManager::instance();
while (true) {
std::vector<Stdout_Request> messages = queue->for_stdout.DequeueAll();
std::vector<Stdout_Request> messages = for_stdout->DequeueAll();
if (messages.empty()) {
waiter->Wait(&queue->for_stdout);
stdout_waiter->Wait(for_stdout);
continue;
}
@ -341,8 +403,7 @@ void LaunchStdoutThread(std::unordered_map<MethodType, Timer>* request_times,
}).detach();
}
void MainLoop(MultiQueueWaiter* querydb_waiter,
MultiQueueWaiter* indexer_waiter) {
void MainLoop() {
Project project;
SemanticHighlightSymbolCache semantic_cache;
WorkingFiles working_files;
@ -362,7 +423,7 @@ void MainLoop(MultiQueueWaiter* querydb_waiter,
out.error.message =
"Dropping completion request; a newer request "
"has come in that will be serviced instead.";
QueueManager::WriteStdout(kMethodType_Unknown, out);
pipeline::WriteStdout(kMethodType_Unknown, out);
}
});
@ -389,11 +450,8 @@ void MainLoop(MultiQueueWaiter* querydb_waiter,
handler->signature_cache = signature_cache.get();
}
// Run query db main loop.
auto* queue = QueueManager::instance();
while (true) {
std::vector<std::unique_ptr<InMessage>> messages =
queue->for_querydb.DequeueAll();
std::vector<std::unique_ptr<InMessage>> messages = on_request->DequeueAll();
bool did_work = messages.size();
for (auto& message : messages) {
// TODO: Consider using std::unordered_map to lookup the handler
@ -409,19 +467,40 @@ void MainLoop(MultiQueueWaiter* querydb_waiter,
}
for (int i = 80; i--;) {
std::optional<Index_OnIndexed> response = queue->on_indexed.TryPopFront();
std::optional<Index_OnIndexed> response = on_indexed->TryPopFront();
if (!response)
break;
did_work = true;
QueryDb_OnIndexed(queue, &db, &semantic_cache, &working_files, &*response);
Main_OnIndexed(&db, &semantic_cache, &working_files, &*response);
}
// Cleanup and free any unused memory.
FreeUnusedMemory();
if (!did_work) {
auto* queue = QueueManager::instance();
querydb_waiter->Wait(&queue->on_indexed, &queue->for_querydb);
}
if (!did_work)
main_waiter->Wait(on_indexed, on_request);
}
}
void Index(const std::string& path,
const std::vector<std::string>& args,
bool interactive,
lsRequestId id) {
index_request->PushBack({path, args, interactive, id}, interactive);
}
std::optional<std::string> LoadCachedFileContents(const std::string& path) {
return ReadContent(GetCachePath(path));
}
void WriteStdout(MethodType method, lsBaseOutMessage& response) {
std::ostringstream sstream;
response.Write(sstream);
Stdout_Request out;
out.content = sstream.str();
out.method = method;
for_stdout->PushBack(std::move(out));
}
}

33
src/pipeline.hh
View File

@ -1,28 +1,35 @@
#pragma once
#include "queue_manager.h"
#include "method.h"
#include "query.h"
#include "timer.h"
#include <atomic>
#include <string>
#include <unordered_map>
#include <vector>
struct ClangTranslationUnit;
class DiagnosticsEngine;
struct VFS;
struct ICacheManager;
struct Project;
struct QueryDatabase;
struct SemanticHighlightSymbolCache;
struct WorkingFiles;
struct lsBaseOutMessage;
namespace ccls::pipeline {
void Init();
void LaunchStdin(std::unordered_map<MethodType, Timer>* request_times);
void LaunchStdout(std::unordered_map<MethodType, Timer>* request_times);
void Indexer_Main(DiagnosticsEngine* diag_engine,
VFS* vfs,
Project* project,
WorkingFiles* working_files,
MultiQueueWaiter* waiter);
WorkingFiles* working_files);
void MainLoop();
void LaunchStdinLoop(std::unordered_map<MethodType, Timer>* request_times);
void LaunchStdoutThread(std::unordered_map<MethodType, Timer>* request_times,
MultiQueueWaiter* waiter);
void MainLoop(MultiQueueWaiter* querydb_waiter,
MultiQueueWaiter* indexer_waiter);
void Index(const std::string& path,
const std::vector<std::string>& args,
bool is_interactive,
lsRequestId id = {});
std::optional<std::string> LoadCachedFileContents(const std::string& path);
void WriteStdout(MethodType method, lsBaseOutMessage& response);
}

10
src/port.cc
View File

@ -1,10 +0,0 @@
#include "port.h"
#include <stdio.h>
#include <stdlib.h>
void ccls_unreachable_internal(const char* msg, const char* file, int line) {
fprintf(stderr, "unreachable %s:%d %s\n", file, line, msg);
CCLS_BUILTIN_UNREACHABLE;
abort();
}

34
src/port.h
View File

@ -1,34 +0,0 @@
#pragma once
#ifndef __has_builtin
#define __has_builtin(x) 0
#endif
#if defined(__GNUC__)
#define ATTRIBUTE_UNUSED __attribute__((unused))
#else
#define ATTRIBUTE_UNUSED
#endif
#ifdef __clang__
#define GUARDED_BY(x) __attribute__((guarded_by(x)))
#else
#define GUARDED_BY(x)
#endif
// TODO GCC
#if __has_builtin(__builtin_unreachable)
#define CCLS_BUILTIN_UNREACHABLE __builtin_unreachable()
#elif defined(_MSC_VER)
#define CCLS_BUILTIN_UNREACHABLE __assume(false)
#else
#define CCLS_BUILTIN_UNREACHABLE
#endif
void ccls_unreachable_internal(const char* msg, const char* file, int line);
#ifndef NDEBUG
#define CCLS_UNREACHABLE(msg) \
ccls_unreachable_internal(msg, __FILE__, __LINE__)
#else
#define CCLS_UNREACHABLE(msg)
#endif

12
src/project.cc
View File

@ -1,17 +1,17 @@
#include "project.h"
#include "cache_manager.h"
#include "clang_utils.h"
#include "filesystem.hh"
#include "language.h"
#include "log.hh"
#include "match.h"
#include "platform.h"
#include "queue_manager.h"
#include "pipeline.hh"
#include "serializers/json.h"
#include "timer.h"
#include "utils.h"
#include "working_files.h"
using namespace ccls;
#include <clang/Driver/Options.h>
#include <llvm/ADT/ArrayRef.h>
@ -454,17 +454,15 @@ void Project::ForAllFilteredFiles(
}
}
void Project::Index(QueueManager* queue,
WorkingFiles* wfiles,
void Project::Index(WorkingFiles* wfiles,
lsRequestId id) {
ForAllFilteredFiles([&](int i, const Project::Entry& entry) {
bool is_interactive = wfiles->GetFileByFilename(entry.filename) != nullptr;
queue->index_request.PushBack(
Index_Request(entry.filename, entry.args, is_interactive, id));
pipeline::Index(entry.filename, entry.args, is_interactive, id);
});
// Dummy request to indicate that project is loaded and
// trigger refreshing semantic highlight for all working files.
queue->index_request.PushBack(Index_Request("", {}, false));
pipeline::Index("", {}, false);
}
TEST_SUITE("Project") {

6
src/project.h
View File

@ -5,12 +5,10 @@
#include <functional>
#include <mutex>
#include <optional>
#include <string>
#include <unordered_map>
#include <vector>
class QueueManager;
struct WorkingFiles;
struct Project {
@ -29,7 +27,7 @@ struct Project {
std::vector<Entry> entries;
std::mutex mutex_;
std::unordered_map<std::string, int> absolute_path_to_entry_index_ GUARDED_BY(mutex_);
std::unordered_map<std::string, int> absolute_path_to_entry_index_;
// Loads a project for the given |directory|.
//
@ -58,5 +56,5 @@ struct Project {
void ForAllFilteredFiles(
std::function<void(int i, const Entry& entry)> action);
void Index(QueueManager* queue, WorkingFiles* wfiles, lsRequestId id);
void Index(WorkingFiles* wfiles, lsRequestId id);
};

2
src/query_utils.cc
View File

@ -1,6 +1,6 @@
#include "query_utils.h"
#include "queue_manager.h"
#include "pipeline.hh"
#include <limits.h>
#include <unordered_set>

46
src/queue_manager.cc
View File

@ -1,46 +0,0 @@
#include "queue_manager.h"
#include "cache_manager.h"
#include "lsp.h"
#include "query.h"
#include <sstream>
Index_Request::Index_Request(const std::string& path,
const std::vector<std::string>& args,
bool is_interactive,
lsRequestId id)
: path(path), args(args), is_interactive(is_interactive), id(id) {}
Index_OnIndexed::Index_OnIndexed(IndexUpdate&& update,
PerformanceImportFile perf)
: update(std::move(update)), perf(perf) {}
std::unique_ptr<QueueManager> QueueManager::instance_;
// static
void QueueManager::Init(MultiQueueWaiter* querydb_waiter,
MultiQueueWaiter* indexer_waiter,
MultiQueueWaiter* stdout_waiter) {
instance_ = std::unique_ptr<QueueManager>(
new QueueManager(querydb_waiter, indexer_waiter, stdout_waiter));
}
// static
void QueueManager::WriteStdout(MethodType method, lsBaseOutMessage& response) {
std::ostringstream sstream;
response.Write(sstream);
Stdout_Request out;
out.content = sstream.str();
out.method = method;
instance()->for_stdout.PushBack(std::move(out));
}
QueueManager::QueueManager(MultiQueueWaiter* querydb_waiter,
MultiQueueWaiter* indexer_waiter,
MultiQueueWaiter* stdout_waiter)
: for_stdout(stdout_waiter),
for_querydb(querydb_waiter),
on_indexed(querydb_waiter),
index_request(indexer_waiter) {}

84
src/queue_manager.h
View File

@ -1,84 +0,0 @@
#pragma once
#include "method.h"
#include "query.h"
#include "threaded_queue.h"
#include <memory>
struct ICacheManager;
struct lsBaseOutMessage;
struct Stdout_Request {
MethodType method;
std::string content;
};
struct Index_Request {
std::string path;
// TODO: make |args| a string that is parsed lazily.
std::vector<std::string> args;
bool is_interactive;
lsRequestId id;
Index_Request(const std::string& path,
const std::vector<std::string>& args,
bool is_interactive,
lsRequestId id = {});
};
struct Index_OnIdMapped {
std::shared_ptr<ICacheManager> cache_manager;
std::unique_ptr<IndexFile> previous;
std::unique_ptr<IndexFile> current;
PerformanceImportFile perf;
bool is_interactive;
bool write_to_disk;
Index_OnIdMapped(const std::shared_ptr<ICacheManager>& cache_manager,
std::unique_ptr<IndexFile> previous,
std::unique_ptr<IndexFile> current,
PerformanceImportFile perf,
bool is_interactive,
bool write_to_disk)
: cache_manager(cache_manager),
previous(std::move(previous)),
current(std::move(current)),
perf(perf),
is_interactive(is_interactive),
write_to_disk(write_to_disk) {}
};
struct Index_OnIndexed {
IndexUpdate update;
PerformanceImportFile perf;
Index_OnIndexed(IndexUpdate&& update, PerformanceImportFile perf);
};
class QueueManager {
static std::unique_ptr<QueueManager> instance_;
public:
static QueueManager* instance() { return instance_.get(); }
static void Init(MultiQueueWaiter* querydb_waiter,
MultiQueueWaiter* indexer_waiter,
MultiQueueWaiter* stdout_waiter);
static void WriteStdout(MethodType method, lsBaseOutMessage& response);
// Messages received by "stdout" thread.
ThreadedQueue<Stdout_Request> for_stdout;
// Runs on querydb thread.
ThreadedQueue<std::unique_ptr<InMessage>> for_querydb;
ThreadedQueue<Index_OnIndexed> on_indexed;
// Runs on indexer threads.
ThreadedQueue<Index_Request> index_request;
private:
explicit QueueManager(MultiQueueWaiter* querydb_waiter,
MultiQueueWaiter* indexer_waiter,
MultiQueueWaiter* stdout_waiter);
};

7
src/serializer.h
View File

@ -2,7 +2,8 @@
#include "maybe.h"
#include "nt_string.h"
#include "port.h"
#include <llvm/Support/Compiler.h>
#include <macro_map.h>
@ -84,12 +85,12 @@ struct IndexFile;
#define MAKE_REFLECT_TYPE_PROXY(type_name) \
MAKE_REFLECT_TYPE_PROXY2(type_name, std::underlying_type_t<type_name>)
#define MAKE_REFLECT_TYPE_PROXY2(type, as_type) \
ATTRIBUTE_UNUSED inline void Reflect(Reader& visitor, type& value) { \
LLVM_ATTRIBUTE_UNUSED inline void Reflect(Reader& visitor, type& value) { \
as_type value0; \
::Reflect(visitor, value0); \
value = static_cast<type>(value0); \
} \
ATTRIBUTE_UNUSED inline void Reflect(Writer& visitor, type& value) { \
LLVM_ATTRIBUTE_UNUSED inline void Reflect(Writer& visitor, type& value) { \
auto value0 = static_cast<as_type>(value); \
::Reflect(visitor, value0); \
}

2
src/test.cc
View File

@ -284,8 +284,6 @@ bool RunIndexTests(const std::string& filter_path, bool enable_update) {
&all_expected_output);
// Build flags.
if (!AnyStartsWith(flags, "-x"))
flags.push_back("-xc++");
flags.push_back("-resource-dir=" + GetDefaultResourceDirectory());
flags.push_back(path);

20
src/threaded_queue.h
View File

@ -94,26 +94,6 @@ struct ThreadedQueue : public BaseThreadQueue {
Push<&std::deque<T>::push_back>(std::move(t), priority);
}
// Add a set of elements to the queue.
void EnqueueAll(std::vector<T>&& elements, bool priority = false) {
if (elements.empty())
return;
std::lock_guard<std::mutex> lock(mutex_);
total_count_ += elements.size();
for (T& element : elements) {
if (priority)
priority_.push_back(std::move(element));
else
queue_.push_back(std::move(element));
}
elements.clear();
waiter_->cv.notify_all();
}
// Return all elements in the queue.
std::vector<T> DequeueAll() {
std::lock_guard<std::mutex> lock(mutex_);

9
src/utils.cc
View File

@ -48,15 +48,6 @@ bool StartsWith(std::string_view s, std::string_view prefix) {
std::equal(prefix.begin(), prefix.end(), s.begin());
}
bool AnyStartsWith(const std::vector<std::string>& xs,
std::string_view prefix) {
return std::any_of(xs.begin(), xs.end(), std::bind(StartsWith, _1, prefix));
}
bool StartsWithAny(std::string_view s, const std::vector<std::string>& ps) {
return std::any_of(ps.begin(), ps.end(), std::bind(StartsWith, s, _1));
}
bool EndsWithAny(std::string_view s, const std::vector<std::string>& ss) {
return std::any_of(ss.begin(), ss.end(), std::bind(EndsWith, s, _1));
}

2
src/utils.h
View File

@ -17,8 +17,6 @@ uint64_t HashUsr(std::string_view s);
// Returns true if |value| starts/ends with |start| or |ending|.
bool StartsWith(std::string_view value, std::string_view start);
bool EndsWith(std::string_view value, std::string_view ending);
bool AnyStartsWith(const std::vector<std::string>& xs, std::string_view prefix);
bool StartsWithAny(std::string_view s, const std::vector<std::string>& ps);
bool EndsWithAny(std::string_view s, const std::vector<std::string>& ss);
bool FindAnyPartial(const std::string& value,
const std::vector<std::string>& values);