Zengtudor/ccls
1
0
Fork 0
mirror of https://github.com/MaskRay/ccls.git synced 2025-11-04 06:15:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Redesign import_pipeline.cc and mitigate race (duplicate Query*::uses for initial indexing)

Browse Source
This commit is contained in:
Fangrui Song 2018-05-05 15:29:17 -07:00
parent 86efddf032
commit 990d10a605
34 changed files with 347 additions and 797 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -244,7 +244,6 @@ target_sources(ccls PRIVATE
src/messages/ccls_derived.cc
src/messages/ccls_file_info.cc
src/messages/ccls_freshen_index.cc
src/messages/ccls_index_file.cc
src/messages/ccls_inheritance_hierarchy.cc
src/messages/ccls_member_hierarchy.cc
src/messages/ccls_random.cc

115
src/cache_manager.cc
View File

@ -11,37 +11,6 @@
#include <unordered_map>
namespace {
// Manages loading caches from file paths for the indexer process.
struct RealCacheManager : ICacheManager {
explicit RealCacheManager() {}
~RealCacheManager() override = default;
void WriteToCache(IndexFile& file) override {
std::string cache_path = GetCachePath(file.path);
WriteToFile(cache_path, file.file_contents);
std::string indexed_content = Serialize(g_config->cacheFormat, file);
WriteToFile(AppendSerializationFormat(cache_path), indexed_content);
}
std::optional<std::string> LoadCachedFileContents(
const std::string& path) override {
return ReadContent(GetCachePath(path));
}
std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path) override {
std::string cache_path = GetCachePath(path);
std::optional<std::string> file_content = ReadContent(cache_path);
std::optional<std::string> serialized_indexed_content =
ReadContent(AppendSerializationFormat(cache_path));
if (!file_content || !serialized_indexed_content)
return nullptr;
return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content,
*file_content, IndexFile::kMajorVersion);
}
std::string GetCachePath(const std::string& source_file) {
assert(!g_config->cacheDirectory.empty());
std::string cache_file;
@ -65,75 +34,31 @@ struct RealCacheManager : ICacheManager {
return base + ".json";
}
}
};
struct FakeCacheManager : ICacheManager {
explicit FakeCacheManager(const std::vector<FakeCacheEntry>& entries)
: entries_(entries) {}
void WriteToCache(IndexFile& file) override { assert(false); }
std::optional<std::string> LoadCachedFileContents(
const std::string& path) override {
for (const FakeCacheEntry& entry : entries_) {
if (entry.path == path) {
return entry.content;
}
}
return std::nullopt;
// Manages loading caches from file paths for the indexer process.
void ICacheManager::WriteToCache(IndexFile& file) {
std::string cache_path = GetCachePath(file.path);
WriteToFile(cache_path, file.file_contents);
std::string indexed_content = Serialize(g_config->cacheFormat, file);
WriteToFile(AppendSerializationFormat(cache_path), indexed_content);
}
std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path) override {
for (const FakeCacheEntry& entry : entries_) {
if (entry.path == path) {
return Deserialize(SerializeFormat::Json, path, entry.json, "<empty>",
std::nullopt);
}
}
return nullptr;
}
std::vector<FakeCacheEntry> entries_;
};
} // namespace
// static
std::shared_ptr<ICacheManager> ICacheManager::Make() {
return std::make_shared<RealCacheManager>();
}
// static
std::shared_ptr<ICacheManager> ICacheManager::MakeFake(
const std::vector<FakeCacheEntry>& entries) {
return std::make_shared<FakeCacheManager>(entries);
}
ICacheManager::~ICacheManager() = default;
IndexFile* ICacheManager::TryLoad(const std::string& path) {
auto it = caches_.find(path);
if (it != caches_.end())
return it->second.get();
std::unique_ptr<IndexFile> cache = RawCacheLoad(path);
if (!cache)
return nullptr;
caches_[path] = std::move(cache);
return caches_[path].get();
}
std::unique_ptr<IndexFile> ICacheManager::TryTakeOrLoad(
std::optional<std::string> ICacheManager::LoadCachedFileContents(
const std::string& path) {
auto it = caches_.find(path);
if (it != caches_.end()) {
auto result = std::move(it->second);
caches_.erase(it);
return result;
return ReadContent(GetCachePath(path));
}
return RawCacheLoad(path);
std::unique_ptr<IndexFile> ICacheManager::RawCacheLoad(
const std::string& path) {
std::string cache_path = GetCachePath(path);
std::optional<std::string> file_content = ReadContent(cache_path);
std::optional<std::string> serialized_indexed_content =
ReadContent(AppendSerializationFormat(cache_path));
if (!file_content || !serialized_indexed_content)
return nullptr;
return Deserialize(g_config->cacheFormat, path, *serialized_indexed_content,
*file_content, IndexFile::kMajorVersion);
}

30
src/cache_manager.h
View File

@ -7,34 +7,12 @@
#include <unordered_map>
#include <vector>
struct Config;
struct IndexFile;
struct ICacheManager {
struct FakeCacheEntry {
std::string path;
std::string content;
std::string json;
};
void WriteToCache(IndexFile& file);
static std::shared_ptr<ICacheManager> Make();
static std::shared_ptr<ICacheManager> MakeFake(
const std::vector<FakeCacheEntry>& entries);
virtual ~ICacheManager();
// Tries to load a cache for |path|, returning null if there is none. The
// cache loader still owns the cache.
IndexFile* TryLoad(const std::string& path);
// Takes the existing cache or loads the cache at |path|. May return null if
// the cache does not exist.
std::unique_ptr<IndexFile> TryTakeOrLoad(const std::string& path);
virtual void WriteToCache(IndexFile& file) = 0;
virtual std::optional<std::string> LoadCachedFileContents(
const std::string& path) = 0;
std::optional<std::string> LoadCachedFileContents(const std::string& path);
template <typename Fn>
void IterateLoadedCaches(Fn fn) {
@ -42,7 +20,7 @@ struct ICacheManager {
fn(cache.second.get());
}
protected:
virtual std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path) = 0;
std::unique_ptr<IndexFile> RawCacheLoad(const std::string& path);
std::unordered_map<std::string, std::unique_ptr<IndexFile>> caches_;
};

2
src/clang_complete.cc
View File

@ -647,12 +647,10 @@ void DiagnosticQueryMain(ClangCompleteManager* completion_manager) {
ClangCompleteManager::ClangCompleteManager(Project* project,
WorkingFiles* working_files,
OnDiagnostic on_diagnostic,
OnIndex on_index,
OnDropped on_dropped)
: project_(project),
working_files_(working_files),
on_diagnostic_(on_diagnostic),
on_index_(on_index),
on_dropped_(on_dropped),
preloaded_sessions_(kMaxPreloadedSessions),
completion_sessions_(kMaxCompletionSessions) {

2
src/clang_complete.h
View File

@ -82,7 +82,6 @@ struct ClangCompleteManager {
ClangCompleteManager(Project* project,
WorkingFiles* working_files,
OnDiagnostic on_diagnostic,
OnIndex on_index,
OnDropped on_dropped);
// Start a code completion at the given location. |on_complete| will run when
@ -127,7 +126,6 @@ struct ClangCompleteManager {
Project* project_;
WorkingFiles* working_files_;
OnDiagnostic on_diagnostic_;
OnIndex on_index_;
OnDropped on_dropped_;
using LruSessionCache = LruCache<std::string, CompletionSession>;

38
src/clang_indexer.cc
View File

@ -9,11 +9,12 @@
#include <loguru.hpp>
#include <assert.h>
#include <inttypes.h>
#include <limits.h>
#include <algorithm>
#include <cassert>
#include <chrono>
#include <unordered_set>
#if CINDEX_VERSION >= 47
#define CINDEX_HAVE_PRETTY 1
@ -279,7 +280,7 @@ struct IndexParam {
std::unordered_set<CXFile> seen_cx_files;
std::vector<std::string> seen_files;
std::unordered_map<std::string, FileContents> file_contents;
std::unordered_map<std::string, int64_t> file_modification_times;
std::unordered_map<std::string, int64_t> file2write_time;
// Only use this when strictly needed (ie, primary translation unit is
// needed). Most logic should get the IndexFile instance via
@ -373,11 +374,11 @@ IndexFile* ConsumeFile(IndexParam* param, CXFile file) {
param->seen_files.push_back(file_name);
// Set modification time.
std::optional<int64_t> modification_time = LastWriteTime(file_name);
LOG_IF_S(ERROR, !modification_time)
<< "Failed fetching modification time for " << file_name;
if (modification_time)
param->file_modification_times[file_name] = *modification_time;
std::optional<int64_t> write_time = LastWriteTime(file_name);
LOG_IF_S(ERROR, !write_time) << "failed to fetch write time for "
<< file_name;
if (write_time)
param->file2write_time[file_name] = *write_time;
}
}
@ -2010,7 +2011,7 @@ void OnIndexReference(CXClientData client_data, const CXIdxEntityRefInfo* ref) {
}
std::vector<std::unique_ptr<IndexFile>> Parse(
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
std::string file,
const std::vector<std::string>& args,
const std::vector<FileContents>& file_contents,
@ -2041,12 +2042,11 @@ std::vector<std::unique_ptr<IndexFile>> Parse(
perf->index_parse = timer.ElapsedMicrosecondsAndReset();
return ParseWithTu(file_consumer_shared, perf, tu.get(), index, file,
args, unsaved_files);
return ParseWithTu(vfs, perf, tu.get(), index, file, args, unsaved_files);
}
std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
PerformanceImportFile* perf,
ClangTranslationUnit* tu,
ClangIndex* index,
@ -2067,7 +2067,7 @@ std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
callback.indexDeclaration = &OnIndexDeclaration;
callback.indexEntityReference = &OnIndexReference;
FileConsumer file_consumer(file_consumer_shared, file);
FileConsumer file_consumer(vfs, file);
IndexParam param(tu, &file_consumer);
for (const CXUnsavedFile& contents : file_contents) {
param.file_contents[contents.Filename] = FileContents(
@ -2140,15 +2140,13 @@ std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
}
// Update file contents and modification time.
entry->last_modification_time = param.file_modification_times[entry->path];
entry->last_write_time = param.file2write_time[entry->path];
// Update dependencies for the file. Do not include the file in its own
// dependency set.
entry->dependencies = param.seen_files;
entry->dependencies.erase(
std::remove(entry->dependencies.begin(), entry->dependencies.end(),
entry->path),
entry->dependencies.end());
for (const std::string& path : param.seen_files)
if (path != entry->path && path != entry->import_file)
entry->dependencies[path] = param.file2write_time[path];
}
return result;
@ -2229,10 +2227,8 @@ struct TestIndexer : IIndexer {
return result;
}
~TestIndexer() override = default;
std::vector<std::unique_ptr<IndexFile>> Index(
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
std::string file,
const std::vector<std::string>& args,
const std::vector<FileContents>& file_contents,

18
src/command_line.cc
View File

@ -110,9 +110,8 @@ See more on https://github.com/MaskRay/ccls/wiki
bool QueryDbMainLoop(QueryDatabase* db,
MultiQueueWaiter* waiter,
Project* project,
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
ImportPipelineStatus* status,
TimestampManager* timestamp_manager,
SemanticHighlightSymbolCache* semantic_cache,
WorkingFiles* working_files,
ClangCompleteManager* clang_complete,
@ -154,7 +153,7 @@ void RunQueryDbThread(const std::string& bin_name,
Project project;
SemanticHighlightSymbolCache semantic_cache;
WorkingFiles working_files;
FileConsumerSharedState file_consumer_shared;
VFS vfs;
DiagnosticsEngine diag_engine;
ClangCompleteManager clang_complete(
@ -162,11 +161,6 @@ void RunQueryDbThread(const std::string& bin_name,
[&](std::string path, std::vector<lsDiagnostic> diagnostics) {
diag_engine.Publish(&working_files, path, diagnostics);
},
[&](ClangTranslationUnit* tu, const std::vector<CXUnsavedFile>& unsaved,
const std::string& path, const std::vector<std::string>& args) {
IndexWithTuFromCodeCompletion(&file_consumer_shared, tu, unsaved, path,
args);
},
[](lsRequestId id) {
if (id.Valid()) {
Out_Error out;
@ -184,7 +178,6 @@ void RunQueryDbThread(const std::string& bin_name,
auto non_global_code_complete_cache = std::make_unique<CodeCompleteCache>();
auto signature_cache = std::make_unique<CodeCompleteCache>();
ImportPipelineStatus import_pipeline_status;
TimestampManager timestamp_manager;
QueryDatabase db;
// Setup shared references.
@ -193,9 +186,8 @@ void RunQueryDbThread(const std::string& bin_name,
handler->waiter = indexer_waiter;
handler->project = &project;
handler->diag_engine = &diag_engine;
handler->file_consumer_shared = &file_consumer_shared;
handler->vfs = &vfs;
handler->import_pipeline_status = &import_pipeline_status;
handler->timestamp_manager = &timestamp_manager;
handler->semantic_cache = &semantic_cache;
handler->working_files = &working_files;
handler->clang_complete = &clang_complete;
@ -210,8 +202,8 @@ void RunQueryDbThread(const std::string& bin_name,
SetThreadName("querydb");
while (true) {
bool did_work = QueryDbMainLoop(
&db, querydb_waiter, &project, &file_consumer_shared,
&import_pipeline_status, &timestamp_manager,
&db, querydb_waiter, &project, &vfs,
&import_pipeline_status,
&semantic_cache, &working_files, &clang_complete, &include_complete,
global_code_complete_cache.get(), non_global_code_complete_cache.get(),
signature_cache.get());

1
src/config.cc
View File

@ -1,3 +1,4 @@
#include "config.h"
std::unique_ptr<Config> g_config;
thread_local int g_thread_id;

1
src/config.h
View File

@ -280,3 +280,4 @@ MAKE_REFLECT_STRUCT(Config,
xref);
extern std::unique_ptr<Config> g_config;
thread_local extern int g_thread_id;

50
src/file_consumer.cc
View File

@ -55,21 +55,48 @@ std::optional<std::string> FileContents::ContentsInRange(Range range) const {
return std::nullopt;
}
bool FileConsumerSharedState::Mark(const std::string& file) {
VFS::State VFS::Get(const std::string& file) {
std::lock_guard<std::mutex> lock(mutex);
return used_files.insert(file).second;
auto it = state.find(file);
if (it != state.end())
return it->second;
return {0, 0, 0};
}
void FileConsumerSharedState::Reset(const std::string& file) {
bool VFS::Mark(const std::string& file, int owner, int stage) {
std::lock_guard<std::mutex> lock(mutex);
auto it = used_files.find(file);
if (it != used_files.end())
used_files.erase(it);
State& st = state[file];
if (st.stage < stage) {
st.owner = owner;
st.stage = stage;
return true;
} else
return false;
}
FileConsumer::FileConsumer(FileConsumerSharedState* shared_state,
const std::string& parse_file)
: shared_(shared_state), parse_file_(parse_file) {}
bool VFS::Stamp(const std::string& file, int64_t ts) {
std::lock_guard<std::mutex> lock(mutex);
State& st = state[file];
if (st.timestamp < ts) {
st.timestamp = ts;
return true;
} else
return false;
}
void VFS::ResetLocked(const std::string& file) {
State& st = state[file];
if (st.owner == g_thread_id)
st.stage = 0;
}
void VFS::Reset(const std::string& file) {
std::lock_guard<std::mutex> lock(mutex);
ResetLocked(file);
}
FileConsumer::FileConsumer(VFS* vfs, const std::string& parse_file)
: vfs_(vfs), parse_file_(parse_file), thread_id_(g_thread_id) {}
IndexFile* FileConsumer::TryConsumeFile(
CXFile file,
@ -96,12 +123,9 @@ IndexFile* FileConsumer::TryConsumeFile(
std::string file_name = FileName(file);
// No result in local; we need to query global.
bool did_insert = shared_->Mark(file_name);
// We did not take the file from global. Cache that we failed so we don't try
// again and return nullptr.
if (!did_insert) {
if (!vfs_->Mark(file_name, thread_id_, 2)) {
local_[file_id] = nullptr;
return nullptr;
}

23
src/file_consumer.h
View File

@ -8,7 +8,6 @@
#include <functional>
#include <mutex>
#include <unordered_map>
#include <unordered_set>
#include <vector>
struct IndexFile;
@ -30,13 +29,19 @@ struct FileContents {
std::vector<int> line_offsets_;
};
struct FileConsumerSharedState {
mutable std::unordered_set<std::string> used_files;
struct VFS {
struct State {
int64_t timestamp;
int owner;
int stage;
};
mutable std::unordered_map<std::string, State> state;
mutable std::mutex mutex;
// Mark the file as used. Returns true if the file was not previously used.
bool Mark(const std::string& file);
// Reset the used state (ie, mark the file as unused).
State Get(const std::string& file);
bool Mark(const std::string& file, int owner, int stage);
bool Stamp(const std::string& file, int64_t ts);
void ResetLocked(const std::string& file);
void Reset(const std::string& file);
};
@ -48,8 +53,7 @@ struct FileConsumerSharedState {
// The indexer does this because header files do not have their own translation
// units but we still want to index them.
struct FileConsumer {
FileConsumer(FileConsumerSharedState* shared_state,
const std::string& parse_file);
FileConsumer(VFS* vfs, const std::string& parse_file);
// Returns true if this instance owns given |file|. This will also attempt to
// take ownership over |file|.
@ -69,6 +73,7 @@ struct FileConsumer {
private:
std::unordered_map<CXFileUniqueID, std::unique_ptr<IndexFile>> local_;
FileConsumerSharedState* shared_;
VFS* vfs_;
std::string parse_file_;
int thread_id_;
};

540
src/import_pipeline.cc
View File

@ -18,115 +18,25 @@
namespace {
struct Out_Progress : public lsOutMessage<Out_Progress> {
struct Params {
int indexRequestCount = 0;
int loadPreviousIndexCount = 0;
int onIdMappedCount = 0;
int onIndexedCount = 0;
int activeThreads = 0;
};
std::string method = "$ccls/progress";
Params params;
};
MAKE_REFLECT_STRUCT(Out_Progress::Params,
indexRequestCount,
loadPreviousIndexCount,
onIdMappedCount,
onIndexedCount,
activeThreads);
MAKE_REFLECT_STRUCT(Out_Progress, jsonrpc, method, params);
long long GetCurrentTimeInMilliseconds() {
auto time_since_epoch = Timer::Clock::now().time_since_epoch();
long long elapsed_milliseconds =
std::chrono::duration_cast<std::chrono::milliseconds>(time_since_epoch)
.count();
return elapsed_milliseconds;
}
struct ActiveThread {
ActiveThread(ImportPipelineStatus* status)
: status_(status) {
if (g_config && g_config->progressReportFrequencyMs < 0)
return;
++status_->num_active_threads;
}
~ActiveThread() {
if (g_config && g_config->progressReportFrequencyMs < 0)
return;
--status_->num_active_threads;
EmitProgress();
}
// Send indexing progress to client if reporting is enabled.
void EmitProgress() {
auto* queue = QueueManager::instance();
Out_Progress out;
out.params.indexRequestCount = queue->index_request.Size();
out.params.onIdMappedCount = queue->on_id_mapped.Size();
out.params.onIndexedCount = queue->on_indexed.Size();
out.params.activeThreads = status_->num_active_threads;
// Ignore this progress update if the last update was too recent.
if (g_config && g_config->progressReportFrequencyMs != 0) {
// Make sure we output a status update if queue lengths are zero.
bool all_zero = out.params.indexRequestCount == 0 &&
out.params.loadPreviousIndexCount == 0 &&
out.params.onIdMappedCount == 0 &&
out.params.onIndexedCount == 0 &&
out.params.activeThreads == 0;
if (!all_zero &&
GetCurrentTimeInMilliseconds() < status_->next_progress_output)
return;
status_->next_progress_output =
GetCurrentTimeInMilliseconds() + g_config->progressReportFrequencyMs;
}
QueueManager::WriteStdout(kMethodType_Unknown, out);
}
ImportPipelineStatus* status_;
};
enum class ShouldParse { Yes, No, NoSuchFile };
// Checks if |path| needs to be reparsed. This will modify cached state
// such that calling this function twice with the same path may return true
// the first time but will return false the second.
//
// |from|: The file which generated the parse request for this file.
ShouldParse FileNeedsParse(
bool FileNeedsParse(int64_t write_time,
VFS* vfs,
bool is_interactive,
TimestampManager* timestamp_manager,
const std::shared_ptr<ICacheManager>& cache_manager,
IndexFile* opt_previous_index,
const std::string& path,
const std::vector<std::string>& args,
const std::optional<std::string>& from) {
auto unwrap_opt = [](const std::optional<std::string>& opt) -> std::string {
if (opt)
return " (via " + *opt + ")";
return "";
};
std::optional<int64_t> modification_timestamp = LastWriteTime(path);
// Cannot find file.
if (!modification_timestamp)
return ShouldParse::NoSuchFile;
std::optional<int64_t> last_cached_modification =
timestamp_manager->GetLastCachedModificationTime(cache_manager.get(),
path);
// File has been changed.
if (!last_cached_modification ||
modification_timestamp != *last_cached_modification) {
LOG_S(INFO) << "Timestamp has changed for " << path << unwrap_opt(from);
return ShouldParse::Yes;
{
std::lock_guard<std::mutex> lock(vfs->mutex);
if (vfs->state[path].timestamp < write_time) {
LOG_S(INFO) << "timestamp changed for " << path
<< (from ? " (via " + *from + ")" : std::string());
return true;
}
}
// Command-line arguments changed.
@ -141,183 +51,96 @@ ShouldParse FileNeedsParse(
(is_file(prev_args[i]) && is_file(args[i]));
}
if (!same) {
LOG_S(INFO) << "Arguments have changed for " << path << unwrap_opt(from);
return ShouldParse::Yes;
LOG_S(INFO) << "args changed for " << path << (from ? " (via " + *from + ")" : std::string());
return true;
}
}
// File has not changed, do not parse it.
return ShouldParse::No;
return false;
};
enum CacheLoadResult { Parse, DoNotParse };
CacheLoadResult TryLoadFromCache(
FileConsumerSharedState* file_consumer_shared,
TimestampManager* timestamp_manager,
const std::shared_ptr<ICacheManager>& cache_manager,
bool is_interactive,
const Project::Entry& entry,
const std::string& path_to_index) {
// Always run this block, even if we are interactive, so we can check
// dependencies and reset files in |file_consumer_shared|.
IndexFile* previous_index = cache_manager->TryLoad(path_to_index);
if (!previous_index)
return CacheLoadResult::Parse;
// If none of the dependencies have changed and the index is not
// interactive (ie, requested by a file save), skip parsing and just load
// from cache.
// Check timestamps and update |file_consumer_shared|.
ShouldParse path_state =
FileNeedsParse(is_interactive, timestamp_manager, cache_manager,
previous_index, path_to_index, entry.args, std::nullopt);
if (path_state == ShouldParse::Yes)
file_consumer_shared->Reset(path_to_index);
// Target file does not exist on disk, do not emit any indexes.
// TODO: Dependencies should be reassigned to other files. We can do this by
// updating the "primary_file" if it doesn't exist. Might not actually be a
// problem in practice.
if (path_state == ShouldParse::NoSuchFile)
return CacheLoadResult::DoNotParse;
bool needs_reparse = is_interactive || path_state == ShouldParse::Yes;
for (const std::string& dependency : previous_index->dependencies) {
assert(!dependency.empty());
if (FileNeedsParse(is_interactive, timestamp_manager, cache_manager,
previous_index, dependency, entry.args,
previous_index->path) == ShouldParse::Yes) {
needs_reparse = true;
// Do not break here, as we need to update |file_consumer_shared| for
// every dependency that needs to be reparsed.
file_consumer_shared->Reset(dependency);
}
}
// FIXME: should we still load from cache?
if (needs_reparse)
return CacheLoadResult::Parse;
// No timestamps changed - load directly from cache.
LOG_S(INFO) << "load index for " << path_to_index;
// TODO/FIXME: real perf
PerformanceImportFile perf;
std::vector<Index_OnIdMapped> result;
result.push_back(Index_OnIdMapped(
cache_manager, nullptr, cache_manager->TryTakeOrLoad(path_to_index), perf,
is_interactive, false /*write_to_disk*/));
for (const std::string& dependency : previous_index->dependencies) {
// Only load a dependency if it is not already loaded.
//
// This is important for perf in large projects where there are lots of
// dependencies shared between many files.
if (!file_consumer_shared->Mark(dependency))
continue;
LOG_S(INFO) << "emit index for " << dependency << " via "
<< previous_index->path;
// |dependency_index| may be null if there is no cache for it but
// another file has already started importing it.
if (std::unique_ptr<IndexFile> dependency_index =
cache_manager->TryTakeOrLoad(dependency)) {
result.push_back(
Index_OnIdMapped(cache_manager, nullptr, std::move(dependency_index),
perf, is_interactive, false /*write_to_disk*/));
}
}
QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result));
return CacheLoadResult::DoNotParse;
}
std::vector<FileContents> PreloadFileContents(
const std::shared_ptr<ICacheManager>& cache_manager,
const Project::Entry& entry,
const std::string& entry_contents,
const std::string& path_to_index) {
// Load file contents for all dependencies into memory. If the dependencies
// for the file changed we may not end up using all of the files we
// preloaded. If a new dependency was added the indexer will grab the file
// contents as soon as possible.
//
// We do this to minimize the race between indexing a file and capturing the
// file contents.
//
// TODO: We might be able to optimize perf by only copying for files in
// working_files. We can pass that same set of files to the indexer as
// well. We then default to a fast file-copy if not in working set.
// index->file_contents comes from cache, so we need to check if that cache is
// still valid. if so, we can use it, otherwise we need to load from disk.
auto get_latest_content = [](const std::string& path, int64_t cached_time,
const std::string& cached) -> std::string {
std::optional<int64_t> mod_time = LastWriteTime(path);
if (!mod_time)
return "";
if (*mod_time == cached_time)
return cached;
std::optional<std::string> fresh_content = ReadContent(path);
if (!fresh_content) {
LOG_S(ERROR) << "Failed to load content for " << path;
return "";
}
return *fresh_content;
};
std::vector<FileContents> file_contents;
file_contents.push_back(FileContents(entry.filename, entry_contents));
cache_manager->IterateLoadedCaches([&](IndexFile* index) {
if (index->path == entry.filename)
return;
file_contents.push_back(FileContents(
index->path,
get_latest_content(index->path, index->last_modification_time,
index->file_contents)));
});
return file_contents;
}
void ParseFile(DiagnosticsEngine* diag_engine,
bool Indexer_Parse(DiagnosticsEngine* diag_engine,
WorkingFiles* working_files,
FileConsumerSharedState* file_consumer_shared,
TimestampManager* timestamp_manager,
IIndexer* indexer,
const Index_Request& request,
const Project::Entry& entry) {
// If the file is inferred, we may not actually be able to parse that file
// directly (ie, a header file, which are not listed in the project). If this
// file is inferred, then try to use the file which originally imported it.
std::string path_to_index = entry.filename;
if (entry.is_inferred) {
IndexFile* entry_cache = request.cache_manager->TryLoad(entry.filename);
if (entry_cache)
path_to_index = entry_cache->import_file;
Project* project,
VFS* vfs,
IIndexer* indexer) {
auto* queue = QueueManager::instance();
std::optional<Index_Request> opt_request = queue->index_request.TryPopFront();
if (!opt_request)
return false;
auto& request = *opt_request;
ICacheManager cache;
Project::Entry entry;
{
std::lock_guard<std::mutex> lock(project->mutex_);
auto it = project->absolute_path_to_entry_index_.find(request.path);
if (it != project->absolute_path_to_entry_index_.end())
entry = project->entries[it->second];
else {
entry.filename = request.path;
entry.args = request.args;
}
}
std::string path_to_index = entry.filename;
std::unique_ptr<IndexFile> prev;
// Try to load the file from cache.
if (TryLoadFromCache(file_consumer_shared, timestamp_manager,
request.cache_manager, request.is_interactive, entry,
path_to_index) == CacheLoadResult::DoNotParse)
return;
std::optional<int64_t> write_time = LastWriteTime(path_to_index);
if (!write_time)
return true;
// FIXME Don't drop
if (!vfs->Mark(path_to_index, g_thread_id, 1))
return true;
int reparse; // request.is_interactive;
prev = cache.RawCacheLoad(path_to_index);
if (!prev)
reparse = 2;
else {
reparse = vfs->Stamp(path_to_index, prev->last_write_time);
if (FileNeedsParse(*write_time, vfs, request.is_interactive, &*prev,
path_to_index, entry.args, std::nullopt))
reparse = 2;
for (const auto& dep : prev->dependencies)
if (auto write_time1 = LastWriteTime(dep.first)) {
if (dep.second < *write_time1) {
reparse = 2;
std::lock_guard<std::mutex> lock(vfs->mutex);
vfs->state[dep.first].stage = 0;
}
} else
reparse = 2;
}
if (reparse < 2) {
PerformanceImportFile perf;
auto dependencies = prev->dependencies;
if (reparse) {
IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf),
request.is_interactive);
}
for (const auto& dep : dependencies)
if (vfs->Mark(dep.first, 0, 2)) {
prev = cache.RawCacheLoad(dep.first);
IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
queue->on_indexed.PushBack(Index_OnIndexed(std::move(update), perf),
request.is_interactive);
}
std::lock_guard<std::mutex> lock(vfs->mutex);
VFS::State& state = vfs->state[path_to_index];
if (state.owner == g_thread_id)
state.stage = 0;
return true;
}
LOG_S(INFO) << "parse " << path_to_index;
std::vector<FileContents> file_contents = PreloadFileContents(
request.cache_manager, entry, request.contents, path_to_index);
std::vector<Index_OnIdMapped> result;
PerformanceImportFile perf;
auto indexes = indexer->Index(file_consumer_shared, path_to_index, entry.args,
file_contents, &perf);
auto indexes = indexer->Index(vfs, path_to_index, entry.args, {}, &perf);
if (indexes.empty()) {
if (g_config->index.enabled && request.id.Valid()) {
@ -327,160 +150,52 @@ void ParseFile(DiagnosticsEngine* diag_engine,
out.error.message = "Failed to index " + path_to_index;
QueueManager::WriteStdout(kMethodType_Unknown, out);
}
return;
vfs->Reset(path_to_index);
return true;
}
for (std::unique_ptr<IndexFile>& new_index : indexes) {
Timer time;
for (std::unique_ptr<IndexFile>& curr : indexes) {
// Only emit diagnostics for non-interactive sessions, which makes it easier
// to identify indexing problems. For interactive sessions, diagnostics are
// handled by code completion.
if (!request.is_interactive)
diag_engine->Publish(working_files, new_index->path,
new_index->diagnostics_);
diag_engine->Publish(working_files, curr->path, curr->diagnostics_);
// When main thread does IdMap request it will request the previous index if
// needed.
LOG_S(INFO) << "emit index for " << new_index->path;
result.push_back(
Index_OnIdMapped(request.cache_manager,
request.cache_manager->TryTakeOrLoad(path_to_index),
std::move(new_index), perf, request.is_interactive,
true /*write_to_disk*/));
}
QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result),
request.is_interactive);
}
bool IndexMain_DoParse(
DiagnosticsEngine* diag_engine,
WorkingFiles* working_files,
FileConsumerSharedState* file_consumer_shared,
TimestampManager* timestamp_manager,
IIndexer* indexer) {
auto* queue = QueueManager::instance();
std::optional<Index_Request> request = queue->index_request.TryPopFront();
if (!request)
return false;
Project::Entry entry;
entry.filename = request->path;
entry.args = request->args;
ParseFile(diag_engine, working_files, file_consumer_shared, timestamp_manager,
indexer, request.value(), entry);
return true;
}
bool IndexMain_DoCreateIndexUpdate(TimestampManager* timestamp_manager) {
auto* queue = QueueManager::instance();
bool did_work = false;
for (int i = 100; i--; ) {
std::optional<Index_OnIdMapped> response = queue->on_id_mapped.TryPopFront();
if (!response)
return did_work;
did_work = true;
Timer time;
std::string path = curr->path;
if (!(vfs->Stamp(path, curr->last_write_time) || path == path_to_index))
continue;
LOG_S(INFO) << "emit index for " << path;
prev = cache.RawCacheLoad(path);
// Write current index to disk if requested.
std::string path = response->current->path;
if (response->write_to_disk) {
LOG_S(INFO) << "store index for " << path;
time.Reset();
response->cache_manager->WriteToCache(*response->current);
response->perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
timestamp_manager->UpdateCachedModificationTime(
path, response->current->last_modification_time);
Timer time;
cache.WriteToCache(*curr);
perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
vfs->Reset(path_to_index);
if (entry.id >= 0) {
std::lock_guard<std::mutex> lock(project->mutex_);
for (auto& dep : curr->dependencies)
project->absolute_path_to_entry_index_[dep.first] = entry.id;
}
// Build delta update.
IndexUpdate update = IndexUpdate::CreateDelta(response->previous.get(),
response->current.get());
response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
LOG_S(INFO) << "built index for " << path
<< " (is_delta=" << !!response->previous << ")";
IndexUpdate update = IndexUpdate::CreateDelta(prev.get(), curr.get());
perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
LOG_S(INFO) << "built index for " << path << " (is_delta=" << !!prev << ")";
Index_OnIndexed reply(std::move(update), response->perf);
queue->on_indexed.PushBack(std::move(reply), response->is_interactive);
Index_OnIndexed reply(std::move(update), perf);
queue->on_indexed.PushBack(std::move(reply), request.is_interactive);
}
return did_work;
return true;
}
} // namespace
std::optional<int64_t> TimestampManager::GetLastCachedModificationTime(
ICacheManager* cache_manager,
const std::string& path) {
{
std::lock_guard<std::mutex> guard(mutex_);
auto it = timestamps_.find(path);
if (it != timestamps_.end())
return it->second;
}
IndexFile* file = cache_manager->TryLoad(path);
if (!file)
return std::nullopt;
UpdateCachedModificationTime(path, file->last_modification_time);
return file->last_modification_time;
}
void TimestampManager::UpdateCachedModificationTime(const std::string& path,
int64_t timestamp) {
std::lock_guard<std::mutex> guard(mutex_);
timestamps_[path] = timestamp;
}
ImportPipelineStatus::ImportPipelineStatus()
: num_active_threads(0), next_progress_output(0) {}
// Index a file using an already-parsed translation unit from code completion.
// Since most of the time for indexing a file comes from parsing, we can do
// real-time indexing.
// TODO: add option to disable this.
void IndexWithTuFromCodeCompletion(
FileConsumerSharedState* file_consumer_shared,
ClangTranslationUnit* tu,
const std::vector<CXUnsavedFile>& file_contents,
const std::string& path,
const std::vector<std::string>& args) {
file_consumer_shared->Reset(path);
PerformanceImportFile perf;
ClangIndex index;
auto indexes = ParseWithTu(file_consumer_shared, &perf, tu, &index, path,
args, file_contents);
if (indexes.empty())
return;
std::vector<Index_OnIdMapped> result;
for (std::unique_ptr<IndexFile>& new_index : indexes) {
Timer time;
std::shared_ptr<ICacheManager> cache_manager;
assert(false && "FIXME cache_manager");
// When main thread does IdMap request it will request the previous index if
// needed.
LOG_S(INFO) << "Emitting index for " << new_index->path;
result.push_back(Index_OnIdMapped(
cache_manager, cache_manager->TryTakeOrLoad(path), std::move(new_index),
perf, true /*is_interactive*/, true /*write_to_disk*/));
}
LOG_IF_S(WARNING, result.size() > 1)
<< "Code completion index update generated more than one index";
QueueManager::instance()->on_id_mapped.EnqueueAll(std::move(result));
}
void Indexer_Main(DiagnosticsEngine* diag_engine,
FileConsumerSharedState* file_consumer_shared,
TimestampManager* timestamp_manager,
VFS* vfs,
ImportPipelineStatus* status,
Project* project,
WorkingFiles* working_files,
@ -489,35 +204,9 @@ void Indexer_Main(DiagnosticsEngine* diag_engine,
// Build one index per-indexer, as building the index acquires a global lock.
auto indexer = std::make_unique<ClangIndexer>();
while (true) {
bool did_work = false;
{
ActiveThread active_thread(status);
// TODO: process all off IndexMain_DoIndex before calling
// IndexMain_DoCreateIndexUpdate for better icache behavior. We need to
// have some threads spinning on both though otherwise memory usage will
// get bad.
// We need to make sure to run both IndexMain_DoParse and
// IndexMain_DoCreateIndexUpdate so we don't starve querydb from doing any
// work. Running both also lets the user query the partially constructed
// index.
did_work = IndexMain_DoParse(diag_engine, working_files,
file_consumer_shared, timestamp_manager,
indexer.get()) ||
did_work;
did_work = IndexMain_DoCreateIndexUpdate(timestamp_manager) || did_work;
}
// We didn't do any work, so wait for a notification.
if (!did_work) {
waiter->Wait(&queue->on_indexed, &queue->index_request,
&queue->on_id_mapped);
}
}
while (true)
if (!Indexer_Parse(diag_engine, working_files, project, vfs, indexer.get()))
waiter->Wait(&queue->index_request);
}
namespace {
@ -559,9 +248,6 @@ bool QueryDb_ImportMain(QueryDatabase* db,
SemanticHighlightSymbolCache* semantic_cache,
WorkingFiles* working_files) {
auto* queue = QueueManager::instance();
ActiveThread active_thread(status);
bool did_work = false;
for (int i = 80; i--; ) {

41
src/import_pipeline.h
View File

@ -1,19 +1,10 @@
#pragma once
// FIXME: do not include clang-c outside of clang_ files.
#include <clang-c/Index.h>
#include <atomic>
#include <memory>
#include <optional>
#include <string>
#include <mutex>
#include <unordered_map>
#include <vector>
struct ClangTranslationUnit;
class DiagnosticsEngine;
struct FileConsumerSharedState;
struct VFS;
struct ICacheManager;
struct MultiQueueWaiter;
struct Project;
@ -21,37 +12,13 @@ struct QueryDatabase;
struct SemanticHighlightSymbolCache;
struct WorkingFiles;
// Caches timestamps of cc files so we can avoid a filesystem reads. This is
// important for import perf, as during dependency checking the same files are
// checked over and over again if they are common headers.
struct TimestampManager {
std::optional<int64_t> GetLastCachedModificationTime(ICacheManager* cache_manager,
const std::string& path);
void UpdateCachedModificationTime(const std::string& path, int64_t timestamp);
// TODO: use std::shared_mutex so we can have multiple readers.
std::mutex mutex_;
std::unordered_map<std::string, int64_t> timestamps_;
};
struct ImportPipelineStatus {
std::atomic<int> num_active_threads;
std::atomic<long long> next_progress_output;
ImportPipelineStatus();
std::atomic<int> num_active_threads = {0};
std::atomic<long long> next_progress_output = {0};
};
void IndexWithTuFromCodeCompletion(
FileConsumerSharedState* file_consumer_shared,
ClangTranslationUnit* tu,
const std::vector<CXUnsavedFile>& file_contents,
const std::string& path,
const std::vector<std::string>& args);
void Indexer_Main(DiagnosticsEngine* diag_engine,
FileConsumerSharedState* file_consumer_shared,
TimestampManager* timestamp_manager,
VFS* vfs,
ImportPipelineStatus* status,
Project* project,
WorkingFiles* working_files,

14
src/indexer.h
View File

@ -285,7 +285,7 @@ struct IndexFile {
std::string path;
std::vector<std::string> args;
int64_t last_modification_time = 0;
int64_t last_write_time = 0;
LanguageId language = LanguageId::Unknown;
// The path to the translation unit cc file which caused the creation of this
@ -298,7 +298,7 @@ struct IndexFile {
std::vector<Range> skipped_by_preprocessor;
std::vector<IndexInclude> includes;
std::vector<std::string> dependencies;
std::unordered_map<std::string, int64_t> dependencies;
std::unordered_map<Usr, IndexFunc> usr2func;
std::unordered_map<Usr, IndexType> usr2type;
std::unordered_map<Usr, IndexVar> usr2var;
@ -334,14 +334,14 @@ struct NamespaceHelper {
// |dependencies| are the existing dependencies of |import_file| if this is a
// reparse.
std::vector<std::unique_ptr<IndexFile>> Parse(
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
std::string file,
const std::vector<std::string>& args,
const std::vector<FileContents>& file_contents,
PerformanceImportFile* perf,
ClangIndex* index);
std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
PerformanceImportFile* perf,
ClangTranslationUnit* tu,
ClangIndex* index,
@ -367,7 +367,7 @@ struct IIndexer {
virtual ~IIndexer() = default;
virtual std::vector<std::unique_ptr<IndexFile>> Index(
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
std::string file,
const std::vector<std::string>& args,
const std::vector<FileContents>& file_contents,
@ -376,12 +376,12 @@ struct IIndexer {
struct ClangIndexer : IIndexer {
std::vector<std::unique_ptr<IndexFile>> Index(
FileConsumerSharedState* file_consumer_shared,
VFS* vfs,
std::string file,
const std::vector<std::string>& args,
const std::vector<FileContents>& file_contents,
PerformanceImportFile* perf) override {
return Parse(file_consumer_shared, file, args, file_contents, perf, &index);
return Parse(vfs, file, args, file_contents, perf, &index);
}
// Note: constructing this acquires a global lock

8
src/message_handler.cc
View File

@ -126,7 +126,7 @@ MessageHandler::MessageHandler() {
std::vector<MessageHandler*>* MessageHandler::message_handlers = nullptr;
bool FindFileOrFail(QueryDatabase* db,
const Project* project,
Project* project,
std::optional<lsRequestId> id,
const std::string& absolute_path,
QueryFile** out_query_file,
@ -147,8 +147,12 @@ bool FindFileOrFail(QueryDatabase* db,
if (out_file_id)
*out_file_id = -1;
bool indexing = project->absolute_path_to_entry_index_.find(absolute_path) !=
bool indexing;
{
std::lock_guard<std::mutex> lock(project->mutex_);
indexing = project->absolute_path_to_entry_index_.find(absolute_path) !=
project->absolute_path_to_entry_index_.end();
}
if (indexing)
LOG_S(INFO) << "\"" << absolute_path << "\" is being indexed.";
else

8
src/message_handler.h
View File

@ -15,14 +15,13 @@ struct ClangCompleteManager;
struct CodeCompleteCache;
struct Config;
class DiagnosticsEngine;
struct FileConsumerSharedState;
struct VFS;
struct ImportManager;
struct ImportPipelineStatus;
struct IncludeComplete;
struct MultiQueueWaiter;
struct Project;
struct QueryDatabase;
struct TimestampManager;
struct WorkingFile;
struct WorkingFiles;
@ -107,10 +106,9 @@ struct MessageHandler {
MultiQueueWaiter* waiter = nullptr;
Project* project = nullptr;
DiagnosticsEngine* diag_engine = nullptr;
FileConsumerSharedState* file_consumer_shared = nullptr;
VFS* vfs = nullptr;
ImportManager* import_manager = nullptr;
ImportPipelineStatus* import_pipeline_status = nullptr;
TimestampManager* timestamp_manager = nullptr;
SemanticHighlightSymbolCache* semantic_cache = nullptr;
WorkingFiles* working_files = nullptr;
ClangCompleteManager* clang_complete = nullptr;
@ -139,7 +137,7 @@ struct BaseMessageHandler : MessageHandler {
};
bool FindFileOrFail(QueryDatabase* db,
const Project* project,
Project* project,
std::optional<lsRequestId> id,
const std::string& absolute_path,
QueryFile** out_query_file,

2
src/messages/ccls_call_hierarchy.cc
View File

@ -4,6 +4,8 @@
#include <loguru.hpp>
#include <unordered_set>
namespace {
MethodType kMethodType = "$ccls/callHierarchy";

24
src/messages/ccls_freshen_index.cc
View File

@ -34,14 +34,8 @@ REGISTER_IN_MESSAGE(In_CclsFreshenIndex);
struct Handler_CclsFreshenIndex : BaseMessageHandler<In_CclsFreshenIndex> {
MethodType GetMethodType() const override { return kMethodType; }
void Run(In_CclsFreshenIndex* request) override {
LOG_S(INFO) << "Freshening " << project->entries.size() << " files";
// TODO: think about this flow and test it more.
GroupMatch matcher(request->params.whitelist, request->params.blacklist);
// Unmark all files whose timestamp has changed.
std::shared_ptr<ICacheManager> cache_manager = ICacheManager::Make();
std::queue<const QueryFile*> q;
// |need_index| stores every filename ever enqueued.
std::unordered_set<std::string> need_index;
@ -65,15 +59,15 @@ struct Handler_CclsFreshenIndex : BaseMessageHandler<In_CclsFreshenIndex> {
q.pop();
need_index.insert(file->def->path);
std::optional<int64_t> modification_timestamp =
LastWriteTime(file->def->path);
if (!modification_timestamp)
std::optional<int64_t> write_time = LastWriteTime(file->def->path);
if (!write_time)
continue;
std::optional<int64_t> cached_modification =
timestamp_manager->GetLastCachedModificationTime(cache_manager.get(),
file->def->path);
if (modification_timestamp != cached_modification)
file_consumer_shared->Reset(file->def->path);
{
std::lock_guard<std::mutex> lock(vfs->mutex);
VFS::State& st = vfs->state[file->def->path];
if (st.timestamp < write_time)
st.stage = 0;
}
if (request->params.dependencies)
for (const std::string& path : graph[file->def->path]) {
@ -85,10 +79,8 @@ struct Handler_CclsFreshenIndex : BaseMessageHandler<In_CclsFreshenIndex> {
}
}
Timer time;
// Send index requests for every file.
project->Index(QueueManager::instance(), working_files, lsRequestId());
time.ResetAndPrint("[perf] Dispatched $ccls/freshenIndex index requests");
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsFreshenIndex);

40
src/messages/ccls_index_file.cc
View File

@ -1,40 +0,0 @@
#include "cache_manager.h"
#include "message_handler.h"
#include "platform.h"
#include "queue_manager.h"
#include <loguru/loguru.hpp>
namespace {
MethodType kMethodType = "$ccls/indexFile";
struct In_CclsIndexFile : public NotificationInMessage {
MethodType GetMethodType() const override { return kMethodType; }
struct Params {
std::string path;
std::vector<std::string> args;
bool is_interactive = false;
std::string contents;
};
Params params;
};
MAKE_REFLECT_STRUCT(In_CclsIndexFile::Params,
path,
args,
is_interactive,
contents);
MAKE_REFLECT_STRUCT(In_CclsIndexFile, params);
REGISTER_IN_MESSAGE(In_CclsIndexFile);
struct Handler_CclsIndexFile : BaseMessageHandler<In_CclsIndexFile> {
MethodType GetMethodType() const override { return kMethodType; }
void Run(In_CclsIndexFile* request) override {
LOG_S(INFO) << "Indexing file " << request->params.path;
QueueManager::instance()->index_request.PushBack(
Index_Request(NormalizePath(request->params.path), request->params.args,
request->params.is_interactive, request->params.contents,
ICacheManager::Make()));
}
};
REGISTER_MESSAGE_HANDLER(Handler_CclsIndexFile);
} // namespace

2
src/messages/ccls_member_hierarchy.cc
View File

@ -2,6 +2,8 @@
#include "query_utils.h"
#include "queue_manager.h"
#include <unordered_set>
namespace {
MethodType kMethodType = "$ccls/memberHierarchy";

4
src/messages/initialize.cc
View File

@ -519,9 +519,9 @@ struct Handler_Initialize : BaseMessageHandler<In_InitializeRequest> {
LOG_S(INFO) << "Starting " << g_config->index.threads << " indexers";
for (int i = 0; i < g_config->index.threads; i++) {
std::thread([=]() {
g_thread_id = i + 1;
SetThreadName("indexer" + std::to_string(i));
Indexer_Main(diag_engine, file_consumer_shared, timestamp_manager,
import_pipeline_status, project,
Indexer_Main(diag_engine, vfs, import_pipeline_status, project,
working_files, waiter);
}).detach();
}

2
src/messages/text_document_did_change.cc
View File

@ -33,7 +33,7 @@ struct Handler_TextDocumentDidChange
Project::Entry entry = project->FindCompilationEntryForFile(path);
QueueManager::instance()->index_request.PushBack(
Index_Request(entry.filename, entry.args, true /*is_interactive*/,
*content, ICacheManager::Make()),
*content),
true);
}
}

15
src/messages/text_document_did_open.cc
View File

@ -38,13 +38,12 @@ struct Handler_TextDocumentDidOpen
// NOTE: This function blocks code lens. If it starts taking a long time
// we will need to find a way to unblock the code lens request.
const auto& params = request->params;
Timer time;
std::string path = params.textDocument.uri.GetPath();
std::shared_ptr<ICacheManager> cache_manager = ICacheManager::Make();
ICacheManager cache;
WorkingFile* working_file = working_files->OnOpen(params.textDocument);
std::optional<std::string> cached_file_contents =
cache_manager->LoadCachedFileContents(path);
cache.LoadCachedFileContents(path);
if (cached_file_contents)
working_file->SetIndexContent(*cached_file_contents);
@ -55,10 +54,6 @@ struct Handler_TextDocumentDidOpen
EmitSemanticHighlighting(db, semantic_cache, working_file, file);
}
time.ResetAndPrint(
"[querydb] Loading cached index file for DidOpen (blocks "
"CodeLens)");
include_complete->AddFile(working_file->filename);
clang_complete->NotifyView(path);
if (params.args.size())
@ -68,9 +63,9 @@ struct Handler_TextDocumentDidOpen
if (SourceFileLanguage(path) != LanguageId::Unknown) {
Project::Entry entry = project->FindCompilationEntryForFile(path);
QueueManager::instance()->index_request.PushBack(
Index_Request(
entry.filename, params.args.size() ? params.args : entry.args,
true /*is_interactive*/, params.textDocument.text, cache_manager),
Index_Request(entry.filename,
params.args.size() ? params.args : entry.args,
true /*is_interactive*/, params.textDocument.text),
true /* priority */);
clang_complete->FlushSession(entry.filename);

2
src/messages/text_document_did_save.cc
View File

@ -55,7 +55,7 @@ struct Handler_TextDocumentDidSave
Project::Entry entry = project->FindCompilationEntryForFile(path);
QueueManager::instance()->index_request.PushBack(
Index_Request(entry.filename, entry.args, true /*is_interactive*/,
*content, ICacheManager::Make()),
*content),
true);
}
}

12
src/messages/workspace_did_change_watched_files.cc
View File

@ -41,10 +41,14 @@ struct Handler_WorkspaceDidChangeWatchedFiles
void Run(In_WorkspaceDidChangeWatchedFiles* request) override {
for (lsFileEvent& event : request->params.changes) {
std::string path = event.uri.GetPath();
Project::Entry entry;
{
std::lock_guard<std::mutex> lock(project->mutex_);
auto it = project->absolute_path_to_entry_index_.find(path);
if (it == project->absolute_path_to_entry_index_.end())
continue;
const Project::Entry& entry = project->entries[it->second];
entry = project->entries[it->second];
}
bool is_interactive =
working_files->GetFileByFilename(entry.filename) != nullptr;
switch (event.type) {
@ -55,8 +59,7 @@ struct Handler_WorkspaceDidChangeWatchedFiles
LOG_S(ERROR) << "Unable to read file content after saving " << path;
else {
QueueManager::instance()->index_request.PushBack(
Index_Request(path, entry.args, is_interactive, *content,
ICacheManager::Make()));
Index_Request(path, entry.args, is_interactive, *content));
if (is_interactive)
clang_complete->NotifySave(path);
}
@ -64,8 +67,7 @@ struct Handler_WorkspaceDidChangeWatchedFiles
}
case lsFileChangeType::Deleted:
QueueManager::instance()->index_request.PushBack(
Index_Request(path, entry.args, is_interactive, std::string(),
ICacheManager::Make()));
Index_Request(path, entry.args, is_interactive, std::string()));
break;
}
}

4
src/port.h
View File

@ -10,6 +10,10 @@
#define ATTRIBUTE_UNUSED
#endif
#ifdef __clang__
#define GUARDED_BY(x) __attribute__((guarded_by(x)))
#endif
// TODO GCC
#if __has_builtin(__builtin_unreachable)
#define CCLS_BUILTIN_UNREACHABLE __builtin_unreachable()

12
src/project.cc
View File

@ -461,14 +461,18 @@ void Project::Load(const std::string& root_directory) {
}
// Setup project entries.
std::lock_guard<std::mutex> lock(mutex_);
absolute_path_to_entry_index_.reserve(entries.size());
for (size_t i = 0; i < entries.size(); ++i)
for (size_t i = 0; i < entries.size(); ++i) {
entries[i].id = i;
absolute_path_to_entry_index_[entries[i].filename] = i;
}
}
void Project::SetFlagsForFile(
const std::vector<std::string>& flags,
const std::string& path) {
std::lock_guard<std::mutex> lock(mutex_);
auto it = absolute_path_to_entry_index_.find(path);
if (it != absolute_path_to_entry_index_.end()) {
// The entry already exists in the project, just set the flags.
@ -485,9 +489,12 @@ void Project::SetFlagsForFile(
Project::Entry Project::FindCompilationEntryForFile(
const std::string& filename) {
{
std::lock_guard<std::mutex> lock(mutex_);
auto it = absolute_path_to_entry_index_.find(filename);
if (it != absolute_path_to_entry_index_.end())
return entries[it->second];
}
// We couldn't find the file. Try to infer it.
// TODO: Cache inferred file in a separate array (using a lock or similar)
@ -554,8 +561,7 @@ void Project::Index(QueueManager* queue,
}
bool is_interactive = wfiles->GetFileByFilename(entry.filename) != nullptr;
queue->index_request.PushBack(Index_Request(entry.filename, entry.args,
is_interactive, *content,
ICacheManager::Make(), id));
is_interactive, *content, id));
});
}

4
src/project.h
View File

@ -19,6 +19,7 @@ struct Project {
std::vector<std::string> args;
// If true, this entry is inferred and was not read from disk.
bool is_inferred = false;
int id = -1;
};
// Include directories for "" headers
@ -27,7 +28,8 @@ struct Project {
std::vector<std::string> angle_include_directories;
std::vector<Entry> entries;
std::unordered_map<std::string, int> absolute_path_to_entry_index_;
std::mutex mutex_;
std::unordered_map<std::string, int> absolute_path_to_entry_index_ GUARDED_BY(mutex_);
// Loads a project for the given |directory|.
//

4
src/query.cc
View File

@ -76,7 +76,9 @@ QueryFile::DefUpdate BuildFileDefUpdate(const IndexFile& indexed) {
def.args = std::move(indexed.args);
def.includes = std::move(indexed.includes);
def.inactive_regions = std::move(indexed.skipped_by_preprocessor);
def.dependencies = std::move(indexed.dependencies);
def.dependencies.reserve(indexed.dependencies.size());
for (auto& dep : indexed.dependencies)
def.dependencies.push_back(dep.first);
def.language = indexed.language;
auto add_all_symbols = [&](Use use, Usr usr, SymbolKind kind) {

10
src/queue_manager.cc
View File

@ -11,13 +11,11 @@ Index_Request::Index_Request(
const std::vector<std::string>& args,
bool is_interactive,
const std::string& contents,
const std::shared_ptr<ICacheManager>& cache_manager,
lsRequestId id)
: path(path),
args(args),
is_interactive(is_interactive),
contents(contents),
cache_manager(cache_manager),
id(id) {}
Index_OnIndexed::Index_OnIndexed(IndexUpdate&& update,
@ -51,10 +49,4 @@ QueueManager::QueueManager(MultiQueueWaiter* querydb_waiter,
: for_stdout(stdout_waiter),
for_querydb(querydb_waiter),
on_indexed(querydb_waiter),
index_request(indexer_waiter),
on_id_mapped(indexer_waiter) {}
bool QueueManager::HasWork() {
return !index_request.IsEmpty() || !on_id_mapped.IsEmpty() ||
!on_indexed.IsEmpty();
}
index_request(indexer_waiter) {}

5
src/queue_manager.h
View File

@ -21,14 +21,12 @@ struct Index_Request {
std::vector<std::string> args;
bool is_interactive;
std::string contents; // Preloaded contents.
std::shared_ptr<ICacheManager> cache_manager;
lsRequestId id;
Index_Request(const std::string& path,
const std::vector<std::string>& args,
bool is_interactive,
const std::string& contents,
const std::shared_ptr<ICacheManager>& cache_manager,
lsRequestId id = {});
};
@ -72,8 +70,6 @@ class QueueManager {
MultiQueueWaiter* stdout_waiter);
static void WriteStdout(MethodType method, lsBaseOutMessage& response);
bool HasWork();
// Messages received by "stdout" thread.
ThreadedQueue<Stdout_Request> for_stdout;
@ -83,7 +79,6 @@ class QueueManager {
// Runs on indexer threads.
ThreadedQueue<Index_Request> index_request;
ThreadedQueue<Index_OnIdMapped> on_id_mapped;
private:
explicit QueueManager(MultiQueueWaiter* querydb_waiter,

26
src/serializer.cc
View File

@ -283,14 +283,13 @@ template <typename TVisitor>
void Reflect(TVisitor& visitor, IndexFile& value) {
REFLECT_MEMBER_START();
if (!gTestOutputMode) {
REFLECT_MEMBER(last_modification_time);
REFLECT_MEMBER(last_write_time);
REFLECT_MEMBER(language);
REFLECT_MEMBER(import_file);
REFLECT_MEMBER(args);
REFLECT_MEMBER(dependencies);
}
REFLECT_MEMBER(includes);
if (!gTestOutputMode)
REFLECT_MEMBER(dependencies);
REFLECT_MEMBER(skipped_by_preprocessor);
REFLECT_MEMBER(usr2func);
REFLECT_MEMBER(usr2type);
@ -314,6 +313,27 @@ void Reflect(Writer& visitor, SerializeFormat& value) {
}
}
void Reflect(Reader& visitor, std::unordered_map<std::string, int64_t>& map) {
visitor.IterArray([&](Reader& entry) {
std::string name;
Reflect(entry, name);
if (visitor.Format() == SerializeFormat::Binary)
Reflect(entry, map[name]);
else
map[name] = 0;
});
}
void Reflect(Writer& visitor, std::unordered_map<std::string, int64_t>& map) {
visitor.StartArray(map.size());
for (auto& it : map) {
std::string key = it.first;
Reflect(visitor, key);
if (visitor.Format() == SerializeFormat::Binary)
Reflect(visitor, it.second);
}
visitor.EndArray();
}
std::string Serialize(SerializeFormat format, IndexFile& file) {
switch (format) {
case SerializeFormat::Binary: {

8
src/serializer.h
View File

@ -262,12 +262,12 @@ void Reflect(Writer& visitor, std::vector<T>& values) {
// std::unordered_map
template <typename V>
void Reflect(Reader& visitor, std::unordered_map<uint64_t, V>& values) {
void Reflect(Reader& visitor, std::unordered_map<uint64_t, V>& map) {
visitor.IterArray([&](Reader& entry) {
V val;
Reflect(entry, val);
auto usr = val.usr;
values[usr] = std::move(val);
map[usr] = std::move(val);
});
}
template <typename V>
@ -281,6 +281,10 @@ void Reflect(Writer& visitor, std::unordered_map<uint64_t, V>& map) {
visitor.EndArray();
}
// Used by IndexFile::dependencies. Timestamps are emitted for Binary.
void Reflect(Reader& visitor, std::unordered_map<std::string, int64_t>& map);
void Reflect(Writer& visitor, std::unordered_map<std::string, int64_t>& map);
// ReflectMember
template <typename T>

4
src/test.cc
View File

@ -292,9 +292,9 @@ bool RunIndexTests(const std::string& filter_path, bool enable_update) {
// Run test.
g_config = std::make_unique<Config>();
FileConsumerSharedState file_consumer_shared;
VFS vfs;
PerformanceImportFile perf;
auto dbs = Parse(&file_consumer_shared, path, flags, {}, &perf, &index);
auto dbs = Parse(&vfs, path, flags, {}, &perf, &index);
for (const auto& entry : all_expected_output) {
const std::string& expected_path = entry.first;