Make import pipeline a bit more flexible w.r.t. writing indexed content to disk.

This commit is contained in:
Jacob Dufault 2017-08-16 19:06:28 -07:00
parent d5bdf8ce0a
commit 4245bac1db
3 changed files with 17 additions and 17 deletions

View File

@ -32,10 +32,7 @@ std::unique_ptr<IndexFile> LoadCachedIndex(Config* config,
if (!file_content) if (!file_content)
return nullptr; return nullptr;
auto result = Deserialize(filename, *file_content, IndexFile::kCurrentVersion); return Deserialize(filename, *file_content, IndexFile::kCurrentVersion);
if (result)
result->is_loaded_from_cache_ = true;
return result;
} }
optional<std::string> LoadCachedFileContents(Config* config, optional<std::string> LoadCachedFileContents(Config* config,

View File

@ -561,15 +561,18 @@ struct Index_DoIdMap {
PerformanceImportFile perf; PerformanceImportFile perf;
bool is_interactive = false; bool is_interactive = false;
bool write_to_disk = false;
bool load_previous = false; bool load_previous = false;
Index_DoIdMap( Index_DoIdMap(
std::unique_ptr<IndexFile> current, std::unique_ptr<IndexFile> current,
PerformanceImportFile perf, PerformanceImportFile perf,
bool is_interactive) bool is_interactive,
bool write_to_disk)
: current(std::move(current)), : current(std::move(current)),
perf(perf), perf(perf),
is_interactive(is_interactive) {} is_interactive(is_interactive),
write_to_disk(write_to_disk) {}
}; };
struct Index_OnIdMapped { struct Index_OnIdMapped {
@ -586,12 +589,15 @@ struct Index_OnIdMapped {
PerformanceImportFile perf; PerformanceImportFile perf;
bool is_interactive; bool is_interactive;
bool write_to_disk;
Index_OnIdMapped( Index_OnIdMapped(
PerformanceImportFile perf, PerformanceImportFile perf,
bool is_interactive) bool is_interactive,
bool write_to_disk)
: perf(perf), : perf(perf),
is_interactive(is_interactive) {} is_interactive(is_interactive),
write_to_disk(write_to_disk) {}
}; };
struct Index_OnIndexed { struct Index_OnIndexed {
@ -876,7 +882,7 @@ std::vector<Index_DoIdMap> DoParseFile(
bool is_interactive = false; bool is_interactive = false;
// TODO/FIXME: real perf // TODO/FIXME: real perf
PerformanceImportFile perf; PerformanceImportFile perf;
result.push_back(Index_DoIdMap(cache_loader->TryTakeOrLoad(path), perf, is_interactive)); result.push_back(Index_DoIdMap(cache_loader->TryTakeOrLoad(path), perf, is_interactive, false /*write_to_disk*/));
for (const std::string& dependency : previous_index->dependencies) { for (const std::string& dependency : previous_index->dependencies) {
// Only actually load the file if we haven't loaded it yet. Important // Only actually load the file if we haven't loaded it yet. Important
// for perf when files have lots of common dependencies. // for perf when files have lots of common dependencies.
@ -884,7 +890,7 @@ std::vector<Index_DoIdMap> DoParseFile(
continue; continue;
LOG_S(INFO) << "Emitting index result for " << dependency; LOG_S(INFO) << "Emitting index result for " << dependency;
result.push_back(Index_DoIdMap(cache_loader->TryTakeOrLoad(dependency), perf, is_interactive)); result.push_back(Index_DoIdMap(cache_loader->TryTakeOrLoad(dependency), perf, is_interactive, false /*write_to_disk*/));
} }
return result; return result;
} }
@ -948,7 +954,7 @@ std::vector<Index_DoIdMap> DoParseFile(
// TODO/FIXME: real is_interactive // TODO/FIXME: real is_interactive
bool is_interactive = false; bool is_interactive = false;
LOG_S(INFO) << "Emitting index result for " << new_index->path; LOG_S(INFO) << "Emitting index result for " << new_index->path;
result.push_back(Index_DoIdMap(std::move(new_index), perf, is_interactive)); result.push_back(Index_DoIdMap(std::move(new_index), perf, is_interactive, true /*write_to_disk*/));
} }
return result; return result;
@ -1024,12 +1030,11 @@ bool IndexMain_DoCreateIndexUpdate(
previous_index, response->current->file.get()); previous_index, response->current->file.get());
response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset(); response->perf.index_make_delta = time.ElapsedMicrosecondsAndReset();
// Write new cache to disk if it is a fresh index. // Write current index to disk if requested.
if (!response->current->file->is_loaded_from_cache_) { if (response->write_to_disk) {
time.Reset(); time.Reset();
WriteToCache(config, *response->current->file); WriteToCache(config, *response->current->file);
response->perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset(); response->perf.index_save_to_disk = time.ElapsedMicrosecondsAndReset();
timestamp_manager->UpdateCachedModificationTime(response->current->file->path, response->current->file->last_modification_time); timestamp_manager->UpdateCachedModificationTime(response->current->file->path, response->current->file->last_modification_time);
} }
@ -1160,7 +1165,7 @@ bool QueryDb_ImportMain(Config* config, QueryDatabase* db, QueueManager* queue,
continue; continue;
} }
Index_OnIdMapped response(request->perf, request->is_interactive); Index_OnIdMapped response(request->perf, request->is_interactive, request->write_to_disk);
Timer time; Timer time;
assert(request->current); assert(request->current);

View File

@ -480,8 +480,6 @@ struct IndexFile {
std::vector<IndexFunc> funcs; std::vector<IndexFunc> funcs;
std::vector<IndexVar> vars; std::vector<IndexVar> vars;
// True if this file was loaded from cache. Not serialized.
bool is_loaded_from_cache_ = false;
// Diagnostics found when indexing this file. Not serialized. // Diagnostics found when indexing this file. Not serialized.
NonElidedVector<lsDiagnostic> diagnostics_; NonElidedVector<lsDiagnostic> diagnostics_;
// File contents at the time of index. Not serialized. // File contents at the time of index. Not serialized.