Be more aggressive about only indexing files once.

This should help fix perf regressions with long import times after syncing.
This commit is contained in:
Jacob Dufault 2017-09-13 23:39:32 -07:00
parent 036c2819f1
commit d6a8071da4
2 changed files with 19 additions and 4 deletions

View File

@ -896,6 +896,10 @@ std::vector<Index_DoIdMap> DoParseFile(
IndexFile* previous_index = cache_loader->TryLoad(path);
if (previous_index) {
// If none of the dependencies have changed, skip parsing and just load from cache.
// Checks if |path| needs to be reparsed. This will modify cached state
// such that calling this function twice with the same path may return true
// the first time but will return false the second.
auto file_needs_parse = [&](const std::string& path) {
optional<int64_t> modification_timestamp = GetLastModificationTime(path);
if (!modification_timestamp)
@ -905,6 +909,7 @@ std::vector<Index_DoIdMap> DoParseFile(
if (!last_cached_modification || modification_timestamp != *last_cached_modification) {
file_consumer_shared->Reset(path);
timestamp_manager->UpdateCachedModificationTime(path, *modification_timestamp);
return FileParseQuery::NeedsParse;
}
return FileParseQuery::DoesNotNeedParse;
@ -1284,7 +1289,9 @@ bool QueryDb_ImportMain(Config* config, QueryDatabase* db, ImportManager* import
time.Reset();
db->ApplyIndexUpdate(&response->update);
time.ResetAndPrint("Applying index update");
time.ResetAndPrint("Applying index update for " + StringJoinMap(response->update.files_def_update, [](const QueryFile::DefUpdate& value) {
return value.path;
}));
}
return did_work;

View File

@ -32,19 +32,27 @@ std::vector<std::string> SplitString(const std::string& str, const std::string&
std::string LowerPathIfCaseInsensitive(const std::string& path);
template <typename TValues>
std::string StringJoin(const TValues& values) {
template <typename TValues, typename TMap>
std::string StringJoinMap(const TValues& values, const TMap& map) {
std::string result;
bool first = true;
for (auto& entry : values) {
if (!first)
result += ", ";
first = false;
result += entry;
result += map(entry);
}
return result;
}
template <typename TValues>
std::string StringJoin(const TValues& values) {
return StringJoinMap(values, [](const std::string& entry) {
return entry;
});
}
// Finds all files in the given folder. This is recursive.
std::vector<std::string> GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path);
void GetFilesInFolder(std::string folder, bool recursive, bool add_folder_to_path, const std::function<void(const std::string&)>& handler);