From d9f0de471971468252488591ebbda040aefb9345 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Thu, 20 Sep 2018 00:31:23 -0700 Subject: [PATCH] Clean up FileConsumer and improve pipeline --- src/file_consumer.cc | 88 ++++++++++---------------------------------- src/file_consumer.h | 39 +++++++------------- src/indexer.cc | 35 ++++++++++-------- src/pipeline.cc | 27 ++++++-------- src/pipeline.hh | 1 - src/utils.cc | 12 +++++- src/utils.h | 1 + 7 files changed, 76 insertions(+), 127 deletions(-) diff --git a/src/file_consumer.cc b/src/file_consumer.cc index a7115ed2..40ad2dc3 100644 --- a/src/file_consumer.cc +++ b/src/file_consumer.cc @@ -9,62 +9,17 @@ #include "platform.h" #include "utils.h" -namespace { - -std::optional -GetFileContents(const std::string &path, - std::unordered_map *file_contents) { - auto it = file_contents->find(path); - if (it == file_contents->end()) { - std::optional content = ReadContent(path); - if (content) - (*file_contents)[path] = FileContents(path, *content); - return content; - } - return it->second.content; +bool VFS::Loaded(const std::string &path) { + std::lock_guard lock(mutex); + return state[path].loaded; } -} // namespace - -FileContents::FileContents(const std::string &path, const std::string &content) - : path(path), content(content) { - line_offsets_.push_back(0); - for (size_t i = 0; i < content.size(); i++) { - if (content[i] == '\n') - line_offsets_.push_back(i + 1); - } -} - -std::optional FileContents::ToOffset(Position p) const { - if (0 <= p.line && size_t(p.line) < line_offsets_.size()) { - int ret = line_offsets_[p.line] + p.column; - if (size_t(ret) < content.size()) - return ret; - } - return std::nullopt; -} - -std::optional FileContents::ContentsInRange(Range range) const { - std::optional start_offset = ToOffset(range.start), - end_offset = ToOffset(range.end); - if (start_offset && end_offset && *start_offset < *end_offset) - return content.substr(*start_offset, *end_offset - *start_offset); - return std::nullopt; -} - -VFS::State VFS::Get(const std::string &file) { +bool VFS::Stamp(const std::string &path, int64_t ts, int step) { std::lock_guard lock(mutex); - auto it = state.find(file); - if (it != state.end()) - return it->second; - return {0, 0}; -} - -bool VFS::Stamp(const std::string &file, int64_t ts, int64_t offset) { - std::lock_guard lock(mutex); - State &st = state[file]; - if (st.timestamp < ts) { - st.timestamp = ts + offset; + State &st = state[path]; + if (st.timestamp < ts || (st.timestamp == ts && st.step < step)) { + st.timestamp = ts; + st.step = step; return true; } else return false; @@ -75,29 +30,26 @@ FileConsumer::FileConsumer(VFS *vfs, const std::string &parse_file) IndexFile *FileConsumer::TryConsumeFile( const clang::FileEntry &File, - std::unordered_map *file_contents_map) { + const std::unordered_map + &UID2File) { auto UniqueID = File.getUniqueID(); - auto it = local_.find(UniqueID); - if (it != local_.end()) - return it->second.get(); + { + auto it = local_.find(UniqueID); + if (it != local_.end()) + return it->second.get(); + } - std::string file_name = FileName(File); - int64_t tim = File.getModificationTime(); - assert(tim); - if (!vfs_->Stamp(file_name, tim, 0)) { + auto it = UID2File.find(UniqueID); + assert(it != UID2File.end()); + assert(it->second.mtime); + if (!vfs_->Stamp(it->second.path, it->second.mtime, 1)) { local_[UniqueID] = nullptr; return nullptr; } - // Read the file contents, if we fail then we cannot index the file. - std::optional contents = - GetFileContents(file_name, file_contents_map); - if (!contents) - return nullptr; - // Build IndexFile instance. local_[UniqueID] = - std::make_unique(UniqueID, file_name, *contents); + std::make_unique(UniqueID, it->second.path, it->second.content); return local_[UniqueID].get(); } diff --git a/src/file_consumer.h b/src/file_consumer.h index fe92f3a9..a2f27298 100644 --- a/src/file_consumer.h +++ b/src/file_consumer.h @@ -15,29 +15,17 @@ struct IndexFile; -struct FileContents { - FileContents() = default; - FileContents(const std::string &path, const std::string &content); - - std::optional ToOffset(Position p) const; - std::optional ContentsInRange(Range range) const; - - std::string path; - std::string content; - // {0, 1 + position of first newline, 1 + position of second newline, ...} - std::vector line_offsets_; -}; - struct VFS { struct State { int64_t timestamp; - bool loaded = false; + int step; + bool loaded; }; mutable std::unordered_map state; mutable std::mutex mutex; - State Get(const std::string &file); - bool Stamp(const std::string &file, int64_t ts, int64_t offset); + bool Loaded(const std::string &path); + bool Stamp(const std::string &path, int64_t ts, int step); }; namespace std { @@ -58,17 +46,18 @@ template <> struct hash { // The indexer does this because header files do not have their own translation // units but we still want to index them. struct FileConsumer { + struct File { + std::string path; + int64_t mtime; + std::string content; + }; + FileConsumer(VFS *vfs, const std::string &parse_file); - // Returns IndexFile for the file or nullptr. |is_first_ownership| is set - // to true iff the function just took ownership over the file. Otherwise it - // is set to false. - // - // note: file_contents is passed as a parameter instead of as a member - // variable since it is large and we do not want to copy it. - IndexFile * - TryConsumeFile(const clang::FileEntry &file, - std::unordered_map *file_contents); + // Returns IndexFile or nullptr for the file or nullptr. + IndexFile *TryConsumeFile( + const clang::FileEntry &file, + const std::unordered_map &); // Returns and passes ownership of all local state. std::vector> TakeLocalState(); diff --git a/src/indexer.cc b/src/indexer.cc index 4a1a1b2c..f1a0ea89 100644 --- a/src/indexer.cc +++ b/src/indexer.cc @@ -37,10 +37,8 @@ constexpr int kInitializerMaxLines = 3; GroupMatch *multiVersionMatcher; struct IndexParam { - std::unordered_map SeenUniqueID; + std::unordered_map UID2File; std::unordered_map UID2multi; - std::unordered_map file_contents; - std::unordered_map file2mtime; struct DeclInfo { Usr usr; std::string short_name; @@ -56,17 +54,22 @@ struct IndexParam { void SeenFile(const FileEntry &File) { // If this is the first time we have seen the file (ignoring if we are // generating an index for it): - auto [it, inserted] = SeenUniqueID.try_emplace(File.getUniqueID()); + auto [it, inserted] = UID2File.try_emplace(File.getUniqueID()); if (inserted) { - std::string file_name = FileName(File); - it->second = file_name; - file2mtime[file_name] = File.getModificationTime(); + std::string path = FileName(File); + it->second.path = path; + it->second.mtime = File.getModificationTime(); + if (!it->second.mtime) + if (auto tim = LastWriteTime(path)) + it->second.mtime = *tim; + if (std::optional content = ReadContent(path)) + it->second.content = *content; } } IndexFile *ConsumeFile(const FileEntry &FE) { SeenFile(FE); - return file_consumer->TryConsumeFile(FE, &file_contents); + return file_consumer->TryConsumeFile(FE, UID2File); } bool UseMultiVersion(const FileEntry &FE) { @@ -1319,15 +1322,15 @@ Index(CompletionManager *completion, WorkingFiles *wfiles, VFS *vfs, for (auto &it : entry->usr2var) Uniquify(it.second.uses); - // Update file contents and modification time. - entry->mtime = param.file2mtime[entry->path]; - - // Update dependencies for the file. Do not include the file in its own - // dependency set. - for (auto &[_, path] : param.SeenUniqueID) - if (path != entry->path && path != entry->import_file) + // Update dependencies for the file. + for (auto &[_, file] : param.UID2File) { + const std::string &path = file.path; + if (path == entry->path) + entry->mtime = file.mtime; + else if (path != entry->import_file) entry->dependencies[llvm::CachedHashStringRef(Intern(path))] = - param.file2mtime[path]; + file.mtime; + } } return result; diff --git a/src/pipeline.cc b/src/pipeline.cc index 0c5ed435..5bc8bc32 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -194,16 +194,21 @@ bool Indexer_Parse(CompletionManager *completion, WorkingFiles *wfiles, std::optional write_time = LastWriteTime(path_to_index); if (!write_time) return true; - int reparse = vfs->Stamp(path_to_index, *write_time, -1); + int reparse = vfs->Stamp(path_to_index, *write_time, 0); if (request.path != path_to_index) { std::optional mtime1 = LastWriteTime(request.path); if (!mtime1) return true; - if (vfs->Stamp(request.path, *mtime1, -1)) + if (vfs->Stamp(request.path, *mtime1, 0)) reparse = 1; } - if (g_config->index.onChange) + if (g_config->index.onChange) { reparse = 2; + std::lock_guard lock(vfs->mutex); + vfs->state[path_to_index].step = 0; + if (request.path != path_to_index) + vfs->state[request.path].step = 0; + } if (!reparse) return true; @@ -230,6 +235,8 @@ bool Indexer_Parse(CompletionManager *completion, WorkingFiles *wfiles, LOG_S(INFO) << "load cache for " << path_to_index; auto dependencies = prev->dependencies; if (reparse) { + if (vfs->Loaded(path_to_index)) + return true; IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get()); on_indexed->PushBack(std::move(update), request.mode != IndexMode::NonInteractive); @@ -297,12 +304,7 @@ bool Indexer_Parse(CompletionManager *completion, WorkingFiles *wfiles, << ")"; { std::lock_guard lock(mutexes[std::hash()(path) % N_MUTEXES]); - bool loaded; - { - std::lock_guard lock1(vfs->mutex); - loaded = vfs->state[path].loaded; - } - if (loaded) + if (vfs->Loaded(path)) prev = RawCacheLoad(path); else prev.reset(); @@ -531,13 +533,6 @@ void Index(const std::string &path, const std::vector &args, index_request->PushBack({path, args, mode, id}, mode != IndexMode::NonInteractive); } -std::optional LastWriteTime(const std::string &path) { - sys::fs::file_status Status; - if (sys::fs::status(path, Status)) - return {}; - return sys::toTimeT(Status.getLastModificationTime()); -} - std::optional LoadIndexedContent(const std::string &path) { if (g_config->cacheDirectory.empty()) { std::shared_lock lock(g_index_mutex); diff --git a/src/pipeline.hh b/src/pipeline.hh index 422bc4a9..c1ffbd75 100644 --- a/src/pipeline.hh +++ b/src/pipeline.hh @@ -49,7 +49,6 @@ void MainLoop(); void Index(const std::string &path, const std::vector &args, IndexMode mode, lsRequestId id = {}); -std::optional LastWriteTime(const std::string &path); std::optional LoadIndexedContent(const std::string& path); void WriteStdout(MethodType method, lsBaseOutMessage &response); } // namespace pipeline diff --git a/src/utils.cc b/src/utils.cc index 1752b420..981e783f 100644 --- a/src/utils.cc +++ b/src/utils.cc @@ -8,7 +8,10 @@ #include -#include "llvm/ADT/StringRef.h" +#include +#include +#include +using namespace llvm; #include #include @@ -112,6 +115,13 @@ std::string EscapeFileName(std::string path) { return path; } +std::optional LastWriteTime(const std::string &path) { + sys::fs::file_status Status; + if (sys::fs::status(path, Status)) + return {}; + return sys::toTimeT(Status.getLastModificationTime()); +} + std::optional ReadContent(const std::string &filename) { char buf[4096]; std::string ret; diff --git a/src/utils.h b/src/utils.h index 127530bc..93f5cafe 100644 --- a/src/utils.h +++ b/src/utils.h @@ -61,6 +61,7 @@ void EnsureEndsInSlash(std::string &path); // e.g. foo/bar.c => foo_bar.c std::string EscapeFileName(std::string path); +std::optional LastWriteTime(const std::string &path); std::optional ReadContent(const std::string &filename); void WriteToFile(const std::string &filename, const std::string &content);