diff --git a/src/config.hh b/src/config.hh
index 94830b66..008c1e24 100644
--- a/src/config.hh
+++ b/src/config.hh
@@ -28,19 +28,41 @@ struct Config {
   std::string compilationDatabaseCommand;
   // Directory containing compile_commands.json.
   std::string compilationDatabaseDirectory;
-  // Cache directory for indexed files, either absolute or relative to the
-  // project root.
-  // If empty, cache will be stored in memory.
-  std::string cacheDirectory = ".ccls-cache";
-  // Cache serialization format.
-  //
-  // "json" generates `cacheDirectory/.../xxx.json` files which can be pretty
-  // printed with jq.
-  //
-  // "binary" uses a compact binary serialization format.
-  // It is not schema-aware and you need to re-index whenever an internal struct
-  // member has changed.
-  SerializeFormat cacheFormat = SerializeFormat::Binary;
+
+  struct Cache {
+    // Cache directory for indexed files, either absolute or relative to the
+    // project root.
+    //
+    // If empty, retainInMemory will be set to 1 and cache will be stored in
+    // memory.
+    std::string directory = ".ccls-cache";
+
+    // Cache serialization format.
+    //
+    // "json" generates $directory/.../xxx.json files which can be pretty
+    // printed with jq.
+    //
+    // "binary" uses a compact binary serialization format.
+    // It is not schema-aware and you need to re-index whenever an internal
+    // struct member has changed.
+    SerializeFormat format = SerializeFormat::Binary;
+
+    // If false, store cache files as $directory/@a@b/c.cc.blob
+    //
+    // If true, $directory/a/b/c.cc.blob. If cache.directory is absolute, make
+    // sure different projects use different cache.directory as they would have
+    // conflicting cache files for system headers.
+    bool hierarchicalPath = false;
+
+    // After this number of loads, keep a copy of file index in memory (which
+    // increases memory usage). During incremental updates, the index subtracted
+    // will come from the in-memory copy, instead of the on-disk file.
+    //
+    // The initial load or a save action is counted as one load.
+    // 0: never retain; 1: retain after initial load; 2: retain after 2 loads
+    // (initial load+first save)
+    int retainInMemory = 2;
+  } cache;
 
   struct ServerCap {
     struct DocumentOnTypeFormattingOptions {
@@ -280,6 +302,8 @@ struct Config {
     int maxNum = 2000;
   } xref;
 };
+REFLECT_STRUCT(Config::Cache, directory, format, hierarchicalPath,
+               retainInMemory);
 REFLECT_STRUCT(Config::ServerCap::DocumentOnTypeFormattingOptions,
                firstTriggerCharacter, moreTriggerCharacter);
 REFLECT_STRUCT(Config::ServerCap::Workspace::WorkspaceFolders, supported,
@@ -309,9 +333,9 @@ REFLECT_STRUCT(Config::Session, maxNum);
 REFLECT_STRUCT(Config::WorkspaceSymbol, caseSensitivity, maxNum, sort);
 REFLECT_STRUCT(Config::Xref, maxNum);
 REFLECT_STRUCT(Config, compilationDatabaseCommand, compilationDatabaseDirectory,
-               cacheDirectory, cacheFormat, capabilities, clang, client,
-               codeLens, completion, diagnostics, highlight, index, request,
-               session, workspaceSymbol, xref);
+               cache, capabilities, clang, client, codeLens, completion,
+               diagnostics, highlight, index, request, session, workspaceSymbol,
+               xref);
 
 extern Config *g_config;
 
diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc
index 3798a19d..d064891e 100644
--- a/src/messages/initialize.cc
+++ b/src/messages/initialize.cc
@@ -282,12 +282,12 @@ void Initialize(MessageHandler *m, InitializeParam &param, ReplyOnce &reply) {
     Reflect(json_writer, *g_config);
     LOG_S(INFO) << "initializationOptions: " << output.GetString();
 
-    if (g_config->cacheDirectory.size()) {
-      SmallString<256> Path(g_config->cacheDirectory);
+    if (g_config->cache.directory.size()) {
+      SmallString<256> Path(g_config->cache.directory);
       sys::fs::make_absolute(project_path, Path);
       // Use upper case for the Driver letter on Windows.
-      g_config->cacheDirectory = NormalizePath(Path.str());
-      EnsureEndsInSlash(g_config->cacheDirectory);
+      g_config->cache.directory = NormalizePath(Path.str());
+      EnsureEndsInSlash(g_config->cache.directory);
     }
   }
 
@@ -334,13 +334,16 @@ void Initialize(MessageHandler *m, InitializeParam &param, ReplyOnce &reply) {
   }
   if (param.workspaceFolders.empty())
     g_config->workspaceFolders.push_back(project_path);
-  if (g_config->cacheDirectory.size())
+
+  if (g_config->cache.directory.empty())
+    g_config->cache.retainInMemory = 1;
+  else if (!g_config->cache.hierarchicalPath)
     for (const std::string &folder : g_config->workspaceFolders) {
       // Create two cache directories for files inside and outside of the
       // project.
       std::string escaped = EscapeFileName(folder.substr(0, folder.size() - 1));
-      sys::fs::create_directories(g_config->cacheDirectory + escaped);
-      sys::fs::create_directories(g_config->cacheDirectory + '@' + escaped);
+      sys::fs::create_directories(g_config->cache.directory + escaped);
+      sys::fs::create_directories(g_config->cache.directory + '@' + escaped);
     }
 
   idx::Init();
diff --git a/src/messages/textDocument_did.cc b/src/messages/textDocument_did.cc
index 1f2a47de..84814e19 100644
--- a/src/messages/textDocument_did.cc
+++ b/src/messages/textDocument_did.cc
@@ -23,6 +23,7 @@ void MessageHandler::textDocument_didClose(TextDocumentParam &param) {
   std::string path = param.textDocument.uri.GetPath();
   wfiles->OnClose(path);
   manager->OnClose(path);
+  pipeline::RemoveCache(path);
 }
 
 void MessageHandler::textDocument_didOpen(DidOpenTextDocumentParam &param) {
diff --git a/src/messages/workspace.cc b/src/messages/workspace.cc
index baea03c2..c7af6136 100644
--- a/src/messages/workspace.cc
+++ b/src/messages/workspace.cc
@@ -34,8 +34,8 @@ void MessageHandler::workspace_didChangeWatchedFiles(
     DidChangeWatchedFilesParam &param) {
   for (auto &event : param.changes) {
     std::string path = event.uri.GetPath();
-    if ((g_config->cacheDirectory.size() &&
-         StringRef(path).startswith(g_config->cacheDirectory)) ||
+    if ((g_config->cache.directory.size() &&
+         StringRef(path).startswith(g_config->cache.directory)) ||
         lookupExtension(path).first == LanguageId::Unknown)
       return;
     for (std::string cur = path; cur.size(); cur = sys::path::parent_path(cur))
diff --git a/src/pipeline.cc b/src/pipeline.cc
index f7831076..4f48b0a0 100644
--- a/src/pipeline.cc
+++ b/src/pipeline.cc
@@ -45,7 +45,7 @@ void VFS::Clear() {
   state.clear();
 }
 
-bool VFS::Loaded(const std::string &path) {
+int VFS::Loaded(const std::string &path) {
   std::lock_guard lock(mutex);
   return state[path].loaded;
 }
@@ -125,7 +125,7 @@ bool CacheInvalid(VFS *vfs, IndexFile *prev, const std::string &path,
 };
 
 std::string AppendSerializationFormat(const std::string &base) {
-  switch (g_config->cacheFormat) {
+  switch (g_config->cache.format) {
   case SerializeFormat::Binary:
     return base + ".blob";
   case SerializeFormat::Json:
@@ -133,27 +133,35 @@ std::string AppendSerializationFormat(const std::string &base) {
   }
 }
 
-std::string GetCachePath(const std::string &source_file) {
+std::string GetCachePath(const std::string &src) {
+  if (g_config->cache.hierarchicalPath) {
+    std::string ret = src[0] == '/' ? src.substr(1) : src;
+#ifdef _WIN32
+    std::replace(ret.begin(), ret.end(), ':', '@');
+#endif
+    return g_config->cache.directory + ret;
+  }
   for (auto &root : g_config->workspaceFolders)
-    if (StringRef(source_file).startswith(root)) {
+    if (StringRef(src).startswith(root)) {
       auto len = root.size();
-      return g_config->cacheDirectory +
+      return g_config->cache.directory +
              EscapeFileName(root.substr(0, len - 1)) + '/' +
-             EscapeFileName(source_file.substr(len));
+             EscapeFileName(src.substr(len));
     }
-  return g_config->cacheDirectory + '@' +
+  return g_config->cache.directory + '@' +
          EscapeFileName(g_config->fallbackFolder.substr(
              0, g_config->fallbackFolder.size() - 1)) +
-         '/' + EscapeFileName(source_file);
+         '/' + EscapeFileName(src);
 }
 
 std::unique_ptr<IndexFile> RawCacheLoad(const std::string &path) {
-  if (g_config->cacheDirectory.empty()) {
+  if (g_config->cache.retainInMemory) {
     std::shared_lock lock(g_index_mutex);
     auto it = g_index.find(path);
-    if (it == g_index.end())
+    if (it != g_index.end())
+      return std::make_unique<IndexFile>(it->second.index);
+    if (g_config->cache.directory.empty())
       return nullptr;
-    return std::make_unique<IndexFile>(it->second.index);
   }
 
   std::string cache_path = GetCachePath(path);
@@ -163,7 +171,7 @@ std::unique_ptr<IndexFile> RawCacheLoad(const std::string &path) {
   if (!file_content || !serialized_indexed_content)
     return nullptr;
 
-  return ccls::Deserialize(g_config->cacheFormat, path,
+  return ccls::Deserialize(g_config->cache.format, path,
                            *serialized_indexed_content, *file_content,
                            IndexFile::kMajorVersion);
 }
@@ -275,7 +283,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
         request.mode != IndexMode::NonInteractive);
       {
         std::lock_guard lock1(vfs->mutex);
-        vfs->state[path_to_index].loaded = true;
+        vfs->state[path_to_index].loaded++;
       }
       lock.unlock();
 
@@ -292,7 +300,7 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
           VFS::State &st = vfs->state[path];
           if (st.loaded)
             continue;
-          st.loaded = true;
+          st.loaded++;
           st.timestamp = prev->mtime;
         }
         IndexUpdate update = IndexUpdate::CreateDelta(nullptr, prev.get());
@@ -355,31 +363,37 @@ bool Indexer_Parse(SemaManager *completion, WorkingFiles *wfiles,
                            << " (delta: " << !!prev << ")";
     {
       std::lock_guard lock(GetFileMutex(path));
-      if (vfs->Loaded(path))
+      int loaded = vfs->Loaded(path), retain = g_config->cache.retainInMemory;
+      if (loaded)
         prev = RawCacheLoad(path);
       else
         prev.reset();
-      if (g_config->cacheDirectory.empty()) {
+      if (retain > 0 && retain <= loaded + 1) {
         std::lock_guard lock(g_index_mutex);
         auto it = g_index.insert_or_assign(
           path, InMemoryIndexFile{curr->file_contents, *curr});
         std::string().swap(it.first->second.index.file_contents);
-      } else {
+      }
+      if (g_config->cache.directory.size()) {
         std::string cache_path = GetCachePath(path);
         if (deleted) {
           (void)sys::fs::remove(cache_path);
           (void)sys::fs::remove(AppendSerializationFormat(cache_path));
         } else {
+          if (g_config->cache.hierarchicalPath)
+            sys::fs::create_directories(
+                sys::path::parent_path(cache_path, sys::path::Style::posix),
+                true);
           WriteToFile(cache_path, curr->file_contents);
           WriteToFile(AppendSerializationFormat(cache_path),
-                      Serialize(g_config->cacheFormat, *curr));
+                      Serialize(g_config->cache.format, *curr));
         }
       }
       on_indexed->PushBack(IndexUpdate::CreateDelta(prev.get(), curr.get()),
                            request.mode != IndexMode::NonInteractive);
       {
         std::lock_guard lock1(vfs->mutex);
-        vfs->state[path].loaded = true;
+        vfs->state[path].loaded++;
       }
       if (entry.id >= 0) {
         std::lock_guard lock(project->mtx);
@@ -706,8 +720,15 @@ void Index(const std::string &path, const std::vector<const char *> &args,
                           mode != IndexMode::NonInteractive);
 }
 
+void RemoveCache(const std::string &path) {
+  if (g_config->cache.directory.size()) {
+    std::lock_guard lock(g_index_mutex);
+    g_index.erase(path);
+  }
+}
+
 std::optional<std::string> LoadIndexedContent(const std::string &path) {
-  if (g_config->cacheDirectory.empty()) {
+  if (g_config->cache.directory.empty()) {
     std::shared_lock lock(g_index_mutex);
     auto it = g_index.find(path);
     if (it == g_index.end())
diff --git a/src/pipeline.hh b/src/pipeline.hh
index 0ec48090..5dd35c08 100644
--- a/src/pipeline.hh
+++ b/src/pipeline.hh
@@ -22,13 +22,13 @@ struct VFS {
   struct State {
     int64_t timestamp;
     int step;
-    bool loaded;
+    int loaded;
   };
   std::unordered_map<std::string, State> state;
   std::mutex mutex;
 
   void Clear();
-  bool Loaded(const std::string &path);
+  int Loaded(const std::string &path);
   bool Stamp(const std::string &path, int64_t ts, int step);
 };
 
@@ -55,7 +55,7 @@ void Standalone(const std::string &root);
 
 void Index(const std::string &path, const std::vector<const char *> &args,
            IndexMode mode, bool must_exist, RequestId id = {});
-
+void RemoveCache(const std::string &path);
 std::optional<std::string> LoadIndexedContent(const std::string& path);
 
 void NotifyOrRequest(const char *method, bool request,
diff --git a/src/utils.cc b/src/utils.cc
index 6dc288b1..c291de01 100644
--- a/src/utils.cc
+++ b/src/utils.cc
@@ -112,9 +112,10 @@ void EnsureEndsInSlash(std::string &path) {
 
 std::string EscapeFileName(std::string path) {
   bool slash = path.size() && path.back() == '/';
-  for (char &c : path)
-    if (c == '\\' || c == '/' || c == ':')
-      c = '@';
+#ifdef _WIN32
+  std::replace(path.begin(), path.end(), ':', '@');
+#endif
+  std::replace(path.begin(), path.end(), '/', '@');
   if (slash)
     path += '/';
   return path;