Better support for removing data from querydb

src/command_line.cc

@@ -293,8 +293,11 @@ void PushBack(NonElidedVector<lsLocation>* result, optional<lsLocation> location
 QueryFile* FindFile(QueryDatabase* db, const std::string& filename, QueryFileId* file_id) {
   auto it = db->usr_to_symbol.find(filename);
   if (it != db->usr_to_symbol.end()) {
-    *file_id = QueryFileId(it->second.idx);
+    optional<QueryFile>& file = db->files[it->second.idx];
-    return &db->files[it->second.idx];
+    if (file) {
+      *file_id = QueryFileId(it->second.idx);
+      return &file.value();
+    }
   }
 
   std::cerr << "Unable to find file " << filename << std::endl;
@@ -305,69 +308,137 @@ QueryFile* FindFile(QueryDatabase* db, const std::string& filename, QueryFileId*
 QueryFile* FindFile(QueryDatabase* db, const std::string& filename) {
   // TODO: consider calling NormalizePath here. It might add too much latency though.
   auto it = db->usr_to_symbol.find(filename);
-  if (it != db->usr_to_symbol.end())
+  if (it != db->usr_to_symbol.end()) {
-    return &db->files[it->second.idx];
+    optional<QueryFile>& file = db->files[it->second.idx];
+    if (file)
+      return &file.value();
+  }
 
   std::cerr << "Unable to find file " << filename << std::endl;
   return nullptr;
 }
 
-QueryFile* GetQuery(QueryDatabase* db, const QueryFileId& id) {
+optional<QueryFile>* GetQuery(QueryDatabase* db, const QueryFileId& id) {
   return &db->files[id.id];
 }
-QueryType* GetQuery(QueryDatabase* db, const QueryTypeId& id) {
+optional<QueryType>* GetQuery(QueryDatabase* db, const QueryTypeId& id) {
   return &db->types[id.id];
 }
-QueryFunc* GetQuery(QueryDatabase* db, const QueryFuncId& id) {
+optional<QueryFunc>* GetQuery(QueryDatabase* db, const QueryFuncId& id) {
   return &db->funcs[id.id];
 }
-QueryVar* GetQuery(QueryDatabase* db, const QueryVarId& id) {
+optional<QueryVar>* GetQuery(QueryDatabase* db, const QueryVarId& id) {
   return &db->vars[id.id];
 }
 
 
 
-
 optional<QueryLocation> GetDefinitionSpellingOfSymbol(QueryDatabase* db, const QueryTypeId& id) {
-  return GetQuery(db, id)->def.definition_spelling;
+  optional<QueryType>& type = db->types[id.id];
+  if (type)
+    return type->def.definition_spelling;
+  return nullopt;
 }
+
 optional<QueryLocation> GetDefinitionSpellingOfSymbol(QueryDatabase* db, const QueryFuncId& id) {
-  return GetQuery(db, id)->def.definition_spelling;
+  optional<QueryFunc>& func = db->funcs[id.id];
+  if (func)
+    return func->def.definition_spelling;
+  return nullopt;
 }
+
 optional<QueryLocation> GetDefinitionSpellingOfSymbol(QueryDatabase* db, const QueryVarId& id) {
-  return GetQuery(db, id)->def.definition_spelling;
+  optional<QueryVar>& var = db->vars[id.id];
+  if (var)
+    return var->def.definition_spelling;
+  return nullopt;
 }
+
 optional<QueryLocation> GetDefinitionSpellingOfSymbol(QueryDatabase* db, const SymbolIdx& symbol) {
   switch (symbol.kind) {
-  case SymbolKind::Type:
+    case SymbolKind::Type: {
-    return db->types[symbol.idx].def.definition_spelling;
+      optional<QueryType>& type = db->types[symbol.idx];
-  case SymbolKind::Func:
+      if (type)
-    return db->funcs[symbol.idx].def.definition_spelling;
+        return type->def.definition_spelling;
-  case SymbolKind::Var:
+      break;
-    return db->vars[symbol.idx].def.definition_spelling;
+    }
-  case SymbolKind::File:
+    case SymbolKind::Func: {
-  case SymbolKind::Invalid: {
+      optional<QueryFunc>& func = db->funcs[symbol.idx];
-    assert(false && "unexpected");
+      if (func)
-    break;
+        return func->def.definition_spelling;
-  }
+      break;
+    }
+    case SymbolKind::Var: {
+      optional<QueryVar>& var = db->vars[symbol.idx];
+      if (var)
+        return var->def.definition_spelling;
+      break;
+    }
+    case SymbolKind::File:
+    case SymbolKind::Invalid: {
+      assert(false && "unexpected");
+      break;
+    }
   }
   return nullopt;
 }
 
+optional<QueryLocation> GetDefinitionExtentOfSymbol(QueryDatabase* db, const SymbolIdx& symbol) {
+  switch (symbol.kind) {
+    case SymbolKind::Type: {
+      optional<QueryType>& type = db->types[symbol.idx];
+      if (type)
+        return type->def.definition_extent;
+      break;
+    }
+    case SymbolKind::Func: {
+      optional<QueryFunc>& func = db->funcs[symbol.idx];
+      if (func)
+        return func->def.definition_extent;
+      break;
+    }
+    case SymbolKind::Var: {
+      optional<QueryVar>& var = db->vars[symbol.idx];
+      if (var)
+        return var->def.definition_extent;
+      break;
+    }
+    case SymbolKind::File: {
+      return QueryLocation(QueryFileId(symbol.idx), Range(Position(1, 1), Position(1, 1)));
+    }
+    case SymbolKind::Invalid: {
+      assert(false && "unexpected");
+      break;
+    }
+  }
+  return nullopt;
+}
 
 std::string GetHoverForSymbol(QueryDatabase* db, const SymbolIdx& symbol) {
   switch (symbol.kind) {
-  case SymbolKind::Type:
+    case SymbolKind::Type: {
-    return db->types[symbol.idx].def.detailed_name;
+      optional<QueryType>& type = db->types[symbol.idx];
-  case SymbolKind::Func:
+      if (type)
-    return db->funcs[symbol.idx].def.detailed_name;
+        return type->def.detailed_name;
-  case SymbolKind::Var:
+      break;
-    return db->vars[symbol.idx].def.detailed_name;
+    }
-  case SymbolKind::File:
+    case SymbolKind::Func: {
-  case SymbolKind::Invalid: {
+      optional<QueryFunc>& func = db->funcs[symbol.idx];
-    assert(false && "unexpected");
+      if (func)
-    break;
+        return func->def.detailed_name;
-  }
+      break;
+    }
+    case SymbolKind::Var: {
+      optional<QueryVar>& var = db->vars[symbol.idx];
+      if (var)
+        return var->def.detailed_name;
+      break;
+    }
+    case SymbolKind::File:
+    case SymbolKind::Invalid: {
+      assert(false && "unexpected");
+      break;
+    }
   }
   return "";
 }
@@ -414,19 +485,30 @@ std::vector<QueryLocation> ToQueryLocation(QueryDatabase* db, const std::vector<
 
 std::vector<QueryLocation> GetUsesOfSymbol(QueryDatabase* db, const SymbolIdx& symbol) {
   switch (symbol.kind) {
-    case SymbolKind::Type:
+    case SymbolKind::Type: {
-      return db->types[symbol.idx].uses;
+      optional<QueryType>& type = db->types[symbol.idx];
+      if (type)
+        return type->uses;
+      break;
+    }
     case SymbolKind::Func: {
       // TODO: the vector allocation could be avoided.
-      const QueryFunc& func = db->funcs[symbol.idx];
+      optional<QueryFunc>& func = db->funcs[symbol.idx];
-      std::vector<QueryLocation> result = ToQueryLocation(db, func.callers);
+      if (func) {
-      AddRange(&result, func.declarations);
+        std::vector<QueryLocation> result = ToQueryLocation(db, func->callers);
-      if (func.def.definition_spelling)
+        AddRange(&result, func->declarations);
-        result.push_back(*func.def.definition_spelling);
+        if (func->def.definition_spelling)
-      return result;
+          result.push_back(*func->def.definition_spelling);
+        return result;
+      }
+      break;
+    }
+    case SymbolKind::Var: {
+      optional<QueryVar>& var = db->vars[symbol.idx];
+      if (var)
+        return var->uses;
+      break;
     }
-    case SymbolKind::Var:
-      return db->vars[symbol.idx].uses;
     case SymbolKind::File:
     case SymbolKind::Invalid: {
       assert(false && "unexpected");
@@ -436,34 +518,6 @@ std::vector<QueryLocation> GetUsesOfSymbol(QueryDatabase* db, const SymbolIdx& s
   return {};
 }
 
-optional<QueryLocation> GetDefinitionExtentOfSymbol(QueryDatabase* db, const QueryTypeId& id) {
-  return GetQuery(db, id)->def.definition_extent;
-}
-optional<QueryLocation> GetDefinitionExtentOfSymbol(QueryDatabase* db, const QueryFuncId& id) {
-  return GetQuery(db, id)->def.definition_extent;
-}
-optional<QueryLocation> GetDefinitionExtentOfSymbol(QueryDatabase* db, const QueryVarId& id) {
-  return GetQuery(db, id)->def.definition_extent;
-}
-optional<QueryLocation> GetDefinitionExtentOfSymbol(QueryDatabase* db, const SymbolIdx& symbol) {
-  switch (symbol.kind) {
-    case SymbolKind::File:
-      // TODO: If line 1 is deleted the file won't show up in, ie, workspace symbol search results.
-      return QueryLocation(QueryFileId(symbol.idx), Range(Position(1, 1), Position(1, 1)));
-    case SymbolKind::Type:
-      return db->types[symbol.idx].def.definition_extent;
-    case SymbolKind::Func:
-      return db->funcs[symbol.idx].def.definition_extent;
-    case SymbolKind::Var:
-      return db->vars[symbol.idx].def.definition_extent;
-    case SymbolKind::Invalid: {
-      assert(false && "unexpected");
-      break;
-    }
-  }
-  return nullopt;
-}
-
 std::vector<QueryLocation> GetDeclarationsOfSymbolForGotoDefinition(QueryDatabase* db, const SymbolIdx& symbol) {
   switch (symbol.kind) {
     case SymbolKind::Type: {
@@ -471,17 +525,27 @@ std::vector<QueryLocation> GetDeclarationsOfSymbolForGotoDefinition(QueryDatabas
       // function has the postfix `ForGotoDefintion`, but it lets the user
       // jump to the start of a type if clicking goto-definition on the same
       // type from within the type definition.
-      optional<QueryLocation> declaration = db->types[symbol.idx].def.definition_spelling;
+      optional<QueryType>& type = db->types[symbol.idx];
-      if (declaration)
+      if (type) {
-        return { *declaration };
+        optional<QueryLocation> declaration = type->def.definition_spelling;
+        if (declaration)
+          return { *declaration };
+      }
+      break;
+    }
+    case SymbolKind::Func: {
+      optional<QueryFunc>& func = db->funcs[symbol.idx];
+      if (func)
+        return func->declarations;
       break;
     }
-    case SymbolKind::Func:
-      return db->funcs[symbol.idx].declarations;
     case SymbolKind::Var: {
-      optional<QueryLocation> declaration = db->vars[symbol.idx].def.declaration;
+      optional<QueryVar>& var = db->vars[symbol.idx];
-      if (declaration)
+      if (var) {
-        return { *declaration };
+        optional<QueryLocation> declaration = var->def.declaration;
+        if (declaration)
+          return { *declaration };
+      }
       break;
     }
   }
@@ -492,10 +556,13 @@ std::vector<QueryLocation> GetDeclarationsOfSymbolForGotoDefinition(QueryDatabas
 optional<QueryLocation> GetBaseDefinitionOrDeclarationSpelling(QueryDatabase* db, QueryFunc& func) {
   if (!func.def.base)
     return nullopt;
-  QueryFunc& base = db->funcs[func.def.base->id];
+  optional<QueryFunc>& base = db->funcs[func.def.base->id];
-  auto def = base.def.definition_spelling;
+  if (!base)
-  if (!def && !base.declarations.empty())
+    return nullopt;
-    def = base.declarations[0];
+
+  auto def = base->def.definition_spelling;
+  if (!def && !base->declarations.empty())
+    def = base->declarations[0];
   return def;
 }
 
@@ -504,9 +571,12 @@ std::vector<QueryFuncRef> GetCallersForAllBaseFunctions(QueryDatabase* db, Query
 
   optional<QueryFuncId> func_id = root.def.base;
   while (func_id) {
-    QueryFunc& func = db->funcs[func_id->id];
+    optional<QueryFunc>& func = db->funcs[func_id->id];
-    AddRange(&callers, func.callers);
+    if (!func)
-    func_id = func.def.base;
+      break;
+
+    AddRange(&callers, func->callers);
+    func_id = func->def.base;
   }
 
   return callers;
@@ -519,11 +589,13 @@ std::vector<QueryFuncRef> GetCallersForAllDerivedFunctions(QueryDatabase* db, Qu
   PushRange(&queue, root.derived);
 
   while (!queue.empty()) {
-    QueryFunc& func = db->funcs[queue.front().id];
+    optional<QueryFunc>& func = db->funcs[queue.front().id];
     queue.pop();
-    PushRange(&queue, func.derived);
+    if (!func)
+      continue;
 
-    AddRange(&callers, func.callers);
+    PushRange(&queue, func->derived);
+    AddRange(&callers, func->callers);
   }
 
   return callers;
@@ -547,13 +619,25 @@ optional<lsRange> GetLsRange(WorkingFile* working_file, const Range& location) {
 }
 
 lsDocumentUri GetLsDocumentUri(QueryDatabase* db, QueryFileId file_id, std::string* path) {
-  *path = db->files[file_id.id].def.path;
+  optional<QueryFile>& file = db->files[file_id.id];
-  return lsDocumentUri::FromPath(*path);
+  if (file) {
+    *path = file->def.path;
+    return lsDocumentUri::FromPath(*path);
+  }
+  else {
+    *path = "";
+    return lsDocumentUri::FromPath("");
+  }
 }
 
 lsDocumentUri GetLsDocumentUri(QueryDatabase* db, QueryFileId file_id) {
-  std::string path = db->files[file_id.id].def.path;
+  optional<QueryFile>& file = db->files[file_id.id];
-  return lsDocumentUri::FromPath(path);
+  if (file) {
+    return lsDocumentUri::FromPath(file->def.path);
+  }
+  else {
+    return lsDocumentUri::FromPath("");
+  }
 }
 
 optional<lsLocation> GetLsLocation(QueryDatabase* db, WorkingFiles* working_files, const QueryLocation& location) {
@@ -567,46 +651,62 @@ optional<lsLocation> GetLsLocation(QueryDatabase* db, WorkingFiles* working_file
 
 // Returns a symbol. The symbol will have *NOT* have a location assigned.
 optional<lsSymbolInformation> GetSymbolInfo(QueryDatabase* db, WorkingFiles* working_files, SymbolIdx symbol) {
-  lsSymbolInformation info;
+    switch (symbol.kind) {
-
-  switch (symbol.kind) {
     case SymbolKind::File: {
-      QueryFile* file = symbol.ResolveFile(db);
+      optional<QueryFile>& file = db->files[symbol.idx];
+      if (!file)
+        return nullopt;
+
+      lsSymbolInformation info;
       info.name = file->def.path;
       info.kind = lsSymbolKind::File;
-      break;
+      return info;
     }
     case SymbolKind::Type: {
-      QueryType* type = symbol.ResolveType(db);
+      optional<QueryType>& type = db->types[symbol.idx];
+      if (!type)
+        return nullopt;
+
+      lsSymbolInformation info;
       info.name = type->def.detailed_name;
       info.kind = lsSymbolKind::Class;
-      break;
+      return info;
     }
     case SymbolKind::Func: {
-      QueryFunc* func = symbol.ResolveFunc(db);
+      optional<QueryFunc>& func = db->funcs[symbol.idx];
+      if (!func)
+        return nullopt;
 
+      lsSymbolInformation info;
       info.name = func->def.detailed_name;
+      info.kind = lsSymbolKind::Function;
+
       if (func->def.declaring_type.has_value()) {
-        info.kind = lsSymbolKind::Method;
+        optional<QueryType>& container = db->types[func->def.declaring_type->id];
-        info.containerName = db->types[func->def.declaring_type->id].def.detailed_name;
+        if (container) {
+          info.kind = lsSymbolKind::Method;
+          info.containerName = container->def.detailed_name;
+        }
       }
-      else {
+
-        info.kind = lsSymbolKind::Function;
+      return info;
-      }
-      break;
     }
     case SymbolKind::Var: {
-      QueryVar* var = symbol.ResolveVar(db);
+      optional<QueryVar>& var = db->vars[symbol.idx];
+      if (!var)
+        return nullopt;
+
+      lsSymbolInformation info;
       info.name += var->def.detailed_name;
       info.kind = lsSymbolKind::Variable;
-      break;
+      return info;
     }
     case SymbolKind::Invalid: {
       return nullopt;
     }
   };
 
-  return info;
+  return nullopt;
 }
 
 struct CommonCodeLensParams {
@@ -669,7 +769,11 @@ lsWorkspaceEdit BuildWorkspaceEdit(QueryDatabase* db, WorkingFiles* working_file
     if (path_to_edit.find(location.path) == path_to_edit.end()) {
       path_to_edit[location.path] = lsTextDocumentEdit();
 
-      const std::string& path = db->files[location.path.id].def.path;
+      optional<QueryFile>& file = db->files[location.path.id];
+      if (!file)
+        continue;
+
+      const std::string& path = file->def.path;
       path_to_edit[location.path].textDocument.uri = lsDocumentUri::FromPath(path);
 
       WorkingFile* working_file = working_files->GetFileByFilename(path);
@@ -1203,26 +1307,28 @@ void QueryDbMainLoop(
     }
 
     case IpcId::TextDocumentDidOpen: {
+      // NOTE: This function blocks code lens. If it starts taking a long time
+      // we will need to find a way to unblock the code lens request.
+
+      Timer time;
       auto msg = static_cast<Ipc_TextDocumentDidOpen*>(message.get());
-      //std::cerr << "Opening " << msg->params.textDocument.uri.GetPath() << std::endl;
       WorkingFile* working_file = working_files->OnOpen(msg->params);
       optional<std::string> cached_file_contents = LoadCachedFileContents(config, msg->params.textDocument.uri.GetPath());
       if (cached_file_contents)
         working_file->SetIndexContent(*cached_file_contents);
       else
         working_file->SetIndexContent(working_file->buffer_content);
+      time.ResetAndPrint("[querydb] Loading cached index file for DidOpen");
 
       break;
     }
     case IpcId::TextDocumentDidChange: {
       auto msg = static_cast<Ipc_TextDocumentDidChange*>(message.get());
       working_files->OnChange(msg->params);
-      //std::cerr << "Changing " << msg->params.textDocument.uri.GetPath() << std::endl;
       break;
     }
     case IpcId::TextDocumentDidClose: {
       auto msg = static_cast<Ipc_TextDocumentDidClose*>(message.get());
-      //std::cerr << "Closing " << msg->params.textDocument.uri.GetPath() << std::endl;
       working_files->OnClose(msg->params);
       break;
     }
@@ -1518,36 +1624,39 @@ void QueryDbMainLoop(
         SymbolIdx symbol = ref.idx;
         switch (symbol.kind) {
         case SymbolKind::Type: {
-          QueryType& type = db->types[symbol.idx];
+          optional<QueryType>& type = db->types[symbol.idx];
-          AddCodeLens(&common, ref.loc.OffsetStartColumn(0), type.uses, "ref", "refs");
+          if (!type)
-          AddCodeLens(&common, ref.loc.OffsetStartColumn(1), ToQueryLocation(db, type.derived), "derived", "derived");
+            continue;
-          AddCodeLens(&common, ref.loc.OffsetStartColumn(2), ToQueryLocation(db, type.instances), "var", "vars");
+          AddCodeLens(&common, ref.loc.OffsetStartColumn(0), type->uses, "ref", "refs");
+          AddCodeLens(&common, ref.loc.OffsetStartColumn(1), ToQueryLocation(db, type->derived), "derived", "derived");
+          AddCodeLens(&common, ref.loc.OffsetStartColumn(2), ToQueryLocation(db, type->instances), "var", "vars");
           break;
         }
         case SymbolKind::Func: {
-          QueryFunc& func = db->funcs[symbol.idx];
+          optional<QueryFunc>& func = db->funcs[symbol.idx];
-
+          if (!func)
+            continue;
 
           int offset = 0;
 
-          std::vector<QueryFuncRef> base_callers = GetCallersForAllBaseFunctions(db, func);
+          std::vector<QueryFuncRef> base_callers = GetCallersForAllBaseFunctions(db, *func);
-          std::vector<QueryFuncRef> derived_callers = GetCallersForAllDerivedFunctions(db, func);
+          std::vector<QueryFuncRef> derived_callers = GetCallersForAllDerivedFunctions(db, *func);
           if (base_callers.empty() && derived_callers.empty()) {
             // set exclude_loc to true to force the code lens to show up
-            AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, func.callers), "call", "calls", true /*exclude_loc*/);
+            AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, func->callers), "call", "calls", true /*exclude_loc*/);
           }
           else {
-            AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, func.callers), "direct call", "direct calls");
+            AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, func->callers), "direct call", "direct calls");
             if (!base_callers.empty())
               AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, base_callers), "base call", "base calls");
             if (!derived_callers.empty())
               AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, derived_callers), "derived call", "derived calls");
           }
 
-          AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, func.derived), "derived", "derived");
+          AddCodeLens(&common, ref.loc.OffsetStartColumn(offset++), ToQueryLocation(db, func->derived), "derived", "derived");
 
           // "Base"
-          optional<QueryLocation> base_loc = GetBaseDefinitionOrDeclarationSpelling(db, func);
+          optional<QueryLocation> base_loc = GetBaseDefinitionOrDeclarationSpelling(db, *func);
           if (base_loc) {
             optional<lsLocation> ls_base = GetLsLocation(db, working_files, *base_loc);
             if (ls_base) {
@@ -1569,8 +1678,11 @@ void QueryDbMainLoop(
           break;
         }
         case SymbolKind::Var: {
-          QueryVar& var = db->vars[symbol.idx];
+          optional<QueryVar>& var = db->vars[symbol.idx];
-          AddCodeLens(&common, ref.loc.OffsetStartColumn(0), var.uses, "ref", "refs", true /*exclude_loc*/);
+          if (!var)
+            continue;
+
+          AddCodeLens(&common, ref.loc.OffsetStartColumn(0), var->uses, "ref", "refs", true /*exclude_loc*/);
           break;
         }
         case SymbolKind::File:

src/query.cc

@@ -69,8 +69,6 @@ void AddMergeableRange(
   // time at the cost of some additional memory.
 
   // Build lookup table.
-  //google::dense_hash_map<TId, size_t, std::hash<TId>> id_to_index;
-  //id_to_index.set_empty_key(TId(-1));
   spp::sparse_hash_map<TId, size_t> id_to_index;
   id_to_index.resize(dest->size());
   for (size_t i = 0; i < dest->size(); ++i)
@@ -232,49 +230,6 @@ QueryFile::Def BuildFileDef(const IdMap& id_map, const IndexedFile& indexed) {
 
 
 
-
-
-
-
-
-
-
-QueryFile* SymbolIdx::ResolveFile(QueryDatabase* db) const {
-  assert(kind == SymbolKind::File);
-  return &db->files[idx];
-}
-QueryType* SymbolIdx::ResolveType(QueryDatabase* db) const {
-  assert(kind == SymbolKind::Type);
-  return &db->types[idx];
-}
-QueryFunc* SymbolIdx::ResolveFunc(QueryDatabase* db) const {
-  assert(kind == SymbolKind::Func);
-  return &db->funcs[idx];
-}
-QueryVar* SymbolIdx::ResolveVar(QueryDatabase* db) const {
-  assert(kind == SymbolKind::Var);
-  return &db->vars[idx];
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 
 
 
@@ -349,17 +304,14 @@ IdMap::IdMap(QueryDatabase* query_db, const IdCache& local_ids)
   : local_ids(local_ids) {
   primary_file = GetQueryFileIdFromPath(query_db, local_ids.primary_file);
 
-  //cached_type_ids_.set_empty_key(IndexTypeId(-1));
   cached_type_ids_.resize(local_ids.type_id_to_usr.size());
   for (const auto& entry : local_ids.type_id_to_usr)
     cached_type_ids_[entry.first] = GetQueryTypeIdFromUsr(query_db, entry.second);
 
-  //cached_func_ids_.set_empty_key(IndexFuncId(-1));
   cached_func_ids_.resize(local_ids.func_id_to_usr.size());
   for (const auto& entry : local_ids.func_id_to_usr)
     cached_func_ids_[entry.first] = GetQueryFuncIdFromUsr(query_db, entry.second);
 
-  //cached_var_ids_.set_empty_key(IndexVarId(-1));
   cached_var_ids_.resize(local_ids.var_id_to_usr.size());
   for (const auto& entry : local_ids.var_id_to_usr)
     cached_var_ids_[entry.first] = GetQueryVarIdFromUsr(query_db, entry.second);
@@ -646,24 +598,51 @@ void IndexUpdate::Merge(const IndexUpdate& update) {
 void QueryDatabase::RemoveUsrs(const std::vector<Usr>& to_remove) {
   // This function runs on the querydb thread.
 
-  // Actually removing data is extremely slow because every offset/index would
+  // When we remove an element, we just erase the state from the storage. We do
-  // have to be updated. Instead, we just accept the memory overhead and mark
+  // not update array indices because that would take a huge amount of time for
-  // the symbol as invalid.
+  // a very large index.
   //
-  // If the user wants to reduce memory usage, they will have to restart the
+  // There means that there is some memory growth that will never be reclaimed,
-  // indexer and load it from cache. Luckily, this doesn't take too long even
+  // but it should be pretty minimal and is solved by simply restarting the
-  // on large projects (1-2 minutes).
+  // indexer and loading from cache, which is a fast operation.
+  //
+  // TODO: Add "cquery: Reload Index" command which unloads all querydb state
+  // and fully reloads from cache. This will address the memory leak above.
 
-  for (Usr usr : to_remove)
+  for (Usr usr : to_remove) {
-    usr_to_symbol[usr].kind = SymbolKind::Invalid;
+    SymbolIdx& symbol = usr_to_symbol[usr];
+    switch (symbol.kind) {
+      case SymbolKind::File:
+        files[symbol.idx] = nullopt;
+        break;
+      case SymbolKind::Type:
+        types[symbol.idx] = nullopt;
+        break;
+      case SymbolKind::Func:
+        funcs[symbol.idx] = nullopt;
+        break;
+      case SymbolKind::Var:
+        vars[symbol.idx] = nullopt;
+        break;
+    }
+
+    symbol.kind = SymbolKind::Invalid;
+  }
 }
 
 void QueryDatabase::ApplyIndexUpdate(IndexUpdate* update) {
   // This function runs on the querydb thread.
 
+  // Example types:
+  //  storage_name       =>  std::vector<optional<QueryType>>
+  //  merge_update       =>  QueryType::DerivedUpdate => MergeableUpdate<QueryTypeId, QueryTypeId>
+  //  def                =>  QueryType
+  //  def->def_var_name  =>  std::vector<QueryTypeId>
 #define HANDLE_MERGEABLE(update_var_name, def_var_name, storage_name) \
   for (auto merge_update : update->update_var_name) { \
-    auto* def = &storage_name[merge_update.id.id]; \
+    auto& def = storage_name[merge_update.id.id]; \
+    if (!def) \
+      continue; /* TODO: Should we continue or create an empty def? */ \
     AddRange(&def->def_var_name, merge_update.to_add); \
     RemoveRange(&def->def_var_name, merge_update.to_remove); \
   }
@@ -697,9 +676,12 @@ void QueryDatabase::ImportOrUpdate(const std::vector<QueryFile::DefUpdate>& upda
     auto it = usr_to_symbol.find(def.path);
     assert(it != usr_to_symbol.end());
 
-    QueryFile& existing = files[it->second.idx];
+    optional<QueryFile>& existing = files[it->second.idx];
-    existing.def = def;
+    if (!existing)
-    UpdateDetailedNames(&existing.detailed_name_idx, SymbolKind::File, it->second.idx, def.path);
+      existing = QueryFile(def.path);
+
+    existing->def = def;
+    UpdateDetailedNames(&existing->detailed_name_idx, SymbolKind::File, it->second.idx, def.path);
   }
 }
 
@@ -712,14 +694,16 @@ void QueryDatabase::ImportOrUpdate(const std::vector<QueryType::DefUpdate>& upda
     auto it = usr_to_symbol.find(def.usr);
     assert(it != usr_to_symbol.end());
 
-    QueryType& existing = types[it->second.idx];
+    optional<QueryType>& existing = types[it->second.idx];
+    if (!existing)
+      existing = QueryType(def.usr);
 
     // Keep the existing definition if it is higher quality.
-    if (existing.def.definition_spelling && !def.definition_spelling)
+    if (existing->def.definition_spelling && !def.definition_spelling)
       continue;
 
-    existing.def = def;
+    existing->def = def;
-    UpdateDetailedNames(&existing.detailed_name_idx, SymbolKind::Type, it->second.idx, def.detailed_name);
+    UpdateDetailedNames(&existing->detailed_name_idx, SymbolKind::Type, it->second.idx, def.detailed_name);
   }
 }
 
@@ -732,14 +716,16 @@ void QueryDatabase::ImportOrUpdate(const std::vector<QueryFunc::DefUpdate>& upda
     auto it = usr_to_symbol.find(def.usr);
     assert(it != usr_to_symbol.end());
 
-    QueryFunc& existing = funcs[it->second.idx];
+    optional<QueryFunc>& existing = funcs[it->second.idx];
+    if (!existing)
+      existing = QueryFunc(def.usr);
 
     // Keep the existing definition if it is higher quality.
-    if (existing.def.definition_spelling && !def.definition_spelling)
+    if (existing->def.definition_spelling && !def.definition_spelling)
       continue;
 
-    existing.def = def;
+    existing->def = def;
-    UpdateDetailedNames(&existing.detailed_name_idx, SymbolKind::Func, it->second.idx, def.detailed_name);
+    UpdateDetailedNames(&existing->detailed_name_idx, SymbolKind::Func, it->second.idx, def.detailed_name);
   }
 }
 
@@ -752,15 +738,17 @@ void QueryDatabase::ImportOrUpdate(const std::vector<QueryVar::DefUpdate>& updat
     auto it = usr_to_symbol.find(def.usr);
     assert(it != usr_to_symbol.end());
 
-    QueryVar& existing = vars[it->second.idx];
+    optional<QueryVar>& existing = vars[it->second.idx];
+    if (!existing)
+      existing = QueryVar(def.usr);
 
     // Keep the existing definition if it is higher quality.
-    if (existing.def.definition_spelling && !def.definition_spelling)
+    if (existing->def.definition_spelling && !def.definition_spelling)
       continue;
 
-    existing.def = def;
+    existing->def = def;
     if (def.declaring_type)
-      UpdateDetailedNames(&existing.detailed_name_idx, SymbolKind::Var, it->second.idx, def.detailed_name);
+      UpdateDetailedNames(&existing->detailed_name_idx, SymbolKind::Var, it->second.idx, def.detailed_name);
   }
 }
 
@@ -773,12 +761,4 @@ void QueryDatabase::UpdateDetailedNames(size_t* qualified_name_index, SymbolKind
   else {
     detailed_names[*qualified_name_index] = name;
   }
-}
+}
-
-
-
-
-
-
-
-// TODO: allow user to decide some indexer choices, ie, do we mark prototype parameters as usages?

src/query.h

@@ -74,11 +74,6 @@ struct SymbolIdx {
   bool operator<(const SymbolIdx& that) const {
     return kind < that.kind || idx < that.idx;
   }
-
-  QueryFile* ResolveFile(QueryDatabase* db) const;
-  QueryType* ResolveType(QueryDatabase* db) const;
-  QueryFunc* ResolveFunc(QueryDatabase* db) const;
-  QueryVar* ResolveVar(QueryDatabase* db) const;
 };
 
 struct SymbolRef {
@@ -188,7 +183,6 @@ struct QueryFunc {
   size_t detailed_name_idx = -1;
 
   QueryFunc(const Usr& usr) : def(usr) {}
-  QueryFunc(const DefUpdate& def) : def(def) {}
 };
 
 struct QueryVar {
@@ -250,20 +244,14 @@ struct QueryDatabase {
   std::vector<std::string> detailed_names;
   std::vector<SymbolIdx> symbols;
 
-  // Raw data storage.
+  // Raw data storage. Accessible via SymbolIdx instances.
-  std::vector<QueryFile> files; // File path is stored as a Usr.
+  std::vector<optional<QueryFile>> files;
-  std::vector<QueryType> types;
+  std::vector<optional<QueryType>> types;
-  std::vector<QueryFunc> funcs;
+  std::vector<optional<QueryFunc>> funcs;
-  std::vector<QueryVar> vars;
+  std::vector<optional<QueryVar>> vars;
 
   // Lookup symbol based on a usr.
   spp::sparse_hash_map<Usr, SymbolIdx> usr_to_symbol;
-  //google::dense_hash_map<Usr, SymbolIdx> usr_to_symbol;
-
-  QueryDatabase() {
-    //usr_to_symbol.set_empty_key("");
-  }
-  //std::unordered_map<Usr, SymbolIdx> usr_to_symbol;
 
   // Marks the given Usrs as invalid.
   void RemoveUsrs(const std::vector<Usr>& to_remove);
@@ -315,7 +303,4 @@ private:
   spp::sparse_hash_map<IndexTypeId, QueryTypeId> cached_type_ids_;
   spp::sparse_hash_map<IndexFuncId, QueryFuncId> cached_func_ids_;
   spp::sparse_hash_map<IndexVarId, QueryVarId> cached_var_ids_;
-  //google::dense_hash_map<IndexTypeId, QueryTypeId, std::hash<IndexTypeId>> cached_type_ids_;
-  //google::dense_hash_map<IndexFuncId, QueryFuncId, std::hash<IndexFuncId>> cached_func_ids_;
-  //google::dense_hash_map<IndexVarId, QueryVarId, std::hash<IndexVarId>> cached_var_ids_;
 };