wip

command_line.cc

@@ -2,6 +2,7 @@
 #include <string>
 #include <unordered_map>
 #include <thread>
+#include <vector>
 
 #include "compilation_database_loader.h"
 #include "indexer.h"
@@ -9,6 +10,8 @@
 #include "query.h"
 #include "language_server_api.h"
 
+#include "third_party/tiny-process-library/process.hpp"
+
 #include <rapidjson/istreamwrapper.h>
 #include <rapidjson/ostreamwrapper.h>
 
@@ -17,40 +20,6 @@
 #include <fcntl.h>
 #endif
 
-bool ParsePreferredSymbolLocation(const std::string& content, PreferredSymbolLocation* obj) {
-#define PARSE_AS(name, string)      \
-  if (content == #string) {         \
-    *obj = name;                    \
-    return true;                    \
-  }
-
-  PARSE_AS(PreferredSymbolLocation::Declaration, "declaration");
-  PARSE_AS(PreferredSymbolLocation::Definition, "definition");
-
-  return false;
-#undef PARSE_AS
-}
-
-bool ParseCommand(const std::string& content, Command* obj) {
-#define PARSE_AS(name, string)      \
-  if (content == #string) {         \
-    *obj = name;                    \
-    return true;                    \
-  }
-
-  PARSE_AS(Command::Callees, "callees");
-  PARSE_AS(Command::Callers, "callers");
-  PARSE_AS(Command::FindAllUsages, "find-all-usages");
-  PARSE_AS(Command::FindInterestingUsages, "find-interesting-usages");
-  PARSE_AS(Command::GotoReferenced, "goto-referenced");
-  PARSE_AS(Command::Hierarchy, "hierarchy");
-  PARSE_AS(Command::Outline, "outline");
-  PARSE_AS(Command::Search, "search");
-
-  return false;
-#undef PARSE_AS
-}
-
 
 std::unordered_map<std::string, std::string> ParseOptions(int argc, char** argv) {
   std::unordered_map<std::string, std::string> output;
@@ -82,118 +51,7 @@ bool HasOption(const std::unordered_map<std::string, std::string>& options, cons
   return options.find(option) != options.end();
 }
 
-/*
 
-// Connects to a running --project-directory instance. Forks
-// and creates it if not running.
-//
-// Implements language server spec.
-indexer.exe --language-server
-
-// Holds the runtime db that the --language-server instance
-// runs queries against.
-indexer.exe --project-directory /work2/chrome/src
-
-// Created from the --project-directory (server) instance
-indexer.exe --index-file /work2/chrome/src/chrome/foo.cc
-
-// Configuration data is read from a JSON file.
-{
-  "max_threads": 40,
-  "cache_directory": "/work/indexer_cache/"
-
-}
-*/
-
-
-struct IpcMessage_IsAlive : public BaseIpcMessage<IpcMessage_IsAlive> {
-  static IpcMessageId id;
-};
-
-IpcMessageId IpcMessage_IsAlive::id = "IsAlive";
-
-
-
-
-
-
-
-
-
-
-
-
-
-struct IpcMessage_DocumentSymbolsRequest : public BaseIpcMessage<IpcMessage_DocumentSymbolsRequest> {
-  std::string document;
-
-  // BaseIpcMessage:
-  static IpcMessageId id;
-  void Serialize(Writer& writer) override {
-    writer.String(document.c_str());
-  }
-  void Deserialize(Reader& reader) override {
-    document = reader.GetString();
-  }
-};
-IpcMessageId IpcMessage_DocumentSymbolsRequest::id = "IpcMessage_DocumentSymbolsRequest";
-
-struct IpcMessage_DocumentSymbolsResponse : public BaseIpcMessage<IpcMessage_DocumentSymbolsResponse> {
-  std::vector<language_server_api::SymbolInformation> symbols;
-
-  // BaseIpcMessage:
-  static IpcMessageId id;
-};
-IpcMessageId IpcMessage_DocumentSymbolsResponse::id = "IpcMessage_DocumentSymbolsResponse";
-
-
-
-
-
-void QueryDbMain() {
-  IpcServer ipc("languageserver");
-
-  while (true) {
-    std::vector<std::unique_ptr<BaseIpcMessageElided>> messages = ipc.TakeMessages();
-
-    for (auto& message : messages) {
-      std::cout << "Processing message " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
-
-      if (message->runtime_id() == IpcMessage_IsAlive::id) {
-        IpcMessage_IsAlive response;
-        ipc.SendToClient(0, &response); // todo: make non-blocking
-        break;
-      }
-      else {
-        std::cerr << "Unhandled IPC message with kind " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
-        exit(1);
-        break;
-      }
-    }
-
-    std::this_thread::sleep_for(std::chrono::milliseconds(20));
-  }
-}
-
-void EmitReferences(IpcClient& ipc) {
-
-}
-
-// Separate thread whose only job is to read from stdin and
-// dispatch read commands to the actual indexer program. This
-// cannot be done on the main thread because reading from std::cin
-// blocks.
-void LanguageServerStdinToServerDispatcher(IpcClient& ipc) {
-  while (true) {
-    std::string input;
-    std::cin >> input;
-
-  }
-}
-
-
-void ParseRpc(const std::string& method, const rapidjson::GenericValue<rapidjson::UTF8<>>& params) {
-}
 
 std::unique_ptr<language_server_api::InMessage> ParseMessage() {
   int content_length = -1;
@@ -216,7 +74,11 @@ std::unique_ptr<language_server_api::InMessage> ParseMessage() {
       break;
   }
 
-  assert(content_length >= 0);
+  // bad input that is not a message.
+  if (content_length < 0) {
+    std::cerr << "parsing command failed (no Content-Length header)" << std::endl;
+    return nullptr;
+  }
 
   std::string content;
   content.reserve(content_length);
@@ -231,41 +93,322 @@ std::unique_ptr<language_server_api::InMessage> ParseMessage() {
   assert(!document.HasParseError());
 
   return language_server_api::MessageRegistry::instance()->Parse(document);
-
-  /*
-  std::string id;
-  if (document["id"].IsString())
-  id = document["id"].GetString();
-  else
-  id = std::to_string(document["id"].GetInt());
-  std::string method = document["method"].GetString();
-  auto& params = document["params"];
-
-
-  // Send initialize response.
-  {
-    std::string content =
-      R"foo({
-      "jsonrpc": "2.0",
-      "id": 0,
-      "result": {
-        "capabilities": {
-          "documentSymbolProvider": true
-        }
-      }
-    })foo";
-    std::cout << "Content-Length: " << content.size();
-    std::cout << (char)13 << char(10) << char(13) << char(10);
-    std::cout << content;
-  }
-  */
 }
 
 
 
-// Main loop for the language server. |ipc| is connected to
+
-// a server.
+
-void LanguageServerLoop(IpcClient* ipc) {
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+struct IpcMessage_Quit : public BaseIpcMessage<IpcMessage_Quit> {
+  static IpcMessageId kId;
+};
+IpcMessageId IpcMessage_Quit::kId = "Quit";
+
+
+struct IpcMessage_IsAlive : public BaseIpcMessage<IpcMessage_IsAlive> {
+  static IpcMessageId kId;
+};
+IpcMessageId IpcMessage_IsAlive::kId = "IsAlive";
+
+
+
+
+
+struct IpcMessage_OpenProject : public BaseIpcMessage<IpcMessage_OpenProject> {
+  static IpcMessageId kId;
+
+  std::string project_path;
+
+  // BaseIpcMessage:
+  void Serialize(Writer& writer) override {
+    writer.String(project_path.c_str(), project_path.size());
+  }
+  void Deserialize(Reader& reader) override {
+    project_path = reader.GetString();
+  }
+};
+IpcMessageId IpcMessage_OpenProject::kId = "OpenProject";
+
+
+
+
+
+
+
+
+struct IpcMessage_DocumentSymbolsRequest : public BaseIpcMessage<IpcMessage_DocumentSymbolsRequest> {
+  language_server_api::RequestId id;
+  std::string document;
+
+  // BaseIpcMessage:
+  static IpcMessageId kId;
+  void Serialize(Writer& writer) override {
+    using namespace language_server_api;
+    auto& value = *this;
+
+    writer.StartObject();
+    SERIALIZE_MEMBER(id);
+    SERIALIZE_MEMBER(document);
+    writer.EndObject();
+  }
+  void Deserialize(Reader& reader) override {
+    using namespace language_server_api;
+    auto& value = *this;
+
+    DESERIALIZE_MEMBER(id);
+    DESERIALIZE_MEMBER(document);
+  }
+};
+IpcMessageId IpcMessage_DocumentSymbolsRequest::kId = "IpcMessage_DocumentSymbolsRequest";
+
+struct IpcMessage_DocumentSymbolsResponse : public BaseIpcMessage<IpcMessage_DocumentSymbolsResponse> {
+  language_server_api::RequestId id;
+  std::vector<language_server_api::SymbolInformation> symbols;
+
+  // BaseIpcMessage:
+  static IpcMessageId kId;
+  void Serialize(Writer& writer) override {
+    using namespace language_server_api;
+    auto& value = *this;
+
+    writer.StartObject();
+    SERIALIZE_MEMBER(id);
+    SERIALIZE_MEMBER(symbols);
+    writer.EndObject();
+  }
+  void Deserialize(Reader& reader) override {
+    using namespace language_server_api;
+    auto& value = *this;
+
+    DESERIALIZE_MEMBER(id);
+    DESERIALIZE_MEMBER(symbols);
+  }
+};
+IpcMessageId IpcMessage_DocumentSymbolsResponse::kId = "IpcMessage_DocumentSymbolsResponse";
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+void QueryDbMainLoop(IpcServer* ipc, QueryableDatabase* db) {
+  using namespace language_server_api;
+
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> messages = ipc->TakeMessages();
+
+  for (auto& message : messages) {
+    std::cerr << "Processing message " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
+
+    if (IpcMessage_Quit::kId == message->runtime_id()) {
+      break;
+    }
+
+    else if (IpcMessage_IsAlive::kId == message->runtime_id()) {
+      IpcMessage_IsAlive response;
+      ipc->SendToClient(0, &response); // todo: make non-blocking
+    }
+
+    else if (IpcMessage_OpenProject::kId == message->runtime_id()) {
+      IpcMessage_OpenProject* msg = static_cast<IpcMessage_OpenProject*>(message.get());
+      std::string path = msg->project_path;
+
+
+      std::vector<CompilationEntry> entries = LoadCompilationEntriesFromDirectory(path);
+      for (int i = 0; i < entries.size(); ++i) {
+        const CompilationEntry& entry = entries[i];
+        std::string filepath = path + "/" + entry.filename;
+        std::cerr << "[" << i << "/" << (entries.size() - 1) << "] Parsing file " << filepath << std::endl;
+        IndexedFile file = Parse(filepath, entry.args);
+        IndexUpdate update(file);
+        db->ApplyIndexUpdate(&update);
+      }
+      std::cerr << "Done" << std::endl;
+    }
+
+
+    else if (IpcMessage_DocumentSymbolsRequest::kId == message->runtime_id()) {
+      auto msg = static_cast<IpcMessage_DocumentSymbolsRequest*>(message.get());
+
+      IpcMessage_DocumentSymbolsResponse response;
+      response.id = msg->id;
+
+      std::cerr << "Wanted file " << msg->document << std::endl;
+      for (auto& file : db->files) {
+        std::cerr << " - Have file " << file.file_id << std::endl;
+
+        // TODO: make sure we normalize ids!
+        // TODO: hashmap lookup.
+        if (file.file_id == msg->document) {
+          std::cerr << "Found file" << std::endl;
+
+
+          for (UsrRef ref : file.outline) {
+            SymbolIdx symbol = db->usr_to_symbol[ref.usr];
+
+            SymbolInformation info;
+            info.location.range.start.line = ref.loc.line - 1; // TODO: cleanup indexer to negate by 1.
+            info.location.range.start.character = ref.loc.column - 1; // TODO: cleanup indexer to negate by 1.
+            // TODO: store range information.
+            info.location.range.end.line = info.location.range.start.line;
+            info.location.range.end.character = info.location.range.start.character;
+
+            // TODO: cleanup namespace/naming so there is only one SymbolKind.
+            switch (symbol.kind) {
+            case ::SymbolKind::Type:
+            {
+              QueryableTypeDef& def = db->types[symbol.idx];
+              info.name = def.def.qualified_name;
+              info.kind = language_server_api::SymbolKind::Class;
+              break;
+            }
+            case ::SymbolKind::Func:
+            {
+              QueryableFuncDef& def = db->funcs[symbol.idx];
+              info.name = def.def.qualified_name;
+              if (def.def.declaring_type.has_value()) {
+                info.kind = language_server_api::SymbolKind::Method;
+                Usr declaring = def.def.declaring_type.value();
+                info.containerName = db->types[db->usr_to_symbol[declaring].idx].def.qualified_name;
+              }
+              else {
+                info.kind = language_server_api::SymbolKind::Function;
+              }
+              break;
+            }
+            case ::SymbolKind::Var:
+            {
+              QueryableVarDef& def = db->vars[symbol.idx];
+              info.name = def.def.qualified_name;
+              info.kind = language_server_api::SymbolKind::Variable;
+              break;
+            }
+            };
+
+            // TODO
+            //info.containerName = "fooey";
+
+            response.symbols.push_back(info);
+
+          }
+          break;
+        }
+      }
+
+
+
+
+      ipc->SendToClient(0, &response);
+    }
+
+
+    else {
+      std::cerr << "Unhandled IPC message with kind " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
+      exit(1);
+    }
+  }
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+// TODO: global lock on stderr output.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+// Separate thread whose only job is to read from stdin and
+// dispatch read commands to the actual indexer program. This
+// cannot be done on the main thread because reading from std::cin
+// blocks.
+//
+// |ipc| is connected to a server.
+void LanguageServerStdinLoop(IpcClient* ipc) {
   using namespace language_server_api;
 
   while (true) {
@@ -279,8 +422,15 @@ void LanguageServerLoop(IpcClient* ipc) {
     switch (message->method_id) {
     case MethodId::Initialize:
     {
-      // TODO: response should take id as input.
+      auto request = static_cast<In_InitializeRequest*>(message.get());
-      // TODO: message should not have top-level id.
+      if (request->params.rootUri) {
+        std::string project_path = request->params.rootUri->GetPath();
+        std::cerr << "Initialize in directory " << project_path << std::endl;
+        IpcMessage_OpenProject open_project;
+        open_project.project_path = project_path;
+        ipc->SendToServer(&open_project);
+      }
+
       auto response = Out_InitializeResponse();
       response.id = message->id.value();
       response.result.capabilities.documentSymbolProvider = true;
@@ -290,157 +440,169 @@ void LanguageServerLoop(IpcClient* ipc) {
 
     case MethodId::TextDocumentDocumentSymbol:
     {
-      auto response = Out_DocumentSymbolResponse();
+      // TODO: response should take id as input.
-      response.id = message->id.value();
+      // TODO: message should not have top-level id.
+      auto request = static_cast<In_DocumentSymbolRequest*>(message.get());
 
-      for (int i = 0; i < 2500; ++i) {
+      IpcMessage_DocumentSymbolsRequest ipc_request;
-        SymbolInformation info;
+      ipc_request.id = request->id.value();
-        info.containerName = "fooContainer";
+      ipc_request.document = request->params.textDocument.uri.GetPath();
-        info.kind = language_server_api::SymbolKind::Field;
+      std::cerr << "Request textDocument=" << ipc_request.document << std::endl;
-        info.location.range.start.line = 5;
+      ipc->SendToServer(&ipc_request);
-        info.location.range.end.character = 20;
-        info.location.range.end.line = 5;
-        info.location.range.end.character = 25;
-        info.name = "Foobar";
-        response.result.push_back(info);
-      }
-
-      response.Send();
       break;
     }
     }
   }
 }
 
+void LanguageServerMainLoop(IpcClient* ipc) {
+  using namespace language_server_api;
+
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> messages = ipc->TakeMessages();
+  for (auto& message : messages) {
+    if (IpcMessage_Quit::kId == message->runtime_id()) {
+      exit(0);
+    }
 
 
+    else if (IpcMessage_DocumentSymbolsResponse::kId == message->runtime_id()) {
+      auto msg = static_cast<IpcMessage_DocumentSymbolsResponse*>(message.get());
 
-void LanguageServerMain() {
+      auto response = Out_DocumentSymbolResponse();
-  IpcClient ipc("languageserver", 0);
+      response.id = msg->id;
+      response.result = msg->symbols;
+      response.Send();
+      std::cerr << "Send symbol response to client (" << response.result.size() << " symbols)" << std::endl;
+    }
+
+    else {
+      std::cerr << "Unhandled IPC message with kind " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
+      exit(1);
+    }
+  }
+}
+
+void LanguageServerMain(std::string process_name) {
+  IpcClient client_ipc("languageserver", 0);
 
   // Discard any left-over messages from previous runs.
-  ipc.TakeMessages();
+  client_ipc.TakeMessages();
 
   // Emit an alive check. Sleep so the server has time to respond.
   IpcMessage_IsAlive check_alive;
-  ipc.SendToServer(&check_alive);
+  client_ipc.SendToServer(&check_alive);
 
   // TODO: Tune this value or make it configurable.
   std::this_thread::sleep_for(std::chrono::milliseconds(20));
 
   // Check if we got an IsAlive message back.
-  std::vector<std::unique_ptr<BaseIpcMessageElided>> messages = ipc.TakeMessages();
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> messages = client_ipc.TakeMessages();
   bool has_server = false;
   for (auto& message : messages) {
-    if (message->runtime_id() == IpcMessage_IsAlive::id) {
+    if (message->runtime_id() == IpcMessage_IsAlive::kId) {
       has_server = true;
       break;
     }
   }
 
   // No server is running. Start it.
-  //if (!has_server) {
+#if false
-  //  std::cerr << "Unable to detect running indexer server" << std::endl;
+  if (!has_server) {
-  //  exit(1);
+    if (process_name.empty())
-  //}
+      return;
 
-  std::thread stdio_reader(&LanguageServerLoop, &ipc);
+    Process p(process_name + " --querydb", "",
+      /*stdout*/[](const char* bytes, size_t n) {
+      for (int i = 0; i < n; ++i)
+        std::cerr << bytes[i];
+    },
+      /*stderr*/[](const char* bytes, size_t n) {
+      for (int i = 0; i < n; ++i)
+        std::cerr << bytes[i];
+},
+/*open_stdin*/false);
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    // Pass empty process name so we only try to start the querydb once.
+    LanguageServerMain("");
+    return;
+}
+#endif
 
-  //std::cout << "Found indexer server" << std::endl;
+  // for debugging attach
-  //LanguageServerLoop(ipc);
-
-  // TODO: This is used for debugging, so we can attach to the client.
-
-
-  //std::cout << "garbagelkadklasldk" << std::endl;
-
-  bool should_break = true;
-  while (should_break)
-    std::this_thread::sleep_for(std::chrono::milliseconds(100));
   //std::this_thread::sleep_for(std::chrono::seconds(4));
 
 
-  //std::cout.flush();
+  std::thread stdio_reader(&LanguageServerStdinLoop, &client_ipc);
-  /*
+
-  language_server_api::ShowMessageOutNotification show;
+
-  show.type = language_server_api::MessageType::Info;
+  // No server. Run it in-process.
-  show.message = "hello";
+  if (!has_server) {
-  show.Send();
+
-  */
+    QueryableDatabase db;
+    IpcServer server_ipc("languageserver");
+
+    while (true) {
+      QueryDbMainLoop(&server_ipc, &db);
+      LanguageServerMainLoop(&client_ipc);
+      // TODO: use a condition variable.
+      std::this_thread::sleep_for(std::chrono::milliseconds(20));
+    }
+  }
+
+  else {
+    while (true) {
+      LanguageServerMainLoop(&client_ipc);
+      // TODO: use a condition variable.
+      std::this_thread::sleep_for(std::chrono::milliseconds(20));
+    }
+  }
 }
 
 
 
 
 
-#if false
-
-struct IpcMessage_IsAlive : public BaseIpcMessage {
-  IpcMessage_IsAlive();
-
-  // BaseIpcMessage:
-  void Serialize(Writer& writer) override;
-  void Deserialize(Reader& reader) override;
-};
-
-struct IpcMessage_ImportIndex : public BaseIpcMessage {
-  std::string path;
-
-  IpcMessage_ImportIndex();
-
-  // BaseMessage:
-  void Serialize(Writer& writer) override;
-  void Deserialize(Reader& reader) override;
-};
-
-struct IpcMessage_CreateIndex : public BaseIpcMessage {
-  std::string path;
-  std::vector<std::string> args;
-
-  IpcMessage_CreateIndex();
-
-  // BaseMessage:
-  void Serialize(Writer& writer) override;
-  void Deserialize(Reader& reader) override;
-};
 
 
-IpcMessage_IsAlive::IpcMessage_IsAlive() {
-  kind = JsonMessage::Kind::IsAlive;
-}
 
-void IpcMessage_IsAlive::Serialize(Writer& writer) {}
 
-void IpcMessage_IsAlive::Deserialize(Reader& reader) {}
 
-IpcMessage_ImportIndex::IpcMessage_ImportIndex() {
-  kind = JsonMessage::Kind::ImportIndex;
-}
 
-void IpcMessage_ImportIndex::Serialize(Writer& writer) {
-  writer.StartObject();
-  ::Serialize(writer, "path", path);
-  writer.EndObject();
-}
-void IpcMessage_ImportIndex::Deserialize(Reader& reader) {
-  ::Deserialize(reader, "path", path);
-}
 
-IpcMessage_CreateIndex::IpcMessage_CreateIndex() {
-  kind = JsonMessage::Kind::CreateIndex;
-}
 
-void IpcMessage_CreateIndex::Serialize(Writer& writer) {
+
-  writer.StartObject();
+
-  ::Serialize(writer, "path", path);
+
-  ::Serialize(writer, "args", args);
+
-  writer.EndObject();
+
-}
+
-void IpcMessage_CreateIndex::Deserialize(Reader& reader) {
+
-  ::Deserialize(reader, "path", path);
+
-  ::Deserialize(reader, "args", args);
+
-}
+
-#endif
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 int main(int argc, char** argv) {
   // We need to write to stdout in binary mode because in Windows, writing
@@ -451,9 +613,13 @@ int main(int argc, char** argv) {
   _setmode(_fileno(stdin), O_BINARY);
 #endif
 
-  std::cerr << "Starting language server" << std::endl;
+
+
+  IpcRegistry::instance()->Register<IpcMessage_Quit>();
 
   IpcRegistry::instance()->Register<IpcMessage_IsAlive>();
+  IpcRegistry::instance()->Register<IpcMessage_OpenProject>();
+
   IpcRegistry::instance()->Register<IpcMessage_DocumentSymbolsRequest>();
   IpcRegistry::instance()->Register<IpcMessage_DocumentSymbolsResponse>();
 
@@ -462,128 +628,34 @@ int main(int argc, char** argv) {
   language_server_api::MessageRegistry::instance()->Register<language_server_api::In_InitializedNotification>();
   language_server_api::MessageRegistry::instance()->Register<language_server_api::In_DocumentSymbolRequest>();
 
+
+
+
+
+
   std::unordered_map<std::string, std::string> options = ParseOptions(argc, argv);
 
   if (HasOption(options, "--language-server")) {
-    LanguageServerMain();
+    std::cerr << "Running language server" << std::endl;
+    LanguageServerMain(argv[0]);
     return 0;
   }
-  if (HasOption(options, "--querydb")) {
+  else if (HasOption(options, "--querydb")) {
-    QueryDbMain();
+    std::cerr << "Running querydb" << std::endl;
-    return 0;
+    QueryableDatabase db;
-  }
+    IpcServer ipc("languageserver");
-
+    while (true) {
-
+      QueryDbMainLoop(&ipc, &db);
-  LanguageServerMain();
+      // TODO: use a condition variable.
-  return 0;
+      std::this_thread::sleep_for(std::chrono::milliseconds(20));
-
-  if (argc == 1 || options.find("--help") != options.end()) {
-    std::cout << R"help(clang-indexer help:
-
-  General:
-    --help        Print this help information.
-    --help-commands
-                  Print all available query commands.
-    --project     Path to compile_commands.json. Needed for the server, and
-                  optionally by clients if there are multiple servers running.
-    --print-config
-                  Emit all configuration data this executable is using.
-
-
-  Server:
-    --server      If present, this binary will run in server mode. The binary
-                  will not return until killed or an exit is requested. The
-                  server computes and caches an index of the entire program
-                  which is then queried by short-lived client processes. A
-                  client is created by running this binary with a --command
-                  flag.
-    --cache-dir   Directory to cache the index and other useful information. If
-                  a previous cache is present, the database will try to reuse
-                  it. If this flag is not present, the database will be
-                  in-memory only.
-    --threads     Number of threads to use for indexing and querying tasks.
-                  This value is optional; a good estimate is computed by
-                  default.
-
-
-  Client:
-    --command     Execute a query command against the index. See
-                  --command-help for a listing of valid commands and a
-                  description of what they do. Presence of this flag indicates
-                  that the indexer is in client mode; this flag is mutually
-                  exclusive with --server.
-    --location    Location of the query. Some commands require only a file,
-                  other require a line and column as well. Format is
-                  filename[:line:column]. For example, "foobar.cc" and
-                  "foobar.cc:1:10" are valid inputs.
-    --preferred-symbol-location
-                  When looking up symbols, try to return either the
-                  'declaration' or the 'definition'. Defaults to 'definition'.
-)help";
-    exit(0);
-  }
-
-  if (HasOption(options, "--help-commands")) {
-    std::cout << R"(Available commands:
-
-  callees:
-  callers:
-    Emit all functions (with location) that this function calls ("callees") or
-    that call this function ("callers"). Requires a location.
-
-  find-all-usages:
-    Emit every usage of the given symbol. This is intended to support a rename
-    refactoring. This output contains many uninteresting usages of symbols;
-    prefer find-interesting-usges. Requires a location.
-
-  find-interesting-usages:
-    Emit only usages of the given symbol which are semantically interesting.
-    Requires a location.
-
-  goto-referenced:
-    Find an associated reference (either definition or declaration) for the
-    given symbol. Requires a location.
-
-  hierarchy:
-    List the type hierarchy (ie, inherited and derived members) for the given
-    method or type. Requires a location.
-
-  outline:
-    Emit a file outline, listing all of the symbols in the file.
-
-  search:
-    Search for a symbol by name.
-)";
-    exit(0);
-  }
-
-  if (HasOption(options, "--project")) {
-    std::vector<CompilationEntry> entries = LoadCompilationEntriesFromDirectory(options["--project"]);
-
-
-    for (const CompilationEntry& entry : entries) {
-      std::cout << "Parsing " << entry.filename << std::endl;
-      QueryableDatabase db;
-      IndexedFile file = Parse(entry.filename, entry.args);
-
-      IndexUpdate update(file);
-      db.ApplyIndexUpdate(&update);
-      //std::cout << db.ToString() << std::endl << std::endl;
     }
-
+    return 0;
-    std::cin.get();
+  }
-    exit(0);
+  else {
+    std::cerr << "Running language server" << std::endl;
+    LanguageServerMain(argv[0]);
+    return 0;
   }
 
-  if (HasOption(options, "--command")) {
+  return 1;
-    Command command;
-    if (!ParseCommand(options["--command"], &command))
-      Fail("Unknown command \"" + options["--command"] + "\"; see --help-commands");
-
-
-  }
-
-  std::cout << "Invalid arguments. Try --help.";
-  exit(1);
-  return 0;
 }

compilation_database_loader.cc

@@ -5,12 +5,42 @@
 
 #include "libclangmm/Utility.h"
 
+#include "utils.h"
+
+// See http://stackoverflow.com/a/2072890
+bool EndsWith(const std::string& value, const std::string& ending) {
+  if (ending.size() > value.size())
+    return false;
+  return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
+}
+
+std::vector<CompilationEntry> LoadFromDirectoryListing(const std::string& project_directory) {
+  std::vector<CompilationEntry> result;
+
+  std::vector<std::string> files = GetFilesInFolder(project_directory, false /*add_folder_to_path*/);
+
+  for (const std::string& file : files) {
+    if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") ||
+      EndsWith(file, ".c") || EndsWith(file, ".h") ||
+      EndsWith(file, ".hpp")) {
+
+      CompilationEntry entry;
+      entry.directory = ".";
+      entry.filename = file;
+      entry.args = {};
+      result.push_back(entry);
+    }
+  }
+
+  return result;
+}
+
 std::vector<CompilationEntry> LoadCompilationEntriesFromDirectory(const std::string& project_directory) {
   CXCompilationDatabase_Error cx_db_load_error;
   CXCompilationDatabase cx_db = clang_CompilationDatabase_fromDirectory(project_directory.c_str(), &cx_db_load_error);
   if (cx_db_load_error == CXCompilationDatabase_CanNotLoadDatabase) {
-    std::cerr << "[FATAL]: Unable to load compile_commands.json located at \"" << project_directory << "\"";
+    std::cerr << "Unable to load compile_commands.json located at \"" << project_directory << "\"; using directory listing instead." << std::endl;
-    exit(1);
+    return LoadFromDirectoryListing(project_directory);
   }
 
   CXCompileCommands cx_commands = clang_CompilationDatabase_getAllCompileCommands(cx_db);

indexer.cpp

@@ -87,7 +87,7 @@ std::string IndexedFile::ToString() {
 
 IndexedTypeDef::IndexedTypeDef(TypeId id, const std::string& usr) : id(id), def(usr) {
   assert(usr.size() > 0);
-  //std::cout << "Creating type with usr " << usr << std::endl;
+  //std::cerr << "Creating type with usr " << usr << std::endl;
 }
 
 void IndexedTypeDef::AddUsage(Location loc, bool insert_if_not_present) {
@@ -196,8 +196,8 @@ CXIdxClientContainer startedTranslationUnit(CXClientData client_data, void *rese
 
 clang::VisiterResult DumpVisitor(clang::Cursor cursor, clang::Cursor parent, int* level) {
   for (int i = 0; i < *level; ++i)
-    std::cout << "  ";
+    std::cerr << "  ";
-  std::cout << clang::ToString(cursor.get_kind()) << " " << cursor.get_spelling() << std::endl;
+  std::cerr << clang::ToString(cursor.get_kind()) << " " << cursor.get_spelling() << std::endl;
 
   *level += 1;
   cursor.VisitChildren(&DumpVisitor, level);
@@ -432,6 +432,8 @@ optional<TypeId> ResolveDeclToType(IndexedFile* db, clang::Cursor decl_cursor,
   //  The second TypeRef is an uninteresting usage.
   bool process_last_type_ref = true;
   if (IsTypeDefinition(semantic_container) && !IsTypeDefinition(lexical_container)) {
+    //if (!decl_cursor.is_definition())
+    //  decl_cursor = decl_cursor.get_definition();
     assert(decl_cursor.is_definition());
     process_last_type_ref = false;
   }
@@ -713,15 +715,15 @@ void indexDeclaration(CXClientData client_data, const CXIdxDeclInfo* decl) {
   }
 
   default:
-    std::cout << "!! Unhandled indexDeclaration:     " << clang::Cursor(decl->cursor).ToString() << " at " << db->id_cache.Resolve(decl->loc, false /*interesting*/).ToString() << std::endl;
+    std::cerr << "!! Unhandled indexDeclaration:     " << clang::Cursor(decl->cursor).ToString() << " at " << db->id_cache.Resolve(decl->loc, false /*interesting*/).ToString() << std::endl;
-    std::cout << "     entityInfo->kind  = " << decl->entityInfo->kind << std::endl;
+    std::cerr << "     entityInfo->kind  = " << decl->entityInfo->kind << std::endl;
-    std::cout << "     entityInfo->USR   = " << decl->entityInfo->USR << std::endl;
+    std::cerr << "     entityInfo->USR   = " << decl->entityInfo->USR << std::endl;
     if (decl->declAsContainer)
-      std::cout << "     declAsContainer   = " << clang::Cursor(decl->declAsContainer->cursor).ToString() << std::endl;
+      std::cerr << "     declAsContainer   = " << clang::Cursor(decl->declAsContainer->cursor).ToString() << std::endl;
     if (decl->semanticContainer)
-      std::cout << "     semanticContainer = " << clang::Cursor(decl->semanticContainer->cursor).ToString() << std::endl;
+      std::cerr << "     semanticContainer = " << clang::Cursor(decl->semanticContainer->cursor).ToString() << std::endl;
     if (decl->lexicalContainer)
-      std::cout << "     lexicalContainer  = " << clang::Cursor(decl->lexicalContainer->cursor).get_usr() << std::endl;
+      std::cerr << "     lexicalContainer  = " << clang::Cursor(decl->lexicalContainer->cursor).get_usr() << std::endl;
     break;
   }
 }
@@ -849,18 +851,18 @@ void indexEntityReference(CXClientData client_data, const CXIdxEntityRefInfo* re
   }
 
   default:
-    std::cout << "!! Unhandled indexEntityReference: " << cursor.ToString() << " at " << db->id_cache.Resolve(ref->loc, false /*interesting*/).ToString() << std::endl;
+    std::cerr << "!! Unhandled indexEntityReference: " << cursor.ToString() << " at " << db->id_cache.Resolve(ref->loc, false /*interesting*/).ToString() << std::endl;
-    std::cout << "     ref->referencedEntity->kind = " << ref->referencedEntity->kind << std::endl;
+    std::cerr << "     ref->referencedEntity->kind = " << ref->referencedEntity->kind << std::endl;
     if (ref->parentEntity)
-      std::cout << "     ref->parentEntity->kind = " << ref->parentEntity->kind << std::endl;
+      std::cerr << "     ref->parentEntity->kind = " << ref->parentEntity->kind << std::endl;
-    std::cout << "     ref->loc          = " << db->id_cache.Resolve(ref->loc, false /*interesting*/).ToString() << std::endl;
+    std::cerr << "     ref->loc          = " << db->id_cache.Resolve(ref->loc, false /*interesting*/).ToString() << std::endl;
-    std::cout << "     ref->kind         = " << ref->kind << std::endl;
+    std::cerr << "     ref->kind         = " << ref->kind << std::endl;
     if (ref->parentEntity)
-      std::cout << "     parentEntity      = " << clang::Cursor(ref->parentEntity->cursor).ToString() << std::endl;
+      std::cerr << "     parentEntity      = " << clang::Cursor(ref->parentEntity->cursor).ToString() << std::endl;
     if (ref->referencedEntity)
-      std::cout << "     referencedEntity  = " << clang::Cursor(ref->referencedEntity->cursor).ToString() << std::endl;
+      std::cerr << "     referencedEntity  = " << clang::Cursor(ref->referencedEntity->cursor).ToString() << std::endl;
     if (ref->container)
-      std::cout << "     container         = " << clang::Cursor(ref->container->cursor).ToString() << std::endl;
+      std::cerr << "     container         = " << clang::Cursor(ref->container->cursor).ToString() << std::endl;
     break;
   }
 }

ipc.cc

@@ -52,7 +52,7 @@ void IpcDirectionalChannel::PushMessage(BaseIpcMessageElided* message) {
   writer.SetIndent(' ', 2);
   message->Serialize(writer);
 
-  //std::cout << "Sending message with id " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
+  //std::cerr << "Sending message with id " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
 
   size_t payload_size = strlen(output.GetString());
   assert(payload_size < shmem_size && "Increase shared memory size, payload will never fit");
@@ -62,7 +62,7 @@ void IpcDirectionalChannel::PushMessage(BaseIpcMessageElided* message) {
   while (true) {
     if (!first) {
       if (!did_log) {
-        std::cout << "[info]: shmem full, waiting" << std::endl; // TODO: remove
+        std::cerr << "[info]: shmem full, waiting" << std::endl; // TODO: remove
         did_log = true;
       }
       std::this_thread::sleep_for(std::chrono::milliseconds(16));

ipc.h

@@ -15,6 +15,7 @@
 using Writer = rapidjson::PrettyWriter<rapidjson::StringBuffer>;
 using Reader = rapidjson::Document;
 
+// TODO: We need to add support for payloads larger than the maximum shared memory buffer size.
 
 // Messages are funky objects. They contain potentially variable amounts of
 // data and are passed between processes. This means that they need to be
@@ -42,12 +43,12 @@ struct BaseIpcMessageElided {
 // Usage:
 //
 //  class IpcMessage_Foo : public BaseIpcMessage<IpcMessage_Foo> {
-//    static IpcMessageId id;
+//    static IpcMessageId kId;
 //
 //    // BaseIpcMessage:
 //    ...
 //  }
-//  IpcMessageId IpcMessage_Foo::id = "Foo";
+//  IpcMessageId IpcMessage_Foo::kId = "Foo";
 //
 //
 //  main() {
@@ -102,7 +103,7 @@ void IpcRegistry::Register() {
     hash_to_id = MakeUnique<std::unordered_map<int, std::string>>();
   }
 
-  IpcMessageId id = T::id;
+  IpcMessageId id = T::kId;
 
   int hash = std::hash<IpcMessageId>()(id);
   auto it = allocators->find(hash);
@@ -163,6 +164,8 @@ struct IpcClient {
   void SendToServer(BaseIpcMessageElided* message);
   std::vector<std::unique_ptr<BaseIpcMessageElided>> TakeMessages();
 
+  IpcDirectionalChannel* client() { return &client_; }
+
 private:
   IpcDirectionalChannel server_;
   IpcDirectionalChannel client_;

language_server_api.h

@@ -512,15 +512,33 @@ namespace language_server_api {
   // Keep all types in the language_server_api namespace in sync with language server spec.
   // TODO
   struct DocumentUri {
-    std::string path;
+    std::string raw_uri;
+
+    std::string GetPath() {
+      // TODO: make this not a hack.
+      std::string result = raw_uri;
+
+      size_t index = result.find("%3A");
+      if (index != -1) {
+        result.replace(result.begin() + index, result.begin() + index + 3, ":");
+      }
+
+      index = result.find("file://");
+      if (index != -1) {
+        result.replace(result.begin() + index, result.begin() + index + 8, "");
+      }
+
+      std::replace(result.begin(), result.end(), '\\', '/');
+      return result;
+    }
   };
 
   void Serialize(Writer& writer, const DocumentUri& value) {
-    Serialize(writer, value.path);
+    Serialize(writer, value.raw_uri);
   }
 
   void Deserialize(const Reader& reader, DocumentUri& value) {
-    Deserialize(reader, value.path);
+    Deserialize(reader, value.raw_uri);
   }
 
 
@@ -1344,7 +1362,7 @@ namespace language_server_api {
 
   struct In_DocumentSymbolRequest : public InRequestMessage {
     const static MethodId kMethod = MethodId::TextDocumentDocumentSymbol;
-    
+
     DocumentSymbolParams params;
 
     In_DocumentSymbolRequest(optional<RequestId> id, const Reader& reader)
@@ -1449,9 +1467,9 @@ namespace language_server_api {
   };
 
 
-#undef SERIALIZE_MEMBER
+//#undef SERIALIZE_MEMBER
-#undef SERIALIZE_MEMBER2
+//#undef SERIALIZE_MEMBER2
-#undef DESERIALIZE_MEMBER
+//#undef DESERIALIZE_MEMBER
 
 
 

old.cc

@@ -0,0 +1,175 @@
+#if false
+
+
+/*
+
+// Connects to a running --project-directory instance. Forks
+// and creates it if not running.
+//
+// Implements language server spec.
+indexer.exe --language-server
+
+// Holds the runtime db that the --language-server instance
+// runs queries against.
+indexer.exe --project-directory /work2/chrome/src
+
+// Created from the --project-directory (server) instance
+indexer.exe --index-file /work2/chrome/src/chrome/foo.cc
+
+// Configuration data is read from a JSON file.
+{
+"max_threads": 40,
+"cache_directory": "/work/indexer_cache/"
+
+}
+*/
+
+
+bool ParsePreferredSymbolLocation(const std::string& content, PreferredSymbolLocation* obj) {
+#define PARSE_AS(name, string)      \
+  if (content == #string) {         \
+    *obj = name;                    \
+    return true;                    \
+  }
+
+  PARSE_AS(PreferredSymbolLocation::Declaration, "declaration");
+  PARSE_AS(PreferredSymbolLocation::Definition, "definition");
+
+  return false;
+#undef PARSE_AS
+}
+
+bool ParseCommand(const std::string& content, Command* obj) {
+#define PARSE_AS(name, string)      \
+  if (content == #string) {         \
+    *obj = name;                    \
+    return true;                    \
+  }
+
+  PARSE_AS(Command::Callees, "callees");
+  PARSE_AS(Command::Callers, "callers");
+  PARSE_AS(Command::FindAllUsages, "find-all-usages");
+  PARSE_AS(Command::FindInterestingUsages, "find-interesting-usages");
+  PARSE_AS(Command::GotoReferenced, "goto-referenced");
+  PARSE_AS(Command::Hierarchy, "hierarchy");
+  PARSE_AS(Command::Outline, "outline");
+  PARSE_AS(Command::Search, "search");
+
+  return false;
+#undef PARSE_AS
+}
+
+
+
+int main(int argc, char** argv) {
+  if (argc == 1 || options.find("--help") != options.end()) {
+    std::cout << R"help(clang-indexer help:
+
+  General:
+    --help        Print this help information.
+    --help-commands
+                  Print all available query commands.
+    --project     Path to compile_commands.json. Needed for the server, and
+                  optionally by clients if there are multiple servers running.
+    --print-config
+                  Emit all configuration data this executable is using.
+
+
+  Server:
+    --server      If present, this binary will run in server mode. The binary
+                  will not return until killed or an exit is requested. The
+                  server computes and caches an index of the entire program
+                  which is then queried by short-lived client processes. A
+                  client is created by running this binary with a --command
+                  flag.
+    --cache-dir   Directory to cache the index and other useful information. If
+                  a previous cache is present, the database will try to reuse
+                  it. If this flag is not present, the database will be
+                  in-memory only.
+    --threads     Number of threads to use for indexing and querying tasks.
+                  This value is optional; a good estimate is computed by
+                  default.
+
+
+  Client:
+    --command     Execute a query command against the index. See
+                  --command-help for a listing of valid commands and a
+                  description of what they do. Presence of this flag indicates
+                  that the indexer is in client mode; this flag is mutually
+                  exclusive with --server.
+    --location    Location of the query. Some commands require only a file,
+                  other require a line and column as well. Format is
+                  filename[:line:column]. For example, "foobar.cc" and
+                  "foobar.cc:1:10" are valid inputs.
+    --preferred-symbol-location
+                  When looking up symbols, try to return either the
+                  'declaration' or the 'definition'. Defaults to 'definition'.
+)help";
+    exit(0);
+  }
+
+  if (HasOption(options, "--help-commands")) {
+    std::cout << R"(Available commands:
+
+  callees:
+  callers:
+    Emit all functions (with location) that this function calls ("callees") or
+    that call this function ("callers"). Requires a location.
+
+  find-all-usages:
+    Emit every usage of the given symbol. This is intended to support a rename
+    refactoring. This output contains many uninteresting usages of symbols;
+    prefer find-interesting-usges. Requires a location.
+
+  find-interesting-usages:
+    Emit only usages of the given symbol which are semantically interesting.
+    Requires a location.
+
+  goto-referenced:
+    Find an associated reference (either definition or declaration) for the
+    given symbol. Requires a location.
+
+  hierarchy:
+    List the type hierarchy (ie, inherited and derived members) for the given
+    method or type. Requires a location.
+
+  outline:
+    Emit a file outline, listing all of the symbols in the file.
+
+  search:
+    Search for a symbol by name.
+)";
+    exit(0);
+  }
+
+  if (HasOption(options, "--project")) {
+    std::vector<CompilationEntry> entries = LoadCompilationEntriesFromDirectory(options["--project"]);
+
+
+    for (const CompilationEntry& entry : entries) {
+      std::cout << "Parsing " << entry.filename << std::endl;
+      QueryableDatabase db;
+      IndexedFile file = Parse(entry.filename, entry.args);
+
+      IndexUpdate update(file);
+      db.ApplyIndexUpdate(&update);
+      //std::cout << db.ToString() << std::endl << std::endl;
+    }
+
+    std::cin.get();
+    exit(0);
+  }
+
+  if (HasOption(options, "--command")) {
+    Command command;
+    if (!ParseCommand(options["--command"], &command))
+      Fail("Unknown command \"" + options["--command"] + "\"; see --help-commands");
+
+
+  }
+
+  std::cout << "Invalid arguments. Try --help.";
+  exit(1);
+  return 0;
+}
+#endif

platform.h

@@ -16,8 +16,10 @@ struct PlatformSharedMemory {
   char* shared_start;
 };
 
-const int shmem_size = 1024;  // number of chars/bytes (256kb)
+const int shmem_size = 1024 * 256;  // number of chars/bytes (256kb)
 
 std::unique_ptr<PlatformMutex> CreatePlatformMutex(const std::string& name);
 std::unique_ptr<PlatformScopedMutexLock> CreatePlatformScopedMutexLock(PlatformMutex* mutex);
 std::unique_ptr<PlatformSharedMemory> CreatePlatformSharedMemory(const std::string& name);
+
+std::string GetWorkingDirectory();

platform_win.cc

@@ -71,3 +71,22 @@ std::unique_ptr<PlatformScopedMutexLock> CreatePlatformScopedMutexLock(PlatformM
 std::unique_ptr<PlatformSharedMemory> CreatePlatformSharedMemory(const std::string& name) {
   return MakeUnique<PlatformSharedMemoryWin>(name);
 }
+
+// See http://stackoverflow.com/a/19535628
+std::string GetWorkingDirectory() {
+  char result[MAX_PATH];
+  return std::string(result, GetModuleFileName(NULL, result, MAX_PATH));
+}
+
+/*
+// linux
+#include <string>
+#include <limits.h>
+#include <unistd.h>
+
+std::string getexepath() {
+  char result[ PATH_MAX ];
+  ssize_t count = readlink( "/proc/self/exe", result, PATH_MAX );
+  return std::string( result, (count > 0) ? count : 0 );
+}
+*/

query.cc

@@ -74,9 +74,11 @@ std::vector<QueryableLocation> MapIdToUsr(const IdCache& id_cache, const std::ve
 }
 QueryableTypeDef::DefUpdate MapIdToUsr(const IdCache& id_cache, const TypeDefDefinitionData<>& def) {
   QueryableTypeDef::DefUpdate result(def.usr);
-  if (result.definition)
+  result.short_name = def.short_name;
+  result.qualified_name = def.qualified_name;
+  if (def.definition)
     result.definition = MapIdToUsr(id_cache, def.definition.value());
-  if (result.alias_of)
+  if (def.alias_of)
     result.alias_of = MapIdToUsr(id_cache, def.alias_of.value());
   result.parents = MapIdToUsr(id_cache, def.parents);
   result.types = MapIdToUsr(id_cache, def.types);
@@ -86,11 +88,13 @@ QueryableTypeDef::DefUpdate MapIdToUsr(const IdCache& id_cache, const TypeDefDef
 }
 QueryableFuncDef::DefUpdate MapIdToUsr(const IdCache& id_cache, const FuncDefDefinitionData<>& def) {
   QueryableFuncDef::DefUpdate result(def.usr);
-  if (result.definition)
+  result.short_name = def.short_name;
+  result.qualified_name = def.qualified_name;
+  if (def.definition)
     result.definition = MapIdToUsr(id_cache, def.definition.value());
-  if (result.declaring_type)
+  if (def.declaring_type)
     result.declaring_type = MapIdToUsr(id_cache, def.declaring_type.value());
-  if (result.base)
+  if (def.base)
     result.base = MapIdToUsr(id_cache, def.base.value());
   result.locals = MapIdToUsr(id_cache, def.locals);
   result.callees = MapIdToUsr(id_cache, def.callees);
@@ -98,13 +102,15 @@ QueryableFuncDef::DefUpdate MapIdToUsr(const IdCache& id_cache, const FuncDefDef
 }
 QueryableVarDef::DefUpdate MapIdToUsr(const IdCache& id_cache, const VarDefDefinitionData<>& def) {
   QueryableVarDef::DefUpdate result(def.usr);
-  if (result.declaration)
+  result.short_name = def.short_name;
+  result.qualified_name = def.qualified_name;
+  if (def.declaration)
     result.declaration = MapIdToUsr(id_cache, def.declaration.value());
-  if (result.definition)
+  if (def.definition)
     result.definition = MapIdToUsr(id_cache, def.definition.value());
-  if (result.variable_type)
+  if (def.variable_type)
     result.variable_type = MapIdToUsr(id_cache, def.variable_type.value());
-  if (result.declaring_type)
+  if (def.declaring_type)
     result.declaring_type = MapIdToUsr(id_cache, def.declaring_type.value());
   return result;
 }
@@ -112,8 +118,10 @@ QueryableVarDef::DefUpdate MapIdToUsr(const IdCache& id_cache, const VarDefDefin
 QueryableFile::QueryableFile(const IndexedFile& indexed)
   : file_id(indexed.path) {
 
-  auto add_outline = [this, &indexed](Usr usr, Location location) {
+  FileId local_file_id = indexed.id_cache.file_path_to_file_id.find(indexed.path)->second;
-    outline.push_back(UsrRef(usr, MapIdToUsr(indexed.id_cache, location)));
+  auto add_outline = [this, &indexed, local_file_id](Usr usr, Location location) {
+    if (location.file_id() == local_file_id)
+      outline.push_back(UsrRef(usr, MapIdToUsr(indexed.id_cache, location)));
   };
 
   for (const IndexedTypeDef& def : indexed.types) {
@@ -357,7 +365,7 @@ IndexUpdate::IndexUpdate(IndexedFile& previous_file, IndexedFile& current_file)
                       previous, current, \
                       &removed, &added); \
     if (did_add) {\
-      std::cout << "Adding mergeable update on " << current_def->def.short_name << " (" << current_def->def.usr << ") for field " << #index_name << std::endl; \
+      std::cerr << "Adding mergeable update on " << current_def->def.short_name << " (" << current_def->def.usr << ") for field " << #index_name << std::endl; \
       query_name.push_back(MergeableUpdate<type>(current_def->def.usr, removed, added)); \
     } \
   }
@@ -376,7 +384,7 @@ IndexUpdate::IndexUpdate(IndexedFile& previous_file, IndexedFile& current_file)
       current_queryable_file.outline,
       &removed, &added);
     if (did_add) {
-      std::cout << "Adding mergeable update on outline (" << current_file.path << ")" << std::endl;
+      std::cerr << "Adding mergeable update on outline (" << current_file.path << ")" << std::endl;
       files_outline.push_back(MergeableUpdate<UsrRef>(current_file.path, removed, added));
     }
   } while (false); // do while false instead of just {} to appease Visual Studio code formatter.
@@ -617,11 +625,11 @@ void QueryableDatabase::ApplyIndexUpdate(IndexUpdate* update) {
 
 int main233(int argc, char** argv) {
   IndexedFile indexed_file_a = Parse("full_tests/index_delta/a_v0.cc", {});
-  std::cout << indexed_file_a.ToString() << std::endl;
+  std::cerr << indexed_file_a.ToString() << std::endl;
 
-  std::cout << std::endl;
+  std::cerr << std::endl;
   IndexedFile indexed_file_b = Parse("full_tests/index_delta/a_v1.cc", {});
-  std::cout << indexed_file_b.ToString() << std::endl;
+  std::cerr << indexed_file_b.ToString() << std::endl;
   // TODO: We don't need to do ID remapping when computting a diff. Well, we need to do it for the IndexUpdate.
   IndexUpdate import(indexed_file_a);
   /*

task.cc

@@ -183,11 +183,11 @@ static void ThreadMain(int id, Config* config, TaskManager* tm) {
       assert(false);
       break;
     case Task::Kind::Exit:
-      std::cout << id << ": Exiting" << std::endl;
+      std::cerr << id << ": Exiting" << std::endl;
       return;
     }
 
-    std::cout << id << ": waking" << std::endl;
+    std::cerr << id << ": waking" << std::endl;
   }
 
 }

test.cc

@@ -1,6 +1,7 @@
 #include "indexer.h"
 #include "serializer.h"
 #include "utils.h"
+#include "platform.h"
 
 void Write(const std::vector<std::string>& strs) {
   for (const std::string& str : strs)
@@ -81,7 +82,7 @@ void VerifySerializeToFrom(IndexedFile* file) {
   }
 }
 
-int main333(int argc, char** argv) {
+int main222(int argc, char** argv) {
   // TODO: Assert that we need to be on clang >= 3.9.1
 
   /*
@@ -92,8 +93,9 @@ int main333(int argc, char** argv) {
   return 0;
   */
 
-  for (std::string path : GetFilesInFolder("tests")) {
+  for (std::string path : GetFilesInFolder("tests", true /*add_folder_to_path*/)) {
     //if (path != "tests/usage/type_usage_declare_field.cc") continue;
+    path = "C:/Users/jacob/Desktop/superindex/indexer/" + path;
 
     // Parse expected output from the test, parse it into JSON document.
     std::string expected_output;

utils.cc

@@ -5,7 +5,7 @@
 
 #include "tinydir.h"
 
-std::vector<std::string> GetFilesInFolder(std::string folder) {
+std::vector<std::string> GetFilesInFolder(std::string folder, bool add_folder_to_path) {
   std::vector<std::string> result;
 
   tinydir_dir dir;
@@ -21,10 +21,16 @@ std::vector<std::string> GetFilesInFolder(std::string folder) {
       goto bail;
     }
 
-    std::string full_path = folder + "/" + file.name;
+    std::string full_path;
+    if (add_folder_to_path)
+      full_path = folder + "/";
+    full_path += file.name;
     if (file.is_dir) {
-      if (strcmp(file.name, ".") != 0 && strcmp(file.name, "..") != 0) {
+      // Ignore all dot directories.
-        for (std::string nested_file : GetFilesInFolder(full_path))
+      // Note that we must always ignore the '.' and '..' directories, otherwise
+      // this will loop infinitely.
+      if (file.name[0] != '.') {
+        for (std::string nested_file : GetFilesInFolder(full_path, true /*add_folder_to_path*/))
           result.push_back(nested_file);
       }
     }

utils.h

@@ -4,7 +4,8 @@
 #include <vector>
 #include <memory>
 
-std::vector<std::string> GetFilesInFolder(std::string folder);
+// Finds all files in the given folder. This is recursive.
+std::vector<std::string> GetFilesInFolder(std::string folder, bool add_folder_to_path);
 std::vector<std::string> ReadLines(std::string filename);
 void ParseTestExpectation(std::string filename, std::string* expected_output);