wip

command_line.cc

@@ -1,11 +1,21 @@
 #include <iostream>
 #include <string>
 #include <unordered_map>
+#include <thread>
 
 #include "compilation_database_loader.h"
 #include "indexer.h"
 #include "ipc.h"
 #include "query.h"
+#include "language_server_api.h"
+
+#include <rapidjson/istreamwrapper.h>
+#include <rapidjson/ostreamwrapper.h>
+
+#ifdef _WIN32
+#include <io.h>
+#include <fcntl.h>
+#endif
 
 bool ParsePreferredSymbolLocation(const std::string& content, PreferredSymbolLocation* obj) {
 #define PARSE_AS(name, string)      \
@@ -96,7 +106,7 @@ indexer.exe --index-file /work2/chrome/src/chrome/foo.cc
 */
 
 
-struct IpcMessage_IsAlive : public BaseIpcMessage {
+struct IpcMessage_IsAlive : public BaseIpcMessage<IpcMessage_IsAlive> {
   static IpcMessageId id;
 };
 
@@ -109,7 +119,12 @@ IpcMessageId IpcMessage_IsAlive::id = "IsAlive";
 
 
 
-struct IpcMessage_DocumentSymbolsRequest : public BaseIpcMessage {
+
+
+
+
+
+struct IpcMessage_DocumentSymbolsRequest : public BaseIpcMessage<IpcMessage_DocumentSymbolsRequest> {
   std::string document;
 
   // BaseIpcMessage:
@@ -123,58 +138,8 @@ struct IpcMessage_DocumentSymbolsRequest : public BaseIpcMessage {
 };
 IpcMessageId IpcMessage_DocumentSymbolsRequest::id = "IpcMessage_DocumentSymbolsRequest";
 
-
+struct IpcMessage_DocumentSymbolsResponse : public BaseIpcMessage<IpcMessage_DocumentSymbolsResponse> {
-// Keep all types in the language_server_api namespace in sync with language server spec.
+  std::vector<language_server_api::SymbolInformation> symbols;
-namespace language_server_api {
-  using DocumentUri = std::string; // TODO
-
-  struct Position {
-    // Note: these are 0-based.
-    int line;
-    int character;
-  };
-
-  struct Range {
-    Position start;
-    Position end;
-  };
-
-  struct Location {
-    DocumentUri uri;
-    Range range;
-  };
-
-  enum class SymbolKind : int {
-    File = 1,
-    Module = 2,
-    Namespace = 3,
-    Package = 4,
-    Class = 5,
-    Method = 6,
-    Property = 7,
-    Field = 8,
-    Constructor = 9,
-    Enum = 10,
-    Interface = 11,
-    Function = 12,
-    Variable = 13,
-    Constant = 14,
-    String = 15,
-    Number = 16,
-    Boolean = 17,
-    Array = 18
-  };
-
-  struct SymbolInfo {
-    std::string name;
-    SymbolKind kind;
-    Location location;
-    std::string containerName;
-  };
-}
-
-struct IpcMessage_DocumentSymbolsResponse : public BaseIpcMessage {
-  std::vector<language_server_api::SymbolInfo> symbols;
 
   // BaseIpcMessage:
   static IpcMessageId id;
@@ -185,22 +150,22 @@ IpcMessageId IpcMessage_DocumentSymbolsResponse::id = "IpcMessage_DocumentSymbol
 
 
 
-
+void QueryDbMain() {
-void IndexerServerMain() {
   IpcServer ipc("languageserver");
 
   while (true) {
-    std::vector<std::unique_ptr<BaseIpcMessage>> messages = ipc.TakeMessages();
+    std::vector<std::unique_ptr<BaseIpcMessageElided>> messages = ipc.TakeMessages();
 
-    std::cout << "Server has " << messages.size() << " messages" << std::endl;
     for (auto& message : messages) {
-      if (message->runtime_id == IpcMessage_IsAlive::id) {
+      std::cout << "Processing message " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
+
+      if (message->runtime_id() == IpcMessage_IsAlive::id) {
         IpcMessage_IsAlive response;
         ipc.SendToClient(0, &response); // todo: make non-blocking
         break;
       }
       else {
-        std::cerr << "Unhandled IPC message with kind " << message->runtime_id << " (hash " << message->hashed_runtime_id << ")" << std::endl;
+        std::cerr << "Unhandled IPC message with kind " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
         exit(1);
         break;
       }
@@ -226,21 +191,130 @@ void LanguageServerStdinToServerDispatcher(IpcClient& ipc) {
   }
 }
 
+
+void ParseRpc(const std::string& method, const rapidjson::GenericValue<rapidjson::UTF8<>>& params) {
+}
+
+std::unique_ptr<language_server_api::InMessage> ParseMessage() {
+  int content_length = -1;
+  int iteration = 0;
+  while (true) {
+    if (++iteration > 10) {
+      assert(false && "bad parser state");
+      exit(1);
+    }
+
+    std::string line;
+    std::getline(std::cin, line);
+    //std::cin >> line;
+
+    if (line.compare(0, 14, "Content-Length") == 0) {
+      content_length = atoi(line.c_str() + 16);
+    }
+
+    if (line == "\r")
+      break;
+  }
+
+  assert(content_length >= 0);
+
+  std::string content;
+  content.reserve(content_length);
+  for (int i = 0; i < content_length; ++i) {
+    char c;
+    std::cin >> c;
+    content += c;
+  }
+
+  rapidjson::Document document;
+  document.Parse(content.c_str(), content_length);
+  assert(!document.HasParseError());
+
+  return language_server_api::MessageRegistry::instance()->Parse(document);
+
+  /*
+  std::string id;
+  if (document["id"].IsString())
+  id = document["id"].GetString();
+  else
+  id = std::to_string(document["id"].GetInt());
+  std::string method = document["method"].GetString();
+  auto& params = document["params"];
+
+
+  // Send initialize response.
+  {
+    std::string content =
+      R"foo({
+      "jsonrpc": "2.0",
+      "id": 0,
+      "result": {
+        "capabilities": {
+          "documentSymbolProvider": true
+        }
+      }
+    })foo";
+    std::cout << "Content-Length: " << content.size();
+    std::cout << (char)13 << char(10) << char(13) << char(10);
+    std::cout << content;
+  }
+  */
+}
+
+
+
 // Main loop for the language server. |ipc| is connected to
 // a server.
-void LanguageServerLoop(IpcClient& ipc) {
+void LanguageServerLoop(IpcClient* ipc) {
+  using namespace language_server_api;
+
   while (true) {
-    std::string input;
+    std::unique_ptr<InMessage> message = ParseMessage();
-    std::cin >> input;
 
-    std::cout << "got input " << input << std::endl << std::endl;
+    // Message parsing can fail if we don't recognize the method.
+    if (!message)
+      continue;
 
-    if (input == "references") {
+    std::cerr << "[info]: Got message of type " << MethodIdToString(message->method_id) << std::endl;
+    switch (message->method_id) {
+    case MethodId::Initialize:
+    {
+      // TODO: response should take id as input.
+      // TODO: message should not have top-level id.
+      auto response = Out_InitializeResponse();
+      response.id = message->id.value();
+      response.result.capabilities.documentSymbolProvider = true;
+      response.Send();
+      break;
+    }
 
+    case MethodId::TextDocumentDocumentSymbol:
+    {
+      auto response = Out_DocumentSymbolResponse();
+      response.id = message->id.value();
+
+      for (int i = 0; i < 2500; ++i) {
+        SymbolInformation info;
+        info.containerName = "fooContainer";
+        info.kind = language_server_api::SymbolKind::Field;
+        info.location.range.start.line = 5;
+        info.location.range.end.character = 20;
+        info.location.range.end.line = 5;
+        info.location.range.end.character = 25;
+        info.name = "Foobar";
+        response.result.push_back(info);
+      }
+
+      response.Send();
+      break;
+    }
     }
   }
 }
 
+
+
+
 void LanguageServerMain() {
   IpcClient ipc("languageserver", 0);
 
@@ -255,23 +329,44 @@ void LanguageServerMain() {
   std::this_thread::sleep_for(std::chrono::milliseconds(20));
 
   // Check if we got an IsAlive message back.
-  std::vector<std::unique_ptr<BaseIpcMessage>> messages = ipc.TakeMessages();
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> messages = ipc.TakeMessages();
   bool has_server = false;
   for (auto& message : messages) {
-    if (message->runtime_id == IpcMessage_IsAlive::id) {
+    if (message->runtime_id() == IpcMessage_IsAlive::id) {
       has_server = true;
       break;
     }
   }
 
   // No server is running. Start it.
-  if (!has_server) {
+  //if (!has_server) {
-    std::cerr << "Unable to detect running indexer server" << std::endl;
+  //  std::cerr << "Unable to detect running indexer server" << std::endl;
-    exit(1);
+  //  exit(1);
-  }
+  //}
 
-  std::cout << "Found indexer server" << std::endl;
+  std::thread stdio_reader(&LanguageServerLoop, &ipc);
-  LanguageServerLoop(ipc);
+
+  //std::cout << "Found indexer server" << std::endl;
+  //LanguageServerLoop(ipc);
+
+  // TODO: This is used for debugging, so we can attach to the client.
+
+
+  //std::cout << "garbagelkadklasldk" << std::endl;
+
+  bool should_break = true;
+  while (should_break)
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+  //std::this_thread::sleep_for(std::chrono::seconds(4));
+
+
+  //std::cout.flush();
+  /*
+  language_server_api::ShowMessageOutNotification show;
+  show.type = language_server_api::MessageType::Info;
+  show.message = "hello";
+  show.Send();
+  */
 }
 
 
@@ -348,19 +443,40 @@ void IpcMessage_CreateIndex::Deserialize(Reader& reader) {
 #endif
 
 int main(int argc, char** argv) {
+  // We need to write to stdout in binary mode because in Windows, writing
+  // \n will implicitly write \r\n. Language server API will ignore a
+  // \r\r\n split request.
+#ifdef _WIN32
+  _setmode(_fileno(stdout), O_BINARY);
+  _setmode(_fileno(stdin), O_BINARY);
+#endif
+
+  std::cerr << "Starting language server" << std::endl;
+
   IpcRegistry::instance()->Register<IpcMessage_IsAlive>();
   IpcRegistry::instance()->Register<IpcMessage_DocumentSymbolsRequest>();
   IpcRegistry::instance()->Register<IpcMessage_DocumentSymbolsResponse>();
 
-
+  language_server_api::MessageRegistry::instance()->Register<language_server_api::In_CancelRequest>();
-  if (argc == 2)
+  language_server_api::MessageRegistry::instance()->Register<language_server_api::In_InitializeRequest>();
-    LanguageServerMain();
+  language_server_api::MessageRegistry::instance()->Register<language_server_api::In_InitializedNotification>();
-  else
+  language_server_api::MessageRegistry::instance()->Register<language_server_api::In_DocumentSymbolRequest>();
-    IndexerServerMain();
-  return 0;
 
   std::unordered_map<std::string, std::string> options = ParseOptions(argc, argv);
 
+  if (HasOption(options, "--language-server")) {
+    LanguageServerMain();
+    return 0;
+  }
+  if (HasOption(options, "--querydb")) {
+    QueryDbMain();
+    return 0;
+  }
+
+
+  LanguageServerMain();
+  return 0;
+
   if (argc == 1 || options.find("--help") != options.end()) {
     std::cout << R"help(clang-indexer help:
 

ipc.cc

@@ -24,27 +24,14 @@ void JsonMessage::SetPayload(size_t payload_size, const char* payload) {
   memcpy(payload_dest, payload, payload_size);
 }
 
-BaseIpcMessage::BaseIpcMessage() {
+void BaseIpcMessageElided::Serialize(Writer& writer) {}
-  assert(!runtime_id_.empty() && "Message is not registered using IpcRegistry::RegisterAllocator");
 
-  runtime_id = runtime_id_;
+void BaseIpcMessageElided::Deserialize(Reader& reader) {}
-  hashed_runtime_id = hashed_runtime_id_;
-}
-
-BaseIpcMessage::~BaseIpcMessage() {}
-
-void BaseIpcMessage::Serialize(Writer& writer) {}
-
-void BaseIpcMessage::Deserialize(Reader& reader) {}
-
-IpcMessageId BaseIpcMessage::runtime_id_;
-
-int BaseIpcMessage::hashed_runtime_id_ = -1;
 
 IpcRegistry* IpcRegistry::instance_ = nullptr;
 
-std::unique_ptr<BaseIpcMessage> IpcRegistry::Allocate(int id) {
+std::unique_ptr<BaseIpcMessageElided> IpcRegistry::Allocate(int id) {
-  return std::unique_ptr<BaseIpcMessage>((*allocators)[id]());
+  return std::unique_ptr<BaseIpcMessageElided>((*allocators)[id]());
 }
 
 IpcDirectionalChannel::IpcDirectionalChannel(const std::string& name) {
@@ -57,7 +44,7 @@ IpcDirectionalChannel::~IpcDirectionalChannel() {
   delete[] local_block;
 }
 
-void IpcDirectionalChannel::PushMessage(BaseIpcMessage* message) {
+void IpcDirectionalChannel::PushMessage(BaseIpcMessageElided* message) {
   rapidjson::StringBuffer output;
   rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(output);
   writer.SetFormatOptions(
@@ -65,6 +52,8 @@ void IpcDirectionalChannel::PushMessage(BaseIpcMessage* message) {
   writer.SetIndent(' ', 2);
   message->Serialize(writer);
 
+  //std::cout << "Sending message with id " << message->runtime_id() << " (hash " << message->hashed_runtime_id() << ")" << std::endl;
+
   size_t payload_size = strlen(output.GetString());
   assert(payload_size < shmem_size && "Increase shared memory size, payload will never fit");
 
@@ -86,7 +75,7 @@ void IpcDirectionalChannel::PushMessage(BaseIpcMessage* message) {
     if ((*shared->shared_bytes_used + sizeof(JsonMessage) + payload_size) >= shmem_size)
       continue;
 
-    get_free_message()->message_id = message->hashed_runtime_id;
+    get_free_message()->message_id = message->hashed_runtime_id();
     get_free_message()->SetPayload(payload_size, output.GetString());
 
     *shared->shared_bytes_used += sizeof(JsonMessage) + get_free_message()->payload_size;
@@ -97,7 +86,7 @@ void IpcDirectionalChannel::PushMessage(BaseIpcMessage* message) {
 
 }
 
-std::vector<std::unique_ptr<BaseIpcMessage>> IpcDirectionalChannel::TakeMessages() {
+std::vector<std::unique_ptr<BaseIpcMessageElided>> IpcDirectionalChannel::TakeMessages() {
   size_t remaining_bytes = 0;
   // Move data from shared memory into a local buffer. Do this
   // before parsing the blocks so that other processes can begin
@@ -111,11 +100,11 @@ std::vector<std::unique_ptr<BaseIpcMessage>> IpcDirectionalChannel::TakeMessages
     get_free_message()->message_id = -1;
   }
 
-  std::vector<std::unique_ptr<BaseIpcMessage>> result;
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> result;
 
   char* message = local_block;
   while (remaining_bytes > 0) {
-    std::unique_ptr<BaseIpcMessage> base_message = IpcRegistry::instance()->Allocate(as_message(message)->message_id);
+    std::unique_ptr<BaseIpcMessageElided> base_message = IpcRegistry::instance()->Allocate(as_message(message)->message_id);
 
     rapidjson::Document document;
     document.Parse(as_message(message)->payload(), as_message(message)->payload_size);
@@ -138,7 +127,7 @@ std::vector<std::unique_ptr<BaseIpcMessage>> IpcDirectionalChannel::TakeMessages
 IpcServer::IpcServer(const std::string& name)
   : name_(name), server_(NameToServerName(name)) {}
 
-void IpcServer::SendToClient(int client_id, BaseIpcMessage* message) {
+void IpcServer::SendToClient(int client_id, BaseIpcMessageElided* message) {
   // Find or create the client.
   auto it = clients_.find(client_id);
   if (it == clients_.end())
@@ -147,17 +136,17 @@ void IpcServer::SendToClient(int client_id, BaseIpcMessage* message) {
   clients_[client_id]->PushMessage(message);
 }
 
-std::vector<std::unique_ptr<BaseIpcMessage>> IpcServer::TakeMessages() {
+std::vector<std::unique_ptr<BaseIpcMessageElided>> IpcServer::TakeMessages() {
   return server_.TakeMessages();
 }
 
 IpcClient::IpcClient(const std::string& name, int client_id)
   : server_(NameToServerName(name)), client_(NameToClientName(name, client_id)) {}
 
-void IpcClient::SendToServer(BaseIpcMessage* message) {
+void IpcClient::SendToServer(BaseIpcMessageElided* message) {
   server_.PushMessage(message);
 }
 
-std::vector<std::unique_ptr<BaseIpcMessage>> IpcClient::TakeMessages() {
+std::vector<std::unique_ptr<BaseIpcMessageElided>> IpcClient::TakeMessages() {
   return client_.TakeMessages();
 }

ipc.h

@@ -31,9 +31,17 @@ struct JsonMessage {
 
 using IpcMessageId = std::string;
 
+struct BaseIpcMessageElided {
+  virtual IpcMessageId runtime_id() const = 0;
+  virtual int hashed_runtime_id() const = 0;
+
+  virtual void Serialize(Writer& writer);
+  virtual void Deserialize(Reader& reader);
+};
+
 // Usage:
 //
-//  class IpcMessage_Foo : public BaseIpcMessage {
+//  class IpcMessage_Foo : public BaseIpcMessage<IpcMessage_Foo> {
 //    static IpcMessageId id;
 //
 //    // BaseIpcMessage:
@@ -45,23 +53,29 @@ using IpcMessageId = std::string;
 //  main() {
 //    IpcRegistry::instance()->Register<IpcMessage_Foo>();
 //  }
-struct BaseIpcMessage {
+//
+// Note: This is a template so that the statics are stored separately
+// per type.
+template<typename T>
+struct BaseIpcMessage : BaseIpcMessageElided {
   BaseIpcMessage();
   virtual ~BaseIpcMessage();
 
-  virtual void Serialize(Writer& writer);
-  virtual void Deserialize(Reader& reader);
-
-  IpcMessageId runtime_id;
-  int hashed_runtime_id = -1;
-
   // Populated by IpcRegistry::RegisterAllocator.
   static IpcMessageId runtime_id_;
   static int hashed_runtime_id_;
+
+  // BaseIpcMessageElided:
+  IpcMessageId runtime_id() const override {
+    return runtime_id_;
+  }
+  int hashed_runtime_id() const override {
+    return hashed_runtime_id_;
+  }
 };
 
 struct IpcRegistry {
-  using Allocator = std::function<BaseIpcMessage*()>;
+  using Allocator = std::function<BaseIpcMessageElided*()>;
 
   // Use unique_ptrs so we can initialize on first use
   // (static init order might not be right).
@@ -71,7 +85,7 @@ struct IpcRegistry {
   template<typename T>
   void Register();
 
-  std::unique_ptr<BaseIpcMessage> Allocate(int id);
+  std::unique_ptr<BaseIpcMessageElided> Allocate(int id);
 
   static IpcRegistry* instance() {
     if (!instance_)
@@ -115,8 +129,8 @@ struct IpcDirectionalChannel {
   explicit IpcDirectionalChannel(const std::string& name);
   ~IpcDirectionalChannel();
 
-  void PushMessage(BaseIpcMessage* message);
+  void PushMessage(BaseIpcMessageElided* message);
-  std::vector<std::unique_ptr<BaseIpcMessage>> TakeMessages();
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> TakeMessages();
 
 private:
   JsonMessage* get_free_message() {
@@ -134,8 +148,8 @@ private:
 struct IpcServer {
   IpcServer(const std::string& name);
 
-  void SendToClient(int client_id, BaseIpcMessage* message);
+  void SendToClient(int client_id, BaseIpcMessageElided* message);
-  std::vector<std::unique_ptr<BaseIpcMessage>> TakeMessages();
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> TakeMessages();
 
 private:
   std::string name_;
@@ -146,10 +160,34 @@ private:
 struct IpcClient {
   IpcClient(const std::string& name, int client_id);
 
-  void SendToServer(BaseIpcMessage* message);
+  void SendToServer(BaseIpcMessageElided* message);
-  std::vector<std::unique_ptr<BaseIpcMessage>> TakeMessages();
+  std::vector<std::unique_ptr<BaseIpcMessageElided>> TakeMessages();
 
 private:
   IpcDirectionalChannel server_;
   IpcDirectionalChannel client_;
 };
+
+
+
+
+
+
+
+
+
+
+
+template<typename T>
+BaseIpcMessage<T>::BaseIpcMessage() {
+  assert(!runtime_id_.empty() && "Message is not registered using IpcRegistry::RegisterAllocator");
+}
+
+template<typename T>
+BaseIpcMessage<T>::~BaseIpcMessage() {}
+
+template<typename T>
+IpcMessageId BaseIpcMessage<T>::runtime_id_;
+
+template<typename T>
+int BaseIpcMessage<T>::hashed_runtime_id_ = -1;

platform_win.cc

@@ -1,10 +1,11 @@
-#if false
 #include "platform.h"
 
 #include <cassert>
 #include <string>
 #include <Windows.h>
 
+#include "utils.h"
+
 struct PlatformMutexWin : public PlatformMutex {
   HANDLE raw_mutex = INVALID_HANDLE_VALUE;
 
@@ -70,4 +71,3 @@ std::unique_ptr<PlatformScopedMutexLock> CreatePlatformScopedMutexLock(PlatformM
 std::unique_ptr<PlatformSharedMemory> CreatePlatformSharedMemory(const std::string& name) {
   return MakeUnique<PlatformSharedMemoryWin>(name);
 }
-#endif

query.h

@@ -166,6 +166,7 @@ struct SymbolIdx {
 // TODO: Instead of passing to/from json, we can probably bass the IndexedFile type almost directly as
 // a raw memory dump - the type has almost zero pointers inside of it. We could do a little bit of fixup
 // so that passing from a separate process to the main db is really fast (no need to go through JSON).
+/*
 namespace foo2 {
   using Usr = size_t;
   struct UsrTable {
@@ -174,7 +175,7 @@ namespace foo2 {
     const char* usrs[];
   };
 }
-
+*/
 
 struct IndexUpdate {
   // File updates.

serializer.cc

@@ -230,7 +230,7 @@ void Deserialize(const Reader& reader, IndexedFile* file) {
 
 std::string Serialize(IndexedFile* file) {
   rapidjson::StringBuffer output;
-  rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(output);
+  Writer writer(output);
   writer.SetFormatOptions(
     rapidjson::PrettyFormatOptions::kFormatSingleLineArray);
   writer.SetIndent(' ', 2);