code completion works (but is still blocking)

foo/_empty_test.cc

@@ -0,0 +1,37 @@
+#include <string>
+#include <vector>
+
+struct MyBar {
+  int x;
+  int aaaa1;
+  int aaaa2;
+  int aaaa3;
+  static int foobez;
+
+  // This is some awesome docs.
+  float MemberFunc(int a, char b, std::vector<int> foo);
+  float MemberFunc2(int a, char b, std::vector<int> foo);
+
+  // The names are some extra state.
+  std::vector<std::string> names;
+};
+
+int MyBar::foobez;
+
+int foo() {
+  int a = 10;
+  MyBar foooo;
+  MyBar f;
+  MyBar f2;
+}
+
+float MyBar::MemberFunc(int a, char b, std::vector<int> foo) {
+  this->x = 100;
+  return 0;
+}
+
+
+/*
+OUTPUT:
+{}
+*/

foo/clang_args

@@ -0,0 +1,13 @@
+-xc++
+-std=c++11
+-IC:/Users/jacob/Desktop/superindex/indexer/third_party
+-IC:/Users/jacob/Desktop/superindex/indexer/third_party/doctest/doctest
+-IC:/Users/jacob/Desktop/superindex/indexer/third_party/rapidjson/include
+-IC:/Program Files/LLVM/include
+
+#--sysrootC:/Users/jacob/Desktop/superindex/indexer/libcxx
+#-IC:/Users/jacob/Desktop/superindex/indexer/libcxx/include
+#-FC:/Users/jacob/Desktop/superindex/indexer/libcxx/include
+
+-fms-compatibility
+-fdelayed-template-parsing

src/code_completion.cc

@@ -1,11 +1,348 @@
-#include <clang-c/Index.h>
+#include "code_completion.h"
+
+#include "libclangmm/Utility.h"
+#include "timer.h"
+
+#include <algorithm>
+
+namespace {
+unsigned Flags() {
+  return
+    CXTranslationUnit_Incomplete |
+    CXTranslationUnit_PrecompiledPreamble |
+    CXTranslationUnit_CacheCompletionResults |
+    //CXTranslationUnit_ForSerialization |
+    CXTranslationUnit_IncludeBriefCommentsInCodeCompletion;
+  CXTranslationUnit_CreatePreambleOnFirstParse |
+    CXTranslationUnit_KeepGoing;
+}
+
+bool StartsWith(const std::string& value, const std::string& start) {
+  if (start.size() > value.size())
+    return false;
+  return std::equal(start.begin(), start.end(), value.begin());
+}
+
+lsCompletionItemKind GetCompletionKind(CXCursorKind cursor_kind) {
+  switch (cursor_kind) {
+
+  case CXCursor_ObjCInstanceMethodDecl:
+  case CXCursor_CXXMethod:
+    return lsCompletionItemKind::Method;
+
+  case CXCursor_FunctionTemplate:
+  case CXCursor_FunctionDecl:
+    return lsCompletionItemKind::Function;
+
+  case CXCursor_Constructor:
+  case CXCursor_Destructor:
+  case CXCursor_ConversionFunction:
+    return lsCompletionItemKind::Constructor;
+
+  case CXCursor_FieldDecl:
+    return lsCompletionItemKind::Field;
+
+  case CXCursor_VarDecl:
+  case CXCursor_ParmDecl:
+    return lsCompletionItemKind::Variable;
+
+  case CXCursor_ClassTemplate:
+  case CXCursor_ClassTemplatePartialSpecialization:
+  case CXCursor_ClassDecl:
+  case CXCursor_StructDecl:
+  case CXCursor_UsingDeclaration:
+  case CXCursor_TypedefDecl:
+  case CXCursor_TypeAliasDecl:
+  case CXCursor_TypeAliasTemplateDecl:
+    return lsCompletionItemKind::Class;
+
+  case CXCursor_EnumConstantDecl:
+  case CXCursor_EnumDecl:
+    return lsCompletionItemKind::Enum;
+
+  case CXCursor_MacroDefinition:
+    return lsCompletionItemKind::Interface;
+
+  case CXCursor_Namespace:
+    return lsCompletionItemKind::Module;
+
+    //return lsCompletionItemKind::Property;
+    //return lsCompletionItemKind::Unit;
+    //return lsCompletionItemKind::Value;
+    //return lsCompletionItemKind::Keyword;
+    //return lsCompletionItemKind::Snippet;
+    //return lsCompletionItemKind::Color;
+    //return lsCompletionItemKind::File;
+    //return lsCompletionItemKind::Reference;
+
+  case CXCursor_NotImplemented:
+    break;
+
+  default:
+    std::cerr << "Unhandled completion kind " << cursor_kind << std::endl;
+    return lsCompletionItemKind::Text;
+  }
+}
+
+std::string BuildDetailString(CXCompletionString completion_string) {
+  std::string detail;
+
+  int num_chunks = clang_getNumCompletionChunks(completion_string);
+  for (unsigned i = 0; i < num_chunks; ++i) {
+    CXCompletionChunkKind kind = clang_getCompletionChunkKind(completion_string, i);
+
+    switch (kind) {
+    case CXCompletionChunk_Optional: {
+      CXCompletionString nested = clang_getCompletionChunkCompletionString(completion_string, i);
+      detail += BuildDetailString(nested);
+      break;
+    }
+
+    case CXCompletionChunk_Placeholder: {
+      // TODO: send this info to vscode.
+      CXString text = clang_getCompletionChunkText(completion_string, i);
+      detail += clang::ToString(text);
+      break;
+    }
+
+    case CXCompletionChunk_TypedText:
+    case CXCompletionChunk_Text:
+    case CXCompletionChunk_Informative:
+    case CXCompletionChunk_CurrentParameter: {
+      CXString text = clang_getCompletionChunkText(completion_string, i);
+      detail += clang::ToString(text);
+      break;
+    }
+
+    case CXCompletionChunk_ResultType: {
+      CXString text = clang_getCompletionChunkText(completion_string, i);
+      std::string new_detail = clang::ToString(text) + detail + " ";
+      detail = new_detail;
+      break;
+    }
+
+    case CXCompletionChunk_LeftParen:
+      detail += "(";
+      break;
+    case CXCompletionChunk_RightParen:
+      detail += ")";
+      break;
+    case CXCompletionChunk_LeftBracket:
+      detail += "]";
+      break;
+    case CXCompletionChunk_RightBracket:
+      detail += "[";
+      break;
+    case CXCompletionChunk_LeftBrace:
+      detail += "{";
+      break;
+    case CXCompletionChunk_RightBrace:
+      detail += "}";
+      break;
+    case CXCompletionChunk_LeftAngle:
+      detail += "<";
+      break;
+    case CXCompletionChunk_RightAngle:
+      detail += ">";
+      break;
+    case CXCompletionChunk_Comma:
+      detail += ", ";
+      break;
+    case CXCompletionChunk_Colon:
+      detail += ":";
+      break;
+    case CXCompletionChunk_SemiColon:
+      detail += ";";
+      break;
+    case CXCompletionChunk_Equal:
+      detail += "=";
+      break;
+    case CXCompletionChunk_HorizontalSpace:
+    case CXCompletionChunk_VerticalSpace:
+      detail += " ";
+      break;
+    }
+  }
+
+  return detail;
+}
+}
+
+CompletionSession::CompletionSession(const CompilationEntry& file, WorkingFiles* working_files) : file(file) {
+  std::vector<CXUnsavedFile> unsaved = working_files->AsUnsavedFiles();
+
+  std::vector<std::string> args = file.args;
+  args.push_back("-fparse-all-comments");
+
+  std::string sent_args = "";
+  for (auto& arg : args)
+    sent_args += arg + ", ";
+  std::cerr << "Creating completion session with arguments " << sent_args << std::endl;
+
+  // TODO: I think we crash when there are syntax errors.
+  active_index = MakeUnique<clang::Index>(0 /*excludeDeclarationsFromPCH*/, 0 /*displayDiagnostics*/);
+  active = MakeUnique<clang::TranslationUnit>(*active_index, file.filename, args, unsaved, Flags());
+  if (active->did_fail) {
+    std::cerr << "Failed to create translation unit; trying again..." << std::endl;
+    active = MakeUnique<clang::TranslationUnit>(*active_index, file.filename, args, unsaved, Flags());
+  }
+
+  // Despite requesting clang create a precompiled header on the first parse in
+  // Flags() via CXTranslationUnit_CreatePreambleOnFirstParse, it doesn't seem
+  // to do so. Immediately reparsing will create one which reduces
+  // clang_codeCompleteAt timing from 200ms to 20ms on simple files.
+  if (!active->did_fail)
+    Refresh(working_files);
+}
+
+CompletionSession::~CompletionSession() {}
+
+void CompletionSession::Refresh(WorkingFiles* working_files) {
+  // TODO: Do this off the code completion thread so we don't block completions.
+  active->ReparseTranslationUnit(working_files->AsUnsavedFiles(), Flags());
+}
+
+CompletionManager::CompletionManager(Project* project, WorkingFiles* working_files) : project(project), working_files(working_files) {}
+
+NonElidedVector<lsCompletionItem> CompletionManager::CodeComplete(const lsTextDocumentPositionParams& completion_location) {
+  NonElidedVector<lsCompletionItem> ls_result;
+
+  CompletionSession* session = GetOrOpenSession(completion_location.textDocument.uri.GetPath());
+
+  unsigned line = completion_location.position.line + 1;
+  unsigned column = completion_location.position.character + 1;
+
+  std::cerr << std::endl;
+  //std::cerr << "Completing at " << line << ":" << column << std::endl;
+
+  std::vector<CXUnsavedFile> unsaved = working_files->AsUnsavedFiles();
+
+
+  Timer timer;
+
+
+  timer.Reset();
+  CXCodeCompleteResults* cx_results = clang_codeCompleteAt(
+    session->active->cx_tu,
+    session->file.filename.c_str(), line, column,
+    unsaved.data(), unsaved.size(),
+    CXCodeComplete_IncludeMacros | CXCodeComplete_IncludeBriefComments);
+  if (!cx_results) {
+    std::cerr << "Code completion failed" << std::endl;
+    return ls_result;
+  }
+  std::cerr << "clang_codeCompleteAt took " << timer.ElapsedMilliseconds() << "ms; got " << cx_results->NumResults << " results" << std::endl;
+
+  // TODO: for comments we could hack the unsaved buffer and transform // into ///
+
+  timer.Reset();
+  ls_result.reserve(cx_results->NumResults);
+  std::cerr << "Reserving results took " << timer.ElapsedMilliseconds() << "ms" << std::endl;
+
+  timer.Reset();
+  for (int i = 0; i < cx_results->NumResults; ++i) {
+    CXCompletionResult& result = cx_results->Results[i];
+
+    //unsigned int is_incomplete;
+    //CXCursorKind kind = clang_codeCompleteGetContainerKind(cx_results, &is_incomplete);
+    //std::cerr << "clang_codeCompleteGetContainerKind kind=" << kind << " is_incomplete=" << is_incomplete << std::endl;
+
+    // CXCursor_InvalidCode fo
+
+    //clang_codeCompleteGetContexts
+    //CXCompletionContext
+    // clang_codeCompleteGetContexts
+    //
+    //CXCursorKind kind;
+    //CXString str = clang_getCompletionParent(result.CompletionString, &kind);
+    //std::cerr << "clang_getCompletionParent kind=" << kind << ", str=" << clang::ToString(str) << std::endl;
+    // if global don't append now, append to end later
+
+    // also clang_codeCompleteGetContainerKind
+
+    // TODO: fill in more data
+    lsCompletionItem ls_completion_item;
+
+    ls_completion_item.kind = GetCompletionKind(result.CursorKind);
+
+
+    // Get the primary text to insert.
+
+    int num_chunks = clang_getNumCompletionChunks(result.CompletionString);
+    for (unsigned i = 0; i < num_chunks; ++i) {
+      CXCompletionChunkKind kind = clang_getCompletionChunkKind(result.CompletionString, i);
+      if (kind == CXCompletionChunk_TypedText) {
+        ls_completion_item.label += clang::ToString(clang_getCompletionChunkText(result.CompletionString, i));
+        break;
+      }
+    }
+
+    ls_completion_item.detail = BuildDetailString(result.CompletionString);
+
+    // Get docs.
+    ls_completion_item.documentation = clang::ToString(clang_getCompletionBriefComment(result.CompletionString));
+
+    // Set priority, and make some adjustments.
+    int priority = clang_getCompletionPriority(result.CompletionString);
+
+    if (result.CursorKind == CXCursor_Destructor) {
+      priority *= 100;
+      //std::cerr << "Bumping[destructor] " << ls_completion_item.label << std::endl;
+    }
+    if (result.CursorKind == CXCursor_ConversionFunction ||
+      (result.CursorKind == CXCursor_CXXMethod && StartsWith(ls_completion_item.label, "operator"))) {
+      //std::cerr << "Bumping[conversion] " << ls_completion_item.label << std::endl;
+      priority *= 100;
+    }
+    if (clang_getCompletionAvailability(result.CompletionString) != CXAvailability_Available) {
+      //std::cerr << "Bumping[notavailable] " << ls_completion_item.label << std::endl;
+      priority *= 100;
+    }
+    //std::cerr << "Adding kind=" << result.CursorKind << ", priority=" << ls_completion_item.priority_ << ", label=" << ls_completion_item.label << std::endl;
+
+    // TODO: we can probably remove priority_ and our sort.
+    ls_completion_item.sortText = uint64_t(priority);// std::to_string(ls_completion_item.priority_);
+
+    ls_result.push_back(ls_completion_item);
+  }
+  std::cerr << "Building completion results took " << timer.ElapsedMilliseconds() << "ms" << std::endl;
+
+  timer.Reset();
+  clang_disposeCodeCompleteResults(cx_results);
+  std::cerr << "clang_disposeCodeCompleteResults took " << timer.ElapsedMilliseconds() << "ms" << std::endl;
+
+  // Score completions by priority.
+  /*
+  timer.Reset();
+  std::sort(
+    ls_result.begin(), ls_result.end(),
+    [](const lsCompletionItem& a, const lsCompletionItem& b) {
+      return a.priority_ < b.priority_;
+    });
+  std::cerr << "Sorting completion results took " << timer.ElapsedMilliseconds() << "ms" << std::endl;
+  */
+  //std::cerr << std::endl;
+  //for (auto& result : ls_result) {
+  //  std::cerr << "SORTED priority=" << result.priority_ << ", sortText=" << result.sortText << ", label=" << result.label << std::endl;
+  //}
+
+  return ls_result;
+  //clang_codeCompleteAt()
 
-void doit() {
   // we should probably main two translation units, one for
   // serving current requests, and one that is reparsing (follow qtcreator)
 
-  // use clang_codeCompleteAt
+  // todo: we need to run code completion on a separate thread from querydb
+  // so thread layout looks like:
+  //    - stdin                   # Reads data from stdin
+  //    - stdout                  # Pushes data to stdout
+  //    - querydb                 # Resolves index database queries.
+  //    - complete_responder      # Handles low-latency code complete requests.
+  //    - complete_parser         # Parses most recent document for future code complete requests.
+  //    - indexer (many)          # Runs index jobs (for querydb updates)
 
+  // use clang_codeCompleteAt
+  //CXUnsavedFile
   // we need to setup CXUnsavedFile
   // The key to doing that is via
   //  - textDocument/didOpen
@@ -14,4 +351,26 @@ void doit() {
 
   // probably don't need
   //  - textDocument/willSave
+}
+
+CompletionSession* CompletionManager::GetOrOpenSession(const std::string& filename) {
+  // Try to find existing session.
+  for (auto& session : sessions) {
+    if (session->file.filename == filename)
+      return session.get();
+  }
+
+  // Create new session. Note that this will block.
+  std::cerr << "Creating new code completion session for " << filename << std::endl;
+  optional<CompilationEntry> entry = project->FindCompilationEntryForFile(filename);
+  if (!entry) {
+    std::cerr << "Unable to find compilation entry" << std::endl;
+    entry = CompilationEntry();
+    entry->filename = filename;
+  }
+  else {
+    std::cerr << "Found compilation entry" << std::endl;
+  }
+  sessions.push_back(MakeUnique<CompletionSession>(*entry, working_files));
+  return sessions[sessions.size() - 1].get();
 }

src/code_completion.h

@@ -0,0 +1,44 @@
+#include "language_server_api.h"
+#include "libclangmm/CompletionString.h"
+#include "libclangmm/Index.h"
+#include "libclangmm/TranslationUnit.h"
+#include "project.h"
+#include "working_files.h"
+
+#include <clang-c/Index.h>
+
+struct CompletionSession {
+  CompilationEntry file;
+
+  // The active translation unit.
+  std::unique_ptr<clang::TranslationUnit> active;
+  std::unique_ptr<clang::Index> active_index;
+
+  // Updated translation unit. If |is_updated_ready| is true, then |updated|
+  // contains more recent state than |active| and the two should be swapped.
+  // TODO: implement this. Needs changes in Refresh and CodeComplete.
+  //bool is_updated_ready = false;
+  //std::unique_ptr<clang::TranslationUnit> updated;
+  //std::unique_ptr<clang::Index> updated_index;
+
+  CompletionSession(const CompilationEntry& file, WorkingFiles* working_files);
+  ~CompletionSession();
+
+  // Refresh file index.
+  void Refresh(WorkingFiles* working_files);
+};
+
+struct CompletionManager {
+  std::vector<std::unique_ptr<CompletionSession>> sessions;
+  Project* project;
+  WorkingFiles* working_files;
+
+  CompletionManager(Project* project, WorkingFiles* working_files);
+
+  // This all should run on complete_responder thread. This will internally run a child thread to
+  // reparse.
+  NonElidedVector<lsCompletionItem> CodeComplete(const lsTextDocumentPositionParams& completion_location);
+
+ private:
+  CompletionSession* GetOrOpenSession(const std::string& filename);
+};

src/command_line.cc

@@ -1,13 +1,16 @@
 // TODO: cleanup includes
+#include "code_completion.h"
 #include "compilation_database_loader.h"
 #include "indexer.h"
 #include "query.h"
 #include "language_server_api.h"
+#include "project.h"
 #include "platform.h"
 #include "test.h"
 #include "timer.h"
 #include "threaded_queue.h"
 #include "typed_bidi_message_queue.h"
+#include "working_files.h"
 
 #include <doctest/doctest.h>
 #include <rapidjson/istreamwrapper.h>
@@ -187,7 +190,7 @@ QueryableFile* FindFile(QueryableDatabase* db, const std::string& filename) {
   for (auto& file : db->files) {
     // std::cerr << " - Have file " << file.file_id << std::endl;
     if (file.file_id == filename) {
-      std::cerr << "Found file " << filename << std::endl;
+      //std::cerr << "Found file " << filename << std::endl;
       return &file;
     }
   }
@@ -301,7 +304,11 @@ void QueryDbMainLoop(
   QueryableDatabase* db,
   IpcMessageQueue* language_client,
   IndexRequestQueue* index_requests,
-  IndexResponseQueue* index_responses) {
+  IndexResponseQueue* index_responses,
+  Project* project,
+  WorkingFiles* working_files,
+  CompletionManager* completion_manager) {
+
   std::vector<std::unique_ptr<BaseIpcMessage>> messages = language_client->GetMessages(&language_client->for_server);
   for (auto& message : messages) {
     // std::cerr << "Processing message " << static_cast<int>(message->ipc_id)
@@ -324,12 +331,12 @@ void QueryDbMainLoop(
       Ipc_OpenProject* msg = static_cast<Ipc_OpenProject*>(message.get());
       std::string path = msg->project_path;
 
-      std::vector<CompilationEntry> entries =
+      project->entries = LoadCompilationEntriesFromDirectory(path);
-        LoadCompilationEntriesFromDirectory(path);
+      for (int i = 0; i < project->entries.size(); ++i) {
-      for (int i = 0; i < entries.size(); ++i) {
+        const CompilationEntry& entry = project->entries[i];
-        const CompilationEntry& entry = entries[i];
+        std::string filepath = entry.filename;
-        std::string filepath = path + "/" + entry.filename;
+
-        std::cerr << "[" << i << "/" << (entries.size() - 1)
+        std::cerr << "[" << i << "/" << (project->entries.size() - 1)
           << "] Dispatching index request for file " << filepath
           << std::endl;
 
@@ -344,17 +351,20 @@ void QueryDbMainLoop(
 
     case IpcId::TextDocumentDidOpen: {
       auto msg = static_cast<Ipc_TextDocumentDidOpen*>(message.get());
-      std::cerr << "Opening " << msg->params.textDocument.uri.GetPath() << std::endl;
+      //std::cerr << "Opening " << msg->params.textDocument.uri.GetPath() << std::endl;
+      working_files->OnOpen(msg->params);
       break;
     }
     case IpcId::TextDocumentDidChange: {
       auto msg = static_cast<Ipc_TextDocumentDidChange*>(message.get());
-      std::cerr << "Changing " << msg->params.textDocument.uri.GetPath() << std::endl;
+      working_files->OnChange(msg->params);
+      //std::cerr << "Changing " << msg->params.textDocument.uri.GetPath() << std::endl;
       break;
     }
     case IpcId::TextDocumentDidClose: {
       auto msg = static_cast<Ipc_TextDocumentDidClose*>(message.get());
       std::cerr << "Closing " << msg->params.textDocument.uri.GetPath() << std::endl;
+      //working_files->OnClose(msg->params);
       break;
     }
 
@@ -363,16 +373,12 @@ void QueryDbMainLoop(
       Out_TextDocumentComplete response;
       response.id = msg->id;
       response.result.isIncomplete = false;
+      response.result.items = completion_manager->CodeComplete(msg->params);
 
-      std::cerr << "!! Code complete";
+      Timer timer;
-      for (int i = 0; i < 50; ++i) {
+      response.Write(std::cout);
-        lsCompletionItem item;
+      std::cerr << "Writing completion results to stdout took " << timer.ElapsedMilliseconds() << "ms" << std::endl;
-        item.label = "Entry#" + std::to_string(i);
+      //SendOutMessageToClient(language_client, response);
-        item.documentation = "this is my doc " + std::to_string(i);
-        response.result.items.push_back(item);
-      }
-
-      SendOutMessageToClient(language_client, response);
       break;
     }
 
@@ -617,6 +623,9 @@ void QueryDbMain() {
   std::unique_ptr<IpcMessageQueue> ipc = BuildIpcMessageQueue(kIpcLanguageClientName, kQueueSizeBytes);
   IndexRequestQueue index_request_queue;
   IndexResponseQueue index_response_queue;
+  Project project;
+  WorkingFiles working_files;
+  CompletionManager completion_manager(&project, &working_files);
 
   // Start indexer threads.
   for (int i = 0; i < kNumIndexers; ++i) {
@@ -628,7 +637,7 @@ void QueryDbMain() {
   // Run query db main loop.
   QueryableDatabase db;
   while (true) {
-    QueryDbMainLoop(&db, ipc.get(), &index_request_queue, &index_response_queue);
+    QueryDbMainLoop(&db, ipc.get(), &index_request_queue, &index_response_queue, &project, &working_files, &completion_manager);
     std::this_thread::sleep_for(std::chrono::milliseconds(10));
   }
 }
@@ -649,8 +658,8 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) {
     if (!message)
       continue;
 
-    std::cerr << "[info]: Got message of type "
+    //std::cerr << "[info]: Got message of type "
-      << IpcIdToString(message->method_id) << std::endl;
+    //  << IpcIdToString(message->method_id) << std::endl;
     switch (message->method_id) {
       // TODO: For simplicitly lets just proxy the initialize request like
       // all other requests so that stdin loop thread becomes super simple.
@@ -677,11 +686,11 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) {
       //response.result.capabilities.textDocumentSync->change = lsTextDocumentSyncKind::Full;
       //response.result.capabilities.textDocumentSync->willSave = true;
       //response.result.capabilities.textDocumentSync->willSaveWaitUntil = true;
-      response.result.capabilities.textDocumentSync = lsTextDocumentSyncKind::Incremental;
+      response.result.capabilities.textDocumentSync = lsTextDocumentSyncKind::Full; // TODO: use incremental at some point
 
       response.result.capabilities.completionProvider = lsCompletionOptions();
       response.result.capabilities.completionProvider->resolveProvider = false;
-      response.result.capabilities.completionProvider->triggerCharacters = { ".", "::", "->" };
+      response.result.capabilities.completionProvider->triggerCharacters = { ".", "::", "->", ":", ">" };
 
       response.result.capabilities.codeLensProvider = lsCodeLensOptions();
       response.result.capabilities.codeLensProvider->resolveProvider = false;
@@ -705,19 +714,13 @@ void LanguageServerStdinLoop(IpcMessageQueue* ipc) {
       break;
     }
 
+    case IpcId::TextDocumentDidOpen:
+    case IpcId::TextDocumentDidChange:
+    case IpcId::TextDocumentDidClose: {
     case IpcId::TextDocumentCompletion:
     case IpcId::TextDocumentDocumentSymbol:
     case IpcId::TextDocumentCodeLens:
     case IpcId::WorkspaceSymbol:
-      break;
-
-    case IpcId::TextDocumentDidOpen:
-    case IpcId::TextDocumentDidChange:
-    case IpcId::TextDocumentDidClose: {
-      //case IpcId::TextDocumentCompletion:
-      //case IpcId::TextDocumentDocumentSymbol:
-      //case IpcId::TextDocumentCodeLens:
-      //case IpcId::WorkspaceSymbol: {
       ipc->SendMessage(&ipc->for_server, message->method_id, *message.get());
       break;
     }

src/compilation_database_loader.cc

@@ -15,6 +15,8 @@ bool EndsWith(const std::string& value, const std::string& ending) {
 }
 
 bool StartsWith(const std::string& value, const std::string& start) {
+  if (start.size() > value.size())
+    return false;
   return std::equal(start.begin(), start.end(), value.begin());
 }
 
@@ -30,14 +32,14 @@ std::vector<CompilationEntry> LoadFromDirectoryListing(const std::string& projec
   }
 
 
-  std::vector<std::string> files = GetFilesInFolder(project_directory, true /*recursive*/, false /*add_folder_to_path*/);
+  std::vector<std::string> files = GetFilesInFolder(project_directory, true /*recursive*/, true /*add_folder_to_path*/);
   for (const std::string& file : files) {
     if (EndsWith(file, ".cc") || EndsWith(file, ".cpp") ||
       EndsWith(file, ".c") || EndsWith(file, ".h") ||
       EndsWith(file, ".hpp")) {
 
       CompilationEntry entry;
-      entry.directory = ".";
+      entry.directory = project_directory;
       entry.filename = file;
       entry.args = args;
       result.push_back(entry);

src/indexer.cpp

@@ -1218,7 +1218,7 @@ IndexedFile Parse(std::string filename,
 
   clang::Index index(0 /*excludeDeclarationsFromPCH*/,
                      0 /*displayDiagnostics*/);
-  clang::TranslationUnit tu(index, filename, args);
+  clang::TranslationUnit tu(index, filename, args, {} /*unsaved_files*/, CXTranslationUnit_None);
 
   if (dump_ast)
     Dump(tu.document_cursor());

src/language_server_api.cc

@@ -130,7 +130,7 @@ void lsDocumentUri::SetPath(const std::string& path) {
   //std::cerr << "Set uri to " << raw_uri << " from " << path;
 }
 
-std::string lsDocumentUri::GetPath() {
+std::string lsDocumentUri::GetPath() const {
   // TODO: make this not a hack.
   std::string result = raw_uri;
 

src/language_server_api.h

@@ -212,7 +212,7 @@ struct lsDocumentUri {
   bool operator==(const lsDocumentUri& other) const;
 
   void SetPath(const std::string& path);
-  std::string GetPath();
+  std::string GetPath() const;
 
   std::string raw_uri;
 };
@@ -413,15 +413,15 @@ struct lsCompletionItem {
 
   // A string that should be used when filtering a set of
   // completion items. When `falsy` the label is used.
-  std::string filterText;
+  //std::string filterText;
 
   // A string that should be inserted a document when selecting
   // this completion. When `falsy` the label is used.
-  std::string insertText;
+  //std::string insertText;
 
   // The format of the insert text. The format applies to both the `insertText` property
   // and the `newText` property of a provided `textEdit`.
-  lsInsertTextFormat insertTextFormat;
+  //lsInsertTextFormat insertTextFormat;
 
   // An edit which is applied to a document when selecting this completion. When an edit is provided the value of
   // `insertText` is ignored.
@@ -449,10 +449,7 @@ MAKE_REFLECT_STRUCT(lsCompletionItem,
   kind,
   detail,
   documentation,
-  sortText,
+  sortText);
-  filterText,
-  insertText,
-  insertTextFormat);
 
 
 struct lsTextDocumentItem {
@@ -1050,7 +1047,7 @@ struct Ipc_TextDocumentDidChange : public IpcMessage<Ipc_TextDocumentDidChange>
     // The range of the document that changed.
     lsRange range;
     // The length of the range that got replaced.
-    int rangeLength = 0;
+    int rangeLength = -1;
     // The new text of the range/document.
     std::string text;
   };

src/libclangmm/TranslationUnit.cc

@@ -8,50 +8,44 @@
 
 namespace clang {
 
-/*
+TranslationUnit::TranslationUnit(
-TranslationUnit::TranslationUnit(Index &index, const std::string &file_path,
+  Index &index,
-  const std::vector<std::string> &command_line_args,
+  const std::string& filepath,
-  const std::string &buffer, unsigned flags) {
+  const std::vector<std::string>& arguments,
-  std::vector<const char*> args;
+  std::vector<CXUnsavedFile> unsaved_files,
-  for (auto& a : command_line_args) {
+  unsigned flags) {
-    args.push_back(a.c_str());
-  }
-
-  CXUnsavedFile files[1];
-  files[0].Filename = file_path.c_str();
-  files[0].Contents = buffer.c_str();
-  files[0].Length = buffer.size();
-
-  CXErrorCode error_code = clang_parseTranslationUnit2(
-    index.cx_index, file_path.c_str(), args.data(), args.size(), files, 1, flags, &cx_tu);
-  assert(!error_code);
-}
-*/
-
-TranslationUnit::TranslationUnit(Index &index, const std::string &file_path,
-  const std::vector<std::string> &command_line_args, unsigned flags) {
 
   std::vector<const char*> args;
-  for (const std::string& a : command_line_args) {
+  for (const std::string& a : arguments) {
     args.push_back(a.c_str());
   }
 
   CXErrorCode error_code = clang_parseTranslationUnit2(
-    index.cx_index, file_path.c_str(), args.data(), args.size(), nullptr, 0, flags, &cx_tu);
+    index.cx_index,
+    filepath.c_str(),
+    args.data(), args.size(),
+    unsaved_files.data(), unsaved_files.size(),
+    flags, &cx_tu);
+  
   switch (error_code) {
   case CXError_Success:
+    did_fail = false;
     break;
   case CXError_Failure:
-    std::cerr << "libclang generic failure for " << file_path << std::endl;
+    std::cerr << "libclang generic failure for " << filepath << std::endl;
+    did_fail = true;
     break;
   case CXError_Crashed:
-    std::cerr << "libclang crashed for " << file_path << std::endl;
+    std::cerr << "libclang crashed for " << filepath << std::endl;
+    did_fail = true;
     break;
   case CXError_InvalidArguments:
-    std::cerr << "libclang had invalid arguments for " << file_path << std::endl;
+    std::cerr << "libclang had invalid arguments for " << filepath << std::endl;
+    did_fail = true;
     break;
   case CXError_ASTReadError:
-    std::cerr << "libclang had ast read error for " << file_path << std::endl;
+    std::cerr << "libclang had ast read error for " << filepath << std::endl;
+    did_fail = true;
     break;
   }
 }
@@ -60,51 +54,33 @@ TranslationUnit::~TranslationUnit() {
   clang_disposeTranslationUnit(cx_tu);
 }
 
-void TranslationUnit::parse(Index &index, const std::string &file_path,
+void TranslationUnit::ReparseTranslationUnit(std::vector<CXUnsavedFile>& unsaved, unsigned flags) {
-  const std::vector<std::string> &command_line_args,
+  int error_code = clang_reparseTranslationUnit(cx_tu, unsaved.size(), unsaved.data(), flags);
-  const std::map<std::string, std::string>  &buffers, unsigned flags) {
+  switch (error_code) {
-  std::vector<CXUnsavedFile> files;
+  case CXError_Success:
-  for (auto &buffer : buffers) {
+    did_fail = false;
-    CXUnsavedFile file;
+    break;
-    file.Filename = buffer.first.c_str();
+  case CXError_Failure:
-    file.Contents = buffer.second.c_str();
+    std::cerr << "libclang reparse generic failure" << std::endl;
-    file.Length = buffer.second.size();
+    did_fail = true;
-    files.push_back(file);
+    break;
+  case CXError_Crashed:
+    std::cerr << "libclang reparse crashed " << std::endl;
+    did_fail = true;
+    break;
+  case CXError_InvalidArguments:
+    std::cerr << "libclang reparse had invalid arguments" << std::endl;
+    did_fail = true;
+    break;
+  case CXError_ASTReadError:
+    std::cerr << "libclang reparse had ast read error" << std::endl;
+    did_fail = true;
+    break;
   }
-  std::vector<const char*> args;
-  for (auto &a : command_line_args) {
-    args.push_back(a.c_str());
-  }
-  cx_tu = clang_parseTranslationUnit(index.cx_index, file_path.c_str(), args.data(),
-    args.size(), files.data(), files.size(), flags);
 }
 
-int TranslationUnit::ReparseTranslationUnit(const std::string &buffer, unsigned flags) {
+CodeCompleteResults TranslationUnit::get_code_completions(
-  CXUnsavedFile files[1];
+  const std::string& buffer, unsigned line_number, unsigned column) {
-
-  auto file_path = ToString(clang_getTranslationUnitSpelling(cx_tu));
-
-  files[0].Filename = file_path.c_str();
-  files[0].Contents = buffer.c_str();
-  files[0].Length = buffer.size();
-
-  return clang_reparseTranslationUnit(cx_tu, 1, files, flags);
-}
-
-unsigned TranslationUnit::DefaultFlags() {
-  auto flags =
-    CXTranslationUnit_CacheCompletionResults |
-    CXTranslationUnit_PrecompiledPreamble |
-    CXTranslationUnit_Incomplete |
-    CXTranslationUnit_IncludeBriefCommentsInCodeCompletion;
-#if CINDEX_VERSION_MAJOR>0 || (CINDEX_VERSION_MAJOR==0 && CINDEX_VERSION_MINOR>=35)
-  flags |= CXTranslationUnit_KeepGoing;
-#endif
-  return flags;
-}
-
-CodeCompleteResults TranslationUnit::get_code_completions(const std::string &buffer,
-  unsigned line_number, unsigned column) {
   CodeCompleteResults results(cx_tu, buffer, line_number, column);
   return results;
 }

src/libclangmm/TranslationUnit.h

@@ -14,26 +14,16 @@
 namespace clang {
   class TranslationUnit {
   public:
-    //TranslationUnit(Index &index,
-    //                const std::string &file_path,
-    //                const std::vector<std::string> &command_line_args,
-    //                const std::string &buffer,
-    //                unsigned flags=DefaultFlags());
     TranslationUnit(Index &index,
-                    const std::string &file_path,
+                    const std::string &filepath,
-                    const std::vector<std::string> &command_line_args,
+                    const std::vector<std::string>& arguments,
-                    unsigned flags=DefaultFlags());
+                    std::vector<CXUnsavedFile> unsaved_files,
+                    unsigned flags);
     ~TranslationUnit();
     
-    int ReparseTranslationUnit(const std::string &buffer, unsigned flags=DefaultFlags());
+    bool did_fail = false;
-    
+
-    static unsigned DefaultFlags();
+    void ReparseTranslationUnit(std::vector<CXUnsavedFile>& unsaved, unsigned flags);
-    
-    void parse(Index &index,
-               const std::string &file_path,
-               const std::vector<std::string> &command_line_args,
-               const std::map<std::string, std::string> &buffers,
-               unsigned flags=DefaultFlags());
 
     clang::CodeCompleteResults get_code_completions(const std::string &buffer,
                                                     unsigned line_number, unsigned column);

src/message_queue.cc

@@ -161,6 +161,7 @@ MessageQueue::MessageQueue(std::unique_ptr<Buffer> buffer, bool buffer_has_data)
 
   local_buffer_ = Buffer::Create(buffer_->capacity - sizeof(BufferMetadata));
   memset(local_buffer_->data, 0, local_buffer_->capacity);
+
 }
 
 void MessageQueue::Enqueue(const Message& message) {

src/project.cc

@@ -0,0 +1,10 @@
+#include "project.h"
+
+optional<CompilationEntry> Project::FindCompilationEntryForFile(const std::string& filename) {
+  for (auto& entry : entries) {
+    if (filename == entry.filename)
+      return entry;
+  }
+
+  return nullopt;
+}

src/project.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "compilation_database_loader.h" // TODO: merge compilation_database_loader into this file.
+
+#include <optional.h>
+
+using std::experimental::optional;
+using std::experimental::nullopt;
+
+struct Project {
+  std::vector<CompilationEntry> entries;
+
+  optional<CompilationEntry> FindCompilationEntryForFile(const std::string& filename);
+};

src/working_files.cc

@@ -0,0 +1,77 @@
+#include "working_files.h"
+
+WorkingFile::WorkingFile(const std::string& filename, const std::string& content)
+  : filename(filename), content(content) {
+}
+
+CXUnsavedFile WorkingFile::AsUnsavedFile() const {
+  CXUnsavedFile result;
+  result.Filename = filename.c_str();
+  result.Contents = content.c_str();
+  result.Length = content.size();
+  return result;
+}
+
+WorkingFile* WorkingFiles::GetFileByFilename(const std::string& filename) {
+  for (auto& file : files) {
+    if (file->filename == filename)
+      return file.get();
+  }
+  return nullptr;
+}
+
+void WorkingFiles::OnOpen(const Ipc_TextDocumentDidOpen::Params& open) {
+  std::string filename = open.textDocument.uri.GetPath();
+  std::string content = open.textDocument.text;
+
+  // The file may already be open.
+  if (WorkingFile* file = GetFileByFilename(filename)) {
+    file->content = content;
+    return;
+  }
+
+  files.push_back(MakeUnique<WorkingFile>(filename, content));
+}
+
+void WorkingFiles::OnChange(const Ipc_TextDocumentDidChange::Params& change) {
+  std::string filename = change.textDocument.uri.GetPath();
+  WorkingFile* file = GetFileByFilename(filename);
+  if (!file) {
+    std::cerr << "Could not change " << filename << " because it was not open" << std::endl;
+    return;
+  }
+
+  // TODO: we should probably pay attention to versioning.
+
+  for (const Ipc_TextDocumentDidChange::lsTextDocumentContentChangeEvent& diff : change.contentChanges) {
+    // If range or rangeLength are emitted we replace everything, per the spec.
+    if (diff.rangeLength == -1) {
+      file->content = diff.text;
+    }
+    else {
+      file->content.replace(file->content.begin(), file->content.begin() + diff.rangeLength, diff.text);
+    }
+  }
+}
+
+void WorkingFiles::OnClose(const Ipc_TextDocumentDidClose::Params& close) {
+  std::string filename = close.textDocument.uri.GetPath();
+
+  for (int i = 0; i < files.size(); ++i) {
+    if (files[i]->filename == filename) {
+      files.erase(files.begin() + i);
+      return;
+    }
+  }
+
+  std::cerr << "Could not close " << filename << " because it was not open" << std::endl;
+}
+
+std::vector<CXUnsavedFile> WorkingFiles::AsUnsavedFiles() const {
+  std::vector<CXUnsavedFile> result;
+  result.reserve(files.size());
+  for (auto& file : files)
+    result.push_back(file->AsUnsavedFile());
+  return result;
+}
+

src/working_files.h

@@ -2,6 +2,29 @@
 
 #include "language_server_api.h"
 
-struct WorkingFiles {
+#include <clang-c/Index.h>
 
+#include <string>
+
+struct WorkingFile {
+  std::string filename;
+  std::string content;
+
+  WorkingFile(const std::string& filename, const std::string& content);
+
+  CXUnsavedFile AsUnsavedFile() const;
+};
+
+struct WorkingFiles {
+  // Find the file with the given filename.
+  WorkingFile* GetFileByFilename(const std::string& filename);
+  void OnOpen(const Ipc_TextDocumentDidOpen::Params& open);
+  void OnChange(const Ipc_TextDocumentDidChange::Params& change);
+  void OnClose(const Ipc_TextDocumentDidClose::Params& close);
+
+  std::vector<CXUnsavedFile> AsUnsavedFiles() const;
+
+  // Use unique_ptrs so we can handout WorkingFile ptrs and not have them
+  // invalidated if we resize files.
+  std::vector<std::unique_ptr<WorkingFile>> files;
 };