First draft: replace libclang indexer with clangIndex

.clang-format

@@ -1 +1 @@
-BasedOnStyle: Chromium
+BasedOnStyle: LLVM

cmake/FindClang.cmake

@@ -69,6 +69,7 @@ set(_Clang_REQUIRED_VARS Clang_LIBRARY Clang_INCLUDE_DIR Clang_EXECUTABLE
                          LLVM_INCLUDE_DIR LLVM_BUILD_INCLUDE_DIR)
 
 _Clang_find_library(Clang_LIBRARY clang)
+_Clang_find_add_library(clangIndex)
 _Clang_find_add_library(clangFrontend)
 _Clang_find_add_library(clangParse)
 _Clang_find_add_library(clangSerialization)

src/clang_tu.cc

@@ -259,67 +259,6 @@ std::string ClangCursor::get_type_description() const {
   return ::ToString(clang_getTypeSpelling(type));
 }
 
-std::string ClangCursor::get_comments() const {
-  CXSourceRange range = clang_Cursor_getCommentRange(cx_cursor);
-  if (clang_Range_isNull(range))
-    return {};
-
-  unsigned start_column;
-  clang_getSpellingLocation(clang_getRangeStart(range), nullptr, nullptr,
-                            &start_column, nullptr);
-
-  // Get associated comment text.
-  CXString cx_raw = clang_Cursor_getRawCommentText(cx_cursor);
-  int pad = -1;
-  std::string ret;
-  for (const char* p = clang_getCString(cx_raw); *p;) {
-    // The first line starts with a comment marker, but the rest needs
-    // un-indenting.
-    unsigned skip = start_column - 1;
-    for (; skip > 0 && (*p == ' ' || *p == '\t'); p++)
-      skip--;
-    const char* q = p;
-    while (*q != '\n' && *q)
-      q++;
-    if (*q)
-      q++;
-    // A minimalist approach to skip Doxygen comment markers.
-    // See https://www.stack.nl/~dimitri/doxygen/manual/docblocks.html
-    if (pad < 0) {
-      // First line, detect the length of comment marker and put into |pad|
-      const char* begin = p;
-      while (*p == '/' || *p == '*')
-        p++;
-      if (*p == '<' || *p == '!')
-        p++;
-      if (*p == ' ')
-        p++;
-      pad = int(p - begin);
-    } else {
-      // Other lines, skip |pad| bytes
-      int prefix = pad;
-      while (prefix > 0 &&
-             (*p == ' ' || *p == '/' || *p == '*' || *p == '<' || *p == '!'))
-        prefix--, p++;
-    }
-    ret.insert(ret.end(), p, q);
-    p = q;
-  }
-  clang_disposeString(cx_raw);
-  while (ret.size() && isspace(ret.back()))
-    ret.pop_back();
-  if (EndsWith(ret, "*/")) {
-    ret.resize(ret.size() - 2);
-  } else if (EndsWith(ret, "\n/")) {
-    ret.resize(ret.size() - 2);
-  }
-  while (ret.size() && isspace(ret.back()))
-    ret.pop_back();
-  if (ret.empty())
-    return {};
-  return ret;
-}
-
 std::string ClangCursor::ToString() const {
   return ::ToString(get_kind()) + " " + get_spell_name();
 }

src/clang_tu.h

@@ -85,7 +85,6 @@ class ClangCursor {
   bool is_valid_kind() const;
 
   std::string get_type_description() const;
-  std::string get_comments() const;
 
   std::string ToString() const;
 

src/clang_utils.cc

@@ -3,6 +3,7 @@
 #include "platform.h"
 
 #include "filesystem.hh"
+using namespace clang;
 using namespace llvm;
 
 namespace {
@@ -125,6 +126,20 @@ std::string FileName(CXFile file) {
   return ret;
 }
 
+std::string FileName(const FileEntry& file) {
+  StringRef Name = file.tryGetRealPathName();
+  if (Name.empty())
+    Name = file.getName();
+  std::string ret = NormalizePath(Name);
+  // Resolve /usr/include/c++/7.3.0 symlink.
+  if (!StartsWith(ret, g_config->projectRoot)) {
+    SmallString<256> dest;
+    sys::fs::real_path(ret, dest);
+    ret = dest.str();
+  }
+  return ret;
+}
+
 std::string ToString(CXString cx_string) {
   std::string string;
   if (cx_string.data != nullptr) {

src/clang_utils.h

@@ -3,6 +3,7 @@
 #include "lsp_diagnostic.h"
 
 #include <clang-c/Index.h>
+#include <clang/Basic/FileManager.h>
 
 #include <optional>
 #include <vector>
@@ -12,6 +13,7 @@ std::optional<lsDiagnostic> BuildAndDisposeDiagnostic(CXDiagnostic diagnostic,
 
 // Returns the absolute path to |file|.
 std::string FileName(CXFile file);
+std::string FileName(const clang::FileEntry& file);
 
 std::string ToString(CXString cx_string);
 

src/file_consumer.cc

@@ -23,11 +23,6 @@ std::optional<std::string> GetFileContents(
 
 }  // namespace
 
-bool operator==(const CXFileUniqueID& a, const CXFileUniqueID& b) {
-  return a.data[0] == b.data[0] && a.data[1] == b.data[1] &&
-         a.data[2] == b.data[2];
-}
-
 FileContents::FileContents(const std::string& path, const std::string& content)
     : path(path), content(content) {
   line_offsets_.push_back(0);
@@ -98,14 +93,10 @@ FileConsumer::FileConsumer(VFS* vfs, const std::string& parse_file)
     : vfs_(vfs), parse_file_(parse_file), thread_id_(g_thread_id) {}
 
 IndexFile* FileConsumer::TryConsumeFile(
-    CXFile file,
+    const clang::FileEntry& File,
-    bool* is_first_ownership,
     std::unordered_map<std::string, FileContents>* file_contents_map) {
-  assert(is_first_ownership);
+  std::string file_name = FileName(File);
-
+  if (!File.isValid()) {
-  CXFileUniqueID file_id;
-  if (clang_getFileUniqueID(file, &file_id) != 0) {
-    std::string file_name = FileName(file);
     if (!file_name.empty()) {
       LOG_S(ERROR) << "Could not get unique file id for " << file_name
                    << " when parsing " << parse_file_;
@@ -114,33 +105,27 @@ IndexFile* FileConsumer::TryConsumeFile(
   }
 
   // Try to find cached local result.
-  auto it = local_.find(file_id);
+  unsigned UID = File.getUID();
-  if (it != local_.end()) {
+  auto it = local_.find(UID);
-    *is_first_ownership = false;
+  if (it != local_.end())
     return it->second.get();
-  }
-
-  std::string file_name = FileName(file);
 
   // We did not take the file from global. Cache that we failed so we don't try
   // again and return nullptr.
   if (!vfs_->Mark(file_name, thread_id_, 2)) {
-    local_[file_id] = nullptr;
+    local_[UID] = nullptr;
     return nullptr;
   }
 
   // Read the file contents, if we fail then we cannot index the file.
   std::optional<std::string> contents =
       GetFileContents(file_name, file_contents_map);
-  if (!contents) {
+  if (!contents)
-    *is_first_ownership = false;
     return nullptr;
-  }
 
   // Build IndexFile instance.
-  *is_first_ownership = true;
+  local_[UID] = std::make_unique<IndexFile>(UID, file_name, *contents);
-  local_[file_id] = std::make_unique<IndexFile>(file_name, *contents);
+  return local_[UID].get();
-  return local_[file_id].get();
 }
 
 std::vector<std::unique_ptr<IndexFile>> FileConsumer::TakeLocalState() {

src/file_consumer.h

@@ -5,6 +5,7 @@
 #include "utils.h"
 
 #include <clang-c/Index.h>
+#include <clang/Basic/FileManager.h>
 
 #include <functional>
 #include <mutex>
@@ -13,10 +14,6 @@
 
 struct IndexFile;
 
-// Needed for unordered_map usage below.
-MAKE_HASHABLE(CXFileUniqueID, t.data[0], t.data[1], t.data[2]);
-bool operator==(const CXFileUniqueID& a, const CXFileUniqueID& b);
-
 struct FileContents {
   FileContents() = default;
   FileContents(const std::string& path, const std::string& content);
@@ -56,24 +53,20 @@ struct VFS {
 struct FileConsumer {
   FileConsumer(VFS* vfs, const std::string& parse_file);
 
-  // Returns true if this instance owns given |file|. This will also attempt to
-  // take ownership over |file|.
-  //
   // Returns IndexFile for the file or nullptr. |is_first_ownership| is set
   // to true iff the function just took ownership over the file. Otherwise it
   // is set to false.
   //
   // note: file_contents is passed as a parameter instead of as a member
   // variable since it is large and we do not want to copy it.
-  IndexFile* TryConsumeFile(CXFile file,
+  IndexFile* TryConsumeFile(const clang::FileEntry& file,
-                            bool* is_first_ownership,
                             std::unordered_map<std::string, FileContents>* file_contents);
 
   // Returns and passes ownership of all local state.
   std::vector<std::unique_ptr<IndexFile>> TakeLocalState();
 
  private:
-  std::unordered_map<CXFileUniqueID, std::unique_ptr<IndexFile>> local_;
+  std::unordered_map<unsigned, std::unique_ptr<IndexFile>> local_;
   VFS* vfs_;
   std::string parse_file_;
   int thread_id_;

src/indexer.h

@@ -248,6 +248,7 @@ struct IndexFile {
   // files accepted by newer ccls.
   static const int kMinorVersion;
 
+  unsigned UID;
   std::string path;
   std::vector<std::string> args;
   int64_t last_write_time = 0;
@@ -260,7 +261,7 @@ struct IndexFile {
   std::string import_file;
 
   // Source ranges that were not processed.
-  std::vector<Range> skipped_by_preprocessor;
+  std::vector<Range> skipped_ranges;
 
   std::vector<IndexInclude> includes;
   llvm::StringMap<int64_t> dependencies;
@@ -273,7 +274,7 @@ struct IndexFile {
   // File contents at the time of index. Not serialized.
   std::string file_contents;
 
-  IndexFile(const std::string& path, const std::string& contents);
+  IndexFile(unsigned UID, const std::string& path, const std::string& contents);
 
   IndexFunc& ToFunc(Usr usr);
   IndexType& ToType(Usr usr);
@@ -293,14 +294,6 @@ struct NamespaceHelper {
       std::string_view unqualified_name);
 };
 
-std::vector<std::unique_ptr<IndexFile>> ParseWithTu(
-    VFS* vfs,
-    ClangTranslationUnit* tu,
-    ClangIndex* index,
-    const std::string& file,
-    const std::vector<std::string>& args,
-    const std::vector<CXUnsavedFile>& file_contents);
-
 bool ConcatTypeAndName(std::string& type, const std::string& name);
 
 void IndexInit();

src/main.cc

@@ -28,7 +28,7 @@ std::string g_init_options;
 namespace {
 opt<bool> opt_help("h", desc("Alias for -help"));
 opt<int> opt_verbose("v", desc("verbosity"), init(0));
-opt<bool> opt_test_index("test-index", desc("run index tests"));
+opt<std::string> opt_test_index("test-index", ValueOptional, init("!"), desc("run index tests"));
 opt<bool> opt_test_unit("test-unit", desc("run unit tests"));
 
 opt<std::string> opt_init("init", desc("extra initialization options"));
@@ -91,9 +91,9 @@ int main(int argc, char** argv) {
       return res;
   }
 
-  if (opt_test_index) {
+  if (opt_test_index != "!") {
     language_server = false;
-    if (!RunIndexTests("", sys::Process::StandardInIsUserInput()))
+    if (!RunIndexTests(opt_test_index, sys::Process::StandardInIsUserInput()))
       return 1;
   }
 

src/message_handler.cc

@@ -175,11 +175,11 @@ bool FindFileOrFail(DB* db,
   return false;
 }
 
-void EmitInactiveLines(WorkingFile* working_file,
+void EmitSkippedRanges(WorkingFile *working_file,
-                       const std::vector<Range>& inactive_regions) {
+                       const std::vector<Range> &skipped_ranges) {
   Out_CclsSetInactiveRegion out;
   out.params.uri = lsDocumentUri::FromPath(working_file->filename);
-  for (Range skipped : inactive_regions) {
+  for (Range skipped : skipped_ranges) {
     std::optional<lsRange> ls_skipped = GetLsRange(working_file, skipped);
     if (ls_skipped)
       out.params.inactiveRegions.push_back(*ls_skipped);

src/message_handler.h

@@ -141,8 +141,8 @@ bool FindFileOrFail(DB* db,
                     QueryFile** out_query_file,
                     int* out_file_id = nullptr);
 
-void EmitInactiveLines(WorkingFile* working_file,
+void EmitSkippedRanges(WorkingFile *working_file,
-                       const std::vector<Range>& inactive_regions);
+                       const std::vector<Range> &skipped_ranges);
 
 void EmitSemanticHighlighting(DB* db,
                               SemanticHighlightSymbolCache* semantic_cache,

src/messages/ccls_fileInfo.cc

@@ -9,7 +9,7 @@ MAKE_REFLECT_STRUCT(QueryFile::Def,
                     language,
                     outline,
                     all_symbols,
-                    inactive_regions,
+                    skipped_ranges,
                     dependencies);
 
 namespace {
@@ -49,7 +49,7 @@ struct Handler_CclsFileInfo : BaseMessageHandler<In_CclsFileInfo> {
     out.result.args = file->def->args;
     out.result.language = file->def->language;
     out.result.includes = file->def->includes;
-    out.result.inactive_regions = file->def->inactive_regions;
+    out.result.skipped_ranges = file->def->skipped_ranges;
     pipeline::WriteStdout(kMethodType, out);
   }
 };

src/messages/textDocument_didOpen.cc

@@ -45,7 +45,7 @@ struct Handler_TextDocumentDidOpen
     QueryFile* file = nullptr;
     FindFileOrFail(db, project, std::nullopt, path, &file);
     if (file && file->def) {
-      EmitInactiveLines(working_file, file->def->inactive_regions);
+      EmitSkippedRanges(working_file, file->def->skipped_ranges);
       EmitSemanticHighlighting(db, semantic_cache, working_file, file);
     }
 

src/pipeline.cc

@@ -347,7 +347,7 @@ void Main_OnIndexed(DB* db,
     if (WorkingFile* working_file =
             working_files->GetFileByFilename(def_u.first.path)) {
       working_file->SetIndexContent(def_u.second);
-      EmitInactiveLines(working_file, def_u.first.inactive_regions);
+      EmitSkippedRanges(working_file, def_u.first.skipped_ranges);
       EmitSemanticHighlighting(db, semantic_cache, working_file,
                                &db->files[update->file_id]);
     }

src/query.cc

@@ -73,7 +73,7 @@ QueryFile::DefUpdate BuildFileDefUpdate(const IndexFile& indexed) {
   def.path = std::move(indexed.path);
   def.args = std::move(indexed.args);
   def.includes = std::move(indexed.includes);
-  def.inactive_regions = std::move(indexed.skipped_by_preprocessor);
+  def.skipped_ranges = std::move(indexed.skipped_ranges);
   def.dependencies.reserve(indexed.dependencies.size());
   for (auto& dep : indexed.dependencies)
     def.dependencies.push_back(dep.first());
@@ -172,7 +172,7 @@ bool TryReplaceDef(llvm::SmallVectorImpl<Q>& def_list, Q&& def) {
 IndexUpdate IndexUpdate::CreateDelta(IndexFile* previous,
                                      IndexFile* current) {
   IndexUpdate r;
-  static IndexFile empty(current->path, "<empty>");
+  static IndexFile empty(0u, current->path, "<empty>");
   if (!previous)
     previous = &empty;
 

src/query.h

@@ -19,7 +19,7 @@ struct QueryFile {
     // Every symbol found in the file (ie, for goto definition)
     std::vector<SymbolRef> all_symbols;
     // Parts of the file which are disabled.
-    std::vector<Range> inactive_regions;
+    std::vector<Range> skipped_ranges;
     // Used by |$ccls/freshenIndex|.
     std::vector<std::string> dependencies;
   };

src/serializer.cc

@@ -335,7 +335,7 @@ void Reflect(TVisitor& visitor, IndexFile& value) {
     REFLECT_MEMBER(dependencies);
   }
   REFLECT_MEMBER(includes);
-  REFLECT_MEMBER(skipped_by_preprocessor);
+  REFLECT_MEMBER(skipped_ranges);
   REFLECT_MEMBER(usr2func);
   REFLECT_MEMBER(usr2type);
   REFLECT_MEMBER(usr2var);
@@ -426,7 +426,7 @@ std::unique_ptr<IndexFile> Deserialize(
         if (major != IndexFile::kMajorVersion ||
             minor != IndexFile::kMinorVersion)
           throw std::invalid_argument("Invalid version");
-        file = std::make_unique<IndexFile>(path, file_content);
+        file = std::make_unique<IndexFile>(0u, path, file_content);
         Reflect(reader, *file);
       } catch (std::invalid_argument& e) {
         LOG_S(INFO) << "failed to deserialize '" << path
@@ -450,7 +450,7 @@ std::unique_ptr<IndexFile> Deserialize(
       if (reader.HasParseError())
         return nullptr;
 
-      file = std::make_unique<IndexFile>(path, file_content);
+      file = std::make_unique<IndexFile>(0u, path, file_content);
       JsonReader json_reader{&reader};
       try {
         Reflect(json_reader, *file);

src/test.cc

@@ -307,8 +307,7 @@ bool RunIndexTests(const std::string& filter_path, bool enable_update) {
 
           // Get output from index operation.
           IndexFile* db = FindDbForPathEnding(expected_path, dbs);
-          assert(db);
+          if (db && !db->diagnostics_.empty()) {
-          if (!db->diagnostics_.empty()) {
             printf("For %s\n", path.c_str());
             for (const lsDiagnostic& diagnostic : db->diagnostics_) {
               printf("  ");