From 503ac951c024d6d59b0bd59fd5c7838f9c0454be Mon Sep 17 00:00:00 2001
From: Jan Lahoda <jlahoda@netbeans.org>
Date: Fri, 25 Dec 2020 11:22:07 +0100
Subject: [PATCH 01/24] First attempt to include the semantic highlight
 feature, introduced in specification 3.16.0.

---
 CMakeLists.txt                             |   1 +
 src/message_handler.cc                     |   1 +
 src/message_handler.hh                     |   5 +
 src/messages/initialize.cc                 |  52 ++++-
 src/messages/textDocument_semanticToken.cc | 259 +++++++++++++++++++++
 5 files changed, 317 insertions(+), 1 deletion(-)
 create mode 100644 src/messages/textDocument_semanticToken.cc

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b3a02386..78897797 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -235,6 +235,7 @@ target_sources(ccls PRIVATE
   src/messages/textDocument_hover.cc
   src/messages/textDocument_references.cc
   src/messages/textDocument_rename.cc
+  src/messages/textDocument_semanticToken.cc
   src/messages/textDocument_signatureHelp.cc
   src/messages/workspace.cc
 )
diff --git a/src/message_handler.cc b/src/message_handler.cc
index 498a72de..2f86fb71 100644
--- a/src/message_handler.cc
+++ b/src/message_handler.cc
@@ -188,6 +188,7 @@ MessageHandler::MessageHandler() {
   bind("textDocument/rename", &MessageHandler::textDocument_rename);
   bind("textDocument/signatureHelp", &MessageHandler::textDocument_signatureHelp);
   bind("textDocument/typeDefinition", &MessageHandler::textDocument_typeDefinition);
+  bind("textDocument/semanticTokens/full", &MessageHandler::textDocument_semanticTokensFull);
   bind("workspace/didChangeConfiguration", &MessageHandler::workspace_didChangeConfiguration);
   bind("workspace/didChangeWatchedFiles", &MessageHandler::workspace_didChangeWatchedFiles);
   bind("workspace/didChangeWorkspaceFolders", &MessageHandler::workspace_didChangeWorkspaceFolders);
diff --git a/src/message_handler.hh b/src/message_handler.hh
index 7718e346..27c684e9 100644
--- a/src/message_handler.hh
+++ b/src/message_handler.hh
@@ -48,6 +48,10 @@ struct TextDocumentPositionParam {
   TextDocumentIdentifier textDocument;
   Position position;
 };
+struct SemanticTokensParams {
+  TextDocumentIdentifier textDocument;
+};
+REFLECT_STRUCT(SemanticTokensParams, textDocument);
 struct TextDocumentEdit {
   VersionedTextDocumentIdentifier textDocument;
   std::vector<TextEdit> edits;
@@ -287,6 +291,7 @@ private:
   void textDocument_rename(RenameParam &, ReplyOnce &);
   void textDocument_signatureHelp(TextDocumentPositionParam &, ReplyOnce &);
   void textDocument_typeDefinition(TextDocumentPositionParam &, ReplyOnce &);
+  void textDocument_semanticTokensFull(SemanticTokensParams &, ReplyOnce &);
   void workspace_didChangeConfiguration(EmptyParam &);
   void workspace_didChangeWatchedFiles(DidChangeWatchedFilesParam &);
   void workspace_didChangeWorkspaceFolders(DidChangeWorkspaceFoldersParam &);
diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc
index 1601db70..cee98460 100644
--- a/src/messages/initialize.cc
+++ b/src/messages/initialize.cc
@@ -25,6 +25,47 @@
 namespace ccls {
 using namespace llvm;
 
+std::vector<const char *> SEMANTIC_TOKENS = {
+  "unknown",
+
+  "file",
+  "module",
+  "namespace",
+  "package",
+  "class",
+  "method",
+  "property",
+  "field",
+  "constructor",
+  "enum",
+  "interface",
+  "function",
+  "variable",
+  "constant",
+  "string",
+  "number",
+  "boolean",
+  "array",
+  "object",
+  "key",
+  "null",
+  "enumMember",
+  "struct",
+  "event",
+  "operator",
+  "typeParameter",
+  "typeAlias", //252 => 27
+  "parameter",
+  "staticMethod",
+  "macro"
+};
+
+std::vector<const char *> SEMANTIC_MODIFIERS = {
+    "declaration", //1
+    "definition",  //2
+    "static"       //4
+};
+
 extern std::vector<std::string> g_init_options;
 
 namespace {
@@ -89,6 +130,13 @@ struct ServerCap {
     std::vector<const char *> commands = {ccls_xref};
   } executeCommandProvider;
   Config::ServerCap::Workspace workspace;
+  struct SemanticTokenProvider {
+      struct SemanticTokensLegend {
+          std::vector<const char *> tokenTypes = SEMANTIC_TOKENS;
+          std::vector<const char *> tokenModifiers = SEMANTIC_MODIFIERS;
+      } legend;
+      bool full = true;
+  } semanticTokensProvider;
 };
 REFLECT_STRUCT(ServerCap::CodeActionOptions, codeActionKinds);
 REFLECT_STRUCT(ServerCap::CodeLensOptions, resolveProvider);
@@ -109,7 +157,9 @@ REFLECT_STRUCT(ServerCap, textDocumentSync, hoverProvider, completionProvider,
                documentRangeFormattingProvider,
                documentOnTypeFormattingProvider, renameProvider,
                documentLinkProvider, foldingRangeProvider,
-               executeCommandProvider, workspace);
+               executeCommandProvider, workspace, semanticTokensProvider);
+REFLECT_STRUCT(ServerCap::SemanticTokenProvider, legend, full);
+REFLECT_STRUCT(ServerCap::SemanticTokenProvider::SemanticTokensLegend, tokenTypes, tokenModifiers);
 
 struct DynamicReg {
   bool dynamicRegistration = false;
diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
new file mode 100644
index 00000000..7a9e94b7
--- /dev/null
+++ b/src/messages/textDocument_semanticToken.cc
@@ -0,0 +1,259 @@
+// Copyright 2017-2020 ccls Authors
+// SPDX-License-Identifier: Apache-2.0
+
+#include "indexer.hh"
+#include "message_handler.hh"
+#include "pipeline.hh"
+#include "sema_manager.hh"
+
+#include <clang/Sema/Sema.h>
+#include <algorithm>
+#include <stdexcept>
+#include <string>
+
+#include <iostream>
+
+MAKE_HASHABLE(ccls::SymbolIdx, t.usr, t.kind);
+
+namespace ccls {
+using namespace clang;
+
+namespace {
+struct SemanticTokens {
+  std::vector<int> data;
+};
+REFLECT_STRUCT(SemanticTokens, data);
+
+struct CclsSemanticHighlightSymbol {
+  int id = 0;
+  SymbolKind parentKind;
+  SymbolKind kind;
+  uint8_t storage;
+  std::vector<std::pair<lsRange, Role>> lsRangeAndRoles;
+};
+
+struct ScanLineEvent {
+  Position pos;
+  Position end_pos; // Second key when there is a tie for insertion events.
+  int id;
+  CclsSemanticHighlightSymbol *symbol;
+  Role role;
+  bool operator<(const ScanLineEvent &o) const {
+    // See the comments below when insertion/deletion events are inserted.
+    if (!(pos == o.pos))
+      return pos < o.pos;
+    if (!(o.end_pos == end_pos))
+      return o.end_pos < end_pos;
+    // This comparison essentially order Macro after non-Macro,
+    // So that macros will not be rendered as Var/Type/...
+    if (symbol->kind != o.symbol->kind)
+      return symbol->kind < o.symbol->kind;
+    // If symbol A and B occupy the same place, we want one to be placed
+    // before the other consistantly.
+    return symbol->id < o.symbol->id;
+  }
+};
+
+}
+
+void MessageHandler::textDocument_semanticTokensFull(
+        SemanticTokensParams &param, ReplyOnce &reply) {
+  std::string path = param.textDocument.uri.getPath();
+  WorkingFile *wfile = wfiles->getFile(path);
+  if (!wfile) {
+    reply.notOpened(path);
+    return;
+  }
+
+  QueryFile *file = findFile(path);
+  if (!file) {
+    reply.notOpened(path); //TODO
+    return;
+  }
+
+  SemanticTokens result;
+
+  static GroupMatch match(g_config->highlight.whitelist,
+                          g_config->highlight.blacklist);
+  assert(file->def);
+  if (wfile->buffer_content.size() > g_config->highlight.largeFileSize ||
+      !match.matches(file->def->path)) {
+    reply.notOpened(path); //TODO
+    return; //TODO?
+  }
+
+  // Group symbols together.
+  std::unordered_map<SymbolIdx, CclsSemanticHighlightSymbol> grouped_symbols;
+  for (auto [sym, refcnt] : file->symbol2refcnt) {
+    if (refcnt <= 0) //TODO: unused?
+      continue;
+    std::string_view detailed_name;
+    SymbolKind parent_kind = SymbolKind::Unknown;
+    SymbolKind kind = SymbolKind::Unknown;
+    uint8_t storage = SC_None;
+    int idx;
+    // This switch statement also filters out symbols that are not highlighted.
+    switch (sym.kind) {
+    case Kind::Func: {
+      idx = db->func_usr[sym.usr];
+      const QueryFunc &func = db->funcs[idx];
+      const QueryFunc::Def *def = func.anyDef();
+      if (!def)
+        continue; // applies to for loop
+      // Don't highlight overloadable operators or implicit lambda ->
+      // std::function constructor.
+      std::string_view short_name = def->name(false);
+      if (short_name.compare(0, 8, "operator") == 0)
+        continue; // applies to for loop
+      kind = def->kind;
+      storage = def->storage;
+      detailed_name = short_name;
+      parent_kind = def->parent_kind;
+
+      // Check whether the function name is actually there.
+      // If not, do not publish the semantic highlight.
+      // E.g. copy-initialization of constructors should not be highlighted
+      // but we still want to keep the range for jumping to definition.
+      std::string_view concise_name =
+          detailed_name.substr(0, detailed_name.find('<'));
+      uint16_t start_line = sym.range.start.line;
+      int16_t start_col = sym.range.start.column;
+      if (start_line >= wfile->index_lines.size())
+        continue;
+      std::string_view line = wfile->index_lines[start_line];
+      sym.range.end.line = start_line;
+      if (!(start_col + concise_name.size() <= line.size() &&
+            line.compare(start_col, concise_name.size(), concise_name) == 0))
+        continue;
+      sym.range.end.column = start_col + concise_name.size();
+      break;
+    }
+    case Kind::Type: {
+      idx = db->type_usr[sym.usr];
+      const QueryType &type = db->types[idx];
+      for (auto &def : type.def) {
+        kind = def.kind;
+        detailed_name = def.detailed_name;
+        if (def.spell) {
+          parent_kind = def.parent_kind;
+          break;
+        }
+      }
+      break;
+    }
+    case Kind::Var: {
+      idx = db->var_usr[sym.usr];
+      const QueryVar &var = db->vars[idx];
+      for (auto &def : var.def) {
+        kind = def.kind;
+        storage = def.storage;
+        detailed_name = def.detailed_name;
+        if (def.spell) {
+          parent_kind = def.parent_kind;
+          break;
+        }
+      }
+      break;
+    }
+    default:
+      continue; // applies to for loop
+    }
+
+    if (std::optional<lsRange> loc = getLsRange(wfile, sym.range)) {
+      auto it = grouped_symbols.find(sym);
+      if (it != grouped_symbols.end()) {
+        it->second.lsRangeAndRoles.push_back({*loc, sym.role});
+      } else {
+        CclsSemanticHighlightSymbol symbol;
+        symbol.id = idx;
+        symbol.parentKind = parent_kind;
+        symbol.kind = kind;
+        symbol.storage = storage;
+        symbol.lsRangeAndRoles.push_back({*loc, sym.role});
+        grouped_symbols[sym] = symbol;
+      }
+    }
+  }
+
+  // Make ranges non-overlapping using a scan line algorithm.
+  std::vector<ScanLineEvent> events;
+  int id = 0;
+  for (auto &entry : grouped_symbols) {
+    CclsSemanticHighlightSymbol &symbol = entry.second;
+    for (auto &loc : symbol.lsRangeAndRoles) {
+      // For ranges sharing the same start point, the one with leftmost end
+      // point comes first.
+      events.push_back({loc.first.start, loc.first.end, id, &symbol, loc.second});
+      // For ranges sharing the same end point, their relative order does not
+      // matter, therefore we arbitrarily assign loc.end to them. We use
+      // negative id to indicate a deletion event.
+      events.push_back({loc.first.end, loc.first.end, ~id, &symbol, loc.second});
+      id++;
+    }
+    symbol.lsRangeAndRoles.clear();
+  }
+  std::sort(events.begin(), events.end());
+
+  std::vector<uint8_t> deleted(id, 0);
+  int top = 0;
+  for (size_t i = 0; i < events.size(); i++) {
+    while (top && deleted[events[top - 1].id])
+      top--;
+    // Order [a, b0) after [a, b1) if b0 < b1. The range comes later overrides
+    // the ealier. The order of [a0, b) [a1, b) does not matter.
+    // The order of [a, b) [b, c) does not as long as we do not emit empty
+    // ranges.
+    // Attribute range [events[i-1].pos, events[i].pos) to events[top-1].symbol
+    // .
+    if (top && !(events[i - 1].pos == events[i].pos))
+      events[top - 1].symbol->lsRangeAndRoles.push_back(
+          {{events[i - 1].pos, events[i].pos}, events[i].role});
+    if (events[i].id >= 0)
+      events[top++] = events[i];
+    else
+      deleted[~events[i].id] = 1;
+  }
+
+  // Transform lsRange into pair<int, int> (offset pairs)
+    std::vector<std::pair<std::pair<lsRange, Role>, CclsSemanticHighlightSymbol *>> scratch;
+    for (auto &entry : grouped_symbols) {
+      for (auto &range : entry.second.lsRangeAndRoles)
+        scratch.emplace_back(range, &entry.second);
+      entry.second.lsRangeAndRoles.clear();
+    }
+    std::sort(scratch.begin(), scratch.end(),
+              [](auto &l, auto &r) { return l.first.first.start < r.first.first.start; });
+    int line = 0;
+    int column = 0;
+    for (auto &entry : scratch) {
+      lsRange &r = entry.first.first;
+      if (r.start.line != line) {
+        column = 0;
+      }
+      result.data.push_back(r.start.line - line); line = r.start.line;
+      result.data.push_back(r.start.character - column); column = r.start.character;
+      result.data.push_back(r.end.character - r.start.character);
+      uint8_t kindId;
+      int modifiers = entry.second->storage == SC_Static ? 4 : 0;
+      if (entry.first.second & Role::Declaration) {
+          modifiers |= 1;
+      }
+      if (entry.first.second & Role::Definition) {
+          modifiers |= 2;
+      }
+      if (entry.second->kind == SymbolKind::StaticMethod) {
+          kindId = (uint8_t) SymbolKind::Method;
+          modifiers = 4;
+      } else {
+          kindId = (uint8_t) entry.second->kind;
+          if (kindId > (uint8_t) SymbolKind::StaticMethod)
+              kindId--;
+          if (kindId >= 252) kindId = 27 + kindId - 252;
+      }
+      result.data.push_back(kindId);
+      result.data.push_back(modifiers);
+    }
+
+  reply(result);
+}
+} // namespace ccls

From 013b655e4c396e12bdb11373a4c2231e251b1d8e Mon Sep 17 00:00:00 2001
From: Jan Lahoda <jlahoda@netbeans.org>
Date: Sat, 26 Dec 2020 22:29:31 +0100
Subject: [PATCH 02/24] Removing unnecessary include.

---
 src/messages/textDocument_semanticToken.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 7a9e94b7..5d3631ef 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -11,8 +11,6 @@
 #include <stdexcept>
 #include <string>
 
-#include <iostream>
-
 MAKE_HASHABLE(ccls::SymbolIdx, t.usr, t.kind);
 
 namespace ccls {

From a4de46b9a0033a712a203cf8476a18c216a5a998 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 12 Apr 2021 14:31:45 -0400
Subject: [PATCH 03/24] name types for (trace|read)ability

---
 src/position.hh | 6 ++++--
 src/query.hh    | 5 ++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/position.hh b/src/position.hh
index e59804d2..fd953cb2 100644
--- a/src/position.hh
+++ b/src/position.hh
@@ -10,8 +10,10 @@
 
 namespace ccls {
 struct Pos {
-  uint16_t line = 0;
-  int16_t column = -1;
+  using Line=uint16_t;
+  Line line = 0;
+  using Column=int16_t;
+  Column column = -1;
 
   static Pos fromString(const std::string &encoded);
 
diff --git a/src/query.hh b/src/query.hh
index cd3a19c1..dee5ff52 100644
--- a/src/query.hh
+++ b/src/query.hh
@@ -146,7 +146,10 @@ using Lid2file_id = std::unordered_map<int, int>;
 struct DB {
   std::vector<QueryFile> files;
   llvm::StringMap<int> name2file_id;
-  llvm::DenseMap<Usr, int, DenseMapInfoForUsr> func_usr, type_usr, var_usr;
+  //! Underlying type used for indexes-of-Usr
+  using UsrIndex=int;
+  //! Usr → index
+  llvm::DenseMap<Usr, UsrIndex, DenseMapInfoForUsr> func_usr, type_usr, var_usr;
   llvm::SmallVector<QueryFunc, 0> funcs;
   llvm::SmallVector<QueryType, 0> types;
   llvm::SmallVector<QueryVar, 0> vars;

From ca51f9eb4a576f6b5eaf636936f0bfe958dc9b6e Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 12 Apr 2021 14:32:54 -0400
Subject: [PATCH 04/24] use tuple for lexicographical comparison

---
 src/position.hh | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/position.hh b/src/position.hh
index fd953cb2..250cdead 100644
--- a/src/position.hh
+++ b/src/position.hh
@@ -23,14 +23,23 @@ struct Pos {
   // Compare two Positions and check if they are equal. Ignores the value of
   // |interesting|.
   bool operator==(const Pos &o) const {
-    return line == o.line && column == o.column;
+    return asTuple() == o.asTuple();
   }
   bool operator<(const Pos &o) const {
-    if (line != o.line)
-      return line < o.line;
-    return column < o.column;
+    return asTuple() < o.asTuple();
   }
-  bool operator<=(const Pos &o) const { return !(o < *this); }
+  bool operator<=(const Pos &o) const {
+    return asTuple() <= o.asTuple();
+  }
+protected:
+  /*!
+   * (line, pos)
+   * use for lexicographic comparison
+   */
+  auto asTuple() const -> std::tuple<Line,Column> {
+    return std::make_tuple(line, tuple);
+  }
+
 };
 
 struct Range {

From 554833abf22a033101bd4e8109b7845a058a8136 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 12 Apr 2021 14:33:19 -0400
Subject: [PATCH 05/24] handle TODOs, imprve readbility

---
 src/messages/textDocument_semanticToken.cc | 116 +++++++++++----------
 1 file changed, 59 insertions(+), 57 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 5d3631ef..2c8cd740 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -23,7 +23,8 @@ struct SemanticTokens {
 REFLECT_STRUCT(SemanticTokens, data);
 
 struct CclsSemanticHighlightSymbol {
-  int id = 0;
+  using Id=int;
+  Id id = 0;
   SymbolKind parentKind;
   SymbolKind kind;
   uint8_t storage;
@@ -33,7 +34,8 @@ struct CclsSemanticHighlightSymbol {
 struct ScanLineEvent {
   Position pos;
   Position end_pos; // Second key when there is a tie for insertion events.
-  int id;
+  using Id=int;
+  Id id;
   CclsSemanticHighlightSymbol *symbol;
   Role role;
   bool operator<(const ScanLineEvent &o) const {
@@ -63,9 +65,9 @@ void MessageHandler::textDocument_semanticTokensFull(
     return;
   }
 
-  QueryFile *file = findFile(path);
-  if (!file) {
-    reply.notOpened(path); //TODO
+  auto [queryFile,wFile] = findOrFail(path, reply);
+  if (!queryFile) {
+    // `findOrFail` already set the reply message
     return;
   }
 
@@ -75,21 +77,21 @@ void MessageHandler::textDocument_semanticTokensFull(
                           g_config->highlight.blacklist);
   assert(file->def);
   if (wfile->buffer_content.size() > g_config->highlight.largeFileSize ||
-      !match.matches(file->def->path)) {
-    reply.notOpened(path); //TODO
-    return; //TODO?
+      !match.matches(queryFile->def->path)) {
+    LOG_V(INFO) << "Not SemTokenizing " << path << "because of allowlist/denylist";
+    return;
   }
 
   // Group symbols together.
   std::unordered_map<SymbolIdx, CclsSemanticHighlightSymbol> grouped_symbols;
-  for (auto [sym, refcnt] : file->symbol2refcnt) {
-    if (refcnt <= 0) //TODO: unused?
+  for (auto [sym, refcnt] : queryFile->symbol2refcnt) {
+    if (refcnt <= 0)
       continue;
     std::string_view detailed_name;
     SymbolKind parent_kind = SymbolKind::Unknown;
     SymbolKind kind = SymbolKind::Unknown;
     uint8_t storage = SC_None;
-    int idx;
+    DB::UsrIndex idx;
     // This switch statement also filters out symbols that are not highlighted.
     switch (sym.kind) {
     case Kind::Func: {
@@ -100,7 +102,7 @@ void MessageHandler::textDocument_semanticTokensFull(
         continue; // applies to for loop
       // Don't highlight overloadable operators or implicit lambda ->
       // std::function constructor.
-      std::string_view short_name = def->name(false);
+      const auto short_name = def->name(false);
       if (short_name.compare(0, 8, "operator") == 0)
         continue; // applies to for loop
       kind = def->kind;
@@ -112,14 +114,14 @@ void MessageHandler::textDocument_semanticTokensFull(
       // If not, do not publish the semantic highlight.
       // E.g. copy-initialization of constructors should not be highlighted
       // but we still want to keep the range for jumping to definition.
-      std::string_view concise_name =
+      const auto concise_name =
           detailed_name.substr(0, detailed_name.find('<'));
-      uint16_t start_line = sym.range.start.line;
-      int16_t start_col = sym.range.start.column;
-      if (start_line >= wfile->index_lines.size())
+      const auto start_line_idx = sym.range.start.line;
+      const auto start_col = sym.range.start.column;
+      if (start_line_idx >= wfile->index_lines.size()) // out-of-range ?
         continue;
-      std::string_view line = wfile->index_lines[start_line];
-      sym.range.end.line = start_line;
+      const auto line = wfile->index_lines[start_line_idx];
+      sym.range.end.line = start_line_idx;
       if (!(start_col + concise_name.size() <= line.size() &&
             line.compare(start_col, concise_name.size(), concise_name) == 0))
         continue;
@@ -157,10 +159,10 @@ void MessageHandler::textDocument_semanticTokensFull(
       continue; // applies to for loop
     }
 
-    if (std::optional<lsRange> loc = getLsRange(wfile, sym.range)) {
+    if (auto maybe_loc = getLsRange(wfile, sym.range)) {
       auto it = grouped_symbols.find(sym);
       if (it != grouped_symbols.end()) {
-        it->second.lsRangeAndRoles.push_back({*loc, sym.role});
+        it->second.lsRangeAndRoles.push_back({*maybe_loc, sym.role});
       } else {
         CclsSemanticHighlightSymbol symbol;
         symbol.id = idx;
@@ -175,7 +177,7 @@ void MessageHandler::textDocument_semanticTokensFull(
 
   // Make ranges non-overlapping using a scan line algorithm.
   std::vector<ScanLineEvent> events;
-  int id = 0;
+  ScanLineEvent::Id id = 0;
   for (auto &entry : grouped_symbols) {
     CclsSemanticHighlightSymbol &symbol = entry.second;
     for (auto &loc : symbol.lsRangeAndRoles) {
@@ -213,44 +215,44 @@ void MessageHandler::textDocument_semanticTokensFull(
   }
 
   // Transform lsRange into pair<int, int> (offset pairs)
-    std::vector<std::pair<std::pair<lsRange, Role>, CclsSemanticHighlightSymbol *>> scratch;
-    for (auto &entry : grouped_symbols) {
-      for (auto &range : entry.second.lsRangeAndRoles)
-        scratch.emplace_back(range, &entry.second);
-      entry.second.lsRangeAndRoles.clear();
+  std::vector<std::pair<std::pair<lsRange, Role>, CclsSemanticHighlightSymbol *>> scratch;
+  for (auto &entry : grouped_symbols) {
+    for (auto &range : entry.second.lsRangeAndRoles)
+      scratch.emplace_back(range, &entry.second);
+    entry.second.lsRangeAndRoles.clear();
+  }
+  std::sort(scratch.begin(), scratch.end(),
+      [](auto &l, auto &r) { return l.first.first.start < r.first.first.start; });
+  int line = 0;
+  int column = 0;
+  for (auto &entry : scratch) {
+    lsRange &r = entry.first.first;
+    if (r.start.line != line) {
+      column = 0;
     }
-    std::sort(scratch.begin(), scratch.end(),
-              [](auto &l, auto &r) { return l.first.first.start < r.first.first.start; });
-    int line = 0;
-    int column = 0;
-    for (auto &entry : scratch) {
-      lsRange &r = entry.first.first;
-      if (r.start.line != line) {
-        column = 0;
-      }
-      result.data.push_back(r.start.line - line); line = r.start.line;
-      result.data.push_back(r.start.character - column); column = r.start.character;
-      result.data.push_back(r.end.character - r.start.character);
-      uint8_t kindId;
-      int modifiers = entry.second->storage == SC_Static ? 4 : 0;
-      if (entry.first.second & Role::Declaration) {
-          modifiers |= 1;
-      }
-      if (entry.first.second & Role::Definition) {
-          modifiers |= 2;
-      }
-      if (entry.second->kind == SymbolKind::StaticMethod) {
-          kindId = (uint8_t) SymbolKind::Method;
-          modifiers = 4;
-      } else {
-          kindId = (uint8_t) entry.second->kind;
-          if (kindId > (uint8_t) SymbolKind::StaticMethod)
-              kindId--;
-          if (kindId >= 252) kindId = 27 + kindId - 252;
-      }
-      result.data.push_back(kindId);
-      result.data.push_back(modifiers);
+    result.data.push_back(r.start.line - line); line = r.start.line;
+    result.data.push_back(r.start.character - column); column = r.start.character;
+    result.data.push_back(r.end.character - r.start.character);
+    uint8_t kindId;
+    int modifiers = entry.second->storage == SC_Static ? 4 : 0;
+    if (entry.first.second & Role::Declaration) {
+      modifiers |= 1;
     }
+    if (entry.first.second & Role::Definition) {
+      modifiers |= 2;
+    }
+    if (entry.second->kind == SymbolKind::StaticMethod) {
+      kindId = (uint8_t) SymbolKind::Method;
+      modifiers = 4;
+    } else {
+      kindId = (uint8_t) entry.second->kind;
+      if (kindId > (uint8_t) SymbolKind::StaticMethod)
+	kindId--;
+      if (kindId >= 252) kindId = 27 + kindId - 252;
+    }
+    result.data.push_back(kindId);
+    result.data.push_back(modifiers);
+  }
 
   reply(result);
 }

From c9860305f0e43b6c2b77d7dac885ca857f8126a9 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 12 Apr 2021 14:47:33 -0400
Subject: [PATCH 06/24] fix type

---
 src/position.hh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/position.hh b/src/position.hh
index 250cdead..a227ae26 100644
--- a/src/position.hh
+++ b/src/position.hh
@@ -37,7 +37,7 @@ protected:
    * use for lexicographic comparison
    */
   auto asTuple() const -> std::tuple<Line,Column> {
-    return std::make_tuple(line, tuple);
+    return std::make_tuple(line, column);
   }
 
 };

From 661870a5028b74d04699035b78f7d4515923d0b1 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 12 Apr 2021 14:47:48 -0400
Subject: [PATCH 07/24] fix compilation

---
 src/messages/textDocument_semanticToken.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 2c8cd740..c51f9013 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "indexer.hh"
+#include "log.hh"
 #include "message_handler.hh"
 #include "pipeline.hh"
 #include "sema_manager.hh"
@@ -78,7 +79,7 @@ void MessageHandler::textDocument_semanticTokensFull(
   assert(file->def);
   if (wfile->buffer_content.size() > g_config->highlight.largeFileSize ||
       !match.matches(queryFile->def->path)) {
-    LOG_V(INFO) << "Not SemTokenizing " << path << "because of allowlist/denylist";
+    LOG_S(INFO) << "Not SemTokenizing " << path << "because of allowlist/denylist";
     return;
   }
 
@@ -161,15 +162,16 @@ void MessageHandler::textDocument_semanticTokensFull(
 
     if (auto maybe_loc = getLsRange(wfile, sym.range)) {
       auto it = grouped_symbols.find(sym);
+      const auto &loc = *maybe_loc;
       if (it != grouped_symbols.end()) {
-        it->second.lsRangeAndRoles.push_back({*maybe_loc, sym.role});
+        it->second.lsRangeAndRoles.push_back({loc, sym.role});
       } else {
         CclsSemanticHighlightSymbol symbol;
         symbol.id = idx;
         symbol.parentKind = parent_kind;
         symbol.kind = kind;
         symbol.storage = storage;
-        symbol.lsRangeAndRoles.push_back({*loc, sym.role});
+        symbol.lsRangeAndRoles.push_back({loc, sym.role});
         grouped_symbols[sym] = symbol;
       }
     }

From 84a00f637126afee0fe09e8e23fd6773055293ee Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 3 May 2021 12:51:04 -0400
Subject: [PATCH 08/24] textDocument/semanticTokens/range

- add overhead code for new method
- move code from "semantic tokens for full document" to
  "semantic tokens for range in document"
- add range delimitation to function
- make "full document" use "range that covers all document"
---
 src/message_handler.cc                     |  1 +
 src/message_handler.hh                     |  6 ++++
 src/messages/initialize.cc                 |  3 +-
 src/messages/textDocument_semanticToken.cc | 42 ++++++++++++++++++++--
 src/query.hh                               |  5 +--
 5 files changed, 52 insertions(+), 5 deletions(-)

diff --git a/src/message_handler.cc b/src/message_handler.cc
index 2f86fb71..6546722a 100644
--- a/src/message_handler.cc
+++ b/src/message_handler.cc
@@ -189,6 +189,7 @@ MessageHandler::MessageHandler() {
   bind("textDocument/signatureHelp", &MessageHandler::textDocument_signatureHelp);
   bind("textDocument/typeDefinition", &MessageHandler::textDocument_typeDefinition);
   bind("textDocument/semanticTokens/full", &MessageHandler::textDocument_semanticTokensFull);
+  bind("textDocument/semanticTokens/range", &MessageHandler::textDocument_semanticTokensRange);
   bind("workspace/didChangeConfiguration", &MessageHandler::workspace_didChangeConfiguration);
   bind("workspace/didChangeWatchedFiles", &MessageHandler::workspace_didChangeWatchedFiles);
   bind("workspace/didChangeWorkspaceFolders", &MessageHandler::workspace_didChangeWorkspaceFolders);
diff --git a/src/message_handler.hh b/src/message_handler.hh
index 27c684e9..f4149b61 100644
--- a/src/message_handler.hh
+++ b/src/message_handler.hh
@@ -52,6 +52,11 @@ struct SemanticTokensParams {
   TextDocumentIdentifier textDocument;
 };
 REFLECT_STRUCT(SemanticTokensParams, textDocument);
+struct SemanticTokensRangeParams {
+  TextDocumentIdentifier textDocument;
+  lsRange range;
+};
+REFLECT_STRUCT(SemanticTokensRangeParams, textDocument, range);
 struct TextDocumentEdit {
   VersionedTextDocumentIdentifier textDocument;
   std::vector<TextEdit> edits;
@@ -292,6 +297,7 @@ private:
   void textDocument_signatureHelp(TextDocumentPositionParam &, ReplyOnce &);
   void textDocument_typeDefinition(TextDocumentPositionParam &, ReplyOnce &);
   void textDocument_semanticTokensFull(SemanticTokensParams &, ReplyOnce &);
+  void textDocument_semanticTokensRange(SemanticTokensRangeParams &, ReplyOnce &);
   void workspace_didChangeConfiguration(EmptyParam &);
   void workspace_didChangeWatchedFiles(DidChangeWatchedFilesParam &);
   void workspace_didChangeWorkspaceFolders(DidChangeWorkspaceFoldersParam &);
diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc
index cee98460..5ad5ef47 100644
--- a/src/messages/initialize.cc
+++ b/src/messages/initialize.cc
@@ -135,6 +135,7 @@ struct ServerCap {
           std::vector<const char *> tokenTypes = SEMANTIC_TOKENS;
           std::vector<const char *> tokenModifiers = SEMANTIC_MODIFIERS;
       } legend;
+      bool range = true;
       bool full = true;
   } semanticTokensProvider;
 };
@@ -158,7 +159,7 @@ REFLECT_STRUCT(ServerCap, textDocumentSync, hoverProvider, completionProvider,
                documentOnTypeFormattingProvider, renameProvider,
                documentLinkProvider, foldingRangeProvider,
                executeCommandProvider, workspace, semanticTokensProvider);
-REFLECT_STRUCT(ServerCap::SemanticTokenProvider, legend, full);
+REFLECT_STRUCT(ServerCap::SemanticTokenProvider, legend, range, full);
 REFLECT_STRUCT(ServerCap::SemanticTokenProvider::SemanticTokensLegend, tokenTypes, tokenModifiers);
 
 struct DynamicReg {
diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index c51f9013..f0034e41 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -56,9 +56,34 @@ struct ScanLineEvent {
 };
 
 }
+constexpr Position documentBegin{0,0};
+constexpr Position documentEnd{
+    std::numeric_limits<decltype(Position::line)>::max(),
+    std::numeric_limits<decltype(Position::character)>::max()};
 
-void MessageHandler::textDocument_semanticTokensFull(
-        SemanticTokensParams &param, ReplyOnce &reply) {
+inline std::ostream &operator<<(std::ostream &s, const Position pos) {
+  s
+    << "{line: " << pos.line
+    << ", end: " << pos.character;
+  return s;
+}
+inline std::ostream &operator<<(std::ostream &s, const lsRange &range) {
+  s
+    << "lsRange(start:" << range.start
+    << ", end:" << range.end
+    << ")";
+  return s;
+}
+
+void MessageHandler::textDocument_semanticTokensRange(
+        SemanticTokensRangeParams &param, ReplyOnce &reply) {
+  if(param.range.start == documentBegin && param.range.end == documentEnd)
+    LOG_S(INFO)
+      << "SemanticToken for all document";
+  else
+    LOG_S(INFO)
+      << "SemanticToken for range "
+      << param.range.start;
   std::string path = param.textDocument.uri.getPath();
   WorkingFile *wfile = wfiles->getFile(path);
   if (!wfile) {
@@ -88,6 +113,13 @@ void MessageHandler::textDocument_semanticTokensFull(
   for (auto [sym, refcnt] : queryFile->symbol2refcnt) {
     if (refcnt <= 0)
       continue;
+    // skip symbols that don't start within range
+    if( sym.range.start.line < param.range.start.line
+            || sym.range.end.line > param.range.end.line
+            // range is within lines here below, let's test if within specified characters/columns
+            || sym.range.start.column < param.range.start.character
+            || sym.range.end.column > param.range.end.character)
+      continue;
     std::string_view detailed_name;
     SymbolKind parent_kind = SymbolKind::Unknown;
     SymbolKind kind = SymbolKind::Unknown;
@@ -258,4 +290,10 @@ void MessageHandler::textDocument_semanticTokensFull(
 
   reply(result);
 }
+void MessageHandler::textDocument_semanticTokensFull(
+        SemanticTokensParams &param, ReplyOnce &reply){
+  lsRange fullRange{documentBegin, documentEnd};
+  SemanticTokensRangeParams fullRangeParameters{param.textDocument, fullRange};
+  textDocument_semanticTokensRange(fullRangeParameters, reply);
+}
 } // namespace ccls
diff --git a/src/query.hh b/src/query.hh
index dee5ff52..37dbd9c3 100644
--- a/src/query.hh
+++ b/src/query.hh
@@ -42,8 +42,9 @@ struct QueryFile {
 
   int id = -1;
   std::optional<Def> def;
-  // `extent` is valid => declaration; invalid => regular reference
-  llvm::DenseMap<ExtentRef, int> symbol2refcnt;
+  //! `extent` is valid => declaration; invalid => regular reference
+  using SymbolToRefCount=llvm::DenseMap<ExtentRef, int>;
+  SymbolToRefCount symbol2refcnt;
 };
 
 template <typename Q, typename QDef> struct QueryEntity {

From 700c1ffa58e769296e54558e9f66e06fc1c2afea Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 3 May 2021 12:51:16 -0400
Subject: [PATCH 09/24] fix debug build

---
 src/messages/textDocument_semanticToken.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index f0034e41..499a0ccd 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -101,7 +101,7 @@ void MessageHandler::textDocument_semanticTokensRange(
 
   static GroupMatch match(g_config->highlight.whitelist,
                           g_config->highlight.blacklist);
-  assert(file->def);
+  assert(queryFile->def);
   if (wfile->buffer_content.size() > g_config->highlight.largeFileSize ||
       !match.matches(queryFile->def->path)) {
     LOG_S(INFO) << "Not SemTokenizing " << path << "because of allowlist/denylist";

From 16d7ef6aa8ed14727d85f80cccd4d362b05246d6 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 10 May 2021 15:41:37 -0400
Subject: [PATCH 10/24] use symbols that "intersect", rather than "begin"
 within range

---
 src/messages/textDocument_semanticToken.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 499a0ccd..c8c98543 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -113,12 +113,12 @@ void MessageHandler::textDocument_semanticTokensRange(
   for (auto [sym, refcnt] : queryFile->symbol2refcnt) {
     if (refcnt <= 0)
       continue;
-    // skip symbols that don't start within range
-    if( sym.range.start.line < param.range.start.line
-            || sym.range.end.line > param.range.end.line
+    // skip symbols that don't intersect range
+    if( sym.range.end.line < param.range.start.line
+            || sym.range.start.line > param.range.end.line
             // range is within lines here below, let's test if within specified characters/columns
-            || sym.range.start.column < param.range.start.character
-            || sym.range.end.column > param.range.end.character)
+            || sym.range.end.column < param.range.start.character
+            || sym.range.start.column > param.range.end.character)
       continue;
     std::string_view detailed_name;
     SymbolKind parent_kind = SymbolKind::Unknown;

From ab53802497b388fa3d3dcc04a71bf60b063345cc Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Fri, 28 May 2021 13:20:50 -0400
Subject: [PATCH 11/24] use SemanticTokensWithId internally

- Move SemanticTokens to QueryFile
- Add SemanticTokensId (though still returning SemanticTokens)
- Save latest SemanticTokensId in QueryFile (latest semantic token sent
is associated to a file)
---
 src/messages/textDocument_semanticToken.cc | 31 ++++++++++++++--------
 src/query.hh                               | 15 +++++++++++
 2 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index c8c98543..726887a5 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -15,13 +15,13 @@
 MAKE_HASHABLE(ccls::SymbolIdx, t.usr, t.kind);
 
 namespace ccls {
+REFLECT_STRUCT(QueryFile::SemanticTokens, data);
+REFLECT_STRUCT(QueryFile::SemanticTokensWithId, tokens, id);
+
 using namespace clang;
 
 namespace {
-struct SemanticTokens {
-  std::vector<int> data;
-};
-REFLECT_STRUCT(SemanticTokens, data);
+
 
 struct CclsSemanticHighlightSymbol {
   using Id=int;
@@ -97,7 +97,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     return;
   }
 
-  SemanticTokens result;
+  QueryFile::SemanticTokensWithId result;
 
   static GroupMatch match(g_config->highlight.whitelist,
                           g_config->highlight.blacklist);
@@ -264,9 +264,13 @@ void MessageHandler::textDocument_semanticTokensRange(
     if (r.start.line != line) {
       column = 0;
     }
-    result.data.push_back(r.start.line - line); line = r.start.line;
-    result.data.push_back(r.start.character - column); column = r.start.character;
-    result.data.push_back(r.end.character - r.start.character);
+
+    auto &serialized = result.tokens.data;
+
+    serialized.push_back(r.start.line - line); line = r.start.line;
+    serialized.push_back(r.start.character - column); column = r.start.character;
+    serialized.push_back(r.end.character - r.start.character);
+
     uint8_t kindId;
     int modifiers = entry.second->storage == SC_Static ? 4 : 0;
     if (entry.first.second & Role::Declaration) {
@@ -284,12 +288,17 @@ void MessageHandler::textDocument_semanticTokensRange(
 	kindId--;
       if (kindId >= 252) kindId = 27 + kindId - 252;
     }
-    result.data.push_back(kindId);
-    result.data.push_back(modifiers);
+    serialized.push_back(kindId);
+    serialized.push_back(modifiers);
   }
+  // tokens ready, let's tag them with "the next id"
+  result.id = queryFile->latestSemanticTokens.id +1;
+  // before sending data, we'll cache the token we're sending
+  queryFile->latestSemanticTokens = result;
 
-  reply(result);
+  reply(result.tokens);
 }
+
 void MessageHandler::textDocument_semanticTokensFull(
         SemanticTokensParams &param, ReplyOnce &reply){
   lsRange fullRange{documentBegin, documentEnd};
diff --git a/src/query.hh b/src/query.hh
index 37dbd9c3..f232ebd7 100644
--- a/src/query.hh
+++ b/src/query.hh
@@ -45,6 +45,21 @@ struct QueryFile {
   //! `extent` is valid => declaration; invalid => regular reference
   using SymbolToRefCount=llvm::DenseMap<ExtentRef, int>;
   SymbolToRefCount symbol2refcnt;
+
+  //! List of 5-integers that describe (line, column, length, token id, token mod)
+  struct SemanticTokens {
+    std::vector<int> data;
+  };
+  //! Semantic tokens with an id
+  struct SemanticTokensWithId {
+    using Id=int;
+    static constexpr Id invalidId=-1;
+    SemanticTokens tokens;
+    //! Id local to a file
+    Id id = invalidId;
+  };
+  //! Latest tokens sent to the client
+  SemanticTokensWithId latestSemanticTokens;
 };
 
 template <typename Q, typename QDef> struct QueryEntity {

From e702d33e48fee81649388ff7d6bf940a7fa4c41e Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Fri, 28 May 2021 13:22:29 -0400
Subject: [PATCH 12/24] indent fix

---
 src/messages/textDocument_semanticToken.cc | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 726887a5..db2d7f64 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -115,10 +115,10 @@ void MessageHandler::textDocument_semanticTokensRange(
       continue;
     // skip symbols that don't intersect range
     if( sym.range.end.line < param.range.start.line
-            || sym.range.start.line > param.range.end.line
-            // range is within lines here below, let's test if within specified characters/columns
-            || sym.range.end.column < param.range.start.character
-            || sym.range.start.column > param.range.end.character)
+        || sym.range.start.line > param.range.end.line
+        // range is within lines here below, let's test if within specified characters/columns
+        || sym.range.end.column < param.range.start.character
+        || sym.range.start.column > param.range.end.character)
       continue;
     std::string_view detailed_name;
     SymbolKind parent_kind = SymbolKind::Unknown;
@@ -148,7 +148,7 @@ void MessageHandler::textDocument_semanticTokensRange(
       // E.g. copy-initialization of constructors should not be highlighted
       // but we still want to keep the range for jumping to definition.
       const auto concise_name =
-          detailed_name.substr(0, detailed_name.find('<'));
+        detailed_name.substr(0, detailed_name.find('<'));
       const auto start_line_idx = sym.range.start.line;
       const auto start_col = sym.range.start.column;
       if (start_line_idx >= wfile->index_lines.size()) // out-of-range ?
@@ -241,7 +241,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     // .
     if (top && !(events[i - 1].pos == events[i].pos))
       events[top - 1].symbol->lsRangeAndRoles.push_back(
-          {{events[i - 1].pos, events[i].pos}, events[i].role});
+        {{events[i - 1].pos, events[i].pos}, events[i].role});
     if (events[i].id >= 0)
       events[top++] = events[i];
     else
@@ -256,7 +256,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     entry.second.lsRangeAndRoles.clear();
   }
   std::sort(scratch.begin(), scratch.end(),
-      [](auto &l, auto &r) { return l.first.first.start < r.first.first.start; });
+            [](auto &l, auto &r) { return l.first.first.start < r.first.first.start; });
   int line = 0;
   int column = 0;
   for (auto &entry : scratch) {
@@ -285,7 +285,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     } else {
       kindId = (uint8_t) entry.second->kind;
       if (kindId > (uint8_t) SymbolKind::StaticMethod)
-	kindId--;
+        kindId--;
       if (kindId >= 252) kindId = 27 + kindId - 252;
     }
     serialized.push_back(kindId);
@@ -300,9 +300,10 @@ void MessageHandler::textDocument_semanticTokensRange(
 }
 
 void MessageHandler::textDocument_semanticTokensFull(
-        SemanticTokensParams &param, ReplyOnce &reply){
-  lsRange fullRange{documentBegin, documentEnd};
+    SemanticTokensParams &param, ReplyOnce &reply){
+    lsRange fullRange{documentBegin, documentEnd};
   SemanticTokensRangeParams fullRangeParameters{param.textDocument, fullRange};
   textDocument_semanticTokensRange(fullRangeParameters, reply);
 }
+
 } // namespace ccls

From de89baedd35c793644817609e5be3fbf23ce8d21 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Wed, 16 Jun 2021 18:03:24 -0400
Subject: [PATCH 13/24] use different container per gh review

---
 src/messages/initialize.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc
index 5ad5ef47..42fba4be 100644
--- a/src/messages/initialize.cc
+++ b/src/messages/initialize.cc
@@ -25,7 +25,7 @@
 namespace ccls {
 using namespace llvm;
 
-std::vector<const char *> SEMANTIC_TOKENS = {
+const char * SEMANTIC_TOKENS[] = {
   "unknown",
 
   "file",
@@ -60,7 +60,7 @@ std::vector<const char *> SEMANTIC_TOKENS = {
   "macro"
 };
 
-std::vector<const char *> SEMANTIC_MODIFIERS = {
+const char * SEMANTIC_MODIFIERS[] = {
     "declaration", //1
     "definition",  //2
     "static"       //4
@@ -132,8 +132,8 @@ struct ServerCap {
   Config::ServerCap::Workspace workspace;
   struct SemanticTokenProvider {
       struct SemanticTokensLegend {
-          std::vector<const char *> tokenTypes = SEMANTIC_TOKENS;
-          std::vector<const char *> tokenModifiers = SEMANTIC_MODIFIERS;
+          std::vector<const char *> tokenTypes{std::begin(SEMANTIC_TOKENS), std::end(SEMANTIC_TOKENS)};
+          std::vector<const char *> tokenModifiers{std::begin(SEMANTIC_MODIFIERS), std::end(SEMANTIC_MODIFIERS)};
       } legend;
       bool range = true;
       bool full = true;

From 1ff4931f2af41ff4e0d980c5480aaa7e7834c8cf Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Wed, 16 Jun 2021 18:04:16 -0400
Subject: [PATCH 14/24] comply with github review comments

---
 src/messages/initialize.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc
index 42fba4be..c719b345 100644
--- a/src/messages/initialize.cc
+++ b/src/messages/initialize.cc
@@ -1,4 +1,4 @@
-// Copyright 2017-2018 ccls Authors
+// Copyright ccls Authors
 // SPDX-License-Identifier: Apache-2.0
 
 #include "filesystem.hh"

From 79c0beedcd4089540184578e2eb8d9ca457f4468 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Thu, 17 Jun 2021 13:37:28 -0400
Subject: [PATCH 15/24] omit copyright year per review

---
 src/messages/textDocument_semanticToken.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index db2d7f64..55c71e2d 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -1,4 +1,4 @@
-// Copyright 2017-2020 ccls Authors
+// Copyright ccls Authors
 // SPDX-License-Identifier: Apache-2.0
 
 #include "indexer.hh"

From c9f36a342d3d497a7ddf4a1ae601b58e4a14eb54 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Fri, 18 Jun 2021 15:09:14 -0400
Subject: [PATCH 16/24] remove $ccls/publishSemanticHighlight

---
 src/message_handler.cc                     | 232 +--------------------
 src/message_handler.hh                     |   3 +-
 src/messages/textDocument_did.cc           |   2 +-
 src/messages/textDocument_semanticToken.cc |   1 +
 src/pipeline.cc                            |   5 +-
 5 files changed, 13 insertions(+), 230 deletions(-)

diff --git a/src/message_handler.cc b/src/message_handler.cc
index 6546722a..c533d834 100644
--- a/src/message_handler.cc
+++ b/src/message_handler.cc
@@ -50,24 +50,6 @@ REFLECT_STRUCT(DidChangeWorkspaceFoldersParam, event);
 REFLECT_STRUCT(WorkspaceSymbolParam, query, folders);
 
 namespace {
-struct CclsSemanticHighlightSymbol {
-  int id = 0;
-  SymbolKind parentKind;
-  SymbolKind kind;
-  uint8_t storage;
-  std::vector<std::pair<int, int>> ranges;
-
-  // `lsRanges` is used to compute `ranges`.
-  std::vector<lsRange> lsRanges;
-};
-
-struct CclsSemanticHighlight {
-  DocumentUri uri;
-  std::vector<CclsSemanticHighlightSymbol> symbols;
-};
-REFLECT_STRUCT(CclsSemanticHighlightSymbol, id, parentKind, kind, storage,
-               ranges, lsRanges);
-REFLECT_STRUCT(CclsSemanticHighlight, uri, symbols);
 
 struct CclsSetSkippedRanges {
   DocumentUri uri;
@@ -75,26 +57,6 @@ struct CclsSetSkippedRanges {
 };
 REFLECT_STRUCT(CclsSetSkippedRanges, uri, skippedRanges);
 
-struct ScanLineEvent {
-  Position pos;
-  Position end_pos; // Second key when there is a tie for insertion events.
-  int id;
-  CclsSemanticHighlightSymbol *symbol;
-  bool operator<(const ScanLineEvent &o) const {
-    // See the comments below when insertion/deletion events are inserted.
-    if (!(pos == o.pos))
-      return pos < o.pos;
-    if (!(o.end_pos == end_pos))
-      return o.end_pos < end_pos;
-    // This comparison essentially order Macro after non-Macro,
-    // So that macros will not be rendered as Var/Type/...
-    if (symbol->kind != o.symbol->kind)
-      return symbol->kind < o.symbol->kind;
-    // If symbol A and B occupy the same place, we want one to be placed
-    // before the other consistantly.
-    return symbol->id < o.symbol->id;
-  }
-};
 } // namespace
 
 void ReplyOnce::notOpened(std::string_view path) {
@@ -279,193 +241,13 @@ void emitSkippedRanges(WorkingFile *wfile, QueryFile &file) {
   pipeline::notify("$ccls/publishSkippedRanges", params);
 }
 
-void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) {
-  static GroupMatch match(g_config->highlight.whitelist,
-                          g_config->highlight.blacklist);
-  assert(file.def);
-  if (wfile->buffer_content.size() > g_config->highlight.largeFileSize ||
-      !match.matches(file.def->path))
-    return;
 
-  // Group symbols together.
-  std::unordered_map<SymbolIdx, CclsSemanticHighlightSymbol> grouped_symbols;
-  for (auto [sym, refcnt] : file.symbol2refcnt) {
-    if (refcnt <= 0)
-      continue;
-    std::string_view detailed_name;
-    SymbolKind parent_kind = SymbolKind::Unknown;
-    SymbolKind kind = SymbolKind::Unknown;
-    uint8_t storage = SC_None;
-    int idx;
-    // This switch statement also filters out symbols that are not highlighted.
-    switch (sym.kind) {
-    case Kind::Func: {
-      idx = db->func_usr[sym.usr];
-      const QueryFunc &func = db->funcs[idx];
-      const QueryFunc::Def *def = func.anyDef();
-      if (!def)
-        continue; // applies to for loop
-      // Don't highlight overloadable operators or implicit lambda ->
-      // std::function constructor.
-      std::string_view short_name = def->name(false);
-      if (short_name.compare(0, 8, "operator") == 0)
-        continue; // applies to for loop
-      kind = def->kind;
-      storage = def->storage;
-      detailed_name = short_name;
-      parent_kind = def->parent_kind;
-
-      // Check whether the function name is actually there.
-      // If not, do not publish the semantic highlight.
-      // E.g. copy-initialization of constructors should not be highlighted
-      // but we still want to keep the range for jumping to definition.
-      std::string_view concise_name =
-          detailed_name.substr(0, detailed_name.find('<'));
-      uint16_t start_line = sym.range.start.line;
-      int16_t start_col = sym.range.start.column;
-      if (start_line >= wfile->index_lines.size())
-        continue;
-      std::string_view line = wfile->index_lines[start_line];
-      sym.range.end.line = start_line;
-      if (!(start_col + concise_name.size() <= line.size() &&
-            line.compare(start_col, concise_name.size(), concise_name) == 0))
-        continue;
-      sym.range.end.column = start_col + concise_name.size();
-      break;
-    }
-    case Kind::Type: {
-      idx = db->type_usr[sym.usr];
-      const QueryType &type = db->types[idx];
-      for (auto &def : type.def) {
-        kind = def.kind;
-        detailed_name = def.detailed_name;
-        if (def.spell) {
-          parent_kind = def.parent_kind;
-          break;
-        }
-      }
-      break;
-    }
-    case Kind::Var: {
-      idx = db->var_usr[sym.usr];
-      const QueryVar &var = db->vars[idx];
-      for (auto &def : var.def) {
-        kind = def.kind;
-        storage = def.storage;
-        detailed_name = def.detailed_name;
-        if (def.spell) {
-          parent_kind = def.parent_kind;
-          break;
-        }
-      }
-      break;
-    }
-    default:
-      continue; // applies to for loop
-    }
-
-    if (std::optional<lsRange> loc = getLsRange(wfile, sym.range)) {
-      auto it = grouped_symbols.find(sym);
-      if (it != grouped_symbols.end()) {
-        it->second.lsRanges.push_back(*loc);
-      } else {
-        CclsSemanticHighlightSymbol symbol;
-        symbol.id = idx;
-        symbol.parentKind = parent_kind;
-        symbol.kind = kind;
-        symbol.storage = storage;
-        symbol.lsRanges.push_back(*loc);
-        grouped_symbols[sym] = symbol;
-      }
-    }
-  }
-
-  // Make ranges non-overlapping using a scan line algorithm.
-  std::vector<ScanLineEvent> events;
-  int id = 0;
-  for (auto &entry : grouped_symbols) {
-    CclsSemanticHighlightSymbol &symbol = entry.second;
-    for (auto &loc : symbol.lsRanges) {
-      // For ranges sharing the same start point, the one with leftmost end
-      // point comes first.
-      events.push_back({loc.start, loc.end, id, &symbol});
-      // For ranges sharing the same end point, their relative order does not
-      // matter, therefore we arbitrarily assign loc.end to them. We use
-      // negative id to indicate a deletion event.
-      events.push_back({loc.end, loc.end, ~id, &symbol});
-      id++;
-    }
-    symbol.lsRanges.clear();
-  }
-  std::sort(events.begin(), events.end());
-
-  std::vector<uint8_t> deleted(id, 0);
-  int top = 0;
-  for (size_t i = 0; i < events.size(); i++) {
-    while (top && deleted[events[top - 1].id])
-      top--;
-    // Order [a, b0) after [a, b1) if b0 < b1. The range comes later overrides
-    // the ealier. The order of [a0, b) [a1, b) does not matter.
-    // The order of [a, b) [b, c) does not as long as we do not emit empty
-    // ranges.
-    // Attribute range [events[i-1].pos, events[i].pos) to events[top-1].symbol
-    // .
-    if (top && !(events[i - 1].pos == events[i].pos))
-      events[top - 1].symbol->lsRanges.push_back(
-          {events[i - 1].pos, events[i].pos});
-    if (events[i].id >= 0)
-      events[top++] = events[i];
-    else
-      deleted[~events[i].id] = 1;
-  }
-
-  CclsSemanticHighlight params;
-  params.uri = DocumentUri::fromPath(wfile->filename);
-  // Transform lsRange into pair<int, int> (offset pairs)
-  if (!g_config->highlight.lsRanges) {
-    std::vector<std::pair<lsRange, CclsSemanticHighlightSymbol *>> scratch;
-    for (auto &entry : grouped_symbols) {
-      for (auto &range : entry.second.lsRanges)
-        scratch.emplace_back(range, &entry.second);
-      entry.second.lsRanges.clear();
-    }
-    std::sort(scratch.begin(), scratch.end(),
-              [](auto &l, auto &r) { return l.first.start < r.first.start; });
-    const auto &buf = wfile->buffer_content;
-    int l = 0, c = 0, i = 0, p = 0;
-    auto mov = [&](int line, int col) {
-      if (l < line)
-        c = 0;
-      for (; l < line && i < buf.size(); i++) {
-        if (buf[i] == '\n')
-          l++;
-        if (uint8_t(buf[i]) < 128 || 192 <= uint8_t(buf[i]))
-          p++;
-      }
-      if (l < line)
-        return true;
-      for (; c < col && i < buf.size() && buf[i] != '\n'; c++)
-        if (p++, uint8_t(buf[i++]) >= 128)
-          // Skip 0b10xxxxxx
-          while (i < buf.size() && uint8_t(buf[i]) >= 128 &&
-                 uint8_t(buf[i]) < 192)
-            i++;
-      return c < col;
-    };
-    for (auto &entry : scratch) {
-      lsRange &r = entry.first;
-      if (mov(r.start.line, r.start.character))
-        continue;
-      int beg = p;
-      if (mov(r.end.line, r.end.character))
-        continue;
-      entry.second->ranges.emplace_back(beg, p);
-    }
-  }
-
-  for (auto &entry : grouped_symbols)
-    if (entry.second.ranges.size() || entry.second.lsRanges.size())
-      params.symbols.push_back(std::move(entry.second));
-  pipeline::notify("$ccls/publishSemanticHighlight", params);
+void emitSemanticHighlightRefresh() {
+  //// tried using `notify`, but won't compile
+  //EmptyParam empty;
+  //pipeline::notify("workspace/semanticTokens/refresh", empty);
+  
+  pipeline::notifyOrRequest(
+   "workspace/semanticTokens/refresh", false, [](JsonWriter &){});
 }
 } // namespace ccls
diff --git a/src/message_handler.hh b/src/message_handler.hh
index f4149b61..b21842fe 100644
--- a/src/message_handler.hh
+++ b/src/message_handler.hh
@@ -307,5 +307,6 @@ private:
 
 void emitSkippedRanges(WorkingFile *wfile, QueryFile &file);
 
-void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file);
+//! Tell client there should be a highlighting refresh
+void emitSemanticHighlightRefresh();
 } // namespace ccls
diff --git a/src/messages/textDocument_did.cc b/src/messages/textDocument_did.cc
index c6f1035b..215c7d43 100644
--- a/src/messages/textDocument_did.cc
+++ b/src/messages/textDocument_did.cc
@@ -36,7 +36,7 @@ void MessageHandler::textDocument_didOpen(DidOpenTextDocumentParam &param) {
   QueryFile *file = findFile(path);
   if (file) {
     emitSkippedRanges(wf, *file);
-    emitSemanticHighlight(db, wf, *file);
+    emitSemanticHighlightRefresh();
   }
   include_complete->addFile(wf->filename);
 
diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 55c71e2d..009c88f1 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -84,6 +84,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     LOG_S(INFO)
       << "SemanticToken for range "
       << param.range.start;
+
   std::string path = param.textDocument.uri.getPath();
   WorkingFile *wfile = wfiles->getFile(path);
   if (!wfile) {
diff --git a/src/pipeline.cc b/src/pipeline.cc
index 6f517b3f..fc38888b 100644
--- a/src/pipeline.cc
+++ b/src/pipeline.cc
@@ -497,8 +497,7 @@ void main_OnIndexed(DB *db, WorkingFiles *wfiles, IndexUpdate *update) {
       std::string path = lowerPathIfInsensitive(f);
       if (db->name2file_id.find(path) == db->name2file_id.end())
         continue;
-      QueryFile &file = db->files[db->name2file_id[path]];
-      emitSemanticHighlight(db, wf.get(), file);
+      emitSemanticHighlightRefresh();
     }
     return;
   }
@@ -515,7 +514,7 @@ void main_OnIndexed(DB *db, WorkingFiles *wfiles, IndexUpdate *update) {
                                                       : def_u.second);
       QueryFile &file = db->files[update->file_id];
       emitSkippedRanges(wfile, file);
-      emitSemanticHighlight(db, wfile, file);
+      emitSemanticHighlightRefresh();
     }
   }
 }

From 5e2ea98cb03d07fdda1507945eb9a934a45f6c73 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Fri, 18 Jun 2021 16:19:21 -0400
Subject: [PATCH 17/24] remove absctractions per github review

---
 src/position.hh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/position.hh b/src/position.hh
index a227ae26..dee98657 100644
--- a/src/position.hh
+++ b/src/position.hh
@@ -10,10 +10,8 @@
 
 namespace ccls {
 struct Pos {
-  using Line=uint16_t;
-  Line line = 0;
-  using Column=int16_t;
-  Column column = -1;
+  uint16_t line = 0;
+  int16_t column = -1;
 
   static Pos fromString(const std::string &encoded);
 
@@ -36,7 +34,7 @@ protected:
    * (line, pos)
    * use for lexicographic comparison
    */
-  auto asTuple() const -> std::tuple<Line,Column> {
+  auto asTuple() const -> std::tuple<decltype(line),decltype(column)> {
     return std::make_tuple(line, column);
   }
 

From 000fd75af4a2af1d1d5cf71ef9a7d0293fc799ac Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Fri, 18 Jun 2021 17:02:09 -0400
Subject: [PATCH 18/24] remove another unnecessary abstraction per gh review

---
 src/messages/textDocument_semanticToken.cc | 2 +-
 src/query.hh                               | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 009c88f1..702d1be1 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -125,7 +125,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     SymbolKind parent_kind = SymbolKind::Unknown;
     SymbolKind kind = SymbolKind::Unknown;
     uint8_t storage = SC_None;
-    DB::UsrIndex idx;
+    decltype(db->func_usr)::key_type idx;
     // This switch statement also filters out symbols that are not highlighted.
     switch (sym.kind) {
     case Kind::Func: {
diff --git a/src/query.hh b/src/query.hh
index f232ebd7..ac88dc78 100644
--- a/src/query.hh
+++ b/src/query.hh
@@ -162,10 +162,8 @@ using Lid2file_id = std::unordered_map<int, int>;
 struct DB {
   std::vector<QueryFile> files;
   llvm::StringMap<int> name2file_id;
-  //! Underlying type used for indexes-of-Usr
-  using UsrIndex=int;
   //! Usr → index
-  llvm::DenseMap<Usr, UsrIndex, DenseMapInfoForUsr> func_usr, type_usr, var_usr;
+  llvm::DenseMap<Usr, int, DenseMapInfoForUsr> func_usr, type_usr, var_usr;
   llvm::SmallVector<QueryFunc, 0> funcs;
   llvm::SmallVector<QueryType, 0> types;
   llvm::SmallVector<QueryVar, 0> vars;

From 03324a122e78b585ea2cb644f0b8e05dbdb9b160 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Tue, 22 Jun 2021 15:25:30 -0400
Subject: [PATCH 19/24] correct usage of "empty (list) of params"

lsp-mode was complaining that generated json was not properly parsed
---
 src/message_handler.cc | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/message_handler.cc b/src/message_handler.cc
index c533d834..15121460 100644
--- a/src/message_handler.cc
+++ b/src/message_handler.cc
@@ -243,11 +243,7 @@ void emitSkippedRanges(WorkingFile *wfile, QueryFile &file) {
 
 
 void emitSemanticHighlightRefresh() {
-  //// tried using `notify`, but won't compile
-  //EmptyParam empty;
-  //pipeline::notify("workspace/semanticTokens/refresh", empty);
-  
-  pipeline::notifyOrRequest(
-   "workspace/semanticTokens/refresh", false, [](JsonWriter &){});
+  std::vector<int> emptyParameters{}; // notification with no parameters (empty list)
+  pipeline::notify("workspace/semanticTokens/refresh", emptyParameters);
 }
 } // namespace ccls

From b07d39b949f241f92d3000162bf407c16c42d8ea Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Wed, 23 Jun 2021 12:29:28 -0400
Subject: [PATCH 20/24] remove `using` per github review

---
 src/messages/textDocument_semanticToken.cc | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 702d1be1..b5ac82e2 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -22,10 +22,8 @@ using namespace clang;
 
 namespace {
 
-
 struct CclsSemanticHighlightSymbol {
-  using Id=int;
-  Id id = 0;
+  int id = 0;
   SymbolKind parentKind;
   SymbolKind kind;
   uint8_t storage;
@@ -35,8 +33,7 @@ struct CclsSemanticHighlightSymbol {
 struct ScanLineEvent {
   Position pos;
   Position end_pos; // Second key when there is a tie for insertion events.
-  using Id=int;
-  Id id;
+  int id;
   CclsSemanticHighlightSymbol *symbol;
   Role role;
   bool operator<(const ScanLineEvent &o) const {

From 20ef17c0a754c651428aa4410f9aaf0b4982514c Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Wed, 23 Jun 2021 12:29:54 -0400
Subject: [PATCH 21/24] clang-format file

---
 src/messages/textDocument_semanticToken.cc | 82 +++++++++++-----------
 1 file changed, 42 insertions(+), 40 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index b5ac82e2..e01974c2 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -7,8 +7,8 @@
 #include "pipeline.hh"
 #include "sema_manager.hh"
 
-#include <clang/Sema/Sema.h>
 #include <algorithm>
+#include <clang/Sema/Sema.h>
 #include <stdexcept>
 #include <string>
 
@@ -52,35 +52,27 @@ struct ScanLineEvent {
   }
 };
 
-}
-constexpr Position documentBegin{0,0};
+} // namespace
+constexpr Position documentBegin{0, 0};
 constexpr Position documentEnd{
     std::numeric_limits<decltype(Position::line)>::max(),
     std::numeric_limits<decltype(Position::character)>::max()};
 
 inline std::ostream &operator<<(std::ostream &s, const Position pos) {
-  s
-    << "{line: " << pos.line
-    << ", end: " << pos.character;
+  s << "{line: " << pos.line << ", end: " << pos.character;
   return s;
 }
 inline std::ostream &operator<<(std::ostream &s, const lsRange &range) {
-  s
-    << "lsRange(start:" << range.start
-    << ", end:" << range.end
-    << ")";
+  s << "lsRange(start:" << range.start << ", end:" << range.end << ")";
   return s;
 }
 
 void MessageHandler::textDocument_semanticTokensRange(
-        SemanticTokensRangeParams &param, ReplyOnce &reply) {
-  if(param.range.start == documentBegin && param.range.end == documentEnd)
-    LOG_S(INFO)
-      << "SemanticToken for all document";
+    SemanticTokensRangeParams &param, ReplyOnce &reply) {
+  if (param.range.start == documentBegin && param.range.end == documentEnd)
+    LOG_S(INFO) << "SemanticToken for all document";
   else
-    LOG_S(INFO)
-      << "SemanticToken for range "
-      << param.range.start;
+    LOG_S(INFO) << "SemanticToken for range " << param.range.start;
 
   std::string path = param.textDocument.uri.getPath();
   WorkingFile *wfile = wfiles->getFile(path);
@@ -89,7 +81,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     return;
   }
 
-  auto [queryFile,wFile] = findOrFail(path, reply);
+  auto [queryFile, wFile] = findOrFail(path, reply);
   if (!queryFile) {
     // `findOrFail` already set the reply message
     return;
@@ -102,7 +94,8 @@ void MessageHandler::textDocument_semanticTokensRange(
   assert(queryFile->def);
   if (wfile->buffer_content.size() > g_config->highlight.largeFileSize ||
       !match.matches(queryFile->def->path)) {
-    LOG_S(INFO) << "Not SemTokenizing " << path << "because of allowlist/denylist";
+    LOG_S(INFO) << "Not SemTokenizing " << path
+                << "because of allowlist/denylist";
     return;
   }
 
@@ -112,11 +105,12 @@ void MessageHandler::textDocument_semanticTokensRange(
     if (refcnt <= 0)
       continue;
     // skip symbols that don't intersect range
-    if( sym.range.end.line < param.range.start.line
-        || sym.range.start.line > param.range.end.line
-        // range is within lines here below, let's test if within specified characters/columns
-        || sym.range.end.column < param.range.start.character
-        || sym.range.start.column > param.range.end.character)
+    if (sym.range.end.line < param.range.start.line ||
+        sym.range.start.line > param.range.end.line
+        // range is within lines here below, let's test if within specified
+        // characters/columns
+        || sym.range.end.column < param.range.start.character ||
+        sym.range.start.column > param.range.end.character)
       continue;
     std::string_view detailed_name;
     SymbolKind parent_kind = SymbolKind::Unknown;
@@ -146,7 +140,7 @@ void MessageHandler::textDocument_semanticTokensRange(
       // E.g. copy-initialization of constructors should not be highlighted
       // but we still want to keep the range for jumping to definition.
       const auto concise_name =
-        detailed_name.substr(0, detailed_name.find('<'));
+          detailed_name.substr(0, detailed_name.find('<'));
       const auto start_line_idx = sym.range.start.line;
       const auto start_col = sym.range.start.column;
       if (start_line_idx >= wfile->index_lines.size()) // out-of-range ?
@@ -215,11 +209,13 @@ void MessageHandler::textDocument_semanticTokensRange(
     for (auto &loc : symbol.lsRangeAndRoles) {
       // For ranges sharing the same start point, the one with leftmost end
       // point comes first.
-      events.push_back({loc.first.start, loc.first.end, id, &symbol, loc.second});
+      events.push_back(
+          {loc.first.start, loc.first.end, id, &symbol, loc.second});
       // For ranges sharing the same end point, their relative order does not
       // matter, therefore we arbitrarily assign loc.end to them. We use
       // negative id to indicate a deletion event.
-      events.push_back({loc.first.end, loc.first.end, ~id, &symbol, loc.second});
+      events.push_back(
+          {loc.first.end, loc.first.end, ~id, &symbol, loc.second});
       id++;
     }
     symbol.lsRangeAndRoles.clear();
@@ -239,7 +235,7 @@ void MessageHandler::textDocument_semanticTokensRange(
     // .
     if (top && !(events[i - 1].pos == events[i].pos))
       events[top - 1].symbol->lsRangeAndRoles.push_back(
-        {{events[i - 1].pos, events[i].pos}, events[i].role});
+          {{events[i - 1].pos, events[i].pos}, events[i].role});
     if (events[i].id >= 0)
       events[top++] = events[i];
     else
@@ -247,14 +243,17 @@ void MessageHandler::textDocument_semanticTokensRange(
   }
 
   // Transform lsRange into pair<int, int> (offset pairs)
-  std::vector<std::pair<std::pair<lsRange, Role>, CclsSemanticHighlightSymbol *>> scratch;
+  std::vector<
+      std::pair<std::pair<lsRange, Role>, CclsSemanticHighlightSymbol *>>
+      scratch;
   for (auto &entry : grouped_symbols) {
     for (auto &range : entry.second.lsRangeAndRoles)
       scratch.emplace_back(range, &entry.second);
     entry.second.lsRangeAndRoles.clear();
   }
-  std::sort(scratch.begin(), scratch.end(),
-            [](auto &l, auto &r) { return l.first.first.start < r.first.first.start; });
+  std::sort(scratch.begin(), scratch.end(), [](auto &l, auto &r) {
+    return l.first.first.start < r.first.first.start;
+  });
   int line = 0;
   int column = 0;
   for (auto &entry : scratch) {
@@ -265,8 +264,10 @@ void MessageHandler::textDocument_semanticTokensRange(
 
     auto &serialized = result.tokens.data;
 
-    serialized.push_back(r.start.line - line); line = r.start.line;
-    serialized.push_back(r.start.character - column); column = r.start.character;
+    serialized.push_back(r.start.line - line);
+    line = r.start.line;
+    serialized.push_back(r.start.character - column);
+    column = r.start.character;
     serialized.push_back(r.end.character - r.start.character);
 
     uint8_t kindId;
@@ -278,19 +279,20 @@ void MessageHandler::textDocument_semanticTokensRange(
       modifiers |= 2;
     }
     if (entry.second->kind == SymbolKind::StaticMethod) {
-      kindId = (uint8_t) SymbolKind::Method;
+      kindId = (uint8_t)SymbolKind::Method;
       modifiers = 4;
     } else {
-      kindId = (uint8_t) entry.second->kind;
-      if (kindId > (uint8_t) SymbolKind::StaticMethod)
+      kindId = (uint8_t)entry.second->kind;
+      if (kindId > (uint8_t)SymbolKind::StaticMethod)
         kindId--;
-      if (kindId >= 252) kindId = 27 + kindId - 252;
+      if (kindId >= 252)
+        kindId = 27 + kindId - 252;
     }
     serialized.push_back(kindId);
     serialized.push_back(modifiers);
   }
   // tokens ready, let's tag them with "the next id"
-  result.id = queryFile->latestSemanticTokens.id +1;
+  result.id = queryFile->latestSemanticTokens.id + 1;
   // before sending data, we'll cache the token we're sending
   queryFile->latestSemanticTokens = result;
 
@@ -298,8 +300,8 @@ void MessageHandler::textDocument_semanticTokensRange(
 }
 
 void MessageHandler::textDocument_semanticTokensFull(
-    SemanticTokensParams &param, ReplyOnce &reply){
-    lsRange fullRange{documentBegin, documentEnd};
+    SemanticTokensParams &param, ReplyOnce &reply) {
+  lsRange fullRange{documentBegin, documentEnd};
   SemanticTokensRangeParams fullRangeParameters{param.textDocument, fullRange};
   textDocument_semanticTokensRange(fullRangeParameters, reply);
 }

From dc9ba782aa263978e1d16d6338fc620b0756d573 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Wed, 23 Jun 2021 12:32:57 -0400
Subject: [PATCH 22/24] remove surrounding braces on single-if-statements

---
 src/messages/textDocument_semanticToken.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index e01974c2..6531fe91 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -272,12 +272,13 @@ void MessageHandler::textDocument_semanticTokensRange(
 
     uint8_t kindId;
     int modifiers = entry.second->storage == SC_Static ? 4 : 0;
-    if (entry.first.second & Role::Declaration) {
+
+    if (entry.first.second & Role::Declaration)
       modifiers |= 1;
-    }
-    if (entry.first.second & Role::Definition) {
+
+    if (entry.first.second & Role::Definition)
       modifiers |= 2;
-    }
+
     if (entry.second->kind == SymbolKind::StaticMethod) {
       kindId = (uint8_t)SymbolKind::Method;
       modifiers = 4;

From b27b1a6d5916ec37d2e6661ef1190998f55200b9 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 16 Aug 2021 18:31:03 -0400
Subject: [PATCH 23/24] fix build (no more `using ID`)

---
 src/messages/textDocument_semanticToken.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 6531fe91..4d67a1f6 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -203,7 +203,7 @@ void MessageHandler::textDocument_semanticTokensRange(
 
   // Make ranges non-overlapping using a scan line algorithm.
   std::vector<ScanLineEvent> events;
-  ScanLineEvent::Id id = 0;
+  int id = 0;
   for (auto &entry : grouped_symbols) {
     CclsSemanticHighlightSymbol &symbol = entry.second;
     for (auto &loc : symbol.lsRangeAndRoles) {

From 7c1d53b186e3e0ee8142efcd25c5cf3b41bb54a6 Mon Sep 17 00:00:00 2001
From: Felipe Lema <1232306+FelipeLema@users.noreply.github.com>
Date: Mon, 16 Aug 2021 18:49:40 -0400
Subject: [PATCH 24/24] improve log messages: add info

---
 src/messages/textDocument_semanticToken.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/messages/textDocument_semanticToken.cc b/src/messages/textDocument_semanticToken.cc
index 4d67a1f6..7b1bd9c9 100644
--- a/src/messages/textDocument_semanticToken.cc
+++ b/src/messages/textDocument_semanticToken.cc
@@ -59,7 +59,7 @@ constexpr Position documentEnd{
     std::numeric_limits<decltype(Position::character)>::max()};
 
 inline std::ostream &operator<<(std::ostream &s, const Position pos) {
-  s << "{line: " << pos.line << ", end: " << pos.character;
+  s << "{line: " << pos.line << ", end: " << pos.character << "}";
   return s;
 }
 inline std::ostream &operator<<(std::ostream &s, const lsRange &range) {
@@ -69,12 +69,12 @@ inline std::ostream &operator<<(std::ostream &s, const lsRange &range) {
 
 void MessageHandler::textDocument_semanticTokensRange(
     SemanticTokensRangeParams &param, ReplyOnce &reply) {
+  const std::string path = param.textDocument.uri.getPath();
   if (param.range.start == documentBegin && param.range.end == documentEnd)
-    LOG_S(INFO) << "SemanticToken for all document";
+    LOG_S(INFO) << "SemanticToken for all document of " << path;
   else
-    LOG_S(INFO) << "SemanticToken for range " << param.range.start;
+    LOG_S(INFO) << "SemanticToken for range " << param.range.start << " of " << path;
 
-  std::string path = param.textDocument.uri.getPath();
   WorkingFile *wfile = wfiles->getFile(path);
   if (!wfile) {
     reply.notOpened(path);