From cc13ced6594596f39b31558ec2eb7755827325d4 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Fri, 25 Dec 2020 11:22:07 +0100 Subject: [PATCH] Support LSP semantic tokens This patch implements `textDocument/semanticTokens/{full,range}`. If the client supports semantic tokens, $ccls/publishSemanticHighlight (now deprecated) is disabled. These token modifiers are mostly useful to emphasize certain symbols: `static, classScope, globalScope, namespaceScope`. To enable a colorful syntax highlighting scheme, set the highlight.rainbow initialization option to 10. https://maskray.me/blog/2024-10-20-ccls-and-lsp-semantic-tokens Note that the older $ccls/publishSemanticHighlight protocol with highlight.lsRanges==true (used by vscode-ccls) is no longer supported. --- src/config.hh | 9 ++- src/enum.inc | 36 +++++++++ src/indexer.hh | 6 ++ src/lsp.hh | 3 + src/message_handler.cc | 162 ++++++++++++++++++++++++++++++------- src/message_handler.hh | 7 ++ src/messages/initialize.cc | 82 ++++++++++++++++--- src/pipeline.cc | 10 +++ 8 files changed, 272 insertions(+), 43 deletions(-) create mode 100644 src/enum.inc diff --git a/src/config.hh b/src/config.hh index 751a712c..b768e476 100644 --- a/src/config.hh +++ b/src/config.hh @@ -117,6 +117,8 @@ struct Config { bool hierarchicalDocumentSymbolSupport = true; // TextDocumentClientCapabilities.definition.linkSupport bool linkSupport = true; + // ClientCapabilities.workspace.semanticTokens.refreshSupport + bool semanticTokensRefresh = true; // If false, disable snippets and complete just the identifier part. // TextDocumentClientCapabilities.completion.completionItem.snippetSupport @@ -226,8 +228,9 @@ struct Config { // Disable semantic highlighting for files larger than the size. int64_t largeFileSize = 2 * 1024 * 1024; - // true: LSP line/character; false: position - bool lsRanges = false; + // If non-zero, enable rainbow semantic tokens by assinging an extra modifier + // indicating the rainbow ID to each symbol. + int rainbow = 0; // Like index.{whitelist,blacklist}, don't publish semantic highlighting to // blacklisted files. @@ -342,7 +345,7 @@ REFLECT_STRUCT(Config::Completion, caseSensitivity, detailedLabel, maxNum, placeholder); REFLECT_STRUCT(Config::Diagnostics, blacklist, onChange, onOpen, onSave, spellChecking, whitelist) -REFLECT_STRUCT(Config::Highlight, largeFileSize, lsRanges, blacklist, whitelist) +REFLECT_STRUCT(Config::Highlight, largeFileSize, rainbow, blacklist, whitelist) REFLECT_STRUCT(Config::Index::Name, suppressUnwrittenScope); REFLECT_STRUCT(Config::Index, blacklist, comments, initialNoLinkage, initialBlacklist, initialWhitelist, maxInitializerLines, diff --git a/src/enum.inc b/src/enum.inc new file mode 100644 index 00000000..55aa89ae --- /dev/null +++ b/src/enum.inc @@ -0,0 +1,36 @@ +#ifndef TOKEN_MODIFIER +#define TOKEN_MODIFIER(name, str) +#endif +// vscode +TOKEN_MODIFIER(Declaration, "declaration") +TOKEN_MODIFIER(Definition, "definition") +TOKEN_MODIFIER(Static, "static") + +// ccls extensions +TOKEN_MODIFIER(Read, "read") +TOKEN_MODIFIER(Write, "write") +TOKEN_MODIFIER(ClassScope, "classScope") +TOKEN_MODIFIER(FunctionScope, "functionScope") +TOKEN_MODIFIER(NamespaceScope, "namespaceScope") + +// Rainbow semantic tokens +TOKEN_MODIFIER(Id0, "id0") +TOKEN_MODIFIER(Id1, "id1") +TOKEN_MODIFIER(Id2, "id2") +TOKEN_MODIFIER(Id3, "id3") +TOKEN_MODIFIER(Id4, "id4") +TOKEN_MODIFIER(Id5, "id5") +TOKEN_MODIFIER(Id6, "id6") +TOKEN_MODIFIER(Id7, "id7") +TOKEN_MODIFIER(Id8, "id8") +TOKEN_MODIFIER(Id9, "id9") +TOKEN_MODIFIER(Id10, "id10") +TOKEN_MODIFIER(Id11, "id11") +TOKEN_MODIFIER(Id12, "id12") +TOKEN_MODIFIER(Id13, "id13") +TOKEN_MODIFIER(Id14, "id14") +TOKEN_MODIFIER(Id15, "id15") +TOKEN_MODIFIER(Id16, "id16") +TOKEN_MODIFIER(Id17, "id17") +TOKEN_MODIFIER(Id18, "id18") +TOKEN_MODIFIER(Id19, "id19") diff --git a/src/indexer.hh b/src/indexer.hh index 723cda8b..8aea31e6 100644 --- a/src/indexer.hh +++ b/src/indexer.hh @@ -132,6 +132,12 @@ void reflect(BinaryWriter &visitor, SymbolRef &value); void reflect(BinaryWriter &visitor, Use &value); void reflect(BinaryWriter &visitor, DeclRef &value); +enum class TokenModifier { +#define TOKEN_MODIFIER(name, str) name, +#include "enum.inc" +#undef TOKEN_MODIFIER +}; + template using VectorAdapter = std::vector>; template struct NameMixin { diff --git a/src/lsp.hh b/src/lsp.hh index e01d7497..51367f55 100644 --- a/src/lsp.hh +++ b/src/lsp.hh @@ -166,6 +166,7 @@ enum class SymbolKind : uint8_t { // For C++, this is interpreted as "template parameter" (including // non-type template parameters). TypeParameter = 26, + FirstNonStandard, // ccls extensions // See also https://github.com/Microsoft/language-server-protocol/issues/344 @@ -174,6 +175,8 @@ enum class SymbolKind : uint8_t { Parameter = 253, StaticMethod = 254, Macro = 255, + FirstExtension = TypeAlias, + LastExtension = Macro, }; struct SymbolInformation { diff --git a/src/message_handler.cc b/src/message_handler.cc index 09ee6cbf..efc3ba60 100644 --- a/src/message_handler.cc +++ b/src/message_handler.cc @@ -11,11 +11,25 @@ #include #include +#include + #include #include using namespace clang; +#if LLVM_VERSION_MAJOR < 15 // llvmorg-15-init-6118-gb39f43775796 +namespace llvm { +template +constexpr bool is_contained(std::initializer_list set, const E &e) { + for (const T &v : set) + if (v == e) + return true; + return false; +} +} +#endif + MAKE_HASHABLE(ccls::SymbolIdx, t.usr, t.kind); namespace ccls { @@ -51,6 +65,10 @@ REFLECT_STRUCT(DidChangeWorkspaceFoldersParam, event); REFLECT_STRUCT(WorkspaceSymbolParam, query, folders); namespace { +struct Occur { + lsRange range; + Role role; +}; struct CclsSemanticHighlightSymbol { int id = 0; SymbolKind parentKind; @@ -58,16 +76,15 @@ struct CclsSemanticHighlightSymbol { uint8_t storage; std::vector> ranges; - // `lsRanges` is used to compute `ranges`. - std::vector lsRanges; + // `lsOccur` is used to compute `ranges`. + std::vector lsOccurs; }; struct CclsSemanticHighlight { DocumentUri uri; std::vector symbols; }; -REFLECT_STRUCT(CclsSemanticHighlightSymbol, id, parentKind, kind, storage, - ranges, lsRanges); +REFLECT_STRUCT(CclsSemanticHighlightSymbol, id, parentKind, kind, storage, ranges); REFLECT_STRUCT(CclsSemanticHighlight, uri, symbols); struct CclsSetSkippedRanges { @@ -76,10 +93,16 @@ struct CclsSetSkippedRanges { }; REFLECT_STRUCT(CclsSetSkippedRanges, uri, skippedRanges); +struct SemanticTokensPartialResult { + std::vector data; +}; +REFLECT_STRUCT(SemanticTokensPartialResult, data); + struct ScanLineEvent { Position pos; Position end_pos; // Second key when there is a tie for insertion events. int id; + Role role; CclsSemanticHighlightSymbol *symbol; bool operator<(const ScanLineEvent &o) const { // See the comments below when insertion/deletion events are inserted. @@ -190,6 +213,8 @@ MessageHandler::MessageHandler() { bind("textDocument/rangeFormatting", &MessageHandler::textDocument_rangeFormatting); bind("textDocument/references", &MessageHandler::textDocument_references); bind("textDocument/rename", &MessageHandler::textDocument_rename); + bind("textDocument/semanticTokens/full", &MessageHandler::textDocument_semanticTokensFull); + bind("textDocument/semanticTokens/range", &MessageHandler::textDocument_semanticTokensRange); bind("textDocument/signatureHelp", &MessageHandler::textDocument_signatureHelp); bind("textDocument/typeDefinition", &MessageHandler::textDocument_typeDefinition); bind("workspace/didChangeConfiguration", &MessageHandler::workspace_didChangeConfiguration); @@ -281,16 +306,16 @@ void emitSkippedRanges(WorkingFile *wfile, QueryFile &file) { pipeline::notify("$ccls/publishSkippedRanges", params); } -void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) { +static std::unordered_map computeSemanticTokens(DB *db, WorkingFile *wfile, + QueryFile &file) { static GroupMatch match(g_config->highlight.whitelist, g_config->highlight.blacklist); assert(file.def); - if (wfile->buffer_content.size() > g_config->highlight.largeFileSize || - !match.matches(file.def->path)) - return; - // Group symbols together. std::unordered_map grouped_symbols; + if (!match.matches(file.def->path)) + return grouped_symbols; + for (auto [sym, refcnt] : file.symbol2refcnt) { if (refcnt <= 0) continue; @@ -369,14 +394,14 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) { if (std::optional loc = getLsRange(wfile, sym.range)) { auto it = grouped_symbols.find(sym); if (it != grouped_symbols.end()) { - it->second.lsRanges.push_back(*loc); + it->second.lsOccurs.push_back({*loc, sym.role}); } else { CclsSemanticHighlightSymbol symbol; symbol.id = idx; symbol.parentKind = parent_kind; symbol.kind = kind; symbol.storage = storage; - symbol.lsRanges.push_back(*loc); + symbol.lsOccurs.push_back({*loc, sym.role}); grouped_symbols[sym] = symbol; } } @@ -387,17 +412,17 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) { int id = 0; for (auto &entry : grouped_symbols) { CclsSemanticHighlightSymbol &symbol = entry.second; - for (auto &loc : symbol.lsRanges) { + for (auto &occur : symbol.lsOccurs) { // For ranges sharing the same start point, the one with leftmost end // point comes first. - events.push_back({loc.start, loc.end, id, &symbol}); + events.push_back({occur.range.start, occur.range.end, id, occur.role, &symbol}); // For ranges sharing the same end point, their relative order does not - // matter, therefore we arbitrarily assign loc.end to them. We use + // matter, therefore we arbitrarily assign occur.range.end to them. We use // negative id to indicate a deletion event. - events.push_back({loc.end, loc.end, ~id, &symbol}); + events.push_back({occur.range.end, occur.range.end, ~id, occur.role, &symbol}); id++; } - symbol.lsRanges.clear(); + symbol.lsOccurs.clear(); } std::sort(events.begin(), events.end()); @@ -413,26 +438,33 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) { // Attribute range [events[i-1].pos, events[i].pos) to events[top-1].symbol // . if (top && !(events[i - 1].pos == events[i].pos)) - events[top - 1].symbol->lsRanges.push_back( - {events[i - 1].pos, events[i].pos}); + events[top - 1].symbol->lsOccurs.push_back({{events[i - 1].pos, events[i].pos}, events[i].role}); if (events[i].id >= 0) events[top++] = events[i]; else deleted[~events[i].id] = 1; } + return grouped_symbols; +} + +void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) { + // Disable $ccls/publishSemanticHighlight if semantic tokens support is + // enabled or the file is too large. + if (g_config->client.semanticTokensRefresh || wfile->buffer_content.size() > g_config->highlight.largeFileSize) + return; + auto grouped_symbols = computeSemanticTokens(db, wfile, file); CclsSemanticHighlight params; params.uri = DocumentUri::fromPath(wfile->filename); // Transform lsRange into pair (offset pairs) - if (!g_config->highlight.lsRanges) { - std::vector> scratch; + { + std::vector> scratch; for (auto &entry : grouped_symbols) { - for (auto &range : entry.second.lsRanges) - scratch.emplace_back(range, &entry.second); - entry.second.lsRanges.clear(); + for (auto &occur : entry.second.lsOccurs) + scratch.push_back({occur, &entry.second}); + entry.second.lsOccurs.clear(); } - std::sort(scratch.begin(), scratch.end(), - [](auto &l, auto &r) { return l.first.start < r.first.start; }); + std::sort(scratch.begin(), scratch.end(), [](auto &l, auto &r) { return l.first.range < r.first.range; }); const auto &buf = wfile->buffer_content; int l = 0, c = 0, i = 0, p = 0; auto mov = [&](int line, int col) { @@ -455,7 +487,7 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) { return c < col; }; for (auto &entry : scratch) { - lsRange &r = entry.first; + lsRange &r = entry.first.range; if (mov(r.start.line, r.start.character)) continue; int beg = p; @@ -466,8 +498,84 @@ void emitSemanticHighlight(DB *db, WorkingFile *wfile, QueryFile &file) { } for (auto &entry : grouped_symbols) - if (entry.second.ranges.size() || entry.second.lsRanges.size()) + if (entry.second.ranges.size() || entry.second.lsOccurs.size()) params.symbols.push_back(std::move(entry.second)); pipeline::notify("$ccls/publishSemanticHighlight", params); } + +void MessageHandler::textDocument_semanticTokensFull(TextDocumentParam ¶m, ReplyOnce &reply) { + SemanticTokensRangeParams parameters{param.textDocument, lsRange{{0, 0}, {UINT16_MAX, INT16_MAX}}}; + textDocument_semanticTokensRange(parameters, reply); +} + +void MessageHandler::textDocument_semanticTokensRange(SemanticTokensRangeParams ¶m, ReplyOnce &reply) { + int file_id; + auto [file, wf] = findOrFail(param.textDocument.uri.getPath(), reply, &file_id); + if (!wf) + return; + + auto grouped_symbols = computeSemanticTokens(db, wf, *file); + std::vector> scratch; + for (auto &entry : grouped_symbols) { + for (auto &occur : entry.second.lsOccurs) + scratch.emplace_back(occur, &entry.second); + entry.second.lsOccurs.clear(); + } + std::sort(scratch.begin(), scratch.end(), [](auto &l, auto &r) { return l.first.range < r.first.range; }); + + SemanticTokensPartialResult result; + int line = 0, column = 0; + for (auto &entry : scratch) { + lsRange &r = entry.first.range; + CclsSemanticHighlightSymbol &symbol = *entry.second; + if (r.start.line != line) + column = 0; + result.data.push_back(r.start.line - line); + line = r.start.line; + result.data.push_back(r.start.character - column); + column = r.start.character; + result.data.push_back(r.end.character - r.start.character); + + int tokenType = (int)symbol.kind, modifier = 0; + if (tokenType == (int)SymbolKind::StaticMethod) { + tokenType = (int)SymbolKind::Method; + modifier |= 1 << (int)TokenModifier::Static; + } else if (tokenType >= (int)SymbolKind::FirstExtension) { + tokenType += (int)SymbolKind::FirstNonStandard - (int)SymbolKind::FirstExtension; + } + + // Set modifiers. + if (entry.first.role & Role::Declaration) + modifier |= 1 << (int)TokenModifier::Declaration; + if (entry.first.role & Role::Definition) + modifier |= 1 << (int)TokenModifier::Definition; + if (entry.first.role & Role::Read) + modifier |= 1 << (int)TokenModifier::Read; + if (entry.first.role & Role::Write) + modifier |= 1 << (int)TokenModifier::Write; + if (symbol.storage == SC_Static) + modifier |= 1 << (int)TokenModifier::Static; + + if (llvm::is_contained({SymbolKind::Constructor, SymbolKind::Field, SymbolKind::Method, SymbolKind::StaticMethod}, + symbol.kind)) + modifier |= 1 << (int)TokenModifier::ClassScope; + else if (llvm::is_contained({SymbolKind::File, SymbolKind::Namespace}, symbol.parentKind)) + modifier |= 1 << (int)TokenModifier::NamespaceScope; + else if (llvm::is_contained( + {SymbolKind::Constructor, SymbolKind::Function, SymbolKind::Method, SymbolKind::StaticMethod}, + symbol.parentKind)) + modifier |= 1 << (int)TokenModifier::FunctionScope; + + // Rainbow semantic tokens + static_assert((int)TokenModifier::Id0 + 20 < 31); + if (int rainbow = g_config->highlight.rainbow) + modifier |= 1 << ((int)TokenModifier::Id0 + symbol.id % std::min(rainbow, 20)); + + result.data.push_back(tokenType); + result.data.push_back(modifier); + } + + reply(result); +} + } // namespace ccls diff --git a/src/message_handler.hh b/src/message_handler.hh index d2f66c93..07044c68 100644 --- a/src/message_handler.hh +++ b/src/message_handler.hh @@ -40,6 +40,11 @@ struct RenameParam { Position position; std::string newName; }; +struct SemanticTokensRangeParams { + TextDocumentIdentifier textDocument; + lsRange range; +}; +REFLECT_STRUCT(SemanticTokensRangeParams, textDocument, range); struct TextDocumentParam { TextDocumentIdentifier textDocument; }; @@ -303,6 +308,8 @@ private: ReplyOnce &); void textDocument_references(JsonReader &, ReplyOnce &); void textDocument_rename(RenameParam &, ReplyOnce &); + void textDocument_semanticTokensFull(TextDocumentParam &, ReplyOnce &); + void textDocument_semanticTokensRange(SemanticTokensRangeParams &, ReplyOnce &); void textDocument_signatureHelp(TextDocumentPositionParam &, ReplyOnce &); void textDocument_typeDefinition(TextDocumentPositionParam &, ReplyOnce &); void workspace_didChangeConfiguration(EmptyParam &); diff --git a/src/messages/initialize.cc b/src/messages/initialize.cc index cae1ca82..8b641388 100644 --- a/src/messages/initialize.cc +++ b/src/messages/initialize.cc @@ -24,6 +24,51 @@ namespace ccls { using namespace llvm; +const char *const kTokenTypes[] = { + "unknown", + + "file", + "module", + "namespace", + "package", + "class", + "method", + "property", + "field", + "constructor", + "enum", + "interface", + "function", + "variable", + "constant", + "string", + "number", + "boolean", + "array", + "object", + "key", + "null", + "enumMember", + "struct", + "event", + "operator", + "typeParameter", + + // 252 => 27 + "typeAlias", + "parameter", + "staticMethod", + "macro", +}; +static_assert(std::size(kTokenTypes) == + int(SymbolKind::FirstNonStandard) + int(SymbolKind::LastExtension) - int(SymbolKind::FirstExtension) + 1); + +const char *const kTokenModifiers[] = { +#define TOKEN_MODIFIER(name, str) str, +#include "enum.inc" +#undef TOKEN_MODIFIER +}; + extern std::vector g_init_options; namespace { @@ -88,6 +133,14 @@ struct ServerCap { std::vector commands = {ccls_xref}; } executeCommandProvider; bool callHierarchyProvider = true; + struct SemanticTokenProvider { + struct SemanticTokensLegend { + std::vector tokenTypes{std::begin(kTokenTypes), std::end(kTokenTypes)}; + std::vector tokenModifiers{std::begin(kTokenModifiers), std::end(kTokenModifiers)}; + } legend; + bool range = true; + bool full = true; + } semanticTokensProvider; Config::ServerCap::Workspace workspace; }; REFLECT_STRUCT(ServerCap::CodeActionOptions, codeActionKinds); @@ -100,16 +153,14 @@ REFLECT_STRUCT(ServerCap::SaveOptions, includeText); REFLECT_STRUCT(ServerCap::SignatureHelpOptions, triggerCharacters); REFLECT_STRUCT(ServerCap::TextDocumentSyncOptions, openClose, change, willSave, willSaveWaitUntil, save); -REFLECT_STRUCT(ServerCap, textDocumentSync, hoverProvider, completionProvider, - signatureHelpProvider, declarationProvider, definitionProvider, - implementationProvider, typeDefinitionProvider, - referencesProvider, documentHighlightProvider, - documentSymbolProvider, workspaceSymbolProvider, - codeActionProvider, codeLensProvider, documentFormattingProvider, - documentRangeFormattingProvider, - documentOnTypeFormattingProvider, renameProvider, - documentLinkProvider, foldingRangeProvider, - executeCommandProvider, callHierarchyProvider, workspace); +REFLECT_STRUCT(ServerCap, textDocumentSync, hoverProvider, completionProvider, signatureHelpProvider, + declarationProvider, definitionProvider, implementationProvider, typeDefinitionProvider, + referencesProvider, documentHighlightProvider, documentSymbolProvider, workspaceSymbolProvider, + codeActionProvider, codeLensProvider, documentFormattingProvider, documentRangeFormattingProvider, + documentOnTypeFormattingProvider, renameProvider, documentLinkProvider, foldingRangeProvider, + executeCommandProvider, callHierarchyProvider, semanticTokensProvider, workspace); +REFLECT_STRUCT(ServerCap::SemanticTokenProvider, legend, range, full); +REFLECT_STRUCT(ServerCap::SemanticTokenProvider::SemanticTokensLegend, tokenTypes, tokenModifiers); struct DynamicReg { bool dynamicRegistration = false; @@ -132,12 +183,16 @@ struct WorkspaceClientCap { DynamicReg didChangeWatchedFiles; DynamicReg symbol; DynamicReg executeCommand; + + struct SemanticTokensWorkspace { + bool refreshSupport = false; + } semanticTokens; }; REFLECT_STRUCT(WorkspaceClientCap::WorkspaceEdit, documentChanges); -REFLECT_STRUCT(WorkspaceClientCap, applyEdit, workspaceEdit, - didChangeConfiguration, didChangeWatchedFiles, symbol, - executeCommand); +REFLECT_STRUCT(WorkspaceClientCap::SemanticTokensWorkspace, refreshSupport); +REFLECT_STRUCT(WorkspaceClientCap, applyEdit, workspaceEdit, didChangeConfiguration, didChangeWatchedFiles, symbol, + executeCommand, semanticTokens); // Text document specific client capabilities. struct TextDocumentClientCap { @@ -319,6 +374,7 @@ void do_initialize(MessageHandler *m, InitializeParam ¶m, capabilities.textDocument.publishDiagnostics.relatedInformation; didChangeWatchedFiles = capabilities.workspace.didChangeWatchedFiles.dynamicRegistration; + g_config->client.semanticTokensRefresh &= capabilities.workspace.semanticTokens.refreshSupport; if (!g_config->client.snippetSupport) g_config->completion.duplicateOptional = false; diff --git a/src/pipeline.cc b/src/pipeline.cc index f63d5b9a..ee3055a1 100644 --- a/src/pipeline.cc +++ b/src/pipeline.cc @@ -517,6 +517,10 @@ void main_OnIndexed(DB *db, WorkingFiles *wfiles, IndexUpdate *update) { QueryFile &file = db->files[db->name2file_id[path]]; emitSemanticHighlight(db, wf.get(), file); } + if (g_config->client.semanticTokensRefresh) { + std::optional param; + request("workspace/semanticTokens/refresh", param); + } return; } @@ -533,6 +537,12 @@ void main_OnIndexed(DB *db, WorkingFiles *wfiles, IndexUpdate *update) { QueryFile &file = db->files[update->file_id]; emitSkippedRanges(wfile, file); emitSemanticHighlight(db, wfile, file); + if (g_config->client.semanticTokensRefresh) { + // Return filename, even if the spec indicates params is none. + TextDocumentIdentifier param; + param.uri = DocumentUri::fromPath(wfile->filename); + request("workspace/semanticTokens/refresh", param); + } } } }