llvm-project
1584 строки · 58.8 Кб
1//===--- SemanticHighlighting.cpp - ------------------------- ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SemanticHighlighting.h"
10#include "Config.h"
11#include "FindTarget.h"
12#include "HeuristicResolver.h"
13#include "ParsedAST.h"
14#include "Protocol.h"
15#include "SourceCode.h"
16#include "support/Logger.h"
17#include "clang/AST/ASTContext.h"
18#include "clang/AST/Decl.h"
19#include "clang/AST/DeclCXX.h"
20#include "clang/AST/DeclObjC.h"
21#include "clang/AST/DeclTemplate.h"
22#include "clang/AST/DeclarationName.h"
23#include "clang/AST/ExprCXX.h"
24#include "clang/AST/RecursiveASTVisitor.h"
25#include "clang/AST/Type.h"
26#include "clang/AST/TypeLoc.h"
27#include "clang/Basic/LangOptions.h"
28#include "clang/Basic/SourceLocation.h"
29#include "clang/Basic/SourceManager.h"
30#include "clang/Tooling/Syntax/Tokens.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/Error.h"
35#include <algorithm>
36#include <optional>
37
38namespace clang {
39namespace clangd {
40namespace {
41
42/// Get the last Position on a given line.
43llvm::Expected<Position> endOfLine(llvm::StringRef Code, int Line) {
44auto StartOfLine = positionToOffset(Code, Position{Line, 0});
45if (!StartOfLine)
46return StartOfLine.takeError();
47StringRef LineText = Code.drop_front(*StartOfLine).take_until([](char C) {
48return C == '\n';
49});
50return Position{Line, static_cast<int>(lspLength(LineText))};
51}
52
53/// Some names are not written in the source code and cannot be highlighted,
54/// e.g. anonymous classes. This function detects those cases.
55bool canHighlightName(DeclarationName Name) {
56switch (Name.getNameKind()) {
57case DeclarationName::Identifier: {
58auto *II = Name.getAsIdentifierInfo();
59return II && !II->getName().empty();
60}
61case DeclarationName::CXXConstructorName:
62case DeclarationName::CXXDestructorName:
63return true;
64case DeclarationName::ObjCZeroArgSelector:
65case DeclarationName::ObjCOneArgSelector:
66case DeclarationName::ObjCMultiArgSelector:
67// Multi-arg selectors need special handling, and we handle 0/1 arg
68// selectors there too.
69return false;
70case DeclarationName::CXXConversionFunctionName:
71case DeclarationName::CXXOperatorName:
72case DeclarationName::CXXDeductionGuideName:
73case DeclarationName::CXXLiteralOperatorName:
74case DeclarationName::CXXUsingDirective:
75return false;
76}
77llvm_unreachable("invalid name kind");
78}
79
80bool isUniqueDefinition(const NamedDecl *Decl) {
81if (auto *Func = dyn_cast<FunctionDecl>(Decl))
82return Func->isThisDeclarationADefinition();
83if (auto *Klass = dyn_cast<CXXRecordDecl>(Decl))
84return Klass->isThisDeclarationADefinition();
85if (auto *Iface = dyn_cast<ObjCInterfaceDecl>(Decl))
86return Iface->isThisDeclarationADefinition();
87if (auto *Proto = dyn_cast<ObjCProtocolDecl>(Decl))
88return Proto->isThisDeclarationADefinition();
89if (auto *Var = dyn_cast<VarDecl>(Decl))
90return Var->isThisDeclarationADefinition();
91return isa<TemplateTypeParmDecl>(Decl) ||
92isa<NonTypeTemplateParmDecl>(Decl) ||
93isa<TemplateTemplateParmDecl>(Decl) || isa<ObjCCategoryDecl>(Decl) ||
94isa<ObjCImplDecl>(Decl);
95}
96
97std::optional<HighlightingKind> kindForType(const Type *TP,
98const HeuristicResolver *Resolver);
99std::optional<HighlightingKind> kindForDecl(const NamedDecl *D,
100const HeuristicResolver *Resolver) {
101if (auto *USD = dyn_cast<UsingShadowDecl>(D)) {
102if (auto *Target = USD->getTargetDecl())
103D = Target;
104}
105if (auto *TD = dyn_cast<TemplateDecl>(D)) {
106if (auto *Templated = TD->getTemplatedDecl())
107D = Templated;
108}
109if (auto *TD = dyn_cast<TypedefNameDecl>(D)) {
110// We try to highlight typedefs as their underlying type.
111if (auto K =
112kindForType(TD->getUnderlyingType().getTypePtrOrNull(), Resolver))
113return K;
114// And fallback to a generic kind if this fails.
115return HighlightingKind::Typedef;
116}
117// We highlight class decls, constructor decls and destructor decls as
118// `Class` type. The destructor decls are handled in `VisitTagTypeLoc` (we
119// will visit a TypeLoc where the underlying Type is a CXXRecordDecl).
120if (auto *RD = llvm::dyn_cast<RecordDecl>(D)) {
121// We don't want to highlight lambdas like classes.
122if (RD->isLambda())
123return std::nullopt;
124return HighlightingKind::Class;
125}
126if (isa<ClassTemplateDecl, RecordDecl, CXXConstructorDecl, ObjCInterfaceDecl,
127ObjCImplementationDecl>(D))
128return HighlightingKind::Class;
129if (isa<ObjCProtocolDecl>(D))
130return HighlightingKind::Interface;
131if (isa<ObjCCategoryDecl, ObjCCategoryImplDecl>(D))
132return HighlightingKind::Namespace;
133if (auto *MD = dyn_cast<CXXMethodDecl>(D))
134return MD->isStatic() ? HighlightingKind::StaticMethod
135: HighlightingKind::Method;
136if (auto *OMD = dyn_cast<ObjCMethodDecl>(D))
137return OMD->isClassMethod() ? HighlightingKind::StaticMethod
138: HighlightingKind::Method;
139if (isa<FieldDecl, IndirectFieldDecl, ObjCPropertyDecl>(D))
140return HighlightingKind::Field;
141if (isa<EnumDecl>(D))
142return HighlightingKind::Enum;
143if (isa<EnumConstantDecl>(D))
144return HighlightingKind::EnumConstant;
145if (isa<ParmVarDecl>(D))
146return HighlightingKind::Parameter;
147if (auto *VD = dyn_cast<VarDecl>(D)) {
148if (isa<ImplicitParamDecl>(VD)) // e.g. ObjC Self
149return std::nullopt;
150return VD->isStaticDataMember()
151? HighlightingKind::StaticField
152: VD->isLocalVarDecl() ? HighlightingKind::LocalVariable
153: HighlightingKind::Variable;
154}
155if (const auto *BD = dyn_cast<BindingDecl>(D))
156return BD->getDeclContext()->isFunctionOrMethod()
157? HighlightingKind::LocalVariable
158: HighlightingKind::Variable;
159if (isa<FunctionDecl>(D))
160return HighlightingKind::Function;
161if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||
162isa<UsingDirectiveDecl>(D))
163return HighlightingKind::Namespace;
164if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
165isa<NonTypeTemplateParmDecl>(D))
166return HighlightingKind::TemplateParameter;
167if (isa<ConceptDecl>(D))
168return HighlightingKind::Concept;
169if (isa<LabelDecl>(D))
170return HighlightingKind::Label;
171if (const auto *UUVD = dyn_cast<UnresolvedUsingValueDecl>(D)) {
172auto Targets = Resolver->resolveUsingValueDecl(UUVD);
173if (!Targets.empty() && Targets[0] != UUVD) {
174return kindForDecl(Targets[0], Resolver);
175}
176return HighlightingKind::Unknown;
177}
178return std::nullopt;
179}
180std::optional<HighlightingKind> kindForType(const Type *TP,
181const HeuristicResolver *Resolver) {
182if (!TP)
183return std::nullopt;
184if (TP->isBuiltinType()) // Builtins are special, they do not have decls.
185return HighlightingKind::Primitive;
186if (auto *TD = dyn_cast<TemplateTypeParmType>(TP))
187return kindForDecl(TD->getDecl(), Resolver);
188if (isa<ObjCObjectPointerType>(TP))
189return HighlightingKind::Class;
190if (auto *TD = TP->getAsTagDecl())
191return kindForDecl(TD, Resolver);
192return std::nullopt;
193}
194
195// Whether T is const in a loose sense - is a variable with this type readonly?
196bool isConst(QualType T) {
197if (T.isNull())
198return false;
199T = T.getNonReferenceType();
200if (T.isConstQualified())
201return true;
202if (const auto *AT = T->getAsArrayTypeUnsafe())
203return isConst(AT->getElementType());
204if (isConst(T->getPointeeType()))
205return true;
206return false;
207}
208
209// Whether D is const in a loose sense (should it be highlighted as such?)
210// FIXME: This is separate from whether *a particular usage* can mutate D.
211// We may want V in V.size() to be readonly even if V is mutable.
212bool isConst(const Decl *D) {
213if (llvm::isa<EnumConstantDecl>(D) || llvm::isa<NonTypeTemplateParmDecl>(D))
214return true;
215if (llvm::isa<FieldDecl>(D) || llvm::isa<VarDecl>(D) ||
216llvm::isa<MSPropertyDecl>(D) || llvm::isa<BindingDecl>(D)) {
217if (isConst(llvm::cast<ValueDecl>(D)->getType()))
218return true;
219}
220if (const auto *OCPD = llvm::dyn_cast<ObjCPropertyDecl>(D)) {
221if (OCPD->isReadOnly())
222return true;
223}
224if (const auto *MPD = llvm::dyn_cast<MSPropertyDecl>(D)) {
225if (!MPD->hasSetter())
226return true;
227}
228if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D)) {
229if (CMD->isConst())
230return true;
231}
232return false;
233}
234
235// "Static" means many things in C++, only some get the "static" modifier.
236//
237// Meanings that do:
238// - Members associated with the class rather than the instance.
239// This is what 'static' most often means across languages.
240// - static local variables
241// These are similarly "detached from their context" by the static keyword.
242// In practice, these are rarely used inside classes, reducing confusion.
243//
244// Meanings that don't:
245// - Namespace-scoped variables, which have static storage class.
246// This is implicit, so the keyword "static" isn't so strongly associated.
247// If we want a modifier for these, "global scope" is probably the concept.
248// - Namespace-scoped variables/functions explicitly marked "static".
249// There the keyword changes *linkage* , which is a totally different concept.
250// If we want to model this, "file scope" would be a nice modifier.
251//
252// This is confusing, and maybe we should use another name, but because "static"
253// is a standard LSP modifier, having one with that name has advantages.
254bool isStatic(const Decl *D) {
255if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
256return CMD->isStatic();
257if (const VarDecl *VD = llvm::dyn_cast<VarDecl>(D))
258return VD->isStaticDataMember() || VD->isStaticLocal();
259if (const auto *OPD = llvm::dyn_cast<ObjCPropertyDecl>(D))
260return OPD->isClassProperty();
261if (const auto *OMD = llvm::dyn_cast<ObjCMethodDecl>(D))
262return OMD->isClassMethod();
263return false;
264}
265
266bool isAbstract(const Decl *D) {
267if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
268return CMD->isPureVirtual();
269if (const auto *CRD = llvm::dyn_cast<CXXRecordDecl>(D))
270return CRD->hasDefinition() && CRD->isAbstract();
271return false;
272}
273
274bool isVirtual(const Decl *D) {
275if (const auto *CMD = llvm::dyn_cast<CXXMethodDecl>(D))
276return CMD->isVirtual();
277return false;
278}
279
280bool isDependent(const Decl *D) {
281if (isa<UnresolvedUsingValueDecl>(D))
282return true;
283return false;
284}
285
286/// Returns true if `Decl` is considered to be from a default/system library.
287/// This currently checks the systemness of the file by include type, although
288/// different heuristics may be used in the future (e.g. sysroot paths).
289bool isDefaultLibrary(const Decl *D) {
290SourceLocation Loc = D->getLocation();
291if (!Loc.isValid())
292return false;
293return D->getASTContext().getSourceManager().isInSystemHeader(Loc);
294}
295
296bool isDefaultLibrary(const Type *T) {
297if (!T)
298return false;
299const Type *Underlying = T->getPointeeOrArrayElementType();
300if (Underlying->isBuiltinType())
301return true;
302if (auto *TD = dyn_cast<TemplateTypeParmType>(Underlying))
303return isDefaultLibrary(TD->getDecl());
304if (auto *TD = Underlying->getAsTagDecl())
305return isDefaultLibrary(TD);
306return false;
307}
308
309// For a macro usage `DUMP(foo)`, we want:
310// - DUMP --> "macro"
311// - foo --> "variable".
312SourceLocation getHighlightableSpellingToken(SourceLocation L,
313const SourceManager &SM) {
314if (L.isFileID())
315return SM.isWrittenInMainFile(L) ? L : SourceLocation{};
316// Tokens expanded from the macro body contribute no highlightings.
317if (!SM.isMacroArgExpansion(L))
318return {};
319// Tokens expanded from macro args are potentially highlightable.
320return getHighlightableSpellingToken(SM.getImmediateSpellingLoc(L), SM);
321}
322
323unsigned evaluateHighlightPriority(const HighlightingToken &Tok) {
324enum HighlightPriority { Dependent = 0, Resolved = 1 };
325return (Tok.Modifiers & (1 << uint32_t(HighlightingModifier::DependentName)))
326? Dependent
327: Resolved;
328}
329
330// Sometimes we get multiple tokens at the same location:
331//
332// - findExplicitReferences() returns a heuristic result for a dependent name
333// (e.g. Method) and CollectExtraHighlighting returning a fallback dependent
334// highlighting (e.g. Unknown+Dependent).
335// - macro arguments are expanded multiple times and have different roles
336// - broken code recovery produces several AST nodes at the same location
337//
338// We should either resolve these to a single token, or drop them all.
339// Our heuristics are:
340//
341// - token kinds that come with "dependent-name" modifiers are less reliable
342// (these tend to be vague, like Type or Unknown)
343// - if we have multiple equally reliable kinds, drop token rather than guess
344// - take the union of modifiers from all tokens
345//
346// In particular, heuristically resolved dependent names get their heuristic
347// kind, plus the dependent modifier.
348std::optional<HighlightingToken> resolveConflict(const HighlightingToken &A,
349const HighlightingToken &B) {
350unsigned Priority1 = evaluateHighlightPriority(A);
351unsigned Priority2 = evaluateHighlightPriority(B);
352if (Priority1 == Priority2 && A.Kind != B.Kind)
353return std::nullopt;
354auto Result = Priority1 > Priority2 ? A : B;
355Result.Modifiers = A.Modifiers | B.Modifiers;
356return Result;
357}
358std::optional<HighlightingToken>
359resolveConflict(ArrayRef<HighlightingToken> Tokens) {
360if (Tokens.size() == 1)
361return Tokens[0];
362
363assert(Tokens.size() >= 2);
364std::optional<HighlightingToken> Winner =
365resolveConflict(Tokens[0], Tokens[1]);
366for (size_t I = 2; Winner && I < Tokens.size(); ++I)
367Winner = resolveConflict(*Winner, Tokens[I]);
368return Winner;
369}
370
371/// Filter to remove particular kinds of highlighting tokens and modifiers from
372/// the output.
373class HighlightingFilter {
374public:
375HighlightingFilter() {
376for (auto &Active : ActiveKindLookup)
377Active = true;
378
379ActiveModifiersMask = ~0;
380}
381
382void disableKind(HighlightingKind Kind) {
383ActiveKindLookup[static_cast<size_t>(Kind)] = false;
384}
385
386void disableModifier(HighlightingModifier Modifier) {
387ActiveModifiersMask &= ~(1 << static_cast<uint32_t>(Modifier));
388}
389
390bool isHighlightKindActive(HighlightingKind Kind) const {
391return ActiveKindLookup[static_cast<size_t>(Kind)];
392}
393
394uint32_t maskModifiers(uint32_t Modifiers) const {
395return Modifiers & ActiveModifiersMask;
396}
397
398static HighlightingFilter fromCurrentConfig() {
399const Config &C = Config::current();
400HighlightingFilter Filter;
401for (const auto &Kind : C.SemanticTokens.DisabledKinds)
402if (auto K = highlightingKindFromString(Kind))
403Filter.disableKind(*K);
404for (const auto &Modifier : C.SemanticTokens.DisabledModifiers)
405if (auto M = highlightingModifierFromString(Modifier))
406Filter.disableModifier(*M);
407
408return Filter;
409}
410
411private:
412bool ActiveKindLookup[static_cast<size_t>(HighlightingKind::LastKind) + 1];
413uint32_t ActiveModifiersMask;
414};
415
416/// Consumes source locations and maps them to text ranges for highlightings.
417class HighlightingsBuilder {
418public:
419HighlightingsBuilder(const ParsedAST &AST, const HighlightingFilter &Filter)
420: TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
421LangOpts(AST.getLangOpts()), Filter(Filter),
422Resolver(AST.getHeuristicResolver()) {}
423
424HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
425auto Range = getRangeForSourceLocation(Loc);
426if (!Range)
427return InvalidHighlightingToken;
428
429return addToken(*Range, Kind);
430}
431
432// Most of this function works around
433// https://github.com/clangd/clangd/issues/871.
434void addAngleBracketTokens(SourceLocation LLoc, SourceLocation RLoc) {
435if (!LLoc.isValid() || !RLoc.isValid())
436return;
437
438auto LRange = getRangeForSourceLocation(LLoc);
439if (!LRange)
440return;
441
442// RLoc might be pointing at a virtual buffer when it's part of a `>>`
443// token.
444RLoc = SourceMgr.getFileLoc(RLoc);
445// Make sure token is part of the main file.
446RLoc = getHighlightableSpellingToken(RLoc, SourceMgr);
447if (!RLoc.isValid())
448return;
449
450const auto *RTok = TB.spelledTokenContaining(RLoc);
451// Handle `>>`. RLoc is either part of `>>` or a spelled token on its own
452// `>`. If it's the former, slice to have length of 1, if latter use the
453// token as-is.
454if (!RTok || RTok->kind() == tok::greatergreater) {
455Position Begin = sourceLocToPosition(SourceMgr, RLoc);
456Position End = sourceLocToPosition(SourceMgr, RLoc.getLocWithOffset(1));
457addToken(*LRange, HighlightingKind::Bracket);
458addToken({Begin, End}, HighlightingKind::Bracket);
459return;
460}
461
462// Easy case, we have the `>` token directly available.
463if (RTok->kind() == tok::greater) {
464if (auto RRange = getRangeForSourceLocation(RLoc)) {
465addToken(*LRange, HighlightingKind::Bracket);
466addToken(*RRange, HighlightingKind::Bracket);
467}
468return;
469}
470}
471
472HighlightingToken &addToken(Range R, HighlightingKind Kind) {
473if (!Filter.isHighlightKindActive(Kind))
474return InvalidHighlightingToken;
475
476HighlightingToken HT;
477HT.R = std::move(R);
478HT.Kind = Kind;
479Tokens.push_back(std::move(HT));
480return Tokens.back();
481}
482
483void addExtraModifier(SourceLocation Loc, HighlightingModifier Modifier) {
484if (auto Range = getRangeForSourceLocation(Loc))
485ExtraModifiers[*Range].push_back(Modifier);
486}
487
488std::vector<HighlightingToken> collect(ParsedAST &AST) && {
489// Initializer lists can give duplicates of tokens, therefore all tokens
490// must be deduplicated.
491llvm::sort(Tokens);
492auto Last = std::unique(Tokens.begin(), Tokens.end());
493Tokens.erase(Last, Tokens.end());
494
495// Macros can give tokens that have the same source range but conflicting
496// kinds. In this case all tokens sharing this source range should be
497// removed.
498std::vector<HighlightingToken> NonConflicting;
499NonConflicting.reserve(Tokens.size());
500for (ArrayRef<HighlightingToken> TokRef = Tokens; !TokRef.empty();) {
501ArrayRef<HighlightingToken> Conflicting =
502TokRef.take_while([&](const HighlightingToken &T) {
503// TokRef is guaranteed at least one element here because otherwise
504// this predicate would never fire.
505return T.R == TokRef.front().R;
506});
507if (auto Resolved = resolveConflict(Conflicting)) {
508// Apply extra collected highlighting modifiers
509auto Modifiers = ExtraModifiers.find(Resolved->R);
510if (Modifiers != ExtraModifiers.end()) {
511for (HighlightingModifier Mod : Modifiers->second) {
512Resolved->addModifier(Mod);
513}
514}
515
516Resolved->Modifiers = Filter.maskModifiers(Resolved->Modifiers);
517NonConflicting.push_back(*Resolved);
518}
519// TokRef[Conflicting.size()] is the next token with a different range (or
520// the end of the Tokens).
521TokRef = TokRef.drop_front(Conflicting.size());
522}
523
524if (!Filter.isHighlightKindActive(HighlightingKind::InactiveCode))
525return NonConflicting;
526
527const auto &SM = AST.getSourceManager();
528StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
529
530// Merge token stream with "inactive line" markers.
531std::vector<HighlightingToken> WithInactiveLines;
532auto SortedInactiveRegions = getInactiveRegions(AST);
533llvm::sort(SortedInactiveRegions);
534auto It = NonConflicting.begin();
535for (const Range &R : SortedInactiveRegions) {
536// Create one token for each line in the inactive range, so it works
537// with line-based diffing.
538assert(R.start.line <= R.end.line);
539for (int Line = R.start.line; Line <= R.end.line; ++Line) {
540// Copy tokens before the inactive line
541for (; It != NonConflicting.end() && It->R.start.line < Line; ++It)
542WithInactiveLines.push_back(std::move(*It));
543// Add a token for the inactive line itself.
544auto EndOfLine = endOfLine(MainCode, Line);
545if (EndOfLine) {
546HighlightingToken HT;
547WithInactiveLines.emplace_back();
548WithInactiveLines.back().Kind = HighlightingKind::InactiveCode;
549WithInactiveLines.back().R.start.line = Line;
550WithInactiveLines.back().R.end = *EndOfLine;
551} else {
552elog("Failed to determine end of line: {0}", EndOfLine.takeError());
553}
554
555// Skip any other tokens on the inactive line. e.g.
556// `#ifndef Foo` is considered as part of an inactive region when Foo is
557// defined, and there is a Foo macro token.
558// FIXME: we should reduce the scope of the inactive region to not
559// include the directive itself.
560while (It != NonConflicting.end() && It->R.start.line == Line)
561++It;
562}
563}
564// Copy tokens after the last inactive line
565for (; It != NonConflicting.end(); ++It)
566WithInactiveLines.push_back(std::move(*It));
567return WithInactiveLines;
568}
569
570const HeuristicResolver *getResolver() const { return Resolver; }
571
572private:
573std::optional<Range> getRangeForSourceLocation(SourceLocation Loc) {
574Loc = getHighlightableSpellingToken(Loc, SourceMgr);
575if (Loc.isInvalid())
576return std::nullopt;
577// We might have offsets in the main file that don't correspond to any
578// spelled tokens.
579const auto *Tok = TB.spelledTokenContaining(Loc);
580if (!Tok)
581return std::nullopt;
582return halfOpenToRange(SourceMgr,
583Tok->range(SourceMgr).toCharRange(SourceMgr));
584}
585
586const syntax::TokenBuffer &TB;
587const SourceManager &SourceMgr;
588const LangOptions &LangOpts;
589HighlightingFilter Filter;
590std::vector<HighlightingToken> Tokens;
591std::map<Range, llvm::SmallVector<HighlightingModifier, 1>> ExtraModifiers;
592const HeuristicResolver *Resolver;
593// returned from addToken(InvalidLoc)
594HighlightingToken InvalidHighlightingToken;
595};
596
597std::optional<HighlightingModifier> scopeModifier(const NamedDecl *D) {
598const DeclContext *DC = D->getDeclContext();
599// Injected "Foo" within the class "Foo" has file scope, not class scope.
600if (auto *R = dyn_cast_or_null<RecordDecl>(D))
601if (R->isInjectedClassName())
602DC = DC->getParent();
603// Lambda captures are considered function scope, not class scope.
604if (llvm::isa<FieldDecl>(D))
605if (const auto *RD = llvm::dyn_cast<RecordDecl>(DC))
606if (RD->isLambda())
607return HighlightingModifier::FunctionScope;
608// Walk up the DeclContext hierarchy until we find something interesting.
609for (; !DC->isFileContext(); DC = DC->getParent()) {
610if (DC->isFunctionOrMethod())
611return HighlightingModifier::FunctionScope;
612if (DC->isRecord())
613return HighlightingModifier::ClassScope;
614}
615// Some template parameters (e.g. those for variable templates) don't have
616// meaningful DeclContexts. That doesn't mean they're global!
617if (DC->isTranslationUnit() && D->isTemplateParameter())
618return std::nullopt;
619// ExternalLinkage threshold could be tweaked, e.g. module-visible as global.
620if (llvm::to_underlying(D->getLinkageInternal()) <
621llvm::to_underlying(Linkage::External))
622return HighlightingModifier::FileScope;
623return HighlightingModifier::GlobalScope;
624}
625
626std::optional<HighlightingModifier> scopeModifier(const Type *T) {
627if (!T)
628return std::nullopt;
629if (T->isBuiltinType())
630return HighlightingModifier::GlobalScope;
631if (auto *TD = dyn_cast<TemplateTypeParmType>(T))
632return scopeModifier(TD->getDecl());
633if (auto *TD = T->getAsTagDecl())
634return scopeModifier(TD);
635return std::nullopt;
636}
637
638/// Produces highlightings, which are not captured by findExplicitReferences,
639/// e.g. highlights dependent names and 'auto' as the underlying type.
640class CollectExtraHighlightings
641: public RecursiveASTVisitor<CollectExtraHighlightings> {
642using Base = RecursiveASTVisitor<CollectExtraHighlightings>;
643
644public:
645CollectExtraHighlightings(HighlightingsBuilder &H) : H(H) {}
646
647bool VisitCXXConstructExpr(CXXConstructExpr *E) {
648highlightMutableReferenceArguments(E->getConstructor(),
649{E->getArgs(), E->getNumArgs()});
650
651return true;
652}
653
654bool TraverseConstructorInitializer(CXXCtorInitializer *Init) {
655if (Init->isMemberInitializer())
656if (auto *Member = Init->getMember())
657highlightMutableReferenceArgument(Member->getType(), Init->getInit());
658return Base::TraverseConstructorInitializer(Init);
659}
660
661bool TraverseTypeConstraint(const TypeConstraint *C) {
662if (auto *Args = C->getTemplateArgsAsWritten())
663H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
664return Base::TraverseTypeConstraint(C);
665}
666
667bool VisitPredefinedExpr(PredefinedExpr *E) {
668H.addToken(E->getLocation(), HighlightingKind::LocalVariable)
669.addModifier(HighlightingModifier::Static)
670.addModifier(HighlightingModifier::Readonly)
671.addModifier(HighlightingModifier::FunctionScope);
672return true;
673}
674
675bool VisitConceptSpecializationExpr(ConceptSpecializationExpr *E) {
676if (auto *Args = E->getTemplateArgsAsWritten())
677H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
678return true;
679}
680
681bool VisitTemplateDecl(TemplateDecl *D) {
682if (auto *TPL = D->getTemplateParameters())
683H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
684return true;
685}
686
687bool VisitTagDecl(TagDecl *D) {
688for (unsigned i = 0; i < D->getNumTemplateParameterLists(); ++i) {
689if (auto *TPL = D->getTemplateParameterList(i))
690H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
691}
692return true;
693}
694
695bool
696VisitClassTemplateSpecializationDecl(ClassTemplateSpecializationDecl *D) {
697if (auto *Args = D->getTemplateArgsAsWritten())
698H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
699return true;
700}
701
702bool VisitClassTemplatePartialSpecializationDecl(
703ClassTemplatePartialSpecializationDecl *D) {
704if (auto *TPL = D->getTemplateParameters())
705H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
706return true;
707}
708
709bool VisitVarTemplateSpecializationDecl(VarTemplateSpecializationDecl *D) {
710if (auto *Args = D->getTemplateArgsAsWritten())
711H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
712return true;
713}
714
715bool VisitVarTemplatePartialSpecializationDecl(
716VarTemplatePartialSpecializationDecl *D) {
717if (auto *TPL = D->getTemplateParameters())
718H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
719return true;
720}
721
722bool VisitDeclRefExpr(DeclRefExpr *E) {
723H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
724return true;
725}
726bool VisitMemberExpr(MemberExpr *E) {
727H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
728return true;
729}
730
731bool VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc L) {
732H.addAngleBracketTokens(L.getLAngleLoc(), L.getRAngleLoc());
733return true;
734}
735
736bool VisitFunctionDecl(FunctionDecl *D) {
737if (D->isOverloadedOperator()) {
738const auto AddOpDeclToken = [&](SourceLocation Loc) {
739auto &Token = H.addToken(Loc, HighlightingKind::Operator)
740.addModifier(HighlightingModifier::Declaration);
741if (D->isThisDeclarationADefinition())
742Token.addModifier(HighlightingModifier::Definition);
743};
744const auto Range = D->getNameInfo().getCXXOperatorNameRange();
745AddOpDeclToken(Range.getBegin());
746const auto Kind = D->getOverloadedOperator();
747if (Kind == OO_Call || Kind == OO_Subscript)
748AddOpDeclToken(Range.getEnd());
749}
750if (auto *Args = D->getTemplateSpecializationArgsAsWritten())
751H.addAngleBracketTokens(Args->getLAngleLoc(), Args->getRAngleLoc());
752return true;
753}
754
755bool VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
756const auto AddOpToken = [&](SourceLocation Loc) {
757H.addToken(Loc, HighlightingKind::Operator)
758.addModifier(HighlightingModifier::UserDefined);
759};
760AddOpToken(E->getOperatorLoc());
761const auto Kind = E->getOperator();
762if (Kind == OO_Call || Kind == OO_Subscript) {
763if (auto *Callee = E->getCallee())
764AddOpToken(Callee->getBeginLoc());
765}
766return true;
767}
768
769bool VisitUnaryOperator(UnaryOperator *Op) {
770auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);
771if (Op->getSubExpr()->isTypeDependent())
772Token.addModifier(HighlightingModifier::UserDefined);
773return true;
774}
775
776bool VisitBinaryOperator(BinaryOperator *Op) {
777auto &Token = H.addToken(Op->getOperatorLoc(), HighlightingKind::Operator);
778if (Op->getLHS()->isTypeDependent() || Op->getRHS()->isTypeDependent())
779Token.addModifier(HighlightingModifier::UserDefined);
780return true;
781}
782
783bool VisitConditionalOperator(ConditionalOperator *Op) {
784H.addToken(Op->getQuestionLoc(), HighlightingKind::Operator);
785H.addToken(Op->getColonLoc(), HighlightingKind::Operator);
786return true;
787}
788
789bool VisitCXXNewExpr(CXXNewExpr *E) {
790auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);
791if (isa_and_present<CXXMethodDecl>(E->getOperatorNew()))
792Token.addModifier(HighlightingModifier::UserDefined);
793return true;
794}
795
796bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
797auto &Token = H.addToken(E->getBeginLoc(), HighlightingKind::Operator);
798if (isa_and_present<CXXMethodDecl>(E->getOperatorDelete()))
799Token.addModifier(HighlightingModifier::UserDefined);
800return true;
801}
802
803bool VisitCXXNamedCastExpr(CXXNamedCastExpr *E) {
804const auto &B = E->getAngleBrackets();
805H.addAngleBracketTokens(B.getBegin(), B.getEnd());
806return true;
807}
808
809bool VisitCallExpr(CallExpr *E) {
810// Highlighting parameters passed by non-const reference does not really
811// make sense for literals...
812if (isa<UserDefinedLiteral>(E))
813return true;
814
815// FIXME: consider highlighting parameters of some other overloaded
816// operators as well
817llvm::ArrayRef<const Expr *> Args = {E->getArgs(), E->getNumArgs()};
818if (auto *CallOp = dyn_cast<CXXOperatorCallExpr>(E)) {
819switch (CallOp->getOperator()) {
820case OO_Call:
821case OO_Subscript:
822Args = Args.drop_front(); // Drop object parameter
823break;
824default:
825return true;
826}
827}
828
829highlightMutableReferenceArguments(
830dyn_cast_or_null<FunctionDecl>(E->getCalleeDecl()), Args);
831
832return true;
833}
834
835void highlightMutableReferenceArgument(QualType T, const Expr *Arg) {
836if (!Arg)
837return;
838
839// Is this parameter passed by non-const pointer or reference?
840// FIXME The condition T->idDependentType() could be relaxed a bit,
841// e.g. std::vector<T>& is dependent but we would want to highlight it
842bool IsRef = T->isLValueReferenceType();
843bool IsPtr = T->isPointerType();
844if ((!IsRef && !IsPtr) || T->getPointeeType().isConstQualified() ||
845T->isDependentType()) {
846return;
847}
848
849std::optional<SourceLocation> Location;
850
851// FIXME Add "unwrapping" for ArraySubscriptExpr,
852// e.g. highlight `a` in `a[i]`
853// FIXME Handle dependent expression types
854if (auto *IC = dyn_cast<ImplicitCastExpr>(Arg))
855Arg = IC->getSubExprAsWritten();
856if (auto *UO = dyn_cast<UnaryOperator>(Arg)) {
857if (UO->getOpcode() == UO_AddrOf)
858Arg = UO->getSubExpr();
859}
860if (auto *DR = dyn_cast<DeclRefExpr>(Arg))
861Location = DR->getLocation();
862else if (auto *M = dyn_cast<MemberExpr>(Arg))
863Location = M->getMemberLoc();
864
865if (Location)
866H.addExtraModifier(*Location,
867IsRef ? HighlightingModifier::UsedAsMutableReference
868: HighlightingModifier::UsedAsMutablePointer);
869}
870
871void
872highlightMutableReferenceArguments(const FunctionDecl *FD,
873llvm::ArrayRef<const Expr *const> Args) {
874if (!FD)
875return;
876
877if (auto *ProtoType = FD->getType()->getAs<FunctionProtoType>()) {
878// Iterate over the types of the function parameters.
879// If any of them are non-const reference paramteres, add it as a
880// highlighting modifier to the corresponding expression
881for (size_t I = 0;
882I < std::min(size_t(ProtoType->getNumParams()), Args.size()); ++I) {
883highlightMutableReferenceArgument(ProtoType->getParamType(I), Args[I]);
884}
885}
886}
887
888bool VisitDecltypeTypeLoc(DecltypeTypeLoc L) {
889if (auto K = kindForType(L.getTypePtr(), H.getResolver())) {
890auto &Tok = H.addToken(L.getBeginLoc(), *K)
891.addModifier(HighlightingModifier::Deduced);
892if (auto Mod = scopeModifier(L.getTypePtr()))
893Tok.addModifier(*Mod);
894if (isDefaultLibrary(L.getTypePtr()))
895Tok.addModifier(HighlightingModifier::DefaultLibrary);
896}
897return true;
898}
899
900bool VisitCXXDestructorDecl(CXXDestructorDecl *D) {
901if (auto *TI = D->getNameInfo().getNamedTypeInfo()) {
902SourceLocation Loc = TI->getTypeLoc().getBeginLoc();
903H.addExtraModifier(Loc, HighlightingModifier::ConstructorOrDestructor);
904H.addExtraModifier(Loc, HighlightingModifier::Declaration);
905if (D->isThisDeclarationADefinition())
906H.addExtraModifier(Loc, HighlightingModifier::Definition);
907}
908return true;
909}
910
911bool VisitCXXMemberCallExpr(CXXMemberCallExpr *CE) {
912// getMethodDecl can return nullptr with member pointers, e.g.
913// `(foo.*pointer_to_member_fun)(arg);`
914if (auto *D = CE->getMethodDecl()) {
915if (isa<CXXDestructorDecl>(D)) {
916if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee())) {
917if (auto *TI = ME->getMemberNameInfo().getNamedTypeInfo()) {
918H.addExtraModifier(TI->getTypeLoc().getBeginLoc(),
919HighlightingModifier::ConstructorOrDestructor);
920}
921}
922} else if (D->isOverloadedOperator()) {
923if (auto *ME = dyn_cast<MemberExpr>(CE->getCallee()))
924H.addToken(
925ME->getMemberNameInfo().getCXXOperatorNameRange().getBegin(),
926HighlightingKind::Operator)
927.addModifier(HighlightingModifier::UserDefined);
928}
929}
930return true;
931}
932
933bool VisitDeclaratorDecl(DeclaratorDecl *D) {
934for (unsigned i = 0; i < D->getNumTemplateParameterLists(); ++i) {
935if (auto *TPL = D->getTemplateParameterList(i))
936H.addAngleBracketTokens(TPL->getLAngleLoc(), TPL->getRAngleLoc());
937}
938auto *AT = D->getType()->getContainedAutoType();
939if (!AT)
940return true;
941auto K =
942kindForType(AT->getDeducedType().getTypePtrOrNull(), H.getResolver());
943if (!K)
944return true;
945auto *TSI = D->getTypeSourceInfo();
946if (!TSI)
947return true;
948SourceLocation StartLoc =
949TSI->getTypeLoc().getContainedAutoTypeLoc().getNameLoc();
950// The AutoType may not have a corresponding token, e.g. in the case of
951// init-captures. In this case, StartLoc overlaps with the location
952// of the decl itself, and producing a token for the type here would result
953// in both it and the token for the decl being dropped due to conflict.
954if (StartLoc == D->getLocation())
955return true;
956
957auto &Tok =
958H.addToken(StartLoc, *K).addModifier(HighlightingModifier::Deduced);
959const Type *Deduced = AT->getDeducedType().getTypePtrOrNull();
960if (auto Mod = scopeModifier(Deduced))
961Tok.addModifier(*Mod);
962if (isDefaultLibrary(Deduced))
963Tok.addModifier(HighlightingModifier::DefaultLibrary);
964return true;
965}
966
967// We handle objective-C selectors specially, because one reference can
968// cover several non-contiguous tokens.
969void highlightObjCSelector(const ArrayRef<SourceLocation> &Locs, bool Decl,
970bool Def, bool Class, bool DefaultLibrary) {
971HighlightingKind Kind =
972Class ? HighlightingKind::StaticMethod : HighlightingKind::Method;
973for (SourceLocation Part : Locs) {
974auto &Tok =
975H.addToken(Part, Kind).addModifier(HighlightingModifier::ClassScope);
976if (Decl)
977Tok.addModifier(HighlightingModifier::Declaration);
978if (Def)
979Tok.addModifier(HighlightingModifier::Definition);
980if (Class)
981Tok.addModifier(HighlightingModifier::Static);
982if (DefaultLibrary)
983Tok.addModifier(HighlightingModifier::DefaultLibrary);
984}
985}
986
987bool VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
988llvm::SmallVector<SourceLocation> Locs;
989OMD->getSelectorLocs(Locs);
990highlightObjCSelector(Locs, /*Decl=*/true,
991OMD->isThisDeclarationADefinition(),
992OMD->isClassMethod(), isDefaultLibrary(OMD));
993return true;
994}
995
996bool VisitObjCMessageExpr(ObjCMessageExpr *OME) {
997llvm::SmallVector<SourceLocation> Locs;
998OME->getSelectorLocs(Locs);
999bool DefaultLibrary = false;
1000if (ObjCMethodDecl *OMD = OME->getMethodDecl())
1001DefaultLibrary = isDefaultLibrary(OMD);
1002highlightObjCSelector(Locs, /*Decl=*/false, /*Def=*/false,
1003OME->isClassMessage(), DefaultLibrary);
1004return true;
1005}
1006
1007// Objective-C allows you to use property syntax `self.prop` as sugar for
1008// `[self prop]` and `[self setProp:]` when there's no explicit `@property`
1009// for `prop` as well as for class properties. We treat this like a property
1010// even though semantically it's equivalent to a method expression.
1011void highlightObjCImplicitPropertyRef(const ObjCMethodDecl *OMD,
1012SourceLocation Loc) {
1013auto &Tok = H.addToken(Loc, HighlightingKind::Field)
1014.addModifier(HighlightingModifier::ClassScope);
1015if (OMD->isClassMethod())
1016Tok.addModifier(HighlightingModifier::Static);
1017if (isDefaultLibrary(OMD))
1018Tok.addModifier(HighlightingModifier::DefaultLibrary);
1019}
1020
1021bool VisitObjCPropertyRefExpr(ObjCPropertyRefExpr *OPRE) {
1022// We need to handle implicit properties here since they will appear to
1023// reference `ObjCMethodDecl` via an implicit `ObjCMessageExpr`, so normal
1024// highlighting will not work.
1025if (!OPRE->isImplicitProperty())
1026return true;
1027// A single property expr can reference both a getter and setter, but we can
1028// only provide a single semantic token, so prefer the getter. In most cases
1029// the end result should be the same, although it's technically possible
1030// that the user defines a setter for a system SDK.
1031if (OPRE->isMessagingGetter()) {
1032highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertyGetter(),
1033OPRE->getLocation());
1034return true;
1035}
1036if (OPRE->isMessagingSetter()) {
1037highlightObjCImplicitPropertyRef(OPRE->getImplicitPropertySetter(),
1038OPRE->getLocation());
1039}
1040return true;
1041}
1042
1043bool VisitOverloadExpr(OverloadExpr *E) {
1044H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
1045if (!E->decls().empty())
1046return true; // handled by findExplicitReferences.
1047auto &Tok = H.addToken(E->getNameLoc(), HighlightingKind::Unknown)
1048.addModifier(HighlightingModifier::DependentName);
1049if (llvm::isa<UnresolvedMemberExpr>(E))
1050Tok.addModifier(HighlightingModifier::ClassScope);
1051// other case is UnresolvedLookupExpr, scope is unknown.
1052return true;
1053}
1054
1055bool VisitCXXDependentScopeMemberExpr(CXXDependentScopeMemberExpr *E) {
1056H.addToken(E->getMemberNameInfo().getLoc(), HighlightingKind::Unknown)
1057.addModifier(HighlightingModifier::DependentName)
1058.addModifier(HighlightingModifier::ClassScope);
1059H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
1060return true;
1061}
1062
1063bool VisitDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E) {
1064H.addToken(E->getNameInfo().getLoc(), HighlightingKind::Unknown)
1065.addModifier(HighlightingModifier::DependentName)
1066.addModifier(HighlightingModifier::ClassScope);
1067H.addAngleBracketTokens(E->getLAngleLoc(), E->getRAngleLoc());
1068return true;
1069}
1070
1071bool VisitAttr(Attr *A) {
1072switch (A->getKind()) {
1073case attr::Override:
1074case attr::Final:
1075H.addToken(A->getLocation(), HighlightingKind::Modifier);
1076break;
1077default:
1078break;
1079}
1080return true;
1081}
1082
1083bool VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
1084H.addToken(L.getNameLoc(), HighlightingKind::Type)
1085.addModifier(HighlightingModifier::DependentName)
1086.addModifier(HighlightingModifier::ClassScope);
1087return true;
1088}
1089
1090bool VisitDependentTemplateSpecializationTypeLoc(
1091DependentTemplateSpecializationTypeLoc L) {
1092H.addToken(L.getTemplateNameLoc(), HighlightingKind::Type)
1093.addModifier(HighlightingModifier::DependentName)
1094.addModifier(HighlightingModifier::ClassScope);
1095H.addAngleBracketTokens(L.getLAngleLoc(), L.getRAngleLoc());
1096return true;
1097}
1098
1099bool TraverseTemplateArgumentLoc(TemplateArgumentLoc L) {
1100// Handle template template arguments only (other arguments are handled by
1101// their Expr, TypeLoc etc values).
1102if (L.getArgument().getKind() != TemplateArgument::Template &&
1103L.getArgument().getKind() != TemplateArgument::TemplateExpansion)
1104return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
1105
1106TemplateName N = L.getArgument().getAsTemplateOrTemplatePattern();
1107switch (N.getKind()) {
1108case TemplateName::OverloadedTemplate:
1109// Template template params must always be class templates.
1110// Don't bother to try to work out the scope here.
1111H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class);
1112break;
1113case TemplateName::DependentTemplate:
1114case TemplateName::AssumedTemplate:
1115H.addToken(L.getTemplateNameLoc(), HighlightingKind::Class)
1116.addModifier(HighlightingModifier::DependentName);
1117break;
1118case TemplateName::Template:
1119case TemplateName::QualifiedTemplate:
1120case TemplateName::SubstTemplateTemplateParm:
1121case TemplateName::SubstTemplateTemplateParmPack:
1122case TemplateName::UsingTemplate:
1123// Names that could be resolved to a TemplateDecl are handled elsewhere.
1124break;
1125}
1126return RecursiveASTVisitor::TraverseTemplateArgumentLoc(L);
1127}
1128
1129// findExplicitReferences will walk nested-name-specifiers and
1130// find anything that can be resolved to a Decl. However, non-leaf
1131// components of nested-name-specifiers which are dependent names
1132// (kind "Identifier") cannot be resolved to a decl, so we visit
1133// them here.
1134bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc Q) {
1135if (NestedNameSpecifier *NNS = Q.getNestedNameSpecifier()) {
1136if (NNS->getKind() == NestedNameSpecifier::Identifier)
1137H.addToken(Q.getLocalBeginLoc(), HighlightingKind::Type)
1138.addModifier(HighlightingModifier::DependentName)
1139.addModifier(HighlightingModifier::ClassScope);
1140}
1141return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(Q);
1142}
1143
1144private:
1145HighlightingsBuilder &H;
1146};
1147} // namespace
1148
1149std::vector<HighlightingToken>
1150getSemanticHighlightings(ParsedAST &AST, bool IncludeInactiveRegionTokens) {
1151auto &C = AST.getASTContext();
1152HighlightingFilter Filter = HighlightingFilter::fromCurrentConfig();
1153if (!IncludeInactiveRegionTokens)
1154Filter.disableKind(HighlightingKind::InactiveCode);
1155// Add highlightings for AST nodes.
1156HighlightingsBuilder Builder(AST, Filter);
1157// Highlight 'decltype' and 'auto' as their underlying types.
1158CollectExtraHighlightings(Builder).TraverseAST(C);
1159// Highlight all decls and references coming from the AST.
1160findExplicitReferences(
1161C,
1162[&](ReferenceLoc R) {
1163for (const NamedDecl *Decl : R.Targets) {
1164if (!canHighlightName(Decl->getDeclName()))
1165continue;
1166auto Kind = kindForDecl(Decl, AST.getHeuristicResolver());
1167if (!Kind)
1168continue;
1169auto &Tok = Builder.addToken(R.NameLoc, *Kind);
1170
1171// The attribute tests don't want to look at the template.
1172if (auto *TD = dyn_cast<TemplateDecl>(Decl)) {
1173if (auto *Templated = TD->getTemplatedDecl())
1174Decl = Templated;
1175}
1176if (auto Mod = scopeModifier(Decl))
1177Tok.addModifier(*Mod);
1178if (isConst(Decl))
1179Tok.addModifier(HighlightingModifier::Readonly);
1180if (isStatic(Decl))
1181Tok.addModifier(HighlightingModifier::Static);
1182if (isAbstract(Decl))
1183Tok.addModifier(HighlightingModifier::Abstract);
1184if (isVirtual(Decl))
1185Tok.addModifier(HighlightingModifier::Virtual);
1186if (isDependent(Decl))
1187Tok.addModifier(HighlightingModifier::DependentName);
1188if (isDefaultLibrary(Decl))
1189Tok.addModifier(HighlightingModifier::DefaultLibrary);
1190if (Decl->isDeprecated())
1191Tok.addModifier(HighlightingModifier::Deprecated);
1192if (isa<CXXConstructorDecl>(Decl))
1193Tok.addModifier(HighlightingModifier::ConstructorOrDestructor);
1194if (R.IsDecl) {
1195// Do not treat an UnresolvedUsingValueDecl as a declaration.
1196// It's more common to think of it as a reference to the
1197// underlying declaration.
1198if (!isa<UnresolvedUsingValueDecl>(Decl))
1199Tok.addModifier(HighlightingModifier::Declaration);
1200if (isUniqueDefinition(Decl))
1201Tok.addModifier(HighlightingModifier::Definition);
1202}
1203}
1204},
1205AST.getHeuristicResolver());
1206// Add highlightings for macro references.
1207auto AddMacro = [&](const MacroOccurrence &M) {
1208auto &T = Builder.addToken(M.toRange(C.getSourceManager()),
1209HighlightingKind::Macro);
1210T.addModifier(HighlightingModifier::GlobalScope);
1211if (M.IsDefinition)
1212T.addModifier(HighlightingModifier::Declaration);
1213};
1214for (const auto &SIDToRefs : AST.getMacros().MacroRefs)
1215for (const auto &M : SIDToRefs.second)
1216AddMacro(M);
1217for (const auto &M : AST.getMacros().UnknownMacros)
1218AddMacro(M);
1219
1220return std::move(Builder).collect(AST);
1221}
1222
1223llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
1224switch (K) {
1225case HighlightingKind::Variable:
1226return OS << "Variable";
1227case HighlightingKind::LocalVariable:
1228return OS << "LocalVariable";
1229case HighlightingKind::Parameter:
1230return OS << "Parameter";
1231case HighlightingKind::Function:
1232return OS << "Function";
1233case HighlightingKind::Method:
1234return OS << "Method";
1235case HighlightingKind::StaticMethod:
1236return OS << "StaticMethod";
1237case HighlightingKind::Field:
1238return OS << "Field";
1239case HighlightingKind::StaticField:
1240return OS << "StaticField";
1241case HighlightingKind::Class:
1242return OS << "Class";
1243case HighlightingKind::Interface:
1244return OS << "Interface";
1245case HighlightingKind::Enum:
1246return OS << "Enum";
1247case HighlightingKind::EnumConstant:
1248return OS << "EnumConstant";
1249case HighlightingKind::Typedef:
1250return OS << "Typedef";
1251case HighlightingKind::Type:
1252return OS << "Type";
1253case HighlightingKind::Unknown:
1254return OS << "Unknown";
1255case HighlightingKind::Namespace:
1256return OS << "Namespace";
1257case HighlightingKind::TemplateParameter:
1258return OS << "TemplateParameter";
1259case HighlightingKind::Concept:
1260return OS << "Concept";
1261case HighlightingKind::Primitive:
1262return OS << "Primitive";
1263case HighlightingKind::Macro:
1264return OS << "Macro";
1265case HighlightingKind::Modifier:
1266return OS << "Modifier";
1267case HighlightingKind::Operator:
1268return OS << "Operator";
1269case HighlightingKind::Bracket:
1270return OS << "Bracket";
1271case HighlightingKind::Label:
1272return OS << "Label";
1273case HighlightingKind::InactiveCode:
1274return OS << "InactiveCode";
1275}
1276llvm_unreachable("invalid HighlightingKind");
1277}
1278std::optional<HighlightingKind>
1279highlightingKindFromString(llvm::StringRef Name) {
1280static llvm::StringMap<HighlightingKind> Lookup = {
1281{"Variable", HighlightingKind::Variable},
1282{"LocalVariable", HighlightingKind::LocalVariable},
1283{"Parameter", HighlightingKind::Parameter},
1284{"Function", HighlightingKind::Function},
1285{"Method", HighlightingKind::Method},
1286{"StaticMethod", HighlightingKind::StaticMethod},
1287{"Field", HighlightingKind::Field},
1288{"StaticField", HighlightingKind::StaticField},
1289{"Class", HighlightingKind::Class},
1290{"Interface", HighlightingKind::Interface},
1291{"Enum", HighlightingKind::Enum},
1292{"EnumConstant", HighlightingKind::EnumConstant},
1293{"Typedef", HighlightingKind::Typedef},
1294{"Type", HighlightingKind::Type},
1295{"Unknown", HighlightingKind::Unknown},
1296{"Namespace", HighlightingKind::Namespace},
1297{"TemplateParameter", HighlightingKind::TemplateParameter},
1298{"Concept", HighlightingKind::Concept},
1299{"Primitive", HighlightingKind::Primitive},
1300{"Macro", HighlightingKind::Macro},
1301{"Modifier", HighlightingKind::Modifier},
1302{"Operator", HighlightingKind::Operator},
1303{"Bracket", HighlightingKind::Bracket},
1304{"InactiveCode", HighlightingKind::InactiveCode},
1305};
1306
1307auto It = Lookup.find(Name);
1308return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
1309}
1310llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
1311switch (K) {
1312case HighlightingModifier::Declaration:
1313return OS << "decl"; // abbreviation for common case
1314case HighlightingModifier::Definition:
1315return OS << "def"; // abbrevation for common case
1316case HighlightingModifier::ConstructorOrDestructor:
1317return OS << "constrDestr";
1318default:
1319return OS << toSemanticTokenModifier(K);
1320}
1321}
1322std::optional<HighlightingModifier>
1323highlightingModifierFromString(llvm::StringRef Name) {
1324static llvm::StringMap<HighlightingModifier> Lookup = {
1325{"Declaration", HighlightingModifier::Declaration},
1326{"Definition", HighlightingModifier::Definition},
1327{"Deprecated", HighlightingModifier::Deprecated},
1328{"Deduced", HighlightingModifier::Deduced},
1329{"Readonly", HighlightingModifier::Readonly},
1330{"Static", HighlightingModifier::Static},
1331{"Abstract", HighlightingModifier::Abstract},
1332{"Virtual", HighlightingModifier::Virtual},
1333{"DependentName", HighlightingModifier::DependentName},
1334{"DefaultLibrary", HighlightingModifier::DefaultLibrary},
1335{"UsedAsMutableReference", HighlightingModifier::UsedAsMutableReference},
1336{"UsedAsMutablePointer", HighlightingModifier::UsedAsMutablePointer},
1337{"ConstructorOrDestructor",
1338HighlightingModifier::ConstructorOrDestructor},
1339{"UserDefined", HighlightingModifier::UserDefined},
1340{"FunctionScope", HighlightingModifier::FunctionScope},
1341{"ClassScope", HighlightingModifier::ClassScope},
1342{"FileScope", HighlightingModifier::FileScope},
1343{"GlobalScope", HighlightingModifier::GlobalScope},
1344};
1345
1346auto It = Lookup.find(Name);
1347return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
1348}
1349
1350bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
1351return std::tie(L.R, L.Kind, L.Modifiers) ==
1352std::tie(R.R, R.Kind, R.Modifiers);
1353}
1354bool operator<(const HighlightingToken &L, const HighlightingToken &R) {
1355return std::tie(L.R, L.Kind, L.Modifiers) <
1356std::tie(R.R, R.Kind, R.Modifiers);
1357}
1358
1359std::vector<SemanticToken>
1360toSemanticTokens(llvm::ArrayRef<HighlightingToken> Tokens,
1361llvm::StringRef Code) {
1362assert(llvm::is_sorted(Tokens));
1363std::vector<SemanticToken> Result;
1364// In case we split a HighlightingToken into multiple tokens (e.g. because it
1365// was spanning multiple lines), this tracks the last one. This prevents
1366// having a copy all the time.
1367HighlightingToken Scratch;
1368const HighlightingToken *Last = nullptr;
1369for (const HighlightingToken &Tok : Tokens) {
1370Result.emplace_back();
1371SemanticToken *Out = &Result.back();
1372// deltaStart/deltaLine are relative if possible.
1373if (Last) {
1374assert(Tok.R.start.line >= Last->R.end.line);
1375Out->deltaLine = Tok.R.start.line - Last->R.end.line;
1376if (Out->deltaLine == 0) {
1377assert(Tok.R.start.character >= Last->R.start.character);
1378Out->deltaStart = Tok.R.start.character - Last->R.start.character;
1379} else {
1380Out->deltaStart = Tok.R.start.character;
1381}
1382} else {
1383Out->deltaLine = Tok.R.start.line;
1384Out->deltaStart = Tok.R.start.character;
1385}
1386Out->tokenType = static_cast<unsigned>(Tok.Kind);
1387Out->tokenModifiers = Tok.Modifiers;
1388Last = &Tok;
1389
1390if (Tok.R.end.line == Tok.R.start.line) {
1391Out->length = Tok.R.end.character - Tok.R.start.character;
1392} else {
1393// If the token spans a line break, split it into multiple pieces for each
1394// line.
1395// This is slow, but multiline tokens are rare.
1396// FIXME: There's a client capability for supporting multiline tokens,
1397// respect that.
1398auto TokStartOffset = llvm::cantFail(positionToOffset(Code, Tok.R.start));
1399// Note that the loop doesn't cover the last line, which has a special
1400// length.
1401for (int I = Tok.R.start.line; I < Tok.R.end.line; ++I) {
1402auto LineEnd = Code.find('\n', TokStartOffset);
1403assert(LineEnd != Code.npos);
1404Out->length = LineEnd - TokStartOffset;
1405// Token continues on next line, right after the line break.
1406TokStartOffset = LineEnd + 1;
1407Result.emplace_back();
1408Out = &Result.back();
1409*Out = Result[Result.size() - 2];
1410// New token starts at the first column of the next line.
1411Out->deltaLine = 1;
1412Out->deltaStart = 0;
1413}
1414// This is the token on last line.
1415Out->length = Tok.R.end.character;
1416// Update the start location for last token, as that's used in the
1417// relative delta calculation for following tokens.
1418Scratch = *Last;
1419Scratch.R.start.line = Tok.R.end.line;
1420Scratch.R.start.character = 0;
1421Last = &Scratch;
1422}
1423}
1424return Result;
1425}
1426llvm::StringRef toSemanticTokenType(HighlightingKind Kind) {
1427switch (Kind) {
1428case HighlightingKind::Variable:
1429case HighlightingKind::LocalVariable:
1430case HighlightingKind::StaticField:
1431return "variable";
1432case HighlightingKind::Parameter:
1433return "parameter";
1434case HighlightingKind::Function:
1435return "function";
1436case HighlightingKind::Method:
1437return "method";
1438case HighlightingKind::StaticMethod:
1439// FIXME: better method with static modifier?
1440return "function";
1441case HighlightingKind::Field:
1442return "property";
1443case HighlightingKind::Class:
1444return "class";
1445case HighlightingKind::Interface:
1446return "interface";
1447case HighlightingKind::Enum:
1448return "enum";
1449case HighlightingKind::EnumConstant:
1450return "enumMember";
1451case HighlightingKind::Typedef:
1452case HighlightingKind::Type:
1453return "type";
1454case HighlightingKind::Unknown:
1455return "unknown"; // nonstandard
1456case HighlightingKind::Namespace:
1457return "namespace";
1458case HighlightingKind::TemplateParameter:
1459return "typeParameter";
1460case HighlightingKind::Concept:
1461return "concept"; // nonstandard
1462case HighlightingKind::Primitive:
1463return "type";
1464case HighlightingKind::Macro:
1465return "macro";
1466case HighlightingKind::Modifier:
1467return "modifier";
1468case HighlightingKind::Operator:
1469return "operator";
1470case HighlightingKind::Bracket:
1471return "bracket";
1472case HighlightingKind::Label:
1473return "label";
1474case HighlightingKind::InactiveCode:
1475return "comment";
1476}
1477llvm_unreachable("unhandled HighlightingKind");
1478}
1479
1480llvm::StringRef toSemanticTokenModifier(HighlightingModifier Modifier) {
1481switch (Modifier) {
1482case HighlightingModifier::Declaration:
1483return "declaration";
1484case HighlightingModifier::Definition:
1485return "definition";
1486case HighlightingModifier::Deprecated:
1487return "deprecated";
1488case HighlightingModifier::Readonly:
1489return "readonly";
1490case HighlightingModifier::Static:
1491return "static";
1492case HighlightingModifier::Deduced:
1493return "deduced"; // nonstandard
1494case HighlightingModifier::Abstract:
1495return "abstract";
1496case HighlightingModifier::Virtual:
1497return "virtual";
1498case HighlightingModifier::DependentName:
1499return "dependentName"; // nonstandard
1500case HighlightingModifier::DefaultLibrary:
1501return "defaultLibrary";
1502case HighlightingModifier::UsedAsMutableReference:
1503return "usedAsMutableReference"; // nonstandard
1504case HighlightingModifier::UsedAsMutablePointer:
1505return "usedAsMutablePointer"; // nonstandard
1506case HighlightingModifier::ConstructorOrDestructor:
1507return "constructorOrDestructor"; // nonstandard
1508case HighlightingModifier::UserDefined:
1509return "userDefined"; // nonstandard
1510case HighlightingModifier::FunctionScope:
1511return "functionScope"; // nonstandard
1512case HighlightingModifier::ClassScope:
1513return "classScope"; // nonstandard
1514case HighlightingModifier::FileScope:
1515return "fileScope"; // nonstandard
1516case HighlightingModifier::GlobalScope:
1517return "globalScope"; // nonstandard
1518}
1519llvm_unreachable("unhandled HighlightingModifier");
1520}
1521
1522std::vector<SemanticTokensEdit>
1523diffTokens(llvm::ArrayRef<SemanticToken> Old,
1524llvm::ArrayRef<SemanticToken> New) {
1525// For now, just replace everything from the first-last modification.
1526// FIXME: use a real diff instead, this is bad with include-insertion.
1527
1528unsigned Offset = 0;
1529while (!Old.empty() && !New.empty() && Old.front() == New.front()) {
1530++Offset;
1531Old = Old.drop_front();
1532New = New.drop_front();
1533}
1534while (!Old.empty() && !New.empty() && Old.back() == New.back()) {
1535Old = Old.drop_back();
1536New = New.drop_back();
1537}
1538
1539if (Old.empty() && New.empty())
1540return {};
1541SemanticTokensEdit Edit;
1542Edit.startToken = Offset;
1543Edit.deleteTokens = Old.size();
1544Edit.tokens = New;
1545return {std::move(Edit)};
1546}
1547
1548std::vector<Range> getInactiveRegions(ParsedAST &AST) {
1549std::vector<Range> SkippedRanges(std::move(AST.getMacros().SkippedRanges));
1550const auto &SM = AST.getSourceManager();
1551StringRef MainCode = SM.getBufferOrFake(SM.getMainFileID()).getBuffer();
1552std::vector<Range> InactiveRegions;
1553for (const Range &Skipped : SkippedRanges) {
1554Range Inactive = Skipped;
1555// Sometimes, SkippedRanges contains a range ending at position 0
1556// of a line. Clients that apply whole-line styles will treat that
1557// line as inactive which is not desirable, so adjust the ending
1558// position to be the end of the previous line.
1559if (Inactive.end.character == 0 && Inactive.end.line > 0) {
1560--Inactive.end.line;
1561}
1562// Exclude the directive lines themselves from the range.
1563if (Inactive.end.line >= Inactive.start.line + 2) {
1564++Inactive.start.line;
1565--Inactive.end.line;
1566} else {
1567// range would be empty, e.g. #endif on next line after #ifdef
1568continue;
1569}
1570// Since we've adjusted the ending line, we need to recompute the
1571// column to reflect the end of that line.
1572if (auto EndOfLine = endOfLine(MainCode, Inactive.end.line)) {
1573Inactive.end = *EndOfLine;
1574} else {
1575elog("Failed to determine end of line: {0}", EndOfLine.takeError());
1576continue;
1577}
1578InactiveRegions.push_back(Inactive);
1579}
1580return InactiveRegions;
1581}
1582
1583} // namespace clangd
1584} // namespace clang
1585