llvm-project

1
//===--- ParsedAST.cpp -------------------------------------------*- C++-*-===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8

9
#include "ParsedAST.h"
10
#include "../clang-tidy/ClangTidyCheck.h"
11
#include "../clang-tidy/ClangTidyDiagnosticConsumer.h"
12
#include "../clang-tidy/ClangTidyModule.h"
13
#include "../clang-tidy/ClangTidyModuleRegistry.h"
14
#include "../clang-tidy/ClangTidyOptions.h"
15
#include "AST.h"
16
#include "CollectMacros.h"
17
#include "Compiler.h"
18
#include "Config.h"
19
#include "Diagnostics.h"
20
#include "Feature.h"
21
#include "FeatureModule.h"
22
#include "Headers.h"
23
#include "HeuristicResolver.h"
24
#include "IncludeCleaner.h"
25
#include "IncludeFixer.h"
26
#include "Preamble.h"
27
#include "SourceCode.h"
28
#include "TidyProvider.h"
29
#include "clang-include-cleaner/Record.h"
30
#include "index/Symbol.h"
31
#include "support/Logger.h"
32
#include "support/Path.h"
33
#include "support/Trace.h"
34
#include "clang/AST/ASTContext.h"
35
#include "clang/AST/Decl.h"
36
#include "clang/AST/DeclGroup.h"
37
#include "clang/AST/ExternalASTSource.h"
38
#include "clang/ASTMatchers/ASTMatchFinder.h"
39
#include "clang/Basic/Diagnostic.h"
40
#include "clang/Basic/DiagnosticIDs.h"
41
#include "clang/Basic/DiagnosticSema.h"
42
#include "clang/Basic/FileEntry.h"
43
#include "clang/Basic/LLVM.h"
44
#include "clang/Basic/LangOptions.h"
45
#include "clang/Basic/SourceLocation.h"
46
#include "clang/Basic/SourceManager.h"
47
#include "clang/Basic/TokenKinds.h"
48
#include "clang/Frontend/CompilerInstance.h"
49
#include "clang/Frontend/CompilerInvocation.h"
50
#include "clang/Frontend/FrontendActions.h"
51
#include "clang/Frontend/FrontendOptions.h"
52
#include "clang/Frontend/PrecompiledPreamble.h"
53
#include "clang/Lex/Lexer.h"
54
#include "clang/Lex/PPCallbacks.h"
55
#include "clang/Lex/Preprocessor.h"
56
#include "clang/Serialization/ASTWriter.h"
57
#include "clang/Tooling/CompilationDatabase.h"
58
#include "clang/Tooling/Core/Diagnostic.h"
59
#include "clang/Tooling/Syntax/Tokens.h"
60
#include "llvm/ADT/ArrayRef.h"
61
#include "llvm/ADT/DenseMap.h"
62
#include "llvm/ADT/DenseSet.h"
63
#include "llvm/ADT/STLExtras.h"
64
#include "llvm/ADT/STLFunctionalExtras.h"
65
#include "llvm/ADT/SmallVector.h"
66
#include "llvm/ADT/StringRef.h"
67
#include "llvm/Support/Error.h"
68
#include "llvm/Support/MemoryBuffer.h"
69
#include <cassert>
70
#include <cstddef>
71
#include <iterator>
72
#include <memory>
73
#include <optional>
74
#include <string>
75
#include <tuple>
76
#include <utility>
77
#include <vector>
78

79
// Force the linker to link in Clang-tidy modules.
80
// clangd doesn't support the static analyzer.
81
#if CLANGD_TIDY_CHECKS
82
#define CLANG_TIDY_DISABLE_STATIC_ANALYZER_CHECKS
83
#include "../clang-tidy/ClangTidyForceLinker.h"
84
#endif
85

86
namespace clang {
87
namespace clangd {
88
namespace {
89

90
template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
91
  return Vec.capacity() * sizeof(T);
92
}
93

94
class DeclTrackingASTConsumer : public ASTConsumer {
95
public:
96
  DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls)
97
      : TopLevelDecls(TopLevelDecls) {}
98

99
  bool HandleTopLevelDecl(DeclGroupRef DG) override {
100
    for (Decl *D : DG) {
101
      auto &SM = D->getASTContext().getSourceManager();
102
      if (!isInsideMainFile(D->getLocation(), SM))
103
        continue;
104
      if (const NamedDecl *ND = dyn_cast<NamedDecl>(D))
105
        if (isImplicitTemplateInstantiation(ND))
106
          continue;
107

108
      // ObjCMethodDecl are not actually top-level decls.
109
      if (isa<ObjCMethodDecl>(D))
110
        continue;
111

112
      TopLevelDecls.push_back(D);
113
    }
114
    return true;
115
  }
116

117
private:
118
  std::vector<Decl *> &TopLevelDecls;
119
};
120

121
class ClangdFrontendAction : public SyntaxOnlyAction {
122
public:
123
  std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
124

125
protected:
126
  std::unique_ptr<ASTConsumer>
127
  CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override {
128
    return std::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls);
129
  }
130

131
private:
132
  std::vector<Decl *> TopLevelDecls;
133
};
134

135
// When using a preamble, only preprocessor events outside its bounds are seen.
136
// This is almost what we want: replaying transitive preprocessing wastes time.
137
// However this confuses clang-tidy checks: they don't see any #includes!
138
// So we replay the *non-transitive* #includes that appear in the main-file.
139
// It would be nice to replay other events (macro definitions, ifdefs etc) but
140
// this addresses the most common cases fairly cheaply.
141
class ReplayPreamble : private PPCallbacks {
142
public:
143
  // Attach preprocessor hooks such that preamble events will be injected at
144
  // the appropriate time.
145
  // Events will be delivered to the *currently registered* PP callbacks.
146
  static void attach(std::vector<Inclusion> Includes, CompilerInstance &Clang,
147
                     const PreambleBounds &PB) {
148
    auto &PP = Clang.getPreprocessor();
149
    auto *ExistingCallbacks = PP.getPPCallbacks();
150
    // No need to replay events if nobody is listening.
151
    if (!ExistingCallbacks)
152
      return;
153
    PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(new ReplayPreamble(
154
        std::move(Includes), ExistingCallbacks, Clang.getSourceManager(), PP,
155
        Clang.getLangOpts(), PB)));
156
    // We're relying on the fact that addPPCallbacks keeps the old PPCallbacks
157
    // around, creating a chaining wrapper. Guard against other implementations.
158
    assert(PP.getPPCallbacks() != ExistingCallbacks &&
159
           "Expected chaining implementation");
160
  }
161

162
private:
163
  ReplayPreamble(std::vector<Inclusion> Includes, PPCallbacks *Delegate,
164
                 const SourceManager &SM, Preprocessor &PP,
165
                 const LangOptions &LangOpts, const PreambleBounds &PB)
166
      : Includes(std::move(Includes)), Delegate(Delegate), SM(SM), PP(PP) {
167
    // Only tokenize the preamble section of the main file, as we are not
168
    // interested in the rest of the tokens.
169
    MainFileTokens = syntax::tokenize(
170
        syntax::FileRange(SM.getMainFileID(), 0, PB.Size), SM, LangOpts);
171
  }
172

173
  // In a normal compile, the preamble traverses the following structure:
174
  //
175
  // mainfile.cpp
176
  //   <built-in>
177
  //     ... macro definitions like __cplusplus ...
178
  //     <command-line>
179
  //       ... macro definitions for args like -Dfoo=bar ...
180
  //   "header1.h"
181
  //     ... header file contents ...
182
  //   "header2.h"
183
  //     ... header file contents ...
184
  //   ... main file contents ...
185
  //
186
  // When using a preamble, the "header1" and "header2" subtrees get skipped.
187
  // We insert them right after the built-in header, which still appears.
188
  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
189
                   SrcMgr::CharacteristicKind Kind, FileID PrevFID) override {
190
    // It'd be nice if there was a better way to identify built-in headers...
191
    if (Reason == FileChangeReason::ExitFile &&
192
        SM.getBufferOrFake(PrevFID).getBufferIdentifier() == "<built-in>")
193
      replay();
194
  }
195

196
  void replay() {
197
    for (const auto &Inc : Includes) {
198
      OptionalFileEntryRef File;
199
      if (Inc.Resolved != "")
200
        File = expectedToOptional(SM.getFileManager().getFileRef(Inc.Resolved));
201

202
      // Re-lex the #include directive to find its interesting parts.
203
      auto HashLoc = SM.getComposedLoc(SM.getMainFileID(), Inc.HashOffset);
204
      auto HashTok = llvm::partition_point(MainFileTokens,
205
                                           [&HashLoc](const syntax::Token &T) {
206
                                             return T.location() < HashLoc;
207
                                           });
208
      assert(HashTok != MainFileTokens.end() && HashTok->kind() == tok::hash);
209

210
      auto IncludeTok = std::next(HashTok);
211
      assert(IncludeTok != MainFileTokens.end());
212

213
      auto FileTok = std::next(IncludeTok);
214
      assert(FileTok != MainFileTokens.end());
215

216
      // Create a fake import/include token, none of the callers seem to care
217
      // about clang::Token::Flags.
218
      Token SynthesizedIncludeTok;
219
      SynthesizedIncludeTok.startToken();
220
      SynthesizedIncludeTok.setLocation(IncludeTok->location());
221
      SynthesizedIncludeTok.setLength(IncludeTok->length());
222
      SynthesizedIncludeTok.setKind(tok::raw_identifier);
223
      SynthesizedIncludeTok.setRawIdentifierData(IncludeTok->text(SM).data());
224
      PP.LookUpIdentifierInfo(SynthesizedIncludeTok);
225

226
      // Same here, create a fake one for Filename, including angles or quotes.
227
      Token SynthesizedFilenameTok;
228
      SynthesizedFilenameTok.startToken();
229
      SynthesizedFilenameTok.setLocation(FileTok->location());
230
      // Note that we can't make use of FileTok->length/text in here as in the
231
      // case of angled includes this will contain tok::less instead of
232
      // filename. Whereas Inc.Written contains the full header name including
233
      // quotes/angles.
234
      SynthesizedFilenameTok.setLength(Inc.Written.length());
235
      SynthesizedFilenameTok.setKind(tok::header_name);
236
      SynthesizedFilenameTok.setLiteralData(Inc.Written.data());
237

238
      llvm::StringRef WrittenFilename =
239
          llvm::StringRef(Inc.Written).drop_front().drop_back();
240
      Delegate->InclusionDirective(
241
          HashTok->location(), SynthesizedIncludeTok, WrittenFilename,
242
          Inc.Written.front() == '<',
243
          syntax::FileRange(SM, SynthesizedFilenameTok.getLocation(),
244
                            SynthesizedFilenameTok.getEndLoc())
245
              .toCharRange(SM),
246
          File, "SearchPath", "RelPath",
247
          /*SuggestedModule=*/nullptr, /*ModuleImported=*/false, Inc.FileKind);
248
      if (File)
249
        Delegate->FileSkipped(*File, SynthesizedFilenameTok, Inc.FileKind);
250
    }
251
  }
252

253
  const std::vector<Inclusion> Includes;
254
  PPCallbacks *Delegate;
255
  const SourceManager &SM;
256
  Preprocessor &PP;
257
  std::vector<syntax::Token> MainFileTokens;
258
};
259

260
// Filter for clang diagnostics groups enabled by CTOptions.Checks.
261
//
262
// These are check names like clang-diagnostics-unused.
263
// Note that unlike -Wunused, clang-diagnostics-unused does not imply
264
// subcategories like clang-diagnostics-unused-function.
265
//
266
// This is used to determine which diagnostics can be enabled by ExtraArgs in
267
// the clang-tidy configuration.
268
class TidyDiagnosticGroups {
269
  // Whether all diagnostic groups are enabled by default.
270
  // True if we've seen clang-diagnostic-*.
271
  bool Default = false;
272
  // Set of diag::Group whose enablement != Default.
273
  // If Default is false, this is foo where we've seen clang-diagnostic-foo.
274
  llvm::DenseSet<unsigned> Exceptions;
275

276
public:
277
  TidyDiagnosticGroups(llvm::StringRef Checks) {
278
    constexpr llvm::StringLiteral CDPrefix = "clang-diagnostic-";
279

280
    llvm::StringRef Check;
281
    while (!Checks.empty()) {
282
      std::tie(Check, Checks) = Checks.split(',');
283
      if (Check.empty())
284
        continue;
285

286
      bool Enable = !Check.consume_front("-");
287
      bool Glob = Check.consume_back("*");
288
      if (Glob) {
289
        // Is this clang-diagnostic-*, or *, or so?
290
        // (We ignore all other types of globs).
291
        if (CDPrefix.starts_with(Check)) {
292
          Default = Enable;
293
          Exceptions.clear();
294
        }
295
        continue;
296
      }
297

298
      // In "*,clang-diagnostic-foo", the latter is a no-op.
299
      if (Default == Enable)
300
        continue;
301
      // The only non-glob entries we care about are clang-diagnostic-foo.
302
      if (!Check.consume_front(CDPrefix))
303
        continue;
304

305
      if (auto Group = DiagnosticIDs::getGroupForWarningOption(Check))
306
        Exceptions.insert(static_cast<unsigned>(*Group));
307
    }
308
  }
309

310
  bool operator()(diag::Group GroupID) const {
311
    return Exceptions.contains(static_cast<unsigned>(GroupID)) ? !Default
312
                                                               : Default;
313
  }
314
};
315

316
// Find -W<group> and -Wno-<group> options in ExtraArgs and apply them to Diags.
317
//
318
// This is used to handle ExtraArgs in clang-tidy configuration.
319
// We don't use clang's standard handling of this as we want slightly different
320
// behavior (e.g. we want to exclude these from -Wno-error).
321
void applyWarningOptions(llvm::ArrayRef<std::string> ExtraArgs,
322
                         llvm::function_ref<bool(diag::Group)> EnabledGroups,
323
                         DiagnosticsEngine &Diags) {
324
  for (llvm::StringRef Group : ExtraArgs) {
325
    // Only handle args that are of the form -W[no-]<group>.
326
    // Other flags are possible but rare and deliberately out of scope.
327
    llvm::SmallVector<diag::kind> Members;
328
    if (!Group.consume_front("-W") || Group.empty())
329
      continue;
330
    bool Enable = !Group.consume_front("no-");
331
    if (Diags.getDiagnosticIDs()->getDiagnosticsInGroup(
332
            diag::Flavor::WarningOrError, Group, Members))
333
      continue;
334

335
    // Upgrade (or downgrade) the severity of each diagnostic in the group.
336
    // If -Werror is on, newly added warnings will be treated as errors.
337
    // We don't want this, so keep track of them to fix afterwards.
338
    bool NeedsWerrorExclusion = false;
339
    for (diag::kind ID : Members) {
340
      if (Enable) {
341
        if (Diags.getDiagnosticLevel(ID, SourceLocation()) <
342
            DiagnosticsEngine::Warning) {
343
          auto Group = DiagnosticIDs::getGroupForDiag(ID);
344
          if (!Group || !EnabledGroups(*Group))
345
            continue;
346
          Diags.setSeverity(ID, diag::Severity::Warning, SourceLocation());
347
          if (Diags.getWarningsAsErrors())
348
            NeedsWerrorExclusion = true;
349
        }
350
      } else {
351
        Diags.setSeverity(ID, diag::Severity::Ignored, SourceLocation());
352
      }
353
    }
354
    if (NeedsWerrorExclusion) {
355
      // FIXME: there's no API to suppress -Werror for single diagnostics.
356
      // In some cases with sub-groups, we may end up erroneously
357
      // downgrading diagnostics that were -Werror in the compile command.
358
      Diags.setDiagnosticGroupWarningAsError(Group, false);
359
    }
360
  }
361
}
362

363
std::vector<Diag> getIncludeCleanerDiags(ParsedAST &AST, llvm::StringRef Code,
364
                                         const ThreadsafeFS &TFS) {
365
  auto &Cfg = Config::current();
366
  if (Cfg.Diagnostics.SuppressAll)
367
    return {};
368
  bool SuppressMissing =
369
      Cfg.Diagnostics.Suppress.contains("missing-includes") ||
370
      Cfg.Diagnostics.MissingIncludes == Config::IncludesPolicy::None;
371
  bool SuppressUnused =
372
      Cfg.Diagnostics.Suppress.contains("unused-includes") ||
373
      Cfg.Diagnostics.UnusedIncludes == Config::IncludesPolicy::None;
374
  if (SuppressMissing && SuppressUnused)
375
    return {};
376
  auto Findings = computeIncludeCleanerFindings(
377
      AST, Cfg.Diagnostics.Includes.AnalyzeAngledIncludes);
378
  if (SuppressMissing)
379
    Findings.MissingIncludes.clear();
380
  if (SuppressUnused)
381
    Findings.UnusedIncludes.clear();
382
  return issueIncludeCleanerDiagnostics(AST, Code, Findings, TFS,
383
                                        Cfg.Diagnostics.Includes.IgnoreHeader);
384
}
385

386
tidy::ClangTidyCheckFactories
387
filterFastTidyChecks(const tidy::ClangTidyCheckFactories &All,
388
                     Config::FastCheckPolicy Policy) {
389
  if (Policy == Config::FastCheckPolicy::None)
390
    return All;
391
  bool AllowUnknown = Policy == Config::FastCheckPolicy::Loose;
392
  tidy::ClangTidyCheckFactories Fast;
393
  for (const auto &Factory : All) {
394
    if (isFastTidyCheck(Factory.getKey()).value_or(AllowUnknown))
395
      Fast.registerCheckFactory(Factory.first(), Factory.second);
396
  }
397
  return Fast;
398
}
399

400
} // namespace
401

402
std::optional<ParsedAST>
403
ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
404
                 std::unique_ptr<clang::CompilerInvocation> CI,
405
                 llvm::ArrayRef<Diag> CompilerInvocationDiags,
406
                 std::shared_ptr<const PreambleData> Preamble) {
407
  trace::Span Tracer("BuildAST");
408
  SPAN_ATTACH(Tracer, "File", Filename);
409
  const Config &Cfg = Config::current();
410

411
  auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
412
  if (Preamble && Preamble->StatCache)
413
    VFS = Preamble->StatCache->getConsumingFS(std::move(VFS));
414

415
  assert(CI);
416

417
  if (CI->getFrontendOpts().Inputs.size() > 0) {
418
    auto Lang = CI->getFrontendOpts().Inputs[0].getKind().getLanguage();
419
    if (Lang == Language::Asm || Lang == Language::LLVM_IR) {
420
      elog("Clangd does not support assembly or IR source files");
421
      return std::nullopt;
422
    }
423
  }
424

425
  // Command-line parsing sets DisableFree to true by default, but we don't want
426
  // to leak memory in clangd.
427
  CI->getFrontendOpts().DisableFree = false;
428
  const PrecompiledPreamble *PreamblePCH =
429
      Preamble ? &Preamble->Preamble : nullptr;
430

431
  // This is on-by-default in windows to allow parsing SDK headers, but it
432
  // breaks many features. Disable it for the main-file (not preamble).
433
  CI->getLangOpts().DelayedTemplateParsing = false;
434

435
  std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
436
  if (Inputs.FeatureModules) {
437
    for (auto &M : *Inputs.FeatureModules) {
438
      if (auto Listener = M.astListeners())
439
        ASTListeners.emplace_back(std::move(Listener));
440
    }
441
  }
442
  StoreDiags ASTDiags;
443
  ASTDiags.setDiagCallback(
444
      [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
445
        for (const auto &L : ASTListeners)
446
          L->sawDiagnostic(D, Diag);
447
      });
448

449
  std::optional<PreamblePatch> Patch;
450
  // We might use an ignoring diagnostic consumer if they are going to be
451
  // dropped later on to not pay for extra latency by processing them.
452
  DiagnosticConsumer *DiagConsumer = &ASTDiags;
453
  IgnoreDiagnostics DropDiags;
454
  if (Preamble) {
455
    Patch = PreamblePatch::createFullPatch(Filename, Inputs, *Preamble);
456
    Patch->apply(*CI);
457
  }
458
  auto Clang = prepareCompilerInstance(
459
      std::move(CI), PreamblePCH,
460
      llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents, Filename), VFS,
461
      *DiagConsumer);
462
  if (!Clang) {
463
    // The last diagnostic contains information about the reason of this
464
    // failure.
465
    std::vector<Diag> Diags(ASTDiags.take());
466
    elog("Failed to prepare a compiler instance: {0}",
467
         !Diags.empty() ? static_cast<DiagBase &>(Diags.back()).Message
468
                        : "unknown error");
469
    return std::nullopt;
470
  }
471
  tidy::ClangTidyOptions ClangTidyOpts;
472
  {
473
    trace::Span Tracer("ClangTidyOpts");
474
    ClangTidyOpts = getTidyOptionsForFile(Inputs.ClangTidyProvider, Filename);
475
    dlog("ClangTidy configuration for file {0}: {1}", Filename,
476
         tidy::configurationAsText(ClangTidyOpts));
477

478
    // If clang-tidy is configured to emit clang warnings, we should too.
479
    //
480
    // Such clang-tidy configuration consists of two parts:
481
    //   - ExtraArgs: ["-Wfoo"] causes clang to produce the warnings
482
    //   - Checks: "clang-diagnostic-foo" prevents clang-tidy filtering them out
483
    //
484
    // In clang-tidy, diagnostics are emitted if they pass both checks.
485
    // When groups contain subgroups, -Wparent includes the child, but
486
    // clang-diagnostic-parent does not.
487
    //
488
    // We *don't* want to change the compile command directly. This can have
489
    // too many unexpected effects: breaking the command, interactions with
490
    // -- and -Werror, etc. Besides, we've already parsed the command.
491
    // Instead we parse the -W<group> flags and handle them directly.
492
    //
493
    // Similarly, we don't want to use Checks to filter clang diagnostics after
494
    // they are generated, as this spreads clang-tidy emulation everywhere.
495
    // Instead, we just use these to filter which extra diagnostics we enable.
496
    auto &Diags = Clang->getDiagnostics();
497
    TidyDiagnosticGroups TidyGroups(ClangTidyOpts.Checks ? *ClangTidyOpts.Checks
498
                                                         : llvm::StringRef());
499
    if (ClangTidyOpts.ExtraArgsBefore)
500
      applyWarningOptions(*ClangTidyOpts.ExtraArgsBefore, TidyGroups, Diags);
501
    if (ClangTidyOpts.ExtraArgs)
502
      applyWarningOptions(*ClangTidyOpts.ExtraArgs, TidyGroups, Diags);
503
  }
504

505
  auto Action = std::make_unique<ClangdFrontendAction>();
506
  const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
507
  if (!Action->BeginSourceFile(*Clang, MainInput)) {
508
    log("BeginSourceFile() failed when building AST for {0}",
509
        MainInput.getFile());
510
    return std::nullopt;
511
  }
512
  // If we saw an include guard in the preamble section of the main file,
513
  // mark the main-file as include-guarded.
514
  // This information is part of the HeaderFileInfo but is not loaded from the
515
  // preamble as the file's size is part of its identity and may have changed.
516
  // (The rest of HeaderFileInfo is not relevant for our purposes).
517
  if (Preamble && Preamble->MainIsIncludeGuarded) {
518
    const SourceManager &SM = Clang->getSourceManager();
519
    OptionalFileEntryRef MainFE = SM.getFileEntryRefForID(SM.getMainFileID());
520
    Clang->getPreprocessor().getHeaderSearchInfo().MarkFileIncludeOnce(*MainFE);
521
  }
522

523
  // Set up ClangTidy. Must happen after BeginSourceFile() so ASTContext exists.
524
  // Clang-tidy has some limitations to ensure reasonable performance:
525
  //  - checks don't see all preprocessor events in the preamble
526
  //  - matchers run only over the main-file top-level decls (and can't see
527
  //    ancestors outside this scope).
528
  // In practice almost all checks work well without modifications.
529
  std::vector<std::unique_ptr<tidy::ClangTidyCheck>> CTChecks;
530
  ast_matchers::MatchFinder CTFinder;
531
  std::optional<tidy::ClangTidyContext> CTContext;
532
  // Must outlive FixIncludes.
533
  auto BuildDir = VFS->getCurrentWorkingDirectory();
534
  std::optional<IncludeFixer> FixIncludes;
535
  llvm::DenseMap<diag::kind, DiagnosticsEngine::Level> OverriddenSeverity;
536
  // No need to run clang-tidy or IncludeFixerif we are not going to surface
537
  // diagnostics.
538
  {
539
    trace::Span Tracer("ClangTidyInit");
540
    static const auto *AllCTFactories = [] {
541
      auto *CTFactories = new tidy::ClangTidyCheckFactories;
542
      for (const auto &E : tidy::ClangTidyModuleRegistry::entries())
543
        E.instantiate()->addCheckFactories(*CTFactories);
544
      return CTFactories;
545
    }();
546
    tidy::ClangTidyCheckFactories FastFactories = filterFastTidyChecks(
547
        *AllCTFactories, Cfg.Diagnostics.ClangTidy.FastCheckFilter);
548
    CTContext.emplace(std::make_unique<tidy::DefaultOptionsProvider>(
549
        tidy::ClangTidyGlobalOptions(), ClangTidyOpts));
550
    CTContext->setDiagnosticsEngine(&Clang->getDiagnostics());
551
    CTContext->setASTContext(&Clang->getASTContext());
552
    CTContext->setCurrentFile(Filename);
553
    CTContext->setSelfContainedDiags(true);
554
    CTChecks = FastFactories.createChecksForLanguage(&*CTContext);
555
    Preprocessor *PP = &Clang->getPreprocessor();
556
    for (const auto &Check : CTChecks) {
557
      Check->registerPPCallbacks(Clang->getSourceManager(), PP, PP);
558
      Check->registerMatchers(&CTFinder);
559
    }
560

561
    // Clang only corrects typos for use of undeclared functions in C if that
562
    // use is an error. Include fixer relies on typo correction, so pretend
563
    // this is an error. (The actual typo correction is nice too).
564
    // We restore the original severity in the level adjuster.
565
    // FIXME: It would be better to have a real API for this, but what?
566
    for (auto ID : {diag::ext_implicit_function_decl_c99,
567
                    diag::ext_implicit_lib_function_decl,
568
                    diag::ext_implicit_lib_function_decl_c99,
569
                    diag::warn_implicit_function_decl}) {
570
      OverriddenSeverity.try_emplace(
571
          ID, Clang->getDiagnostics().getDiagnosticLevel(ID, SourceLocation()));
572
      Clang->getDiagnostics().setSeverity(ID, diag::Severity::Error,
573
                                          SourceLocation());
574
    }
575

576
    ASTDiags.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
577
                                  const clang::Diagnostic &Info) {
578
      if (Cfg.Diagnostics.SuppressAll ||
579
          isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
580
                                        Clang->getLangOpts()))
581
        return DiagnosticsEngine::Ignored;
582

583
      auto It = OverriddenSeverity.find(Info.getID());
584
      if (It != OverriddenSeverity.end())
585
        DiagLevel = It->second;
586

587
      if (!CTChecks.empty()) {
588
        std::string CheckName = CTContext->getCheckName(Info.getID());
589
        bool IsClangTidyDiag = !CheckName.empty();
590
        if (IsClangTidyDiag) {
591
          if (Cfg.Diagnostics.Suppress.contains(CheckName))
592
            return DiagnosticsEngine::Ignored;
593
          // Check for suppression comment. Skip the check for diagnostics not
594
          // in the main file, because we don't want that function to query the
595
          // source buffer for preamble files. For the same reason, we ask
596
          // shouldSuppressDiagnostic to avoid I/O.
597
          // We let suppression comments take precedence over warning-as-error
598
          // to match clang-tidy's behaviour.
599
          bool IsInsideMainFile =
600
              Info.hasSourceManager() &&
601
              isInsideMainFile(Info.getLocation(), Info.getSourceManager());
602
          SmallVector<tooling::Diagnostic, 1> TidySuppressedErrors;
603
          if (IsInsideMainFile && CTContext->shouldSuppressDiagnostic(
604
                                      DiagLevel, Info, TidySuppressedErrors,
605
                                      /*AllowIO=*/false,
606
                                      /*EnableNolintBlocks=*/true)) {
607
            // FIXME: should we expose the suppression error (invalid use of
608
            // NOLINT comments)?
609
            return DiagnosticsEngine::Ignored;
610
          }
611
          if (!CTContext->getOptions().SystemHeaders.value_or(false) &&
612
              Info.hasSourceManager() &&
613
              Info.getSourceManager().isInSystemMacro(Info.getLocation()))
614
            return DiagnosticsEngine::Ignored;
615

616
          // Check for warning-as-error.
617
          if (DiagLevel == DiagnosticsEngine::Warning &&
618
              CTContext->treatAsError(CheckName)) {
619
            return DiagnosticsEngine::Error;
620
          }
621
        }
622
      }
623
      return DiagLevel;
624
    });
625

626
    // Add IncludeFixer which can recover diagnostics caused by missing includes
627
    // (e.g. incomplete type) and attach include insertion fixes to diagnostics.
628
    if (Inputs.Index && !BuildDir.getError()) {
629
      auto Style =
630
          getFormatStyleForFile(Filename, Inputs.Contents, *Inputs.TFS, false);
631
      auto Inserter = std::make_shared<IncludeInserter>(
632
          Filename, Inputs.Contents, Style, BuildDir.get(),
633
          &Clang->getPreprocessor().getHeaderSearchInfo());
634
      ArrayRef<Inclusion> MainFileIncludes;
635
      if (Preamble) {
636
        MainFileIncludes = Preamble->Includes.MainFileIncludes;
637
        for (const auto &Inc : Preamble->Includes.MainFileIncludes)
638
          Inserter->addExisting(Inc);
639
      }
640
      // FIXME: Consider piping through ASTSignals to fetch this to handle the
641
      // case where a header file contains ObjC decls but no #imports.
642
      Symbol::IncludeDirective Directive =
643
          Inputs.Opts.ImportInsertions
644
              ? preferredIncludeDirective(Filename, Clang->getLangOpts(),
645
                                          MainFileIncludes, {})
646
              : Symbol::Include;
647
      FixIncludes.emplace(Filename, Inserter, *Inputs.Index,
648
                          /*IndexRequestLimit=*/5, Directive);
649
      ASTDiags.contributeFixes([&FixIncludes](DiagnosticsEngine::Level DiagLevl,
650
                                              const clang::Diagnostic &Info) {
651
        return FixIncludes->fix(DiagLevl, Info);
652
      });
653
      Clang->setExternalSemaSource(FixIncludes->unresolvedNameRecorder());
654
    }
655
  }
656

657
  IncludeStructure Includes;
658
  include_cleaner::PragmaIncludes PI;
659
  // If we are using a preamble, copy existing includes.
660
  if (Preamble) {
661
    Includes = Preamble->Includes;
662
    Includes.MainFileIncludes = Patch->preambleIncludes();
663
    // Replay the preamble includes so that clang-tidy checks can see them.
664
    ReplayPreamble::attach(Patch->preambleIncludes(), *Clang,
665
                           Patch->modifiedBounds());
666
    PI = *Preamble->Pragmas;
667
  }
668
  // Important: collectIncludeStructure is registered *after* ReplayPreamble!
669
  // Otherwise we would collect the replayed includes again...
670
  // (We can't *just* use the replayed includes, they don't have Resolved path).
671
  Includes.collect(*Clang);
672
  // Same for pragma-includes, we're already inheriting preamble includes, so we
673
  // should only receive callbacks for non-preamble mainfile includes.
674
  PI.record(*Clang);
675
  // Copy over the macros in the preamble region of the main file, and combine
676
  // with non-preamble macros below.
677
  MainFileMacros Macros;
678
  std::vector<PragmaMark> Marks;
679
  if (Preamble) {
680
    Macros = Patch->mainFileMacros();
681
    Marks = Patch->marks();
682
  }
683
  auto &PP = Clang->getPreprocessor();
684
  PP.addPPCallbacks(std::make_unique<CollectMainFileMacros>(PP, Macros));
685

686
  PP.addPPCallbacks(
687
      collectPragmaMarksCallback(Clang->getSourceManager(), Marks));
688

689
  // FIXME: Attach a comment handler to take care of
690
  // keep/export/no_include etc. IWYU pragmas.
691

692
  // Collect tokens of the main file.
693
  syntax::TokenCollector CollectTokens(PP);
694

695
  // To remain consistent with preamble builds, these callbacks must be called
696
  // exactly here, after preprocessor is initialized and BeginSourceFile() was
697
  // called already.
698
  for (const auto &L : ASTListeners)
699
    L->beforeExecute(*Clang);
700

701
  if (llvm::Error Err = Action->Execute())
702
    log("Execute() failed when building AST for {0}: {1}", MainInput.getFile(),
703
        toString(std::move(Err)));
704

705
  // We have to consume the tokens before running clang-tidy to avoid collecting
706
  // tokens from running the preprocessor inside the checks (only
707
  // modernize-use-trailing-return-type does that today).
708
  syntax::TokenBuffer Tokens = std::move(CollectTokens).consume();
709
  // Makes SelectionTree build much faster.
710
  Tokens.indexExpandedTokens();
711
  std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
712
  // AST traversals should exclude the preamble, to avoid performance cliffs.
713
  Clang->getASTContext().setTraversalScope(ParsedDecls);
714
  if (!CTChecks.empty()) {
715
    // Run the AST-dependent part of the clang-tidy checks.
716
    // (The preprocessor part ran already, via PPCallbacks).
717
    trace::Span Tracer("ClangTidyMatch");
718
    CTFinder.matchAST(Clang->getASTContext());
719
  }
720

721
  // XXX: This is messy: clang-tidy checks flush some diagnostics at EOF.
722
  // However Action->EndSourceFile() would destroy the ASTContext!
723
  // So just inform the preprocessor of EOF, while keeping everything alive.
724
  PP.EndSourceFile();
725
  // UnitDiagsConsumer is local, we can not store it in CompilerInstance that
726
  // has a longer lifetime.
727
  Clang->getDiagnostics().setClient(new IgnoreDiagnostics);
728
  // CompilerInstance won't run this callback, do it directly.
729
  ASTDiags.EndSourceFile();
730

731
  std::vector<Diag> Diags = CompilerInvocationDiags;
732
  // FIXME: Also skip generation of diagnostics altogether to speed up ast
733
  // builds when we are patching a stale preamble.
734
  // Add diagnostics from the preamble, if any.
735
  if (Preamble)
736
    llvm::append_range(Diags, Patch->patchedDiags());
737
  // Finally, add diagnostics coming from the AST.
738
  {
739
    std::vector<Diag> D = ASTDiags.take(&*CTContext);
740
    Diags.insert(Diags.end(), D.begin(), D.end());
741
  }
742
  ParsedAST Result(Filename, Inputs.Version, std::move(Preamble),
743
                   std::move(Clang), std::move(Action), std::move(Tokens),
744
                   std::move(Macros), std::move(Marks), std::move(ParsedDecls),
745
                   std::move(Diags), std::move(Includes), std::move(PI));
746
  llvm::move(getIncludeCleanerDiags(Result, Inputs.Contents, *Inputs.TFS),
747
             std::back_inserter(Result.Diags));
748
  return std::move(Result);
749
}
750

751
ParsedAST::ParsedAST(ParsedAST &&Other) = default;
752

753
ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
754

755
ParsedAST::~ParsedAST() {
756
  if (Action) {
757
    // We already notified the PP of end-of-file earlier, so detach it first.
758
    // We must keep it alive until after EndSourceFile(), Sema relies on this.
759
    auto PP = Clang->getPreprocessorPtr(); // Keep PP alive for now.
760
    Clang->setPreprocessor(nullptr);       // Detach so we don't send EOF again.
761
    Action->EndSourceFile();               // Destroy ASTContext and Sema.
762
    // Now Sema is gone, it's safe for PP to go out of scope.
763
  }
764
}
765

766
ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
767

768
const ASTContext &ParsedAST::getASTContext() const {
769
  return Clang->getASTContext();
770
}
771

772
Sema &ParsedAST::getSema() { return Clang->getSema(); }
773

774
Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
775

776
std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() {
777
  return Clang->getPreprocessorPtr();
778
}
779

780
const Preprocessor &ParsedAST::getPreprocessor() const {
781
  return Clang->getPreprocessor();
782
}
783

784
llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
785
  return LocalTopLevelDecls;
786
}
787

788
llvm::ArrayRef<const Decl *> ParsedAST::getLocalTopLevelDecls() const {
789
  return LocalTopLevelDecls;
790
}
791

792
const MainFileMacros &ParsedAST::getMacros() const { return Macros; }
793
const std::vector<PragmaMark> &ParsedAST::getMarks() const { return Marks; }
794

795
std::size_t ParsedAST::getUsedBytes() const {
796
  auto &AST = getASTContext();
797
  // FIXME(ibiryukov): we do not account for the dynamically allocated part of
798
  // Message and Fixes inside each diagnostic.
799
  std::size_t Total =
800
      clangd::getUsedBytes(LocalTopLevelDecls) + clangd::getUsedBytes(Diags);
801

802
  // FIXME: the rest of the function is almost a direct copy-paste from
803
  // libclang's clang_getCXTUResourceUsage. We could share the implementation.
804

805
  // Sum up various allocators inside the ast context and the preprocessor.
806
  Total += AST.getASTAllocatedMemory();
807
  Total += AST.getSideTableAllocatedMemory();
808
  Total += AST.Idents.getAllocator().getTotalMemory();
809
  Total += AST.Selectors.getTotalMemory();
810

811
  Total += AST.getSourceManager().getContentCacheSize();
812
  Total += AST.getSourceManager().getDataStructureSizes();
813
  Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
814

815
  if (ExternalASTSource *Ext = AST.getExternalSource())
816
    Total += Ext->getMemoryBufferSizes().malloc_bytes;
817

818
  const Preprocessor &PP = getPreprocessor();
819
  Total += PP.getTotalMemory();
820
  if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
821
    Total += PRec->getTotalMemory();
822
  Total += PP.getHeaderSearchInfo().getTotalMemory();
823

824
  return Total;
825
}
826

827
const IncludeStructure &ParsedAST::getIncludeStructure() const {
828
  return Includes;
829
}
830

831
ParsedAST::ParsedAST(PathRef TUPath, llvm::StringRef Version,
832
                     std::shared_ptr<const PreambleData> Preamble,
833
                     std::unique_ptr<CompilerInstance> Clang,
834
                     std::unique_ptr<FrontendAction> Action,
835
                     syntax::TokenBuffer Tokens, MainFileMacros Macros,
836
                     std::vector<PragmaMark> Marks,
837
                     std::vector<Decl *> LocalTopLevelDecls,
838
                     std::vector<Diag> Diags, IncludeStructure Includes,
839
                     include_cleaner::PragmaIncludes PI)
840
    : TUPath(TUPath), Version(Version), Preamble(std::move(Preamble)),
841
      Clang(std::move(Clang)), Action(std::move(Action)),
842
      Tokens(std::move(Tokens)), Macros(std::move(Macros)),
843
      Marks(std::move(Marks)), Diags(std::move(Diags)),
844
      LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
845
      Includes(std::move(Includes)), PI(std::move(PI)),
846
      Resolver(std::make_unique<HeuristicResolver>(getASTContext())) {
847
  assert(this->Clang);
848
  assert(this->Action);
849
}
850

851
const include_cleaner::PragmaIncludes &ParsedAST::getPragmaIncludes() const {
852
  return PI;
853
}
854

855
std::optional<llvm::StringRef> ParsedAST::preambleVersion() const {
856
  if (!Preamble)
857
    return std::nullopt;
858
  return llvm::StringRef(Preamble->Version);
859
}
860

861
llvm::ArrayRef<Diag> ParsedAST::getDiagnostics() const { return Diags; }
862
} // namespace clangd
863
} // namespace clang
864