llvm-project

1
//===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8

9
#include "Preamble.h"
10
#include "CollectMacros.h"
11
#include "Compiler.h"
12
#include "Config.h"
13
#include "Diagnostics.h"
14
#include "FS.h"
15
#include "FeatureModule.h"
16
#include "Headers.h"
17
#include "Protocol.h"
18
#include "SourceCode.h"
19
#include "clang-include-cleaner/Record.h"
20
#include "support/Logger.h"
21
#include "support/Path.h"
22
#include "support/ThreadsafeFS.h"
23
#include "support/Trace.h"
24
#include "clang/AST/DeclTemplate.h"
25
#include "clang/AST/Type.h"
26
#include "clang/Basic/Diagnostic.h"
27
#include "clang/Basic/DiagnosticLex.h"
28
#include "clang/Basic/DiagnosticOptions.h"
29
#include "clang/Basic/LangOptions.h"
30
#include "clang/Basic/SourceLocation.h"
31
#include "clang/Basic/SourceManager.h"
32
#include "clang/Basic/TargetInfo.h"
33
#include "clang/Basic/TokenKinds.h"
34
#include "clang/Frontend/CompilerInstance.h"
35
#include "clang/Frontend/CompilerInvocation.h"
36
#include "clang/Frontend/FrontendActions.h"
37
#include "clang/Frontend/PrecompiledPreamble.h"
38
#include "clang/Lex/HeaderSearch.h"
39
#include "clang/Lex/Lexer.h"
40
#include "clang/Lex/PPCallbacks.h"
41
#include "clang/Lex/Preprocessor.h"
42
#include "clang/Lex/PreprocessorOptions.h"
43
#include "clang/Serialization/ASTReader.h"
44
#include "clang/Tooling/CompilationDatabase.h"
45
#include "llvm/ADT/ArrayRef.h"
46
#include "llvm/ADT/DenseMap.h"
47
#include "llvm/ADT/IntrusiveRefCntPtr.h"
48
#include "llvm/ADT/STLExtras.h"
49
#include "llvm/ADT/SmallString.h"
50
#include "llvm/ADT/SmallVector.h"
51
#include "llvm/ADT/StringExtras.h"
52
#include "llvm/ADT/StringMap.h"
53
#include "llvm/ADT/StringRef.h"
54
#include "llvm/Support/Casting.h"
55
#include "llvm/Support/Error.h"
56
#include "llvm/Support/ErrorHandling.h"
57
#include "llvm/Support/ErrorOr.h"
58
#include "llvm/Support/FormatVariadic.h"
59
#include "llvm/Support/MemoryBuffer.h"
60
#include "llvm/Support/Path.h"
61
#include "llvm/Support/VirtualFileSystem.h"
62
#include "llvm/Support/raw_ostream.h"
63
#include <cassert>
64
#include <chrono>
65
#include <cstddef>
66
#include <cstdint>
67
#include <cstdlib>
68
#include <functional>
69
#include <memory>
70
#include <optional>
71
#include <string>
72
#include <system_error>
73
#include <tuple>
74
#include <utility>
75
#include <vector>
76

77
namespace clang {
78
namespace clangd {
79
namespace {
80

81
bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
82
                             const tooling::CompileCommand &RHS) {
83
  // We don't check for Output, it should not matter to clangd.
84
  return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
85
         llvm::ArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
86
}
87

88
class CppFilePreambleCallbacks : public PreambleCallbacks {
89
public:
90
  CppFilePreambleCallbacks(
91
      PathRef File, PreambleBuildStats *Stats, bool ParseForwardingFunctions,
92
      std::function<void(CompilerInstance &)> BeforeExecuteCallback)
93
      : File(File), Stats(Stats),
94
        ParseForwardingFunctions(ParseForwardingFunctions),
95
        BeforeExecuteCallback(std::move(BeforeExecuteCallback)) {}
96

97
  IncludeStructure takeIncludes() { return std::move(Includes); }
98

99
  MainFileMacros takeMacros() { return std::move(Macros); }
100

101
  std::vector<PragmaMark> takeMarks() { return std::move(Marks); }
102

103
  include_cleaner::PragmaIncludes takePragmaIncludes() {
104
    return std::move(Pragmas);
105
  }
106

107
  std::optional<CapturedASTCtx> takeLife() { return std::move(CapturedCtx); }
108

109
  bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; }
110

111
  void AfterExecute(CompilerInstance &CI) override {
112
    // As part of the Preamble compilation, ASTConsumer
113
    // PrecompilePreambleConsumer/PCHGenerator is setup. This would be called
114
    // when Preamble consists of modules. Therefore while capturing AST context,
115
    // we have to reset ast consumer and ASTMutationListener.
116
    if (CI.getASTReader()) {
117
      CI.getASTReader()->setDeserializationListener(nullptr);
118
      // This just sets consumer to null when DeserializationListener is null.
119
      CI.getASTReader()->StartTranslationUnit(nullptr);
120
    }
121
    CI.getASTContext().setASTMutationListener(nullptr);
122
    CapturedCtx.emplace(CI);
123

124
    const SourceManager &SM = CI.getSourceManager();
125
    OptionalFileEntryRef MainFE = SM.getFileEntryRefForID(SM.getMainFileID());
126
    IsMainFileIncludeGuarded =
127
        CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded(
128
            *MainFE);
129

130
    if (Stats) {
131
      const ASTContext &AST = CI.getASTContext();
132
      Stats->BuildSize = AST.getASTAllocatedMemory();
133
      Stats->BuildSize += AST.getSideTableAllocatedMemory();
134
      Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory();
135
      Stats->BuildSize += AST.Selectors.getTotalMemory();
136

137
      Stats->BuildSize += AST.getSourceManager().getContentCacheSize();
138
      Stats->BuildSize += AST.getSourceManager().getDataStructureSizes();
139
      Stats->BuildSize +=
140
          AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
141

142
      const Preprocessor &PP = CI.getPreprocessor();
143
      Stats->BuildSize += PP.getTotalMemory();
144
      if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
145
        Stats->BuildSize += PRec->getTotalMemory();
146
      Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory();
147
    }
148
  }
149

150
  void BeforeExecute(CompilerInstance &CI) override {
151
    LangOpts = &CI.getLangOpts();
152
    SourceMgr = &CI.getSourceManager();
153
    PP = &CI.getPreprocessor();
154
    Includes.collect(CI);
155
    Pragmas.record(CI);
156
    if (BeforeExecuteCallback)
157
      BeforeExecuteCallback(CI);
158
  }
159

160
  std::unique_ptr<PPCallbacks> createPPCallbacks() override {
161
    assert(SourceMgr && LangOpts && PP &&
162
           "SourceMgr, LangOpts and PP must be set at this point");
163

164
    return std::make_unique<PPChainedCallbacks>(
165
        std::make_unique<CollectMainFileMacros>(*PP, Macros),
166
        collectPragmaMarksCallback(*SourceMgr, Marks));
167
  }
168

169
  static bool isLikelyForwardingFunction(FunctionTemplateDecl *FT) {
170
    const auto *FD = FT->getTemplatedDecl();
171
    const auto NumParams = FD->getNumParams();
172
    // Check whether its last parameter is a parameter pack...
173
    if (NumParams > 0) {
174
      const auto *LastParam = FD->getParamDecl(NumParams - 1);
175
      if (const auto *PET = dyn_cast<PackExpansionType>(LastParam->getType())) {
176
        // ... of the type T&&... or T...
177
        const auto BaseType = PET->getPattern().getNonReferenceType();
178
        if (const auto *TTPT =
179
                dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) {
180
          // ... whose template parameter comes from the function directly
181
          if (FT->getTemplateParameters()->getDepth() == TTPT->getDepth()) {
182
            return true;
183
          }
184
        }
185
      }
186
    }
187
    return false;
188
  }
189

190
  bool shouldSkipFunctionBody(Decl *D) override {
191
    // Usually we don't need to look inside the bodies of header functions
192
    // to understand the program. However when forwarding function like
193
    // emplace() forward their arguments to some other function, the
194
    // interesting overload resolution happens inside the forwarding
195
    // function's body. To provide more meaningful diagnostics,
196
    // code completion, and parameter hints we should parse (and later
197
    // instantiate) the bodies.
198
    if (auto *FT = llvm::dyn_cast<clang::FunctionTemplateDecl>(D)) {
199
      if (ParseForwardingFunctions) {
200
        // Don't skip parsing the body if it looks like a forwarding function
201
        if (isLikelyForwardingFunction(FT))
202
          return false;
203
      } else {
204
        // By default, only take care of make_unique
205
        // std::make_unique is trivial, and we diagnose bad constructor calls.
206
        if (const auto *II = FT->getDeclName().getAsIdentifierInfo()) {
207
          if (II->isStr("make_unique") && FT->isInStdNamespace())
208
            return false;
209
        }
210
      }
211
    }
212
    return true;
213
  }
214

215
private:
216
  PathRef File;
217
  IncludeStructure Includes;
218
  include_cleaner::PragmaIncludes Pragmas;
219
  MainFileMacros Macros;
220
  std::vector<PragmaMark> Marks;
221
  bool IsMainFileIncludeGuarded = false;
222
  const clang::LangOptions *LangOpts = nullptr;
223
  const SourceManager *SourceMgr = nullptr;
224
  const Preprocessor *PP = nullptr;
225
  PreambleBuildStats *Stats;
226
  bool ParseForwardingFunctions;
227
  std::function<void(CompilerInstance &)> BeforeExecuteCallback;
228
  std::optional<CapturedASTCtx> CapturedCtx;
229
};
230

231
// Represents directives other than includes, where basic textual information is
232
// enough.
233
struct TextualPPDirective {
234
  unsigned DirectiveLine;
235
  // Full text that's representing the directive, including the `#`.
236
  std::string Text;
237
  unsigned Offset;
238
  tok::PPKeywordKind Directive = tok::PPKeywordKind::pp_not_keyword;
239
  // Name of the macro being defined in the case of a #define directive.
240
  std::string MacroName;
241

242
  bool operator==(const TextualPPDirective &RHS) const {
243
    return std::tie(DirectiveLine, Offset, Text) ==
244
           std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text);
245
  }
246
};
247

248
// Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X
249
// 10"). The formatting is copied so that the tokens in Body have PresumedLocs
250
// with correct columns and lines.
251
std::string spellDirective(llvm::StringRef Prefix,
252
                           CharSourceRange DirectiveRange,
253
                           const LangOptions &LangOpts, const SourceManager &SM,
254
                           unsigned &DirectiveLine, unsigned &Offset) {
255
  std::string SpelledDirective;
256
  llvm::raw_string_ostream OS(SpelledDirective);
257
  OS << Prefix;
258

259
  // Make sure DirectiveRange is a char range and doesn't contain macro ids.
260
  DirectiveRange = SM.getExpansionRange(DirectiveRange);
261
  if (DirectiveRange.isTokenRange()) {
262
    DirectiveRange.setEnd(
263
        Lexer::getLocForEndOfToken(DirectiveRange.getEnd(), 0, SM, LangOpts));
264
  }
265

266
  auto DecompLoc = SM.getDecomposedLoc(DirectiveRange.getBegin());
267
  DirectiveLine = SM.getLineNumber(DecompLoc.first, DecompLoc.second);
268
  Offset = DecompLoc.second;
269
  auto TargetColumn = SM.getColumnNumber(DecompLoc.first, DecompLoc.second) - 1;
270

271
  // Pad with spaces before DirectiveRange to make sure it will be on right
272
  // column when patched.
273
  if (Prefix.size() <= TargetColumn) {
274
    // There is enough space for Prefix and space before directive, use it.
275
    // We try to squeeze the Prefix into the same line whenever we can, as
276
    // putting onto a separate line won't work at the beginning of the file.
277
    OS << std::string(TargetColumn - Prefix.size(), ' ');
278
  } else {
279
    // Prefix was longer than the space we had. We produce e.g.:
280
    // #line N-1
281
    // #define \
282
    //    X 10
283
    OS << "\\\n" << std::string(TargetColumn, ' ');
284
    // Decrement because we put an additional line break before
285
    // DirectiveRange.begin().
286
    --DirectiveLine;
287
  }
288
  OS << toSourceCode(SM, DirectiveRange.getAsRange());
289
  return OS.str();
290
}
291

292
// Collects #define directives inside the main file.
293
struct DirectiveCollector : public PPCallbacks {
294
  DirectiveCollector(const Preprocessor &PP,
295
                     std::vector<TextualPPDirective> &TextualDirectives)
296
      : LangOpts(PP.getLangOpts()), SM(PP.getSourceManager()),
297
        TextualDirectives(TextualDirectives) {}
298

299
  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
300
                   SrcMgr::CharacteristicKind FileType,
301
                   FileID PrevFID) override {
302
    InMainFile = SM.isWrittenInMainFile(Loc);
303
  }
304

305
  void MacroDefined(const Token &MacroNameTok,
306
                    const MacroDirective *MD) override {
307
    if (!InMainFile)
308
      return;
309
    TextualDirectives.emplace_back();
310
    TextualPPDirective &TD = TextualDirectives.back();
311
    TD.Directive = tok::pp_define;
312
    TD.MacroName = MacroNameTok.getIdentifierInfo()->getName().str();
313

314
    const auto *MI = MD->getMacroInfo();
315
    TD.Text =
316
        spellDirective("#define ",
317
                       CharSourceRange::getTokenRange(
318
                           MI->getDefinitionLoc(), MI->getDefinitionEndLoc()),
319
                       LangOpts, SM, TD.DirectiveLine, TD.Offset);
320
  }
321

322
private:
323
  bool InMainFile = true;
324
  const LangOptions &LangOpts;
325
  const SourceManager &SM;
326
  std::vector<TextualPPDirective> &TextualDirectives;
327
};
328

329
struct ScannedPreamble {
330
  std::vector<Inclusion> Includes;
331
  std::vector<TextualPPDirective> TextualDirectives;
332
  // Literal lines of the preamble contents.
333
  std::vector<llvm::StringRef> Lines;
334
  PreambleBounds Bounds = {0, false};
335
  std::vector<PragmaMark> Marks;
336
  MainFileMacros Macros;
337
};
338

339
/// Scans the preprocessor directives in the preamble section of the file by
340
/// running preprocessor over \p Contents. Returned includes do not contain
341
/// resolved paths. \p Cmd is used to build the compiler invocation, which might
342
/// stat/read files.
343
llvm::Expected<ScannedPreamble>
344
scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
345
  class EmptyFS : public ThreadsafeFS {
346
  private:
347
    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
348
      return new llvm::vfs::InMemoryFileSystem;
349
    }
350
  };
351
  EmptyFS FS;
352
  // Build and run Preprocessor over the preamble.
353
  ParseInputs PI;
354
  // Memory buffers below expect null-terminated && non-null strings. So make
355
  // sure to always use PI.Contents!
356
  PI.Contents = Contents.str();
357
  PI.TFS = &FS;
358
  PI.CompileCommand = Cmd;
359
  IgnoringDiagConsumer IgnoreDiags;
360
  auto CI = buildCompilerInvocation(PI, IgnoreDiags);
361
  if (!CI)
362
    return error("failed to create compiler invocation");
363
  CI->getDiagnosticOpts().IgnoreWarnings = true;
364
  auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(PI.Contents);
365
  // This means we're scanning (though not preprocessing) the preamble section
366
  // twice. However, it's important to precisely follow the preamble bounds used
367
  // elsewhere.
368
  auto Bounds = ComputePreambleBounds(CI->getLangOpts(), *ContentsBuffer, 0);
369
  auto PreambleContents = llvm::MemoryBuffer::getMemBufferCopy(
370
      llvm::StringRef(PI.Contents).take_front(Bounds.Size));
371
  auto Clang = prepareCompilerInstance(
372
      std::move(CI), nullptr, std::move(PreambleContents),
373
      // Provide an empty FS to prevent preprocessor from performing IO. This
374
      // also implies missing resolved paths for includes.
375
      FS.view(std::nullopt), IgnoreDiags);
376
  if (Clang->getFrontendOpts().Inputs.empty())
377
    return error("compiler instance had no inputs");
378
  // We are only interested in main file includes.
379
  Clang->getPreprocessorOpts().SingleFileParseMode = true;
380
  Clang->getPreprocessorOpts().UsePredefines = false;
381
  PreprocessOnlyAction Action;
382
  if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
383
    return error("failed BeginSourceFile");
384
  Preprocessor &PP = Clang->getPreprocessor();
385
  const auto &SM = PP.getSourceManager();
386
  IncludeStructure Includes;
387
  Includes.collect(*Clang);
388
  ScannedPreamble SP;
389
  SP.Bounds = Bounds;
390
  PP.addPPCallbacks(
391
      std::make_unique<DirectiveCollector>(PP, SP.TextualDirectives));
392
  PP.addPPCallbacks(collectPragmaMarksCallback(SM, SP.Marks));
393
  PP.addPPCallbacks(std::make_unique<CollectMainFileMacros>(PP, SP.Macros));
394
  if (llvm::Error Err = Action.Execute())
395
    return std::move(Err);
396
  Action.EndSourceFile();
397
  SP.Includes = std::move(Includes.MainFileIncludes);
398
  llvm::append_range(SP.Lines, llvm::split(Contents, "\n"));
399
  return SP;
400
}
401

402
const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
403
  switch (IncludeDirective) {
404
  case tok::pp_include:
405
    return "include";
406
  case tok::pp_import:
407
    return "import";
408
  case tok::pp_include_next:
409
    return "include_next";
410
  default:
411
    break;
412
  }
413
  llvm_unreachable("not an include directive");
414
}
415

416
// Accumulating wall time timer. Similar to llvm::Timer, but much cheaper,
417
// it only tracks wall time.
418
// Since this is a generic timer, We may want to move this to support/ if we
419
// find a use case outside of FS time tracking.
420
class WallTimer {
421
public:
422
  WallTimer() : TotalTime(std::chrono::steady_clock::duration::zero()) {}
423
  // [Re-]Start the timer.
424
  void startTimer() { StartTime = std::chrono::steady_clock::now(); }
425
  // Stop the timer and update total time.
426
  void stopTimer() {
427
    TotalTime += std::chrono::steady_clock::now() - StartTime;
428
  }
429
  // Return total time, in seconds.
430
  double getTime() { return std::chrono::duration<double>(TotalTime).count(); }
431

432
private:
433
  std::chrono::steady_clock::duration TotalTime;
434
  std::chrono::steady_clock::time_point StartTime;
435
};
436

437
class WallTimerRegion {
438
public:
439
  WallTimerRegion(WallTimer &T) : T(T) { T.startTimer(); }
440
  ~WallTimerRegion() { T.stopTimer(); }
441

442
private:
443
  WallTimer &T;
444
};
445

446
// Used by TimerFS, tracks time spent in status() and getBuffer() calls while
447
// proxying to underlying File implementation.
448
class TimerFile : public llvm::vfs::File {
449
public:
450
  TimerFile(WallTimer &Timer, std::unique_ptr<File> InnerFile)
451
      : Timer(Timer), InnerFile(std::move(InnerFile)) {}
452

453
  llvm::ErrorOr<llvm::vfs::Status> status() override {
454
    WallTimerRegion T(Timer);
455
    return InnerFile->status();
456
  }
457
  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
458
  getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
459
            bool IsVolatile) override {
460
    WallTimerRegion T(Timer);
461
    return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
462
                                IsVolatile);
463
  }
464
  std::error_code close() override {
465
    WallTimerRegion T(Timer);
466
    return InnerFile->close();
467
  }
468

469
private:
470
  WallTimer &Timer;
471
  std::unique_ptr<llvm::vfs::File> InnerFile;
472
};
473

474
// A wrapper for FileSystems that tracks the amount of time spent in status()
475
// and openFileForRead() calls.
476
class TimerFS : public llvm::vfs::ProxyFileSystem {
477
public:
478
  TimerFS(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
479
      : ProxyFileSystem(std::move(FS)) {}
480

481
  llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
482
  openFileForRead(const llvm::Twine &Path) override {
483
    WallTimerRegion T(Timer);
484
    auto FileOr = getUnderlyingFS().openFileForRead(Path);
485
    if (!FileOr)
486
      return FileOr;
487
    return std::make_unique<TimerFile>(Timer, std::move(FileOr.get()));
488
  }
489

490
  llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
491
    WallTimerRegion T(Timer);
492
    return getUnderlyingFS().status(Path);
493
  }
494

495
  double getTime() { return Timer.getTime(); }
496

497
private:
498
  WallTimer Timer;
499
};
500

501
// Helpers for patching diagnostics between two versions of file contents.
502
class DiagPatcher {
503
  llvm::ArrayRef<llvm::StringRef> OldLines;
504
  llvm::ArrayRef<llvm::StringRef> CurrentLines;
505
  llvm::StringMap<llvm::SmallVector<int>> CurrentContentsToLine;
506

507
  // Translates a range from old lines to current lines.
508
  // Finds the consecutive set of lines that corresponds to the same contents in
509
  // old and current, and applies the same translation to the range.
510
  // Returns true if translation succeeded.
511
  bool translateRange(Range &R) {
512
    int OldStart = R.start.line;
513
    int OldEnd = R.end.line;
514
    assert(OldStart <= OldEnd);
515

516
    size_t RangeLen = OldEnd - OldStart + 1;
517
    auto RangeContents = OldLines.slice(OldStart).take_front(RangeLen);
518
    // Make sure the whole range is covered in old contents.
519
    if (RangeContents.size() < RangeLen)
520
      return false;
521

522
    std::optional<int> Closest;
523
    for (int AlternateLine : CurrentContentsToLine.lookup(RangeContents[0])) {
524
      // Check if AlternateLine matches all lines in the range.
525
      if (RangeContents !=
526
          CurrentLines.slice(AlternateLine).take_front(RangeLen))
527
        continue;
528
      int Delta = AlternateLine - OldStart;
529
      if (!Closest.has_value() || abs(Delta) < abs(*Closest))
530
        Closest = Delta;
531
    }
532
    // Couldn't find any viable matches in the current contents.
533
    if (!Closest.has_value())
534
      return false;
535
    R.start.line += *Closest;
536
    R.end.line += *Closest;
537
    return true;
538
  }
539

540
  // Translates a Note by patching its range when inside main file. Returns true
541
  // on success.
542
  bool translateNote(Note &N) {
543
    if (!N.InsideMainFile)
544
      return true;
545
    if (translateRange(N.Range))
546
      return true;
547
    return false;
548
  }
549

550
  // Tries to translate all the edit ranges inside the fix. Returns true on
551
  // success. On failure fixes might be in an invalid state.
552
  bool translateFix(Fix &F) {
553
    return llvm::all_of(
554
        F.Edits, [this](TextEdit &E) { return translateRange(E.range); });
555
  }
556

557
public:
558
  DiagPatcher(llvm::ArrayRef<llvm::StringRef> OldLines,
559
              llvm::ArrayRef<llvm::StringRef> CurrentLines) {
560
    this->OldLines = OldLines;
561
    this->CurrentLines = CurrentLines;
562
    for (int Line = 0, E = CurrentLines.size(); Line != E; ++Line) {
563
      llvm::StringRef Contents = CurrentLines[Line];
564
      CurrentContentsToLine[Contents].push_back(Line);
565
    }
566
  }
567
  // Translate diagnostic by moving its main range to new location (if inside
568
  // the main file). Preserve all the notes and fixes that can be translated to
569
  // new contents.
570
  // Drops the whole diagnostic if main range can't be patched.
571
  std::optional<Diag> translateDiag(const Diag &D) {
572
    Range NewRange = D.Range;
573
    // Patch range if it's inside main file.
574
    if (D.InsideMainFile && !translateRange(NewRange)) {
575
      // Drop the diagnostic if we couldn't patch the range.
576
      return std::nullopt;
577
    }
578

579
    Diag NewD = D;
580
    NewD.Range = NewRange;
581
    // Translate ranges inside notes and fixes too, dropping the ones that are
582
    // no longer relevant.
583
    llvm::erase_if(NewD.Notes, [this](Note &N) { return !translateNote(N); });
584
    llvm::erase_if(NewD.Fixes, [this](Fix &F) { return !translateFix(F); });
585
    return NewD;
586
  }
587
};
588
} // namespace
589

590
std::shared_ptr<const PreambleData>
591
buildPreamble(PathRef FileName, CompilerInvocation CI,
592
              const ParseInputs &Inputs, bool StoreInMemory,
593
              PreambleParsedCallback PreambleCallback,
594
              PreambleBuildStats *Stats) {
595
  // Note that we don't need to copy the input contents, preamble can live
596
  // without those.
597
  auto ContentsBuffer =
598
      llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
599
  auto Bounds = ComputePreambleBounds(CI.getLangOpts(), *ContentsBuffer, 0);
600

601
  trace::Span Tracer("BuildPreamble");
602
  SPAN_ATTACH(Tracer, "File", FileName);
603
  std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
604
  if (Inputs.FeatureModules) {
605
    for (auto &M : *Inputs.FeatureModules) {
606
      if (auto Listener = M.astListeners())
607
        ASTListeners.emplace_back(std::move(Listener));
608
    }
609
  }
610
  StoreDiags PreambleDiagnostics;
611
  PreambleDiagnostics.setDiagCallback(
612
      [&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
613
        for (const auto &L : ASTListeners)
614
          L->sawDiagnostic(D, Diag);
615
      });
616
  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
617
      CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
618
                                          &PreambleDiagnostics,
619
                                          /*ShouldOwnClient=*/false);
620
  const Config &Cfg = Config::current();
621
  PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
622
                                           const clang::Diagnostic &Info) {
623
    if (Cfg.Diagnostics.SuppressAll ||
624
        isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
625
                                      CI.getLangOpts()))
626
      return DiagnosticsEngine::Ignored;
627
    switch (Info.getID()) {
628
    case diag::warn_no_newline_eof:
629
    case diag::warn_cxx98_compat_no_newline_eof:
630
    case diag::ext_no_newline_eof:
631
      // If the preamble doesn't span the whole file, drop the no newline at
632
      // eof warnings.
633
      return Bounds.Size != ContentsBuffer->getBufferSize()
634
                 ? DiagnosticsEngine::Level::Ignored
635
                 : DiagLevel;
636
    }
637
    return DiagLevel;
638
  });
639

640
  // Skip function bodies when building the preamble to speed up building
641
  // the preamble and make it smaller.
642
  assert(!CI.getFrontendOpts().SkipFunctionBodies);
643
  CI.getFrontendOpts().SkipFunctionBodies = true;
644
  // We don't want to write comment locations into PCH. They are racy and slow
645
  // to read back. We rely on dynamic index for the comments instead.
646
  CI.getPreprocessorOpts().WriteCommentListToPCH = false;
647

648
  CppFilePreambleCallbacks CapturedInfo(
649
      FileName, Stats, Inputs.Opts.PreambleParseForwardingFunctions,
650
      [&ASTListeners](CompilerInstance &CI) {
651
        for (const auto &L : ASTListeners)
652
          L->beforeExecute(CI);
653
      });
654
  auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
655
  llvm::SmallString<32> AbsFileName(FileName);
656
  VFS->makeAbsolute(AbsFileName);
657
  auto StatCache = std::make_shared<PreambleFileStatusCache>(AbsFileName);
658
  auto StatCacheFS = StatCache->getProducingFS(VFS);
659
  llvm::IntrusiveRefCntPtr<TimerFS> TimedFS(new TimerFS(StatCacheFS));
660

661
  WallTimer PreambleTimer;
662
  PreambleTimer.startTimer();
663
  auto BuiltPreamble = PrecompiledPreamble::Build(
664
      CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
665
      Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(),
666
      StoreInMemory, /*StoragePath=*/"", CapturedInfo);
667
  PreambleTimer.stopTimer();
668

669
  // We have to setup DiagnosticConsumer that will be alife
670
  // while preamble callback is executed
671
  PreambleDiagsEngine->setClient(new IgnoringDiagConsumer, true);
672
  // Reset references to ref-counted-ptrs before executing the callbacks, to
673
  // prevent resetting them concurrently.
674
  PreambleDiagsEngine.reset();
675
  CI.DiagnosticOpts.reset();
676

677
  // When building the AST for the main file, we do want the function
678
  // bodies.
679
  CI.getFrontendOpts().SkipFunctionBodies = false;
680

681
  if (Stats != nullptr) {
682
    Stats->TotalBuildTime = PreambleTimer.getTime();
683
    Stats->FileSystemTime = TimedFS->getTime();
684
    Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0;
685
  }
686

687
  if (BuiltPreamble) {
688
    log("Built preamble of size {0} for file {1} version {2} in {3} seconds",
689
        BuiltPreamble->getSize(), FileName, Inputs.Version,
690
        PreambleTimer.getTime());
691
    std::vector<Diag> Diags = PreambleDiagnostics.take();
692
    auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble));
693
    Result->Version = Inputs.Version;
694
    Result->CompileCommand = Inputs.CompileCommand;
695
    Result->Diags = std::move(Diags);
696
    Result->Includes = CapturedInfo.takeIncludes();
697
    Result->Pragmas = std::make_shared<const include_cleaner::PragmaIncludes>(
698
        CapturedInfo.takePragmaIncludes());
699
    Result->Macros = CapturedInfo.takeMacros();
700
    Result->Marks = CapturedInfo.takeMarks();
701
    Result->StatCache = StatCache;
702
    Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded();
703
    Result->TargetOpts = CI.TargetOpts;
704
    if (PreambleCallback) {
705
      trace::Span Tracer("Running PreambleCallback");
706
      auto Ctx = CapturedInfo.takeLife();
707
      // Stat cache is thread safe only when there are no producers. Hence
708
      // change the VFS underneath to a consuming fs.
709
      Ctx->getFileManager().setVirtualFileSystem(
710
          Result->StatCache->getConsumingFS(VFS));
711
      // While extending the life of FileMgr and VFS, StatCache should also be
712
      // extended.
713
      Ctx->setStatCache(Result->StatCache);
714

715
      PreambleCallback(std::move(*Ctx), Result->Pragmas);
716
    }
717
    return Result;
718
  }
719

720
  elog("Could not build a preamble for file {0} version {1}: {2}", FileName,
721
       Inputs.Version, BuiltPreamble.getError().message());
722
  for (const Diag &D : PreambleDiagnostics.take()) {
723
    if (D.Severity < DiagnosticsEngine::Error)
724
      continue;
725
    // Not an ideal way to show errors, but better than nothing!
726
    elog("  error: {0}", D.Message);
727
  }
728
  return nullptr;
729
}
730

731
bool isPreambleCompatible(const PreambleData &Preamble,
732
                          const ParseInputs &Inputs, PathRef FileName,
733
                          const CompilerInvocation &CI) {
734
  auto ContentsBuffer =
735
      llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
736
  auto Bounds = ComputePreambleBounds(CI.getLangOpts(), *ContentsBuffer, 0);
737
  auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
738
  return compileCommandsAreEqual(Inputs.CompileCommand,
739
                                 Preamble.CompileCommand) &&
740
         Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS);
741
}
742

743
void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
744
  for (char C : Text) {
745
    switch (C) {
746
    case '\\':
747
    case '"':
748
      OS << '\\';
749
      break;
750
    default:
751
      break;
752
    }
753
    OS << C;
754
  }
755
}
756

757
// Translate diagnostics from baseline into modified for the lines that have the
758
// same spelling.
759
static std::vector<Diag> patchDiags(llvm::ArrayRef<Diag> BaselineDiags,
760
                                    const ScannedPreamble &BaselineScan,
761
                                    const ScannedPreamble &ModifiedScan) {
762
  std::vector<Diag> PatchedDiags;
763
  if (BaselineDiags.empty())
764
    return PatchedDiags;
765
  DiagPatcher Patcher(BaselineScan.Lines, ModifiedScan.Lines);
766
  for (auto &D : BaselineDiags) {
767
    if (auto NewD = Patcher.translateDiag(D))
768
      PatchedDiags.emplace_back(std::move(*NewD));
769
  }
770
  return PatchedDiags;
771
}
772

773
static std::string getPatchName(llvm::StringRef FileName) {
774
  // This shouldn't coincide with any real file name.
775
  llvm::SmallString<128> PatchName;
776
  llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
777
                          PreamblePatch::HeaderName);
778
  return PatchName.str().str();
779
}
780

781
PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
782
                                    const ParseInputs &Modified,
783
                                    const PreambleData &Baseline,
784
                                    PatchType PatchType) {
785
  trace::Span Tracer("CreatePreamblePatch");
786
  SPAN_ATTACH(Tracer, "File", FileName);
787
  assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!");
788
  // First scan preprocessor directives in Baseline and Modified. These will be
789
  // used to figure out newly added directives in Modified. Scanning can fail,
790
  // the code just bails out and creates an empty patch in such cases, as:
791
  // - If scanning for Baseline fails, no knowledge of existing includes hence
792
  //   patch will contain all the includes in Modified. Leading to rebuild of
793
  //   whole preamble, which is terribly slow.
794
  // - If scanning for Modified fails, cannot figure out newly added ones so
795
  //   there's nothing to do but generate an empty patch.
796
  auto BaselineScan =
797
      scanPreamble(Baseline.Preamble.getContents(), Modified.CompileCommand);
798
  if (!BaselineScan) {
799
    elog("Failed to scan baseline of {0}: {1}", FileName,
800
         BaselineScan.takeError());
801
    return PreamblePatch::unmodified(Baseline);
802
  }
803
  auto ModifiedScan = scanPreamble(Modified.Contents, Modified.CompileCommand);
804
  if (!ModifiedScan) {
805
    elog("Failed to scan modified contents of {0}: {1}", FileName,
806
         ModifiedScan.takeError());
807
    return PreamblePatch::unmodified(Baseline);
808
  }
809

810
  bool IncludesChanged = BaselineScan->Includes != ModifiedScan->Includes;
811
  bool DirectivesChanged =
812
      BaselineScan->TextualDirectives != ModifiedScan->TextualDirectives;
813
  if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) &&
814
      !DirectivesChanged)
815
    return PreamblePatch::unmodified(Baseline);
816

817
  PreamblePatch PP;
818
  PP.Baseline = &Baseline;
819
  PP.PatchFileName = getPatchName(FileName);
820
  PP.ModifiedBounds = ModifiedScan->Bounds;
821

822
  llvm::raw_string_ostream Patch(PP.PatchContents);
823
  // Set default filename for subsequent #line directives
824
  Patch << "#line 0 \"";
825
  // FileName part of a line directive is subject to backslash escaping, which
826
  // might lead to problems on windows especially.
827
  escapeBackslashAndQuotes(FileName, Patch);
828
  Patch << "\"\n";
829

830
  if (IncludesChanged && PatchType == PatchType::All) {
831
    // We are only interested in newly added includes, record the ones in
832
    // Baseline for exclusion.
833
    llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>,
834
                   const Inclusion *>
835
        ExistingIncludes;
836
    for (const auto &Inc : Baseline.Includes.MainFileIncludes)
837
      ExistingIncludes[{Inc.Directive, Inc.Written}] = &Inc;
838
    // There might be includes coming from disabled regions, record these for
839
    // exclusion too. note that we don't have resolved paths for those.
840
    for (const auto &Inc : BaselineScan->Includes)
841
      ExistingIncludes.try_emplace({Inc.Directive, Inc.Written});
842
    // Calculate extra includes that needs to be inserted.
843
    for (auto &Inc : ModifiedScan->Includes) {
844
      auto It = ExistingIncludes.find({Inc.Directive, Inc.Written});
845
      // Include already present in the baseline preamble. Set resolved path and
846
      // put into preamble includes.
847
      if (It != ExistingIncludes.end()) {
848
        if (It->second) {
849
          // If this header is included in an active region of the baseline
850
          // preamble, preserve it.
851
          auto &PatchedInc = PP.PreambleIncludes.emplace_back();
852
          // Copy everything from existing include, apart from the location,
853
          // when it's coming from baseline preamble.
854
          PatchedInc = *It->second;
855
          PatchedInc.HashLine = Inc.HashLine;
856
          PatchedInc.HashOffset = Inc.HashOffset;
857
        }
858
        continue;
859
      }
860
      // Include is new in the modified preamble. Inject it into the patch and
861
      // use #line to set the presumed location to where it is spelled.
862
      auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset);
863
      Patch << llvm::formatv("#line {0}\n", LineCol.first);
864
      Patch << llvm::formatv(
865
          "#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written);
866
    }
867
  } else {
868
    // Make sure we have the full set of includes available even when we're not
869
    // patching. As these are used by features we provide afterwards like hover,
870
    // go-to-def or include-cleaner when preamble is stale.
871
    PP.PreambleIncludes = Baseline.Includes.MainFileIncludes;
872
  }
873

874
  if (DirectivesChanged) {
875
    // We need to patch all the directives, since they are order dependent. e.g:
876
    // #define BAR(X) NEW(X) // Newly introduced in Modified
877
    // #define BAR(X) OLD(X) // Exists in the Baseline
878
    //
879
    // If we've patched only the first directive, the macro definition would've
880
    // been wrong for the rest of the file, since patch is applied after the
881
    // baseline preamble.
882
    //
883
    // Note that we deliberately ignore conditional directives and undefs to
884
    // reduce complexity. The former might cause problems because scanning is
885
    // imprecise and might pick directives from disabled regions.
886
    for (const auto &TD : ModifiedScan->TextualDirectives) {
887
      // Introduce an #undef directive before #defines to suppress any
888
      // re-definition warnings.
889
      if (TD.Directive == tok::pp_define)
890
        Patch << "#undef " << TD.MacroName << '\n';
891
      Patch << "#line " << TD.DirectiveLine << '\n';
892
      Patch << TD.Text << '\n';
893
    }
894
  }
895

896
  PP.PatchedDiags = patchDiags(Baseline.Diags, *BaselineScan, *ModifiedScan);
897
  PP.PatchedMarks = std::move(ModifiedScan->Marks);
898
  PP.PatchedMacros = std::move(ModifiedScan->Macros);
899
  dlog("Created preamble patch: {0}", Patch.str());
900
  Patch.flush();
901
  return PP;
902
}
903

904
PreamblePatch PreamblePatch::createFullPatch(llvm::StringRef FileName,
905
                                             const ParseInputs &Modified,
906
                                             const PreambleData &Baseline) {
907
  return create(FileName, Modified, Baseline, PatchType::All);
908
}
909

910
PreamblePatch PreamblePatch::createMacroPatch(llvm::StringRef FileName,
911
                                              const ParseInputs &Modified,
912
                                              const PreambleData &Baseline) {
913
  return create(FileName, Modified, Baseline, PatchType::MacroDirectives);
914
}
915

916
void PreamblePatch::apply(CompilerInvocation &CI) const {
917
  // Make sure the compilation uses same target opts as the preamble. Clang has
918
  // no guarantees around using arbitrary options when reusing PCHs, and
919
  // different target opts can result in crashes, see
920
  // ParsedASTTest.PreambleWithDifferentTarget.
921
  // Make sure this is a deep copy, as the same Baseline might be used
922
  // concurrently.
923
  *CI.TargetOpts = *Baseline->TargetOpts;
924

925
  // No need to map an empty file.
926
  if (PatchContents.empty())
927
    return;
928
  auto &PPOpts = CI.getPreprocessorOpts();
929
  auto PatchBuffer =
930
      // we copy here to ensure contents are still valid if CI outlives the
931
      // PreamblePatch.
932
      llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName);
933
  // CI will take care of the lifetime of the buffer.
934
  PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release());
935
  // The patch will be parsed after loading the preamble ast and before parsing
936
  // the main file.
937
  PPOpts.Includes.push_back(PatchFileName);
938
}
939

940
std::vector<Inclusion> PreamblePatch::preambleIncludes() const {
941
  return PreambleIncludes;
942
}
943

944
PreamblePatch PreamblePatch::unmodified(const PreambleData &Preamble) {
945
  PreamblePatch PP;
946
  PP.Baseline = &Preamble;
947
  PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
948
  PP.ModifiedBounds = Preamble.Preamble.getBounds();
949
  PP.PatchedDiags = Preamble.Diags;
950
  return PP;
951
}
952

953
llvm::ArrayRef<PragmaMark> PreamblePatch::marks() const {
954
  if (PatchContents.empty())
955
    return Baseline->Marks;
956
  return PatchedMarks;
957
}
958

959
const MainFileMacros &PreamblePatch::mainFileMacros() const {
960
  if (PatchContents.empty())
961
    return Baseline->Macros;
962
  return PatchedMacros;
963
}
964

965
OptionalFileEntryRef PreamblePatch::getPatchEntry(llvm::StringRef MainFilePath,
966
                                                  const SourceManager &SM) {
967
  auto PatchFilePath = getPatchName(MainFilePath);
968
  return SM.getFileManager().getOptionalFileRef(PatchFilePath);
969
}
970
} // namespace clangd
971
} // namespace clang
972