llvm-project
971 строка · 37.1 Кб
1//===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Preamble.h"
10#include "CollectMacros.h"
11#include "Compiler.h"
12#include "Config.h"
13#include "Diagnostics.h"
14#include "FS.h"
15#include "FeatureModule.h"
16#include "Headers.h"
17#include "Protocol.h"
18#include "SourceCode.h"
19#include "clang-include-cleaner/Record.h"
20#include "support/Logger.h"
21#include "support/Path.h"
22#include "support/ThreadsafeFS.h"
23#include "support/Trace.h"
24#include "clang/AST/DeclTemplate.h"
25#include "clang/AST/Type.h"
26#include "clang/Basic/Diagnostic.h"
27#include "clang/Basic/DiagnosticLex.h"
28#include "clang/Basic/DiagnosticOptions.h"
29#include "clang/Basic/LangOptions.h"
30#include "clang/Basic/SourceLocation.h"
31#include "clang/Basic/SourceManager.h"
32#include "clang/Basic/TargetInfo.h"
33#include "clang/Basic/TokenKinds.h"
34#include "clang/Frontend/CompilerInstance.h"
35#include "clang/Frontend/CompilerInvocation.h"
36#include "clang/Frontend/FrontendActions.h"
37#include "clang/Frontend/PrecompiledPreamble.h"
38#include "clang/Lex/HeaderSearch.h"
39#include "clang/Lex/Lexer.h"
40#include "clang/Lex/PPCallbacks.h"
41#include "clang/Lex/Preprocessor.h"
42#include "clang/Lex/PreprocessorOptions.h"
43#include "clang/Serialization/ASTReader.h"
44#include "clang/Tooling/CompilationDatabase.h"
45#include "llvm/ADT/ArrayRef.h"
46#include "llvm/ADT/DenseMap.h"
47#include "llvm/ADT/IntrusiveRefCntPtr.h"
48#include "llvm/ADT/STLExtras.h"
49#include "llvm/ADT/SmallString.h"
50#include "llvm/ADT/SmallVector.h"
51#include "llvm/ADT/StringExtras.h"
52#include "llvm/ADT/StringMap.h"
53#include "llvm/ADT/StringRef.h"
54#include "llvm/Support/Casting.h"
55#include "llvm/Support/Error.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/ErrorOr.h"
58#include "llvm/Support/FormatVariadic.h"
59#include "llvm/Support/MemoryBuffer.h"
60#include "llvm/Support/Path.h"
61#include "llvm/Support/VirtualFileSystem.h"
62#include "llvm/Support/raw_ostream.h"
63#include <cassert>
64#include <chrono>
65#include <cstddef>
66#include <cstdint>
67#include <cstdlib>
68#include <functional>
69#include <memory>
70#include <optional>
71#include <string>
72#include <system_error>
73#include <tuple>
74#include <utility>
75#include <vector>
76
77namespace clang {
78namespace clangd {
79namespace {
80
81bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
82const tooling::CompileCommand &RHS) {
83// We don't check for Output, it should not matter to clangd.
84return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
85llvm::ArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
86}
87
88class CppFilePreambleCallbacks : public PreambleCallbacks {
89public:
90CppFilePreambleCallbacks(
91PathRef File, PreambleBuildStats *Stats, bool ParseForwardingFunctions,
92std::function<void(CompilerInstance &)> BeforeExecuteCallback)
93: File(File), Stats(Stats),
94ParseForwardingFunctions(ParseForwardingFunctions),
95BeforeExecuteCallback(std::move(BeforeExecuteCallback)) {}
96
97IncludeStructure takeIncludes() { return std::move(Includes); }
98
99MainFileMacros takeMacros() { return std::move(Macros); }
100
101std::vector<PragmaMark> takeMarks() { return std::move(Marks); }
102
103include_cleaner::PragmaIncludes takePragmaIncludes() {
104return std::move(Pragmas);
105}
106
107std::optional<CapturedASTCtx> takeLife() { return std::move(CapturedCtx); }
108
109bool isMainFileIncludeGuarded() const { return IsMainFileIncludeGuarded; }
110
111void AfterExecute(CompilerInstance &CI) override {
112// As part of the Preamble compilation, ASTConsumer
113// PrecompilePreambleConsumer/PCHGenerator is setup. This would be called
114// when Preamble consists of modules. Therefore while capturing AST context,
115// we have to reset ast consumer and ASTMutationListener.
116if (CI.getASTReader()) {
117CI.getASTReader()->setDeserializationListener(nullptr);
118// This just sets consumer to null when DeserializationListener is null.
119CI.getASTReader()->StartTranslationUnit(nullptr);
120}
121CI.getASTContext().setASTMutationListener(nullptr);
122CapturedCtx.emplace(CI);
123
124const SourceManager &SM = CI.getSourceManager();
125OptionalFileEntryRef MainFE = SM.getFileEntryRefForID(SM.getMainFileID());
126IsMainFileIncludeGuarded =
127CI.getPreprocessor().getHeaderSearchInfo().isFileMultipleIncludeGuarded(
128*MainFE);
129
130if (Stats) {
131const ASTContext &AST = CI.getASTContext();
132Stats->BuildSize = AST.getASTAllocatedMemory();
133Stats->BuildSize += AST.getSideTableAllocatedMemory();
134Stats->BuildSize += AST.Idents.getAllocator().getTotalMemory();
135Stats->BuildSize += AST.Selectors.getTotalMemory();
136
137Stats->BuildSize += AST.getSourceManager().getContentCacheSize();
138Stats->BuildSize += AST.getSourceManager().getDataStructureSizes();
139Stats->BuildSize +=
140AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
141
142const Preprocessor &PP = CI.getPreprocessor();
143Stats->BuildSize += PP.getTotalMemory();
144if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
145Stats->BuildSize += PRec->getTotalMemory();
146Stats->BuildSize += PP.getHeaderSearchInfo().getTotalMemory();
147}
148}
149
150void BeforeExecute(CompilerInstance &CI) override {
151LangOpts = &CI.getLangOpts();
152SourceMgr = &CI.getSourceManager();
153PP = &CI.getPreprocessor();
154Includes.collect(CI);
155Pragmas.record(CI);
156if (BeforeExecuteCallback)
157BeforeExecuteCallback(CI);
158}
159
160std::unique_ptr<PPCallbacks> createPPCallbacks() override {
161assert(SourceMgr && LangOpts && PP &&
162"SourceMgr, LangOpts and PP must be set at this point");
163
164return std::make_unique<PPChainedCallbacks>(
165std::make_unique<CollectMainFileMacros>(*PP, Macros),
166collectPragmaMarksCallback(*SourceMgr, Marks));
167}
168
169static bool isLikelyForwardingFunction(FunctionTemplateDecl *FT) {
170const auto *FD = FT->getTemplatedDecl();
171const auto NumParams = FD->getNumParams();
172// Check whether its last parameter is a parameter pack...
173if (NumParams > 0) {
174const auto *LastParam = FD->getParamDecl(NumParams - 1);
175if (const auto *PET = dyn_cast<PackExpansionType>(LastParam->getType())) {
176// ... of the type T&&... or T...
177const auto BaseType = PET->getPattern().getNonReferenceType();
178if (const auto *TTPT =
179dyn_cast<TemplateTypeParmType>(BaseType.getTypePtr())) {
180// ... whose template parameter comes from the function directly
181if (FT->getTemplateParameters()->getDepth() == TTPT->getDepth()) {
182return true;
183}
184}
185}
186}
187return false;
188}
189
190bool shouldSkipFunctionBody(Decl *D) override {
191// Usually we don't need to look inside the bodies of header functions
192// to understand the program. However when forwarding function like
193// emplace() forward their arguments to some other function, the
194// interesting overload resolution happens inside the forwarding
195// function's body. To provide more meaningful diagnostics,
196// code completion, and parameter hints we should parse (and later
197// instantiate) the bodies.
198if (auto *FT = llvm::dyn_cast<clang::FunctionTemplateDecl>(D)) {
199if (ParseForwardingFunctions) {
200// Don't skip parsing the body if it looks like a forwarding function
201if (isLikelyForwardingFunction(FT))
202return false;
203} else {
204// By default, only take care of make_unique
205// std::make_unique is trivial, and we diagnose bad constructor calls.
206if (const auto *II = FT->getDeclName().getAsIdentifierInfo()) {
207if (II->isStr("make_unique") && FT->isInStdNamespace())
208return false;
209}
210}
211}
212return true;
213}
214
215private:
216PathRef File;
217IncludeStructure Includes;
218include_cleaner::PragmaIncludes Pragmas;
219MainFileMacros Macros;
220std::vector<PragmaMark> Marks;
221bool IsMainFileIncludeGuarded = false;
222const clang::LangOptions *LangOpts = nullptr;
223const SourceManager *SourceMgr = nullptr;
224const Preprocessor *PP = nullptr;
225PreambleBuildStats *Stats;
226bool ParseForwardingFunctions;
227std::function<void(CompilerInstance &)> BeforeExecuteCallback;
228std::optional<CapturedASTCtx> CapturedCtx;
229};
230
231// Represents directives other than includes, where basic textual information is
232// enough.
233struct TextualPPDirective {
234unsigned DirectiveLine;
235// Full text that's representing the directive, including the `#`.
236std::string Text;
237unsigned Offset;
238tok::PPKeywordKind Directive = tok::PPKeywordKind::pp_not_keyword;
239// Name of the macro being defined in the case of a #define directive.
240std::string MacroName;
241
242bool operator==(const TextualPPDirective &RHS) const {
243return std::tie(DirectiveLine, Offset, Text) ==
244std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text);
245}
246};
247
248// Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X
249// 10"). The formatting is copied so that the tokens in Body have PresumedLocs
250// with correct columns and lines.
251std::string spellDirective(llvm::StringRef Prefix,
252CharSourceRange DirectiveRange,
253const LangOptions &LangOpts, const SourceManager &SM,
254unsigned &DirectiveLine, unsigned &Offset) {
255std::string SpelledDirective;
256llvm::raw_string_ostream OS(SpelledDirective);
257OS << Prefix;
258
259// Make sure DirectiveRange is a char range and doesn't contain macro ids.
260DirectiveRange = SM.getExpansionRange(DirectiveRange);
261if (DirectiveRange.isTokenRange()) {
262DirectiveRange.setEnd(
263Lexer::getLocForEndOfToken(DirectiveRange.getEnd(), 0, SM, LangOpts));
264}
265
266auto DecompLoc = SM.getDecomposedLoc(DirectiveRange.getBegin());
267DirectiveLine = SM.getLineNumber(DecompLoc.first, DecompLoc.second);
268Offset = DecompLoc.second;
269auto TargetColumn = SM.getColumnNumber(DecompLoc.first, DecompLoc.second) - 1;
270
271// Pad with spaces before DirectiveRange to make sure it will be on right
272// column when patched.
273if (Prefix.size() <= TargetColumn) {
274// There is enough space for Prefix and space before directive, use it.
275// We try to squeeze the Prefix into the same line whenever we can, as
276// putting onto a separate line won't work at the beginning of the file.
277OS << std::string(TargetColumn - Prefix.size(), ' ');
278} else {
279// Prefix was longer than the space we had. We produce e.g.:
280// #line N-1
281// #define \
282// X 10
283OS << "\\\n" << std::string(TargetColumn, ' ');
284// Decrement because we put an additional line break before
285// DirectiveRange.begin().
286--DirectiveLine;
287}
288OS << toSourceCode(SM, DirectiveRange.getAsRange());
289return OS.str();
290}
291
292// Collects #define directives inside the main file.
293struct DirectiveCollector : public PPCallbacks {
294DirectiveCollector(const Preprocessor &PP,
295std::vector<TextualPPDirective> &TextualDirectives)
296: LangOpts(PP.getLangOpts()), SM(PP.getSourceManager()),
297TextualDirectives(TextualDirectives) {}
298
299void FileChanged(SourceLocation Loc, FileChangeReason Reason,
300SrcMgr::CharacteristicKind FileType,
301FileID PrevFID) override {
302InMainFile = SM.isWrittenInMainFile(Loc);
303}
304
305void MacroDefined(const Token &MacroNameTok,
306const MacroDirective *MD) override {
307if (!InMainFile)
308return;
309TextualDirectives.emplace_back();
310TextualPPDirective &TD = TextualDirectives.back();
311TD.Directive = tok::pp_define;
312TD.MacroName = MacroNameTok.getIdentifierInfo()->getName().str();
313
314const auto *MI = MD->getMacroInfo();
315TD.Text =
316spellDirective("#define ",
317CharSourceRange::getTokenRange(
318MI->getDefinitionLoc(), MI->getDefinitionEndLoc()),
319LangOpts, SM, TD.DirectiveLine, TD.Offset);
320}
321
322private:
323bool InMainFile = true;
324const LangOptions &LangOpts;
325const SourceManager &SM;
326std::vector<TextualPPDirective> &TextualDirectives;
327};
328
329struct ScannedPreamble {
330std::vector<Inclusion> Includes;
331std::vector<TextualPPDirective> TextualDirectives;
332// Literal lines of the preamble contents.
333std::vector<llvm::StringRef> Lines;
334PreambleBounds Bounds = {0, false};
335std::vector<PragmaMark> Marks;
336MainFileMacros Macros;
337};
338
339/// Scans the preprocessor directives in the preamble section of the file by
340/// running preprocessor over \p Contents. Returned includes do not contain
341/// resolved paths. \p Cmd is used to build the compiler invocation, which might
342/// stat/read files.
343llvm::Expected<ScannedPreamble>
344scanPreamble(llvm::StringRef Contents, const tooling::CompileCommand &Cmd) {
345class EmptyFS : public ThreadsafeFS {
346private:
347llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> viewImpl() const override {
348return new llvm::vfs::InMemoryFileSystem;
349}
350};
351EmptyFS FS;
352// Build and run Preprocessor over the preamble.
353ParseInputs PI;
354// Memory buffers below expect null-terminated && non-null strings. So make
355// sure to always use PI.Contents!
356PI.Contents = Contents.str();
357PI.TFS = &FS;
358PI.CompileCommand = Cmd;
359IgnoringDiagConsumer IgnoreDiags;
360auto CI = buildCompilerInvocation(PI, IgnoreDiags);
361if (!CI)
362return error("failed to create compiler invocation");
363CI->getDiagnosticOpts().IgnoreWarnings = true;
364auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(PI.Contents);
365// This means we're scanning (though not preprocessing) the preamble section
366// twice. However, it's important to precisely follow the preamble bounds used
367// elsewhere.
368auto Bounds = ComputePreambleBounds(CI->getLangOpts(), *ContentsBuffer, 0);
369auto PreambleContents = llvm::MemoryBuffer::getMemBufferCopy(
370llvm::StringRef(PI.Contents).take_front(Bounds.Size));
371auto Clang = prepareCompilerInstance(
372std::move(CI), nullptr, std::move(PreambleContents),
373// Provide an empty FS to prevent preprocessor from performing IO. This
374// also implies missing resolved paths for includes.
375FS.view(std::nullopt), IgnoreDiags);
376if (Clang->getFrontendOpts().Inputs.empty())
377return error("compiler instance had no inputs");
378// We are only interested in main file includes.
379Clang->getPreprocessorOpts().SingleFileParseMode = true;
380Clang->getPreprocessorOpts().UsePredefines = false;
381PreprocessOnlyAction Action;
382if (!Action.BeginSourceFile(*Clang, Clang->getFrontendOpts().Inputs[0]))
383return error("failed BeginSourceFile");
384Preprocessor &PP = Clang->getPreprocessor();
385const auto &SM = PP.getSourceManager();
386IncludeStructure Includes;
387Includes.collect(*Clang);
388ScannedPreamble SP;
389SP.Bounds = Bounds;
390PP.addPPCallbacks(
391std::make_unique<DirectiveCollector>(PP, SP.TextualDirectives));
392PP.addPPCallbacks(collectPragmaMarksCallback(SM, SP.Marks));
393PP.addPPCallbacks(std::make_unique<CollectMainFileMacros>(PP, SP.Macros));
394if (llvm::Error Err = Action.Execute())
395return std::move(Err);
396Action.EndSourceFile();
397SP.Includes = std::move(Includes.MainFileIncludes);
398llvm::append_range(SP.Lines, llvm::split(Contents, "\n"));
399return SP;
400}
401
402const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
403switch (IncludeDirective) {
404case tok::pp_include:
405return "include";
406case tok::pp_import:
407return "import";
408case tok::pp_include_next:
409return "include_next";
410default:
411break;
412}
413llvm_unreachable("not an include directive");
414}
415
416// Accumulating wall time timer. Similar to llvm::Timer, but much cheaper,
417// it only tracks wall time.
418// Since this is a generic timer, We may want to move this to support/ if we
419// find a use case outside of FS time tracking.
420class WallTimer {
421public:
422WallTimer() : TotalTime(std::chrono::steady_clock::duration::zero()) {}
423// [Re-]Start the timer.
424void startTimer() { StartTime = std::chrono::steady_clock::now(); }
425// Stop the timer and update total time.
426void stopTimer() {
427TotalTime += std::chrono::steady_clock::now() - StartTime;
428}
429// Return total time, in seconds.
430double getTime() { return std::chrono::duration<double>(TotalTime).count(); }
431
432private:
433std::chrono::steady_clock::duration TotalTime;
434std::chrono::steady_clock::time_point StartTime;
435};
436
437class WallTimerRegion {
438public:
439WallTimerRegion(WallTimer &T) : T(T) { T.startTimer(); }
440~WallTimerRegion() { T.stopTimer(); }
441
442private:
443WallTimer &T;
444};
445
446// Used by TimerFS, tracks time spent in status() and getBuffer() calls while
447// proxying to underlying File implementation.
448class TimerFile : public llvm::vfs::File {
449public:
450TimerFile(WallTimer &Timer, std::unique_ptr<File> InnerFile)
451: Timer(Timer), InnerFile(std::move(InnerFile)) {}
452
453llvm::ErrorOr<llvm::vfs::Status> status() override {
454WallTimerRegion T(Timer);
455return InnerFile->status();
456}
457llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
458getBuffer(const Twine &Name, int64_t FileSize, bool RequiresNullTerminator,
459bool IsVolatile) override {
460WallTimerRegion T(Timer);
461return InnerFile->getBuffer(Name, FileSize, RequiresNullTerminator,
462IsVolatile);
463}
464std::error_code close() override {
465WallTimerRegion T(Timer);
466return InnerFile->close();
467}
468
469private:
470WallTimer &Timer;
471std::unique_ptr<llvm::vfs::File> InnerFile;
472};
473
474// A wrapper for FileSystems that tracks the amount of time spent in status()
475// and openFileForRead() calls.
476class TimerFS : public llvm::vfs::ProxyFileSystem {
477public:
478TimerFS(llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
479: ProxyFileSystem(std::move(FS)) {}
480
481llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
482openFileForRead(const llvm::Twine &Path) override {
483WallTimerRegion T(Timer);
484auto FileOr = getUnderlyingFS().openFileForRead(Path);
485if (!FileOr)
486return FileOr;
487return std::make_unique<TimerFile>(Timer, std::move(FileOr.get()));
488}
489
490llvm::ErrorOr<llvm::vfs::Status> status(const llvm::Twine &Path) override {
491WallTimerRegion T(Timer);
492return getUnderlyingFS().status(Path);
493}
494
495double getTime() { return Timer.getTime(); }
496
497private:
498WallTimer Timer;
499};
500
501// Helpers for patching diagnostics between two versions of file contents.
502class DiagPatcher {
503llvm::ArrayRef<llvm::StringRef> OldLines;
504llvm::ArrayRef<llvm::StringRef> CurrentLines;
505llvm::StringMap<llvm::SmallVector<int>> CurrentContentsToLine;
506
507// Translates a range from old lines to current lines.
508// Finds the consecutive set of lines that corresponds to the same contents in
509// old and current, and applies the same translation to the range.
510// Returns true if translation succeeded.
511bool translateRange(Range &R) {
512int OldStart = R.start.line;
513int OldEnd = R.end.line;
514assert(OldStart <= OldEnd);
515
516size_t RangeLen = OldEnd - OldStart + 1;
517auto RangeContents = OldLines.slice(OldStart).take_front(RangeLen);
518// Make sure the whole range is covered in old contents.
519if (RangeContents.size() < RangeLen)
520return false;
521
522std::optional<int> Closest;
523for (int AlternateLine : CurrentContentsToLine.lookup(RangeContents[0])) {
524// Check if AlternateLine matches all lines in the range.
525if (RangeContents !=
526CurrentLines.slice(AlternateLine).take_front(RangeLen))
527continue;
528int Delta = AlternateLine - OldStart;
529if (!Closest.has_value() || abs(Delta) < abs(*Closest))
530Closest = Delta;
531}
532// Couldn't find any viable matches in the current contents.
533if (!Closest.has_value())
534return false;
535R.start.line += *Closest;
536R.end.line += *Closest;
537return true;
538}
539
540// Translates a Note by patching its range when inside main file. Returns true
541// on success.
542bool translateNote(Note &N) {
543if (!N.InsideMainFile)
544return true;
545if (translateRange(N.Range))
546return true;
547return false;
548}
549
550// Tries to translate all the edit ranges inside the fix. Returns true on
551// success. On failure fixes might be in an invalid state.
552bool translateFix(Fix &F) {
553return llvm::all_of(
554F.Edits, [this](TextEdit &E) { return translateRange(E.range); });
555}
556
557public:
558DiagPatcher(llvm::ArrayRef<llvm::StringRef> OldLines,
559llvm::ArrayRef<llvm::StringRef> CurrentLines) {
560this->OldLines = OldLines;
561this->CurrentLines = CurrentLines;
562for (int Line = 0, E = CurrentLines.size(); Line != E; ++Line) {
563llvm::StringRef Contents = CurrentLines[Line];
564CurrentContentsToLine[Contents].push_back(Line);
565}
566}
567// Translate diagnostic by moving its main range to new location (if inside
568// the main file). Preserve all the notes and fixes that can be translated to
569// new contents.
570// Drops the whole diagnostic if main range can't be patched.
571std::optional<Diag> translateDiag(const Diag &D) {
572Range NewRange = D.Range;
573// Patch range if it's inside main file.
574if (D.InsideMainFile && !translateRange(NewRange)) {
575// Drop the diagnostic if we couldn't patch the range.
576return std::nullopt;
577}
578
579Diag NewD = D;
580NewD.Range = NewRange;
581// Translate ranges inside notes and fixes too, dropping the ones that are
582// no longer relevant.
583llvm::erase_if(NewD.Notes, [this](Note &N) { return !translateNote(N); });
584llvm::erase_if(NewD.Fixes, [this](Fix &F) { return !translateFix(F); });
585return NewD;
586}
587};
588} // namespace
589
590std::shared_ptr<const PreambleData>
591buildPreamble(PathRef FileName, CompilerInvocation CI,
592const ParseInputs &Inputs, bool StoreInMemory,
593PreambleParsedCallback PreambleCallback,
594PreambleBuildStats *Stats) {
595// Note that we don't need to copy the input contents, preamble can live
596// without those.
597auto ContentsBuffer =
598llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
599auto Bounds = ComputePreambleBounds(CI.getLangOpts(), *ContentsBuffer, 0);
600
601trace::Span Tracer("BuildPreamble");
602SPAN_ATTACH(Tracer, "File", FileName);
603std::vector<std::unique_ptr<FeatureModule::ASTListener>> ASTListeners;
604if (Inputs.FeatureModules) {
605for (auto &M : *Inputs.FeatureModules) {
606if (auto Listener = M.astListeners())
607ASTListeners.emplace_back(std::move(Listener));
608}
609}
610StoreDiags PreambleDiagnostics;
611PreambleDiagnostics.setDiagCallback(
612[&ASTListeners](const clang::Diagnostic &D, clangd::Diag &Diag) {
613for (const auto &L : ASTListeners)
614L->sawDiagnostic(D, Diag);
615});
616llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
617CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
618&PreambleDiagnostics,
619/*ShouldOwnClient=*/false);
620const Config &Cfg = Config::current();
621PreambleDiagnostics.setLevelAdjuster([&](DiagnosticsEngine::Level DiagLevel,
622const clang::Diagnostic &Info) {
623if (Cfg.Diagnostics.SuppressAll ||
624isBuiltinDiagnosticSuppressed(Info.getID(), Cfg.Diagnostics.Suppress,
625CI.getLangOpts()))
626return DiagnosticsEngine::Ignored;
627switch (Info.getID()) {
628case diag::warn_no_newline_eof:
629case diag::warn_cxx98_compat_no_newline_eof:
630case diag::ext_no_newline_eof:
631// If the preamble doesn't span the whole file, drop the no newline at
632// eof warnings.
633return Bounds.Size != ContentsBuffer->getBufferSize()
634? DiagnosticsEngine::Level::Ignored
635: DiagLevel;
636}
637return DiagLevel;
638});
639
640// Skip function bodies when building the preamble to speed up building
641// the preamble and make it smaller.
642assert(!CI.getFrontendOpts().SkipFunctionBodies);
643CI.getFrontendOpts().SkipFunctionBodies = true;
644// We don't want to write comment locations into PCH. They are racy and slow
645// to read back. We rely on dynamic index for the comments instead.
646CI.getPreprocessorOpts().WriteCommentListToPCH = false;
647
648CppFilePreambleCallbacks CapturedInfo(
649FileName, Stats, Inputs.Opts.PreambleParseForwardingFunctions,
650[&ASTListeners](CompilerInstance &CI) {
651for (const auto &L : ASTListeners)
652L->beforeExecute(CI);
653});
654auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
655llvm::SmallString<32> AbsFileName(FileName);
656VFS->makeAbsolute(AbsFileName);
657auto StatCache = std::make_shared<PreambleFileStatusCache>(AbsFileName);
658auto StatCacheFS = StatCache->getProducingFS(VFS);
659llvm::IntrusiveRefCntPtr<TimerFS> TimedFS(new TimerFS(StatCacheFS));
660
661WallTimer PreambleTimer;
662PreambleTimer.startTimer();
663auto BuiltPreamble = PrecompiledPreamble::Build(
664CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
665Stats ? TimedFS : StatCacheFS, std::make_shared<PCHContainerOperations>(),
666StoreInMemory, /*StoragePath=*/"", CapturedInfo);
667PreambleTimer.stopTimer();
668
669// We have to setup DiagnosticConsumer that will be alife
670// while preamble callback is executed
671PreambleDiagsEngine->setClient(new IgnoringDiagConsumer, true);
672// Reset references to ref-counted-ptrs before executing the callbacks, to
673// prevent resetting them concurrently.
674PreambleDiagsEngine.reset();
675CI.DiagnosticOpts.reset();
676
677// When building the AST for the main file, we do want the function
678// bodies.
679CI.getFrontendOpts().SkipFunctionBodies = false;
680
681if (Stats != nullptr) {
682Stats->TotalBuildTime = PreambleTimer.getTime();
683Stats->FileSystemTime = TimedFS->getTime();
684Stats->SerializedSize = BuiltPreamble ? BuiltPreamble->getSize() : 0;
685}
686
687if (BuiltPreamble) {
688log("Built preamble of size {0} for file {1} version {2} in {3} seconds",
689BuiltPreamble->getSize(), FileName, Inputs.Version,
690PreambleTimer.getTime());
691std::vector<Diag> Diags = PreambleDiagnostics.take();
692auto Result = std::make_shared<PreambleData>(std::move(*BuiltPreamble));
693Result->Version = Inputs.Version;
694Result->CompileCommand = Inputs.CompileCommand;
695Result->Diags = std::move(Diags);
696Result->Includes = CapturedInfo.takeIncludes();
697Result->Pragmas = std::make_shared<const include_cleaner::PragmaIncludes>(
698CapturedInfo.takePragmaIncludes());
699Result->Macros = CapturedInfo.takeMacros();
700Result->Marks = CapturedInfo.takeMarks();
701Result->StatCache = StatCache;
702Result->MainIsIncludeGuarded = CapturedInfo.isMainFileIncludeGuarded();
703Result->TargetOpts = CI.TargetOpts;
704if (PreambleCallback) {
705trace::Span Tracer("Running PreambleCallback");
706auto Ctx = CapturedInfo.takeLife();
707// Stat cache is thread safe only when there are no producers. Hence
708// change the VFS underneath to a consuming fs.
709Ctx->getFileManager().setVirtualFileSystem(
710Result->StatCache->getConsumingFS(VFS));
711// While extending the life of FileMgr and VFS, StatCache should also be
712// extended.
713Ctx->setStatCache(Result->StatCache);
714
715PreambleCallback(std::move(*Ctx), Result->Pragmas);
716}
717return Result;
718}
719
720elog("Could not build a preamble for file {0} version {1}: {2}", FileName,
721Inputs.Version, BuiltPreamble.getError().message());
722for (const Diag &D : PreambleDiagnostics.take()) {
723if (D.Severity < DiagnosticsEngine::Error)
724continue;
725// Not an ideal way to show errors, but better than nothing!
726elog(" error: {0}", D.Message);
727}
728return nullptr;
729}
730
731bool isPreambleCompatible(const PreambleData &Preamble,
732const ParseInputs &Inputs, PathRef FileName,
733const CompilerInvocation &CI) {
734auto ContentsBuffer =
735llvm::MemoryBuffer::getMemBuffer(Inputs.Contents, FileName);
736auto Bounds = ComputePreambleBounds(CI.getLangOpts(), *ContentsBuffer, 0);
737auto VFS = Inputs.TFS->view(Inputs.CompileCommand.Directory);
738return compileCommandsAreEqual(Inputs.CompileCommand,
739Preamble.CompileCommand) &&
740Preamble.Preamble.CanReuse(CI, *ContentsBuffer, Bounds, *VFS);
741}
742
743void escapeBackslashAndQuotes(llvm::StringRef Text, llvm::raw_ostream &OS) {
744for (char C : Text) {
745switch (C) {
746case '\\':
747case '"':
748OS << '\\';
749break;
750default:
751break;
752}
753OS << C;
754}
755}
756
757// Translate diagnostics from baseline into modified for the lines that have the
758// same spelling.
759static std::vector<Diag> patchDiags(llvm::ArrayRef<Diag> BaselineDiags,
760const ScannedPreamble &BaselineScan,
761const ScannedPreamble &ModifiedScan) {
762std::vector<Diag> PatchedDiags;
763if (BaselineDiags.empty())
764return PatchedDiags;
765DiagPatcher Patcher(BaselineScan.Lines, ModifiedScan.Lines);
766for (auto &D : BaselineDiags) {
767if (auto NewD = Patcher.translateDiag(D))
768PatchedDiags.emplace_back(std::move(*NewD));
769}
770return PatchedDiags;
771}
772
773static std::string getPatchName(llvm::StringRef FileName) {
774// This shouldn't coincide with any real file name.
775llvm::SmallString<128> PatchName;
776llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
777PreamblePatch::HeaderName);
778return PatchName.str().str();
779}
780
781PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
782const ParseInputs &Modified,
783const PreambleData &Baseline,
784PatchType PatchType) {
785trace::Span Tracer("CreatePreamblePatch");
786SPAN_ATTACH(Tracer, "File", FileName);
787assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!");
788// First scan preprocessor directives in Baseline and Modified. These will be
789// used to figure out newly added directives in Modified. Scanning can fail,
790// the code just bails out and creates an empty patch in such cases, as:
791// - If scanning for Baseline fails, no knowledge of existing includes hence
792// patch will contain all the includes in Modified. Leading to rebuild of
793// whole preamble, which is terribly slow.
794// - If scanning for Modified fails, cannot figure out newly added ones so
795// there's nothing to do but generate an empty patch.
796auto BaselineScan =
797scanPreamble(Baseline.Preamble.getContents(), Modified.CompileCommand);
798if (!BaselineScan) {
799elog("Failed to scan baseline of {0}: {1}", FileName,
800BaselineScan.takeError());
801return PreamblePatch::unmodified(Baseline);
802}
803auto ModifiedScan = scanPreamble(Modified.Contents, Modified.CompileCommand);
804if (!ModifiedScan) {
805elog("Failed to scan modified contents of {0}: {1}", FileName,
806ModifiedScan.takeError());
807return PreamblePatch::unmodified(Baseline);
808}
809
810bool IncludesChanged = BaselineScan->Includes != ModifiedScan->Includes;
811bool DirectivesChanged =
812BaselineScan->TextualDirectives != ModifiedScan->TextualDirectives;
813if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) &&
814!DirectivesChanged)
815return PreamblePatch::unmodified(Baseline);
816
817PreamblePatch PP;
818PP.Baseline = &Baseline;
819PP.PatchFileName = getPatchName(FileName);
820PP.ModifiedBounds = ModifiedScan->Bounds;
821
822llvm::raw_string_ostream Patch(PP.PatchContents);
823// Set default filename for subsequent #line directives
824Patch << "#line 0 \"";
825// FileName part of a line directive is subject to backslash escaping, which
826// might lead to problems on windows especially.
827escapeBackslashAndQuotes(FileName, Patch);
828Patch << "\"\n";
829
830if (IncludesChanged && PatchType == PatchType::All) {
831// We are only interested in newly added includes, record the ones in
832// Baseline for exclusion.
833llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>,
834const Inclusion *>
835ExistingIncludes;
836for (const auto &Inc : Baseline.Includes.MainFileIncludes)
837ExistingIncludes[{Inc.Directive, Inc.Written}] = &Inc;
838// There might be includes coming from disabled regions, record these for
839// exclusion too. note that we don't have resolved paths for those.
840for (const auto &Inc : BaselineScan->Includes)
841ExistingIncludes.try_emplace({Inc.Directive, Inc.Written});
842// Calculate extra includes that needs to be inserted.
843for (auto &Inc : ModifiedScan->Includes) {
844auto It = ExistingIncludes.find({Inc.Directive, Inc.Written});
845// Include already present in the baseline preamble. Set resolved path and
846// put into preamble includes.
847if (It != ExistingIncludes.end()) {
848if (It->second) {
849// If this header is included in an active region of the baseline
850// preamble, preserve it.
851auto &PatchedInc = PP.PreambleIncludes.emplace_back();
852// Copy everything from existing include, apart from the location,
853// when it's coming from baseline preamble.
854PatchedInc = *It->second;
855PatchedInc.HashLine = Inc.HashLine;
856PatchedInc.HashOffset = Inc.HashOffset;
857}
858continue;
859}
860// Include is new in the modified preamble. Inject it into the patch and
861// use #line to set the presumed location to where it is spelled.
862auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset);
863Patch << llvm::formatv("#line {0}\n", LineCol.first);
864Patch << llvm::formatv(
865"#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written);
866}
867} else {
868// Make sure we have the full set of includes available even when we're not
869// patching. As these are used by features we provide afterwards like hover,
870// go-to-def or include-cleaner when preamble is stale.
871PP.PreambleIncludes = Baseline.Includes.MainFileIncludes;
872}
873
874if (DirectivesChanged) {
875// We need to patch all the directives, since they are order dependent. e.g:
876// #define BAR(X) NEW(X) // Newly introduced in Modified
877// #define BAR(X) OLD(X) // Exists in the Baseline
878//
879// If we've patched only the first directive, the macro definition would've
880// been wrong for the rest of the file, since patch is applied after the
881// baseline preamble.
882//
883// Note that we deliberately ignore conditional directives and undefs to
884// reduce complexity. The former might cause problems because scanning is
885// imprecise and might pick directives from disabled regions.
886for (const auto &TD : ModifiedScan->TextualDirectives) {
887// Introduce an #undef directive before #defines to suppress any
888// re-definition warnings.
889if (TD.Directive == tok::pp_define)
890Patch << "#undef " << TD.MacroName << '\n';
891Patch << "#line " << TD.DirectiveLine << '\n';
892Patch << TD.Text << '\n';
893}
894}
895
896PP.PatchedDiags = patchDiags(Baseline.Diags, *BaselineScan, *ModifiedScan);
897PP.PatchedMarks = std::move(ModifiedScan->Marks);
898PP.PatchedMacros = std::move(ModifiedScan->Macros);
899dlog("Created preamble patch: {0}", Patch.str());
900Patch.flush();
901return PP;
902}
903
904PreamblePatch PreamblePatch::createFullPatch(llvm::StringRef FileName,
905const ParseInputs &Modified,
906const PreambleData &Baseline) {
907return create(FileName, Modified, Baseline, PatchType::All);
908}
909
910PreamblePatch PreamblePatch::createMacroPatch(llvm::StringRef FileName,
911const ParseInputs &Modified,
912const PreambleData &Baseline) {
913return create(FileName, Modified, Baseline, PatchType::MacroDirectives);
914}
915
916void PreamblePatch::apply(CompilerInvocation &CI) const {
917// Make sure the compilation uses same target opts as the preamble. Clang has
918// no guarantees around using arbitrary options when reusing PCHs, and
919// different target opts can result in crashes, see
920// ParsedASTTest.PreambleWithDifferentTarget.
921// Make sure this is a deep copy, as the same Baseline might be used
922// concurrently.
923*CI.TargetOpts = *Baseline->TargetOpts;
924
925// No need to map an empty file.
926if (PatchContents.empty())
927return;
928auto &PPOpts = CI.getPreprocessorOpts();
929auto PatchBuffer =
930// we copy here to ensure contents are still valid if CI outlives the
931// PreamblePatch.
932llvm::MemoryBuffer::getMemBufferCopy(PatchContents, PatchFileName);
933// CI will take care of the lifetime of the buffer.
934PPOpts.addRemappedFile(PatchFileName, PatchBuffer.release());
935// The patch will be parsed after loading the preamble ast and before parsing
936// the main file.
937PPOpts.Includes.push_back(PatchFileName);
938}
939
940std::vector<Inclusion> PreamblePatch::preambleIncludes() const {
941return PreambleIncludes;
942}
943
944PreamblePatch PreamblePatch::unmodified(const PreambleData &Preamble) {
945PreamblePatch PP;
946PP.Baseline = &Preamble;
947PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
948PP.ModifiedBounds = Preamble.Preamble.getBounds();
949PP.PatchedDiags = Preamble.Diags;
950return PP;
951}
952
953llvm::ArrayRef<PragmaMark> PreamblePatch::marks() const {
954if (PatchContents.empty())
955return Baseline->Marks;
956return PatchedMarks;
957}
958
959const MainFileMacros &PreamblePatch::mainFileMacros() const {
960if (PatchContents.empty())
961return Baseline->Macros;
962return PatchedMacros;
963}
964
965OptionalFileEntryRef PreamblePatch::getPatchEntry(llvm::StringRef MainFilePath,
966const SourceManager &SM) {
967auto PatchFilePath = getPatchName(MainFilePath);
968return SM.getFileManager().getOptionalFileRef(PatchFilePath);
969}
970} // namespace clangd
971} // namespace clang
972