llvm-project

1
//===--- CodeGenModule.cpp - Emit LLVM Code from ASTs for a Module --------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This coordinates the per-module state used while generating code.
10
//
11
//===----------------------------------------------------------------------===//
12

13
#include "CodeGenModule.h"
14
#include "ABIInfo.h"
15
#include "CGBlocks.h"
16
#include "CGCUDARuntime.h"
17
#include "CGCXXABI.h"
18
#include "CGCall.h"
19
#include "CGDebugInfo.h"
20
#include "CGHLSLRuntime.h"
21
#include "CGObjCRuntime.h"
22
#include "CGOpenCLRuntime.h"
23
#include "CGOpenMPRuntime.h"
24
#include "CGOpenMPRuntimeGPU.h"
25
#include "CodeGenFunction.h"
26
#include "CodeGenPGO.h"
27
#include "ConstantEmitter.h"
28
#include "CoverageMappingGen.h"
29
#include "TargetInfo.h"
30
#include "clang/AST/ASTContext.h"
31
#include "clang/AST/ASTLambda.h"
32
#include "clang/AST/CharUnits.h"
33
#include "clang/AST/Decl.h"
34
#include "clang/AST/DeclCXX.h"
35
#include "clang/AST/DeclObjC.h"
36
#include "clang/AST/DeclTemplate.h"
37
#include "clang/AST/Mangle.h"
38
#include "clang/AST/RecursiveASTVisitor.h"
39
#include "clang/AST/StmtVisitor.h"
40
#include "clang/Basic/Builtins.h"
41
#include "clang/Basic/CharInfo.h"
42
#include "clang/Basic/CodeGenOptions.h"
43
#include "clang/Basic/Diagnostic.h"
44
#include "clang/Basic/FileManager.h"
45
#include "clang/Basic/Module.h"
46
#include "clang/Basic/SourceManager.h"
47
#include "clang/Basic/TargetInfo.h"
48
#include "clang/Basic/Version.h"
49
#include "clang/CodeGen/BackendUtil.h"
50
#include "clang/CodeGen/ConstantInitBuilder.h"
51
#include "clang/Frontend/FrontendDiagnostic.h"
52
#include "llvm/ADT/STLExtras.h"
53
#include "llvm/ADT/StringExtras.h"
54
#include "llvm/ADT/StringSwitch.h"
55
#include "llvm/Analysis/TargetLibraryInfo.h"
56
#include "llvm/BinaryFormat/ELF.h"
57
#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
58
#include "llvm/IR/AttributeMask.h"
59
#include "llvm/IR/CallingConv.h"
60
#include "llvm/IR/DataLayout.h"
61
#include "llvm/IR/Intrinsics.h"
62
#include "llvm/IR/LLVMContext.h"
63
#include "llvm/IR/Module.h"
64
#include "llvm/IR/ProfileSummary.h"
65
#include "llvm/ProfileData/InstrProfReader.h"
66
#include "llvm/ProfileData/SampleProf.h"
67
#include "llvm/Support/CRC.h"
68
#include "llvm/Support/CodeGen.h"
69
#include "llvm/Support/CommandLine.h"
70
#include "llvm/Support/ConvertUTF.h"
71
#include "llvm/Support/ErrorHandling.h"
72
#include "llvm/Support/TimeProfiler.h"
73
#include "llvm/Support/xxhash.h"
74
#include "llvm/TargetParser/RISCVISAInfo.h"
75
#include "llvm/TargetParser/Triple.h"
76
#include "llvm/TargetParser/X86TargetParser.h"
77
#include "llvm/Transforms/Utils/BuildLibCalls.h"
78
#include <optional>
79

80
using namespace clang;
81
using namespace CodeGen;
82

83
static llvm::cl::opt<bool> LimitedCoverage(
84
    "limited-coverage-experimental", llvm::cl::Hidden,
85
    llvm::cl::desc("Emit limited coverage mapping information (experimental)"));
86

87
static const char AnnotationSection[] = "llvm.metadata";
88

89
static CGCXXABI *createCXXABI(CodeGenModule &CGM) {
90
  switch (CGM.getContext().getCXXABIKind()) {
91
  case TargetCXXABI::AppleARM64:
92
  case TargetCXXABI::Fuchsia:
93
  case TargetCXXABI::GenericAArch64:
94
  case TargetCXXABI::GenericARM:
95
  case TargetCXXABI::iOS:
96
  case TargetCXXABI::WatchOS:
97
  case TargetCXXABI::GenericMIPS:
98
  case TargetCXXABI::GenericItanium:
99
  case TargetCXXABI::WebAssembly:
100
  case TargetCXXABI::XL:
101
    return CreateItaniumCXXABI(CGM);
102
  case TargetCXXABI::Microsoft:
103
    return CreateMicrosoftCXXABI(CGM);
104
  }
105

106
  llvm_unreachable("invalid C++ ABI kind");
107
}
108

109
static std::unique_ptr<TargetCodeGenInfo>
110
createTargetCodeGenInfo(CodeGenModule &CGM) {
111
  const TargetInfo &Target = CGM.getTarget();
112
  const llvm::Triple &Triple = Target.getTriple();
113
  const CodeGenOptions &CodeGenOpts = CGM.getCodeGenOpts();
114

115
  switch (Triple.getArch()) {
116
  default:
117
    return createDefaultTargetCodeGenInfo(CGM);
118

119
  case llvm::Triple::le32:
120
    return createPNaClTargetCodeGenInfo(CGM);
121
  case llvm::Triple::m68k:
122
    return createM68kTargetCodeGenInfo(CGM);
123
  case llvm::Triple::mips:
124
  case llvm::Triple::mipsel:
125
    if (Triple.getOS() == llvm::Triple::NaCl)
126
      return createPNaClTargetCodeGenInfo(CGM);
127
    return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/true);
128

129
  case llvm::Triple::mips64:
130
  case llvm::Triple::mips64el:
131
    return createMIPSTargetCodeGenInfo(CGM, /*IsOS32=*/false);
132

133
  case llvm::Triple::avr: {
134
    // For passing parameters, R8~R25 are used on avr, and R18~R25 are used
135
    // on avrtiny. For passing return value, R18~R25 are used on avr, and
136
    // R22~R25 are used on avrtiny.
137
    unsigned NPR = Target.getABI() == "avrtiny" ? 6 : 18;
138
    unsigned NRR = Target.getABI() == "avrtiny" ? 4 : 8;
139
    return createAVRTargetCodeGenInfo(CGM, NPR, NRR);
140
  }
141

142
  case llvm::Triple::aarch64:
143
  case llvm::Triple::aarch64_32:
144
  case llvm::Triple::aarch64_be: {
145
    AArch64ABIKind Kind = AArch64ABIKind::AAPCS;
146
    if (Target.getABI() == "darwinpcs")
147
      Kind = AArch64ABIKind::DarwinPCS;
148
    else if (Triple.isOSWindows())
149
      return createWindowsAArch64TargetCodeGenInfo(CGM, AArch64ABIKind::Win64);
150
    else if (Target.getABI() == "aapcs-soft")
151
      Kind = AArch64ABIKind::AAPCSSoft;
152

153
    return createAArch64TargetCodeGenInfo(CGM, Kind);
154
  }
155

156
  case llvm::Triple::wasm32:
157
  case llvm::Triple::wasm64: {
158
    WebAssemblyABIKind Kind = WebAssemblyABIKind::MVP;
159
    if (Target.getABI() == "experimental-mv")
160
      Kind = WebAssemblyABIKind::ExperimentalMV;
161
    return createWebAssemblyTargetCodeGenInfo(CGM, Kind);
162
  }
163

164
  case llvm::Triple::arm:
165
  case llvm::Triple::armeb:
166
  case llvm::Triple::thumb:
167
  case llvm::Triple::thumbeb: {
168
    if (Triple.getOS() == llvm::Triple::Win32)
169
      return createWindowsARMTargetCodeGenInfo(CGM, ARMABIKind::AAPCS_VFP);
170

171
    ARMABIKind Kind = ARMABIKind::AAPCS;
172
    StringRef ABIStr = Target.getABI();
173
    if (ABIStr == "apcs-gnu")
174
      Kind = ARMABIKind::APCS;
175
    else if (ABIStr == "aapcs16")
176
      Kind = ARMABIKind::AAPCS16_VFP;
177
    else if (CodeGenOpts.FloatABI == "hard" ||
178
             (CodeGenOpts.FloatABI != "soft" &&
179
              (Triple.getEnvironment() == llvm::Triple::GNUEABIHF ||
180
               Triple.getEnvironment() == llvm::Triple::MuslEABIHF ||
181
               Triple.getEnvironment() == llvm::Triple::EABIHF)))
182
      Kind = ARMABIKind::AAPCS_VFP;
183

184
    return createARMTargetCodeGenInfo(CGM, Kind);
185
  }
186

187
  case llvm::Triple::ppc: {
188
    if (Triple.isOSAIX())
189
      return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/false);
190

191
    bool IsSoftFloat =
192
        CodeGenOpts.FloatABI == "soft" || Target.hasFeature("spe");
193
    return createPPC32TargetCodeGenInfo(CGM, IsSoftFloat);
194
  }
195
  case llvm::Triple::ppcle: {
196
    bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
197
    return createPPC32TargetCodeGenInfo(CGM, IsSoftFloat);
198
  }
199
  case llvm::Triple::ppc64:
200
    if (Triple.isOSAIX())
201
      return createAIXTargetCodeGenInfo(CGM, /*Is64Bit=*/true);
202

203
    if (Triple.isOSBinFormatELF()) {
204
      PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv1;
205
      if (Target.getABI() == "elfv2")
206
        Kind = PPC64_SVR4_ABIKind::ELFv2;
207
      bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
208

209
      return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, IsSoftFloat);
210
    }
211
    return createPPC64TargetCodeGenInfo(CGM);
212
  case llvm::Triple::ppc64le: {
213
    assert(Triple.isOSBinFormatELF() && "PPC64 LE non-ELF not supported!");
214
    PPC64_SVR4_ABIKind Kind = PPC64_SVR4_ABIKind::ELFv2;
215
    if (Target.getABI() == "elfv1")
216
      Kind = PPC64_SVR4_ABIKind::ELFv1;
217
    bool IsSoftFloat = CodeGenOpts.FloatABI == "soft";
218

219
    return createPPC64_SVR4_TargetCodeGenInfo(CGM, Kind, IsSoftFloat);
220
  }
221

222
  case llvm::Triple::nvptx:
223
  case llvm::Triple::nvptx64:
224
    return createNVPTXTargetCodeGenInfo(CGM);
225

226
  case llvm::Triple::msp430:
227
    return createMSP430TargetCodeGenInfo(CGM);
228

229
  case llvm::Triple::riscv32:
230
  case llvm::Triple::riscv64: {
231
    StringRef ABIStr = Target.getABI();
232
    unsigned XLen = Target.getPointerWidth(LangAS::Default);
233
    unsigned ABIFLen = 0;
234
    if (ABIStr.ends_with("f"))
235
      ABIFLen = 32;
236
    else if (ABIStr.ends_with("d"))
237
      ABIFLen = 64;
238
    bool EABI = ABIStr.ends_with("e");
239
    return createRISCVTargetCodeGenInfo(CGM, XLen, ABIFLen, EABI);
240
  }
241

242
  case llvm::Triple::systemz: {
243
    bool SoftFloat = CodeGenOpts.FloatABI == "soft";
244
    bool HasVector = !SoftFloat && Target.getABI() == "vector";
245
    return createSystemZTargetCodeGenInfo(CGM, HasVector, SoftFloat);
246
  }
247

248
  case llvm::Triple::tce:
249
  case llvm::Triple::tcele:
250
    return createTCETargetCodeGenInfo(CGM);
251

252
  case llvm::Triple::x86: {
253
    bool IsDarwinVectorABI = Triple.isOSDarwin();
254
    bool IsWin32FloatStructABI = Triple.isOSWindows() && !Triple.isOSCygMing();
255

256
    if (Triple.getOS() == llvm::Triple::Win32) {
257
      return createWinX86_32TargetCodeGenInfo(
258
          CGM, IsDarwinVectorABI, IsWin32FloatStructABI,
259
          CodeGenOpts.NumRegisterParameters);
260
    }
261
    return createX86_32TargetCodeGenInfo(
262
        CGM, IsDarwinVectorABI, IsWin32FloatStructABI,
263
        CodeGenOpts.NumRegisterParameters, CodeGenOpts.FloatABI == "soft");
264
  }
265

266
  case llvm::Triple::x86_64: {
267
    StringRef ABI = Target.getABI();
268
    X86AVXABILevel AVXLevel = (ABI == "avx512" ? X86AVXABILevel::AVX512
269
                               : ABI == "avx"  ? X86AVXABILevel::AVX
270
                                               : X86AVXABILevel::None);
271

272
    switch (Triple.getOS()) {
273
    case llvm::Triple::Win32:
274
      return createWinX86_64TargetCodeGenInfo(CGM, AVXLevel);
275
    default:
276
      return createX86_64TargetCodeGenInfo(CGM, AVXLevel);
277
    }
278
  }
279
  case llvm::Triple::hexagon:
280
    return createHexagonTargetCodeGenInfo(CGM);
281
  case llvm::Triple::lanai:
282
    return createLanaiTargetCodeGenInfo(CGM);
283
  case llvm::Triple::r600:
284
    return createAMDGPUTargetCodeGenInfo(CGM);
285
  case llvm::Triple::amdgcn:
286
    return createAMDGPUTargetCodeGenInfo(CGM);
287
  case llvm::Triple::sparc:
288
    return createSparcV8TargetCodeGenInfo(CGM);
289
  case llvm::Triple::sparcv9:
290
    return createSparcV9TargetCodeGenInfo(CGM);
291
  case llvm::Triple::xcore:
292
    return createXCoreTargetCodeGenInfo(CGM);
293
  case llvm::Triple::arc:
294
    return createARCTargetCodeGenInfo(CGM);
295
  case llvm::Triple::spir:
296
  case llvm::Triple::spir64:
297
    return createCommonSPIRTargetCodeGenInfo(CGM);
298
  case llvm::Triple::spirv32:
299
  case llvm::Triple::spirv64:
300
    return createSPIRVTargetCodeGenInfo(CGM);
301
  case llvm::Triple::ve:
302
    return createVETargetCodeGenInfo(CGM);
303
  case llvm::Triple::csky: {
304
    bool IsSoftFloat = !Target.hasFeature("hard-float-abi");
305
    bool hasFP64 =
306
        Target.hasFeature("fpuv2_df") || Target.hasFeature("fpuv3_df");
307
    return createCSKYTargetCodeGenInfo(CGM, IsSoftFloat ? 0
308
                                            : hasFP64   ? 64
309
                                                        : 32);
310
  }
311
  case llvm::Triple::bpfeb:
312
  case llvm::Triple::bpfel:
313
    return createBPFTargetCodeGenInfo(CGM);
314
  case llvm::Triple::loongarch32:
315
  case llvm::Triple::loongarch64: {
316
    StringRef ABIStr = Target.getABI();
317
    unsigned ABIFRLen = 0;
318
    if (ABIStr.ends_with("f"))
319
      ABIFRLen = 32;
320
    else if (ABIStr.ends_with("d"))
321
      ABIFRLen = 64;
322
    return createLoongArchTargetCodeGenInfo(
323
        CGM, Target.getPointerWidth(LangAS::Default), ABIFRLen);
324
  }
325
  }
326
}
327

328
const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
329
  if (!TheTargetCodeGenInfo)
330
    TheTargetCodeGenInfo = createTargetCodeGenInfo(*this);
331
  return *TheTargetCodeGenInfo;
332
}
333

334
CodeGenModule::CodeGenModule(ASTContext &C,
335
                             IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
336
                             const HeaderSearchOptions &HSO,
337
                             const PreprocessorOptions &PPO,
338
                             const CodeGenOptions &CGO, llvm::Module &M,
339
                             DiagnosticsEngine &diags,
340
                             CoverageSourceInfo *CoverageInfo)
341
    : Context(C), LangOpts(C.getLangOpts()), FS(FS), HeaderSearchOpts(HSO),
342
      PreprocessorOpts(PPO), CodeGenOpts(CGO), TheModule(M), Diags(diags),
343
      Target(C.getTargetInfo()), ABI(createCXXABI(*this)),
344
      VMContext(M.getContext()), Types(*this), VTables(*this),
345
      SanitizerMD(new SanitizerMetadata(*this)) {
346

347
  // Initialize the type cache.
348
  llvm::LLVMContext &LLVMContext = M.getContext();
349
  VoidTy = llvm::Type::getVoidTy(LLVMContext);
350
  Int8Ty = llvm::Type::getInt8Ty(LLVMContext);
351
  Int16Ty = llvm::Type::getInt16Ty(LLVMContext);
352
  Int32Ty = llvm::Type::getInt32Ty(LLVMContext);
353
  Int64Ty = llvm::Type::getInt64Ty(LLVMContext);
354
  HalfTy = llvm::Type::getHalfTy(LLVMContext);
355
  BFloatTy = llvm::Type::getBFloatTy(LLVMContext);
356
  FloatTy = llvm::Type::getFloatTy(LLVMContext);
357
  DoubleTy = llvm::Type::getDoubleTy(LLVMContext);
358
  PointerWidthInBits = C.getTargetInfo().getPointerWidth(LangAS::Default);
359
  PointerAlignInBytes =
360
      C.toCharUnitsFromBits(C.getTargetInfo().getPointerAlign(LangAS::Default))
361
          .getQuantity();
362
  SizeSizeInBytes =
363
    C.toCharUnitsFromBits(C.getTargetInfo().getMaxPointerWidth()).getQuantity();
364
  IntAlignInBytes =
365
    C.toCharUnitsFromBits(C.getTargetInfo().getIntAlign()).getQuantity();
366
  CharTy =
367
    llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getCharWidth());
368
  IntTy = llvm::IntegerType::get(LLVMContext, C.getTargetInfo().getIntWidth());
369
  IntPtrTy = llvm::IntegerType::get(LLVMContext,
370
    C.getTargetInfo().getMaxPointerWidth());
371
  Int8PtrTy = llvm::PointerType::get(LLVMContext,
372
                                     C.getTargetAddressSpace(LangAS::Default));
373
  const llvm::DataLayout &DL = M.getDataLayout();
374
  AllocaInt8PtrTy =
375
      llvm::PointerType::get(LLVMContext, DL.getAllocaAddrSpace());
376
  GlobalsInt8PtrTy =
377
      llvm::PointerType::get(LLVMContext, DL.getDefaultGlobalsAddressSpace());
378
  ConstGlobalsPtrTy = llvm::PointerType::get(
379
      LLVMContext, C.getTargetAddressSpace(GetGlobalConstantAddressSpace()));
380
  ASTAllocaAddressSpace = getTargetCodeGenInfo().getASTAllocaAddressSpace();
381

382
  // Build C++20 Module initializers.
383
  // TODO: Add Microsoft here once we know the mangling required for the
384
  // initializers.
385
  CXX20ModuleInits =
386
      LangOpts.CPlusPlusModules && getCXXABI().getMangleContext().getKind() ==
387
                                       ItaniumMangleContext::MK_Itanium;
388

389
  RuntimeCC = getTargetCodeGenInfo().getABIInfo().getRuntimeCC();
390

391
  if (LangOpts.ObjC)
392
    createObjCRuntime();
393
  if (LangOpts.OpenCL)
394
    createOpenCLRuntime();
395
  if (LangOpts.OpenMP)
396
    createOpenMPRuntime();
397
  if (LangOpts.CUDA)
398
    createCUDARuntime();
399
  if (LangOpts.HLSL)
400
    createHLSLRuntime();
401

402
  // Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0.
403
  if (LangOpts.Sanitize.has(SanitizerKind::Thread) ||
404
      (!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0))
405
    TBAA.reset(new CodeGenTBAA(Context, getTypes(), TheModule, CodeGenOpts,
406
                               getLangOpts(), getCXXABI().getMangleContext()));
407

408
  // If debug info or coverage generation is enabled, create the CGDebugInfo
409
  // object.
410
  if (CodeGenOpts.getDebugInfo() != llvm::codegenoptions::NoDebugInfo ||
411
      CodeGenOpts.CoverageNotesFile.size() ||
412
      CodeGenOpts.CoverageDataFile.size())
413
    DebugInfo.reset(new CGDebugInfo(*this));
414

415
  Block.GlobalUniqueCount = 0;
416

417
  if (C.getLangOpts().ObjC)
418
    ObjCData.reset(new ObjCEntrypoints());
419

420
  if (CodeGenOpts.hasProfileClangUse()) {
421
    auto ReaderOrErr = llvm::IndexedInstrProfReader::create(
422
        CodeGenOpts.ProfileInstrumentUsePath, *FS,
423
        CodeGenOpts.ProfileRemappingFile);
424
    // We're checking for profile read errors in CompilerInvocation, so if
425
    // there was an error it should've already been caught. If it hasn't been
426
    // somehow, trip an assertion.
427
    assert(ReaderOrErr);
428
    PGOReader = std::move(ReaderOrErr.get());
429
  }
430

431
  // If coverage mapping generation is enabled, create the
432
  // CoverageMappingModuleGen object.
433
  if (CodeGenOpts.CoverageMapping)
434
    CoverageMapping.reset(new CoverageMappingModuleGen(*this, *CoverageInfo));
435

436
  // Generate the module name hash here if needed.
437
  if (CodeGenOpts.UniqueInternalLinkageNames &&
438
      !getModule().getSourceFileName().empty()) {
439
    std::string Path = getModule().getSourceFileName();
440
    // Check if a path substitution is needed from the MacroPrefixMap.
441
    for (const auto &Entry : LangOpts.MacroPrefixMap)
442
      if (Path.rfind(Entry.first, 0) != std::string::npos) {
443
        Path = Entry.second + Path.substr(Entry.first.size());
444
        break;
445
      }
446
    ModuleNameHash = llvm::getUniqueInternalLinkagePostfix(Path);
447
  }
448

449
  // Record mregparm value now so it is visible through all of codegen.
450
  if (Context.getTargetInfo().getTriple().getArch() == llvm::Triple::x86)
451
    getModule().addModuleFlag(llvm::Module::Error, "NumRegisterParameters",
452
                              CodeGenOpts.NumRegisterParameters);
453
}
454

455
CodeGenModule::~CodeGenModule() {}
456

457
void CodeGenModule::createObjCRuntime() {
458
  // This is just isGNUFamily(), but we want to force implementors of
459
  // new ABIs to decide how best to do this.
460
  switch (LangOpts.ObjCRuntime.getKind()) {
461
  case ObjCRuntime::GNUstep:
462
  case ObjCRuntime::GCC:
463
  case ObjCRuntime::ObjFW:
464
    ObjCRuntime.reset(CreateGNUObjCRuntime(*this));
465
    return;
466

467
  case ObjCRuntime::FragileMacOSX:
468
  case ObjCRuntime::MacOSX:
469
  case ObjCRuntime::iOS:
470
  case ObjCRuntime::WatchOS:
471
    ObjCRuntime.reset(CreateMacObjCRuntime(*this));
472
    return;
473
  }
474
  llvm_unreachable("bad runtime kind");
475
}
476

477
void CodeGenModule::createOpenCLRuntime() {
478
  OpenCLRuntime.reset(new CGOpenCLRuntime(*this));
479
}
480

481
void CodeGenModule::createOpenMPRuntime() {
482
  // Select a specialized code generation class based on the target, if any.
483
  // If it does not exist use the default implementation.
484
  switch (getTriple().getArch()) {
485
  case llvm::Triple::nvptx:
486
  case llvm::Triple::nvptx64:
487
  case llvm::Triple::amdgcn:
488
    assert(getLangOpts().OpenMPIsTargetDevice &&
489
           "OpenMP AMDGPU/NVPTX is only prepared to deal with device code.");
490
    OpenMPRuntime.reset(new CGOpenMPRuntimeGPU(*this));
491
    break;
492
  default:
493
    if (LangOpts.OpenMPSimd)
494
      OpenMPRuntime.reset(new CGOpenMPSIMDRuntime(*this));
495
    else
496
      OpenMPRuntime.reset(new CGOpenMPRuntime(*this));
497
    break;
498
  }
499
}
500

501
void CodeGenModule::createCUDARuntime() {
502
  CUDARuntime.reset(CreateNVCUDARuntime(*this));
503
}
504

505
void CodeGenModule::createHLSLRuntime() {
506
  HLSLRuntime.reset(new CGHLSLRuntime(*this));
507
}
508

509
void CodeGenModule::addReplacement(StringRef Name, llvm::Constant *C) {
510
  Replacements[Name] = C;
511
}
512

513
void CodeGenModule::applyReplacements() {
514
  for (auto &I : Replacements) {
515
    StringRef MangledName = I.first;
516
    llvm::Constant *Replacement = I.second;
517
    llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
518
    if (!Entry)
519
      continue;
520
    auto *OldF = cast<llvm::Function>(Entry);
521
    auto *NewF = dyn_cast<llvm::Function>(Replacement);
522
    if (!NewF) {
523
      if (auto *Alias = dyn_cast<llvm::GlobalAlias>(Replacement)) {
524
        NewF = dyn_cast<llvm::Function>(Alias->getAliasee());
525
      } else {
526
        auto *CE = cast<llvm::ConstantExpr>(Replacement);
527
        assert(CE->getOpcode() == llvm::Instruction::BitCast ||
528
               CE->getOpcode() == llvm::Instruction::GetElementPtr);
529
        NewF = dyn_cast<llvm::Function>(CE->getOperand(0));
530
      }
531
    }
532

533
    // Replace old with new, but keep the old order.
534
    OldF->replaceAllUsesWith(Replacement);
535
    if (NewF) {
536
      NewF->removeFromParent();
537
      OldF->getParent()->getFunctionList().insertAfter(OldF->getIterator(),
538
                                                       NewF);
539
    }
540
    OldF->eraseFromParent();
541
  }
542
}
543

544
void CodeGenModule::addGlobalValReplacement(llvm::GlobalValue *GV, llvm::Constant *C) {
545
  GlobalValReplacements.push_back(std::make_pair(GV, C));
546
}
547

548
void CodeGenModule::applyGlobalValReplacements() {
549
  for (auto &I : GlobalValReplacements) {
550
    llvm::GlobalValue *GV = I.first;
551
    llvm::Constant *C = I.second;
552

553
    GV->replaceAllUsesWith(C);
554
    GV->eraseFromParent();
555
  }
556
}
557

558
// This is only used in aliases that we created and we know they have a
559
// linear structure.
560
static const llvm::GlobalValue *getAliasedGlobal(const llvm::GlobalValue *GV) {
561
  const llvm::Constant *C;
562
  if (auto *GA = dyn_cast<llvm::GlobalAlias>(GV))
563
    C = GA->getAliasee();
564
  else if (auto *GI = dyn_cast<llvm::GlobalIFunc>(GV))
565
    C = GI->getResolver();
566
  else
567
    return GV;
568

569
  const auto *AliaseeGV = dyn_cast<llvm::GlobalValue>(C->stripPointerCasts());
570
  if (!AliaseeGV)
571
    return nullptr;
572

573
  const llvm::GlobalValue *FinalGV = AliaseeGV->getAliaseeObject();
574
  if (FinalGV == GV)
575
    return nullptr;
576

577
  return FinalGV;
578
}
579

580
static bool checkAliasedGlobal(
581
    const ASTContext &Context, DiagnosticsEngine &Diags, SourceLocation Location,
582
    bool IsIFunc, const llvm::GlobalValue *Alias, const llvm::GlobalValue *&GV,
583
    const llvm::MapVector<GlobalDecl, StringRef> &MangledDeclNames,
584
    SourceRange AliasRange) {
585
  GV = getAliasedGlobal(Alias);
586
  if (!GV) {
587
    Diags.Report(Location, diag::err_cyclic_alias) << IsIFunc;
588
    return false;
589
  }
590

591
  if (GV->hasCommonLinkage()) {
592
    const llvm::Triple &Triple = Context.getTargetInfo().getTriple();
593
    if (Triple.getObjectFormat() == llvm::Triple::XCOFF) {
594
      Diags.Report(Location, diag::err_alias_to_common);
595
      return false;
596
    }
597
  }
598

599
  if (GV->isDeclaration()) {
600
    Diags.Report(Location, diag::err_alias_to_undefined) << IsIFunc << IsIFunc;
601
    Diags.Report(Location, diag::note_alias_requires_mangled_name)
602
        << IsIFunc << IsIFunc;
603
    // Provide a note if the given function is not found and exists as a
604
    // mangled name.
605
    for (const auto &[Decl, Name] : MangledDeclNames) {
606
      if (const auto *ND = dyn_cast<NamedDecl>(Decl.getDecl())) {
607
        if (ND->getName() == GV->getName()) {
608
          Diags.Report(Location, diag::note_alias_mangled_name_alternative)
609
              << Name
610
              << FixItHint::CreateReplacement(
611
                     AliasRange,
612
                     (Twine(IsIFunc ? "ifunc" : "alias") + "(\"" + Name + "\")")
613
                         .str());
614
        }
615
      }
616
    }
617
    return false;
618
  }
619

620
  if (IsIFunc) {
621
    // Check resolver function type.
622
    const auto *F = dyn_cast<llvm::Function>(GV);
623
    if (!F) {
624
      Diags.Report(Location, diag::err_alias_to_undefined)
625
          << IsIFunc << IsIFunc;
626
      return false;
627
    }
628

629
    llvm::FunctionType *FTy = F->getFunctionType();
630
    if (!FTy->getReturnType()->isPointerTy()) {
631
      Diags.Report(Location, diag::err_ifunc_resolver_return);
632
      return false;
633
    }
634
  }
635

636
  return true;
637
}
638

639
// Emit a warning if toc-data attribute is requested for global variables that
640
// have aliases and remove the toc-data attribute.
641
static void checkAliasForTocData(llvm::GlobalVariable *GVar,
642
                                 const CodeGenOptions &CodeGenOpts,
643
                                 DiagnosticsEngine &Diags,
644
                                 SourceLocation Location) {
645
  if (GVar->hasAttribute("toc-data")) {
646
    auto GVId = GVar->getName();
647
    // Is this a global variable specified by the user as local?
648
    if ((llvm::binary_search(CodeGenOpts.TocDataVarsUserSpecified, GVId))) {
649
      Diags.Report(Location, diag::warn_toc_unsupported_type)
650
          << GVId << "the variable has an alias";
651
    }
652
    llvm::AttributeSet CurrAttributes = GVar->getAttributes();
653
    llvm::AttributeSet NewAttributes =
654
        CurrAttributes.removeAttribute(GVar->getContext(), "toc-data");
655
    GVar->setAttributes(NewAttributes);
656
  }
657
}
658

659
void CodeGenModule::checkAliases() {
660
  // Check if the constructed aliases are well formed. It is really unfortunate
661
  // that we have to do this in CodeGen, but we only construct mangled names
662
  // and aliases during codegen.
663
  bool Error = false;
664
  DiagnosticsEngine &Diags = getDiags();
665
  for (const GlobalDecl &GD : Aliases) {
666
    const auto *D = cast<ValueDecl>(GD.getDecl());
667
    SourceLocation Location;
668
    SourceRange Range;
669
    bool IsIFunc = D->hasAttr<IFuncAttr>();
670
    if (const Attr *A = D->getDefiningAttr()) {
671
      Location = A->getLocation();
672
      Range = A->getRange();
673
    } else
674
      llvm_unreachable("Not an alias or ifunc?");
675

676
    StringRef MangledName = getMangledName(GD);
677
    llvm::GlobalValue *Alias = GetGlobalValue(MangledName);
678
    const llvm::GlobalValue *GV = nullptr;
679
    if (!checkAliasedGlobal(getContext(), Diags, Location, IsIFunc, Alias, GV,
680
                            MangledDeclNames, Range)) {
681
      Error = true;
682
      continue;
683
    }
684

685
    if (getContext().getTargetInfo().getTriple().isOSAIX())
686
      if (const llvm::GlobalVariable *GVar =
687
              dyn_cast<const llvm::GlobalVariable>(GV))
688
        checkAliasForTocData(const_cast<llvm::GlobalVariable *>(GVar),
689
                             getCodeGenOpts(), Diags, Location);
690

691
    llvm::Constant *Aliasee =
692
        IsIFunc ? cast<llvm::GlobalIFunc>(Alias)->getResolver()
693
                : cast<llvm::GlobalAlias>(Alias)->getAliasee();
694

695
    llvm::GlobalValue *AliaseeGV;
696
    if (auto CE = dyn_cast<llvm::ConstantExpr>(Aliasee))
697
      AliaseeGV = cast<llvm::GlobalValue>(CE->getOperand(0));
698
    else
699
      AliaseeGV = cast<llvm::GlobalValue>(Aliasee);
700

701
    if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
702
      StringRef AliasSection = SA->getName();
703
      if (AliasSection != AliaseeGV->getSection())
704
        Diags.Report(SA->getLocation(), diag::warn_alias_with_section)
705
            << AliasSection << IsIFunc << IsIFunc;
706
    }
707

708
    // We have to handle alias to weak aliases in here. LLVM itself disallows
709
    // this since the object semantics would not match the IL one. For
710
    // compatibility with gcc we implement it by just pointing the alias
711
    // to its aliasee's aliasee. We also warn, since the user is probably
712
    // expecting the link to be weak.
713
    if (auto *GA = dyn_cast<llvm::GlobalAlias>(AliaseeGV)) {
714
      if (GA->isInterposable()) {
715
        Diags.Report(Location, diag::warn_alias_to_weak_alias)
716
            << GV->getName() << GA->getName() << IsIFunc;
717
        Aliasee = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
718
            GA->getAliasee(), Alias->getType());
719

720
        if (IsIFunc)
721
          cast<llvm::GlobalIFunc>(Alias)->setResolver(Aliasee);
722
        else
723
          cast<llvm::GlobalAlias>(Alias)->setAliasee(Aliasee);
724
      }
725
    }
726
  }
727
  if (!Error)
728
    return;
729

730
  for (const GlobalDecl &GD : Aliases) {
731
    StringRef MangledName = getMangledName(GD);
732
    llvm::GlobalValue *Alias = GetGlobalValue(MangledName);
733
    Alias->replaceAllUsesWith(llvm::UndefValue::get(Alias->getType()));
734
    Alias->eraseFromParent();
735
  }
736
}
737

738
void CodeGenModule::clear() {
739
  DeferredDeclsToEmit.clear();
740
  EmittedDeferredDecls.clear();
741
  DeferredAnnotations.clear();
742
  if (OpenMPRuntime)
743
    OpenMPRuntime->clear();
744
}
745

746
void InstrProfStats::reportDiagnostics(DiagnosticsEngine &Diags,
747
                                       StringRef MainFile) {
748
  if (!hasDiagnostics())
749
    return;
750
  if (VisitedInMainFile > 0 && VisitedInMainFile == MissingInMainFile) {
751
    if (MainFile.empty())
752
      MainFile = "<stdin>";
753
    Diags.Report(diag::warn_profile_data_unprofiled) << MainFile;
754
  } else {
755
    if (Mismatched > 0)
756
      Diags.Report(diag::warn_profile_data_out_of_date) << Visited << Mismatched;
757

758
    if (Missing > 0)
759
      Diags.Report(diag::warn_profile_data_missing) << Visited << Missing;
760
  }
761
}
762

763
static std::optional<llvm::GlobalValue::VisibilityTypes>
764
getLLVMVisibility(clang::LangOptions::VisibilityFromDLLStorageClassKinds K) {
765
  // Map to LLVM visibility.
766
  switch (K) {
767
  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Keep:
768
    return std::nullopt;
769
  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Default:
770
    return llvm::GlobalValue::DefaultVisibility;
771
  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Hidden:
772
    return llvm::GlobalValue::HiddenVisibility;
773
  case clang::LangOptions::VisibilityFromDLLStorageClassKinds::Protected:
774
    return llvm::GlobalValue::ProtectedVisibility;
775
  }
776
  llvm_unreachable("unknown option value!");
777
}
778

779
void setLLVMVisibility(llvm::GlobalValue &GV,
780
                       std::optional<llvm::GlobalValue::VisibilityTypes> V) {
781
  if (!V)
782
    return;
783

784
  // Reset DSO locality before setting the visibility. This removes
785
  // any effects that visibility options and annotations may have
786
  // had on the DSO locality. Setting the visibility will implicitly set
787
  // appropriate globals to DSO Local; however, this will be pessimistic
788
  // w.r.t. to the normal compiler IRGen.
789
  GV.setDSOLocal(false);
790
  GV.setVisibility(*V);
791
}
792

793
static void setVisibilityFromDLLStorageClass(const clang::LangOptions &LO,
794
                                             llvm::Module &M) {
795
  if (!LO.VisibilityFromDLLStorageClass)
796
    return;
797

798
  std::optional<llvm::GlobalValue::VisibilityTypes> DLLExportVisibility =
799
      getLLVMVisibility(LO.getDLLExportVisibility());
800

801
  std::optional<llvm::GlobalValue::VisibilityTypes>
802
      NoDLLStorageClassVisibility =
803
          getLLVMVisibility(LO.getNoDLLStorageClassVisibility());
804

805
  std::optional<llvm::GlobalValue::VisibilityTypes>
806
      ExternDeclDLLImportVisibility =
807
          getLLVMVisibility(LO.getExternDeclDLLImportVisibility());
808

809
  std::optional<llvm::GlobalValue::VisibilityTypes>
810
      ExternDeclNoDLLStorageClassVisibility =
811
          getLLVMVisibility(LO.getExternDeclNoDLLStorageClassVisibility());
812

813
  for (llvm::GlobalValue &GV : M.global_values()) {
814
    if (GV.hasAppendingLinkage() || GV.hasLocalLinkage())
815
      continue;
816

817
    if (GV.isDeclarationForLinker())
818
      setLLVMVisibility(GV, GV.getDLLStorageClass() ==
819
                                    llvm::GlobalValue::DLLImportStorageClass
820
                                ? ExternDeclDLLImportVisibility
821
                                : ExternDeclNoDLLStorageClassVisibility);
822
    else
823
      setLLVMVisibility(GV, GV.getDLLStorageClass() ==
824
                                    llvm::GlobalValue::DLLExportStorageClass
825
                                ? DLLExportVisibility
826
                                : NoDLLStorageClassVisibility);
827

828
    GV.setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
829
  }
830
}
831

832
static bool isStackProtectorOn(const LangOptions &LangOpts,
833
                               const llvm::Triple &Triple,
834
                               clang::LangOptions::StackProtectorMode Mode) {
835
  if (Triple.isAMDGPU() || Triple.isNVPTX())
836
    return false;
837
  return LangOpts.getStackProtector() == Mode;
838
}
839

840
void CodeGenModule::Release() {
841
  Module *Primary = getContext().getCurrentNamedModule();
842
  if (CXX20ModuleInits && Primary && !Primary->isHeaderLikeModule())
843
    EmitModuleInitializers(Primary);
844
  EmitDeferred();
845
  DeferredDecls.insert(EmittedDeferredDecls.begin(),
846
                       EmittedDeferredDecls.end());
847
  EmittedDeferredDecls.clear();
848
  EmitVTablesOpportunistically();
849
  applyGlobalValReplacements();
850
  applyReplacements();
851
  emitMultiVersionFunctions();
852

853
  if (Context.getLangOpts().IncrementalExtensions &&
854
      GlobalTopLevelStmtBlockInFlight.first) {
855
    const TopLevelStmtDecl *TLSD = GlobalTopLevelStmtBlockInFlight.second;
856
    GlobalTopLevelStmtBlockInFlight.first->FinishFunction(TLSD->getEndLoc());
857
    GlobalTopLevelStmtBlockInFlight = {nullptr, nullptr};
858
  }
859

860
  // Module implementations are initialized the same way as a regular TU that
861
  // imports one or more modules.
862
  if (CXX20ModuleInits && Primary && Primary->isInterfaceOrPartition())
863
    EmitCXXModuleInitFunc(Primary);
864
  else
865
    EmitCXXGlobalInitFunc();
866
  EmitCXXGlobalCleanUpFunc();
867
  registerGlobalDtorsWithAtExit();
868
  EmitCXXThreadLocalInitFunc();
869
  if (ObjCRuntime)
870
    if (llvm::Function *ObjCInitFunction = ObjCRuntime->ModuleInitFunction())
871
      AddGlobalCtor(ObjCInitFunction);
872
  if (Context.getLangOpts().CUDA && CUDARuntime) {
873
    if (llvm::Function *CudaCtorFunction = CUDARuntime->finalizeModule())
874
      AddGlobalCtor(CudaCtorFunction);
875
  }
876
  if (OpenMPRuntime) {
877
    OpenMPRuntime->createOffloadEntriesAndInfoMetadata();
878
    OpenMPRuntime->clear();
879
  }
880
  if (PGOReader) {
881
    getModule().setProfileSummary(
882
        PGOReader->getSummary(/* UseCS */ false).getMD(VMContext),
883
        llvm::ProfileSummary::PSK_Instr);
884
    if (PGOStats.hasDiagnostics())
885
      PGOStats.reportDiagnostics(getDiags(), getCodeGenOpts().MainFileName);
886
  }
887
  llvm::stable_sort(GlobalCtors, [](const Structor &L, const Structor &R) {
888
    return L.LexOrder < R.LexOrder;
889
  });
890
  EmitCtorList(GlobalCtors, "llvm.global_ctors");
891
  EmitCtorList(GlobalDtors, "llvm.global_dtors");
892
  EmitGlobalAnnotations();
893
  EmitStaticExternCAliases();
894
  checkAliases();
895
  EmitDeferredUnusedCoverageMappings();
896
  CodeGenPGO(*this).setValueProfilingFlag(getModule());
897
  CodeGenPGO(*this).setProfileVersion(getModule());
898
  if (CoverageMapping)
899
    CoverageMapping->emit();
900
  if (CodeGenOpts.SanitizeCfiCrossDso) {
901
    CodeGenFunction(*this).EmitCfiCheckFail();
902
    CodeGenFunction(*this).EmitCfiCheckStub();
903
  }
904
  if (LangOpts.Sanitize.has(SanitizerKind::KCFI))
905
    finalizeKCFITypes();
906
  emitAtAvailableLinkGuard();
907
  if (Context.getTargetInfo().getTriple().isWasm())
908
    EmitMainVoidAlias();
909

910
  if (getTriple().isAMDGPU() ||
911
      (getTriple().isSPIRV() && getTriple().getVendor() == llvm::Triple::AMD)) {
912
    // Emit amdhsa_code_object_version module flag, which is code object version
913
    // times 100.
914
    if (getTarget().getTargetOpts().CodeObjectVersion !=
915
        llvm::CodeObjectVersionKind::COV_None) {
916
      getModule().addModuleFlag(llvm::Module::Error,
917
                                "amdhsa_code_object_version",
918
                                getTarget().getTargetOpts().CodeObjectVersion);
919
    }
920

921
    // Currently, "-mprintf-kind" option is only supported for HIP
922
    if (LangOpts.HIP) {
923
      auto *MDStr = llvm::MDString::get(
924
          getLLVMContext(), (getTarget().getTargetOpts().AMDGPUPrintfKindVal ==
925
                             TargetOptions::AMDGPUPrintfKind::Hostcall)
926
                                ? "hostcall"
927
                                : "buffered");
928
      getModule().addModuleFlag(llvm::Module::Error, "amdgpu_printf_kind",
929
                                MDStr);
930
    }
931
  }
932

933
  // Emit a global array containing all external kernels or device variables
934
  // used by host functions and mark it as used for CUDA/HIP. This is necessary
935
  // to get kernels or device variables in archives linked in even if these
936
  // kernels or device variables are only used in host functions.
937
  if (!Context.CUDAExternalDeviceDeclODRUsedByHost.empty()) {
938
    SmallVector<llvm::Constant *, 8> UsedArray;
939
    for (auto D : Context.CUDAExternalDeviceDeclODRUsedByHost) {
940
      GlobalDecl GD;
941
      if (auto *FD = dyn_cast<FunctionDecl>(D))
942
        GD = GlobalDecl(FD, KernelReferenceKind::Kernel);
943
      else
944
        GD = GlobalDecl(D);
945
      UsedArray.push_back(llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
946
          GetAddrOfGlobal(GD), Int8PtrTy));
947
    }
948

949
    llvm::ArrayType *ATy = llvm::ArrayType::get(Int8PtrTy, UsedArray.size());
950

951
    auto *GV = new llvm::GlobalVariable(
952
        getModule(), ATy, false, llvm::GlobalValue::InternalLinkage,
953
        llvm::ConstantArray::get(ATy, UsedArray), "__clang_gpu_used_external");
954
    addCompilerUsedGlobal(GV);
955
  }
956
  if (LangOpts.HIP && !getLangOpts().OffloadingNewDriver) {
957
    // Emit a unique ID so that host and device binaries from the same
958
    // compilation unit can be associated.
959
    auto *GV = new llvm::GlobalVariable(
960
        getModule(), Int8Ty, false, llvm::GlobalValue::ExternalLinkage,
961
        llvm::Constant::getNullValue(Int8Ty),
962
        "__hip_cuid_" + getContext().getCUIDHash());
963
    addCompilerUsedGlobal(GV);
964
  }
965
  emitLLVMUsed();
966
  if (SanStats)
967
    SanStats->finish();
968

969
  if (CodeGenOpts.Autolink &&
970
      (Context.getLangOpts().Modules || !LinkerOptionsMetadata.empty())) {
971
    EmitModuleLinkOptions();
972
  }
973

974
  // On ELF we pass the dependent library specifiers directly to the linker
975
  // without manipulating them. This is in contrast to other platforms where
976
  // they are mapped to a specific linker option by the compiler. This
977
  // difference is a result of the greater variety of ELF linkers and the fact
978
  // that ELF linkers tend to handle libraries in a more complicated fashion
979
  // than on other platforms. This forces us to defer handling the dependent
980
  // libs to the linker.
981
  //
982
  // CUDA/HIP device and host libraries are different. Currently there is no
983
  // way to differentiate dependent libraries for host or device. Existing
984
  // usage of #pragma comment(lib, *) is intended for host libraries on
985
  // Windows. Therefore emit llvm.dependent-libraries only for host.
986
  if (!ELFDependentLibraries.empty() && !Context.getLangOpts().CUDAIsDevice) {
987
    auto *NMD = getModule().getOrInsertNamedMetadata("llvm.dependent-libraries");
988
    for (auto *MD : ELFDependentLibraries)
989
      NMD->addOperand(MD);
990
  }
991

992
  if (CodeGenOpts.DwarfVersion) {
993
    getModule().addModuleFlag(llvm::Module::Max, "Dwarf Version",
994
                              CodeGenOpts.DwarfVersion);
995
  }
996

997
  if (CodeGenOpts.Dwarf64)
998
    getModule().addModuleFlag(llvm::Module::Max, "DWARF64", 1);
999

1000
  if (Context.getLangOpts().SemanticInterposition)
1001
    // Require various optimization to respect semantic interposition.
1002
    getModule().setSemanticInterposition(true);
1003

1004
  if (CodeGenOpts.EmitCodeView) {
1005
    // Indicate that we want CodeView in the metadata.
1006
    getModule().addModuleFlag(llvm::Module::Warning, "CodeView", 1);
1007
  }
1008
  if (CodeGenOpts.CodeViewGHash) {
1009
    getModule().addModuleFlag(llvm::Module::Warning, "CodeViewGHash", 1);
1010
  }
1011
  if (CodeGenOpts.ControlFlowGuard) {
1012
    // Function ID tables and checks for Control Flow Guard (cfguard=2).
1013
    getModule().addModuleFlag(llvm::Module::Warning, "cfguard", 2);
1014
  } else if (CodeGenOpts.ControlFlowGuardNoChecks) {
1015
    // Function ID tables for Control Flow Guard (cfguard=1).
1016
    getModule().addModuleFlag(llvm::Module::Warning, "cfguard", 1);
1017
  }
1018
  if (CodeGenOpts.EHContGuard) {
1019
    // Function ID tables for EH Continuation Guard.
1020
    getModule().addModuleFlag(llvm::Module::Warning, "ehcontguard", 1);
1021
  }
1022
  if (Context.getLangOpts().Kernel) {
1023
    // Note if we are compiling with /kernel.
1024
    getModule().addModuleFlag(llvm::Module::Warning, "ms-kernel", 1);
1025
  }
1026
  if (CodeGenOpts.OptimizationLevel > 0 && CodeGenOpts.StrictVTablePointers) {
1027
    // We don't support LTO with 2 with different StrictVTablePointers
1028
    // FIXME: we could support it by stripping all the information introduced
1029
    // by StrictVTablePointers.
1030

1031
    getModule().addModuleFlag(llvm::Module::Error, "StrictVTablePointers",1);
1032

1033
    llvm::Metadata *Ops[2] = {
1034
              llvm::MDString::get(VMContext, "StrictVTablePointers"),
1035
              llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
1036
                  llvm::Type::getInt32Ty(VMContext), 1))};
1037

1038
    getModule().addModuleFlag(llvm::Module::Require,
1039
                              "StrictVTablePointersRequirement",
1040
                              llvm::MDNode::get(VMContext, Ops));
1041
  }
1042
  if (getModuleDebugInfo())
1043
    // We support a single version in the linked module. The LLVM
1044
    // parser will drop debug info with a different version number
1045
    // (and warn about it, too).
1046
    getModule().addModuleFlag(llvm::Module::Warning, "Debug Info Version",
1047
                              llvm::DEBUG_METADATA_VERSION);
1048

1049
  // We need to record the widths of enums and wchar_t, so that we can generate
1050
  // the correct build attributes in the ARM backend. wchar_size is also used by
1051
  // TargetLibraryInfo.
1052
  uint64_t WCharWidth =
1053
      Context.getTypeSizeInChars(Context.getWideCharType()).getQuantity();
1054
  getModule().addModuleFlag(llvm::Module::Error, "wchar_size", WCharWidth);
1055

1056
  if (getTriple().isOSzOS()) {
1057
    getModule().addModuleFlag(llvm::Module::Warning,
1058
                              "zos_product_major_version",
1059
                              uint32_t(CLANG_VERSION_MAJOR));
1060
    getModule().addModuleFlag(llvm::Module::Warning,
1061
                              "zos_product_minor_version",
1062
                              uint32_t(CLANG_VERSION_MINOR));
1063
    getModule().addModuleFlag(llvm::Module::Warning, "zos_product_patchlevel",
1064
                              uint32_t(CLANG_VERSION_PATCHLEVEL));
1065
    std::string ProductId = getClangVendor() + "clang";
1066
    getModule().addModuleFlag(llvm::Module::Error, "zos_product_id",
1067
                              llvm::MDString::get(VMContext, ProductId));
1068

1069
    // Record the language because we need it for the PPA2.
1070
    StringRef lang_str = languageToString(
1071
        LangStandard::getLangStandardForKind(LangOpts.LangStd).Language);
1072
    getModule().addModuleFlag(llvm::Module::Error, "zos_cu_language",
1073
                              llvm::MDString::get(VMContext, lang_str));
1074

1075
    time_t TT = PreprocessorOpts.SourceDateEpoch
1076
                    ? *PreprocessorOpts.SourceDateEpoch
1077
                    : std::time(nullptr);
1078
    getModule().addModuleFlag(llvm::Module::Max, "zos_translation_time",
1079
                              static_cast<uint64_t>(TT));
1080

1081
    // Multiple modes will be supported here.
1082
    getModule().addModuleFlag(llvm::Module::Error, "zos_le_char_mode",
1083
                              llvm::MDString::get(VMContext, "ascii"));
1084
  }
1085

1086
  llvm::Triple T = Context.getTargetInfo().getTriple();
1087
  if (T.isARM() || T.isThumb()) {
1088
    // The minimum width of an enum in bytes
1089
    uint64_t EnumWidth = Context.getLangOpts().ShortEnums ? 1 : 4;
1090
    getModule().addModuleFlag(llvm::Module::Error, "min_enum_size", EnumWidth);
1091
  }
1092

1093
  if (T.isRISCV()) {
1094
    StringRef ABIStr = Target.getABI();
1095
    llvm::LLVMContext &Ctx = TheModule.getContext();
1096
    getModule().addModuleFlag(llvm::Module::Error, "target-abi",
1097
                              llvm::MDString::get(Ctx, ABIStr));
1098

1099
    // Add the canonical ISA string as metadata so the backend can set the ELF
1100
    // attributes correctly. We use AppendUnique so LTO will keep all of the
1101
    // unique ISA strings that were linked together.
1102
    const std::vector<std::string> &Features =
1103
        getTarget().getTargetOpts().Features;
1104
    auto ParseResult =
1105
        llvm::RISCVISAInfo::parseFeatures(T.isRISCV64() ? 64 : 32, Features);
1106
    if (!errorToBool(ParseResult.takeError()))
1107
      getModule().addModuleFlag(
1108
          llvm::Module::AppendUnique, "riscv-isa",
1109
          llvm::MDNode::get(
1110
              Ctx, llvm::MDString::get(Ctx, (*ParseResult)->toString())));
1111
  }
1112

1113
  if (CodeGenOpts.SanitizeCfiCrossDso) {
1114
    // Indicate that we want cross-DSO control flow integrity checks.
1115
    getModule().addModuleFlag(llvm::Module::Override, "Cross-DSO CFI", 1);
1116
  }
1117

1118
  if (CodeGenOpts.WholeProgramVTables) {
1119
    // Indicate whether VFE was enabled for this module, so that the
1120
    // vcall_visibility metadata added under whole program vtables is handled
1121
    // appropriately in the optimizer.
1122
    getModule().addModuleFlag(llvm::Module::Error, "Virtual Function Elim",
1123
                              CodeGenOpts.VirtualFunctionElimination);
1124
  }
1125

1126
  if (LangOpts.Sanitize.has(SanitizerKind::CFIICall)) {
1127
    getModule().addModuleFlag(llvm::Module::Override,
1128
                              "CFI Canonical Jump Tables",
1129
                              CodeGenOpts.SanitizeCfiCanonicalJumpTables);
1130
  }
1131

1132
  if (LangOpts.Sanitize.has(SanitizerKind::KCFI)) {
1133
    getModule().addModuleFlag(llvm::Module::Override, "kcfi", 1);
1134
    // KCFI assumes patchable-function-prefix is the same for all indirectly
1135
    // called functions. Store the expected offset for code generation.
1136
    if (CodeGenOpts.PatchableFunctionEntryOffset)
1137
      getModule().addModuleFlag(llvm::Module::Override, "kcfi-offset",
1138
                                CodeGenOpts.PatchableFunctionEntryOffset);
1139
  }
1140

1141
  if (CodeGenOpts.CFProtectionReturn &&
1142
      Target.checkCFProtectionReturnSupported(getDiags())) {
1143
    // Indicate that we want to instrument return control flow protection.
1144
    getModule().addModuleFlag(llvm::Module::Min, "cf-protection-return",
1145
                              1);
1146
  }
1147

1148
  if (CodeGenOpts.CFProtectionBranch &&
1149
      Target.checkCFProtectionBranchSupported(getDiags())) {
1150
    // Indicate that we want to instrument branch control flow protection.
1151
    getModule().addModuleFlag(llvm::Module::Min, "cf-protection-branch",
1152
                              1);
1153
  }
1154

1155
  if (CodeGenOpts.FunctionReturnThunks)
1156
    getModule().addModuleFlag(llvm::Module::Override, "function_return_thunk_extern", 1);
1157

1158
  if (CodeGenOpts.IndirectBranchCSPrefix)
1159
    getModule().addModuleFlag(llvm::Module::Override, "indirect_branch_cs_prefix", 1);
1160

1161
  // Add module metadata for return address signing (ignoring
1162
  // non-leaf/all) and stack tagging. These are actually turned on by function
1163
  // attributes, but we use module metadata to emit build attributes. This is
1164
  // needed for LTO, where the function attributes are inside bitcode
1165
  // serialised into a global variable by the time build attributes are
1166
  // emitted, so we can't access them. LTO objects could be compiled with
1167
  // different flags therefore module flags are set to "Min" behavior to achieve
1168
  // the same end result of the normal build where e.g BTI is off if any object
1169
  // doesn't support it.
1170
  if (Context.getTargetInfo().hasFeature("ptrauth") &&
1171
      LangOpts.getSignReturnAddressScope() !=
1172
          LangOptions::SignReturnAddressScopeKind::None)
1173
    getModule().addModuleFlag(llvm::Module::Override,
1174
                              "sign-return-address-buildattr", 1);
1175
  if (LangOpts.Sanitize.has(SanitizerKind::MemtagStack))
1176
    getModule().addModuleFlag(llvm::Module::Override,
1177
                              "tag-stack-memory-buildattr", 1);
1178

1179
  if (T.isARM() || T.isThumb() || T.isAArch64()) {
1180
    if (LangOpts.BranchTargetEnforcement)
1181
      getModule().addModuleFlag(llvm::Module::Min, "branch-target-enforcement",
1182
                                1);
1183
    if (LangOpts.BranchProtectionPAuthLR)
1184
      getModule().addModuleFlag(llvm::Module::Min, "branch-protection-pauth-lr",
1185
                                1);
1186
    if (LangOpts.GuardedControlStack)
1187
      getModule().addModuleFlag(llvm::Module::Min, "guarded-control-stack", 1);
1188
    if (LangOpts.hasSignReturnAddress())
1189
      getModule().addModuleFlag(llvm::Module::Min, "sign-return-address", 1);
1190
    if (LangOpts.isSignReturnAddressScopeAll())
1191
      getModule().addModuleFlag(llvm::Module::Min, "sign-return-address-all",
1192
                                1);
1193
    if (!LangOpts.isSignReturnAddressWithAKey())
1194
      getModule().addModuleFlag(llvm::Module::Min,
1195
                                "sign-return-address-with-bkey", 1);
1196

1197
    if (getTriple().isOSLinux()) {
1198
      assert(getTriple().isOSBinFormatELF());
1199
      using namespace llvm::ELF;
1200
      uint64_t PAuthABIVersion =
1201
          (LangOpts.PointerAuthIntrinsics
1202
           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INTRINSICS) |
1203
          (LangOpts.PointerAuthCalls
1204
           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_CALLS) |
1205
          (LangOpts.PointerAuthReturns
1206
           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_RETURNS) |
1207
          (LangOpts.PointerAuthAuthTraps
1208
           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_AUTHTRAPS) |
1209
          (LangOpts.PointerAuthVTPtrAddressDiscrimination
1210
           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRADDRDISCR) |
1211
          (LangOpts.PointerAuthVTPtrTypeDiscrimination
1212
           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_VPTRTYPEDISCR) |
1213
          (LangOpts.PointerAuthInitFini
1214
           << AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI);
1215
      static_assert(AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_INITFINI ==
1216
                        AARCH64_PAUTH_PLATFORM_LLVM_LINUX_VERSION_LAST,
1217
                    "Update when new enum items are defined");
1218
      if (PAuthABIVersion != 0) {
1219
        getModule().addModuleFlag(llvm::Module::Error,
1220
                                  "aarch64-elf-pauthabi-platform",
1221
                                  AARCH64_PAUTH_PLATFORM_LLVM_LINUX);
1222
        getModule().addModuleFlag(llvm::Module::Error,
1223
                                  "aarch64-elf-pauthabi-version",
1224
                                  PAuthABIVersion);
1225
      }
1226
    }
1227
  }
1228

1229
  if (CodeGenOpts.StackClashProtector)
1230
    getModule().addModuleFlag(
1231
        llvm::Module::Override, "probe-stack",
1232
        llvm::MDString::get(TheModule.getContext(), "inline-asm"));
1233

1234
  if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096)
1235
    getModule().addModuleFlag(llvm::Module::Min, "stack-probe-size",
1236
                              CodeGenOpts.StackProbeSize);
1237

1238
  if (!CodeGenOpts.MemoryProfileOutput.empty()) {
1239
    llvm::LLVMContext &Ctx = TheModule.getContext();
1240
    getModule().addModuleFlag(
1241
        llvm::Module::Error, "MemProfProfileFilename",
1242
        llvm::MDString::get(Ctx, CodeGenOpts.MemoryProfileOutput));
1243
  }
1244

1245
  if (LangOpts.CUDAIsDevice && getTriple().isNVPTX()) {
1246
    // Indicate whether __nvvm_reflect should be configured to flush denormal
1247
    // floating point values to 0.  (This corresponds to its "__CUDA_FTZ"
1248
    // property.)
1249
    getModule().addModuleFlag(llvm::Module::Override, "nvvm-reflect-ftz",
1250
                              CodeGenOpts.FP32DenormalMode.Output !=
1251
                                  llvm::DenormalMode::IEEE);
1252
  }
1253

1254
  if (LangOpts.EHAsynch)
1255
    getModule().addModuleFlag(llvm::Module::Warning, "eh-asynch", 1);
1256

1257
  // Indicate whether this Module was compiled with -fopenmp
1258
  if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
1259
    getModule().addModuleFlag(llvm::Module::Max, "openmp", LangOpts.OpenMP);
1260
  if (getLangOpts().OpenMPIsTargetDevice)
1261
    getModule().addModuleFlag(llvm::Module::Max, "openmp-device",
1262
                              LangOpts.OpenMP);
1263

1264
  // Emit OpenCL specific module metadata: OpenCL/SPIR version.
1265
  if (LangOpts.OpenCL || (LangOpts.CUDAIsDevice && getTriple().isSPIRV())) {
1266
    EmitOpenCLMetadata();
1267
    // Emit SPIR version.
1268
    if (getTriple().isSPIR()) {
1269
      // SPIR v2.0 s2.12 - The SPIR version used by the module is stored in the
1270
      // opencl.spir.version named metadata.
1271
      // C++ for OpenCL has a distinct mapping for version compatibility with
1272
      // OpenCL.
1273
      auto Version = LangOpts.getOpenCLCompatibleVersion();
1274
      llvm::Metadata *SPIRVerElts[] = {
1275
          llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
1276
              Int32Ty, Version / 100)),
1277
          llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
1278
              Int32Ty, (Version / 100 > 1) ? 0 : 2))};
1279
      llvm::NamedMDNode *SPIRVerMD =
1280
          TheModule.getOrInsertNamedMetadata("opencl.spir.version");
1281
      llvm::LLVMContext &Ctx = TheModule.getContext();
1282
      SPIRVerMD->addOperand(llvm::MDNode::get(Ctx, SPIRVerElts));
1283
    }
1284
  }
1285

1286
  // HLSL related end of code gen work items.
1287
  if (LangOpts.HLSL)
1288
    getHLSLRuntime().finishCodeGen();
1289

1290
  if (uint32_t PLevel = Context.getLangOpts().PICLevel) {
1291
    assert(PLevel < 3 && "Invalid PIC Level");
1292
    getModule().setPICLevel(static_cast<llvm::PICLevel::Level>(PLevel));
1293
    if (Context.getLangOpts().PIE)
1294
      getModule().setPIELevel(static_cast<llvm::PIELevel::Level>(PLevel));
1295
  }
1296

1297
  if (getCodeGenOpts().CodeModel.size() > 0) {
1298
    unsigned CM = llvm::StringSwitch<unsigned>(getCodeGenOpts().CodeModel)
1299
                  .Case("tiny", llvm::CodeModel::Tiny)
1300
                  .Case("small", llvm::CodeModel::Small)
1301
                  .Case("kernel", llvm::CodeModel::Kernel)
1302
                  .Case("medium", llvm::CodeModel::Medium)
1303
                  .Case("large", llvm::CodeModel::Large)
1304
                  .Default(~0u);
1305
    if (CM != ~0u) {
1306
      llvm::CodeModel::Model codeModel = static_cast<llvm::CodeModel::Model>(CM);
1307
      getModule().setCodeModel(codeModel);
1308

1309
      if ((CM == llvm::CodeModel::Medium || CM == llvm::CodeModel::Large) &&
1310
          Context.getTargetInfo().getTriple().getArch() ==
1311
              llvm::Triple::x86_64) {
1312
        getModule().setLargeDataThreshold(getCodeGenOpts().LargeDataThreshold);
1313
      }
1314
    }
1315
  }
1316

1317
  if (CodeGenOpts.NoPLT)
1318
    getModule().setRtLibUseGOT();
1319
  if (getTriple().isOSBinFormatELF() &&
1320
      CodeGenOpts.DirectAccessExternalData !=
1321
          getModule().getDirectAccessExternalData()) {
1322
    getModule().setDirectAccessExternalData(
1323
        CodeGenOpts.DirectAccessExternalData);
1324
  }
1325
  if (CodeGenOpts.UnwindTables)
1326
    getModule().setUwtable(llvm::UWTableKind(CodeGenOpts.UnwindTables));
1327

1328
  switch (CodeGenOpts.getFramePointer()) {
1329
  case CodeGenOptions::FramePointerKind::None:
1330
    // 0 ("none") is the default.
1331
    break;
1332
  case CodeGenOptions::FramePointerKind::Reserved:
1333
    getModule().setFramePointer(llvm::FramePointerKind::Reserved);
1334
    break;
1335
  case CodeGenOptions::FramePointerKind::NonLeaf:
1336
    getModule().setFramePointer(llvm::FramePointerKind::NonLeaf);
1337
    break;
1338
  case CodeGenOptions::FramePointerKind::All:
1339
    getModule().setFramePointer(llvm::FramePointerKind::All);
1340
    break;
1341
  }
1342

1343
  SimplifyPersonality();
1344

1345
  if (getCodeGenOpts().EmitDeclMetadata)
1346
    EmitDeclMetadata();
1347

1348
  if (getCodeGenOpts().CoverageNotesFile.size() ||
1349
      getCodeGenOpts().CoverageDataFile.size())
1350
    EmitCoverageFile();
1351

1352
  if (CGDebugInfo *DI = getModuleDebugInfo())
1353
    DI->finalize();
1354

1355
  if (getCodeGenOpts().EmitVersionIdentMetadata)
1356
    EmitVersionIdentMetadata();
1357

1358
  if (!getCodeGenOpts().RecordCommandLine.empty())
1359
    EmitCommandLineMetadata();
1360

1361
  if (!getCodeGenOpts().StackProtectorGuard.empty())
1362
    getModule().setStackProtectorGuard(getCodeGenOpts().StackProtectorGuard);
1363
  if (!getCodeGenOpts().StackProtectorGuardReg.empty())
1364
    getModule().setStackProtectorGuardReg(
1365
        getCodeGenOpts().StackProtectorGuardReg);
1366
  if (!getCodeGenOpts().StackProtectorGuardSymbol.empty())
1367
    getModule().setStackProtectorGuardSymbol(
1368
        getCodeGenOpts().StackProtectorGuardSymbol);
1369
  if (getCodeGenOpts().StackProtectorGuardOffset != INT_MAX)
1370
    getModule().setStackProtectorGuardOffset(
1371
        getCodeGenOpts().StackProtectorGuardOffset);
1372
  if (getCodeGenOpts().StackAlignment)
1373
    getModule().setOverrideStackAlignment(getCodeGenOpts().StackAlignment);
1374
  if (getCodeGenOpts().SkipRaxSetup)
1375
    getModule().addModuleFlag(llvm::Module::Override, "SkipRaxSetup", 1);
1376
  if (getLangOpts().RegCall4)
1377
    getModule().addModuleFlag(llvm::Module::Override, "RegCallv4", 1);
1378

1379
  if (getContext().getTargetInfo().getMaxTLSAlign())
1380
    getModule().addModuleFlag(llvm::Module::Error, "MaxTLSAlign",
1381
                              getContext().getTargetInfo().getMaxTLSAlign());
1382

1383
  getTargetCodeGenInfo().emitTargetGlobals(*this);
1384

1385
  getTargetCodeGenInfo().emitTargetMetadata(*this, MangledDeclNames);
1386

1387
  EmitBackendOptionsMetadata(getCodeGenOpts());
1388

1389
  // If there is device offloading code embed it in the host now.
1390
  EmbedObject(&getModule(), CodeGenOpts, getDiags());
1391

1392
  // Set visibility from DLL storage class
1393
  // We do this at the end of LLVM IR generation; after any operation
1394
  // that might affect the DLL storage class or the visibility, and
1395
  // before anything that might act on these.
1396
  setVisibilityFromDLLStorageClass(LangOpts, getModule());
1397

1398
  // Check the tail call symbols are truly undefined.
1399
  if (getTriple().isPPC() && !MustTailCallUndefinedGlobals.empty()) {
1400
    for (auto &I : MustTailCallUndefinedGlobals) {
1401
      if (!I.first->isDefined())
1402
        getDiags().Report(I.second, diag::err_ppc_impossible_musttail) << 2;
1403
      else {
1404
        StringRef MangledName = getMangledName(GlobalDecl(I.first));
1405
        llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
1406
        if (!Entry || Entry->isWeakForLinker() ||
1407
            Entry->isDeclarationForLinker())
1408
          getDiags().Report(I.second, diag::err_ppc_impossible_musttail) << 2;
1409
      }
1410
    }
1411
  }
1412
}
1413

1414
void CodeGenModule::EmitOpenCLMetadata() {
1415
  // SPIR v2.0 s2.13 - The OpenCL version used by the module is stored in the
1416
  // opencl.ocl.version named metadata node.
1417
  // C++ for OpenCL has a distinct mapping for versions compatible with OpenCL.
1418
  auto CLVersion = LangOpts.getOpenCLCompatibleVersion();
1419

1420
  auto EmitVersion = [this](StringRef MDName, int Version) {
1421
    llvm::Metadata *OCLVerElts[] = {
1422
        llvm::ConstantAsMetadata::get(
1423
            llvm::ConstantInt::get(Int32Ty, Version / 100)),
1424
        llvm::ConstantAsMetadata::get(
1425
            llvm::ConstantInt::get(Int32Ty, (Version % 100) / 10))};
1426
    llvm::NamedMDNode *OCLVerMD = TheModule.getOrInsertNamedMetadata(MDName);
1427
    llvm::LLVMContext &Ctx = TheModule.getContext();
1428
    OCLVerMD->addOperand(llvm::MDNode::get(Ctx, OCLVerElts));
1429
  };
1430

1431
  EmitVersion("opencl.ocl.version", CLVersion);
1432
  if (LangOpts.OpenCLCPlusPlus) {
1433
    // In addition to the OpenCL compatible version, emit the C++ version.
1434
    EmitVersion("opencl.cxx.version", LangOpts.OpenCLCPlusPlusVersion);
1435
  }
1436
}
1437

1438
void CodeGenModule::EmitBackendOptionsMetadata(
1439
    const CodeGenOptions &CodeGenOpts) {
1440
  if (getTriple().isRISCV()) {
1441
    getModule().addModuleFlag(llvm::Module::Min, "SmallDataLimit",
1442
                              CodeGenOpts.SmallDataLimit);
1443
  }
1444
}
1445

1446
void CodeGenModule::UpdateCompletedType(const TagDecl *TD) {
1447
  // Make sure that this type is translated.
1448
  Types.UpdateCompletedType(TD);
1449
}
1450

1451
void CodeGenModule::RefreshTypeCacheForClass(const CXXRecordDecl *RD) {
1452
  // Make sure that this type is translated.
1453
  Types.RefreshTypeCacheForClass(RD);
1454
}
1455

1456
llvm::MDNode *CodeGenModule::getTBAATypeInfo(QualType QTy) {
1457
  if (!TBAA)
1458
    return nullptr;
1459
  return TBAA->getTypeInfo(QTy);
1460
}
1461

1462
TBAAAccessInfo CodeGenModule::getTBAAAccessInfo(QualType AccessType) {
1463
  if (!TBAA)
1464
    return TBAAAccessInfo();
1465
  if (getLangOpts().CUDAIsDevice) {
1466
    // As CUDA builtin surface/texture types are replaced, skip generating TBAA
1467
    // access info.
1468
    if (AccessType->isCUDADeviceBuiltinSurfaceType()) {
1469
      if (getTargetCodeGenInfo().getCUDADeviceBuiltinSurfaceDeviceType() !=
1470
          nullptr)
1471
        return TBAAAccessInfo();
1472
    } else if (AccessType->isCUDADeviceBuiltinTextureType()) {
1473
      if (getTargetCodeGenInfo().getCUDADeviceBuiltinTextureDeviceType() !=
1474
          nullptr)
1475
        return TBAAAccessInfo();
1476
    }
1477
  }
1478
  return TBAA->getAccessInfo(AccessType);
1479
}
1480

1481
TBAAAccessInfo
1482
CodeGenModule::getTBAAVTablePtrAccessInfo(llvm::Type *VTablePtrType) {
1483
  if (!TBAA)
1484
    return TBAAAccessInfo();
1485
  return TBAA->getVTablePtrAccessInfo(VTablePtrType);
1486
}
1487

1488
llvm::MDNode *CodeGenModule::getTBAAStructInfo(QualType QTy) {
1489
  if (!TBAA)
1490
    return nullptr;
1491
  return TBAA->getTBAAStructInfo(QTy);
1492
}
1493

1494
llvm::MDNode *CodeGenModule::getTBAABaseTypeInfo(QualType QTy) {
1495
  if (!TBAA)
1496
    return nullptr;
1497
  return TBAA->getBaseTypeInfo(QTy);
1498
}
1499

1500
llvm::MDNode *CodeGenModule::getTBAAAccessTagInfo(TBAAAccessInfo Info) {
1501
  if (!TBAA)
1502
    return nullptr;
1503
  return TBAA->getAccessTagInfo(Info);
1504
}
1505

1506
TBAAAccessInfo CodeGenModule::mergeTBAAInfoForCast(TBAAAccessInfo SourceInfo,
1507
                                                   TBAAAccessInfo TargetInfo) {
1508
  if (!TBAA)
1509
    return TBAAAccessInfo();
1510
  return TBAA->mergeTBAAInfoForCast(SourceInfo, TargetInfo);
1511
}
1512

1513
TBAAAccessInfo
1514
CodeGenModule::mergeTBAAInfoForConditionalOperator(TBAAAccessInfo InfoA,
1515
                                                   TBAAAccessInfo InfoB) {
1516
  if (!TBAA)
1517
    return TBAAAccessInfo();
1518
  return TBAA->mergeTBAAInfoForConditionalOperator(InfoA, InfoB);
1519
}
1520

1521
TBAAAccessInfo
1522
CodeGenModule::mergeTBAAInfoForMemoryTransfer(TBAAAccessInfo DestInfo,
1523
                                              TBAAAccessInfo SrcInfo) {
1524
  if (!TBAA)
1525
    return TBAAAccessInfo();
1526
  return TBAA->mergeTBAAInfoForConditionalOperator(DestInfo, SrcInfo);
1527
}
1528

1529
void CodeGenModule::DecorateInstructionWithTBAA(llvm::Instruction *Inst,
1530
                                                TBAAAccessInfo TBAAInfo) {
1531
  if (llvm::MDNode *Tag = getTBAAAccessTagInfo(TBAAInfo))
1532
    Inst->setMetadata(llvm::LLVMContext::MD_tbaa, Tag);
1533
}
1534

1535
void CodeGenModule::DecorateInstructionWithInvariantGroup(
1536
    llvm::Instruction *I, const CXXRecordDecl *RD) {
1537
  I->setMetadata(llvm::LLVMContext::MD_invariant_group,
1538
                 llvm::MDNode::get(getLLVMContext(), {}));
1539
}
1540

1541
void CodeGenModule::Error(SourceLocation loc, StringRef message) {
1542
  unsigned diagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error, "%0");
1543
  getDiags().Report(Context.getFullLoc(loc), diagID) << message;
1544
}
1545

1546
/// ErrorUnsupported - Print out an error that codegen doesn't support the
1547
/// specified stmt yet.
1548
void CodeGenModule::ErrorUnsupported(const Stmt *S, const char *Type) {
1549
  unsigned DiagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error,
1550
                                               "cannot compile this %0 yet");
1551
  std::string Msg = Type;
1552
  getDiags().Report(Context.getFullLoc(S->getBeginLoc()), DiagID)
1553
      << Msg << S->getSourceRange();
1554
}
1555

1556
/// ErrorUnsupported - Print out an error that codegen doesn't support the
1557
/// specified decl yet.
1558
void CodeGenModule::ErrorUnsupported(const Decl *D, const char *Type) {
1559
  unsigned DiagID = getDiags().getCustomDiagID(DiagnosticsEngine::Error,
1560
                                               "cannot compile this %0 yet");
1561
  std::string Msg = Type;
1562
  getDiags().Report(Context.getFullLoc(D->getLocation()), DiagID) << Msg;
1563
}
1564

1565
llvm::ConstantInt *CodeGenModule::getSize(CharUnits size) {
1566
  return llvm::ConstantInt::get(SizeTy, size.getQuantity());
1567
}
1568

1569
void CodeGenModule::setGlobalVisibility(llvm::GlobalValue *GV,
1570
                                        const NamedDecl *D) const {
1571
  // Internal definitions always have default visibility.
1572
  if (GV->hasLocalLinkage()) {
1573
    GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
1574
    return;
1575
  }
1576
  if (!D)
1577
    return;
1578

1579
  // Set visibility for definitions, and for declarations if requested globally
1580
  // or set explicitly.
1581
  LinkageInfo LV = D->getLinkageAndVisibility();
1582

1583
  // OpenMP declare target variables must be visible to the host so they can
1584
  // be registered. We require protected visibility unless the variable has
1585
  // the DT_nohost modifier and does not need to be registered.
1586
  if (Context.getLangOpts().OpenMP &&
1587
      Context.getLangOpts().OpenMPIsTargetDevice && isa<VarDecl>(D) &&
1588
      D->hasAttr<OMPDeclareTargetDeclAttr>() &&
1589
      D->getAttr<OMPDeclareTargetDeclAttr>()->getDevType() !=
1590
          OMPDeclareTargetDeclAttr::DT_NoHost &&
1591
      LV.getVisibility() == HiddenVisibility) {
1592
    GV->setVisibility(llvm::GlobalValue::ProtectedVisibility);
1593
    return;
1594
  }
1595

1596
  if (GV->hasDLLExportStorageClass() || GV->hasDLLImportStorageClass()) {
1597
    // Reject incompatible dlllstorage and visibility annotations.
1598
    if (!LV.isVisibilityExplicit())
1599
      return;
1600
    if (GV->hasDLLExportStorageClass()) {
1601
      if (LV.getVisibility() == HiddenVisibility)
1602
        getDiags().Report(D->getLocation(),
1603
                          diag::err_hidden_visibility_dllexport);
1604
    } else if (LV.getVisibility() != DefaultVisibility) {
1605
      getDiags().Report(D->getLocation(),
1606
                        diag::err_non_default_visibility_dllimport);
1607
    }
1608
    return;
1609
  }
1610

1611
  if (LV.isVisibilityExplicit() || getLangOpts().SetVisibilityForExternDecls ||
1612
      !GV->isDeclarationForLinker())
1613
    GV->setVisibility(GetLLVMVisibility(LV.getVisibility()));
1614
}
1615

1616
static bool shouldAssumeDSOLocal(const CodeGenModule &CGM,
1617
                                 llvm::GlobalValue *GV) {
1618
  if (GV->hasLocalLinkage())
1619
    return true;
1620

1621
  if (!GV->hasDefaultVisibility() && !GV->hasExternalWeakLinkage())
1622
    return true;
1623

1624
  // DLLImport explicitly marks the GV as external.
1625
  if (GV->hasDLLImportStorageClass())
1626
    return false;
1627

1628
  const llvm::Triple &TT = CGM.getTriple();
1629
  const auto &CGOpts = CGM.getCodeGenOpts();
1630
  if (TT.isWindowsGNUEnvironment()) {
1631
    // In MinGW, variables without DLLImport can still be automatically
1632
    // imported from a DLL by the linker; don't mark variables that
1633
    // potentially could come from another DLL as DSO local.
1634

1635
    // With EmulatedTLS, TLS variables can be autoimported from other DLLs
1636
    // (and this actually happens in the public interface of libstdc++), so
1637
    // such variables can't be marked as DSO local. (Native TLS variables
1638
    // can't be dllimported at all, though.)
1639
    if (GV->isDeclarationForLinker() && isa<llvm::GlobalVariable>(GV) &&
1640
        (!GV->isThreadLocal() || CGM.getCodeGenOpts().EmulatedTLS) &&
1641
        CGOpts.AutoImport)
1642
      return false;
1643
  }
1644

1645
  // On COFF, don't mark 'extern_weak' symbols as DSO local. If these symbols
1646
  // remain unresolved in the link, they can be resolved to zero, which is
1647
  // outside the current DSO.
1648
  if (TT.isOSBinFormatCOFF() && GV->hasExternalWeakLinkage())
1649
    return false;
1650

1651
  // Every other GV is local on COFF.
1652
  // Make an exception for windows OS in the triple: Some firmware builds use
1653
  // *-win32-macho triples. This (accidentally?) produced windows relocations
1654
  // without GOT tables in older clang versions; Keep this behaviour.
1655
  // FIXME: even thread local variables?
1656
  if (TT.isOSBinFormatCOFF() || (TT.isOSWindows() && TT.isOSBinFormatMachO()))
1657
    return true;
1658

1659
  // Only handle COFF and ELF for now.
1660
  if (!TT.isOSBinFormatELF())
1661
    return false;
1662

1663
  // If this is not an executable, don't assume anything is local.
1664
  llvm::Reloc::Model RM = CGOpts.RelocationModel;
1665
  const auto &LOpts = CGM.getLangOpts();
1666
  if (RM != llvm::Reloc::Static && !LOpts.PIE) {
1667
    // On ELF, if -fno-semantic-interposition is specified and the target
1668
    // supports local aliases, there will be neither CC1
1669
    // -fsemantic-interposition nor -fhalf-no-semantic-interposition. Set
1670
    // dso_local on the function if using a local alias is preferable (can avoid
1671
    // PLT indirection).
1672
    if (!(isa<llvm::Function>(GV) && GV->canBenefitFromLocalAlias()))
1673
      return false;
1674
    return !(CGM.getLangOpts().SemanticInterposition ||
1675
             CGM.getLangOpts().HalfNoSemanticInterposition);
1676
  }
1677

1678
  // A definition cannot be preempted from an executable.
1679
  if (!GV->isDeclarationForLinker())
1680
    return true;
1681

1682
  // Most PIC code sequences that assume that a symbol is local cannot produce a
1683
  // 0 if it turns out the symbol is undefined. While this is ABI and relocation
1684
  // depended, it seems worth it to handle it here.
1685
  if (RM == llvm::Reloc::PIC_ && GV->hasExternalWeakLinkage())
1686
    return false;
1687

1688
  // PowerPC64 prefers TOC indirection to avoid copy relocations.
1689
  if (TT.isPPC64())
1690
    return false;
1691

1692
  if (CGOpts.DirectAccessExternalData) {
1693
    // If -fdirect-access-external-data (default for -fno-pic), set dso_local
1694
    // for non-thread-local variables. If the symbol is not defined in the
1695
    // executable, a copy relocation will be needed at link time. dso_local is
1696
    // excluded for thread-local variables because they generally don't support
1697
    // copy relocations.
1698
    if (auto *Var = dyn_cast<llvm::GlobalVariable>(GV))
1699
      if (!Var->isThreadLocal())
1700
        return true;
1701

1702
    // -fno-pic sets dso_local on a function declaration to allow direct
1703
    // accesses when taking its address (similar to a data symbol). If the
1704
    // function is not defined in the executable, a canonical PLT entry will be
1705
    // needed at link time. -fno-direct-access-external-data can avoid the
1706
    // canonical PLT entry. We don't generalize this condition to -fpie/-fpic as
1707
    // it could just cause trouble without providing perceptible benefits.
1708
    if (isa<llvm::Function>(GV) && !CGOpts.NoPLT && RM == llvm::Reloc::Static)
1709
      return true;
1710
  }
1711

1712
  // If we can use copy relocations we can assume it is local.
1713

1714
  // Otherwise don't assume it is local.
1715
  return false;
1716
}
1717

1718
void CodeGenModule::setDSOLocal(llvm::GlobalValue *GV) const {
1719
  GV->setDSOLocal(shouldAssumeDSOLocal(*this, GV));
1720
}
1721

1722
void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
1723
                                          GlobalDecl GD) const {
1724
  const auto *D = dyn_cast<NamedDecl>(GD.getDecl());
1725
  // C++ destructors have a few C++ ABI specific special cases.
1726
  if (const auto *Dtor = dyn_cast_or_null<CXXDestructorDecl>(D)) {
1727
    getCXXABI().setCXXDestructorDLLStorage(GV, Dtor, GD.getDtorType());
1728
    return;
1729
  }
1730
  setDLLImportDLLExport(GV, D);
1731
}
1732

1733
void CodeGenModule::setDLLImportDLLExport(llvm::GlobalValue *GV,
1734
                                          const NamedDecl *D) const {
1735
  if (D && D->isExternallyVisible()) {
1736
    if (D->hasAttr<DLLImportAttr>())
1737
      GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
1738
    else if ((D->hasAttr<DLLExportAttr>() ||
1739
              shouldMapVisibilityToDLLExport(D)) &&
1740
             !GV->isDeclarationForLinker())
1741
      GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
1742
  }
1743
}
1744

1745
void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
1746
                                    GlobalDecl GD) const {
1747
  setDLLImportDLLExport(GV, GD);
1748
  setGVPropertiesAux(GV, dyn_cast<NamedDecl>(GD.getDecl()));
1749
}
1750

1751
void CodeGenModule::setGVProperties(llvm::GlobalValue *GV,
1752
                                    const NamedDecl *D) const {
1753
  setDLLImportDLLExport(GV, D);
1754
  setGVPropertiesAux(GV, D);
1755
}
1756

1757
void CodeGenModule::setGVPropertiesAux(llvm::GlobalValue *GV,
1758
                                       const NamedDecl *D) const {
1759
  setGlobalVisibility(GV, D);
1760
  setDSOLocal(GV);
1761
  GV->setPartition(CodeGenOpts.SymbolPartition);
1762
}
1763

1764
static llvm::GlobalVariable::ThreadLocalMode GetLLVMTLSModel(StringRef S) {
1765
  return llvm::StringSwitch<llvm::GlobalVariable::ThreadLocalMode>(S)
1766
      .Case("global-dynamic", llvm::GlobalVariable::GeneralDynamicTLSModel)
1767
      .Case("local-dynamic", llvm::GlobalVariable::LocalDynamicTLSModel)
1768
      .Case("initial-exec", llvm::GlobalVariable::InitialExecTLSModel)
1769
      .Case("local-exec", llvm::GlobalVariable::LocalExecTLSModel);
1770
}
1771

1772
llvm::GlobalVariable::ThreadLocalMode
1773
CodeGenModule::GetDefaultLLVMTLSModel() const {
1774
  switch (CodeGenOpts.getDefaultTLSModel()) {
1775
  case CodeGenOptions::GeneralDynamicTLSModel:
1776
    return llvm::GlobalVariable::GeneralDynamicTLSModel;
1777
  case CodeGenOptions::LocalDynamicTLSModel:
1778
    return llvm::GlobalVariable::LocalDynamicTLSModel;
1779
  case CodeGenOptions::InitialExecTLSModel:
1780
    return llvm::GlobalVariable::InitialExecTLSModel;
1781
  case CodeGenOptions::LocalExecTLSModel:
1782
    return llvm::GlobalVariable::LocalExecTLSModel;
1783
  }
1784
  llvm_unreachable("Invalid TLS model!");
1785
}
1786

1787
void CodeGenModule::setTLSMode(llvm::GlobalValue *GV, const VarDecl &D) const {
1788
  assert(D.getTLSKind() && "setting TLS mode on non-TLS var!");
1789

1790
  llvm::GlobalValue::ThreadLocalMode TLM;
1791
  TLM = GetDefaultLLVMTLSModel();
1792

1793
  // Override the TLS model if it is explicitly specified.
1794
  if (const TLSModelAttr *Attr = D.getAttr<TLSModelAttr>()) {
1795
    TLM = GetLLVMTLSModel(Attr->getModel());
1796
  }
1797

1798
  GV->setThreadLocalMode(TLM);
1799
}
1800

1801
static std::string getCPUSpecificMangling(const CodeGenModule &CGM,
1802
                                          StringRef Name) {
1803
  const TargetInfo &Target = CGM.getTarget();
1804
  return (Twine('.') + Twine(Target.CPUSpecificManglingCharacter(Name))).str();
1805
}
1806

1807
static void AppendCPUSpecificCPUDispatchMangling(const CodeGenModule &CGM,
1808
                                                 const CPUSpecificAttr *Attr,
1809
                                                 unsigned CPUIndex,
1810
                                                 raw_ostream &Out) {
1811
  // cpu_specific gets the current name, dispatch gets the resolver if IFunc is
1812
  // supported.
1813
  if (Attr)
1814
    Out << getCPUSpecificMangling(CGM, Attr->getCPUName(CPUIndex)->getName());
1815
  else if (CGM.getTarget().supportsIFunc())
1816
    Out << ".resolver";
1817
}
1818

1819
// Returns true if GD is a function decl with internal linkage and
1820
// needs a unique suffix after the mangled name.
1821
static bool isUniqueInternalLinkageDecl(GlobalDecl GD,
1822
                                        CodeGenModule &CGM) {
1823
  const Decl *D = GD.getDecl();
1824
  return !CGM.getModuleNameHash().empty() && isa<FunctionDecl>(D) &&
1825
         (CGM.getFunctionLinkage(GD) == llvm::GlobalValue::InternalLinkage);
1826
}
1827

1828
static std::string getMangledNameImpl(CodeGenModule &CGM, GlobalDecl GD,
1829
                                      const NamedDecl *ND,
1830
                                      bool OmitMultiVersionMangling = false) {
1831
  SmallString<256> Buffer;
1832
  llvm::raw_svector_ostream Out(Buffer);
1833
  MangleContext &MC = CGM.getCXXABI().getMangleContext();
1834
  if (!CGM.getModuleNameHash().empty())
1835
    MC.needsUniqueInternalLinkageNames();
1836
  bool ShouldMangle = MC.shouldMangleDeclName(ND);
1837
  if (ShouldMangle)
1838
    MC.mangleName(GD.getWithDecl(ND), Out);
1839
  else {
1840
    IdentifierInfo *II = ND->getIdentifier();
1841
    assert(II && "Attempt to mangle unnamed decl.");
1842
    const auto *FD = dyn_cast<FunctionDecl>(ND);
1843

1844
    if (FD &&
1845
        FD->getType()->castAs<FunctionType>()->getCallConv() == CC_X86RegCall) {
1846
      if (CGM.getLangOpts().RegCall4)
1847
        Out << "__regcall4__" << II->getName();
1848
      else
1849
        Out << "__regcall3__" << II->getName();
1850
    } else if (FD && FD->hasAttr<CUDAGlobalAttr>() &&
1851
               GD.getKernelReferenceKind() == KernelReferenceKind::Stub) {
1852
      Out << "__device_stub__" << II->getName();
1853
    } else {
1854
      Out << II->getName();
1855
    }
1856
  }
1857

1858
  // Check if the module name hash should be appended for internal linkage
1859
  // symbols.   This should come before multi-version target suffixes are
1860
  // appended. This is to keep the name and module hash suffix of the
1861
  // internal linkage function together.  The unique suffix should only be
1862
  // added when name mangling is done to make sure that the final name can
1863
  // be properly demangled.  For example, for C functions without prototypes,
1864
  // name mangling is not done and the unique suffix should not be appeneded
1865
  // then.
1866
  if (ShouldMangle && isUniqueInternalLinkageDecl(GD, CGM)) {
1867
    assert(CGM.getCodeGenOpts().UniqueInternalLinkageNames &&
1868
           "Hash computed when not explicitly requested");
1869
    Out << CGM.getModuleNameHash();
1870
  }
1871

1872
  if (const auto *FD = dyn_cast<FunctionDecl>(ND))
1873
    if (FD->isMultiVersion() && !OmitMultiVersionMangling) {
1874
      switch (FD->getMultiVersionKind()) {
1875
      case MultiVersionKind::CPUDispatch:
1876
      case MultiVersionKind::CPUSpecific:
1877
        AppendCPUSpecificCPUDispatchMangling(CGM,
1878
                                             FD->getAttr<CPUSpecificAttr>(),
1879
                                             GD.getMultiVersionIndex(), Out);
1880
        break;
1881
      case MultiVersionKind::Target: {
1882
        auto *Attr = FD->getAttr<TargetAttr>();
1883
        assert(Attr && "Expected TargetAttr to be present "
1884
                       "for attribute mangling");
1885
        const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
1886
        Info.appendAttributeMangling(Attr, Out);
1887
        break;
1888
      }
1889
      case MultiVersionKind::TargetVersion: {
1890
        auto *Attr = FD->getAttr<TargetVersionAttr>();
1891
        assert(Attr && "Expected TargetVersionAttr to be present "
1892
                       "for attribute mangling");
1893
        const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
1894
        Info.appendAttributeMangling(Attr, Out);
1895
        break;
1896
      }
1897
      case MultiVersionKind::TargetClones: {
1898
        auto *Attr = FD->getAttr<TargetClonesAttr>();
1899
        assert(Attr && "Expected TargetClonesAttr to be present "
1900
                       "for attribute mangling");
1901
        unsigned Index = GD.getMultiVersionIndex();
1902
        const ABIInfo &Info = CGM.getTargetCodeGenInfo().getABIInfo();
1903
        Info.appendAttributeMangling(Attr, Index, Out);
1904
        break;
1905
      }
1906
      case MultiVersionKind::None:
1907
        llvm_unreachable("None multiversion type isn't valid here");
1908
      }
1909
    }
1910

1911
  // Make unique name for device side static file-scope variable for HIP.
1912
  if (CGM.getContext().shouldExternalize(ND) &&
1913
      CGM.getLangOpts().GPURelocatableDeviceCode &&
1914
      CGM.getLangOpts().CUDAIsDevice)
1915
    CGM.printPostfixForExternalizedDecl(Out, ND);
1916

1917
  return std::string(Out.str());
1918
}
1919

1920
void CodeGenModule::UpdateMultiVersionNames(GlobalDecl GD,
1921
                                            const FunctionDecl *FD,
1922
                                            StringRef &CurName) {
1923
  if (!FD->isMultiVersion())
1924
    return;
1925

1926
  // Get the name of what this would be without the 'target' attribute.  This
1927
  // allows us to lookup the version that was emitted when this wasn't a
1928
  // multiversion function.
1929
  std::string NonTargetName =
1930
      getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
1931
  GlobalDecl OtherGD;
1932
  if (lookupRepresentativeDecl(NonTargetName, OtherGD)) {
1933
    assert(OtherGD.getCanonicalDecl()
1934
               .getDecl()
1935
               ->getAsFunction()
1936
               ->isMultiVersion() &&
1937
           "Other GD should now be a multiversioned function");
1938
    // OtherFD is the version of this function that was mangled BEFORE
1939
    // becoming a MultiVersion function.  It potentially needs to be updated.
1940
    const FunctionDecl *OtherFD = OtherGD.getCanonicalDecl()
1941
                                      .getDecl()
1942
                                      ->getAsFunction()
1943
                                      ->getMostRecentDecl();
1944
    std::string OtherName = getMangledNameImpl(*this, OtherGD, OtherFD);
1945
    // This is so that if the initial version was already the 'default'
1946
    // version, we don't try to update it.
1947
    if (OtherName != NonTargetName) {
1948
      // Remove instead of erase, since others may have stored the StringRef
1949
      // to this.
1950
      const auto ExistingRecord = Manglings.find(NonTargetName);
1951
      if (ExistingRecord != std::end(Manglings))
1952
        Manglings.remove(&(*ExistingRecord));
1953
      auto Result = Manglings.insert(std::make_pair(OtherName, OtherGD));
1954
      StringRef OtherNameRef = MangledDeclNames[OtherGD.getCanonicalDecl()] =
1955
          Result.first->first();
1956
      // If this is the current decl is being created, make sure we update the name.
1957
      if (GD.getCanonicalDecl() == OtherGD.getCanonicalDecl())
1958
        CurName = OtherNameRef;
1959
      if (llvm::GlobalValue *Entry = GetGlobalValue(NonTargetName))
1960
        Entry->setName(OtherName);
1961
    }
1962
  }
1963
}
1964

1965
StringRef CodeGenModule::getMangledName(GlobalDecl GD) {
1966
  GlobalDecl CanonicalGD = GD.getCanonicalDecl();
1967

1968
  // Some ABIs don't have constructor variants.  Make sure that base and
1969
  // complete constructors get mangled the same.
1970
  if (const auto *CD = dyn_cast<CXXConstructorDecl>(CanonicalGD.getDecl())) {
1971
    if (!getTarget().getCXXABI().hasConstructorVariants()) {
1972
      CXXCtorType OrigCtorType = GD.getCtorType();
1973
      assert(OrigCtorType == Ctor_Base || OrigCtorType == Ctor_Complete);
1974
      if (OrigCtorType == Ctor_Base)
1975
        CanonicalGD = GlobalDecl(CD, Ctor_Complete);
1976
    }
1977
  }
1978

1979
  // In CUDA/HIP device compilation with -fgpu-rdc, the mangled name of a
1980
  // static device variable depends on whether the variable is referenced by
1981
  // a host or device host function. Therefore the mangled name cannot be
1982
  // cached.
1983
  if (!LangOpts.CUDAIsDevice || !getContext().mayExternalize(GD.getDecl())) {
1984
    auto FoundName = MangledDeclNames.find(CanonicalGD);
1985
    if (FoundName != MangledDeclNames.end())
1986
      return FoundName->second;
1987
  }
1988

1989
  // Keep the first result in the case of a mangling collision.
1990
  const auto *ND = cast<NamedDecl>(GD.getDecl());
1991
  std::string MangledName = getMangledNameImpl(*this, GD, ND);
1992

1993
  // Ensure either we have different ABIs between host and device compilations,
1994
  // says host compilation following MSVC ABI but device compilation follows
1995
  // Itanium C++ ABI or, if they follow the same ABI, kernel names after
1996
  // mangling should be the same after name stubbing. The later checking is
1997
  // very important as the device kernel name being mangled in host-compilation
1998
  // is used to resolve the device binaries to be executed. Inconsistent naming
1999
  // result in undefined behavior. Even though we cannot check that naming
2000
  // directly between host- and device-compilations, the host- and
2001
  // device-mangling in host compilation could help catching certain ones.
2002
  assert(!isa<FunctionDecl>(ND) || !ND->hasAttr<CUDAGlobalAttr>() ||
2003
         getContext().shouldExternalize(ND) || getLangOpts().CUDAIsDevice ||
2004
         (getContext().getAuxTargetInfo() &&
2005
          (getContext().getAuxTargetInfo()->getCXXABI() !=
2006
           getContext().getTargetInfo().getCXXABI())) ||
2007
         getCUDARuntime().getDeviceSideName(ND) ==
2008
             getMangledNameImpl(
2009
                 *this,
2010
                 GD.getWithKernelReferenceKind(KernelReferenceKind::Kernel),
2011
                 ND));
2012

2013
  auto Result = Manglings.insert(std::make_pair(MangledName, GD));
2014
  return MangledDeclNames[CanonicalGD] = Result.first->first();
2015
}
2016

2017
StringRef CodeGenModule::getBlockMangledName(GlobalDecl GD,
2018
                                             const BlockDecl *BD) {
2019
  MangleContext &MangleCtx = getCXXABI().getMangleContext();
2020
  const Decl *D = GD.getDecl();
2021

2022
  SmallString<256> Buffer;
2023
  llvm::raw_svector_ostream Out(Buffer);
2024
  if (!D)
2025
    MangleCtx.mangleGlobalBlock(BD,
2026
      dyn_cast_or_null<VarDecl>(initializedGlobalDecl.getDecl()), Out);
2027
  else if (const auto *CD = dyn_cast<CXXConstructorDecl>(D))
2028
    MangleCtx.mangleCtorBlock(CD, GD.getCtorType(), BD, Out);
2029
  else if (const auto *DD = dyn_cast<CXXDestructorDecl>(D))
2030
    MangleCtx.mangleDtorBlock(DD, GD.getDtorType(), BD, Out);
2031
  else
2032
    MangleCtx.mangleBlock(cast<DeclContext>(D), BD, Out);
2033

2034
  auto Result = Manglings.insert(std::make_pair(Out.str(), BD));
2035
  return Result.first->first();
2036
}
2037

2038
const GlobalDecl CodeGenModule::getMangledNameDecl(StringRef Name) {
2039
  auto it = MangledDeclNames.begin();
2040
  while (it != MangledDeclNames.end()) {
2041
    if (it->second == Name)
2042
      return it->first;
2043
    it++;
2044
  }
2045
  return GlobalDecl();
2046
}
2047

2048
llvm::GlobalValue *CodeGenModule::GetGlobalValue(StringRef Name) {
2049
  return getModule().getNamedValue(Name);
2050
}
2051

2052
/// AddGlobalCtor - Add a function to the list that will be called before
2053
/// main() runs.
2054
void CodeGenModule::AddGlobalCtor(llvm::Function *Ctor, int Priority,
2055
                                  unsigned LexOrder,
2056
                                  llvm::Constant *AssociatedData) {
2057
  // FIXME: Type coercion of void()* types.
2058
  GlobalCtors.push_back(Structor(Priority, LexOrder, Ctor, AssociatedData));
2059
}
2060

2061
/// AddGlobalDtor - Add a function to the list that will be called
2062
/// when the module is unloaded.
2063
void CodeGenModule::AddGlobalDtor(llvm::Function *Dtor, int Priority,
2064
                                  bool IsDtorAttrFunc) {
2065
  if (CodeGenOpts.RegisterGlobalDtorsWithAtExit &&
2066
      (!getContext().getTargetInfo().getTriple().isOSAIX() || IsDtorAttrFunc)) {
2067
    DtorsUsingAtExit[Priority].push_back(Dtor);
2068
    return;
2069
  }
2070

2071
  // FIXME: Type coercion of void()* types.
2072
  GlobalDtors.push_back(Structor(Priority, ~0U, Dtor, nullptr));
2073
}
2074

2075
void CodeGenModule::EmitCtorList(CtorList &Fns, const char *GlobalName) {
2076
  if (Fns.empty()) return;
2077

2078
  // Ctor function type is void()*.
2079
  llvm::FunctionType* CtorFTy = llvm::FunctionType::get(VoidTy, false);
2080
  llvm::Type *CtorPFTy = llvm::PointerType::get(CtorFTy,
2081
      TheModule.getDataLayout().getProgramAddressSpace());
2082

2083
  // Get the type of a ctor entry, { i32, void ()*, i8* }.
2084
  llvm::StructType *CtorStructTy = llvm::StructType::get(
2085
      Int32Ty, CtorPFTy, VoidPtrTy);
2086

2087
  // Construct the constructor and destructor arrays.
2088
  ConstantInitBuilder builder(*this);
2089
  auto ctors = builder.beginArray(CtorStructTy);
2090
  for (const auto &I : Fns) {
2091
    auto ctor = ctors.beginStruct(CtorStructTy);
2092
    ctor.addInt(Int32Ty, I.Priority);
2093
    ctor.add(I.Initializer);
2094
    if (I.AssociatedData)
2095
      ctor.add(I.AssociatedData);
2096
    else
2097
      ctor.addNullPointer(VoidPtrTy);
2098
    ctor.finishAndAddTo(ctors);
2099
  }
2100

2101
  auto list =
2102
    ctors.finishAndCreateGlobal(GlobalName, getPointerAlign(),
2103
                                /*constant*/ false,
2104
                                llvm::GlobalValue::AppendingLinkage);
2105

2106
  // The LTO linker doesn't seem to like it when we set an alignment
2107
  // on appending variables.  Take it off as a workaround.
2108
  list->setAlignment(std::nullopt);
2109

2110
  Fns.clear();
2111
}
2112

2113
llvm::GlobalValue::LinkageTypes
2114
CodeGenModule::getFunctionLinkage(GlobalDecl GD) {
2115
  const auto *D = cast<FunctionDecl>(GD.getDecl());
2116

2117
  GVALinkage Linkage = getContext().GetGVALinkageForFunction(D);
2118

2119
  if (const auto *Dtor = dyn_cast<CXXDestructorDecl>(D))
2120
    return getCXXABI().getCXXDestructorLinkage(Linkage, Dtor, GD.getDtorType());
2121

2122
  return getLLVMLinkageForDeclarator(D, Linkage);
2123
}
2124

2125
llvm::ConstantInt *CodeGenModule::CreateCrossDsoCfiTypeId(llvm::Metadata *MD) {
2126
  llvm::MDString *MDS = dyn_cast<llvm::MDString>(MD);
2127
  if (!MDS) return nullptr;
2128

2129
  return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString()));
2130
}
2131

2132
llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) {
2133
  if (auto *FnType = T->getAs<FunctionProtoType>())
2134
    T = getContext().getFunctionType(
2135
        FnType->getReturnType(), FnType->getParamTypes(),
2136
        FnType->getExtProtoInfo().withExceptionSpec(EST_None));
2137

2138
  std::string OutName;
2139
  llvm::raw_string_ostream Out(OutName);
2140
  getCXXABI().getMangleContext().mangleCanonicalTypeName(
2141
      T, Out, getCodeGenOpts().SanitizeCfiICallNormalizeIntegers);
2142

2143
  if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers)
2144
    Out << ".normalized";
2145

2146
  return llvm::ConstantInt::get(Int32Ty,
2147
                                static_cast<uint32_t>(llvm::xxHash64(OutName)));
2148
}
2149

2150
void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD,
2151
                                              const CGFunctionInfo &Info,
2152
                                              llvm::Function *F, bool IsThunk) {
2153
  unsigned CallingConv;
2154
  llvm::AttributeList PAL;
2155
  ConstructAttributeList(F->getName(), Info, GD, PAL, CallingConv,
2156
                         /*AttrOnCallSite=*/false, IsThunk);
2157
  if (CallingConv == llvm::CallingConv::X86_VectorCall &&
2158
      getTarget().getTriple().isWindowsArm64EC()) {
2159
    SourceLocation Loc;
2160
    if (const Decl *D = GD.getDecl())
2161
      Loc = D->getLocation();
2162

2163
    Error(Loc, "__vectorcall calling convention is not currently supported");
2164
  }
2165
  F->setAttributes(PAL);
2166
  F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
2167
}
2168

2169
static void removeImageAccessQualifier(std::string& TyName) {
2170
  std::string ReadOnlyQual("__read_only");
2171
  std::string::size_type ReadOnlyPos = TyName.find(ReadOnlyQual);
2172
  if (ReadOnlyPos != std::string::npos)
2173
    // "+ 1" for the space after access qualifier.
2174
    TyName.erase(ReadOnlyPos, ReadOnlyQual.size() + 1);
2175
  else {
2176
    std::string WriteOnlyQual("__write_only");
2177
    std::string::size_type WriteOnlyPos = TyName.find(WriteOnlyQual);
2178
    if (WriteOnlyPos != std::string::npos)
2179
      TyName.erase(WriteOnlyPos, WriteOnlyQual.size() + 1);
2180
    else {
2181
      std::string ReadWriteQual("__read_write");
2182
      std::string::size_type ReadWritePos = TyName.find(ReadWriteQual);
2183
      if (ReadWritePos != std::string::npos)
2184
        TyName.erase(ReadWritePos, ReadWriteQual.size() + 1);
2185
    }
2186
  }
2187
}
2188

2189
// Returns the address space id that should be produced to the
2190
// kernel_arg_addr_space metadata. This is always fixed to the ids
2191
// as specified in the SPIR 2.0 specification in order to differentiate
2192
// for example in clGetKernelArgInfo() implementation between the address
2193
// spaces with targets without unique mapping to the OpenCL address spaces
2194
// (basically all single AS CPUs).
2195
static unsigned ArgInfoAddressSpace(LangAS AS) {
2196
  switch (AS) {
2197
  case LangAS::opencl_global:
2198
    return 1;
2199
  case LangAS::opencl_constant:
2200
    return 2;
2201
  case LangAS::opencl_local:
2202
    return 3;
2203
  case LangAS::opencl_generic:
2204
    return 4; // Not in SPIR 2.0 specs.
2205
  case LangAS::opencl_global_device:
2206
    return 5;
2207
  case LangAS::opencl_global_host:
2208
    return 6;
2209
  default:
2210
    return 0; // Assume private.
2211
  }
2212
}
2213

2214
void CodeGenModule::GenKernelArgMetadata(llvm::Function *Fn,
2215
                                         const FunctionDecl *FD,
2216
                                         CodeGenFunction *CGF) {
2217
  assert(((FD && CGF) || (!FD && !CGF)) &&
2218
         "Incorrect use - FD and CGF should either be both null or not!");
2219
  // Create MDNodes that represent the kernel arg metadata.
2220
  // Each MDNode is a list in the form of "key", N number of values which is
2221
  // the same number of values as their are kernel arguments.
2222

2223
  const PrintingPolicy &Policy = Context.getPrintingPolicy();
2224

2225
  // MDNode for the kernel argument address space qualifiers.
2226
  SmallVector<llvm::Metadata *, 8> addressQuals;
2227

2228
  // MDNode for the kernel argument access qualifiers (images only).
2229
  SmallVector<llvm::Metadata *, 8> accessQuals;
2230

2231
  // MDNode for the kernel argument type names.
2232
  SmallVector<llvm::Metadata *, 8> argTypeNames;
2233

2234
  // MDNode for the kernel argument base type names.
2235
  SmallVector<llvm::Metadata *, 8> argBaseTypeNames;
2236

2237
  // MDNode for the kernel argument type qualifiers.
2238
  SmallVector<llvm::Metadata *, 8> argTypeQuals;
2239

2240
  // MDNode for the kernel argument names.
2241
  SmallVector<llvm::Metadata *, 8> argNames;
2242

2243
  if (FD && CGF)
2244
    for (unsigned i = 0, e = FD->getNumParams(); i != e; ++i) {
2245
      const ParmVarDecl *parm = FD->getParamDecl(i);
2246
      // Get argument name.
2247
      argNames.push_back(llvm::MDString::get(VMContext, parm->getName()));
2248

2249
      if (!getLangOpts().OpenCL)
2250
        continue;
2251
      QualType ty = parm->getType();
2252
      std::string typeQuals;
2253

2254
      // Get image and pipe access qualifier:
2255
      if (ty->isImageType() || ty->isPipeType()) {
2256
        const Decl *PDecl = parm;
2257
        if (const auto *TD = ty->getAs<TypedefType>())
2258
          PDecl = TD->getDecl();
2259
        const OpenCLAccessAttr *A = PDecl->getAttr<OpenCLAccessAttr>();
2260
        if (A && A->isWriteOnly())
2261
          accessQuals.push_back(llvm::MDString::get(VMContext, "write_only"));
2262
        else if (A && A->isReadWrite())
2263
          accessQuals.push_back(llvm::MDString::get(VMContext, "read_write"));
2264
        else
2265
          accessQuals.push_back(llvm::MDString::get(VMContext, "read_only"));
2266
      } else
2267
        accessQuals.push_back(llvm::MDString::get(VMContext, "none"));
2268

2269
      auto getTypeSpelling = [&](QualType Ty) {
2270
        auto typeName = Ty.getUnqualifiedType().getAsString(Policy);
2271

2272
        if (Ty.isCanonical()) {
2273
          StringRef typeNameRef = typeName;
2274
          // Turn "unsigned type" to "utype"
2275
          if (typeNameRef.consume_front("unsigned "))
2276
            return std::string("u") + typeNameRef.str();
2277
          if (typeNameRef.consume_front("signed "))
2278
            return typeNameRef.str();
2279
        }
2280

2281
        return typeName;
2282
      };
2283

2284
      if (ty->isPointerType()) {
2285
        QualType pointeeTy = ty->getPointeeType();
2286

2287
        // Get address qualifier.
2288
        addressQuals.push_back(
2289
            llvm::ConstantAsMetadata::get(CGF->Builder.getInt32(
2290
                ArgInfoAddressSpace(pointeeTy.getAddressSpace()))));
2291

2292
        // Get argument type name.
2293
        std::string typeName = getTypeSpelling(pointeeTy) + "*";
2294
        std::string baseTypeName =
2295
            getTypeSpelling(pointeeTy.getCanonicalType()) + "*";
2296
        argTypeNames.push_back(llvm::MDString::get(VMContext, typeName));
2297
        argBaseTypeNames.push_back(
2298
            llvm::MDString::get(VMContext, baseTypeName));
2299

2300
        // Get argument type qualifiers:
2301
        if (ty.isRestrictQualified())
2302
          typeQuals = "restrict";
2303
        if (pointeeTy.isConstQualified() ||
2304
            (pointeeTy.getAddressSpace() == LangAS::opencl_constant))
2305
          typeQuals += typeQuals.empty() ? "const" : " const";
2306
        if (pointeeTy.isVolatileQualified())
2307
          typeQuals += typeQuals.empty() ? "volatile" : " volatile";
2308
      } else {
2309
        uint32_t AddrSpc = 0;
2310
        bool isPipe = ty->isPipeType();
2311
        if (ty->isImageType() || isPipe)
2312
          AddrSpc = ArgInfoAddressSpace(LangAS::opencl_global);
2313

2314
        addressQuals.push_back(
2315
            llvm::ConstantAsMetadata::get(CGF->Builder.getInt32(AddrSpc)));
2316

2317
        // Get argument type name.
2318
        ty = isPipe ? ty->castAs<PipeType>()->getElementType() : ty;
2319
        std::string typeName = getTypeSpelling(ty);
2320
        std::string baseTypeName = getTypeSpelling(ty.getCanonicalType());
2321

2322
        // Remove access qualifiers on images
2323
        // (as they are inseparable from type in clang implementation,
2324
        // but OpenCL spec provides a special query to get access qualifier
2325
        // via clGetKernelArgInfo with CL_KERNEL_ARG_ACCESS_QUALIFIER):
2326
        if (ty->isImageType()) {
2327
          removeImageAccessQualifier(typeName);
2328
          removeImageAccessQualifier(baseTypeName);
2329
        }
2330

2331
        argTypeNames.push_back(llvm::MDString::get(VMContext, typeName));
2332
        argBaseTypeNames.push_back(
2333
            llvm::MDString::get(VMContext, baseTypeName));
2334

2335
        if (isPipe)
2336
          typeQuals = "pipe";
2337
      }
2338
      argTypeQuals.push_back(llvm::MDString::get(VMContext, typeQuals));
2339
    }
2340

2341
  if (getLangOpts().OpenCL) {
2342
    Fn->setMetadata("kernel_arg_addr_space",
2343
                    llvm::MDNode::get(VMContext, addressQuals));
2344
    Fn->setMetadata("kernel_arg_access_qual",
2345
                    llvm::MDNode::get(VMContext, accessQuals));
2346
    Fn->setMetadata("kernel_arg_type",
2347
                    llvm::MDNode::get(VMContext, argTypeNames));
2348
    Fn->setMetadata("kernel_arg_base_type",
2349
                    llvm::MDNode::get(VMContext, argBaseTypeNames));
2350
    Fn->setMetadata("kernel_arg_type_qual",
2351
                    llvm::MDNode::get(VMContext, argTypeQuals));
2352
  }
2353
  if (getCodeGenOpts().EmitOpenCLArgMetadata ||
2354
      getCodeGenOpts().HIPSaveKernelArgName)
2355
    Fn->setMetadata("kernel_arg_name",
2356
                    llvm::MDNode::get(VMContext, argNames));
2357
}
2358

2359
/// Determines whether the language options require us to model
2360
/// unwind exceptions.  We treat -fexceptions as mandating this
2361
/// except under the fragile ObjC ABI with only ObjC exceptions
2362
/// enabled.  This means, for example, that C with -fexceptions
2363
/// enables this.
2364
static bool hasUnwindExceptions(const LangOptions &LangOpts) {
2365
  // If exceptions are completely disabled, obviously this is false.
2366
  if (!LangOpts.Exceptions) return false;
2367

2368
  // If C++ exceptions are enabled, this is true.
2369
  if (LangOpts.CXXExceptions) return true;
2370

2371
  // If ObjC exceptions are enabled, this depends on the ABI.
2372
  if (LangOpts.ObjCExceptions) {
2373
    return LangOpts.ObjCRuntime.hasUnwindExceptions();
2374
  }
2375

2376
  return true;
2377
}
2378

2379
static bool requiresMemberFunctionPointerTypeMetadata(CodeGenModule &CGM,
2380
                                                      const CXXMethodDecl *MD) {
2381
  // Check that the type metadata can ever actually be used by a call.
2382
  if (!CGM.getCodeGenOpts().LTOUnit ||
2383
      !CGM.HasHiddenLTOVisibility(MD->getParent()))
2384
    return false;
2385

2386
  // Only functions whose address can be taken with a member function pointer
2387
  // need this sort of type metadata.
2388
  return MD->isImplicitObjectMemberFunction() && !MD->isVirtual() &&
2389
         !isa<CXXConstructorDecl, CXXDestructorDecl>(MD);
2390
}
2391

2392
SmallVector<const CXXRecordDecl *, 0>
2393
CodeGenModule::getMostBaseClasses(const CXXRecordDecl *RD) {
2394
  llvm::SetVector<const CXXRecordDecl *> MostBases;
2395

2396
  std::function<void (const CXXRecordDecl *)> CollectMostBases;
2397
  CollectMostBases = [&](const CXXRecordDecl *RD) {
2398
    if (RD->getNumBases() == 0)
2399
      MostBases.insert(RD);
2400
    for (const CXXBaseSpecifier &B : RD->bases())
2401
      CollectMostBases(B.getType()->getAsCXXRecordDecl());
2402
  };
2403
  CollectMostBases(RD);
2404
  return MostBases.takeVector();
2405
}
2406

2407
void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
2408
                                                           llvm::Function *F) {
2409
  llvm::AttrBuilder B(F->getContext());
2410

2411
  if ((!D || !D->hasAttr<NoUwtableAttr>()) && CodeGenOpts.UnwindTables)
2412
    B.addUWTableAttr(llvm::UWTableKind(CodeGenOpts.UnwindTables));
2413

2414
  if (CodeGenOpts.StackClashProtector)
2415
    B.addAttribute("probe-stack", "inline-asm");
2416

2417
  if (CodeGenOpts.StackProbeSize && CodeGenOpts.StackProbeSize != 4096)
2418
    B.addAttribute("stack-probe-size",
2419
                   std::to_string(CodeGenOpts.StackProbeSize));
2420

2421
  if (!hasUnwindExceptions(LangOpts))
2422
    B.addAttribute(llvm::Attribute::NoUnwind);
2423

2424
  if (D && D->hasAttr<NoStackProtectorAttr>())
2425
    ; // Do nothing.
2426
  else if (D && D->hasAttr<StrictGuardStackCheckAttr>() &&
2427
           isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPOn))
2428
    B.addAttribute(llvm::Attribute::StackProtectStrong);
2429
  else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPOn))
2430
    B.addAttribute(llvm::Attribute::StackProtect);
2431
  else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPStrong))
2432
    B.addAttribute(llvm::Attribute::StackProtectStrong);
2433
  else if (isStackProtectorOn(LangOpts, getTriple(), LangOptions::SSPReq))
2434
    B.addAttribute(llvm::Attribute::StackProtectReq);
2435

2436
  if (!D) {
2437
    // If we don't have a declaration to control inlining, the function isn't
2438
    // explicitly marked as alwaysinline for semantic reasons, and inlining is
2439
    // disabled, mark the function as noinline.
2440
    if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline) &&
2441
        CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining)
2442
      B.addAttribute(llvm::Attribute::NoInline);
2443

2444
    F->addFnAttrs(B);
2445
    return;
2446
  }
2447

2448
  // Handle SME attributes that apply to function definitions,
2449
  // rather than to function prototypes.
2450
  if (D->hasAttr<ArmLocallyStreamingAttr>())
2451
    B.addAttribute("aarch64_pstate_sm_body");
2452

2453
  if (auto *Attr = D->getAttr<ArmNewAttr>()) {
2454
    if (Attr->isNewZA())
2455
      B.addAttribute("aarch64_new_za");
2456
    if (Attr->isNewZT0())
2457
      B.addAttribute("aarch64_new_zt0");
2458
  }
2459

2460
  // Track whether we need to add the optnone LLVM attribute,
2461
  // starting with the default for this optimization level.
2462
  bool ShouldAddOptNone =
2463
      !CodeGenOpts.DisableO0ImplyOptNone && CodeGenOpts.OptimizationLevel == 0;
2464
  // We can't add optnone in the following cases, it won't pass the verifier.
2465
  ShouldAddOptNone &= !D->hasAttr<MinSizeAttr>();
2466
  ShouldAddOptNone &= !D->hasAttr<AlwaysInlineAttr>();
2467

2468
  // Add optnone, but do so only if the function isn't always_inline.
2469
  if ((ShouldAddOptNone || D->hasAttr<OptimizeNoneAttr>()) &&
2470
      !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
2471
    B.addAttribute(llvm::Attribute::OptimizeNone);
2472

2473
    // OptimizeNone implies noinline; we should not be inlining such functions.
2474
    B.addAttribute(llvm::Attribute::NoInline);
2475

2476
    // We still need to handle naked functions even though optnone subsumes
2477
    // much of their semantics.
2478
    if (D->hasAttr<NakedAttr>())
2479
      B.addAttribute(llvm::Attribute::Naked);
2480

2481
    // OptimizeNone wins over OptimizeForSize and MinSize.
2482
    F->removeFnAttr(llvm::Attribute::OptimizeForSize);
2483
    F->removeFnAttr(llvm::Attribute::MinSize);
2484
  } else if (D->hasAttr<NakedAttr>()) {
2485
    // Naked implies noinline: we should not be inlining such functions.
2486
    B.addAttribute(llvm::Attribute::Naked);
2487
    B.addAttribute(llvm::Attribute::NoInline);
2488
  } else if (D->hasAttr<NoDuplicateAttr>()) {
2489
    B.addAttribute(llvm::Attribute::NoDuplicate);
2490
  } else if (D->hasAttr<NoInlineAttr>() && !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
2491
    // Add noinline if the function isn't always_inline.
2492
    B.addAttribute(llvm::Attribute::NoInline);
2493
  } else if (D->hasAttr<AlwaysInlineAttr>() &&
2494
             !F->hasFnAttribute(llvm::Attribute::NoInline)) {
2495
    // (noinline wins over always_inline, and we can't specify both in IR)
2496
    B.addAttribute(llvm::Attribute::AlwaysInline);
2497
  } else if (CodeGenOpts.getInlining() == CodeGenOptions::OnlyAlwaysInlining) {
2498
    // If we're not inlining, then force everything that isn't always_inline to
2499
    // carry an explicit noinline attribute.
2500
    if (!F->hasFnAttribute(llvm::Attribute::AlwaysInline))
2501
      B.addAttribute(llvm::Attribute::NoInline);
2502
  } else {
2503
    // Otherwise, propagate the inline hint attribute and potentially use its
2504
    // absence to mark things as noinline.
2505
    if (auto *FD = dyn_cast<FunctionDecl>(D)) {
2506
      // Search function and template pattern redeclarations for inline.
2507
      auto CheckForInline = [](const FunctionDecl *FD) {
2508
        auto CheckRedeclForInline = [](const FunctionDecl *Redecl) {
2509
          return Redecl->isInlineSpecified();
2510
        };
2511
        if (any_of(FD->redecls(), CheckRedeclForInline))
2512
          return true;
2513
        const FunctionDecl *Pattern = FD->getTemplateInstantiationPattern();
2514
        if (!Pattern)
2515
          return false;
2516
        return any_of(Pattern->redecls(), CheckRedeclForInline);
2517
      };
2518
      if (CheckForInline(FD)) {
2519
        B.addAttribute(llvm::Attribute::InlineHint);
2520
      } else if (CodeGenOpts.getInlining() ==
2521
                     CodeGenOptions::OnlyHintInlining &&
2522
                 !FD->isInlined() &&
2523
                 !F->hasFnAttribute(llvm::Attribute::AlwaysInline)) {
2524
        B.addAttribute(llvm::Attribute::NoInline);
2525
      }
2526
    }
2527
  }
2528

2529
  // Add other optimization related attributes if we are optimizing this
2530
  // function.
2531
  if (!D->hasAttr<OptimizeNoneAttr>()) {
2532
    if (D->hasAttr<ColdAttr>()) {
2533
      if (!ShouldAddOptNone)
2534
        B.addAttribute(llvm::Attribute::OptimizeForSize);
2535
      B.addAttribute(llvm::Attribute::Cold);
2536
    }
2537
    if (D->hasAttr<HotAttr>())
2538
      B.addAttribute(llvm::Attribute::Hot);
2539
    if (D->hasAttr<MinSizeAttr>())
2540
      B.addAttribute(llvm::Attribute::MinSize);
2541
  }
2542

2543
  F->addFnAttrs(B);
2544

2545
  unsigned alignment = D->getMaxAlignment() / Context.getCharWidth();
2546
  if (alignment)
2547
    F->setAlignment(llvm::Align(alignment));
2548

2549
  if (!D->hasAttr<AlignedAttr>())
2550
    if (LangOpts.FunctionAlignment)
2551
      F->setAlignment(llvm::Align(1ull << LangOpts.FunctionAlignment));
2552

2553
  // Some C++ ABIs require 2-byte alignment for member functions, in order to
2554
  // reserve a bit for differentiating between virtual and non-virtual member
2555
  // functions. If the current target's C++ ABI requires this and this is a
2556
  // member function, set its alignment accordingly.
2557
  if (getTarget().getCXXABI().areMemberFunctionsAligned()) {
2558
    if (isa<CXXMethodDecl>(D) && F->getPointerAlignment(getDataLayout()) < 2)
2559
      F->setAlignment(std::max(llvm::Align(2), F->getAlign().valueOrOne()));
2560
  }
2561

2562
  // In the cross-dso CFI mode with canonical jump tables, we want !type
2563
  // attributes on definitions only.
2564
  if (CodeGenOpts.SanitizeCfiCrossDso &&
2565
      CodeGenOpts.SanitizeCfiCanonicalJumpTables) {
2566
    if (auto *FD = dyn_cast<FunctionDecl>(D)) {
2567
      // Skip available_externally functions. They won't be codegen'ed in the
2568
      // current module anyway.
2569
      if (getContext().GetGVALinkageForFunction(FD) != GVA_AvailableExternally)
2570
        CreateFunctionTypeMetadataForIcall(FD, F);
2571
    }
2572
  }
2573

2574
  // Emit type metadata on member functions for member function pointer checks.
2575
  // These are only ever necessary on definitions; we're guaranteed that the
2576
  // definition will be present in the LTO unit as a result of LTO visibility.
2577
  auto *MD = dyn_cast<CXXMethodDecl>(D);
2578
  if (MD && requiresMemberFunctionPointerTypeMetadata(*this, MD)) {
2579
    for (const CXXRecordDecl *Base : getMostBaseClasses(MD->getParent())) {
2580
      llvm::Metadata *Id =
2581
          CreateMetadataIdentifierForType(Context.getMemberPointerType(
2582
              MD->getType(), Context.getRecordType(Base).getTypePtr()));
2583
      F->addTypeMetadata(0, Id);
2584
    }
2585
  }
2586
}
2587

2588
void CodeGenModule::SetCommonAttributes(GlobalDecl GD, llvm::GlobalValue *GV) {
2589
  const Decl *D = GD.getDecl();
2590
  if (isa_and_nonnull<NamedDecl>(D))
2591
    setGVProperties(GV, GD);
2592
  else
2593
    GV->setVisibility(llvm::GlobalValue::DefaultVisibility);
2594

2595
  if (D && D->hasAttr<UsedAttr>())
2596
    addUsedOrCompilerUsedGlobal(GV);
2597

2598
  if (const auto *VD = dyn_cast_if_present<VarDecl>(D);
2599
      VD &&
2600
      ((CodeGenOpts.KeepPersistentStorageVariables &&
2601
        (VD->getStorageDuration() == SD_Static ||
2602
         VD->getStorageDuration() == SD_Thread)) ||
2603
       (CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static &&
2604
        VD->getType().isConstQualified())))
2605
    addUsedOrCompilerUsedGlobal(GV);
2606
}
2607

2608
bool CodeGenModule::GetCPUAndFeaturesAttributes(GlobalDecl GD,
2609
                                                llvm::AttrBuilder &Attrs,
2610
                                                bool SetTargetFeatures) {
2611
  // Add target-cpu and target-features attributes to functions. If
2612
  // we have a decl for the function and it has a target attribute then
2613
  // parse that and add it to the feature set.
2614
  StringRef TargetCPU = getTarget().getTargetOpts().CPU;
2615
  StringRef TuneCPU = getTarget().getTargetOpts().TuneCPU;
2616
  std::vector<std::string> Features;
2617
  const auto *FD = dyn_cast_or_null<FunctionDecl>(GD.getDecl());
2618
  FD = FD ? FD->getMostRecentDecl() : FD;
2619
  const auto *TD = FD ? FD->getAttr<TargetAttr>() : nullptr;
2620
  const auto *TV = FD ? FD->getAttr<TargetVersionAttr>() : nullptr;
2621
  assert((!TD || !TV) && "both target_version and target specified");
2622
  const auto *SD = FD ? FD->getAttr<CPUSpecificAttr>() : nullptr;
2623
  const auto *TC = FD ? FD->getAttr<TargetClonesAttr>() : nullptr;
2624
  bool AddedAttr = false;
2625
  if (TD || TV || SD || TC) {
2626
    llvm::StringMap<bool> FeatureMap;
2627
    getContext().getFunctionFeatureMap(FeatureMap, GD);
2628

2629
    // Produce the canonical string for this set of features.
2630
    for (const llvm::StringMap<bool>::value_type &Entry : FeatureMap)
2631
      Features.push_back((Entry.getValue() ? "+" : "-") + Entry.getKey().str());
2632

2633
    // Now add the target-cpu and target-features to the function.
2634
    // While we populated the feature map above, we still need to
2635
    // get and parse the target attribute so we can get the cpu for
2636
    // the function.
2637
    if (TD) {
2638
      ParsedTargetAttr ParsedAttr =
2639
          Target.parseTargetAttr(TD->getFeaturesStr());
2640
      if (!ParsedAttr.CPU.empty() &&
2641
          getTarget().isValidCPUName(ParsedAttr.CPU)) {
2642
        TargetCPU = ParsedAttr.CPU;
2643
        TuneCPU = ""; // Clear the tune CPU.
2644
      }
2645
      if (!ParsedAttr.Tune.empty() &&
2646
          getTarget().isValidCPUName(ParsedAttr.Tune))
2647
        TuneCPU = ParsedAttr.Tune;
2648
    }
2649

2650
    if (SD) {
2651
      // Apply the given CPU name as the 'tune-cpu' so that the optimizer can
2652
      // favor this processor.
2653
      TuneCPU = SD->getCPUName(GD.getMultiVersionIndex())->getName();
2654
    }
2655
  } else {
2656
    // Otherwise just add the existing target cpu and target features to the
2657
    // function.
2658
    Features = getTarget().getTargetOpts().Features;
2659
  }
2660

2661
  if (!TargetCPU.empty()) {
2662
    Attrs.addAttribute("target-cpu", TargetCPU);
2663
    AddedAttr = true;
2664
  }
2665
  if (!TuneCPU.empty()) {
2666
    Attrs.addAttribute("tune-cpu", TuneCPU);
2667
    AddedAttr = true;
2668
  }
2669
  if (!Features.empty() && SetTargetFeatures) {
2670
    llvm::erase_if(Features, [&](const std::string& F) {
2671
       return getTarget().isReadOnlyFeature(F.substr(1));
2672
    });
2673
    llvm::sort(Features);
2674
    Attrs.addAttribute("target-features", llvm::join(Features, ","));
2675
    AddedAttr = true;
2676
  }
2677

2678
  return AddedAttr;
2679
}
2680

2681
void CodeGenModule::setNonAliasAttributes(GlobalDecl GD,
2682
                                          llvm::GlobalObject *GO) {
2683
  const Decl *D = GD.getDecl();
2684
  SetCommonAttributes(GD, GO);
2685

2686
  if (D) {
2687
    if (auto *GV = dyn_cast<llvm::GlobalVariable>(GO)) {
2688
      if (D->hasAttr<RetainAttr>())
2689
        addUsedGlobal(GV);
2690
      if (auto *SA = D->getAttr<PragmaClangBSSSectionAttr>())
2691
        GV->addAttribute("bss-section", SA->getName());
2692
      if (auto *SA = D->getAttr<PragmaClangDataSectionAttr>())
2693
        GV->addAttribute("data-section", SA->getName());
2694
      if (auto *SA = D->getAttr<PragmaClangRodataSectionAttr>())
2695
        GV->addAttribute("rodata-section", SA->getName());
2696
      if (auto *SA = D->getAttr<PragmaClangRelroSectionAttr>())
2697
        GV->addAttribute("relro-section", SA->getName());
2698
    }
2699

2700
    if (auto *F = dyn_cast<llvm::Function>(GO)) {
2701
      if (D->hasAttr<RetainAttr>())
2702
        addUsedGlobal(F);
2703
      if (auto *SA = D->getAttr<PragmaClangTextSectionAttr>())
2704
        if (!D->getAttr<SectionAttr>())
2705
          F->setSection(SA->getName());
2706

2707
      llvm::AttrBuilder Attrs(F->getContext());
2708
      if (GetCPUAndFeaturesAttributes(GD, Attrs)) {
2709
        // We know that GetCPUAndFeaturesAttributes will always have the
2710
        // newest set, since it has the newest possible FunctionDecl, so the
2711
        // new ones should replace the old.
2712
        llvm::AttributeMask RemoveAttrs;
2713
        RemoveAttrs.addAttribute("target-cpu");
2714
        RemoveAttrs.addAttribute("target-features");
2715
        RemoveAttrs.addAttribute("tune-cpu");
2716
        F->removeFnAttrs(RemoveAttrs);
2717
        F->addFnAttrs(Attrs);
2718
      }
2719
    }
2720

2721
    if (const auto *CSA = D->getAttr<CodeSegAttr>())
2722
      GO->setSection(CSA->getName());
2723
    else if (const auto *SA = D->getAttr<SectionAttr>())
2724
      GO->setSection(SA->getName());
2725
  }
2726

2727
  getTargetCodeGenInfo().setTargetAttributes(D, GO, *this);
2728
}
2729

2730
void CodeGenModule::SetInternalFunctionAttributes(GlobalDecl GD,
2731
                                                  llvm::Function *F,
2732
                                                  const CGFunctionInfo &FI) {
2733
  const Decl *D = GD.getDecl();
2734
  SetLLVMFunctionAttributes(GD, FI, F, /*IsThunk=*/false);
2735
  SetLLVMFunctionAttributesForDefinition(D, F);
2736

2737
  F->setLinkage(llvm::Function::InternalLinkage);
2738

2739
  setNonAliasAttributes(GD, F);
2740
}
2741

2742
static void setLinkageForGV(llvm::GlobalValue *GV, const NamedDecl *ND) {
2743
  // Set linkage and visibility in case we never see a definition.
2744
  LinkageInfo LV = ND->getLinkageAndVisibility();
2745
  // Don't set internal linkage on declarations.
2746
  // "extern_weak" is overloaded in LLVM; we probably should have
2747
  // separate linkage types for this.
2748
  if (isExternallyVisible(LV.getLinkage()) &&
2749
      (ND->hasAttr<WeakAttr>() || ND->isWeakImported()))
2750
    GV->setLinkage(llvm::GlobalValue::ExternalWeakLinkage);
2751
}
2752

2753
void CodeGenModule::CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD,
2754
                                                       llvm::Function *F) {
2755
  // Only if we are checking indirect calls.
2756
  if (!LangOpts.Sanitize.has(SanitizerKind::CFIICall))
2757
    return;
2758

2759
  // Non-static class methods are handled via vtable or member function pointer
2760
  // checks elsewhere.
2761
  if (isa<CXXMethodDecl>(FD) && !cast<CXXMethodDecl>(FD)->isStatic())
2762
    return;
2763

2764
  llvm::Metadata *MD = CreateMetadataIdentifierForType(FD->getType());
2765
  F->addTypeMetadata(0, MD);
2766
  F->addTypeMetadata(0, CreateMetadataIdentifierGeneralized(FD->getType()));
2767

2768
  // Emit a hash-based bit set entry for cross-DSO calls.
2769
  if (CodeGenOpts.SanitizeCfiCrossDso)
2770
    if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
2771
      F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId));
2772
}
2773

2774
void CodeGenModule::setKCFIType(const FunctionDecl *FD, llvm::Function *F) {
2775
  llvm::LLVMContext &Ctx = F->getContext();
2776
  llvm::MDBuilder MDB(Ctx);
2777
  F->setMetadata(llvm::LLVMContext::MD_kcfi_type,
2778
                 llvm::MDNode::get(
2779
                     Ctx, MDB.createConstant(CreateKCFITypeId(FD->getType()))));
2780
}
2781

2782
static bool allowKCFIIdentifier(StringRef Name) {
2783
  // KCFI type identifier constants are only necessary for external assembly
2784
  // functions, which means it's safe to skip unusual names. Subset of
2785
  // MCAsmInfo::isAcceptableChar() and MCAsmInfoXCOFF::isAcceptableChar().
2786
  return llvm::all_of(Name, [](const char &C) {
2787
    return llvm::isAlnum(C) || C == '_' || C == '.';
2788
  });
2789
}
2790

2791
void CodeGenModule::finalizeKCFITypes() {
2792
  llvm::Module &M = getModule();
2793
  for (auto &F : M.functions()) {
2794
    // Remove KCFI type metadata from non-address-taken local functions.
2795
    bool AddressTaken = F.hasAddressTaken();
2796
    if (!AddressTaken && F.hasLocalLinkage())
2797
      F.eraseMetadata(llvm::LLVMContext::MD_kcfi_type);
2798

2799
    // Generate a constant with the expected KCFI type identifier for all
2800
    // address-taken function declarations to support annotating indirectly
2801
    // called assembly functions.
2802
    if (!AddressTaken || !F.isDeclaration())
2803
      continue;
2804

2805
    const llvm::ConstantInt *Type;
2806
    if (const llvm::MDNode *MD = F.getMetadata(llvm::LLVMContext::MD_kcfi_type))
2807
      Type = llvm::mdconst::extract<llvm::ConstantInt>(MD->getOperand(0));
2808
    else
2809
      continue;
2810

2811
    StringRef Name = F.getName();
2812
    if (!allowKCFIIdentifier(Name))
2813
      continue;
2814

2815
    std::string Asm = (".weak __kcfi_typeid_" + Name + "\n.set __kcfi_typeid_" +
2816
                       Name + ", " + Twine(Type->getZExtValue()) + "\n")
2817
                          .str();
2818
    M.appendModuleInlineAsm(Asm);
2819
  }
2820
}
2821

2822
void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F,
2823
                                          bool IsIncompleteFunction,
2824
                                          bool IsThunk) {
2825

2826
  if (llvm::Intrinsic::ID IID = F->getIntrinsicID()) {
2827
    // If this is an intrinsic function, set the function's attributes
2828
    // to the intrinsic's attributes.
2829
    F->setAttributes(llvm::Intrinsic::getAttributes(getLLVMContext(), IID));
2830
    return;
2831
  }
2832

2833
  const auto *FD = cast<FunctionDecl>(GD.getDecl());
2834

2835
  if (!IsIncompleteFunction)
2836
    SetLLVMFunctionAttributes(GD, getTypes().arrangeGlobalDeclaration(GD), F,
2837
                              IsThunk);
2838

2839
  // Add the Returned attribute for "this", except for iOS 5 and earlier
2840
  // where substantial code, including the libstdc++ dylib, was compiled with
2841
  // GCC and does not actually return "this".
2842
  if (!IsThunk && getCXXABI().HasThisReturn(GD) &&
2843
      !(getTriple().isiOS() && getTriple().isOSVersionLT(6))) {
2844
    assert(!F->arg_empty() &&
2845
           F->arg_begin()->getType()
2846
             ->canLosslesslyBitCastTo(F->getReturnType()) &&
2847
           "unexpected this return");
2848
    F->addParamAttr(0, llvm::Attribute::Returned);
2849
  }
2850

2851
  // Only a few attributes are set on declarations; these may later be
2852
  // overridden by a definition.
2853

2854
  setLinkageForGV(F, FD);
2855
  setGVProperties(F, FD);
2856

2857
  // Setup target-specific attributes.
2858
  if (!IsIncompleteFunction && F->isDeclaration())
2859
    getTargetCodeGenInfo().setTargetAttributes(FD, F, *this);
2860

2861
  if (const auto *CSA = FD->getAttr<CodeSegAttr>())
2862
    F->setSection(CSA->getName());
2863
  else if (const auto *SA = FD->getAttr<SectionAttr>())
2864
     F->setSection(SA->getName());
2865

2866
  if (const auto *EA = FD->getAttr<ErrorAttr>()) {
2867
    if (EA->isError())
2868
      F->addFnAttr("dontcall-error", EA->getUserDiagnostic());
2869
    else if (EA->isWarning())
2870
      F->addFnAttr("dontcall-warn", EA->getUserDiagnostic());
2871
  }
2872

2873
  // If we plan on emitting this inline builtin, we can't treat it as a builtin.
2874
  if (FD->isInlineBuiltinDeclaration()) {
2875
    const FunctionDecl *FDBody;
2876
    bool HasBody = FD->hasBody(FDBody);
2877
    (void)HasBody;
2878
    assert(HasBody && "Inline builtin declarations should always have an "
2879
                      "available body!");
2880
    if (shouldEmitFunction(FDBody))
2881
      F->addFnAttr(llvm::Attribute::NoBuiltin);
2882
  }
2883

2884
  if (FD->isReplaceableGlobalAllocationFunction()) {
2885
    // A replaceable global allocation function does not act like a builtin by
2886
    // default, only if it is invoked by a new-expression or delete-expression.
2887
    F->addFnAttr(llvm::Attribute::NoBuiltin);
2888
  }
2889

2890
  if (isa<CXXConstructorDecl>(FD) || isa<CXXDestructorDecl>(FD))
2891
    F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2892
  else if (const auto *MD = dyn_cast<CXXMethodDecl>(FD))
2893
    if (MD->isVirtual())
2894
      F->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2895

2896
  // Don't emit entries for function declarations in the cross-DSO mode. This
2897
  // is handled with better precision by the receiving DSO. But if jump tables
2898
  // are non-canonical then we need type metadata in order to produce the local
2899
  // jump table.
2900
  if (!CodeGenOpts.SanitizeCfiCrossDso ||
2901
      !CodeGenOpts.SanitizeCfiCanonicalJumpTables)
2902
    CreateFunctionTypeMetadataForIcall(FD, F);
2903

2904
  if (LangOpts.Sanitize.has(SanitizerKind::KCFI))
2905
    setKCFIType(FD, F);
2906

2907
  if (getLangOpts().OpenMP && FD->hasAttr<OMPDeclareSimdDeclAttr>())
2908
    getOpenMPRuntime().emitDeclareSimdFunction(FD, F);
2909

2910
  if (CodeGenOpts.InlineMaxStackSize != UINT_MAX)
2911
    F->addFnAttr("inline-max-stacksize", llvm::utostr(CodeGenOpts.InlineMaxStackSize));
2912

2913
  if (const auto *CB = FD->getAttr<CallbackAttr>()) {
2914
    // Annotate the callback behavior as metadata:
2915
    //  - The callback callee (as argument number).
2916
    //  - The callback payloads (as argument numbers).
2917
    llvm::LLVMContext &Ctx = F->getContext();
2918
    llvm::MDBuilder MDB(Ctx);
2919

2920
    // The payload indices are all but the first one in the encoding. The first
2921
    // identifies the callback callee.
2922
    int CalleeIdx = *CB->encoding_begin();
2923
    ArrayRef<int> PayloadIndices(CB->encoding_begin() + 1, CB->encoding_end());
2924
    F->addMetadata(llvm::LLVMContext::MD_callback,
2925
                   *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding(
2926
                                               CalleeIdx, PayloadIndices,
2927
                                               /* VarArgsArePassed */ false)}));
2928
  }
2929
}
2930

2931
void CodeGenModule::addUsedGlobal(llvm::GlobalValue *GV) {
2932
  assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
2933
         "Only globals with definition can force usage.");
2934
  LLVMUsed.emplace_back(GV);
2935
}
2936

2937
void CodeGenModule::addCompilerUsedGlobal(llvm::GlobalValue *GV) {
2938
  assert(!GV->isDeclaration() &&
2939
         "Only globals with definition can force usage.");
2940
  LLVMCompilerUsed.emplace_back(GV);
2941
}
2942

2943
void CodeGenModule::addUsedOrCompilerUsedGlobal(llvm::GlobalValue *GV) {
2944
  assert((isa<llvm::Function>(GV) || !GV->isDeclaration()) &&
2945
         "Only globals with definition can force usage.");
2946
  if (getTriple().isOSBinFormatELF())
2947
    LLVMCompilerUsed.emplace_back(GV);
2948
  else
2949
    LLVMUsed.emplace_back(GV);
2950
}
2951

2952
static void emitUsed(CodeGenModule &CGM, StringRef Name,
2953
                     std::vector<llvm::WeakTrackingVH> &List) {
2954
  // Don't create llvm.used if there is no need.
2955
  if (List.empty())
2956
    return;
2957

2958
  // Convert List to what ConstantArray needs.
2959
  SmallVector<llvm::Constant*, 8> UsedArray;
2960
  UsedArray.resize(List.size());
2961
  for (unsigned i = 0, e = List.size(); i != e; ++i) {
2962
    UsedArray[i] =
2963
        llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
2964
            cast<llvm::Constant>(&*List[i]), CGM.Int8PtrTy);
2965
  }
2966

2967
  if (UsedArray.empty())
2968
    return;
2969
  llvm::ArrayType *ATy = llvm::ArrayType::get(CGM.Int8PtrTy, UsedArray.size());
2970

2971
  auto *GV = new llvm::GlobalVariable(
2972
      CGM.getModule(), ATy, false, llvm::GlobalValue::AppendingLinkage,
2973
      llvm::ConstantArray::get(ATy, UsedArray), Name);
2974

2975
  GV->setSection("llvm.metadata");
2976
}
2977

2978
void CodeGenModule::emitLLVMUsed() {
2979
  emitUsed(*this, "llvm.used", LLVMUsed);
2980
  emitUsed(*this, "llvm.compiler.used", LLVMCompilerUsed);
2981
}
2982

2983
void CodeGenModule::AppendLinkerOptions(StringRef Opts) {
2984
  auto *MDOpts = llvm::MDString::get(getLLVMContext(), Opts);
2985
  LinkerOptionsMetadata.push_back(llvm::MDNode::get(getLLVMContext(), MDOpts));
2986
}
2987

2988
void CodeGenModule::AddDetectMismatch(StringRef Name, StringRef Value) {
2989
  llvm::SmallString<32> Opt;
2990
  getTargetCodeGenInfo().getDetectMismatchOption(Name, Value, Opt);
2991
  if (Opt.empty())
2992
    return;
2993
  auto *MDOpts = llvm::MDString::get(getLLVMContext(), Opt);
2994
  LinkerOptionsMetadata.push_back(llvm::MDNode::get(getLLVMContext(), MDOpts));
2995
}
2996

2997
void CodeGenModule::AddDependentLib(StringRef Lib) {
2998
  auto &C = getLLVMContext();
2999
  if (getTarget().getTriple().isOSBinFormatELF()) {
3000
      ELFDependentLibraries.push_back(
3001
        llvm::MDNode::get(C, llvm::MDString::get(C, Lib)));
3002
    return;
3003
  }
3004

3005
  llvm::SmallString<24> Opt;
3006
  getTargetCodeGenInfo().getDependentLibraryOption(Lib, Opt);
3007
  auto *MDOpts = llvm::MDString::get(getLLVMContext(), Opt);
3008
  LinkerOptionsMetadata.push_back(llvm::MDNode::get(C, MDOpts));
3009
}
3010

3011
/// Add link options implied by the given module, including modules
3012
/// it depends on, using a postorder walk.
3013
static void addLinkOptionsPostorder(CodeGenModule &CGM, Module *Mod,
3014
                                    SmallVectorImpl<llvm::MDNode *> &Metadata,
3015
                                    llvm::SmallPtrSet<Module *, 16> &Visited) {
3016
  // Import this module's parent.
3017
  if (Mod->Parent && Visited.insert(Mod->Parent).second) {
3018
    addLinkOptionsPostorder(CGM, Mod->Parent, Metadata, Visited);
3019
  }
3020

3021
  // Import this module's dependencies.
3022
  for (Module *Import : llvm::reverse(Mod->Imports)) {
3023
    if (Visited.insert(Import).second)
3024
      addLinkOptionsPostorder(CGM, Import, Metadata, Visited);
3025
  }
3026

3027
  // Add linker options to link against the libraries/frameworks
3028
  // described by this module.
3029
  llvm::LLVMContext &Context = CGM.getLLVMContext();
3030
  bool IsELF = CGM.getTarget().getTriple().isOSBinFormatELF();
3031

3032
  // For modules that use export_as for linking, use that module
3033
  // name instead.
3034
  if (Mod->UseExportAsModuleLinkName)
3035
    return;
3036

3037
  for (const Module::LinkLibrary &LL : llvm::reverse(Mod->LinkLibraries)) {
3038
    // Link against a framework.  Frameworks are currently Darwin only, so we
3039
    // don't to ask TargetCodeGenInfo for the spelling of the linker option.
3040
    if (LL.IsFramework) {
3041
      llvm::Metadata *Args[2] = {llvm::MDString::get(Context, "-framework"),
3042
                                 llvm::MDString::get(Context, LL.Library)};
3043

3044
      Metadata.push_back(llvm::MDNode::get(Context, Args));
3045
      continue;
3046
    }
3047

3048
    // Link against a library.
3049
    if (IsELF) {
3050
      llvm::Metadata *Args[2] = {
3051
          llvm::MDString::get(Context, "lib"),
3052
          llvm::MDString::get(Context, LL.Library),
3053
      };
3054
      Metadata.push_back(llvm::MDNode::get(Context, Args));
3055
    } else {
3056
      llvm::SmallString<24> Opt;
3057
      CGM.getTargetCodeGenInfo().getDependentLibraryOption(LL.Library, Opt);
3058
      auto *OptString = llvm::MDString::get(Context, Opt);
3059
      Metadata.push_back(llvm::MDNode::get(Context, OptString));
3060
    }
3061
  }
3062
}
3063

3064
void CodeGenModule::EmitModuleInitializers(clang::Module *Primary) {
3065
  assert(Primary->isNamedModuleUnit() &&
3066
         "We should only emit module initializers for named modules.");
3067

3068
  // Emit the initializers in the order that sub-modules appear in the
3069
  // source, first Global Module Fragments, if present.
3070
  if (auto GMF = Primary->getGlobalModuleFragment()) {
3071
    for (Decl *D : getContext().getModuleInitializers(GMF)) {
3072
      if (isa<ImportDecl>(D))
3073
        continue;
3074
      assert(isa<VarDecl>(D) && "GMF initializer decl is not a var?");
3075
      EmitTopLevelDecl(D);
3076
    }
3077
  }
3078
  // Second any associated with the module, itself.
3079
  for (Decl *D : getContext().getModuleInitializers(Primary)) {
3080
    // Skip import decls, the inits for those are called explicitly.
3081
    if (isa<ImportDecl>(D))
3082
      continue;
3083
    EmitTopLevelDecl(D);
3084
  }
3085
  // Third any associated with the Privat eMOdule Fragment, if present.
3086
  if (auto PMF = Primary->getPrivateModuleFragment()) {
3087
    for (Decl *D : getContext().getModuleInitializers(PMF)) {
3088
      // Skip import decls, the inits for those are called explicitly.
3089
      if (isa<ImportDecl>(D))
3090
        continue;
3091
      assert(isa<VarDecl>(D) && "PMF initializer decl is not a var?");
3092
      EmitTopLevelDecl(D);
3093
    }
3094
  }
3095
}
3096

3097
void CodeGenModule::EmitModuleLinkOptions() {
3098
  // Collect the set of all of the modules we want to visit to emit link
3099
  // options, which is essentially the imported modules and all of their
3100
  // non-explicit child modules.
3101
  llvm::SetVector<clang::Module *> LinkModules;
3102
  llvm::SmallPtrSet<clang::Module *, 16> Visited;
3103
  SmallVector<clang::Module *, 16> Stack;
3104

3105
  // Seed the stack with imported modules.
3106
  for (Module *M : ImportedModules) {
3107
    // Do not add any link flags when an implementation TU of a module imports
3108
    // a header of that same module.
3109
    if (M->getTopLevelModuleName() == getLangOpts().CurrentModule &&
3110
        !getLangOpts().isCompilingModule())
3111
      continue;
3112
    if (Visited.insert(M).second)
3113
      Stack.push_back(M);
3114
  }
3115

3116
  // Find all of the modules to import, making a little effort to prune
3117
  // non-leaf modules.
3118
  while (!Stack.empty()) {
3119
    clang::Module *Mod = Stack.pop_back_val();
3120

3121
    bool AnyChildren = false;
3122

3123
    // Visit the submodules of this module.
3124
    for (const auto &SM : Mod->submodules()) {
3125
      // Skip explicit children; they need to be explicitly imported to be
3126
      // linked against.
3127
      if (SM->IsExplicit)
3128
        continue;
3129

3130
      if (Visited.insert(SM).second) {
3131
        Stack.push_back(SM);
3132
        AnyChildren = true;
3133
      }
3134
    }
3135

3136
    // We didn't find any children, so add this module to the list of
3137
    // modules to link against.
3138
    if (!AnyChildren) {
3139
      LinkModules.insert(Mod);
3140
    }
3141
  }
3142

3143
  // Add link options for all of the imported modules in reverse topological
3144
  // order.  We don't do anything to try to order import link flags with respect
3145
  // to linker options inserted by things like #pragma comment().
3146
  SmallVector<llvm::MDNode *, 16> MetadataArgs;
3147
  Visited.clear();
3148
  for (Module *M : LinkModules)
3149
    if (Visited.insert(M).second)
3150
      addLinkOptionsPostorder(*this, M, MetadataArgs, Visited);
3151
  std::reverse(MetadataArgs.begin(), MetadataArgs.end());
3152
  LinkerOptionsMetadata.append(MetadataArgs.begin(), MetadataArgs.end());
3153

3154
  // Add the linker options metadata flag.
3155
  auto *NMD = getModule().getOrInsertNamedMetadata("llvm.linker.options");
3156
  for (auto *MD : LinkerOptionsMetadata)
3157
    NMD->addOperand(MD);
3158
}
3159

3160
void CodeGenModule::EmitDeferred() {
3161
  // Emit deferred declare target declarations.
3162
  if (getLangOpts().OpenMP && !getLangOpts().OpenMPSimd)
3163
    getOpenMPRuntime().emitDeferredTargetDecls();
3164

3165
  // Emit code for any potentially referenced deferred decls.  Since a
3166
  // previously unused static decl may become used during the generation of code
3167
  // for a static function, iterate until no changes are made.
3168

3169
  if (!DeferredVTables.empty()) {
3170
    EmitDeferredVTables();
3171

3172
    // Emitting a vtable doesn't directly cause more vtables to
3173
    // become deferred, although it can cause functions to be
3174
    // emitted that then need those vtables.
3175
    assert(DeferredVTables.empty());
3176
  }
3177

3178
  // Emit CUDA/HIP static device variables referenced by host code only.
3179
  // Note we should not clear CUDADeviceVarODRUsedByHost since it is still
3180
  // needed for further handling.
3181
  if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice)
3182
    llvm::append_range(DeferredDeclsToEmit,
3183
                       getContext().CUDADeviceVarODRUsedByHost);
3184

3185
  // Stop if we're out of both deferred vtables and deferred declarations.
3186
  if (DeferredDeclsToEmit.empty())
3187
    return;
3188

3189
  // Grab the list of decls to emit. If EmitGlobalDefinition schedules more
3190
  // work, it will not interfere with this.
3191
  std::vector<GlobalDecl> CurDeclsToEmit;
3192
  CurDeclsToEmit.swap(DeferredDeclsToEmit);
3193

3194
  for (GlobalDecl &D : CurDeclsToEmit) {
3195
    // We should call GetAddrOfGlobal with IsForDefinition set to true in order
3196
    // to get GlobalValue with exactly the type we need, not something that
3197
    // might had been created for another decl with the same mangled name but
3198
    // different type.
3199
    llvm::GlobalValue *GV = dyn_cast<llvm::GlobalValue>(
3200
        GetAddrOfGlobal(D, ForDefinition));
3201

3202
    // In case of different address spaces, we may still get a cast, even with
3203
    // IsForDefinition equal to true. Query mangled names table to get
3204
    // GlobalValue.
3205
    if (!GV)
3206
      GV = GetGlobalValue(getMangledName(D));
3207

3208
    // Make sure GetGlobalValue returned non-null.
3209
    assert(GV);
3210

3211
    // Check to see if we've already emitted this.  This is necessary
3212
    // for a couple of reasons: first, decls can end up in the
3213
    // deferred-decls queue multiple times, and second, decls can end
3214
    // up with definitions in unusual ways (e.g. by an extern inline
3215
    // function acquiring a strong function redefinition).  Just
3216
    // ignore these cases.
3217
    if (!GV->isDeclaration())
3218
      continue;
3219

3220
    // If this is OpenMP, check if it is legal to emit this global normally.
3221
    if (LangOpts.OpenMP && OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(D))
3222
      continue;
3223

3224
    // Otherwise, emit the definition and move on to the next one.
3225
    EmitGlobalDefinition(D, GV);
3226

3227
    // If we found out that we need to emit more decls, do that recursively.
3228
    // This has the advantage that the decls are emitted in a DFS and related
3229
    // ones are close together, which is convenient for testing.
3230
    if (!DeferredVTables.empty() || !DeferredDeclsToEmit.empty()) {
3231
      EmitDeferred();
3232
      assert(DeferredVTables.empty() && DeferredDeclsToEmit.empty());
3233
    }
3234
  }
3235
}
3236

3237
void CodeGenModule::EmitVTablesOpportunistically() {
3238
  // Try to emit external vtables as available_externally if they have emitted
3239
  // all inlined virtual functions.  It runs after EmitDeferred() and therefore
3240
  // is not allowed to create new references to things that need to be emitted
3241
  // lazily. Note that it also uses fact that we eagerly emitting RTTI.
3242

3243
  assert((OpportunisticVTables.empty() || shouldOpportunisticallyEmitVTables())
3244
         && "Only emit opportunistic vtables with optimizations");
3245

3246
  for (const CXXRecordDecl *RD : OpportunisticVTables) {
3247
    assert(getVTables().isVTableExternal(RD) &&
3248
           "This queue should only contain external vtables");
3249
    if (getCXXABI().canSpeculativelyEmitVTable(RD))
3250
      VTables.GenerateClassData(RD);
3251
  }
3252
  OpportunisticVTables.clear();
3253
}
3254

3255
void CodeGenModule::EmitGlobalAnnotations() {
3256
  for (const auto& [MangledName, VD] : DeferredAnnotations) {
3257
    llvm::GlobalValue *GV = GetGlobalValue(MangledName);
3258
    if (GV)
3259
      AddGlobalAnnotations(VD, GV);
3260
  }
3261
  DeferredAnnotations.clear();
3262

3263
  if (Annotations.empty())
3264
    return;
3265

3266
  // Create a new global variable for the ConstantStruct in the Module.
3267
  llvm::Constant *Array = llvm::ConstantArray::get(llvm::ArrayType::get(
3268
    Annotations[0]->getType(), Annotations.size()), Annotations);
3269
  auto *gv = new llvm::GlobalVariable(getModule(), Array->getType(), false,
3270
                                      llvm::GlobalValue::AppendingLinkage,
3271
                                      Array, "llvm.global.annotations");
3272
  gv->setSection(AnnotationSection);
3273
}
3274

3275
llvm::Constant *CodeGenModule::EmitAnnotationString(StringRef Str) {
3276
  llvm::Constant *&AStr = AnnotationStrings[Str];
3277
  if (AStr)
3278
    return AStr;
3279

3280
  // Not found yet, create a new global.
3281
  llvm::Constant *s = llvm::ConstantDataArray::getString(getLLVMContext(), Str);
3282
  auto *gv = new llvm::GlobalVariable(
3283
      getModule(), s->getType(), true, llvm::GlobalValue::PrivateLinkage, s,
3284
      ".str", nullptr, llvm::GlobalValue::NotThreadLocal,
3285
      ConstGlobalsPtrTy->getAddressSpace());
3286
  gv->setSection(AnnotationSection);
3287
  gv->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3288
  AStr = gv;
3289
  return gv;
3290
}
3291

3292
llvm::Constant *CodeGenModule::EmitAnnotationUnit(SourceLocation Loc) {
3293
  SourceManager &SM = getContext().getSourceManager();
3294
  PresumedLoc PLoc = SM.getPresumedLoc(Loc);
3295
  if (PLoc.isValid())
3296
    return EmitAnnotationString(PLoc.getFilename());
3297
  return EmitAnnotationString(SM.getBufferName(Loc));
3298
}
3299

3300
llvm::Constant *CodeGenModule::EmitAnnotationLineNo(SourceLocation L) {
3301
  SourceManager &SM = getContext().getSourceManager();
3302
  PresumedLoc PLoc = SM.getPresumedLoc(L);
3303
  unsigned LineNo = PLoc.isValid() ? PLoc.getLine() :
3304
    SM.getExpansionLineNumber(L);
3305
  return llvm::ConstantInt::get(Int32Ty, LineNo);
3306
}
3307

3308
llvm::Constant *CodeGenModule::EmitAnnotationArgs(const AnnotateAttr *Attr) {
3309
  ArrayRef<Expr *> Exprs = {Attr->args_begin(), Attr->args_size()};
3310
  if (Exprs.empty())
3311
    return llvm::ConstantPointerNull::get(ConstGlobalsPtrTy);
3312

3313
  llvm::FoldingSetNodeID ID;
3314
  for (Expr *E : Exprs) {
3315
    ID.Add(cast<clang::ConstantExpr>(E)->getAPValueResult());
3316
  }
3317
  llvm::Constant *&Lookup = AnnotationArgs[ID.ComputeHash()];
3318
  if (Lookup)
3319
    return Lookup;
3320

3321
  llvm::SmallVector<llvm::Constant *, 4> LLVMArgs;
3322
  LLVMArgs.reserve(Exprs.size());
3323
  ConstantEmitter ConstEmiter(*this);
3324
  llvm::transform(Exprs, std::back_inserter(LLVMArgs), [&](const Expr *E) {
3325
    const auto *CE = cast<clang::ConstantExpr>(E);
3326
    return ConstEmiter.emitAbstract(CE->getBeginLoc(), CE->getAPValueResult(),
3327
                                    CE->getType());
3328
  });
3329
  auto *Struct = llvm::ConstantStruct::getAnon(LLVMArgs);
3330
  auto *GV = new llvm::GlobalVariable(getModule(), Struct->getType(), true,
3331
                                      llvm::GlobalValue::PrivateLinkage, Struct,
3332
                                      ".args");
3333
  GV->setSection(AnnotationSection);
3334
  GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3335

3336
  Lookup = GV;
3337
  return GV;
3338
}
3339

3340
llvm::Constant *CodeGenModule::EmitAnnotateAttr(llvm::GlobalValue *GV,
3341
                                                const AnnotateAttr *AA,
3342
                                                SourceLocation L) {
3343
  // Get the globals for file name, annotation, and the line number.
3344
  llvm::Constant *AnnoGV = EmitAnnotationString(AA->getAnnotation()),
3345
                 *UnitGV = EmitAnnotationUnit(L),
3346
                 *LineNoCst = EmitAnnotationLineNo(L),
3347
                 *Args = EmitAnnotationArgs(AA);
3348

3349
  llvm::Constant *GVInGlobalsAS = GV;
3350
  if (GV->getAddressSpace() !=
3351
      getDataLayout().getDefaultGlobalsAddressSpace()) {
3352
    GVInGlobalsAS = llvm::ConstantExpr::getAddrSpaceCast(
3353
        GV,
3354
        llvm::PointerType::get(
3355
            GV->getContext(), getDataLayout().getDefaultGlobalsAddressSpace()));
3356
  }
3357

3358
  // Create the ConstantStruct for the global annotation.
3359
  llvm::Constant *Fields[] = {
3360
      GVInGlobalsAS, AnnoGV, UnitGV, LineNoCst, Args,
3361
  };
3362
  return llvm::ConstantStruct::getAnon(Fields);
3363
}
3364

3365
void CodeGenModule::AddGlobalAnnotations(const ValueDecl *D,
3366
                                         llvm::GlobalValue *GV) {
3367
  assert(D->hasAttr<AnnotateAttr>() && "no annotate attribute");
3368
  // Get the struct elements for these annotations.
3369
  for (const auto *I : D->specific_attrs<AnnotateAttr>())
3370
    Annotations.push_back(EmitAnnotateAttr(GV, I, D->getLocation()));
3371
}
3372

3373
bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind, llvm::Function *Fn,
3374
                                       SourceLocation Loc) const {
3375
  const auto &NoSanitizeL = getContext().getNoSanitizeList();
3376
  // NoSanitize by function name.
3377
  if (NoSanitizeL.containsFunction(Kind, Fn->getName()))
3378
    return true;
3379
  // NoSanitize by location. Check "mainfile" prefix.
3380
  auto &SM = Context.getSourceManager();
3381
  FileEntryRef MainFile = *SM.getFileEntryRefForID(SM.getMainFileID());
3382
  if (NoSanitizeL.containsMainFile(Kind, MainFile.getName()))
3383
    return true;
3384

3385
  // Check "src" prefix.
3386
  if (Loc.isValid())
3387
    return NoSanitizeL.containsLocation(Kind, Loc);
3388
  // If location is unknown, this may be a compiler-generated function. Assume
3389
  // it's located in the main file.
3390
  return NoSanitizeL.containsFile(Kind, MainFile.getName());
3391
}
3392

3393
bool CodeGenModule::isInNoSanitizeList(SanitizerMask Kind,
3394
                                       llvm::GlobalVariable *GV,
3395
                                       SourceLocation Loc, QualType Ty,
3396
                                       StringRef Category) const {
3397
  const auto &NoSanitizeL = getContext().getNoSanitizeList();
3398
  if (NoSanitizeL.containsGlobal(Kind, GV->getName(), Category))
3399
    return true;
3400
  auto &SM = Context.getSourceManager();
3401
  if (NoSanitizeL.containsMainFile(
3402
          Kind, SM.getFileEntryRefForID(SM.getMainFileID())->getName(),
3403
          Category))
3404
    return true;
3405
  if (NoSanitizeL.containsLocation(Kind, Loc, Category))
3406
    return true;
3407

3408
  // Check global type.
3409
  if (!Ty.isNull()) {
3410
    // Drill down the array types: if global variable of a fixed type is
3411
    // not sanitized, we also don't instrument arrays of them.
3412
    while (auto AT = dyn_cast<ArrayType>(Ty.getTypePtr()))
3413
      Ty = AT->getElementType();
3414
    Ty = Ty.getCanonicalType().getUnqualifiedType();
3415
    // Only record types (classes, structs etc.) are ignored.
3416
    if (Ty->isRecordType()) {
3417
      std::string TypeStr = Ty.getAsString(getContext().getPrintingPolicy());
3418
      if (NoSanitizeL.containsType(Kind, TypeStr, Category))
3419
        return true;
3420
    }
3421
  }
3422
  return false;
3423
}
3424

3425
bool CodeGenModule::imbueXRayAttrs(llvm::Function *Fn, SourceLocation Loc,
3426
                                   StringRef Category) const {
3427
  const auto &XRayFilter = getContext().getXRayFilter();
3428
  using ImbueAttr = XRayFunctionFilter::ImbueAttribute;
3429
  auto Attr = ImbueAttr::NONE;
3430
  if (Loc.isValid())
3431
    Attr = XRayFilter.shouldImbueLocation(Loc, Category);
3432
  if (Attr == ImbueAttr::NONE)
3433
    Attr = XRayFilter.shouldImbueFunction(Fn->getName());
3434
  switch (Attr) {
3435
  case ImbueAttr::NONE:
3436
    return false;
3437
  case ImbueAttr::ALWAYS:
3438
    Fn->addFnAttr("function-instrument", "xray-always");
3439
    break;
3440
  case ImbueAttr::ALWAYS_ARG1:
3441
    Fn->addFnAttr("function-instrument", "xray-always");
3442
    Fn->addFnAttr("xray-log-args", "1");
3443
    break;
3444
  case ImbueAttr::NEVER:
3445
    Fn->addFnAttr("function-instrument", "xray-never");
3446
    break;
3447
  }
3448
  return true;
3449
}
3450

3451
ProfileList::ExclusionType
3452
CodeGenModule::isFunctionBlockedByProfileList(llvm::Function *Fn,
3453
                                              SourceLocation Loc) const {
3454
  const auto &ProfileList = getContext().getProfileList();
3455
  // If the profile list is empty, then instrument everything.
3456
  if (ProfileList.isEmpty())
3457
    return ProfileList::Allow;
3458
  CodeGenOptions::ProfileInstrKind Kind = getCodeGenOpts().getProfileInstr();
3459
  // First, check the function name.
3460
  if (auto V = ProfileList.isFunctionExcluded(Fn->getName(), Kind))
3461
    return *V;
3462
  // Next, check the source location.
3463
  if (Loc.isValid())
3464
    if (auto V = ProfileList.isLocationExcluded(Loc, Kind))
3465
      return *V;
3466
  // If location is unknown, this may be a compiler-generated function. Assume
3467
  // it's located in the main file.
3468
  auto &SM = Context.getSourceManager();
3469
  if (auto MainFile = SM.getFileEntryRefForID(SM.getMainFileID()))
3470
    if (auto V = ProfileList.isFileExcluded(MainFile->getName(), Kind))
3471
      return *V;
3472
  return ProfileList.getDefault(Kind);
3473
}
3474

3475
ProfileList::ExclusionType
3476
CodeGenModule::isFunctionBlockedFromProfileInstr(llvm::Function *Fn,
3477
                                                 SourceLocation Loc) const {
3478
  auto V = isFunctionBlockedByProfileList(Fn, Loc);
3479
  if (V != ProfileList::Allow)
3480
    return V;
3481

3482
  auto NumGroups = getCodeGenOpts().ProfileTotalFunctionGroups;
3483
  if (NumGroups > 1) {
3484
    auto Group = llvm::crc32(arrayRefFromStringRef(Fn->getName())) % NumGroups;
3485
    if (Group != getCodeGenOpts().ProfileSelectedFunctionGroup)
3486
      return ProfileList::Skip;
3487
  }
3488
  return ProfileList::Allow;
3489
}
3490

3491
bool CodeGenModule::MustBeEmitted(const ValueDecl *Global) {
3492
  // Never defer when EmitAllDecls is specified.
3493
  if (LangOpts.EmitAllDecls)
3494
    return true;
3495

3496
  const auto *VD = dyn_cast<VarDecl>(Global);
3497
  if (VD &&
3498
      ((CodeGenOpts.KeepPersistentStorageVariables &&
3499
        (VD->getStorageDuration() == SD_Static ||
3500
         VD->getStorageDuration() == SD_Thread)) ||
3501
       (CodeGenOpts.KeepStaticConsts && VD->getStorageDuration() == SD_Static &&
3502
        VD->getType().isConstQualified())))
3503
    return true;
3504

3505
  return getContext().DeclMustBeEmitted(Global);
3506
}
3507

3508
bool CodeGenModule::MayBeEmittedEagerly(const ValueDecl *Global) {
3509
  // In OpenMP 5.0 variables and function may be marked as
3510
  // device_type(host/nohost) and we should not emit them eagerly unless we sure
3511
  // that they must be emitted on the host/device. To be sure we need to have
3512
  // seen a declare target with an explicit mentioning of the function, we know
3513
  // we have if the level of the declare target attribute is -1. Note that we
3514
  // check somewhere else if we should emit this at all.
3515
  if (LangOpts.OpenMP >= 50 && !LangOpts.OpenMPSimd) {
3516
    std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =
3517
        OMPDeclareTargetDeclAttr::getActiveAttr(Global);
3518
    if (!ActiveAttr || (*ActiveAttr)->getLevel() != (unsigned)-1)
3519
      return false;
3520
  }
3521

3522
  if (const auto *FD = dyn_cast<FunctionDecl>(Global)) {
3523
    if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
3524
      // Implicit template instantiations may change linkage if they are later
3525
      // explicitly instantiated, so they should not be emitted eagerly.
3526
      return false;
3527
    // Defer until all versions have been semantically checked.
3528
    if (FD->hasAttr<TargetVersionAttr>() && !FD->isMultiVersion())
3529
      return false;
3530
  }
3531
  if (const auto *VD = dyn_cast<VarDecl>(Global)) {
3532
    if (Context.getInlineVariableDefinitionKind(VD) ==
3533
        ASTContext::InlineVariableDefinitionKind::WeakUnknown)
3534
      // A definition of an inline constexpr static data member may change
3535
      // linkage later if it's redeclared outside the class.
3536
      return false;
3537
    if (CXX20ModuleInits && VD->getOwningModule() &&
3538
        !VD->getOwningModule()->isModuleMapModule()) {
3539
      // For CXX20, module-owned initializers need to be deferred, since it is
3540
      // not known at this point if they will be run for the current module or
3541
      // as part of the initializer for an imported one.
3542
      return false;
3543
    }
3544
  }
3545
  // If OpenMP is enabled and threadprivates must be generated like TLS, delay
3546
  // codegen for global variables, because they may be marked as threadprivate.
3547
  if (LangOpts.OpenMP && LangOpts.OpenMPUseTLS &&
3548
      getContext().getTargetInfo().isTLSSupported() && isa<VarDecl>(Global) &&
3549
      !Global->getType().isConstantStorage(getContext(), false, false) &&
3550
      !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(Global))
3551
    return false;
3552

3553
  return true;
3554
}
3555

3556
ConstantAddress CodeGenModule::GetAddrOfMSGuidDecl(const MSGuidDecl *GD) {
3557
  StringRef Name = getMangledName(GD);
3558

3559
  // The UUID descriptor should be pointer aligned.
3560
  CharUnits Alignment = CharUnits::fromQuantity(PointerAlignInBytes);
3561

3562
  // Look for an existing global.
3563
  if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
3564
    return ConstantAddress(GV, GV->getValueType(), Alignment);
3565

3566
  ConstantEmitter Emitter(*this);
3567
  llvm::Constant *Init;
3568

3569
  APValue &V = GD->getAsAPValue();
3570
  if (!V.isAbsent()) {
3571
    // If possible, emit the APValue version of the initializer. In particular,
3572
    // this gets the type of the constant right.
3573
    Init = Emitter.emitForInitializer(
3574
        GD->getAsAPValue(), GD->getType().getAddressSpace(), GD->getType());
3575
  } else {
3576
    // As a fallback, directly construct the constant.
3577
    // FIXME: This may get padding wrong under esoteric struct layout rules.
3578
    // MSVC appears to create a complete type 'struct __s_GUID' that it
3579
    // presumably uses to represent these constants.
3580
    MSGuidDecl::Parts Parts = GD->getParts();
3581
    llvm::Constant *Fields[4] = {
3582
        llvm::ConstantInt::get(Int32Ty, Parts.Part1),
3583
        llvm::ConstantInt::get(Int16Ty, Parts.Part2),
3584
        llvm::ConstantInt::get(Int16Ty, Parts.Part3),
3585
        llvm::ConstantDataArray::getRaw(
3586
            StringRef(reinterpret_cast<char *>(Parts.Part4And5), 8), 8,
3587
            Int8Ty)};
3588
    Init = llvm::ConstantStruct::getAnon(Fields);
3589
  }
3590

3591
  auto *GV = new llvm::GlobalVariable(
3592
      getModule(), Init->getType(),
3593
      /*isConstant=*/true, llvm::GlobalValue::LinkOnceODRLinkage, Init, Name);
3594
  if (supportsCOMDAT())
3595
    GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
3596
  setDSOLocal(GV);
3597

3598
  if (!V.isAbsent()) {
3599
    Emitter.finalize(GV);
3600
    return ConstantAddress(GV, GV->getValueType(), Alignment);
3601
  }
3602

3603
  llvm::Type *Ty = getTypes().ConvertTypeForMem(GD->getType());
3604
  return ConstantAddress(GV, Ty, Alignment);
3605
}
3606

3607
ConstantAddress CodeGenModule::GetAddrOfUnnamedGlobalConstantDecl(
3608
    const UnnamedGlobalConstantDecl *GCD) {
3609
  CharUnits Alignment = getContext().getTypeAlignInChars(GCD->getType());
3610

3611
  llvm::GlobalVariable **Entry = nullptr;
3612
  Entry = &UnnamedGlobalConstantDeclMap[GCD];
3613
  if (*Entry)
3614
    return ConstantAddress(*Entry, (*Entry)->getValueType(), Alignment);
3615

3616
  ConstantEmitter Emitter(*this);
3617
  llvm::Constant *Init;
3618

3619
  const APValue &V = GCD->getValue();
3620

3621
  assert(!V.isAbsent());
3622
  Init = Emitter.emitForInitializer(V, GCD->getType().getAddressSpace(),
3623
                                    GCD->getType());
3624

3625
  auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
3626
                                      /*isConstant=*/true,
3627
                                      llvm::GlobalValue::PrivateLinkage, Init,
3628
                                      ".constant");
3629
  GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
3630
  GV->setAlignment(Alignment.getAsAlign());
3631

3632
  Emitter.finalize(GV);
3633

3634
  *Entry = GV;
3635
  return ConstantAddress(GV, GV->getValueType(), Alignment);
3636
}
3637

3638
ConstantAddress CodeGenModule::GetAddrOfTemplateParamObject(
3639
    const TemplateParamObjectDecl *TPO) {
3640
  StringRef Name = getMangledName(TPO);
3641
  CharUnits Alignment = getNaturalTypeAlignment(TPO->getType());
3642

3643
  if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
3644
    return ConstantAddress(GV, GV->getValueType(), Alignment);
3645

3646
  ConstantEmitter Emitter(*this);
3647
  llvm::Constant *Init = Emitter.emitForInitializer(
3648
        TPO->getValue(), TPO->getType().getAddressSpace(), TPO->getType());
3649

3650
  if (!Init) {
3651
    ErrorUnsupported(TPO, "template parameter object");
3652
    return ConstantAddress::invalid();
3653
  }
3654

3655
  llvm::GlobalValue::LinkageTypes Linkage =
3656
      isExternallyVisible(TPO->getLinkageAndVisibility().getLinkage())
3657
          ? llvm::GlobalValue::LinkOnceODRLinkage
3658
          : llvm::GlobalValue::InternalLinkage;
3659
  auto *GV = new llvm::GlobalVariable(getModule(), Init->getType(),
3660
                                      /*isConstant=*/true, Linkage, Init, Name);
3661
  setGVProperties(GV, TPO);
3662
  if (supportsCOMDAT())
3663
    GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
3664
  Emitter.finalize(GV);
3665

3666
    return ConstantAddress(GV, GV->getValueType(), Alignment);
3667
}
3668

3669
ConstantAddress CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
3670
  const AliasAttr *AA = VD->getAttr<AliasAttr>();
3671
  assert(AA && "No alias?");
3672

3673
  CharUnits Alignment = getContext().getDeclAlign(VD);
3674
  llvm::Type *DeclTy = getTypes().ConvertTypeForMem(VD->getType());
3675

3676
  // See if there is already something with the target's name in the module.
3677
  llvm::GlobalValue *Entry = GetGlobalValue(AA->getAliasee());
3678
  if (Entry)
3679
    return ConstantAddress(Entry, DeclTy, Alignment);
3680

3681
  llvm::Constant *Aliasee;
3682
  if (isa<llvm::FunctionType>(DeclTy))
3683
    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy,
3684
                                      GlobalDecl(cast<FunctionDecl>(VD)),
3685
                                      /*ForVTable=*/false);
3686
  else
3687
    Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, LangAS::Default,
3688
                                    nullptr);
3689

3690
  auto *F = cast<llvm::GlobalValue>(Aliasee);
3691
  F->setLinkage(llvm::Function::ExternalWeakLinkage);
3692
  WeakRefReferences.insert(F);
3693

3694
  return ConstantAddress(Aliasee, DeclTy, Alignment);
3695
}
3696

3697
template <typename AttrT> static bool hasImplicitAttr(const ValueDecl *D) {
3698
  if (!D)
3699
    return false;
3700
  if (auto *A = D->getAttr<AttrT>())
3701
    return A->isImplicit();
3702
  return D->isImplicit();
3703
}
3704

3705
void CodeGenModule::EmitGlobal(GlobalDecl GD) {
3706
  const auto *Global = cast<ValueDecl>(GD.getDecl());
3707

3708
  // Weak references don't produce any output by themselves.
3709
  if (Global->hasAttr<WeakRefAttr>())
3710
    return;
3711

3712
  // If this is an alias definition (which otherwise looks like a declaration)
3713
  // emit it now.
3714
  if (Global->hasAttr<AliasAttr>())
3715
    return EmitAliasDefinition(GD);
3716

3717
  // IFunc like an alias whose value is resolved at runtime by calling resolver.
3718
  if (Global->hasAttr<IFuncAttr>())
3719
    return emitIFuncDefinition(GD);
3720

3721
  // If this is a cpu_dispatch multiversion function, emit the resolver.
3722
  if (Global->hasAttr<CPUDispatchAttr>())
3723
    return emitCPUDispatchDefinition(GD);
3724

3725
  // If this is CUDA, be selective about which declarations we emit.
3726
  // Non-constexpr non-lambda implicit host device functions are not emitted
3727
  // unless they are used on device side.
3728
  if (LangOpts.CUDA) {
3729
    if (LangOpts.CUDAIsDevice) {
3730
      const auto *FD = dyn_cast<FunctionDecl>(Global);
3731
      if ((!Global->hasAttr<CUDADeviceAttr>() ||
3732
           (LangOpts.OffloadImplicitHostDeviceTemplates && FD &&
3733
            hasImplicitAttr<CUDAHostAttr>(FD) &&
3734
            hasImplicitAttr<CUDADeviceAttr>(FD) && !FD->isConstexpr() &&
3735
            !isLambdaCallOperator(FD) &&
3736
            !getContext().CUDAImplicitHostDeviceFunUsedByDevice.count(FD))) &&
3737
          !Global->hasAttr<CUDAGlobalAttr>() &&
3738
          !Global->hasAttr<CUDAConstantAttr>() &&
3739
          !Global->hasAttr<CUDASharedAttr>() &&
3740
          !Global->getType()->isCUDADeviceBuiltinSurfaceType() &&
3741
          !Global->getType()->isCUDADeviceBuiltinTextureType() &&
3742
          !(LangOpts.HIPStdPar && isa<FunctionDecl>(Global) &&
3743
            !Global->hasAttr<CUDAHostAttr>()))
3744
        return;
3745
    } else {
3746
      // We need to emit host-side 'shadows' for all global
3747
      // device-side variables because the CUDA runtime needs their
3748
      // size and host-side address in order to provide access to
3749
      // their device-side incarnations.
3750

3751
      // So device-only functions are the only things we skip.
3752
      if (isa<FunctionDecl>(Global) && !Global->hasAttr<CUDAHostAttr>() &&
3753
          Global->hasAttr<CUDADeviceAttr>())
3754
        return;
3755

3756
      assert((isa<FunctionDecl>(Global) || isa<VarDecl>(Global)) &&
3757
             "Expected Variable or Function");
3758
    }
3759
  }
3760

3761
  if (LangOpts.OpenMP) {
3762
    // If this is OpenMP, check if it is legal to emit this global normally.
3763
    if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD))
3764
      return;
3765
    if (auto *DRD = dyn_cast<OMPDeclareReductionDecl>(Global)) {
3766
      if (MustBeEmitted(Global))
3767
        EmitOMPDeclareReduction(DRD);
3768
      return;
3769
    }
3770
    if (auto *DMD = dyn_cast<OMPDeclareMapperDecl>(Global)) {
3771
      if (MustBeEmitted(Global))
3772
        EmitOMPDeclareMapper(DMD);
3773
      return;
3774
    }
3775
  }
3776

3777
  // Ignore declarations, they will be emitted on their first use.
3778
  if (const auto *FD = dyn_cast<FunctionDecl>(Global)) {
3779
    // Update deferred annotations with the latest declaration if the function
3780
    // function was already used or defined.
3781
    if (FD->hasAttr<AnnotateAttr>()) {
3782
      StringRef MangledName = getMangledName(GD);
3783
      if (GetGlobalValue(MangledName))
3784
        DeferredAnnotations[MangledName] = FD;
3785
    }
3786

3787
    // Forward declarations are emitted lazily on first use.
3788
    if (!FD->doesThisDeclarationHaveABody()) {
3789
      if (!FD->doesDeclarationForceExternallyVisibleDefinition() &&
3790
          (!FD->isMultiVersion() ||
3791
           !FD->getASTContext().getTargetInfo().getTriple().isAArch64()))
3792
        return;
3793

3794
      StringRef MangledName = getMangledName(GD);
3795

3796
      // Compute the function info and LLVM type.
3797
      const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
3798
      llvm::Type *Ty = getTypes().GetFunctionType(FI);
3799

3800
      GetOrCreateLLVMFunction(MangledName, Ty, GD, /*ForVTable=*/false,
3801
                              /*DontDefer=*/false);
3802
      return;
3803
    }
3804
  } else {
3805
    const auto *VD = cast<VarDecl>(Global);
3806
    assert(VD->isFileVarDecl() && "Cannot emit local var decl as global.");
3807
    if (VD->isThisDeclarationADefinition() != VarDecl::Definition &&
3808
        !Context.isMSStaticDataMemberInlineDefinition(VD)) {
3809
      if (LangOpts.OpenMP) {
3810
        // Emit declaration of the must-be-emitted declare target variable.
3811
        if (std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
3812
                OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {
3813

3814
          // If this variable has external storage and doesn't require special
3815
          // link handling we defer to its canonical definition.
3816
          if (VD->hasExternalStorage() &&
3817
              Res != OMPDeclareTargetDeclAttr::MT_Link)
3818
            return;
3819

3820
          bool UnifiedMemoryEnabled =
3821
              getOpenMPRuntime().hasRequiresUnifiedSharedMemory();
3822
          if ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3823
               *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3824
              !UnifiedMemoryEnabled) {
3825
            (void)GetAddrOfGlobalVar(VD);
3826
          } else {
3827
            assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||
3828
                    ((*Res == OMPDeclareTargetDeclAttr::MT_To ||
3829
                      *Res == OMPDeclareTargetDeclAttr::MT_Enter) &&
3830
                     UnifiedMemoryEnabled)) &&
3831
                   "Link clause or to clause with unified memory expected.");
3832
            (void)getOpenMPRuntime().getAddrOfDeclareTargetVar(VD);
3833
          }
3834

3835
          return;
3836
        }
3837
      }
3838
      // If this declaration may have caused an inline variable definition to
3839
      // change linkage, make sure that it's emitted.
3840
      if (Context.getInlineVariableDefinitionKind(VD) ==
3841
          ASTContext::InlineVariableDefinitionKind::Strong)
3842
        GetAddrOfGlobalVar(VD);
3843
      return;
3844
    }
3845
  }
3846

3847
  // Defer code generation to first use when possible, e.g. if this is an inline
3848
  // function. If the global must always be emitted, do it eagerly if possible
3849
  // to benefit from cache locality.
3850
  if (MustBeEmitted(Global) && MayBeEmittedEagerly(Global)) {
3851
    // Emit the definition if it can't be deferred.
3852
    EmitGlobalDefinition(GD);
3853
    addEmittedDeferredDecl(GD);
3854
    return;
3855
  }
3856

3857
  // If we're deferring emission of a C++ variable with an
3858
  // initializer, remember the order in which it appeared in the file.
3859
  if (getLangOpts().CPlusPlus && isa<VarDecl>(Global) &&
3860
      cast<VarDecl>(Global)->hasInit()) {
3861
    DelayedCXXInitPosition[Global] = CXXGlobalInits.size();
3862
    CXXGlobalInits.push_back(nullptr);
3863
  }
3864

3865
  StringRef MangledName = getMangledName(GD);
3866
  if (GetGlobalValue(MangledName) != nullptr) {
3867
    // The value has already been used and should therefore be emitted.
3868
    addDeferredDeclToEmit(GD);
3869
  } else if (MustBeEmitted(Global)) {
3870
    // The value must be emitted, but cannot be emitted eagerly.
3871
    assert(!MayBeEmittedEagerly(Global));
3872
    addDeferredDeclToEmit(GD);
3873
  } else {
3874
    // Otherwise, remember that we saw a deferred decl with this name.  The
3875
    // first use of the mangled name will cause it to move into
3876
    // DeferredDeclsToEmit.
3877
    DeferredDecls[MangledName] = GD;
3878
  }
3879
}
3880

3881
// Check if T is a class type with a destructor that's not dllimport.
3882
static bool HasNonDllImportDtor(QualType T) {
3883
  if (const auto *RT = T->getBaseElementTypeUnsafe()->getAs<RecordType>())
3884
    if (CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(RT->getDecl()))
3885
      if (RD->getDestructor() && !RD->getDestructor()->hasAttr<DLLImportAttr>())
3886
        return true;
3887

3888
  return false;
3889
}
3890

3891
namespace {
3892
  struct FunctionIsDirectlyRecursive
3893
      : public ConstStmtVisitor<FunctionIsDirectlyRecursive, bool> {
3894
    const StringRef Name;
3895
    const Builtin::Context &BI;
3896
    FunctionIsDirectlyRecursive(StringRef N, const Builtin::Context &C)
3897
        : Name(N), BI(C) {}
3898

3899
    bool VisitCallExpr(const CallExpr *E) {
3900
      const FunctionDecl *FD = E->getDirectCallee();
3901
      if (!FD)
3902
        return false;
3903
      AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
3904
      if (Attr && Name == Attr->getLabel())
3905
        return true;
3906
      unsigned BuiltinID = FD->getBuiltinID();
3907
      if (!BuiltinID || !BI.isLibFunction(BuiltinID))
3908
        return false;
3909
      StringRef BuiltinName = BI.getName(BuiltinID);
3910
      if (BuiltinName.starts_with("__builtin_") &&
3911
          Name == BuiltinName.slice(strlen("__builtin_"), StringRef::npos)) {
3912
        return true;
3913
      }
3914
      return false;
3915
    }
3916

3917
    bool VisitStmt(const Stmt *S) {
3918
      for (const Stmt *Child : S->children())
3919
        if (Child && this->Visit(Child))
3920
          return true;
3921
      return false;
3922
    }
3923
  };
3924

3925
  // Make sure we're not referencing non-imported vars or functions.
3926
  struct DLLImportFunctionVisitor
3927
      : public RecursiveASTVisitor<DLLImportFunctionVisitor> {
3928
    bool SafeToInline = true;
3929

3930
    bool shouldVisitImplicitCode() const { return true; }
3931

3932
    bool VisitVarDecl(VarDecl *VD) {
3933
      if (VD->getTLSKind()) {
3934
        // A thread-local variable cannot be imported.
3935
        SafeToInline = false;
3936
        return SafeToInline;
3937
      }
3938

3939
      // A variable definition might imply a destructor call.
3940
      if (VD->isThisDeclarationADefinition())
3941
        SafeToInline = !HasNonDllImportDtor(VD->getType());
3942

3943
      return SafeToInline;
3944
    }
3945

3946
    bool VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
3947
      if (const auto *D = E->getTemporary()->getDestructor())
3948
        SafeToInline = D->hasAttr<DLLImportAttr>();
3949
      return SafeToInline;
3950
    }
3951

3952
    bool VisitDeclRefExpr(DeclRefExpr *E) {
3953
      ValueDecl *VD = E->getDecl();
3954
      if (isa<FunctionDecl>(VD))
3955
        SafeToInline = VD->hasAttr<DLLImportAttr>();
3956
      else if (VarDecl *V = dyn_cast<VarDecl>(VD))
3957
        SafeToInline = !V->hasGlobalStorage() || V->hasAttr<DLLImportAttr>();
3958
      return SafeToInline;
3959
    }
3960

3961
    bool VisitCXXConstructExpr(CXXConstructExpr *E) {
3962
      SafeToInline = E->getConstructor()->hasAttr<DLLImportAttr>();
3963
      return SafeToInline;
3964
    }
3965

3966
    bool VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
3967
      CXXMethodDecl *M = E->getMethodDecl();
3968
      if (!M) {
3969
        // Call through a pointer to member function. This is safe to inline.
3970
        SafeToInline = true;
3971
      } else {
3972
        SafeToInline = M->hasAttr<DLLImportAttr>();
3973
      }
3974
      return SafeToInline;
3975
    }
3976

3977
    bool VisitCXXDeleteExpr(CXXDeleteExpr *E) {
3978
      SafeToInline = E->getOperatorDelete()->hasAttr<DLLImportAttr>();
3979
      return SafeToInline;
3980
    }
3981

3982
    bool VisitCXXNewExpr(CXXNewExpr *E) {
3983
      SafeToInline = E->getOperatorNew()->hasAttr<DLLImportAttr>();
3984
      return SafeToInline;
3985
    }
3986
  };
3987
}
3988

3989
// isTriviallyRecursive - Check if this function calls another
3990
// decl that, because of the asm attribute or the other decl being a builtin,
3991
// ends up pointing to itself.
3992
bool
3993
CodeGenModule::isTriviallyRecursive(const FunctionDecl *FD) {
3994
  StringRef Name;
3995
  if (getCXXABI().getMangleContext().shouldMangleDeclName(FD)) {
3996
    // asm labels are a special kind of mangling we have to support.
3997
    AsmLabelAttr *Attr = FD->getAttr<AsmLabelAttr>();
3998
    if (!Attr)
3999
      return false;
4000
    Name = Attr->getLabel();
4001
  } else {
4002
    Name = FD->getName();
4003
  }
4004

4005
  FunctionIsDirectlyRecursive Walker(Name, Context.BuiltinInfo);
4006
  const Stmt *Body = FD->getBody();
4007
  return Body ? Walker.Visit(Body) : false;
4008
}
4009

4010
bool CodeGenModule::shouldEmitFunction(GlobalDecl GD) {
4011
  if (getFunctionLinkage(GD) != llvm::Function::AvailableExternallyLinkage)
4012
    return true;
4013

4014
  const auto *F = cast<FunctionDecl>(GD.getDecl());
4015
  if (CodeGenOpts.OptimizationLevel == 0 && !F->hasAttr<AlwaysInlineAttr>())
4016
    return false;
4017

4018
  // We don't import function bodies from other named module units since that
4019
  // behavior may break ABI compatibility of the current unit.
4020
  if (const Module *M = F->getOwningModule();
4021
      M && M->getTopLevelModule()->isNamedModule() &&
4022
      getContext().getCurrentNamedModule() != M->getTopLevelModule()) {
4023
    // There are practices to mark template member function as always-inline
4024
    // and mark the template as extern explicit instantiation but not give
4025
    // the definition for member function. So we have to emit the function
4026
    // from explicitly instantiation with always-inline.
4027
    //
4028
    // See https://github.com/llvm/llvm-project/issues/86893 for details.
4029
    //
4030
    // TODO: Maybe it is better to give it a warning if we call a non-inline
4031
    // function from other module units which is marked as always-inline.
4032
    if (!F->isTemplateInstantiation() || !F->hasAttr<AlwaysInlineAttr>()) {
4033
      return false;
4034
    }
4035
  }
4036

4037
  if (F->hasAttr<NoInlineAttr>())
4038
    return false;
4039

4040
  if (F->hasAttr<DLLImportAttr>() && !F->hasAttr<AlwaysInlineAttr>()) {
4041
    // Check whether it would be safe to inline this dllimport function.
4042
    DLLImportFunctionVisitor Visitor;
4043
    Visitor.TraverseFunctionDecl(const_cast<FunctionDecl*>(F));
4044
    if (!Visitor.SafeToInline)
4045
      return false;
4046

4047
    if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(F)) {
4048
      // Implicit destructor invocations aren't captured in the AST, so the
4049
      // check above can't see them. Check for them manually here.
4050
      for (const Decl *Member : Dtor->getParent()->decls())
4051
        if (isa<FieldDecl>(Member))
4052
          if (HasNonDllImportDtor(cast<FieldDecl>(Member)->getType()))
4053
            return false;
4054
      for (const CXXBaseSpecifier &B : Dtor->getParent()->bases())
4055
        if (HasNonDllImportDtor(B.getType()))
4056
          return false;
4057
    }
4058
  }
4059

4060
  // Inline builtins declaration must be emitted. They often are fortified
4061
  // functions.
4062
  if (F->isInlineBuiltinDeclaration())
4063
    return true;
4064

4065
  // PR9614. Avoid cases where the source code is lying to us. An available
4066
  // externally function should have an equivalent function somewhere else,
4067
  // but a function that calls itself through asm label/`__builtin_` trickery is
4068
  // clearly not equivalent to the real implementation.
4069
  // This happens in glibc's btowc and in some configure checks.
4070
  return !isTriviallyRecursive(F);
4071
}
4072

4073
bool CodeGenModule::shouldOpportunisticallyEmitVTables() {
4074
  return CodeGenOpts.OptimizationLevel > 0;
4075
}
4076

4077
void CodeGenModule::EmitMultiVersionFunctionDefinition(GlobalDecl GD,
4078
                                                       llvm::GlobalValue *GV) {
4079
  const auto *FD = cast<FunctionDecl>(GD.getDecl());
4080

4081
  if (FD->isCPUSpecificMultiVersion()) {
4082
    auto *Spec = FD->getAttr<CPUSpecificAttr>();
4083
    for (unsigned I = 0; I < Spec->cpus_size(); ++I)
4084
      EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
4085
  } else if (auto *TC = FD->getAttr<TargetClonesAttr>()) {
4086
    for (unsigned I = 0; I < TC->featuresStrs_size(); ++I)
4087
      // AArch64 favors the default target version over the clone if any.
4088
      if ((!TC->isDefaultVersion(I) || !getTarget().getTriple().isAArch64()) &&
4089
          TC->isFirstOfVersion(I))
4090
        EmitGlobalFunctionDefinition(GD.getWithMultiVersionIndex(I), nullptr);
4091
    // Ensure that the resolver function is also emitted.
4092
    GetOrCreateMultiVersionResolver(GD);
4093
  } else
4094
    EmitGlobalFunctionDefinition(GD, GV);
4095

4096
  // Defer the resolver emission until we can reason whether the TU
4097
  // contains a default target version implementation.
4098
  if (FD->isTargetVersionMultiVersion())
4099
    AddDeferredMultiVersionResolverToEmit(GD);
4100
}
4101

4102
void CodeGenModule::EmitGlobalDefinition(GlobalDecl GD, llvm::GlobalValue *GV) {
4103
  const auto *D = cast<ValueDecl>(GD.getDecl());
4104

4105
  PrettyStackTraceDecl CrashInfo(const_cast<ValueDecl *>(D), D->getLocation(),
4106
                                 Context.getSourceManager(),
4107
                                 "Generating code for declaration");
4108

4109
  if (const auto *FD = dyn_cast<FunctionDecl>(D)) {
4110
    // At -O0, don't generate IR for functions with available_externally
4111
    // linkage.
4112
    if (!shouldEmitFunction(GD))
4113
      return;
4114

4115
    llvm::TimeTraceScope TimeScope("CodeGen Function", [&]() {
4116
      std::string Name;
4117
      llvm::raw_string_ostream OS(Name);
4118
      FD->getNameForDiagnostic(OS, getContext().getPrintingPolicy(),
4119
                               /*Qualified=*/true);
4120
      return Name;
4121
    });
4122

4123
    if (const auto *Method = dyn_cast<CXXMethodDecl>(D)) {
4124
      // Make sure to emit the definition(s) before we emit the thunks.
4125
      // This is necessary for the generation of certain thunks.
4126
      if (isa<CXXConstructorDecl>(Method) || isa<CXXDestructorDecl>(Method))
4127
        ABI->emitCXXStructor(GD);
4128
      else if (FD->isMultiVersion())
4129
        EmitMultiVersionFunctionDefinition(GD, GV);
4130
      else
4131
        EmitGlobalFunctionDefinition(GD, GV);
4132

4133
      if (Method->isVirtual())
4134
        getVTables().EmitThunks(GD);
4135

4136
      return;
4137
    }
4138

4139
    if (FD->isMultiVersion())
4140
      return EmitMultiVersionFunctionDefinition(GD, GV);
4141
    return EmitGlobalFunctionDefinition(GD, GV);
4142
  }
4143

4144
  if (const auto *VD = dyn_cast<VarDecl>(D))
4145
    return EmitGlobalVarDefinition(VD, !VD->hasDefinition());
4146

4147
  llvm_unreachable("Invalid argument to EmitGlobalDefinition()");
4148
}
4149

4150
static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
4151
                                                      llvm::Function *NewFn);
4152

4153
static unsigned
4154
TargetMVPriority(const TargetInfo &TI,
4155
                 const CodeGenFunction::MultiVersionResolverOption &RO) {
4156
  unsigned Priority = 0;
4157
  unsigned NumFeatures = 0;
4158
  for (StringRef Feat : RO.Conditions.Features) {
4159
    Priority = std::max(Priority, TI.multiVersionSortPriority(Feat));
4160
    NumFeatures++;
4161
  }
4162

4163
  if (!RO.Conditions.Architecture.empty())
4164
    Priority = std::max(
4165
        Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture));
4166

4167
  Priority += TI.multiVersionFeatureCost() * NumFeatures;
4168

4169
  return Priority;
4170
}
4171

4172
// Multiversion functions should be at most 'WeakODRLinkage' so that a different
4173
// TU can forward declare the function without causing problems.  Particularly
4174
// in the cases of CPUDispatch, this causes issues. This also makes sure we
4175
// work with internal linkage functions, so that the same function name can be
4176
// used with internal linkage in multiple TUs.
4177
llvm::GlobalValue::LinkageTypes getMultiversionLinkage(CodeGenModule &CGM,
4178
                                                       GlobalDecl GD) {
4179
  const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
4180
  if (FD->getFormalLinkage() == Linkage::Internal)
4181
    return llvm::GlobalValue::InternalLinkage;
4182
  return llvm::GlobalValue::WeakODRLinkage;
4183
}
4184

4185
static FunctionDecl *createDefaultTargetVersionFrom(const FunctionDecl *FD) {
4186
  auto *DeclCtx = const_cast<DeclContext *>(FD->getDeclContext());
4187
  TypeSourceInfo *TInfo = FD->getTypeSourceInfo();
4188
  StorageClass SC = FD->getStorageClass();
4189
  DeclarationName Name = FD->getNameInfo().getName();
4190

4191
  FunctionDecl *NewDecl =
4192
      FunctionDecl::Create(FD->getASTContext(), DeclCtx, FD->getBeginLoc(),
4193
                           FD->getEndLoc(), Name, TInfo->getType(), TInfo, SC);
4194

4195
  NewDecl->setIsMultiVersion();
4196
  NewDecl->addAttr(TargetVersionAttr::CreateImplicit(
4197
      NewDecl->getASTContext(), "default", NewDecl->getSourceRange()));
4198

4199
  return NewDecl;
4200
}
4201

4202
void CodeGenModule::emitMultiVersionFunctions() {
4203
  std::vector<GlobalDecl> MVFuncsToEmit;
4204
  MultiVersionFuncs.swap(MVFuncsToEmit);
4205
  for (GlobalDecl GD : MVFuncsToEmit) {
4206
    const auto *FD = cast<FunctionDecl>(GD.getDecl());
4207
    assert(FD && "Expected a FunctionDecl");
4208

4209
    auto createFunction = [&](const FunctionDecl *Decl, unsigned MVIdx = 0) {
4210
      GlobalDecl CurGD{Decl->isDefined() ? Decl->getDefinition() : Decl, MVIdx};
4211
      StringRef MangledName = getMangledName(CurGD);
4212
      llvm::Constant *Func = GetGlobalValue(MangledName);
4213
      if (!Func) {
4214
        if (Decl->isDefined()) {
4215
          EmitGlobalFunctionDefinition(CurGD, nullptr);
4216
          Func = GetGlobalValue(MangledName);
4217
        } else {
4218
          const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(CurGD);
4219
          llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
4220
          Func = GetAddrOfFunction(CurGD, Ty, /*ForVTable=*/false,
4221
                                   /*DontDefer=*/false, ForDefinition);
4222
        }
4223
        assert(Func && "This should have just been created");
4224
      }
4225
      return cast<llvm::Function>(Func);
4226
    };
4227

4228
    bool HasDefaultDecl = !FD->isTargetVersionMultiVersion();
4229
    bool ShouldEmitResolver =
4230
        !getContext().getTargetInfo().getTriple().isAArch64();
4231
    SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
4232

4233
    getContext().forEachMultiversionedFunctionVersion(
4234
        FD, [&](const FunctionDecl *CurFD) {
4235
          llvm::SmallVector<StringRef, 8> Feats;
4236

4237
          if (const auto *TA = CurFD->getAttr<TargetAttr>()) {
4238
            TA->getAddedFeatures(Feats);
4239
            llvm::Function *Func = createFunction(CurFD);
4240
            Options.emplace_back(Func, TA->getArchitecture(), Feats);
4241
          } else if (const auto *TVA = CurFD->getAttr<TargetVersionAttr>()) {
4242
            bool HasDefaultDef = TVA->isDefaultVersion() &&
4243
                                 CurFD->doesThisDeclarationHaveABody();
4244
            HasDefaultDecl |= TVA->isDefaultVersion();
4245
            ShouldEmitResolver |= (CurFD->isUsed() || HasDefaultDef);
4246
            TVA->getFeatures(Feats);
4247
            llvm::Function *Func = createFunction(CurFD);
4248
            Options.emplace_back(Func, /*Architecture*/ "", Feats);
4249
          } else if (const auto *TC = CurFD->getAttr<TargetClonesAttr>()) {
4250
            ShouldEmitResolver |= CurFD->doesThisDeclarationHaveABody();
4251
            for (unsigned I = 0; I < TC->featuresStrs_size(); ++I) {
4252
              if (!TC->isFirstOfVersion(I))
4253
                continue;
4254

4255
              llvm::Function *Func = createFunction(CurFD, I);
4256
              StringRef Architecture;
4257
              Feats.clear();
4258
              if (getTarget().getTriple().isAArch64())
4259
                TC->getFeatures(Feats, I);
4260
              else {
4261
                StringRef Version = TC->getFeatureStr(I);
4262
                if (Version.starts_with("arch="))
4263
                  Architecture = Version.drop_front(sizeof("arch=") - 1);
4264
                else if (Version != "default")
4265
                  Feats.push_back(Version);
4266
              }
4267
              Options.emplace_back(Func, Architecture, Feats);
4268
            }
4269
          } else
4270
            llvm_unreachable("unexpected MultiVersionKind");
4271
        });
4272

4273
    if (!ShouldEmitResolver)
4274
      continue;
4275

4276
    if (!HasDefaultDecl) {
4277
      FunctionDecl *NewFD = createDefaultTargetVersionFrom(FD);
4278
      llvm::Function *Func = createFunction(NewFD);
4279
      llvm::SmallVector<StringRef, 1> Feats;
4280
      Options.emplace_back(Func, /*Architecture*/ "", Feats);
4281
    }
4282

4283
    llvm::Constant *ResolverConstant = GetOrCreateMultiVersionResolver(GD);
4284
    if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(ResolverConstant)) {
4285
      ResolverConstant = IFunc->getResolver();
4286
      if (FD->isTargetClonesMultiVersion() &&
4287
          !getTarget().getTriple().isAArch64()) {
4288
        std::string MangledName = getMangledNameImpl(
4289
            *this, GD, FD, /*OmitMultiVersionMangling=*/true);
4290
        if (!GetGlobalValue(MangledName + ".ifunc")) {
4291
          const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4292
          llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
4293
          // In prior versions of Clang, the mangling for ifuncs incorrectly
4294
          // included an .ifunc suffix. This alias is generated for backward
4295
          // compatibility. It is deprecated, and may be removed in the future.
4296
          auto *Alias = llvm::GlobalAlias::create(
4297
              DeclTy, 0, getMultiversionLinkage(*this, GD),
4298
              MangledName + ".ifunc", IFunc, &getModule());
4299
          SetCommonAttributes(FD, Alias);
4300
        }
4301
      }
4302
    }
4303
    llvm::Function *ResolverFunc = cast<llvm::Function>(ResolverConstant);
4304

4305
    ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD));
4306

4307
    if (!ResolverFunc->hasLocalLinkage() && supportsCOMDAT())
4308
      ResolverFunc->setComdat(
4309
          getModule().getOrInsertComdat(ResolverFunc->getName()));
4310

4311
    const TargetInfo &TI = getTarget();
4312
    llvm::stable_sort(
4313
        Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
4314
                       const CodeGenFunction::MultiVersionResolverOption &RHS) {
4315
          return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
4316
        });
4317
    CodeGenFunction CGF(*this);
4318
    CGF.EmitMultiVersionResolver(ResolverFunc, Options);
4319
  }
4320

4321
  // Ensure that any additions to the deferred decls list caused by emitting a
4322
  // variant are emitted.  This can happen when the variant itself is inline and
4323
  // calls a function without linkage.
4324
  if (!MVFuncsToEmit.empty())
4325
    EmitDeferred();
4326

4327
  // Ensure that any additions to the multiversion funcs list from either the
4328
  // deferred decls or the multiversion functions themselves are emitted.
4329
  if (!MultiVersionFuncs.empty())
4330
    emitMultiVersionFunctions();
4331
}
4332

4333
void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
4334
  const auto *FD = cast<FunctionDecl>(GD.getDecl());
4335
  assert(FD && "Not a FunctionDecl?");
4336
  assert(FD->isCPUDispatchMultiVersion() && "Not a multiversion function?");
4337
  const auto *DD = FD->getAttr<CPUDispatchAttr>();
4338
  assert(DD && "Not a cpu_dispatch Function?");
4339

4340
  const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4341
  llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
4342

4343
  StringRef ResolverName = getMangledName(GD);
4344
  UpdateMultiVersionNames(GD, FD, ResolverName);
4345

4346
  llvm::Type *ResolverType;
4347
  GlobalDecl ResolverGD;
4348
  if (getTarget().supportsIFunc()) {
4349
    ResolverType = llvm::FunctionType::get(
4350
        llvm::PointerType::get(DeclTy,
4351
                               getTypes().getTargetAddressSpace(FD->getType())),
4352
        false);
4353
  }
4354
  else {
4355
    ResolverType = DeclTy;
4356
    ResolverGD = GD;
4357
  }
4358

4359
  auto *ResolverFunc = cast<llvm::Function>(GetOrCreateLLVMFunction(
4360
      ResolverName, ResolverType, ResolverGD, /*ForVTable=*/false));
4361
  ResolverFunc->setLinkage(getMultiversionLinkage(*this, GD));
4362
  if (supportsCOMDAT())
4363
    ResolverFunc->setComdat(
4364
        getModule().getOrInsertComdat(ResolverFunc->getName()));
4365

4366
  SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
4367
  const TargetInfo &Target = getTarget();
4368
  unsigned Index = 0;
4369
  for (const IdentifierInfo *II : DD->cpus()) {
4370
    // Get the name of the target function so we can look it up/create it.
4371
    std::string MangledName = getMangledNameImpl(*this, GD, FD, true) +
4372
                              getCPUSpecificMangling(*this, II->getName());
4373

4374
    llvm::Constant *Func = GetGlobalValue(MangledName);
4375

4376
    if (!Func) {
4377
      GlobalDecl ExistingDecl = Manglings.lookup(MangledName);
4378
      if (ExistingDecl.getDecl() &&
4379
          ExistingDecl.getDecl()->getAsFunction()->isDefined()) {
4380
        EmitGlobalFunctionDefinition(ExistingDecl, nullptr);
4381
        Func = GetGlobalValue(MangledName);
4382
      } else {
4383
        if (!ExistingDecl.getDecl())
4384
          ExistingDecl = GD.getWithMultiVersionIndex(Index);
4385

4386
      Func = GetOrCreateLLVMFunction(
4387
          MangledName, DeclTy, ExistingDecl,
4388
          /*ForVTable=*/false, /*DontDefer=*/true,
4389
          /*IsThunk=*/false, llvm::AttributeList(), ForDefinition);
4390
      }
4391
    }
4392

4393
    llvm::SmallVector<StringRef, 32> Features;
4394
    Target.getCPUSpecificCPUDispatchFeatures(II->getName(), Features);
4395
    llvm::transform(Features, Features.begin(),
4396
                    [](StringRef Str) { return Str.substr(1); });
4397
    llvm::erase_if(Features, [&Target](StringRef Feat) {
4398
      return !Target.validateCpuSupports(Feat);
4399
    });
4400
    Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features);
4401
    ++Index;
4402
  }
4403

4404
  llvm::stable_sort(
4405
      Options, [](const CodeGenFunction::MultiVersionResolverOption &LHS,
4406
                  const CodeGenFunction::MultiVersionResolverOption &RHS) {
4407
        return llvm::X86::getCpuSupportsMask(LHS.Conditions.Features) >
4408
               llvm::X86::getCpuSupportsMask(RHS.Conditions.Features);
4409
      });
4410

4411
  // If the list contains multiple 'default' versions, such as when it contains
4412
  // 'pentium' and 'generic', don't emit the call to the generic one (since we
4413
  // always run on at least a 'pentium'). We do this by deleting the 'least
4414
  // advanced' (read, lowest mangling letter).
4415
  while (Options.size() > 1 &&
4416
         llvm::all_of(llvm::X86::getCpuSupportsMask(
4417
                          (Options.end() - 2)->Conditions.Features),
4418
                      [](auto X) { return X == 0; })) {
4419
    StringRef LHSName = (Options.end() - 2)->Function->getName();
4420
    StringRef RHSName = (Options.end() - 1)->Function->getName();
4421
    if (LHSName.compare(RHSName) < 0)
4422
      Options.erase(Options.end() - 2);
4423
    else
4424
      Options.erase(Options.end() - 1);
4425
  }
4426

4427
  CodeGenFunction CGF(*this);
4428
  CGF.EmitMultiVersionResolver(ResolverFunc, Options);
4429

4430
  if (getTarget().supportsIFunc()) {
4431
    llvm::GlobalValue::LinkageTypes Linkage = getMultiversionLinkage(*this, GD);
4432
    auto *IFunc = cast<llvm::GlobalValue>(GetOrCreateMultiVersionResolver(GD));
4433

4434
    // Fix up function declarations that were created for cpu_specific before
4435
    // cpu_dispatch was known
4436
    if (!isa<llvm::GlobalIFunc>(IFunc)) {
4437
      assert(cast<llvm::Function>(IFunc)->isDeclaration());
4438
      auto *GI = llvm::GlobalIFunc::create(DeclTy, 0, Linkage, "", ResolverFunc,
4439
                                           &getModule());
4440
      GI->takeName(IFunc);
4441
      IFunc->replaceAllUsesWith(GI);
4442
      IFunc->eraseFromParent();
4443
      IFunc = GI;
4444
    }
4445

4446
    std::string AliasName = getMangledNameImpl(
4447
        *this, GD, FD, /*OmitMultiVersionMangling=*/true);
4448
    llvm::Constant *AliasFunc = GetGlobalValue(AliasName);
4449
    if (!AliasFunc) {
4450
      auto *GA = llvm::GlobalAlias::create(DeclTy, 0, Linkage, AliasName, IFunc,
4451
                                           &getModule());
4452
      SetCommonAttributes(GD, GA);
4453
    }
4454
  }
4455
}
4456

4457
/// Adds a declaration to the list of multi version functions if not present.
4458
void CodeGenModule::AddDeferredMultiVersionResolverToEmit(GlobalDecl GD) {
4459
  const auto *FD = cast<FunctionDecl>(GD.getDecl());
4460
  assert(FD && "Not a FunctionDecl?");
4461

4462
  if (FD->isTargetVersionMultiVersion() || FD->isTargetClonesMultiVersion()) {
4463
    std::string MangledName =
4464
        getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
4465
    if (!DeferredResolversToEmit.insert(MangledName).second)
4466
      return;
4467
  }
4468
  MultiVersionFuncs.push_back(GD);
4469
}
4470

4471
/// If a dispatcher for the specified mangled name is not in the module, create
4472
/// and return an llvm Function with the specified type.
4473
llvm::Constant *CodeGenModule::GetOrCreateMultiVersionResolver(GlobalDecl GD) {
4474
  const auto *FD = cast<FunctionDecl>(GD.getDecl());
4475
  assert(FD && "Not a FunctionDecl?");
4476

4477
  std::string MangledName =
4478
      getMangledNameImpl(*this, GD, FD, /*OmitMultiVersionMangling=*/true);
4479

4480
  // Holds the name of the resolver, in ifunc mode this is the ifunc (which has
4481
  // a separate resolver).
4482
  std::string ResolverName = MangledName;
4483
  if (getTarget().supportsIFunc()) {
4484
    switch (FD->getMultiVersionKind()) {
4485
    case MultiVersionKind::None:
4486
      llvm_unreachable("unexpected MultiVersionKind::None for resolver");
4487
    case MultiVersionKind::Target:
4488
    case MultiVersionKind::CPUSpecific:
4489
    case MultiVersionKind::CPUDispatch:
4490
      ResolverName += ".ifunc";
4491
      break;
4492
    case MultiVersionKind::TargetClones:
4493
    case MultiVersionKind::TargetVersion:
4494
      break;
4495
    }
4496
  } else if (FD->isTargetMultiVersion()) {
4497
    ResolverName += ".resolver";
4498
  }
4499

4500
  // If the resolver has already been created, just return it.
4501
  if (llvm::GlobalValue *ResolverGV = GetGlobalValue(ResolverName))
4502
    return ResolverGV;
4503

4504
  const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
4505
  llvm::FunctionType *DeclTy = getTypes().GetFunctionType(FI);
4506

4507
  // The resolver needs to be created. For target and target_clones, defer
4508
  // creation until the end of the TU.
4509
  if (FD->isTargetMultiVersion() || FD->isTargetClonesMultiVersion())
4510
    AddDeferredMultiVersionResolverToEmit(GD);
4511

4512
  // For cpu_specific, don't create an ifunc yet because we don't know if the
4513
  // cpu_dispatch will be emitted in this translation unit.
4514
  if (getTarget().supportsIFunc() && !FD->isCPUSpecificMultiVersion()) {
4515
    llvm::Type *ResolverType = llvm::FunctionType::get(
4516
        llvm::PointerType::get(DeclTy,
4517
                               getTypes().getTargetAddressSpace(FD->getType())),
4518
        false);
4519
    llvm::Constant *Resolver = GetOrCreateLLVMFunction(
4520
        MangledName + ".resolver", ResolverType, GlobalDecl{},
4521
        /*ForVTable=*/false);
4522
    llvm::GlobalIFunc *GIF =
4523
        llvm::GlobalIFunc::create(DeclTy, 0, getMultiversionLinkage(*this, GD),
4524
                                  "", Resolver, &getModule());
4525
    GIF->setName(ResolverName);
4526
    SetCommonAttributes(FD, GIF);
4527

4528
    return GIF;
4529
  }
4530

4531
  llvm::Constant *Resolver = GetOrCreateLLVMFunction(
4532
      ResolverName, DeclTy, GlobalDecl{}, /*ForVTable=*/false);
4533
  assert(isa<llvm::GlobalValue>(Resolver) &&
4534
         "Resolver should be created for the first time");
4535
  SetCommonAttributes(FD, cast<llvm::GlobalValue>(Resolver));
4536
  return Resolver;
4537
}
4538

4539
bool CodeGenModule::shouldDropDLLAttribute(const Decl *D,
4540
                                           const llvm::GlobalValue *GV) const {
4541
  auto SC = GV->getDLLStorageClass();
4542
  if (SC == llvm::GlobalValue::DefaultStorageClass)
4543
    return false;
4544
  const Decl *MRD = D->getMostRecentDecl();
4545
  return (((SC == llvm::GlobalValue::DLLImportStorageClass &&
4546
            !MRD->hasAttr<DLLImportAttr>()) ||
4547
           (SC == llvm::GlobalValue::DLLExportStorageClass &&
4548
            !MRD->hasAttr<DLLExportAttr>())) &&
4549
          !shouldMapVisibilityToDLLExport(cast<NamedDecl>(MRD)));
4550
}
4551

4552
/// GetOrCreateLLVMFunction - If the specified mangled name is not in the
4553
/// module, create and return an llvm Function with the specified type. If there
4554
/// is something in the module with the specified name, return it potentially
4555
/// bitcasted to the right type.
4556
///
4557
/// If D is non-null, it specifies a decl that correspond to this.  This is used
4558
/// to set the attributes on the function when it is first created.
4559
llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
4560
    StringRef MangledName, llvm::Type *Ty, GlobalDecl GD, bool ForVTable,
4561
    bool DontDefer, bool IsThunk, llvm::AttributeList ExtraAttrs,
4562
    ForDefinition_t IsForDefinition) {
4563
  const Decl *D = GD.getDecl();
4564

4565
  // Any attempts to use a MultiVersion function should result in retrieving
4566
  // the iFunc instead. Name Mangling will handle the rest of the changes.
4567
  if (const FunctionDecl *FD = cast_or_null<FunctionDecl>(D)) {
4568
    // For the device mark the function as one that should be emitted.
4569
    if (getLangOpts().OpenMPIsTargetDevice && OpenMPRuntime &&
4570
        !OpenMPRuntime->markAsGlobalTarget(GD) && FD->isDefined() &&
4571
        !DontDefer && !IsForDefinition) {
4572
      if (const FunctionDecl *FDDef = FD->getDefinition()) {
4573
        GlobalDecl GDDef;
4574
        if (const auto *CD = dyn_cast<CXXConstructorDecl>(FDDef))
4575
          GDDef = GlobalDecl(CD, GD.getCtorType());
4576
        else if (const auto *DD = dyn_cast<CXXDestructorDecl>(FDDef))
4577
          GDDef = GlobalDecl(DD, GD.getDtorType());
4578
        else
4579
          GDDef = GlobalDecl(FDDef);
4580
        EmitGlobal(GDDef);
4581
      }
4582
    }
4583

4584
    if (FD->isMultiVersion()) {
4585
      UpdateMultiVersionNames(GD, FD, MangledName);
4586
      if (FD->getASTContext().getTargetInfo().getTriple().isAArch64() &&
4587
          !FD->isUsed())
4588
        AddDeferredMultiVersionResolverToEmit(GD);
4589
      else if (!IsForDefinition)
4590
        return GetOrCreateMultiVersionResolver(GD);
4591
    }
4592
  }
4593

4594
  // Lookup the entry, lazily creating it if necessary.
4595
  llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
4596
  if (Entry) {
4597
    if (WeakRefReferences.erase(Entry)) {
4598
      const FunctionDecl *FD = cast_or_null<FunctionDecl>(D);
4599
      if (FD && !FD->hasAttr<WeakAttr>())
4600
        Entry->setLinkage(llvm::Function::ExternalLinkage);
4601
    }
4602

4603
    // Handle dropped DLL attributes.
4604
    if (D && shouldDropDLLAttribute(D, Entry)) {
4605
      Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
4606
      setDSOLocal(Entry);
4607
    }
4608

4609
    // If there are two attempts to define the same mangled name, issue an
4610
    // error.
4611
    if (IsForDefinition && !Entry->isDeclaration()) {
4612
      GlobalDecl OtherGD;
4613
      // Check that GD is not yet in DiagnosedConflictingDefinitions is required
4614
      // to make sure that we issue an error only once.
4615
      if (lookupRepresentativeDecl(MangledName, OtherGD) &&
4616
          (GD.getCanonicalDecl().getDecl() !=
4617
           OtherGD.getCanonicalDecl().getDecl()) &&
4618
          DiagnosedConflictingDefinitions.insert(GD).second) {
4619
        getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name)
4620
            << MangledName;
4621
        getDiags().Report(OtherGD.getDecl()->getLocation(),
4622
                          diag::note_previous_definition);
4623
      }
4624
    }
4625

4626
    if ((isa<llvm::Function>(Entry) || isa<llvm::GlobalAlias>(Entry)) &&
4627
        (Entry->getValueType() == Ty)) {
4628
      return Entry;
4629
    }
4630

4631
    // Make sure the result is of the correct type.
4632
    // (If function is requested for a definition, we always need to create a new
4633
    // function, not just return a bitcast.)
4634
    if (!IsForDefinition)
4635
      return Entry;
4636
  }
4637

4638
  // This function doesn't have a complete type (for example, the return
4639
  // type is an incomplete struct). Use a fake type instead, and make
4640
  // sure not to try to set attributes.
4641
  bool IsIncompleteFunction = false;
4642

4643
  llvm::FunctionType *FTy;
4644
  if (isa<llvm::FunctionType>(Ty)) {
4645
    FTy = cast<llvm::FunctionType>(Ty);
4646
  } else {
4647
    FTy = llvm::FunctionType::get(VoidTy, false);
4648
    IsIncompleteFunction = true;
4649
  }
4650

4651
  llvm::Function *F =
4652
      llvm::Function::Create(FTy, llvm::Function::ExternalLinkage,
4653
                             Entry ? StringRef() : MangledName, &getModule());
4654

4655
  // Store the declaration associated with this function so it is potentially
4656
  // updated by further declarations or definitions and emitted at the end.
4657
  if (D && D->hasAttr<AnnotateAttr>())
4658
    DeferredAnnotations[MangledName] = cast<ValueDecl>(D);
4659

4660
  // If we already created a function with the same mangled name (but different
4661
  // type) before, take its name and add it to the list of functions to be
4662
  // replaced with F at the end of CodeGen.
4663
  //
4664
  // This happens if there is a prototype for a function (e.g. "int f()") and
4665
  // then a definition of a different type (e.g. "int f(int x)").
4666
  if (Entry) {
4667
    F->takeName(Entry);
4668

4669
    // This might be an implementation of a function without a prototype, in
4670
    // which case, try to do special replacement of calls which match the new
4671
    // prototype.  The really key thing here is that we also potentially drop
4672
    // arguments from the call site so as to make a direct call, which makes the
4673
    // inliner happier and suppresses a number of optimizer warnings (!) about
4674
    // dropping arguments.
4675
    if (!Entry->use_empty()) {
4676
      ReplaceUsesOfNonProtoTypeWithRealFunction(Entry, F);
4677
      Entry->removeDeadConstantUsers();
4678
    }
4679

4680
    addGlobalValReplacement(Entry, F);
4681
  }
4682

4683
  assert(F->getName() == MangledName && "name was uniqued!");
4684
  if (D)
4685
    SetFunctionAttributes(GD, F, IsIncompleteFunction, IsThunk);
4686
  if (ExtraAttrs.hasFnAttrs()) {
4687
    llvm::AttrBuilder B(F->getContext(), ExtraAttrs.getFnAttrs());
4688
    F->addFnAttrs(B);
4689
  }
4690

4691
  if (!DontDefer) {
4692
    // All MSVC dtors other than the base dtor are linkonce_odr and delegate to
4693
    // each other bottoming out with the base dtor.  Therefore we emit non-base
4694
    // dtors on usage, even if there is no dtor definition in the TU.
4695
    if (isa_and_nonnull<CXXDestructorDecl>(D) &&
4696
        getCXXABI().useThunkForDtorVariant(cast<CXXDestructorDecl>(D),
4697
                                           GD.getDtorType()))
4698
      addDeferredDeclToEmit(GD);
4699

4700
    // This is the first use or definition of a mangled name.  If there is a
4701
    // deferred decl with this name, remember that we need to emit it at the end
4702
    // of the file.
4703
    auto DDI = DeferredDecls.find(MangledName);
4704
    if (DDI != DeferredDecls.end()) {
4705
      // Move the potentially referenced deferred decl to the
4706
      // DeferredDeclsToEmit list, and remove it from DeferredDecls (since we
4707
      // don't need it anymore).
4708
      addDeferredDeclToEmit(DDI->second);
4709
      DeferredDecls.erase(DDI);
4710

4711
      // Otherwise, there are cases we have to worry about where we're
4712
      // using a declaration for which we must emit a definition but where
4713
      // we might not find a top-level definition:
4714
      //   - member functions defined inline in their classes
4715
      //   - friend functions defined inline in some class
4716
      //   - special member functions with implicit definitions
4717
      // If we ever change our AST traversal to walk into class methods,
4718
      // this will be unnecessary.
4719
      //
4720
      // We also don't emit a definition for a function if it's going to be an
4721
      // entry in a vtable, unless it's already marked as used.
4722
    } else if (getLangOpts().CPlusPlus && D) {
4723
      // Look for a declaration that's lexically in a record.
4724
      for (const auto *FD = cast<FunctionDecl>(D)->getMostRecentDecl(); FD;
4725
           FD = FD->getPreviousDecl()) {
4726
        if (isa<CXXRecordDecl>(FD->getLexicalDeclContext())) {
4727
          if (FD->doesThisDeclarationHaveABody()) {
4728
            addDeferredDeclToEmit(GD.getWithDecl(FD));
4729
            break;
4730
          }
4731
        }
4732
      }
4733
    }
4734
  }
4735

4736
  // Make sure the result is of the requested type.
4737
  if (!IsIncompleteFunction) {
4738
    assert(F->getFunctionType() == Ty);
4739
    return F;
4740
  }
4741

4742
  return F;
4743
}
4744

4745
/// GetAddrOfFunction - Return the address of the given function.  If Ty is
4746
/// non-null, then this function will use the specified type if it has to
4747
/// create it (this occurs when we see a definition of the function).
4748
llvm::Constant *
4749
CodeGenModule::GetAddrOfFunction(GlobalDecl GD, llvm::Type *Ty, bool ForVTable,
4750
                                 bool DontDefer,
4751
                                 ForDefinition_t IsForDefinition) {
4752
  // If there was no specific requested type, just convert it now.
4753
  if (!Ty) {
4754
    const auto *FD = cast<FunctionDecl>(GD.getDecl());
4755
    Ty = getTypes().ConvertType(FD->getType());
4756
  }
4757

4758
  // Devirtualized destructor calls may come through here instead of via
4759
  // getAddrOfCXXStructor. Make sure we use the MS ABI base destructor instead
4760
  // of the complete destructor when necessary.
4761
  if (const auto *DD = dyn_cast<CXXDestructorDecl>(GD.getDecl())) {
4762
    if (getTarget().getCXXABI().isMicrosoft() &&
4763
        GD.getDtorType() == Dtor_Complete &&
4764
        DD->getParent()->getNumVBases() == 0)
4765
      GD = GlobalDecl(DD, Dtor_Base);
4766
  }
4767

4768
  StringRef MangledName = getMangledName(GD);
4769
  auto *F = GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer,
4770
                                    /*IsThunk=*/false, llvm::AttributeList(),
4771
                                    IsForDefinition);
4772
  // Returns kernel handle for HIP kernel stub function.
4773
  if (LangOpts.CUDA && !LangOpts.CUDAIsDevice &&
4774
      cast<FunctionDecl>(GD.getDecl())->hasAttr<CUDAGlobalAttr>()) {
4775
    auto *Handle = getCUDARuntime().getKernelHandle(
4776
        cast<llvm::Function>(F->stripPointerCasts()), GD);
4777
    if (IsForDefinition)
4778
      return F;
4779
    return Handle;
4780
  }
4781
  return F;
4782
}
4783

4784
llvm::Constant *CodeGenModule::GetFunctionStart(const ValueDecl *Decl) {
4785
  llvm::GlobalValue *F =
4786
      cast<llvm::GlobalValue>(GetAddrOfFunction(Decl)->stripPointerCasts());
4787

4788
  return llvm::NoCFIValue::get(F);
4789
}
4790

4791
static const FunctionDecl *
4792
GetRuntimeFunctionDecl(ASTContext &C, StringRef Name) {
4793
  TranslationUnitDecl *TUDecl = C.getTranslationUnitDecl();
4794
  DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
4795

4796
  IdentifierInfo &CII = C.Idents.get(Name);
4797
  for (const auto *Result : DC->lookup(&CII))
4798
    if (const auto *FD = dyn_cast<FunctionDecl>(Result))
4799
      return FD;
4800

4801
  if (!C.getLangOpts().CPlusPlus)
4802
    return nullptr;
4803

4804
  // Demangle the premangled name from getTerminateFn()
4805
  IdentifierInfo &CXXII =
4806
      (Name == "_ZSt9terminatev" || Name == "?terminate@@YAXXZ")
4807
          ? C.Idents.get("terminate")
4808
          : C.Idents.get(Name);
4809

4810
  for (const auto &N : {"__cxxabiv1", "std"}) {
4811
    IdentifierInfo &NS = C.Idents.get(N);
4812
    for (const auto *Result : DC->lookup(&NS)) {
4813
      const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(Result);
4814
      if (auto *LSD = dyn_cast<LinkageSpecDecl>(Result))
4815
        for (const auto *Result : LSD->lookup(&NS))
4816
          if ((ND = dyn_cast<NamespaceDecl>(Result)))
4817
            break;
4818

4819
      if (ND)
4820
        for (const auto *Result : ND->lookup(&CXXII))
4821
          if (const auto *FD = dyn_cast<FunctionDecl>(Result))
4822
            return FD;
4823
    }
4824
  }
4825

4826
  return nullptr;
4827
}
4828

4829
/// CreateRuntimeFunction - Create a new runtime function with the specified
4830
/// type and name.
4831
llvm::FunctionCallee
4832
CodeGenModule::CreateRuntimeFunction(llvm::FunctionType *FTy, StringRef Name,
4833
                                     llvm::AttributeList ExtraAttrs, bool Local,
4834
                                     bool AssumeConvergent) {
4835
  if (AssumeConvergent) {
4836
    ExtraAttrs =
4837
        ExtraAttrs.addFnAttribute(VMContext, llvm::Attribute::Convergent);
4838
  }
4839

4840
  llvm::Constant *C =
4841
      GetOrCreateLLVMFunction(Name, FTy, GlobalDecl(), /*ForVTable=*/false,
4842
                              /*DontDefer=*/false, /*IsThunk=*/false,
4843
                              ExtraAttrs);
4844

4845
  if (auto *F = dyn_cast<llvm::Function>(C)) {
4846
    if (F->empty()) {
4847
      F->setCallingConv(getRuntimeCC());
4848

4849
      // In Windows Itanium environments, try to mark runtime functions
4850
      // dllimport. For Mingw and MSVC, don't. We don't really know if the user
4851
      // will link their standard library statically or dynamically. Marking
4852
      // functions imported when they are not imported can cause linker errors
4853
      // and warnings.
4854
      if (!Local && getTriple().isWindowsItaniumEnvironment() &&
4855
          !getCodeGenOpts().LTOVisibilityPublicStd) {
4856
        const FunctionDecl *FD = GetRuntimeFunctionDecl(Context, Name);
4857
        if (!FD || FD->hasAttr<DLLImportAttr>()) {
4858
          F->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
4859
          F->setLinkage(llvm::GlobalValue::ExternalLinkage);
4860
        }
4861
      }
4862
      setDSOLocal(F);
4863
      // FIXME: We should use CodeGenModule::SetLLVMFunctionAttributes() instead
4864
      // of trying to approximate the attributes using the LLVM function
4865
      // signature. This requires revising the API of CreateRuntimeFunction().
4866
      markRegisterParameterAttributes(F);
4867
    }
4868
  }
4869

4870
  return {FTy, C};
4871
}
4872

4873
/// GetOrCreateLLVMGlobal - If the specified mangled name is not in the module,
4874
/// create and return an llvm GlobalVariable with the specified type and address
4875
/// space. If there is something in the module with the specified name, return
4876
/// it potentially bitcasted to the right type.
4877
///
4878
/// If D is non-null, it specifies a decl that correspond to this.  This is used
4879
/// to set the attributes on the global when it is first created.
4880
///
4881
/// If IsForDefinition is true, it is guaranteed that an actual global with
4882
/// type Ty will be returned, not conversion of a variable with the same
4883
/// mangled name but some other type.
4884
llvm::Constant *
4885
CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName, llvm::Type *Ty,
4886
                                     LangAS AddrSpace, const VarDecl *D,
4887
                                     ForDefinition_t IsForDefinition) {
4888
  // Lookup the entry, lazily creating it if necessary.
4889
  llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
4890
  unsigned TargetAS = getContext().getTargetAddressSpace(AddrSpace);
4891
  if (Entry) {
4892
    if (WeakRefReferences.erase(Entry)) {
4893
      if (D && !D->hasAttr<WeakAttr>())
4894
        Entry->setLinkage(llvm::Function::ExternalLinkage);
4895
    }
4896

4897
    // Handle dropped DLL attributes.
4898
    if (D && shouldDropDLLAttribute(D, Entry))
4899
      Entry->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
4900

4901
    if (LangOpts.OpenMP && !LangOpts.OpenMPSimd && D)
4902
      getOpenMPRuntime().registerTargetGlobalVariable(D, Entry);
4903

4904
    if (Entry->getValueType() == Ty && Entry->getAddressSpace() == TargetAS)
4905
      return Entry;
4906

4907
    // If there are two attempts to define the same mangled name, issue an
4908
    // error.
4909
    if (IsForDefinition && !Entry->isDeclaration()) {
4910
      GlobalDecl OtherGD;
4911
      const VarDecl *OtherD;
4912

4913
      // Check that D is not yet in DiagnosedConflictingDefinitions is required
4914
      // to make sure that we issue an error only once.
4915
      if (D && lookupRepresentativeDecl(MangledName, OtherGD) &&
4916
          (D->getCanonicalDecl() != OtherGD.getCanonicalDecl().getDecl()) &&
4917
          (OtherD = dyn_cast<VarDecl>(OtherGD.getDecl())) &&
4918
          OtherD->hasInit() &&
4919
          DiagnosedConflictingDefinitions.insert(D).second) {
4920
        getDiags().Report(D->getLocation(), diag::err_duplicate_mangled_name)
4921
            << MangledName;
4922
        getDiags().Report(OtherGD.getDecl()->getLocation(),
4923
                          diag::note_previous_definition);
4924
      }
4925
    }
4926

4927
    // Make sure the result is of the correct type.
4928
    if (Entry->getType()->getAddressSpace() != TargetAS)
4929
      return llvm::ConstantExpr::getAddrSpaceCast(
4930
          Entry, llvm::PointerType::get(Ty->getContext(), TargetAS));
4931

4932
    // (If global is requested for a definition, we always need to create a new
4933
    // global, not just return a bitcast.)
4934
    if (!IsForDefinition)
4935
      return Entry;
4936
  }
4937

4938
  auto DAddrSpace = GetGlobalVarAddressSpace(D);
4939

4940
  auto *GV = new llvm::GlobalVariable(
4941
      getModule(), Ty, false, llvm::GlobalValue::ExternalLinkage, nullptr,
4942
      MangledName, nullptr, llvm::GlobalVariable::NotThreadLocal,
4943
      getContext().getTargetAddressSpace(DAddrSpace));
4944

4945
  // If we already created a global with the same mangled name (but different
4946
  // type) before, take its name and remove it from its parent.
4947
  if (Entry) {
4948
    GV->takeName(Entry);
4949

4950
    if (!Entry->use_empty()) {
4951
      Entry->replaceAllUsesWith(GV);
4952
    }
4953

4954
    Entry->eraseFromParent();
4955
  }
4956

4957
  // This is the first use or definition of a mangled name.  If there is a
4958
  // deferred decl with this name, remember that we need to emit it at the end
4959
  // of the file.
4960
  auto DDI = DeferredDecls.find(MangledName);
4961
  if (DDI != DeferredDecls.end()) {
4962
    // Move the potentially referenced deferred decl to the DeferredDeclsToEmit
4963
    // list, and remove it from DeferredDecls (since we don't need it anymore).
4964
    addDeferredDeclToEmit(DDI->second);
4965
    DeferredDecls.erase(DDI);
4966
  }
4967

4968
  // Handle things which are present even on external declarations.
4969
  if (D) {
4970
    if (LangOpts.OpenMP && !LangOpts.OpenMPSimd)
4971
      getOpenMPRuntime().registerTargetGlobalVariable(D, GV);
4972

4973
    // FIXME: This code is overly simple and should be merged with other global
4974
    // handling.
4975
    GV->setConstant(D->getType().isConstantStorage(getContext(), false, false));
4976

4977
    GV->setAlignment(getContext().getDeclAlign(D).getAsAlign());
4978

4979
    setLinkageForGV(GV, D);
4980

4981
    if (D->getTLSKind()) {
4982
      if (D->getTLSKind() == VarDecl::TLS_Dynamic)
4983
        CXXThreadLocals.push_back(D);
4984
      setTLSMode(GV, *D);
4985
    }
4986

4987
    setGVProperties(GV, D);
4988

4989
    // If required by the ABI, treat declarations of static data members with
4990
    // inline initializers as definitions.
4991
    if (getContext().isMSStaticDataMemberInlineDefinition(D)) {
4992
      EmitGlobalVarDefinition(D);
4993
    }
4994

4995
    // Emit section information for extern variables.
4996
    if (D->hasExternalStorage()) {
4997
      if (const SectionAttr *SA = D->getAttr<SectionAttr>())
4998
        GV->setSection(SA->getName());
4999
    }
5000

5001
    // Handle XCore specific ABI requirements.
5002
    if (getTriple().getArch() == llvm::Triple::xcore &&
5003
        D->getLanguageLinkage() == CLanguageLinkage &&
5004
        D->getType().isConstant(Context) &&
5005
        isExternallyVisible(D->getLinkageAndVisibility().getLinkage()))
5006
      GV->setSection(".cp.rodata");
5007

5008
    // Handle code model attribute
5009
    if (const auto *CMA = D->getAttr<CodeModelAttr>())
5010
      GV->setCodeModel(CMA->getModel());
5011

5012
    // Check if we a have a const declaration with an initializer, we may be
5013
    // able to emit it as available_externally to expose it's value to the
5014
    // optimizer.
5015
    if (Context.getLangOpts().CPlusPlus && GV->hasExternalLinkage() &&
5016
        D->getType().isConstQualified() && !GV->hasInitializer() &&
5017
        !D->hasDefinition() && D->hasInit() && !D->hasAttr<DLLImportAttr>()) {
5018
      const auto *Record =
5019
          Context.getBaseElementType(D->getType())->getAsCXXRecordDecl();
5020
      bool HasMutableFields = Record && Record->hasMutableFields();
5021
      if (!HasMutableFields) {
5022
        const VarDecl *InitDecl;
5023
        const Expr *InitExpr = D->getAnyInitializer(InitDecl);
5024
        if (InitExpr) {
5025
          ConstantEmitter emitter(*this);
5026
          llvm::Constant *Init = emitter.tryEmitForInitializer(*InitDecl);
5027
          if (Init) {
5028
            auto *InitType = Init->getType();
5029
            if (GV->getValueType() != InitType) {
5030
              // The type of the initializer does not match the definition.
5031
              // This happens when an initializer has a different type from
5032
              // the type of the global (because of padding at the end of a
5033
              // structure for instance).
5034
              GV->setName(StringRef());
5035
              // Make a new global with the correct type, this is now guaranteed
5036
              // to work.
5037
              auto *NewGV = cast<llvm::GlobalVariable>(
5038
                  GetAddrOfGlobalVar(D, InitType, IsForDefinition)
5039
                      ->stripPointerCasts());
5040

5041
              // Erase the old global, since it is no longer used.
5042
              GV->eraseFromParent();
5043
              GV = NewGV;
5044
            } else {
5045
              GV->setInitializer(Init);
5046
              GV->setConstant(true);
5047
              GV->setLinkage(llvm::GlobalValue::AvailableExternallyLinkage);
5048
            }
5049
            emitter.finalize(GV);
5050
          }
5051
        }
5052
      }
5053
    }
5054
  }
5055

5056
  if (D &&
5057
      D->isThisDeclarationADefinition(Context) == VarDecl::DeclarationOnly) {
5058
    getTargetCodeGenInfo().setTargetAttributes(D, GV, *this);
5059
    // External HIP managed variables needed to be recorded for transformation
5060
    // in both device and host compilations.
5061
    if (getLangOpts().CUDA && D && D->hasAttr<HIPManagedAttr>() &&
5062
        D->hasExternalStorage())
5063
      getCUDARuntime().handleVarRegistration(D, *GV);
5064
  }
5065

5066
  if (D)
5067
    SanitizerMD->reportGlobal(GV, *D);
5068

5069
  LangAS ExpectedAS =
5070
      D ? D->getType().getAddressSpace()
5071
        : (LangOpts.OpenCL ? LangAS::opencl_global : LangAS::Default);
5072
  assert(getContext().getTargetAddressSpace(ExpectedAS) == TargetAS);
5073
  if (DAddrSpace != ExpectedAS) {
5074
    return getTargetCodeGenInfo().performAddrSpaceCast(
5075
        *this, GV, DAddrSpace, ExpectedAS,
5076
        llvm::PointerType::get(getLLVMContext(), TargetAS));
5077
  }
5078

5079
  return GV;
5080
}
5081

5082
llvm::Constant *
5083
CodeGenModule::GetAddrOfGlobal(GlobalDecl GD, ForDefinition_t IsForDefinition) {
5084
  const Decl *D = GD.getDecl();
5085

5086
  if (isa<CXXConstructorDecl>(D) || isa<CXXDestructorDecl>(D))
5087
    return getAddrOfCXXStructor(GD, /*FnInfo=*/nullptr, /*FnType=*/nullptr,
5088
                                /*DontDefer=*/false, IsForDefinition);
5089

5090
  if (isa<CXXMethodDecl>(D)) {
5091
    auto FInfo =
5092
        &getTypes().arrangeCXXMethodDeclaration(cast<CXXMethodDecl>(D));
5093
    auto Ty = getTypes().GetFunctionType(*FInfo);
5094
    return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
5095
                             IsForDefinition);
5096
  }
5097

5098
  if (isa<FunctionDecl>(D)) {
5099
    const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
5100
    llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
5101
    return GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, /*DontDefer=*/false,
5102
                             IsForDefinition);
5103
  }
5104

5105
  return GetAddrOfGlobalVar(cast<VarDecl>(D), /*Ty=*/nullptr, IsForDefinition);
5106
}
5107

5108
llvm::GlobalVariable *CodeGenModule::CreateOrReplaceCXXRuntimeVariable(
5109
    StringRef Name, llvm::Type *Ty, llvm::GlobalValue::LinkageTypes Linkage,
5110
    llvm::Align Alignment) {
5111
  llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name);
5112
  llvm::GlobalVariable *OldGV = nullptr;
5113

5114
  if (GV) {
5115
    // Check if the variable has the right type.
5116
    if (GV->getValueType() == Ty)
5117
      return GV;
5118

5119
    // Because C++ name mangling, the only way we can end up with an already
5120
    // existing global with the same name is if it has been declared extern "C".
5121
    assert(GV->isDeclaration() && "Declaration has wrong type!");
5122
    OldGV = GV;
5123
  }
5124

5125
  // Create a new variable.
5126
  GV = new llvm::GlobalVariable(getModule(), Ty, /*isConstant=*/true,
5127
                                Linkage, nullptr, Name);
5128

5129
  if (OldGV) {
5130
    // Replace occurrences of the old variable if needed.
5131
    GV->takeName(OldGV);
5132

5133
    if (!OldGV->use_empty()) {
5134
      OldGV->replaceAllUsesWith(GV);
5135
    }
5136

5137
    OldGV->eraseFromParent();
5138
  }
5139

5140
  if (supportsCOMDAT() && GV->isWeakForLinker() &&
5141
      !GV->hasAvailableExternallyLinkage())
5142
    GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
5143

5144
  GV->setAlignment(Alignment);
5145

5146
  return GV;
5147
}
5148

5149
/// GetAddrOfGlobalVar - Return the llvm::Constant for the address of the
5150
/// given global variable.  If Ty is non-null and if the global doesn't exist,
5151
/// then it will be created with the specified type instead of whatever the
5152
/// normal requested type would be. If IsForDefinition is true, it is guaranteed
5153
/// that an actual global with type Ty will be returned, not conversion of a
5154
/// variable with the same mangled name but some other type.
5155
llvm::Constant *CodeGenModule::GetAddrOfGlobalVar(const VarDecl *D,
5156
                                                  llvm::Type *Ty,
5157
                                           ForDefinition_t IsForDefinition) {
5158
  assert(D->hasGlobalStorage() && "Not a global variable");
5159
  QualType ASTTy = D->getType();
5160
  if (!Ty)
5161
    Ty = getTypes().ConvertTypeForMem(ASTTy);
5162

5163
  StringRef MangledName = getMangledName(D);
5164
  return GetOrCreateLLVMGlobal(MangledName, Ty, ASTTy.getAddressSpace(), D,
5165
                               IsForDefinition);
5166
}
5167

5168
/// CreateRuntimeVariable - Create a new runtime global variable with the
5169
/// specified type and name.
5170
llvm::Constant *
5171
CodeGenModule::CreateRuntimeVariable(llvm::Type *Ty,
5172
                                     StringRef Name) {
5173
  LangAS AddrSpace = getContext().getLangOpts().OpenCL ? LangAS::opencl_global
5174
                                                       : LangAS::Default;
5175
  auto *Ret = GetOrCreateLLVMGlobal(Name, Ty, AddrSpace, nullptr);
5176
  setDSOLocal(cast<llvm::GlobalValue>(Ret->stripPointerCasts()));
5177
  return Ret;
5178
}
5179

5180
void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) {
5181
  assert(!D->getInit() && "Cannot emit definite definitions here!");
5182

5183
  StringRef MangledName = getMangledName(D);
5184
  llvm::GlobalValue *GV = GetGlobalValue(MangledName);
5185

5186
  // We already have a definition, not declaration, with the same mangled name.
5187
  // Emitting of declaration is not required (and actually overwrites emitted
5188
  // definition).
5189
  if (GV && !GV->isDeclaration())
5190
    return;
5191

5192
  // If we have not seen a reference to this variable yet, place it into the
5193
  // deferred declarations table to be emitted if needed later.
5194
  if (!MustBeEmitted(D) && !GV) {
5195
      DeferredDecls[MangledName] = D;
5196
      return;
5197
  }
5198

5199
  // The tentative definition is the only definition.
5200
  EmitGlobalVarDefinition(D);
5201
}
5202

5203
void CodeGenModule::EmitExternalDeclaration(const DeclaratorDecl *D) {
5204
  if (auto const *V = dyn_cast<const VarDecl>(D))
5205
    EmitExternalVarDeclaration(V);
5206
  if (auto const *FD = dyn_cast<const FunctionDecl>(D))
5207
    EmitExternalFunctionDeclaration(FD);
5208
}
5209

5210
CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const {
5211
  return Context.toCharUnitsFromBits(
5212
      getDataLayout().getTypeStoreSizeInBits(Ty));
5213
}
5214

5215
LangAS CodeGenModule::GetGlobalVarAddressSpace(const VarDecl *D) {
5216
  if (LangOpts.OpenCL) {
5217
    LangAS AS = D ? D->getType().getAddressSpace() : LangAS::opencl_global;
5218
    assert(AS == LangAS::opencl_global ||
5219
           AS == LangAS::opencl_global_device ||
5220
           AS == LangAS::opencl_global_host ||
5221
           AS == LangAS::opencl_constant ||
5222
           AS == LangAS::opencl_local ||
5223
           AS >= LangAS::FirstTargetAddressSpace);
5224
    return AS;
5225
  }
5226

5227
  if (LangOpts.SYCLIsDevice &&
5228
      (!D || D->getType().getAddressSpace() == LangAS::Default))
5229
    return LangAS::sycl_global;
5230

5231
  if (LangOpts.CUDA && LangOpts.CUDAIsDevice) {
5232
    if (D) {
5233
      if (D->hasAttr<CUDAConstantAttr>())
5234
        return LangAS::cuda_constant;
5235
      if (D->hasAttr<CUDASharedAttr>())
5236
        return LangAS::cuda_shared;
5237
      if (D->hasAttr<CUDADeviceAttr>())
5238
        return LangAS::cuda_device;
5239
      if (D->getType().isConstQualified())
5240
        return LangAS::cuda_constant;
5241
    }
5242
    return LangAS::cuda_device;
5243
  }
5244

5245
  if (LangOpts.OpenMP) {
5246
    LangAS AS;
5247
    if (OpenMPRuntime->hasAllocateAttributeForGlobalVar(D, AS))
5248
      return AS;
5249
  }
5250
  return getTargetCodeGenInfo().getGlobalVarAddressSpace(*this, D);
5251
}
5252

5253
LangAS CodeGenModule::GetGlobalConstantAddressSpace() const {
5254
  // OpenCL v1.2 s6.5.3: a string literal is in the constant address space.
5255
  if (LangOpts.OpenCL)
5256
    return LangAS::opencl_constant;
5257
  if (LangOpts.SYCLIsDevice)
5258
    return LangAS::sycl_global;
5259
  if (LangOpts.HIP && LangOpts.CUDAIsDevice && getTriple().isSPIRV())
5260
    // For HIPSPV map literals to cuda_device (maps to CrossWorkGroup in SPIR-V)
5261
    // instead of default AS (maps to Generic in SPIR-V). Otherwise, we end up
5262
    // with OpVariable instructions with Generic storage class which is not
5263
    // allowed (SPIR-V V1.6 s3.42.8). Also, mapping literals to SPIR-V
5264
    // UniformConstant storage class is not viable as pointers to it may not be
5265
    // casted to Generic pointers which are used to model HIP's "flat" pointers.
5266
    return LangAS::cuda_device;
5267
  if (auto AS = getTarget().getConstantAddressSpace())
5268
    return *AS;
5269
  return LangAS::Default;
5270
}
5271

5272
// In address space agnostic languages, string literals are in default address
5273
// space in AST. However, certain targets (e.g. amdgcn) request them to be
5274
// emitted in constant address space in LLVM IR. To be consistent with other
5275
// parts of AST, string literal global variables in constant address space
5276
// need to be casted to default address space before being put into address
5277
// map and referenced by other part of CodeGen.
5278
// In OpenCL, string literals are in constant address space in AST, therefore
5279
// they should not be casted to default address space.
5280
static llvm::Constant *
5281
castStringLiteralToDefaultAddressSpace(CodeGenModule &CGM,
5282
                                       llvm::GlobalVariable *GV) {
5283
  llvm::Constant *Cast = GV;
5284
  if (!CGM.getLangOpts().OpenCL) {
5285
    auto AS = CGM.GetGlobalConstantAddressSpace();
5286
    if (AS != LangAS::Default)
5287
      Cast = CGM.getTargetCodeGenInfo().performAddrSpaceCast(
5288
          CGM, GV, AS, LangAS::Default,
5289
          llvm::PointerType::get(
5290
              CGM.getLLVMContext(),
5291
              CGM.getContext().getTargetAddressSpace(LangAS::Default)));
5292
  }
5293
  return Cast;
5294
}
5295

5296
template<typename SomeDecl>
5297
void CodeGenModule::MaybeHandleStaticInExternC(const SomeDecl *D,
5298
                                               llvm::GlobalValue *GV) {
5299
  if (!getLangOpts().CPlusPlus)
5300
    return;
5301

5302
  // Must have 'used' attribute, or else inline assembly can't rely on
5303
  // the name existing.
5304
  if (!D->template hasAttr<UsedAttr>())
5305
    return;
5306

5307
  // Must have internal linkage and an ordinary name.
5308
  if (!D->getIdentifier() || D->getFormalLinkage() != Linkage::Internal)
5309
    return;
5310

5311
  // Must be in an extern "C" context. Entities declared directly within
5312
  // a record are not extern "C" even if the record is in such a context.
5313
  const SomeDecl *First = D->getFirstDecl();
5314
  if (First->getDeclContext()->isRecord() || !First->isInExternCContext())
5315
    return;
5316

5317
  // OK, this is an internal linkage entity inside an extern "C" linkage
5318
  // specification. Make a note of that so we can give it the "expected"
5319
  // mangled name if nothing else is using that name.
5320
  std::pair<StaticExternCMap::iterator, bool> R =
5321
      StaticExternCValues.insert(std::make_pair(D->getIdentifier(), GV));
5322

5323
  // If we have multiple internal linkage entities with the same name
5324
  // in extern "C" regions, none of them gets that name.
5325
  if (!R.second)
5326
    R.first->second = nullptr;
5327
}
5328

5329
static bool shouldBeInCOMDAT(CodeGenModule &CGM, const Decl &D) {
5330
  if (!CGM.supportsCOMDAT())
5331
    return false;
5332

5333
  if (D.hasAttr<SelectAnyAttr>())
5334
    return true;
5335

5336
  GVALinkage Linkage;
5337
  if (auto *VD = dyn_cast<VarDecl>(&D))
5338
    Linkage = CGM.getContext().GetGVALinkageForVariable(VD);
5339
  else
5340
    Linkage = CGM.getContext().GetGVALinkageForFunction(cast<FunctionDecl>(&D));
5341

5342
  switch (Linkage) {
5343
  case GVA_Internal:
5344
  case GVA_AvailableExternally:
5345
  case GVA_StrongExternal:
5346
    return false;
5347
  case GVA_DiscardableODR:
5348
  case GVA_StrongODR:
5349
    return true;
5350
  }
5351
  llvm_unreachable("No such linkage");
5352
}
5353

5354
bool CodeGenModule::supportsCOMDAT() const {
5355
  return getTriple().supportsCOMDAT();
5356
}
5357

5358
void CodeGenModule::maybeSetTrivialComdat(const Decl &D,
5359
                                          llvm::GlobalObject &GO) {
5360
  if (!shouldBeInCOMDAT(*this, D))
5361
    return;
5362
  GO.setComdat(TheModule.getOrInsertComdat(GO.getName()));
5363
}
5364

5365
/// Pass IsTentative as true if you want to create a tentative definition.
5366
void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
5367
                                            bool IsTentative) {
5368
  // OpenCL global variables of sampler type are translated to function calls,
5369
  // therefore no need to be translated.
5370
  QualType ASTTy = D->getType();
5371
  if (getLangOpts().OpenCL && ASTTy->isSamplerT())
5372
    return;
5373

5374
  // If this is OpenMP device, check if it is legal to emit this global
5375
  // normally.
5376
  if (LangOpts.OpenMPIsTargetDevice && OpenMPRuntime &&
5377
      OpenMPRuntime->emitTargetGlobalVariable(D))
5378
    return;
5379

5380
  llvm::TrackingVH<llvm::Constant> Init;
5381
  bool NeedsGlobalCtor = false;
5382
  // Whether the definition of the variable is available externally.
5383
  // If yes, we shouldn't emit the GloablCtor and GlobalDtor for the variable
5384
  // since this is the job for its original source.
5385
  bool IsDefinitionAvailableExternally =
5386
      getContext().GetGVALinkageForVariable(D) == GVA_AvailableExternally;
5387
  bool NeedsGlobalDtor =
5388
      !IsDefinitionAvailableExternally &&
5389
      D->needsDestruction(getContext()) == QualType::DK_cxx_destructor;
5390

5391
  // It is helpless to emit the definition for an available_externally variable
5392
  // which can't be marked as const.
5393
  // We don't need to check if it needs global ctor or dtor. See the above
5394
  // comment for ideas.
5395
  if (IsDefinitionAvailableExternally &&
5396
      (!D->hasConstantInitialization() ||
5397
       // TODO: Update this when we have interface to check constexpr
5398
       // destructor.
5399
       D->needsDestruction(getContext()) ||
5400
       !D->getType().isConstantStorage(getContext(), true, true)))
5401
    return;
5402

5403
  const VarDecl *InitDecl;
5404
  const Expr *InitExpr = D->getAnyInitializer(InitDecl);
5405

5406
  std::optional<ConstantEmitter> emitter;
5407

5408
  // CUDA E.2.4.1 "__shared__ variables cannot have an initialization
5409
  // as part of their declaration."  Sema has already checked for
5410
  // error cases, so we just need to set Init to UndefValue.
5411
  bool IsCUDASharedVar =
5412
      getLangOpts().CUDAIsDevice && D->hasAttr<CUDASharedAttr>();
5413
  // Shadows of initialized device-side global variables are also left
5414
  // undefined.
5415
  // Managed Variables should be initialized on both host side and device side.
5416
  bool IsCUDAShadowVar =
5417
      !getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
5418
      (D->hasAttr<CUDAConstantAttr>() || D->hasAttr<CUDADeviceAttr>() ||
5419
       D->hasAttr<CUDASharedAttr>());
5420
  bool IsCUDADeviceShadowVar =
5421
      getLangOpts().CUDAIsDevice && !D->hasAttr<HIPManagedAttr>() &&
5422
      (D->getType()->isCUDADeviceBuiltinSurfaceType() ||
5423
       D->getType()->isCUDADeviceBuiltinTextureType());
5424
  if (getLangOpts().CUDA &&
5425
      (IsCUDASharedVar || IsCUDAShadowVar || IsCUDADeviceShadowVar))
5426
    Init = llvm::UndefValue::get(getTypes().ConvertTypeForMem(ASTTy));
5427
  else if (D->hasAttr<LoaderUninitializedAttr>())
5428
    Init = llvm::UndefValue::get(getTypes().ConvertTypeForMem(ASTTy));
5429
  else if (!InitExpr) {
5430
    // This is a tentative definition; tentative definitions are
5431
    // implicitly initialized with { 0 }.
5432
    //
5433
    // Note that tentative definitions are only emitted at the end of
5434
    // a translation unit, so they should never have incomplete
5435
    // type. In addition, EmitTentativeDefinition makes sure that we
5436
    // never attempt to emit a tentative definition if a real one
5437
    // exists. A use may still exists, however, so we still may need
5438
    // to do a RAUW.
5439
    assert(!ASTTy->isIncompleteType() && "Unexpected incomplete type");
5440
    Init = EmitNullConstant(D->getType());
5441
  } else {
5442
    initializedGlobalDecl = GlobalDecl(D);
5443
    emitter.emplace(*this);
5444
    llvm::Constant *Initializer = emitter->tryEmitForInitializer(*InitDecl);
5445
    if (!Initializer) {
5446
      QualType T = InitExpr->getType();
5447
      if (D->getType()->isReferenceType())
5448
        T = D->getType();
5449

5450
      if (getLangOpts().CPlusPlus) {
5451
        if (InitDecl->hasFlexibleArrayInit(getContext()))
5452
          ErrorUnsupported(D, "flexible array initializer");
5453
        Init = EmitNullConstant(T);
5454

5455
        if (!IsDefinitionAvailableExternally)
5456
          NeedsGlobalCtor = true;
5457
      } else {
5458
        ErrorUnsupported(D, "static initializer");
5459
        Init = llvm::UndefValue::get(getTypes().ConvertType(T));
5460
      }
5461
    } else {
5462
      Init = Initializer;
5463
      // We don't need an initializer, so remove the entry for the delayed
5464
      // initializer position (just in case this entry was delayed) if we
5465
      // also don't need to register a destructor.
5466
      if (getLangOpts().CPlusPlus && !NeedsGlobalDtor)
5467
        DelayedCXXInitPosition.erase(D);
5468

5469
#ifndef NDEBUG
5470
      CharUnits VarSize = getContext().getTypeSizeInChars(ASTTy) +
5471
                          InitDecl->getFlexibleArrayInitChars(getContext());
5472
      CharUnits CstSize = CharUnits::fromQuantity(
5473
          getDataLayout().getTypeAllocSize(Init->getType()));
5474
      assert(VarSize == CstSize && "Emitted constant has unexpected size");
5475
#endif
5476
    }
5477
  }
5478

5479
  llvm::Type* InitType = Init->getType();
5480
  llvm::Constant *Entry =
5481
      GetAddrOfGlobalVar(D, InitType, ForDefinition_t(!IsTentative));
5482

5483
  // Strip off pointer casts if we got them.
5484
  Entry = Entry->stripPointerCasts();
5485

5486
  // Entry is now either a Function or GlobalVariable.
5487
  auto *GV = dyn_cast<llvm::GlobalVariable>(Entry);
5488

5489
  // We have a definition after a declaration with the wrong type.
5490
  // We must make a new GlobalVariable* and update everything that used OldGV
5491
  // (a declaration or tentative definition) with the new GlobalVariable*
5492
  // (which will be a definition).
5493
  //
5494
  // This happens if there is a prototype for a global (e.g.
5495
  // "extern int x[];") and then a definition of a different type (e.g.
5496
  // "int x[10];"). This also happens when an initializer has a different type
5497
  // from the type of the global (this happens with unions).
5498
  if (!GV || GV->getValueType() != InitType ||
5499
      GV->getType()->getAddressSpace() !=
5500
          getContext().getTargetAddressSpace(GetGlobalVarAddressSpace(D))) {
5501

5502
    // Move the old entry aside so that we'll create a new one.
5503
    Entry->setName(StringRef());
5504

5505
    // Make a new global with the correct type, this is now guaranteed to work.
5506
    GV = cast<llvm::GlobalVariable>(
5507
        GetAddrOfGlobalVar(D, InitType, ForDefinition_t(!IsTentative))
5508
            ->stripPointerCasts());
5509

5510
    // Replace all uses of the old global with the new global
5511
    llvm::Constant *NewPtrForOldDecl =
5512
        llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(GV,
5513
                                                             Entry->getType());
5514
    Entry->replaceAllUsesWith(NewPtrForOldDecl);
5515

5516
    // Erase the old global, since it is no longer used.
5517
    cast<llvm::GlobalValue>(Entry)->eraseFromParent();
5518
  }
5519

5520
  MaybeHandleStaticInExternC(D, GV);
5521

5522
  if (D->hasAttr<AnnotateAttr>())
5523
    AddGlobalAnnotations(D, GV);
5524

5525
  // Set the llvm linkage type as appropriate.
5526
  llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(D);
5527

5528
  // CUDA B.2.1 "The __device__ qualifier declares a variable that resides on
5529
  // the device. [...]"
5530
  // CUDA B.2.2 "The __constant__ qualifier, optionally used together with
5531
  // __device__, declares a variable that: [...]
5532
  // Is accessible from all the threads within the grid and from the host
5533
  // through the runtime library (cudaGetSymbolAddress() / cudaGetSymbolSize()
5534
  // / cudaMemcpyToSymbol() / cudaMemcpyFromSymbol())."
5535
  if (LangOpts.CUDA) {
5536
    if (LangOpts.CUDAIsDevice) {
5537
      if (Linkage != llvm::GlobalValue::InternalLinkage &&
5538
          (D->hasAttr<CUDADeviceAttr>() || D->hasAttr<CUDAConstantAttr>() ||
5539
           D->getType()->isCUDADeviceBuiltinSurfaceType() ||
5540
           D->getType()->isCUDADeviceBuiltinTextureType()))
5541
        GV->setExternallyInitialized(true);
5542
    } else {
5543
      getCUDARuntime().internalizeDeviceSideVar(D, Linkage);
5544
    }
5545
    getCUDARuntime().handleVarRegistration(D, *GV);
5546
  }
5547

5548
  GV->setInitializer(Init);
5549
  if (emitter)
5550
    emitter->finalize(GV);
5551

5552
  // If it is safe to mark the global 'constant', do so now.
5553
  GV->setConstant(!NeedsGlobalCtor && !NeedsGlobalDtor &&
5554
                  D->getType().isConstantStorage(getContext(), true, true));
5555

5556
  // If it is in a read-only section, mark it 'constant'.
5557
  if (const SectionAttr *SA = D->getAttr<SectionAttr>()) {
5558
    const ASTContext::SectionInfo &SI = Context.SectionInfos[SA->getName()];
5559
    if ((SI.SectionFlags & ASTContext::PSF_Write) == 0)
5560
      GV->setConstant(true);
5561
  }
5562

5563
  CharUnits AlignVal = getContext().getDeclAlign(D);
5564
  // Check for alignment specifed in an 'omp allocate' directive.
5565
  if (std::optional<CharUnits> AlignValFromAllocate =
5566
          getOMPAllocateAlignment(D))
5567
    AlignVal = *AlignValFromAllocate;
5568
  GV->setAlignment(AlignVal.getAsAlign());
5569

5570
  // On Darwin, unlike other Itanium C++ ABI platforms, the thread-wrapper
5571
  // function is only defined alongside the variable, not also alongside
5572
  // callers. Normally, all accesses to a thread_local go through the
5573
  // thread-wrapper in order to ensure initialization has occurred, underlying
5574
  // variable will never be used other than the thread-wrapper, so it can be
5575
  // converted to internal linkage.
5576
  //
5577
  // However, if the variable has the 'constinit' attribute, it _can_ be
5578
  // referenced directly, without calling the thread-wrapper, so the linkage
5579
  // must not be changed.
5580
  //
5581
  // Additionally, if the variable isn't plain external linkage, e.g. if it's
5582
  // weak or linkonce, the de-duplication semantics are important to preserve,
5583
  // so we don't change the linkage.
5584
  if (D->getTLSKind() == VarDecl::TLS_Dynamic &&
5585
      Linkage == llvm::GlobalValue::ExternalLinkage &&
5586
      Context.getTargetInfo().getTriple().isOSDarwin() &&
5587
      !D->hasAttr<ConstInitAttr>())
5588
    Linkage = llvm::GlobalValue::InternalLinkage;
5589

5590
  GV->setLinkage(Linkage);
5591
  if (D->hasAttr<DLLImportAttr>())
5592
    GV->setDLLStorageClass(llvm::GlobalVariable::DLLImportStorageClass);
5593
  else if (D->hasAttr<DLLExportAttr>())
5594
    GV->setDLLStorageClass(llvm::GlobalVariable::DLLExportStorageClass);
5595
  else
5596
    GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
5597

5598
  if (Linkage == llvm::GlobalVariable::CommonLinkage) {
5599
    // common vars aren't constant even if declared const.
5600
    GV->setConstant(false);
5601
    // Tentative definition of global variables may be initialized with
5602
    // non-zero null pointers. In this case they should have weak linkage
5603
    // since common linkage must have zero initializer and must not have
5604
    // explicit section therefore cannot have non-zero initial value.
5605
    if (!GV->getInitializer()->isNullValue())
5606
      GV->setLinkage(llvm::GlobalVariable::WeakAnyLinkage);
5607
  }
5608

5609
  setNonAliasAttributes(D, GV);
5610

5611
  if (D->getTLSKind() && !GV->isThreadLocal()) {
5612
    if (D->getTLSKind() == VarDecl::TLS_Dynamic)
5613
      CXXThreadLocals.push_back(D);
5614
    setTLSMode(GV, *D);
5615
  }
5616

5617
  maybeSetTrivialComdat(*D, *GV);
5618

5619
  // Emit the initializer function if necessary.
5620
  if (NeedsGlobalCtor || NeedsGlobalDtor)
5621
    EmitCXXGlobalVarDeclInitFunc(D, GV, NeedsGlobalCtor);
5622

5623
  SanitizerMD->reportGlobal(GV, *D, NeedsGlobalCtor);
5624

5625
  // Emit global variable debug information.
5626
  if (CGDebugInfo *DI = getModuleDebugInfo())
5627
    if (getCodeGenOpts().hasReducedDebugInfo())
5628
      DI->EmitGlobalVariable(GV, D);
5629
}
5630

5631
void CodeGenModule::EmitExternalVarDeclaration(const VarDecl *D) {
5632
  if (CGDebugInfo *DI = getModuleDebugInfo())
5633
    if (getCodeGenOpts().hasReducedDebugInfo()) {
5634
      QualType ASTTy = D->getType();
5635
      llvm::Type *Ty = getTypes().ConvertTypeForMem(D->getType());
5636
      llvm::Constant *GV =
5637
          GetOrCreateLLVMGlobal(D->getName(), Ty, ASTTy.getAddressSpace(), D);
5638
      DI->EmitExternalVariable(
5639
          cast<llvm::GlobalVariable>(GV->stripPointerCasts()), D);
5640
    }
5641
}
5642

5643
void CodeGenModule::EmitExternalFunctionDeclaration(const FunctionDecl *FD) {
5644
  if (CGDebugInfo *DI = getModuleDebugInfo())
5645
    if (getCodeGenOpts().hasReducedDebugInfo()) {
5646
      auto *Ty = getTypes().ConvertType(FD->getType());
5647
      StringRef MangledName = getMangledName(FD);
5648
      auto *Fn = dyn_cast<llvm::Function>(
5649
          GetOrCreateLLVMFunction(MangledName, Ty, FD, /* ForVTable */ false));
5650
      if (!Fn->getSubprogram())
5651
        DI->EmitFunctionDecl(FD, FD->getLocation(), FD->getType(), Fn);
5652
    }
5653
}
5654

5655
static bool isVarDeclStrongDefinition(const ASTContext &Context,
5656
                                      CodeGenModule &CGM, const VarDecl *D,
5657
                                      bool NoCommon) {
5658
  // Don't give variables common linkage if -fno-common was specified unless it
5659
  // was overridden by a NoCommon attribute.
5660
  if ((NoCommon || D->hasAttr<NoCommonAttr>()) && !D->hasAttr<CommonAttr>())
5661
    return true;
5662

5663
  // C11 6.9.2/2:
5664
  //   A declaration of an identifier for an object that has file scope without
5665
  //   an initializer, and without a storage-class specifier or with the
5666
  //   storage-class specifier static, constitutes a tentative definition.
5667
  if (D->getInit() || D->hasExternalStorage())
5668
    return true;
5669

5670
  // A variable cannot be both common and exist in a section.
5671
  if (D->hasAttr<SectionAttr>())
5672
    return true;
5673

5674
  // A variable cannot be both common and exist in a section.
5675
  // We don't try to determine which is the right section in the front-end.
5676
  // If no specialized section name is applicable, it will resort to default.
5677
  if (D->hasAttr<PragmaClangBSSSectionAttr>() ||
5678
      D->hasAttr<PragmaClangDataSectionAttr>() ||
5679
      D->hasAttr<PragmaClangRelroSectionAttr>() ||
5680
      D->hasAttr<PragmaClangRodataSectionAttr>())
5681
    return true;
5682

5683
  // Thread local vars aren't considered common linkage.
5684
  if (D->getTLSKind())
5685
    return true;
5686

5687
  // Tentative definitions marked with WeakImportAttr are true definitions.
5688
  if (D->hasAttr<WeakImportAttr>())
5689
    return true;
5690

5691
  // A variable cannot be both common and exist in a comdat.
5692
  if (shouldBeInCOMDAT(CGM, *D))
5693
    return true;
5694

5695
  // Declarations with a required alignment do not have common linkage in MSVC
5696
  // mode.
5697
  if (Context.getTargetInfo().getCXXABI().isMicrosoft()) {
5698
    if (D->hasAttr<AlignedAttr>())
5699
      return true;
5700
    QualType VarType = D->getType();
5701
    if (Context.isAlignmentRequired(VarType))
5702
      return true;
5703

5704
    if (const auto *RT = VarType->getAs<RecordType>()) {
5705
      const RecordDecl *RD = RT->getDecl();
5706
      for (const FieldDecl *FD : RD->fields()) {
5707
        if (FD->isBitField())
5708
          continue;
5709
        if (FD->hasAttr<AlignedAttr>())
5710
          return true;
5711
        if (Context.isAlignmentRequired(FD->getType()))
5712
          return true;
5713
      }
5714
    }
5715
  }
5716

5717
  // Microsoft's link.exe doesn't support alignments greater than 32 bytes for
5718
  // common symbols, so symbols with greater alignment requirements cannot be
5719
  // common.
5720
  // Other COFF linkers (ld.bfd and LLD) support arbitrary power-of-two
5721
  // alignments for common symbols via the aligncomm directive, so this
5722
  // restriction only applies to MSVC environments.
5723
  if (Context.getTargetInfo().getTriple().isKnownWindowsMSVCEnvironment() &&
5724
      Context.getTypeAlignIfKnown(D->getType()) >
5725
          Context.toBits(CharUnits::fromQuantity(32)))
5726
    return true;
5727

5728
  return false;
5729
}
5730

5731
llvm::GlobalValue::LinkageTypes
5732
CodeGenModule::getLLVMLinkageForDeclarator(const DeclaratorDecl *D,
5733
                                           GVALinkage Linkage) {
5734
  if (Linkage == GVA_Internal)
5735
    return llvm::Function::InternalLinkage;
5736

5737
  if (D->hasAttr<WeakAttr>())
5738
    return llvm::GlobalVariable::WeakAnyLinkage;
5739

5740
  if (const auto *FD = D->getAsFunction())
5741
    if (FD->isMultiVersion() && Linkage == GVA_AvailableExternally)
5742
      return llvm::GlobalVariable::LinkOnceAnyLinkage;
5743

5744
  // We are guaranteed to have a strong definition somewhere else,
5745
  // so we can use available_externally linkage.
5746
  if (Linkage == GVA_AvailableExternally)
5747
    return llvm::GlobalValue::AvailableExternallyLinkage;
5748

5749
  // Note that Apple's kernel linker doesn't support symbol
5750
  // coalescing, so we need to avoid linkonce and weak linkages there.
5751
  // Normally, this means we just map to internal, but for explicit
5752
  // instantiations we'll map to external.
5753

5754
  // In C++, the compiler has to emit a definition in every translation unit
5755
  // that references the function.  We should use linkonce_odr because
5756
  // a) if all references in this translation unit are optimized away, we
5757
  // don't need to codegen it.  b) if the function persists, it needs to be
5758
  // merged with other definitions. c) C++ has the ODR, so we know the
5759
  // definition is dependable.
5760
  if (Linkage == GVA_DiscardableODR)
5761
    return !Context.getLangOpts().AppleKext ? llvm::Function::LinkOnceODRLinkage
5762
                                            : llvm::Function::InternalLinkage;
5763

5764
  // An explicit instantiation of a template has weak linkage, since
5765
  // explicit instantiations can occur in multiple translation units
5766
  // and must all be equivalent. However, we are not allowed to
5767
  // throw away these explicit instantiations.
5768
  //
5769
  // CUDA/HIP: For -fno-gpu-rdc case, device code is limited to one TU,
5770
  // so say that CUDA templates are either external (for kernels) or internal.
5771
  // This lets llvm perform aggressive inter-procedural optimizations. For
5772
  // -fgpu-rdc case, device function calls across multiple TU's are allowed,
5773
  // therefore we need to follow the normal linkage paradigm.
5774
  if (Linkage == GVA_StrongODR) {
5775
    if (getLangOpts().AppleKext)
5776
      return llvm::Function::ExternalLinkage;
5777
    if (getLangOpts().CUDA && getLangOpts().CUDAIsDevice &&
5778
        !getLangOpts().GPURelocatableDeviceCode)
5779
      return D->hasAttr<CUDAGlobalAttr>() ? llvm::Function::ExternalLinkage
5780
                                          : llvm::Function::InternalLinkage;
5781
    return llvm::Function::WeakODRLinkage;
5782
  }
5783

5784
  // C++ doesn't have tentative definitions and thus cannot have common
5785
  // linkage.
5786
  if (!getLangOpts().CPlusPlus && isa<VarDecl>(D) &&
5787
      !isVarDeclStrongDefinition(Context, *this, cast<VarDecl>(D),
5788
                                 CodeGenOpts.NoCommon))
5789
    return llvm::GlobalVariable::CommonLinkage;
5790

5791
  // selectany symbols are externally visible, so use weak instead of
5792
  // linkonce.  MSVC optimizes away references to const selectany globals, so
5793
  // all definitions should be the same and ODR linkage should be used.
5794
  // http://msdn.microsoft.com/en-us/library/5tkz6s71.aspx
5795
  if (D->hasAttr<SelectAnyAttr>())
5796
    return llvm::GlobalVariable::WeakODRLinkage;
5797

5798
  // Otherwise, we have strong external linkage.
5799
  assert(Linkage == GVA_StrongExternal);
5800
  return llvm::GlobalVariable::ExternalLinkage;
5801
}
5802

5803
llvm::GlobalValue::LinkageTypes
5804
CodeGenModule::getLLVMLinkageVarDefinition(const VarDecl *VD) {
5805
  GVALinkage Linkage = getContext().GetGVALinkageForVariable(VD);
5806
  return getLLVMLinkageForDeclarator(VD, Linkage);
5807
}
5808

5809
/// Replace the uses of a function that was declared with a non-proto type.
5810
/// We want to silently drop extra arguments from call sites
5811
static void replaceUsesOfNonProtoConstant(llvm::Constant *old,
5812
                                          llvm::Function *newFn) {
5813
  // Fast path.
5814
  if (old->use_empty())
5815
    return;
5816

5817
  llvm::Type *newRetTy = newFn->getReturnType();
5818
  SmallVector<llvm::Value *, 4> newArgs;
5819

5820
  SmallVector<llvm::CallBase *> callSitesToBeRemovedFromParent;
5821

5822
  for (llvm::Value::use_iterator ui = old->use_begin(), ue = old->use_end();
5823
       ui != ue; ui++) {
5824
    llvm::User *user = ui->getUser();
5825

5826
    // Recognize and replace uses of bitcasts.  Most calls to
5827
    // unprototyped functions will use bitcasts.
5828
    if (auto *bitcast = dyn_cast<llvm::ConstantExpr>(user)) {
5829
      if (bitcast->getOpcode() == llvm::Instruction::BitCast)
5830
        replaceUsesOfNonProtoConstant(bitcast, newFn);
5831
      continue;
5832
    }
5833

5834
    // Recognize calls to the function.
5835
    llvm::CallBase *callSite = dyn_cast<llvm::CallBase>(user);
5836
    if (!callSite)
5837
      continue;
5838
    if (!callSite->isCallee(&*ui))
5839
      continue;
5840

5841
    // If the return types don't match exactly, then we can't
5842
    // transform this call unless it's dead.
5843
    if (callSite->getType() != newRetTy && !callSite->use_empty())
5844
      continue;
5845

5846
    // Get the call site's attribute list.
5847
    SmallVector<llvm::AttributeSet, 8> newArgAttrs;
5848
    llvm::AttributeList oldAttrs = callSite->getAttributes();
5849

5850
    // If the function was passed too few arguments, don't transform.
5851
    unsigned newNumArgs = newFn->arg_size();
5852
    if (callSite->arg_size() < newNumArgs)
5853
      continue;
5854

5855
    // If extra arguments were passed, we silently drop them.
5856
    // If any of the types mismatch, we don't transform.
5857
    unsigned argNo = 0;
5858
    bool dontTransform = false;
5859
    for (llvm::Argument &A : newFn->args()) {
5860
      if (callSite->getArgOperand(argNo)->getType() != A.getType()) {
5861
        dontTransform = true;
5862
        break;
5863
      }
5864

5865
      // Add any parameter attributes.
5866
      newArgAttrs.push_back(oldAttrs.getParamAttrs(argNo));
5867
      argNo++;
5868
    }
5869
    if (dontTransform)
5870
      continue;
5871

5872
    // Okay, we can transform this.  Create the new call instruction and copy
5873
    // over the required information.
5874
    newArgs.append(callSite->arg_begin(), callSite->arg_begin() + argNo);
5875

5876
    // Copy over any operand bundles.
5877
    SmallVector<llvm::OperandBundleDef, 1> newBundles;
5878
    callSite->getOperandBundlesAsDefs(newBundles);
5879

5880
    llvm::CallBase *newCall;
5881
    if (isa<llvm::CallInst>(callSite)) {
5882
      newCall =
5883
          llvm::CallInst::Create(newFn, newArgs, newBundles, "", callSite);
5884
    } else {
5885
      auto *oldInvoke = cast<llvm::InvokeInst>(callSite);
5886
      newCall = llvm::InvokeInst::Create(newFn, oldInvoke->getNormalDest(),
5887
                                         oldInvoke->getUnwindDest(), newArgs,
5888
                                         newBundles, "", callSite);
5889
    }
5890
    newArgs.clear(); // for the next iteration
5891

5892
    if (!newCall->getType()->isVoidTy())
5893
      newCall->takeName(callSite);
5894
    newCall->setAttributes(
5895
        llvm::AttributeList::get(newFn->getContext(), oldAttrs.getFnAttrs(),
5896
                                 oldAttrs.getRetAttrs(), newArgAttrs));
5897
    newCall->setCallingConv(callSite->getCallingConv());
5898

5899
    // Finally, remove the old call, replacing any uses with the new one.
5900
    if (!callSite->use_empty())
5901
      callSite->replaceAllUsesWith(newCall);
5902

5903
    // Copy debug location attached to CI.
5904
    if (callSite->getDebugLoc())
5905
      newCall->setDebugLoc(callSite->getDebugLoc());
5906

5907
    callSitesToBeRemovedFromParent.push_back(callSite);
5908
  }
5909

5910
  for (auto *callSite : callSitesToBeRemovedFromParent) {
5911
    callSite->eraseFromParent();
5912
  }
5913
}
5914

5915
/// ReplaceUsesOfNonProtoTypeWithRealFunction - This function is called when we
5916
/// implement a function with no prototype, e.g. "int foo() {}".  If there are
5917
/// existing call uses of the old function in the module, this adjusts them to
5918
/// call the new function directly.
5919
///
5920
/// This is not just a cleanup: the always_inline pass requires direct calls to
5921
/// functions to be able to inline them.  If there is a bitcast in the way, it
5922
/// won't inline them.  Instcombine normally deletes these calls, but it isn't
5923
/// run at -O0.
5924
static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
5925
                                                      llvm::Function *NewFn) {
5926
  // If we're redefining a global as a function, don't transform it.
5927
  if (!isa<llvm::Function>(Old)) return;
5928

5929
  replaceUsesOfNonProtoConstant(Old, NewFn);
5930
}
5931

5932
void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) {
5933
  auto DK = VD->isThisDeclarationADefinition();
5934
  if (DK == VarDecl::Definition && VD->hasAttr<DLLImportAttr>())
5935
    return;
5936

5937
  TemplateSpecializationKind TSK = VD->getTemplateSpecializationKind();
5938
  // If we have a definition, this might be a deferred decl. If the
5939
  // instantiation is explicit, make sure we emit it at the end.
5940
  if (VD->getDefinition() && TSK == TSK_ExplicitInstantiationDefinition)
5941
    GetAddrOfGlobalVar(VD);
5942

5943
  EmitTopLevelDecl(VD);
5944
}
5945

5946
void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
5947
                                                 llvm::GlobalValue *GV) {
5948
  const auto *D = cast<FunctionDecl>(GD.getDecl());
5949

5950
  // Compute the function info and LLVM type.
5951
  const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD);
5952
  llvm::FunctionType *Ty = getTypes().GetFunctionType(FI);
5953

5954
  // Get or create the prototype for the function.
5955
  if (!GV || (GV->getValueType() != Ty))
5956
    GV = cast<llvm::GlobalValue>(GetAddrOfFunction(GD, Ty, /*ForVTable=*/false,
5957
                                                   /*DontDefer=*/true,
5958
                                                   ForDefinition));
5959

5960
  // Already emitted.
5961
  if (!GV->isDeclaration())
5962
    return;
5963

5964
  // We need to set linkage and visibility on the function before
5965
  // generating code for it because various parts of IR generation
5966
  // want to propagate this information down (e.g. to local static
5967
  // declarations).
5968
  auto *Fn = cast<llvm::Function>(GV);
5969
  setFunctionLinkage(GD, Fn);
5970

5971
  // FIXME: this is redundant with part of setFunctionDefinitionAttributes
5972
  setGVProperties(Fn, GD);
5973

5974
  MaybeHandleStaticInExternC(D, Fn);
5975

5976
  maybeSetTrivialComdat(*D, *Fn);
5977

5978
  CodeGenFunction(*this).GenerateCode(GD, Fn, FI);
5979

5980
  setNonAliasAttributes(GD, Fn);
5981
  SetLLVMFunctionAttributesForDefinition(D, Fn);
5982

5983
  if (const ConstructorAttr *CA = D->getAttr<ConstructorAttr>())
5984
    AddGlobalCtor(Fn, CA->getPriority());
5985
  if (const DestructorAttr *DA = D->getAttr<DestructorAttr>())
5986
    AddGlobalDtor(Fn, DA->getPriority(), true);
5987
  if (getLangOpts().OpenMP && D->hasAttr<OMPDeclareTargetDeclAttr>())
5988
    getOpenMPRuntime().emitDeclareTargetFunction(D, GV);
5989
}
5990

5991
void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
5992
  const auto *D = cast<ValueDecl>(GD.getDecl());
5993
  const AliasAttr *AA = D->getAttr<AliasAttr>();
5994
  assert(AA && "Not an alias?");
5995

5996
  StringRef MangledName = getMangledName(GD);
5997

5998
  if (AA->getAliasee() == MangledName) {
5999
    Diags.Report(AA->getLocation(), diag::err_cyclic_alias) << 0;
6000
    return;
6001
  }
6002

6003
  // If there is a definition in the module, then it wins over the alias.
6004
  // This is dubious, but allow it to be safe.  Just ignore the alias.
6005
  llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
6006
  if (Entry && !Entry->isDeclaration())
6007
    return;
6008

6009
  Aliases.push_back(GD);
6010

6011
  llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType());
6012

6013
  // Create a reference to the named value.  This ensures that it is emitted
6014
  // if a deferred decl.
6015
  llvm::Constant *Aliasee;
6016
  llvm::GlobalValue::LinkageTypes LT;
6017
  if (isa<llvm::FunctionType>(DeclTy)) {
6018
    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GD,
6019
                                      /*ForVTable=*/false);
6020
    LT = getFunctionLinkage(GD);
6021
  } else {
6022
    Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(), DeclTy, LangAS::Default,
6023
                                    /*D=*/nullptr);
6024
    if (const auto *VD = dyn_cast<VarDecl>(GD.getDecl()))
6025
      LT = getLLVMLinkageVarDefinition(VD);
6026
    else
6027
      LT = getFunctionLinkage(GD);
6028
  }
6029

6030
  // Create the new alias itself, but don't set a name yet.
6031
  unsigned AS = Aliasee->getType()->getPointerAddressSpace();
6032
  auto *GA =
6033
      llvm::GlobalAlias::create(DeclTy, AS, LT, "", Aliasee, &getModule());
6034

6035
  if (Entry) {
6036
    if (GA->getAliasee() == Entry) {
6037
      Diags.Report(AA->getLocation(), diag::err_cyclic_alias) << 0;
6038
      return;
6039
    }
6040

6041
    assert(Entry->isDeclaration());
6042

6043
    // If there is a declaration in the module, then we had an extern followed
6044
    // by the alias, as in:
6045
    //   extern int test6();
6046
    //   ...
6047
    //   int test6() __attribute__((alias("test7")));
6048
    //
6049
    // Remove it and replace uses of it with the alias.
6050
    GA->takeName(Entry);
6051

6052
    Entry->replaceAllUsesWith(GA);
6053
    Entry->eraseFromParent();
6054
  } else {
6055
    GA->setName(MangledName);
6056
  }
6057

6058
  // Set attributes which are particular to an alias; this is a
6059
  // specialization of the attributes which may be set on a global
6060
  // variable/function.
6061
  if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
6062
      D->isWeakImported()) {
6063
    GA->setLinkage(llvm::Function::WeakAnyLinkage);
6064
  }
6065

6066
  if (const auto *VD = dyn_cast<VarDecl>(D))
6067
    if (VD->getTLSKind())
6068
      setTLSMode(GA, *VD);
6069

6070
  SetCommonAttributes(GD, GA);
6071

6072
  // Emit global alias debug information.
6073
  if (isa<VarDecl>(D))
6074
    if (CGDebugInfo *DI = getModuleDebugInfo())
6075
      DI->EmitGlobalAlias(cast<llvm::GlobalValue>(GA->getAliasee()->stripPointerCasts()), GD);
6076
}
6077

6078
void CodeGenModule::emitIFuncDefinition(GlobalDecl GD) {
6079
  const auto *D = cast<ValueDecl>(GD.getDecl());
6080
  const IFuncAttr *IFA = D->getAttr<IFuncAttr>();
6081
  assert(IFA && "Not an ifunc?");
6082

6083
  StringRef MangledName = getMangledName(GD);
6084

6085
  if (IFA->getResolver() == MangledName) {
6086
    Diags.Report(IFA->getLocation(), diag::err_cyclic_alias) << 1;
6087
    return;
6088
  }
6089

6090
  // Report an error if some definition overrides ifunc.
6091
  llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
6092
  if (Entry && !Entry->isDeclaration()) {
6093
    GlobalDecl OtherGD;
6094
    if (lookupRepresentativeDecl(MangledName, OtherGD) &&
6095
        DiagnosedConflictingDefinitions.insert(GD).second) {
6096
      Diags.Report(D->getLocation(), diag::err_duplicate_mangled_name)
6097
          << MangledName;
6098
      Diags.Report(OtherGD.getDecl()->getLocation(),
6099
                   diag::note_previous_definition);
6100
    }
6101
    return;
6102
  }
6103

6104
  Aliases.push_back(GD);
6105

6106
  llvm::Type *DeclTy = getTypes().ConvertTypeForMem(D->getType());
6107
  llvm::Type *ResolverTy = llvm::GlobalIFunc::getResolverFunctionType(DeclTy);
6108
  llvm::Constant *Resolver =
6109
      GetOrCreateLLVMFunction(IFA->getResolver(), ResolverTy, {},
6110
                              /*ForVTable=*/false);
6111
  llvm::GlobalIFunc *GIF =
6112
      llvm::GlobalIFunc::create(DeclTy, 0, llvm::Function::ExternalLinkage,
6113
                                "", Resolver, &getModule());
6114
  if (Entry) {
6115
    if (GIF->getResolver() == Entry) {
6116
      Diags.Report(IFA->getLocation(), diag::err_cyclic_alias) << 1;
6117
      return;
6118
    }
6119
    assert(Entry->isDeclaration());
6120

6121
    // If there is a declaration in the module, then we had an extern followed
6122
    // by the ifunc, as in:
6123
    //   extern int test();
6124
    //   ...
6125
    //   int test() __attribute__((ifunc("resolver")));
6126
    //
6127
    // Remove it and replace uses of it with the ifunc.
6128
    GIF->takeName(Entry);
6129

6130
    Entry->replaceAllUsesWith(GIF);
6131
    Entry->eraseFromParent();
6132
  } else
6133
    GIF->setName(MangledName);
6134
  if (auto *F = dyn_cast<llvm::Function>(Resolver)) {
6135
    F->addFnAttr(llvm::Attribute::DisableSanitizerInstrumentation);
6136
  }
6137
  SetCommonAttributes(GD, GIF);
6138
}
6139

6140
llvm::Function *CodeGenModule::getIntrinsic(unsigned IID,
6141
                                            ArrayRef<llvm::Type*> Tys) {
6142
  return llvm::Intrinsic::getDeclaration(&getModule(), (llvm::Intrinsic::ID)IID,
6143
                                         Tys);
6144
}
6145

6146
static llvm::StringMapEntry<llvm::GlobalVariable *> &
6147
GetConstantCFStringEntry(llvm::StringMap<llvm::GlobalVariable *> &Map,
6148
                         const StringLiteral *Literal, bool TargetIsLSB,
6149
                         bool &IsUTF16, unsigned &StringLength) {
6150
  StringRef String = Literal->getString();
6151
  unsigned NumBytes = String.size();
6152

6153
  // Check for simple case.
6154
  if (!Literal->containsNonAsciiOrNull()) {
6155
    StringLength = NumBytes;
6156
    return *Map.insert(std::make_pair(String, nullptr)).first;
6157
  }
6158

6159
  // Otherwise, convert the UTF8 literals into a string of shorts.
6160
  IsUTF16 = true;
6161

6162
  SmallVector<llvm::UTF16, 128> ToBuf(NumBytes + 1); // +1 for ending nulls.
6163
  const llvm::UTF8 *FromPtr = (const llvm::UTF8 *)String.data();
6164
  llvm::UTF16 *ToPtr = &ToBuf[0];
6165

6166
  (void)llvm::ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes, &ToPtr,
6167
                                 ToPtr + NumBytes, llvm::strictConversion);
6168

6169
  // ConvertUTF8toUTF16 returns the length in ToPtr.
6170
  StringLength = ToPtr - &ToBuf[0];
6171

6172
  // Add an explicit null.
6173
  *ToPtr = 0;
6174
  return *Map.insert(std::make_pair(
6175
                         StringRef(reinterpret_cast<const char *>(ToBuf.data()),
6176
                                   (StringLength + 1) * 2),
6177
                         nullptr)).first;
6178
}
6179

6180
ConstantAddress
6181
CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
6182
  unsigned StringLength = 0;
6183
  bool isUTF16 = false;
6184
  llvm::StringMapEntry<llvm::GlobalVariable *> &Entry =
6185
      GetConstantCFStringEntry(CFConstantStringMap, Literal,
6186
                               getDataLayout().isLittleEndian(), isUTF16,
6187
                               StringLength);
6188

6189
  if (auto *C = Entry.second)
6190
    return ConstantAddress(
6191
        C, C->getValueType(), CharUnits::fromQuantity(C->getAlignment()));
6192

6193
  const ASTContext &Context = getContext();
6194
  const llvm::Triple &Triple = getTriple();
6195

6196
  const auto CFRuntime = getLangOpts().CFRuntime;
6197
  const bool IsSwiftABI =
6198
      static_cast<unsigned>(CFRuntime) >=
6199
      static_cast<unsigned>(LangOptions::CoreFoundationABI::Swift);
6200
  const bool IsSwift4_1 = CFRuntime == LangOptions::CoreFoundationABI::Swift4_1;
6201

6202
  // If we don't already have it, get __CFConstantStringClassReference.
6203
  if (!CFConstantStringClassRef) {
6204
    const char *CFConstantStringClassName = "__CFConstantStringClassReference";
6205
    llvm::Type *Ty = getTypes().ConvertType(getContext().IntTy);
6206
    Ty = llvm::ArrayType::get(Ty, 0);
6207

6208
    switch (CFRuntime) {
6209
    default: break;
6210
    case LangOptions::CoreFoundationABI::Swift: [[fallthrough]];
6211
    case LangOptions::CoreFoundationABI::Swift5_0:
6212
      CFConstantStringClassName =
6213
          Triple.isOSDarwin() ? "$s15SwiftFoundation19_NSCFConstantStringCN"
6214
                              : "$s10Foundation19_NSCFConstantStringCN";
6215
      Ty = IntPtrTy;
6216
      break;
6217
    case LangOptions::CoreFoundationABI::Swift4_2:
6218
      CFConstantStringClassName =
6219
          Triple.isOSDarwin() ? "$S15SwiftFoundation19_NSCFConstantStringCN"
6220
                              : "$S10Foundation19_NSCFConstantStringCN";
6221
      Ty = IntPtrTy;
6222
      break;
6223
    case LangOptions::CoreFoundationABI::Swift4_1:
6224
      CFConstantStringClassName =
6225
          Triple.isOSDarwin() ? "__T015SwiftFoundation19_NSCFConstantStringCN"
6226
                              : "__T010Foundation19_NSCFConstantStringCN";
6227
      Ty = IntPtrTy;
6228
      break;
6229
    }
6230

6231
    llvm::Constant *C = CreateRuntimeVariable(Ty, CFConstantStringClassName);
6232

6233
    if (Triple.isOSBinFormatELF() || Triple.isOSBinFormatCOFF()) {
6234
      llvm::GlobalValue *GV = nullptr;
6235

6236
      if ((GV = dyn_cast<llvm::GlobalValue>(C))) {
6237
        IdentifierInfo &II = Context.Idents.get(GV->getName());
6238
        TranslationUnitDecl *TUDecl = Context.getTranslationUnitDecl();
6239
        DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
6240

6241
        const VarDecl *VD = nullptr;
6242
        for (const auto *Result : DC->lookup(&II))
6243
          if ((VD = dyn_cast<VarDecl>(Result)))
6244
            break;
6245

6246
        if (Triple.isOSBinFormatELF()) {
6247
          if (!VD)
6248
            GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
6249
        } else {
6250
          GV->setLinkage(llvm::GlobalValue::ExternalLinkage);
6251
          if (!VD || !VD->hasAttr<DLLExportAttr>())
6252
            GV->setDLLStorageClass(llvm::GlobalValue::DLLImportStorageClass);
6253
          else
6254
            GV->setDLLStorageClass(llvm::GlobalValue::DLLExportStorageClass);
6255
        }
6256

6257
        setDSOLocal(GV);
6258
      }
6259
    }
6260

6261
    // Decay array -> ptr
6262
    CFConstantStringClassRef =
6263
        IsSwiftABI ? llvm::ConstantExpr::getPtrToInt(C, Ty) : C;
6264
  }
6265

6266
  QualType CFTy = Context.getCFConstantStringType();
6267

6268
  auto *STy = cast<llvm::StructType>(getTypes().ConvertType(CFTy));
6269

6270
  ConstantInitBuilder Builder(*this);
6271
  auto Fields = Builder.beginStruct(STy);
6272

6273
  // Class pointer.
6274
  Fields.add(cast<llvm::Constant>(CFConstantStringClassRef));
6275

6276
  // Flags.
6277
  if (IsSwiftABI) {
6278
    Fields.addInt(IntPtrTy, IsSwift4_1 ? 0x05 : 0x01);
6279
    Fields.addInt(Int64Ty, isUTF16 ? 0x07d0 : 0x07c8);
6280
  } else {
6281
    Fields.addInt(IntTy, isUTF16 ? 0x07d0 : 0x07C8);
6282
  }
6283

6284
  // String pointer.
6285
  llvm::Constant *C = nullptr;
6286
  if (isUTF16) {
6287
    auto Arr = llvm::ArrayRef(
6288
        reinterpret_cast<uint16_t *>(const_cast<char *>(Entry.first().data())),
6289
        Entry.first().size() / 2);
6290
    C = llvm::ConstantDataArray::get(VMContext, Arr);
6291
  } else {
6292
    C = llvm::ConstantDataArray::getString(VMContext, Entry.first());
6293
  }
6294

6295
  // Note: -fwritable-strings doesn't make the backing store strings of
6296
  // CFStrings writable.
6297
  auto *GV =
6298
      new llvm::GlobalVariable(getModule(), C->getType(), /*isConstant=*/true,
6299
                               llvm::GlobalValue::PrivateLinkage, C, ".str");
6300
  GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6301
  // Don't enforce the target's minimum global alignment, since the only use
6302
  // of the string is via this class initializer.
6303
  CharUnits Align = isUTF16 ? Context.getTypeAlignInChars(Context.ShortTy)
6304
                            : Context.getTypeAlignInChars(Context.CharTy);
6305
  GV->setAlignment(Align.getAsAlign());
6306

6307
  // FIXME: We set the section explicitly to avoid a bug in ld64 224.1.
6308
  // Without it LLVM can merge the string with a non unnamed_addr one during
6309
  // LTO.  Doing that changes the section it ends in, which surprises ld64.
6310
  if (Triple.isOSBinFormatMachO())
6311
    GV->setSection(isUTF16 ? "__TEXT,__ustring"
6312
                           : "__TEXT,__cstring,cstring_literals");
6313
  // Make sure the literal ends up in .rodata to allow for safe ICF and for
6314
  // the static linker to adjust permissions to read-only later on.
6315
  else if (Triple.isOSBinFormatELF())
6316
    GV->setSection(".rodata");
6317

6318
  // String.
6319
  Fields.add(GV);
6320

6321
  // String length.
6322
  llvm::IntegerType *LengthTy =
6323
      llvm::IntegerType::get(getModule().getContext(),
6324
                             Context.getTargetInfo().getLongWidth());
6325
  if (IsSwiftABI) {
6326
    if (CFRuntime == LangOptions::CoreFoundationABI::Swift4_1 ||
6327
        CFRuntime == LangOptions::CoreFoundationABI::Swift4_2)
6328
      LengthTy = Int32Ty;
6329
    else
6330
      LengthTy = IntPtrTy;
6331
  }
6332
  Fields.addInt(LengthTy, StringLength);
6333

6334
  // Swift ABI requires 8-byte alignment to ensure that the _Atomic(uint64_t) is
6335
  // properly aligned on 32-bit platforms.
6336
  CharUnits Alignment =
6337
      IsSwiftABI ? Context.toCharUnitsFromBits(64) : getPointerAlign();
6338

6339
  // The struct.
6340
  GV = Fields.finishAndCreateGlobal("_unnamed_cfstring_", Alignment,
6341
                                    /*isConstant=*/false,
6342
                                    llvm::GlobalVariable::PrivateLinkage);
6343
  GV->addAttribute("objc_arc_inert");
6344
  switch (Triple.getObjectFormat()) {
6345
  case llvm::Triple::UnknownObjectFormat:
6346
    llvm_unreachable("unknown file format");
6347
  case llvm::Triple::DXContainer:
6348
  case llvm::Triple::GOFF:
6349
  case llvm::Triple::SPIRV:
6350
  case llvm::Triple::XCOFF:
6351
    llvm_unreachable("unimplemented");
6352
  case llvm::Triple::COFF:
6353
  case llvm::Triple::ELF:
6354
  case llvm::Triple::Wasm:
6355
    GV->setSection("cfstring");
6356
    break;
6357
  case llvm::Triple::MachO:
6358
    GV->setSection("__DATA,__cfstring");
6359
    break;
6360
  }
6361
  Entry.second = GV;
6362

6363
  return ConstantAddress(GV, GV->getValueType(), Alignment);
6364
}
6365

6366
bool CodeGenModule::getExpressionLocationsEnabled() const {
6367
  return !CodeGenOpts.EmitCodeView || CodeGenOpts.DebugColumnInfo;
6368
}
6369

6370
QualType CodeGenModule::getObjCFastEnumerationStateType() {
6371
  if (ObjCFastEnumerationStateType.isNull()) {
6372
    RecordDecl *D = Context.buildImplicitRecord("__objcFastEnumerationState");
6373
    D->startDefinition();
6374

6375
    QualType FieldTypes[] = {
6376
        Context.UnsignedLongTy, Context.getPointerType(Context.getObjCIdType()),
6377
        Context.getPointerType(Context.UnsignedLongTy),
6378
        Context.getConstantArrayType(Context.UnsignedLongTy, llvm::APInt(32, 5),
6379
                                     nullptr, ArraySizeModifier::Normal, 0)};
6380

6381
    for (size_t i = 0; i < 4; ++i) {
6382
      FieldDecl *Field = FieldDecl::Create(Context,
6383
                                           D,
6384
                                           SourceLocation(),
6385
                                           SourceLocation(), nullptr,
6386
                                           FieldTypes[i], /*TInfo=*/nullptr,
6387
                                           /*BitWidth=*/nullptr,
6388
                                           /*Mutable=*/false,
6389
                                           ICIS_NoInit);
6390
      Field->setAccess(AS_public);
6391
      D->addDecl(Field);
6392
    }
6393

6394
    D->completeDefinition();
6395
    ObjCFastEnumerationStateType = Context.getTagDeclType(D);
6396
  }
6397

6398
  return ObjCFastEnumerationStateType;
6399
}
6400

6401
llvm::Constant *
6402
CodeGenModule::GetConstantArrayFromStringLiteral(const StringLiteral *E) {
6403
  assert(!E->getType()->isPointerType() && "Strings are always arrays");
6404

6405
  // Don't emit it as the address of the string, emit the string data itself
6406
  // as an inline array.
6407
  if (E->getCharByteWidth() == 1) {
6408
    SmallString<64> Str(E->getString());
6409

6410
    // Resize the string to the right size, which is indicated by its type.
6411
    const ConstantArrayType *CAT = Context.getAsConstantArrayType(E->getType());
6412
    assert(CAT && "String literal not of constant array type!");
6413
    Str.resize(CAT->getZExtSize());
6414
    return llvm::ConstantDataArray::getString(VMContext, Str, false);
6415
  }
6416

6417
  auto *AType = cast<llvm::ArrayType>(getTypes().ConvertType(E->getType()));
6418
  llvm::Type *ElemTy = AType->getElementType();
6419
  unsigned NumElements = AType->getNumElements();
6420

6421
  // Wide strings have either 2-byte or 4-byte elements.
6422
  if (ElemTy->getPrimitiveSizeInBits() == 16) {
6423
    SmallVector<uint16_t, 32> Elements;
6424
    Elements.reserve(NumElements);
6425

6426
    for(unsigned i = 0, e = E->getLength(); i != e; ++i)
6427
      Elements.push_back(E->getCodeUnit(i));
6428
    Elements.resize(NumElements);
6429
    return llvm::ConstantDataArray::get(VMContext, Elements);
6430
  }
6431

6432
  assert(ElemTy->getPrimitiveSizeInBits() == 32);
6433
  SmallVector<uint32_t, 32> Elements;
6434
  Elements.reserve(NumElements);
6435

6436
  for(unsigned i = 0, e = E->getLength(); i != e; ++i)
6437
    Elements.push_back(E->getCodeUnit(i));
6438
  Elements.resize(NumElements);
6439
  return llvm::ConstantDataArray::get(VMContext, Elements);
6440
}
6441

6442
static llvm::GlobalVariable *
6443
GenerateStringLiteral(llvm::Constant *C, llvm::GlobalValue::LinkageTypes LT,
6444
                      CodeGenModule &CGM, StringRef GlobalName,
6445
                      CharUnits Alignment) {
6446
  unsigned AddrSpace = CGM.getContext().getTargetAddressSpace(
6447
      CGM.GetGlobalConstantAddressSpace());
6448

6449
  llvm::Module &M = CGM.getModule();
6450
  // Create a global variable for this string
6451
  auto *GV = new llvm::GlobalVariable(
6452
      M, C->getType(), !CGM.getLangOpts().WritableStrings, LT, C, GlobalName,
6453
      nullptr, llvm::GlobalVariable::NotThreadLocal, AddrSpace);
6454
  GV->setAlignment(Alignment.getAsAlign());
6455
  GV->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
6456
  if (GV->isWeakForLinker()) {
6457
    assert(CGM.supportsCOMDAT() && "Only COFF uses weak string literals");
6458
    GV->setComdat(M.getOrInsertComdat(GV->getName()));
6459
  }
6460
  CGM.setDSOLocal(GV);
6461

6462
  return GV;
6463
}
6464

6465
/// GetAddrOfConstantStringFromLiteral - Return a pointer to a
6466
/// constant array for the given string literal.
6467
ConstantAddress
6468
CodeGenModule::GetAddrOfConstantStringFromLiteral(const StringLiteral *S,
6469
                                                  StringRef Name) {
6470
  CharUnits Alignment =
6471
      getContext().getAlignOfGlobalVarInChars(S->getType(), /*VD=*/nullptr);
6472

6473
  llvm::Constant *C = GetConstantArrayFromStringLiteral(S);
6474
  llvm::GlobalVariable **Entry = nullptr;
6475
  if (!LangOpts.WritableStrings) {
6476
    Entry = &ConstantStringMap[C];
6477
    if (auto GV = *Entry) {
6478
      if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
6479
        GV->setAlignment(Alignment.getAsAlign());
6480
      return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
6481
                             GV->getValueType(), Alignment);
6482
    }
6483
  }
6484

6485
  SmallString<256> MangledNameBuffer;
6486
  StringRef GlobalVariableName;
6487
  llvm::GlobalValue::LinkageTypes LT;
6488

6489
  // Mangle the string literal if that's how the ABI merges duplicate strings.
6490
  // Don't do it if they are writable, since we don't want writes in one TU to
6491
  // affect strings in another.
6492
  if (getCXXABI().getMangleContext().shouldMangleStringLiteral(S) &&
6493
      !LangOpts.WritableStrings) {
6494
    llvm::raw_svector_ostream Out(MangledNameBuffer);
6495
    getCXXABI().getMangleContext().mangleStringLiteral(S, Out);
6496
    LT = llvm::GlobalValue::LinkOnceODRLinkage;
6497
    GlobalVariableName = MangledNameBuffer;
6498
  } else {
6499
    LT = llvm::GlobalValue::PrivateLinkage;
6500
    GlobalVariableName = Name;
6501
  }
6502

6503
  auto GV = GenerateStringLiteral(C, LT, *this, GlobalVariableName, Alignment);
6504

6505
  CGDebugInfo *DI = getModuleDebugInfo();
6506
  if (DI && getCodeGenOpts().hasReducedDebugInfo())
6507
    DI->AddStringLiteralDebugInfo(GV, S);
6508

6509
  if (Entry)
6510
    *Entry = GV;
6511

6512
  SanitizerMD->reportGlobal(GV, S->getStrTokenLoc(0), "<string literal>");
6513

6514
  return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
6515
                         GV->getValueType(), Alignment);
6516
}
6517

6518
/// GetAddrOfConstantStringFromObjCEncode - Return a pointer to a constant
6519
/// array for the given ObjCEncodeExpr node.
6520
ConstantAddress
6521
CodeGenModule::GetAddrOfConstantStringFromObjCEncode(const ObjCEncodeExpr *E) {
6522
  std::string Str;
6523
  getContext().getObjCEncodingForType(E->getEncodedType(), Str);
6524

6525
  return GetAddrOfConstantCString(Str);
6526
}
6527

6528
/// GetAddrOfConstantCString - Returns a pointer to a character array containing
6529
/// the literal and a terminating '\0' character.
6530
/// The result has pointer to array type.
6531
ConstantAddress CodeGenModule::GetAddrOfConstantCString(
6532
    const std::string &Str, const char *GlobalName) {
6533
  StringRef StrWithNull(Str.c_str(), Str.size() + 1);
6534
  CharUnits Alignment = getContext().getAlignOfGlobalVarInChars(
6535
      getContext().CharTy, /*VD=*/nullptr);
6536

6537
  llvm::Constant *C =
6538
      llvm::ConstantDataArray::getString(getLLVMContext(), StrWithNull, false);
6539

6540
  // Don't share any string literals if strings aren't constant.
6541
  llvm::GlobalVariable **Entry = nullptr;
6542
  if (!LangOpts.WritableStrings) {
6543
    Entry = &ConstantStringMap[C];
6544
    if (auto GV = *Entry) {
6545
      if (uint64_t(Alignment.getQuantity()) > GV->getAlignment())
6546
        GV->setAlignment(Alignment.getAsAlign());
6547
      return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
6548
                             GV->getValueType(), Alignment);
6549
    }
6550
  }
6551

6552
  // Get the default prefix if a name wasn't specified.
6553
  if (!GlobalName)
6554
    GlobalName = ".str";
6555
  // Create a global variable for this.
6556
  auto GV = GenerateStringLiteral(C, llvm::GlobalValue::PrivateLinkage, *this,
6557
                                  GlobalName, Alignment);
6558
  if (Entry)
6559
    *Entry = GV;
6560

6561
  return ConstantAddress(castStringLiteralToDefaultAddressSpace(*this, GV),
6562
                         GV->getValueType(), Alignment);
6563
}
6564

6565
ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary(
6566
    const MaterializeTemporaryExpr *E, const Expr *Init) {
6567
  assert((E->getStorageDuration() == SD_Static ||
6568
          E->getStorageDuration() == SD_Thread) && "not a global temporary");
6569
  const auto *VD = cast<VarDecl>(E->getExtendingDecl());
6570

6571
  // If we're not materializing a subobject of the temporary, keep the
6572
  // cv-qualifiers from the type of the MaterializeTemporaryExpr.
6573
  QualType MaterializedType = Init->getType();
6574
  if (Init == E->getSubExpr())
6575
    MaterializedType = E->getType();
6576

6577
  CharUnits Align = getContext().getTypeAlignInChars(MaterializedType);
6578

6579
  auto InsertResult = MaterializedGlobalTemporaryMap.insert({E, nullptr});
6580
  if (!InsertResult.second) {
6581
    // We've seen this before: either we already created it or we're in the
6582
    // process of doing so.
6583
    if (!InsertResult.first->second) {
6584
      // We recursively re-entered this function, probably during emission of
6585
      // the initializer. Create a placeholder. We'll clean this up in the
6586
      // outer call, at the end of this function.
6587
      llvm::Type *Type = getTypes().ConvertTypeForMem(MaterializedType);
6588
      InsertResult.first->second = new llvm::GlobalVariable(
6589
          getModule(), Type, false, llvm::GlobalVariable::InternalLinkage,
6590
          nullptr);
6591
    }
6592
    return ConstantAddress(InsertResult.first->second,
6593
                           llvm::cast<llvm::GlobalVariable>(
6594
                               InsertResult.first->second->stripPointerCasts())
6595
                               ->getValueType(),
6596
                           Align);
6597
  }
6598

6599
  // FIXME: If an externally-visible declaration extends multiple temporaries,
6600
  // we need to give each temporary the same name in every translation unit (and
6601
  // we also need to make the temporaries externally-visible).
6602
  SmallString<256> Name;
6603
  llvm::raw_svector_ostream Out(Name);
6604
  getCXXABI().getMangleContext().mangleReferenceTemporary(
6605
      VD, E->getManglingNumber(), Out);
6606

6607
  APValue *Value = nullptr;
6608
  if (E->getStorageDuration() == SD_Static && VD->evaluateValue()) {
6609
    // If the initializer of the extending declaration is a constant
6610
    // initializer, we should have a cached constant initializer for this
6611
    // temporary. Note that this might have a different value from the value
6612
    // computed by evaluating the initializer if the surrounding constant
6613
    // expression modifies the temporary.
6614
    Value = E->getOrCreateValue(false);
6615
  }
6616

6617
  // Try evaluating it now, it might have a constant initializer.
6618
  Expr::EvalResult EvalResult;
6619
  if (!Value && Init->EvaluateAsRValue(EvalResult, getContext()) &&
6620
      !EvalResult.hasSideEffects())
6621
    Value = &EvalResult.Val;
6622

6623
  LangAS AddrSpace = GetGlobalVarAddressSpace(VD);
6624

6625
  std::optional<ConstantEmitter> emitter;
6626
  llvm::Constant *InitialValue = nullptr;
6627
  bool Constant = false;
6628
  llvm::Type *Type;
6629
  if (Value) {
6630
    // The temporary has a constant initializer, use it.
6631
    emitter.emplace(*this);
6632
    InitialValue = emitter->emitForInitializer(*Value, AddrSpace,
6633
                                               MaterializedType);
6634
    Constant =
6635
        MaterializedType.isConstantStorage(getContext(), /*ExcludeCtor*/ Value,
6636
                                           /*ExcludeDtor*/ false);
6637
    Type = InitialValue->getType();
6638
  } else {
6639
    // No initializer, the initialization will be provided when we
6640
    // initialize the declaration which performed lifetime extension.
6641
    Type = getTypes().ConvertTypeForMem(MaterializedType);
6642
  }
6643

6644
  // Create a global variable for this lifetime-extended temporary.
6645
  llvm::GlobalValue::LinkageTypes Linkage = getLLVMLinkageVarDefinition(VD);
6646
  if (Linkage == llvm::GlobalVariable::ExternalLinkage) {
6647
    const VarDecl *InitVD;
6648
    if (VD->isStaticDataMember() && VD->getAnyInitializer(InitVD) &&
6649
        isa<CXXRecordDecl>(InitVD->getLexicalDeclContext())) {
6650
      // Temporaries defined inside a class get linkonce_odr linkage because the
6651
      // class can be defined in multiple translation units.
6652
      Linkage = llvm::GlobalVariable::LinkOnceODRLinkage;
6653
    } else {
6654
      // There is no need for this temporary to have external linkage if the
6655
      // VarDecl has external linkage.
6656
      Linkage = llvm::GlobalVariable::InternalLinkage;
6657
    }
6658
  }
6659
  auto TargetAS = getContext().getTargetAddressSpace(AddrSpace);
6660
  auto *GV = new llvm::GlobalVariable(
6661
      getModule(), Type, Constant, Linkage, InitialValue, Name.c_str(),
6662
      /*InsertBefore=*/nullptr, llvm::GlobalVariable::NotThreadLocal, TargetAS);
6663
  if (emitter) emitter->finalize(GV);
6664
  // Don't assign dllimport or dllexport to local linkage globals.
6665
  if (!llvm::GlobalValue::isLocalLinkage(Linkage)) {
6666
    setGVProperties(GV, VD);
6667
    if (GV->getDLLStorageClass() == llvm::GlobalVariable::DLLExportStorageClass)
6668
      // The reference temporary should never be dllexport.
6669
      GV->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
6670
  }
6671
  GV->setAlignment(Align.getAsAlign());
6672
  if (supportsCOMDAT() && GV->isWeakForLinker())
6673
    GV->setComdat(TheModule.getOrInsertComdat(GV->getName()));
6674
  if (VD->getTLSKind())
6675
    setTLSMode(GV, *VD);
6676
  llvm::Constant *CV = GV;
6677
  if (AddrSpace != LangAS::Default)
6678
    CV = getTargetCodeGenInfo().performAddrSpaceCast(
6679
        *this, GV, AddrSpace, LangAS::Default,
6680
        llvm::PointerType::get(
6681
            getLLVMContext(),
6682
            getContext().getTargetAddressSpace(LangAS::Default)));
6683

6684
  // Update the map with the new temporary. If we created a placeholder above,
6685
  // replace it with the new global now.
6686
  llvm::Constant *&Entry = MaterializedGlobalTemporaryMap[E];
6687
  if (Entry) {
6688
    Entry->replaceAllUsesWith(CV);
6689
    llvm::cast<llvm::GlobalVariable>(Entry)->eraseFromParent();
6690
  }
6691
  Entry = CV;
6692

6693
  return ConstantAddress(CV, Type, Align);
6694
}
6695

6696
/// EmitObjCPropertyImplementations - Emit information for synthesized
6697
/// properties for an implementation.
6698
void CodeGenModule::EmitObjCPropertyImplementations(const
6699
                                                    ObjCImplementationDecl *D) {
6700
  for (const auto *PID : D->property_impls()) {
6701
    // Dynamic is just for type-checking.
6702
    if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Synthesize) {
6703
      ObjCPropertyDecl *PD = PID->getPropertyDecl();
6704

6705
      // Determine which methods need to be implemented, some may have
6706
      // been overridden. Note that ::isPropertyAccessor is not the method
6707
      // we want, that just indicates if the decl came from a
6708
      // property. What we want to know is if the method is defined in
6709
      // this implementation.
6710
      auto *Getter = PID->getGetterMethodDecl();
6711
      if (!Getter || Getter->isSynthesizedAccessorStub())
6712
        CodeGenFunction(*this).GenerateObjCGetter(
6713
            const_cast<ObjCImplementationDecl *>(D), PID);
6714
      auto *Setter = PID->getSetterMethodDecl();
6715
      if (!PD->isReadOnly() && (!Setter || Setter->isSynthesizedAccessorStub()))
6716
        CodeGenFunction(*this).GenerateObjCSetter(
6717
                                 const_cast<ObjCImplementationDecl *>(D), PID);
6718
    }
6719
  }
6720
}
6721

6722
static bool needsDestructMethod(ObjCImplementationDecl *impl) {
6723
  const ObjCInterfaceDecl *iface = impl->getClassInterface();
6724
  for (const ObjCIvarDecl *ivar = iface->all_declared_ivar_begin();
6725
       ivar; ivar = ivar->getNextIvar())
6726
    if (ivar->getType().isDestructedType())
6727
      return true;
6728

6729
  return false;
6730
}
6731

6732
static bool AllTrivialInitializers(CodeGenModule &CGM,
6733
                                   ObjCImplementationDecl *D) {
6734
  CodeGenFunction CGF(CGM);
6735
  for (ObjCImplementationDecl::init_iterator B = D->init_begin(),
6736
       E = D->init_end(); B != E; ++B) {
6737
    CXXCtorInitializer *CtorInitExp = *B;
6738
    Expr *Init = CtorInitExp->getInit();
6739
    if (!CGF.isTrivialInitializer(Init))
6740
      return false;
6741
  }
6742
  return true;
6743
}
6744

6745
/// EmitObjCIvarInitializations - Emit information for ivar initialization
6746
/// for an implementation.
6747
void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
6748
  // We might need a .cxx_destruct even if we don't have any ivar initializers.
6749
  if (needsDestructMethod(D)) {
6750
    const IdentifierInfo *II = &getContext().Idents.get(".cxx_destruct");
6751
    Selector cxxSelector = getContext().Selectors.getSelector(0, &II);
6752
    ObjCMethodDecl *DTORMethod = ObjCMethodDecl::Create(
6753
        getContext(), D->getLocation(), D->getLocation(), cxxSelector,
6754
        getContext().VoidTy, nullptr, D,
6755
        /*isInstance=*/true, /*isVariadic=*/false,
6756
        /*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
6757
        /*isImplicitlyDeclared=*/true,
6758
        /*isDefined=*/false, ObjCImplementationControl::Required);
6759
    D->addInstanceMethod(DTORMethod);
6760
    CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, DTORMethod, false);
6761
    D->setHasDestructors(true);
6762
  }
6763

6764
  // If the implementation doesn't have any ivar initializers, we don't need
6765
  // a .cxx_construct.
6766
  if (D->getNumIvarInitializers() == 0 ||
6767
      AllTrivialInitializers(*this, D))
6768
    return;
6769

6770
  const IdentifierInfo *II = &getContext().Idents.get(".cxx_construct");
6771
  Selector cxxSelector = getContext().Selectors.getSelector(0, &II);
6772
  // The constructor returns 'self'.
6773
  ObjCMethodDecl *CTORMethod = ObjCMethodDecl::Create(
6774
      getContext(), D->getLocation(), D->getLocation(), cxxSelector,
6775
      getContext().getObjCIdType(), nullptr, D, /*isInstance=*/true,
6776
      /*isVariadic=*/false,
6777
      /*isPropertyAccessor=*/true, /*isSynthesizedAccessorStub=*/false,
6778
      /*isImplicitlyDeclared=*/true,
6779
      /*isDefined=*/false, ObjCImplementationControl::Required);
6780
  D->addInstanceMethod(CTORMethod);
6781
  CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, CTORMethod, true);
6782
  D->setHasNonZeroConstructors(true);
6783
}
6784

6785
// EmitLinkageSpec - Emit all declarations in a linkage spec.
6786
void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) {
6787
  if (LSD->getLanguage() != LinkageSpecLanguageIDs::C &&
6788
      LSD->getLanguage() != LinkageSpecLanguageIDs::CXX) {
6789
    ErrorUnsupported(LSD, "linkage spec");
6790
    return;
6791
  }
6792

6793
  EmitDeclContext(LSD);
6794
}
6795

6796
void CodeGenModule::EmitTopLevelStmt(const TopLevelStmtDecl *D) {
6797
  // Device code should not be at top level.
6798
  if (LangOpts.CUDA && LangOpts.CUDAIsDevice)
6799
    return;
6800

6801
  std::unique_ptr<CodeGenFunction> &CurCGF =
6802
      GlobalTopLevelStmtBlockInFlight.first;
6803

6804
  // We emitted a top-level stmt but after it there is initialization.
6805
  // Stop squashing the top-level stmts into a single function.
6806
  if (CurCGF && CXXGlobalInits.back() != CurCGF->CurFn) {
6807
    CurCGF->FinishFunction(D->getEndLoc());
6808
    CurCGF = nullptr;
6809
  }
6810

6811
  if (!CurCGF) {
6812
    // void __stmts__N(void)
6813
    // FIXME: Ask the ABI name mangler to pick a name.
6814
    std::string Name = "__stmts__" + llvm::utostr(CXXGlobalInits.size());
6815
    FunctionArgList Args;
6816
    QualType RetTy = getContext().VoidTy;
6817
    const CGFunctionInfo &FnInfo =
6818
        getTypes().arrangeBuiltinFunctionDeclaration(RetTy, Args);
6819
    llvm::FunctionType *FnTy = getTypes().GetFunctionType(FnInfo);
6820
    llvm::Function *Fn = llvm::Function::Create(
6821
        FnTy, llvm::GlobalValue::InternalLinkage, Name, &getModule());
6822

6823
    CurCGF.reset(new CodeGenFunction(*this));
6824
    GlobalTopLevelStmtBlockInFlight.second = D;
6825
    CurCGF->StartFunction(GlobalDecl(), RetTy, Fn, FnInfo, Args,
6826
                          D->getBeginLoc(), D->getBeginLoc());
6827
    CXXGlobalInits.push_back(Fn);
6828
  }
6829

6830
  CurCGF->EmitStmt(D->getStmt());
6831
}
6832

6833
void CodeGenModule::EmitDeclContext(const DeclContext *DC) {
6834
  for (auto *I : DC->decls()) {
6835
    // Unlike other DeclContexts, the contents of an ObjCImplDecl at TU scope
6836
    // are themselves considered "top-level", so EmitTopLevelDecl on an
6837
    // ObjCImplDecl does not recursively visit them. We need to do that in
6838
    // case they're nested inside another construct (LinkageSpecDecl /
6839
    // ExportDecl) that does stop them from being considered "top-level".
6840
    if (auto *OID = dyn_cast<ObjCImplDecl>(I)) {
6841
      for (auto *M : OID->methods())
6842
        EmitTopLevelDecl(M);
6843
    }
6844

6845
    EmitTopLevelDecl(I);
6846
  }
6847
}
6848

6849
/// EmitTopLevelDecl - Emit code for a single top level declaration.
6850
void CodeGenModule::EmitTopLevelDecl(Decl *D) {
6851
  // Ignore dependent declarations.
6852
  if (D->isTemplated())
6853
    return;
6854

6855
  // Consteval function shouldn't be emitted.
6856
  if (auto *FD = dyn_cast<FunctionDecl>(D); FD && FD->isImmediateFunction())
6857
    return;
6858

6859
  switch (D->getKind()) {
6860
  case Decl::CXXConversion:
6861
  case Decl::CXXMethod:
6862
  case Decl::Function:
6863
    EmitGlobal(cast<FunctionDecl>(D));
6864
    // Always provide some coverage mapping
6865
    // even for the functions that aren't emitted.
6866
    AddDeferredUnusedCoverageMapping(D);
6867
    break;
6868

6869
  case Decl::CXXDeductionGuide:
6870
    // Function-like, but does not result in code emission.
6871
    break;
6872

6873
  case Decl::Var:
6874
  case Decl::Decomposition:
6875
  case Decl::VarTemplateSpecialization:
6876
    EmitGlobal(cast<VarDecl>(D));
6877
    if (auto *DD = dyn_cast<DecompositionDecl>(D))
6878
      for (auto *B : DD->bindings())
6879
        if (auto *HD = B->getHoldingVar())
6880
          EmitGlobal(HD);
6881
    break;
6882

6883
  // Indirect fields from global anonymous structs and unions can be
6884
  // ignored; only the actual variable requires IR gen support.
6885
  case Decl::IndirectField:
6886
    break;
6887

6888
  // C++ Decls
6889
  case Decl::Namespace:
6890
    EmitDeclContext(cast<NamespaceDecl>(D));
6891
    break;
6892
  case Decl::ClassTemplateSpecialization: {
6893
    const auto *Spec = cast<ClassTemplateSpecializationDecl>(D);
6894
    if (CGDebugInfo *DI = getModuleDebugInfo())
6895
      if (Spec->getSpecializationKind() ==
6896
              TSK_ExplicitInstantiationDefinition &&
6897
          Spec->hasDefinition())
6898
        DI->completeTemplateDefinition(*Spec);
6899
  } [[fallthrough]];
6900
  case Decl::CXXRecord: {
6901
    CXXRecordDecl *CRD = cast<CXXRecordDecl>(D);
6902
    if (CGDebugInfo *DI = getModuleDebugInfo()) {
6903
      if (CRD->hasDefinition())
6904
        DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(D)));
6905
      if (auto *ES = D->getASTContext().getExternalSource())
6906
        if (ES->hasExternalDefinitions(D) == ExternalASTSource::EK_Never)
6907
          DI->completeUnusedClass(*CRD);
6908
    }
6909
    // Emit any static data members, they may be definitions.
6910
    for (auto *I : CRD->decls())
6911
      if (isa<VarDecl>(I) || isa<CXXRecordDecl>(I))
6912
        EmitTopLevelDecl(I);
6913
    break;
6914
  }
6915
    // No code generation needed.
6916
  case Decl::UsingShadow:
6917
  case Decl::ClassTemplate:
6918
  case Decl::VarTemplate:
6919
  case Decl::Concept:
6920
  case Decl::VarTemplatePartialSpecialization:
6921
  case Decl::FunctionTemplate:
6922
  case Decl::TypeAliasTemplate:
6923
  case Decl::Block:
6924
  case Decl::Empty:
6925
  case Decl::Binding:
6926
    break;
6927
  case Decl::Using:          // using X; [C++]
6928
    if (CGDebugInfo *DI = getModuleDebugInfo())
6929
        DI->EmitUsingDecl(cast<UsingDecl>(*D));
6930
    break;
6931
  case Decl::UsingEnum: // using enum X; [C++]
6932
    if (CGDebugInfo *DI = getModuleDebugInfo())
6933
      DI->EmitUsingEnumDecl(cast<UsingEnumDecl>(*D));
6934
    break;
6935
  case Decl::NamespaceAlias:
6936
    if (CGDebugInfo *DI = getModuleDebugInfo())
6937
        DI->EmitNamespaceAlias(cast<NamespaceAliasDecl>(*D));
6938
    break;
6939
  case Decl::UsingDirective: // using namespace X; [C++]
6940
    if (CGDebugInfo *DI = getModuleDebugInfo())
6941
      DI->EmitUsingDirective(cast<UsingDirectiveDecl>(*D));
6942
    break;
6943
  case Decl::CXXConstructor:
6944
    getCXXABI().EmitCXXConstructors(cast<CXXConstructorDecl>(D));
6945
    break;
6946
  case Decl::CXXDestructor:
6947
    getCXXABI().EmitCXXDestructors(cast<CXXDestructorDecl>(D));
6948
    break;
6949

6950
  case Decl::StaticAssert:
6951
    // Nothing to do.
6952
    break;
6953

6954
  // Objective-C Decls
6955

6956
  // Forward declarations, no (immediate) code generation.
6957
  case Decl::ObjCInterface:
6958
  case Decl::ObjCCategory:
6959
    break;
6960

6961
  case Decl::ObjCProtocol: {
6962
    auto *Proto = cast<ObjCProtocolDecl>(D);
6963
    if (Proto->isThisDeclarationADefinition())
6964
      ObjCRuntime->GenerateProtocol(Proto);
6965
    break;
6966
  }
6967

6968
  case Decl::ObjCCategoryImpl:
6969
    // Categories have properties but don't support synthesize so we
6970
    // can ignore them here.
6971
    ObjCRuntime->GenerateCategory(cast<ObjCCategoryImplDecl>(D));
6972
    break;
6973

6974
  case Decl::ObjCImplementation: {
6975
    auto *OMD = cast<ObjCImplementationDecl>(D);
6976
    EmitObjCPropertyImplementations(OMD);
6977
    EmitObjCIvarInitializations(OMD);
6978
    ObjCRuntime->GenerateClass(OMD);
6979
    // Emit global variable debug information.
6980
    if (CGDebugInfo *DI = getModuleDebugInfo())
6981
      if (getCodeGenOpts().hasReducedDebugInfo())
6982
        DI->getOrCreateInterfaceType(getContext().getObjCInterfaceType(
6983
            OMD->getClassInterface()), OMD->getLocation());
6984
    break;
6985
  }
6986
  case Decl::ObjCMethod: {
6987
    auto *OMD = cast<ObjCMethodDecl>(D);
6988
    // If this is not a prototype, emit the body.
6989
    if (OMD->getBody())
6990
      CodeGenFunction(*this).GenerateObjCMethod(OMD);
6991
    break;
6992
  }
6993
  case Decl::ObjCCompatibleAlias:
6994
    ObjCRuntime->RegisterAlias(cast<ObjCCompatibleAliasDecl>(D));
6995
    break;
6996

6997
  case Decl::PragmaComment: {
6998
    const auto *PCD = cast<PragmaCommentDecl>(D);
6999
    switch (PCD->getCommentKind()) {
7000
    case PCK_Unknown:
7001
      llvm_unreachable("unexpected pragma comment kind");
7002
    case PCK_Linker:
7003
      AppendLinkerOptions(PCD->getArg());
7004
      break;
7005
    case PCK_Lib:
7006
        AddDependentLib(PCD->getArg());
7007
      break;
7008
    case PCK_Compiler:
7009
    case PCK_ExeStr:
7010
    case PCK_User:
7011
      break; // We ignore all of these.
7012
    }
7013
    break;
7014
  }
7015

7016
  case Decl::PragmaDetectMismatch: {
7017
    const auto *PDMD = cast<PragmaDetectMismatchDecl>(D);
7018
    AddDetectMismatch(PDMD->getName(), PDMD->getValue());
7019
    break;
7020
  }
7021

7022
  case Decl::LinkageSpec:
7023
    EmitLinkageSpec(cast<LinkageSpecDecl>(D));
7024
    break;
7025

7026
  case Decl::FileScopeAsm: {
7027
    // File-scope asm is ignored during device-side CUDA compilation.
7028
    if (LangOpts.CUDA && LangOpts.CUDAIsDevice)
7029
      break;
7030
    // File-scope asm is ignored during device-side OpenMP compilation.
7031
    if (LangOpts.OpenMPIsTargetDevice)
7032
      break;
7033
    // File-scope asm is ignored during device-side SYCL compilation.
7034
    if (LangOpts.SYCLIsDevice)
7035
      break;
7036
    auto *AD = cast<FileScopeAsmDecl>(D);
7037
    getModule().appendModuleInlineAsm(AD->getAsmString()->getString());
7038
    break;
7039
  }
7040

7041
  case Decl::TopLevelStmt:
7042
    EmitTopLevelStmt(cast<TopLevelStmtDecl>(D));
7043
    break;
7044

7045
  case Decl::Import: {
7046
    auto *Import = cast<ImportDecl>(D);
7047

7048
    // If we've already imported this module, we're done.
7049
    if (!ImportedModules.insert(Import->getImportedModule()))
7050
      break;
7051

7052
    // Emit debug information for direct imports.
7053
    if (!Import->getImportedOwningModule()) {
7054
      if (CGDebugInfo *DI = getModuleDebugInfo())
7055
        DI->EmitImportDecl(*Import);
7056
    }
7057

7058
    // For C++ standard modules we are done - we will call the module
7059
    // initializer for imported modules, and that will likewise call those for
7060
    // any imports it has.
7061
    if (CXX20ModuleInits && Import->getImportedOwningModule() &&
7062
        !Import->getImportedOwningModule()->isModuleMapModule())
7063
      break;
7064

7065
    // For clang C++ module map modules the initializers for sub-modules are
7066
    // emitted here.
7067

7068
    // Find all of the submodules and emit the module initializers.
7069
    llvm::SmallPtrSet<clang::Module *, 16> Visited;
7070
    SmallVector<clang::Module *, 16> Stack;
7071
    Visited.insert(Import->getImportedModule());
7072
    Stack.push_back(Import->getImportedModule());
7073

7074
    while (!Stack.empty()) {
7075
      clang::Module *Mod = Stack.pop_back_val();
7076
      if (!EmittedModuleInitializers.insert(Mod).second)
7077
        continue;
7078

7079
      for (auto *D : Context.getModuleInitializers(Mod))
7080
        EmitTopLevelDecl(D);
7081

7082
      // Visit the submodules of this module.
7083
      for (auto *Submodule : Mod->submodules()) {
7084
        // Skip explicit children; they need to be explicitly imported to emit
7085
        // the initializers.
7086
        if (Submodule->IsExplicit)
7087
          continue;
7088

7089
        if (Visited.insert(Submodule).second)
7090
          Stack.push_back(Submodule);
7091
      }
7092
    }
7093
    break;
7094
  }
7095

7096
  case Decl::Export:
7097
    EmitDeclContext(cast<ExportDecl>(D));
7098
    break;
7099

7100
  case Decl::OMPThreadPrivate:
7101
    EmitOMPThreadPrivateDecl(cast<OMPThreadPrivateDecl>(D));
7102
    break;
7103

7104
  case Decl::OMPAllocate:
7105
    EmitOMPAllocateDecl(cast<OMPAllocateDecl>(D));
7106
    break;
7107

7108
  case Decl::OMPDeclareReduction:
7109
    EmitOMPDeclareReduction(cast<OMPDeclareReductionDecl>(D));
7110
    break;
7111

7112
  case Decl::OMPDeclareMapper:
7113
    EmitOMPDeclareMapper(cast<OMPDeclareMapperDecl>(D));
7114
    break;
7115

7116
  case Decl::OMPRequires:
7117
    EmitOMPRequiresDecl(cast<OMPRequiresDecl>(D));
7118
    break;
7119

7120
  case Decl::Typedef:
7121
  case Decl::TypeAlias: // using foo = bar; [C++11]
7122
    if (CGDebugInfo *DI = getModuleDebugInfo())
7123
      DI->EmitAndRetainType(
7124
          getContext().getTypedefType(cast<TypedefNameDecl>(D)));
7125
    break;
7126

7127
  case Decl::Record:
7128
    if (CGDebugInfo *DI = getModuleDebugInfo())
7129
      if (cast<RecordDecl>(D)->getDefinition())
7130
        DI->EmitAndRetainType(getContext().getRecordType(cast<RecordDecl>(D)));
7131
    break;
7132

7133
  case Decl::Enum:
7134
    if (CGDebugInfo *DI = getModuleDebugInfo())
7135
      if (cast<EnumDecl>(D)->getDefinition())
7136
        DI->EmitAndRetainType(getContext().getEnumType(cast<EnumDecl>(D)));
7137
    break;
7138

7139
  case Decl::HLSLBuffer:
7140
    getHLSLRuntime().addBuffer(cast<HLSLBufferDecl>(D));
7141
    break;
7142

7143
  default:
7144
    // Make sure we handled everything we should, every other kind is a
7145
    // non-top-level decl.  FIXME: Would be nice to have an isTopLevelDeclKind
7146
    // function. Need to recode Decl::Kind to do that easily.
7147
    assert(isa<TypeDecl>(D) && "Unsupported decl kind");
7148
    break;
7149
  }
7150
}
7151

7152
void CodeGenModule::AddDeferredUnusedCoverageMapping(Decl *D) {
7153
  // Do we need to generate coverage mapping?
7154
  if (!CodeGenOpts.CoverageMapping)
7155
    return;
7156
  switch (D->getKind()) {
7157
  case Decl::CXXConversion:
7158
  case Decl::CXXMethod:
7159
  case Decl::Function:
7160
  case Decl::ObjCMethod:
7161
  case Decl::CXXConstructor:
7162
  case Decl::CXXDestructor: {
7163
    if (!cast<FunctionDecl>(D)->doesThisDeclarationHaveABody())
7164
      break;
7165
    SourceManager &SM = getContext().getSourceManager();
7166
    if (LimitedCoverage && SM.getMainFileID() != SM.getFileID(D->getBeginLoc()))
7167
      break;
7168
    DeferredEmptyCoverageMappingDecls.try_emplace(D, true);
7169
    break;
7170
  }
7171
  default:
7172
    break;
7173
  };
7174
}
7175

7176
void CodeGenModule::ClearUnusedCoverageMapping(const Decl *D) {
7177
  // Do we need to generate coverage mapping?
7178
  if (!CodeGenOpts.CoverageMapping)
7179
    return;
7180
  if (const auto *Fn = dyn_cast<FunctionDecl>(D)) {
7181
    if (Fn->isTemplateInstantiation())
7182
      ClearUnusedCoverageMapping(Fn->getTemplateInstantiationPattern());
7183
  }
7184
  DeferredEmptyCoverageMappingDecls.insert_or_assign(D, false);
7185
}
7186

7187
void CodeGenModule::EmitDeferredUnusedCoverageMappings() {
7188
  // We call takeVector() here to avoid use-after-free.
7189
  // FIXME: DeferredEmptyCoverageMappingDecls is getting mutated because
7190
  // we deserialize function bodies to emit coverage info for them, and that
7191
  // deserializes more declarations. How should we handle that case?
7192
  for (const auto &Entry : DeferredEmptyCoverageMappingDecls.takeVector()) {
7193
    if (!Entry.second)
7194
      continue;
7195
    const Decl *D = Entry.first;
7196
    switch (D->getKind()) {
7197
    case Decl::CXXConversion:
7198
    case Decl::CXXMethod:
7199
    case Decl::Function:
7200
    case Decl::ObjCMethod: {
7201
      CodeGenPGO PGO(*this);
7202
      GlobalDecl GD(cast<FunctionDecl>(D));
7203
      PGO.emitEmptyCounterMapping(D, getMangledName(GD),
7204
                                  getFunctionLinkage(GD));
7205
      break;
7206
    }
7207
    case Decl::CXXConstructor: {
7208
      CodeGenPGO PGO(*this);
7209
      GlobalDecl GD(cast<CXXConstructorDecl>(D), Ctor_Base);
7210
      PGO.emitEmptyCounterMapping(D, getMangledName(GD),
7211
                                  getFunctionLinkage(GD));
7212
      break;
7213
    }
7214
    case Decl::CXXDestructor: {
7215
      CodeGenPGO PGO(*this);
7216
      GlobalDecl GD(cast<CXXDestructorDecl>(D), Dtor_Base);
7217
      PGO.emitEmptyCounterMapping(D, getMangledName(GD),
7218
                                  getFunctionLinkage(GD));
7219
      break;
7220
    }
7221
    default:
7222
      break;
7223
    };
7224
  }
7225
}
7226

7227
void CodeGenModule::EmitMainVoidAlias() {
7228
  // In order to transition away from "__original_main" gracefully, emit an
7229
  // alias for "main" in the no-argument case so that libc can detect when
7230
  // new-style no-argument main is in used.
7231
  if (llvm::Function *F = getModule().getFunction("main")) {
7232
    if (!F->isDeclaration() && F->arg_size() == 0 && !F->isVarArg() &&
7233
        F->getReturnType()->isIntegerTy(Context.getTargetInfo().getIntWidth())) {
7234
      auto *GA = llvm::GlobalAlias::create("__main_void", F);
7235
      GA->setVisibility(llvm::GlobalValue::HiddenVisibility);
7236
    }
7237
  }
7238
}
7239

7240
/// Turns the given pointer into a constant.
7241
static llvm::Constant *GetPointerConstant(llvm::LLVMContext &Context,
7242
                                          const void *Ptr) {
7243
  uintptr_t PtrInt = reinterpret_cast<uintptr_t>(Ptr);
7244
  llvm::Type *i64 = llvm::Type::getInt64Ty(Context);
7245
  return llvm::ConstantInt::get(i64, PtrInt);
7246
}
7247

7248
static void EmitGlobalDeclMetadata(CodeGenModule &CGM,
7249
                                   llvm::NamedMDNode *&GlobalMetadata,
7250
                                   GlobalDecl D,
7251
                                   llvm::GlobalValue *Addr) {
7252
  if (!GlobalMetadata)
7253
    GlobalMetadata =
7254
      CGM.getModule().getOrInsertNamedMetadata("clang.global.decl.ptrs");
7255

7256
  // TODO: should we report variant information for ctors/dtors?
7257
  llvm::Metadata *Ops[] = {llvm::ConstantAsMetadata::get(Addr),
7258
                           llvm::ConstantAsMetadata::get(GetPointerConstant(
7259
                               CGM.getLLVMContext(), D.getDecl()))};
7260
  GlobalMetadata->addOperand(llvm::MDNode::get(CGM.getLLVMContext(), Ops));
7261
}
7262

7263
bool CodeGenModule::CheckAndReplaceExternCIFuncs(llvm::GlobalValue *Elem,
7264
                                                 llvm::GlobalValue *CppFunc) {
7265
  // Store the list of ifuncs we need to replace uses in.
7266
  llvm::SmallVector<llvm::GlobalIFunc *> IFuncs;
7267
  // List of ConstantExprs that we should be able to delete when we're done
7268
  // here.
7269
  llvm::SmallVector<llvm::ConstantExpr *> CEs;
7270

7271
  // It isn't valid to replace the extern-C ifuncs if all we find is itself!
7272
  if (Elem == CppFunc)
7273
    return false;
7274

7275
  // First make sure that all users of this are ifuncs (or ifuncs via a
7276
  // bitcast), and collect the list of ifuncs and CEs so we can work on them
7277
  // later.
7278
  for (llvm::User *User : Elem->users()) {
7279
    // Users can either be a bitcast ConstExpr that is used by the ifuncs, OR an
7280
    // ifunc directly. In any other case, just give up, as we don't know what we
7281
    // could break by changing those.
7282
    if (auto *ConstExpr = dyn_cast<llvm::ConstantExpr>(User)) {
7283
      if (ConstExpr->getOpcode() != llvm::Instruction::BitCast)
7284
        return false;
7285

7286
      for (llvm::User *CEUser : ConstExpr->users()) {
7287
        if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(CEUser)) {
7288
          IFuncs.push_back(IFunc);
7289
        } else {
7290
          return false;
7291
        }
7292
      }
7293
      CEs.push_back(ConstExpr);
7294
    } else if (auto *IFunc = dyn_cast<llvm::GlobalIFunc>(User)) {
7295
      IFuncs.push_back(IFunc);
7296
    } else {
7297
      // This user is one we don't know how to handle, so fail redirection. This
7298
      // will result in an ifunc retaining a resolver name that will ultimately
7299
      // fail to be resolved to a defined function.
7300
      return false;
7301
    }
7302
  }
7303

7304
  // Now we know this is a valid case where we can do this alias replacement, we
7305
  // need to remove all of the references to Elem (and the bitcasts!) so we can
7306
  // delete it.
7307
  for (llvm::GlobalIFunc *IFunc : IFuncs)
7308
    IFunc->setResolver(nullptr);
7309
  for (llvm::ConstantExpr *ConstExpr : CEs)
7310
    ConstExpr->destroyConstant();
7311

7312
  // We should now be out of uses for the 'old' version of this function, so we
7313
  // can erase it as well.
7314
  Elem->eraseFromParent();
7315

7316
  for (llvm::GlobalIFunc *IFunc : IFuncs) {
7317
    // The type of the resolver is always just a function-type that returns the
7318
    // type of the IFunc, so create that here. If the type of the actual
7319
    // resolver doesn't match, it just gets bitcast to the right thing.
7320
    auto *ResolverTy =
7321
        llvm::FunctionType::get(IFunc->getType(), /*isVarArg*/ false);
7322
    llvm::Constant *Resolver = GetOrCreateLLVMFunction(
7323
        CppFunc->getName(), ResolverTy, {}, /*ForVTable*/ false);
7324
    IFunc->setResolver(Resolver);
7325
  }
7326
  return true;
7327
}
7328

7329
/// For each function which is declared within an extern "C" region and marked
7330
/// as 'used', but has internal linkage, create an alias from the unmangled
7331
/// name to the mangled name if possible. People expect to be able to refer
7332
/// to such functions with an unmangled name from inline assembly within the
7333
/// same translation unit.
7334
void CodeGenModule::EmitStaticExternCAliases() {
7335
  if (!getTargetCodeGenInfo().shouldEmitStaticExternCAliases())
7336
    return;
7337
  for (auto &I : StaticExternCValues) {
7338
    const IdentifierInfo *Name = I.first;
7339
    llvm::GlobalValue *Val = I.second;
7340

7341
    // If Val is null, that implies there were multiple declarations that each
7342
    // had a claim to the unmangled name. In this case, generation of the alias
7343
    // is suppressed. See CodeGenModule::MaybeHandleStaticInExternC.
7344
    if (!Val)
7345
      break;
7346

7347
    llvm::GlobalValue *ExistingElem =
7348
        getModule().getNamedValue(Name->getName());
7349

7350
    // If there is either not something already by this name, or we were able to
7351
    // replace all uses from IFuncs, create the alias.
7352
    if (!ExistingElem || CheckAndReplaceExternCIFuncs(ExistingElem, Val))
7353
      addCompilerUsedGlobal(llvm::GlobalAlias::create(Name->getName(), Val));
7354
  }
7355
}
7356

7357
bool CodeGenModule::lookupRepresentativeDecl(StringRef MangledName,
7358
                                             GlobalDecl &Result) const {
7359
  auto Res = Manglings.find(MangledName);
7360
  if (Res == Manglings.end())
7361
    return false;
7362
  Result = Res->getValue();
7363
  return true;
7364
}
7365

7366
/// Emits metadata nodes associating all the global values in the
7367
/// current module with the Decls they came from.  This is useful for
7368
/// projects using IR gen as a subroutine.
7369
///
7370
/// Since there's currently no way to associate an MDNode directly
7371
/// with an llvm::GlobalValue, we create a global named metadata
7372
/// with the name 'clang.global.decl.ptrs'.
7373
void CodeGenModule::EmitDeclMetadata() {
7374
  llvm::NamedMDNode *GlobalMetadata = nullptr;
7375

7376
  for (auto &I : MangledDeclNames) {
7377
    llvm::GlobalValue *Addr = getModule().getNamedValue(I.second);
7378
    // Some mangled names don't necessarily have an associated GlobalValue
7379
    // in this module, e.g. if we mangled it for DebugInfo.
7380
    if (Addr)
7381
      EmitGlobalDeclMetadata(*this, GlobalMetadata, I.first, Addr);
7382
  }
7383
}
7384

7385
/// Emits metadata nodes for all the local variables in the current
7386
/// function.
7387
void CodeGenFunction::EmitDeclMetadata() {
7388
  if (LocalDeclMap.empty()) return;
7389

7390
  llvm::LLVMContext &Context = getLLVMContext();
7391

7392
  // Find the unique metadata ID for this name.
7393
  unsigned DeclPtrKind = Context.getMDKindID("clang.decl.ptr");
7394

7395
  llvm::NamedMDNode *GlobalMetadata = nullptr;
7396

7397
  for (auto &I : LocalDeclMap) {
7398
    const Decl *D = I.first;
7399
    llvm::Value *Addr = I.second.emitRawPointer(*this);
7400
    if (auto *Alloca = dyn_cast<llvm::AllocaInst>(Addr)) {
7401
      llvm::Value *DAddr = GetPointerConstant(getLLVMContext(), D);
7402
      Alloca->setMetadata(
7403
          DeclPtrKind, llvm::MDNode::get(
7404
                           Context, llvm::ValueAsMetadata::getConstant(DAddr)));
7405
    } else if (auto *GV = dyn_cast<llvm::GlobalValue>(Addr)) {
7406
      GlobalDecl GD = GlobalDecl(cast<VarDecl>(D));
7407
      EmitGlobalDeclMetadata(CGM, GlobalMetadata, GD, GV);
7408
    }
7409
  }
7410
}
7411

7412
void CodeGenModule::EmitVersionIdentMetadata() {
7413
  llvm::NamedMDNode *IdentMetadata =
7414
    TheModule.getOrInsertNamedMetadata("llvm.ident");
7415
  std::string Version = getClangFullVersion();
7416
  llvm::LLVMContext &Ctx = TheModule.getContext();
7417

7418
  llvm::Metadata *IdentNode[] = {llvm::MDString::get(Ctx, Version)};
7419
  IdentMetadata->addOperand(llvm::MDNode::get(Ctx, IdentNode));
7420
}
7421

7422
void CodeGenModule::EmitCommandLineMetadata() {
7423
  llvm::NamedMDNode *CommandLineMetadata =
7424
    TheModule.getOrInsertNamedMetadata("llvm.commandline");
7425
  std::string CommandLine = getCodeGenOpts().RecordCommandLine;
7426
  llvm::LLVMContext &Ctx = TheModule.getContext();
7427

7428
  llvm::Metadata *CommandLineNode[] = {llvm::MDString::get(Ctx, CommandLine)};
7429
  CommandLineMetadata->addOperand(llvm::MDNode::get(Ctx, CommandLineNode));
7430
}
7431

7432
void CodeGenModule::EmitCoverageFile() {
7433
  llvm::NamedMDNode *CUNode = TheModule.getNamedMetadata("llvm.dbg.cu");
7434
  if (!CUNode)
7435
    return;
7436

7437
  llvm::NamedMDNode *GCov = TheModule.getOrInsertNamedMetadata("llvm.gcov");
7438
  llvm::LLVMContext &Ctx = TheModule.getContext();
7439
  auto *CoverageDataFile =
7440
      llvm::MDString::get(Ctx, getCodeGenOpts().CoverageDataFile);
7441
  auto *CoverageNotesFile =
7442
      llvm::MDString::get(Ctx, getCodeGenOpts().CoverageNotesFile);
7443
  for (int i = 0, e = CUNode->getNumOperands(); i != e; ++i) {
7444
    llvm::MDNode *CU = CUNode->getOperand(i);
7445
    llvm::Metadata *Elts[] = {CoverageNotesFile, CoverageDataFile, CU};
7446
    GCov->addOperand(llvm::MDNode::get(Ctx, Elts));
7447
  }
7448
}
7449

7450
llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty,
7451
                                                       bool ForEH) {
7452
  // Return a bogus pointer if RTTI is disabled, unless it's for EH.
7453
  // FIXME: should we even be calling this method if RTTI is disabled
7454
  // and it's not for EH?
7455
  if (!shouldEmitRTTI(ForEH))
7456
    return llvm::Constant::getNullValue(GlobalsInt8PtrTy);
7457

7458
  if (ForEH && Ty->isObjCObjectPointerType() &&
7459
      LangOpts.ObjCRuntime.isGNUFamily())
7460
    return ObjCRuntime->GetEHType(Ty);
7461

7462
  return getCXXABI().getAddrOfRTTIDescriptor(Ty);
7463
}
7464

7465
void CodeGenModule::EmitOMPThreadPrivateDecl(const OMPThreadPrivateDecl *D) {
7466
  // Do not emit threadprivates in simd-only mode.
7467
  if (LangOpts.OpenMP && LangOpts.OpenMPSimd)
7468
    return;
7469
  for (auto RefExpr : D->varlists()) {
7470
    auto *VD = cast<VarDecl>(cast<DeclRefExpr>(RefExpr)->getDecl());
7471
    bool PerformInit =
7472
        VD->getAnyInitializer() &&
7473
        !VD->getAnyInitializer()->isConstantInitializer(getContext(),
7474
                                                        /*ForRef=*/false);
7475

7476
    Address Addr(GetAddrOfGlobalVar(VD),
7477
                 getTypes().ConvertTypeForMem(VD->getType()),
7478
                 getContext().getDeclAlign(VD));
7479
    if (auto InitFunction = getOpenMPRuntime().emitThreadPrivateVarDefinition(
7480
            VD, Addr, RefExpr->getBeginLoc(), PerformInit))
7481
      CXXGlobalInits.push_back(InitFunction);
7482
  }
7483
}
7484

7485
llvm::Metadata *
7486
CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
7487
                                            StringRef Suffix) {
7488
  if (auto *FnType = T->getAs<FunctionProtoType>())
7489
    T = getContext().getFunctionType(
7490
        FnType->getReturnType(), FnType->getParamTypes(),
7491
        FnType->getExtProtoInfo().withExceptionSpec(EST_None));
7492

7493
  llvm::Metadata *&InternalId = Map[T.getCanonicalType()];
7494
  if (InternalId)
7495
    return InternalId;
7496

7497
  if (isExternallyVisible(T->getLinkage())) {
7498
    std::string OutName;
7499
    llvm::raw_string_ostream Out(OutName);
7500
    getCXXABI().getMangleContext().mangleCanonicalTypeName(
7501
        T, Out, getCodeGenOpts().SanitizeCfiICallNormalizeIntegers);
7502

7503
    if (getCodeGenOpts().SanitizeCfiICallNormalizeIntegers)
7504
      Out << ".normalized";
7505

7506
    Out << Suffix;
7507

7508
    InternalId = llvm::MDString::get(getLLVMContext(), Out.str());
7509
  } else {
7510
    InternalId = llvm::MDNode::getDistinct(getLLVMContext(),
7511
                                           llvm::ArrayRef<llvm::Metadata *>());
7512
  }
7513

7514
  return InternalId;
7515
}
7516

7517
llvm::Metadata *CodeGenModule::CreateMetadataIdentifierForType(QualType T) {
7518
  return CreateMetadataIdentifierImpl(T, MetadataIdMap, "");
7519
}
7520

7521
llvm::Metadata *
7522
CodeGenModule::CreateMetadataIdentifierForVirtualMemPtrType(QualType T) {
7523
  return CreateMetadataIdentifierImpl(T, VirtualMetadataIdMap, ".virtual");
7524
}
7525

7526
// Generalize pointer types to a void pointer with the qualifiers of the
7527
// originally pointed-to type, e.g. 'const char *' and 'char * const *'
7528
// generalize to 'const void *' while 'char *' and 'const char **' generalize to
7529
// 'void *'.
7530
static QualType GeneralizeType(ASTContext &Ctx, QualType Ty) {
7531
  if (!Ty->isPointerType())
7532
    return Ty;
7533

7534
  return Ctx.getPointerType(
7535
      QualType(Ctx.VoidTy).withCVRQualifiers(
7536
          Ty->getPointeeType().getCVRQualifiers()));
7537
}
7538

7539
// Apply type generalization to a FunctionType's return and argument types
7540
static QualType GeneralizeFunctionType(ASTContext &Ctx, QualType Ty) {
7541
  if (auto *FnType = Ty->getAs<FunctionProtoType>()) {
7542
    SmallVector<QualType, 8> GeneralizedParams;
7543
    for (auto &Param : FnType->param_types())
7544
      GeneralizedParams.push_back(GeneralizeType(Ctx, Param));
7545

7546
    return Ctx.getFunctionType(
7547
        GeneralizeType(Ctx, FnType->getReturnType()),
7548
        GeneralizedParams, FnType->getExtProtoInfo());
7549
  }
7550

7551
  if (auto *FnType = Ty->getAs<FunctionNoProtoType>())
7552
    return Ctx.getFunctionNoProtoType(
7553
        GeneralizeType(Ctx, FnType->getReturnType()));
7554

7555
  llvm_unreachable("Encountered unknown FunctionType");
7556
}
7557

7558
llvm::Metadata *CodeGenModule::CreateMetadataIdentifierGeneralized(QualType T) {
7559
  return CreateMetadataIdentifierImpl(GeneralizeFunctionType(getContext(), T),
7560
                                      GeneralizedMetadataIdMap, ".generalized");
7561
}
7562

7563
/// Returns whether this module needs the "all-vtables" type identifier.
7564
bool CodeGenModule::NeedAllVtablesTypeId() const {
7565
  // Returns true if at least one of vtable-based CFI checkers is enabled and
7566
  // is not in the trapping mode.
7567
  return ((LangOpts.Sanitize.has(SanitizerKind::CFIVCall) &&
7568
           !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFIVCall)) ||
7569
          (LangOpts.Sanitize.has(SanitizerKind::CFINVCall) &&
7570
           !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFINVCall)) ||
7571
          (LangOpts.Sanitize.has(SanitizerKind::CFIDerivedCast) &&
7572
           !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFIDerivedCast)) ||
7573
          (LangOpts.Sanitize.has(SanitizerKind::CFIUnrelatedCast) &&
7574
           !CodeGenOpts.SanitizeTrap.has(SanitizerKind::CFIUnrelatedCast)));
7575
}
7576

7577
void CodeGenModule::AddVTableTypeMetadata(llvm::GlobalVariable *VTable,
7578
                                          CharUnits Offset,
7579
                                          const CXXRecordDecl *RD) {
7580
  llvm::Metadata *MD =
7581
      CreateMetadataIdentifierForType(QualType(RD->getTypeForDecl(), 0));
7582
  VTable->addTypeMetadata(Offset.getQuantity(), MD);
7583

7584
  if (CodeGenOpts.SanitizeCfiCrossDso)
7585
    if (auto CrossDsoTypeId = CreateCrossDsoCfiTypeId(MD))
7586
      VTable->addTypeMetadata(Offset.getQuantity(),
7587
                              llvm::ConstantAsMetadata::get(CrossDsoTypeId));
7588

7589
  if (NeedAllVtablesTypeId()) {
7590
    llvm::Metadata *MD = llvm::MDString::get(getLLVMContext(), "all-vtables");
7591
    VTable->addTypeMetadata(Offset.getQuantity(), MD);
7592
  }
7593
}
7594

7595
llvm::SanitizerStatReport &CodeGenModule::getSanStats() {
7596
  if (!SanStats)
7597
    SanStats = std::make_unique<llvm::SanitizerStatReport>(&getModule());
7598

7599
  return *SanStats;
7600
}
7601

7602
llvm::Value *
7603
CodeGenModule::createOpenCLIntToSamplerConversion(const Expr *E,
7604
                                                  CodeGenFunction &CGF) {
7605
  llvm::Constant *C = ConstantEmitter(CGF).emitAbstract(E, E->getType());
7606
  auto *SamplerT = getOpenCLRuntime().getSamplerType(E->getType().getTypePtr());
7607
  auto *FTy = llvm::FunctionType::get(SamplerT, {C->getType()}, false);
7608
  auto *Call = CGF.EmitRuntimeCall(
7609
      CreateRuntimeFunction(FTy, "__translate_sampler_initializer"), {C});
7610
  return Call;
7611
}
7612

7613
CharUnits CodeGenModule::getNaturalPointeeTypeAlignment(
7614
    QualType T, LValueBaseInfo *BaseInfo, TBAAAccessInfo *TBAAInfo) {
7615
  return getNaturalTypeAlignment(T->getPointeeType(), BaseInfo, TBAAInfo,
7616
                                 /* forPointeeType= */ true);
7617
}
7618

7619
CharUnits CodeGenModule::getNaturalTypeAlignment(QualType T,
7620
                                                 LValueBaseInfo *BaseInfo,
7621
                                                 TBAAAccessInfo *TBAAInfo,
7622
                                                 bool forPointeeType) {
7623
  if (TBAAInfo)
7624
    *TBAAInfo = getTBAAAccessInfo(T);
7625

7626
  // FIXME: This duplicates logic in ASTContext::getTypeAlignIfKnown. But
7627
  // that doesn't return the information we need to compute BaseInfo.
7628

7629
  // Honor alignment typedef attributes even on incomplete types.
7630
  // We also honor them straight for C++ class types, even as pointees;
7631
  // there's an expressivity gap here.
7632
  if (auto TT = T->getAs<TypedefType>()) {
7633
    if (auto Align = TT->getDecl()->getMaxAlignment()) {
7634
      if (BaseInfo)
7635
        *BaseInfo = LValueBaseInfo(AlignmentSource::AttributedType);
7636
      return getContext().toCharUnitsFromBits(Align);
7637
    }
7638
  }
7639

7640
  bool AlignForArray = T->isArrayType();
7641

7642
  // Analyze the base element type, so we don't get confused by incomplete
7643
  // array types.
7644
  T = getContext().getBaseElementType(T);
7645

7646
  if (T->isIncompleteType()) {
7647
    // We could try to replicate the logic from
7648
    // ASTContext::getTypeAlignIfKnown, but nothing uses the alignment if the
7649
    // type is incomplete, so it's impossible to test. We could try to reuse
7650
    // getTypeAlignIfKnown, but that doesn't return the information we need
7651
    // to set BaseInfo.  So just ignore the possibility that the alignment is
7652
    // greater than one.
7653
    if (BaseInfo)
7654
      *BaseInfo = LValueBaseInfo(AlignmentSource::Type);
7655
    return CharUnits::One();
7656
  }
7657

7658
  if (BaseInfo)
7659
    *BaseInfo = LValueBaseInfo(AlignmentSource::Type);
7660

7661
  CharUnits Alignment;
7662
  const CXXRecordDecl *RD;
7663
  if (T.getQualifiers().hasUnaligned()) {
7664
    Alignment = CharUnits::One();
7665
  } else if (forPointeeType && !AlignForArray &&
7666
             (RD = T->getAsCXXRecordDecl())) {
7667
    // For C++ class pointees, we don't know whether we're pointing at a
7668
    // base or a complete object, so we generally need to use the
7669
    // non-virtual alignment.
7670
    Alignment = getClassPointerAlignment(RD);
7671
  } else {
7672
    Alignment = getContext().getTypeAlignInChars(T);
7673
  }
7674

7675
  // Cap to the global maximum type alignment unless the alignment
7676
  // was somehow explicit on the type.
7677
  if (unsigned MaxAlign = getLangOpts().MaxTypeAlign) {
7678
    if (Alignment.getQuantity() > MaxAlign &&
7679
        !getContext().isAlignmentRequired(T))
7680
      Alignment = CharUnits::fromQuantity(MaxAlign);
7681
  }
7682
  return Alignment;
7683
}
7684

7685
bool CodeGenModule::stopAutoInit() {
7686
  unsigned StopAfter = getContext().getLangOpts().TrivialAutoVarInitStopAfter;
7687
  if (StopAfter) {
7688
    // This number is positive only when -ftrivial-auto-var-init-stop-after=* is
7689
    // used
7690
    if (NumAutoVarInit >= StopAfter) {
7691
      return true;
7692
    }
7693
    if (!NumAutoVarInit) {
7694
      unsigned DiagID = getDiags().getCustomDiagID(
7695
          DiagnosticsEngine::Warning,
7696
          "-ftrivial-auto-var-init-stop-after=%0 has been enabled to limit the "
7697
          "number of times ftrivial-auto-var-init=%1 gets applied.");
7698
      getDiags().Report(DiagID)
7699
          << StopAfter
7700
          << (getContext().getLangOpts().getTrivialAutoVarInit() ==
7701
                      LangOptions::TrivialAutoVarInitKind::Zero
7702
                  ? "zero"
7703
                  : "pattern");
7704
    }
7705
    ++NumAutoVarInit;
7706
  }
7707
  return false;
7708
}
7709

7710
void CodeGenModule::printPostfixForExternalizedDecl(llvm::raw_ostream &OS,
7711
                                                    const Decl *D) const {
7712
  // ptxas does not allow '.' in symbol names. On the other hand, HIP prefers
7713
  // postfix beginning with '.' since the symbol name can be demangled.
7714
  if (LangOpts.HIP)
7715
    OS << (isa<VarDecl>(D) ? ".static." : ".intern.");
7716
  else
7717
    OS << (isa<VarDecl>(D) ? "__static__" : "__intern__");
7718

7719
  // If the CUID is not specified we try to generate a unique postfix.
7720
  if (getLangOpts().CUID.empty()) {
7721
    SourceManager &SM = getContext().getSourceManager();
7722
    PresumedLoc PLoc = SM.getPresumedLoc(D->getLocation());
7723
    assert(PLoc.isValid() && "Source location is expected to be valid.");
7724

7725
    // Get the hash of the user defined macros.
7726
    llvm::MD5 Hash;
7727
    llvm::MD5::MD5Result Result;
7728
    for (const auto &Arg : PreprocessorOpts.Macros)
7729
      Hash.update(Arg.first);
7730
    Hash.final(Result);
7731

7732
    // Get the UniqueID for the file containing the decl.
7733
    llvm::sys::fs::UniqueID ID;
7734
    if (llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID)) {
7735
      PLoc = SM.getPresumedLoc(D->getLocation(), /*UseLineDirectives=*/false);
7736
      assert(PLoc.isValid() && "Source location is expected to be valid.");
7737
      if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID))
7738
        SM.getDiagnostics().Report(diag::err_cannot_open_file)
7739
            << PLoc.getFilename() << EC.message();
7740
    }
7741
    OS << llvm::format("%x", ID.getFile()) << llvm::format("%x", ID.getDevice())
7742
       << "_" << llvm::utohexstr(Result.low(), /*LowerCase=*/true, /*Width=*/8);
7743
  } else {
7744
    OS << getContext().getCUIDHash();
7745
  }
7746
}
7747

7748
void CodeGenModule::moveLazyEmissionStates(CodeGenModule *NewBuilder) {
7749
  assert(DeferredDeclsToEmit.empty() &&
7750
         "Should have emitted all decls deferred to emit.");
7751
  assert(NewBuilder->DeferredDecls.empty() &&
7752
         "Newly created module should not have deferred decls");
7753
  NewBuilder->DeferredDecls = std::move(DeferredDecls);
7754
  assert(EmittedDeferredDecls.empty() &&
7755
         "Still have (unmerged) EmittedDeferredDecls deferred decls");
7756

7757
  assert(NewBuilder->DeferredVTables.empty() &&
7758
         "Newly created module should not have deferred vtables");
7759
  NewBuilder->DeferredVTables = std::move(DeferredVTables);
7760

7761
  assert(NewBuilder->MangledDeclNames.empty() &&
7762
         "Newly created module should not have mangled decl names");
7763
  assert(NewBuilder->Manglings.empty() &&
7764
         "Newly created module should not have manglings");
7765
  NewBuilder->Manglings = std::move(Manglings);
7766

7767
  NewBuilder->WeakRefReferences = std::move(WeakRefReferences);
7768

7769
  NewBuilder->TBAA = std::move(TBAA);
7770

7771
  NewBuilder->ABI->MangleCtx = std::move(ABI->MangleCtx);
7772
}
7773