llvm-project

1
//===- bolt/Passes/PatchEntries.cpp - Pass for patching function entries --===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements the PatchEntries class that is used for patching
10
// the original function entry points.
11
//
12
//===----------------------------------------------------------------------===//
13

14
#include "bolt/Passes/PatchEntries.h"
15
#include "bolt/Utils/NameResolver.h"
16
#include "llvm/ADT/STLExtras.h"
17
#include "llvm/Support/CommandLine.h"
18

19
namespace opts {
20

21
extern llvm::cl::OptionCategory BoltCategory;
22

23
extern llvm::cl::opt<unsigned> Verbosity;
24

25
llvm::cl::opt<bool>
26
    ForcePatch("force-patch",
27
               llvm::cl::desc("force patching of original entry points"),
28
               llvm::cl::Hidden, llvm::cl::cat(BoltCategory));
29
}
30

31
namespace llvm {
32
namespace bolt {
33

34
Error PatchEntries::runOnFunctions(BinaryContext &BC) {
35
  if (!opts::ForcePatch) {
36
    // Mark the binary for patching if we did not create external references
37
    // for original code in any of functions we are not going to emit.
38
    bool NeedsPatching = llvm::any_of(
39
        llvm::make_second_range(BC.getBinaryFunctions()),
40
        [&](BinaryFunction &BF) {
41
          return !BC.shouldEmit(BF) && !BF.hasExternalRefRelocations();
42
        });
43

44
    if (!NeedsPatching)
45
      return Error::success();
46
  }
47

48
  if (opts::Verbosity >= 1)
49
    BC.outs() << "BOLT-INFO: patching entries in original code\n";
50

51
  // Calculate the size of the patch.
52
  static size_t PatchSize = 0;
53
  if (!PatchSize) {
54
    InstructionListType Seq;
55
    BC.MIB->createLongTailCall(Seq, BC.Ctx->createTempSymbol(), BC.Ctx.get());
56
    PatchSize = BC.computeCodeSize(Seq.begin(), Seq.end());
57
  }
58

59
  for (auto &BFI : BC.getBinaryFunctions()) {
60
    BinaryFunction &Function = BFI.second;
61

62
    // Patch original code only for functions that will be emitted.
63
    if (!BC.shouldEmit(Function))
64
      continue;
65

66
    // Check if we can skip patching the function.
67
    if (!opts::ForcePatch && !Function.hasEHRanges() &&
68
        Function.getSize() < PatchThreshold)
69
      continue;
70

71
    // List of patches for function entries. We either successfully patch
72
    // all entries or, if we cannot patch one or more, do no patch any and
73
    // mark the function as ignorable.
74
    std::vector<Patch> PendingPatches;
75

76
    uint64_t NextValidByte = 0; // offset of the byte past the last patch
77
    bool Success = Function.forEachEntryPoint([&](uint64_t Offset,
78
                                                  const MCSymbol *Symbol) {
79
      if (Offset < NextValidByte) {
80
        if (opts::Verbosity >= 1)
81
          BC.outs() << "BOLT-INFO: unable to patch entry point in " << Function
82
                    << " at offset 0x" << Twine::utohexstr(Offset) << '\n';
83
        return false;
84
      }
85

86
      PendingPatches.emplace_back(Patch{Symbol, Function.getAddress() + Offset,
87
                                        Function.getFileOffset() + Offset,
88
                                        Function.getOriginSection()});
89
      NextValidByte = Offset + PatchSize;
90
      if (NextValidByte > Function.getMaxSize()) {
91
        if (opts::Verbosity >= 1)
92
          BC.outs() << "BOLT-INFO: function " << Function
93
                    << " too small to patch its entry point\n";
94
        return false;
95
      }
96

97
      return true;
98
    });
99

100
    if (!Success) {
101
      // We can't change output layout for AArch64 due to LongJmp pass
102
      if (BC.isAArch64()) {
103
        if (opts::ForcePatch) {
104
          BC.errs() << "BOLT-ERROR: unable to patch entries in " << Function
105
                    << "\n";
106
          return createFatalBOLTError("");
107
        }
108

109
        continue;
110
      }
111

112
      // If the original function entries cannot be patched, then we cannot
113
      // safely emit new function body.
114
      BC.errs() << "BOLT-WARNING: failed to patch entries in " << Function
115
                << ". The function will not be optimized.\n";
116
      Function.setIgnored();
117
      continue;
118
    }
119

120
    for (Patch &Patch : PendingPatches) {
121
      BinaryFunction *PatchFunction = BC.createInjectedBinaryFunction(
122
          NameResolver::append(Patch.Symbol->getName(), ".org.0"));
123
      // Force the function to be emitted at the given address.
124
      PatchFunction->setOutputAddress(Patch.Address);
125
      PatchFunction->setFileOffset(Patch.FileOffset);
126
      PatchFunction->setOriginSection(Patch.Section);
127

128
      InstructionListType Seq;
129
      BC.MIB->createLongTailCall(Seq, Patch.Symbol, BC.Ctx.get());
130
      PatchFunction->addBasicBlock()->addInstructions(Seq);
131

132
      // Verify the size requirements.
133
      uint64_t HotSize, ColdSize;
134
      std::tie(HotSize, ColdSize) = BC.calculateEmittedSize(*PatchFunction);
135
      assert(!ColdSize && "unexpected cold code");
136
      assert(HotSize <= PatchSize && "max patch size exceeded");
137
    }
138

139
    Function.setIsPatched(true);
140
  }
141
  return Error::success();
142
}
143

144
} // end namespace bolt
145
} // end namespace llvm
146