llvm-project

1
//===- bolt/Passes/RetpolineInsertion.cpp ---------------------------------===//
2
//
3
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4
// See https://llvm.org/LICENSE.txt for license information.
5
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6
//
7
//===----------------------------------------------------------------------===//
8
//
9
// This file implements RetpolineInsertion class, which replaces indirect
10
// branches (calls and jumps) with calls to retpolines to protect against branch
11
// target injection attacks.
12
// A unique retpoline is created for each register holding the address of the
13
// callee, if the callee address is in memory %r11 is used if available to
14
// hold the address of the callee before calling the retpoline, otherwise an
15
// address pattern specific retpoline is called where the callee address is
16
// loaded inside the retpoline.
17
// The user can determine when to assume %r11 available using r11-availability
18
// option, by default %r11 is assumed not available.
19
// Adding lfence instruction to the body of the speculate code is enabled by
20
// default and can be controlled by the user using retpoline-lfence option.
21
//
22
//===----------------------------------------------------------------------===//
23

24
#include "bolt/Passes/RetpolineInsertion.h"
25
#include "llvm/MC/MCInstPrinter.h"
26
#include "llvm/Support/raw_ostream.h"
27

28
#define DEBUG_TYPE "bolt-retpoline"
29

30
using namespace llvm;
31
using namespace bolt;
32
namespace opts {
33

34
extern cl::OptionCategory BoltCategory;
35

36
llvm::cl::opt<bool> InsertRetpolines("insert-retpolines",
37
                                     cl::desc("run retpoline insertion pass"),
38
                                     cl::cat(BoltCategory));
39

40
llvm::cl::opt<bool>
41
RetpolineLfence("retpoline-lfence",
42
  cl::desc("determine if lfence instruction should exist in the retpoline"),
43
  cl::init(true),
44
  cl::ZeroOrMore,
45
  cl::Hidden,
46
  cl::cat(BoltCategory));
47

48
cl::opt<RetpolineInsertion::AvailabilityOptions> R11Availability(
49
    "r11-availability",
50
    cl::desc("determine the availability of r11 before indirect branches"),
51
    cl::init(RetpolineInsertion::AvailabilityOptions::NEVER),
52
    cl::values(clEnumValN(RetpolineInsertion::AvailabilityOptions::NEVER,
53
                          "never", "r11 not available"),
54
               clEnumValN(RetpolineInsertion::AvailabilityOptions::ALWAYS,
55
                          "always", "r11 available before calls and jumps"),
56
               clEnumValN(RetpolineInsertion::AvailabilityOptions::ABI, "abi",
57
                          "r11 available before calls but not before jumps")),
58
    cl::ZeroOrMore, cl::cat(BoltCategory));
59

60
} // namespace opts
61

62
namespace llvm {
63
namespace bolt {
64

65
// Retpoline function structure:
66
// BB0: call BB2
67
// BB1: pause
68
//      lfence
69
//      jmp BB1
70
// BB2: mov %reg, (%rsp)
71
//      ret
72
// or
73
// BB2: push %r11
74
//      mov Address, %r11
75
//      mov %r11, 8(%rsp)
76
//      pop %r11
77
//      ret
78
BinaryFunction *createNewRetpoline(BinaryContext &BC,
79
                                   const std::string &RetpolineTag,
80
                                   const IndirectBranchInfo &BrInfo,
81
                                   bool R11Available) {
82
  auto &MIB = *BC.MIB;
83
  MCContext &Ctx = *BC.Ctx.get();
84
  LLVM_DEBUG(dbgs() << "BOLT-DEBUG: Creating a new retpoline function["
85
                    << RetpolineTag << "]\n");
86

87
  BinaryFunction *NewRetpoline =
88
      BC.createInjectedBinaryFunction(RetpolineTag, true);
89
  std::vector<std::unique_ptr<BinaryBasicBlock>> NewBlocks(3);
90
  for (int I = 0; I < 3; I++) {
91
    MCSymbol *Symbol =
92
        Ctx.createNamedTempSymbol(Twine(RetpolineTag + "_BB" + to_string(I)));
93
    NewBlocks[I] = NewRetpoline->createBasicBlock(Symbol);
94
    NewBlocks[I].get()->setCFIState(0);
95
  }
96

97
  BinaryBasicBlock &BB0 = *NewBlocks[0].get();
98
  BinaryBasicBlock &BB1 = *NewBlocks[1].get();
99
  BinaryBasicBlock &BB2 = *NewBlocks[2].get();
100

101
  BB0.addSuccessor(&BB2, 0, 0);
102
  BB1.addSuccessor(&BB1, 0, 0);
103

104
  // Build BB0
105
  MCInst DirectCall;
106
  MIB.createDirectCall(DirectCall, BB2.getLabel(), &Ctx, /*IsTailCall*/ false);
107
  BB0.addInstruction(DirectCall);
108

109
  // Build BB1
110
  MCInst Pause;
111
  MIB.createPause(Pause);
112
  BB1.addInstruction(Pause);
113

114
  if (opts::RetpolineLfence) {
115
    MCInst Lfence;
116
    MIB.createLfence(Lfence);
117
    BB1.addInstruction(Lfence);
118
  }
119

120
  InstructionListType Seq;
121
  MIB.createShortJmp(Seq, BB1.getLabel(), &Ctx);
122
  BB1.addInstructions(Seq.begin(), Seq.end());
123

124
  // Build BB2
125
  if (BrInfo.isMem()) {
126
    if (R11Available) {
127
      MCInst StoreToStack;
128
      MIB.createSaveToStack(StoreToStack, MIB.getStackPointer(), 0,
129
                            MIB.getX86R11(), 8);
130
      BB2.addInstruction(StoreToStack);
131
    } else {
132
      MCInst PushR11;
133
      MIB.createPushRegister(PushR11, MIB.getX86R11(), 8);
134
      BB2.addInstruction(PushR11);
135

136
      MCInst LoadCalleeAddrs;
137
      const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
138
      MIB.createLoad(LoadCalleeAddrs, MemRef.BaseRegNum, MemRef.ScaleImm,
139
                     MemRef.IndexRegNum, MemRef.DispImm, MemRef.DispExpr,
140
                     MemRef.SegRegNum, MIB.getX86R11(), 8);
141

142
      BB2.addInstruction(LoadCalleeAddrs);
143

144
      MCInst StoreToStack;
145
      MIB.createSaveToStack(StoreToStack, MIB.getStackPointer(), 8,
146
                            MIB.getX86R11(), 8);
147
      BB2.addInstruction(StoreToStack);
148

149
      MCInst PopR11;
150
      MIB.createPopRegister(PopR11, MIB.getX86R11(), 8);
151
      BB2.addInstruction(PopR11);
152
    }
153
  } else if (BrInfo.isReg()) {
154
    MCInst StoreToStack;
155
    MIB.createSaveToStack(StoreToStack, MIB.getStackPointer(), 0,
156
                          BrInfo.BranchReg, 8);
157
    BB2.addInstruction(StoreToStack);
158
  } else {
159
    llvm_unreachable("not expected");
160
  }
161

162
  // return
163
  MCInst Return;
164
  MIB.createReturn(Return);
165
  BB2.addInstruction(Return);
166
  NewRetpoline->insertBasicBlocks(nullptr, std::move(NewBlocks),
167
                                  /* UpdateLayout */ true,
168
                                  /* UpdateCFIState */ false);
169

170
  NewRetpoline->updateState(BinaryFunction::State::CFG_Finalized);
171
  return NewRetpoline;
172
}
173

174
std::string createRetpolineFunctionTag(BinaryContext &BC,
175
                                       const IndirectBranchInfo &BrInfo,
176
                                       bool R11Available) {
177
  std::string Tag;
178
  llvm::raw_string_ostream TagOS(Tag);
179
  TagOS << "__retpoline_";
180

181
  if (BrInfo.isReg()) {
182
    BC.InstPrinter->printRegName(TagOS, BrInfo.BranchReg);
183
    TagOS << "_";
184
    TagOS.flush();
185
    return Tag;
186
  }
187

188
  // Memory Branch
189
  if (R11Available)
190
    return "__retpoline_r11";
191

192
  const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
193

194
  TagOS << "mem_";
195

196
  if (MemRef.BaseRegNum != BC.MIB->getNoRegister())
197
    BC.InstPrinter->printRegName(TagOS, MemRef.BaseRegNum);
198

199
  TagOS << "+";
200
  if (MemRef.DispExpr)
201
    MemRef.DispExpr->print(TagOS, BC.AsmInfo.get());
202
  else
203
    TagOS << MemRef.DispImm;
204

205
  if (MemRef.IndexRegNum != BC.MIB->getNoRegister()) {
206
    TagOS << "+" << MemRef.ScaleImm << "*";
207
    BC.InstPrinter->printRegName(TagOS, MemRef.IndexRegNum);
208
  }
209

210
  if (MemRef.SegRegNum != BC.MIB->getNoRegister()) {
211
    TagOS << "_seg_";
212
    BC.InstPrinter->printRegName(TagOS, MemRef.SegRegNum);
213
  }
214

215
  TagOS.flush();
216
  return Tag;
217
}
218

219
BinaryFunction *RetpolineInsertion::getOrCreateRetpoline(
220
    BinaryContext &BC, const IndirectBranchInfo &BrInfo, bool R11Available) {
221
  const std::string RetpolineTag =
222
      createRetpolineFunctionTag(BC, BrInfo, R11Available);
223

224
  if (CreatedRetpolines.count(RetpolineTag))
225
    return CreatedRetpolines[RetpolineTag];
226

227
  return CreatedRetpolines[RetpolineTag] =
228
             createNewRetpoline(BC, RetpolineTag, BrInfo, R11Available);
229
}
230

231
void createBranchReplacement(BinaryContext &BC,
232
                             const IndirectBranchInfo &BrInfo,
233
                             bool R11Available,
234
                             InstructionListType &Replacement,
235
                             const MCSymbol *RetpolineSymbol) {
236
  auto &MIB = *BC.MIB;
237
  // Load the branch address in r11 if available
238
  if (BrInfo.isMem() && R11Available) {
239
    const IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
240
    MCInst LoadCalleeAddrs;
241
    MIB.createLoad(LoadCalleeAddrs, MemRef.BaseRegNum, MemRef.ScaleImm,
242
                   MemRef.IndexRegNum, MemRef.DispImm, MemRef.DispExpr,
243
                   MemRef.SegRegNum, MIB.getX86R11(), 8);
244
    Replacement.push_back(LoadCalleeAddrs);
245
  }
246

247
  // Call the retpoline
248
  MCInst RetpolineCall;
249
  MIB.createDirectCall(RetpolineCall, RetpolineSymbol, BC.Ctx.get(),
250
                       BrInfo.isJump() || BrInfo.isTailCall());
251

252
  Replacement.push_back(RetpolineCall);
253
}
254

255
IndirectBranchInfo::IndirectBranchInfo(MCInst &Inst, MCPlusBuilder &MIB) {
256
  IsCall = MIB.isCall(Inst);
257
  IsTailCall = MIB.isTailCall(Inst);
258

259
  if (MIB.isBranchOnMem(Inst)) {
260
    IsMem = true;
261
    std::optional<MCPlusBuilder::X86MemOperand> MO =
262
        MIB.evaluateX86MemoryOperand(Inst);
263
    if (!MO)
264
      llvm_unreachable("not expected");
265
    Memory = MO.value();
266
  } else if (MIB.isBranchOnReg(Inst)) {
267
    assert(MCPlus::getNumPrimeOperands(Inst) == 1 && "expect 1 operand");
268
    BranchReg = Inst.getOperand(0).getReg();
269
  } else {
270
    llvm_unreachable("unexpected instruction");
271
  }
272
}
273

274
Error RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
275
  if (!opts::InsertRetpolines)
276
    return Error::success();
277

278
  assert(BC.isX86() &&
279
         "retpoline insertion not supported for target architecture");
280

281
  assert(BC.HasRelocations && "retpoline mode not supported in non-reloc");
282

283
  auto &MIB = *BC.MIB;
284
  uint32_t RetpolinedBranches = 0;
285
  for (auto &It : BC.getBinaryFunctions()) {
286
    BinaryFunction &Function = It.second;
287
    for (BinaryBasicBlock &BB : Function) {
288
      for (auto It = BB.begin(); It != BB.end(); ++It) {
289
        MCInst &Inst = *It;
290

291
        if (!MIB.isIndirectCall(Inst) && !MIB.isIndirectBranch(Inst))
292
          continue;
293

294
        IndirectBranchInfo BrInfo(Inst, MIB);
295
        bool R11Available = false;
296
        BinaryFunction *TargetRetpoline;
297
        InstructionListType Replacement;
298

299
        // Determine if r11 is available before this instruction
300
        if (BrInfo.isMem()) {
301
          if (MIB.hasAnnotation(Inst, "PLTCall"))
302
            R11Available = true;
303
          else if (opts::R11Availability == AvailabilityOptions::ALWAYS)
304
            R11Available = true;
305
          else if (opts::R11Availability == AvailabilityOptions::ABI)
306
            R11Available = BrInfo.isCall();
307
        }
308

309
        // If the instruction addressing pattern uses rsp and the retpoline
310
        // loads the callee address then displacement needs to be updated
311
        if (BrInfo.isMem() && !R11Available) {
312
          IndirectBranchInfo::MemOpInfo &MemRef = BrInfo.Memory;
313
          int Addend = (BrInfo.isJump() || BrInfo.isTailCall()) ? 8 : 16;
314
          if (MemRef.BaseRegNum == MIB.getStackPointer())
315
            MemRef.DispImm += Addend;
316
          if (MemRef.IndexRegNum == MIB.getStackPointer())
317
            MemRef.DispImm += Addend * MemRef.ScaleImm;
318
        }
319

320
        TargetRetpoline = getOrCreateRetpoline(BC, BrInfo, R11Available);
321

322
        createBranchReplacement(BC, BrInfo, R11Available, Replacement,
323
                                TargetRetpoline->getSymbol());
324

325
        It = BB.replaceInstruction(It, Replacement.begin(), Replacement.end());
326
        RetpolinedBranches++;
327
      }
328
    }
329
  }
330
  BC.outs() << "BOLT-INFO: The number of created retpoline functions is : "
331
            << CreatedRetpolines.size()
332
            << "\nBOLT-INFO: The number of retpolined branches is : "
333
            << RetpolinedBranches << "\n";
334
  return Error::success();
335
}
336

337
} // namespace bolt
338
} // namespace llvm
339