llvm-project
658 строк · 23.3 Кб
1//===- bolt/Passes/LongJmp.cpp --------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the LongJmpPass class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "bolt/Passes/LongJmp.h"14
15#define DEBUG_TYPE "longjmp"16
17using namespace llvm;18
19namespace opts {20extern cl::OptionCategory BoltOptCategory;21extern llvm::cl::opt<unsigned> AlignText;22extern cl::opt<unsigned> AlignFunctions;23extern cl::opt<bool> UseOldText;24extern cl::opt<bool> HotFunctionsAtEnd;25
26static cl::opt<bool> GroupStubs("group-stubs",27cl::desc("share stubs across functions"),28cl::init(true), cl::cat(BoltOptCategory));29}
30
31namespace llvm {32namespace bolt {33
34constexpr unsigned ColdFragAlign = 16;35
36static void relaxStubToShortJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {37const BinaryContext &BC = StubBB.getFunction()->getBinaryContext();38InstructionListType Seq;39BC.MIB->createShortJmp(Seq, Tgt, BC.Ctx.get());40StubBB.clear();41StubBB.addInstructions(Seq.begin(), Seq.end());42}
43
44static void relaxStubToLongJmp(BinaryBasicBlock &StubBB, const MCSymbol *Tgt) {45const BinaryContext &BC = StubBB.getFunction()->getBinaryContext();46InstructionListType Seq;47BC.MIB->createLongJmp(Seq, Tgt, BC.Ctx.get());48StubBB.clear();49StubBB.addInstructions(Seq.begin(), Seq.end());50}
51
52static BinaryBasicBlock *getBBAtHotColdSplitPoint(BinaryFunction &Func) {53if (!Func.isSplit() || Func.empty())54return nullptr;55
56assert(!(*Func.begin()).isCold() && "Entry cannot be cold");57for (auto I = Func.getLayout().block_begin(),58E = Func.getLayout().block_end();59I != E; ++I) {60auto Next = std::next(I);61if (Next != E && (*Next)->isCold())62return *I;63}64llvm_unreachable("No hot-colt split point found");65}
66
67static bool shouldInsertStub(const BinaryContext &BC, const MCInst &Inst) {68return (BC.MIB->isBranch(Inst) || BC.MIB->isCall(Inst)) &&69!BC.MIB->isIndirectBranch(Inst) && !BC.MIB->isIndirectCall(Inst);70}
71
72std::pair<std::unique_ptr<BinaryBasicBlock>, MCSymbol *>73LongJmpPass::createNewStub(BinaryBasicBlock &SourceBB, const MCSymbol *TgtSym,74bool TgtIsFunc, uint64_t AtAddress) {75BinaryFunction &Func = *SourceBB.getFunction();76const BinaryContext &BC = Func.getBinaryContext();77const bool IsCold = SourceBB.isCold();78MCSymbol *StubSym = BC.Ctx->createNamedTempSymbol("Stub");79std::unique_ptr<BinaryBasicBlock> StubBB = Func.createBasicBlock(StubSym);80MCInst Inst;81BC.MIB->createUncondBranch(Inst, TgtSym, BC.Ctx.get());82if (TgtIsFunc)83BC.MIB->convertJmpToTailCall(Inst);84StubBB->addInstruction(Inst);85StubBB->setExecutionCount(0);86
87// Register this in stubs maps88auto registerInMap = [&](StubGroupsTy &Map) {89StubGroupTy &StubGroup = Map[TgtSym];90StubGroup.insert(91llvm::lower_bound(92StubGroup, std::make_pair(AtAddress, nullptr),93[&](const std::pair<uint64_t, BinaryBasicBlock *> &LHS,94const std::pair<uint64_t, BinaryBasicBlock *> &RHS) {95return LHS.first < RHS.first;96}),97std::make_pair(AtAddress, StubBB.get()));98};99
100Stubs[&Func].insert(StubBB.get());101StubBits[StubBB.get()] = BC.MIB->getUncondBranchEncodingSize();102if (IsCold) {103registerInMap(ColdLocalStubs[&Func]);104if (opts::GroupStubs && TgtIsFunc)105registerInMap(ColdStubGroups);106++NumColdStubs;107} else {108registerInMap(HotLocalStubs[&Func]);109if (opts::GroupStubs && TgtIsFunc)110registerInMap(HotStubGroups);111++NumHotStubs;112}113
114return std::make_pair(std::move(StubBB), StubSym);115}
116
117BinaryBasicBlock *LongJmpPass::lookupStubFromGroup(118const StubGroupsTy &StubGroups, const BinaryFunction &Func,119const MCInst &Inst, const MCSymbol *TgtSym, uint64_t DotAddress) const {120const BinaryContext &BC = Func.getBinaryContext();121auto CandidatesIter = StubGroups.find(TgtSym);122if (CandidatesIter == StubGroups.end())123return nullptr;124const StubGroupTy &Candidates = CandidatesIter->second;125if (Candidates.empty())126return nullptr;127auto Cand = llvm::lower_bound(128Candidates, std::make_pair(DotAddress, nullptr),129[&](const std::pair<uint64_t, BinaryBasicBlock *> &LHS,130const std::pair<uint64_t, BinaryBasicBlock *> &RHS) {131return LHS.first < RHS.first;132});133if (Cand == Candidates.end())134return nullptr;135if (Cand != Candidates.begin()) {136const StubTy *LeftCand = std::prev(Cand);137if (Cand->first - DotAddress > DotAddress - LeftCand->first)138Cand = LeftCand;139}140int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;141assert(BitsAvail < 63 && "PCRelEncodingSize is too large to use int64_t to"142"check for out-of-bounds.");143int64_t MaxVal = (1ULL << BitsAvail) - 1;144int64_t MinVal = -(1ULL << BitsAvail);145uint64_t PCRelTgtAddress = Cand->first;146int64_t PCOffset = (int64_t)(PCRelTgtAddress - DotAddress);147
148LLVM_DEBUG({149if (Candidates.size() > 1)150dbgs() << "Considering stub group with " << Candidates.size()151<< " candidates. DotAddress is " << Twine::utohexstr(DotAddress)152<< ", chosen candidate address is "153<< Twine::utohexstr(Cand->first) << "\n";154});155return (PCOffset < MinVal || PCOffset > MaxVal) ? nullptr : Cand->second;156}
157
158BinaryBasicBlock *159LongJmpPass::lookupGlobalStub(const BinaryBasicBlock &SourceBB,160const MCInst &Inst, const MCSymbol *TgtSym,161uint64_t DotAddress) const {162const BinaryFunction &Func = *SourceBB.getFunction();163const StubGroupsTy &StubGroups =164SourceBB.isCold() ? ColdStubGroups : HotStubGroups;165return lookupStubFromGroup(StubGroups, Func, Inst, TgtSym, DotAddress);166}
167
168BinaryBasicBlock *LongJmpPass::lookupLocalStub(const BinaryBasicBlock &SourceBB,169const MCInst &Inst,170const MCSymbol *TgtSym,171uint64_t DotAddress) const {172const BinaryFunction &Func = *SourceBB.getFunction();173const DenseMap<const BinaryFunction *, StubGroupsTy> &StubGroups =174SourceBB.isCold() ? ColdLocalStubs : HotLocalStubs;175const auto Iter = StubGroups.find(&Func);176if (Iter == StubGroups.end())177return nullptr;178return lookupStubFromGroup(Iter->second, Func, Inst, TgtSym, DotAddress);179}
180
181std::unique_ptr<BinaryBasicBlock>182LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst,183uint64_t DotAddress,184uint64_t StubCreationAddress) {185const BinaryFunction &Func = *BB.getFunction();186const BinaryContext &BC = Func.getBinaryContext();187std::unique_ptr<BinaryBasicBlock> NewBB;188const MCSymbol *TgtSym = BC.MIB->getTargetSymbol(Inst);189assert(TgtSym && "getTargetSymbol failed");190
191BinaryBasicBlock::BinaryBranchInfo BI{0, 0};192BinaryBasicBlock *TgtBB = BB.getSuccessor(TgtSym, BI);193auto LocalStubsIter = Stubs.find(&Func);194
195// If already using stub and the stub is from another function, create a local196// stub, since the foreign stub is now out of range197if (!TgtBB) {198auto SSIter = SharedStubs.find(TgtSym);199if (SSIter != SharedStubs.end()) {200TgtSym = BC.MIB->getTargetSymbol(*SSIter->second->begin());201--NumSharedStubs;202}203} else if (LocalStubsIter != Stubs.end() &&204LocalStubsIter->second.count(TgtBB)) {205// The TgtBB and TgtSym now are the local out-of-range stub and its label.206// So, we are attempting to restore BB to its previous state without using207// this stub.208TgtSym = BC.MIB->getTargetSymbol(*TgtBB->begin());209assert(TgtSym &&210"First instruction is expected to contain a target symbol.");211BinaryBasicBlock *TgtBBSucc = TgtBB->getSuccessor(TgtSym, BI);212
213// TgtBB might have no successor. e.g. a stub for a function call.214if (TgtBBSucc) {215BB.replaceSuccessor(TgtBB, TgtBBSucc, BI.Count, BI.MispredictedCount);216assert(TgtBB->getExecutionCount() >= BI.Count &&217"At least equal or greater than the branch count.");218TgtBB->setExecutionCount(TgtBB->getExecutionCount() - BI.Count);219}220
221TgtBB = TgtBBSucc;222}223
224BinaryBasicBlock *StubBB = lookupLocalStub(BB, Inst, TgtSym, DotAddress);225// If not found, look it up in globally shared stub maps if it is a function226// call (TgtBB is not set)227if (!StubBB && !TgtBB) {228StubBB = lookupGlobalStub(BB, Inst, TgtSym, DotAddress);229if (StubBB) {230SharedStubs[StubBB->getLabel()] = StubBB;231++NumSharedStubs;232}233}234MCSymbol *StubSymbol = StubBB ? StubBB->getLabel() : nullptr;235
236if (!StubBB) {237std::tie(NewBB, StubSymbol) =238createNewStub(BB, TgtSym, /*is func?*/ !TgtBB, StubCreationAddress);239StubBB = NewBB.get();240}241
242// Local branch243if (TgtBB) {244uint64_t OrigCount = BI.Count;245uint64_t OrigMispreds = BI.MispredictedCount;246BB.replaceSuccessor(TgtBB, StubBB, OrigCount, OrigMispreds);247StubBB->setExecutionCount(StubBB->getExecutionCount() + OrigCount);248if (NewBB) {249StubBB->addSuccessor(TgtBB, OrigCount, OrigMispreds);250StubBB->setIsCold(BB.isCold());251}252// Call / tail call253} else {254StubBB->setExecutionCount(StubBB->getExecutionCount() +255BB.getExecutionCount());256if (NewBB) {257assert(TgtBB == nullptr);258StubBB->setIsCold(BB.isCold());259// Set as entry point because this block is valid but we have no preds260StubBB->getFunction()->addEntryPoint(*StubBB);261}262}263BC.MIB->replaceBranchTarget(Inst, StubSymbol, BC.Ctx.get());264
265return NewBB;266}
267
268void LongJmpPass::updateStubGroups() {269auto update = [&](StubGroupsTy &StubGroups) {270for (auto &KeyVal : StubGroups) {271for (StubTy &Elem : KeyVal.second)272Elem.first = BBAddresses[Elem.second];273llvm::sort(KeyVal.second, llvm::less_first());274}275};276
277for (auto &KeyVal : HotLocalStubs)278update(KeyVal.second);279for (auto &KeyVal : ColdLocalStubs)280update(KeyVal.second);281update(HotStubGroups);282update(ColdStubGroups);283}
284
285void LongJmpPass::tentativeBBLayout(const BinaryFunction &Func) {286const BinaryContext &BC = Func.getBinaryContext();287uint64_t HotDot = HotAddresses[&Func];288uint64_t ColdDot = ColdAddresses[&Func];289bool Cold = false;290for (const BinaryBasicBlock *BB : Func.getLayout().blocks()) {291if (Cold || BB->isCold()) {292Cold = true;293BBAddresses[BB] = ColdDot;294ColdDot += BC.computeCodeSize(BB->begin(), BB->end());295} else {296BBAddresses[BB] = HotDot;297HotDot += BC.computeCodeSize(BB->begin(), BB->end());298}299}300}
301
302uint64_t LongJmpPass::tentativeLayoutRelocColdPart(303const BinaryContext &BC, std::vector<BinaryFunction *> &SortedFunctions,304uint64_t DotAddress) {305DotAddress = alignTo(DotAddress, llvm::Align(opts::AlignFunctions));306for (BinaryFunction *Func : SortedFunctions) {307if (!Func->isSplit())308continue;309DotAddress = alignTo(DotAddress, Func->getMinAlignment());310uint64_t Pad =311offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));312if (Pad <= Func->getMaxColdAlignmentBytes())313DotAddress += Pad;314ColdAddresses[Func] = DotAddress;315LLVM_DEBUG(dbgs() << Func->getPrintName() << " cold tentative: "316<< Twine::utohexstr(DotAddress) << "\n");317DotAddress += Func->estimateColdSize();318DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());319DotAddress += Func->estimateConstantIslandSize();320}321return DotAddress;322}
323
324uint64_t LongJmpPass::tentativeLayoutRelocMode(325const BinaryContext &BC, std::vector<BinaryFunction *> &SortedFunctions,326uint64_t DotAddress) {327
328// Compute hot cold frontier329uint32_t LastHotIndex = -1u;330uint32_t CurrentIndex = 0;331if (opts::HotFunctionsAtEnd) {332for (BinaryFunction *BF : SortedFunctions) {333if (BF->hasValidIndex()) {334LastHotIndex = CurrentIndex;335break;336}337
338++CurrentIndex;339}340} else {341for (BinaryFunction *BF : SortedFunctions) {342if (!BF->hasValidIndex()) {343LastHotIndex = CurrentIndex;344break;345}346
347++CurrentIndex;348}349}350
351// Hot352CurrentIndex = 0;353bool ColdLayoutDone = false;354for (BinaryFunction *Func : SortedFunctions) {355if (!BC.shouldEmit(*Func)) {356HotAddresses[Func] = Func->getAddress();357continue;358}359
360if (!ColdLayoutDone && CurrentIndex >= LastHotIndex) {361DotAddress =362tentativeLayoutRelocColdPart(BC, SortedFunctions, DotAddress);363ColdLayoutDone = true;364if (opts::HotFunctionsAtEnd)365DotAddress = alignTo(DotAddress, opts::AlignText);366}367
368DotAddress = alignTo(DotAddress, Func->getMinAlignment());369uint64_t Pad =370offsetToAlignment(DotAddress, llvm::Align(Func->getAlignment()));371if (Pad <= Func->getMaxAlignmentBytes())372DotAddress += Pad;373HotAddresses[Func] = DotAddress;374LLVM_DEBUG(dbgs() << Func->getPrintName() << " tentative: "375<< Twine::utohexstr(DotAddress) << "\n");376if (!Func->isSplit())377DotAddress += Func->estimateSize();378else379DotAddress += Func->estimateHotSize();380
381DotAddress = alignTo(DotAddress, Func->getConstantIslandAlignment());382DotAddress += Func->estimateConstantIslandSize();383++CurrentIndex;384}385// BBs386for (BinaryFunction *Func : SortedFunctions)387tentativeBBLayout(*Func);388
389return DotAddress;390}
391
392void LongJmpPass::tentativeLayout(393const BinaryContext &BC, std::vector<BinaryFunction *> &SortedFunctions) {394uint64_t DotAddress = BC.LayoutStartAddress;395
396if (!BC.HasRelocations) {397for (BinaryFunction *Func : SortedFunctions) {398HotAddresses[Func] = Func->getAddress();399DotAddress = alignTo(DotAddress, ColdFragAlign);400ColdAddresses[Func] = DotAddress;401if (Func->isSplit())402DotAddress += Func->estimateColdSize();403tentativeBBLayout(*Func);404}405
406return;407}408
409// Relocation mode410uint64_t EstimatedTextSize = 0;411if (opts::UseOldText) {412EstimatedTextSize = tentativeLayoutRelocMode(BC, SortedFunctions, 0);413
414// Initial padding415if (EstimatedTextSize <= BC.OldTextSectionSize) {416DotAddress = BC.OldTextSectionAddress;417uint64_t Pad =418offsetToAlignment(DotAddress, llvm::Align(opts::AlignText));419if (Pad + EstimatedTextSize <= BC.OldTextSectionSize) {420DotAddress += Pad;421}422}423}424
425if (!EstimatedTextSize || EstimatedTextSize > BC.OldTextSectionSize)426DotAddress = alignTo(BC.LayoutStartAddress, opts::AlignText);427
428tentativeLayoutRelocMode(BC, SortedFunctions, DotAddress);429}
430
431bool LongJmpPass::usesStub(const BinaryFunction &Func,432const MCInst &Inst) const {433const MCSymbol *TgtSym = Func.getBinaryContext().MIB->getTargetSymbol(Inst);434const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(TgtSym);435auto Iter = Stubs.find(&Func);436if (Iter != Stubs.end())437return Iter->second.count(TgtBB);438return false;439}
440
441uint64_t LongJmpPass::getSymbolAddress(const BinaryContext &BC,442const MCSymbol *Target,443const BinaryBasicBlock *TgtBB) const {444if (TgtBB) {445auto Iter = BBAddresses.find(TgtBB);446assert(Iter != BBAddresses.end() && "Unrecognized BB");447return Iter->second;448}449uint64_t EntryID = 0;450const BinaryFunction *TargetFunc = BC.getFunctionForSymbol(Target, &EntryID);451auto Iter = HotAddresses.find(TargetFunc);452if (Iter == HotAddresses.end() || (TargetFunc && EntryID)) {453// Look at BinaryContext's resolution for this symbol - this is a symbol not454// mapped to a BinaryFunction455ErrorOr<uint64_t> ValueOrError = BC.getSymbolValue(*Target);456assert(ValueOrError && "Unrecognized symbol");457return *ValueOrError;458}459return Iter->second;460}
461
462Error LongJmpPass::relaxStub(BinaryBasicBlock &StubBB, bool &Modified) {463const BinaryFunction &Func = *StubBB.getFunction();464const BinaryContext &BC = Func.getBinaryContext();465const int Bits = StubBits[&StubBB];466// Already working with the largest range?467if (Bits == static_cast<int>(BC.AsmInfo->getCodePointerSize() * 8))468return Error::success();469
470const static int RangeShortJmp = BC.MIB->getShortJmpEncodingSize();471const static int RangeSingleInstr = BC.MIB->getUncondBranchEncodingSize();472const static uint64_t ShortJmpMask = ~((1ULL << RangeShortJmp) - 1);473const static uint64_t SingleInstrMask =474~((1ULL << (RangeSingleInstr - 1)) - 1);475
476const MCSymbol *RealTargetSym = BC.MIB->getTargetSymbol(*StubBB.begin());477const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(RealTargetSym);478uint64_t TgtAddress = getSymbolAddress(BC, RealTargetSym, TgtBB);479uint64_t DotAddress = BBAddresses[&StubBB];480uint64_t PCRelTgtAddress = DotAddress > TgtAddress ? DotAddress - TgtAddress481: TgtAddress - DotAddress;482// If it fits in one instruction, do not relax483if (!(PCRelTgtAddress & SingleInstrMask))484return Error::success();485
486// Fits short jmp487if (!(PCRelTgtAddress & ShortJmpMask)) {488if (Bits >= RangeShortJmp)489return Error::success();490
491LLVM_DEBUG(dbgs() << "Relaxing stub to short jump. PCRelTgtAddress = "492<< Twine::utohexstr(PCRelTgtAddress)493<< " RealTargetSym = " << RealTargetSym->getName()494<< "\n");495relaxStubToShortJmp(StubBB, RealTargetSym);496StubBits[&StubBB] = RangeShortJmp;497Modified = true;498return Error::success();499}500
501// The long jmp uses absolute address on AArch64502// So we could not use it for PIC binaries503if (BC.isAArch64() && !BC.HasFixedLoadAddress)504return createFatalBOLTError(505"BOLT-ERROR: Unable to relax stub for PIC binary\n");506
507LLVM_DEBUG(dbgs() << "Relaxing stub to long jump. PCRelTgtAddress = "508<< Twine::utohexstr(PCRelTgtAddress)509<< " RealTargetSym = " << RealTargetSym->getName() << "\n");510relaxStubToLongJmp(StubBB, RealTargetSym);511StubBits[&StubBB] = static_cast<int>(BC.AsmInfo->getCodePointerSize() * 8);512Modified = true;513return Error::success();514}
515
516bool LongJmpPass::needsStub(const BinaryBasicBlock &BB, const MCInst &Inst,517uint64_t DotAddress) const {518const BinaryFunction &Func = *BB.getFunction();519const BinaryContext &BC = Func.getBinaryContext();520const MCSymbol *TgtSym = BC.MIB->getTargetSymbol(Inst);521assert(TgtSym && "getTargetSymbol failed");522
523const BinaryBasicBlock *TgtBB = Func.getBasicBlockForLabel(TgtSym);524// Check for shared stubs from foreign functions525if (!TgtBB) {526auto SSIter = SharedStubs.find(TgtSym);527if (SSIter != SharedStubs.end())528TgtBB = SSIter->second;529}530
531int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;532assert(BitsAvail < 63 && "PCRelEncodingSize is too large to use int64_t to"533"check for out-of-bounds.");534int64_t MaxVal = (1ULL << BitsAvail) - 1;535int64_t MinVal = -(1ULL << BitsAvail);536
537uint64_t PCRelTgtAddress = getSymbolAddress(BC, TgtSym, TgtBB);538int64_t PCOffset = (int64_t)(PCRelTgtAddress - DotAddress);539
540return PCOffset < MinVal || PCOffset > MaxVal;541}
542
543Error LongJmpPass::relax(BinaryFunction &Func, bool &Modified) {544const BinaryContext &BC = Func.getBinaryContext();545
546assert(BC.isAArch64() && "Unsupported arch");547constexpr int InsnSize = 4; // AArch64548std::vector<std::pair<BinaryBasicBlock *, std::unique_ptr<BinaryBasicBlock>>>549Insertions;550
551BinaryBasicBlock *Frontier = getBBAtHotColdSplitPoint(Func);552uint64_t FrontierAddress = Frontier ? BBAddresses[Frontier] : 0;553if (FrontierAddress)554FrontierAddress += Frontier->getNumNonPseudos() * InsnSize;555
556// Add necessary stubs for branch targets we know we can't fit in the557// instruction558for (BinaryBasicBlock &BB : Func) {559uint64_t DotAddress = BBAddresses[&BB];560// Stubs themselves are relaxed on the next loop561if (Stubs[&Func].count(&BB))562continue;563
564for (MCInst &Inst : BB) {565if (BC.MIB->isPseudo(Inst))566continue;567
568if (!shouldInsertStub(BC, Inst)) {569DotAddress += InsnSize;570continue;571}572
573// Check and relax direct branch or call574if (!needsStub(BB, Inst, DotAddress)) {575DotAddress += InsnSize;576continue;577}578Modified = true;579
580// Insert stubs close to the patched BB if call, but far away from the581// hot path if a branch, since this branch target is the cold region582// (but first check that the far away stub will be in range).583BinaryBasicBlock *InsertionPoint = &BB;584if (Func.isSimple() && !BC.MIB->isCall(Inst) && FrontierAddress &&585!BB.isCold()) {586int BitsAvail = BC.MIB->getPCRelEncodingSize(Inst) - 1;587uint64_t Mask = ~((1ULL << BitsAvail) - 1);588assert(FrontierAddress > DotAddress &&589"Hot code should be before the frontier");590uint64_t PCRelTgt = FrontierAddress - DotAddress;591if (!(PCRelTgt & Mask))592InsertionPoint = Frontier;593}594// Always put stubs at the end of the function if non-simple. We can't595// change the layout of non-simple functions because it has jump tables596// that we do not control.597if (!Func.isSimple())598InsertionPoint = &*std::prev(Func.end());599
600// Create a stub to handle a far-away target601Insertions.emplace_back(InsertionPoint,602replaceTargetWithStub(BB, Inst, DotAddress,603InsertionPoint == Frontier604? FrontierAddress605: DotAddress));606
607DotAddress += InsnSize;608}609}610
611// Relax stubs if necessary612for (BinaryBasicBlock &BB : Func) {613if (!Stubs[&Func].count(&BB) || !BB.isValid())614continue;615
616if (auto E = relaxStub(BB, Modified))617return Error(std::move(E));618}619
620for (std::pair<BinaryBasicBlock *, std::unique_ptr<BinaryBasicBlock>> &Elmt :621Insertions) {622if (!Elmt.second)623continue;624std::vector<std::unique_ptr<BinaryBasicBlock>> NewBBs;625NewBBs.emplace_back(std::move(Elmt.second));626Func.insertBasicBlocks(Elmt.first, std::move(NewBBs), true);627}628
629return Error::success();630}
631
632Error LongJmpPass::runOnFunctions(BinaryContext &BC) {633BC.outs() << "BOLT-INFO: Starting stub-insertion pass\n";634std::vector<BinaryFunction *> Sorted = BC.getSortedFunctions();635bool Modified;636uint32_t Iterations = 0;637do {638++Iterations;639Modified = false;640tentativeLayout(BC, Sorted);641updateStubGroups();642for (BinaryFunction *Func : Sorted) {643if (auto E = relax(*Func, Modified))644return Error(std::move(E));645// Don't ruin non-simple functions, they can't afford to have the layout646// changed.647if (Modified && Func->isSimple())648Func->fixBranches();649}650} while (Modified);651BC.outs() << "BOLT-INFO: Inserted " << NumHotStubs652<< " stubs in the hot area and " << NumColdStubs653<< " stubs in the cold area. Shared " << NumSharedStubs654<< " times, iterated " << Iterations << " times.\n";655return Error::success();656}
657} // namespace bolt658} // namespace llvm659