1 //===- ARMSLSHardening.cpp - Harden Straight Line Missspeculation ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a pass to insert code to mitigate against side channel
10 // vulnerabilities that may happen under straight line miss-speculation.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "ARM.h"
15 #include "ARMInstrInfo.h"
16 #include "ARMSubtarget.h"
17 #include "llvm/CodeGen/IndirectThunks.h"
18 #include "llvm/CodeGen/MachineBasicBlock.h"
19 #include "llvm/CodeGen/MachineFunction.h"
20 #include "llvm/CodeGen/MachineFunctionPass.h"
21 #include "llvm/CodeGen/MachineInstr.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineOperand.h"
24 #include "llvm/IR/DebugLoc.h"
25 #include <cassert>
26
27 using namespace llvm;
28
29 #define DEBUG_TYPE "arm-sls-hardening"
30
31 #define ARM_SLS_HARDENING_NAME "ARM sls hardening pass"
32
33 namespace {
34
35 class ARMSLSHardening : public MachineFunctionPass {
36 public:
37 const TargetInstrInfo *TII;
38 const ARMSubtarget *ST;
39
40 static char ID;
41
ARMSLSHardening()42 ARMSLSHardening() : MachineFunctionPass(ID) {
43 initializeARMSLSHardeningPass(*PassRegistry::getPassRegistry());
44 }
45
46 bool runOnMachineFunction(MachineFunction &Fn) override;
47
getPassName() const48 StringRef getPassName() const override { return ARM_SLS_HARDENING_NAME; }
49
getAnalysisUsage(AnalysisUsage & AU) const50 void getAnalysisUsage(AnalysisUsage &AU) const override {
51 AU.setPreservesCFG();
52 MachineFunctionPass::getAnalysisUsage(AU);
53 }
54
55 private:
56 bool hardenReturnsAndBRs(MachineBasicBlock &MBB) const;
57 bool hardenIndirectCalls(MachineBasicBlock &MBB) const;
58 MachineBasicBlock &
59 ConvertIndirectCallToIndirectJump(MachineBasicBlock &MBB,
60 MachineBasicBlock::iterator) const;
61 };
62
63 } // end anonymous namespace
64
65 char ARMSLSHardening::ID = 0;
66
67 INITIALIZE_PASS(ARMSLSHardening, "arm-sls-hardening",
68 ARM_SLS_HARDENING_NAME, false, false)
69
insertSpeculationBarrier(const ARMSubtarget * ST,MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI,DebugLoc DL,bool AlwaysUseISBDSB=false)70 static void insertSpeculationBarrier(const ARMSubtarget *ST,
71 MachineBasicBlock &MBB,
72 MachineBasicBlock::iterator MBBI,
73 DebugLoc DL,
74 bool AlwaysUseISBDSB = false) {
75 assert(MBBI != MBB.begin() &&
76 "Must not insert SpeculationBarrierEndBB as only instruction in MBB.");
77 assert(std::prev(MBBI)->isBarrier() &&
78 "SpeculationBarrierEndBB must only follow unconditional control flow "
79 "instructions.");
80 assert(std::prev(MBBI)->isTerminator() &&
81 "SpeculationBarrierEndBB must only follow terminators.");
82 const TargetInstrInfo *TII = ST->getInstrInfo();
83 assert(ST->hasDataBarrier() || ST->hasSB());
84 bool ProduceSB = ST->hasSB() && !AlwaysUseISBDSB;
85 unsigned BarrierOpc =
86 ProduceSB ? (ST->isThumb() ? ARM::t2SpeculationBarrierSBEndBB
87 : ARM::SpeculationBarrierSBEndBB)
88 : (ST->isThumb() ? ARM::t2SpeculationBarrierISBDSBEndBB
89 : ARM::SpeculationBarrierISBDSBEndBB);
90 if (MBBI == MBB.end() || !isSpeculationBarrierEndBBOpcode(MBBI->getOpcode()))
91 BuildMI(MBB, MBBI, DL, TII->get(BarrierOpc));
92 }
93
runOnMachineFunction(MachineFunction & MF)94 bool ARMSLSHardening::runOnMachineFunction(MachineFunction &MF) {
95 ST = &MF.getSubtarget<ARMSubtarget>();
96 TII = MF.getSubtarget().getInstrInfo();
97
98 bool Modified = false;
99 for (auto &MBB : MF) {
100 Modified |= hardenReturnsAndBRs(MBB);
101 Modified |= hardenIndirectCalls(MBB);
102 }
103
104 return Modified;
105 }
106
hardenReturnsAndBRs(MachineBasicBlock & MBB) const107 bool ARMSLSHardening::hardenReturnsAndBRs(MachineBasicBlock &MBB) const {
108 if (!ST->hardenSlsRetBr())
109 return false;
110 assert(!ST->isThumb1Only());
111 bool Modified = false;
112 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(), E = MBB.end();
113 MachineBasicBlock::iterator NextMBBI;
114 for (; MBBI != E; MBBI = NextMBBI) {
115 MachineInstr &MI = *MBBI;
116 NextMBBI = std::next(MBBI);
117 if (isIndirectControlFlowNotComingBack(MI)) {
118 assert(MI.isTerminator());
119 assert(!TII->isPredicated(MI));
120 insertSpeculationBarrier(ST, MBB, std::next(MBBI), MI.getDebugLoc());
121 Modified = true;
122 }
123 }
124 return Modified;
125 }
126
127 static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
128
129 static const struct ThunkNameRegMode {
130 const char* Name;
131 Register Reg;
132 bool isThumb;
133 } SLSBLRThunks[] = {
134 {"__llvm_slsblr_thunk_arm_r0", ARM::R0, false},
135 {"__llvm_slsblr_thunk_arm_r1", ARM::R1, false},
136 {"__llvm_slsblr_thunk_arm_r2", ARM::R2, false},
137 {"__llvm_slsblr_thunk_arm_r3", ARM::R3, false},
138 {"__llvm_slsblr_thunk_arm_r4", ARM::R4, false},
139 {"__llvm_slsblr_thunk_arm_r5", ARM::R5, false},
140 {"__llvm_slsblr_thunk_arm_r6", ARM::R6, false},
141 {"__llvm_slsblr_thunk_arm_r7", ARM::R7, false},
142 {"__llvm_slsblr_thunk_arm_r8", ARM::R8, false},
143 {"__llvm_slsblr_thunk_arm_r9", ARM::R9, false},
144 {"__llvm_slsblr_thunk_arm_r10", ARM::R10, false},
145 {"__llvm_slsblr_thunk_arm_r11", ARM::R11, false},
146 {"__llvm_slsblr_thunk_arm_sp", ARM::SP, false},
147 {"__llvm_slsblr_thunk_arm_pc", ARM::PC, false},
148 {"__llvm_slsblr_thunk_thumb_r0", ARM::R0, true},
149 {"__llvm_slsblr_thunk_thumb_r1", ARM::R1, true},
150 {"__llvm_slsblr_thunk_thumb_r2", ARM::R2, true},
151 {"__llvm_slsblr_thunk_thumb_r3", ARM::R3, true},
152 {"__llvm_slsblr_thunk_thumb_r4", ARM::R4, true},
153 {"__llvm_slsblr_thunk_thumb_r5", ARM::R5, true},
154 {"__llvm_slsblr_thunk_thumb_r6", ARM::R6, true},
155 {"__llvm_slsblr_thunk_thumb_r7", ARM::R7, true},
156 {"__llvm_slsblr_thunk_thumb_r8", ARM::R8, true},
157 {"__llvm_slsblr_thunk_thumb_r9", ARM::R9, true},
158 {"__llvm_slsblr_thunk_thumb_r10", ARM::R10, true},
159 {"__llvm_slsblr_thunk_thumb_r11", ARM::R11, true},
160 {"__llvm_slsblr_thunk_thumb_sp", ARM::SP, true},
161 {"__llvm_slsblr_thunk_thumb_pc", ARM::PC, true},
162 };
163
164 // An enum for tracking whether Arm and Thumb thunks have been inserted into the
165 // current module so far.
166 enum ArmInsertedThunks { ArmThunk = 1, ThumbThunk = 2 };
167
operator |=(ArmInsertedThunks & X,ArmInsertedThunks Y)168 inline ArmInsertedThunks &operator|=(ArmInsertedThunks &X,
169 ArmInsertedThunks Y) {
170 return X = static_cast<ArmInsertedThunks>(X | Y);
171 }
172
173 namespace {
174 struct SLSBLRThunkInserter
175 : ThunkInserter<SLSBLRThunkInserter, ArmInsertedThunks> {
getThunkPrefix__anon772ea6ed0211::SLSBLRThunkInserter176 const char *getThunkPrefix() { return SLSBLRNamePrefix; }
mayUseThunk__anon772ea6ed0211::SLSBLRThunkInserter177 bool mayUseThunk(const MachineFunction &MF,
178 ArmInsertedThunks InsertedThunks) {
179 if ((InsertedThunks & ArmThunk &&
180 !MF.getSubtarget<ARMSubtarget>().isThumb()) ||
181 (InsertedThunks & ThumbThunk &&
182 MF.getSubtarget<ARMSubtarget>().isThumb()))
183 return false;
184 ComdatThunks &= !MF.getSubtarget<ARMSubtarget>().hardenSlsNoComdat();
185 // FIXME: This could also check if there are any indirect calls in the
186 // function to more accurately reflect if a thunk will be needed.
187 return MF.getSubtarget<ARMSubtarget>().hardenSlsBlr();
188 }
189 ArmInsertedThunks insertThunks(MachineModuleInfo &MMI, MachineFunction &MF);
190 void populateThunk(MachineFunction &MF);
191
192 private:
193 bool ComdatThunks = true;
194 };
195 } // namespace
196
insertThunks(MachineModuleInfo & MMI,MachineFunction & MF)197 ArmInsertedThunks SLSBLRThunkInserter::insertThunks(MachineModuleInfo &MMI,
198 MachineFunction &MF) {
199 // FIXME: It probably would be possible to filter which thunks to produce
200 // based on which registers are actually used in indirect calls in this
201 // function. But would that be a worthwhile optimization?
202 const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
203 for (auto T : SLSBLRThunks)
204 if (ST->isThumb() == T.isThumb)
205 createThunkFunction(MMI, T.Name, ComdatThunks,
206 T.isThumb ? "+thumb-mode" : "");
207 return ST->isThumb() ? ThumbThunk : ArmThunk;
208 }
209
populateThunk(MachineFunction & MF)210 void SLSBLRThunkInserter::populateThunk(MachineFunction &MF) {
211 // FIXME: How to better communicate Register number, rather than through
212 // name and lookup table?
213 assert(MF.getName().starts_with(getThunkPrefix()));
214 auto ThunkIt = llvm::find_if(
215 SLSBLRThunks, [&MF](auto T) { return T.Name == MF.getName(); });
216 assert(ThunkIt != std::end(SLSBLRThunks));
217 Register ThunkReg = ThunkIt->Reg;
218 bool isThumb = ThunkIt->isThumb;
219
220 const TargetInstrInfo *TII = MF.getSubtarget<ARMSubtarget>().getInstrInfo();
221 MachineBasicBlock *Entry = &MF.front();
222 Entry->clear();
223
224 // These thunks need to consist of the following instructions:
225 // __llvm_slsblr_thunk_(arm/thumb)_rN:
226 // bx rN
227 // barrierInsts
228 Entry->addLiveIn(ThunkReg);
229 if (isThumb)
230 BuildMI(Entry, DebugLoc(), TII->get(ARM::tBX))
231 .addReg(ThunkReg)
232 .add(predOps(ARMCC::AL));
233 else
234 BuildMI(Entry, DebugLoc(), TII->get(ARM::BX))
235 .addReg(ThunkReg);
236
237 // Make sure the thunks do not make use of the SB extension in case there is
238 // a function somewhere that will call to it that for some reason disabled
239 // the SB extension locally on that function, even though it's enabled for
240 // the module otherwise. Therefore set AlwaysUseISBSDB to true.
241 insertSpeculationBarrier(&MF.getSubtarget<ARMSubtarget>(), *Entry,
242 Entry->end(), DebugLoc(), true /*AlwaysUseISBDSB*/);
243 }
244
ConvertIndirectCallToIndirectJump(MachineBasicBlock & MBB,MachineBasicBlock::iterator MBBI) const245 MachineBasicBlock &ARMSLSHardening::ConvertIndirectCallToIndirectJump(
246 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
247 // Transform an indirect call to an indirect jump as follows:
248 // Before:
249 // |-----------------------------|
250 // | ... |
251 // | instI |
252 // | BLX rN |
253 // | instJ |
254 // | ... |
255 // |-----------------------------|
256 //
257 // After:
258 // |---------- -------------------------|
259 // | ... |
260 // | instI |
261 // | *call* __llvm_slsblr_thunk_mode_xN |
262 // | instJ |
263 // | ... |
264 // |--------------------------------------|
265 //
266 // __llvm_slsblr_thunk_mode_xN:
267 // |-----------------------------|
268 // | BX rN |
269 // | barrierInsts |
270 // |-----------------------------|
271 //
272 // The __llvm_slsblr_thunk_mode_xN thunks are created by the
273 // SLSBLRThunkInserter.
274 // This function merely needs to transform an indirect call to a direct call
275 // to __llvm_slsblr_thunk_xN.
276 MachineInstr &IndirectCall = *MBBI;
277 assert(isIndirectCall(IndirectCall) && !IndirectCall.isReturn());
278 int RegOpIdxOnIndirectCall = -1;
279 bool isThumb;
280 switch (IndirectCall.getOpcode()) {
281 case ARM::BLX: // !isThumb2
282 case ARM::BLX_noip: // !isThumb2
283 isThumb = false;
284 RegOpIdxOnIndirectCall = 0;
285 break;
286 case ARM::tBLXr: // isThumb2
287 case ARM::tBLXr_noip: // isThumb2
288 isThumb = true;
289 RegOpIdxOnIndirectCall = 2;
290 break;
291 default:
292 llvm_unreachable("unhandled Indirect Call");
293 }
294
295 Register Reg = IndirectCall.getOperand(RegOpIdxOnIndirectCall).getReg();
296 // Since linkers are allowed to clobber R12 on function calls, the above
297 // mitigation only works if the original indirect call instruction was not
298 // using R12. Code generation before must make sure that no indirect call
299 // using R12 was produced if the mitigation is enabled.
300 // Also, the transformation is incorrect if the indirect call uses LR, so
301 // also have to avoid that.
302 assert(Reg != ARM::R12 && Reg != ARM::LR);
303 bool RegIsKilled = IndirectCall.getOperand(RegOpIdxOnIndirectCall).isKill();
304
305 DebugLoc DL = IndirectCall.getDebugLoc();
306
307 MachineFunction &MF = *MBBI->getMF();
308 auto ThunkIt = llvm::find_if(SLSBLRThunks, [Reg, isThumb](auto T) {
309 return T.Reg == Reg && T.isThumb == isThumb;
310 });
311 assert(ThunkIt != std::end(SLSBLRThunks));
312 Module *M = MF.getFunction().getParent();
313 const GlobalValue *GV = cast<GlobalValue>(M->getNamedValue(ThunkIt->Name));
314
315 MachineInstr *BL =
316 isThumb ? BuildMI(MBB, MBBI, DL, TII->get(ARM::tBL))
317 .addImm(IndirectCall.getOperand(0).getImm())
318 .addReg(IndirectCall.getOperand(1).getReg())
319 .addGlobalAddress(GV)
320 : BuildMI(MBB, MBBI, DL, TII->get(ARM::BL)).addGlobalAddress(GV);
321
322 // Now copy the implicit operands from IndirectCall to BL and copy other
323 // necessary info.
324 // However, both IndirectCall and BL instructions implictly use SP and
325 // implicitly define LR. Blindly copying implicit operands would result in SP
326 // and LR operands to be present multiple times. While this may not be too
327 // much of an issue, let's avoid that for cleanliness, by removing those
328 // implicit operands from the BL created above before we copy over all
329 // implicit operands from the IndirectCall.
330 int ImpLROpIdx = -1;
331 int ImpSPOpIdx = -1;
332 for (unsigned OpIdx = BL->getNumExplicitOperands();
333 OpIdx < BL->getNumOperands(); OpIdx++) {
334 MachineOperand Op = BL->getOperand(OpIdx);
335 if (!Op.isReg())
336 continue;
337 if (Op.getReg() == ARM::LR && Op.isDef())
338 ImpLROpIdx = OpIdx;
339 if (Op.getReg() == ARM::SP && !Op.isDef())
340 ImpSPOpIdx = OpIdx;
341 }
342 assert(ImpLROpIdx != -1);
343 assert(ImpSPOpIdx != -1);
344 int FirstOpIdxToRemove = std::max(ImpLROpIdx, ImpSPOpIdx);
345 int SecondOpIdxToRemove = std::min(ImpLROpIdx, ImpSPOpIdx);
346 BL->removeOperand(FirstOpIdxToRemove);
347 BL->removeOperand(SecondOpIdxToRemove);
348 // Now copy over the implicit operands from the original IndirectCall
349 BL->copyImplicitOps(MF, IndirectCall);
350 MF.moveCallSiteInfo(&IndirectCall, BL);
351 // Also add the register called in the IndirectCall as being used in the
352 // called thunk.
353 BL->addOperand(MachineOperand::CreateReg(Reg, false /*isDef*/, true /*isImp*/,
354 RegIsKilled /*isKill*/));
355 // Remove IndirectCallinstruction
356 MBB.erase(MBBI);
357 return MBB;
358 }
359
hardenIndirectCalls(MachineBasicBlock & MBB) const360 bool ARMSLSHardening::hardenIndirectCalls(MachineBasicBlock &MBB) const {
361 if (!ST->hardenSlsBlr())
362 return false;
363 bool Modified = false;
364 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
365 MachineBasicBlock::iterator NextMBBI;
366 for (; MBBI != E; MBBI = NextMBBI) {
367 MachineInstr &MI = *MBBI;
368 NextMBBI = std::next(MBBI);
369 // Tail calls are both indirect calls and "returns".
370 // They are also indirect jumps, so should be handled by sls-harden-retbr,
371 // rather than sls-harden-blr.
372 if (isIndirectCall(MI) && !MI.isReturn()) {
373 ConvertIndirectCallToIndirectJump(MBB, MBBI);
374 Modified = true;
375 }
376 }
377 return Modified;
378 }
379
380
381
createARMSLSHardeningPass()382 FunctionPass *llvm::createARMSLSHardeningPass() {
383 return new ARMSLSHardening();
384 }
385
386 namespace {
387 class ARMIndirectThunks : public MachineFunctionPass {
388 public:
389 static char ID;
390
ARMIndirectThunks()391 ARMIndirectThunks() : MachineFunctionPass(ID) {}
392
getPassName() const393 StringRef getPassName() const override { return "ARM Indirect Thunks"; }
394
395 bool doInitialization(Module &M) override;
396 bool runOnMachineFunction(MachineFunction &MF) override;
397
getAnalysisUsage(AnalysisUsage & AU) const398 void getAnalysisUsage(AnalysisUsage &AU) const override {
399 MachineFunctionPass::getAnalysisUsage(AU);
400 AU.addRequired<MachineModuleInfoWrapperPass>();
401 AU.addPreserved<MachineModuleInfoWrapperPass>();
402 }
403
404 private:
405 std::tuple<SLSBLRThunkInserter> TIs;
406
407 // FIXME: When LLVM moves to C++17, these can become folds
408 template <typename... ThunkInserterT>
initTIs(Module & M,std::tuple<ThunkInserterT...> & ThunkInserters)409 static void initTIs(Module &M,
410 std::tuple<ThunkInserterT...> &ThunkInserters) {
411 (void)std::initializer_list<int>{
412 (std::get<ThunkInserterT>(ThunkInserters).init(M), 0)...};
413 }
414 template <typename... ThunkInserterT>
runTIs(MachineModuleInfo & MMI,MachineFunction & MF,std::tuple<ThunkInserterT...> & ThunkInserters)415 static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF,
416 std::tuple<ThunkInserterT...> &ThunkInserters) {
417 bool Modified = false;
418 (void)std::initializer_list<int>{
419 Modified |= std::get<ThunkInserterT>(ThunkInserters).run(MMI, MF)...};
420 return Modified;
421 }
422 };
423
424 } // end anonymous namespace
425
426 char ARMIndirectThunks::ID = 0;
427
createARMIndirectThunks()428 FunctionPass *llvm::createARMIndirectThunks() {
429 return new ARMIndirectThunks();
430 }
431
doInitialization(Module & M)432 bool ARMIndirectThunks::doInitialization(Module &M) {
433 initTIs(M, TIs);
434 return false;
435 }
436
runOnMachineFunction(MachineFunction & MF)437 bool ARMIndirectThunks::runOnMachineFunction(MachineFunction &MF) {
438 LLVM_DEBUG(dbgs() << getPassName() << '\n');
439 auto &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
440 return runTIs(MMI, MF, TIs);
441 }
442