1 //===- ARMSLSHardening.cpp - Harden Straight Line Missspeculation ---------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file contains a pass to insert code to mitigate against side channel
10 // vulnerabilities that may happen under straight line miss-speculation.
12 //===----------------------------------------------------------------------===//
15 #include "ARMInstrInfo.h"
16 #include "ARMSubtarget.h"
17 #include "llvm/CodeGen/IndirectThunks.h"
18 #include "llvm/CodeGen/MachineBasicBlock.h"
19 #include "llvm/CodeGen/MachineFunction.h"
20 #include "llvm/CodeGen/MachineFunctionPass.h"
21 #include "llvm/CodeGen/MachineInstr.h"
22 #include "llvm/CodeGen/MachineInstrBuilder.h"
23 #include "llvm/CodeGen/MachineOperand.h"
24 #include "llvm/IR/DebugLoc.h"
29 #define DEBUG_TYPE "arm-sls-hardening"
31 #define ARM_SLS_HARDENING_NAME "ARM sls hardening pass"
35 class ARMSLSHardening
: public MachineFunctionPass
{
37 const TargetInstrInfo
*TII
;
38 const ARMSubtarget
*ST
;
42 ARMSLSHardening() : MachineFunctionPass(ID
) {
43 initializeARMSLSHardeningPass(*PassRegistry::getPassRegistry());
46 bool runOnMachineFunction(MachineFunction
&Fn
) override
;
48 StringRef
getPassName() const override
{ return ARM_SLS_HARDENING_NAME
; }
50 void getAnalysisUsage(AnalysisUsage
&AU
) const override
{
52 MachineFunctionPass::getAnalysisUsage(AU
);
56 bool hardenReturnsAndBRs(MachineBasicBlock
&MBB
) const;
57 bool hardenIndirectCalls(MachineBasicBlock
&MBB
) const;
59 ConvertIndirectCallToIndirectJump(MachineBasicBlock
&MBB
,
60 MachineBasicBlock::iterator
) const;
63 } // end anonymous namespace
65 char ARMSLSHardening::ID
= 0;
67 INITIALIZE_PASS(ARMSLSHardening
, "arm-sls-hardening",
68 ARM_SLS_HARDENING_NAME
, false, false)
70 static void insertSpeculationBarrier(const ARMSubtarget
*ST
,
71 MachineBasicBlock
&MBB
,
72 MachineBasicBlock::iterator MBBI
,
74 bool AlwaysUseISBDSB
= false) {
75 assert(MBBI
!= MBB
.begin() &&
76 "Must not insert SpeculationBarrierEndBB as only instruction in MBB.");
77 assert(std::prev(MBBI
)->isBarrier() &&
78 "SpeculationBarrierEndBB must only follow unconditional control flow "
80 assert(std::prev(MBBI
)->isTerminator() &&
81 "SpeculationBarrierEndBB must only follow terminators.");
82 const TargetInstrInfo
*TII
= ST
->getInstrInfo();
83 assert(ST
->hasDataBarrier() || ST
->hasSB());
84 bool ProduceSB
= ST
->hasSB() && !AlwaysUseISBDSB
;
86 ProduceSB
? (ST
->isThumb() ? ARM::t2SpeculationBarrierSBEndBB
87 : ARM::SpeculationBarrierSBEndBB
)
88 : (ST
->isThumb() ? ARM::t2SpeculationBarrierISBDSBEndBB
89 : ARM::SpeculationBarrierISBDSBEndBB
);
90 if (MBBI
== MBB
.end() || !isSpeculationBarrierEndBBOpcode(MBBI
->getOpcode()))
91 BuildMI(MBB
, MBBI
, DL
, TII
->get(BarrierOpc
));
94 bool ARMSLSHardening::runOnMachineFunction(MachineFunction
&MF
) {
95 ST
= &MF
.getSubtarget
<ARMSubtarget
>();
96 TII
= MF
.getSubtarget().getInstrInfo();
98 bool Modified
= false;
99 for (auto &MBB
: MF
) {
100 Modified
|= hardenReturnsAndBRs(MBB
);
101 Modified
|= hardenIndirectCalls(MBB
);
107 bool ARMSLSHardening::hardenReturnsAndBRs(MachineBasicBlock
&MBB
) const {
108 if (!ST
->hardenSlsRetBr())
110 assert(!ST
->isThumb1Only());
111 bool Modified
= false;
112 MachineBasicBlock::iterator MBBI
= MBB
.getFirstTerminator(), E
= MBB
.end();
113 MachineBasicBlock::iterator NextMBBI
;
114 for (; MBBI
!= E
; MBBI
= NextMBBI
) {
115 MachineInstr
&MI
= *MBBI
;
116 NextMBBI
= std::next(MBBI
);
117 if (isIndirectControlFlowNotComingBack(MI
)) {
118 assert(MI
.isTerminator());
119 assert(!TII
->isPredicated(MI
));
120 insertSpeculationBarrier(ST
, MBB
, std::next(MBBI
), MI
.getDebugLoc());
127 static const char SLSBLRNamePrefix
[] = "__llvm_slsblr_thunk_";
129 static const struct ThunkNameRegMode
{
134 {"__llvm_slsblr_thunk_arm_r0", ARM::R0
, false},
135 {"__llvm_slsblr_thunk_arm_r1", ARM::R1
, false},
136 {"__llvm_slsblr_thunk_arm_r2", ARM::R2
, false},
137 {"__llvm_slsblr_thunk_arm_r3", ARM::R3
, false},
138 {"__llvm_slsblr_thunk_arm_r4", ARM::R4
, false},
139 {"__llvm_slsblr_thunk_arm_r5", ARM::R5
, false},
140 {"__llvm_slsblr_thunk_arm_r6", ARM::R6
, false},
141 {"__llvm_slsblr_thunk_arm_r7", ARM::R7
, false},
142 {"__llvm_slsblr_thunk_arm_r8", ARM::R8
, false},
143 {"__llvm_slsblr_thunk_arm_r9", ARM::R9
, false},
144 {"__llvm_slsblr_thunk_arm_r10", ARM::R10
, false},
145 {"__llvm_slsblr_thunk_arm_r11", ARM::R11
, false},
146 {"__llvm_slsblr_thunk_arm_sp", ARM::SP
, false},
147 {"__llvm_slsblr_thunk_arm_pc", ARM::PC
, false},
148 {"__llvm_slsblr_thunk_thumb_r0", ARM::R0
, true},
149 {"__llvm_slsblr_thunk_thumb_r1", ARM::R1
, true},
150 {"__llvm_slsblr_thunk_thumb_r2", ARM::R2
, true},
151 {"__llvm_slsblr_thunk_thumb_r3", ARM::R3
, true},
152 {"__llvm_slsblr_thunk_thumb_r4", ARM::R4
, true},
153 {"__llvm_slsblr_thunk_thumb_r5", ARM::R5
, true},
154 {"__llvm_slsblr_thunk_thumb_r6", ARM::R6
, true},
155 {"__llvm_slsblr_thunk_thumb_r7", ARM::R7
, true},
156 {"__llvm_slsblr_thunk_thumb_r8", ARM::R8
, true},
157 {"__llvm_slsblr_thunk_thumb_r9", ARM::R9
, true},
158 {"__llvm_slsblr_thunk_thumb_r10", ARM::R10
, true},
159 {"__llvm_slsblr_thunk_thumb_r11", ARM::R11
, true},
160 {"__llvm_slsblr_thunk_thumb_sp", ARM::SP
, true},
161 {"__llvm_slsblr_thunk_thumb_pc", ARM::PC
, true},
164 // An enum for tracking whether Arm and Thumb thunks have been inserted into the
165 // current module so far.
166 enum ArmInsertedThunks
{ NoThunk
= 0, ArmThunk
= 1, ThumbThunk
= 2 };
168 inline ArmInsertedThunks
&operator|=(ArmInsertedThunks
&X
,
169 ArmInsertedThunks Y
) {
170 return X
= static_cast<ArmInsertedThunks
>(X
| Y
);
174 struct SLSBLRThunkInserter
175 : ThunkInserter
<SLSBLRThunkInserter
, ArmInsertedThunks
> {
176 const char *getThunkPrefix() { return SLSBLRNamePrefix
; }
177 bool mayUseThunk(const MachineFunction
&MF
) {
178 ComdatThunks
&= !MF
.getSubtarget
<ARMSubtarget
>().hardenSlsNoComdat();
179 return MF
.getSubtarget
<ARMSubtarget
>().hardenSlsBlr();
181 ArmInsertedThunks
insertThunks(MachineModuleInfo
&MMI
, MachineFunction
&MF
,
182 ArmInsertedThunks InsertedThunks
);
183 void populateThunk(MachineFunction
&MF
);
186 bool ComdatThunks
= true;
191 SLSBLRThunkInserter::insertThunks(MachineModuleInfo
&MMI
, MachineFunction
&MF
,
192 ArmInsertedThunks InsertedThunks
) {
193 if ((InsertedThunks
& ArmThunk
&&
194 !MF
.getSubtarget
<ARMSubtarget
>().isThumb()) ||
195 (InsertedThunks
& ThumbThunk
&&
196 MF
.getSubtarget
<ARMSubtarget
>().isThumb()))
198 // FIXME: It probably would be possible to filter which thunks to produce
199 // based on which registers are actually used in indirect calls in this
200 // function. But would that be a worthwhile optimization?
201 const ARMSubtarget
*ST
= &MF
.getSubtarget
<ARMSubtarget
>();
202 for (auto T
: SLSBLRThunks
)
203 if (ST
->isThumb() == T
.isThumb
)
204 createThunkFunction(MMI
, T
.Name
, ComdatThunks
,
205 T
.isThumb
? "+thumb-mode" : "");
206 return ST
->isThumb() ? ThumbThunk
: ArmThunk
;
209 void SLSBLRThunkInserter::populateThunk(MachineFunction
&MF
) {
210 assert(MF
.getFunction().hasComdat() == ComdatThunks
&&
211 "ComdatThunks value changed since MF creation");
212 // FIXME: How to better communicate Register number, rather than through
213 // name and lookup table?
214 assert(MF
.getName().starts_with(getThunkPrefix()));
215 auto ThunkIt
= llvm::find_if(
216 SLSBLRThunks
, [&MF
](auto T
) { return T
.Name
== MF
.getName(); });
217 assert(ThunkIt
!= std::end(SLSBLRThunks
));
218 Register ThunkReg
= ThunkIt
->Reg
;
219 bool isThumb
= ThunkIt
->isThumb
;
221 const TargetInstrInfo
*TII
= MF
.getSubtarget
<ARMSubtarget
>().getInstrInfo();
222 MachineBasicBlock
*Entry
= &MF
.front();
225 // These thunks need to consist of the following instructions:
226 // __llvm_slsblr_thunk_(arm/thumb)_rN:
229 Entry
->addLiveIn(ThunkReg
);
231 BuildMI(Entry
, DebugLoc(), TII
->get(ARM::tBX
))
233 .add(predOps(ARMCC::AL
));
235 BuildMI(Entry
, DebugLoc(), TII
->get(ARM::BX
))
238 // Make sure the thunks do not make use of the SB extension in case there is
239 // a function somewhere that will call to it that for some reason disabled
240 // the SB extension locally on that function, even though it's enabled for
241 // the module otherwise. Therefore set AlwaysUseISBSDB to true.
242 insertSpeculationBarrier(&MF
.getSubtarget
<ARMSubtarget
>(), *Entry
,
243 Entry
->end(), DebugLoc(), true /*AlwaysUseISBDSB*/);
246 MachineBasicBlock
&ARMSLSHardening::ConvertIndirectCallToIndirectJump(
247 MachineBasicBlock
&MBB
, MachineBasicBlock::iterator MBBI
) const {
248 // Transform an indirect call to an indirect jump as follows:
250 // |-----------------------------|
256 // |-----------------------------|
259 // |---------- -------------------------|
262 // | *call* __llvm_slsblr_thunk_mode_xN |
265 // |--------------------------------------|
267 // __llvm_slsblr_thunk_mode_xN:
268 // |-----------------------------|
271 // |-----------------------------|
273 // The __llvm_slsblr_thunk_mode_xN thunks are created by the
274 // SLSBLRThunkInserter.
275 // This function merely needs to transform an indirect call to a direct call
276 // to __llvm_slsblr_thunk_xN.
277 MachineInstr
&IndirectCall
= *MBBI
;
278 assert(isIndirectCall(IndirectCall
) && !IndirectCall
.isReturn());
279 int RegOpIdxOnIndirectCall
= -1;
281 switch (IndirectCall
.getOpcode()) {
282 case ARM::BLX
: // !isThumb2
283 case ARM::BLX_noip
: // !isThumb2
285 RegOpIdxOnIndirectCall
= 0;
287 case ARM::tBLXr
: // isThumb2
288 case ARM::tBLXr_noip
: // isThumb2
290 RegOpIdxOnIndirectCall
= 2;
293 llvm_unreachable("unhandled Indirect Call");
296 Register Reg
= IndirectCall
.getOperand(RegOpIdxOnIndirectCall
).getReg();
297 // Since linkers are allowed to clobber R12 on function calls, the above
298 // mitigation only works if the original indirect call instruction was not
299 // using R12. Code generation before must make sure that no indirect call
300 // using R12 was produced if the mitigation is enabled.
301 // Also, the transformation is incorrect if the indirect call uses LR, so
302 // also have to avoid that.
303 assert(Reg
!= ARM::R12
&& Reg
!= ARM::LR
);
304 bool RegIsKilled
= IndirectCall
.getOperand(RegOpIdxOnIndirectCall
).isKill();
306 DebugLoc DL
= IndirectCall
.getDebugLoc();
308 MachineFunction
&MF
= *MBBI
->getMF();
309 auto ThunkIt
= llvm::find_if(SLSBLRThunks
, [Reg
, isThumb
](auto T
) {
310 return T
.Reg
== Reg
&& T
.isThumb
== isThumb
;
312 assert(ThunkIt
!= std::end(SLSBLRThunks
));
313 Module
*M
= MF
.getFunction().getParent();
314 const GlobalValue
*GV
= cast
<GlobalValue
>(M
->getNamedValue(ThunkIt
->Name
));
317 isThumb
? BuildMI(MBB
, MBBI
, DL
, TII
->get(ARM::tBL
))
318 .addImm(IndirectCall
.getOperand(0).getImm())
319 .addReg(IndirectCall
.getOperand(1).getReg())
320 .addGlobalAddress(GV
)
321 : BuildMI(MBB
, MBBI
, DL
, TII
->get(ARM::BL
)).addGlobalAddress(GV
);
323 // Now copy the implicit operands from IndirectCall to BL and copy other
325 // However, both IndirectCall and BL instructions implictly use SP and
326 // implicitly define LR. Blindly copying implicit operands would result in SP
327 // and LR operands to be present multiple times. While this may not be too
328 // much of an issue, let's avoid that for cleanliness, by removing those
329 // implicit operands from the BL created above before we copy over all
330 // implicit operands from the IndirectCall.
333 for (unsigned OpIdx
= BL
->getNumExplicitOperands();
334 OpIdx
< BL
->getNumOperands(); OpIdx
++) {
335 MachineOperand Op
= BL
->getOperand(OpIdx
);
338 if (Op
.getReg() == ARM::LR
&& Op
.isDef())
340 if (Op
.getReg() == ARM::SP
&& !Op
.isDef())
343 assert(ImpLROpIdx
!= -1);
344 assert(ImpSPOpIdx
!= -1);
345 int FirstOpIdxToRemove
= std::max(ImpLROpIdx
, ImpSPOpIdx
);
346 int SecondOpIdxToRemove
= std::min(ImpLROpIdx
, ImpSPOpIdx
);
347 BL
->removeOperand(FirstOpIdxToRemove
);
348 BL
->removeOperand(SecondOpIdxToRemove
);
349 // Now copy over the implicit operands from the original IndirectCall
350 BL
->copyImplicitOps(MF
, IndirectCall
);
351 MF
.moveCallSiteInfo(&IndirectCall
, BL
);
352 // Also add the register called in the IndirectCall as being used in the
354 BL
->addOperand(MachineOperand::CreateReg(Reg
, false /*isDef*/, true /*isImp*/,
355 RegIsKilled
/*isKill*/));
356 // Remove IndirectCallinstruction
361 bool ARMSLSHardening::hardenIndirectCalls(MachineBasicBlock
&MBB
) const {
362 if (!ST
->hardenSlsBlr())
364 bool Modified
= false;
365 MachineBasicBlock::iterator MBBI
= MBB
.begin(), E
= MBB
.end();
366 MachineBasicBlock::iterator NextMBBI
;
367 for (; MBBI
!= E
; MBBI
= NextMBBI
) {
368 MachineInstr
&MI
= *MBBI
;
369 NextMBBI
= std::next(MBBI
);
370 // Tail calls are both indirect calls and "returns".
371 // They are also indirect jumps, so should be handled by sls-harden-retbr,
372 // rather than sls-harden-blr.
373 if (isIndirectCall(MI
) && !MI
.isReturn()) {
374 ConvertIndirectCallToIndirectJump(MBB
, MBBI
);
383 FunctionPass
*llvm::createARMSLSHardeningPass() {
384 return new ARMSLSHardening();
388 class ARMIndirectThunks
: public ThunkInserterPass
<SLSBLRThunkInserter
> {
392 ARMIndirectThunks() : ThunkInserterPass(ID
) {}
394 StringRef
getPassName() const override
{ return "ARM Indirect Thunks"; }
396 } // end anonymous namespace
398 char ARMIndirectThunks::ID
= 0;
400 FunctionPass
*llvm::createARMIndirectThunks() {
401 return new ARMIndirectThunks();