llvm/lib/Target/X86/X86IndirectThunks.cpp

   1 //==- X86IndirectThunks.cpp - Construct indirect call/jump thunks for x86  --=//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 ///
  10 /// Pass that injects an MI thunk that is used to lower indirect calls in a way
  11 /// that prevents speculation on some x86 processors and can be used to mitigate
  12 /// security vulnerabilities due to targeted speculative execution and side
  13 /// channels such as CVE-2017-5715.
  14 ///
  15 /// Currently supported thunks include:
  16 /// - Retpoline -- A RET-implemented trampoline that lowers indirect calls
  17 /// - LVI Thunk -- A CALL/JMP-implemented thunk that forces load serialization
  18 ///   before making an indirect call/jump
  19 ///
  20 /// Note that the reason that this is implemented as a MachineFunctionPass and
  21 /// not a ModulePass is that ModulePasses at this point in the LLVM X86 pipeline
  22 /// serialize all transformations, which can consume lots of memory.
  23 ///
  24 /// TODO(chandlerc): All of this code could use better comments and
  25 /// documentation.
  26 ///
  27 //===----------------------------------------------------------------------===//
  28
  29 #include "X86.h"
  30 #include "X86InstrBuilder.h"
  31 #include "X86Subtarget.h"
  32 #include "llvm/CodeGen/IndirectThunks.h"
  33 #include "llvm/CodeGen/MachineFunction.h"
  34 #include "llvm/CodeGen/MachineFunctionPass.h"
  35 #include "llvm/CodeGen/MachineInstrBuilder.h"
  36 #include "llvm/CodeGen/MachineModuleInfo.h"
  37 #include "llvm/CodeGen/Passes.h"
  38 #include "llvm/CodeGen/TargetPassConfig.h"
  39 #include "llvm/IR/IRBuilder.h"
  40 #include "llvm/IR/Instructions.h"
  41 #include "llvm/IR/Module.h"
  42 #include "llvm/Support/CommandLine.h"
  43 #include "llvm/Support/Debug.h"
  44 #include "llvm/Support/raw_ostream.h"
  45 #include "llvm/Target/TargetMachine.h"
  46
  47 using namespace llvm;
  48
  49 #define DEBUG_TYPE "x86-retpoline-thunks"
  50
  51 static const char RetpolineNamePrefix[] = "__llvm_retpoline_";
  52 static const char R11RetpolineName[] = "__llvm_retpoline_r11";
  53 static const char EAXRetpolineName[] = "__llvm_retpoline_eax";
  54 static const char ECXRetpolineName[] = "__llvm_retpoline_ecx";
  55 static const char EDXRetpolineName[] = "__llvm_retpoline_edx";
  56 static const char EDIRetpolineName[] = "__llvm_retpoline_edi";
  57
  58 static const char LVIThunkNamePrefix[] = "__llvm_lvi_thunk_";
  59 static const char R11LVIThunkName[] = "__llvm_lvi_thunk_r11";
  60
  61 namespace {
  62 struct RetpolineThunkInserter : ThunkInserter<RetpolineThunkInserter> {
  63   const char *getThunkPrefix() { return RetpolineNamePrefix; }
  64   bool mayUseThunk(const MachineFunction &MF, bool InsertedThunks) {
  65     if (InsertedThunks)
  66       return false;
  67     const auto &STI = MF.getSubtarget<X86Subtarget>();
  68     return (STI.useRetpolineIndirectCalls() ||
  69             STI.useRetpolineIndirectBranches()) &&
  70            !STI.useRetpolineExternalThunk();
  71   }
  72   bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF);
  73   void populateThunk(MachineFunction &MF);
  74 };
  75
  76 struct LVIThunkInserter : ThunkInserter<LVIThunkInserter> {
  77   const char *getThunkPrefix() { return LVIThunkNamePrefix; }
  78   bool mayUseThunk(const MachineFunction &MF, bool InsertedThunks) {
  79     if (InsertedThunks)
  80       return false;
  81     return MF.getSubtarget<X86Subtarget>().useLVIControlFlowIntegrity();
  82   }
  83   bool insertThunks(MachineModuleInfo &MMI, MachineFunction &MF) {
  84     createThunkFunction(MMI, R11LVIThunkName);
  85     return true;
  86   }
  87   void populateThunk(MachineFunction &MF) {
  88     assert (MF.size() == 1);
  89     MachineBasicBlock *Entry = &MF.front();
  90     Entry->clear();
  91
  92     // This code mitigates LVI by replacing each indirect call/jump with a
  93     // direct call/jump to a thunk that looks like:
  94     // ```
  95     // lfence
  96     // jmpq *%r11
  97     // ```
  98     // This ensures that if the value in register %r11 was loaded from memory,
  99     // then the value in %r11 is (architecturally) correct prior to the jump.
 100     const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
 101     BuildMI(&MF.front(), DebugLoc(), TII->get(X86::LFENCE));
 102     BuildMI(&MF.front(), DebugLoc(), TII->get(X86::JMP64r)).addReg(X86::R11);
 103     MF.front().addLiveIn(X86::R11);
 104   }
 105 };
 106
 107 class X86IndirectThunks : public MachineFunctionPass {
 108 public:
 109   static char ID;
 110
 111   X86IndirectThunks() : MachineFunctionPass(ID) {}
 112
 113   StringRef getPassName() const override { return "X86 Indirect Thunks"; }
 114
 115   bool doInitialization(Module &M) override;
 116   bool runOnMachineFunction(MachineFunction &MF) override;
 117
 118 private:
 119   std::tuple<RetpolineThunkInserter, LVIThunkInserter> TIs;
 120
 121   // FIXME: When LLVM moves to C++17, these can become folds
 122   template <typename... ThunkInserterT>
 123   static void initTIs(Module &M,
 124                       std::tuple<ThunkInserterT...> &ThunkInserters) {
 125     (void)std::initializer_list<int>{
 126         (std::get<ThunkInserterT>(ThunkInserters).init(M), 0)...};
 127   }
 128   template <typename... ThunkInserterT>
 129   static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF,
 130                      std::tuple<ThunkInserterT...> &ThunkInserters) {
 131     bool Modified = false;
 132     (void)std::initializer_list<int>{
 133         Modified |= std::get<ThunkInserterT>(ThunkInserters).run(MMI, MF)...};
 134     return Modified;
 135   }
 136 };
 137
 138 } // end anonymous namespace
 139
 140 bool RetpolineThunkInserter::insertThunks(MachineModuleInfo &MMI,
 141                                           MachineFunction &MF) {
 142   if (MMI.getTarget().getTargetTriple().getArch() == Triple::x86_64)
 143     createThunkFunction(MMI, R11RetpolineName);
 144   else
 145     for (StringRef Name : {EAXRetpolineName, ECXRetpolineName, EDXRetpolineName,
 146                            EDIRetpolineName})
 147       createThunkFunction(MMI, Name);
 148   return true;
 149 }
 150
 151 void RetpolineThunkInserter::populateThunk(MachineFunction &MF) {
 152   bool Is64Bit = MF.getTarget().getTargetTriple().getArch() == Triple::x86_64;
 153   Register ThunkReg;
 154   if (Is64Bit) {
 155     assert(MF.getName() == "__llvm_retpoline_r11" &&
 156            "Should only have an r11 thunk on 64-bit targets");
 157
 158     // __llvm_retpoline_r11:
 159     //   callq .Lr11_call_target
 160     // .Lr11_capture_spec:
 161     //   pause
 162     //   lfence
 163     //   jmp .Lr11_capture_spec
 164     // .align 16
 165     // .Lr11_call_target:
 166     //   movq %r11, (%rsp)
 167     //   retq
 168     ThunkReg = X86::R11;
 169   } else {
 170     // For 32-bit targets we need to emit a collection of thunks for various
 171     // possible scratch registers as well as a fallback that uses EDI, which is
 172     // normally callee saved.
 173     //   __llvm_retpoline_eax:
 174     //         calll .Leax_call_target
 175     //   .Leax_capture_spec:
 176     //         pause
 177     //         jmp .Leax_capture_spec
 178     //   .align 16
 179     //   .Leax_call_target:
 180     //         movl %eax, (%esp)  # Clobber return addr
 181     //         retl
 182     //
 183     //   __llvm_retpoline_ecx:
 184     //   ... # Same setup
 185     //         movl %ecx, (%esp)
 186     //         retl
 187     //
 188     //   __llvm_retpoline_edx:
 189     //   ... # Same setup
 190     //         movl %edx, (%esp)
 191     //         retl
 192     //
 193     //   __llvm_retpoline_edi:
 194     //   ... # Same setup
 195     //         movl %edi, (%esp)
 196     //         retl
 197     if (MF.getName() == EAXRetpolineName)
 198       ThunkReg = X86::EAX;
 199     else if (MF.getName() == ECXRetpolineName)
 200       ThunkReg = X86::ECX;
 201     else if (MF.getName() == EDXRetpolineName)
 202       ThunkReg = X86::EDX;
 203     else if (MF.getName() == EDIRetpolineName)
 204       ThunkReg = X86::EDI;
 205     else
 206       llvm_unreachable("Invalid thunk name on x86-32!");
 207   }
 208
 209   const TargetInstrInfo *TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
 210   assert (MF.size() == 1);
 211   MachineBasicBlock *Entry = &MF.front();
 212   Entry->clear();
 213
 214   MachineBasicBlock *CaptureSpec =
 215       MF.CreateMachineBasicBlock(Entry->getBasicBlock());
 216   MachineBasicBlock *CallTarget =
 217       MF.CreateMachineBasicBlock(Entry->getBasicBlock());
 218   MCSymbol *TargetSym = MF.getContext().createTempSymbol();
 219   MF.push_back(CaptureSpec);
 220   MF.push_back(CallTarget);
 221
 222   const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
 223   const unsigned RetOpc = Is64Bit ? X86::RET64 : X86::RET32;
 224
 225   Entry->addLiveIn(ThunkReg);
 226   BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym);
 227
 228   // The MIR verifier thinks that the CALL in the entry block will fall through
 229   // to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is
 230   // the successor, but the MIR verifier doesn't know how to cope with that.
 231   Entry->addSuccessor(CaptureSpec);
 232
 233   // In the capture loop for speculation, we want to stop the processor from
 234   // speculating as fast as possible. On Intel processors, the PAUSE instruction
 235   // will block speculation without consuming any execution resources. On AMD
 236   // processors, the PAUSE instruction is (essentially) a nop, so we also use an
 237   // LFENCE instruction which they have advised will stop speculation as well
 238   // with minimal resource utilization. We still end the capture with a jump to
 239   // form an infinite loop to fully guarantee that no matter what implementation
 240   // of the x86 ISA, speculating this code path never escapes.
 241   BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE));
 242   BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE));
 243   BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec);
 244   CaptureSpec->setMachineBlockAddressTaken();
 245   CaptureSpec->addSuccessor(CaptureSpec);
 246
 247   CallTarget->addLiveIn(ThunkReg);
 248   CallTarget->setMachineBlockAddressTaken();
 249   CallTarget->setAlignment(Align(16));
 250
 251   // Insert return address clobber
 252   const unsigned MovOpc = Is64Bit ? X86::MOV64mr : X86::MOV32mr;
 253   const Register SPReg = Is64Bit ? X86::RSP : X86::ESP;
 254   addRegOffset(BuildMI(CallTarget, DebugLoc(), TII->get(MovOpc)), SPReg, false,
 255                0)
 256       .addReg(ThunkReg);
 257
 258   CallTarget->back().setPreInstrSymbol(MF, TargetSym);
 259   BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
 260 }
 261
 262 FunctionPass *llvm::createX86IndirectThunksPass() {
 263   return new X86IndirectThunks();
 264 }
 265
 266 char X86IndirectThunks::ID = 0;
 267
 268 bool X86IndirectThunks::doInitialization(Module &M) {
 269   initTIs(M, TIs);
 270   return false;
 271 }
 272
 273 bool X86IndirectThunks::runOnMachineFunction(MachineFunction &MF) {
 274   LLVM_DEBUG(dbgs() << getPassName() << '\n');
 275   auto &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
 276   return runTIs(MMI, MF, TIs);
 277 }