llvm/lib/Target/X86/X86AvoidTrailingCall.cpp

   1 //===----- X86AvoidTrailingCall.cpp - Insert int3 after trailing calls ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // The Windows x64 unwinder decodes the instruction stream during unwinding.
  10 // The unwinder decodes forward from the current PC to detect epilogue code
  11 // patterns.
  12 //
  13 // First, this means that there must be an instruction after every
  14 // call instruction for the unwinder to decode. LLVM must maintain the invariant
  15 // that the last instruction of a function or funclet is not a call, or the
  16 // unwinder may decode into the next function. Similarly, a call may not
  17 // immediately precede an epilogue code pattern. As of this writing, the
  18 // SEH_Epilogue pseudo instruction takes care of that.
  19 //
  20 // Second, all non-tail call jump targets must be within the *half-open*
  21 // interval of the bounds of the function. The unwinder distinguishes between
  22 // internal jump instructions and tail calls in an epilogue sequence by checking
  23 // the jump target against the function bounds from the .pdata section. This
  24 // means that the last regular MBB of an LLVM function must not be empty if
  25 // there are regular jumps targeting it.
  26 //
  27 // This pass upholds these invariants by ensuring that blocks at the end of a
  28 // function or funclet are a) not empty and b) do not end in a CALL instruction.
  29 //
  30 // Unwinder implementation for reference:
  31 // https://github.com/dotnet/coreclr/blob/a9f3fc16483eecfc47fb79c362811d870be02249/src/unwinder/amd64/unwinder_amd64.cpp#L1015
  32 //
  33 //===----------------------------------------------------------------------===//
  34
  35 #include "X86.h"
  36 #include "X86InstrInfo.h"
  37 #include "X86Subtarget.h"
  38 #include "llvm/CodeGen/MachineFunctionPass.h"
  39 #include "llvm/CodeGen/MachineInstrBuilder.h"
  40
  41 #define AVOIDCALL_DESC "X86 avoid trailing call pass"
  42 #define AVOIDCALL_NAME "x86-avoid-trailing-call"
  43
  44 #define DEBUG_TYPE AVOIDCALL_NAME
  45
  46 using namespace llvm;
  47
  48 namespace {
  49 class X86AvoidTrailingCallPass : public MachineFunctionPass {
  50 public:
  51   X86AvoidTrailingCallPass() : MachineFunctionPass(ID) {}
  52
  53   bool runOnMachineFunction(MachineFunction &MF) override;
  54
  55   static char ID;
  56
  57 private:
  58   StringRef getPassName() const override { return AVOIDCALL_DESC; }
  59 };
  60 } // end anonymous namespace
  61
  62 char X86AvoidTrailingCallPass::ID = 0;
  63
  64 FunctionPass *llvm::createX86AvoidTrailingCallPass() {
  65   return new X86AvoidTrailingCallPass();
  66 }
  67
  68 INITIALIZE_PASS(X86AvoidTrailingCallPass, AVOIDCALL_NAME, AVOIDCALL_DESC, false, false)
  69
  70 // A real instruction is a non-meta, non-pseudo instruction.  Some pseudos
  71 // expand to nothing, and some expand to code. This logic conservatively assumes
  72 // they might expand to nothing.
  73 static bool isCallOrRealInstruction(MachineInstr &MI) {
  74   return MI.isCall() || (!MI.isPseudo() && !MI.isMetaInstruction());
  75 }
  76
  77 // Return true if this is a call instruction, but not a tail call.
  78 static bool isCallInstruction(const MachineInstr &MI) {
  79   return MI.isCall() && !MI.isReturn();
  80 }
  81
  82 bool X86AvoidTrailingCallPass::runOnMachineFunction(MachineFunction &MF) {
  83   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
  84   const X86InstrInfo &TII = *STI.getInstrInfo();
  85   assert(STI.isTargetWin64() && "pass only runs on Win64");
  86
  87   // We don't need to worry about any of the invariants described above if there
  88   // is no unwind info (CFI).
  89   if (!MF.hasWinCFI())
  90     return false;
  91
  92   // FIXME: Perhaps this pass should also replace SEH_Epilogue by inserting nops
  93   // before epilogues.
  94
  95   bool Changed = false;
  96   for (MachineBasicBlock &MBB : MF) {
  97     // Look for basic blocks that precede funclet entries or are at the end of
  98     // the function.
  99     MachineBasicBlock *NextMBB = MBB.getNextNode();
 100     if (NextMBB && !NextMBB->isEHFuncletEntry())
 101       continue;
 102
 103     // Find the last real instruction in this block.
 104     auto LastRealInstr = llvm::find_if(reverse(MBB), isCallOrRealInstruction);
 105
 106     // If the block is empty or the last real instruction is a call instruction,
 107     // insert an int3. If there is a call instruction, insert the int3 between
 108     // the call and any labels or other meta instructions. If the block is
 109     // empty, insert at block end.
 110     bool IsEmpty = LastRealInstr == MBB.rend();
 111     bool IsCall = !IsEmpty && isCallInstruction(*LastRealInstr);
 112     if (IsEmpty || IsCall) {
 113       LLVM_DEBUG({
 114         if (IsCall) {
 115           dbgs() << "inserting int3 after trailing call instruction:\n";
 116           LastRealInstr->dump();
 117           dbgs() << '\n';
 118         } else {
 119           dbgs() << "inserting int3 in trailing empty MBB:\n";
 120           MBB.dump();
 121         }
 122       });
 123
 124       MachineBasicBlock::iterator MBBI = MBB.end();
 125       DebugLoc DL;
 126       if (IsCall) {
 127         MBBI = std::next(LastRealInstr.getReverse());
 128         DL = LastRealInstr->getDebugLoc();
 129       }
 130       BuildMI(MBB, MBBI, DL, TII.get(X86::INT3));
 131       Changed = true;
 132     }
 133   }
 134
 135   return Changed;
 136 }