lib/Target/X86/X86AvoidTrailingCall.cpp

   1 //===----- X86AvoidTrailingCall.cpp - Insert int3 after trailing calls ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // The Windows x64 unwinder has trouble unwinding the stack when a return
  10 // address points to the end of the function. This pass maintains the invariant
  11 // that every return address is inside the bounds of its parent function or
  12 // funclet by inserting int3 if the last instruction would otherwise be a call.
  13 //
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "X86.h"
  17 #include "X86InstrInfo.h"
  18 #include "X86Subtarget.h"
  19 #include "llvm/CodeGen/MachineInstrBuilder.h"
  20
  21 #define DEBUG_TYPE "x86-avoid-trailing-call"
  22
  23 using namespace llvm;
  24
  25 namespace {
  26
  27 class X86AvoidTrailingCallPass : public MachineFunctionPass {
  28 public:
  29   X86AvoidTrailingCallPass() : MachineFunctionPass(ID) {}
  30
  31   bool runOnMachineFunction(MachineFunction &MF) override;
  32
  33 private:
  34   StringRef getPassName() const override {
  35     return "X86 avoid trailing call pass";
  36   }
  37   static char ID;
  38 };
  39
  40 char X86AvoidTrailingCallPass::ID = 0;
  41
  42 } // end anonymous namespace
  43
  44 FunctionPass *llvm::createX86AvoidTrailingCallPass() {
  45   return new X86AvoidTrailingCallPass();
  46 }
  47
  48 // A real instruction is a non-meta, non-pseudo instruction.  Some pseudos
  49 // expand to nothing, and some expand to code. This logic conservatively assumes
  50 // they might expand to nothing.
  51 static bool isRealInstruction(MachineInstr &MI) {
  52   return !MI.isPseudo() && !MI.isMetaInstruction();
  53 }
  54
  55 // Return true if this is a call instruction, but not a tail call.
  56 static bool isCallInstruction(const MachineInstr &MI) {
  57   return MI.isCall() && !MI.isReturn();
  58 }
  59
  60 bool X86AvoidTrailingCallPass::runOnMachineFunction(MachineFunction &MF) {
  61   const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>();
  62   const X86InstrInfo &TII = *STI.getInstrInfo();
  63   assert(STI.isTargetWin64() && "pass only runs on Win64");
  64
  65   // FIXME: Perhaps this pass should also replace SEH_Epilogue by inserting nops
  66   // before epilogues.
  67
  68   bool Changed = false;
  69   for (MachineBasicBlock &MBB : MF) {
  70     // Look for basic blocks that precede funclet entries or are at the end of
  71     // the function.
  72     MachineBasicBlock *NextMBB = MBB.getNextNode();
  73     if (NextMBB && !NextMBB->isEHFuncletEntry())
  74       continue;
  75
  76     // Find the last real instruction in this block, or previous blocks if this
  77     // block is empty.
  78     MachineBasicBlock::reverse_iterator LastRealInstr;
  79     for (MachineBasicBlock &RMBB :
  80          make_range(MBB.getReverseIterator(), MF.rend())) {
  81       LastRealInstr = llvm::find_if(reverse(RMBB), isRealInstruction);
  82       if (LastRealInstr != RMBB.rend())
  83         break;
  84     }
  85
  86     // Do nothing if this function or funclet has no instructions.
  87     if (LastRealInstr == MF.begin()->rend())
  88       continue;
  89
  90     // If this is a call instruction, insert int3 right after it with the same
  91     // DebugLoc. Convert back to a forward iterator and advance the insertion
  92     // position once.
  93     if (isCallInstruction(*LastRealInstr)) {
  94       LLVM_DEBUG({
  95         dbgs() << "inserting int3 after trailing call instruction:\n";
  96         LastRealInstr->dump();
  97         dbgs() << '\n';
  98       });
  99
 100       MachineBasicBlock::iterator MBBI = std::next(LastRealInstr.getReverse());
 101       BuildMI(*LastRealInstr->getParent(), MBBI, LastRealInstr->getDebugLoc(),
 102               TII.get(X86::INT3));
 103       Changed = true;
 104     }
 105   }
 106
 107   return Changed;
 108 }