llvm/lib/Target/X86/X86VZeroUpper.cpp

   1 //===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines the pass which inserts x86 AVX vzeroupper instructions
  10 // before calls to SSE encoded functions. This avoids transition latency
  11 // penalty when transferring control between AVX encoded instructions and old
  12 // SSE encoding mode.
  13 //
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "X86.h"
  17 #include "X86InstrInfo.h"
  18 #include "X86Subtarget.h"
  19 #include "llvm/ADT/SmallVector.h"
  20 #include "llvm/ADT/Statistic.h"
  21 #include "llvm/CodeGen/MachineBasicBlock.h"
  22 #include "llvm/CodeGen/MachineFunction.h"
  23 #include "llvm/CodeGen/MachineFunctionPass.h"
  24 #include "llvm/CodeGen/MachineInstr.h"
  25 #include "llvm/CodeGen/MachineInstrBuilder.h"
  26 #include "llvm/CodeGen/MachineOperand.h"
  27 #include "llvm/CodeGen/MachineRegisterInfo.h"
  28 #include "llvm/CodeGen/TargetInstrInfo.h"
  29 #include "llvm/CodeGen/TargetRegisterInfo.h"
  30 #include "llvm/IR/CallingConv.h"
  31 #include "llvm/IR/DebugLoc.h"
  32 #include "llvm/IR/Function.h"
  33 #include "llvm/Support/Debug.h"
  34 #include "llvm/Support/ErrorHandling.h"
  35 #include "llvm/Support/raw_ostream.h"
  36 #include <cassert>
  37
  38 using namespace llvm;
  39
  40 #define DEBUG_TYPE "x86-vzeroupper"
  41
  42 static cl::opt<bool>
  43 UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
  44   cl::desc("Minimize AVX to SSE transition penalty"),
  45   cl::init(true));
  46
  47 STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
  48
  49 namespace {
  50
  51   class VZeroUpperInserter : public MachineFunctionPass {
  52   public:
  53     VZeroUpperInserter() : MachineFunctionPass(ID) {}
  54
  55     bool runOnMachineFunction(MachineFunction &MF) override;
  56
  57     MachineFunctionProperties getRequiredProperties() const override {
  58       return MachineFunctionProperties().set(
  59           MachineFunctionProperties::Property::NoVRegs);
  60     }
  61
  62     StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
  63
  64   private:
  65     void processBasicBlock(MachineBasicBlock &MBB);
  66     void insertVZeroUpper(MachineBasicBlock::iterator I,
  67                           MachineBasicBlock &MBB);
  68     void addDirtySuccessor(MachineBasicBlock &MBB);
  69
  70     using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
  71
  72     static const char* getBlockExitStateName(BlockExitState ST);
  73
  74     // Core algorithm state:
  75     // BlockState - Each block is either:
  76     //   - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
  77     //                   vzeroupper instructions in this block.
  78     //   - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
  79     //                  block that will ensure that YMM/ZMM is clean on exit.
  80     //   - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
  81     //                  subsequent vzeroupper in the block clears it.
  82     //
  83     // AddedToDirtySuccessors - This flag is raised when a block is added to the
  84     //                          DirtySuccessors list to ensure that it's not
  85     //                          added multiple times.
  86     //
  87     // FirstUnguardedCall - Records the location of the first unguarded call in
  88     //                      each basic block that may need to be guarded by a
  89     //                      vzeroupper. We won't know whether it actually needs
  90     //                      to be guarded until we discover a predecessor that
  91     //                      is DIRTY_OUT.
  92     struct BlockState {
  93       BlockExitState ExitState = PASS_THROUGH;
  94       bool AddedToDirtySuccessors = false;
  95       MachineBasicBlock::iterator FirstUnguardedCall;
  96
  97       BlockState() = default;
  98     };
  99
 100     using BlockStateMap = SmallVector<BlockState, 8>;
 101     using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
 102
 103     BlockStateMap BlockStates;
 104     DirtySuccessorsWorkList DirtySuccessors;
 105     bool EverMadeChange;
 106     bool IsX86INTR;
 107     const TargetInstrInfo *TII;
 108
 109     static char ID;
 110   };
 111
 112 } // end anonymous namespace
 113
 114 char VZeroUpperInserter::ID = 0;
 115
 116 FunctionPass *llvm::createX86IssueVZeroUpperPass() {
 117   return new VZeroUpperInserter();
 118 }
 119
 120 #ifndef NDEBUG
 121 const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
 122   switch (ST) {
 123     case PASS_THROUGH: return "Pass-through";
 124     case EXITS_DIRTY: return "Exits-dirty";
 125     case EXITS_CLEAN: return "Exits-clean";
 126   }
 127   llvm_unreachable("Invalid block exit state.");
 128 }
 129 #endif
 130
 131 /// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
 132 /// Thus, there is no need to check for Y/ZMM16 and above.
 133 static bool isYmmOrZmmReg(unsigned Reg) {
 134   return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
 135          (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
 136 }
 137
 138 static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
 139   for (std::pair<MCRegister, Register> LI : MRI.liveins())
 140     if (isYmmOrZmmReg(LI.first))
 141       return true;
 142
 143   return false;
 144 }
 145
 146 static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
 147   for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
 148     if (!MO.clobbersPhysReg(reg))
 149       return false;
 150   }
 151   for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
 152     if (!MO.clobbersPhysReg(reg))
 153       return false;
 154   }
 155   return true;
 156 }
 157
 158 static bool hasYmmOrZmmReg(MachineInstr &MI) {
 159   for (const MachineOperand &MO : MI.operands()) {
 160     if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
 161       return true;
 162     if (!MO.isReg())
 163       continue;
 164     if (MO.isDebug())
 165       continue;
 166     if (isYmmOrZmmReg(MO.getReg()))
 167       return true;
 168   }
 169   return false;
 170 }
 171
 172 /// Check if given call instruction has a RegMask operand.
 173 static bool callHasRegMask(MachineInstr &MI) {
 174   assert(MI.isCall() && "Can only be called on call instructions.");
 175   for (const MachineOperand &MO : MI.operands()) {
 176     if (MO.isRegMask())
 177       return true;
 178   }
 179   return false;
 180 }
 181
 182 /// Insert a vzeroupper instruction before I.
 183 void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
 184                                           MachineBasicBlock &MBB) {
 185   BuildMI(MBB, I, I->getDebugLoc(), TII->get(X86::VZEROUPPER));
 186   ++NumVZU;
 187   EverMadeChange = true;
 188 }
 189
 190 /// Add MBB to the DirtySuccessors list if it hasn't already been added.
 191 void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
 192   if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
 193     DirtySuccessors.push_back(&MBB);
 194     BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
 195   }
 196 }
 197
 198 /// Loop over all of the instructions in the basic block, inserting vzeroupper
 199 /// instructions before function calls.
 200 void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
 201   // Start by assuming that the block is PASS_THROUGH which implies no unguarded
 202   // calls.
 203   BlockExitState CurState = PASS_THROUGH;
 204   BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
 205
 206   for (MachineInstr &MI : MBB) {
 207     bool IsCall = MI.isCall();
 208     bool IsReturn = MI.isReturn();
 209     bool IsControlFlow = IsCall || IsReturn;
 210
 211     // No need for vzeroupper before iret in interrupt handler function,
 212     // epilogue will restore YMM/ZMM registers if needed.
 213     if (IsX86INTR && IsReturn)
 214       continue;
 215
 216     // An existing VZERO* instruction resets the state.
 217     if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
 218       CurState = EXITS_CLEAN;
 219       continue;
 220     }
 221
 222     // Shortcut: don't need to check regular instructions in dirty state.
 223     if (!IsControlFlow && CurState == EXITS_DIRTY)
 224       continue;
 225
 226     if (hasYmmOrZmmReg(MI)) {
 227       // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
 228       // instruction, or it could be control flow.
 229       CurState = EXITS_DIRTY;
 230       continue;
 231     }
 232
 233     // Check for control-flow out of the current function (which might
 234     // indirectly execute SSE instructions).
 235     if (!IsControlFlow)
 236       continue;
 237
 238     // If the call has no RegMask, skip it as well. It usually happens on
 239     // helper function calls (such as '_chkstk', '_ftol2') where standard
 240     // calling convention is not used (RegMask is not used to mark register
 241     // clobbered and register usage (def/implicit-def/use) is well-defined and
 242     // explicitly specified.
 243     if (IsCall && !callHasRegMask(MI))
 244       continue;
 245
 246     // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
 247     // registers. In addition, the processor changes back to Clean state, after
 248     // which execution of SSE instructions or AVX instructions has no transition
 249     // penalty. Add the VZEROUPPER instruction before any function call/return
 250     // that might execute SSE code.
 251     // FIXME: In some cases, we may want to move the VZEROUPPER into a
 252     // predecessor block.
 253     if (CurState == EXITS_DIRTY) {
 254       // After the inserted VZEROUPPER the state becomes clean again, but
 255       // other YMM/ZMM may appear before other subsequent calls or even before
 256       // the end of the BB.
 257       insertVZeroUpper(MI, MBB);
 258       CurState = EXITS_CLEAN;
 259     } else if (CurState == PASS_THROUGH) {
 260       // If this block is currently in pass-through state and we encounter a
 261       // call then whether we need a vzeroupper or not depends on whether this
 262       // block has successors that exit dirty. Record the location of the call,
 263       // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
 264       // It will be inserted later if necessary.
 265       BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
 266       CurState = EXITS_CLEAN;
 267     }
 268   }
 269
 270   LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
 271                     << getBlockExitStateName(CurState) << '\n');
 272
 273   if (CurState == EXITS_DIRTY)
 274     for (MachineBasicBlock *Succ : MBB.successors())
 275       addDirtySuccessor(*Succ);
 276
 277   BlockStates[MBB.getNumber()].ExitState = CurState;
 278 }
 279
 280 /// Loop over all of the basic blocks, inserting vzeroupper instructions before
 281 /// function calls.
 282 bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
 283   if (!UseVZeroUpper)
 284     return false;
 285
 286   const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
 287   if (!ST.hasAVX() || !ST.insertVZEROUPPER())
 288     return false;
 289   TII = ST.getInstrInfo();
 290   MachineRegisterInfo &MRI = MF.getRegInfo();
 291   EverMadeChange = false;
 292   IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
 293
 294   bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
 295
 296   // Fast check: if the function doesn't use any ymm/zmm registers, we don't
 297   // need to insert any VZEROUPPER instructions.  This is constant-time, so it
 298   // is cheap in the common case of no ymm/zmm use.
 299   bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
 300   for (const auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
 301     if (!YmmOrZmmUsed) {
 302       for (MCPhysReg R : *RC) {
 303         if (!MRI.reg_nodbg_empty(R)) {
 304           YmmOrZmmUsed = true;
 305           break;
 306         }
 307       }
 308     }
 309   }
 310   if (!YmmOrZmmUsed)
 311     return false;
 312
 313   assert(BlockStates.empty() && DirtySuccessors.empty() &&
 314          "X86VZeroUpper state should be clear");
 315   BlockStates.resize(MF.getNumBlockIDs());
 316
 317   // Process all blocks. This will compute block exit states, record the first
 318   // unguarded call in each block, and add successors of dirty blocks to the
 319   // DirtySuccessors list.
 320   for (MachineBasicBlock &MBB : MF)
 321     processBasicBlock(MBB);
 322
 323   // If any YMM/ZMM regs are live-in to this function, add the entry block to
 324   // the DirtySuccessors list
 325   if (FnHasLiveInYmmOrZmm)
 326     addDirtySuccessor(MF.front());
 327
 328   // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
 329   // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
 330   // through PASS_THROUGH blocks.
 331   while (!DirtySuccessors.empty()) {
 332     MachineBasicBlock &MBB = *DirtySuccessors.back();
 333     DirtySuccessors.pop_back();
 334     BlockState &BBState = BlockStates[MBB.getNumber()];
 335
 336     // MBB is a successor of a dirty block, so its first call needs to be
 337     // guarded.
 338     if (BBState.FirstUnguardedCall != MBB.end())
 339       insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
 340
 341     // If this successor was a pass-through block, then it is now dirty. Its
 342     // successors need to be added to the worklist (if they haven't been
 343     // already).
 344     if (BBState.ExitState == PASS_THROUGH) {
 345       LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
 346                         << " was Pass-through, is now Dirty-out.\n");
 347       for (MachineBasicBlock *Succ : MBB.successors())
 348         addDirtySuccessor(*Succ);
 349     }
 350   }
 351
 352   BlockStates.clear();
 353   return EverMadeChange;
 354 }