lib/Target/ARM/MLxExpansionPass.cpp

   1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
  10 // multiple and add / sub instructions) when special VMLx hazards are detected.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "ARM.h"
  15 #include "ARMBaseInstrInfo.h"
  16 #include "ARMSubtarget.h"
  17 #include "llvm/ADT/SmallPtrSet.h"
  18 #include "llvm/ADT/Statistic.h"
  19 #include "llvm/CodeGen/MachineFunctionPass.h"
  20 #include "llvm/CodeGen/MachineInstr.h"
  21 #include "llvm/CodeGen/MachineInstrBuilder.h"
  22 #include "llvm/CodeGen/MachineRegisterInfo.h"
  23 #include "llvm/CodeGen/TargetRegisterInfo.h"
  24 #include "llvm/Support/CommandLine.h"
  25 #include "llvm/Support/Debug.h"
  26 #include "llvm/Support/raw_ostream.h"
  27 using namespace llvm;
  28
  29 #define DEBUG_TYPE "mlx-expansion"
  30
  31 static cl::opt<bool>
  32 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
  33 static cl::opt<unsigned>
  34 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
  35
  36 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
  37
  38 namespace {
  39   struct MLxExpansion : public MachineFunctionPass {
  40     static char ID;
  41     MLxExpansion() : MachineFunctionPass(ID) {}
  42
  43     bool runOnMachineFunction(MachineFunction &Fn) override;
  44
  45     StringRef getPassName() const override {
  46       return "ARM MLA / MLS expansion pass";
  47     }
  48
  49   private:
  50     const ARMBaseInstrInfo *TII;
  51     const TargetRegisterInfo *TRI;
  52     MachineRegisterInfo *MRI;
  53
  54     bool isLikeA9;
  55     bool isSwift;
  56     unsigned MIIdx;
  57     MachineInstr* LastMIs[4];
  58     SmallPtrSet<MachineInstr*, 4> IgnoreStall;
  59
  60     void clearStack();
  61     void pushStack(MachineInstr *MI);
  62     MachineInstr *getAccDefMI(MachineInstr *MI) const;
  63     unsigned getDefReg(MachineInstr *MI) const;
  64     bool hasLoopHazard(MachineInstr *MI) const;
  65     bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
  66     bool FindMLxHazard(MachineInstr *MI);
  67     void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
  68                                 unsigned MulOpc, unsigned AddSubOpc,
  69                                 bool NegAcc, bool HasLane);
  70     bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
  71   };
  72   char MLxExpansion::ID = 0;
  73 }
  74
  75 void MLxExpansion::clearStack() {
  76   std::fill(LastMIs, LastMIs + 4, nullptr);
  77   MIIdx = 0;
  78 }
  79
  80 void MLxExpansion::pushStack(MachineInstr *MI) {
  81   LastMIs[MIIdx] = MI;
  82   if (++MIIdx == 4)
  83     MIIdx = 0;
  84 }
  85
  86 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
  87   // Look past COPY and INSERT_SUBREG instructions to find the
  88   // real definition MI. This is important for _sfp instructions.
  89   unsigned Reg = MI->getOperand(1).getReg();
  90   if (TargetRegisterInfo::isPhysicalRegister(Reg))
  91     return nullptr;
  92
  93   MachineBasicBlock *MBB = MI->getParent();
  94   MachineInstr *DefMI = MRI->getVRegDef(Reg);
  95   while (true) {
  96     if (DefMI->getParent() != MBB)
  97       break;
  98     if (DefMI->isCopyLike()) {
  99       Reg = DefMI->getOperand(1).getReg();
 100       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
 101         DefMI = MRI->getVRegDef(Reg);
 102         continue;
 103       }
 104     } else if (DefMI->isInsertSubreg()) {
 105       Reg = DefMI->getOperand(2).getReg();
 106       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
 107         DefMI = MRI->getVRegDef(Reg);
 108         continue;
 109       }
 110     }
 111     break;
 112   }
 113   return DefMI;
 114 }
 115
 116 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
 117   unsigned Reg = MI->getOperand(0).getReg();
 118   if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
 119       !MRI->hasOneNonDBGUse(Reg))
 120     return Reg;
 121
 122   MachineBasicBlock *MBB = MI->getParent();
 123   MachineInstr *UseMI = &*MRI->use_instr_nodbg_begin(Reg);
 124   if (UseMI->getParent() != MBB)
 125     return Reg;
 126
 127   while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
 128     Reg = UseMI->getOperand(0).getReg();
 129     if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
 130         !MRI->hasOneNonDBGUse(Reg))
 131       return Reg;
 132     UseMI = &*MRI->use_instr_nodbg_begin(Reg);
 133     if (UseMI->getParent() != MBB)
 134       return Reg;
 135   }
 136
 137   return Reg;
 138 }
 139
 140 /// hasLoopHazard - Check whether an MLx instruction is chained to itself across
 141 /// a single-MBB loop.
 142 bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
 143   unsigned Reg = MI->getOperand(1).getReg();
 144   if (TargetRegisterInfo::isPhysicalRegister(Reg))
 145     return false;
 146
 147   MachineBasicBlock *MBB = MI->getParent();
 148   MachineInstr *DefMI = MRI->getVRegDef(Reg);
 149   while (true) {
 150 outer_continue:
 151     if (DefMI->getParent() != MBB)
 152       break;
 153
 154     if (DefMI->isPHI()) {
 155       for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
 156         if (DefMI->getOperand(i + 1).getMBB() == MBB) {
 157           unsigned SrcReg = DefMI->getOperand(i).getReg();
 158           if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
 159             DefMI = MRI->getVRegDef(SrcReg);
 160             goto outer_continue;
 161           }
 162         }
 163       }
 164     } else if (DefMI->isCopyLike()) {
 165       Reg = DefMI->getOperand(1).getReg();
 166       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
 167         DefMI = MRI->getVRegDef(Reg);
 168         continue;
 169       }
 170     } else if (DefMI->isInsertSubreg()) {
 171       Reg = DefMI->getOperand(2).getReg();
 172       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
 173         DefMI = MRI->getVRegDef(Reg);
 174         continue;
 175       }
 176     }
 177
 178     break;
 179   }
 180
 181   return DefMI == MI;
 182 }
 183
 184 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
 185   // FIXME: Detect integer instructions properly.
 186   const MCInstrDesc &MCID = MI->getDesc();
 187   unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
 188   if (MI->mayStore())
 189     return false;
 190   unsigned Opcode = MCID.getOpcode();
 191   if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
 192     return false;
 193   if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
 194     return MI->readsRegister(Reg, TRI);
 195   return false;
 196 }
 197
 198 static bool isFpMulInstruction(unsigned Opcode) {
 199   switch (Opcode) {
 200   case ARM::VMULS:
 201   case ARM::VMULfd:
 202   case ARM::VMULfq:
 203   case ARM::VMULD:
 204   case ARM::VMULslfd:
 205   case ARM::VMULslfq:
 206     return true;
 207   default:
 208     return false;
 209   }
 210 }
 211
 212 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
 213   if (NumExpand >= ExpandLimit)
 214     return false;
 215
 216   if (ForceExapnd)
 217     return true;
 218
 219   MachineInstr *DefMI = getAccDefMI(MI);
 220   if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
 221     // r0 = vmla
 222     // r3 = vmla r0, r1, r2
 223     // takes 16 - 17 cycles
 224     //
 225     // r0 = vmla
 226     // r4 = vmul r1, r2
 227     // r3 = vadd r0, r4
 228     // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
 229     IgnoreStall.insert(DefMI);
 230     return true;
 231   }
 232
 233   // On Swift, we mostly care about hazards from multiplication instructions
 234   // writing the accumulator and the pipelining of loop iterations by out-of-
 235   // order execution.
 236   if (isSwift)
 237     return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
 238
 239   if (IgnoreStall.count(MI))
 240     return false;
 241
 242   // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
 243   // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
 244   // preserves the in-order retirement of the instructions.
 245   // Look at the next few instructions, if *most* of them can cause hazards,
 246   // then the scheduler can't *fix* this, we'd better break up the VMLA.
 247   unsigned Limit1 = isLikeA9 ? 1 : 4;
 248   unsigned Limit2 = isLikeA9 ? 1 : 4;
 249   for (unsigned i = 1; i <= 4; ++i) {
 250     int Idx = ((int)MIIdx - i + 4) % 4;
 251     MachineInstr *NextMI = LastMIs[Idx];
 252     if (!NextMI)
 253       continue;
 254
 255     if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
 256       if (i <= Limit1)
 257         return true;
 258     }
 259
 260     // Look for VMLx RAW hazard.
 261     if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
 262       return true;
 263   }
 264
 265   return false;
 266 }
 267
 268 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
 269 /// of MUL + ADD / SUB instructions.
 270 void
 271 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
 272                                      unsigned MulOpc, unsigned AddSubOpc,
 273                                      bool NegAcc, bool HasLane) {
 274   unsigned DstReg = MI->getOperand(0).getReg();
 275   bool DstDead = MI->getOperand(0).isDead();
 276   unsigned AccReg = MI->getOperand(1).getReg();
 277   unsigned Src1Reg = MI->getOperand(2).getReg();
 278   unsigned Src2Reg = MI->getOperand(3).getReg();
 279   bool Src1Kill = MI->getOperand(2).isKill();
 280   bool Src2Kill = MI->getOperand(3).isKill();
 281   unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
 282   unsigned NextOp = HasLane ? 5 : 4;
 283   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
 284   unsigned PredReg = MI->getOperand(++NextOp).getReg();
 285
 286   const MCInstrDesc &MCID1 = TII->get(MulOpc);
 287   const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
 288   const MachineFunction &MF = *MI->getParent()->getParent();
 289   unsigned TmpReg = MRI->createVirtualRegister(
 290                       TII->getRegClass(MCID1, 0, TRI, MF));
 291
 292   MachineInstrBuilder MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID1, TmpReg)
 293     .addReg(Src1Reg, getKillRegState(Src1Kill))
 294     .addReg(Src2Reg, getKillRegState(Src2Kill));
 295   if (HasLane)
 296     MIB.addImm(LaneImm);
 297   MIB.addImm(Pred).addReg(PredReg);
 298
 299   MIB = BuildMI(MBB, MI, MI->getDebugLoc(), MCID2)
 300     .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
 301
 302   if (NegAcc) {
 303     bool AccKill = MRI->hasOneNonDBGUse(AccReg);
 304     MIB.addReg(TmpReg, getKillRegState(true))
 305        .addReg(AccReg, getKillRegState(AccKill));
 306   } else {
 307     MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
 308   }
 309   MIB.addImm(Pred).addReg(PredReg);
 310
 311   LLVM_DEBUG({
 312     dbgs() << "Expanding: " << *MI;
 313     dbgs() << "  to:\n";
 314     MachineBasicBlock::iterator MII = MI;
 315     MII = std::prev(MII);
 316     MachineInstr &MI2 = *MII;
 317     MII = std::prev(MII);
 318     MachineInstr &MI1 = *MII;
 319     dbgs() << "    " << MI1;
 320     dbgs() << "    " << MI2;
 321   });
 322
 323   MI->eraseFromParent();
 324   ++NumExpand;
 325 }
 326
 327 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
 328   bool Changed = false;
 329
 330   clearStack();
 331   IgnoreStall.clear();
 332
 333   unsigned Skip = 0;
 334   MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
 335   while (MII != E) {
 336     MachineInstr *MI = &*MII++;
 337
 338     if (MI->isPosition() || MI->isImplicitDef() || MI->isCopy())
 339       continue;
 340
 341     const MCInstrDesc &MCID = MI->getDesc();
 342     if (MI->isBarrier()) {
 343       clearStack();
 344       Skip = 0;
 345       continue;
 346     }
 347
 348     unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
 349     if (Domain == ARMII::DomainGeneral) {
 350       if (++Skip == 2)
 351         // Assume dual issues of non-VFP / NEON instructions.
 352         pushStack(nullptr);
 353     } else {
 354       Skip = 0;
 355
 356       unsigned MulOpc, AddSubOpc;
 357       bool NegAcc, HasLane;
 358       if (!TII->isFpMLxInstruction(MCID.getOpcode(),
 359                                    MulOpc, AddSubOpc, NegAcc, HasLane) ||
 360           !FindMLxHazard(MI))
 361         pushStack(MI);
 362       else {
 363         ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
 364         Changed = true;
 365       }
 366     }
 367   }
 368
 369   return Changed;
 370 }
 371
 372 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
 373   if (skipFunction(Fn.getFunction()))
 374     return false;
 375
 376   TII = static_cast<const ARMBaseInstrInfo *>(Fn.getSubtarget().getInstrInfo());
 377   TRI = Fn.getSubtarget().getRegisterInfo();
 378   MRI = &Fn.getRegInfo();
 379   const ARMSubtarget *STI = &Fn.getSubtarget<ARMSubtarget>();
 380   if (!STI->expandMLx())
 381     return false;
 382   isLikeA9 = STI->isLikeA9() || STI->isSwift();
 383   isSwift = STI->isSwift();
 384
 385   bool Modified = false;
 386   for (MachineBasicBlock &MBB : Fn)
 387     Modified |= ExpandFPMLxInstructions(MBB);
 388
 389   return Modified;
 390 }
 391
 392 FunctionPass *llvm::createMLxExpansionPass() {
 393   return new MLxExpansion();
 394 }