lib/Target/ARM/MLxExpansionPass.cpp

   1 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ----------=//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // Expand VFP / NEON floating point MLA / MLS instructions (each to a pair of
  11 // multiple and add / sub instructions) when special VMLx hazards are detected.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #define DEBUG_TYPE "mlx-expansion"
  16 #include "ARM.h"
  17 #include "ARMBaseInstrInfo.h"
  18 #include "ARMSubtarget.h"
  19 #include "llvm/CodeGen/MachineInstr.h"
  20 #include "llvm/CodeGen/MachineInstrBuilder.h"
  21 #include "llvm/CodeGen/MachineFunctionPass.h"
  22 #include "llvm/CodeGen/MachineRegisterInfo.h"
  23 #include "llvm/Target/TargetRegisterInfo.h"
  24 #include "llvm/ADT/SmallPtrSet.h"
  25 #include "llvm/ADT/Statistic.h"
  26 #include "llvm/Support/CommandLine.h"
  27 #include "llvm/Support/Debug.h"
  28 #include "llvm/Support/raw_ostream.h"
  29 using namespace llvm;
  30
  31 static cl::opt<bool>
  32 ForceExapnd("expand-all-fp-mlx", cl::init(false), cl::Hidden);
  33 static cl::opt<unsigned>
  34 ExpandLimit("expand-limit", cl::init(~0U), cl::Hidden);
  35
  36 STATISTIC(NumExpand, "Number of fp MLA / MLS instructions expanded");
  37
  38 namespace {
  39   struct MLxExpansion : public MachineFunctionPass {
  40     static char ID;
  41     MLxExpansion() : MachineFunctionPass(ID) {}
  42
  43     virtual bool runOnMachineFunction(MachineFunction &Fn);
  44
  45     virtual const char *getPassName() const {
  46       return "ARM MLA / MLS expansion pass";
  47     }
  48
  49   private:
  50     const ARMBaseInstrInfo *TII;
  51     const TargetRegisterInfo *TRI;
  52     MachineRegisterInfo *MRI;
  53
  54     bool isA9;
  55     unsigned MIIdx;
  56     MachineInstr* LastMIs[4];
  57     SmallPtrSet<MachineInstr*, 4> IgnoreStall;
  58
  59     void clearStack();
  60     void pushStack(MachineInstr *MI);
  61     MachineInstr *getAccDefMI(MachineInstr *MI) const;
  62     unsigned getDefReg(MachineInstr *MI) const;
  63     bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
  64     bool FindMLxHazard(MachineInstr *MI);
  65     void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
  66                                 unsigned MulOpc, unsigned AddSubOpc,
  67                                 bool NegAcc, bool HasLane);
  68     bool ExpandFPMLxInstructions(MachineBasicBlock &MBB);
  69   };
  70   char MLxExpansion::ID = 0;
  71 }
  72
  73 void MLxExpansion::clearStack() {
  74   std::fill(LastMIs, LastMIs + 4, (MachineInstr*)0);
  75   MIIdx = 0;
  76 }
  77
  78 void MLxExpansion::pushStack(MachineInstr *MI) {
  79   LastMIs[MIIdx] = MI;
  80   if (++MIIdx == 4)
  81     MIIdx = 0;
  82 }
  83
  84 MachineInstr *MLxExpansion::getAccDefMI(MachineInstr *MI) const {
  85   // Look past COPY and INSERT_SUBREG instructions to find the
  86   // real definition MI. This is important for _sfp instructions.
  87   unsigned Reg = MI->getOperand(1).getReg();
  88   if (TargetRegisterInfo::isPhysicalRegister(Reg))
  89     return 0;
  90
  91   MachineBasicBlock *MBB = MI->getParent();
  92   MachineInstr *DefMI = MRI->getVRegDef(Reg);
  93   while (true) {
  94     if (DefMI->getParent() != MBB)
  95       break;
  96     if (DefMI->isCopyLike()) {
  97       Reg = DefMI->getOperand(1).getReg();
  98       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
  99         DefMI = MRI->getVRegDef(Reg);
 100         continue;
 101       }
 102     } else if (DefMI->isInsertSubreg()) {
 103       Reg = DefMI->getOperand(2).getReg();
 104       if (TargetRegisterInfo::isVirtualRegister(Reg)) {
 105         DefMI = MRI->getVRegDef(Reg);
 106         continue;
 107       }
 108     }
 109     break;
 110   }
 111   return DefMI;
 112 }
 113
 114 unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
 115   unsigned Reg = MI->getOperand(0).getReg();
 116   if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
 117       !MRI->hasOneNonDBGUse(Reg))
 118     return Reg;
 119
 120   MachineBasicBlock *MBB = MI->getParent();
 121   MachineInstr *UseMI = &*MRI->use_nodbg_begin(Reg);
 122   if (UseMI->getParent() != MBB)
 123     return Reg;
 124
 125   while (UseMI->isCopy() || UseMI->isInsertSubreg()) {
 126     Reg = UseMI->getOperand(0).getReg();
 127     if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
 128         !MRI->hasOneNonDBGUse(Reg))
 129       return Reg;
 130     UseMI = &*MRI->use_nodbg_begin(Reg);
 131     if (UseMI->getParent() != MBB)
 132       return Reg;
 133   }
 134
 135   return Reg;
 136 }
 137
 138 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
 139   // FIXME: Detect integer instructions properly.
 140   const MCInstrDesc &MCID = MI->getDesc();
 141   unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
 142   if (MCID.mayStore())
 143     return false;
 144   unsigned Opcode = MCID.getOpcode();
 145   if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
 146     return false;
 147   if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON))
 148     return MI->readsRegister(Reg, TRI);
 149   return false;
 150 }
 151
 152
 153 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
 154   if (NumExpand >= ExpandLimit)
 155     return false;
 156
 157   if (ForceExapnd)
 158     return true;
 159
 160   MachineInstr *DefMI = getAccDefMI(MI);
 161   if (TII->isFpMLxInstruction(DefMI->getOpcode())) {
 162     // r0 = vmla
 163     // r3 = vmla r0, r1, r2
 164     // takes 16 - 17 cycles
 165     //
 166     // r0 = vmla
 167     // r4 = vmul r1, r2
 168     // r3 = vadd r0, r4
 169     // takes about 14 - 15 cycles even with vmul stalling for 4 cycles.
 170     IgnoreStall.insert(DefMI);
 171     return true;
 172   }
 173
 174   if (IgnoreStall.count(MI))
 175     return false;
 176
 177   // If a VMLA.F is followed by an VADD.F or VMUL.F with no RAW hazard, the
 178   // VADD.F or VMUL.F will stall 4 cycles before issue. The 4 cycle stall
 179   // preserves the in-order retirement of the instructions.
 180   // Look at the next few instructions, if *most* of them can cause hazards,
 181   // then the scheduler can't *fix* this, we'd better break up the VMLA.
 182   unsigned Limit1 = isA9 ? 1 : 4;
 183   unsigned Limit2 = isA9 ? 1 : 4;
 184   for (unsigned i = 1; i <= 4; ++i) {
 185     int Idx = ((int)MIIdx - i + 4) % 4;
 186     MachineInstr *NextMI = LastMIs[Idx];
 187     if (!NextMI)
 188       continue;
 189
 190     if (TII->canCauseFpMLxStall(NextMI->getOpcode())) {
 191       if (i <= Limit1)
 192         return true;
 193     }
 194
 195     // Look for VMLx RAW hazard.
 196     if (i <= Limit2 && hasRAWHazard(getDefReg(MI), NextMI))
 197       return true;
 198   }
 199
 200   return false;
 201 }
 202
 203 /// ExpandFPMLxInstructions - Expand a MLA / MLS instruction into a pair
 204 /// of MUL + ADD / SUB instructions.
 205 void
 206 MLxExpansion::ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
 207                                      unsigned MulOpc, unsigned AddSubOpc,
 208                                      bool NegAcc, bool HasLane) {
 209   unsigned DstReg = MI->getOperand(0).getReg();
 210   bool DstDead = MI->getOperand(0).isDead();
 211   unsigned AccReg = MI->getOperand(1).getReg();
 212   unsigned Src1Reg = MI->getOperand(2).getReg();
 213   unsigned Src2Reg = MI->getOperand(3).getReg();
 214   bool Src1Kill = MI->getOperand(2).isKill();
 215   bool Src2Kill = MI->getOperand(3).isKill();
 216   unsigned LaneImm = HasLane ? MI->getOperand(4).getImm() : 0;
 217   unsigned NextOp = HasLane ? 5 : 4;
 218   ARMCC::CondCodes Pred = (ARMCC::CondCodes)MI->getOperand(NextOp).getImm();
 219   unsigned PredReg = MI->getOperand(++NextOp).getReg();
 220
 221   const MCInstrDesc &MCID1 = TII->get(MulOpc);
 222   const MCInstrDesc &MCID2 = TII->get(AddSubOpc);
 223   unsigned TmpReg = MRI->createVirtualRegister(TII->getRegClass(MCID1, 0, TRI));
 224
 225   MachineInstrBuilder MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), MCID1, TmpReg)
 226     .addReg(Src1Reg, getKillRegState(Src1Kill))
 227     .addReg(Src2Reg, getKillRegState(Src2Kill));
 228   if (HasLane)
 229     MIB.addImm(LaneImm);
 230   MIB.addImm(Pred).addReg(PredReg);
 231
 232   MIB = BuildMI(MBB, *MI, MI->getDebugLoc(), MCID2)
 233     .addReg(DstReg, getDefRegState(true) | getDeadRegState(DstDead));
 234
 235   if (NegAcc) {
 236     bool AccKill = MRI->hasOneNonDBGUse(AccReg);
 237     MIB.addReg(TmpReg, getKillRegState(true))
 238        .addReg(AccReg, getKillRegState(AccKill));
 239   } else {
 240     MIB.addReg(AccReg).addReg(TmpReg, getKillRegState(true));
 241   }
 242   MIB.addImm(Pred).addReg(PredReg);
 243
 244   DEBUG({
 245       dbgs() << "Expanding: " << *MI;
 246       dbgs() << "  to:\n";
 247       MachineBasicBlock::iterator MII = MI;
 248       MII = llvm::prior(MII);
 249       MachineInstr &MI2 = *MII;
 250       MII = llvm::prior(MII);
 251       MachineInstr &MI1 = *MII;
 252       dbgs() << "    " << MI1;
 253       dbgs() << "    " << MI2;
 254    });
 255
 256   MI->eraseFromParent();
 257   ++NumExpand;
 258 }
 259
 260 bool MLxExpansion::ExpandFPMLxInstructions(MachineBasicBlock &MBB) {
 261   bool Changed = false;
 262
 263   clearStack();
 264   IgnoreStall.clear();
 265
 266   unsigned Skip = 0;
 267   MachineBasicBlock::reverse_iterator MII = MBB.rbegin(), E = MBB.rend();
 268   while (MII != E) {
 269     MachineInstr *MI = &*MII;
 270
 271     if (MI->isLabel() || MI->isImplicitDef() || MI->isCopy()) {
 272       ++MII;
 273       continue;
 274     }
 275
 276     const MCInstrDesc &MCID = MI->getDesc();
 277     if (MCID.isBarrier()) {
 278       clearStack();
 279       Skip = 0;
 280       ++MII;
 281       continue;
 282     }
 283
 284     unsigned Domain = MCID.TSFlags & ARMII::DomainMask;
 285     if (Domain == ARMII::DomainGeneral) {
 286       if (++Skip == 2)
 287         // Assume dual issues of non-VFP / NEON instructions.
 288         pushStack(0);
 289     } else {
 290       Skip = 0;
 291
 292       unsigned MulOpc, AddSubOpc;
 293       bool NegAcc, HasLane;
 294       if (!TII->isFpMLxInstruction(MCID.getOpcode(),
 295                                    MulOpc, AddSubOpc, NegAcc, HasLane) ||
 296           !FindMLxHazard(MI))
 297         pushStack(MI);
 298       else {
 299         ExpandFPMLxInstruction(MBB, MI, MulOpc, AddSubOpc, NegAcc, HasLane);
 300         E = MBB.rend(); // May have changed if MI was the 1st instruction.
 301         Changed = true;
 302         continue;
 303       }
 304     }
 305
 306     ++MII;
 307   }
 308
 309   return Changed;
 310 }
 311
 312 bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
 313   TII = static_cast<const ARMBaseInstrInfo*>(Fn.getTarget().getInstrInfo());
 314   TRI = Fn.getTarget().getRegisterInfo();
 315   MRI = &Fn.getRegInfo();
 316   const ARMSubtarget *STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
 317   isA9 = STI->isCortexA9();
 318
 319   bool Modified = false;
 320   for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
 321        ++MFI) {
 322     MachineBasicBlock &MBB = *MFI;
 323     Modified |= ExpandFPMLxInstructions(MBB);
 324   }
 325
 326   return Modified;
 327 }
 328
 329 FunctionPass *llvm::createMLxExpansionPass() {
 330   return new MLxExpansion();
 331 }