lib/Target/AMDGPU/SIPeepholeSDWA.cpp

   1 //===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file This pass tries to apply several peephole SDWA patterns.
  11 ///
  12 /// E.g. original:
  13 ///   V_LSHRREV_B32_e32 %0, 16, %1
  14 ///   V_ADD_I32_e32 %2, %0, %3
  15 ///   V_LSHLREV_B32_e32 %4, 16, %2
  16 ///
  17 /// Replace:
  18 ///   V_ADD_I32_sdwa %4, %1, %3
  19 ///       dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
  20 ///
  21 //===----------------------------------------------------------------------===//
  22
  23 #include "AMDGPU.h"
  24 #include "AMDGPUSubtarget.h"
  25 #include "SIDefines.h"
  26 #include "SIInstrInfo.h"
  27 #include "SIRegisterInfo.h"
  28 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  29 #include "Utils/AMDGPUBaseInfo.h"
  30 #include "llvm/ADT/None.h"
  31 #include "llvm/ADT/Optional.h"
  32 #include "llvm/ADT/STLExtras.h"
  33 #include "llvm/ADT/SmallVector.h"
  34 #include "llvm/ADT/Statistic.h"
  35 #include "llvm/CodeGen/MachineBasicBlock.h"
  36 #include "llvm/CodeGen/MachineFunction.h"
  37 #include "llvm/CodeGen/MachineFunctionPass.h"
  38 #include "llvm/CodeGen/MachineInstr.h"
  39 #include "llvm/CodeGen/MachineInstrBuilder.h"
  40 #include "llvm/CodeGen/MachineOperand.h"
  41 #include "llvm/CodeGen/MachineRegisterInfo.h"
  42 #include "llvm/CodeGen/TargetRegisterInfo.h"
  43 #include "llvm/Config/llvm-config.h"
  44 #include "llvm/MC/LaneBitmask.h"
  45 #include "llvm/MC/MCInstrDesc.h"
  46 #include "llvm/Pass.h"
  47 #include "llvm/Support/Debug.h"
  48 #include "llvm/Support/raw_ostream.h"
  49 #include <algorithm>
  50 #include <cassert>
  51 #include <cstdint>
  52 #include <memory>
  53 #include <unordered_map>
  54
  55 using namespace llvm;
  56
  57 #define DEBUG_TYPE "si-peephole-sdwa"
  58
  59 STATISTIC(NumSDWAPatternsFound, "Number of SDWA patterns found.");
  60 STATISTIC(NumSDWAInstructionsPeepholed,
  61           "Number of instruction converted to SDWA.");
  62
  63 namespace {
  64
  65 class SDWAOperand;
  66 class SDWADstOperand;
  67
  68 class SIPeepholeSDWA : public MachineFunctionPass {
  69 public:
  70   using SDWAOperandsVector = SmallVector<SDWAOperand *, 4>;
  71
  72 private:
  73   MachineRegisterInfo *MRI;
  74   const SIRegisterInfo *TRI;
  75   const SIInstrInfo *TII;
  76
  77   std::unordered_map<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands;
  78   std::unordered_map<MachineInstr *, SDWAOperandsVector> PotentialMatches;
  79   SmallVector<MachineInstr *, 8> ConvertedInstructions;
  80
  81   Optional<int64_t> foldToImm(const MachineOperand &Op) const;
  82
  83 public:
  84   static char ID;
  85
  86   SIPeepholeSDWA() : MachineFunctionPass(ID) {
  87     initializeSIPeepholeSDWAPass(*PassRegistry::getPassRegistry());
  88   }
  89
  90   bool runOnMachineFunction(MachineFunction &MF) override;
  91   void matchSDWAOperands(MachineBasicBlock &MBB);
  92   std::unique_ptr<SDWAOperand> matchSDWAOperand(MachineInstr &MI);
  93   bool isConvertibleToSDWA(const MachineInstr &MI, const GCNSubtarget &ST) const;
  94   bool convertToSDWA(MachineInstr &MI, const SDWAOperandsVector &SDWAOperands);
  95   void legalizeScalarOperands(MachineInstr &MI, const GCNSubtarget &ST) const;
  96
  97   StringRef getPassName() const override { return "SI Peephole SDWA"; }
  98
  99   void getAnalysisUsage(AnalysisUsage &AU) const override {
 100     AU.setPreservesCFG();
 101     MachineFunctionPass::getAnalysisUsage(AU);
 102   }
 103 };
 104
 105 class SDWAOperand {
 106 private:
 107   MachineOperand *Target; // Operand that would be used in converted instruction
 108   MachineOperand *Replaced; // Operand that would be replace by Target
 109
 110 public:
 111   SDWAOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp)
 112       : Target(TargetOp), Replaced(ReplacedOp) {
 113     assert(Target->isReg());
 114     assert(Replaced->isReg());
 115   }
 116
 117   virtual ~SDWAOperand() = default;
 118
 119   virtual MachineInstr *potentialToConvert(const SIInstrInfo *TII) = 0;
 120   virtual bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) = 0;
 121
 122   MachineOperand *getTargetOperand() const { return Target; }
 123   MachineOperand *getReplacedOperand() const { return Replaced; }
 124   MachineInstr *getParentInst() const { return Target->getParent(); }
 125
 126   MachineRegisterInfo *getMRI() const {
 127     return &getParentInst()->getParent()->getParent()->getRegInfo();
 128   }
 129
 130 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 131   virtual void print(raw_ostream& OS) const = 0;
 132   void dump() const { print(dbgs()); }
 133 #endif
 134 };
 135
 136 using namespace AMDGPU::SDWA;
 137
 138 class SDWASrcOperand : public SDWAOperand {
 139 private:
 140   SdwaSel SrcSel;
 141   bool Abs;
 142   bool Neg;
 143   bool Sext;
 144
 145 public:
 146   SDWASrcOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
 147                  SdwaSel SrcSel_ = DWORD, bool Abs_ = false, bool Neg_ = false,
 148                  bool Sext_ = false)
 149       : SDWAOperand(TargetOp, ReplacedOp),
 150         SrcSel(SrcSel_), Abs(Abs_), Neg(Neg_), Sext(Sext_) {}
 151
 152   MachineInstr *potentialToConvert(const SIInstrInfo *TII) override;
 153   bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
 154
 155   SdwaSel getSrcSel() const { return SrcSel; }
 156   bool getAbs() const { return Abs; }
 157   bool getNeg() const { return Neg; }
 158   bool getSext() const { return Sext; }
 159
 160   uint64_t getSrcMods(const SIInstrInfo *TII,
 161                       const MachineOperand *SrcOp) const;
 162
 163 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 164   void print(raw_ostream& OS) const override;
 165 #endif
 166 };
 167
 168 class SDWADstOperand : public SDWAOperand {
 169 private:
 170   SdwaSel DstSel;
 171   DstUnused DstUn;
 172
 173 public:
 174
 175   SDWADstOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
 176                  SdwaSel DstSel_ = DWORD, DstUnused DstUn_ = UNUSED_PAD)
 177     : SDWAOperand(TargetOp, ReplacedOp), DstSel(DstSel_), DstUn(DstUn_) {}
 178
 179   MachineInstr *potentialToConvert(const SIInstrInfo *TII) override;
 180   bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
 181
 182   SdwaSel getDstSel() const { return DstSel; }
 183   DstUnused getDstUnused() const { return DstUn; }
 184
 185 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 186   void print(raw_ostream& OS) const override;
 187 #endif
 188 };
 189
 190 class SDWADstPreserveOperand : public SDWADstOperand {
 191 private:
 192   MachineOperand *Preserve;
 193
 194 public:
 195   SDWADstPreserveOperand(MachineOperand *TargetOp, MachineOperand *ReplacedOp,
 196                          MachineOperand *PreserveOp, SdwaSel DstSel_ = DWORD)
 197       : SDWADstOperand(TargetOp, ReplacedOp, DstSel_, UNUSED_PRESERVE),
 198         Preserve(PreserveOp) {}
 199
 200   bool convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) override;
 201
 202   MachineOperand *getPreservedOperand() const { return Preserve; }
 203
 204 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 205   void print(raw_ostream& OS) const override;
 206 #endif
 207 };
 208
 209 } // end anonymous namespace
 210
 211 INITIALIZE_PASS(SIPeepholeSDWA, DEBUG_TYPE, "SI Peephole SDWA", false, false)
 212
 213 char SIPeepholeSDWA::ID = 0;
 214
 215 char &llvm::SIPeepholeSDWAID = SIPeepholeSDWA::ID;
 216
 217 FunctionPass *llvm::createSIPeepholeSDWAPass() {
 218   return new SIPeepholeSDWA();
 219 }
 220
 221
 222 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 223 static raw_ostream& operator<<(raw_ostream &OS, SdwaSel Sel) {
 224   switch(Sel) {
 225   case BYTE_0: OS << "BYTE_0"; break;
 226   case BYTE_1: OS << "BYTE_1"; break;
 227   case BYTE_2: OS << "BYTE_2"; break;
 228   case BYTE_3: OS << "BYTE_3"; break;
 229   case WORD_0: OS << "WORD_0"; break;
 230   case WORD_1: OS << "WORD_1"; break;
 231   case DWORD:  OS << "DWORD"; break;
 232   }
 233   return OS;
 234 }
 235
 236 static raw_ostream& operator<<(raw_ostream &OS, const DstUnused &Un) {
 237   switch(Un) {
 238   case UNUSED_PAD: OS << "UNUSED_PAD"; break;
 239   case UNUSED_SEXT: OS << "UNUSED_SEXT"; break;
 240   case UNUSED_PRESERVE: OS << "UNUSED_PRESERVE"; break;
 241   }
 242   return OS;
 243 }
 244
 245 static raw_ostream& operator<<(raw_ostream &OS, const SDWAOperand &Operand) {
 246   Operand.print(OS);
 247   return OS;
 248 }
 249
 250 LLVM_DUMP_METHOD
 251 void SDWASrcOperand::print(raw_ostream& OS) const {
 252   OS << "SDWA src: " << *getTargetOperand()
 253     << " src_sel:" << getSrcSel()
 254     << " abs:" << getAbs() << " neg:" << getNeg()
 255     << " sext:" << getSext() << '\n';
 256 }
 257
 258 LLVM_DUMP_METHOD
 259 void SDWADstOperand::print(raw_ostream& OS) const {
 260   OS << "SDWA dst: " << *getTargetOperand()
 261     << " dst_sel:" << getDstSel()
 262     << " dst_unused:" << getDstUnused() << '\n';
 263 }
 264
 265 LLVM_DUMP_METHOD
 266 void SDWADstPreserveOperand::print(raw_ostream& OS) const {
 267   OS << "SDWA preserve dst: " << *getTargetOperand()
 268     << " dst_sel:" << getDstSel()
 269     << " preserve:" << *getPreservedOperand() << '\n';
 270 }
 271
 272 #endif
 273
 274 static void copyRegOperand(MachineOperand &To, const MachineOperand &From) {
 275   assert(To.isReg() && From.isReg());
 276   To.setReg(From.getReg());
 277   To.setSubReg(From.getSubReg());
 278   To.setIsUndef(From.isUndef());
 279   if (To.isUse()) {
 280     To.setIsKill(From.isKill());
 281   } else {
 282     To.setIsDead(From.isDead());
 283   }
 284 }
 285
 286 static bool isSameReg(const MachineOperand &LHS, const MachineOperand &RHS) {
 287   return LHS.isReg() &&
 288          RHS.isReg() &&
 289          LHS.getReg() == RHS.getReg() &&
 290          LHS.getSubReg() == RHS.getSubReg();
 291 }
 292
 293 static MachineOperand *findSingleRegUse(const MachineOperand *Reg,
 294                                         const MachineRegisterInfo *MRI) {
 295   if (!Reg->isReg() || !Reg->isDef())
 296     return nullptr;
 297
 298   MachineOperand *ResMO = nullptr;
 299   for (MachineOperand &UseMO : MRI->use_nodbg_operands(Reg->getReg())) {
 300     // If there exist use of subreg of Reg then return nullptr
 301     if (!isSameReg(UseMO, *Reg))
 302       return nullptr;
 303
 304     // Check that there is only one instruction that uses Reg
 305     if (!ResMO) {
 306       ResMO = &UseMO;
 307     } else if (ResMO->getParent() != UseMO.getParent()) {
 308       return nullptr;
 309     }
 310   }
 311
 312   return ResMO;
 313 }
 314
 315 static MachineOperand *findSingleRegDef(const MachineOperand *Reg,
 316                                         const MachineRegisterInfo *MRI) {
 317   if (!Reg->isReg())
 318     return nullptr;
 319
 320   MachineInstr *DefInstr = MRI->getUniqueVRegDef(Reg->getReg());
 321   if (!DefInstr)
 322     return nullptr;
 323
 324   for (auto &DefMO : DefInstr->defs()) {
 325     if (DefMO.isReg() && DefMO.getReg() == Reg->getReg())
 326       return &DefMO;
 327   }
 328
 329   // Ignore implicit defs.
 330   return nullptr;
 331 }
 332
 333 uint64_t SDWASrcOperand::getSrcMods(const SIInstrInfo *TII,
 334                                     const MachineOperand *SrcOp) const {
 335   uint64_t Mods = 0;
 336   const auto *MI = SrcOp->getParent();
 337   if (TII->getNamedOperand(*MI, AMDGPU::OpName::src0) == SrcOp) {
 338     if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src0_modifiers)) {
 339       Mods = Mod->getImm();
 340     }
 341   } else if (TII->getNamedOperand(*MI, AMDGPU::OpName::src1) == SrcOp) {
 342     if (auto *Mod = TII->getNamedOperand(*MI, AMDGPU::OpName::src1_modifiers)) {
 343       Mods = Mod->getImm();
 344     }
 345   }
 346   if (Abs || Neg) {
 347     assert(!Sext &&
 348            "Float and integer src modifiers can't be set simulteniously");
 349     Mods |= Abs ? SISrcMods::ABS : 0;
 350     Mods ^= Neg ? SISrcMods::NEG : 0;
 351   } else if (Sext) {
 352     Mods |= SISrcMods::SEXT;
 353   }
 354
 355   return Mods;
 356 }
 357
 358 MachineInstr *SDWASrcOperand::potentialToConvert(const SIInstrInfo *TII) {
 359   // For SDWA src operand potential instruction is one that use register
 360   // defined by parent instruction
 361   MachineOperand *PotentialMO = findSingleRegUse(getReplacedOperand(), getMRI());
 362   if (!PotentialMO)
 363     return nullptr;
 364
 365   return PotentialMO->getParent();
 366 }
 367
 368 bool SDWASrcOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
 369   // Find operand in instruction that matches source operand and replace it with
 370   // target operand. Set corresponding src_sel
 371   bool IsPreserveSrc = false;
 372   MachineOperand *Src = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 373   MachineOperand *SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel);
 374   MachineOperand *SrcMods =
 375       TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
 376   assert(Src && (Src->isReg() || Src->isImm()));
 377   if (!isSameReg(*Src, *getReplacedOperand())) {
 378     // If this is not src0 then it could be src1
 379     Src = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 380     SrcSel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel);
 381     SrcMods = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
 382
 383     if (!Src ||
 384         !isSameReg(*Src, *getReplacedOperand())) {
 385       // It's possible this Src is a tied operand for
 386       // UNUSED_PRESERVE, in which case we can either
 387       // abandon the peephole attempt, or if legal we can
 388       // copy the target operand into the tied slot
 389       // if the preserve operation will effectively cause the same
 390       // result by overwriting the rest of the dst.
 391       MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 392       MachineOperand *DstUnused =
 393         TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);
 394
 395       if (Dst &&
 396           DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {
 397         // This will work if the tied src is acessing WORD_0, and the dst is
 398         // writing WORD_1. Modifiers don't matter because all the bits that
 399         // would be impacted are being overwritten by the dst.
 400         // Any other case will not work.
 401         SdwaSel DstSel = static_cast<SdwaSel>(
 402             TII->getNamedImmOperand(MI, AMDGPU::OpName::dst_sel));
 403         if (DstSel == AMDGPU::SDWA::SdwaSel::WORD_1 &&
 404             getSrcSel() == AMDGPU::SDWA::SdwaSel::WORD_0) {
 405           IsPreserveSrc = true;
 406           auto DstIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
 407                                                    AMDGPU::OpName::vdst);
 408           auto TiedIdx = MI.findTiedOperandIdx(DstIdx);
 409           Src = &MI.getOperand(TiedIdx);
 410           SrcSel = nullptr;
 411           SrcMods = nullptr;
 412         } else {
 413           // Not legal to convert this src
 414           return false;
 415         }
 416       }
 417     }
 418     assert(Src && Src->isReg());
 419
 420     if ((MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
 421          MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
 422          !isSameReg(*Src, *getReplacedOperand())) {
 423       // In case of v_mac_f16/32_sdwa this pass can try to apply src operand to
 424       // src2. This is not allowed.
 425       return false;
 426     }
 427
 428     assert(isSameReg(*Src, *getReplacedOperand()) &&
 429            (IsPreserveSrc || (SrcSel && SrcMods)));
 430   }
 431   copyRegOperand(*Src, *getTargetOperand());
 432   if (!IsPreserveSrc) {
 433     SrcSel->setImm(getSrcSel());
 434     SrcMods->setImm(getSrcMods(TII, Src));
 435   }
 436   getTargetOperand()->setIsKill(false);
 437   return true;
 438 }
 439
 440 MachineInstr *SDWADstOperand::potentialToConvert(const SIInstrInfo *TII) {
 441   // For SDWA dst operand potential instruction is one that defines register
 442   // that this operand uses
 443   MachineRegisterInfo *MRI = getMRI();
 444   MachineInstr *ParentMI = getParentInst();
 445
 446   MachineOperand *PotentialMO = findSingleRegDef(getReplacedOperand(), MRI);
 447   if (!PotentialMO)
 448     return nullptr;
 449
 450   // Check that ParentMI is the only instruction that uses replaced register
 451   for (MachineInstr &UseInst : MRI->use_nodbg_instructions(PotentialMO->getReg())) {
 452     if (&UseInst != ParentMI)
 453       return nullptr;
 454   }
 455
 456   return PotentialMO->getParent();
 457 }
 458
 459 bool SDWADstOperand::convertToSDWA(MachineInstr &MI, const SIInstrInfo *TII) {
 460   // Replace vdst operand in MI with target operand. Set dst_sel and dst_unused
 461
 462   if ((MI.getOpcode() == AMDGPU::V_MAC_F16_sdwa ||
 463        MI.getOpcode() == AMDGPU::V_MAC_F32_sdwa) &&
 464       getDstSel() != AMDGPU::SDWA::DWORD) {
 465     // v_mac_f16/32_sdwa allow dst_sel to be equal only to DWORD
 466     return false;
 467   }
 468
 469   MachineOperand *Operand = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 470   assert(Operand &&
 471          Operand->isReg() &&
 472          isSameReg(*Operand, *getReplacedOperand()));
 473   copyRegOperand(*Operand, *getTargetOperand());
 474   MachineOperand *DstSel= TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel);
 475   assert(DstSel);
 476   DstSel->setImm(getDstSel());
 477   MachineOperand *DstUnused= TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);
 478   assert(DstUnused);
 479   DstUnused->setImm(getDstUnused());
 480
 481   // Remove original instruction  because it would conflict with our new
 482   // instruction by register definition
 483   getParentInst()->eraseFromParent();
 484   return true;
 485 }
 486
 487 bool SDWADstPreserveOperand::convertToSDWA(MachineInstr &MI,
 488                                            const SIInstrInfo *TII) {
 489   // MI should be moved right before v_or_b32.
 490   // For this we should clear all kill flags on uses of MI src-operands or else
 491   // we can encounter problem with use of killed operand.
 492   for (MachineOperand &MO : MI.uses()) {
 493     if (!MO.isReg())
 494       continue;
 495     getMRI()->clearKillFlags(MO.getReg());
 496   }
 497
 498   // Move MI before v_or_b32
 499   auto MBB = MI.getParent();
 500   MBB->remove(&MI);
 501   MBB->insert(getParentInst(), &MI);
 502
 503   // Add Implicit use of preserved register
 504   MachineInstrBuilder MIB(*MBB->getParent(), MI);
 505   MIB.addReg(getPreservedOperand()->getReg(),
 506              RegState::ImplicitKill,
 507              getPreservedOperand()->getSubReg());
 508
 509   // Tie dst to implicit use
 510   MI.tieOperands(AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdst),
 511                  MI.getNumOperands() - 1);
 512
 513   // Convert MI as any other SDWADstOperand and remove v_or_b32
 514   return SDWADstOperand::convertToSDWA(MI, TII);
 515 }
 516
 517 Optional<int64_t> SIPeepholeSDWA::foldToImm(const MachineOperand &Op) const {
 518   if (Op.isImm()) {
 519     return Op.getImm();
 520   }
 521
 522   // If this is not immediate then it can be copy of immediate value, e.g.:
 523   // %1 = S_MOV_B32 255;
 524   if (Op.isReg()) {
 525     for (const MachineOperand &Def : MRI->def_operands(Op.getReg())) {
 526       if (!isSameReg(Op, Def))
 527         continue;
 528
 529       const MachineInstr *DefInst = Def.getParent();
 530       if (!TII->isFoldableCopy(*DefInst))
 531         return None;
 532
 533       const MachineOperand &Copied = DefInst->getOperand(1);
 534       if (!Copied.isImm())
 535         return None;
 536
 537       return Copied.getImm();
 538     }
 539   }
 540
 541   return None;
 542 }
 543
 544 std::unique_ptr<SDWAOperand>
 545 SIPeepholeSDWA::matchSDWAOperand(MachineInstr &MI) {
 546   unsigned Opcode = MI.getOpcode();
 547   switch (Opcode) {
 548   case AMDGPU::V_LSHRREV_B32_e32:
 549   case AMDGPU::V_ASHRREV_I32_e32:
 550   case AMDGPU::V_LSHLREV_B32_e32:
 551   case AMDGPU::V_LSHRREV_B32_e64:
 552   case AMDGPU::V_ASHRREV_I32_e64:
 553   case AMDGPU::V_LSHLREV_B32_e64: {
 554     // from: v_lshrrev_b32_e32 v1, 16/24, v0
 555     // to SDWA src:v0 src_sel:WORD_1/BYTE_3
 556
 557     // from: v_ashrrev_i32_e32 v1, 16/24, v0
 558     // to SDWA src:v0 src_sel:WORD_1/BYTE_3 sext:1
 559
 560     // from: v_lshlrev_b32_e32 v1, 16/24, v0
 561     // to SDWA dst:v1 dst_sel:WORD_1/BYTE_3 dst_unused:UNUSED_PAD
 562     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 563     auto Imm = foldToImm(*Src0);
 564     if (!Imm)
 565       break;
 566
 567     if (*Imm != 16 && *Imm != 24)
 568       break;
 569
 570     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 571     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 572     if (TRI->isPhysicalRegister(Src1->getReg()) ||
 573         TRI->isPhysicalRegister(Dst->getReg()))
 574       break;
 575
 576     if (Opcode == AMDGPU::V_LSHLREV_B32_e32 ||
 577         Opcode == AMDGPU::V_LSHLREV_B32_e64) {
 578       return make_unique<SDWADstOperand>(
 579           Dst, Src1, *Imm == 16 ? WORD_1 : BYTE_3, UNUSED_PAD);
 580     } else {
 581       return make_unique<SDWASrcOperand>(
 582           Src1, Dst, *Imm == 16 ? WORD_1 : BYTE_3, false, false,
 583           Opcode != AMDGPU::V_LSHRREV_B32_e32 &&
 584           Opcode != AMDGPU::V_LSHRREV_B32_e64);
 585     }
 586     break;
 587   }
 588
 589   case AMDGPU::V_LSHRREV_B16_e32:
 590   case AMDGPU::V_ASHRREV_I16_e32:
 591   case AMDGPU::V_LSHLREV_B16_e32:
 592   case AMDGPU::V_LSHRREV_B16_e64:
 593   case AMDGPU::V_ASHRREV_I16_e64:
 594   case AMDGPU::V_LSHLREV_B16_e64: {
 595     // from: v_lshrrev_b16_e32 v1, 8, v0
 596     // to SDWA src:v0 src_sel:BYTE_1
 597
 598     // from: v_ashrrev_i16_e32 v1, 8, v0
 599     // to SDWA src:v0 src_sel:BYTE_1 sext:1
 600
 601     // from: v_lshlrev_b16_e32 v1, 8, v0
 602     // to SDWA dst:v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD
 603     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 604     auto Imm = foldToImm(*Src0);
 605     if (!Imm || *Imm != 8)
 606       break;
 607
 608     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 609     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 610
 611     if (TRI->isPhysicalRegister(Src1->getReg()) ||
 612         TRI->isPhysicalRegister(Dst->getReg()))
 613       break;
 614
 615     if (Opcode == AMDGPU::V_LSHLREV_B16_e32 ||
 616         Opcode == AMDGPU::V_LSHLREV_B16_e64) {
 617       return make_unique<SDWADstOperand>(Dst, Src1, BYTE_1, UNUSED_PAD);
 618     } else {
 619       return make_unique<SDWASrcOperand>(
 620             Src1, Dst, BYTE_1, false, false,
 621             Opcode != AMDGPU::V_LSHRREV_B16_e32 &&
 622             Opcode != AMDGPU::V_LSHRREV_B16_e64);
 623     }
 624     break;
 625   }
 626
 627   case AMDGPU::V_BFE_I32:
 628   case AMDGPU::V_BFE_U32: {
 629     // e.g.:
 630     // from: v_bfe_u32 v1, v0, 8, 8
 631     // to SDWA src:v0 src_sel:BYTE_1
 632
 633     // offset | width | src_sel
 634     // ------------------------
 635     // 0      | 8     | BYTE_0
 636     // 0      | 16    | WORD_0
 637     // 0      | 32    | DWORD ?
 638     // 8      | 8     | BYTE_1
 639     // 16     | 8     | BYTE_2
 640     // 16     | 16    | WORD_1
 641     // 24     | 8     | BYTE_3
 642
 643     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 644     auto Offset = foldToImm(*Src1);
 645     if (!Offset)
 646       break;
 647
 648     MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
 649     auto Width = foldToImm(*Src2);
 650     if (!Width)
 651       break;
 652
 653     SdwaSel SrcSel = DWORD;
 654
 655     if (*Offset == 0 && *Width == 8)
 656       SrcSel = BYTE_0;
 657     else if (*Offset == 0 && *Width == 16)
 658       SrcSel = WORD_0;
 659     else if (*Offset == 0 && *Width == 32)
 660       SrcSel = DWORD;
 661     else if (*Offset == 8 && *Width == 8)
 662       SrcSel = BYTE_1;
 663     else if (*Offset == 16 && *Width == 8)
 664       SrcSel = BYTE_2;
 665     else if (*Offset == 16 && *Width == 16)
 666       SrcSel = WORD_1;
 667     else if (*Offset == 24 && *Width == 8)
 668       SrcSel = BYTE_3;
 669     else
 670       break;
 671
 672     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 673     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 674
 675     if (TRI->isPhysicalRegister(Src0->getReg()) ||
 676         TRI->isPhysicalRegister(Dst->getReg()))
 677       break;
 678
 679     return make_unique<SDWASrcOperand>(
 680           Src0, Dst, SrcSel, false, false, Opcode != AMDGPU::V_BFE_U32);
 681   }
 682
 683   case AMDGPU::V_AND_B32_e32:
 684   case AMDGPU::V_AND_B32_e64: {
 685     // e.g.:
 686     // from: v_and_b32_e32 v1, 0x0000ffff/0x000000ff, v0
 687     // to SDWA src:v0 src_sel:WORD_0/BYTE_0
 688
 689     MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 690     MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 691     auto ValSrc = Src1;
 692     auto Imm = foldToImm(*Src0);
 693
 694     if (!Imm) {
 695       Imm = foldToImm(*Src1);
 696       ValSrc = Src0;
 697     }
 698
 699     if (!Imm || (*Imm != 0x0000ffff && *Imm != 0x000000ff))
 700       break;
 701
 702     MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 703
 704     if (TRI->isPhysicalRegister(ValSrc->getReg()) ||
 705         TRI->isPhysicalRegister(Dst->getReg()))
 706       break;
 707
 708     return make_unique<SDWASrcOperand>(
 709         ValSrc, Dst, *Imm == 0x0000ffff ? WORD_0 : BYTE_0);
 710   }
 711
 712   case AMDGPU::V_OR_B32_e32:
 713   case AMDGPU::V_OR_B32_e64: {
 714     // Patterns for dst_unused:UNUSED_PRESERVE.
 715     // e.g., from:
 716     // v_add_f16_sdwa v0, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD
 717     //                           src1_sel:WORD_1 src2_sel:WORD1
 718     // v_add_f16_e32 v3, v1, v2
 719     // v_or_b32_e32 v4, v0, v3
 720     // to SDWA preserve dst:v4 dst_sel:WORD_1 dst_unused:UNUSED_PRESERVE preserve:v3
 721
 722     // Check if one of operands of v_or_b32 is SDWA instruction
 723     using CheckRetType = Optional<std::pair<MachineOperand *, MachineOperand *>>;
 724     auto CheckOROperandsForSDWA =
 725       [&](const MachineOperand *Op1, const MachineOperand *Op2) -> CheckRetType {
 726         if (!Op1 || !Op1->isReg() || !Op2 || !Op2->isReg())
 727           return CheckRetType(None);
 728
 729         MachineOperand *Op1Def = findSingleRegDef(Op1, MRI);
 730         if (!Op1Def)
 731           return CheckRetType(None);
 732
 733         MachineInstr *Op1Inst = Op1Def->getParent();
 734         if (!TII->isSDWA(*Op1Inst))
 735           return CheckRetType(None);
 736
 737         MachineOperand *Op2Def = findSingleRegDef(Op2, MRI);
 738         if (!Op2Def)
 739           return CheckRetType(None);
 740
 741         return CheckRetType(std::make_pair(Op1Def, Op2Def));
 742       };
 743
 744     MachineOperand *OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 745     MachineOperand *OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 746     assert(OrSDWA && OrOther);
 747     auto Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
 748     if (!Res) {
 749       OrSDWA = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 750       OrOther = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 751       assert(OrSDWA && OrOther);
 752       Res = CheckOROperandsForSDWA(OrSDWA, OrOther);
 753       if (!Res)
 754         break;
 755     }
 756
 757     MachineOperand *OrSDWADef = Res->first;
 758     MachineOperand *OrOtherDef = Res->second;
 759     assert(OrSDWADef && OrOtherDef);
 760
 761     MachineInstr *SDWAInst = OrSDWADef->getParent();
 762     MachineInstr *OtherInst = OrOtherDef->getParent();
 763
 764     // Check that OtherInstr is actually bitwise compatible with SDWAInst = their
 765     // destination patterns don't overlap. Compatible instruction can be either
 766     // regular instruction with compatible bitness or SDWA instruction with
 767     // correct dst_sel
 768     // SDWAInst | OtherInst bitness / OtherInst dst_sel
 769     // -----------------------------------------------------
 770     // DWORD    | no                    / no
 771     // WORD_0   | no                    / BYTE_2/3, WORD_1
 772     // WORD_1   | 8/16-bit instructions / BYTE_0/1, WORD_0
 773     // BYTE_0   | no                    / BYTE_1/2/3, WORD_1
 774     // BYTE_1   | 8-bit                 / BYTE_0/2/3, WORD_1
 775     // BYTE_2   | 8/16-bit              / BYTE_0/1/3. WORD_0
 776     // BYTE_3   | 8/16/24-bit           / BYTE_0/1/2, WORD_0
 777     // E.g. if SDWAInst is v_add_f16_sdwa dst_sel:WORD_1 then v_add_f16 is OK
 778     // but v_add_f32 is not.
 779
 780     // TODO: add support for non-SDWA instructions as OtherInst.
 781     // For now this only works with SDWA instructions. For regular instructions
 782     // there is no way to determine if the instruction writes only 8/16/24-bit
 783     // out of full register size and all registers are at min 32-bit wide.
 784     if (!TII->isSDWA(*OtherInst))
 785       break;
 786
 787     SdwaSel DstSel = static_cast<SdwaSel>(
 788       TII->getNamedImmOperand(*SDWAInst, AMDGPU::OpName::dst_sel));;
 789     SdwaSel OtherDstSel = static_cast<SdwaSel>(
 790       TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_sel));
 791
 792     bool DstSelAgree = false;
 793     switch (DstSel) {
 794     case WORD_0: DstSelAgree = ((OtherDstSel == BYTE_2) ||
 795                                 (OtherDstSel == BYTE_3) ||
 796                                 (OtherDstSel == WORD_1));
 797       break;
 798     case WORD_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
 799                                 (OtherDstSel == BYTE_1) ||
 800                                 (OtherDstSel == WORD_0));
 801       break;
 802     case BYTE_0: DstSelAgree = ((OtherDstSel == BYTE_1) ||
 803                                 (OtherDstSel == BYTE_2) ||
 804                                 (OtherDstSel == BYTE_3) ||
 805                                 (OtherDstSel == WORD_1));
 806       break;
 807     case BYTE_1: DstSelAgree = ((OtherDstSel == BYTE_0) ||
 808                                 (OtherDstSel == BYTE_2) ||
 809                                 (OtherDstSel == BYTE_3) ||
 810                                 (OtherDstSel == WORD_1));
 811       break;
 812     case BYTE_2: DstSelAgree = ((OtherDstSel == BYTE_0) ||
 813                                 (OtherDstSel == BYTE_1) ||
 814                                 (OtherDstSel == BYTE_3) ||
 815                                 (OtherDstSel == WORD_0));
 816       break;
 817     case BYTE_3: DstSelAgree = ((OtherDstSel == BYTE_0) ||
 818                                 (OtherDstSel == BYTE_1) ||
 819                                 (OtherDstSel == BYTE_2) ||
 820                                 (OtherDstSel == WORD_0));
 821       break;
 822     default: DstSelAgree = false;
 823     }
 824
 825     if (!DstSelAgree)
 826       break;
 827
 828     // Also OtherInst dst_unused should be UNUSED_PAD
 829     DstUnused OtherDstUnused = static_cast<DstUnused>(
 830       TII->getNamedImmOperand(*OtherInst, AMDGPU::OpName::dst_unused));
 831     if (OtherDstUnused != DstUnused::UNUSED_PAD)
 832       break;
 833
 834     // Create DstPreserveOperand
 835     MachineOperand *OrDst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 836     assert(OrDst && OrDst->isReg());
 837
 838     return make_unique<SDWADstPreserveOperand>(
 839       OrDst, OrSDWADef, OrOtherDef, DstSel);
 840
 841   }
 842   }
 843
 844   return std::unique_ptr<SDWAOperand>(nullptr);
 845 }
 846
 847 void SIPeepholeSDWA::matchSDWAOperands(MachineBasicBlock &MBB) {
 848   for (MachineInstr &MI : MBB) {
 849     if (auto Operand = matchSDWAOperand(MI)) {
 850       LLVM_DEBUG(dbgs() << "Match: " << MI << "To: " << *Operand << '\n');
 851       SDWAOperands[&MI] = std::move(Operand);
 852       ++NumSDWAPatternsFound;
 853     }
 854   }
 855 }
 856
 857 bool SIPeepholeSDWA::isConvertibleToSDWA(const MachineInstr &MI,
 858                                          const GCNSubtarget &ST) const {
 859   // Check if this is already an SDWA instruction
 860   unsigned Opc = MI.getOpcode();
 861   if (TII->isSDWA(Opc))
 862     return true;
 863
 864   // Check if this instruction has opcode that supports SDWA
 865   if (AMDGPU::getSDWAOp(Opc) == -1)
 866     Opc = AMDGPU::getVOPe32(Opc);
 867
 868   if (AMDGPU::getSDWAOp(Opc) == -1)
 869     return false;
 870
 871   if (!ST.hasSDWAOmod() && TII->hasModifiersSet(MI, AMDGPU::OpName::omod))
 872     return false;
 873
 874   if (TII->isVOPC(Opc)) {
 875     if (!ST.hasSDWASdst()) {
 876       const MachineOperand *SDst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);
 877       if (SDst && SDst->getReg() != AMDGPU::VCC)
 878         return false;
 879     }
 880
 881     if (!ST.hasSDWAOutModsVOPC() &&
 882         (TII->hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
 883          TII->hasModifiersSet(MI, AMDGPU::OpName::omod)))
 884       return false;
 885
 886   } else if (TII->getNamedOperand(MI, AMDGPU::OpName::sdst) ||
 887              !TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
 888     return false;
 889   }
 890
 891   if (!ST.hasSDWAMac() && (Opc == AMDGPU::V_MAC_F16_e32 ||
 892                            Opc == AMDGPU::V_MAC_F32_e32))
 893     return false;
 894
 895   // FIXME: has SDWA but require handling of implicit VCC use
 896   if (Opc == AMDGPU::V_CNDMASK_B32_e32)
 897     return false;
 898
 899   return true;
 900 }
 901
 902 bool SIPeepholeSDWA::convertToSDWA(MachineInstr &MI,
 903                                    const SDWAOperandsVector &SDWAOperands) {
 904
 905   LLVM_DEBUG(dbgs() << "Convert instruction:" << MI);
 906
 907   // Convert to sdwa
 908   int SDWAOpcode;
 909   unsigned Opcode = MI.getOpcode();
 910   if (TII->isSDWA(Opcode)) {
 911     SDWAOpcode = Opcode;
 912   } else {
 913     SDWAOpcode = AMDGPU::getSDWAOp(Opcode);
 914     if (SDWAOpcode == -1)
 915       SDWAOpcode = AMDGPU::getSDWAOp(AMDGPU::getVOPe32(Opcode));
 916   }
 917   assert(SDWAOpcode != -1);
 918
 919   const MCInstrDesc &SDWADesc = TII->get(SDWAOpcode);
 920
 921   // Create SDWA version of instruction MI and initialize its operands
 922   MachineInstrBuilder SDWAInst =
 923     BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), SDWADesc);
 924
 925   // Copy dst, if it is present in original then should also be present in SDWA
 926   MachineOperand *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
 927   if (Dst) {
 928     assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst) != -1);
 929     SDWAInst.add(*Dst);
 930   } else if ((Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst))) {
 931     assert(Dst &&
 932            AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1);
 933     SDWAInst.add(*Dst);
 934   } else {
 935     assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::sdst) != -1);
 936     SDWAInst.addReg(AMDGPU::VCC, RegState::Define);
 937   }
 938
 939   // Copy src0, initialize src0_modifiers. All sdwa instructions has src0 and
 940   // src0_modifiers (except for v_nop_sdwa, but it can't get here)
 941   MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
 942   assert(
 943     Src0 &&
 944     AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0) != -1 &&
 945     AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_modifiers) != -1);
 946   if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src0_modifiers))
 947     SDWAInst.addImm(Mod->getImm());
 948   else
 949     SDWAInst.addImm(0);
 950   SDWAInst.add(*Src0);
 951
 952   // Copy src1 if present, initialize src1_modifiers.
 953   MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 954   if (Src1) {
 955     assert(
 956       AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1) != -1 &&
 957       AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_modifiers) != -1);
 958     if (auto *Mod = TII->getNamedOperand(MI, AMDGPU::OpName::src1_modifiers))
 959       SDWAInst.addImm(Mod->getImm());
 960     else
 961       SDWAInst.addImm(0);
 962     SDWAInst.add(*Src1);
 963   }
 964
 965   if (SDWAOpcode == AMDGPU::V_MAC_F16_sdwa ||
 966       SDWAOpcode == AMDGPU::V_MAC_F32_sdwa) {
 967     // v_mac_f16/32 has additional src2 operand tied to vdst
 968     MachineOperand *Src2 = TII->getNamedOperand(MI, AMDGPU::OpName::src2);
 969     assert(Src2);
 970     SDWAInst.add(*Src2);
 971   }
 972
 973   // Copy clamp if present, initialize otherwise
 974   assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::clamp) != -1);
 975   MachineOperand *Clamp = TII->getNamedOperand(MI, AMDGPU::OpName::clamp);
 976   if (Clamp) {
 977     SDWAInst.add(*Clamp);
 978   } else {
 979     SDWAInst.addImm(0);
 980   }
 981
 982   // Copy omod if present, initialize otherwise if needed
 983   if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::omod) != -1) {
 984     MachineOperand *OMod = TII->getNamedOperand(MI, AMDGPU::OpName::omod);
 985     if (OMod) {
 986       SDWAInst.add(*OMod);
 987     } else {
 988       SDWAInst.addImm(0);
 989     }
 990   }
 991
 992   // Copy dst_sel if present, initialize otherwise if needed
 993   if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_sel) != -1) {
 994     MachineOperand *DstSel = TII->getNamedOperand(MI, AMDGPU::OpName::dst_sel);
 995     if (DstSel) {
 996       SDWAInst.add(*DstSel);
 997     } else {
 998       SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD);
 999     }
1000   }
1001
1002   // Copy dst_unused if present, initialize otherwise if needed
1003   if (AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::dst_unused) != -1) {
1004     MachineOperand *DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);
1005     if (DstUnused) {
1006       SDWAInst.add(*DstUnused);
1007     } else {
1008       SDWAInst.addImm(AMDGPU::SDWA::DstUnused::UNUSED_PAD);
1009     }
1010   }
1011
1012   // Copy src0_sel if present, initialize otherwise
1013   assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src0_sel) != -1);
1014   MachineOperand *Src0Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src0_sel);
1015   if (Src0Sel) {
1016     SDWAInst.add(*Src0Sel);
1017   } else {
1018     SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD);
1019   }
1020
1021   // Copy src1_sel if present, initialize otherwise if needed
1022   if (Src1) {
1023     assert(AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::src1_sel) != -1);
1024     MachineOperand *Src1Sel = TII->getNamedOperand(MI, AMDGPU::OpName::src1_sel);
1025     if (Src1Sel) {
1026       SDWAInst.add(*Src1Sel);
1027     } else {
1028       SDWAInst.addImm(AMDGPU::SDWA::SdwaSel::DWORD);
1029     }
1030   }
1031
1032   // Check for a preserved register that needs to be copied.
1033   auto DstUnused = TII->getNamedOperand(MI, AMDGPU::OpName::dst_unused);
1034   if (DstUnused &&
1035       DstUnused->getImm() == AMDGPU::SDWA::DstUnused::UNUSED_PRESERVE) {
1036     // We expect, if we are here, that the instruction was already in it's SDWA form,
1037     // with a tied operand.
1038     assert(Dst && Dst->isTied());
1039     assert(Opcode == static_cast<unsigned int>(SDWAOpcode));
1040     // We also expect a vdst, since sdst can't preserve.
1041     auto PreserveDstIdx = AMDGPU::getNamedOperandIdx(SDWAOpcode, AMDGPU::OpName::vdst);
1042     assert(PreserveDstIdx != -1);
1043
1044     auto TiedIdx = MI.findTiedOperandIdx(PreserveDstIdx);
1045     auto Tied = MI.getOperand(TiedIdx);
1046
1047     SDWAInst.add(Tied);
1048     SDWAInst->tieOperands(PreserveDstIdx, SDWAInst->getNumOperands() - 1);
1049   }
1050
1051   // Apply all sdwa operand patterns.
1052   bool Converted = false;
1053   for (auto &Operand : SDWAOperands) {
1054     LLVM_DEBUG(dbgs() << *SDWAInst << "\nOperand: " << *Operand);
1055     // There should be no intesection between SDWA operands and potential MIs
1056     // e.g.:
1057     // v_and_b32 v0, 0xff, v1 -> src:v1 sel:BYTE_0
1058     // v_and_b32 v2, 0xff, v0 -> src:v0 sel:BYTE_0
1059     // v_add_u32 v3, v4, v2
1060     //
1061     // In that example it is possible that we would fold 2nd instruction into 3rd
1062     // (v_add_u32_sdwa) and then try to fold 1st instruction into 2nd (that was
1063     // already destroyed). So if SDWAOperand is also a potential MI then do not
1064     // apply it.
1065     if (PotentialMatches.count(Operand->getParentInst()) == 0)
1066       Converted |= Operand->convertToSDWA(*SDWAInst, TII);
1067   }
1068   if (Converted) {
1069     ConvertedInstructions.push_back(SDWAInst);
1070   } else {
1071     SDWAInst->eraseFromParent();
1072     return false;
1073   }
1074
1075   LLVM_DEBUG(dbgs() << "\nInto:" << *SDWAInst << '\n');
1076   ++NumSDWAInstructionsPeepholed;
1077
1078   MI.eraseFromParent();
1079   return true;
1080 }
1081
1082 // If an instruction was converted to SDWA it should not have immediates or SGPR
1083 // operands (allowed one SGPR on GFX9). Copy its scalar operands into VGPRs.
1084 void SIPeepholeSDWA::legalizeScalarOperands(MachineInstr &MI,
1085                                             const GCNSubtarget &ST) const {
1086   const MCInstrDesc &Desc = TII->get(MI.getOpcode());
1087   unsigned ConstantBusCount = 0;
1088   for (MachineOperand &Op : MI.explicit_uses()) {
1089     if (!Op.isImm() && !(Op.isReg() && !TRI->isVGPR(*MRI, Op.getReg())))
1090       continue;
1091
1092     unsigned I = MI.getOperandNo(&Op);
1093     if (Desc.OpInfo[I].RegClass == -1 ||
1094        !TRI->hasVGPRs(TRI->getRegClass(Desc.OpInfo[I].RegClass)))
1095       continue;
1096
1097     if (ST.hasSDWAScalar() && ConstantBusCount == 0 && Op.isReg() &&
1098         TRI->isSGPRReg(*MRI, Op.getReg())) {
1099       ++ConstantBusCount;
1100       continue;
1101     }
1102
1103     unsigned VGPR = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1104     auto Copy = BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(),
1105                         TII->get(AMDGPU::V_MOV_B32_e32), VGPR);
1106     if (Op.isImm())
1107       Copy.addImm(Op.getImm());
1108     else if (Op.isReg())
1109       Copy.addReg(Op.getReg(), Op.isKill() ? RegState::Kill : 0,
1110                   Op.getSubReg());
1111     Op.ChangeToRegister(VGPR, false);
1112   }
1113 }
1114
1115 bool SIPeepholeSDWA::runOnMachineFunction(MachineFunction &MF) {
1116   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1117
1118   if (!ST.hasSDWA() || skipFunction(MF.getFunction()))
1119     return false;
1120
1121   MRI = &MF.getRegInfo();
1122   TRI = ST.getRegisterInfo();
1123   TII = ST.getInstrInfo();
1124
1125   // Find all SDWA operands in MF.
1126   bool Ret = false;
1127   for (MachineBasicBlock &MBB : MF) {
1128     bool Changed = false;
1129     do {
1130       matchSDWAOperands(MBB);
1131
1132       for (const auto &OperandPair : SDWAOperands) {
1133         const auto &Operand = OperandPair.second;
1134         MachineInstr *PotentialMI = Operand->potentialToConvert(TII);
1135         if (PotentialMI && isConvertibleToSDWA(*PotentialMI, ST)) {
1136           PotentialMatches[PotentialMI].push_back(Operand.get());
1137         }
1138       }
1139
1140       for (auto &PotentialPair : PotentialMatches) {
1141         MachineInstr &PotentialMI = *PotentialPair.first;
1142         convertToSDWA(PotentialMI, PotentialPair.second);
1143       }
1144
1145       PotentialMatches.clear();
1146       SDWAOperands.clear();
1147
1148       Changed = !ConvertedInstructions.empty();
1149
1150       if (Changed)
1151         Ret = true;
1152       while (!ConvertedInstructions.empty())
1153         legalizeScalarOperands(*ConvertedInstructions.pop_back_val(), ST);
1154     } while (Changed);
1155   }
1156
1157   return Ret;
1158 }