lib/Target/AMDGPU/R600EmitClauseMarkers.cpp

   1 //===-- R600EmitClauseMarkers.cpp - Emit CF_ALU ---------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 /// \file
  11 /// Add CF_ALU. R600 Alu instructions are grouped in clause which can hold
  12 /// 128 Alu instructions ; these instructions can access up to 4 prefetched
  13 /// 4 lines of 16 registers from constant buffers. Such ALU clauses are
  14 /// initiated by CF_ALU instructions.
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "AMDGPU.h"
  18 #include "AMDGPUSubtarget.h"
  19 #include "R600Defines.h"
  20 #include "R600InstrInfo.h"
  21 #include "R600RegisterInfo.h"
  22 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
  23 #include "llvm/ADT/SmallVector.h"
  24 #include "llvm/ADT/StringRef.h"
  25 #include "llvm/CodeGen/MachineBasicBlock.h"
  26 #include "llvm/CodeGen/MachineFunction.h"
  27 #include "llvm/CodeGen/MachineFunctionPass.h"
  28 #include "llvm/CodeGen/MachineInstr.h"
  29 #include "llvm/CodeGen/MachineInstrBuilder.h"
  30 #include "llvm/CodeGen/MachineOperand.h"
  31 #include "llvm/Pass.h"
  32 #include "llvm/Support/ErrorHandling.h"
  33 #include <cassert>
  34 #include <cstdint>
  35 #include <utility>
  36 #include <vector>
  37
  38 using namespace llvm;
  39
  40 namespace llvm {
  41
  42   void initializeR600EmitClauseMarkersPass(PassRegistry&);
  43
  44 } // end namespace llvm
  45
  46 namespace {
  47
  48 class R600EmitClauseMarkers : public MachineFunctionPass {
  49 private:
  50   const R600InstrInfo *TII = nullptr;
  51   int Address = 0;
  52
  53   unsigned OccupiedDwords(MachineInstr &MI) const {
  54     switch (MI.getOpcode()) {
  55     case R600::INTERP_PAIR_XY:
  56     case R600::INTERP_PAIR_ZW:
  57     case R600::INTERP_VEC_LOAD:
  58     case R600::DOT_4:
  59       return 4;
  60     case R600::KILL:
  61       return 0;
  62     default:
  63       break;
  64     }
  65
  66     // These will be expanded to two ALU instructions in the
  67     // ExpandSpecialInstructions pass.
  68     if (TII->isLDSRetInstr(MI.getOpcode()))
  69       return 2;
  70
  71     if (TII->isVector(MI) || TII->isCubeOp(MI.getOpcode()) ||
  72         TII->isReductionOp(MI.getOpcode()))
  73       return 4;
  74
  75     unsigned NumLiteral = 0;
  76     for (MachineInstr::mop_iterator It = MI.operands_begin(),
  77                                     E = MI.operands_end();
  78          It != E; ++It) {
  79       MachineOperand &MO = *It;
  80       if (MO.isReg() && MO.getReg() == R600::ALU_LITERAL_X)
  81         ++NumLiteral;
  82     }
  83     return 1 + NumLiteral;
  84   }
  85
  86   bool isALU(const MachineInstr &MI) const {
  87     if (TII->isALUInstr(MI.getOpcode()))
  88       return true;
  89     if (TII->isVector(MI) || TII->isCubeOp(MI.getOpcode()))
  90       return true;
  91     switch (MI.getOpcode()) {
  92     case R600::PRED_X:
  93     case R600::INTERP_PAIR_XY:
  94     case R600::INTERP_PAIR_ZW:
  95     case R600::INTERP_VEC_LOAD:
  96     case R600::COPY:
  97     case R600::DOT_4:
  98       return true;
  99     default:
 100       return false;
 101     }
 102   }
 103
 104   bool IsTrivialInst(MachineInstr &MI) const {
 105     switch (MI.getOpcode()) {
 106     case R600::KILL:
 107     case R600::RETURN:
 108     case R600::IMPLICIT_DEF:
 109       return true;
 110     default:
 111       return false;
 112     }
 113   }
 114
 115   std::pair<unsigned, unsigned> getAccessedBankLine(unsigned Sel) const {
 116     // Sel is (512 + (kc_bank << 12) + ConstIndex) << 2
 117     // (See also R600ISelLowering.cpp)
 118     // ConstIndex value is in [0, 4095];
 119     return std::pair<unsigned, unsigned>(
 120         ((Sel >> 2) - 512) >> 12, // KC_BANK
 121         // Line Number of ConstIndex
 122         // A line contains 16 constant registers however KCX bank can lock
 123         // two line at the same time ; thus we want to get an even line number.
 124         // Line number can be retrieved with (>>4), using (>>5) <<1 generates
 125         // an even number.
 126         ((((Sel >> 2) - 512) & 4095) >> 5) << 1);
 127   }
 128
 129   bool
 130   SubstituteKCacheBank(MachineInstr &MI,
 131                        std::vector<std::pair<unsigned, unsigned>> &CachedConsts,
 132                        bool UpdateInstr = true) const {
 133     std::vector<std::pair<unsigned, unsigned>> UsedKCache;
 134
 135     if (!TII->isALUInstr(MI.getOpcode()) && MI.getOpcode() != R600::DOT_4)
 136       return true;
 137
 138     const SmallVectorImpl<std::pair<MachineOperand *, int64_t>> &Consts =
 139         TII->getSrcs(MI);
 140     assert(
 141         (TII->isALUInstr(MI.getOpcode()) || MI.getOpcode() == R600::DOT_4) &&
 142         "Can't assign Const");
 143     for (unsigned i = 0, n = Consts.size(); i < n; ++i) {
 144       if (Consts[i].first->getReg() != R600::ALU_CONST)
 145         continue;
 146       unsigned Sel = Consts[i].second;
 147       unsigned Chan = Sel & 3, Index = ((Sel >> 2) - 512) & 31;
 148       unsigned KCacheIndex = Index * 4 + Chan;
 149       const std::pair<unsigned, unsigned> &BankLine = getAccessedBankLine(Sel);
 150       if (CachedConsts.empty()) {
 151         CachedConsts.push_back(BankLine);
 152         UsedKCache.push_back(std::pair<unsigned, unsigned>(0, KCacheIndex));
 153         continue;
 154       }
 155       if (CachedConsts[0] == BankLine) {
 156         UsedKCache.push_back(std::pair<unsigned, unsigned>(0, KCacheIndex));
 157         continue;
 158       }
 159       if (CachedConsts.size() == 1) {
 160         CachedConsts.push_back(BankLine);
 161         UsedKCache.push_back(std::pair<unsigned, unsigned>(1, KCacheIndex));
 162         continue;
 163       }
 164       if (CachedConsts[1] == BankLine) {
 165         UsedKCache.push_back(std::pair<unsigned, unsigned>(1, KCacheIndex));
 166         continue;
 167       }
 168       return false;
 169     }
 170
 171     if (!UpdateInstr)
 172       return true;
 173
 174     for (unsigned i = 0, j = 0, n = Consts.size(); i < n; ++i) {
 175       if (Consts[i].first->getReg() != R600::ALU_CONST)
 176         continue;
 177       switch(UsedKCache[j].first) {
 178       case 0:
 179         Consts[i].first->setReg(
 180             R600::R600_KC0RegClass.getRegister(UsedKCache[j].second));
 181         break;
 182       case 1:
 183         Consts[i].first->setReg(
 184             R600::R600_KC1RegClass.getRegister(UsedKCache[j].second));
 185         break;
 186       default:
 187         llvm_unreachable("Wrong Cache Line");
 188       }
 189       j++;
 190     }
 191     return true;
 192   }
 193
 194   bool canClauseLocalKillFitInClause(
 195                         unsigned AluInstCount,
 196                         std::vector<std::pair<unsigned, unsigned>> KCacheBanks,
 197                         MachineBasicBlock::iterator Def,
 198                         MachineBasicBlock::iterator BBEnd) {
 199     const R600RegisterInfo &TRI = TII->getRegisterInfo();
 200     //TODO: change this to defs?
 201     for (MachineInstr::const_mop_iterator
 202            MOI = Def->operands_begin(),
 203            MOE = Def->operands_end(); MOI != MOE; ++MOI) {
 204       if (!MOI->isReg() || !MOI->isDef() ||
 205           TRI.isPhysRegLiveAcrossClauses(MOI->getReg()))
 206         continue;
 207
 208       // Def defines a clause local register, so check that its use will fit
 209       // in the clause.
 210       unsigned LastUseCount = 0;
 211       for (MachineBasicBlock::iterator UseI = Def; UseI != BBEnd; ++UseI) {
 212         AluInstCount += OccupiedDwords(*UseI);
 213         // Make sure we won't need to end the clause due to KCache limitations.
 214         if (!SubstituteKCacheBank(*UseI, KCacheBanks, false))
 215           return false;
 216
 217         // We have reached the maximum instruction limit before finding the
 218         // use that kills this register, so we cannot use this def in the
 219         // current clause.
 220         if (AluInstCount >= TII->getMaxAlusPerClause())
 221           return false;
 222
 223         // TODO: Is this true? kill flag appears to work OK below
 224         // Register kill flags have been cleared by the time we get to this
 225         // pass, but it is safe to assume that all uses of this register
 226         // occur in the same basic block as its definition, because
 227         // it is illegal for the scheduler to schedule them in
 228         // different blocks.
 229         if (UseI->readsRegister(MOI->getReg()))
 230           LastUseCount = AluInstCount;
 231
 232         // Exit early if the current use kills the register
 233         if (UseI != Def && UseI->killsRegister(MOI->getReg()))
 234           break;
 235       }
 236       if (LastUseCount)
 237         return LastUseCount <= TII->getMaxAlusPerClause();
 238       llvm_unreachable("Clause local register live at end of clause.");
 239     }
 240     return true;
 241   }
 242
 243   MachineBasicBlock::iterator
 244   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator I) {
 245     MachineBasicBlock::iterator ClauseHead = I;
 246     std::vector<std::pair<unsigned, unsigned>> KCacheBanks;
 247     bool PushBeforeModifier = false;
 248     unsigned AluInstCount = 0;
 249     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
 250       if (IsTrivialInst(*I))
 251         continue;
 252       if (!isALU(*I))
 253         break;
 254       if (AluInstCount > TII->getMaxAlusPerClause())
 255         break;
 256       if (I->getOpcode() == R600::PRED_X) {
 257         // We put PRED_X in its own clause to ensure that ifcvt won't create
 258         // clauses with more than 128 insts.
 259         // IfCvt is indeed checking that "then" and "else" branches of an if
 260         // statement have less than ~60 insts thus converted clauses can't be
 261         // bigger than ~121 insts (predicate setter needs to be in the same
 262         // clause as predicated alus).
 263         if (AluInstCount > 0)
 264           break;
 265         if (TII->getFlagOp(*I).getImm() & MO_FLAG_PUSH)
 266           PushBeforeModifier = true;
 267         AluInstCount ++;
 268         continue;
 269       }
 270       // XXX: GROUP_BARRIER instructions cannot be in the same ALU clause as:
 271       //
 272       // * KILL or INTERP instructions
 273       // * Any instruction that sets UPDATE_EXEC_MASK or UPDATE_PRED bits
 274       // * Uses waterfalling (i.e. INDEX_MODE = AR.X)
 275       //
 276       // XXX: These checks have not been implemented yet.
 277       if (TII->mustBeLastInClause(I->getOpcode())) {
 278         I++;
 279         break;
 280       }
 281
 282       // If this instruction defines a clause local register, make sure
 283       // its use can fit in this clause.
 284       if (!canClauseLocalKillFitInClause(AluInstCount, KCacheBanks, I, E))
 285         break;
 286
 287       if (!SubstituteKCacheBank(*I, KCacheBanks))
 288         break;
 289       AluInstCount += OccupiedDwords(*I);
 290     }
 291     unsigned Opcode = PushBeforeModifier ?
 292         R600::CF_ALU_PUSH_BEFORE : R600::CF_ALU;
 293     BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead), TII->get(Opcode))
 294     // We don't use the ADDR field until R600ControlFlowFinalizer pass, where
 295     // it is safe to assume it is 0. However if we always put 0 here, the ifcvt
 296     // pass may assume that identical ALU clause starter at the beginning of a
 297     // true and false branch can be factorized which is not the case.
 298         .addImm(Address++) // ADDR
 299         .addImm(KCacheBanks.empty()?0:KCacheBanks[0].first) // KB0
 300         .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].first) // KB1
 301         .addImm(KCacheBanks.empty()?0:2) // KM0
 302         .addImm((KCacheBanks.size() < 2)?0:2) // KM1
 303         .addImm(KCacheBanks.empty()?0:KCacheBanks[0].second) // KLINE0
 304         .addImm((KCacheBanks.size() < 2)?0:KCacheBanks[1].second) // KLINE1
 305         .addImm(AluInstCount) // COUNT
 306         .addImm(1); // Enabled
 307     return I;
 308   }
 309
 310 public:
 311   static char ID;
 312
 313   R600EmitClauseMarkers() : MachineFunctionPass(ID) {
 314     initializeR600EmitClauseMarkersPass(*PassRegistry::getPassRegistry());
 315   }
 316
 317   bool runOnMachineFunction(MachineFunction &MF) override {
 318     const R600Subtarget &ST = MF.getSubtarget<R600Subtarget>();
 319     TII = ST.getInstrInfo();
 320
 321     for (MachineFunction::iterator BB = MF.begin(), BB_E = MF.end();
 322                                                     BB != BB_E; ++BB) {
 323       MachineBasicBlock &MBB = *BB;
 324       MachineBasicBlock::iterator I = MBB.begin();
 325       if (I != MBB.end() && I->getOpcode() == R600::CF_ALU)
 326         continue; // BB was already parsed
 327       for (MachineBasicBlock::iterator E = MBB.end(); I != E;) {
 328         if (isALU(*I)) {
 329           auto next = MakeALUClause(MBB, I);
 330           assert(next != I);
 331           I = next;
 332         } else
 333           ++I;
 334       }
 335     }
 336     return false;
 337   }
 338
 339   StringRef getPassName() const override {
 340     return "R600 Emit Clause Markers Pass";
 341   }
 342 };
 343
 344 char R600EmitClauseMarkers::ID = 0;
 345
 346 } // end anonymous namespace
 347
 348 INITIALIZE_PASS_BEGIN(R600EmitClauseMarkers, "emitclausemarkers",
 349                       "R600 Emit Clause Markters", false, false)
 350 INITIALIZE_PASS_END(R600EmitClauseMarkers, "emitclausemarkers",
 351                       "R600 Emit Clause Markters", false, false)
 352
 353 FunctionPass *llvm::createR600EmitClauseMarkers() {
 354   return new R600EmitClauseMarkers();
 355 }