llvm/lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Definitions ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def MUBUFAddr64 : ComplexPattern<i64, 4, "SelectMUBUFAddr64">;
  10 def MUBUFOffset : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  11
  12 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  13 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  14
  15 def BUFAddrKind {
  16   int Offset = 0;
  17   int OffEn  = 1;
  18   int IdxEn  = 2;
  19   int BothEn = 3;
  20   int Addr64 = 4;
  21 }
  22
  23 class getAddrName<int addrKind> {
  24   string ret =
  25     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  26     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  27     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  28     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  29     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  30     "")))));
  31 }
  32
  33 class MUBUFAddr64Table <bit is_addr64, string Name> {
  34   bit IsAddr64 = is_addr64;
  35   string OpName = Name;
  36 }
  37
  38 class MUBUFLdsTable <bit is_lds, string Name> {
  39   bit IsLds = is_lds;
  40   string OpName = Name;
  41 }
  42
  43 class MTBUFAddr64Table <bit is_addr64, string Name> {
  44   bit IsAddr64 = is_addr64;
  45   string OpName = Name;
  46 }
  47
  48 //===----------------------------------------------------------------------===//
  49 // MTBUF classes
  50 //===----------------------------------------------------------------------===//
  51
  52 class MTBUFGetBaseOpcode<string Op> {
  53   string ret = !subst("FORMAT_XY", "FORMAT_X",
  54     !subst("FORMAT_XYZ", "FORMAT_X",
  55     !subst("FORMAT_XYZW", "FORMAT_X", Op)));
  56 }
  57
  58 class getMTBUFElements<string Op> {
  59   int ret = 1;
  60 }
  61
  62
  63 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  64                     string asmOps, list<dag> pattern=[]> :
  65   InstSI<outs, ins, "", pattern>,
  66   SIMCInstr<opName, SIEncodingFamily.NONE> {
  67
  68   let isPseudo = 1;
  69   let isCodeGenOnly = 1;
  70   let Size = 8;
  71   let UseNamedOperandTable = 1;
  72
  73   string Mnemonic = opName;
  74   string AsmOperands = asmOps;
  75
  76   Instruction Opcode = !cast<Instruction>(NAME);
  77   Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret);
  78
  79   let VM_CNT = 1;
  80   let EXP_CNT = 1;
  81   let MTBUF = 1;
  82   let Uses = [EXEC];
  83   let hasSideEffects = 0;
  84   let SchedRW = [WriteVMEM];
  85
  86   let AsmMatchConverter = "cvtMtbuf";
  87
  88   bits<1> offen       = 0;
  89   bits<1> idxen       = 0;
  90   bits<1> addr64      = 0;
  91   bits<1> has_vdata   = 1;
  92   bits<1> has_vaddr   = 1;
  93   bits<1> has_glc     = 1;
  94   bits<1> has_dlc     = 1;
  95   bits<1> glc_value   = 0; // the value for glc if no such operand
  96   bits<1> dlc_value   = 0; // the value for dlc if no such operand
  97   bits<1> has_srsrc   = 1;
  98   bits<1> has_soffset = 1;
  99   bits<1> has_offset  = 1;
 100   bits<1> has_slc     = 1;
 101   bits<1> has_tfe     = 1;
 102   bits<4> elements    = 0;
 103   bits<1> has_sccb    = 1;
 104   bits<1> sccb_value  = 0;
 105 }
 106
 107 class MTBUF_Real <MTBUF_Pseudo ps> :
 108   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 109
 110   let isPseudo = 0;
 111   let isCodeGenOnly = 0;
 112
 113   let VM_CNT = 1;
 114   let EXP_CNT = 1;
 115   let MTBUF = 1;
 116
 117   // copy relevant pseudo op flags
 118   let UseNamedOperandTable = ps.UseNamedOperandTable;
 119   let SubtargetPredicate = ps.SubtargetPredicate;
 120   let AsmMatchConverter  = ps.AsmMatchConverter;
 121   let Constraints        = ps.Constraints;
 122   let DisableEncoding    = ps.DisableEncoding;
 123   let TSFlags            = ps.TSFlags;
 124   let SchedRW            = ps.SchedRW;
 125   let mayLoad            = ps.mayLoad;
 126   let mayStore           = ps.mayStore;
 127   let IsAtomicRet        = ps.IsAtomicRet;
 128   let IsAtomicNoRet      = ps.IsAtomicNoRet;
 129
 130   bits<12> offset;
 131   bits<5>  cpol;
 132   bits<7>  format;
 133   bits<8>  vaddr;
 134   bits<10> vdata;
 135   bits<7>  srsrc;
 136   bits<1>  tfe;
 137   bits<8>  soffset;
 138
 139   bits<4> dfmt = format{3-0};
 140   bits<3> nfmt = format{6-4};
 141
 142   // GFX90A+ only: instruction uses AccVGPR for data
 143   // Bit superceedes tfe.
 144   bits<1> acc = !if(ps.has_vdata, vdata{9}, 0);
 145 }
 146
 147 class getMTBUFInsDA<list<RegisterClass> vdataList,
 148                     list<RegisterClass> vaddrList=[]> {
 149   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 150   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 151   RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
 152   dag InsNoData = !if(!empty(vaddrList),
 153     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 154          offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz),
 155     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 156          offset:$offset, FORMAT:$format, CPol:$cpol, TFE:$tfe, SWZ:$swz)
 157   );
 158   dag InsData = !if(!empty(vaddrList),
 159     (ins vdata_op:$vdata,                    SReg_128:$srsrc,
 160          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol,
 161          TFE:$tfe, SWZ:$swz),
 162     (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 163          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol,
 164          TFE:$tfe, SWZ:$swz)
 165   );
 166   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 167 }
 168
 169 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 170   dag ret =
 171     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 172     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 173     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 174     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 175     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 176     (ins))))));
 177 }
 178
 179 class getMTBUFAsmOps<int addrKind> {
 180   string Pfx =
 181     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc,$format $soffset",
 182     !if(!eq(addrKind, BUFAddrKind.OffEn),
 183             "$vaddr, $srsrc,$format $soffset offen",
 184     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 185             "$vaddr, $srsrc,$format $soffset idxen",
 186     !if(!eq(addrKind, BUFAddrKind.BothEn),
 187             "$vaddr, $srsrc,$format $soffset idxen offen",
 188     !if(!eq(addrKind, BUFAddrKind.Addr64),
 189             "$vaddr, $srsrc,$format $soffset addr64",
 190     "")))));
 191   string ret = Pfx # "$offset";
 192 }
 193
 194 class MTBUF_SetupAddr<int addrKind> {
 195   bits<1> offen  = !or(!eq(addrKind, BUFAddrKind.OffEn),
 196                        !eq(addrKind, BUFAddrKind.BothEn));
 197
 198   bits<1> idxen  = !or(!eq(addrKind, BUFAddrKind.IdxEn),
 199                        !eq(addrKind, BUFAddrKind.BothEn));
 200
 201   bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64);
 202
 203   bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset);
 204 }
 205
 206 class MTBUF_Load_Pseudo <string opName,
 207                          int addrKind,
 208                          RegisterClass vdataClass,
 209                          int elems,
 210                          list<dag> pattern=[],
 211                          // Workaround bug bz30254
 212                          int addrKindCopy = addrKind>
 213   : MTBUF_Pseudo<opName,
 214                  (outs getLdStRegisterOperand<vdataClass>.ret:$vdata),
 215                  getMTBUFIns<addrKindCopy>.ret,
 216                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz",
 217                  pattern>,
 218     MTBUF_SetupAddr<addrKindCopy> {
 219   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 220   let mayLoad = 1;
 221   let mayStore = 0;
 222   let elements = elems;
 223 }
 224
 225 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 226                               int elems, ValueType load_vt = i32,
 227                               SDPatternOperator ld = null_frag> {
 228
 229   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>,
 230                 MTBUFAddr64Table<0, NAME>;
 231
 232   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>,
 233                 MTBUFAddr64Table<1, NAME>;
 234
 235   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 236   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 237   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 238
 239   let DisableWQM = 1 in {
 240     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 241     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 242     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 243     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 244   }
 245 }
 246
 247 class MTBUF_Store_Pseudo <string opName,
 248                           int addrKind,
 249                           RegisterClass vdataClass,
 250                           int elems,
 251                           list<dag> pattern=[],
 252                           // Workaround bug bz30254
 253                           int addrKindCopy = addrKind,
 254                           RegisterClass vdataClassCopy = vdataClass>
 255   : MTBUF_Pseudo<opName,
 256                  (outs),
 257                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 258                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz",
 259                  pattern>,
 260     MTBUF_SetupAddr<addrKindCopy> {
 261   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 262   let mayLoad = 0;
 263   let mayStore = 1;
 264   let elements = elems;
 265 }
 266
 267 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 268                                int elems, ValueType store_vt = i32,
 269                                SDPatternOperator st = null_frag> {
 270
 271   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>,
 272     MTBUFAddr64Table<0, NAME>;
 273
 274   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>,
 275     MTBUFAddr64Table<1, NAME>;
 276
 277   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 278   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 279   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 280
 281   let DisableWQM = 1 in {
 282     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 283     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 284     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 285     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 286   }
 287 }
 288
 289
 290 //===----------------------------------------------------------------------===//
 291 // MUBUF classes
 292 //===----------------------------------------------------------------------===//
 293
 294 class MUBUFGetBaseOpcode<string Op> {
 295   string ret = !subst("DWORDX2", "DWORD",
 296     !subst("DWORDX3", "DWORD",
 297     !subst("DWORDX4", "DWORD", Op)));
 298 }
 299
 300 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 301                     string asmOps, list<dag> pattern=[]> :
 302   InstSI<outs, ins, "", pattern>,
 303   SIMCInstr<opName, SIEncodingFamily.NONE> {
 304
 305   let isPseudo = 1;
 306   let isCodeGenOnly = 1;
 307   let Size = 8;
 308   let UseNamedOperandTable = 1;
 309
 310   string Mnemonic = opName;
 311   string AsmOperands = asmOps;
 312
 313   Instruction Opcode = !cast<Instruction>(NAME);
 314   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
 315
 316   let VM_CNT = 1;
 317   let EXP_CNT = 1;
 318   let MUBUF = 1;
 319   let Uses = [EXEC];
 320   let hasSideEffects = 0;
 321   let SchedRW = [WriteVMEM];
 322
 323   let AsmMatchConverter = "cvtMubuf";
 324
 325   bits<1> offen       = 0;
 326   bits<1> idxen       = 0;
 327   bits<1> addr64      = 0;
 328   bits<1> lds         = 0;
 329   bits<1> has_vdata   = 1;
 330   bits<1> has_vaddr   = 1;
 331   bits<1> has_glc     = 1;
 332   bits<1> has_dlc     = 1;
 333   bits<1> glc_value   = 0; // the value for glc if no such operand
 334   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 335   bits<1> has_srsrc   = 1;
 336   bits<1> has_soffset = 1;
 337   bits<1> has_offset  = 1;
 338   bits<1> has_slc     = 1;
 339   bits<1> has_tfe     = 1;
 340   bits<4> elements    = 0;
 341   bits<1> has_sccb    = 1;
 342   bits<1> sccb_value  = 0;
 343   bits<1> IsBufferInv = 0;
 344 }
 345
 346 class MUBUF_Real <MUBUF_Pseudo ps> :
 347   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 348
 349   let isPseudo = 0;
 350   let isCodeGenOnly = 0;
 351
 352   let VM_CNT = 1;
 353   let EXP_CNT = 1;
 354   let MUBUF = 1;
 355
 356   // copy relevant pseudo op flags
 357   let SubtargetPredicate   = ps.SubtargetPredicate;
 358   let AsmMatchConverter    = ps.AsmMatchConverter;
 359   let OtherPredicates      = ps.OtherPredicates;
 360   let Constraints          = ps.Constraints;
 361   let DisableEncoding      = ps.DisableEncoding;
 362   let TSFlags              = ps.TSFlags;
 363   let UseNamedOperandTable = ps.UseNamedOperandTable;
 364   let SchedRW              = ps.SchedRW;
 365   let mayLoad              = ps.mayLoad;
 366   let mayStore             = ps.mayStore;
 367   let IsAtomicRet          = ps.IsAtomicRet;
 368   let IsAtomicNoRet        = ps.IsAtomicNoRet;
 369
 370   bits<12> offset;
 371   bits<5>  cpol;
 372   bits<8>  vaddr;
 373   bits<10> vdata;
 374   bits<7>  srsrc;
 375   bits<1>  tfe;
 376   bits<8>  soffset;
 377
 378   // GFX90A+ only: instruction uses AccVGPR for data
 379   // Bit superceedes tfe.
 380   bits<1> acc = !if(ps.has_vdata, vdata{9}, 0);
 381 }
 382
 383
 384 // For cache invalidation instructions.
 385 class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
 386   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 387
 388   let AsmMatchConverter = "";
 389
 390   let hasSideEffects = 1;
 391   let mayLoad = 0;
 392   let mayStore = 0;
 393
 394   let IsBufferInv = 1;
 395   // Set everything else to 0.
 396   let offen       = 0;
 397   let idxen       = 0;
 398   let addr64      = 0;
 399   let has_vdata   = 0;
 400   let has_vaddr   = 0;
 401   let has_glc     = 0;
 402   let has_dlc     = 0;
 403   let glc_value   = 0;
 404   let dlc_value   = 0;
 405   let has_srsrc   = 0;
 406   let has_soffset = 0;
 407   let has_offset  = 0;
 408   let has_slc     = 0;
 409   let has_tfe     = 0;
 410   let has_sccb    = 0;
 411   let sccb_value  = 0;
 412 }
 413
 414 class getMUBUFInsDA<list<RegisterClass> vdataList,
 415                     list<RegisterClass> vaddrList=[],
 416                     bit isLds = 0> {
 417   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 418   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 419   RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
 420   dag InsNoData = !if(!empty(vaddrList),
 421     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 422          offset:$offset, CPol_0:$cpol),
 423     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 424          offset:$offset, CPol_0:$cpol)
 425   );
 426   dag InsData = !if(!empty(vaddrList),
 427     (ins vdata_op:$vdata,                    SReg_128:$srsrc,
 428          SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol),
 429     (ins vdata_op:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 430          SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol)
 431   );
 432   dag ret = !con(
 433               !if(!empty(vdataList), InsNoData, InsData),
 434               !if(isLds, (ins SWZ_0:$swz), (ins TFE_0:$tfe, SWZ_0:$swz))
 435              );
 436 }
 437
 438 class getMUBUFElements<ValueType vt> {
 439   int ret =
 440     !if(!eq(vt, f16), 1,
 441       !if(!eq(vt, v2f16), 2,
 442         !if(!eq(vt, v3f16), 3,
 443           !if(!eq(vt, v4f16), 4,
 444             !if(!eq(vt.Size, 32), 1,
 445               !if(!eq(vt.Size, 64), 2,
 446                 !if(!eq(vt.Size, 96), 3,
 447                   !if(!eq(vt.Size, 128), 4, 0)
 448                 )
 449               )
 450             )
 451           )
 452         )
 453       )
 454     );
 455 }
 456
 457 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> {
 458   dag ret =
 459     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret,
 460     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 461     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 462     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 463     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 464     (ins))))));
 465 }
 466
 467 class getMUBUFAsmOps<int addrKind> {
 468   string Pfx =
 469     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 470     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 471     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 472     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 473     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 474     "")))));
 475   string ret = Pfx # "$offset";
 476 }
 477
 478 class MUBUF_SetupAddr<int addrKind> {
 479   bits<1> offen  = !or(!eq(addrKind, BUFAddrKind.OffEn),
 480                        !eq(addrKind, BUFAddrKind.BothEn));
 481
 482   bits<1> idxen  = !or(!eq(addrKind, BUFAddrKind.IdxEn),
 483                        !eq(addrKind, BUFAddrKind.BothEn));
 484
 485   bits<1> addr64 = !eq(addrKind, BUFAddrKind.Addr64);
 486
 487   bits<1> has_vaddr = !ne(addrKind, BUFAddrKind.Offset);
 488 }
 489
 490 class MUBUF_Load_Pseudo <string opName,
 491                          int addrKind,
 492                          ValueType vdata_vt,
 493                          bit HasTiedDest = 0,
 494                          bit isLds = 0,
 495                          list<dag> pattern=[],
 496                          // Workaround bug bz30254
 497                          int addrKindCopy = addrKind,
 498                          RegisterClass vdata_rc = getVregSrcForVT<vdata_vt>.ret,
 499                          RegisterOperand vdata_op = getLdStRegisterOperand<vdata_rc>.ret>
 500   : MUBUF_Pseudo<opName,
 501                  (outs vdata_op:$vdata),
 502                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
 503                       !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))),
 504                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol" #
 505                    !if(isLds, " lds", "$tfe") # "$swz",
 506                  pattern>,
 507     MUBUF_SetupAddr<addrKindCopy> {
 508   let PseudoInstr = opName # !if(isLds, "_lds", "") #
 509                     "_" # getAddrName<addrKindCopy>.ret;
 510   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 511
 512   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 513   let mayLoad = 1;
 514   let mayStore = 0;
 515   let maybeAtomic = 1;
 516   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
 517   let has_tfe = !not(isLds);
 518   let lds = isLds;
 519   let elements = getMUBUFElements<vdata_vt>.ret;
 520 }
 521
 522 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
 523   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))),
 524   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset))
 525 >;
 526
 527 class MUBUF_Addr64_Load_Pat <Instruction inst,
 528                             ValueType load_vt = i32,
 529                             SDPatternOperator ld = null_frag> : Pat <
 530   (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))),
 531   (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset))
 532 >;
 533
 534 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
 535   def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
 536   def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
 537 }
 538
 539
 540 // FIXME: tfe can't be an operand because it requires a separate
 541 // opcode because it needs an N+1 register class dest register.
 542 multiclass MUBUF_Pseudo_Loads<string opName,
 543                               ValueType load_vt = i32,
 544                               SDPatternOperator ld = null_frag,
 545                               bit TiedDest = 0,
 546                               bit isLds = 0> {
 547
 548   defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt);
 549
 550   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>,
 551     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 552
 553   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds>,
 554     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 555
 556   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>;
 557   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>;
 558   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>;
 559
 560   let DisableWQM = 1 in {
 561     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds>;
 562     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds>;
 563     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds>;
 564     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds>;
 565   }
 566 }
 567
 568 multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32,
 569                                   SDPatternOperator ld_nolds = null_frag,
 570                                   SDPatternOperator ld_lds = null_frag> {
 571   defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>;
 572   defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>;
 573 }
 574
 575 class MUBUF_Store_Pseudo <string opName,
 576                           int addrKind,
 577                           ValueType store_vt,
 578                           list<dag> pattern=[],
 579                           // Workaround bug bz30254
 580                           int addrKindCopy = addrKind>
 581   : MUBUF_Pseudo<opName,
 582                  (outs),
 583                  getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret,
 584                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol$tfe$swz",
 585                  pattern>,
 586     MUBUF_SetupAddr<addrKindCopy> {
 587   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 588   let mayLoad = 0;
 589   let mayStore = 1;
 590   let maybeAtomic = 1;
 591   let elements = getMUBUFElements<store_vt>.ret;
 592 }
 593
 594 multiclass MUBUF_Pseudo_Stores<string opName,
 595                                ValueType store_vt = i32,
 596                                SDPatternOperator st = null_frag> {
 597
 598   defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt);
 599
 600   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt,
 601     [(st legal_store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 602                                              i16:$offset))]>,
 603     MUBUFAddr64Table<0, NAME>;
 604
 605   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt,
 606     [(st legal_store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 607                                              i16:$offset))]>,
 608     MUBUFAddr64Table<1, NAME>;
 609
 610   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>;
 611   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>;
 612   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>;
 613
 614   let DisableWQM = 1 in {
 615     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt>;
 616     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt>;
 617     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt>;
 618     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt>;
 619   }
 620 }
 621
 622 class MUBUF_Pseudo_Store_Lds<string opName>
 623   : MUBUF_Pseudo<opName,
 624                  (outs),
 625                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol:$cpol, SWZ:$swz),
 626                  " $srsrc, $soffset$offset lds$cpol$swz"> {
 627   let mayLoad = 0;
 628   let mayStore = 1;
 629   let maybeAtomic = 1;
 630
 631   let has_vdata = 0;
 632   let has_vaddr = 0;
 633   let has_tfe = 0;
 634   let lds = 1;
 635
 636   let Uses = [EXEC, M0];
 637   let AsmMatchConverter = "cvtMubufLds";
 638 }
 639
 640 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 641                           list<RegisterClass> vaddrList=[]> {
 642   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 643   RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
 644   dag ret = !if(vdata_in,
 645     !if(!empty(vaddrList),
 646       (ins vdata_op:$vdata_in,
 647            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol),
 648       (ins vdata_op:$vdata_in, vaddrClass:$vaddr,
 649            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_GLC1:$cpol)
 650     ),
 651     !if(!empty(vaddrList),
 652       (ins vdata_op:$vdata,
 653            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol),
 654       (ins vdata_op:$vdata, vaddrClass:$vaddr,
 655            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol)
 656   ));
 657 }
 658
 659 class getMUBUFAtomicIns<int addrKind,
 660                         RegisterClass vdataClass,
 661                         bit vdata_in,
 662                         // Workaround bug bz30254
 663                         RegisterClass vdataClassCopy=vdataClass> {
 664   dag ret =
 665     !if(!eq(addrKind, BUFAddrKind.Offset),
 666             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 667     !if(!eq(addrKind, BUFAddrKind.OffEn),
 668             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 669     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 670             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 671     !if(!eq(addrKind, BUFAddrKind.BothEn),
 672             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 673     !if(!eq(addrKind, BUFAddrKind.Addr64),
 674             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 675     (ins))))));
 676 }
 677
 678 class MUBUF_Atomic_Pseudo<string opName,
 679                           int addrKind,
 680                           dag outs,
 681                           dag ins,
 682                           string asmOps,
 683                           list<dag> pattern=[],
 684                           // Workaround bug bz30254
 685                           int addrKindCopy = addrKind>
 686   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 687     MUBUF_SetupAddr<addrKindCopy> {
 688   let mayStore = 1;
 689   let mayLoad = 1;
 690   let hasPostISelHook = 1;
 691   let hasSideEffects = 1;
 692   let DisableWQM = 1;
 693   let has_glc = 0;
 694   let has_dlc = 0;
 695   let has_tfe = 0;
 696   let has_sccb = 1;
 697   let maybeAtomic = 1;
 698   let AsmMatchConverter = "cvtMubufAtomic";
 699 }
 700
 701 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 702                                RegisterClass vdataClass,
 703                                list<dag> pattern=[],
 704                                // Workaround bug bz30254
 705                                int addrKindCopy = addrKind,
 706                                RegisterClass vdataClassCopy = vdataClass>
 707   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 708                         (outs),
 709                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 710                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol",
 711                         pattern>,
 712     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 713   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 714   let glc_value = 0;
 715   let dlc_value = 0;
 716   let sccb_value = 0;
 717   let IsAtomicNoRet = 1;
 718 }
 719
 720 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 721                              RegisterClass vdataClass,
 722                              list<dag> pattern=[],
 723                              // Workaround bug bz30254
 724                              int addrKindCopy = addrKind,
 725                              RegisterClass vdataClassCopy = vdataClass,
 726                              RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret>
 727   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 728                         (outs vdata_op:$vdata),
 729                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 730                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$cpol",
 731                         pattern>,
 732     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 733   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 734   let glc_value = 1;
 735   let dlc_value = 0;
 736   let sccb_value = 0;
 737   let IsAtomicRet = 1;
 738   let Constraints = "$vdata = $vdata_in";
 739   let DisableEncoding = "$vdata_in";
 740 }
 741
 742 multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
 743                                         RegisterClass vdataClass,
 744                                         ValueType vdataType,
 745                                         SDPatternOperator atomic,
 746                                         bit isFP = isFloatType<vdataType>.ret> {
 747   let FPAtomic = isFP in
 748   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 749                 MUBUFAddr64Table <0, NAME>;
 750
 751   let FPAtomic = isFP in
 752   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 753                 MUBUFAddr64Table <1, NAME>;
 754
 755   let FPAtomic = isFP in
 756   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 757
 758   let FPAtomic = isFP in
 759
 760   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 761
 762   let FPAtomic = isFP in
 763   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 764 }
 765
 766 multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
 767                                      RegisterClass vdataClass,
 768                                      ValueType vdataType,
 769                                      SDPatternOperator atomic,
 770                                      bit isFP = isFloatType<vdataType>.ret> {
 771   let FPAtomic = isFP in
 772   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 773     [(set vdataType:$vdata,
 774      (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset),
 775              vdataType:$vdata_in))]>,
 776     MUBUFAddr64Table <0, NAME # "_RTN">;
 777
 778   let FPAtomic = isFP in
 779   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 780     [(set vdataType:$vdata,
 781      (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset),
 782               vdataType:$vdata_in))]>,
 783     MUBUFAddr64Table <1, NAME # "_RTN">;
 784
 785   let FPAtomic = isFP in
 786   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 787
 788   let FPAtomic = isFP in
 789   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 790
 791   let FPAtomic = isFP in
 792   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 793 }
 794
 795 multiclass MUBUF_Pseudo_Atomics <string opName,
 796                                  RegisterClass vdataClass,
 797                                  ValueType vdataType,
 798                                  SDPatternOperator atomic> :
 799   MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>,
 800   MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
 801
 802
 803 //===----------------------------------------------------------------------===//
 804 // MUBUF Instructions
 805 //===----------------------------------------------------------------------===//
 806
 807 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
 808   "buffer_load_format_x", f32
 809 >;
 810 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 811   "buffer_load_format_xy", v2f32
 812 >;
 813 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 814   "buffer_load_format_xyz", v3f32
 815 >;
 816 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 817   "buffer_load_format_xyzw", v4f32
 818 >;
 819 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 820   "buffer_store_format_x", f32
 821 >;
 822 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 823   "buffer_store_format_xy", v2f32
 824 >;
 825 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 826   "buffer_store_format_xyz", v3f32
 827 >;
 828 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 829   "buffer_store_format_xyzw", v4f32
 830 >;
 831
 832 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
 833   defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
 834     "buffer_load_format_d16_x", i32
 835   >;
 836   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
 837     "buffer_load_format_d16_xy", v2i32
 838   >;
 839   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
 840     "buffer_load_format_d16_xyz", v3i32
 841   >;
 842   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
 843    "buffer_load_format_d16_xyzw", v4i32
 844   >;
 845   defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
 846     "buffer_store_format_d16_x", i32
 847   >;
 848   defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
 849     "buffer_store_format_d16_xy", v2i32
 850   >;
 851   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
 852     "buffer_store_format_d16_xyz", v3i32
 853   >;
 854   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
 855     "buffer_store_format_d16_xyzw", v4i32
 856   >;
 857 } // End HasUnpackedD16VMem.
 858
 859 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
 860   defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
 861     "buffer_load_format_d16_x", f16
 862   >;
 863   defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
 864     "buffer_load_format_d16_xy", v2f16
 865   >;
 866   defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
 867     "buffer_load_format_d16_xyz", v3f16
 868   >;
 869   defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
 870     "buffer_load_format_d16_xyzw", v4f16
 871   >;
 872   defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
 873     "buffer_store_format_d16_x", f16
 874   >;
 875   defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
 876     "buffer_store_format_d16_xy", v2f16
 877   >;
 878   defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
 879     "buffer_store_format_d16_xyz", v3f16
 880   >;
 881   defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
 882     "buffer_store_format_d16_xyzw", v4f16
 883   >;
 884 } // End HasPackedD16VMem.
 885
 886 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
 887   "buffer_load_ubyte", i32
 888 >;
 889 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
 890   "buffer_load_sbyte", i32
 891 >;
 892 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
 893   "buffer_load_ushort", i32
 894 >;
 895 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
 896   "buffer_load_sshort", i32
 897 >;
 898 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
 899   "buffer_load_dword", i32
 900 >;
 901 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 902   "buffer_load_dwordx2", v2i32
 903 >;
 904 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 905   "buffer_load_dwordx3", v3i32
 906 >;
 907 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 908   "buffer_load_dwordx4", v4i32
 909 >;
 910
 911 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
 912 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
 913 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
 914 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
 915 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
 916 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
 917 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>;
 918 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>;
 919 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>;
 920 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>;
 921
 922 // This is not described in AMD documentation,
 923 // but 'lds' versions of these opcodes are available
 924 // in at least GFX8+ chips. See Bug 37653.
 925 let SubtargetPredicate = isGFX8GFX9 in {
 926 defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
 927   "buffer_load_dwordx2", v2i32, null_frag, 0, 1
 928 >;
 929 defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
 930   "buffer_load_dwordx3", v3i32, null_frag, 0, 1
 931 >;
 932 defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
 933   "buffer_load_dwordx4", v4i32, null_frag, 0, 1
 934 >;
 935 }
 936
 937 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 938   "buffer_store_byte", i32, truncstorei8_global
 939 >;
 940 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 941   "buffer_store_short", i32, truncstorei16_global
 942 >;
 943 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 944   "buffer_store_dword", i32, store_global
 945 >;
 946 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 947   "buffer_store_dwordx2", v2i32, store_global
 948 >;
 949 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 950   "buffer_store_dwordx3", v3i32, store_global
 951 >;
 952 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 953   "buffer_store_dwordx4", v4i32, store_global
 954 >;
 955 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 956   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32
 957 >;
 958 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 959   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 960 >;
 961 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 962   "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32
 963 >;
 964 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 965   "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32
 966 >;
 967 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 968   "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32
 969 >;
 970 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 971   "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32
 972 >;
 973 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 974   "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32
 975 >;
 976 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 977   "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32
 978 >;
 979 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 980   "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32
 981 >;
 982 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 983   "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32
 984 >;
 985 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 986   "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32
 987 >;
 988 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 989   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32
 990 >;
 991 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 992   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32
 993 >;
 994 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 995   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64
 996 >;
 997 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 998   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 999 >;
1000 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
1001   "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64
1002 >;
1003 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
1004   "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64
1005 >;
1006 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
1007   "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64
1008 >;
1009 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
1010   "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64
1011 >;
1012 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
1013   "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64
1014 >;
1015 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
1016   "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64
1017 >;
1018 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
1019   "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64
1020 >;
1021 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
1022   "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64
1023 >;
1024 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
1025   "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64
1026 >;
1027 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
1028   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64
1029 >;
1030 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
1031   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64
1032 >;
1033
1034 let SubtargetPredicate = HasGFX10_BEncoding in
1035 defm BUFFER_ATOMIC_CSUB : MUBUF_Pseudo_Atomics_RTN <
1036   "buffer_atomic_csub", VGPR_32, i32, int_amdgcn_global_atomic_csub
1037 >;
1038
1039 let SubtargetPredicate = isGFX8GFX9 in {
1040 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
1041 }
1042
1043 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
1044 /*
1045 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
1046 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
1047 */
1048
1049 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
1050                                           int_amdgcn_buffer_wbinvl1_sc>;
1051 }
1052
1053 let SubtargetPredicate = isGFX6GFX7GFX10 in {
1054
1055 defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <
1056   "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag
1057 >;
1058 defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <
1059   "buffer_atomic_fmin", VGPR_32, f32, null_frag
1060 >;
1061 defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <
1062   "buffer_atomic_fmax", VGPR_32, f32, null_frag
1063 >;
1064 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <
1065   "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag
1066 >;
1067 defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <
1068   "buffer_atomic_fmin_x2", VReg_64, f64, null_frag
1069 >;
1070 defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <
1071   "buffer_atomic_fmax_x2", VReg_64, f64, null_frag
1072 >;
1073
1074 }
1075
1076 let SubtargetPredicate = HasD16LoadStore in {
1077
1078 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
1079   "buffer_load_ubyte_d16", i32, null_frag, 1
1080 >;
1081
1082 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
1083   "buffer_load_ubyte_d16_hi", i32, null_frag, 1
1084 >;
1085
1086 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
1087   "buffer_load_sbyte_d16", i32, null_frag, 1
1088 >;
1089
1090 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
1091   "buffer_load_sbyte_d16_hi", i32, null_frag, 1
1092 >;
1093
1094 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
1095   "buffer_load_short_d16", i32, null_frag, 1
1096 >;
1097
1098 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
1099   "buffer_load_short_d16_hi", i32, null_frag, 1
1100 >;
1101
1102 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
1103   "buffer_store_byte_d16_hi", i32
1104 >;
1105
1106 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
1107   "buffer_store_short_d16_hi", i32
1108 >;
1109
1110 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
1111   "buffer_load_format_d16_hi_x", i32
1112 >;
1113 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
1114   "buffer_store_format_d16_hi_x", i32
1115 >;
1116
1117 } // End HasD16LoadStore
1118
1119 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
1120                                        int_amdgcn_buffer_wbinvl1>;
1121
1122 let SubtargetPredicate = HasAtomicFaddInsts in {
1123 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
1124   "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_noret_32
1125 >;
1126 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
1127   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_noret_32
1128 >;
1129
1130 let OtherPredicates = [isGFX90APlus] in {
1131 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_RTN <
1132   "buffer_atomic_add_f32", VGPR_32, f32, atomic_load_fadd_global_32
1133 >;
1134 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_RTN <
1135   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_load_fadd_v2f16_global_32
1136 >;
1137 }
1138 } // End SubtargetPredicate = HasAtomicFaddInsts
1139
1140 //===----------------------------------------------------------------------===//
1141 // MTBUF Instructions
1142 //===----------------------------------------------------------------------===//
1143
1144 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32,  1>;
1145 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64,  2>;
1146 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96,  3>;
1147 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128, 4>;
1148 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32,  1>;
1149 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64,  2>;
1150 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96,  3>;
1151 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>;
1152
1153 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
1154   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32,  1>;
1155   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64,  2>;
1156   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96,  3>;
1157   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128, 4>;
1158   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32,  1>;
1159   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64,  2>;
1160   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96,  3>;
1161   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>;
1162 } // End HasUnpackedD16VMem.
1163
1164 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
1165   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32, 1>;
1166   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32, 2>;
1167   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64, 3>;
1168   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64, 4>;
1169   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32, 1>;
1170   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32, 2>;
1171   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64, 3>;
1172   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>;
1173 } // End HasPackedD16VMem.
1174
1175 let SubtargetPredicate = isGFX7Plus in {
1176
1177 //===----------------------------------------------------------------------===//
1178 // Instruction definitions for CI and newer.
1179 //===----------------------------------------------------------------------===//
1180
1181 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
1182                                            int_amdgcn_buffer_wbinvl1_vol>;
1183
1184 } // End let SubtargetPredicate = isGFX7Plus
1185
1186 let SubtargetPredicate = isGFX90APlus in {
1187   def BUFFER_WBL2  : MUBUF_Invalidate<"buffer_wbl2"> {
1188   }
1189   def BUFFER_INVL2 : MUBUF_Invalidate<"buffer_invl2"> {
1190   }
1191
1192   defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_add_f64", VReg_64, f64, int_amdgcn_global_atomic_fadd>;
1193   defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_min_f64", VReg_64, f64, int_amdgcn_global_atomic_fmin>;
1194   defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Pseudo_Atomics<"buffer_atomic_max_f64", VReg_64, f64, int_amdgcn_global_atomic_fmax>;
1195 } // End SubtargetPredicate = isGFX90APlus
1196
1197 let SubtargetPredicate = isGFX10Plus in {
1198   def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
1199   def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
1200 } // End SubtargetPredicate = isGFX10Plus
1201
1202 //===----------------------------------------------------------------------===//
1203 // MUBUF Patterns
1204 //===----------------------------------------------------------------------===//
1205
1206 //===----------------------------------------------------------------------===//
1207 // buffer_load/store_format patterns
1208 //===----------------------------------------------------------------------===//
1209
1210 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1211                                   string opcode, ValueType memoryVt = vt> {
1212   defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_load<name, memoryVt>);
1213
1214   def : GCNPat<
1215     (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1216               timm:$auxiliary, 0)),
1217     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1218       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1219   >;
1220
1221   def : GCNPat<
1222     (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1223               timm:$auxiliary, 0)),
1224     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1225       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1226   >;
1227
1228   def : GCNPat<
1229     (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1230               timm:$auxiliary, timm)),
1231     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1232       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1233   >;
1234
1235   def : GCNPat<
1236     (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1237               timm:$auxiliary, timm)),
1238     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1239       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1240       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1241       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1242   >;
1243 }
1244
1245 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
1246 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
1247 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
1248 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
1249 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
1250 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
1251 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
1252 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
1253
1254 let SubtargetPredicate = HasUnpackedD16VMem in {
1255   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1256   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1257   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1258   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1259   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v3i32, "BUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
1260   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1261 } // End HasUnpackedD16VMem.
1262
1263 let SubtargetPredicate = HasPackedD16VMem in {
1264   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
1265   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
1266   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i32, "BUFFER_LOAD_FORMAT_D16_X">;
1267   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
1268   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
1269   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;
1270   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZ", v3i16>;
1271   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1272   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1273 } // End HasPackedD16VMem.
1274
1275 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
1276 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
1277 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">;
1278 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">;
1279 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
1280 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
1281 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">;
1282 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">;
1283 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
1284 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
1285 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
1286 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
1287 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
1288 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
1289 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
1290 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
1291
1292 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1293                                    string opcode, ValueType memoryVt = vt> {
1294   defvar st = !if(!eq(memoryVt, vt), name, mubuf_intrinsic_store<name, memoryVt>);
1295
1296   def : GCNPat<
1297     (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1298               timm:$auxiliary, 0),
1299     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1300       (extract_cpol $auxiliary), 0,  (extract_swz $auxiliary))
1301   >;
1302
1303   def : GCNPat<
1304     (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1305               timm:$auxiliary, 0),
1306     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
1307       (as_i16timm $offset), (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1308   >;
1309
1310   def : GCNPat<
1311     (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1312               timm:$auxiliary, timm),
1313     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
1314       (as_i16timm $offset), (extract_cpol $auxiliary), 0,  (extract_swz $auxiliary))
1315   >;
1316
1317   def : GCNPat<
1318     (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1319               timm:$auxiliary, timm),
1320     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
1321       getVregSrcForVT<vt>.ret:$vdata,
1322       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1323       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (extract_cpol $auxiliary),
1324       0,  (extract_swz $auxiliary))
1325   >;
1326 }
1327
1328 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
1329 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
1330 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
1331 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
1332 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
1333 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
1334 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
1335 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
1336
1337 let SubtargetPredicate = HasUnpackedD16VMem in {
1338   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1339   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1340   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1341   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
1342   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v3i32, "BUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
1343   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1344 } // End HasUnpackedD16VMem.
1345
1346 let SubtargetPredicate = HasPackedD16VMem in {
1347   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
1348   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
1349   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i32, "BUFFER_STORE_FORMAT_D16_X">;
1350   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
1351   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
1352   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZ", v3f16>;
1353   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZ", v3i16>;
1354   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1355   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1356 } // End HasPackedD16VMem.
1357
1358 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
1359 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
1360 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">;
1361 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">;
1362 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
1363 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
1364 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">;
1365 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">;
1366 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
1367 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
1368 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
1369 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
1370 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
1371 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
1372
1373 //===----------------------------------------------------------------------===//
1374 // buffer_atomic patterns
1375 //===----------------------------------------------------------------------===//
1376
1377 multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
1378                                 string opcode> {
1379   def : GCNPat<
1380     (vt (name vt:$vdata_in, v4i32:$rsrc, 0, 0, i32:$soffset,
1381               timm:$offset, timm:$cachepolicy, 0)),
1382     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN)
1383       getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset,
1384       (as_i16timm $offset), (set_glc $cachepolicy))
1385   >;
1386
1387   def : GCNPat<
1388     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset,
1389               timm:$offset, timm:$cachepolicy, timm)),
1390     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in,
1391       VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
1392       (as_i16timm $offset), (set_glc $cachepolicy))
1393   >;
1394
1395   def : GCNPat<
1396     (vt (name vt:$vdata_in, v4i32:$rsrc, 0, i32:$voffset,
1397               i32:$soffset, timm:$offset, timm:$cachepolicy, 0)),
1398     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) getVregSrcForVT<vt>.ret:$vdata_in,
1399       VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
1400       (as_i16timm $offset), (set_glc $cachepolicy))
1401   >;
1402
1403   def : GCNPat<
1404     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex, i32:$voffset,
1405               i32:$soffset, timm:$offset, timm:$cachepolicy, timm)),
1406     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1407       getVregSrcForVT<vt>.ret:$vdata_in,
1408       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1409         SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1410         (set_glc $cachepolicy))
1411   >;
1412 }
1413
1414 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">;
1415 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, f32, "BUFFER_ATOMIC_SWAP">;
1416 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">;
1417 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">;
1418 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">;
1419 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">;
1420 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">;
1421 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">;
1422 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">;
1423 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">;
1424 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">;
1425 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">;
1426 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">;
1427 defm : BufferAtomicPatterns<SIbuffer_atomic_csub, i32, "BUFFER_ATOMIC_CSUB">;
1428 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">;
1429 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64,  "BUFFER_ATOMIC_ADD_X2">;
1430 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">;
1431 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
1432 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
1433 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
1434 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
1435 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
1436 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
1437 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
1438 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">;
1439 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">;
1440
1441 let SubtargetPredicate = isGFX6GFX7GFX10 in {
1442   defm : BufferAtomicPatterns<SIbuffer_atomic_fmin, f32, "BUFFER_ATOMIC_FMIN">;
1443   defm : BufferAtomicPatterns<SIbuffer_atomic_fmax, f32, "BUFFER_ATOMIC_FMAX">;
1444   defm : BufferAtomicPatterns<SIbuffer_atomic_fmin, f64, "BUFFER_ATOMIC_FMIN_X2">;
1445   defm : BufferAtomicPatterns<SIbuffer_atomic_fmax, f64, "BUFFER_ATOMIC_FMAX_X2">;
1446 }
1447
1448 class NoUseBufferAtomic<SDPatternOperator Op, ValueType vt> : PatFrag <
1449   (ops node:$src0, node:$src1, node:$src2, node:$src3, node:$src4, node:$src5, node:$src6, node:$src7),
1450   (vt (Op $src0, $src1, $src2, $src3, $src4, $src5, $src6, $src7)),
1451   [{ return SDValue(N, 0).use_empty(); }]> {
1452
1453   let GISelPredicateCode = [{
1454     return MRI.use_nodbg_empty(MI.getOperand(0).getReg());
1455   }];
1456 }
1457
1458 multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
1459                                        string opcode> {
1460   def : GCNPat<
1461     (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0,
1462                                  0, i32:$soffset, timm:$offset,
1463                                  timm:$cachepolicy, 0),
1464     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) getVregSrcForVT<vt>.ret:$vdata_in, SReg_128:$rsrc, SCSrc_b32:$soffset,
1465                                           (as_i16timm $offset), $cachepolicy)
1466   >;
1467
1468   def : GCNPat<
1469     (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1470                                  0, i32:$soffset, timm:$offset,
1471                                  timm:$cachepolicy, timm),
1472     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
1473                                           (as_i16timm $offset), $cachepolicy)
1474   >;
1475
1476   def : GCNPat<
1477     (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, 0,
1478                                  i32:$voffset, i32:$soffset, timm:$offset,
1479                                  timm:$cachepolicy, 0),
1480     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) getVregSrcForVT<vt>.ret:$vdata_in, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
1481                                           (as_i16timm $offset), $cachepolicy)
1482   >;
1483
1484   def : GCNPat<
1485     (NoUseBufferAtomic<name, vt> vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1486                                  i32:$voffset, i32:$soffset, timm:$offset,
1487                                  timm:$cachepolicy, timm),
1488     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1489       getVregSrcForVT<vt>.ret:$vdata_in,
1490       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1491       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), $cachepolicy)
1492   >;
1493 }
1494
1495 let SubtargetPredicate = HasAtomicFaddInsts in {
1496 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
1497 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1498 }
1499
1500 let SubtargetPredicate = isGFX90APlus in {
1501   defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f32,   "BUFFER_ATOMIC_ADD_F32">;
1502   defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1503
1504   defm : BufferAtomicPatterns<SIbuffer_atomic_fadd, f64, "BUFFER_ATOMIC_ADD_F64">;
1505   defm : BufferAtomicPatterns<SIbuffer_atomic_fmin, f64, "BUFFER_ATOMIC_MIN_F64">;
1506   defm : BufferAtomicPatterns<SIbuffer_atomic_fmax, f64, "BUFFER_ATOMIC_MAX_F64">;
1507 } // End SubtargetPredicate = isGFX90APlus
1508
1509 def : GCNPat<
1510   (SIbuffer_atomic_cmpswap
1511       i32:$data, i32:$cmp, v4i32:$rsrc, 0, 0, i32:$soffset,
1512       timm:$offset, timm:$cachepolicy, 0),
1513   (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS
1514     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1515       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1516         SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1517         (set_glc $cachepolicy)), VReg_64)), sub0)
1518 >;
1519
1520 def : GCNPat<
1521   (SIbuffer_atomic_cmpswap
1522       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1523       0, i32:$soffset, timm:$offset,
1524       timm:$cachepolicy, timm),
1525   (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS
1526     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1527       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1528       VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1529       (set_glc $cachepolicy)), VReg_64)),
1530     sub0)
1531 >;
1532
1533 def : GCNPat<
1534   (SIbuffer_atomic_cmpswap
1535       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1536       i32:$voffset, i32:$soffset, timm:$offset,
1537       timm:$cachepolicy, 0),
1538   (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS
1539     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1540       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1541       VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1542       (set_glc $cachepolicy)), VReg_64)),
1543     sub0)
1544 >;
1545
1546 def : GCNPat<
1547   (SIbuffer_atomic_cmpswap
1548       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1549       i32:$voffset, i32:$soffset, timm:$offset,
1550       timm:$cachepolicy, timm),
1551   (EXTRACT_SUBREG (i64 (COPY_TO_REGCLASS
1552     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1553       (REG_SEQUENCE VReg_64, VGPR_32:$data, sub0, VGPR_32:$cmp, sub1),
1554       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1555       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1556       (set_glc $cachepolicy)), VReg_64)),
1557     sub0)
1558 >;
1559
1560 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1561                               PatFrag constant_ld> : GCNPat <
1562      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1563                                    i16:$offset))),
1564      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
1565   >;
1566
1567 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1568                                      ValueType vt, PatFrag atomic_ld> {
1569   def : GCNPat <
1570      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))),
1571      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset)
1572   >;
1573
1574   def : GCNPat <
1575     (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))),
1576     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset))
1577   >;
1578 }
1579
1580 let SubtargetPredicate = isGFX6GFX7 in {
1581 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1582 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
1583 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
1584 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1585 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
1586 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
1587
1588 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
1589 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
1590 } // End SubtargetPredicate = isGFX6GFX7
1591
1592 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1593                                PatFrag ld> {
1594
1595   def : GCNPat <
1596     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))),
1597     (Instr_OFFSET $srsrc, $soffset, $offset)
1598   >;
1599 }
1600
1601 let OtherPredicates = [Has16BitInsts] in {
1602
1603 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1604 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>;
1605 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>;
1606 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>;
1607 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>;
1608 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>;
1609
1610 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>;
1611
1612 } // End OtherPredicates = [Has16BitInsts]
1613
1614 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1615                                 MUBUF_Pseudo InstrOffset,
1616                                 ValueType vt, PatFrag ld> {
1617   def : GCNPat <
1618     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1619                                i32:$soffset, u16imm:$offset))),
1620     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1621   >;
1622
1623   def : GCNPat <
1624     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1625     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0)
1626   >;
1627 }
1628
1629 // XXX - Is it possible to have a complex pattern in a PatFrag?
1630 multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
1631                                 MUBUF_Pseudo InstrOffset,
1632                                 ValueType vt, PatFrag ld_frag> {
1633   def : GCNPat <
1634     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
1635     (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in)
1636   >;
1637
1638   def : GCNPat <
1639     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
1640     (InstrOffset $srsrc, $soffset, $offset, $in)
1641   >;
1642 }
1643
1644 let OtherPredicates = [DisableFlatScratch] in {
1645 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1646 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1647 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>;
1648 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1649 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1650 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>;
1651 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1652 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1653 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
1654 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1655
1656 foreach vt = Reg32Types.types in {
1657 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, vt, load_private>;
1658 }
1659 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1660 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
1661 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1662
1663 let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in {
1664 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>;
1665 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>;
1666 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>;
1667 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>;
1668 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>;
1669 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>;
1670
1671 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>;
1672 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>;
1673 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>;
1674 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>;
1675 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>;
1676 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>;
1677 }
1678
1679 } // End OtherPredicates = [DisableFlatScratch]
1680
1681 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1682                                       ValueType vt, PatFrag atomic_st> {
1683   // Store follows atomic op convention so address is first
1684   def : GCNPat <
1685      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val),
1686      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset)
1687   >;
1688
1689   def : GCNPat <
1690     (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1691     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset))
1692   >;
1693 }
1694 let SubtargetPredicate = isGFX6GFX7 in {
1695 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>;
1696 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>;
1697 } // End Predicates = isGFX6GFX7
1698
1699
1700 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1701                                PatFrag st> {
1702
1703   def : GCNPat <
1704     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)),
1705     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset)
1706   >;
1707 }
1708
1709 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1710 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1711
1712 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1713                                  MUBUF_Pseudo InstrOffset,
1714                                  ValueType vt, PatFrag st,
1715                                  RegisterClass rc = VGPR_32> {
1716   def : GCNPat <
1717     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1718                                       i32:$soffset, u16imm:$offset)),
1719     (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0)
1720   >;
1721
1722   def : GCNPat <
1723     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1724                                        u16imm:$offset)),
1725     (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0)
1726   >;
1727 }
1728
1729 let OtherPredicates = [DisableFlatScratch] in {
1730 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1731 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1732 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1733 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1734
1735 foreach vt = Reg32Types.types in {
1736 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>;
1737 }
1738
1739 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
1740 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
1741 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
1742
1743
1744 let OtherPredicates = [D16PreservesUnusedBits, DisableFlatScratch] in {
1745  // Hiding the extract high pattern in the PatFrag seems to not
1746  // automatically increase the complexity.
1747 let AddedComplexity = 1 in {
1748 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1749 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1750 }
1751 }
1752 } // End OtherPredicates = [DisableFlatScratch]
1753
1754 //===----------------------------------------------------------------------===//
1755 // MTBUF Patterns
1756 //===----------------------------------------------------------------------===//
1757
1758 //===----------------------------------------------------------------------===//
1759 // tbuffer_load/store_format patterns
1760 //===----------------------------------------------------------------------===//
1761
1762 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1763                                   string opcode, ValueType memoryVt = vt> {
1764   defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_load<name, memoryVt>);
1765
1766   def : GCNPat<
1767     (vt (st v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1768               timm:$format, timm:$auxiliary, 0)),
1769     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1770       (as_i8timm $format),
1771       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1772   >;
1773
1774   def : GCNPat<
1775     (vt (st v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1776               timm:$format, timm:$auxiliary, timm)),
1777     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1778       (as_i8timm $format),
1779       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1780   >;
1781
1782   def : GCNPat<
1783     (vt (st v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1784               timm:$format, timm:$auxiliary, 0)),
1785     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1786       (as_i8timm $format),
1787       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1788   >;
1789
1790   def : GCNPat<
1791     (vt (st v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1792               timm:$format, timm:$auxiliary, timm)),
1793     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1794       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1795       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset),
1796       (as_i8timm $format),
1797       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1798   >;
1799 }
1800
1801 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1802 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1803 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
1804 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1805 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1806 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1807 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
1808 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1809
1810 let SubtargetPredicate = HasUnpackedD16VMem in {
1811   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1812   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1813   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1814   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v3i32, "TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80">;
1815   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1816 } // End HasUnpackedD16VMem.
1817
1818 let SubtargetPredicate = HasPackedD16VMem in {
1819   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
1820   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, i32,   "TBUFFER_LOAD_FORMAT_D16_X">;
1821   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
1822   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZ", v3f16>;
1823   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
1824 } // End HasPackedD16VMem.
1825
1826 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1827                                         string opcode, ValueType memoryVt = vt> {
1828   defvar st = !if(!eq(memoryVt, vt), name, mtbuf_intrinsic_store<name, memoryVt>);
1829
1830   def : GCNPat<
1831     (st vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1832           timm:$format, timm:$auxiliary, 0),
1833     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) getVregSrcForVT<vt>.ret:$vdata, SReg_128:$rsrc, SCSrc_b32:$soffset,
1834       (as_i16timm $offset), (as_i8timm $format),
1835       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1836   >;
1837
1838   def : GCNPat<
1839     (st vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1840           timm:$format, timm:$auxiliary, timm),
1841     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$vindex, SReg_128:$rsrc, SCSrc_b32:$soffset,
1842       (as_i16timm $offset), (as_i8timm $format),
1843       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1844   >;
1845
1846   def : GCNPat<
1847     (st vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1848           timm:$format, timm:$auxiliary, 0),
1849     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) getVregSrcForVT<vt>.ret:$vdata, VGPR_32:$voffset, SReg_128:$rsrc, SCSrc_b32:$soffset,
1850       (as_i16timm $offset), (as_i8timm $format),
1851       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1852   >;
1853
1854   def : GCNPat<
1855     (st vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1856           timm:$offset, timm:$format, timm:$auxiliary, timm),
1857     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1858       getVregSrcForVT<vt>.ret:$vdata,
1859       (REG_SEQUENCE VReg_64, VGPR_32:$vindex, sub0, VGPR_32:$voffset, sub1),
1860       SReg_128:$rsrc, SCSrc_b32:$soffset, (as_i16timm $offset), (as_i8timm $format),
1861       (extract_cpol $auxiliary), 0, (extract_swz $auxiliary))
1862   >;
1863 }
1864
1865 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1866 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1867 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
1868 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1869 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1870 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1871 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
1872 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1873
1874 let SubtargetPredicate = HasUnpackedD16VMem in {
1875   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1876   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1877   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
1878   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v3i32, "TBUFFER_STORE_FORMAT_D16_XYZ_gfx80">;
1879   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1880 } // End HasUnpackedD16VMem.
1881
1882 let SubtargetPredicate = HasPackedD16VMem in {
1883   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
1884   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, i32,   "TBUFFER_STORE_FORMAT_D16_X">;
1885   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
1886   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZ", v3f16>;
1887   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
1888 } // End HasPackedD16VMem.
1889
1890 //===----------------------------------------------------------------------===//
1891 // Target-specific instruction encodings.
1892 //===----------------------------------------------------------------------===//
1893
1894 //===----------------------------------------------------------------------===//
1895 // Base ENC_MUBUF for GFX6, GFX7, GFX10.
1896 //===----------------------------------------------------------------------===//
1897
1898 class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
1899     MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1900   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1901   let Inst{12}    = ps.offen;
1902   let Inst{13}    = ps.idxen;
1903   let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
1904   let Inst{16}    = ps.lds;
1905   let Inst{24-18} = op;
1906   let Inst{31-26} = 0x38;
1907   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1908   let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
1909   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1910   let Inst{54}    = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
1911   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1912   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1913 }
1914
1915 class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> :
1916     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
1917   let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);
1918   let Inst{25} = op{7};
1919 }
1920
1921 class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
1922     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
1923   let Inst{15} = ps.addr64;
1924 }
1925
1926 //===----------------------------------------------------------------------===//
1927 // MUBUF - GFX10.
1928 //===----------------------------------------------------------------------===//
1929
1930 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
1931   multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
1932     def _BOTHEN_gfx10 :
1933       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1934     def _IDXEN_gfx10 :
1935       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1936     def _OFFEN_gfx10 :
1937       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1938     def _OFFSET_gfx10 :
1939       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1940   }
1941   multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> {
1942     def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1943                         MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">;
1944     def _OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1945                         MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">;
1946     def _IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1947                         MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">;
1948     def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1949                         MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">;
1950
1951     def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1952                             MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">;
1953     def _LDS_OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1954                             MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">;
1955     def _LDS_IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1956                             MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">;
1957     def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1958                             MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">;
1959   }
1960   multiclass MUBUF_Real_Atomics_RTN_gfx10<bits<8> op> {
1961     def _BOTHEN_RTN_gfx10 :
1962       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>,
1963       AtomicNoRet<NAME # "_BOTHEN_gfx10", 1>;
1964     def _IDXEN_RTN_gfx10 :
1965       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>,
1966       AtomicNoRet<NAME # "_IDXEN_gfx10", 1>;
1967     def _OFFEN_RTN_gfx10 :
1968       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>,
1969       AtomicNoRet<NAME # "_OFFEN_gfx10", 1>;
1970     def _OFFSET_RTN_gfx10 :
1971       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>,
1972       AtomicNoRet<NAME # "_OFFSET_gfx10", 1>;
1973   }
1974   multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
1975       MUBUF_Real_Atomics_RTN_gfx10<op> {
1976     def _BOTHEN_gfx10 :
1977       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1978       AtomicNoRet<NAME # "_BOTHEN_gfx10", 0>;
1979     def _IDXEN_gfx10 :
1980       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1981       AtomicNoRet<NAME # "_IDXEN_gfx10", 0>;
1982     def _OFFEN_gfx10 :
1983       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1984       AtomicNoRet<NAME # "_OFFEN_gfx10", 0>;
1985     def _OFFSET_gfx10 :
1986       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1987       AtomicNoRet<NAME # "_OFFSET_gfx10", 0>;
1988   }
1989 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
1990
1991 defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x019>;
1992 defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx10<0x01b>;
1993 defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x020>;
1994 defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x021>;
1995 defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x022>;
1996 defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x023>;
1997 defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx10<0x024>;
1998 defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x025>;
1999 // FIXME-GFX10: Add following instructions:
2000 //defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx10<0x026>;
2001 //defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
2002 defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx10<0x080>;
2003 defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx10<0x081>;
2004 defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx10<0x082>;
2005 defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx10<0x083>;
2006 defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx10<0x084>;
2007 defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx10<0x085>;
2008 defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx10<0x086>;
2009 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
2010
2011 def BUFFER_GL0_INV_gfx10 :
2012   MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>;
2013 def BUFFER_GL1_INV_gfx10 :
2014   MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>;
2015
2016 //===----------------------------------------------------------------------===//
2017 // MUBUF - GFX6, GFX7, GFX10.
2018 //===----------------------------------------------------------------------===//
2019
2020 let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in {
2021   multiclass MUBUF_Real_gfx6<bits<8> op> {
2022     def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
2023   }
2024 } // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6"
2025
2026 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
2027   multiclass MUBUF_Real_gfx7<bits<8> op> {
2028     def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
2029   }
2030 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
2031
2032 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2033   multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
2034     def _ADDR64_gfx6_gfx7 :
2035       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
2036     def _BOTHEN_gfx6_gfx7 :
2037       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2038     def _IDXEN_gfx6_gfx7 :
2039       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2040     def _OFFEN_gfx6_gfx7 :
2041       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2042     def _OFFSET_gfx6_gfx7 :
2043       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2044   }
2045   multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> {
2046     def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2047                             MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">;
2048     def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
2049                             MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">;
2050     def _OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2051                             MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">;
2052     def _IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2053                             MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">;
2054     def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2055                             MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">;
2056
2057     def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2058                                 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">;
2059     def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>,
2060                                 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">;
2061     def _LDS_OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2062                                 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">;
2063     def _LDS_IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2064                                 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">;
2065     def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2066                                 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">;
2067   }
2068   multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> {
2069     def _ADDR64_gfx6_gfx7 :
2070       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
2071       AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 0>;
2072     def _BOTHEN_gfx6_gfx7 :
2073       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2074       AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 0>;
2075     def _IDXEN_gfx6_gfx7 :
2076       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2077       AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 0>;
2078     def _OFFEN_gfx6_gfx7 :
2079       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2080       AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 0>;
2081     def _OFFSET_gfx6_gfx7 :
2082       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2083       AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 0>;
2084
2085     def _ADDR64_RTN_gfx6_gfx7 :
2086       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>,
2087       AtomicNoRet<NAME # "_ADDR64_gfx6_gfx7", 1>;
2088     def _BOTHEN_RTN_gfx6_gfx7 :
2089       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>,
2090       AtomicNoRet<NAME # "_BOTHEN_gfx6_gfx7", 1>;
2091     def _IDXEN_RTN_gfx6_gfx7 :
2092       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>,
2093       AtomicNoRet<NAME # "_IDXEN_gfx6_gfx7", 1>;
2094     def _OFFEN_RTN_gfx6_gfx7 :
2095       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>,
2096       AtomicNoRet<NAME # "_OFFEN_gfx6_gfx7", 1>;
2097     def _OFFSET_RTN_gfx6_gfx7 :
2098       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>,
2099       AtomicNoRet<NAME # "_OFFSET_gfx6_gfx7", 1>;
2100   }
2101 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2102
2103 multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
2104   MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
2105
2106 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> :
2107   MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>;
2108
2109 multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
2110   MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
2111
2112 // FIXME-GFX6: Following instructions are available only on GFX6.
2113 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomics_gfx6 <0x034>;
2114 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomics_gfx6 <0x054>;
2115
2116 defm BUFFER_LOAD_FORMAT_X     : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
2117 defm BUFFER_LOAD_FORMAT_XY    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2118 defm BUFFER_LOAD_FORMAT_XYZ   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2119 defm BUFFER_LOAD_FORMAT_XYZW  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2120 defm BUFFER_STORE_FORMAT_X    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2121 defm BUFFER_STORE_FORMAT_XY   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2122 defm BUFFER_STORE_FORMAT_XYZ  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2123 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2124 defm BUFFER_LOAD_UBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
2125 defm BUFFER_LOAD_SBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
2126 defm BUFFER_LOAD_USHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
2127 defm BUFFER_LOAD_SSHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
2128 defm BUFFER_LOAD_DWORD        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
2129 defm BUFFER_LOAD_DWORDX2      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
2130 defm BUFFER_LOAD_DWORDX4      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
2131 defm BUFFER_LOAD_DWORDX3      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
2132 defm BUFFER_STORE_BYTE        : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
2133 defm BUFFER_STORE_SHORT       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
2134 defm BUFFER_STORE_DWORD       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
2135 defm BUFFER_STORE_DWORDX2     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
2136 defm BUFFER_STORE_DWORDX4     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
2137 defm BUFFER_STORE_DWORDX3     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
2138
2139 defm BUFFER_ATOMIC_SWAP        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
2140 defm BUFFER_ATOMIC_CMPSWAP     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
2141 defm BUFFER_ATOMIC_ADD         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
2142 defm BUFFER_ATOMIC_SUB         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
2143 defm BUFFER_ATOMIC_SMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
2144 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
2145 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
2146 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
2147 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
2148 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
2149 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
2150 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
2151 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
2152 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
2153 defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
2154 defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
2155 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
2156 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
2157 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
2158 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
2159 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
2160 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
2161 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
2162 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
2163 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
2164 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
2165 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
2166 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
2167 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
2168 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
2169 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
2170 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
2171 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
2172
2173 defm BUFFER_ATOMIC_CSUB       : MUBUF_Real_Atomics_RTN_gfx10<0x034>;
2174
2175 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
2176 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
2177 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
2178
2179 //===----------------------------------------------------------------------===//
2180 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
2181 //===----------------------------------------------------------------------===//
2182
2183 class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
2184     MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
2185   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2186   let Inst{12}    = ps.offen;
2187   let Inst{13}    = ps.idxen;
2188   let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
2189   let Inst{18-16} = op;
2190   let Inst{31-26} = 0x3a; //encoding
2191   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2192   let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
2193   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2194   let Inst{54}    = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
2195   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2196   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2197 }
2198
2199 //===----------------------------------------------------------------------===//
2200 // MTBUF - GFX10.
2201 //===----------------------------------------------------------------------===//
2202
2203 class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> :
2204     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
2205   let Inst{15} = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlc_value);
2206   let Inst{25-19} = format;
2207   let Inst{53} = op{3};
2208 }
2209
2210 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
2211   multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
2212     def _BOTHEN_gfx10 :
2213       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2214     def _IDXEN_gfx10 :
2215       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2216     def _OFFEN_gfx10 :
2217       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2218     def _OFFSET_gfx10 :
2219       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2220   }
2221 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
2222
2223 defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx10<0x008>;
2224 defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx10<0x009>;
2225 defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx10<0x00a>;
2226 defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx10<0x00b>;
2227 defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx10<0x00c>;
2228 defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx10<0x00d>;
2229 defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx10<0x00e>;
2230 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
2231
2232 //===----------------------------------------------------------------------===//
2233 // MTBUF - GFX6, GFX7, GFX10.
2234 //===----------------------------------------------------------------------===//
2235
2236 class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> :
2237     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
2238   let Inst{15} = ps.addr64;
2239   let Inst{22-19} = dfmt;
2240   let Inst{25-23} = nfmt;
2241 }
2242
2243 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2244   multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
2245     def _ADDR64_gfx6_gfx7 :
2246       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
2247     def _BOTHEN_gfx6_gfx7 :
2248       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2249     def _IDXEN_gfx6_gfx7 :
2250       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2251     def _OFFEN_gfx6_gfx7 :
2252       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2253     def _OFFSET_gfx6_gfx7 :
2254       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2255   }
2256 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2257
2258 multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
2259   MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
2260
2261 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
2262 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2263 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2264 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2265 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2266 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2267 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2268 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2269
2270 //===----------------------------------------------------------------------===//
2271 // GFX8, GFX9 (VI).
2272 //===----------------------------------------------------------------------===//
2273
2274 class MUBUF_Real_Base_vi <bits<7> op, MUBUF_Pseudo ps, int Enc,
2275                           bit has_sccb = ps.has_sccb> :
2276   MUBUF_Real<ps>,
2277   Enc64,
2278   SIMCInstr<ps.PseudoInstr, Enc>,
2279   AtomicNoRet<!subst("_RTN","",NAME), !if(ps.IsAtomicNoRet, 0,
2280                                         !if(ps.IsAtomicRet, 1, ?))> {
2281
2282   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2283   let Inst{12}    = ps.offen;
2284   let Inst{13}    = ps.idxen;
2285   let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
2286   let Inst{15}    = !if(has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);
2287   let Inst{16}    = ps.lds;
2288   let Inst{17}    = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
2289   let Inst{24-18} = op;
2290   let Inst{31-26} = 0x38; //encoding
2291   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2292   let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
2293   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2294   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2295 }
2296
2297 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps, bit has_sccb = ps.has_sccb> :
2298   MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.VI, has_sccb> {
2299   let AssemblerPredicate = isGFX8GFX9NotGFX90A;
2300   let DecoderNamespace = "GFX8";
2301
2302   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2303 }
2304
2305 class MUBUF_Real_gfx90a <bits<7> op, MUBUF_Pseudo ps,
2306                          bit has_sccb = ps.has_sccb> :
2307   MUBUF_Real_Base_vi<op, ps, SIEncodingFamily.GFX90A, has_sccb> {
2308   let AssemblerPredicate = isGFX90APlus;
2309   let DecoderNamespace = "GFX90A";
2310   let AsmString = ps.Mnemonic # !subst("$sccb", !if(has_sccb, "$sccb",""),
2311                                 !subst("$tfe", "", ps.AsmOperands));
2312
2313   let Inst{55}    = acc;
2314 }
2315
2316 multiclass MUBUF_Real_vi_gfx90a<bits<7> op, MUBUF_Pseudo ps> {
2317   def _vi :     MUBUF_Real_vi<op, ps>;
2318   def _gfx90a : MUBUF_Real_gfx90a<op, ps, !and(ps.has_sccb,!not(ps.FPAtomic))>;
2319 }
2320
2321 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
2322   defm _OFFSET : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2323   defm _OFFEN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2324   defm _IDXEN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2325   defm _BOTHEN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2326 }
2327
2328 multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
2329
2330   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2331                    MUBUFLdsTable<0, NAME # "_OFFSET_vi">;
2332   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2333                    MUBUFLdsTable<0, NAME # "_OFFEN_vi">;
2334   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2335                    MUBUFLdsTable<0, NAME # "_IDXEN_vi">;
2336   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2337                    MUBUFLdsTable<0, NAME # "_BOTHEN_vi">;
2338
2339   def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2340                        MUBUFLdsTable<1, NAME # "_OFFSET_vi">;
2341   def _LDS_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2342                        MUBUFLdsTable<1, NAME # "_OFFEN_vi">;
2343   def _LDS_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2344                        MUBUFLdsTable<1, NAME # "_IDXEN_vi">;
2345   def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2346                        MUBUFLdsTable<1, NAME # "_BOTHEN_vi">;
2347
2348   def _OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2349                    MUBUFLdsTable<0, NAME # "_OFFSET_gfx90a">;
2350   def _OFFEN_gfx90a  : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2351                    MUBUFLdsTable<0, NAME # "_OFFEN_gfx90a">;
2352   def _IDXEN_gfx90a  : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2353                    MUBUFLdsTable<0, NAME # "_IDXEN_gfx90a">;
2354   def _BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2355                    MUBUFLdsTable<0, NAME # "_BOTHEN_gfx90a">;
2356
2357   def _LDS_OFFSET_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2358                        MUBUFLdsTable<1, NAME # "_OFFSET_gfx90a">;
2359   def _LDS_OFFEN_gfx90a  : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2360                        MUBUFLdsTable<1, NAME # "_OFFEN_gfx90a">;
2361   def _LDS_IDXEN_gfx90a  : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2362                        MUBUFLdsTable<1, NAME # "_IDXEN_gfx90a">;
2363   def _LDS_BOTHEN_gfx90a : MUBUF_Real_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2364                        MUBUFLdsTable<1, NAME # "_BOTHEN_gfx90a">;
2365 }
2366
2367 class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> :
2368   MUBUF_Real<ps>,
2369   Enc64,
2370   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2371   let AssemblerPredicate=HasUnpackedD16VMem;
2372   let DecoderNamespace="GFX80_UNPACKED";
2373
2374   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2375   let Inst{12}    = ps.offen;
2376   let Inst{13}    = ps.idxen;
2377   let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
2378   let Inst{16}    = ps.lds;
2379   let Inst{17}    = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
2380   let Inst{24-18} = op;
2381   let Inst{31-26} = 0x38; //encoding
2382   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2383   let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
2384   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2385   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2386   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2387 }
2388
2389 multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
2390   def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2391   def _OFFEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2392   def _IDXEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2393   def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2394 }
2395
2396 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
2397   MUBUF_Real_AllAddr_vi<op> {
2398   defm _OFFSET_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
2399   defm _OFFEN_RTN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
2400   defm _IDXEN_RTN  : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
2401   defm _BOTHEN_RTN : MUBUF_Real_vi_gfx90a <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
2402 }
2403
2404 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_Lds_vi <0x00>;
2405 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
2406 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
2407 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
2408 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
2409 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
2410 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
2411 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
2412 let SubtargetPredicate = HasUnpackedD16VMem in {
2413   defm BUFFER_LOAD_FORMAT_D16_X_gfx80       : MUBUF_Real_AllAddr_gfx80 <0x08>;
2414   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x09>;
2415   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0a>;
2416   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0b>;
2417   defm BUFFER_STORE_FORMAT_D16_X_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x0c>;
2418   defm BUFFER_STORE_FORMAT_D16_XY_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0d>;
2419   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0e>;
2420   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80   : MUBUF_Real_AllAddr_gfx80 <0x0f>;
2421 } // End HasUnpackedD16VMem.
2422 let SubtargetPredicate = HasPackedD16VMem in {
2423   defm BUFFER_LOAD_FORMAT_D16_X       : MUBUF_Real_AllAddr_vi <0x08>;
2424   defm BUFFER_LOAD_FORMAT_D16_XY      : MUBUF_Real_AllAddr_vi <0x09>;
2425   defm BUFFER_LOAD_FORMAT_D16_XYZ     : MUBUF_Real_AllAddr_vi <0x0a>;
2426   defm BUFFER_LOAD_FORMAT_D16_XYZW    : MUBUF_Real_AllAddr_vi <0x0b>;
2427   defm BUFFER_STORE_FORMAT_D16_X      : MUBUF_Real_AllAddr_vi <0x0c>;
2428   defm BUFFER_STORE_FORMAT_D16_XY     : MUBUF_Real_AllAddr_vi <0x0d>;
2429   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
2430   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
2431 } // End HasPackedD16VMem.
2432 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
2433 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
2434 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
2435 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
2436 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
2437 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
2438 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
2439 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
2440 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
2441 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
2442 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
2443 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
2444 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
2445 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
2446 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
2447 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
2448
2449 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
2450 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
2451 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
2452 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
2453 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
2454 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
2455
2456 defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_vi <0x26>;
2457 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
2458
2459 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
2460 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
2461 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
2462 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
2463 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
2464 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
2465 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
2466 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
2467 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
2468 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
2469 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
2470 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
2471 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
2472
2473 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
2474 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
2475 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
2476 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
2477 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
2478 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
2479 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
2480 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
2481 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
2482 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
2483 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
2484 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
2485 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
2486
2487 defm BUFFER_STORE_LDS_DWORD     : MUBUF_Real_vi_gfx90a <0x3d, BUFFER_STORE_LDS_DWORD>;
2488
2489 let AssemblerPredicate = isGFX8GFX9 in {
2490 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
2491 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
2492 } // End AssemblerPredicate = isGFX8GFX9
2493
2494 let SubtargetPredicate = HasAtomicFaddInsts in {
2495
2496 defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_Atomic_vi <0x4d>;
2497 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_Atomic_vi <0x4e>;
2498
2499 } // End SubtargetPredicate = HasAtomicFaddInsts
2500
2501 let SubtargetPredicate = isGFX90APlus in {
2502   defm BUFFER_ATOMIC_ADD_F64 : MUBUF_Real_Atomic_vi<0x4f>;
2503   defm BUFFER_ATOMIC_MIN_F64 : MUBUF_Real_Atomic_vi<0x50>;
2504   defm BUFFER_ATOMIC_MAX_F64 : MUBUF_Real_Atomic_vi<0x51>;
2505 } // End SubtargetPredicate = isGFX90APlus, AssemblerPredicate = isGFX90APlus
2506
2507 def BUFFER_WBL2_gfx90a  : MUBUF_Real_gfx90a<0x28, BUFFER_WBL2> {
2508 }
2509 def BUFFER_INVL2_gfx90a : MUBUF_Real_gfx90a<0x29, BUFFER_INVL2>;
2510
2511 class MTBUF_Real_Base_vi <bits<4> op, MTBUF_Pseudo ps, int Enc> :
2512   MTBUF_Real<ps>,
2513   Enc64,
2514   SIMCInstr<ps.PseudoInstr, Enc> {
2515
2516   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2517   let Inst{12}    = ps.offen;
2518   let Inst{13}    = ps.idxen;
2519   let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
2520   let Inst{18-15} = op;
2521   let Inst{22-19} = dfmt;
2522   let Inst{25-23} = nfmt;
2523   let Inst{31-26} = 0x3a; //encoding
2524   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2525   let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
2526   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2527   let Inst{53}    = !if(ps.has_sccb, cpol{CPolBit.SCC}, ps.sccb_value);
2528   let Inst{54}    = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
2529   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2530   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2531 }
2532
2533 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
2534   MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.VI> {
2535   let AssemblerPredicate = isGFX8GFX9NotGFX90A;
2536   let DecoderNamespace = "GFX8";
2537
2538   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2539 }
2540
2541 class MTBUF_Real_gfx90a <bits<4> op, MTBUF_Pseudo ps> :
2542   MTBUF_Real_Base_vi <op, ps, SIEncodingFamily.GFX90A> {
2543   let AssemblerPredicate = isGFX90APlus;
2544   let DecoderNamespace = "GFX90A";
2545   let AsmString = ps.Mnemonic # !subst("$tfe", "", ps.AsmOperands);
2546
2547   let Inst{55}    = acc;
2548 }
2549
2550 multiclass MTBUF_Real_vi_gfx90a<bits<4> op, MTBUF_Pseudo ps> {
2551   def _vi :     MTBUF_Real_vi<op, ps>;
2552   def _gfx90a : MTBUF_Real_gfx90a<op, ps>;
2553 }
2554
2555 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
2556   defm _OFFSET : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2557   defm _OFFEN  : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2558   defm _IDXEN  : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2559   defm _BOTHEN : MTBUF_Real_vi_gfx90a <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2560 }
2561
2562 class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> :
2563   MTBUF_Real<ps>,
2564   Enc64,
2565   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2566   let AssemblerPredicate=HasUnpackedD16VMem;
2567   let DecoderNamespace="GFX80_UNPACKED";
2568
2569   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2570   let Inst{12}    = ps.offen;
2571   let Inst{13}    = ps.idxen;
2572   let Inst{14}    = !if(ps.has_glc, cpol{CPolBit.GLC}, ps.glc_value);
2573   let Inst{18-15} = op;
2574   let Inst{22-19} = dfmt;
2575   let Inst{25-23} = nfmt;
2576   let Inst{31-26} = 0x3a; //encoding
2577   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2578   let Inst{47-40} = !if(ps.has_vdata, vdata{7-0}, ?);
2579   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2580   let Inst{54}    = !if(ps.has_slc, cpol{CPolBit.SLC}, ?);
2581   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2582   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2583 }
2584
2585 multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
2586   def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2587   def _OFFEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2588   def _IDXEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2589   def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2590 }
2591
2592 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0x00>;
2593 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <0x01>;
2594 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <0x02>;
2595 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <0x03>;
2596 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <0x04>;
2597 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <0x05>;
2598 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <0x06>;
2599 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
2600 let SubtargetPredicate = HasUnpackedD16VMem in {
2601   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Real_AllAddr_gfx80 <0x08>;
2602   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x09>;
2603   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0a>;
2604   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0b>;
2605   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x0c>;
2606   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0d>;
2607   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0e>;
2608   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
2609 } // End HasUnpackedD16VMem.
2610 let SubtargetPredicate = HasPackedD16VMem in {
2611   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_vi <0x08>;
2612   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_vi <0x09>;
2613   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_vi <0x0a>;
2614   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_vi <0x0b>;
2615   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_vi <0x0c>;
2616   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_vi <0x0d>;
2617   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_vi <0x0e>;
2618   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
2619 } // End HasUnpackedD16VMem.
2620
2621 def MUBUFInfoTable : GenericTable {
2622   let FilterClass = "MUBUF_Pseudo";
2623   let CppTypeName = "MUBUFInfo";
2624   let Fields = [
2625     "Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset",
2626     "IsBufferInv"
2627   ];
2628
2629   let PrimaryKey = ["Opcode"];
2630   let PrimaryKeyName = "getMUBUFOpcodeHelper";
2631 }
2632
2633 def getMUBUFInfoFromOpcode : SearchIndex {
2634   let Table = MUBUFInfoTable;
2635   let Key = ["Opcode"];
2636 }
2637
2638 def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2639   let Table = MUBUFInfoTable;
2640   let Key = ["BaseOpcode", "elements"];
2641 }
2642
2643 def MTBUFInfoTable : GenericTable {
2644   let FilterClass = "MTBUF_Pseudo";
2645   let CppTypeName = "MTBUFInfo";
2646   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2647
2648   let PrimaryKey = ["Opcode"];
2649   let PrimaryKeyName = "getMTBUFOpcodeHelper";
2650 }
2651
2652 def getMTBUFInfoFromOpcode : SearchIndex {
2653   let Table = MTBUFInfoTable;
2654   let Key = ["Opcode"];
2655 }
2656
2657 def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2658   let Table = MTBUFInfoTable;
2659   let Key = ["BaseOpcode", "elements"];
2660 }