lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  10 def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">;
  11 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  12
  13 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  14 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  15
  16 def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">;
  17 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  18 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  19
  20 def BUFAddrKind {
  21   int Offset = 0;
  22   int OffEn  = 1;
  23   int IdxEn  = 2;
  24   int BothEn = 3;
  25   int Addr64 = 4;
  26 }
  27
  28 class getAddrName<int addrKind> {
  29   string ret =
  30     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  31     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  32     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  33     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  34     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  35     "")))));
  36 }
  37
  38 class MUBUFAddr64Table <bit is_addr64, string Name> {
  39   bit IsAddr64 = is_addr64;
  40   string OpName = Name;
  41 }
  42
  43 class MUBUFLdsTable <bit is_lds, string Name> {
  44   bit IsLds = is_lds;
  45   string OpName = Name;
  46 }
  47
  48 class MTBUFAddr64Table <bit is_addr64, string Name> {
  49   bit IsAddr64 = is_addr64;
  50   string OpName = Name;
  51 }
  52
  53 //===----------------------------------------------------------------------===//
  54 // MTBUF classes
  55 //===----------------------------------------------------------------------===//
  56
  57 class MTBUFGetBaseOpcode<string Op> {
  58   string ret = !subst("FORMAT_XY", "FORMAT_X",
  59     !subst("FORMAT_XYZ", "FORMAT_X",
  60     !subst("FORMAT_XYZW", "FORMAT_X", Op)));
  61 }
  62
  63 class getMTBUFElements<string Op> {
  64   int ret = 1;
  65 }
  66
  67
  68 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  69                     string asmOps, list<dag> pattern=[]> :
  70   InstSI<outs, ins, "", pattern>,
  71   SIMCInstr<opName, SIEncodingFamily.NONE> {
  72
  73   let isPseudo = 1;
  74   let isCodeGenOnly = 1;
  75   let Size = 8;
  76   let UseNamedOperandTable = 1;
  77
  78   string Mnemonic = opName;
  79   string AsmOperands = asmOps;
  80
  81   Instruction Opcode = !cast<Instruction>(NAME);
  82   Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret);
  83
  84   let VM_CNT = 1;
  85   let EXP_CNT = 1;
  86   let MTBUF = 1;
  87   let Uses = [EXEC];
  88   let hasSideEffects = 0;
  89   let SchedRW = [WriteVMEM];
  90
  91   let AsmMatchConverter = "cvtMtbuf";
  92
  93   bits<1> offen       = 0;
  94   bits<1> idxen       = 0;
  95   bits<1> addr64      = 0;
  96   bits<1> has_vdata   = 1;
  97   bits<1> has_vaddr   = 1;
  98   bits<1> has_glc     = 1;
  99   bits<1> has_dlc     = 1;
 100   bits<1> glc_value   = 0; // the value for glc if no such operand
 101   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 102   bits<1> has_srsrc   = 1;
 103   bits<1> has_soffset = 1;
 104   bits<1> has_offset  = 1;
 105   bits<1> has_slc     = 1;
 106   bits<1> has_tfe     = 1;
 107   bits<4> elements    = 0;
 108 }
 109
 110 class MTBUF_Real <MTBUF_Pseudo ps> :
 111   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 112
 113   let isPseudo = 0;
 114   let isCodeGenOnly = 0;
 115
 116   // copy relevant pseudo op flags
 117   let SubtargetPredicate = ps.SubtargetPredicate;
 118   let AsmMatchConverter  = ps.AsmMatchConverter;
 119   let Constraints        = ps.Constraints;
 120   let DisableEncoding    = ps.DisableEncoding;
 121   let TSFlags            = ps.TSFlags;
 122
 123   bits<12> offset;
 124   bits<1>  glc;
 125   bits<1>  dlc;
 126   bits<7>  format;
 127   bits<8>  vaddr;
 128   bits<8>  vdata;
 129   bits<7>  srsrc;
 130   bits<1>  slc;
 131   bits<1>  tfe;
 132   bits<8>  soffset;
 133
 134   bits<4> dfmt = format{3-0};
 135   bits<3> nfmt = format{6-4};
 136 }
 137
 138 class getMTBUFInsDA<list<RegisterClass> vdataList,
 139                     list<RegisterClass> vaddrList=[]> {
 140   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 141   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 142   dag InsNoData = !if(!empty(vaddrList),
 143     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 144          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
 145     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 146          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
 147   );
 148   dag InsData = !if(!empty(vaddrList),
 149     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 150          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 151          SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
 152     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 153          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 154          SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
 155   );
 156   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 157 }
 158
 159 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 160   dag ret =
 161     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 162     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 163     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 164     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 165     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 166     (ins))))));
 167 }
 168
 169 class getMTBUFAsmOps<int addrKind> {
 170   string Pfx =
 171     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset",
 172     !if(!eq(addrKind, BUFAddrKind.OffEn),
 173             "$vaddr, $srsrc, $format, $soffset offen",
 174     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 175             "$vaddr, $srsrc, $format, $soffset idxen",
 176     !if(!eq(addrKind, BUFAddrKind.BothEn),
 177             "$vaddr, $srsrc, $format, $soffset idxen offen",
 178     !if(!eq(addrKind, BUFAddrKind.Addr64),
 179             "$vaddr, $srsrc, $format, $soffset addr64",
 180     "")))));
 181   string ret = Pfx # "$offset";
 182 }
 183
 184 class MTBUF_SetupAddr<int addrKind> {
 185   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 186                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 187
 188   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 189                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 190
 191   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 192
 193   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 194 }
 195
 196 class MTBUF_Load_Pseudo <string opName,
 197                          int addrKind,
 198                          RegisterClass vdataClass,
 199                          int elems,
 200                          list<dag> pattern=[],
 201                          // Workaround bug bz30254
 202                          int addrKindCopy = addrKind>
 203   : MTBUF_Pseudo<opName,
 204                  (outs vdataClass:$vdata),
 205                  getMTBUFIns<addrKindCopy>.ret,
 206                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 207                  pattern>,
 208     MTBUF_SetupAddr<addrKindCopy> {
 209   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 210   let mayLoad = 1;
 211   let mayStore = 0;
 212   let elements = elems;
 213 }
 214
 215 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 216                               int elems, ValueType load_vt = i32,
 217                               SDPatternOperator ld = null_frag> {
 218
 219   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
 220     [(set load_vt:$vdata,
 221      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format,
 222                       i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
 223     MTBUFAddr64Table<0, NAME>;
 224
 225   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
 226     [(set load_vt:$vdata,
 227      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 228                       i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
 229     MTBUFAddr64Table<1, NAME>;
 230
 231   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 232   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 233   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 234
 235   let DisableWQM = 1 in {
 236     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 237     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 238     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 239     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 240   }
 241 }
 242
 243 class MTBUF_Store_Pseudo <string opName,
 244                           int addrKind,
 245                           RegisterClass vdataClass,
 246                           int elems,
 247                           list<dag> pattern=[],
 248                           // Workaround bug bz30254
 249                           int addrKindCopy = addrKind,
 250                           RegisterClass vdataClassCopy = vdataClass>
 251   : MTBUF_Pseudo<opName,
 252                  (outs),
 253                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 254                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 255                  pattern>,
 256     MTBUF_SetupAddr<addrKindCopy> {
 257   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 258   let mayLoad = 0;
 259   let mayStore = 1;
 260   let elements = elems;
 261 }
 262
 263 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 264                                int elems, ValueType store_vt = i32,
 265                                SDPatternOperator st = null_frag> {
 266
 267   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
 268     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 269                                        i16:$offset, i8:$format, i1:$glc,
 270                                        i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 271     MTBUFAddr64Table<0, NAME>;
 272
 273   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
 274     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 275                                        i16:$offset, i8:$format, i1:$glc,
 276                                        i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 277     MTBUFAddr64Table<1, NAME>;
 278
 279   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 280   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 281   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 282
 283   let DisableWQM = 1 in {
 284     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 285     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 286     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 287     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 288   }
 289 }
 290
 291
 292 //===----------------------------------------------------------------------===//
 293 // MUBUF classes
 294 //===----------------------------------------------------------------------===//
 295
 296 class MUBUFGetBaseOpcode<string Op> {
 297   string ret = !subst("DWORDX2", "DWORD",
 298     !subst("DWORDX3", "DWORD",
 299     !subst("DWORDX4", "DWORD", Op)));
 300 }
 301
 302 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 303                     string asmOps, list<dag> pattern=[]> :
 304   InstSI<outs, ins, "", pattern>,
 305   SIMCInstr<opName, SIEncodingFamily.NONE> {
 306
 307   let isPseudo = 1;
 308   let isCodeGenOnly = 1;
 309   let Size = 8;
 310   let UseNamedOperandTable = 1;
 311
 312   string Mnemonic = opName;
 313   string AsmOperands = asmOps;
 314
 315   Instruction Opcode = !cast<Instruction>(NAME);
 316   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
 317
 318   let VM_CNT = 1;
 319   let EXP_CNT = 1;
 320   let MUBUF = 1;
 321   let Uses = [EXEC];
 322   let hasSideEffects = 0;
 323   let SchedRW = [WriteVMEM];
 324
 325   let AsmMatchConverter = "cvtMubuf";
 326
 327   bits<1> offen       = 0;
 328   bits<1> idxen       = 0;
 329   bits<1> addr64      = 0;
 330   bits<1> lds         = 0;
 331   bits<1> has_vdata   = 1;
 332   bits<1> has_vaddr   = 1;
 333   bits<1> has_glc     = 1;
 334   bits<1> has_dlc     = 1;
 335   bits<1> glc_value   = 0; // the value for glc if no such operand
 336   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 337   bits<1> has_srsrc   = 1;
 338   bits<1> has_soffset = 1;
 339   bits<1> has_offset  = 1;
 340   bits<1> has_slc     = 1;
 341   bits<1> has_tfe     = 1;
 342   bits<4> elements    = 0;
 343 }
 344
 345 class MUBUF_Real <MUBUF_Pseudo ps> :
 346   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 347
 348   let isPseudo = 0;
 349   let isCodeGenOnly = 0;
 350
 351   // copy relevant pseudo op flags
 352   let SubtargetPredicate = ps.SubtargetPredicate;
 353   let AsmMatchConverter  = ps.AsmMatchConverter;
 354   let Constraints        = ps.Constraints;
 355   let DisableEncoding    = ps.DisableEncoding;
 356   let TSFlags            = ps.TSFlags;
 357
 358   bits<12> offset;
 359   bits<1>  glc;
 360   bits<1>  dlc;
 361   bits<8>  vaddr;
 362   bits<8>  vdata;
 363   bits<7>  srsrc;
 364   bits<1>  slc;
 365   bits<1>  tfe;
 366   bits<8>  soffset;
 367 }
 368
 369
 370 // For cache invalidation instructions.
 371 class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
 372   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 373
 374   let AsmMatchConverter = "";
 375
 376   let hasSideEffects = 1;
 377   let mayStore = 1;
 378
 379   // Set everything to 0.
 380   let offen       = 0;
 381   let idxen       = 0;
 382   let addr64      = 0;
 383   let has_vdata   = 0;
 384   let has_vaddr   = 0;
 385   let has_glc     = 0;
 386   let has_dlc     = 0;
 387   let glc_value   = 0;
 388   let dlc_value   = 0;
 389   let has_srsrc   = 0;
 390   let has_soffset = 0;
 391   let has_offset  = 0;
 392   let has_slc     = 0;
 393   let has_tfe     = 0;
 394 }
 395
 396 class getMUBUFInsDA<list<RegisterClass> vdataList,
 397                     list<RegisterClass> vaddrList=[],
 398                     bit isLds = 0> {
 399   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 400   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 401   dag InsNoData = !if(!empty(vaddrList),
 402     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 403          offset:$offset, GLC:$glc, SLC:$slc),
 404     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 405          offset:$offset, GLC:$glc, SLC:$slc)
 406   );
 407   dag InsData = !if(!empty(vaddrList),
 408     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 409          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 410     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 411          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc)
 412   );
 413   dag ret = !con(
 414               !if(!empty(vdataList), InsNoData, InsData),
 415               !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz))
 416              );
 417 }
 418
 419 class getMUBUFElements<ValueType vt> {
 420   // eq does not support ValueType for some reason.
 421   string vtAsStr = !cast<string>(vt);
 422
 423   int ret =
 424     !if(!eq(vtAsStr, "f16"), 1,
 425       !if(!eq(vtAsStr, "v2f16"), 2,
 426         !if(!eq(vtAsStr, "v3f16"), 3,
 427           !if(!eq(vtAsStr, "v4f16"), 4,
 428             !if(!eq(vt.Size, 32), 1,
 429               !if(!eq(vt.Size, 64), 2,
 430                 !if(!eq(vt.Size, 96), 3,
 431                   !if(!eq(vt.Size, 128), 4, 0)
 432                 )
 433               )
 434             )
 435           )
 436         )
 437       )
 438     );
 439 }
 440
 441 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> {
 442   dag ret =
 443     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret,
 444     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 445     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 446     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 447     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 448     (ins))))));
 449 }
 450
 451 class getMUBUFAsmOps<int addrKind> {
 452   string Pfx =
 453     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 454     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 455     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 456     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 457     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 458     "")))));
 459   string ret = Pfx # "$offset";
 460 }
 461
 462 class MUBUF_SetupAddr<int addrKind> {
 463   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 464                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 465
 466   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 467                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 468
 469   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 470
 471   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 472 }
 473
 474 class MUBUF_Load_Pseudo <string opName,
 475                          int addrKind,
 476                          ValueType vdata_vt,
 477                          bit HasTiedDest = 0,
 478                          bit isLds = 0,
 479                          list<dag> pattern=[],
 480                          // Workaround bug bz30254
 481                          int addrKindCopy = addrKind>
 482   : MUBUF_Pseudo<opName,
 483                  (outs getVregSrcForVT<vdata_vt>.ret:$vdata),
 484                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
 485                       !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))),
 486                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" #
 487                    !if(isLds, " lds", "$tfe") # "$dlc" # "$swz",
 488                  pattern>,
 489     MUBUF_SetupAddr<addrKindCopy> {
 490   let PseudoInstr = opName # !if(isLds, "_lds", "") #
 491                     "_" # getAddrName<addrKindCopy>.ret;
 492   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 493
 494   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 495   let mayLoad = 1;
 496   let mayStore = 0;
 497   let maybeAtomic = 1;
 498   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
 499   let has_tfe = !if(isLds, 0, 1);
 500   let lds = isLds;
 501   let elements = getMUBUFElements<vdata_vt>.ret;
 502 }
 503
 504 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
 505   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
 506   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 507 >;
 508
 509 class MUBUF_Addr64_Load_Pat <Instruction inst,
 510                             ValueType load_vt = i32,
 511                             SDPatternOperator ld = null_frag> : Pat <
 512   (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
 513   (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 514 >;
 515
 516 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
 517   def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
 518   def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
 519 }
 520
 521
 522 // FIXME: tfe can't be an operand because it requires a separate
 523 // opcode because it needs an N+1 register class dest register.
 524 multiclass MUBUF_Pseudo_Loads<string opName,
 525                               ValueType load_vt = i32,
 526                               SDPatternOperator ld = null_frag,
 527                               bit TiedDest = 0,
 528                               bit isLds = 0> {
 529
 530   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>,
 531     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 532
 533   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>,
 534     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 535
 536   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 537   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 538   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 539
 540   let DisableWQM = 1 in {
 541     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>;
 542     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 543     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 544     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 545   }
 546 }
 547
 548 multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32,
 549                                   SDPatternOperator ld_nolds = null_frag,
 550                                   SDPatternOperator ld_lds = null_frag> {
 551   defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>;
 552   defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>;
 553 }
 554
 555 class MUBUF_Store_Pseudo <string opName,
 556                           int addrKind,
 557                           ValueType store_vt,
 558                           list<dag> pattern=[],
 559                           // Workaround bug bz30254
 560                           int addrKindCopy = addrKind>
 561   : MUBUF_Pseudo<opName,
 562                  (outs),
 563                  getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret,
 564                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 565                  pattern>,
 566     MUBUF_SetupAddr<addrKindCopy> {
 567   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 568   let mayLoad = 0;
 569   let mayStore = 1;
 570   let maybeAtomic = 1;
 571   let elements = getMUBUFElements<store_vt>.ret;
 572 }
 573
 574 multiclass MUBUF_Pseudo_Stores<string opName,
 575                                ValueType store_vt = i32,
 576                                SDPatternOperator st = null_frag> {
 577
 578   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt,
 579     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 580                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 581     MUBUFAddr64Table<0, NAME>;
 582
 583   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt,
 584     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 585                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 586     MUBUFAddr64Table<1, NAME>;
 587
 588   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 589   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 590   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 591
 592   let DisableWQM = 1 in {
 593     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>;
 594     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 595     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 596     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 597   }
 598 }
 599
 600 class MUBUF_Pseudo_Store_Lds<string opName>
 601   : MUBUF_Pseudo<opName,
 602                  (outs),
 603                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz),
 604                  " $srsrc, $soffset$offset lds$glc$slc$swz"> {
 605   let mayLoad = 0;
 606   let mayStore = 1;
 607   let maybeAtomic = 1;
 608
 609   let has_vdata = 0;
 610   let has_vaddr = 0;
 611   let has_tfe = 0;
 612   let lds = 1;
 613
 614   let Uses = [EXEC, M0];
 615   let AsmMatchConverter = "cvtMubufLds";
 616 }
 617
 618 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 619                           list<RegisterClass> vaddrList=[]> {
 620   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 621   dag ret = !if(vdata_in,
 622     !if(!empty(vaddrList),
 623       (ins vdataClass:$vdata_in,
 624            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 625       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 626            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 627     ),
 628     !if(!empty(vaddrList),
 629       (ins vdataClass:$vdata,
 630            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 631       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 632            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 633   ));
 634 }
 635
 636 class getMUBUFAtomicIns<int addrKind,
 637                         RegisterClass vdataClass,
 638                         bit vdata_in,
 639                         // Workaround bug bz30254
 640                         RegisterClass vdataClassCopy=vdataClass> {
 641   dag ret =
 642     !if(!eq(addrKind, BUFAddrKind.Offset),
 643             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 644     !if(!eq(addrKind, BUFAddrKind.OffEn),
 645             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 646     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 647             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 648     !if(!eq(addrKind, BUFAddrKind.BothEn),
 649             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 650     !if(!eq(addrKind, BUFAddrKind.Addr64),
 651             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 652     (ins))))));
 653 }
 654
 655 class MUBUF_Atomic_Pseudo<string opName,
 656                           int addrKind,
 657                           dag outs,
 658                           dag ins,
 659                           string asmOps,
 660                           list<dag> pattern=[],
 661                           // Workaround bug bz30254
 662                           int addrKindCopy = addrKind>
 663   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 664     MUBUF_SetupAddr<addrKindCopy> {
 665   let mayStore = 1;
 666   let mayLoad = 1;
 667   let hasPostISelHook = 1;
 668   let hasSideEffects = 1;
 669   let DisableWQM = 1;
 670   let has_glc = 0;
 671   let has_dlc = 0;
 672   let has_tfe = 0;
 673   let maybeAtomic = 1;
 674 }
 675
 676 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 677                                RegisterClass vdataClass,
 678                                list<dag> pattern=[],
 679                                // Workaround bug bz30254
 680                                int addrKindCopy = addrKind,
 681                                RegisterClass vdataClassCopy = vdataClass>
 682   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 683                         (outs),
 684                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 685                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 686                         pattern>,
 687     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 688   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 689   let glc_value = 0;
 690   let dlc_value = 0;
 691   let AsmMatchConverter = "cvtMubufAtomic";
 692 }
 693
 694 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 695                              RegisterClass vdataClass,
 696                              list<dag> pattern=[],
 697                              // Workaround bug bz30254
 698                              int addrKindCopy = addrKind,
 699                              RegisterClass vdataClassCopy = vdataClass>
 700   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 701                         (outs vdataClassCopy:$vdata),
 702                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 703                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 704                         pattern>,
 705     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 706   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 707   let glc_value = 1;
 708   let dlc_value = 0;
 709   let Constraints = "$vdata = $vdata_in";
 710   let DisableEncoding = "$vdata_in";
 711   let AsmMatchConverter = "cvtMubufAtomicReturn";
 712 }
 713
 714 multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
 715                                         RegisterClass vdataClass,
 716                                         ValueType vdataType,
 717                                         SDPatternOperator atomic,
 718                                         bit isFP = isFloatType<vdataType>.ret> {
 719   let FPAtomic = isFP in
 720   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 721                 MUBUFAddr64Table <0, NAME>;
 722
 723   let FPAtomic = isFP in
 724   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 725                 MUBUFAddr64Table <1, NAME>;
 726
 727   let FPAtomic = isFP in
 728   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 729
 730   let FPAtomic = isFP in
 731
 732   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 733
 734   let FPAtomic = isFP in
 735   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 736 }
 737
 738 multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
 739                                      RegisterClass vdataClass,
 740                                      ValueType vdataType,
 741                                      SDPatternOperator atomic,
 742                                      bit isFP = isFloatType<vdataType>.ret> {
 743   let FPAtomic = isFP in
 744   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 745     [(set vdataType:$vdata,
 746      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 747              vdataType:$vdata_in))]>,
 748     MUBUFAddr64Table <0, NAME # "_RTN">;
 749
 750   let FPAtomic = isFP in
 751   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 752     [(set vdataType:$vdata,
 753      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 754              vdataType:$vdata_in))]>,
 755     MUBUFAddr64Table <1, NAME # "_RTN">;
 756
 757   let FPAtomic = isFP in
 758   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 759
 760   let FPAtomic = isFP in
 761   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 762
 763   let FPAtomic = isFP in
 764   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 765 }
 766
 767 multiclass MUBUF_Pseudo_Atomics <string opName,
 768                                  RegisterClass vdataClass,
 769                                  ValueType vdataType,
 770                                  SDPatternOperator atomic> :
 771   MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>,
 772   MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
 773
 774
 775 //===----------------------------------------------------------------------===//
 776 // MUBUF Instructions
 777 //===----------------------------------------------------------------------===//
 778
 779 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
 780   "buffer_load_format_x", f32
 781 >;
 782 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 783   "buffer_load_format_xy", v2f32
 784 >;
 785 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 786   "buffer_load_format_xyz", v3f32
 787 >;
 788 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 789   "buffer_load_format_xyzw", v4f32
 790 >;
 791 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 792   "buffer_store_format_x", f32
 793 >;
 794 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 795   "buffer_store_format_xy", v2f32
 796 >;
 797 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 798   "buffer_store_format_xyz", v3f32
 799 >;
 800 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 801   "buffer_store_format_xyzw", v4f32
 802 >;
 803
 804 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
 805   defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
 806     "buffer_load_format_d16_x", i32
 807   >;
 808   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
 809     "buffer_load_format_d16_xy", v2i32
 810   >;
 811   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
 812     "buffer_load_format_d16_xyz", v3i32
 813   >;
 814   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
 815    "buffer_load_format_d16_xyzw", v4i32
 816   >;
 817   defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
 818     "buffer_store_format_d16_x", i32
 819   >;
 820   defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
 821     "buffer_store_format_d16_xy", v2i32
 822   >;
 823   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
 824     "buffer_store_format_d16_xyz", v3i32
 825   >;
 826   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
 827     "buffer_store_format_d16_xyzw", v4i32
 828   >;
 829 } // End HasUnpackedD16VMem.
 830
 831 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
 832   defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
 833     "buffer_load_format_d16_x", f16
 834   >;
 835   defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
 836     "buffer_load_format_d16_xy", v2f16
 837   >;
 838   defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
 839     "buffer_load_format_d16_xyz", v3f16
 840   >;
 841   defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
 842     "buffer_load_format_d16_xyzw", v4f16
 843   >;
 844   defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
 845     "buffer_store_format_d16_x", f16
 846   >;
 847   defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
 848     "buffer_store_format_d16_xy", v2f16
 849   >;
 850   defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
 851     "buffer_store_format_d16_xyz", v3f16
 852   >;
 853   defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
 854     "buffer_store_format_d16_xyzw", v4f16
 855   >;
 856 } // End HasPackedD16VMem.
 857
 858 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
 859   "buffer_load_ubyte", i32
 860 >;
 861 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
 862   "buffer_load_sbyte", i32
 863 >;
 864 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
 865   "buffer_load_ushort", i32
 866 >;
 867 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
 868   "buffer_load_sshort", i32
 869 >;
 870 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
 871   "buffer_load_dword", i32
 872 >;
 873 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 874   "buffer_load_dwordx2", v2i32
 875 >;
 876 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 877   "buffer_load_dwordx3", v3i32
 878 >;
 879 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 880   "buffer_load_dwordx4", v4i32
 881 >;
 882
 883 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
 884 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
 885 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
 886 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
 887 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
 888 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
 889 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>;
 890 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>;
 891 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>;
 892 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>;
 893
 894 // This is not described in AMD documentation,
 895 // but 'lds' versions of these opcodes are available
 896 // in at least GFX8+ chips. See Bug 37653.
 897 let SubtargetPredicate = isGFX8GFX9 in {
 898 defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
 899   "buffer_load_dwordx2", v2i32, null_frag, 0, 1
 900 >;
 901 defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
 902   "buffer_load_dwordx3", v3i32, null_frag, 0, 1
 903 >;
 904 defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
 905   "buffer_load_dwordx4", v4i32, null_frag, 0, 1
 906 >;
 907 }
 908
 909 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 910   "buffer_store_byte", i32, truncstorei8_global
 911 >;
 912 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 913   "buffer_store_short", i32, truncstorei16_global
 914 >;
 915 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 916   "buffer_store_dword", i32, store_global
 917 >;
 918 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 919   "buffer_store_dwordx2", v2i32, store_global
 920 >;
 921 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 922   "buffer_store_dwordx3", v3i32, store_global
 923 >;
 924 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 925   "buffer_store_dwordx4", v4i32, store_global
 926 >;
 927 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 928   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32
 929 >;
 930 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 931   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 932 >;
 933 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 934   "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32
 935 >;
 936 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 937   "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32
 938 >;
 939 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 940   "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32
 941 >;
 942 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 943   "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32
 944 >;
 945 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 946   "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32
 947 >;
 948 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 949   "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32
 950 >;
 951 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 952   "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32
 953 >;
 954 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 955   "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32
 956 >;
 957 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 958   "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32
 959 >;
 960 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 961   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32
 962 >;
 963 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 964   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32
 965 >;
 966 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 967   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64
 968 >;
 969 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 970   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 971 >;
 972 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 973   "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64
 974 >;
 975 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 976   "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64
 977 >;
 978 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 979   "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64
 980 >;
 981 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 982   "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64
 983 >;
 984 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 985   "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64
 986 >;
 987 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 988   "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64
 989 >;
 990 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 991   "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64
 992 >;
 993 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 994   "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64
 995 >;
 996 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 997   "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64
 998 >;
 999 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
1000   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64
1001 >;
1002 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
1003   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64
1004 >;
1005
1006 let SubtargetPredicate = isGFX8GFX9 in {
1007 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
1008 }
1009
1010 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
1011 /*
1012 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
1013 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
1014 */
1015
1016 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
1017                                           int_amdgcn_buffer_wbinvl1_sc>;
1018 }
1019
1020 let SubtargetPredicate = isGFX6GFX7GFX10 in {
1021
1022 defm BUFFER_ATOMIC_FCMPSWAP : MUBUF_Pseudo_Atomics <
1023   "buffer_atomic_fcmpswap", VReg_64, v2f32, null_frag
1024 >;
1025 defm BUFFER_ATOMIC_FMIN : MUBUF_Pseudo_Atomics <
1026   "buffer_atomic_fmin", VGPR_32, f32, null_frag
1027 >;
1028 defm BUFFER_ATOMIC_FMAX : MUBUF_Pseudo_Atomics <
1029   "buffer_atomic_fmax", VGPR_32, f32, null_frag
1030 >;
1031 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <
1032   "buffer_atomic_fcmpswap_x2", VReg_128, v2f64, null_frag
1033 >;
1034 defm BUFFER_ATOMIC_FMIN_X2 : MUBUF_Pseudo_Atomics <
1035   "buffer_atomic_fmin_x2", VReg_64, f64, null_frag
1036 >;
1037 defm BUFFER_ATOMIC_FMAX_X2 : MUBUF_Pseudo_Atomics <
1038   "buffer_atomic_fmax_x2", VReg_64, f64, null_frag
1039 >;
1040
1041 }
1042
1043 let SubtargetPredicate = HasD16LoadStore in {
1044
1045 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
1046   "buffer_load_ubyte_d16", i32, null_frag, 1
1047 >;
1048
1049 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
1050   "buffer_load_ubyte_d16_hi", i32, null_frag, 1
1051 >;
1052
1053 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
1054   "buffer_load_sbyte_d16", i32, null_frag, 1
1055 >;
1056
1057 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
1058   "buffer_load_sbyte_d16_hi", i32, null_frag, 1
1059 >;
1060
1061 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
1062   "buffer_load_short_d16", i32, null_frag, 1
1063 >;
1064
1065 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
1066   "buffer_load_short_d16_hi", i32, null_frag, 1
1067 >;
1068
1069 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
1070   "buffer_store_byte_d16_hi", i32
1071 >;
1072
1073 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
1074   "buffer_store_short_d16_hi", i32
1075 >;
1076
1077 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
1078   "buffer_load_format_d16_hi_x", i32
1079 >;
1080 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
1081   "buffer_store_format_d16_hi_x", i32
1082 >;
1083
1084 } // End HasD16LoadStore
1085
1086 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
1087                                        int_amdgcn_buffer_wbinvl1>;
1088
1089 let SubtargetPredicate = HasAtomicFaddInsts in {
1090
1091 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
1092   "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret
1093 >;
1094 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
1095   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret
1096 >;
1097
1098 } // End SubtargetPredicate = HasAtomicFaddInsts
1099
1100 //===----------------------------------------------------------------------===//
1101 // MTBUF Instructions
1102 //===----------------------------------------------------------------------===//
1103
1104 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32,  1>;
1105 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64,  2>;
1106 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96,  3>;
1107 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128, 4>;
1108 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32,  1>;
1109 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64,  2>;
1110 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96,  3>;
1111 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>;
1112
1113 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
1114   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32,  1>;
1115   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64,  2>;
1116   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96,  3>;
1117   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128, 4>;
1118   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32,  1>;
1119   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64,  2>;
1120   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96,  3>;
1121   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>;
1122 } // End HasUnpackedD16VMem.
1123
1124 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
1125   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32, 1>;
1126   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32, 2>;
1127   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64, 3>;
1128   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64, 4>;
1129   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32, 1>;
1130   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32, 2>;
1131   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64, 3>;
1132   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>;
1133 } // End HasPackedD16VMem.
1134
1135 let SubtargetPredicate = isGFX7Plus in {
1136
1137 //===----------------------------------------------------------------------===//
1138 // Instruction definitions for CI and newer.
1139 //===----------------------------------------------------------------------===//
1140
1141 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
1142                                            int_amdgcn_buffer_wbinvl1_vol>;
1143
1144 } // End let SubtargetPredicate = isGFX7Plus
1145
1146 let SubtargetPredicate = isGFX10Plus in {
1147   def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
1148   def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
1149 } // End SubtargetPredicate = isGFX10Plus
1150
1151 //===----------------------------------------------------------------------===//
1152 // MUBUF Patterns
1153 //===----------------------------------------------------------------------===//
1154
1155 def extract_glc : SDNodeXForm<imm, [{
1156   return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8);
1157 }]>;
1158
1159 def extract_slc : SDNodeXForm<imm, [{
1160   return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8);
1161 }]>;
1162
1163 def extract_dlc : SDNodeXForm<imm, [{
1164   return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8);
1165 }]>;
1166
1167 def extract_swz : SDNodeXForm<imm, [{
1168   return CurDAG->getTargetConstant((N->getZExtValue() >> 3) & 1, SDLoc(N), MVT::i8);
1169 }]>;
1170
1171 //===----------------------------------------------------------------------===//
1172 // buffer_load/store_format patterns
1173 //===----------------------------------------------------------------------===//
1174
1175 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1176                                   string opcode> {
1177   def : GCNPat<
1178     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1179               timm:$auxiliary, 0)),
1180     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1181       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1182       (extract_swz $auxiliary))
1183   >;
1184
1185   def : GCNPat<
1186     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1187               timm:$auxiliary, 0)),
1188     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1189       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1190       (extract_swz $auxiliary))
1191   >;
1192
1193   def : GCNPat<
1194     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1195               timm:$auxiliary, timm)),
1196     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1197       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1198       (extract_swz $auxiliary))
1199   >;
1200
1201   def : GCNPat<
1202     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1203               timm:$auxiliary, timm)),
1204     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1205       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1206       $rsrc, $soffset, (as_i16imm $offset),
1207       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1208       (extract_swz $auxiliary))
1209   >;
1210 }
1211
1212 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
1213 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
1214 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
1215 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
1216 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
1217 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
1218 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
1219 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
1220
1221 let SubtargetPredicate = HasUnpackedD16VMem in {
1222   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1223   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1224   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1225   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1226 } // End HasUnpackedD16VMem.
1227
1228 let SubtargetPredicate = HasPackedD16VMem in {
1229   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
1230   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
1231   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
1232   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
1233   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1234   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1235 } // End HasPackedD16VMem.
1236
1237 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
1238 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
1239 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">;
1240 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">;
1241 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
1242 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
1243 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">;
1244 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">;
1245 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
1246 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
1247 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
1248 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
1249 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
1250 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
1251 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
1252 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
1253
1254 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1255                                    string opcode> {
1256   def : GCNPat<
1257     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1258               timm:$auxiliary, 0),
1259     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
1260       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1261       (extract_swz $auxiliary))
1262   >;
1263
1264   def : GCNPat<
1265     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1266               timm:$auxiliary, 0),
1267     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1268       (as_i16imm $offset), (extract_glc $auxiliary),
1269       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1270       (extract_swz $auxiliary))
1271   >;
1272
1273   def : GCNPat<
1274     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1275               timm:$auxiliary, timm),
1276     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1277       (as_i16imm $offset), (extract_glc $auxiliary),
1278       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1279       (extract_swz $auxiliary))
1280   >;
1281
1282   def : GCNPat<
1283     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1284               timm:$auxiliary, timm),
1285     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
1286       $vdata,
1287       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1288       $rsrc, $soffset, (as_i16imm $offset), (extract_glc $auxiliary),
1289       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1290       (extract_swz $auxiliary))
1291   >;
1292 }
1293
1294 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
1295 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
1296 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
1297 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
1298 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
1299 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
1300 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
1301 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
1302
1303 let SubtargetPredicate = HasUnpackedD16VMem in {
1304   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1305   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1306   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
1307   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1308 } // End HasUnpackedD16VMem.
1309
1310 let SubtargetPredicate = HasPackedD16VMem in {
1311   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
1312   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
1313   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
1314   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
1315   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1316   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1317 } // End HasPackedD16VMem.
1318
1319 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
1320 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
1321 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">;
1322 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">;
1323 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
1324 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
1325 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">;
1326 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">;
1327 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
1328 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
1329 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
1330 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
1331 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
1332 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
1333
1334 //===----------------------------------------------------------------------===//
1335 // buffer_atomic patterns
1336 //===----------------------------------------------------------------------===//
1337
1338 multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
1339                                 string opcode> {
1340   def : GCNPat<
1341     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1342           0, i32:$soffset, timm:$offset,
1343           timm:$cachepolicy, 0)),
1344     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset,
1345                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1346   >;
1347
1348   def : GCNPat<
1349     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1350           0, i32:$soffset, timm:$offset,
1351           timm:$cachepolicy, timm)),
1352     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset,
1353                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1354   >;
1355
1356   def : GCNPat<
1357     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1358           i32:$voffset, i32:$soffset, timm:$offset,
1359           timm:$cachepolicy, 0)),
1360     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset,
1361                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1362   >;
1363
1364   def : GCNPat<
1365     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1366           i32:$voffset, i32:$soffset, timm:$offset,
1367           timm:$cachepolicy, timm)),
1368     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1369       $vdata_in,
1370       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1371       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1372   >;
1373 }
1374
1375 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">;
1376 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">;
1377 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">;
1378 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">;
1379 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">;
1380 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">;
1381 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">;
1382 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">;
1383 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">;
1384 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">;
1385 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">;
1386 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">;
1387 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">;
1388 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64,  "BUFFER_ATOMIC_ADD_X2">;
1389 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">;
1390 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
1391 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
1392 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
1393 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
1394 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
1395 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
1396 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
1397 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">;
1398 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">;
1399
1400 multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
1401                                        string opcode> {
1402   def : GCNPat<
1403     (name vt:$vdata_in, v4i32:$rsrc, 0,
1404           0, i32:$soffset, timm:$offset,
1405           timm:$cachepolicy, 0),
1406     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
1407                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1408   >;
1409
1410   def : GCNPat<
1411     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1412           0, i32:$soffset, timm:$offset,
1413           timm:$cachepolicy, timm),
1414     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
1415                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1416   >;
1417
1418   def : GCNPat<
1419     (name vt:$vdata_in, v4i32:$rsrc, 0,
1420           i32:$voffset, i32:$soffset, timm:$offset,
1421           timm:$cachepolicy, 0),
1422     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
1423                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1424   >;
1425
1426   def : GCNPat<
1427     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1428           i32:$voffset, i32:$soffset, timm:$offset,
1429           timm:$cachepolicy, timm),
1430     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1431       $vdata_in,
1432       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1433       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1434   >;
1435 }
1436
1437 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
1438 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1439
1440 def : GCNPat<
1441   (SIbuffer_atomic_cmpswap
1442       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1443       0, i32:$soffset, timm:$offset,
1444       timm:$cachepolicy, 0),
1445   (EXTRACT_SUBREG
1446     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1447       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1448       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1449     sub0)
1450 >;
1451
1452 def : GCNPat<
1453   (SIbuffer_atomic_cmpswap
1454       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1455       0, i32:$soffset, timm:$offset,
1456       timm:$cachepolicy, timm),
1457   (EXTRACT_SUBREG
1458     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1459       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1460       $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1461     sub0)
1462 >;
1463
1464 def : GCNPat<
1465   (SIbuffer_atomic_cmpswap
1466       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1467       i32:$voffset, i32:$soffset, timm:$offset,
1468       timm:$cachepolicy, 0),
1469   (EXTRACT_SUBREG
1470     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1471       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1472       $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1473     sub0)
1474 >;
1475
1476 def : GCNPat<
1477   (SIbuffer_atomic_cmpswap
1478       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1479       i32:$voffset, i32:$soffset, timm:$offset,
1480       timm:$cachepolicy, timm),
1481   (EXTRACT_SUBREG
1482     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1483       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1484       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1485       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1486     sub0)
1487 >;
1488
1489 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1490                               PatFrag constant_ld> : GCNPat <
1491      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1492                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
1493      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1494   >;
1495
1496 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1497                                      ValueType vt, PatFrag atomic_ld> {
1498   def : GCNPat <
1499      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1500                                    i16:$offset, i1:$slc))),
1501      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
1502   >;
1503
1504   def : GCNPat <
1505     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1506     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
1507   >;
1508 }
1509
1510 let SubtargetPredicate = isGFX6GFX7 in {
1511 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1512 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
1513 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
1514 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1515 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
1516 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
1517
1518 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
1519 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
1520 } // End SubtargetPredicate = isGFX6GFX7
1521
1522 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1523                                PatFrag ld> {
1524
1525   def : GCNPat <
1526     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1527                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
1528     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1529   >;
1530 }
1531
1532 let OtherPredicates = [Has16BitInsts] in {
1533
1534 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1535 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>;
1536 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>;
1537 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>;
1538 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>;
1539 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>;
1540
1541 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>;
1542
1543 } // End OtherPredicates = [Has16BitInsts]
1544
1545 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1546                                 MUBUF_Pseudo InstrOffset,
1547                                 ValueType vt, PatFrag ld> {
1548   def : GCNPat <
1549     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1550                                i32:$soffset, u16imm:$offset))),
1551     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1552   >;
1553
1554   def : GCNPat <
1555     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1556     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1557   >;
1558 }
1559
1560 // XXX - Is it possible to have a complex pattern in a PatFrag?
1561 multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
1562                                 MUBUF_Pseudo InstrOffset,
1563                                 ValueType vt, PatFrag ld_frag> {
1564   def : GCNPat <
1565     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
1566     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
1567   >;
1568
1569   def : GCNPat <
1570     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
1571     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
1572   >;
1573 }
1574
1575 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1576 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1577 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>;
1578 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1579 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1580 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>;
1581 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1582 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1583 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
1584 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1585
1586 foreach vt = Reg32Types.types in {
1587 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1588 }
1589 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1590 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
1591 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1592
1593 let OtherPredicates = [D16PreservesUnusedBits] in {
1594 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>;
1595 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>;
1596 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>;
1597 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>;
1598 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>;
1599 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>;
1600
1601 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>;
1602 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>;
1603 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>;
1604 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>;
1605 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>;
1606 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>;
1607 }
1608
1609 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1610                                       ValueType vt, PatFrag atomic_st> {
1611   // Store follows atomic op convention so address is first
1612   def : GCNPat <
1613      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1614                                    i16:$offset, i1:$slc), vt:$val),
1615      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
1616   >;
1617
1618   def : GCNPat <
1619     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1620     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
1621   >;
1622 }
1623 let SubtargetPredicate = isGFX6GFX7 in {
1624 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>;
1625 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>;
1626 } // End Predicates = isGFX6GFX7
1627
1628
1629 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1630                                PatFrag st> {
1631
1632   def : GCNPat <
1633     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1634                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)),
1635     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1636   >;
1637 }
1638
1639 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1640 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1641
1642 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1643                                  MUBUF_Pseudo InstrOffset,
1644                                  ValueType vt, PatFrag st,
1645                                  RegisterClass rc = VGPR_32> {
1646   def : GCNPat <
1647     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1648                                       i32:$soffset, u16imm:$offset)),
1649     (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1650   >;
1651
1652   def : GCNPat <
1653     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1654                                        u16imm:$offset)),
1655     (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1656   >;
1657 }
1658
1659 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1660 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1661 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1662 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1663
1664 foreach vt = Reg32Types.types in {
1665 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>;
1666 }
1667
1668 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
1669 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
1670 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
1671
1672
1673 let OtherPredicates = [D16PreservesUnusedBits] in {
1674  // Hiding the extract high pattern in the PatFrag seems to not
1675  // automatically increase the complexity.
1676 let AddedComplexity = 1 in {
1677 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1678 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1679 }
1680 }
1681
1682 //===----------------------------------------------------------------------===//
1683 // MTBUF Patterns
1684 //===----------------------------------------------------------------------===//
1685
1686 //===----------------------------------------------------------------------===//
1687 // tbuffer_load/store_format patterns
1688 //===----------------------------------------------------------------------===//
1689
1690 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1691                                   string opcode> {
1692   def : GCNPat<
1693     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1694               timm:$format, timm:$auxiliary, 0)),
1695     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1696       (as_i8imm $format),
1697       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1698       (extract_swz $auxiliary))
1699   >;
1700
1701   def : GCNPat<
1702     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1703               timm:$format, timm:$auxiliary, timm)),
1704     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1705       (as_i8imm $format),
1706       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1707       (extract_swz $auxiliary))
1708   >;
1709
1710   def : GCNPat<
1711     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1712               timm:$format, timm:$auxiliary, 0)),
1713     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1714       (as_i8imm $format),
1715       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1716       (extract_swz $auxiliary))
1717   >;
1718
1719   def : GCNPat<
1720     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1721               timm:$format, timm:$auxiliary, timm)),
1722     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1723       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1724       $rsrc, $soffset, (as_i16imm $offset),
1725       (as_i8imm $format),
1726       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1727       (extract_swz $auxiliary))
1728   >;
1729 }
1730
1731 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1732 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1733 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
1734 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1735 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1736 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1737 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
1738 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1739
1740 let SubtargetPredicate = HasUnpackedD16VMem in {
1741   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1742   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1743   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1744 } // End HasUnpackedD16VMem.
1745
1746 let SubtargetPredicate = HasPackedD16VMem in {
1747   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
1748   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
1749   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
1750 } // End HasPackedD16VMem.
1751
1752 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1753                                    string opcode> {
1754   def : GCNPat<
1755     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1756           timm:$format, timm:$auxiliary, 0),
1757     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1758       (as_i16imm $offset), (as_i8imm $format),
1759       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1760       (extract_swz $auxiliary))
1761   >;
1762
1763   def : GCNPat<
1764     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1765           timm:$format, timm:$auxiliary, timm),
1766     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1767       (as_i16imm $offset), (as_i8imm $format),
1768       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1769       (extract_swz $auxiliary))
1770   >;
1771
1772   def : GCNPat<
1773     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1774           timm:$format, timm:$auxiliary, 0),
1775     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1776       (as_i16imm $offset), (as_i8imm $format),
1777       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1778       (extract_swz $auxiliary))
1779   >;
1780
1781   def : GCNPat<
1782     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1783           timm:$offset, timm:$format, timm:$auxiliary, timm),
1784     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1785       $vdata,
1786       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1787       $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format),
1788       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1789       (extract_swz $auxiliary))
1790   >;
1791 }
1792
1793 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1794 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1795 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
1796 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1797 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1798 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1799 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
1800 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1801
1802 let SubtargetPredicate = HasUnpackedD16VMem in {
1803   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1804   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
1805   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1806 } // End HasUnpackedD16VMem.
1807
1808 let SubtargetPredicate = HasPackedD16VMem in {
1809   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
1810   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
1811   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
1812 } // End HasPackedD16VMem.
1813
1814 //===----------------------------------------------------------------------===//
1815 // Target-specific instruction encodings.
1816 //===----------------------------------------------------------------------===//
1817
1818 //===----------------------------------------------------------------------===//
1819 // Base ENC_MUBUF for GFX6, GFX7, GFX10.
1820 //===----------------------------------------------------------------------===//
1821
1822 class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
1823     MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1824   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1825   let Inst{12}    = ps.offen;
1826   let Inst{13}    = ps.idxen;
1827   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1828   let Inst{16}    = !if(ps.lds, 1, 0);
1829   let Inst{24-18} = op;
1830   let Inst{31-26} = 0x38;
1831   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1832   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1833   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1834   let Inst{54}    = !if(ps.has_slc, slc, ?);
1835   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1836   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1837 }
1838
1839 class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> :
1840     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
1841   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
1842   let Inst{25} = op{7};
1843 }
1844
1845 class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
1846     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
1847   let Inst{15} = ps.addr64;
1848 }
1849
1850 //===----------------------------------------------------------------------===//
1851 // MUBUF - GFX10.
1852 //===----------------------------------------------------------------------===//
1853
1854 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
1855   multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName,
1856                                         string asmName> {
1857     def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> {
1858       MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName);
1859       let AsmString = asmName # ps.AsmOperands;
1860     }
1861   }
1862   multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
1863     def _BOTHEN_gfx10 :
1864       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1865     def _IDXEN_gfx10 :
1866       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1867     def _OFFEN_gfx10 :
1868       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1869     def _OFFSET_gfx10 :
1870       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1871   }
1872   multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> {
1873     def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1874                         MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">;
1875     def _OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1876                         MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">;
1877     def _IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1878                         MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">;
1879     def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1880                         MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">;
1881
1882     def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1883                             MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">;
1884     def _LDS_OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1885                             MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">;
1886     def _LDS_IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1887                             MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">;
1888     def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1889                             MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">;
1890   }
1891   multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
1892       MUBUF_Real_AllAddr_gfx10<op> {
1893     def _BOTHEN_RTN_gfx10 :
1894       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1895     def _IDXEN_RTN_gfx10 :
1896       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1897     def _OFFEN_RTN_gfx10 :
1898       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1899     def _OFFSET_RTN_gfx10 :
1900       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1901   }
1902 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
1903
1904 defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x019>;
1905 defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx10<0x01b>;
1906 defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x020>;
1907 defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x021>;
1908 defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x022>;
1909 defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x023>;
1910 defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx10<0x024>;
1911 defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x025>;
1912 // FIXME-GFX10: Add following instructions:
1913 //defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx10<0x026>;
1914 //defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
1915 defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx10<0x080>;
1916 defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx10<0x081>;
1917 defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx10<0x082>;
1918 defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx10<0x083>;
1919 defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx10<0x084>;
1920 defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx10<0x085>;
1921 defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx10<0x086>;
1922 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
1923
1924 def BUFFER_GL0_INV_gfx10 :
1925   MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>;
1926 def BUFFER_GL1_INV_gfx10 :
1927   MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>;
1928
1929 //===----------------------------------------------------------------------===//
1930 // MUBUF - GFX6, GFX7, GFX10.
1931 //===----------------------------------------------------------------------===//
1932
1933 let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in {
1934   multiclass MUBUF_Real_gfx6<bits<8> op> {
1935     def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1936   }
1937 } // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6"
1938
1939 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1940   multiclass MUBUF_Real_gfx7<bits<8> op> {
1941     def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1942   }
1943 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1944
1945 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1946   multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
1947     def _ADDR64_gfx6_gfx7 :
1948       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1949     def _BOTHEN_gfx6_gfx7 :
1950       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1951     def _IDXEN_gfx6_gfx7 :
1952       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1953     def _OFFEN_gfx6_gfx7 :
1954       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1955     def _OFFSET_gfx6_gfx7 :
1956       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1957   }
1958   multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> {
1959     def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1960                             MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">;
1961     def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
1962                             MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">;
1963     def _OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1964                             MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">;
1965     def _IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1966                             MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">;
1967     def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1968                             MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">;
1969
1970     def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1971                                 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">;
1972     def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>,
1973                                 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">;
1974     def _LDS_OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1975                                 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">;
1976     def _LDS_IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1977                                 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">;
1978     def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1979                                 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">;
1980   }
1981   multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> :
1982       MUBUF_Real_AllAddr_gfx6_gfx7<op> {
1983     def _ADDR64_RTN_gfx6_gfx7 :
1984       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>;
1985     def _BOTHEN_RTN_gfx6_gfx7 :
1986       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1987     def _IDXEN_RTN_gfx6_gfx7 :
1988       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1989     def _OFFEN_RTN_gfx6_gfx7 :
1990       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1991     def _OFFSET_RTN_gfx6_gfx7 :
1992       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1993   }
1994 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1995
1996 multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
1997   MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
1998
1999 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> :
2000   MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>;
2001
2002 multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
2003   MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
2004
2005 // FIXME-GFX6: Following instructions are available only on GFX6.
2006 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomics_gfx6 <0x034>;
2007 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomics_gfx6 <0x054>;
2008
2009 defm BUFFER_LOAD_FORMAT_X     : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
2010 defm BUFFER_LOAD_FORMAT_XY    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2011 defm BUFFER_LOAD_FORMAT_XYZ   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2012 defm BUFFER_LOAD_FORMAT_XYZW  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2013 defm BUFFER_STORE_FORMAT_X    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2014 defm BUFFER_STORE_FORMAT_XY   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2015 defm BUFFER_STORE_FORMAT_XYZ  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2016 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2017 defm BUFFER_LOAD_UBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
2018 defm BUFFER_LOAD_SBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
2019 defm BUFFER_LOAD_USHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
2020 defm BUFFER_LOAD_SSHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
2021 defm BUFFER_LOAD_DWORD        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
2022 defm BUFFER_LOAD_DWORDX2      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
2023 defm BUFFER_LOAD_DWORDX4      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
2024 defm BUFFER_LOAD_DWORDX3      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
2025 defm BUFFER_STORE_BYTE        : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
2026 defm BUFFER_STORE_SHORT       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
2027 defm BUFFER_STORE_DWORD       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
2028 defm BUFFER_STORE_DWORDX2     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
2029 defm BUFFER_STORE_DWORDX4     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
2030 defm BUFFER_STORE_DWORDX3     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
2031
2032 defm BUFFER_ATOMIC_SWAP        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
2033 defm BUFFER_ATOMIC_CMPSWAP     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
2034 defm BUFFER_ATOMIC_ADD         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
2035 defm BUFFER_ATOMIC_SUB         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
2036 defm BUFFER_ATOMIC_SMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
2037 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
2038 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
2039 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
2040 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
2041 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
2042 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
2043 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
2044 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
2045 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
2046 defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
2047 defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
2048 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
2049 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
2050 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
2051 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
2052 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
2053 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
2054 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
2055 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
2056 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
2057 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
2058 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
2059 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
2060 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
2061 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
2062 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
2063 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
2064 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
2065
2066 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
2067 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
2068 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
2069
2070 //===----------------------------------------------------------------------===//
2071 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
2072 //===----------------------------------------------------------------------===//
2073
2074 class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
2075     MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
2076   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2077   let Inst{12}    = ps.offen;
2078   let Inst{13}    = ps.idxen;
2079   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2080   let Inst{18-16} = op;
2081   let Inst{31-26} = 0x3a; //encoding
2082   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2083   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2084   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2085   let Inst{54}    = !if(ps.has_slc, slc, ?);
2086   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2087   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2088 }
2089
2090 //===----------------------------------------------------------------------===//
2091 // MTBUF - GFX10.
2092 //===----------------------------------------------------------------------===//
2093
2094 class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> :
2095     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
2096   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
2097   let Inst{25-19} = format;
2098   let Inst{53} = op{3};
2099 }
2100
2101 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
2102   multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
2103     def _BOTHEN_gfx10 :
2104       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2105     def _IDXEN_gfx10 :
2106       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2107     def _OFFEN_gfx10 :
2108       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2109     def _OFFSET_gfx10 :
2110       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2111   }
2112 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
2113
2114 defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx10<0x008>;
2115 defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx10<0x009>;
2116 defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx10<0x00a>;
2117 defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx10<0x00b>;
2118 defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx10<0x00c>;
2119 defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx10<0x00d>;
2120 defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx10<0x00e>;
2121 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
2122
2123 //===----------------------------------------------------------------------===//
2124 // MTBUF - GFX6, GFX7, GFX10.
2125 //===----------------------------------------------------------------------===//
2126
2127 class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> :
2128     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
2129   let Inst{15} = ps.addr64;
2130   let Inst{22-19} = dfmt;
2131   let Inst{25-23} = nfmt;
2132 }
2133
2134 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2135   multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
2136     def _ADDR64_gfx6_gfx7 :
2137       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
2138     def _BOTHEN_gfx6_gfx7 :
2139       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2140     def _IDXEN_gfx6_gfx7 :
2141       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2142     def _OFFEN_gfx6_gfx7 :
2143       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2144     def _OFFSET_gfx6_gfx7 :
2145       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2146   }
2147 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2148
2149 multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
2150   MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
2151
2152 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
2153 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2154 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2155 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2156 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2157 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2158 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2159 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2160
2161 //===----------------------------------------------------------------------===//
2162 // GFX8, GFX9 (VI).
2163 //===----------------------------------------------------------------------===//
2164
2165 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
2166   MUBUF_Real<ps>,
2167   Enc64,
2168   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2169   let AssemblerPredicate = isGFX8GFX9;
2170   let DecoderNamespace = "GFX8";
2171
2172   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2173   let Inst{12}    = ps.offen;
2174   let Inst{13}    = ps.idxen;
2175   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2176   let Inst{16}    = !if(ps.lds, 1, 0);
2177   let Inst{17}    = !if(ps.has_slc, slc, ?);
2178   let Inst{24-18} = op;
2179   let Inst{31-26} = 0x38; //encoding
2180   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2181   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2182   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2183   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2184   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2185 }
2186
2187 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
2188   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2189   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2190   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2191   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2192 }
2193
2194 multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
2195
2196   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2197                    MUBUFLdsTable<0, NAME # "_OFFSET_vi">;
2198   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2199                    MUBUFLdsTable<0, NAME # "_OFFEN_vi">;
2200   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2201                    MUBUFLdsTable<0, NAME # "_IDXEN_vi">;
2202   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2203                    MUBUFLdsTable<0, NAME # "_BOTHEN_vi">;
2204
2205   def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2206                        MUBUFLdsTable<1, NAME # "_OFFSET_vi">;
2207   def _LDS_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2208                        MUBUFLdsTable<1, NAME # "_OFFEN_vi">;
2209   def _LDS_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2210                        MUBUFLdsTable<1, NAME # "_IDXEN_vi">;
2211   def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2212                        MUBUFLdsTable<1, NAME # "_BOTHEN_vi">;
2213 }
2214
2215 class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> :
2216   MUBUF_Real<ps>,
2217   Enc64,
2218   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2219   let AssemblerPredicate=HasUnpackedD16VMem;
2220   let DecoderNamespace="GFX80_UNPACKED";
2221
2222   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2223   let Inst{12}    = ps.offen;
2224   let Inst{13}    = ps.idxen;
2225   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2226   let Inst{16}    = !if(ps.lds, 1, 0);
2227   let Inst{17}    = !if(ps.has_slc, slc, ?);
2228   let Inst{24-18} = op;
2229   let Inst{31-26} = 0x38; //encoding
2230   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2231   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2232   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2233   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2234   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2235 }
2236
2237 multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
2238   def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2239   def _OFFEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2240   def _IDXEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2241   def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2242 }
2243
2244 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
2245   MUBUF_Real_AllAddr_vi<op> {
2246   def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
2247   def _OFFEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
2248   def _IDXEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
2249   def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
2250 }
2251
2252 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_Lds_vi <0x00>;
2253 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
2254 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
2255 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
2256 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
2257 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
2258 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
2259 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
2260 let SubtargetPredicate = HasUnpackedD16VMem in {
2261   defm BUFFER_LOAD_FORMAT_D16_X_gfx80       : MUBUF_Real_AllAddr_gfx80 <0x08>;
2262   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x09>;
2263   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0a>;
2264   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0b>;
2265   defm BUFFER_STORE_FORMAT_D16_X_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x0c>;
2266   defm BUFFER_STORE_FORMAT_D16_XY_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0d>;
2267   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0e>;
2268   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80   : MUBUF_Real_AllAddr_gfx80 <0x0f>;
2269 } // End HasUnpackedD16VMem.
2270 let SubtargetPredicate = HasPackedD16VMem in {
2271   defm BUFFER_LOAD_FORMAT_D16_X       : MUBUF_Real_AllAddr_vi <0x08>;
2272   defm BUFFER_LOAD_FORMAT_D16_XY      : MUBUF_Real_AllAddr_vi <0x09>;
2273   defm BUFFER_LOAD_FORMAT_D16_XYZ     : MUBUF_Real_AllAddr_vi <0x0a>;
2274   defm BUFFER_LOAD_FORMAT_D16_XYZW    : MUBUF_Real_AllAddr_vi <0x0b>;
2275   defm BUFFER_STORE_FORMAT_D16_X      : MUBUF_Real_AllAddr_vi <0x0c>;
2276   defm BUFFER_STORE_FORMAT_D16_XY     : MUBUF_Real_AllAddr_vi <0x0d>;
2277   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
2278   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
2279 } // End HasPackedD16VMem.
2280 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
2281 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
2282 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
2283 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
2284 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
2285 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
2286 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
2287 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
2288 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
2289 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
2290 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
2291 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
2292 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
2293 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
2294 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
2295 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
2296
2297 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
2298 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
2299 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
2300 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
2301 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
2302 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
2303
2304 defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_vi <0x26>;
2305 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
2306
2307 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
2308 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
2309 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
2310 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
2311 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
2312 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
2313 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
2314 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
2315 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
2316 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
2317 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
2318 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
2319 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
2320
2321 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
2322 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
2323 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
2324 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
2325 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
2326 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
2327 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
2328 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
2329 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
2330 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
2331 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
2332 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
2333 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
2334
2335 def BUFFER_STORE_LDS_DWORD_vi   : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>;
2336
2337 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
2338 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
2339
2340 let SubtargetPredicate = HasAtomicFaddInsts in {
2341
2342 defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_AllAddr_vi <0x4d>;
2343 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>;
2344
2345 } // End SubtargetPredicate = HasAtomicFaddInsts
2346
2347 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
2348   MTBUF_Real<ps>,
2349   Enc64,
2350   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2351   let AssemblerPredicate = isGFX8GFX9;
2352   let DecoderNamespace = "GFX8";
2353
2354   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2355   let Inst{12}    = ps.offen;
2356   let Inst{13}    = ps.idxen;
2357   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2358   let Inst{18-15} = op;
2359   let Inst{22-19} = dfmt;
2360   let Inst{25-23} = nfmt;
2361   let Inst{31-26} = 0x3a; //encoding
2362   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2363   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2364   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2365   let Inst{54}    = !if(ps.has_slc, slc, ?);
2366   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2367   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2368 }
2369
2370 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
2371   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2372   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2373   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2374   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2375 }
2376
2377 class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> :
2378   MTBUF_Real<ps>,
2379   Enc64,
2380   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2381   let AssemblerPredicate=HasUnpackedD16VMem;
2382   let DecoderNamespace="GFX80_UNPACKED";
2383
2384   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2385   let Inst{12}    = ps.offen;
2386   let Inst{13}    = ps.idxen;
2387   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2388   let Inst{18-15} = op;
2389   let Inst{22-19} = dfmt;
2390   let Inst{25-23} = nfmt;
2391   let Inst{31-26} = 0x3a; //encoding
2392   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2393   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2394   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2395   let Inst{54}    = !if(ps.has_slc, slc, ?);
2396   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2397   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2398 }
2399
2400 multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
2401   def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2402   def _OFFEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2403   def _IDXEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2404   def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2405 }
2406
2407 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0x00>;
2408 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <0x01>;
2409 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <0x02>;
2410 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <0x03>;
2411 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <0x04>;
2412 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <0x05>;
2413 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <0x06>;
2414 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
2415 let SubtargetPredicate = HasUnpackedD16VMem in {
2416   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Real_AllAddr_gfx80 <0x08>;
2417   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x09>;
2418   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0a>;
2419   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0b>;
2420   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x0c>;
2421   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0d>;
2422   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0e>;
2423   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
2424 } // End HasUnpackedD16VMem.
2425 let SubtargetPredicate = HasPackedD16VMem in {
2426   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_vi <0x08>;
2427   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_vi <0x09>;
2428   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_vi <0x0a>;
2429   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_vi <0x0b>;
2430   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_vi <0x0c>;
2431   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_vi <0x0d>;
2432   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_vi <0x0e>;
2433   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
2434 } // End HasUnpackedD16VMem.
2435
2436 def MUBUFInfoTable : GenericTable {
2437   let FilterClass = "MUBUF_Pseudo";
2438   let CppTypeName = "MUBUFInfo";
2439   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2440
2441   let PrimaryKey = ["Opcode"];
2442   let PrimaryKeyName = "getMUBUFOpcodeHelper";
2443 }
2444
2445 def getMUBUFInfoFromOpcode : SearchIndex {
2446   let Table = MUBUFInfoTable;
2447   let Key = ["Opcode"];
2448 }
2449
2450 def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2451   let Table = MUBUFInfoTable;
2452   let Key = ["BaseOpcode", "elements"];
2453 }
2454
2455 def MTBUFInfoTable : GenericTable {
2456   let FilterClass = "MTBUF_Pseudo";
2457   let CppTypeName = "MTBUFInfo";
2458   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2459
2460   let PrimaryKey = ["Opcode"];
2461   let PrimaryKeyName = "getMTBUFOpcodeHelper";
2462 }
2463
2464 def getMTBUFInfoFromOpcode : SearchIndex {
2465   let Table = MTBUFInfoTable;
2466   let Key = ["Opcode"];
2467 }
2468
2469 def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2470   let Table = MTBUFInfoTable;
2471   let Key = ["BaseOpcode", "elements"];
2472 }