lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  10 def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">;
  11 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  12
  13 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  14 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  15
  16 def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">;
  17 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  18 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  19
  20 def BUFAddrKind {
  21   int Offset = 0;
  22   int OffEn  = 1;
  23   int IdxEn  = 2;
  24   int BothEn = 3;
  25   int Addr64 = 4;
  26 }
  27
  28 class getAddrName<int addrKind> {
  29   string ret =
  30     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  31     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  32     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  33     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  34     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  35     "")))));
  36 }
  37
  38 class MUBUFAddr64Table <bit is_addr64, string Name> {
  39   bit IsAddr64 = is_addr64;
  40   string OpName = Name;
  41 }
  42
  43 class MUBUFLdsTable <bit is_lds, string Name> {
  44   bit IsLds = is_lds;
  45   string OpName = Name;
  46 }
  47
  48 class MTBUFAddr64Table <bit is_addr64, string Name> {
  49   bit IsAddr64 = is_addr64;
  50   string OpName = Name;
  51 }
  52
  53 //===----------------------------------------------------------------------===//
  54 // MTBUF classes
  55 //===----------------------------------------------------------------------===//
  56
  57 class MTBUFGetBaseOpcode<string Op> {
  58   string ret = !subst("FORMAT_XY", "FORMAT_X",
  59     !subst("FORMAT_XYZ", "FORMAT_X",
  60     !subst("FORMAT_XYZW", "FORMAT_X", Op)));
  61 }
  62
  63 class getMTBUFElements<string Op> {
  64   int ret = 1;
  65 }
  66
  67
  68 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  69                     string asmOps, list<dag> pattern=[]> :
  70   InstSI<outs, ins, "", pattern>,
  71   SIMCInstr<opName, SIEncodingFamily.NONE> {
  72
  73   let isPseudo = 1;
  74   let isCodeGenOnly = 1;
  75   let Size = 8;
  76   let UseNamedOperandTable = 1;
  77
  78   string Mnemonic = opName;
  79   string AsmOperands = asmOps;
  80
  81   Instruction Opcode = !cast<Instruction>(NAME);
  82   Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret);
  83
  84   let VM_CNT = 1;
  85   let EXP_CNT = 1;
  86   let MTBUF = 1;
  87   let Uses = [EXEC];
  88   let hasSideEffects = 0;
  89   let SchedRW = [WriteVMEM];
  90
  91   let AsmMatchConverter = "cvtMtbuf";
  92
  93   bits<1> offen       = 0;
  94   bits<1> idxen       = 0;
  95   bits<1> addr64      = 0;
  96   bits<1> has_vdata   = 1;
  97   bits<1> has_vaddr   = 1;
  98   bits<1> has_glc     = 1;
  99   bits<1> has_dlc     = 1;
 100   bits<1> glc_value   = 0; // the value for glc if no such operand
 101   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 102   bits<1> has_srsrc   = 1;
 103   bits<1> has_soffset = 1;
 104   bits<1> has_offset  = 1;
 105   bits<1> has_slc     = 1;
 106   bits<1> has_tfe     = 1;
 107   bits<4> elements    = 0;
 108 }
 109
 110 class MTBUF_Real <MTBUF_Pseudo ps> :
 111   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 112
 113   let isPseudo = 0;
 114   let isCodeGenOnly = 0;
 115
 116   // copy relevant pseudo op flags
 117   let SubtargetPredicate = ps.SubtargetPredicate;
 118   let AsmMatchConverter  = ps.AsmMatchConverter;
 119   let Constraints        = ps.Constraints;
 120   let DisableEncoding    = ps.DisableEncoding;
 121   let TSFlags            = ps.TSFlags;
 122
 123   bits<12> offset;
 124   bits<1>  glc;
 125   bits<1>  dlc;
 126   bits<7>  format;
 127   bits<8>  vaddr;
 128   bits<8>  vdata;
 129   bits<7>  srsrc;
 130   bits<1>  slc;
 131   bits<1>  tfe;
 132   bits<8>  soffset;
 133
 134   bits<4> dfmt = format{3-0};
 135   bits<3> nfmt = format{6-4};
 136 }
 137
 138 class getMTBUFInsDA<list<RegisterClass> vdataList,
 139                     list<RegisterClass> vaddrList=[]> {
 140   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 141   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 142   dag InsNoData = !if(!empty(vaddrList),
 143     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 144          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
 145     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 146          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
 147   );
 148   dag InsData = !if(!empty(vaddrList),
 149     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 150          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 151          SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
 152     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 153          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 154          SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
 155   );
 156   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 157 }
 158
 159 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 160   dag ret =
 161     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 162     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 163     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 164     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 165     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 166     (ins))))));
 167 }
 168
 169 class getMTBUFAsmOps<int addrKind> {
 170   string Pfx =
 171     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset",
 172     !if(!eq(addrKind, BUFAddrKind.OffEn),
 173             "$vaddr, $srsrc, $format, $soffset offen",
 174     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 175             "$vaddr, $srsrc, $format, $soffset idxen",
 176     !if(!eq(addrKind, BUFAddrKind.BothEn),
 177             "$vaddr, $srsrc, $format, $soffset idxen offen",
 178     !if(!eq(addrKind, BUFAddrKind.Addr64),
 179             "$vaddr, $srsrc, $format, $soffset addr64",
 180     "")))));
 181   string ret = Pfx # "$offset";
 182 }
 183
 184 class MTBUF_SetupAddr<int addrKind> {
 185   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 186                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 187
 188   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 189                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 190
 191   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 192
 193   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 194 }
 195
 196 class MTBUF_Load_Pseudo <string opName,
 197                          int addrKind,
 198                          RegisterClass vdataClass,
 199                          int elems,
 200                          list<dag> pattern=[],
 201                          // Workaround bug bz30254
 202                          int addrKindCopy = addrKind>
 203   : MTBUF_Pseudo<opName,
 204                  (outs vdataClass:$vdata),
 205                  getMTBUFIns<addrKindCopy>.ret,
 206                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 207                  pattern>,
 208     MTBUF_SetupAddr<addrKindCopy> {
 209   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 210   let mayLoad = 1;
 211   let mayStore = 0;
 212   let elements = elems;
 213 }
 214
 215 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 216                               int elems, ValueType load_vt = i32,
 217                               SDPatternOperator ld = null_frag> {
 218
 219   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
 220     [(set load_vt:$vdata,
 221      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format,
 222                       i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
 223     MTBUFAddr64Table<0, NAME>;
 224
 225   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
 226     [(set load_vt:$vdata,
 227      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 228                       i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
 229     MTBUFAddr64Table<1, NAME>;
 230
 231   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 232   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 233   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 234
 235   let DisableWQM = 1 in {
 236     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 237     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 238     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 239     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 240   }
 241 }
 242
 243 class MTBUF_Store_Pseudo <string opName,
 244                           int addrKind,
 245                           RegisterClass vdataClass,
 246                           int elems,
 247                           list<dag> pattern=[],
 248                           // Workaround bug bz30254
 249                           int addrKindCopy = addrKind,
 250                           RegisterClass vdataClassCopy = vdataClass>
 251   : MTBUF_Pseudo<opName,
 252                  (outs),
 253                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 254                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 255                  pattern>,
 256     MTBUF_SetupAddr<addrKindCopy> {
 257   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 258   let mayLoad = 0;
 259   let mayStore = 1;
 260   let elements = elems;
 261 }
 262
 263 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 264                                int elems, ValueType store_vt = i32,
 265                                SDPatternOperator st = null_frag> {
 266
 267   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
 268     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 269                                        i16:$offset, i8:$format, i1:$glc,
 270                                        i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 271     MTBUFAddr64Table<0, NAME>;
 272
 273   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
 274     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 275                                        i16:$offset, i8:$format, i1:$glc,
 276                                        i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 277     MTBUFAddr64Table<1, NAME>;
 278
 279   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 280   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 281   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 282
 283   let DisableWQM = 1 in {
 284     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
 285     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
 286     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
 287     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 288   }
 289 }
 290
 291
 292 //===----------------------------------------------------------------------===//
 293 // MUBUF classes
 294 //===----------------------------------------------------------------------===//
 295
 296 class MUBUFGetBaseOpcode<string Op> {
 297   string ret = !subst("DWORDX2", "DWORD",
 298     !subst("DWORDX3", "DWORD",
 299     !subst("DWORDX4", "DWORD", Op)));
 300 }
 301
 302 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 303                     string asmOps, list<dag> pattern=[]> :
 304   InstSI<outs, ins, "", pattern>,
 305   SIMCInstr<opName, SIEncodingFamily.NONE> {
 306
 307   let isPseudo = 1;
 308   let isCodeGenOnly = 1;
 309   let Size = 8;
 310   let UseNamedOperandTable = 1;
 311
 312   string Mnemonic = opName;
 313   string AsmOperands = asmOps;
 314
 315   Instruction Opcode = !cast<Instruction>(NAME);
 316   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
 317
 318   let VM_CNT = 1;
 319   let EXP_CNT = 1;
 320   let MUBUF = 1;
 321   let Uses = [EXEC];
 322   let hasSideEffects = 0;
 323   let SchedRW = [WriteVMEM];
 324
 325   let AsmMatchConverter = "cvtMubuf";
 326
 327   bits<1> offen       = 0;
 328   bits<1> idxen       = 0;
 329   bits<1> addr64      = 0;
 330   bits<1> lds         = 0;
 331   bits<1> has_vdata   = 1;
 332   bits<1> has_vaddr   = 1;
 333   bits<1> has_glc     = 1;
 334   bits<1> has_dlc     = 1;
 335   bits<1> glc_value   = 0; // the value for glc if no such operand
 336   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 337   bits<1> has_srsrc   = 1;
 338   bits<1> has_soffset = 1;
 339   bits<1> has_offset  = 1;
 340   bits<1> has_slc     = 1;
 341   bits<1> has_tfe     = 1;
 342   bits<4> elements    = 0;
 343 }
 344
 345 class MUBUF_Real <MUBUF_Pseudo ps> :
 346   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 347
 348   let isPseudo = 0;
 349   let isCodeGenOnly = 0;
 350
 351   // copy relevant pseudo op flags
 352   let SubtargetPredicate = ps.SubtargetPredicate;
 353   let AsmMatchConverter  = ps.AsmMatchConverter;
 354   let Constraints        = ps.Constraints;
 355   let DisableEncoding    = ps.DisableEncoding;
 356   let TSFlags            = ps.TSFlags;
 357
 358   bits<12> offset;
 359   bits<1>  glc;
 360   bits<1>  dlc;
 361   bits<8>  vaddr;
 362   bits<8>  vdata;
 363   bits<7>  srsrc;
 364   bits<1>  slc;
 365   bits<1>  tfe;
 366   bits<8>  soffset;
 367 }
 368
 369
 370 // For cache invalidation instructions.
 371 class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
 372   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 373
 374   let AsmMatchConverter = "";
 375
 376   let hasSideEffects = 1;
 377   let mayStore = 1;
 378
 379   // Set everything to 0.
 380   let offen       = 0;
 381   let idxen       = 0;
 382   let addr64      = 0;
 383   let has_vdata   = 0;
 384   let has_vaddr   = 0;
 385   let has_glc     = 0;
 386   let has_dlc     = 0;
 387   let glc_value   = 0;
 388   let dlc_value   = 0;
 389   let has_srsrc   = 0;
 390   let has_soffset = 0;
 391   let has_offset  = 0;
 392   let has_slc     = 0;
 393   let has_tfe     = 0;
 394 }
 395
 396 class getMUBUFInsDA<list<RegisterClass> vdataList,
 397                     list<RegisterClass> vaddrList=[],
 398                     bit isLds = 0> {
 399   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 400   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 401   dag InsNoData = !if(!empty(vaddrList),
 402     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 403          offset:$offset, GLC:$glc, SLC:$slc),
 404     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 405          offset:$offset, GLC:$glc, SLC:$slc)
 406   );
 407   dag InsData = !if(!empty(vaddrList),
 408     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 409          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 410     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 411          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc)
 412   );
 413   dag ret = !con(
 414               !if(!empty(vdataList), InsNoData, InsData),
 415               !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz))
 416              );
 417 }
 418
 419 class getMUBUFElements<ValueType vt> {
 420   // eq does not support ValueType for some reason.
 421   string vtAsStr = !cast<string>(vt);
 422
 423   int ret =
 424     !if(!eq(vtAsStr, "f16"), 1,
 425       !if(!eq(vtAsStr, "v2f16"), 2,
 426         !if(!eq(vtAsStr, "v3f16"), 3,
 427           !if(!eq(vtAsStr, "v4f16"), 4,
 428             !if(!eq(vt.Size, 32), 1,
 429               !if(!eq(vt.Size, 64), 2,
 430                 !if(!eq(vt.Size, 96), 3,
 431                   !if(!eq(vt.Size, 128), 4, 0)
 432                 )
 433               )
 434             )
 435           )
 436         )
 437       )
 438     );
 439 }
 440
 441 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> {
 442   dag ret =
 443     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret,
 444     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 445     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 446     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 447     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 448     (ins))))));
 449 }
 450
 451 class getMUBUFAsmOps<int addrKind> {
 452   string Pfx =
 453     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 454     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 455     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 456     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 457     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 458     "")))));
 459   string ret = Pfx # "$offset";
 460 }
 461
 462 class MUBUF_SetupAddr<int addrKind> {
 463   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 464                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 465
 466   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 467                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 468
 469   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 470
 471   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 472 }
 473
 474 class MUBUF_Load_Pseudo <string opName,
 475                          int addrKind,
 476                          ValueType vdata_vt,
 477                          bit HasTiedDest = 0,
 478                          bit isLds = 0,
 479                          list<dag> pattern=[],
 480                          // Workaround bug bz30254
 481                          int addrKindCopy = addrKind>
 482   : MUBUF_Pseudo<opName,
 483                  (outs getVregSrcForVT<vdata_vt>.ret:$vdata),
 484                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
 485                       !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))),
 486                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" #
 487                    !if(isLds, " lds", "$tfe") # "$dlc" # "$swz",
 488                  pattern>,
 489     MUBUF_SetupAddr<addrKindCopy> {
 490   let PseudoInstr = opName # !if(isLds, "_lds", "") #
 491                     "_" # getAddrName<addrKindCopy>.ret;
 492   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 493
 494   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 495   let mayLoad = 1;
 496   let mayStore = 0;
 497   let maybeAtomic = 1;
 498   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
 499   let has_tfe = !if(isLds, 0, 1);
 500   let lds = isLds;
 501   let elements = getMUBUFElements<vdata_vt>.ret;
 502 }
 503
 504 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
 505   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
 506   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 507 >;
 508
 509 class MUBUF_Addr64_Load_Pat <Instruction inst,
 510                             ValueType load_vt = i32,
 511                             SDPatternOperator ld = null_frag> : Pat <
 512   (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
 513   (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 514 >;
 515
 516 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
 517   def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
 518   def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
 519 }
 520
 521
 522 // FIXME: tfe can't be an operand because it requires a separate
 523 // opcode because it needs an N+1 register class dest register.
 524 multiclass MUBUF_Pseudo_Loads<string opName,
 525                               ValueType load_vt = i32,
 526                               SDPatternOperator ld = null_frag,
 527                               bit TiedDest = 0,
 528                               bit isLds = 0> {
 529
 530   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>,
 531     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 532
 533   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>,
 534     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 535
 536   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 537   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 538   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 539
 540   let DisableWQM = 1 in {
 541     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>;
 542     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 543     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 544     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 545   }
 546 }
 547
 548 multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32,
 549                                   SDPatternOperator ld_nolds = null_frag,
 550                                   SDPatternOperator ld_lds = null_frag> {
 551   defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>;
 552   defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>;
 553 }
 554
 555 class MUBUF_Store_Pseudo <string opName,
 556                           int addrKind,
 557                           ValueType store_vt,
 558                           list<dag> pattern=[],
 559                           // Workaround bug bz30254
 560                           int addrKindCopy = addrKind>
 561   : MUBUF_Pseudo<opName,
 562                  (outs),
 563                  getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret,
 564                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
 565                  pattern>,
 566     MUBUF_SetupAddr<addrKindCopy> {
 567   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 568   let mayLoad = 0;
 569   let mayStore = 1;
 570   let maybeAtomic = 1;
 571   let elements = getMUBUFElements<store_vt>.ret;
 572 }
 573
 574 multiclass MUBUF_Pseudo_Stores<string opName,
 575                                ValueType store_vt = i32,
 576                                SDPatternOperator st = null_frag> {
 577
 578   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt,
 579     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 580                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 581     MUBUFAddr64Table<0, NAME>;
 582
 583   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt,
 584     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 585                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
 586     MUBUFAddr64Table<1, NAME>;
 587
 588   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 589   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 590   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 591
 592   let DisableWQM = 1 in {
 593     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>;
 594     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 595     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 596     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 597   }
 598 }
 599
 600 class MUBUF_Pseudo_Store_Lds<string opName>
 601   : MUBUF_Pseudo<opName,
 602                  (outs),
 603                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz),
 604                  " $srsrc, $soffset$offset lds$glc$slc$swz"> {
 605   let mayLoad = 0;
 606   let mayStore = 1;
 607   let maybeAtomic = 1;
 608
 609   let has_vdata = 0;
 610   let has_vaddr = 0;
 611   let has_tfe = 0;
 612   let lds = 1;
 613
 614   let Uses = [EXEC, M0];
 615   let AsmMatchConverter = "cvtMubufLds";
 616 }
 617
 618 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 619                           list<RegisterClass> vaddrList=[]> {
 620   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 621   dag ret = !if(vdata_in,
 622     !if(!empty(vaddrList),
 623       (ins vdataClass:$vdata_in,
 624            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 625       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 626            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 627     ),
 628     !if(!empty(vaddrList),
 629       (ins vdataClass:$vdata,
 630            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 631       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 632            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 633   ));
 634 }
 635
 636 class getMUBUFAtomicIns<int addrKind,
 637                         RegisterClass vdataClass,
 638                         bit vdata_in,
 639                         // Workaround bug bz30254
 640                         RegisterClass vdataClassCopy=vdataClass> {
 641   dag ret =
 642     !if(!eq(addrKind, BUFAddrKind.Offset),
 643             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 644     !if(!eq(addrKind, BUFAddrKind.OffEn),
 645             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 646     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 647             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 648     !if(!eq(addrKind, BUFAddrKind.BothEn),
 649             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 650     !if(!eq(addrKind, BUFAddrKind.Addr64),
 651             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 652     (ins))))));
 653 }
 654
 655 class MUBUF_Atomic_Pseudo<string opName,
 656                           int addrKind,
 657                           dag outs,
 658                           dag ins,
 659                           string asmOps,
 660                           list<dag> pattern=[],
 661                           // Workaround bug bz30254
 662                           int addrKindCopy = addrKind>
 663   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 664     MUBUF_SetupAddr<addrKindCopy> {
 665   let mayStore = 1;
 666   let mayLoad = 1;
 667   let hasPostISelHook = 1;
 668   let hasSideEffects = 1;
 669   let DisableWQM = 1;
 670   let has_glc = 0;
 671   let has_dlc = 0;
 672   let has_tfe = 0;
 673   let maybeAtomic = 1;
 674 }
 675
 676 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 677                                RegisterClass vdataClass,
 678                                list<dag> pattern=[],
 679                                // Workaround bug bz30254
 680                                int addrKindCopy = addrKind,
 681                                RegisterClass vdataClassCopy = vdataClass>
 682   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 683                         (outs),
 684                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 685                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 686                         pattern>,
 687     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 688   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 689   let glc_value = 0;
 690   let dlc_value = 0;
 691   let AsmMatchConverter = "cvtMubufAtomic";
 692 }
 693
 694 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 695                              RegisterClass vdataClass,
 696                              list<dag> pattern=[],
 697                              // Workaround bug bz30254
 698                              int addrKindCopy = addrKind,
 699                              RegisterClass vdataClassCopy = vdataClass>
 700   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 701                         (outs vdataClassCopy:$vdata),
 702                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 703                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 704                         pattern>,
 705     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 706   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 707   let glc_value = 1;
 708   let dlc_value = 0;
 709   let Constraints = "$vdata = $vdata_in";
 710   let DisableEncoding = "$vdata_in";
 711   let AsmMatchConverter = "cvtMubufAtomicReturn";
 712 }
 713
 714 multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
 715                                         RegisterClass vdataClass,
 716                                         ValueType vdataType,
 717                                         SDPatternOperator atomic,
 718                                         bit isFP = getIsFP<vdataType>.ret> {
 719   let FPAtomic = isFP in
 720   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 721                 MUBUFAddr64Table <0, NAME>;
 722
 723   let FPAtomic = isFP in
 724   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 725                 MUBUFAddr64Table <1, NAME>;
 726
 727   let FPAtomic = isFP in
 728   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 729
 730   let FPAtomic = isFP in
 731
 732   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 733
 734   let FPAtomic = isFP in
 735   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 736 }
 737
 738 multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
 739                                      RegisterClass vdataClass,
 740                                      ValueType vdataType,
 741                                      SDPatternOperator atomic,
 742                                      bit isFP = getIsFP<vdataType>.ret> {
 743   let FPAtomic = isFP in
 744   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 745     [(set vdataType:$vdata,
 746      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 747              vdataType:$vdata_in))]>,
 748     MUBUFAddr64Table <0, NAME # "_RTN">;
 749
 750   let FPAtomic = isFP in
 751   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 752     [(set vdataType:$vdata,
 753      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 754              vdataType:$vdata_in))]>,
 755     MUBUFAddr64Table <1, NAME # "_RTN">;
 756
 757   let FPAtomic = isFP in
 758   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 759
 760   let FPAtomic = isFP in
 761   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 762
 763   let FPAtomic = isFP in
 764   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 765 }
 766
 767 multiclass MUBUF_Pseudo_Atomics <string opName,
 768                                  RegisterClass vdataClass,
 769                                  ValueType vdataType,
 770                                  SDPatternOperator atomic> :
 771   MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>,
 772   MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
 773
 774
 775 //===----------------------------------------------------------------------===//
 776 // MUBUF Instructions
 777 //===----------------------------------------------------------------------===//
 778
 779 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
 780   "buffer_load_format_x", f32
 781 >;
 782 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 783   "buffer_load_format_xy", v2f32
 784 >;
 785 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 786   "buffer_load_format_xyz", v3f32
 787 >;
 788 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 789   "buffer_load_format_xyzw", v4f32
 790 >;
 791 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 792   "buffer_store_format_x", f32
 793 >;
 794 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 795   "buffer_store_format_xy", v2f32
 796 >;
 797 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 798   "buffer_store_format_xyz", v3f32
 799 >;
 800 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 801   "buffer_store_format_xyzw", v4f32
 802 >;
 803
 804 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
 805   defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
 806     "buffer_load_format_d16_x", i32
 807   >;
 808   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
 809     "buffer_load_format_d16_xy", v2i32
 810   >;
 811   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
 812     "buffer_load_format_d16_xyz", v3i32
 813   >;
 814   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
 815    "buffer_load_format_d16_xyzw", v4i32
 816   >;
 817   defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
 818     "buffer_store_format_d16_x", i32
 819   >;
 820   defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
 821     "buffer_store_format_d16_xy", v2i32
 822   >;
 823   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
 824     "buffer_store_format_d16_xyz", v3i32
 825   >;
 826   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
 827     "buffer_store_format_d16_xyzw", v4i32
 828   >;
 829 } // End HasUnpackedD16VMem.
 830
 831 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
 832   defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
 833     "buffer_load_format_d16_x", f16
 834   >;
 835   defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
 836     "buffer_load_format_d16_xy", v2f16
 837   >;
 838   defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
 839     "buffer_load_format_d16_xyz", v3f16
 840   >;
 841   defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
 842     "buffer_load_format_d16_xyzw", v4f16
 843   >;
 844   defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
 845     "buffer_store_format_d16_x", f16
 846   >;
 847   defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
 848     "buffer_store_format_d16_xy", v2f16
 849   >;
 850   defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
 851     "buffer_store_format_d16_xyz", v3f16
 852   >;
 853   defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
 854     "buffer_store_format_d16_xyzw", v4f16
 855   >;
 856 } // End HasPackedD16VMem.
 857
 858 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
 859   "buffer_load_ubyte", i32
 860 >;
 861 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
 862   "buffer_load_sbyte", i32
 863 >;
 864 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
 865   "buffer_load_ushort", i32
 866 >;
 867 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
 868   "buffer_load_sshort", i32
 869 >;
 870 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
 871   "buffer_load_dword", i32
 872 >;
 873 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 874   "buffer_load_dwordx2", v2i32
 875 >;
 876 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 877   "buffer_load_dwordx3", v3i32
 878 >;
 879 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 880   "buffer_load_dwordx4", v4i32
 881 >;
 882
 883 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
 884 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
 885 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
 886 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
 887 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
 888 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
 889 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>;
 890 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>;
 891 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>;
 892 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>;
 893
 894 // This is not described in AMD documentation,
 895 // but 'lds' versions of these opcodes are available
 896 // in at least GFX8+ chips. See Bug 37653.
 897 let SubtargetPredicate = isGFX8GFX9 in {
 898 defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
 899   "buffer_load_dwordx2", v2i32, null_frag, 0, 1
 900 >;
 901 defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
 902   "buffer_load_dwordx3", v3i32, null_frag, 0, 1
 903 >;
 904 defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
 905   "buffer_load_dwordx4", v4i32, null_frag, 0, 1
 906 >;
 907 }
 908
 909 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 910   "buffer_store_byte", i32, truncstorei8_global
 911 >;
 912 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 913   "buffer_store_short", i32, truncstorei16_global
 914 >;
 915 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 916   "buffer_store_dword", i32, store_global
 917 >;
 918 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 919   "buffer_store_dwordx2", v2i32, store_global
 920 >;
 921 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 922   "buffer_store_dwordx3", v3i32, store_global
 923 >;
 924 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 925   "buffer_store_dwordx4", v4i32, store_global
 926 >;
 927 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 928   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32
 929 >;
 930 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 931   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 932 >;
 933 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 934   "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32
 935 >;
 936 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 937   "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32
 938 >;
 939 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 940   "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32
 941 >;
 942 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 943   "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32
 944 >;
 945 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 946   "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32
 947 >;
 948 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 949   "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32
 950 >;
 951 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 952   "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32
 953 >;
 954 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 955   "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32
 956 >;
 957 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 958   "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32
 959 >;
 960 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 961   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32
 962 >;
 963 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 964   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32
 965 >;
 966 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 967   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64
 968 >;
 969 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 970   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 971 >;
 972 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 973   "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64
 974 >;
 975 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 976   "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64
 977 >;
 978 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 979   "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64
 980 >;
 981 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 982   "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64
 983 >;
 984 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 985   "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64
 986 >;
 987 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 988   "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64
 989 >;
 990 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 991   "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64
 992 >;
 993 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 994   "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64
 995 >;
 996 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 997   "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64
 998 >;
 999 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
1000   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64
1001 >;
1002 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
1003   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64
1004 >;
1005
1006 let SubtargetPredicate = isGFX8GFX9 in {
1007 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
1008 }
1009
1010 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
1011 /*
1012 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
1013 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
1014 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
1015 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
1016 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
1017 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
1018 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
1019 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
1020 */
1021
1022 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
1023                                           int_amdgcn_buffer_wbinvl1_sc>;
1024 }
1025
1026 let SubtargetPredicate = HasD16LoadStore in {
1027
1028 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
1029   "buffer_load_ubyte_d16", i32, null_frag, 1
1030 >;
1031
1032 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
1033   "buffer_load_ubyte_d16_hi", i32, null_frag, 1
1034 >;
1035
1036 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
1037   "buffer_load_sbyte_d16", i32, null_frag, 1
1038 >;
1039
1040 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
1041   "buffer_load_sbyte_d16_hi", i32, null_frag, 1
1042 >;
1043
1044 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
1045   "buffer_load_short_d16", i32, null_frag, 1
1046 >;
1047
1048 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
1049   "buffer_load_short_d16_hi", i32, null_frag, 1
1050 >;
1051
1052 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
1053   "buffer_store_byte_d16_hi", i32
1054 >;
1055
1056 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
1057   "buffer_store_short_d16_hi", i32
1058 >;
1059
1060 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
1061   "buffer_load_format_d16_hi_x", i32
1062 >;
1063 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
1064   "buffer_store_format_d16_hi_x", i32
1065 >;
1066
1067 } // End HasD16LoadStore
1068
1069 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
1070                                        int_amdgcn_buffer_wbinvl1>;
1071
1072 let SubtargetPredicate = HasAtomicFaddInsts in {
1073
1074 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
1075   "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret
1076 >;
1077 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
1078   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret
1079 >;
1080
1081 } // End SubtargetPredicate = HasAtomicFaddInsts
1082
1083 //===----------------------------------------------------------------------===//
1084 // MTBUF Instructions
1085 //===----------------------------------------------------------------------===//
1086
1087 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32,  1>;
1088 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64,  2>;
1089 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96,  3>;
1090 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128, 4>;
1091 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32,  1>;
1092 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64,  2>;
1093 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96,  3>;
1094 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>;
1095
1096 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
1097   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32,  1>;
1098   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64,  2>;
1099   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96,  3>;
1100   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128, 4>;
1101   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32,  1>;
1102   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64,  2>;
1103   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96,  3>;
1104   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>;
1105 } // End HasUnpackedD16VMem.
1106
1107 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
1108   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32, 1>;
1109   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32, 2>;
1110   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64, 3>;
1111   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64, 4>;
1112   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32, 1>;
1113   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32, 2>;
1114   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64, 3>;
1115   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>;
1116 } // End HasPackedD16VMem.
1117
1118 let SubtargetPredicate = isGFX7Plus in {
1119
1120 //===----------------------------------------------------------------------===//
1121 // Instruction definitions for CI and newer.
1122 //===----------------------------------------------------------------------===//
1123
1124 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
1125                                            int_amdgcn_buffer_wbinvl1_vol>;
1126
1127 } // End let SubtargetPredicate = isGFX7Plus
1128
1129 let SubtargetPredicate = isGFX10Plus in {
1130   def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
1131   def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
1132 } // End SubtargetPredicate = isGFX10Plus
1133
1134 //===----------------------------------------------------------------------===//
1135 // MUBUF Patterns
1136 //===----------------------------------------------------------------------===//
1137
1138 def extract_glc : SDNodeXForm<imm, [{
1139   return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8);
1140 }]>;
1141
1142 def extract_slc : SDNodeXForm<imm, [{
1143   return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8);
1144 }]>;
1145
1146 def extract_dlc : SDNodeXForm<imm, [{
1147   return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8);
1148 }]>;
1149
1150 def extract_swz : SDNodeXForm<imm, [{
1151   return CurDAG->getTargetConstant((N->getZExtValue() >> 3) & 1, SDLoc(N), MVT::i8);
1152 }]>;
1153
1154 //===----------------------------------------------------------------------===//
1155 // buffer_load/store_format patterns
1156 //===----------------------------------------------------------------------===//
1157
1158 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1159                                   string opcode> {
1160   def : GCNPat<
1161     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1162               timm:$auxiliary, 0)),
1163     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1164       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1165       (extract_swz $auxiliary))
1166   >;
1167
1168   def : GCNPat<
1169     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1170               timm:$auxiliary, 0)),
1171     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1172       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1173       (extract_swz $auxiliary))
1174   >;
1175
1176   def : GCNPat<
1177     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1178               timm:$auxiliary, timm)),
1179     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1180       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1181       (extract_swz $auxiliary))
1182   >;
1183
1184   def : GCNPat<
1185     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1186               timm:$auxiliary, timm)),
1187     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1188       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1189       $rsrc, $soffset, (as_i16imm $offset),
1190       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1191       (extract_swz $auxiliary))
1192   >;
1193 }
1194
1195 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
1196 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
1197 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
1198 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
1199 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
1200 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
1201 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
1202 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
1203
1204 let SubtargetPredicate = HasUnpackedD16VMem in {
1205   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1206   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1207   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1208   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1209 } // End HasUnpackedD16VMem.
1210
1211 let SubtargetPredicate = HasPackedD16VMem in {
1212   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
1213   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
1214   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
1215   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
1216   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1217   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1218 } // End HasPackedD16VMem.
1219
1220 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
1221 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
1222 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">;
1223 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">;
1224 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
1225 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
1226 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">;
1227 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">;
1228 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
1229 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
1230 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
1231 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
1232 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
1233 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
1234 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
1235 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
1236
1237 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1238                                    string opcode> {
1239   def : GCNPat<
1240     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1241               timm:$auxiliary, 0),
1242     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
1243       (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1244       (extract_swz $auxiliary))
1245   >;
1246
1247   def : GCNPat<
1248     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1249               timm:$auxiliary, 0),
1250     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1251       (as_i16imm $offset), (extract_glc $auxiliary),
1252       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1253       (extract_swz $auxiliary))
1254   >;
1255
1256   def : GCNPat<
1257     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1258               timm:$auxiliary, timm),
1259     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1260       (as_i16imm $offset), (extract_glc $auxiliary),
1261       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1262       (extract_swz $auxiliary))
1263   >;
1264
1265   def : GCNPat<
1266     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1267               timm:$auxiliary, timm),
1268     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
1269       $vdata,
1270       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1271       $rsrc, $soffset, (as_i16imm $offset), (extract_glc $auxiliary),
1272       (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
1273       (extract_swz $auxiliary))
1274   >;
1275 }
1276
1277 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
1278 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
1279 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
1280 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
1281 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
1282 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
1283 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
1284 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
1285
1286 let SubtargetPredicate = HasUnpackedD16VMem in {
1287   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1288   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1289   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
1290   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1291 } // End HasUnpackedD16VMem.
1292
1293 let SubtargetPredicate = HasPackedD16VMem in {
1294   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
1295   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
1296   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
1297   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
1298   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1299   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1300 } // End HasPackedD16VMem.
1301
1302 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
1303 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
1304 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">;
1305 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">;
1306 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
1307 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
1308 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">;
1309 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">;
1310 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
1311 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
1312 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
1313 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
1314 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
1315 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
1316
1317 //===----------------------------------------------------------------------===//
1318 // buffer_atomic patterns
1319 //===----------------------------------------------------------------------===//
1320
1321 multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
1322                                 string opcode> {
1323   def : GCNPat<
1324     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1325           0, i32:$soffset, timm:$offset,
1326           timm:$cachepolicy, 0)),
1327     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset,
1328                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1329   >;
1330
1331   def : GCNPat<
1332     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1333           0, i32:$soffset, timm:$offset,
1334           timm:$cachepolicy, timm)),
1335     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset,
1336                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1337   >;
1338
1339   def : GCNPat<
1340     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1341           i32:$voffset, i32:$soffset, timm:$offset,
1342           timm:$cachepolicy, 0)),
1343     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset,
1344                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1345   >;
1346
1347   def : GCNPat<
1348     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1349           i32:$voffset, i32:$soffset, timm:$offset,
1350           timm:$cachepolicy, timm)),
1351     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1352       $vdata_in,
1353       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1354       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1355   >;
1356 }
1357
1358 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">;
1359 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">;
1360 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">;
1361 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">;
1362 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">;
1363 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">;
1364 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">;
1365 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">;
1366 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">;
1367 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">;
1368 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">;
1369 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">;
1370 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">;
1371 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64,  "BUFFER_ATOMIC_ADD_X2">;
1372 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">;
1373 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
1374 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
1375 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
1376 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
1377 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
1378 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
1379 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
1380 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">;
1381 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">;
1382
1383 multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
1384                                        string opcode> {
1385   def : GCNPat<
1386     (name vt:$vdata_in, v4i32:$rsrc, 0,
1387           0, i32:$soffset, timm:$offset,
1388           timm:$cachepolicy, 0),
1389     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
1390                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1391   >;
1392
1393   def : GCNPat<
1394     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1395           0, i32:$soffset, timm:$offset,
1396           timm:$cachepolicy, timm),
1397     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
1398                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1399   >;
1400
1401   def : GCNPat<
1402     (name vt:$vdata_in, v4i32:$rsrc, 0,
1403           i32:$voffset, i32:$soffset, timm:$offset,
1404           timm:$cachepolicy, 0),
1405     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
1406                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1407   >;
1408
1409   def : GCNPat<
1410     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1411           i32:$voffset, i32:$soffset, timm:$offset,
1412           timm:$cachepolicy, timm),
1413     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1414       $vdata_in,
1415       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1416       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1417   >;
1418 }
1419
1420 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
1421 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1422
1423 def : GCNPat<
1424   (SIbuffer_atomic_cmpswap
1425       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1426       0, i32:$soffset, timm:$offset,
1427       timm:$cachepolicy, 0),
1428   (EXTRACT_SUBREG
1429     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1430       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1431       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1432     sub0)
1433 >;
1434
1435 def : GCNPat<
1436   (SIbuffer_atomic_cmpswap
1437       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1438       0, i32:$soffset, timm:$offset,
1439       timm:$cachepolicy, timm),
1440   (EXTRACT_SUBREG
1441     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1442       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1443       $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1444     sub0)
1445 >;
1446
1447 def : GCNPat<
1448   (SIbuffer_atomic_cmpswap
1449       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1450       i32:$voffset, i32:$soffset, timm:$offset,
1451       timm:$cachepolicy, 0),
1452   (EXTRACT_SUBREG
1453     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1454       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1455       $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1456     sub0)
1457 >;
1458
1459 def : GCNPat<
1460   (SIbuffer_atomic_cmpswap
1461       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1462       i32:$voffset, i32:$soffset, timm:$offset,
1463       timm:$cachepolicy, timm),
1464   (EXTRACT_SUBREG
1465     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1466       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1467       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1468       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1469     sub0)
1470 >;
1471
1472 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1473                               PatFrag constant_ld> : GCNPat <
1474      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1475                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
1476      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1477   >;
1478
1479 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1480                                      ValueType vt, PatFrag atomic_ld> {
1481   def : GCNPat <
1482      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1483                                    i16:$offset, i1:$slc))),
1484      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
1485   >;
1486
1487   def : GCNPat <
1488     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1489     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
1490   >;
1491 }
1492
1493 let SubtargetPredicate = isGFX6GFX7 in {
1494 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1495 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
1496 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
1497 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1498 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
1499 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
1500
1501 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
1502 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
1503 } // End SubtargetPredicate = isGFX6GFX7
1504
1505 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1506                                PatFrag ld> {
1507
1508   def : GCNPat <
1509     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1510                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
1511     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1512   >;
1513 }
1514
1515 let OtherPredicates = [Has16BitInsts] in {
1516
1517 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1518 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>;
1519 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>;
1520 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>;
1521 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>;
1522 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>;
1523
1524 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>;
1525
1526 } // End OtherPredicates = [Has16BitInsts]
1527
1528 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1529                                 MUBUF_Pseudo InstrOffset,
1530                                 ValueType vt, PatFrag ld> {
1531   def : GCNPat <
1532     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1533                                i32:$soffset, u16imm:$offset))),
1534     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1535   >;
1536
1537   def : GCNPat <
1538     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1539     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1540   >;
1541 }
1542
1543 // XXX - Is it possible to have a complex pattern in a PatFrag?
1544 multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
1545                                 MUBUF_Pseudo InstrOffset,
1546                                 ValueType vt, PatFrag ld_frag> {
1547   def : GCNPat <
1548     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
1549     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
1550   >;
1551
1552   def : GCNPat <
1553     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
1554     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
1555   >;
1556 }
1557
1558 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1559 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1560 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>;
1561 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1562 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1563 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>;
1564 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1565 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1566 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
1567 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1568
1569 foreach vt = Reg32Types.types in {
1570 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1571 }
1572 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1573 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
1574 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1575
1576 let OtherPredicates = [D16PreservesUnusedBits] in {
1577 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>;
1578 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>;
1579 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>;
1580 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>;
1581 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>;
1582 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>;
1583
1584 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>;
1585 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>;
1586 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>;
1587 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>;
1588 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>;
1589 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>;
1590 }
1591
1592 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1593                                       ValueType vt, PatFrag atomic_st> {
1594   // Store follows atomic op convention so address is first
1595   def : GCNPat <
1596      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1597                                    i16:$offset, i1:$slc), vt:$val),
1598      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
1599   >;
1600
1601   def : GCNPat <
1602     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1603     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
1604   >;
1605 }
1606 let SubtargetPredicate = isGFX6GFX7 in {
1607 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>;
1608 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>;
1609 } // End Predicates = isGFX6GFX7
1610
1611
1612 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1613                                PatFrag st> {
1614
1615   def : GCNPat <
1616     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1617                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)),
1618     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
1619   >;
1620 }
1621
1622 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1623 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1624
1625 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1626                                  MUBUF_Pseudo InstrOffset,
1627                                  ValueType vt, PatFrag st,
1628                                  RegisterClass rc = VGPR_32> {
1629   def : GCNPat <
1630     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1631                                       i32:$soffset, u16imm:$offset)),
1632     (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1633   >;
1634
1635   def : GCNPat <
1636     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1637                                        u16imm:$offset)),
1638     (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
1639   >;
1640 }
1641
1642 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1643 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1644 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1645 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1646
1647 foreach vt = Reg32Types.types in {
1648 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>;
1649 }
1650
1651 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
1652 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
1653 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
1654
1655
1656 let OtherPredicates = [D16PreservesUnusedBits] in {
1657  // Hiding the extract high pattern in the PatFrag seems to not
1658  // automatically increase the complexity.
1659 let AddedComplexity = 1 in {
1660 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1661 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1662 }
1663 }
1664
1665 //===----------------------------------------------------------------------===//
1666 // MTBUF Patterns
1667 //===----------------------------------------------------------------------===//
1668
1669 //===----------------------------------------------------------------------===//
1670 // tbuffer_load/store_format patterns
1671 //===----------------------------------------------------------------------===//
1672
1673 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1674                                   string opcode> {
1675   def : GCNPat<
1676     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1677               timm:$format, timm:$auxiliary, 0)),
1678     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1679       (as_i8imm $format),
1680       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1681       (extract_swz $auxiliary))
1682   >;
1683
1684   def : GCNPat<
1685     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1686               timm:$format, timm:$auxiliary, timm)),
1687     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1688       (as_i8imm $format),
1689       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1690       (extract_swz $auxiliary))
1691   >;
1692
1693   def : GCNPat<
1694     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1695               timm:$format, timm:$auxiliary, 0)),
1696     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1697       (as_i8imm $format),
1698       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1699       (extract_swz $auxiliary))
1700   >;
1701
1702   def : GCNPat<
1703     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1704               timm:$format, timm:$auxiliary, timm)),
1705     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1706       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1707       $rsrc, $soffset, (as_i16imm $offset),
1708       (as_i8imm $format),
1709       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1710       (extract_swz $auxiliary))
1711   >;
1712 }
1713
1714 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1715 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1716 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
1717 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1718 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1719 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1720 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
1721 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1722
1723 let SubtargetPredicate = HasUnpackedD16VMem in {
1724   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1725   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1726   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1727 } // End HasUnpackedD16VMem.
1728
1729 let SubtargetPredicate = HasPackedD16VMem in {
1730   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
1731   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
1732   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
1733 } // End HasPackedD16VMem.
1734
1735 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1736                                    string opcode> {
1737   def : GCNPat<
1738     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1739           timm:$format, timm:$auxiliary, 0),
1740     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1741       (as_i16imm $offset), (as_i8imm $format),
1742       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1743       (extract_swz $auxiliary))
1744   >;
1745
1746   def : GCNPat<
1747     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1748           timm:$format, timm:$auxiliary, timm),
1749     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1750       (as_i16imm $offset), (as_i8imm $format),
1751       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1752       (extract_swz $auxiliary))
1753   >;
1754
1755   def : GCNPat<
1756     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1757           timm:$format, timm:$auxiliary, 0),
1758     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1759       (as_i16imm $offset), (as_i8imm $format),
1760       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1761       (extract_swz $auxiliary))
1762   >;
1763
1764   def : GCNPat<
1765     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1766           timm:$offset, timm:$format, timm:$auxiliary, timm),
1767     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1768       $vdata,
1769       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1770       $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format),
1771       (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
1772       (extract_swz $auxiliary))
1773   >;
1774 }
1775
1776 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1777 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1778 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
1779 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1780 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1781 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1782 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
1783 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1784
1785 let SubtargetPredicate = HasUnpackedD16VMem in {
1786   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1787   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
1788   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1789 } // End HasUnpackedD16VMem.
1790
1791 let SubtargetPredicate = HasPackedD16VMem in {
1792   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
1793   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
1794   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
1795 } // End HasPackedD16VMem.
1796
1797 //===----------------------------------------------------------------------===//
1798 // Target-specific instruction encodings.
1799 //===----------------------------------------------------------------------===//
1800
1801 //===----------------------------------------------------------------------===//
1802 // Base ENC_MUBUF for GFX6, GFX7, GFX10.
1803 //===----------------------------------------------------------------------===//
1804
1805 class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
1806     MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1807   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1808   let Inst{12}    = ps.offen;
1809   let Inst{13}    = ps.idxen;
1810   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1811   let Inst{16}    = !if(ps.lds, 1, 0);
1812   let Inst{24-18} = op;
1813   let Inst{31-26} = 0x38;
1814   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1815   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1816   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1817   let Inst{54}    = !if(ps.has_slc, slc, ?);
1818   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1819   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1820 }
1821
1822 class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> :
1823     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
1824   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
1825   let Inst{25} = op{7};
1826 }
1827
1828 class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
1829     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
1830   let Inst{15} = ps.addr64;
1831 }
1832
1833 //===----------------------------------------------------------------------===//
1834 // MUBUF - GFX10.
1835 //===----------------------------------------------------------------------===//
1836
1837 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
1838   multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName,
1839                                         string asmName> {
1840     def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> {
1841       MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName);
1842       let AsmString = asmName # ps.AsmOperands;
1843     }
1844   }
1845   multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
1846     def _BOTHEN_gfx10 :
1847       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1848     def _IDXEN_gfx10 :
1849       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1850     def _OFFEN_gfx10 :
1851       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1852     def _OFFSET_gfx10 :
1853       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1854   }
1855   multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> {
1856     def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1857                         MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">;
1858     def _OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1859                         MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">;
1860     def _IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1861                         MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">;
1862     def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1863                         MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">;
1864
1865     def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1866                             MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">;
1867     def _LDS_OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1868                             MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">;
1869     def _LDS_IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1870                             MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">;
1871     def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1872                             MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">;
1873   }
1874   multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
1875       MUBUF_Real_AllAddr_gfx10<op> {
1876     def _BOTHEN_RTN_gfx10 :
1877       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1878     def _IDXEN_RTN_gfx10 :
1879       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1880     def _OFFEN_RTN_gfx10 :
1881       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1882     def _OFFSET_RTN_gfx10 :
1883       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1884   }
1885 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
1886
1887 defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x019>;
1888 defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx10<0x01b>;
1889 defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x020>;
1890 defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x021>;
1891 defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x022>;
1892 defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x023>;
1893 defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx10<0x024>;
1894 defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x025>;
1895 // FIXME-GFX10: Add following instructions:
1896 //defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx10<0x026>;
1897 //defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
1898 defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx10<0x080>;
1899 defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx10<0x081>;
1900 defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx10<0x082>;
1901 defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx10<0x083>;
1902 defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx10<0x084>;
1903 defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx10<0x085>;
1904 defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx10<0x086>;
1905 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
1906
1907 def BUFFER_GL0_INV_gfx10 :
1908   MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>;
1909 def BUFFER_GL1_INV_gfx10 :
1910   MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>;
1911
1912 //===----------------------------------------------------------------------===//
1913 // MUBUF - GFX6, GFX7, GFX10.
1914 //===----------------------------------------------------------------------===//
1915
1916 let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in {
1917   multiclass MUBUF_Real_gfx6<bits<8> op> {
1918     def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1919   }
1920 } // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6"
1921
1922 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1923   multiclass MUBUF_Real_gfx7<bits<8> op> {
1924     def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1925   }
1926 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1927
1928 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1929   multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
1930     def _ADDR64_gfx6_gfx7 :
1931       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1932     def _BOTHEN_gfx6_gfx7 :
1933       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1934     def _IDXEN_gfx6_gfx7 :
1935       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1936     def _OFFEN_gfx6_gfx7 :
1937       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1938     def _OFFSET_gfx6_gfx7 :
1939       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1940   }
1941   multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> {
1942     def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1943                             MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">;
1944     def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
1945                             MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">;
1946     def _OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1947                             MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">;
1948     def _IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1949                             MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">;
1950     def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1951                             MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">;
1952
1953     def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1954                                 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">;
1955     def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>,
1956                                 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">;
1957     def _LDS_OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1958                                 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">;
1959     def _LDS_IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1960                                 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">;
1961     def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1962                                 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">;
1963   }
1964   multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> :
1965       MUBUF_Real_AllAddr_gfx6_gfx7<op> {
1966     def _ADDR64_RTN_gfx6_gfx7 :
1967       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>;
1968     def _BOTHEN_RTN_gfx6_gfx7 :
1969       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1970     def _IDXEN_RTN_gfx6_gfx7 :
1971       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1972     def _OFFEN_RTN_gfx6_gfx7 :
1973       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1974     def _OFFSET_RTN_gfx6_gfx7 :
1975       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1976   }
1977 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1978
1979 multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
1980   MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
1981
1982 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> :
1983   MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>;
1984
1985 multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
1986   MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
1987
1988 // FIXME-GFX6: Following instructions are available only on GFX6.
1989 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomics_gfx6 <0x034>;
1990 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomics_gfx6 <0x054>;
1991
1992 defm BUFFER_LOAD_FORMAT_X     : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
1993 defm BUFFER_LOAD_FORMAT_XY    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
1994 defm BUFFER_LOAD_FORMAT_XYZ   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
1995 defm BUFFER_LOAD_FORMAT_XYZW  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
1996 defm BUFFER_STORE_FORMAT_X    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
1997 defm BUFFER_STORE_FORMAT_XY   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
1998 defm BUFFER_STORE_FORMAT_XYZ  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
1999 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2000 defm BUFFER_LOAD_UBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
2001 defm BUFFER_LOAD_SBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
2002 defm BUFFER_LOAD_USHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
2003 defm BUFFER_LOAD_SSHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
2004 defm BUFFER_LOAD_DWORD        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
2005 defm BUFFER_LOAD_DWORDX2      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
2006 defm BUFFER_LOAD_DWORDX4      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
2007 defm BUFFER_LOAD_DWORDX3      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
2008 defm BUFFER_STORE_BYTE        : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
2009 defm BUFFER_STORE_SHORT       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
2010 defm BUFFER_STORE_DWORD       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
2011 defm BUFFER_STORE_DWORDX2     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
2012 defm BUFFER_STORE_DWORDX4     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
2013 defm BUFFER_STORE_DWORDX3     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
2014
2015 defm BUFFER_ATOMIC_SWAP        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
2016 defm BUFFER_ATOMIC_CMPSWAP     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
2017 defm BUFFER_ATOMIC_ADD         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
2018 defm BUFFER_ATOMIC_SUB         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
2019 defm BUFFER_ATOMIC_SMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
2020 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
2021 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
2022 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
2023 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
2024 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
2025 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
2026 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
2027 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
2028 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
2029 //defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
2030 //defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
2031 //defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
2032 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
2033 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
2034 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
2035 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
2036 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
2037 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
2038 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
2039 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
2040 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
2041 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
2042 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
2043 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
2044 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
2045 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
2046 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
2047 //defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
2048 //defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
2049 //defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
2050
2051 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
2052 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
2053 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
2054
2055 //===----------------------------------------------------------------------===//
2056 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
2057 //===----------------------------------------------------------------------===//
2058
2059 class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
2060     MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
2061   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2062   let Inst{12}    = ps.offen;
2063   let Inst{13}    = ps.idxen;
2064   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2065   let Inst{18-16} = op;
2066   let Inst{31-26} = 0x3a; //encoding
2067   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2068   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2069   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2070   let Inst{54}    = !if(ps.has_slc, slc, ?);
2071   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2072   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2073 }
2074
2075 //===----------------------------------------------------------------------===//
2076 // MTBUF - GFX10.
2077 //===----------------------------------------------------------------------===//
2078
2079 class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> :
2080     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
2081   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
2082   let Inst{25-19} = format;
2083   let Inst{53} = op{3};
2084 }
2085
2086 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
2087   multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
2088     def _BOTHEN_gfx10 :
2089       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2090     def _IDXEN_gfx10 :
2091       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2092     def _OFFEN_gfx10 :
2093       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2094     def _OFFSET_gfx10 :
2095       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2096   }
2097 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
2098
2099 defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx10<0x008>;
2100 defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx10<0x009>;
2101 defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx10<0x00a>;
2102 defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx10<0x00b>;
2103 defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx10<0x00c>;
2104 defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx10<0x00d>;
2105 defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx10<0x00e>;
2106 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
2107
2108 //===----------------------------------------------------------------------===//
2109 // MTBUF - GFX6, GFX7, GFX10.
2110 //===----------------------------------------------------------------------===//
2111
2112 class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> :
2113     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
2114   let Inst{15} = ps.addr64;
2115   let Inst{22-19} = dfmt;
2116   let Inst{25-23} = nfmt;
2117 }
2118
2119 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2120   multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
2121     def _ADDR64_gfx6_gfx7 :
2122       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
2123     def _BOTHEN_gfx6_gfx7 :
2124       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2125     def _IDXEN_gfx6_gfx7 :
2126       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2127     def _OFFEN_gfx6_gfx7 :
2128       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2129     def _OFFSET_gfx6_gfx7 :
2130       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2131   }
2132 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2133
2134 multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
2135   MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
2136
2137 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
2138 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2139 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2140 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2141 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2142 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2143 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2144 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2145
2146 //===----------------------------------------------------------------------===//
2147 // GFX8, GFX9 (VI).
2148 //===----------------------------------------------------------------------===//
2149
2150 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
2151   MUBUF_Real<ps>,
2152   Enc64,
2153   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2154   let AssemblerPredicate = isGFX8GFX9;
2155   let DecoderNamespace = "GFX8";
2156
2157   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2158   let Inst{12}    = ps.offen;
2159   let Inst{13}    = ps.idxen;
2160   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2161   let Inst{16}    = !if(ps.lds, 1, 0);
2162   let Inst{17}    = !if(ps.has_slc, slc, ?);
2163   let Inst{24-18} = op;
2164   let Inst{31-26} = 0x38; //encoding
2165   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2166   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2167   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2168   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2169   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2170 }
2171
2172 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
2173   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2174   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2175   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2176   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2177 }
2178
2179 multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
2180
2181   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2182                    MUBUFLdsTable<0, NAME # "_OFFSET_vi">;
2183   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2184                    MUBUFLdsTable<0, NAME # "_OFFEN_vi">;
2185   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2186                    MUBUFLdsTable<0, NAME # "_IDXEN_vi">;
2187   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2188                    MUBUFLdsTable<0, NAME # "_BOTHEN_vi">;
2189
2190   def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2191                        MUBUFLdsTable<1, NAME # "_OFFSET_vi">;
2192   def _LDS_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2193                        MUBUFLdsTable<1, NAME # "_OFFEN_vi">;
2194   def _LDS_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2195                        MUBUFLdsTable<1, NAME # "_IDXEN_vi">;
2196   def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2197                        MUBUFLdsTable<1, NAME # "_BOTHEN_vi">;
2198 }
2199
2200 class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> :
2201   MUBUF_Real<ps>,
2202   Enc64,
2203   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2204   let AssemblerPredicate=HasUnpackedD16VMem;
2205   let DecoderNamespace="GFX80_UNPACKED";
2206
2207   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2208   let Inst{12}    = ps.offen;
2209   let Inst{13}    = ps.idxen;
2210   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2211   let Inst{16}    = !if(ps.lds, 1, 0);
2212   let Inst{17}    = !if(ps.has_slc, slc, ?);
2213   let Inst{24-18} = op;
2214   let Inst{31-26} = 0x38; //encoding
2215   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2216   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2217   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2218   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2219   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2220 }
2221
2222 multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
2223   def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2224   def _OFFEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2225   def _IDXEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2226   def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2227 }
2228
2229 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
2230   MUBUF_Real_AllAddr_vi<op> {
2231   def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
2232   def _OFFEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
2233   def _IDXEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
2234   def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
2235 }
2236
2237 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_Lds_vi <0x00>;
2238 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
2239 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
2240 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
2241 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
2242 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
2243 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
2244 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
2245 let SubtargetPredicate = HasUnpackedD16VMem in {
2246   defm BUFFER_LOAD_FORMAT_D16_X_gfx80       : MUBUF_Real_AllAddr_gfx80 <0x08>;
2247   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x09>;
2248   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0a>;
2249   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0b>;
2250   defm BUFFER_STORE_FORMAT_D16_X_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x0c>;
2251   defm BUFFER_STORE_FORMAT_D16_XY_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0d>;
2252   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0e>;
2253   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80   : MUBUF_Real_AllAddr_gfx80 <0x0f>;
2254 } // End HasUnpackedD16VMem.
2255 let SubtargetPredicate = HasPackedD16VMem in {
2256   defm BUFFER_LOAD_FORMAT_D16_X       : MUBUF_Real_AllAddr_vi <0x08>;
2257   defm BUFFER_LOAD_FORMAT_D16_XY      : MUBUF_Real_AllAddr_vi <0x09>;
2258   defm BUFFER_LOAD_FORMAT_D16_XYZ     : MUBUF_Real_AllAddr_vi <0x0a>;
2259   defm BUFFER_LOAD_FORMAT_D16_XYZW    : MUBUF_Real_AllAddr_vi <0x0b>;
2260   defm BUFFER_STORE_FORMAT_D16_X      : MUBUF_Real_AllAddr_vi <0x0c>;
2261   defm BUFFER_STORE_FORMAT_D16_XY     : MUBUF_Real_AllAddr_vi <0x0d>;
2262   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
2263   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
2264 } // End HasPackedD16VMem.
2265 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
2266 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
2267 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
2268 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
2269 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
2270 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
2271 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
2272 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
2273 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
2274 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
2275 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
2276 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
2277 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
2278 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
2279 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
2280 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
2281
2282 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
2283 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
2284 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
2285 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
2286 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
2287 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
2288
2289 defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_vi <0x26>;
2290 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
2291
2292 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
2293 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
2294 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
2295 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
2296 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
2297 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
2298 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
2299 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
2300 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
2301 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
2302 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
2303 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
2304 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
2305
2306 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
2307 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
2308 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
2309 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
2310 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
2311 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
2312 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
2313 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
2314 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
2315 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
2316 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
2317 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
2318 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
2319
2320 def BUFFER_STORE_LDS_DWORD_vi   : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>;
2321
2322 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
2323 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
2324
2325 let SubtargetPredicate = HasAtomicFaddInsts in {
2326
2327 defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_AllAddr_vi <0x4d>;
2328 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>;
2329
2330 } // End SubtargetPredicate = HasAtomicFaddInsts
2331
2332 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
2333   MTBUF_Real<ps>,
2334   Enc64,
2335   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2336   let AssemblerPredicate = isGFX8GFX9;
2337   let DecoderNamespace = "GFX8";
2338
2339   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2340   let Inst{12}    = ps.offen;
2341   let Inst{13}    = ps.idxen;
2342   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2343   let Inst{18-15} = op;
2344   let Inst{22-19} = dfmt;
2345   let Inst{25-23} = nfmt;
2346   let Inst{31-26} = 0x3a; //encoding
2347   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2348   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2349   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2350   let Inst{54}    = !if(ps.has_slc, slc, ?);
2351   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2352   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2353 }
2354
2355 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
2356   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2357   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2358   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2359   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2360 }
2361
2362 class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> :
2363   MTBUF_Real<ps>,
2364   Enc64,
2365   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2366   let AssemblerPredicate=HasUnpackedD16VMem;
2367   let DecoderNamespace="GFX80_UNPACKED";
2368
2369   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2370   let Inst{12}    = ps.offen;
2371   let Inst{13}    = ps.idxen;
2372   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2373   let Inst{18-15} = op;
2374   let Inst{22-19} = dfmt;
2375   let Inst{25-23} = nfmt;
2376   let Inst{31-26} = 0x3a; //encoding
2377   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2378   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2379   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2380   let Inst{54}    = !if(ps.has_slc, slc, ?);
2381   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2382   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2383 }
2384
2385 multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
2386   def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2387   def _OFFEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2388   def _IDXEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2389   def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2390 }
2391
2392 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0x00>;
2393 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <0x01>;
2394 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <0x02>;
2395 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <0x03>;
2396 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <0x04>;
2397 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <0x05>;
2398 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <0x06>;
2399 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
2400 let SubtargetPredicate = HasUnpackedD16VMem in {
2401   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Real_AllAddr_gfx80 <0x08>;
2402   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x09>;
2403   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0a>;
2404   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0b>;
2405   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x0c>;
2406   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0d>;
2407   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0e>;
2408   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
2409 } // End HasUnpackedD16VMem.
2410 let SubtargetPredicate = HasPackedD16VMem in {
2411   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_vi <0x08>;
2412   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_vi <0x09>;
2413   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_vi <0x0a>;
2414   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_vi <0x0b>;
2415   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_vi <0x0c>;
2416   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_vi <0x0d>;
2417   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_vi <0x0e>;
2418   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
2419 } // End HasUnpackedD16VMem.
2420
2421 def MUBUFInfoTable : GenericTable {
2422   let FilterClass = "MUBUF_Pseudo";
2423   let CppTypeName = "MUBUFInfo";
2424   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2425
2426   let PrimaryKey = ["Opcode"];
2427   let PrimaryKeyName = "getMUBUFOpcodeHelper";
2428 }
2429
2430 def getMUBUFInfoFromOpcode : SearchIndex {
2431   let Table = MUBUFInfoTable;
2432   let Key = ["Opcode"];
2433 }
2434
2435 def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2436   let Table = MUBUFInfoTable;
2437   let Key = ["BaseOpcode", "elements"];
2438 }
2439
2440 def MTBUFInfoTable : GenericTable {
2441   let FilterClass = "MTBUF_Pseudo";
2442   let CppTypeName = "MTBUFInfo";
2443   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2444
2445   let PrimaryKey = ["Opcode"];
2446   let PrimaryKeyName = "getMTBUFOpcodeHelper";
2447 }
2448
2449 def getMTBUFInfoFromOpcode : SearchIndex {
2450   let Table = MTBUFInfoTable;
2451   let Key = ["Opcode"];
2452 }
2453
2454 def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2455   let Table = MTBUFInfoTable;
2456   let Key = ["BaseOpcode", "elements"];
2457 }