lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  10 def MUBUFAddr64 : ComplexPattern<i64, 8, "SelectMUBUFAddr64">;
  11 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  12
  13 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  14 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  15
  16 def MUBUFOffset : ComplexPattern<i64, 7, "SelectMUBUFOffset">;
  17 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  18 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  19
  20 def BUFAddrKind {
  21   int Offset = 0;
  22   int OffEn  = 1;
  23   int IdxEn  = 2;
  24   int BothEn = 3;
  25   int Addr64 = 4;
  26 }
  27
  28 class getAddrName<int addrKind> {
  29   string ret =
  30     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  31     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  32     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  33     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  34     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  35     "")))));
  36 }
  37
  38 class MUBUFAddr64Table <bit is_addr64, string Name> {
  39   bit IsAddr64 = is_addr64;
  40   string OpName = Name;
  41 }
  42
  43 class MUBUFLdsTable <bit is_lds, string Name> {
  44   bit IsLds = is_lds;
  45   string OpName = Name;
  46 }
  47
  48 class MTBUFAddr64Table <bit is_addr64, string Name> {
  49   bit IsAddr64 = is_addr64;
  50   string OpName = Name;
  51 }
  52
  53 //===----------------------------------------------------------------------===//
  54 // MTBUF classes
  55 //===----------------------------------------------------------------------===//
  56
  57 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  58                     string asmOps, list<dag> pattern=[]> :
  59   InstSI<outs, ins, "", pattern>,
  60   SIMCInstr<opName, SIEncodingFamily.NONE> {
  61
  62   let isPseudo = 1;
  63   let isCodeGenOnly = 1;
  64   let Size = 8;
  65   let UseNamedOperandTable = 1;
  66
  67   string Mnemonic = opName;
  68   string AsmOperands = asmOps;
  69
  70   let VM_CNT = 1;
  71   let EXP_CNT = 1;
  72   let MTBUF = 1;
  73   let Uses = [EXEC];
  74   let hasSideEffects = 0;
  75   let SchedRW = [WriteVMEM];
  76
  77   let AsmMatchConverter = "cvtMtbuf";
  78
  79   bits<1> offen       = 0;
  80   bits<1> idxen       = 0;
  81   bits<1> addr64      = 0;
  82   bits<1> has_vdata   = 1;
  83   bits<1> has_vaddr   = 1;
  84   bits<1> has_glc     = 1;
  85   bits<1> has_dlc     = 1;
  86   bits<1> glc_value   = 0; // the value for glc if no such operand
  87   bits<1> dlc_value   = 0; // the value for dlc if no such operand
  88   bits<1> has_srsrc   = 1;
  89   bits<1> has_soffset = 1;
  90   bits<1> has_offset  = 1;
  91   bits<1> has_slc     = 1;
  92   bits<1> has_tfe     = 1;
  93 }
  94
  95 class MTBUF_Real <MTBUF_Pseudo ps> :
  96   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
  97
  98   let isPseudo = 0;
  99   let isCodeGenOnly = 0;
 100
 101   // copy relevant pseudo op flags
 102   let SubtargetPredicate = ps.SubtargetPredicate;
 103   let AsmMatchConverter  = ps.AsmMatchConverter;
 104   let Constraints        = ps.Constraints;
 105   let DisableEncoding    = ps.DisableEncoding;
 106   let TSFlags            = ps.TSFlags;
 107
 108   bits<12> offset;
 109   bits<1>  glc;
 110   bits<1>  dlc;
 111   bits<7>  format;
 112   bits<8>  vaddr;
 113   bits<8>  vdata;
 114   bits<7>  srsrc;
 115   bits<1>  slc;
 116   bits<1>  tfe;
 117   bits<8>  soffset;
 118
 119   bits<4> dfmt = format{3-0};
 120   bits<3> nfmt = format{6-4};
 121 }
 122
 123 class getMTBUFInsDA<list<RegisterClass> vdataList,
 124                     list<RegisterClass> vaddrList=[]> {
 125   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 126   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 127   dag InsNoData = !if(!empty(vaddrList),
 128     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 129          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc),
 130     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 131          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc)
 132   );
 133   dag InsData = !if(!empty(vaddrList),
 134     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 135          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 136          SLC:$slc, TFE:$tfe, DLC:$dlc),
 137     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 138          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 139          SLC:$slc, TFE:$tfe, DLC:$dlc)
 140   );
 141   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 142 }
 143
 144 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 145   dag ret =
 146     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 147     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 148     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 149     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 150     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 151     (ins))))));
 152 }
 153
 154 class getMTBUFAsmOps<int addrKind> {
 155   string Pfx =
 156     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset",
 157     !if(!eq(addrKind, BUFAddrKind.OffEn),
 158             "$vaddr, $srsrc, $format, $soffset offen",
 159     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 160             "$vaddr, $srsrc, $format, $soffset idxen",
 161     !if(!eq(addrKind, BUFAddrKind.BothEn),
 162             "$vaddr, $srsrc, $format, $soffset idxen offen",
 163     !if(!eq(addrKind, BUFAddrKind.Addr64),
 164             "$vaddr, $srsrc, $format, $soffset addr64",
 165     "")))));
 166   string ret = Pfx # "$offset";
 167 }
 168
 169 class MTBUF_SetupAddr<int addrKind> {
 170   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 171                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 172
 173   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 174                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 175
 176   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 177
 178   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 179 }
 180
 181 class MTBUF_Load_Pseudo <string opName,
 182                          int addrKind,
 183                          RegisterClass vdataClass,
 184                          list<dag> pattern=[],
 185                          // Workaround bug bz30254
 186                          int addrKindCopy = addrKind>
 187   : MTBUF_Pseudo<opName,
 188                  (outs vdataClass:$vdata),
 189                  getMTBUFIns<addrKindCopy>.ret,
 190                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 191                  pattern>,
 192     MTBUF_SetupAddr<addrKindCopy> {
 193   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 194   let mayLoad = 1;
 195   let mayStore = 0;
 196 }
 197
 198 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 199                               ValueType load_vt = i32,
 200                               SDPatternOperator ld = null_frag> {
 201
 202   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 203     [(set load_vt:$vdata,
 204      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format,
 205                       i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
 206     MTBUFAddr64Table<0, NAME>;
 207
 208   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 209     [(set load_vt:$vdata,
 210      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 211                       i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
 212     MTBUFAddr64Table<1, NAME>;
 213
 214   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 215   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 216   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 217
 218   let DisableWQM = 1 in {
 219     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 220     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 221     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 222     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 223   }
 224 }
 225
 226 class MTBUF_Store_Pseudo <string opName,
 227                           int addrKind,
 228                           RegisterClass vdataClass,
 229                           list<dag> pattern=[],
 230                           // Workaround bug bz30254
 231                           int addrKindCopy = addrKind,
 232                           RegisterClass vdataClassCopy = vdataClass>
 233   : MTBUF_Pseudo<opName,
 234                  (outs),
 235                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 236                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 237                  pattern>,
 238     MTBUF_SetupAddr<addrKindCopy> {
 239   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 240   let mayLoad = 0;
 241   let mayStore = 1;
 242 }
 243
 244 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 245                                ValueType store_vt = i32,
 246                                SDPatternOperator st = null_frag> {
 247
 248   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 249     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 250                                        i16:$offset, i8:$format, i1:$glc,
 251                                        i1:$slc, i1:$tfe, i1:$dlc))]>,
 252     MTBUFAddr64Table<0, NAME>;
 253
 254   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 255     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 256                                        i16:$offset, i8:$format, i1:$glc,
 257                                        i1:$slc, i1:$tfe, i1:$dlc))]>,
 258     MTBUFAddr64Table<1, NAME>;
 259
 260   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 261   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 262   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 263
 264   let DisableWQM = 1 in {
 265     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 266     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 267     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 268     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 269   }
 270 }
 271
 272
 273 //===----------------------------------------------------------------------===//
 274 // MUBUF classes
 275 //===----------------------------------------------------------------------===//
 276
 277 class MUBUFGetBaseOpcode<string Op> {
 278   string ret = !subst("DWORDX2", "DWORD",
 279     !subst("DWORDX3", "DWORD",
 280     !subst("DWORDX4", "DWORD", Op)));
 281 }
 282
 283 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 284                     string asmOps, list<dag> pattern=[]> :
 285   InstSI<outs, ins, "", pattern>,
 286   SIMCInstr<opName, SIEncodingFamily.NONE> {
 287
 288   let isPseudo = 1;
 289   let isCodeGenOnly = 1;
 290   let Size = 8;
 291   let UseNamedOperandTable = 1;
 292
 293   string Mnemonic = opName;
 294   string AsmOperands = asmOps;
 295
 296   Instruction Opcode = !cast<Instruction>(NAME);
 297   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
 298
 299   let VM_CNT = 1;
 300   let EXP_CNT = 1;
 301   let MUBUF = 1;
 302   let Uses = [EXEC];
 303   let hasSideEffects = 0;
 304   let SchedRW = [WriteVMEM];
 305
 306   let AsmMatchConverter = "cvtMubuf";
 307
 308   bits<1> offen       = 0;
 309   bits<1> idxen       = 0;
 310   bits<1> addr64      = 0;
 311   bits<1> lds         = 0;
 312   bits<1> has_vdata   = 1;
 313   bits<1> has_vaddr   = 1;
 314   bits<1> has_glc     = 1;
 315   bits<1> has_dlc     = 1;
 316   bits<1> glc_value   = 0; // the value for glc if no such operand
 317   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 318   bits<1> has_srsrc   = 1;
 319   bits<1> has_soffset = 1;
 320   bits<1> has_offset  = 1;
 321   bits<1> has_slc     = 1;
 322   bits<1> has_tfe     = 1;
 323   bits<4> elements    = 0;
 324 }
 325
 326 class MUBUF_Real <MUBUF_Pseudo ps> :
 327   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 328
 329   let isPseudo = 0;
 330   let isCodeGenOnly = 0;
 331
 332   // copy relevant pseudo op flags
 333   let SubtargetPredicate = ps.SubtargetPredicate;
 334   let AsmMatchConverter  = ps.AsmMatchConverter;
 335   let Constraints        = ps.Constraints;
 336   let DisableEncoding    = ps.DisableEncoding;
 337   let TSFlags            = ps.TSFlags;
 338
 339   bits<12> offset;
 340   bits<1>  glc;
 341   bits<1>  dlc;
 342   bits<8>  vaddr;
 343   bits<8>  vdata;
 344   bits<7>  srsrc;
 345   bits<1>  slc;
 346   bits<1>  tfe;
 347   bits<8>  soffset;
 348 }
 349
 350
 351 // For cache invalidation instructions.
 352 class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
 353   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 354
 355   let AsmMatchConverter = "";
 356
 357   let hasSideEffects = 1;
 358   let mayStore = 1;
 359
 360   // Set everything to 0.
 361   let offen       = 0;
 362   let idxen       = 0;
 363   let addr64      = 0;
 364   let has_vdata   = 0;
 365   let has_vaddr   = 0;
 366   let has_glc     = 0;
 367   let has_dlc     = 0;
 368   let glc_value   = 0;
 369   let dlc_value   = 0;
 370   let has_srsrc   = 0;
 371   let has_soffset = 0;
 372   let has_offset  = 0;
 373   let has_slc     = 0;
 374   let has_tfe     = 0;
 375 }
 376
 377 class getMUBUFInsDA<list<RegisterClass> vdataList,
 378                     list<RegisterClass> vaddrList=[],
 379                     bit isLds = 0> {
 380   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 381   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 382   dag InsNoData = !if(!empty(vaddrList),
 383     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 384          offset:$offset, GLC:$glc, SLC:$slc),
 385     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 386          offset:$offset, GLC:$glc, SLC:$slc)
 387   );
 388   dag InsData = !if(!empty(vaddrList),
 389     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 390          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 391     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 392          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc)
 393   );
 394   dag ret = !con(
 395               !if(!empty(vdataList), InsNoData, InsData),
 396               !if(isLds, (ins DLC:$dlc), (ins TFE:$tfe, DLC:$dlc))
 397              );
 398 }
 399
 400 class getMUBUFElements<ValueType vt> {
 401   // eq does not support ValueType for some reason.
 402   string vtAsStr = !cast<string>(vt);
 403
 404   int ret =
 405     !if(!eq(vtAsStr, "f16"), 1,
 406       !if(!eq(vtAsStr, "v2f16"), 2,
 407         !if(!eq(vtAsStr, "v3f16"), 3,
 408           !if(!eq(vtAsStr, "v4f16"), 4,
 409             !if(!eq(vt.Size, 32), 1,
 410               !if(!eq(vt.Size, 64), 2,
 411                 !if(!eq(vt.Size, 96), 3,
 412                   !if(!eq(vt.Size, 128), 4, 0)
 413                 )
 414               )
 415             )
 416           )
 417         )
 418       )
 419     );
 420 }
 421
 422 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> {
 423   dag ret =
 424     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret,
 425     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 426     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 427     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 428     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 429     (ins))))));
 430 }
 431
 432 class getMUBUFAsmOps<int addrKind> {
 433   string Pfx =
 434     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 435     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 436     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 437     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 438     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 439     "")))));
 440   string ret = Pfx # "$offset";
 441 }
 442
 443 class MUBUF_SetupAddr<int addrKind> {
 444   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 445                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 446
 447   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 448                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 449
 450   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 451
 452   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 453 }
 454
 455 class MUBUF_Load_Pseudo <string opName,
 456                          int addrKind,
 457                          ValueType vdata_vt,
 458                          bit HasTiedDest = 0,
 459                          bit isLds = 0,
 460                          list<dag> pattern=[],
 461                          // Workaround bug bz30254
 462                          int addrKindCopy = addrKind>
 463   : MUBUF_Pseudo<opName,
 464                  (outs getVregSrcForVT<vdata_vt>.ret:$vdata),
 465                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
 466                       !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))),
 467                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" #
 468                    !if(isLds, " lds", "$tfe") # "$dlc",
 469                  pattern>,
 470     MUBUF_SetupAddr<addrKindCopy> {
 471   let PseudoInstr = opName # !if(isLds, "_lds", "") #
 472                     "_" # getAddrName<addrKindCopy>.ret;
 473   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 474
 475   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 476   let mayLoad = 1;
 477   let mayStore = 0;
 478   let maybeAtomic = 1;
 479   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
 480   let has_tfe = !if(isLds, 0, 1);
 481   let lds = isLds;
 482   let elements = getMUBUFElements<vdata_vt>.ret;
 483 }
 484
 485 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
 486   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
 487   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
 488 >;
 489
 490 class MUBUF_Addr64_Load_Pat <Instruction inst,
 491                             ValueType load_vt = i32,
 492                             SDPatternOperator ld = null_frag> : Pat <
 493   (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
 494   (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
 495 >;
 496
 497 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
 498   def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
 499   def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
 500 }
 501
 502
 503 // FIXME: tfe can't be an operand because it requires a separate
 504 // opcode because it needs an N+1 register class dest register.
 505 multiclass MUBUF_Pseudo_Loads<string opName,
 506                               ValueType load_vt = i32,
 507                               SDPatternOperator ld = null_frag,
 508                               bit TiedDest = 0,
 509                               bit isLds = 0> {
 510
 511   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>,
 512     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 513
 514   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, load_vt, TiedDest, isLds>,
 515     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 516
 517   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 518   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 519   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 520
 521   let DisableWQM = 1 in {
 522     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, load_vt, TiedDest, isLds>;
 523     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, load_vt, TiedDest, isLds>;
 524     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, load_vt, TiedDest, isLds>;
 525     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, load_vt, TiedDest, isLds>;
 526   }
 527 }
 528
 529 multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32,
 530                                   SDPatternOperator ld_nolds = null_frag,
 531                                   SDPatternOperator ld_lds = null_frag> {
 532   defm NAME : MUBUF_Pseudo_Loads<opName, load_vt, ld_nolds>;
 533   defm _LDS : MUBUF_Pseudo_Loads<opName, load_vt, ld_lds, 0, 1>;
 534 }
 535
 536 class MUBUF_Store_Pseudo <string opName,
 537                           int addrKind,
 538                           ValueType store_vt,
 539                           list<dag> pattern=[],
 540                           // Workaround bug bz30254
 541                           int addrKindCopy = addrKind>
 542   : MUBUF_Pseudo<opName,
 543                  (outs),
 544                  getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret,
 545                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 546                  pattern>,
 547     MUBUF_SetupAddr<addrKindCopy> {
 548   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 549   let mayLoad = 0;
 550   let mayStore = 1;
 551   let maybeAtomic = 1;
 552   let elements = getMUBUFElements<store_vt>.ret;
 553 }
 554
 555 multiclass MUBUF_Pseudo_Stores<string opName,
 556                                ValueType store_vt = i32,
 557                                SDPatternOperator st = null_frag> {
 558
 559   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt,
 560     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 561                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
 562     MUBUFAddr64Table<0, NAME>;
 563
 564   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt,
 565     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 566                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
 567     MUBUFAddr64Table<1, NAME>;
 568
 569   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 570   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 571   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 572
 573   let DisableWQM = 1 in {
 574     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt>;
 575     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
 576     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, store_vt>;
 577     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, store_vt>;
 578   }
 579 }
 580
 581 class MUBUF_Pseudo_Store_Lds<string opName>
 582   : MUBUF_Pseudo<opName,
 583                  (outs),
 584                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 585                  " $srsrc, $soffset$offset lds$glc$slc"> {
 586   let mayLoad = 0;
 587   let mayStore = 1;
 588   let maybeAtomic = 1;
 589
 590   let has_vdata = 0;
 591   let has_vaddr = 0;
 592   let has_tfe = 0;
 593   let lds = 1;
 594
 595   let Uses = [EXEC, M0];
 596   let AsmMatchConverter = "cvtMubufLds";
 597 }
 598
 599 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 600                           list<RegisterClass> vaddrList=[]> {
 601   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 602   dag ret = !if(vdata_in,
 603     !if(!empty(vaddrList),
 604       (ins vdataClass:$vdata_in,
 605            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 606       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 607            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 608     ),
 609     !if(!empty(vaddrList),
 610       (ins vdataClass:$vdata,
 611            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 612       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 613            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 614   ));
 615 }
 616
 617 class getMUBUFAtomicIns<int addrKind,
 618                         RegisterClass vdataClass,
 619                         bit vdata_in,
 620                         // Workaround bug bz30254
 621                         RegisterClass vdataClassCopy=vdataClass> {
 622   dag ret =
 623     !if(!eq(addrKind, BUFAddrKind.Offset),
 624             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 625     !if(!eq(addrKind, BUFAddrKind.OffEn),
 626             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 627     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 628             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 629     !if(!eq(addrKind, BUFAddrKind.BothEn),
 630             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 631     !if(!eq(addrKind, BUFAddrKind.Addr64),
 632             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 633     (ins))))));
 634 }
 635
 636 class MUBUF_Atomic_Pseudo<string opName,
 637                           int addrKind,
 638                           dag outs,
 639                           dag ins,
 640                           string asmOps,
 641                           list<dag> pattern=[],
 642                           // Workaround bug bz30254
 643                           int addrKindCopy = addrKind>
 644   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 645     MUBUF_SetupAddr<addrKindCopy> {
 646   let mayStore = 1;
 647   let mayLoad = 1;
 648   let hasPostISelHook = 1;
 649   let hasSideEffects = 1;
 650   let DisableWQM = 1;
 651   let has_glc = 0;
 652   let has_dlc = 0;
 653   let has_tfe = 0;
 654   let maybeAtomic = 1;
 655 }
 656
 657 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 658                                RegisterClass vdataClass,
 659                                list<dag> pattern=[],
 660                                // Workaround bug bz30254
 661                                int addrKindCopy = addrKind,
 662                                RegisterClass vdataClassCopy = vdataClass>
 663   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 664                         (outs),
 665                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 666                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 667                         pattern>,
 668     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 669   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 670   let glc_value = 0;
 671   let dlc_value = 0;
 672   let AsmMatchConverter = "cvtMubufAtomic";
 673 }
 674
 675 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 676                              RegisterClass vdataClass,
 677                              list<dag> pattern=[],
 678                              // Workaround bug bz30254
 679                              int addrKindCopy = addrKind,
 680                              RegisterClass vdataClassCopy = vdataClass>
 681   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 682                         (outs vdataClassCopy:$vdata),
 683                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 684                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 685                         pattern>,
 686     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 687   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 688   let glc_value = 1;
 689   let dlc_value = 0;
 690   let Constraints = "$vdata = $vdata_in";
 691   let DisableEncoding = "$vdata_in";
 692   let AsmMatchConverter = "cvtMubufAtomicReturn";
 693 }
 694
 695 multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
 696                                         RegisterClass vdataClass,
 697                                         ValueType vdataType,
 698                                         SDPatternOperator atomic,
 699                                         bit isFP = getIsFP<vdataType>.ret> {
 700   let FPAtomic = isFP in
 701   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 702                 MUBUFAddr64Table <0, NAME>;
 703
 704   let FPAtomic = isFP in
 705   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 706                 MUBUFAddr64Table <1, NAME>;
 707
 708   let FPAtomic = isFP in
 709   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 710
 711   let FPAtomic = isFP in
 712
 713   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 714
 715   let FPAtomic = isFP in
 716   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 717 }
 718
 719 multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
 720                                      RegisterClass vdataClass,
 721                                      ValueType vdataType,
 722                                      SDPatternOperator atomic,
 723                                      bit isFP = getIsFP<vdataType>.ret> {
 724   let FPAtomic = isFP in
 725   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 726     [(set vdataType:$vdata,
 727      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 728              vdataType:$vdata_in))]>,
 729     MUBUFAddr64Table <0, NAME # "_RTN">;
 730
 731   let FPAtomic = isFP in
 732   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 733     [(set vdataType:$vdata,
 734      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 735              vdataType:$vdata_in))]>,
 736     MUBUFAddr64Table <1, NAME # "_RTN">;
 737
 738   let FPAtomic = isFP in
 739   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 740
 741   let FPAtomic = isFP in
 742   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 743
 744   let FPAtomic = isFP in
 745   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 746 }
 747
 748 multiclass MUBUF_Pseudo_Atomics <string opName,
 749                                  RegisterClass vdataClass,
 750                                  ValueType vdataType,
 751                                  SDPatternOperator atomic> :
 752   MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>,
 753   MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
 754
 755
 756 //===----------------------------------------------------------------------===//
 757 // MUBUF Instructions
 758 //===----------------------------------------------------------------------===//
 759
 760 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
 761   "buffer_load_format_x", f32
 762 >;
 763 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 764   "buffer_load_format_xy", v2f32
 765 >;
 766 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 767   "buffer_load_format_xyz", v3f32
 768 >;
 769 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 770   "buffer_load_format_xyzw", v4f32
 771 >;
 772 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 773   "buffer_store_format_x", f32
 774 >;
 775 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 776   "buffer_store_format_xy", v2f32
 777 >;
 778 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 779   "buffer_store_format_xyz", v3f32
 780 >;
 781 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 782   "buffer_store_format_xyzw", v4f32
 783 >;
 784
 785 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
 786   defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
 787     "buffer_load_format_d16_x", i32
 788   >;
 789   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
 790     "buffer_load_format_d16_xy", v2i32
 791   >;
 792   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
 793     "buffer_load_format_d16_xyz", v3i32
 794   >;
 795   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
 796    "buffer_load_format_d16_xyzw", v4i32
 797   >;
 798   defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
 799     "buffer_store_format_d16_x", i32
 800   >;
 801   defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
 802     "buffer_store_format_d16_xy", v2i32
 803   >;
 804   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
 805     "buffer_store_format_d16_xyz", v3i32
 806   >;
 807   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
 808     "buffer_store_format_d16_xyzw", v4i32
 809   >;
 810 } // End HasUnpackedD16VMem.
 811
 812 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
 813   defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
 814     "buffer_load_format_d16_x", f16
 815   >;
 816   defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
 817     "buffer_load_format_d16_xy", v2f16
 818   >;
 819   defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
 820     "buffer_load_format_d16_xyz", v3f16
 821   >;
 822   defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
 823     "buffer_load_format_d16_xyzw", v4f16
 824   >;
 825   defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
 826     "buffer_store_format_d16_x", f16
 827   >;
 828   defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
 829     "buffer_store_format_d16_xy", v2f16
 830   >;
 831   defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
 832     "buffer_store_format_d16_xyz", v3f16
 833   >;
 834   defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
 835     "buffer_store_format_d16_xyzw", v4f16
 836   >;
 837 } // End HasPackedD16VMem.
 838
 839 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
 840   "buffer_load_ubyte", i32
 841 >;
 842 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
 843   "buffer_load_sbyte", i32
 844 >;
 845 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
 846   "buffer_load_ushort", i32
 847 >;
 848 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
 849   "buffer_load_sshort", i32
 850 >;
 851 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
 852   "buffer_load_dword", i32
 853 >;
 854 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 855   "buffer_load_dwordx2", v2i32
 856 >;
 857 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 858   "buffer_load_dwordx3", v3i32
 859 >;
 860 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 861   "buffer_load_dwordx4", v4i32
 862 >;
 863
 864 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
 865 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
 866 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
 867 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
 868 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
 869 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
 870 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>;
 871 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>;
 872 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>;
 873 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>;
 874
 875 // This is not described in AMD documentation,
 876 // but 'lds' versions of these opcodes are available
 877 // in at least GFX8+ chips. See Bug 37653.
 878 let SubtargetPredicate = isGFX8GFX9 in {
 879 defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
 880   "buffer_load_dwordx2", v2i32, null_frag, 0, 1
 881 >;
 882 defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
 883   "buffer_load_dwordx3", v3i32, null_frag, 0, 1
 884 >;
 885 defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
 886   "buffer_load_dwordx4", v4i32, null_frag, 0, 1
 887 >;
 888 }
 889
 890 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 891   "buffer_store_byte", i32, truncstorei8_global
 892 >;
 893 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 894   "buffer_store_short", i32, truncstorei16_global
 895 >;
 896 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 897   "buffer_store_dword", i32, store_global
 898 >;
 899 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 900   "buffer_store_dwordx2", v2i32, store_global
 901 >;
 902 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 903   "buffer_store_dwordx3", v3i32, store_global
 904 >;
 905 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 906   "buffer_store_dwordx4", v4i32, store_global
 907 >;
 908 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 909   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32
 910 >;
 911 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 912   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 913 >;
 914 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 915   "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32
 916 >;
 917 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 918   "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32
 919 >;
 920 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 921   "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32
 922 >;
 923 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 924   "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32
 925 >;
 926 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 927   "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32
 928 >;
 929 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 930   "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32
 931 >;
 932 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 933   "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32
 934 >;
 935 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 936   "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32
 937 >;
 938 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 939   "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32
 940 >;
 941 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 942   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32
 943 >;
 944 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 945   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32
 946 >;
 947 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 948   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64
 949 >;
 950 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 951   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 952 >;
 953 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 954   "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64
 955 >;
 956 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 957   "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64
 958 >;
 959 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 960   "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64
 961 >;
 962 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 963   "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64
 964 >;
 965 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 966   "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64
 967 >;
 968 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 969   "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64
 970 >;
 971 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 972   "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64
 973 >;
 974 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 975   "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64
 976 >;
 977 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 978   "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64
 979 >;
 980 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
 981   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64
 982 >;
 983 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 984   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64
 985 >;
 986
 987 let SubtargetPredicate = isGFX8GFX9 in {
 988 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
 989 }
 990
 991 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
 992 /*
 993 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
 994 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
 995 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
 996 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 997 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
 998 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
 999 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
1000 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
1001 */
1002
1003 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
1004                                           int_amdgcn_buffer_wbinvl1_sc>;
1005 }
1006
1007 let SubtargetPredicate = HasD16LoadStore in {
1008
1009 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
1010   "buffer_load_ubyte_d16", i32, null_frag, 1
1011 >;
1012
1013 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
1014   "buffer_load_ubyte_d16_hi", i32, null_frag, 1
1015 >;
1016
1017 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
1018   "buffer_load_sbyte_d16", i32, null_frag, 1
1019 >;
1020
1021 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
1022   "buffer_load_sbyte_d16_hi", i32, null_frag, 1
1023 >;
1024
1025 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
1026   "buffer_load_short_d16", i32, null_frag, 1
1027 >;
1028
1029 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
1030   "buffer_load_short_d16_hi", i32, null_frag, 1
1031 >;
1032
1033 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
1034   "buffer_store_byte_d16_hi", i32
1035 >;
1036
1037 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
1038   "buffer_store_short_d16_hi", i32
1039 >;
1040
1041 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
1042   "buffer_load_format_d16_hi_x", i32
1043 >;
1044 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
1045   "buffer_store_format_d16_hi_x", i32
1046 >;
1047
1048 } // End HasD16LoadStore
1049
1050 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
1051                                        int_amdgcn_buffer_wbinvl1>;
1052
1053 let SubtargetPredicate = HasAtomicFaddInsts in {
1054
1055 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
1056   "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret
1057 >;
1058 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
1059   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret
1060 >;
1061
1062 } // End SubtargetPredicate = HasAtomicFaddInsts
1063
1064 //===----------------------------------------------------------------------===//
1065 // MTBUF Instructions
1066 //===----------------------------------------------------------------------===//
1067
1068 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;
1069 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64>;
1070 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96>;
1071 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128>;
1072 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32>;
1073 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64>;
1074 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96>;
1075 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>;
1076
1077 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
1078   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
1079   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64>;
1080   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96>;
1081   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128>;
1082   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
1083   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64>;
1084   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96>;
1085   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>;
1086 } // End HasUnpackedD16VMem.
1087
1088 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
1089   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
1090   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32>;
1091   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64>;
1092   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64>;
1093   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
1094   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32>;
1095   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64>;
1096   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>;
1097 } // End HasPackedD16VMem.
1098
1099 let SubtargetPredicate = isGFX7Plus in {
1100
1101 //===----------------------------------------------------------------------===//
1102 // Instruction definitions for CI and newer.
1103 //===----------------------------------------------------------------------===//
1104
1105 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
1106                                            int_amdgcn_buffer_wbinvl1_vol>;
1107
1108 } // End let SubtargetPredicate = isGFX7Plus
1109
1110 let SubtargetPredicate = isGFX10Plus in {
1111   def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
1112   def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
1113 } // End SubtargetPredicate = isGFX10Plus
1114
1115 //===----------------------------------------------------------------------===//
1116 // MUBUF Patterns
1117 //===----------------------------------------------------------------------===//
1118
1119 def extract_glc : SDNodeXForm<imm, [{
1120   return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8);
1121 }]>;
1122
1123 def extract_slc : SDNodeXForm<imm, [{
1124   return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8);
1125 }]>;
1126
1127 def extract_dlc : SDNodeXForm<imm, [{
1128   return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8);
1129 }]>;
1130
1131 //===----------------------------------------------------------------------===//
1132 // buffer_load/store_format patterns
1133 //===----------------------------------------------------------------------===//
1134
1135 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1136                                   string opcode> {
1137   def : GCNPat<
1138     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1139               timm:$cachepolicy, 0)),
1140     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1141       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1142   >;
1143
1144   def : GCNPat<
1145     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1146               timm:$cachepolicy, 0)),
1147     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1148       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1149   >;
1150
1151   def : GCNPat<
1152     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1153               timm:$cachepolicy, timm)),
1154     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1155       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1156   >;
1157
1158   def : GCNPat<
1159     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1160               timm:$cachepolicy, timm)),
1161     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1162       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1163       $rsrc, $soffset, (as_i16imm $offset),
1164       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1165   >;
1166 }
1167
1168 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
1169 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
1170 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
1171 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
1172 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
1173 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
1174 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
1175 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
1176
1177 let SubtargetPredicate = HasUnpackedD16VMem in {
1178   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1179   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1180   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1181   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1182 } // End HasUnpackedD16VMem.
1183
1184 let SubtargetPredicate = HasPackedD16VMem in {
1185   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
1186   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
1187   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
1188   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
1189   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1190   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1191 } // End HasPackedD16VMem.
1192
1193 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
1194 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
1195 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">;
1196 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">;
1197 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
1198 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
1199 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">;
1200 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">;
1201 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
1202 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
1203 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
1204 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
1205 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
1206 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
1207 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
1208 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
1209
1210 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1211                                    string opcode> {
1212   def : GCNPat<
1213     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1214               timm:$cachepolicy, 0),
1215     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
1216       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1217   >;
1218
1219   def : GCNPat<
1220     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1221               timm:$cachepolicy, 0),
1222     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1223       (as_i16imm $offset), (extract_glc $cachepolicy),
1224       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1225   >;
1226
1227   def : GCNPat<
1228     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1229               timm:$cachepolicy, timm),
1230     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1231       (as_i16imm $offset), (extract_glc $cachepolicy),
1232       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1233   >;
1234
1235   def : GCNPat<
1236     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1237               timm:$cachepolicy, timm),
1238     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
1239       $vdata,
1240       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1241       $rsrc, $soffset, (as_i16imm $offset), (extract_glc $cachepolicy),
1242       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1243   >;
1244 }
1245
1246 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
1247 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
1248 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
1249 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
1250 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
1251 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
1252 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
1253 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
1254
1255 let SubtargetPredicate = HasUnpackedD16VMem in {
1256   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1257   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1258   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
1259   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1260 } // End HasUnpackedD16VMem.
1261
1262 let SubtargetPredicate = HasPackedD16VMem in {
1263   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
1264   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
1265   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
1266   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
1267   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1268   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1269 } // End HasPackedD16VMem.
1270
1271 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
1272 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
1273 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">;
1274 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">;
1275 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
1276 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
1277 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">;
1278 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">;
1279 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
1280 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
1281 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
1282 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
1283 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
1284 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
1285
1286 //===----------------------------------------------------------------------===//
1287 // buffer_atomic patterns
1288 //===----------------------------------------------------------------------===//
1289
1290 multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
1291                                 string opcode> {
1292   def : GCNPat<
1293     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1294           0, i32:$soffset, timm:$offset,
1295           timm:$cachepolicy, 0)),
1296     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset,
1297                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1298   >;
1299
1300   def : GCNPat<
1301     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1302           0, i32:$soffset, timm:$offset,
1303           timm:$cachepolicy, timm)),
1304     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset,
1305                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1306   >;
1307
1308   def : GCNPat<
1309     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1310           i32:$voffset, i32:$soffset, timm:$offset,
1311           timm:$cachepolicy, 0)),
1312     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset,
1313                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1314   >;
1315
1316   def : GCNPat<
1317     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1318           i32:$voffset, i32:$soffset, timm:$offset,
1319           timm:$cachepolicy, timm)),
1320     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1321       $vdata_in,
1322       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1323       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1324   >;
1325 }
1326
1327 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">;
1328 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">;
1329 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">;
1330 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">;
1331 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">;
1332 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">;
1333 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">;
1334 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">;
1335 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">;
1336 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">;
1337 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">;
1338 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">;
1339 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">;
1340 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64,  "BUFFER_ATOMIC_ADD_X2">;
1341 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">;
1342 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
1343 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
1344 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
1345 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
1346 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
1347 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
1348 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
1349 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">;
1350 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">;
1351
1352 multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
1353                                        string opcode> {
1354   def : GCNPat<
1355     (name vt:$vdata_in, v4i32:$rsrc, 0,
1356           0, i32:$soffset, timm:$offset,
1357           timm:$cachepolicy, 0),
1358     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
1359                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1360   >;
1361
1362   def : GCNPat<
1363     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1364           0, i32:$soffset, timm:$offset,
1365           timm:$cachepolicy, timm),
1366     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
1367                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1368   >;
1369
1370   def : GCNPat<
1371     (name vt:$vdata_in, v4i32:$rsrc, 0,
1372           i32:$voffset, i32:$soffset, timm:$offset,
1373           timm:$cachepolicy, 0),
1374     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
1375                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1376   >;
1377
1378   def : GCNPat<
1379     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1380           i32:$voffset, i32:$soffset, timm:$offset,
1381           timm:$cachepolicy, timm),
1382     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1383       $vdata_in,
1384       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1385       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1386   >;
1387 }
1388
1389 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
1390 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1391
1392 def : GCNPat<
1393   (SIbuffer_atomic_cmpswap
1394       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1395       0, i32:$soffset, timm:$offset,
1396       timm:$cachepolicy, 0),
1397   (EXTRACT_SUBREG
1398     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1399       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1400       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1401     sub0)
1402 >;
1403
1404 def : GCNPat<
1405   (SIbuffer_atomic_cmpswap
1406       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1407       0, i32:$soffset, timm:$offset,
1408       timm:$cachepolicy, timm),
1409   (EXTRACT_SUBREG
1410     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1411       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1412       $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1413     sub0)
1414 >;
1415
1416 def : GCNPat<
1417   (SIbuffer_atomic_cmpswap
1418       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1419       i32:$voffset, i32:$soffset, timm:$offset,
1420       timm:$cachepolicy, 0),
1421   (EXTRACT_SUBREG
1422     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1423       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1424       $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1425     sub0)
1426 >;
1427
1428 def : GCNPat<
1429   (SIbuffer_atomic_cmpswap
1430       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1431       i32:$voffset, i32:$soffset, timm:$offset,
1432       timm:$cachepolicy, timm),
1433   (EXTRACT_SUBREG
1434     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1435       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1436       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1437       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1438     sub0)
1439 >;
1440
1441 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1442                               PatFrag constant_ld> : GCNPat <
1443      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1444                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
1445      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1446   >;
1447
1448 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1449                                      ValueType vt, PatFrag atomic_ld> {
1450   def : GCNPat <
1451      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1452                                    i16:$offset, i1:$slc))),
1453      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
1454   >;
1455
1456   def : GCNPat <
1457     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1458     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
1459   >;
1460 }
1461
1462 let SubtargetPredicate = isGFX6GFX7 in {
1463 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1464 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
1465 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
1466 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1467 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
1468 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
1469
1470 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
1471 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
1472 } // End SubtargetPredicate = isGFX6GFX7
1473
1474 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1475                                PatFrag ld> {
1476
1477   def : GCNPat <
1478     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1479                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
1480     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1481   >;
1482 }
1483
1484 let OtherPredicates = [Has16BitInsts] in {
1485
1486 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1487 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>;
1488 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>;
1489 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>;
1490 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>;
1491 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>;
1492
1493 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>;
1494
1495 } // End OtherPredicates = [Has16BitInsts]
1496
1497 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1498                                 MUBUF_Pseudo InstrOffset,
1499                                 ValueType vt, PatFrag ld> {
1500   def : GCNPat <
1501     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1502                                i32:$soffset, u16imm:$offset))),
1503     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1504   >;
1505
1506   def : GCNPat <
1507     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1508     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0)
1509   >;
1510 }
1511
1512 // XXX - Is it possible to have a complex pattern in a PatFrag?
1513 multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
1514                                 MUBUF_Pseudo InstrOffset,
1515                                 ValueType vt, PatFrag ld_frag> {
1516   def : GCNPat <
1517     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
1518     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
1519   >;
1520
1521   def : GCNPat <
1522     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
1523     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
1524   >;
1525 }
1526
1527 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1528 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1529 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>;
1530 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1531 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1532 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>;
1533 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1534 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1535 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
1536 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1537
1538 foreach vt = Reg32Types.types in {
1539 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1540 }
1541 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1542 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
1543 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1544
1545 let OtherPredicates = [D16PreservesUnusedBits] in {
1546 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>;
1547 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>;
1548 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>;
1549 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>;
1550 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>;
1551 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>;
1552
1553 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>;
1554 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>;
1555 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>;
1556 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>;
1557 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>;
1558 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>;
1559 }
1560
1561 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1562                                       ValueType vt, PatFrag atomic_st> {
1563   // Store follows atomic op convention so address is forst
1564   def : GCNPat <
1565      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1566                                    i16:$offset, i1:$slc), vt:$val),
1567      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
1568   >;
1569
1570   def : GCNPat <
1571     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1572     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
1573   >;
1574 }
1575 let SubtargetPredicate = isGFX6GFX7 in {
1576 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>;
1577 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>;
1578 } // End Predicates = isGFX6GFX7
1579
1580
1581 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1582                                PatFrag st> {
1583
1584   def : GCNPat <
1585     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1586                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)),
1587     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1588   >;
1589 }
1590
1591 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1592 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1593
1594 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1595                                  MUBUF_Pseudo InstrOffset,
1596                                  ValueType vt, PatFrag st,
1597                                  RegisterClass rc = VGPR_32> {
1598   def : GCNPat <
1599     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1600                                       i32:$soffset, u16imm:$offset)),
1601     (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1602   >;
1603
1604   def : GCNPat <
1605     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1606                                        u16imm:$offset)),
1607     (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1608   >;
1609 }
1610
1611 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1612 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1613 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1614 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1615
1616 foreach vt = Reg32Types.types in {
1617 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>;
1618 }
1619
1620 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
1621 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
1622 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
1623
1624
1625 let OtherPredicates = [D16PreservesUnusedBits] in {
1626  // Hiding the extract high pattern in the PatFrag seems to not
1627  // automatically increase the complexity.
1628 let AddedComplexity = 1 in {
1629 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1630 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1631 }
1632 }
1633
1634 //===----------------------------------------------------------------------===//
1635 // MTBUF Patterns
1636 //===----------------------------------------------------------------------===//
1637
1638 //===----------------------------------------------------------------------===//
1639 // tbuffer_load/store_format patterns
1640 //===----------------------------------------------------------------------===//
1641
1642 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1643                                   string opcode> {
1644   def : GCNPat<
1645     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1646               timm:$format, timm:$cachepolicy, 0)),
1647     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1648       (as_i8imm $format),
1649       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1650   >;
1651
1652   def : GCNPat<
1653     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1654               timm:$format, timm:$cachepolicy, timm)),
1655     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1656       (as_i8imm $format),
1657       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1658   >;
1659
1660   def : GCNPat<
1661     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1662               timm:$format, timm:$cachepolicy, 0)),
1663     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1664       (as_i8imm $format),
1665       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1666   >;
1667
1668   def : GCNPat<
1669     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
1670               timm:$format, timm:$cachepolicy, timm)),
1671     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1672       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1673       $rsrc, $soffset, (as_i16imm $offset),
1674       (as_i8imm $format),
1675       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1676   >;
1677 }
1678
1679 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1680 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1681 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
1682 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1683 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1684 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1685 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
1686 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1687
1688 let SubtargetPredicate = HasUnpackedD16VMem in {
1689   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1690   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1691   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1692 } // End HasUnpackedD16VMem.
1693
1694 let SubtargetPredicate = HasPackedD16VMem in {
1695   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
1696   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
1697   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
1698 } // End HasPackedD16VMem.
1699
1700 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1701                                    string opcode> {
1702   def : GCNPat<
1703     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
1704           timm:$format, timm:$cachepolicy, 0),
1705     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1706       (as_i16imm $offset), (as_i8imm $format),
1707       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1708   >;
1709
1710   def : GCNPat<
1711     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
1712           timm:$format, timm:$cachepolicy, timm),
1713     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1714       (as_i16imm $offset), (as_i8imm $format),
1715       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1716   >;
1717
1718   def : GCNPat<
1719     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
1720           timm:$format, timm:$cachepolicy, 0),
1721     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1722       (as_i16imm $offset), (as_i8imm $format),
1723       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1724   >;
1725
1726   def : GCNPat<
1727     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1728           timm:$offset, timm:$format, timm:$cachepolicy, timm),
1729     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1730       $vdata,
1731       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1732       $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format),
1733       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1734   >;
1735 }
1736
1737 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1738 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1739 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
1740 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1741 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1742 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1743 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
1744 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1745
1746 let SubtargetPredicate = HasUnpackedD16VMem in {
1747   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1748   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
1749   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1750 } // End HasUnpackedD16VMem.
1751
1752 let SubtargetPredicate = HasPackedD16VMem in {
1753   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
1754   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
1755   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
1756 } // End HasPackedD16VMem.
1757
1758 //===----------------------------------------------------------------------===//
1759 // Target-specific instruction encodings.
1760 //===----------------------------------------------------------------------===//
1761
1762 //===----------------------------------------------------------------------===//
1763 // Base ENC_MUBUF for GFX6, GFX7, GFX10.
1764 //===----------------------------------------------------------------------===//
1765
1766 class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
1767     MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1768   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1769   let Inst{12}    = ps.offen;
1770   let Inst{13}    = ps.idxen;
1771   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1772   let Inst{16}    = !if(ps.lds, 1, 0);
1773   let Inst{24-18} = op;
1774   let Inst{31-26} = 0x38;
1775   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1776   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1777   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1778   let Inst{54}    = !if(ps.has_slc, slc, ?);
1779   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1780   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1781 }
1782
1783 class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> :
1784     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
1785   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
1786   let Inst{25} = op{7};
1787 }
1788
1789 class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
1790     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
1791   let Inst{15} = ps.addr64;
1792 }
1793
1794 //===----------------------------------------------------------------------===//
1795 // MUBUF - GFX10.
1796 //===----------------------------------------------------------------------===//
1797
1798 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
1799   multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName,
1800                                         string asmName> {
1801     def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> {
1802       MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName);
1803       let AsmString = asmName # ps.AsmOperands;
1804     }
1805   }
1806   multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
1807     def _BOTHEN_gfx10 :
1808       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1809     def _IDXEN_gfx10 :
1810       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1811     def _OFFEN_gfx10 :
1812       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1813     def _OFFSET_gfx10 :
1814       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1815   }
1816   multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> {
1817     def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1818                         MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">;
1819     def _OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1820                         MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">;
1821     def _IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1822                         MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">;
1823     def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1824                         MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">;
1825
1826     def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1827                             MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">;
1828     def _LDS_OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1829                             MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">;
1830     def _LDS_IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1831                             MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">;
1832     def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1833                             MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">;
1834   }
1835   multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
1836       MUBUF_Real_AllAddr_gfx10<op> {
1837     def _BOTHEN_RTN_gfx10 :
1838       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1839     def _IDXEN_RTN_gfx10 :
1840       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1841     def _OFFEN_RTN_gfx10 :
1842       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1843     def _OFFSET_RTN_gfx10 :
1844       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1845   }
1846 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
1847
1848 defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x019>;
1849 defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx10<0x01b>;
1850 defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x020>;
1851 defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x021>;
1852 defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x022>;
1853 defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x023>;
1854 defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx10<0x024>;
1855 defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x025>;
1856 // FIXME-GFX10: Add following instructions:
1857 //defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx10<0x026>;
1858 //defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
1859 defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx10<0x080>;
1860 defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx10<0x081>;
1861 defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx10<0x082>;
1862 defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx10<0x083>;
1863 defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx10<0x084>;
1864 defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx10<0x085>;
1865 defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx10<0x086>;
1866 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
1867
1868 def BUFFER_GL0_INV_gfx10 :
1869   MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>;
1870 def BUFFER_GL1_INV_gfx10 :
1871   MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>;
1872
1873 //===----------------------------------------------------------------------===//
1874 // MUBUF - GFX6, GFX7, GFX10.
1875 //===----------------------------------------------------------------------===//
1876
1877 let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in {
1878   multiclass MUBUF_Real_gfx6<bits<8> op> {
1879     def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1880   }
1881 } // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6"
1882
1883 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1884   multiclass MUBUF_Real_gfx7<bits<8> op> {
1885     def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1886   }
1887 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1888
1889 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1890   multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
1891     def _ADDR64_gfx6_gfx7 :
1892       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1893     def _BOTHEN_gfx6_gfx7 :
1894       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1895     def _IDXEN_gfx6_gfx7 :
1896       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1897     def _OFFEN_gfx6_gfx7 :
1898       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1899     def _OFFSET_gfx6_gfx7 :
1900       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1901   }
1902   multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> {
1903     def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1904                             MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">;
1905     def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
1906                             MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">;
1907     def _OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1908                             MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">;
1909     def _IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1910                             MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">;
1911     def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1912                             MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">;
1913
1914     def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1915                                 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">;
1916     def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>,
1917                                 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">;
1918     def _LDS_OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1919                                 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">;
1920     def _LDS_IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1921                                 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">;
1922     def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1923                                 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">;
1924   }
1925   multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> :
1926       MUBUF_Real_AllAddr_gfx6_gfx7<op> {
1927     def _ADDR64_RTN_gfx6_gfx7 :
1928       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>;
1929     def _BOTHEN_RTN_gfx6_gfx7 :
1930       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1931     def _IDXEN_RTN_gfx6_gfx7 :
1932       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1933     def _OFFEN_RTN_gfx6_gfx7 :
1934       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1935     def _OFFSET_RTN_gfx6_gfx7 :
1936       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1937   }
1938 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1939
1940 multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
1941   MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
1942
1943 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> :
1944   MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>;
1945
1946 multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
1947   MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
1948
1949 // FIXME-GFX6: Following instructions are available only on GFX6.
1950 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomics_gfx6 <0x034>;
1951 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomics_gfx6 <0x054>;
1952
1953 defm BUFFER_LOAD_FORMAT_X     : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
1954 defm BUFFER_LOAD_FORMAT_XY    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
1955 defm BUFFER_LOAD_FORMAT_XYZ   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
1956 defm BUFFER_LOAD_FORMAT_XYZW  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
1957 defm BUFFER_STORE_FORMAT_X    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
1958 defm BUFFER_STORE_FORMAT_XY   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
1959 defm BUFFER_STORE_FORMAT_XYZ  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
1960 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
1961 defm BUFFER_LOAD_UBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
1962 defm BUFFER_LOAD_SBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
1963 defm BUFFER_LOAD_USHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
1964 defm BUFFER_LOAD_SSHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
1965 defm BUFFER_LOAD_DWORD        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
1966 defm BUFFER_LOAD_DWORDX2      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
1967 defm BUFFER_LOAD_DWORDX4      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
1968 defm BUFFER_LOAD_DWORDX3      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
1969 defm BUFFER_STORE_BYTE        : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
1970 defm BUFFER_STORE_SHORT       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
1971 defm BUFFER_STORE_DWORD       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
1972 defm BUFFER_STORE_DWORDX2     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
1973 defm BUFFER_STORE_DWORDX4     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
1974 defm BUFFER_STORE_DWORDX3     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
1975
1976 defm BUFFER_ATOMIC_SWAP        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
1977 defm BUFFER_ATOMIC_CMPSWAP     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
1978 defm BUFFER_ATOMIC_ADD         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
1979 defm BUFFER_ATOMIC_SUB         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
1980 defm BUFFER_ATOMIC_SMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
1981 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
1982 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
1983 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
1984 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
1985 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
1986 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
1987 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
1988 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
1989 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
1990 //defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
1991 //defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
1992 //defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
1993 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
1994 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
1995 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
1996 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
1997 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
1998 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
1999 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
2000 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
2001 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
2002 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
2003 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
2004 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
2005 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
2006 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
2007 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
2008 //defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
2009 //defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
2010 //defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
2011
2012 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
2013 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
2014 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
2015
2016 //===----------------------------------------------------------------------===//
2017 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
2018 //===----------------------------------------------------------------------===//
2019
2020 class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
2021     MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
2022   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2023   let Inst{12}    = ps.offen;
2024   let Inst{13}    = ps.idxen;
2025   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2026   let Inst{18-16} = op;
2027   let Inst{31-26} = 0x3a; //encoding
2028   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2029   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2030   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2031   let Inst{54}    = !if(ps.has_slc, slc, ?);
2032   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2033   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2034 }
2035
2036 //===----------------------------------------------------------------------===//
2037 // MTBUF - GFX10.
2038 //===----------------------------------------------------------------------===//
2039
2040 class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> :
2041     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
2042   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
2043   let Inst{25-19} = format;
2044   let Inst{53} = op{3};
2045 }
2046
2047 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
2048   multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
2049     def _BOTHEN_gfx10 :
2050       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2051     def _IDXEN_gfx10 :
2052       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2053     def _OFFEN_gfx10 :
2054       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2055     def _OFFSET_gfx10 :
2056       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2057   }
2058 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
2059
2060 defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx10<0x008>;
2061 defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx10<0x009>;
2062 defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx10<0x00a>;
2063 defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx10<0x00b>;
2064 defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx10<0x00c>;
2065 defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx10<0x00d>;
2066 defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx10<0x00e>;
2067 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
2068
2069 //===----------------------------------------------------------------------===//
2070 // MTBUF - GFX6, GFX7, GFX10.
2071 //===----------------------------------------------------------------------===//
2072
2073 class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> :
2074     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
2075   let Inst{15} = ps.addr64;
2076   let Inst{22-19} = dfmt;
2077   let Inst{25-23} = nfmt;
2078 }
2079
2080 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2081   multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
2082     def _ADDR64_gfx6_gfx7 :
2083       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
2084     def _BOTHEN_gfx6_gfx7 :
2085       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2086     def _IDXEN_gfx6_gfx7 :
2087       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2088     def _OFFEN_gfx6_gfx7 :
2089       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2090     def _OFFSET_gfx6_gfx7 :
2091       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2092   }
2093 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2094
2095 multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
2096   MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
2097
2098 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
2099 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2100 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2101 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2102 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2103 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2104 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2105 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2106
2107 //===----------------------------------------------------------------------===//
2108 // GFX8, GFX9 (VI).
2109 //===----------------------------------------------------------------------===//
2110
2111 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
2112   MUBUF_Real<ps>,
2113   Enc64,
2114   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2115   let AssemblerPredicate = isGFX8GFX9;
2116   let DecoderNamespace = "GFX8";
2117
2118   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2119   let Inst{12}    = ps.offen;
2120   let Inst{13}    = ps.idxen;
2121   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2122   let Inst{16}    = !if(ps.lds, 1, 0);
2123   let Inst{17}    = !if(ps.has_slc, slc, ?);
2124   let Inst{24-18} = op;
2125   let Inst{31-26} = 0x38; //encoding
2126   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2127   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2128   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2129   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2130   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2131 }
2132
2133 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
2134   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2135   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2136   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2137   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2138 }
2139
2140 multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
2141
2142   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2143                    MUBUFLdsTable<0, NAME # "_OFFSET_vi">;
2144   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2145                    MUBUFLdsTable<0, NAME # "_OFFEN_vi">;
2146   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2147                    MUBUFLdsTable<0, NAME # "_IDXEN_vi">;
2148   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2149                    MUBUFLdsTable<0, NAME # "_BOTHEN_vi">;
2150
2151   def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2152                        MUBUFLdsTable<1, NAME # "_OFFSET_vi">;
2153   def _LDS_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2154                        MUBUFLdsTable<1, NAME # "_OFFEN_vi">;
2155   def _LDS_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2156                        MUBUFLdsTable<1, NAME # "_IDXEN_vi">;
2157   def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2158                        MUBUFLdsTable<1, NAME # "_BOTHEN_vi">;
2159 }
2160
2161 class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> :
2162   MUBUF_Real<ps>,
2163   Enc64,
2164   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2165   let AssemblerPredicate=HasUnpackedD16VMem;
2166   let DecoderNamespace="GFX80_UNPACKED";
2167
2168   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2169   let Inst{12}    = ps.offen;
2170   let Inst{13}    = ps.idxen;
2171   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2172   let Inst{16}    = !if(ps.lds, 1, 0);
2173   let Inst{17}    = !if(ps.has_slc, slc, ?);
2174   let Inst{24-18} = op;
2175   let Inst{31-26} = 0x38; //encoding
2176   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2177   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2178   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2179   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2180   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2181 }
2182
2183 multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
2184   def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2185   def _OFFEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2186   def _IDXEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2187   def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2188 }
2189
2190 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
2191   MUBUF_Real_AllAddr_vi<op> {
2192   def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
2193   def _OFFEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
2194   def _IDXEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
2195   def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
2196 }
2197
2198 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_Lds_vi <0x00>;
2199 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
2200 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
2201 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
2202 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
2203 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
2204 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
2205 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
2206 let SubtargetPredicate = HasUnpackedD16VMem in {
2207   defm BUFFER_LOAD_FORMAT_D16_X_gfx80       : MUBUF_Real_AllAddr_gfx80 <0x08>;
2208   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x09>;
2209   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0a>;
2210   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0b>;
2211   defm BUFFER_STORE_FORMAT_D16_X_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x0c>;
2212   defm BUFFER_STORE_FORMAT_D16_XY_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0d>;
2213   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0e>;
2214   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80   : MUBUF_Real_AllAddr_gfx80 <0x0f>;
2215 } // End HasUnpackedD16VMem.
2216 let SubtargetPredicate = HasPackedD16VMem in {
2217   defm BUFFER_LOAD_FORMAT_D16_X       : MUBUF_Real_AllAddr_vi <0x08>;
2218   defm BUFFER_LOAD_FORMAT_D16_XY      : MUBUF_Real_AllAddr_vi <0x09>;
2219   defm BUFFER_LOAD_FORMAT_D16_XYZ     : MUBUF_Real_AllAddr_vi <0x0a>;
2220   defm BUFFER_LOAD_FORMAT_D16_XYZW    : MUBUF_Real_AllAddr_vi <0x0b>;
2221   defm BUFFER_STORE_FORMAT_D16_X      : MUBUF_Real_AllAddr_vi <0x0c>;
2222   defm BUFFER_STORE_FORMAT_D16_XY     : MUBUF_Real_AllAddr_vi <0x0d>;
2223   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
2224   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
2225 } // End HasPackedD16VMem.
2226 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
2227 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
2228 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
2229 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
2230 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
2231 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
2232 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
2233 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
2234 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
2235 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
2236 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
2237 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
2238 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
2239 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
2240 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
2241 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
2242
2243 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
2244 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
2245 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
2246 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
2247 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
2248 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
2249
2250 defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_vi <0x26>;
2251 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
2252
2253 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
2254 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
2255 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
2256 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
2257 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
2258 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
2259 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
2260 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
2261 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
2262 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
2263 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
2264 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
2265 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
2266
2267 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
2268 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
2269 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
2270 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
2271 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
2272 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
2273 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
2274 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
2275 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
2276 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
2277 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
2278 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
2279 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
2280
2281 def BUFFER_STORE_LDS_DWORD_vi   : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>;
2282
2283 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
2284 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
2285
2286 let SubtargetPredicate = HasAtomicFaddInsts in {
2287
2288 defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_AllAddr_vi <0x4d>;
2289 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>;
2290
2291 } // End SubtargetPredicate = HasAtomicFaddInsts
2292
2293 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
2294   MTBUF_Real<ps>,
2295   Enc64,
2296   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2297   let AssemblerPredicate = isGFX8GFX9;
2298   let DecoderNamespace = "GFX8";
2299
2300   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2301   let Inst{12}    = ps.offen;
2302   let Inst{13}    = ps.idxen;
2303   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2304   let Inst{18-15} = op;
2305   let Inst{22-19} = dfmt;
2306   let Inst{25-23} = nfmt;
2307   let Inst{31-26} = 0x3a; //encoding
2308   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2309   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2310   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2311   let Inst{54}    = !if(ps.has_slc, slc, ?);
2312   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2313   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2314 }
2315
2316 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
2317   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2318   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2319   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2320   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2321 }
2322
2323 class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> :
2324   MTBUF_Real<ps>,
2325   Enc64,
2326   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2327   let AssemblerPredicate=HasUnpackedD16VMem;
2328   let DecoderNamespace="GFX80_UNPACKED";
2329
2330   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2331   let Inst{12}    = ps.offen;
2332   let Inst{13}    = ps.idxen;
2333   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2334   let Inst{18-15} = op;
2335   let Inst{22-19} = dfmt;
2336   let Inst{25-23} = nfmt;
2337   let Inst{31-26} = 0x3a; //encoding
2338   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2339   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2340   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2341   let Inst{54}    = !if(ps.has_slc, slc, ?);
2342   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2343   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2344 }
2345
2346 multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
2347   def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2348   def _OFFEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2349   def _IDXEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2350   def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2351 }
2352
2353 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0x00>;
2354 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <0x01>;
2355 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <0x02>;
2356 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <0x03>;
2357 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <0x04>;
2358 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <0x05>;
2359 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <0x06>;
2360 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
2361 let SubtargetPredicate = HasUnpackedD16VMem in {
2362   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Real_AllAddr_gfx80 <0x08>;
2363   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x09>;
2364   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0a>;
2365   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0b>;
2366   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x0c>;
2367   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0d>;
2368   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0e>;
2369   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
2370 } // End HasUnpackedD16VMem.
2371 let SubtargetPredicate = HasPackedD16VMem in {
2372   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_vi <0x08>;
2373   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_vi <0x09>;
2374   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_vi <0x0a>;
2375   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_vi <0x0b>;
2376   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_vi <0x0c>;
2377   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_vi <0x0d>;
2378   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_vi <0x0e>;
2379   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
2380 } // End HasUnpackedD16VMem.
2381
2382 def MUBUFInfoTable : GenericTable {
2383   let FilterClass = "MUBUF_Pseudo";
2384   let CppTypeName = "MUBUFInfo";
2385   let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
2386
2387   let PrimaryKey = ["Opcode"];
2388   let PrimaryKeyName = "getMUBUFOpcodeHelper";
2389 }
2390
2391 def getMUBUFInfoFromOpcode : SearchIndex {
2392   let Table = MUBUFInfoTable;
2393   let Key = ["Opcode"];
2394 }
2395
2396 def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {
2397   let Table = MUBUFInfoTable;
2398   let Key = ["BaseOpcode", "elements"];
2399 }