lib/Target/AMDGPU/BUFInstructions.td

   1 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
  10 def MUBUFAddr64 : ComplexPattern<i64, 8, "SelectMUBUFAddr64">;
  11 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
  12
  13 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
  14 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
  15
  16 def MUBUFOffset : ComplexPattern<i64, 7, "SelectMUBUFOffset">;
  17 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
  18 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
  19
  20 def BUFAddrKind {
  21   int Offset = 0;
  22   int OffEn  = 1;
  23   int IdxEn  = 2;
  24   int BothEn = 3;
  25   int Addr64 = 4;
  26 }
  27
  28 class getAddrName<int addrKind> {
  29   string ret =
  30     !if(!eq(addrKind, BUFAddrKind.Offset), "offset",
  31     !if(!eq(addrKind, BUFAddrKind.OffEn),  "offen",
  32     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "idxen",
  33     !if(!eq(addrKind, BUFAddrKind.BothEn), "bothen",
  34     !if(!eq(addrKind, BUFAddrKind.Addr64), "addr64",
  35     "")))));
  36 }
  37
  38 class MUBUFAddr64Table <bit is_addr64, string Name> {
  39   bit IsAddr64 = is_addr64;
  40   string OpName = Name;
  41 }
  42
  43 class MUBUFLdsTable <bit is_lds, string Name> {
  44   bit IsLds = is_lds;
  45   string OpName = Name;
  46 }
  47
  48 class MTBUFAddr64Table <bit is_addr64, string Name> {
  49   bit IsAddr64 = is_addr64;
  50   string OpName = Name;
  51 }
  52
  53 //===----------------------------------------------------------------------===//
  54 // MTBUF classes
  55 //===----------------------------------------------------------------------===//
  56
  57 class MTBUF_Pseudo <string opName, dag outs, dag ins,
  58                     string asmOps, list<dag> pattern=[]> :
  59   InstSI<outs, ins, "", pattern>,
  60   SIMCInstr<opName, SIEncodingFamily.NONE> {
  61
  62   let isPseudo = 1;
  63   let isCodeGenOnly = 1;
  64   let Size = 8;
  65   let UseNamedOperandTable = 1;
  66
  67   string Mnemonic = opName;
  68   string AsmOperands = asmOps;
  69
  70   let VM_CNT = 1;
  71   let EXP_CNT = 1;
  72   let MTBUF = 1;
  73   let Uses = [EXEC];
  74   let hasSideEffects = 0;
  75   let SchedRW = [WriteVMEM];
  76
  77   let AsmMatchConverter = "cvtMtbuf";
  78
  79   bits<1> offen       = 0;
  80   bits<1> idxen       = 0;
  81   bits<1> addr64      = 0;
  82   bits<1> has_vdata   = 1;
  83   bits<1> has_vaddr   = 1;
  84   bits<1> has_glc     = 1;
  85   bits<1> has_dlc     = 1;
  86   bits<1> glc_value   = 0; // the value for glc if no such operand
  87   bits<1> dlc_value   = 0; // the value for dlc if no such operand
  88   bits<1> has_srsrc   = 1;
  89   bits<1> has_soffset = 1;
  90   bits<1> has_offset  = 1;
  91   bits<1> has_slc     = 1;
  92   bits<1> has_tfe     = 1;
  93 }
  94
  95 class MTBUF_Real <MTBUF_Pseudo ps> :
  96   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
  97
  98   let isPseudo = 0;
  99   let isCodeGenOnly = 0;
 100
 101   // copy relevant pseudo op flags
 102   let SubtargetPredicate = ps.SubtargetPredicate;
 103   let AsmMatchConverter  = ps.AsmMatchConverter;
 104   let Constraints        = ps.Constraints;
 105   let DisableEncoding    = ps.DisableEncoding;
 106   let TSFlags            = ps.TSFlags;
 107
 108   bits<12> offset;
 109   bits<1>  glc;
 110   bits<1>  dlc;
 111   bits<7>  format;
 112   bits<8>  vaddr;
 113   bits<8>  vdata;
 114   bits<7>  srsrc;
 115   bits<1>  slc;
 116   bits<1>  tfe;
 117   bits<8>  soffset;
 118
 119   bits<4> dfmt = format{3-0};
 120   bits<3> nfmt = format{6-4};
 121 }
 122
 123 class getMTBUFInsDA<list<RegisterClass> vdataList,
 124                     list<RegisterClass> vaddrList=[]> {
 125   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 126   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 127   dag InsNoData = !if(!empty(vaddrList),
 128     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 129          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc),
 130     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 131          offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc)
 132   );
 133   dag InsData = !if(!empty(vaddrList),
 134     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 135          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 136          SLC:$slc, TFE:$tfe, DLC:$dlc),
 137     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 138          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
 139          SLC:$slc, TFE:$tfe, DLC:$dlc)
 140   );
 141   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 142 }
 143
 144 class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
 145   dag ret =
 146     !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
 147     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 148     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
 149     !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 150     !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
 151     (ins))))));
 152 }
 153
 154 class getMTBUFAsmOps<int addrKind> {
 155   string Pfx =
 156     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $format, $soffset",
 157     !if(!eq(addrKind, BUFAddrKind.OffEn),
 158             "$vaddr, $srsrc, $format, $soffset offen",
 159     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 160             "$vaddr, $srsrc, $format, $soffset idxen",
 161     !if(!eq(addrKind, BUFAddrKind.BothEn),
 162             "$vaddr, $srsrc, $format, $soffset idxen offen",
 163     !if(!eq(addrKind, BUFAddrKind.Addr64),
 164             "$vaddr, $srsrc, $format, $soffset addr64",
 165     "")))));
 166   string ret = Pfx # "$offset";
 167 }
 168
 169 class MTBUF_SetupAddr<int addrKind> {
 170   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 171                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 172
 173   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 174                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 175
 176   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 177
 178   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 179 }
 180
 181 class MTBUF_Load_Pseudo <string opName,
 182                          int addrKind,
 183                          RegisterClass vdataClass,
 184                          list<dag> pattern=[],
 185                          // Workaround bug bz30254
 186                          int addrKindCopy = addrKind>
 187   : MTBUF_Pseudo<opName,
 188                  (outs vdataClass:$vdata),
 189                  getMTBUFIns<addrKindCopy>.ret,
 190                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 191                  pattern>,
 192     MTBUF_SetupAddr<addrKindCopy> {
 193   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 194   let mayLoad = 1;
 195   let mayStore = 0;
 196 }
 197
 198 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 199                               ValueType load_vt = i32,
 200                               SDPatternOperator ld = null_frag> {
 201
 202   def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 203     [(set load_vt:$vdata,
 204      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format,
 205                       i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
 206     MTBUFAddr64Table<0, NAME>;
 207
 208   def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 209     [(set load_vt:$vdata,
 210      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
 211                       i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
 212     MTBUFAddr64Table<1, NAME>;
 213
 214   def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 215   def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 216   def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 217
 218   let DisableWQM = 1 in {
 219     def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 220     def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 221     def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 222     def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 223   }
 224 }
 225
 226 class MTBUF_Store_Pseudo <string opName,
 227                           int addrKind,
 228                           RegisterClass vdataClass,
 229                           list<dag> pattern=[],
 230                           // Workaround bug bz30254
 231                           int addrKindCopy = addrKind,
 232                           RegisterClass vdataClassCopy = vdataClass>
 233   : MTBUF_Pseudo<opName,
 234                  (outs),
 235                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 236                  " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 237                  pattern>,
 238     MTBUF_SetupAddr<addrKindCopy> {
 239   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 240   let mayLoad = 0;
 241   let mayStore = 1;
 242 }
 243
 244 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 245                                ValueType store_vt = i32,
 246                                SDPatternOperator st = null_frag> {
 247
 248   def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 249     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 250                                        i16:$offset, i8:$format, i1:$glc,
 251                                        i1:$slc, i1:$tfe, i1:$dlc))]>,
 252     MTBUFAddr64Table<0, NAME>;
 253
 254   def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 255     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 256                                        i16:$offset, i8:$format, i1:$glc,
 257                                        i1:$slc, i1:$tfe, i1:$dlc))]>,
 258     MTBUFAddr64Table<1, NAME>;
 259
 260   def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 261   def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 262   def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 263
 264   let DisableWQM = 1 in {
 265     def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 266     def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 267     def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 268     def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 269   }
 270 }
 271
 272
 273 //===----------------------------------------------------------------------===//
 274 // MUBUF classes
 275 //===----------------------------------------------------------------------===//
 276
 277 class MUBUFGetBaseOpcode<string Op> {
 278   string ret = !subst("DWORDX2", "DWORD",
 279     !subst("DWORDX3", "DWORD",
 280     !subst("DWORDX4", "DWORD", Op)));
 281 }
 282
 283 class MUBUF_Pseudo <string opName, dag outs, dag ins,
 284                     string asmOps, list<dag> pattern=[]> :
 285   InstSI<outs, ins, "", pattern>,
 286   SIMCInstr<opName, SIEncodingFamily.NONE> {
 287
 288   let isPseudo = 1;
 289   let isCodeGenOnly = 1;
 290   let Size = 8;
 291   let UseNamedOperandTable = 1;
 292
 293   string Mnemonic = opName;
 294   string AsmOperands = asmOps;
 295
 296   Instruction Opcode = !cast<Instruction>(NAME);
 297   Instruction BaseOpcode = !cast<Instruction>(MUBUFGetBaseOpcode<NAME>.ret);
 298
 299   let VM_CNT = 1;
 300   let EXP_CNT = 1;
 301   let MUBUF = 1;
 302   let Uses = [EXEC];
 303   let hasSideEffects = 0;
 304   let SchedRW = [WriteVMEM];
 305
 306   let AsmMatchConverter = "cvtMubuf";
 307
 308   bits<1> offen       = 0;
 309   bits<1> idxen       = 0;
 310   bits<1> addr64      = 0;
 311   bits<1> lds         = 0;
 312   bits<1> has_vdata   = 1;
 313   bits<1> has_vaddr   = 1;
 314   bits<1> has_glc     = 1;
 315   bits<1> has_dlc     = 1;
 316   bits<1> glc_value   = 0; // the value for glc if no such operand
 317   bits<1> dlc_value   = 0; // the value for dlc if no such operand
 318   bits<1> has_srsrc   = 1;
 319   bits<1> has_soffset = 1;
 320   bits<1> has_offset  = 1;
 321   bits<1> has_slc     = 1;
 322   bits<1> has_tfe     = 1;
 323   bits<4> dwords      = 0;
 324 }
 325
 326 class MUBUF_Real <MUBUF_Pseudo ps> :
 327   InstSI <ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
 328
 329   let isPseudo = 0;
 330   let isCodeGenOnly = 0;
 331
 332   // copy relevant pseudo op flags
 333   let SubtargetPredicate = ps.SubtargetPredicate;
 334   let AsmMatchConverter  = ps.AsmMatchConverter;
 335   let Constraints        = ps.Constraints;
 336   let DisableEncoding    = ps.DisableEncoding;
 337   let TSFlags            = ps.TSFlags;
 338
 339   bits<12> offset;
 340   bits<1>  glc;
 341   bits<1>  dlc;
 342   bits<8>  vaddr;
 343   bits<8>  vdata;
 344   bits<7>  srsrc;
 345   bits<1>  slc;
 346   bits<1>  tfe;
 347   bits<8>  soffset;
 348 }
 349
 350
 351 // For cache invalidation instructions.
 352 class MUBUF_Invalidate <string opName, SDPatternOperator node = null_frag> :
 353   MUBUF_Pseudo<opName, (outs), (ins), "", [(node)]> {
 354
 355   let AsmMatchConverter = "";
 356
 357   let hasSideEffects = 1;
 358   let mayStore = 1;
 359
 360   // Set everything to 0.
 361   let offen       = 0;
 362   let idxen       = 0;
 363   let addr64      = 0;
 364   let has_vdata   = 0;
 365   let has_vaddr   = 0;
 366   let has_glc     = 0;
 367   let has_dlc     = 0;
 368   let glc_value   = 0;
 369   let dlc_value   = 0;
 370   let has_srsrc   = 0;
 371   let has_soffset = 0;
 372   let has_offset  = 0;
 373   let has_slc     = 0;
 374   let has_tfe     = 0;
 375 }
 376
 377 class getMUBUFInsDA<list<RegisterClass> vdataList,
 378                     list<RegisterClass> vaddrList=[],
 379                     bit isLds = 0> {
 380   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
 381   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 382   dag InsNoData = !if(!empty(vaddrList),
 383     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
 384          offset:$offset, GLC:$glc, SLC:$slc),
 385     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
 386          offset:$offset, GLC:$glc, SLC:$slc)
 387   );
 388   dag InsData = !if(!empty(vaddrList),
 389     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
 390          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 391     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
 392          SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc)
 393   );
 394   dag ret = !con(
 395               !if(!empty(vdataList), InsNoData, InsData),
 396               !if(isLds, (ins DLC:$dlc), (ins TFE:$tfe, DLC:$dlc))
 397              );
 398 }
 399
 400 class getMUBUFDwords<RegisterClass regClass> {
 401   string regClassAsInt = !cast<string>(regClass);
 402   int ret =
 403     !if(!eq(regClassAsInt, !cast<string>(VGPR_32)), 1,
 404     !if(!eq(regClassAsInt, !cast<string>(VReg_64)), 2,
 405     !if(!eq(regClassAsInt, !cast<string>(VReg_96)), 3,
 406     !if(!eq(regClassAsInt, !cast<string>(VReg_128)), 4,
 407     0))));
 408 }
 409
 410 class getMUBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit isLds = 0> {
 411   dag ret =
 412     !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isLds>.ret,
 413     !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 414     !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isLds>.ret,
 415     !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 416     !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isLds>.ret,
 417     (ins))))));
 418 }
 419
 420 class getMUBUFAsmOps<int addrKind> {
 421   string Pfx =
 422     !if(!eq(addrKind, BUFAddrKind.Offset), "off, $srsrc, $soffset",
 423     !if(!eq(addrKind, BUFAddrKind.OffEn),  "$vaddr, $srsrc, $soffset offen",
 424     !if(!eq(addrKind, BUFAddrKind.IdxEn),  "$vaddr, $srsrc, $soffset idxen",
 425     !if(!eq(addrKind, BUFAddrKind.BothEn), "$vaddr, $srsrc, $soffset idxen offen",
 426     !if(!eq(addrKind, BUFAddrKind.Addr64), "$vaddr, $srsrc, $soffset addr64",
 427     "")))));
 428   string ret = Pfx # "$offset";
 429 }
 430
 431 class MUBUF_SetupAddr<int addrKind> {
 432   bits<1> offen  = !if(!eq(addrKind, BUFAddrKind.OffEn), 1,
 433                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 434
 435   bits<1> idxen  = !if(!eq(addrKind, BUFAddrKind.IdxEn), 1,
 436                    !if(!eq(addrKind, BUFAddrKind.BothEn), 1 , 0));
 437
 438   bits<1> addr64 = !if(!eq(addrKind, BUFAddrKind.Addr64), 1, 0);
 439
 440   bits<1> has_vaddr = !if(!eq(addrKind, BUFAddrKind.Offset), 0, 1);
 441 }
 442
 443 class MUBUF_Load_Pseudo <string opName,
 444                          int addrKind,
 445                          RegisterClass vdataClass,
 446                          bit HasTiedDest = 0,
 447                          bit isLds = 0,
 448                          list<dag> pattern=[],
 449                          // Workaround bug bz30254
 450                          int addrKindCopy = addrKind>
 451   : MUBUF_Pseudo<opName,
 452                  (outs vdataClass:$vdata),
 453                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
 454                       !if(HasTiedDest, (ins vdataClass:$vdata_in), (ins))),
 455                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" #
 456                    !if(isLds, " lds", "$tfe") # "$dlc",
 457                  pattern>,
 458     MUBUF_SetupAddr<addrKindCopy> {
 459   let PseudoInstr = opName # !if(isLds, "_lds", "") #
 460                     "_" # getAddrName<addrKindCopy>.ret;
 461   let AsmMatchConverter = !if(isLds, "cvtMubufLds", "cvtMubuf");
 462
 463   let Constraints = !if(HasTiedDest, "$vdata = $vdata_in", "");
 464   let mayLoad = 1;
 465   let mayStore = 0;
 466   let maybeAtomic = 1;
 467   let Uses = !if(isLds, [EXEC, M0], [EXEC]);
 468   let has_tfe = !if(isLds, 0, 1);
 469   let lds = isLds;
 470   let dwords = getMUBUFDwords<vdataClass>.ret;
 471 }
 472
 473 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
 474   (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
 475   (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
 476 >;
 477
 478 class MUBUF_Addr64_Load_Pat <Instruction inst,
 479                             ValueType load_vt = i32,
 480                             SDPatternOperator ld = null_frag> : Pat <
 481   (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
 482   (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
 483 >;
 484
 485 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
 486   def : MUBUF_Offset_Load_Pat<!cast<Instruction>(BaseInst#"_OFFSET"), load_vt, ld>;
 487   def : MUBUF_Addr64_Load_Pat<!cast<Instruction>(BaseInst#"_ADDR64"), load_vt, ld>;
 488 }
 489
 490
 491 // FIXME: tfe can't be an operand because it requires a separate
 492 // opcode because it needs an N+1 register class dest register.
 493 multiclass MUBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
 494                               ValueType load_vt = i32,
 495                               SDPatternOperator ld = null_frag,
 496                               bit TiedDest = 0,
 497                               bit isLds = 0> {
 498
 499   def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>,
 500     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 501
 502   def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, TiedDest, isLds>,
 503     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 504
 505   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>;
 506   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>;
 507   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>;
 508
 509   let DisableWQM = 1 in {
 510     def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, TiedDest, isLds>;
 511     def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, TiedDest, isLds>;
 512     def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, TiedDest, isLds>;
 513     def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, TiedDest, isLds>;
 514   }
 515 }
 516
 517 multiclass MUBUF_Pseudo_Loads_Lds<string opName, RegisterClass vdataClass,
 518                                   ValueType load_vt = i32,
 519                                   SDPatternOperator ld_nolds = null_frag,
 520                                   SDPatternOperator ld_lds = null_frag> {
 521   defm NAME : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_nolds>;
 522   defm _LDS : MUBUF_Pseudo_Loads<opName, vdataClass, load_vt, ld_lds, 0, 1>;
 523 }
 524
 525 class MUBUF_Store_Pseudo <string opName,
 526                           int addrKind,
 527                           RegisterClass vdataClass,
 528                           list<dag> pattern=[],
 529                           // Workaround bug bz30254
 530                           int addrKindCopy = addrKind,
 531                           RegisterClass vdataClassCopy = vdataClass>
 532   : MUBUF_Pseudo<opName,
 533                  (outs),
 534                  getMUBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
 535                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
 536                  pattern>,
 537     MUBUF_SetupAddr<addrKindCopy> {
 538   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 539   let mayLoad = 0;
 540   let mayStore = 1;
 541   let maybeAtomic = 1;
 542   let dwords = getMUBUFDwords<vdataClass>.ret;
 543 }
 544
 545 multiclass MUBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
 546                                ValueType store_vt = i32,
 547                                SDPatternOperator st = null_frag> {
 548
 549   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 550     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
 551                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
 552     MUBUFAddr64Table<0, NAME>;
 553
 554   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 555     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
 556                                        i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
 557     MUBUFAddr64Table<1, NAME>;
 558
 559   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 560   def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 561   def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 562
 563   let DisableWQM = 1 in {
 564     def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
 565     def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
 566     def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
 567     def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 568   }
 569 }
 570
 571 class MUBUF_Pseudo_Store_Lds<string opName>
 572   : MUBUF_Pseudo<opName,
 573                  (outs),
 574                  (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
 575                  " $srsrc, $soffset$offset lds$glc$slc"> {
 576   let mayLoad = 0;
 577   let mayStore = 1;
 578   let maybeAtomic = 1;
 579
 580   let has_vdata = 0;
 581   let has_vaddr = 0;
 582   let has_tfe = 0;
 583   let lds = 1;
 584
 585   let Uses = [EXEC, M0];
 586   let AsmMatchConverter = "cvtMubufLds";
 587 }
 588
 589 class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 590                           list<RegisterClass> vaddrList=[]> {
 591   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
 592   dag ret = !if(vdata_in,
 593     !if(!empty(vaddrList),
 594       (ins vdataClass:$vdata_in,
 595            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 596       (ins vdataClass:$vdata_in, vaddrClass:$vaddr,
 597            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 598     ),
 599     !if(!empty(vaddrList),
 600       (ins vdataClass:$vdata,
 601            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc),
 602       (ins vdataClass:$vdata, vaddrClass:$vaddr,
 603            SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, SLC:$slc)
 604   ));
 605 }
 606
 607 class getMUBUFAtomicIns<int addrKind,
 608                         RegisterClass vdataClass,
 609                         bit vdata_in,
 610                         // Workaround bug bz30254
 611                         RegisterClass vdataClassCopy=vdataClass> {
 612   dag ret =
 613     !if(!eq(addrKind, BUFAddrKind.Offset),
 614             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
 615     !if(!eq(addrKind, BUFAddrKind.OffEn),
 616             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 617     !if(!eq(addrKind, BUFAddrKind.IdxEn),
 618             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
 619     !if(!eq(addrKind, BUFAddrKind.BothEn),
 620             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 621     !if(!eq(addrKind, BUFAddrKind.Addr64),
 622             getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
 623     (ins))))));
 624 }
 625
 626 class MUBUF_Atomic_Pseudo<string opName,
 627                           int addrKind,
 628                           dag outs,
 629                           dag ins,
 630                           string asmOps,
 631                           list<dag> pattern=[],
 632                           // Workaround bug bz30254
 633                           int addrKindCopy = addrKind>
 634   : MUBUF_Pseudo<opName, outs, ins, asmOps, pattern>,
 635     MUBUF_SetupAddr<addrKindCopy> {
 636   let mayStore = 1;
 637   let mayLoad = 1;
 638   let hasPostISelHook = 1;
 639   let hasSideEffects = 1;
 640   let DisableWQM = 1;
 641   let has_glc = 0;
 642   let has_dlc = 0;
 643   let has_tfe = 0;
 644   let maybeAtomic = 1;
 645 }
 646
 647 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 648                                RegisterClass vdataClass,
 649                                list<dag> pattern=[],
 650                                // Workaround bug bz30254
 651                                int addrKindCopy = addrKind,
 652                                RegisterClass vdataClassCopy = vdataClass>
 653   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 654                         (outs),
 655                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
 656                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$slc",
 657                         pattern>,
 658     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
 659   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
 660   let glc_value = 0;
 661   let dlc_value = 0;
 662   let AsmMatchConverter = "cvtMubufAtomic";
 663 }
 664
 665 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
 666                              RegisterClass vdataClass,
 667                              list<dag> pattern=[],
 668                              // Workaround bug bz30254
 669                              int addrKindCopy = addrKind,
 670                              RegisterClass vdataClassCopy = vdataClass>
 671   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
 672                         (outs vdataClassCopy:$vdata),
 673                         getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
 674                         " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # " glc$slc",
 675                         pattern>,
 676     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
 677   let PseudoInstr = opName # "_rtn_" # getAddrName<addrKindCopy>.ret;
 678   let glc_value = 1;
 679   let dlc_value = 0;
 680   let Constraints = "$vdata = $vdata_in";
 681   let DisableEncoding = "$vdata_in";
 682   let AsmMatchConverter = "cvtMubufAtomicReturn";
 683 }
 684
 685 multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
 686                                         RegisterClass vdataClass,
 687                                         ValueType vdataType,
 688                                         SDPatternOperator atomic,
 689                                         bit isFP = getIsFP<vdataType>.ret> {
 690   let FPAtomic = isFP in
 691   def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
 692                 MUBUFAddr64Table <0, NAME>;
 693
 694   let FPAtomic = isFP in
 695   def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
 696                 MUBUFAddr64Table <1, NAME>;
 697
 698   let FPAtomic = isFP in
 699   def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 700
 701   let FPAtomic = isFP in
 702
 703   def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 704
 705   let FPAtomic = isFP in
 706   def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 707 }
 708
 709 multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
 710                                      RegisterClass vdataClass,
 711                                      ValueType vdataType,
 712                                      SDPatternOperator atomic,
 713                                      bit isFP = getIsFP<vdataType>.ret> {
 714   let FPAtomic = isFP in
 715   def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
 716     [(set vdataType:$vdata,
 717      (atomic (MUBUFOffsetAtomic v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$slc),
 718              vdataType:$vdata_in))]>,
 719     MUBUFAddr64Table <0, NAME # "_RTN">;
 720
 721   let FPAtomic = isFP in
 722   def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
 723     [(set vdataType:$vdata,
 724      (atomic (MUBUFAddr64Atomic v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$slc),
 725              vdataType:$vdata_in))]>,
 726     MUBUFAddr64Table <1, NAME # "_RTN">;
 727
 728   let FPAtomic = isFP in
 729   def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
 730
 731   let FPAtomic = isFP in
 732   def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
 733
 734   let FPAtomic = isFP in
 735   def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
 736 }
 737
 738 multiclass MUBUF_Pseudo_Atomics <string opName,
 739                                  RegisterClass vdataClass,
 740                                  ValueType vdataType,
 741                                  SDPatternOperator atomic> :
 742   MUBUF_Pseudo_Atomics_NO_RTN<opName, vdataClass, vdataType, atomic>,
 743   MUBUF_Pseudo_Atomics_RTN<opName, vdataClass, vdataType, atomic>;
 744
 745
 746 //===----------------------------------------------------------------------===//
 747 // MUBUF Instructions
 748 //===----------------------------------------------------------------------===//
 749
 750 defm BUFFER_LOAD_FORMAT_X : MUBUF_Pseudo_Loads_Lds <
 751   "buffer_load_format_x", VGPR_32
 752 >;
 753 defm BUFFER_LOAD_FORMAT_XY : MUBUF_Pseudo_Loads <
 754   "buffer_load_format_xy", VReg_64
 755 >;
 756 defm BUFFER_LOAD_FORMAT_XYZ : MUBUF_Pseudo_Loads <
 757   "buffer_load_format_xyz", VReg_96
 758 >;
 759 defm BUFFER_LOAD_FORMAT_XYZW : MUBUF_Pseudo_Loads <
 760   "buffer_load_format_xyzw", VReg_128
 761 >;
 762 defm BUFFER_STORE_FORMAT_X : MUBUF_Pseudo_Stores <
 763   "buffer_store_format_x", VGPR_32
 764 >;
 765 defm BUFFER_STORE_FORMAT_XY : MUBUF_Pseudo_Stores <
 766   "buffer_store_format_xy", VReg_64
 767 >;
 768 defm BUFFER_STORE_FORMAT_XYZ : MUBUF_Pseudo_Stores <
 769   "buffer_store_format_xyz", VReg_96
 770 >;
 771 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Pseudo_Stores <
 772   "buffer_store_format_xyzw", VReg_128
 773 >;
 774
 775 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
 776   defm BUFFER_LOAD_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Loads <
 777     "buffer_load_format_d16_x", VGPR_32
 778   >;
 779   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Loads <
 780     "buffer_load_format_d16_xy", VReg_64
 781   >;
 782   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Loads <
 783     "buffer_load_format_d16_xyz", VReg_96
 784   >;
 785   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Loads <
 786    "buffer_load_format_d16_xyzw", VReg_128
 787   >;
 788   defm BUFFER_STORE_FORMAT_D16_X_gfx80 : MUBUF_Pseudo_Stores <
 789     "buffer_store_format_d16_x", VGPR_32
 790   >;
 791   defm BUFFER_STORE_FORMAT_D16_XY_gfx80 : MUBUF_Pseudo_Stores <
 792     "buffer_store_format_d16_xy", VReg_64
 793   >;
 794   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80 : MUBUF_Pseudo_Stores <
 795     "buffer_store_format_d16_xyz", VReg_96
 796   >;
 797   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MUBUF_Pseudo_Stores <
 798     "buffer_store_format_d16_xyzw", VReg_128
 799   >;
 800 } // End HasUnpackedD16VMem.
 801
 802 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
 803   defm BUFFER_LOAD_FORMAT_D16_X : MUBUF_Pseudo_Loads <
 804     "buffer_load_format_d16_x", VGPR_32
 805   >;
 806   defm BUFFER_LOAD_FORMAT_D16_XY : MUBUF_Pseudo_Loads <
 807     "buffer_load_format_d16_xy", VGPR_32
 808   >;
 809   defm BUFFER_LOAD_FORMAT_D16_XYZ : MUBUF_Pseudo_Loads <
 810     "buffer_load_format_d16_xyz", VReg_64
 811   >;
 812   defm BUFFER_LOAD_FORMAT_D16_XYZW : MUBUF_Pseudo_Loads <
 813     "buffer_load_format_d16_xyzw", VReg_64
 814   >;
 815   defm BUFFER_STORE_FORMAT_D16_X : MUBUF_Pseudo_Stores <
 816     "buffer_store_format_d16_x", VGPR_32
 817   >;
 818   defm BUFFER_STORE_FORMAT_D16_XY : MUBUF_Pseudo_Stores <
 819     "buffer_store_format_d16_xy", VGPR_32
 820   >;
 821   defm BUFFER_STORE_FORMAT_D16_XYZ : MUBUF_Pseudo_Stores <
 822     "buffer_store_format_d16_xyz", VReg_64
 823   >;
 824   defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Pseudo_Stores <
 825     "buffer_store_format_d16_xyzw", VReg_64
 826   >;
 827 } // End HasPackedD16VMem.
 828
 829 defm BUFFER_LOAD_UBYTE : MUBUF_Pseudo_Loads_Lds <
 830   "buffer_load_ubyte", VGPR_32, i32
 831 >;
 832 defm BUFFER_LOAD_SBYTE : MUBUF_Pseudo_Loads_Lds <
 833   "buffer_load_sbyte", VGPR_32, i32
 834 >;
 835 defm BUFFER_LOAD_USHORT : MUBUF_Pseudo_Loads_Lds <
 836   "buffer_load_ushort", VGPR_32, i32
 837 >;
 838 defm BUFFER_LOAD_SSHORT : MUBUF_Pseudo_Loads_Lds <
 839   "buffer_load_sshort", VGPR_32, i32
 840 >;
 841 defm BUFFER_LOAD_DWORD : MUBUF_Pseudo_Loads_Lds <
 842   "buffer_load_dword", VGPR_32, i32
 843 >;
 844 defm BUFFER_LOAD_DWORDX2 : MUBUF_Pseudo_Loads <
 845   "buffer_load_dwordx2", VReg_64, v2i32
 846 >;
 847 defm BUFFER_LOAD_DWORDX3 : MUBUF_Pseudo_Loads <
 848   "buffer_load_dwordx3", VReg_96, v3i32
 849 >;
 850 defm BUFFER_LOAD_DWORDX4 : MUBUF_Pseudo_Loads <
 851   "buffer_load_dwordx4", VReg_128, v4i32
 852 >;
 853
 854 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, extloadi8_global>;
 855 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_UBYTE", i32, zextloadi8_global>;
 856 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SBYTE", i32, sextloadi8_global>;
 857 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, extloadi16_global>;
 858 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_USHORT", i32, zextloadi16_global>;
 859 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_SSHORT", i32, sextloadi16_global>;
 860 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORD", i32, load_global>;
 861 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX2", v2i32, load_global>;
 862 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX3", v3i32, load_global>;
 863 defm : MUBUF_Pseudo_Load_Pats<"BUFFER_LOAD_DWORDX4", v4i32, load_global>;
 864
 865 // This is not described in AMD documentation,
 866 // but 'lds' versions of these opcodes are available
 867 // in at least GFX8+ chips. See Bug 37653.
 868 let SubtargetPredicate = isGFX8GFX9 in {
 869 defm BUFFER_LOAD_DWORDX2_LDS : MUBUF_Pseudo_Loads <
 870   "buffer_load_dwordx2", VReg_64, v2i32, null_frag, 0, 1
 871 >;
 872 defm BUFFER_LOAD_DWORDX3_LDS : MUBUF_Pseudo_Loads <
 873   "buffer_load_dwordx3", VReg_96, untyped, null_frag, 0, 1
 874 >;
 875 defm BUFFER_LOAD_DWORDX4_LDS : MUBUF_Pseudo_Loads <
 876   "buffer_load_dwordx4", VReg_128, v4i32, null_frag, 0, 1
 877 >;
 878 }
 879
 880 defm BUFFER_STORE_BYTE : MUBUF_Pseudo_Stores <
 881   "buffer_store_byte", VGPR_32, i32, truncstorei8_global
 882 >;
 883 defm BUFFER_STORE_SHORT : MUBUF_Pseudo_Stores <
 884   "buffer_store_short", VGPR_32, i32, truncstorei16_global
 885 >;
 886 defm BUFFER_STORE_DWORD : MUBUF_Pseudo_Stores <
 887   "buffer_store_dword", VGPR_32, i32, store_global
 888 >;
 889 defm BUFFER_STORE_DWORDX2 : MUBUF_Pseudo_Stores <
 890   "buffer_store_dwordx2", VReg_64, v2i32, store_global
 891 >;
 892 defm BUFFER_STORE_DWORDX3 : MUBUF_Pseudo_Stores <
 893   "buffer_store_dwordx3", VReg_96, v3i32, store_global
 894 >;
 895 defm BUFFER_STORE_DWORDX4 : MUBUF_Pseudo_Stores <
 896   "buffer_store_dwordx4", VReg_128, v4i32, store_global
 897 >;
 898 defm BUFFER_ATOMIC_SWAP : MUBUF_Pseudo_Atomics <
 899   "buffer_atomic_swap", VGPR_32, i32, atomic_swap_global_32
 900 >;
 901 defm BUFFER_ATOMIC_CMPSWAP : MUBUF_Pseudo_Atomics <
 902   "buffer_atomic_cmpswap", VReg_64, v2i32, null_frag
 903 >;
 904 defm BUFFER_ATOMIC_ADD : MUBUF_Pseudo_Atomics <
 905   "buffer_atomic_add", VGPR_32, i32, atomic_load_add_global_32
 906 >;
 907 defm BUFFER_ATOMIC_SUB : MUBUF_Pseudo_Atomics <
 908   "buffer_atomic_sub", VGPR_32, i32, atomic_load_sub_global_32
 909 >;
 910 defm BUFFER_ATOMIC_SMIN : MUBUF_Pseudo_Atomics <
 911   "buffer_atomic_smin", VGPR_32, i32, atomic_load_min_global_32
 912 >;
 913 defm BUFFER_ATOMIC_UMIN : MUBUF_Pseudo_Atomics <
 914   "buffer_atomic_umin", VGPR_32, i32, atomic_load_umin_global_32
 915 >;
 916 defm BUFFER_ATOMIC_SMAX : MUBUF_Pseudo_Atomics <
 917   "buffer_atomic_smax", VGPR_32, i32, atomic_load_max_global_32
 918 >;
 919 defm BUFFER_ATOMIC_UMAX : MUBUF_Pseudo_Atomics <
 920   "buffer_atomic_umax", VGPR_32, i32, atomic_load_umax_global_32
 921 >;
 922 defm BUFFER_ATOMIC_AND : MUBUF_Pseudo_Atomics <
 923   "buffer_atomic_and", VGPR_32, i32, atomic_load_and_global_32
 924 >;
 925 defm BUFFER_ATOMIC_OR : MUBUF_Pseudo_Atomics <
 926   "buffer_atomic_or", VGPR_32, i32, atomic_load_or_global_32
 927 >;
 928 defm BUFFER_ATOMIC_XOR : MUBUF_Pseudo_Atomics <
 929   "buffer_atomic_xor", VGPR_32, i32, atomic_load_xor_global_32
 930 >;
 931 defm BUFFER_ATOMIC_INC : MUBUF_Pseudo_Atomics <
 932   "buffer_atomic_inc", VGPR_32, i32, atomic_inc_global_32
 933 >;
 934 defm BUFFER_ATOMIC_DEC : MUBUF_Pseudo_Atomics <
 935   "buffer_atomic_dec", VGPR_32, i32, atomic_dec_global_32
 936 >;
 937 defm BUFFER_ATOMIC_SWAP_X2 : MUBUF_Pseudo_Atomics <
 938   "buffer_atomic_swap_x2", VReg_64, i64, atomic_swap_global_64
 939 >;
 940 defm BUFFER_ATOMIC_CMPSWAP_X2 : MUBUF_Pseudo_Atomics <
 941   "buffer_atomic_cmpswap_x2", VReg_128, v2i64, null_frag
 942 >;
 943 defm BUFFER_ATOMIC_ADD_X2 : MUBUF_Pseudo_Atomics <
 944   "buffer_atomic_add_x2", VReg_64, i64, atomic_load_add_global_64
 945 >;
 946 defm BUFFER_ATOMIC_SUB_X2 : MUBUF_Pseudo_Atomics <
 947   "buffer_atomic_sub_x2", VReg_64, i64, atomic_load_sub_global_64
 948 >;
 949 defm BUFFER_ATOMIC_SMIN_X2 : MUBUF_Pseudo_Atomics <
 950   "buffer_atomic_smin_x2", VReg_64, i64, atomic_load_min_global_64
 951 >;
 952 defm BUFFER_ATOMIC_UMIN_X2 : MUBUF_Pseudo_Atomics <
 953   "buffer_atomic_umin_x2", VReg_64, i64, atomic_load_umin_global_64
 954 >;
 955 defm BUFFER_ATOMIC_SMAX_X2 : MUBUF_Pseudo_Atomics <
 956   "buffer_atomic_smax_x2", VReg_64, i64, atomic_load_max_global_64
 957 >;
 958 defm BUFFER_ATOMIC_UMAX_X2 : MUBUF_Pseudo_Atomics <
 959   "buffer_atomic_umax_x2", VReg_64, i64, atomic_load_umax_global_64
 960 >;
 961 defm BUFFER_ATOMIC_AND_X2 : MUBUF_Pseudo_Atomics <
 962   "buffer_atomic_and_x2", VReg_64, i64, atomic_load_and_global_64
 963 >;
 964 defm BUFFER_ATOMIC_OR_X2 : MUBUF_Pseudo_Atomics <
 965   "buffer_atomic_or_x2", VReg_64, i64, atomic_load_or_global_64
 966 >;
 967 defm BUFFER_ATOMIC_XOR_X2 : MUBUF_Pseudo_Atomics <
 968   "buffer_atomic_xor_x2", VReg_64, i64, atomic_load_xor_global_64
 969 >;
 970 defm BUFFER_ATOMIC_INC_X2 : MUBUF_Pseudo_Atomics <
 971   "buffer_atomic_inc_x2", VReg_64, i64, atomic_inc_global_64
 972 >;
 973 defm BUFFER_ATOMIC_DEC_X2 : MUBUF_Pseudo_Atomics <
 974   "buffer_atomic_dec_x2", VReg_64, i64, atomic_dec_global_64
 975 >;
 976
 977 let SubtargetPredicate = isGFX8GFX9 in {
 978 def BUFFER_STORE_LDS_DWORD : MUBUF_Pseudo_Store_Lds <"buffer_store_lds_dword">;
 979 }
 980
 981 let SubtargetPredicate = isGFX6 in { // isn't on CI & VI
 982 /*
 983 defm BUFFER_ATOMIC_RSUB        : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub">;
 984 defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap">;
 985 defm BUFFER_ATOMIC_FMIN        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin">;
 986 defm BUFFER_ATOMIC_FMAX        : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax">;
 987 defm BUFFER_ATOMIC_RSUB_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_rsub_x2">;
 988 defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Pseudo_Atomics <"buffer_atomic_fcmpswap_x2">;
 989 defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmin_x2">;
 990 defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Pseudo_Atomics <"buffer_atomic_fmax_x2">;
 991 */
 992
 993 def BUFFER_WBINVL1_SC : MUBUF_Invalidate <"buffer_wbinvl1_sc",
 994                                           int_amdgcn_buffer_wbinvl1_sc>;
 995 }
 996
 997 let SubtargetPredicate = HasD16LoadStore in {
 998
 999 defm BUFFER_LOAD_UBYTE_D16 : MUBUF_Pseudo_Loads <
1000   "buffer_load_ubyte_d16", VGPR_32, i32, null_frag, 1
1001 >;
1002
1003 defm BUFFER_LOAD_UBYTE_D16_HI : MUBUF_Pseudo_Loads <
1004   "buffer_load_ubyte_d16_hi", VGPR_32, i32, null_frag, 1
1005 >;
1006
1007 defm BUFFER_LOAD_SBYTE_D16 : MUBUF_Pseudo_Loads <
1008   "buffer_load_sbyte_d16", VGPR_32, i32, null_frag, 1
1009 >;
1010
1011 defm BUFFER_LOAD_SBYTE_D16_HI : MUBUF_Pseudo_Loads <
1012   "buffer_load_sbyte_d16_hi", VGPR_32, i32, null_frag, 1
1013 >;
1014
1015 defm BUFFER_LOAD_SHORT_D16 : MUBUF_Pseudo_Loads <
1016   "buffer_load_short_d16", VGPR_32, i32, null_frag, 1
1017 >;
1018
1019 defm BUFFER_LOAD_SHORT_D16_HI : MUBUF_Pseudo_Loads <
1020   "buffer_load_short_d16_hi", VGPR_32, i32, null_frag, 1
1021 >;
1022
1023 defm BUFFER_STORE_BYTE_D16_HI : MUBUF_Pseudo_Stores <
1024   "buffer_store_byte_d16_hi", VGPR_32, i32
1025 >;
1026
1027 defm BUFFER_STORE_SHORT_D16_HI : MUBUF_Pseudo_Stores <
1028   "buffer_store_short_d16_hi", VGPR_32, i32
1029 >;
1030
1031 defm BUFFER_LOAD_FORMAT_D16_HI_X : MUBUF_Pseudo_Loads <
1032   "buffer_load_format_d16_hi_x", VGPR_32
1033 >;
1034 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Pseudo_Stores <
1035   "buffer_store_format_d16_hi_x", VGPR_32
1036 >;
1037
1038 } // End HasD16LoadStore
1039
1040 def BUFFER_WBINVL1 : MUBUF_Invalidate <"buffer_wbinvl1",
1041                                        int_amdgcn_buffer_wbinvl1>;
1042
1043 let SubtargetPredicate = HasAtomicFaddInsts in {
1044
1045 defm BUFFER_ATOMIC_ADD_F32 : MUBUF_Pseudo_Atomics_NO_RTN <
1046   "buffer_atomic_add_f32", VGPR_32, f32, atomic_fadd_global_noret
1047 >;
1048 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
1049   "buffer_atomic_pk_add_f16", VGPR_32, v2f16, atomic_pk_fadd_global_noret
1050 >;
1051
1052 } // End SubtargetPredicate = HasAtomicFaddInsts
1053
1054 //===----------------------------------------------------------------------===//
1055 // MTBUF Instructions
1056 //===----------------------------------------------------------------------===//
1057
1058 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;
1059 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64>;
1060 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96>;
1061 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128>;
1062 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32>;
1063 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64>;
1064 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96>;
1065 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>;
1066
1067 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
1068   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
1069   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64>;
1070   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96>;
1071   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128>;
1072   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
1073   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64>;
1074   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96>;
1075   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>;
1076 } // End HasUnpackedD16VMem.
1077
1078 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
1079   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
1080   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32>;
1081   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64>;
1082   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64>;
1083   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
1084   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32>;
1085   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64>;
1086   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>;
1087 } // End HasPackedD16VMem.
1088
1089 let SubtargetPredicate = isGFX7Plus in {
1090
1091 //===----------------------------------------------------------------------===//
1092 // Instruction definitions for CI and newer.
1093 //===----------------------------------------------------------------------===//
1094
1095 def BUFFER_WBINVL1_VOL : MUBUF_Invalidate <"buffer_wbinvl1_vol",
1096                                            int_amdgcn_buffer_wbinvl1_vol>;
1097
1098 } // End let SubtargetPredicate = isGFX7Plus
1099
1100 let SubtargetPredicate = isGFX10Plus in {
1101   def BUFFER_GL0_INV : MUBUF_Invalidate<"buffer_gl0_inv">;
1102   def BUFFER_GL1_INV : MUBUF_Invalidate<"buffer_gl1_inv">;
1103 } // End SubtargetPredicate = isGFX10Plus
1104
1105 //===----------------------------------------------------------------------===//
1106 // MUBUF Patterns
1107 //===----------------------------------------------------------------------===//
1108
1109 def extract_glc : SDNodeXForm<imm, [{
1110   return CurDAG->getTargetConstant(N->getZExtValue() & 1, SDLoc(N), MVT::i8);
1111 }]>;
1112
1113 def extract_slc : SDNodeXForm<imm, [{
1114   return CurDAG->getTargetConstant((N->getZExtValue() >> 1) & 1, SDLoc(N), MVT::i8);
1115 }]>;
1116
1117 def extract_dlc : SDNodeXForm<imm, [{
1118   return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8);
1119 }]>;
1120
1121 //===----------------------------------------------------------------------===//
1122 // buffer_load/store_format patterns
1123 //===----------------------------------------------------------------------===//
1124
1125 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1126                                   string opcode> {
1127   def : GCNPat<
1128     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1129               imm:$cachepolicy, 0)),
1130     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1131       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1132   >;
1133
1134   def : GCNPat<
1135     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1136               imm:$cachepolicy, 0)),
1137     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1138       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1139   >;
1140
1141   def : GCNPat<
1142     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1143               imm:$cachepolicy, imm)),
1144     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1145       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1146   >;
1147
1148   def : GCNPat<
1149     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1150               imm:$cachepolicy, imm)),
1151     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1152       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1153       $rsrc, $soffset, (as_i16imm $offset),
1154       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1155   >;
1156 }
1157
1158 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, f32, "BUFFER_LOAD_FORMAT_X">;
1159 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, i32, "BUFFER_LOAD_FORMAT_X">;
1160 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2f32, "BUFFER_LOAD_FORMAT_XY">;
1161 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v2i32, "BUFFER_LOAD_FORMAT_XY">;
1162 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3f32, "BUFFER_LOAD_FORMAT_XYZ">;
1163 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v3i32, "BUFFER_LOAD_FORMAT_XYZ">;
1164 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4f32, "BUFFER_LOAD_FORMAT_XYZW">;
1165 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format, v4i32, "BUFFER_LOAD_FORMAT_XYZW">;
1166
1167 let SubtargetPredicate = HasUnpackedD16VMem in {
1168   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1169   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X_gfx80">;
1170   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i32, "BUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1171   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i32, "BUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1172 } // End HasUnpackedD16VMem.
1173
1174 let SubtargetPredicate = HasPackedD16VMem in {
1175   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, f16, "BUFFER_LOAD_FORMAT_D16_X">;
1176   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, i16, "BUFFER_LOAD_FORMAT_D16_X">;
1177   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2f16, "BUFFER_LOAD_FORMAT_D16_XY">;
1178   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v2i16, "BUFFER_LOAD_FORMAT_D16_XY">;
1179   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4f16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1180   defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_format_d16, v4i16, "BUFFER_LOAD_FORMAT_D16_XYZW">;
1181 } // End HasPackedD16VMem.
1182
1183 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, f32, "BUFFER_LOAD_DWORD">;
1184 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, i32, "BUFFER_LOAD_DWORD">;
1185 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i16, "BUFFER_LOAD_DWORD">;
1186 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f16, "BUFFER_LOAD_DWORD">;
1187 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2f32, "BUFFER_LOAD_DWORDX2">;
1188 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v2i32, "BUFFER_LOAD_DWORDX2">;
1189 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i16, "BUFFER_LOAD_DWORDX2">;
1190 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f16, "BUFFER_LOAD_DWORDX2">;
1191 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3f32, "BUFFER_LOAD_DWORDX3">;
1192 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v3i32, "BUFFER_LOAD_DWORDX3">;
1193 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4f32, "BUFFER_LOAD_DWORDX4">;
1194 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load, v4i32, "BUFFER_LOAD_DWORDX4">;
1195 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_byte, i32, "BUFFER_LOAD_SBYTE">;
1196 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_short, i32, "BUFFER_LOAD_SSHORT">;
1197 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ubyte, i32, "BUFFER_LOAD_UBYTE">;
1198 defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
1199
1200 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1201                                    string opcode> {
1202   def : GCNPat<
1203     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1204               imm:$cachepolicy, 0),
1205     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
1206       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1207   >;
1208
1209   def : GCNPat<
1210     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1211               imm:$cachepolicy, 0),
1212     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1213       (as_i16imm $offset), (extract_glc $cachepolicy),
1214       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1215   >;
1216
1217   def : GCNPat<
1218     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1219               imm:$cachepolicy, imm),
1220     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1221       (as_i16imm $offset), (extract_glc $cachepolicy),
1222       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1223   >;
1224
1225   def : GCNPat<
1226     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1227               imm:$cachepolicy, imm),
1228     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
1229       $vdata,
1230       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1231       $rsrc, $soffset, (as_i16imm $offset), (extract_glc $cachepolicy),
1232       (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
1233   >;
1234 }
1235
1236 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, f32, "BUFFER_STORE_FORMAT_X">;
1237 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, i32, "BUFFER_STORE_FORMAT_X">;
1238 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2f32, "BUFFER_STORE_FORMAT_XY">;
1239 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v2i32, "BUFFER_STORE_FORMAT_XY">;
1240 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3f32, "BUFFER_STORE_FORMAT_XYZ">;
1241 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v3i32, "BUFFER_STORE_FORMAT_XYZ">;
1242 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4f32, "BUFFER_STORE_FORMAT_XYZW">;
1243 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format, v4i32, "BUFFER_STORE_FORMAT_XYZW">;
1244
1245 let SubtargetPredicate = HasUnpackedD16VMem in {
1246   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1247   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X_gfx80">;
1248   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i32, "BUFFER_STORE_FORMAT_D16_XY_gfx80">;
1249   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i32, "BUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1250 } // End HasUnpackedD16VMem.
1251
1252 let SubtargetPredicate = HasPackedD16VMem in {
1253   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, f16, "BUFFER_STORE_FORMAT_D16_X">;
1254   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, i16, "BUFFER_STORE_FORMAT_D16_X">;
1255   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2f16, "BUFFER_STORE_FORMAT_D16_XY">;
1256   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v2i16, "BUFFER_STORE_FORMAT_D16_XY">;
1257   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4f16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1258   defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_format_d16, v4i16, "BUFFER_STORE_FORMAT_D16_XYZW">;
1259 } // End HasPackedD16VMem.
1260
1261 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, f32, "BUFFER_STORE_DWORD">;
1262 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, i32, "BUFFER_STORE_DWORD">;
1263 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i16, "BUFFER_STORE_DWORD">;
1264 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f16, "BUFFER_STORE_DWORD">;
1265 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2f32, "BUFFER_STORE_DWORDX2">;
1266 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v2i32, "BUFFER_STORE_DWORDX2">;
1267 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i16, "BUFFER_STORE_DWORDX2">;
1268 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f16, "BUFFER_STORE_DWORDX2">;
1269 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3f32, "BUFFER_STORE_DWORDX3">;
1270 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v3i32, "BUFFER_STORE_DWORDX3">;
1271 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4f32, "BUFFER_STORE_DWORDX4">;
1272 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store, v4i32, "BUFFER_STORE_DWORDX4">;
1273 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_byte, i32, "BUFFER_STORE_BYTE">;
1274 defm : MUBUF_StoreIntrinsicPat<SIbuffer_store_short, i32, "BUFFER_STORE_SHORT">;
1275
1276 //===----------------------------------------------------------------------===//
1277 // buffer_atomic patterns
1278 //===----------------------------------------------------------------------===//
1279
1280 multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
1281                                 string opcode> {
1282   def : GCNPat<
1283     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1284           0, i32:$soffset, imm:$offset,
1285           imm:$cachepolicy, 0)),
1286     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset,
1287                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1288   >;
1289
1290   def : GCNPat<
1291     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1292           0, i32:$soffset, imm:$offset,
1293           imm:$cachepolicy, imm)),
1294     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset,
1295                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1296   >;
1297
1298   def : GCNPat<
1299     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
1300           i32:$voffset, i32:$soffset, imm:$offset,
1301           imm:$cachepolicy, 0)),
1302     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset,
1303                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1304   >;
1305
1306   def : GCNPat<
1307     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1308           i32:$voffset, i32:$soffset, imm:$offset,
1309           imm:$cachepolicy, imm)),
1310     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
1311       $vdata_in,
1312       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1313       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1314   >;
1315 }
1316
1317 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i32, "BUFFER_ATOMIC_SWAP">;
1318 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i32, "BUFFER_ATOMIC_ADD">;
1319 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i32, "BUFFER_ATOMIC_SUB">;
1320 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i32, "BUFFER_ATOMIC_SMIN">;
1321 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i32, "BUFFER_ATOMIC_UMIN">;
1322 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i32, "BUFFER_ATOMIC_SMAX">;
1323 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i32, "BUFFER_ATOMIC_UMAX">;
1324 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i32, "BUFFER_ATOMIC_AND">;
1325 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i32, "BUFFER_ATOMIC_OR">;
1326 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i32, "BUFFER_ATOMIC_XOR">;
1327 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i32, "BUFFER_ATOMIC_INC">;
1328 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i32, "BUFFER_ATOMIC_DEC">;
1329 defm : BufferAtomicPatterns<SIbuffer_atomic_swap, i64, "BUFFER_ATOMIC_SWAP_X2">;
1330 defm : BufferAtomicPatterns<SIbuffer_atomic_add, i64,  "BUFFER_ATOMIC_ADD_X2">;
1331 defm : BufferAtomicPatterns<SIbuffer_atomic_sub, i64, "BUFFER_ATOMIC_SUB_X2">;
1332 defm : BufferAtomicPatterns<SIbuffer_atomic_smin, i64, "BUFFER_ATOMIC_SMIN_X2">;
1333 defm : BufferAtomicPatterns<SIbuffer_atomic_umin, i64, "BUFFER_ATOMIC_UMIN_X2">;
1334 defm : BufferAtomicPatterns<SIbuffer_atomic_smax, i64, "BUFFER_ATOMIC_SMAX_X2">;
1335 defm : BufferAtomicPatterns<SIbuffer_atomic_umax, i64, "BUFFER_ATOMIC_UMAX_X2">;
1336 defm : BufferAtomicPatterns<SIbuffer_atomic_and, i64, "BUFFER_ATOMIC_AND_X2">;
1337 defm : BufferAtomicPatterns<SIbuffer_atomic_or, i64, "BUFFER_ATOMIC_OR_X2">;
1338 defm : BufferAtomicPatterns<SIbuffer_atomic_xor, i64, "BUFFER_ATOMIC_XOR_X2">;
1339 defm : BufferAtomicPatterns<SIbuffer_atomic_inc, i64, "BUFFER_ATOMIC_INC_X2">;
1340 defm : BufferAtomicPatterns<SIbuffer_atomic_dec, i64, "BUFFER_ATOMIC_DEC_X2">;
1341
1342 multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
1343                                        string opcode> {
1344   def : GCNPat<
1345     (name vt:$vdata_in, v4i32:$rsrc, 0,
1346           0, i32:$soffset, imm:$offset,
1347           imm:$cachepolicy, 0),
1348     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
1349                                         (as_i16imm $offset), (extract_slc $cachepolicy))
1350   >;
1351
1352   def : GCNPat<
1353     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1354           0, i32:$soffset, imm:$offset,
1355           imm:$cachepolicy, imm),
1356     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
1357                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1358   >;
1359
1360   def : GCNPat<
1361     (name vt:$vdata_in, v4i32:$rsrc, 0,
1362           i32:$voffset, i32:$soffset, imm:$offset,
1363           imm:$cachepolicy, 0),
1364     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
1365                                        (as_i16imm $offset), (extract_slc $cachepolicy))
1366   >;
1367
1368   def : GCNPat<
1369     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
1370           i32:$voffset, i32:$soffset, imm:$offset,
1371           imm:$cachepolicy, imm),
1372     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
1373       $vdata_in,
1374       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1375       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy))
1376   >;
1377 }
1378
1379 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_fadd, f32, "BUFFER_ATOMIC_ADD_F32">;
1380 defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMIC_PK_ADD_F16">;
1381
1382 def : GCNPat<
1383   (SIbuffer_atomic_cmpswap
1384       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1385       0, i32:$soffset, imm:$offset,
1386       imm:$cachepolicy, 0),
1387   (EXTRACT_SUBREG
1388     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
1389       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1390       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1391     sub0)
1392 >;
1393
1394 def : GCNPat<
1395   (SIbuffer_atomic_cmpswap
1396       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1397       0, i32:$soffset, imm:$offset,
1398       imm:$cachepolicy, imm),
1399   (EXTRACT_SUBREG
1400     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
1401       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1402       $vindex, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1403     sub0)
1404 >;
1405
1406 def : GCNPat<
1407   (SIbuffer_atomic_cmpswap
1408       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
1409       i32:$voffset, i32:$soffset, imm:$offset,
1410       imm:$cachepolicy, 0),
1411   (EXTRACT_SUBREG
1412     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
1413       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1414       $voffset, $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1415     sub0)
1416 >;
1417
1418 def : GCNPat<
1419   (SIbuffer_atomic_cmpswap
1420       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
1421       i32:$voffset, i32:$soffset, imm:$offset,
1422       imm:$cachepolicy, imm),
1423   (EXTRACT_SUBREG
1424     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
1425       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
1426       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1427       $rsrc, $soffset, (as_i16imm $offset), (extract_slc $cachepolicy)),
1428     sub0)
1429 >;
1430
1431 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
1432                               PatFrag constant_ld> : GCNPat <
1433      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1434                                    i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
1435      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1436   >;
1437
1438 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1439                                      ValueType vt, PatFrag atomic_ld> {
1440   def : GCNPat <
1441      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1442                                    i16:$offset, i1:$slc))),
1443      (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
1444   >;
1445
1446   def : GCNPat <
1447     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
1448     (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
1449   >;
1450 }
1451
1452 let SubtargetPredicate = isGFX6GFX7 in {
1453 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SBYTE_ADDR64, i32, sextloadi8_constant>;
1454 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, extloadi8_constant>;
1455 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_UBYTE_ADDR64, i32, zextloadi8_constant>;
1456 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_SSHORT_ADDR64, i32, sextloadi16_constant>;
1457 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, extloadi16_constant>;
1458 def : MUBUFLoad_PatternADDR64 <BUFFER_LOAD_USHORT_ADDR64, i32, zextloadi16_constant>;
1459
1460 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORD_ADDR64, BUFFER_LOAD_DWORD_OFFSET, i32, atomic_load_32_global>;
1461 defm : MUBUFLoad_Atomic_Pattern <BUFFER_LOAD_DWORDX2_ADDR64, BUFFER_LOAD_DWORDX2_OFFSET, i64, atomic_load_64_global>;
1462 } // End SubtargetPredicate = isGFX6GFX7
1463
1464 multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1465                                PatFrag ld> {
1466
1467   def : GCNPat <
1468     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1469                           i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
1470     (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1471   >;
1472 }
1473
1474 let OtherPredicates = [Has16BitInsts] in {
1475
1476 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_constant>;
1477 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_constant>;
1478 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_constant>;
1479 defm : MUBUFLoad_Pattern <BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_global>;
1480 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_global>;
1481 defm : MUBUFLoad_Pattern <BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_global>;
1482
1483 defm : MUBUFLoad_Pattern <BUFFER_LOAD_USHORT_OFFSET, i16, load_global>;
1484
1485 } // End OtherPredicates = [Has16BitInsts]
1486
1487 multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
1488                                 MUBUF_Pseudo InstrOffset,
1489                                 ValueType vt, PatFrag ld> {
1490   def : GCNPat <
1491     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1492                                i32:$soffset, u16imm:$offset))),
1493     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1494   >;
1495
1496   def : GCNPat <
1497     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
1498     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0)
1499   >;
1500 }
1501
1502 // XXX - Is it possible to have a complex pattern in a PatFrag?
1503 multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
1504                                 MUBUF_Pseudo InstrOffset,
1505                                 ValueType vt, PatFrag ld_frag> {
1506   def : GCNPat <
1507     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
1508     (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
1509   >;
1510
1511   def : GCNPat <
1512     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
1513     (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
1514   >;
1515 }
1516
1517 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i32, sextloadi8_private>;
1518 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, extloadi8_private>;
1519 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i32, zextloadi8_private>;
1520 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SBYTE_OFFEN, BUFFER_LOAD_SBYTE_OFFSET, i16, sextloadi8_private>;
1521 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, extloadi8_private>;
1522 defm : MUBUFScratchLoadPat <BUFFER_LOAD_UBYTE_OFFEN, BUFFER_LOAD_UBYTE_OFFSET, i16, zextloadi8_private>;
1523 defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET, i32, sextloadi16_private>;
1524 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
1525 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
1526 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
1527 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
1528 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
1529 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
1530 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
1531
1532 let OtherPredicates = [D16PreservesUnusedBits] in {
1533 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2i16, load_d16_hi_private>;
1534 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2i16, az_extloadi8_d16_hi_private>;
1535 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2i16, sextloadi8_d16_hi_private>;
1536 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_HI_OFFEN, BUFFER_LOAD_SHORT_D16_HI_OFFSET, v2f16, load_d16_hi_private>;
1537 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_HI_OFFEN, BUFFER_LOAD_UBYTE_D16_HI_OFFSET, v2f16, az_extloadi8_d16_hi_private>;
1538 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_HI_OFFEN, BUFFER_LOAD_SBYTE_D16_HI_OFFSET, v2f16, sextloadi8_d16_hi_private>;
1539
1540 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2i16, load_d16_lo_private>;
1541 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2i16, az_extloadi8_d16_lo_private>;
1542 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2i16, sextloadi8_d16_lo_private>;
1543 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SHORT_D16_OFFEN, BUFFER_LOAD_SHORT_D16_OFFSET, v2f16, load_d16_lo_private>;
1544 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_UBYTE_D16_OFFEN, BUFFER_LOAD_UBYTE_D16_OFFSET, v2f16, az_extloadi8_d16_lo_private>;
1545 defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D16_OFFSET, v2f16, sextloadi8_d16_lo_private>;
1546 }
1547
1548 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
1549                                       ValueType vt, PatFrag atomic_st> {
1550   // Store follows atomic op convention so address is forst
1551   def : GCNPat <
1552      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
1553                                    i16:$offset, i1:$slc), vt:$val),
1554      (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
1555   >;
1556
1557   def : GCNPat <
1558     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
1559     (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
1560   >;
1561 }
1562 let SubtargetPredicate = isGFX6GFX7 in {
1563 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>;
1564 defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>;
1565 } // End Predicates = isGFX6GFX7
1566
1567
1568 multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
1569                                PatFrag st> {
1570
1571   def : GCNPat <
1572     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
1573                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)),
1574     (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
1575   >;
1576 }
1577
1578 defm : MUBUFStore_Pattern <BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_global>;
1579 defm : MUBUFStore_Pattern <BUFFER_STORE_SHORT_OFFSET, i16, store_global>;
1580
1581 multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
1582                                  MUBUF_Pseudo InstrOffset,
1583                                  ValueType vt, PatFrag st,
1584                                  RegisterClass rc = VGPR_32> {
1585   def : GCNPat <
1586     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
1587                                       i32:$soffset, u16imm:$offset)),
1588     (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1589   >;
1590
1591   def : GCNPat <
1592     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
1593                                        u16imm:$offset)),
1594     (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0)
1595   >;
1596 }
1597
1598 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i32, truncstorei8_private>;
1599 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
1600 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
1601 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
1602 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
1603 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
1604 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
1605 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
1606
1607
1608 let OtherPredicates = [D16PreservesUnusedBits] in {
1609  // Hiding the extract high pattern in the PatFrag seems to not
1610  // automatically increase the complexity.
1611 let AddedComplexity = 1 in {
1612 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_D16_HI_OFFEN, BUFFER_STORE_SHORT_D16_HI_OFFSET, i32, store_hi16_private>;
1613 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D16_HI_OFFSET, i32, truncstorei8_hi16_private>;
1614 }
1615 }
1616
1617 //===----------------------------------------------------------------------===//
1618 // MTBUF Patterns
1619 //===----------------------------------------------------------------------===//
1620
1621 //===----------------------------------------------------------------------===//
1622 // tbuffer_load/store_format patterns
1623 //===----------------------------------------------------------------------===//
1624
1625 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
1626                                   string opcode> {
1627   def : GCNPat<
1628     (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1629               imm:$format, imm:$cachepolicy, 0)),
1630     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
1631       (as_i8imm $format),
1632       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1633   >;
1634
1635   def : GCNPat<
1636     (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1637               imm:$format, imm:$cachepolicy, imm)),
1638     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
1639       (as_i8imm $format),
1640       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1641   >;
1642
1643   def : GCNPat<
1644     (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1645               imm:$format, imm:$cachepolicy, 0)),
1646     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
1647       (as_i8imm $format),
1648       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1649   >;
1650
1651   def : GCNPat<
1652     (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
1653               imm:$format, imm:$cachepolicy, imm)),
1654     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
1655       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1656       $rsrc, $soffset, (as_i16imm $offset),
1657       (as_i8imm $format),
1658       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1659   >;
1660 }
1661
1662 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, i32,   "TBUFFER_LOAD_FORMAT_X">;
1663 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2i32, "TBUFFER_LOAD_FORMAT_XY">;
1664 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3i32, "TBUFFER_LOAD_FORMAT_XYZ">;
1665 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4i32, "TBUFFER_LOAD_FORMAT_XYZW">;
1666 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, f32,   "TBUFFER_LOAD_FORMAT_X">;
1667 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v2f32, "TBUFFER_LOAD_FORMAT_XY">;
1668 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v3f32, "TBUFFER_LOAD_FORMAT_XYZ">;
1669 defm : MTBUF_LoadIntrinsicPat<SItbuffer_load, v4f32, "TBUFFER_LOAD_FORMAT_XYZW">;
1670
1671 let SubtargetPredicate = HasUnpackedD16VMem in {
1672   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X_gfx80">;
1673   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2i32, "TBUFFER_LOAD_FORMAT_D16_XY_gfx80">;
1674   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4i32, "TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80">;
1675 } // End HasUnpackedD16VMem.
1676
1677 let SubtargetPredicate = HasPackedD16VMem in {
1678   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, f16,   "TBUFFER_LOAD_FORMAT_D16_X">;
1679   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v2f16, "TBUFFER_LOAD_FORMAT_D16_XY">;
1680   defm : MTBUF_LoadIntrinsicPat<SItbuffer_load_d16, v4f16, "TBUFFER_LOAD_FORMAT_D16_XYZW">;
1681 } // End HasPackedD16VMem.
1682
1683 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
1684                                    string opcode> {
1685   def : GCNPat<
1686     (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
1687           imm:$format, imm:$cachepolicy, 0),
1688     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
1689       (as_i16imm $offset), (as_i8imm $format),
1690       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1691   >;
1692
1693   def : GCNPat<
1694     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
1695           imm:$format, imm:$cachepolicy, imm),
1696     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
1697       (as_i16imm $offset), (as_i8imm $format),
1698       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1699   >;
1700
1701   def : GCNPat<
1702     (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
1703           imm:$format, imm:$cachepolicy, 0),
1704     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
1705       (as_i16imm $offset), (as_i8imm $format),
1706       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1707   >;
1708
1709   def : GCNPat<
1710     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
1711           imm:$offset, imm:$format, imm:$cachepolicy, imm),
1712     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
1713       $vdata,
1714       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
1715       $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format),
1716       (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
1717   >;
1718 }
1719
1720 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, i32,   "TBUFFER_STORE_FORMAT_X">;
1721 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2i32, "TBUFFER_STORE_FORMAT_XY">;
1722 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3i32, "TBUFFER_STORE_FORMAT_XYZ">;
1723 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4i32, "TBUFFER_STORE_FORMAT_XYZW">;
1724 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, f32,   "TBUFFER_STORE_FORMAT_X">;
1725 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v2f32, "TBUFFER_STORE_FORMAT_XY">;
1726 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v3f32, "TBUFFER_STORE_FORMAT_XYZ">;
1727 defm : MTBUF_StoreIntrinsicPat<SItbuffer_store, v4f32, "TBUFFER_STORE_FORMAT_XYZW">;
1728
1729 let SubtargetPredicate = HasUnpackedD16VMem in {
1730   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X_gfx80">;
1731   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2i32, "TBUFFER_STORE_FORMAT_D16_XY_gfx80">;
1732   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4i32, "TBUFFER_STORE_FORMAT_D16_XYZW_gfx80">;
1733 } // End HasUnpackedD16VMem.
1734
1735 let SubtargetPredicate = HasPackedD16VMem in {
1736   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, f16,   "TBUFFER_STORE_FORMAT_D16_X">;
1737   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v2f16, "TBUFFER_STORE_FORMAT_D16_XY">;
1738   defm : MTBUF_StoreIntrinsicPat<SItbuffer_store_d16, v4f16, "TBUFFER_STORE_FORMAT_D16_XYZW">;
1739 } // End HasPackedD16VMem.
1740
1741 //===----------------------------------------------------------------------===//
1742 // Target-specific instruction encodings.
1743 //===----------------------------------------------------------------------===//
1744
1745 //===----------------------------------------------------------------------===//
1746 // Base ENC_MUBUF for GFX6, GFX7, GFX10.
1747 //===----------------------------------------------------------------------===//
1748
1749 class Base_MUBUF_Real_gfx6_gfx7_gfx10<bits<7> op, MUBUF_Pseudo ps, int ef> :
1750     MUBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
1751   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
1752   let Inst{12}    = ps.offen;
1753   let Inst{13}    = ps.idxen;
1754   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
1755   let Inst{16}    = !if(ps.lds, 1, 0);
1756   let Inst{24-18} = op;
1757   let Inst{31-26} = 0x38;
1758   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
1759   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
1760   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
1761   let Inst{54}    = !if(ps.has_slc, slc, ?);
1762   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
1763   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
1764 }
1765
1766 class MUBUF_Real_gfx10<bits<8> op, MUBUF_Pseudo ps> :
1767     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.GFX10> {
1768   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
1769   let Inst{25} = op{7};
1770 }
1771
1772 class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
1773     Base_MUBUF_Real_gfx6_gfx7_gfx10<op{6-0}, ps, SIEncodingFamily.SI> {
1774   let Inst{15} = ps.addr64;
1775 }
1776
1777 //===----------------------------------------------------------------------===//
1778 // MUBUF - GFX10.
1779 //===----------------------------------------------------------------------===//
1780
1781 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
1782   multiclass MUBUF_Real_gfx10_with_name<bits<8> op, string opName,
1783                                         string asmName> {
1784     def _gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(opName)> {
1785       MUBUF_Pseudo ps = !cast<MUBUF_Pseudo>(opName);
1786       let AsmString = asmName # ps.AsmOperands;
1787     }
1788   }
1789   multiclass MUBUF_Real_AllAddr_gfx10<bits<8> op> {
1790     def _BOTHEN_gfx10 :
1791       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1792     def _IDXEN_gfx10 :
1793       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1794     def _OFFEN_gfx10 :
1795       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1796     def _OFFSET_gfx10 :
1797       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1798   }
1799   multiclass MUBUF_Real_AllAddr_Lds_gfx10<bits<8> op> {
1800     def _OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1801                         MUBUFLdsTable<0, NAME # "_OFFSET_gfx10">;
1802     def _OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1803                         MUBUFLdsTable<0, NAME # "_OFFEN_gfx10">;
1804     def _IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1805                         MUBUFLdsTable<0, NAME # "_IDXEN_gfx10">;
1806     def _BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1807                         MUBUFLdsTable<0, NAME # "_BOTHEN_gfx10">;
1808
1809     def _LDS_OFFSET_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1810                             MUBUFLdsTable<1, NAME # "_OFFSET_gfx10">;
1811     def _LDS_OFFEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1812                             MUBUFLdsTable<1, NAME # "_OFFEN_gfx10">;
1813     def _LDS_IDXEN_gfx10  : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1814                             MUBUFLdsTable<1, NAME # "_IDXEN_gfx10">;
1815     def _LDS_BOTHEN_gfx10 : MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1816                             MUBUFLdsTable<1, NAME # "_BOTHEN_gfx10">;
1817   }
1818   multiclass MUBUF_Real_Atomics_gfx10<bits<8> op> :
1819       MUBUF_Real_AllAddr_gfx10<op> {
1820     def _BOTHEN_RTN_gfx10 :
1821       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1822     def _IDXEN_RTN_gfx10 :
1823       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1824     def _OFFEN_RTN_gfx10 :
1825       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1826     def _OFFSET_RTN_gfx10 :
1827       MUBUF_Real_gfx10<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1828   }
1829 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
1830
1831 defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x019>;
1832 defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx10<0x01b>;
1833 defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x020>;
1834 defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x021>;
1835 defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx10<0x022>;
1836 defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x023>;
1837 defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx10<0x024>;
1838 defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx10<0x025>;
1839 // FIXME-GFX10: Add following instructions:
1840 //defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx10<0x026>;
1841 //defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx10<0x027>;
1842 defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx10<0x080>;
1843 defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx10<0x081>;
1844 defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx10<0x082>;
1845 defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx10<0x083>;
1846 defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx10<0x084>;
1847 defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx10<0x085>;
1848 defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx10<0x086>;
1849 defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx10<0x087>;
1850
1851 def BUFFER_GL0_INV_gfx10 :
1852   MUBUF_Real_gfx10<0x071, BUFFER_GL0_INV>;
1853 def BUFFER_GL1_INV_gfx10 :
1854   MUBUF_Real_gfx10<0x072, BUFFER_GL1_INV>;
1855
1856 //===----------------------------------------------------------------------===//
1857 // MUBUF - GFX6, GFX7, GFX10.
1858 //===----------------------------------------------------------------------===//
1859
1860 let AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6" in {
1861   multiclass MUBUF_Real_gfx6<bits<8> op> {
1862     def _gfx6 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1863   }
1864 } // End AssemblerPredicate = isGFX6, DecoderNamespace = "GFX6"
1865
1866 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
1867   multiclass MUBUF_Real_gfx7<bits<8> op> {
1868     def _gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME)>;
1869   }
1870 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
1871
1872 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
1873   multiclass MUBUF_Real_AllAddr_gfx6_gfx7<bits<8> op> {
1874     def _ADDR64_gfx6_gfx7 :
1875       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>;
1876     def _BOTHEN_gfx6_gfx7 :
1877       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
1878     def _IDXEN_gfx6_gfx7 :
1879       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
1880     def _OFFEN_gfx6_gfx7 :
1881       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
1882     def _OFFSET_gfx6_gfx7 :
1883       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
1884   }
1885   multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7<bits<8> op> {
1886     def _OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
1887                             MUBUFLdsTable<0, NAME # "_OFFSET_gfx6_gfx7">;
1888     def _ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64")>,
1889                             MUBUFLdsTable<0, NAME # "_ADDR64_gfx6_gfx7">;
1890     def _OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
1891                             MUBUFLdsTable<0, NAME # "_OFFEN_gfx6_gfx7">;
1892     def _IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
1893                             MUBUFLdsTable<0, NAME # "_IDXEN_gfx6_gfx7">;
1894     def _BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
1895                             MUBUFLdsTable<0, NAME # "_BOTHEN_gfx6_gfx7">;
1896
1897     def _LDS_OFFSET_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
1898                                 MUBUFLdsTable<1, NAME # "_OFFSET_gfx6_gfx7">;
1899     def _LDS_ADDR64_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_ADDR64")>,
1900                                 MUBUFLdsTable<1, NAME # "_ADDR64_gfx6_gfx7">;
1901     def _LDS_OFFEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
1902                                 MUBUFLdsTable<1, NAME # "_OFFEN_gfx6_gfx7">;
1903     def _LDS_IDXEN_gfx6_gfx7  : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
1904                                 MUBUFLdsTable<1, NAME # "_IDXEN_gfx6_gfx7">;
1905     def _LDS_BOTHEN_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
1906                                 MUBUFLdsTable<1, NAME # "_BOTHEN_gfx6_gfx7">;
1907   }
1908   multiclass MUBUF_Real_Atomics_gfx6_gfx7<bits<8> op> :
1909       MUBUF_Real_AllAddr_gfx6_gfx7<op> {
1910     def _ADDR64_RTN_gfx6_gfx7 :
1911       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_ADDR64_RTN")>;
1912     def _BOTHEN_RTN_gfx6_gfx7 :
1913       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
1914     def _IDXEN_RTN_gfx6_gfx7 :
1915       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
1916     def _OFFEN_RTN_gfx6_gfx7 :
1917       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
1918     def _OFFSET_RTN_gfx6_gfx7 :
1919       MUBUF_Real_gfx6_gfx7<op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
1920   }
1921 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
1922
1923 multiclass MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<8> op> :
1924   MUBUF_Real_AllAddr_gfx6_gfx7<op>, MUBUF_Real_AllAddr_gfx10<op>;
1925
1926 multiclass MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<bits<8> op> :
1927   MUBUF_Real_AllAddr_Lds_gfx6_gfx7<op>, MUBUF_Real_AllAddr_Lds_gfx10<op>;
1928
1929 multiclass MUBUF_Real_Atomics_gfx6_gfx7_gfx10<bits<8> op> :
1930   MUBUF_Real_Atomics_gfx6_gfx7<op>, MUBUF_Real_Atomics_gfx10<op>;
1931
1932 // FIXME-GFX6: Following instructions are available only on GFX6.
1933 //defm BUFFER_ATOMIC_RSUB         : MUBUF_Real_Atomics_gfx6 <0x034>;
1934 //defm BUFFER_ATOMIC_RSUB_X2      : MUBUF_Real_Atomics_gfx6 <0x054>;
1935
1936 defm BUFFER_LOAD_FORMAT_X     : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x000>;
1937 defm BUFFER_LOAD_FORMAT_XY    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
1938 defm BUFFER_LOAD_FORMAT_XYZ   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
1939 defm BUFFER_LOAD_FORMAT_XYZW  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
1940 defm BUFFER_STORE_FORMAT_X    : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
1941 defm BUFFER_STORE_FORMAT_XY   : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
1942 defm BUFFER_STORE_FORMAT_XYZ  : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
1943 defm BUFFER_STORE_FORMAT_XYZW : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
1944 defm BUFFER_LOAD_UBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x008>;
1945 defm BUFFER_LOAD_SBYTE        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x009>;
1946 defm BUFFER_LOAD_USHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00a>;
1947 defm BUFFER_LOAD_SSHORT       : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00b>;
1948 defm BUFFER_LOAD_DWORD        : MUBUF_Real_AllAddr_Lds_gfx6_gfx7_gfx10<0x00c>;
1949 defm BUFFER_LOAD_DWORDX2      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00d>;
1950 defm BUFFER_LOAD_DWORDX4      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00e>;
1951 defm BUFFER_LOAD_DWORDX3      : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x00f>;
1952 defm BUFFER_STORE_BYTE        : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x018>;
1953 defm BUFFER_STORE_SHORT       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01a>;
1954 defm BUFFER_STORE_DWORD       : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01c>;
1955 defm BUFFER_STORE_DWORDX2     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01d>;
1956 defm BUFFER_STORE_DWORDX4     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01e>;
1957 defm BUFFER_STORE_DWORDX3     : MUBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x01f>;
1958
1959 defm BUFFER_ATOMIC_SWAP        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x030>;
1960 defm BUFFER_ATOMIC_CMPSWAP     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x031>;
1961 defm BUFFER_ATOMIC_ADD         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x032>;
1962 defm BUFFER_ATOMIC_SUB         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x033>;
1963 defm BUFFER_ATOMIC_SMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x035>;
1964 defm BUFFER_ATOMIC_UMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x036>;
1965 defm BUFFER_ATOMIC_SMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x037>;
1966 defm BUFFER_ATOMIC_UMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x038>;
1967 defm BUFFER_ATOMIC_AND         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x039>;
1968 defm BUFFER_ATOMIC_OR          : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03a>;
1969 defm BUFFER_ATOMIC_XOR         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03b>;
1970 defm BUFFER_ATOMIC_INC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03c>;
1971 defm BUFFER_ATOMIC_DEC         : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03d>;
1972 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
1973 //defm BUFFER_ATOMIC_FCMPSWAP    : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03e>;
1974 //defm BUFFER_ATOMIC_FMIN        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x03f>;
1975 //defm BUFFER_ATOMIC_FMAX        : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x040>;
1976 defm BUFFER_ATOMIC_SWAP_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x050>;
1977 defm BUFFER_ATOMIC_CMPSWAP_X2  : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x051>;
1978 defm BUFFER_ATOMIC_ADD_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x052>;
1979 defm BUFFER_ATOMIC_SUB_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x053>;
1980 defm BUFFER_ATOMIC_SMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x055>;
1981 defm BUFFER_ATOMIC_UMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x056>;
1982 defm BUFFER_ATOMIC_SMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x057>;
1983 defm BUFFER_ATOMIC_UMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x058>;
1984 defm BUFFER_ATOMIC_AND_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x059>;
1985 defm BUFFER_ATOMIC_OR_X2       : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05a>;
1986 defm BUFFER_ATOMIC_XOR_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05b>;
1987 defm BUFFER_ATOMIC_INC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05c>;
1988 defm BUFFER_ATOMIC_DEC_X2      : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05d>;
1989 // FIXME-GFX7: Need to handle hazard for BUFFER_ATOMIC_FCMPSWAP_X2 on GFX7.
1990 // FIXME-GFX6-GFX7-GFX10: Add following instructions:
1991 //defm BUFFER_ATOMIC_FCMPSWAP_X2 : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05e>;
1992 //defm BUFFER_ATOMIC_FMIN_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x05f>;
1993 //defm BUFFER_ATOMIC_FMAX_X2     : MUBUF_Real_Atomics_gfx6_gfx7_gfx10<0x060>;
1994
1995 defm BUFFER_WBINVL1_SC        : MUBUF_Real_gfx6<0x070>;
1996 defm BUFFER_WBINVL1_VOL       : MUBUF_Real_gfx7<0x070>;
1997 def  BUFFER_WBINVL1_gfx6_gfx7 : MUBUF_Real_gfx6_gfx7<0x071, BUFFER_WBINVL1>;
1998
1999 //===----------------------------------------------------------------------===//
2000 // Base ENC_MTBUF for GFX6, GFX7, GFX10.
2001 //===----------------------------------------------------------------------===//
2002
2003 class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
2004     MTBUF_Real<ps>, Enc64, SIMCInstr<ps.PseudoInstr, ef> {
2005   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2006   let Inst{12}    = ps.offen;
2007   let Inst{13}    = ps.idxen;
2008   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2009   let Inst{18-16} = op;
2010   let Inst{31-26} = 0x3a; //encoding
2011   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2012   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2013   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2014   let Inst{54}    = !if(ps.has_slc, slc, ?);
2015   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2016   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2017 }
2018
2019 //===----------------------------------------------------------------------===//
2020 // MTBUF - GFX10.
2021 //===----------------------------------------------------------------------===//
2022
2023 class MTBUF_Real_gfx10<bits<4> op, MTBUF_Pseudo ps> :
2024     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.GFX10> {
2025   let Inst{15} = !if(ps.has_dlc, dlc, ps.dlc_value);
2026   let Inst{25-19} = format;
2027   let Inst{53} = op{3};
2028 }
2029
2030 let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in {
2031   multiclass MTBUF_Real_AllAddr_gfx10<bits<4> op> {
2032     def _BOTHEN_gfx10 :
2033       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2034     def _IDXEN_gfx10 :
2035       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2036     def _OFFEN_gfx10 :
2037       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2038     def _OFFSET_gfx10 :
2039       MTBUF_Real_gfx10<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2040   }
2041 } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10"
2042
2043 defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx10<0x008>;
2044 defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx10<0x009>;
2045 defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx10<0x00a>;
2046 defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx10<0x00b>;
2047 defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx10<0x00c>;
2048 defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx10<0x00d>;
2049 defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx10<0x00e>;
2050 defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx10<0x00f>;
2051
2052 //===----------------------------------------------------------------------===//
2053 // MTBUF - GFX6, GFX7, GFX10.
2054 //===----------------------------------------------------------------------===//
2055
2056 class MTBUF_Real_gfx6_gfx7<bits<4> op, MTBUF_Pseudo ps> :
2057     Base_MTBUF_Real_gfx6_gfx7_gfx10<op{2-0}, ps, SIEncodingFamily.SI> {
2058   let Inst{15} = ps.addr64;
2059   let Inst{22-19} = dfmt;
2060   let Inst{25-23} = nfmt;
2061 }
2062
2063 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
2064   multiclass MTBUF_Real_AllAddr_gfx6_gfx7<bits<4> op> {
2065     def _ADDR64_gfx6_gfx7 :
2066       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_ADDR64")>;
2067     def _BOTHEN_gfx6_gfx7 :
2068       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2069     def _IDXEN_gfx6_gfx7 :
2070       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2071     def _OFFEN_gfx6_gfx7 :
2072       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2073     def _OFFSET_gfx6_gfx7 :
2074       MTBUF_Real_gfx6_gfx7<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2075   }
2076 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
2077
2078 multiclass MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<bits<4> op> :
2079   MTBUF_Real_AllAddr_gfx6_gfx7<op>, MTBUF_Real_AllAddr_gfx10<op>;
2080
2081 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x000>;
2082 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x001>;
2083 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x002>;
2084 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x003>;
2085 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x004>;
2086 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x005>;
2087 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x006>;
2088 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_gfx6_gfx7_gfx10<0x007>;
2089
2090 //===----------------------------------------------------------------------===//
2091 // GFX8, GFX9 (VI).
2092 //===----------------------------------------------------------------------===//
2093
2094 class MUBUF_Real_vi <bits<7> op, MUBUF_Pseudo ps> :
2095   MUBUF_Real<ps>,
2096   Enc64,
2097   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2098   let AssemblerPredicate = isGFX8GFX9;
2099   let DecoderNamespace = "GFX8";
2100
2101   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2102   let Inst{12}    = ps.offen;
2103   let Inst{13}    = ps.idxen;
2104   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2105   let Inst{16}    = !if(ps.lds, 1, 0);
2106   let Inst{17}    = !if(ps.has_slc, slc, ?);
2107   let Inst{24-18} = op;
2108   let Inst{31-26} = 0x38; //encoding
2109   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2110   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2111   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2112   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2113   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2114 }
2115
2116 multiclass MUBUF_Real_AllAddr_vi<bits<7> op> {
2117   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2118   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2119   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2120   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2121 }
2122
2123 multiclass MUBUF_Real_AllAddr_Lds_vi<bits<7> op> {
2124
2125   def _OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>,
2126                    MUBUFLdsTable<0, NAME # "_OFFSET_vi">;
2127   def _OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>,
2128                    MUBUFLdsTable<0, NAME # "_OFFEN_vi">;
2129   def _IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>,
2130                    MUBUFLdsTable<0, NAME # "_IDXEN_vi">;
2131   def _BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>,
2132                    MUBUFLdsTable<0, NAME # "_BOTHEN_vi">;
2133
2134   def _LDS_OFFSET_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFSET")>,
2135                        MUBUFLdsTable<1, NAME # "_OFFSET_vi">;
2136   def _LDS_OFFEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_OFFEN")>,
2137                        MUBUFLdsTable<1, NAME # "_OFFEN_vi">;
2138   def _LDS_IDXEN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_IDXEN")>,
2139                        MUBUFLdsTable<1, NAME # "_IDXEN_vi">;
2140   def _LDS_BOTHEN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_LDS_BOTHEN")>,
2141                        MUBUFLdsTable<1, NAME # "_BOTHEN_vi">;
2142 }
2143
2144 class MUBUF_Real_gfx80 <bits<7> op, MUBUF_Pseudo ps> :
2145   MUBUF_Real<ps>,
2146   Enc64,
2147   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2148   let AssemblerPredicate=HasUnpackedD16VMem;
2149   let DecoderNamespace="GFX80_UNPACKED";
2150
2151   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2152   let Inst{12}    = ps.offen;
2153   let Inst{13}    = ps.idxen;
2154   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2155   let Inst{16}    = !if(ps.lds, 1, 0);
2156   let Inst{17}    = !if(ps.has_slc, slc, ?);
2157   let Inst{24-18} = op;
2158   let Inst{31-26} = 0x38; //encoding
2159   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2160   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2161   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2162   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2163   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2164 }
2165
2166 multiclass MUBUF_Real_AllAddr_gfx80<bits<7> op> {
2167   def _OFFSET_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET")>;
2168   def _OFFEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN")>;
2169   def _IDXEN_gfx80  : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN")>;
2170   def _BOTHEN_gfx80 : MUBUF_Real_gfx80 <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN")>;
2171 }
2172
2173 multiclass MUBUF_Real_Atomic_vi<bits<7> op> :
2174   MUBUF_Real_AllAddr_vi<op> {
2175   def _OFFSET_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFSET_RTN")>;
2176   def _OFFEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_OFFEN_RTN")>;
2177   def _IDXEN_RTN_vi  : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_IDXEN_RTN")>;
2178   def _BOTHEN_RTN_vi : MUBUF_Real_vi <op, !cast<MUBUF_Pseudo>(NAME#"_BOTHEN_RTN")>;
2179 }
2180
2181 defm BUFFER_LOAD_FORMAT_X       : MUBUF_Real_AllAddr_Lds_vi <0x00>;
2182 defm BUFFER_LOAD_FORMAT_XY      : MUBUF_Real_AllAddr_vi <0x01>;
2183 defm BUFFER_LOAD_FORMAT_XYZ     : MUBUF_Real_AllAddr_vi <0x02>;
2184 defm BUFFER_LOAD_FORMAT_XYZW    : MUBUF_Real_AllAddr_vi <0x03>;
2185 defm BUFFER_STORE_FORMAT_X      : MUBUF_Real_AllAddr_vi <0x04>;
2186 defm BUFFER_STORE_FORMAT_XY     : MUBUF_Real_AllAddr_vi <0x05>;
2187 defm BUFFER_STORE_FORMAT_XYZ    : MUBUF_Real_AllAddr_vi <0x06>;
2188 defm BUFFER_STORE_FORMAT_XYZW   : MUBUF_Real_AllAddr_vi <0x07>;
2189 let SubtargetPredicate = HasUnpackedD16VMem in {
2190   defm BUFFER_LOAD_FORMAT_D16_X_gfx80       : MUBUF_Real_AllAddr_gfx80 <0x08>;
2191   defm BUFFER_LOAD_FORMAT_D16_XY_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x09>;
2192   defm BUFFER_LOAD_FORMAT_D16_XYZ_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0a>;
2193   defm BUFFER_LOAD_FORMAT_D16_XYZW_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0b>;
2194   defm BUFFER_STORE_FORMAT_D16_X_gfx80      : MUBUF_Real_AllAddr_gfx80 <0x0c>;
2195   defm BUFFER_STORE_FORMAT_D16_XY_gfx80     : MUBUF_Real_AllAddr_gfx80 <0x0d>;
2196   defm BUFFER_STORE_FORMAT_D16_XYZ_gfx80    : MUBUF_Real_AllAddr_gfx80 <0x0e>;
2197   defm BUFFER_STORE_FORMAT_D16_XYZW_gfx80   : MUBUF_Real_AllAddr_gfx80 <0x0f>;
2198 } // End HasUnpackedD16VMem.
2199 let SubtargetPredicate = HasPackedD16VMem in {
2200   defm BUFFER_LOAD_FORMAT_D16_X       : MUBUF_Real_AllAddr_vi <0x08>;
2201   defm BUFFER_LOAD_FORMAT_D16_XY      : MUBUF_Real_AllAddr_vi <0x09>;
2202   defm BUFFER_LOAD_FORMAT_D16_XYZ     : MUBUF_Real_AllAddr_vi <0x0a>;
2203   defm BUFFER_LOAD_FORMAT_D16_XYZW    : MUBUF_Real_AllAddr_vi <0x0b>;
2204   defm BUFFER_STORE_FORMAT_D16_X      : MUBUF_Real_AllAddr_vi <0x0c>;
2205   defm BUFFER_STORE_FORMAT_D16_XY     : MUBUF_Real_AllAddr_vi <0x0d>;
2206   defm BUFFER_STORE_FORMAT_D16_XYZ    : MUBUF_Real_AllAddr_vi <0x0e>;
2207   defm BUFFER_STORE_FORMAT_D16_XYZW   : MUBUF_Real_AllAddr_vi <0x0f>;
2208 } // End HasPackedD16VMem.
2209 defm BUFFER_LOAD_UBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x10>;
2210 defm BUFFER_LOAD_SBYTE          : MUBUF_Real_AllAddr_Lds_vi <0x11>;
2211 defm BUFFER_LOAD_USHORT         : MUBUF_Real_AllAddr_Lds_vi <0x12>;
2212 defm BUFFER_LOAD_SSHORT         : MUBUF_Real_AllAddr_Lds_vi <0x13>;
2213 defm BUFFER_LOAD_DWORD          : MUBUF_Real_AllAddr_Lds_vi <0x14>;
2214 defm BUFFER_LOAD_DWORDX2        : MUBUF_Real_AllAddr_Lds_vi <0x15>;
2215 defm BUFFER_LOAD_DWORDX3        : MUBUF_Real_AllAddr_Lds_vi <0x16>;
2216 defm BUFFER_LOAD_DWORDX4        : MUBUF_Real_AllAddr_Lds_vi <0x17>;
2217 defm BUFFER_STORE_BYTE          : MUBUF_Real_AllAddr_vi <0x18>;
2218 defm BUFFER_STORE_BYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x19>;
2219 defm BUFFER_STORE_SHORT         : MUBUF_Real_AllAddr_vi <0x1a>;
2220 defm BUFFER_STORE_SHORT_D16_HI  : MUBUF_Real_AllAddr_vi <0x1b>;
2221 defm BUFFER_STORE_DWORD         : MUBUF_Real_AllAddr_vi <0x1c>;
2222 defm BUFFER_STORE_DWORDX2       : MUBUF_Real_AllAddr_vi <0x1d>;
2223 defm BUFFER_STORE_DWORDX3       : MUBUF_Real_AllAddr_vi <0x1e>;
2224 defm BUFFER_STORE_DWORDX4       : MUBUF_Real_AllAddr_vi <0x1f>;
2225
2226 defm BUFFER_LOAD_UBYTE_D16      : MUBUF_Real_AllAddr_vi <0x20>;
2227 defm BUFFER_LOAD_UBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x21>;
2228 defm BUFFER_LOAD_SBYTE_D16      : MUBUF_Real_AllAddr_vi <0x22>;
2229 defm BUFFER_LOAD_SBYTE_D16_HI   : MUBUF_Real_AllAddr_vi <0x23>;
2230 defm BUFFER_LOAD_SHORT_D16      : MUBUF_Real_AllAddr_vi <0x24>;
2231 defm BUFFER_LOAD_SHORT_D16_HI   : MUBUF_Real_AllAddr_vi <0x25>;
2232
2233 defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_vi <0x26>;
2234 defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_vi <0x27>;
2235
2236 defm BUFFER_ATOMIC_SWAP         : MUBUF_Real_Atomic_vi <0x40>;
2237 defm BUFFER_ATOMIC_CMPSWAP      : MUBUF_Real_Atomic_vi <0x41>;
2238 defm BUFFER_ATOMIC_ADD          : MUBUF_Real_Atomic_vi <0x42>;
2239 defm BUFFER_ATOMIC_SUB          : MUBUF_Real_Atomic_vi <0x43>;
2240 defm BUFFER_ATOMIC_SMIN         : MUBUF_Real_Atomic_vi <0x44>;
2241 defm BUFFER_ATOMIC_UMIN         : MUBUF_Real_Atomic_vi <0x45>;
2242 defm BUFFER_ATOMIC_SMAX         : MUBUF_Real_Atomic_vi <0x46>;
2243 defm BUFFER_ATOMIC_UMAX         : MUBUF_Real_Atomic_vi <0x47>;
2244 defm BUFFER_ATOMIC_AND          : MUBUF_Real_Atomic_vi <0x48>;
2245 defm BUFFER_ATOMIC_OR           : MUBUF_Real_Atomic_vi <0x49>;
2246 defm BUFFER_ATOMIC_XOR          : MUBUF_Real_Atomic_vi <0x4a>;
2247 defm BUFFER_ATOMIC_INC          : MUBUF_Real_Atomic_vi <0x4b>;
2248 defm BUFFER_ATOMIC_DEC          : MUBUF_Real_Atomic_vi <0x4c>;
2249
2250 defm BUFFER_ATOMIC_SWAP_X2      : MUBUF_Real_Atomic_vi <0x60>;
2251 defm BUFFER_ATOMIC_CMPSWAP_X2   : MUBUF_Real_Atomic_vi <0x61>;
2252 defm BUFFER_ATOMIC_ADD_X2       : MUBUF_Real_Atomic_vi <0x62>;
2253 defm BUFFER_ATOMIC_SUB_X2       : MUBUF_Real_Atomic_vi <0x63>;
2254 defm BUFFER_ATOMIC_SMIN_X2      : MUBUF_Real_Atomic_vi <0x64>;
2255 defm BUFFER_ATOMIC_UMIN_X2      : MUBUF_Real_Atomic_vi <0x65>;
2256 defm BUFFER_ATOMIC_SMAX_X2      : MUBUF_Real_Atomic_vi <0x66>;
2257 defm BUFFER_ATOMIC_UMAX_X2      : MUBUF_Real_Atomic_vi <0x67>;
2258 defm BUFFER_ATOMIC_AND_X2       : MUBUF_Real_Atomic_vi <0x68>;
2259 defm BUFFER_ATOMIC_OR_X2        : MUBUF_Real_Atomic_vi <0x69>;
2260 defm BUFFER_ATOMIC_XOR_X2       : MUBUF_Real_Atomic_vi <0x6a>;
2261 defm BUFFER_ATOMIC_INC_X2       : MUBUF_Real_Atomic_vi <0x6b>;
2262 defm BUFFER_ATOMIC_DEC_X2       : MUBUF_Real_Atomic_vi <0x6c>;
2263
2264 def BUFFER_STORE_LDS_DWORD_vi   : MUBUF_Real_vi <0x3d, BUFFER_STORE_LDS_DWORD>;
2265
2266 def BUFFER_WBINVL1_vi           : MUBUF_Real_vi <0x3e, BUFFER_WBINVL1>;
2267 def BUFFER_WBINVL1_VOL_vi       : MUBUF_Real_vi <0x3f, BUFFER_WBINVL1_VOL>;
2268
2269 let SubtargetPredicate = HasAtomicFaddInsts in {
2270
2271 defm BUFFER_ATOMIC_ADD_F32    : MUBUF_Real_AllAddr_vi <0x4d>;
2272 defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Real_AllAddr_vi <0x4e>;
2273
2274 } // End SubtargetPredicate = HasAtomicFaddInsts
2275
2276 class MTBUF_Real_vi <bits<4> op, MTBUF_Pseudo ps> :
2277   MTBUF_Real<ps>,
2278   Enc64,
2279   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI> {
2280   let AssemblerPredicate = isGFX8GFX9;
2281   let DecoderNamespace = "GFX8";
2282
2283   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2284   let Inst{12}    = ps.offen;
2285   let Inst{13}    = ps.idxen;
2286   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2287   let Inst{18-15} = op;
2288   let Inst{22-19} = dfmt;
2289   let Inst{25-23} = nfmt;
2290   let Inst{31-26} = 0x3a; //encoding
2291   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2292   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2293   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2294   let Inst{54}    = !if(ps.has_slc, slc, ?);
2295   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2296   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2297 }
2298
2299 multiclass MTBUF_Real_AllAddr_vi<bits<4> op> {
2300   def _OFFSET_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2301   def _OFFEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2302   def _IDXEN_vi  : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2303   def _BOTHEN_vi : MTBUF_Real_vi <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2304 }
2305
2306 class MTBUF_Real_gfx80 <bits<4> op, MTBUF_Pseudo ps> :
2307   MTBUF_Real<ps>,
2308   Enc64,
2309   SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX80> {
2310   let AssemblerPredicate=HasUnpackedD16VMem;
2311   let DecoderNamespace="GFX80_UNPACKED";
2312
2313   let Inst{11-0}  = !if(ps.has_offset, offset, ?);
2314   let Inst{12}    = ps.offen;
2315   let Inst{13}    = ps.idxen;
2316   let Inst{14}    = !if(ps.has_glc, glc, ps.glc_value);
2317   let Inst{18-15} = op;
2318   let Inst{22-19} = dfmt;
2319   let Inst{25-23} = nfmt;
2320   let Inst{31-26} = 0x3a; //encoding
2321   let Inst{39-32} = !if(ps.has_vaddr, vaddr, ?);
2322   let Inst{47-40} = !if(ps.has_vdata, vdata, ?);
2323   let Inst{52-48} = !if(ps.has_srsrc, srsrc{6-2}, ?);
2324   let Inst{54}    = !if(ps.has_slc, slc, ?);
2325   let Inst{55}    = !if(ps.has_tfe, tfe, ?);
2326   let Inst{63-56} = !if(ps.has_soffset, soffset, ?);
2327 }
2328
2329 multiclass MTBUF_Real_AllAddr_gfx80<bits<4> op> {
2330   def _OFFSET_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET")>;
2331   def _OFFEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_OFFEN")>;
2332   def _IDXEN_gfx80  : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_IDXEN")>;
2333   def _BOTHEN_gfx80 : MTBUF_Real_gfx80 <op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
2334 }
2335
2336 defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Real_AllAddr_vi <0x00>;
2337 defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Real_AllAddr_vi <0x01>;
2338 defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Real_AllAddr_vi <0x02>;
2339 defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Real_AllAddr_vi <0x03>;
2340 defm TBUFFER_STORE_FORMAT_X    : MTBUF_Real_AllAddr_vi <0x04>;
2341 defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Real_AllAddr_vi <0x05>;
2342 defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Real_AllAddr_vi <0x06>;
2343 defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Real_AllAddr_vi <0x07>;
2344 let SubtargetPredicate = HasUnpackedD16VMem in {
2345   defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Real_AllAddr_gfx80 <0x08>;
2346   defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x09>;
2347   defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0a>;
2348   defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0b>;
2349   defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Real_AllAddr_gfx80 <0x0c>;
2350   defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Real_AllAddr_gfx80 <0x0d>;
2351   defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Real_AllAddr_gfx80 <0x0e>;
2352   defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Real_AllAddr_gfx80 <0x0f>;
2353 } // End HasUnpackedD16VMem.
2354 let SubtargetPredicate = HasPackedD16VMem in {
2355   defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_vi <0x08>;
2356   defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_vi <0x09>;
2357   defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_vi <0x0a>;
2358   defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_vi <0x0b>;
2359   defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_vi <0x0c>;
2360   defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_vi <0x0d>;
2361   defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_vi <0x0e>;
2362   defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_vi <0x0f>;
2363 } // End HasUnpackedD16VMem.
2364
2365 def MUBUFInfoTable : GenericTable {
2366   let FilterClass = "MUBUF_Pseudo";
2367   let CppTypeName = "MUBUFInfo";
2368   let Fields = ["Opcode", "BaseOpcode", "dwords", "has_vaddr", "has_srsrc", "has_soffset"];
2369
2370   let PrimaryKey = ["Opcode"];
2371   let PrimaryKeyName = "getMUBUFOpcodeHelper";
2372 }
2373
2374 def getMUBUFInfoFromOpcode : SearchIndex {
2375   let Table = MUBUFInfoTable;
2376   let Key = ["Opcode"];
2377 }
2378
2379 def getMUBUFInfoFromBaseOpcodeAndDwords : SearchIndex {
2380   let Table = MUBUFInfoTable;
2381   let Key = ["BaseOpcode", "dwords"];
2382 }