lib/Target/AMDGPU/SMInstructions.td

   1 //===---- SMInstructions.td - Scalar Memory Instruction Defintions --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 def smrd_offset_8 : NamedOperandU32<"SMRDOffset8",
  10                                   NamedMatchClass<"SMRDOffset8">> {
  11   let OperandType = "OPERAND_IMMEDIATE";
  12 }
  13
  14 def smrd_offset_20 : NamedOperandU32<"SMRDOffset20",
  15                                   NamedMatchClass<"SMRDOffset20">> {
  16   let OperandType = "OPERAND_IMMEDIATE";
  17 }
  18
  19 //===----------------------------------------------------------------------===//
  20 // Scalar Memory classes
  21 //===----------------------------------------------------------------------===//
  22
  23 class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]> :
  24   InstSI <outs, ins, "", pattern>,
  25   SIMCInstr<opName, SIEncodingFamily.NONE> {
  26   let isPseudo = 1;
  27   let isCodeGenOnly = 1;
  28
  29   let LGKM_CNT = 1;
  30   let SMRD = 1;
  31   let mayStore = 0;
  32   let mayLoad = 1;
  33   let hasSideEffects = 0;
  34   let UseNamedOperandTable = 1;
  35   let SchedRW = [WriteSMEM];
  36
  37   string Mnemonic = opName;
  38   string AsmOperands = asmOps;
  39
  40   bits<1> has_sbase = 1;
  41   bits<1> has_sdst = 1;
  42   bit has_glc = 0;
  43   bits<1> has_offset = 1;
  44   bits<1> offset_is_imm = 0;
  45 }
  46
  47 class SM_Real <SM_Pseudo ps>
  48   : InstSI<ps.OutOperandList, ps.InOperandList, ps.Mnemonic # ps.AsmOperands, []> {
  49
  50   let isPseudo = 0;
  51   let isCodeGenOnly = 0;
  52
  53   // copy relevant pseudo op flags
  54   let SubtargetPredicate = ps.SubtargetPredicate;
  55   let AsmMatchConverter  = ps.AsmMatchConverter;
  56
  57   // encoding
  58   bits<7>  sbase;
  59   bits<7>  sdst;
  60   bits<32> offset;
  61   bits<1> imm = !if(ps.has_offset, ps.offset_is_imm, 0);
  62 }
  63
  64 class SM_Probe_Pseudo <string opName, dag ins, bit isImm>
  65   : SM_Pseudo<opName, (outs), ins, " $sdata, $sbase, $offset"> {
  66   let mayLoad = 0;
  67   let mayStore = 0;
  68   let has_glc = 0;
  69   let LGKM_CNT = 0;
  70   let ScalarStore = 0;
  71   let hasSideEffects = 1;
  72   let offset_is_imm = isImm;
  73   let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR");
  74 }
  75
  76 class SM_Load_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> pattern=[]>
  77   : SM_Pseudo<opName, outs, ins, asmOps, pattern> {
  78   RegisterClass BaseClass;
  79   let mayLoad = 1;
  80   let mayStore = 0;
  81   let has_glc = 1;
  82 }
  83
  84 class SM_Store_Pseudo <string opName, dag ins, string asmOps, list<dag> pattern = []>
  85   : SM_Pseudo<opName, (outs), ins, asmOps, pattern> {
  86   RegisterClass BaseClass;
  87   RegisterClass SrcClass;
  88   let mayLoad = 0;
  89   let mayStore = 1;
  90   let has_glc = 1;
  91   let ScalarStore = 1;
  92 }
  93
  94 class SM_Discard_Pseudo <string opName, dag ins, bit isImm>
  95   : SM_Pseudo<opName, (outs), ins, " $sbase, $offset"> {
  96   let mayLoad = 0;
  97   let mayStore = 0;
  98   let has_glc = 0;
  99   let has_sdst = 0;
 100   let ScalarStore = 0;
 101   let hasSideEffects = 1;
 102   let offset_is_imm = isImm;
 103   let PseudoInstr = opName # !if(isImm, "_IMM", "_SGPR");
 104 }
 105
 106 multiclass SM_Pseudo_Loads<string opName,
 107                            RegisterClass baseClass,
 108                            RegisterClass dstClass> {
 109   def _IMM  : SM_Load_Pseudo <opName,
 110                               (outs dstClass:$sdst),
 111                               (ins baseClass:$sbase, i32imm:$offset, i1imm:$glc),
 112                               " $sdst, $sbase, $offset$glc", []> {
 113     let offset_is_imm = 1;
 114     let BaseClass = baseClass;
 115     let PseudoInstr = opName # "_IMM";
 116     let has_glc = 1;
 117   }
 118
 119   def _SGPR  : SM_Load_Pseudo <opName,
 120                               (outs dstClass:$sdst),
 121                               (ins baseClass:$sbase, SReg_32:$soff, i1imm:$glc),
 122                               " $sdst, $sbase, $offset$glc", []> {
 123     let BaseClass = baseClass;
 124     let PseudoInstr = opName # "_SGPR";
 125     let has_glc = 1;
 126   }
 127 }
 128
 129 multiclass SM_Pseudo_Stores<string opName,
 130                            RegisterClass baseClass,
 131                            RegisterClass srcClass> {
 132   def _IMM  : SM_Store_Pseudo <opName,
 133     (ins srcClass:$sdata, baseClass:$sbase, i32imm:$offset, i1imm:$glc),
 134     " $sdata, $sbase, $offset$glc", []> {
 135     let offset_is_imm = 1;
 136     let BaseClass = baseClass;
 137     let SrcClass = srcClass;
 138     let PseudoInstr = opName # "_IMM";
 139   }
 140
 141   def _SGPR  : SM_Store_Pseudo <opName,
 142     (ins srcClass:$sdata, baseClass:$sbase, SReg_32:$soff, i1imm:$glc),
 143     " $sdata, $sbase, $offset$glc", []> {
 144     let BaseClass = baseClass;
 145     let SrcClass = srcClass;
 146     let PseudoInstr = opName # "_SGPR";
 147   }
 148 }
 149
 150 multiclass SM_Pseudo_Discards<string opName> {
 151   def _IMM  : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, smrd_offset_20:$offset), 1>;
 152   def _SGPR : SM_Discard_Pseudo <opName, (ins SReg_64:$sbase, SReg_32:$offset), 0>;
 153 }
 154
 155 class SM_Time_Pseudo<string opName, SDPatternOperator node> : SM_Pseudo<
 156   opName, (outs SReg_64_XEXEC:$sdst), (ins),
 157   " $sdst", [(set i64:$sdst, (node))]> {
 158   let hasSideEffects = 1;
 159   let mayStore = 0;
 160   let mayLoad = 1;
 161   let has_sbase = 0;
 162   let has_offset = 0;
 163 }
 164
 165 class SM_Inval_Pseudo <string opName, SDPatternOperator node> : SM_Pseudo<
 166   opName, (outs), (ins), "", [(node)]> {
 167   let hasSideEffects = 1;
 168   let mayStore = 1;
 169   let has_sdst = 0;
 170   let has_sbase = 0;
 171   let has_offset = 0;
 172 }
 173
 174 multiclass SM_Pseudo_Probe<string opName, RegisterClass baseClass> {
 175   def _IMM  : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, smrd_offset_20:$offset), 1>;
 176   def _SGPR : SM_Probe_Pseudo <opName, (ins i8imm:$sdata, baseClass:$sbase, SReg_32:$offset), 0>;
 177 }
 178
 179 //===----------------------------------------------------------------------===//
 180 // Scalar Atomic Memory Classes
 181 //===----------------------------------------------------------------------===//
 182
 183 class SM_Atomic_Pseudo <string opName,
 184                         dag outs, dag ins, string asmOps, bit isRet>
 185   : SM_Pseudo<opName, outs, ins, asmOps, []> {
 186
 187   bit glc = isRet;
 188
 189   let mayLoad = 1;
 190   let mayStore = 1;
 191   let has_glc = 1;
 192
 193   // Should these be set?
 194   let ScalarStore = 1;
 195   let hasSideEffects = 1;
 196   let maybeAtomic = 1;
 197 }
 198
 199 class SM_Pseudo_Atomic<string opName,
 200                        RegisterClass baseClass,
 201                        RegisterClass dataClass,
 202                        bit isImm,
 203                        bit isRet> :
 204   SM_Atomic_Pseudo<opName,
 205                    !if(isRet, (outs dataClass:$sdst), (outs)),
 206                    !if(isImm,
 207                        (ins dataClass:$sdata, baseClass:$sbase, smrd_offset_20:$offset),
 208                        (ins dataClass:$sdata, baseClass:$sbase, SReg_32:$offset)),
 209                    !if(isRet, " $sdst", " $sdata") # ", $sbase, $offset" # !if(isRet, " glc", ""),
 210                    isRet> {
 211   let offset_is_imm = isImm;
 212   let PseudoInstr = opName # !if(isImm,
 213                                  !if(isRet, "_IMM_RTN", "_IMM"),
 214                                  !if(isRet, "_SGPR_RTN", "_SGPR"));
 215
 216   let Constraints = !if(isRet, "$sdst = $sdata", "");
 217   let DisableEncoding = !if(isRet, "$sdata", "");
 218 }
 219
 220 multiclass SM_Pseudo_Atomics<string opName,
 221                              RegisterClass baseClass,
 222                              RegisterClass dataClass> {
 223   def _IMM      : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 0>;
 224   def _SGPR     : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 0>;
 225   def _IMM_RTN  : SM_Pseudo_Atomic <opName, baseClass, dataClass, 1, 1>;
 226   def _SGPR_RTN : SM_Pseudo_Atomic <opName, baseClass, dataClass, 0, 1>;
 227 }
 228
 229 //===----------------------------------------------------------------------===//
 230 // Scalar Memory Instructions
 231 //===----------------------------------------------------------------------===//
 232
 233 // We are using the SReg_32_XM0 and not the SReg_32 register class for 32-bit
 234 // SMRD instructions, because the SReg_32_XM0 register class does not include M0
 235 // and writing to M0 from an SMRD instruction will hang the GPU.
 236
 237 // XXX - SMEM instructions do not allow exec for data operand, but
 238 // does sdst for SMRD on SI/CI?
 239 defm S_LOAD_DWORD    : SM_Pseudo_Loads <"s_load_dword", SReg_64, SReg_32_XM0_XEXEC>;
 240 defm S_LOAD_DWORDX2  : SM_Pseudo_Loads <"s_load_dwordx2", SReg_64, SReg_64_XEXEC>;
 241 defm S_LOAD_DWORDX4  : SM_Pseudo_Loads <"s_load_dwordx4", SReg_64, SReg_128>;
 242 defm S_LOAD_DWORDX8  : SM_Pseudo_Loads <"s_load_dwordx8", SReg_64, SReg_256>;
 243 defm S_LOAD_DWORDX16 : SM_Pseudo_Loads <"s_load_dwordx16", SReg_64, SReg_512>;
 244
 245 defm S_BUFFER_LOAD_DWORD : SM_Pseudo_Loads <
 246   "s_buffer_load_dword", SReg_128, SReg_32_XM0_XEXEC
 247 >;
 248
 249 // FIXME: exec_lo/exec_hi appear to be allowed for SMRD loads on
 250 // SI/CI, bit disallowed for SMEM on VI.
 251 defm S_BUFFER_LOAD_DWORDX2 : SM_Pseudo_Loads <
 252   "s_buffer_load_dwordx2", SReg_128, SReg_64_XEXEC
 253 >;
 254
 255 defm S_BUFFER_LOAD_DWORDX4 : SM_Pseudo_Loads <
 256   "s_buffer_load_dwordx4", SReg_128, SReg_128
 257 >;
 258
 259 defm S_BUFFER_LOAD_DWORDX8 : SM_Pseudo_Loads <
 260   "s_buffer_load_dwordx8", SReg_128, SReg_256
 261 >;
 262
 263 defm S_BUFFER_LOAD_DWORDX16 : SM_Pseudo_Loads <
 264   "s_buffer_load_dwordx16", SReg_128, SReg_512
 265 >;
 266
 267 defm S_STORE_DWORD : SM_Pseudo_Stores <"s_store_dword", SReg_64, SReg_32_XM0_XEXEC>;
 268 defm S_STORE_DWORDX2 : SM_Pseudo_Stores <"s_store_dwordx2", SReg_64, SReg_64_XEXEC>;
 269 defm S_STORE_DWORDX4 : SM_Pseudo_Stores <"s_store_dwordx4", SReg_64, SReg_128>;
 270
 271 defm S_BUFFER_STORE_DWORD : SM_Pseudo_Stores <
 272   "s_buffer_store_dword", SReg_128, SReg_32_XM0_XEXEC
 273 >;
 274
 275 defm S_BUFFER_STORE_DWORDX2 : SM_Pseudo_Stores <
 276   "s_buffer_store_dwordx2", SReg_128, SReg_64_XEXEC
 277 >;
 278
 279 defm S_BUFFER_STORE_DWORDX4 : SM_Pseudo_Stores <
 280   "s_buffer_store_dwordx4", SReg_128, SReg_128
 281 >;
 282
 283
 284 def S_MEMTIME : SM_Time_Pseudo <"s_memtime", int_amdgcn_s_memtime>;
 285 def S_DCACHE_INV : SM_Inval_Pseudo <"s_dcache_inv", int_amdgcn_s_dcache_inv>;
 286
 287 let SubtargetPredicate = isCIVI in {
 288 def S_DCACHE_INV_VOL : SM_Inval_Pseudo <"s_dcache_inv_vol", int_amdgcn_s_dcache_inv_vol>;
 289 } // let SubtargetPredicate = isCIVI
 290
 291 let SubtargetPredicate = isVI in {
 292 def S_DCACHE_WB     : SM_Inval_Pseudo <"s_dcache_wb", int_amdgcn_s_dcache_wb>;
 293 def S_DCACHE_WB_VOL : SM_Inval_Pseudo <"s_dcache_wb_vol", int_amdgcn_s_dcache_wb_vol>;
 294 def S_MEMREALTIME   : SM_Time_Pseudo <"s_memrealtime", int_amdgcn_s_memrealtime>;
 295
 296 defm S_ATC_PROBE        : SM_Pseudo_Probe <"s_atc_probe", SReg_64>;
 297 defm S_ATC_PROBE_BUFFER : SM_Pseudo_Probe <"s_atc_probe_buffer", SReg_128>;
 298 } // SubtargetPredicate = isVI
 299
 300 let SubtargetPredicate = HasFlatScratchInsts, Uses = [FLAT_SCR] in {
 301 defm S_SCRATCH_LOAD_DWORD    : SM_Pseudo_Loads <"s_scratch_load_dword",   SReg_64, SReg_32_XM0_XEXEC>;
 302 defm S_SCRATCH_LOAD_DWORDX2  : SM_Pseudo_Loads <"s_scratch_load_dwordx2", SReg_64, SReg_64_XEXEC>;
 303 defm S_SCRATCH_LOAD_DWORDX4  : SM_Pseudo_Loads <"s_scratch_load_dwordx4", SReg_64, SReg_128>;
 304
 305 defm S_SCRATCH_STORE_DWORD   : SM_Pseudo_Stores <"s_scratch_store_dword",   SReg_64, SReg_32_XM0_XEXEC>;
 306 defm S_SCRATCH_STORE_DWORDX2 : SM_Pseudo_Stores <"s_scratch_store_dwordx2", SReg_64, SReg_64_XEXEC>;
 307 defm S_SCRATCH_STORE_DWORDX4 : SM_Pseudo_Stores <"s_scratch_store_dwordx4", SReg_64, SReg_128>;
 308 } // SubtargetPredicate = HasFlatScratchInsts
 309
 310 let SubtargetPredicate = HasScalarAtomics in {
 311
 312 defm S_BUFFER_ATOMIC_SWAP         : SM_Pseudo_Atomics <"s_buffer_atomic_swap", SReg_128, SReg_32_XM0_XEXEC>;
 313 defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap", SReg_128, SReg_64_XEXEC>;
 314 defm S_BUFFER_ATOMIC_ADD          : SM_Pseudo_Atomics <"s_buffer_atomic_add", SReg_128, SReg_32_XM0_XEXEC>;
 315 defm S_BUFFER_ATOMIC_SUB          : SM_Pseudo_Atomics <"s_buffer_atomic_sub", SReg_128, SReg_32_XM0_XEXEC>;
 316 defm S_BUFFER_ATOMIC_SMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_smin", SReg_128, SReg_32_XM0_XEXEC>;
 317 defm S_BUFFER_ATOMIC_UMIN         : SM_Pseudo_Atomics <"s_buffer_atomic_umin", SReg_128, SReg_32_XM0_XEXEC>;
 318 defm S_BUFFER_ATOMIC_SMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_smax", SReg_128, SReg_32_XM0_XEXEC>;
 319 defm S_BUFFER_ATOMIC_UMAX         : SM_Pseudo_Atomics <"s_buffer_atomic_umax", SReg_128, SReg_32_XM0_XEXEC>;
 320 defm S_BUFFER_ATOMIC_AND          : SM_Pseudo_Atomics <"s_buffer_atomic_and", SReg_128, SReg_32_XM0_XEXEC>;
 321 defm S_BUFFER_ATOMIC_OR           : SM_Pseudo_Atomics <"s_buffer_atomic_or", SReg_128, SReg_32_XM0_XEXEC>;
 322 defm S_BUFFER_ATOMIC_XOR          : SM_Pseudo_Atomics <"s_buffer_atomic_xor", SReg_128, SReg_32_XM0_XEXEC>;
 323 defm S_BUFFER_ATOMIC_INC          : SM_Pseudo_Atomics <"s_buffer_atomic_inc", SReg_128, SReg_32_XM0_XEXEC>;
 324 defm S_BUFFER_ATOMIC_DEC          : SM_Pseudo_Atomics <"s_buffer_atomic_dec", SReg_128, SReg_32_XM0_XEXEC>;
 325
 326 defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_swap_x2", SReg_128, SReg_64_XEXEC>;
 327 defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Pseudo_Atomics <"s_buffer_atomic_cmpswap_x2", SReg_128, SReg_128>;
 328 defm S_BUFFER_ATOMIC_ADD_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_add_x2", SReg_128, SReg_64_XEXEC>;
 329 defm S_BUFFER_ATOMIC_SUB_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_sub_x2", SReg_128, SReg_64_XEXEC>;
 330 defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_smin_x2", SReg_128, SReg_64_XEXEC>;
 331 defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umin_x2", SReg_128, SReg_64_XEXEC>;
 332 defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_smax_x2", SReg_128, SReg_64_XEXEC>;
 333 defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Pseudo_Atomics <"s_buffer_atomic_umax_x2", SReg_128, SReg_64_XEXEC>;
 334 defm S_BUFFER_ATOMIC_AND_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_and_x2", SReg_128, SReg_64_XEXEC>;
 335 defm S_BUFFER_ATOMIC_OR_X2        : SM_Pseudo_Atomics <"s_buffer_atomic_or_x2", SReg_128, SReg_64_XEXEC>;
 336 defm S_BUFFER_ATOMIC_XOR_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_xor_x2", SReg_128, SReg_64_XEXEC>;
 337 defm S_BUFFER_ATOMIC_INC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_inc_x2", SReg_128, SReg_64_XEXEC>;
 338 defm S_BUFFER_ATOMIC_DEC_X2       : SM_Pseudo_Atomics <"s_buffer_atomic_dec_x2", SReg_128, SReg_64_XEXEC>;
 339
 340 defm S_ATOMIC_SWAP                : SM_Pseudo_Atomics <"s_atomic_swap", SReg_64, SReg_32_XM0_XEXEC>;
 341 defm S_ATOMIC_CMPSWAP             : SM_Pseudo_Atomics <"s_atomic_cmpswap", SReg_64, SReg_64_XEXEC>;
 342 defm S_ATOMIC_ADD                 : SM_Pseudo_Atomics <"s_atomic_add", SReg_64, SReg_32_XM0_XEXEC>;
 343 defm S_ATOMIC_SUB                 : SM_Pseudo_Atomics <"s_atomic_sub", SReg_64, SReg_32_XM0_XEXEC>;
 344 defm S_ATOMIC_SMIN                : SM_Pseudo_Atomics <"s_atomic_smin", SReg_64, SReg_32_XM0_XEXEC>;
 345 defm S_ATOMIC_UMIN                : SM_Pseudo_Atomics <"s_atomic_umin", SReg_64, SReg_32_XM0_XEXEC>;
 346 defm S_ATOMIC_SMAX                : SM_Pseudo_Atomics <"s_atomic_smax", SReg_64, SReg_32_XM0_XEXEC>;
 347 defm S_ATOMIC_UMAX                : SM_Pseudo_Atomics <"s_atomic_umax", SReg_64, SReg_32_XM0_XEXEC>;
 348 defm S_ATOMIC_AND                 : SM_Pseudo_Atomics <"s_atomic_and", SReg_64, SReg_32_XM0_XEXEC>;
 349 defm S_ATOMIC_OR                  : SM_Pseudo_Atomics <"s_atomic_or", SReg_64, SReg_32_XM0_XEXEC>;
 350 defm S_ATOMIC_XOR                 : SM_Pseudo_Atomics <"s_atomic_xor", SReg_64, SReg_32_XM0_XEXEC>;
 351 defm S_ATOMIC_INC                 : SM_Pseudo_Atomics <"s_atomic_inc", SReg_64, SReg_32_XM0_XEXEC>;
 352 defm S_ATOMIC_DEC                 : SM_Pseudo_Atomics <"s_atomic_dec", SReg_64, SReg_32_XM0_XEXEC>;
 353
 354 defm S_ATOMIC_SWAP_X2             : SM_Pseudo_Atomics <"s_atomic_swap_x2", SReg_64, SReg_64_XEXEC>;
 355 defm S_ATOMIC_CMPSWAP_X2          : SM_Pseudo_Atomics <"s_atomic_cmpswap_x2", SReg_64, SReg_128>;
 356 defm S_ATOMIC_ADD_X2              : SM_Pseudo_Atomics <"s_atomic_add_x2", SReg_64, SReg_64_XEXEC>;
 357 defm S_ATOMIC_SUB_X2              : SM_Pseudo_Atomics <"s_atomic_sub_x2", SReg_64, SReg_64_XEXEC>;
 358 defm S_ATOMIC_SMIN_X2             : SM_Pseudo_Atomics <"s_atomic_smin_x2", SReg_64, SReg_64_XEXEC>;
 359 defm S_ATOMIC_UMIN_X2             : SM_Pseudo_Atomics <"s_atomic_umin_x2", SReg_64, SReg_64_XEXEC>;
 360 defm S_ATOMIC_SMAX_X2             : SM_Pseudo_Atomics <"s_atomic_smax_x2", SReg_64, SReg_64_XEXEC>;
 361 defm S_ATOMIC_UMAX_X2             : SM_Pseudo_Atomics <"s_atomic_umax_x2", SReg_64, SReg_64_XEXEC>;
 362 defm S_ATOMIC_AND_X2              : SM_Pseudo_Atomics <"s_atomic_and_x2", SReg_64, SReg_64_XEXEC>;
 363 defm S_ATOMIC_OR_X2               : SM_Pseudo_Atomics <"s_atomic_or_x2", SReg_64, SReg_64_XEXEC>;
 364 defm S_ATOMIC_XOR_X2              : SM_Pseudo_Atomics <"s_atomic_xor_x2", SReg_64, SReg_64_XEXEC>;
 365 defm S_ATOMIC_INC_X2              : SM_Pseudo_Atomics <"s_atomic_inc_x2", SReg_64, SReg_64_XEXEC>;
 366 defm S_ATOMIC_DEC_X2              : SM_Pseudo_Atomics <"s_atomic_dec_x2", SReg_64, SReg_64_XEXEC>;
 367
 368 } // let SubtargetPredicate = HasScalarAtomics
 369
 370 let SubtargetPredicate = isGFX9 in {
 371 defm S_DCACHE_DISCARD    : SM_Pseudo_Discards <"s_dcache_discard">;
 372 defm S_DCACHE_DISCARD_X2 : SM_Pseudo_Discards <"s_dcache_discard_x2">;
 373 }
 374
 375 //===----------------------------------------------------------------------===//
 376 // Targets
 377 //===----------------------------------------------------------------------===//
 378
 379 //===----------------------------------------------------------------------===//
 380 // SI
 381 //===----------------------------------------------------------------------===//
 382
 383 class SMRD_Real_si <bits<5> op, SM_Pseudo ps>
 384   : SM_Real<ps>
 385   , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>
 386   , Enc32 {
 387
 388   let AssemblerPredicates = [isSICI];
 389   let DecoderNamespace = "SICI";
 390
 391   let Inst{7-0}   = !if(ps.has_offset, offset{7-0}, ?);
 392   let Inst{8}     = imm;
 393   let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
 394   let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
 395   let Inst{26-22} = op;
 396   let Inst{31-27} = 0x18; //encoding
 397 }
 398
 399 // FIXME: Assembler should reject trying to use glc on SMRD
 400 // instructions on SI.
 401 multiclass SM_Real_Loads_si<bits<5> op, string ps,
 402                             SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM),
 403                             SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> {
 404
 405   def _IMM_si : SMRD_Real_si <op, immPs> {
 406     let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_8:$offset, GLC:$glc);
 407   }
 408
 409   // FIXME: The operand name $offset is inconsistent with $soff used
 410   // in the pseudo
 411   def _SGPR_si : SMRD_Real_si <op, sgprPs> {
 412     let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc);
 413   }
 414
 415 }
 416
 417 defm S_LOAD_DWORD           : SM_Real_Loads_si <0x00, "S_LOAD_DWORD">;
 418 defm S_LOAD_DWORDX2         : SM_Real_Loads_si <0x01, "S_LOAD_DWORDX2">;
 419 defm S_LOAD_DWORDX4         : SM_Real_Loads_si <0x02, "S_LOAD_DWORDX4">;
 420 defm S_LOAD_DWORDX8         : SM_Real_Loads_si <0x03, "S_LOAD_DWORDX8">;
 421 defm S_LOAD_DWORDX16        : SM_Real_Loads_si <0x04, "S_LOAD_DWORDX16">;
 422 defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_si <0x08, "S_BUFFER_LOAD_DWORD">;
 423 defm S_BUFFER_LOAD_DWORDX2  : SM_Real_Loads_si <0x09, "S_BUFFER_LOAD_DWORDX2">;
 424 defm S_BUFFER_LOAD_DWORDX4  : SM_Real_Loads_si <0x0a, "S_BUFFER_LOAD_DWORDX4">;
 425 defm S_BUFFER_LOAD_DWORDX8  : SM_Real_Loads_si <0x0b, "S_BUFFER_LOAD_DWORDX8">;
 426 defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_si <0x0c, "S_BUFFER_LOAD_DWORDX16">;
 427
 428 def S_MEMTIME_si    : SMRD_Real_si <0x1e, S_MEMTIME>;
 429 def S_DCACHE_INV_si : SMRD_Real_si <0x1f, S_DCACHE_INV>;
 430
 431
 432 //===----------------------------------------------------------------------===//
 433 // VI
 434 //===----------------------------------------------------------------------===//
 435
 436 class SMEM_Real_vi <bits<8> op, SM_Pseudo ps>
 437   : SM_Real<ps>
 438   , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.VI>
 439   , Enc64 {
 440   bit glc;
 441
 442   let AssemblerPredicates = [isVI];
 443   let DecoderNamespace = "VI";
 444
 445   let Inst{5-0}   = !if(ps.has_sbase, sbase{6-1}, ?);
 446   let Inst{12-6}  = !if(ps.has_sdst, sdst{6-0}, ?);
 447
 448   let Inst{16} = !if(ps.has_glc, glc, ?);
 449   let Inst{17} = imm;
 450   let Inst{25-18} = op;
 451   let Inst{31-26} = 0x30; //encoding
 452   let Inst{51-32} = !if(ps.has_offset, offset{19-0}, ?);
 453 }
 454
 455 multiclass SM_Real_Loads_vi<bits<8> op, string ps,
 456                             SM_Load_Pseudo immPs = !cast<SM_Load_Pseudo>(ps#_IMM),
 457                             SM_Load_Pseudo sgprPs = !cast<SM_Load_Pseudo>(ps#_SGPR)> {
 458   def _IMM_vi : SMEM_Real_vi <op, immPs> {
 459     let InOperandList = (ins immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc);
 460   }
 461   def _SGPR_vi : SMEM_Real_vi <op, sgprPs> {
 462     let InOperandList = (ins sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc);
 463   }
 464 }
 465
 466 class SMEM_Real_Store_vi <bits<8> op, SM_Pseudo ps> : SMEM_Real_vi <op, ps> {
 467   // encoding
 468   bits<7> sdata;
 469
 470   let sdst = ?;
 471   let Inst{12-6}  = !if(ps.has_sdst, sdata{6-0}, ?);
 472 }
 473
 474 multiclass SM_Real_Stores_vi<bits<8> op, string ps,
 475                             SM_Store_Pseudo immPs = !cast<SM_Store_Pseudo>(ps#_IMM),
 476                             SM_Store_Pseudo sgprPs = !cast<SM_Store_Pseudo>(ps#_SGPR)> {
 477   // FIXME: The operand name $offset is inconsistent with $soff used
 478   // in the pseudo
 479   def _IMM_vi : SMEM_Real_Store_vi <op, immPs> {
 480     let InOperandList = (ins immPs.SrcClass:$sdata, immPs.BaseClass:$sbase, smrd_offset_20:$offset, GLC:$glc);
 481   }
 482
 483   def _SGPR_vi : SMEM_Real_Store_vi <op, sgprPs> {
 484     let InOperandList = (ins sgprPs.SrcClass:$sdata, sgprPs.BaseClass:$sbase, SReg_32:$offset, GLC:$glc);
 485   }
 486 }
 487
 488 multiclass SM_Real_Probe_vi<bits<8> op, string ps> {
 489   def _IMM_vi  : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_IMM)>;
 490   def _SGPR_vi : SMEM_Real_Store_vi <op, !cast<SM_Probe_Pseudo>(ps#_SGPR)>;
 491 }
 492
 493 defm S_LOAD_DWORD           : SM_Real_Loads_vi <0x00, "S_LOAD_DWORD">;
 494 defm S_LOAD_DWORDX2         : SM_Real_Loads_vi <0x01, "S_LOAD_DWORDX2">;
 495 defm S_LOAD_DWORDX4         : SM_Real_Loads_vi <0x02, "S_LOAD_DWORDX4">;
 496 defm S_LOAD_DWORDX8         : SM_Real_Loads_vi <0x03, "S_LOAD_DWORDX8">;
 497 defm S_LOAD_DWORDX16        : SM_Real_Loads_vi <0x04, "S_LOAD_DWORDX16">;
 498 defm S_BUFFER_LOAD_DWORD    : SM_Real_Loads_vi <0x08, "S_BUFFER_LOAD_DWORD">;
 499 defm S_BUFFER_LOAD_DWORDX2  : SM_Real_Loads_vi <0x09, "S_BUFFER_LOAD_DWORDX2">;
 500 defm S_BUFFER_LOAD_DWORDX4  : SM_Real_Loads_vi <0x0a, "S_BUFFER_LOAD_DWORDX4">;
 501 defm S_BUFFER_LOAD_DWORDX8  : SM_Real_Loads_vi <0x0b, "S_BUFFER_LOAD_DWORDX8">;
 502 defm S_BUFFER_LOAD_DWORDX16 : SM_Real_Loads_vi <0x0c, "S_BUFFER_LOAD_DWORDX16">;
 503
 504 defm S_STORE_DWORD : SM_Real_Stores_vi <0x10, "S_STORE_DWORD">;
 505 defm S_STORE_DWORDX2 : SM_Real_Stores_vi <0x11, "S_STORE_DWORDX2">;
 506 defm S_STORE_DWORDX4 : SM_Real_Stores_vi <0x12, "S_STORE_DWORDX4">;
 507
 508 defm S_BUFFER_STORE_DWORD    : SM_Real_Stores_vi <0x18, "S_BUFFER_STORE_DWORD">;
 509 defm S_BUFFER_STORE_DWORDX2  : SM_Real_Stores_vi <0x19, "S_BUFFER_STORE_DWORDX2">;
 510 defm S_BUFFER_STORE_DWORDX4  : SM_Real_Stores_vi <0x1a, "S_BUFFER_STORE_DWORDX4">;
 511
 512 // These instructions use same encoding
 513 def S_DCACHE_INV_vi         : SMEM_Real_vi <0x20, S_DCACHE_INV>;
 514 def S_DCACHE_WB_vi          : SMEM_Real_vi <0x21, S_DCACHE_WB>;
 515 def S_DCACHE_INV_VOL_vi     : SMEM_Real_vi <0x22, S_DCACHE_INV_VOL>;
 516 def S_DCACHE_WB_VOL_vi      : SMEM_Real_vi <0x23, S_DCACHE_WB_VOL>;
 517 def S_MEMTIME_vi            : SMEM_Real_vi <0x24, S_MEMTIME>;
 518 def S_MEMREALTIME_vi        : SMEM_Real_vi <0x25, S_MEMREALTIME>;
 519
 520 defm S_SCRATCH_LOAD_DWORD    : SM_Real_Loads_vi <0x05, "S_SCRATCH_LOAD_DWORD">;
 521 defm S_SCRATCH_LOAD_DWORDX2  : SM_Real_Loads_vi <0x06, "S_SCRATCH_LOAD_DWORDX2">;
 522 defm S_SCRATCH_LOAD_DWORDX4  : SM_Real_Loads_vi <0x07, "S_SCRATCH_LOAD_DWORDX4">;
 523
 524 defm S_SCRATCH_STORE_DWORD   : SM_Real_Stores_vi <0x15, "S_SCRATCH_STORE_DWORD">;
 525 defm S_SCRATCH_STORE_DWORDX2 : SM_Real_Stores_vi <0x16, "S_SCRATCH_STORE_DWORDX2">;
 526 defm S_SCRATCH_STORE_DWORDX4 : SM_Real_Stores_vi <0x17, "S_SCRATCH_STORE_DWORDX4">;
 527
 528 defm S_ATC_PROBE        : SM_Real_Probe_vi <0x26, "S_ATC_PROBE">;
 529 defm S_ATC_PROBE_BUFFER : SM_Real_Probe_vi <0x27, "S_ATC_PROBE_BUFFER">;
 530
 531 //===----------------------------------------------------------------------===//
 532 // GFX9
 533 //===----------------------------------------------------------------------===//
 534
 535 class SMEM_Atomic_Real_vi <bits<8> op, SM_Atomic_Pseudo ps>
 536   : SMEM_Real_vi <op, ps> {
 537
 538   bits<7> sdata;
 539
 540   let Constraints = ps.Constraints;
 541   let DisableEncoding = ps.DisableEncoding;
 542
 543   let glc = ps.glc;
 544   let Inst{12-6} = !if(glc, sdst{6-0}, sdata{6-0});
 545 }
 546
 547 multiclass SM_Real_Atomics_vi<bits<8> op, string ps> {
 548   def _IMM_vi       : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM)>;
 549   def _SGPR_vi      : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR)>;
 550   def _IMM_RTN_vi   : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_IMM_RTN)>;
 551   def _SGPR_RTN_vi  : SMEM_Atomic_Real_vi <op, !cast<SM_Atomic_Pseudo>(ps#_SGPR_RTN)>;
 552 }
 553
 554 defm S_BUFFER_ATOMIC_SWAP         : SM_Real_Atomics_vi <0x40, "S_BUFFER_ATOMIC_SWAP">;
 555 defm S_BUFFER_ATOMIC_CMPSWAP      : SM_Real_Atomics_vi <0x41, "S_BUFFER_ATOMIC_CMPSWAP">;
 556 defm S_BUFFER_ATOMIC_ADD          : SM_Real_Atomics_vi <0x42, "S_BUFFER_ATOMIC_ADD">;
 557 defm S_BUFFER_ATOMIC_SUB          : SM_Real_Atomics_vi <0x43, "S_BUFFER_ATOMIC_SUB">;
 558 defm S_BUFFER_ATOMIC_SMIN         : SM_Real_Atomics_vi <0x44, "S_BUFFER_ATOMIC_SMIN">;
 559 defm S_BUFFER_ATOMIC_UMIN         : SM_Real_Atomics_vi <0x45, "S_BUFFER_ATOMIC_UMIN">;
 560 defm S_BUFFER_ATOMIC_SMAX         : SM_Real_Atomics_vi <0x46, "S_BUFFER_ATOMIC_SMAX">;
 561 defm S_BUFFER_ATOMIC_UMAX         : SM_Real_Atomics_vi <0x47, "S_BUFFER_ATOMIC_UMAX">;
 562 defm S_BUFFER_ATOMIC_AND          : SM_Real_Atomics_vi <0x48, "S_BUFFER_ATOMIC_AND">;
 563 defm S_BUFFER_ATOMIC_OR           : SM_Real_Atomics_vi <0x49, "S_BUFFER_ATOMIC_OR">;
 564 defm S_BUFFER_ATOMIC_XOR          : SM_Real_Atomics_vi <0x4a, "S_BUFFER_ATOMIC_XOR">;
 565 defm S_BUFFER_ATOMIC_INC          : SM_Real_Atomics_vi <0x4b, "S_BUFFER_ATOMIC_INC">;
 566 defm S_BUFFER_ATOMIC_DEC          : SM_Real_Atomics_vi <0x4c, "S_BUFFER_ATOMIC_DEC">;
 567
 568 defm S_BUFFER_ATOMIC_SWAP_X2      : SM_Real_Atomics_vi <0x60, "S_BUFFER_ATOMIC_SWAP_X2">;
 569 defm S_BUFFER_ATOMIC_CMPSWAP_X2   : SM_Real_Atomics_vi <0x61, "S_BUFFER_ATOMIC_CMPSWAP_X2">;
 570 defm S_BUFFER_ATOMIC_ADD_X2       : SM_Real_Atomics_vi <0x62, "S_BUFFER_ATOMIC_ADD_X2">;
 571 defm S_BUFFER_ATOMIC_SUB_X2       : SM_Real_Atomics_vi <0x63, "S_BUFFER_ATOMIC_SUB_X2">;
 572 defm S_BUFFER_ATOMIC_SMIN_X2      : SM_Real_Atomics_vi <0x64, "S_BUFFER_ATOMIC_SMIN_X2">;
 573 defm S_BUFFER_ATOMIC_UMIN_X2      : SM_Real_Atomics_vi <0x65, "S_BUFFER_ATOMIC_UMIN_X2">;
 574 defm S_BUFFER_ATOMIC_SMAX_X2      : SM_Real_Atomics_vi <0x66, "S_BUFFER_ATOMIC_SMAX_X2">;
 575 defm S_BUFFER_ATOMIC_UMAX_X2      : SM_Real_Atomics_vi <0x67, "S_BUFFER_ATOMIC_UMAX_X2">;
 576 defm S_BUFFER_ATOMIC_AND_X2       : SM_Real_Atomics_vi <0x68, "S_BUFFER_ATOMIC_AND_X2">;
 577 defm S_BUFFER_ATOMIC_OR_X2        : SM_Real_Atomics_vi <0x69, "S_BUFFER_ATOMIC_OR_X2">;
 578 defm S_BUFFER_ATOMIC_XOR_X2       : SM_Real_Atomics_vi <0x6a, "S_BUFFER_ATOMIC_XOR_X2">;
 579 defm S_BUFFER_ATOMIC_INC_X2       : SM_Real_Atomics_vi <0x6b, "S_BUFFER_ATOMIC_INC_X2">;
 580 defm S_BUFFER_ATOMIC_DEC_X2       : SM_Real_Atomics_vi <0x6c, "S_BUFFER_ATOMIC_DEC_X2">;
 581
 582 defm S_ATOMIC_SWAP                : SM_Real_Atomics_vi <0x80, "S_ATOMIC_SWAP">;
 583 defm S_ATOMIC_CMPSWAP             : SM_Real_Atomics_vi <0x81, "S_ATOMIC_CMPSWAP">;
 584 defm S_ATOMIC_ADD                 : SM_Real_Atomics_vi <0x82, "S_ATOMIC_ADD">;
 585 defm S_ATOMIC_SUB                 : SM_Real_Atomics_vi <0x83, "S_ATOMIC_SUB">;
 586 defm S_ATOMIC_SMIN                : SM_Real_Atomics_vi <0x84, "S_ATOMIC_SMIN">;
 587 defm S_ATOMIC_UMIN                : SM_Real_Atomics_vi <0x85, "S_ATOMIC_UMIN">;
 588 defm S_ATOMIC_SMAX                : SM_Real_Atomics_vi <0x86, "S_ATOMIC_SMAX">;
 589 defm S_ATOMIC_UMAX                : SM_Real_Atomics_vi <0x87, "S_ATOMIC_UMAX">;
 590 defm S_ATOMIC_AND                 : SM_Real_Atomics_vi <0x88, "S_ATOMIC_AND">;
 591 defm S_ATOMIC_OR                  : SM_Real_Atomics_vi <0x89, "S_ATOMIC_OR">;
 592 defm S_ATOMIC_XOR                 : SM_Real_Atomics_vi <0x8a, "S_ATOMIC_XOR">;
 593 defm S_ATOMIC_INC                 : SM_Real_Atomics_vi <0x8b, "S_ATOMIC_INC">;
 594 defm S_ATOMIC_DEC                 : SM_Real_Atomics_vi <0x8c, "S_ATOMIC_DEC">;
 595
 596 defm S_ATOMIC_SWAP_X2             : SM_Real_Atomics_vi <0xa0, "S_ATOMIC_SWAP_X2">;
 597 defm S_ATOMIC_CMPSWAP_X2          : SM_Real_Atomics_vi <0xa1, "S_ATOMIC_CMPSWAP_X2">;
 598 defm S_ATOMIC_ADD_X2              : SM_Real_Atomics_vi <0xa2, "S_ATOMIC_ADD_X2">;
 599 defm S_ATOMIC_SUB_X2              : SM_Real_Atomics_vi <0xa3, "S_ATOMIC_SUB_X2">;
 600 defm S_ATOMIC_SMIN_X2             : SM_Real_Atomics_vi <0xa4, "S_ATOMIC_SMIN_X2">;
 601 defm S_ATOMIC_UMIN_X2             : SM_Real_Atomics_vi <0xa5, "S_ATOMIC_UMIN_X2">;
 602 defm S_ATOMIC_SMAX_X2             : SM_Real_Atomics_vi <0xa6, "S_ATOMIC_SMAX_X2">;
 603 defm S_ATOMIC_UMAX_X2             : SM_Real_Atomics_vi <0xa7, "S_ATOMIC_UMAX_X2">;
 604 defm S_ATOMIC_AND_X2              : SM_Real_Atomics_vi <0xa8, "S_ATOMIC_AND_X2">;
 605 defm S_ATOMIC_OR_X2               : SM_Real_Atomics_vi <0xa9, "S_ATOMIC_OR_X2">;
 606 defm S_ATOMIC_XOR_X2              : SM_Real_Atomics_vi <0xaa, "S_ATOMIC_XOR_X2">;
 607 defm S_ATOMIC_INC_X2              : SM_Real_Atomics_vi <0xab, "S_ATOMIC_INC_X2">;
 608 defm S_ATOMIC_DEC_X2              : SM_Real_Atomics_vi <0xac, "S_ATOMIC_DEC_X2">;
 609
 610 multiclass SM_Real_Discard_vi<bits<8> op, string ps> {
 611   def _IMM_vi  : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_IMM)>;
 612   def _SGPR_vi : SMEM_Real_vi <op, !cast<SM_Discard_Pseudo>(ps#_SGPR)>;
 613 }
 614
 615 defm S_DCACHE_DISCARD    : SM_Real_Discard_vi <0x28, "S_DCACHE_DISCARD">;
 616 defm S_DCACHE_DISCARD_X2 : SM_Real_Discard_vi <0x29, "S_DCACHE_DISCARD_X2">;
 617
 618 //===----------------------------------------------------------------------===//
 619 // CI
 620 //===----------------------------------------------------------------------===//
 621
 622 def smrd_literal_offset : NamedOperandU32<"SMRDLiteralOffset",
 623                                           NamedMatchClass<"SMRDLiteralOffset">> {
 624   let OperandType = "OPERAND_IMMEDIATE";
 625 }
 626
 627 class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> :
 628   SM_Real<ps>,
 629   Enc64 {
 630
 631   let AssemblerPredicates = [isCIOnly];
 632   let DecoderNamespace = "CI";
 633   let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc);
 634
 635   let LGKM_CNT = ps.LGKM_CNT;
 636   let SMRD = ps.SMRD;
 637   let mayLoad = ps.mayLoad;
 638   let mayStore = ps.mayStore;
 639   let hasSideEffects = ps.hasSideEffects;
 640   let SchedRW = ps.SchedRW;
 641   let UseNamedOperandTable = ps.UseNamedOperandTable;
 642
 643   let Inst{7-0}   = 0xff;
 644   let Inst{8}     = 0;
 645   let Inst{14-9}  = sbase{6-1};
 646   let Inst{21-15} = sdst{6-0};
 647   let Inst{26-22} = op;
 648   let Inst{31-27} = 0x18; //encoding
 649   let Inst{63-32} = offset{31-0};
 650 }
 651
 652 def S_LOAD_DWORD_IMM_ci           : SMRD_Real_Load_IMM_ci <0x00, S_LOAD_DWORD_IMM>;
 653 def S_LOAD_DWORDX2_IMM_ci         : SMRD_Real_Load_IMM_ci <0x01, S_LOAD_DWORDX2_IMM>;
 654 def S_LOAD_DWORDX4_IMM_ci         : SMRD_Real_Load_IMM_ci <0x02, S_LOAD_DWORDX4_IMM>;
 655 def S_LOAD_DWORDX8_IMM_ci         : SMRD_Real_Load_IMM_ci <0x03, S_LOAD_DWORDX8_IMM>;
 656 def S_LOAD_DWORDX16_IMM_ci        : SMRD_Real_Load_IMM_ci <0x04, S_LOAD_DWORDX16_IMM>;
 657 def S_BUFFER_LOAD_DWORD_IMM_ci    : SMRD_Real_Load_IMM_ci <0x08, S_BUFFER_LOAD_DWORD_IMM>;
 658 def S_BUFFER_LOAD_DWORDX2_IMM_ci  : SMRD_Real_Load_IMM_ci <0x09, S_BUFFER_LOAD_DWORDX2_IMM>;
 659 def S_BUFFER_LOAD_DWORDX4_IMM_ci  : SMRD_Real_Load_IMM_ci <0x0a, S_BUFFER_LOAD_DWORDX4_IMM>;
 660 def S_BUFFER_LOAD_DWORDX8_IMM_ci  : SMRD_Real_Load_IMM_ci <0x0b, S_BUFFER_LOAD_DWORDX8_IMM>;
 661 def S_BUFFER_LOAD_DWORDX16_IMM_ci : SMRD_Real_Load_IMM_ci <0x0c, S_BUFFER_LOAD_DWORDX16_IMM>;
 662
 663 class SMRD_Real_ci <bits<5> op, SM_Pseudo ps>
 664   : SM_Real<ps>
 665   , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI>
 666   , Enc32 {
 667
 668   let AssemblerPredicates = [isCIOnly];
 669   let DecoderNamespace = "CI";
 670
 671   let Inst{7-0}   = !if(ps.has_offset, offset{7-0}, ?);
 672   let Inst{8}     = imm;
 673   let Inst{14-9}  = !if(ps.has_sbase, sbase{6-1}, ?);
 674   let Inst{21-15} = !if(ps.has_sdst, sdst{6-0}, ?);
 675   let Inst{26-22} = op;
 676   let Inst{31-27} = 0x18; //encoding
 677 }
 678
 679 def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
 680
 681 //===----------------------------------------------------------------------===//
 682 // Scalar Memory Patterns
 683 //===----------------------------------------------------------------------===//
 684
 685 def smrd_load : PatFrag <(ops node:$ptr), (load node:$ptr), [{ return isUniformLoad(N);}]>;
 686
 687 def SMRDImm         : ComplexPattern<i64, 2, "SelectSMRDImm">;
 688 def SMRDImm32       : ComplexPattern<i64, 2, "SelectSMRDImm32">;
 689 def SMRDSgpr        : ComplexPattern<i64, 2, "SelectSMRDSgpr">;
 690 def SMRDBufferImm   : ComplexPattern<i32, 1, "SelectSMRDBufferImm">;
 691 def SMRDBufferImm32 : ComplexPattern<i32, 1, "SelectSMRDBufferImm32">;
 692
 693 multiclass SMRD_Pattern <string Instr, ValueType vt> {
 694
 695   // 1. IMM offset
 696   def : GCNPat <
 697     (smrd_load (SMRDImm i64:$sbase, i32:$offset)),
 698     (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, 0))
 699   >;
 700
 701   // 2. 32-bit IMM offset on CI
 702   def : GCNPat <
 703     (smrd_load (SMRDImm32 i64:$sbase, i32:$offset)),
 704     (vt (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, 0))> {
 705     let OtherPredicates = [isCIOnly];
 706   }
 707
 708   // 3. SGPR offset
 709   def : GCNPat <
 710     (smrd_load (SMRDSgpr i64:$sbase, i32:$offset)),
 711     (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, 0))
 712   >;
 713 }
 714
 715 multiclass SMLoad_Pattern <string Instr, ValueType vt> {
 716   // 1. Offset as an immediate
 717   def : GCNPat <
 718     (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm i32:$offset), i1:$glc),
 719     (vt (!cast<SM_Pseudo>(Instr#"_IMM") $sbase, $offset, (as_i1imm $glc)))
 720   >;
 721
 722   // 2. 32-bit IMM offset on CI
 723   def : GCNPat <
 724     (vt (SIsbuffer_load v4i32:$sbase, (SMRDBufferImm32 i32:$offset), i1:$glc)),
 725     (!cast<InstSI>(Instr#"_IMM_ci") $sbase, $offset, (as_i1imm $glc))> {
 726     let OtherPredicates = [isCIOnly];
 727   }
 728
 729   // 3. Offset loaded in an 32bit SGPR
 730   def : GCNPat <
 731     (SIsbuffer_load v4i32:$sbase, i32:$offset, i1:$glc),
 732     (vt (!cast<SM_Pseudo>(Instr#"_SGPR") $sbase, $offset, (as_i1imm $glc)))
 733   >;
 734 }
 735
 736 // Global and constant loads can be selected to either MUBUF or SMRD
 737 // instructions, but SMRD instructions are faster so we want the instruction
 738 // selector to prefer those.
 739 let AddedComplexity = 100 in {
 740
 741 defm : SMRD_Pattern <"S_LOAD_DWORD",    i32>;
 742 defm : SMRD_Pattern <"S_LOAD_DWORDX2",  v2i32>;
 743 defm : SMRD_Pattern <"S_LOAD_DWORDX4",  v4i32>;
 744 defm : SMRD_Pattern <"S_LOAD_DWORDX8",  v8i32>;
 745 defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>;
 746
 747 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD",     i32>;
 748 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2",   v2i32>;
 749 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4",   v4i32>;
 750 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8",   v8i32>;
 751 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16",  v16i32>;
 752
 753 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORD",     f32>;
 754 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX2",   v2f32>;
 755 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX4",   v4f32>;
 756 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX8",   v8f32>;
 757 defm : SMLoad_Pattern <"S_BUFFER_LOAD_DWORDX16",  v16f32>;
 758 } // End let AddedComplexity = 100
 759
 760 let OtherPredicates = [isSICI] in {
 761 def : GCNPat <
 762   (i64 (readcyclecounter)),
 763   (S_MEMTIME)
 764 >;
 765 }
 766
 767 let OtherPredicates = [isVI] in {
 768
 769 def : GCNPat <
 770   (i64 (readcyclecounter)),
 771   (S_MEMREALTIME)
 772 >;
 773
 774 } // let OtherPredicates = [isVI]