llvm/test/CodeGen/AMDGPU/memory-legalizer-global-system.ll

   1 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
   2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX6 %s
   3 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX7 %s
   4 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-WGP %s
   5 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr=+cumode -verify-machineinstrs < %s | FileCheck --check-prefixes=GFX10-CU %s
   6 ; RUN: llc -mtriple=amdgcn-amd-amdpal -mcpu=gfx700 -amdgcn-skip-cache-invalidations -verify-machineinstrs < %s | FileCheck --check-prefixes=SKIP-CACHE-INV %s
   7 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-NOTTGSPLIT %s
   8 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a -mattr=+tgsplit -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX90A-TGSPLIT %s
   9
  10 define amdgpu_kernel void @global_system_unordered_load(
  11 ; GFX6-LABEL: global_system_unordered_load:
  12 ; GFX6:       ; %bb.0: ; %entry
  13 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
  14 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
  15 ; GFX6-NEXT:    s_mov_b32 s2, -1
  16 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
  17 ; GFX6-NEXT:    s_mov_b32 s0, s4
  18 ; GFX6-NEXT:    s_mov_b32 s1, s5
  19 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
  20 ; GFX6-NEXT:    s_mov_b32 s4, s6
  21 ; GFX6-NEXT:    s_mov_b32 s5, s7
  22 ; GFX6-NEXT:    s_mov_b32 s6, s2
  23 ; GFX6-NEXT:    s_mov_b32 s7, s3
  24 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
  25 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
  26 ; GFX6-NEXT:    s_endpgm
  27 ;
  28 ; GFX7-LABEL: global_system_unordered_load:
  29 ; GFX7:       ; %bb.0: ; %entry
  30 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
  31 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
  32 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
  33 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
  34 ; GFX7-NEXT:    flat_load_dword v0, v[0:1]
  35 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
  36 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
  37 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
  38 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
  39 ; GFX7-NEXT:    s_endpgm
  40 ;
  41 ; GFX10-WGP-LABEL: global_system_unordered_load:
  42 ; GFX10-WGP:       ; %bb.0: ; %entry
  43 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
  44 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
  45 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
  46 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
  47 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
  48 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
  49 ; GFX10-WGP-NEXT:    s_endpgm
  50 ;
  51 ; GFX10-CU-LABEL: global_system_unordered_load:
  52 ; GFX10-CU:       ; %bb.0: ; %entry
  53 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
  54 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
  55 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
  56 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
  57 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
  58 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
  59 ; GFX10-CU-NEXT:    s_endpgm
  60 ;
  61 ; SKIP-CACHE-INV-LABEL: global_system_unordered_load:
  62 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
  63 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
  64 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
  65 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
  66 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
  67 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
  68 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
  69 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
  70 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
  71 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
  72 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
  73 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
  74 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
  75 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
  76 ; SKIP-CACHE-INV-NEXT:    s_endpgm
  77 ;
  78 ; GFX90A-NOTTGSPLIT-LABEL: global_system_unordered_load:
  79 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
  80 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
  81 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
  82 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
  83 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
  84 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
  85 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
  86 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
  87 ;
  88 ; GFX90A-TGSPLIT-LABEL: global_system_unordered_load:
  89 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
  90 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
  91 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
  92 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
  93 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
  94 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
  95 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
  96 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
  97     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
  98 entry:
  99   %val = load atomic i32, i32 addrspace(1)* %in unordered, align 4
 100   store i32 %val, i32 addrspace(1)* %out
 101   ret void
 102 }
 103
 104 define amdgpu_kernel void @global_system_monotonic_load(
 105 ; GFX6-LABEL: global_system_monotonic_load:
 106 ; GFX6:       ; %bb.0: ; %entry
 107 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
 108 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 109 ; GFX6-NEXT:    s_mov_b32 s2, -1
 110 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 111 ; GFX6-NEXT:    s_mov_b32 s0, s4
 112 ; GFX6-NEXT:    s_mov_b32 s1, s5
 113 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
 114 ; GFX6-NEXT:    s_mov_b32 s4, s6
 115 ; GFX6-NEXT:    s_mov_b32 s5, s7
 116 ; GFX6-NEXT:    s_mov_b32 s6, s2
 117 ; GFX6-NEXT:    s_mov_b32 s7, s3
 118 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 119 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 120 ; GFX6-NEXT:    s_endpgm
 121 ;
 122 ; GFX7-LABEL: global_system_monotonic_load:
 123 ; GFX7:       ; %bb.0: ; %entry
 124 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 125 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 126 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 127 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 128 ; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
 129 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 130 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 131 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 132 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
 133 ; GFX7-NEXT:    s_endpgm
 134 ;
 135 ; GFX10-WGP-LABEL: global_system_monotonic_load:
 136 ; GFX10-WGP:       ; %bb.0: ; %entry
 137 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 138 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 139 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 140 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
 141 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
 142 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
 143 ; GFX10-WGP-NEXT:    s_endpgm
 144 ;
 145 ; GFX10-CU-LABEL: global_system_monotonic_load:
 146 ; GFX10-CU:       ; %bb.0: ; %entry
 147 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 148 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 149 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 150 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
 151 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
 152 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
 153 ; GFX10-CU-NEXT:    s_endpgm
 154 ;
 155 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_load:
 156 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 157 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 158 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
 159 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
 160 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 161 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
 162 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
 163 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
 164 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
 165 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
 166 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
 167 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
 168 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 169 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 170 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 171 ;
 172 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_load:
 173 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 174 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 175 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 176 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 177 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
 178 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 179 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
 180 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 181 ;
 182 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_load:
 183 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 184 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 185 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 186 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 187 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
 188 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 189 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
 190 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 191     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
 192 entry:
 193   %val = load atomic i32, i32 addrspace(1)* %in monotonic, align 4
 194   store i32 %val, i32 addrspace(1)* %out
 195   ret void
 196 }
 197
 198 define amdgpu_kernel void @global_system_acquire_load(
 199 ; GFX6-LABEL: global_system_acquire_load:
 200 ; GFX6:       ; %bb.0: ; %entry
 201 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
 202 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 203 ; GFX6-NEXT:    s_mov_b32 s2, -1
 204 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 205 ; GFX6-NEXT:    s_mov_b32 s0, s4
 206 ; GFX6-NEXT:    s_mov_b32 s1, s5
 207 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
 208 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 209 ; GFX6-NEXT:    buffer_wbinvl1
 210 ; GFX6-NEXT:    s_mov_b32 s4, s6
 211 ; GFX6-NEXT:    s_mov_b32 s5, s7
 212 ; GFX6-NEXT:    s_mov_b32 s6, s2
 213 ; GFX6-NEXT:    s_mov_b32 s7, s3
 214 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 215 ; GFX6-NEXT:    s_endpgm
 216 ;
 217 ; GFX7-LABEL: global_system_acquire_load:
 218 ; GFX7:       ; %bb.0: ; %entry
 219 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 220 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 221 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 222 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 223 ; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
 224 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 225 ; GFX7-NEXT:    buffer_wbinvl1_vol
 226 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 227 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 228 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
 229 ; GFX7-NEXT:    s_endpgm
 230 ;
 231 ; GFX10-WGP-LABEL: global_system_acquire_load:
 232 ; GFX10-WGP:       ; %bb.0: ; %entry
 233 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 234 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 235 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 236 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
 237 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
 238 ; GFX10-WGP-NEXT:    buffer_gl0_inv
 239 ; GFX10-WGP-NEXT:    buffer_gl1_inv
 240 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
 241 ; GFX10-WGP-NEXT:    s_endpgm
 242 ;
 243 ; GFX10-CU-LABEL: global_system_acquire_load:
 244 ; GFX10-CU:       ; %bb.0: ; %entry
 245 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 246 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 247 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 248 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
 249 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
 250 ; GFX10-CU-NEXT:    buffer_gl0_inv
 251 ; GFX10-CU-NEXT:    buffer_gl1_inv
 252 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
 253 ; GFX10-CU-NEXT:    s_endpgm
 254 ;
 255 ; SKIP-CACHE-INV-LABEL: global_system_acquire_load:
 256 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 257 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 258 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
 259 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
 260 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 261 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
 262 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
 263 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
 264 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 265 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
 266 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
 267 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
 268 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
 269 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 270 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 271 ;
 272 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_load:
 273 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 274 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 275 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 276 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 277 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
 278 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 279 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
 280 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
 281 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
 282 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 283 ;
 284 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_load:
 285 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 286 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 287 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 288 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 289 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
 290 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 291 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
 292 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
 293 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
 294 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 295     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
 296 entry:
 297   %val = load atomic i32, i32 addrspace(1)* %in acquire, align 4
 298   store i32 %val, i32 addrspace(1)* %out
 299   ret void
 300 }
 301
 302 define amdgpu_kernel void @global_system_seq_cst_load(
 303 ; GFX6-LABEL: global_system_seq_cst_load:
 304 ; GFX6:       ; %bb.0: ; %entry
 305 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
 306 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 307 ; GFX6-NEXT:    s_mov_b32 s2, -1
 308 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 309 ; GFX6-NEXT:    s_mov_b32 s0, s4
 310 ; GFX6-NEXT:    s_mov_b32 s1, s5
 311 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 312 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
 313 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 314 ; GFX6-NEXT:    buffer_wbinvl1
 315 ; GFX6-NEXT:    s_mov_b32 s4, s6
 316 ; GFX6-NEXT:    s_mov_b32 s5, s7
 317 ; GFX6-NEXT:    s_mov_b32 s6, s2
 318 ; GFX6-NEXT:    s_mov_b32 s7, s3
 319 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 320 ; GFX6-NEXT:    s_endpgm
 321 ;
 322 ; GFX7-LABEL: global_system_seq_cst_load:
 323 ; GFX7:       ; %bb.0: ; %entry
 324 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 325 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 326 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 327 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 328 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 329 ; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
 330 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 331 ; GFX7-NEXT:    buffer_wbinvl1_vol
 332 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 333 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
 334 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
 335 ; GFX7-NEXT:    s_endpgm
 336 ;
 337 ; GFX10-WGP-LABEL: global_system_seq_cst_load:
 338 ; GFX10-WGP:       ; %bb.0: ; %entry
 339 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 340 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 341 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 342 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 343 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
 344 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
 345 ; GFX10-WGP-NEXT:    buffer_gl0_inv
 346 ; GFX10-WGP-NEXT:    buffer_gl1_inv
 347 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
 348 ; GFX10-WGP-NEXT:    s_endpgm
 349 ;
 350 ; GFX10-CU-LABEL: global_system_seq_cst_load:
 351 ; GFX10-CU:       ; %bb.0: ; %entry
 352 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 353 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 354 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 355 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 356 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
 357 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
 358 ; GFX10-CU-NEXT:    buffer_gl0_inv
 359 ; GFX10-CU-NEXT:    buffer_gl1_inv
 360 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
 361 ; GFX10-CU-NEXT:    s_endpgm
 362 ;
 363 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_load:
 364 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 365 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
 366 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
 367 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
 368 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 369 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
 370 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
 371 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 372 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
 373 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 374 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
 375 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
 376 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
 377 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
 378 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
 379 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 380 ;
 381 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_load:
 382 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 383 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 384 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 385 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 386 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
 387 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 388 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
 389 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
 390 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
 391 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 392 ;
 393 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_load:
 394 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 395 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 396 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 397 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 398 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
 399 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 400 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
 401 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
 402 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
 403 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 404     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
 405 entry:
 406   %val = load atomic i32, i32 addrspace(1)* %in seq_cst, align 4
 407   store i32 %val, i32 addrspace(1)* %out
 408   ret void
 409 }
 410
 411 define amdgpu_kernel void @global_system_unordered_store(
 412 ; GFX6-LABEL: global_system_unordered_store:
 413 ; GFX6:       ; %bb.0: ; %entry
 414 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
 415 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 416 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 417 ; GFX6-NEXT:    s_mov_b32 s2, -1
 418 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 419 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
 420 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 421 ; GFX6-NEXT:    s_endpgm
 422 ;
 423 ; GFX7-LABEL: global_system_unordered_store:
 424 ; GFX7:       ; %bb.0: ; %entry
 425 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
 426 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 427 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 428 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 429 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 430 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 431 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
 432 ; GFX7-NEXT:    s_endpgm
 433 ;
 434 ; GFX10-WGP-LABEL: global_system_unordered_store:
 435 ; GFX10-WGP:       ; %bb.0: ; %entry
 436 ; GFX10-WGP-NEXT:    s_clause 0x1
 437 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
 438 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 439 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 440 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 441 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
 442 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
 443 ; GFX10-WGP-NEXT:    s_endpgm
 444 ;
 445 ; GFX10-CU-LABEL: global_system_unordered_store:
 446 ; GFX10-CU:       ; %bb.0: ; %entry
 447 ; GFX10-CU-NEXT:    s_clause 0x1
 448 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
 449 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 450 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 451 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 452 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
 453 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
 454 ; GFX10-CU-NEXT:    s_endpgm
 455 ;
 456 ; SKIP-CACHE-INV-LABEL: global_system_unordered_store:
 457 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 458 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
 459 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 460 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
 461 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
 462 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 463 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
 464 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 465 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 466 ;
 467 ; GFX90A-NOTTGSPLIT-LABEL: global_system_unordered_store:
 468 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 469 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 470 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 471 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 472 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 473 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 474 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 475 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 476 ;
 477 ; GFX90A-TGSPLIT-LABEL: global_system_unordered_store:
 478 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 479 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 480 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 481 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 482 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 483 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 484 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 485 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 486     i32 %in, i32 addrspace(1)* %out) {
 487 entry:
 488   store atomic i32 %in, i32 addrspace(1)* %out unordered, align 4
 489   ret void
 490 }
 491
 492 define amdgpu_kernel void @global_system_monotonic_store(
 493 ; GFX6-LABEL: global_system_monotonic_store:
 494 ; GFX6:       ; %bb.0: ; %entry
 495 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
 496 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 497 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 498 ; GFX6-NEXT:    s_mov_b32 s2, -1
 499 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 500 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
 501 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 502 ; GFX6-NEXT:    s_endpgm
 503 ;
 504 ; GFX7-LABEL: global_system_monotonic_store:
 505 ; GFX7:       ; %bb.0: ; %entry
 506 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
 507 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 508 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 509 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 510 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 511 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 512 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
 513 ; GFX7-NEXT:    s_endpgm
 514 ;
 515 ; GFX10-WGP-LABEL: global_system_monotonic_store:
 516 ; GFX10-WGP:       ; %bb.0: ; %entry
 517 ; GFX10-WGP-NEXT:    s_clause 0x1
 518 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
 519 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 520 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 521 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 522 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
 523 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
 524 ; GFX10-WGP-NEXT:    s_endpgm
 525 ;
 526 ; GFX10-CU-LABEL: global_system_monotonic_store:
 527 ; GFX10-CU:       ; %bb.0: ; %entry
 528 ; GFX10-CU-NEXT:    s_clause 0x1
 529 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
 530 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 531 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 532 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 533 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
 534 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
 535 ; GFX10-CU-NEXT:    s_endpgm
 536 ;
 537 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_store:
 538 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 539 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
 540 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 541 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
 542 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
 543 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 544 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
 545 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 546 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 547 ;
 548 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_store:
 549 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 550 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 551 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 552 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 553 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 554 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 555 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 556 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 557 ;
 558 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_store:
 559 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 560 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 561 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 562 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 563 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 564 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 565 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 566 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 567     i32 %in, i32 addrspace(1)* %out) {
 568 entry:
 569   store atomic i32 %in, i32 addrspace(1)* %out monotonic, align 4
 570   ret void
 571 }
 572
 573 define amdgpu_kernel void @global_system_release_store(
 574 ; GFX6-LABEL: global_system_release_store:
 575 ; GFX6:       ; %bb.0: ; %entry
 576 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
 577 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 578 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 579 ; GFX6-NEXT:    s_mov_b32 s2, -1
 580 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 581 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
 582 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 583 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 584 ; GFX6-NEXT:    s_endpgm
 585 ;
 586 ; GFX7-LABEL: global_system_release_store:
 587 ; GFX7:       ; %bb.0: ; %entry
 588 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
 589 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 590 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 591 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 592 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 593 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 594 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 595 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
 596 ; GFX7-NEXT:    s_endpgm
 597 ;
 598 ; GFX10-WGP-LABEL: global_system_release_store:
 599 ; GFX10-WGP:       ; %bb.0: ; %entry
 600 ; GFX10-WGP-NEXT:    s_clause 0x1
 601 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
 602 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 603 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 604 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 605 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
 606 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 607 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 608 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
 609 ; GFX10-WGP-NEXT:    s_endpgm
 610 ;
 611 ; GFX10-CU-LABEL: global_system_release_store:
 612 ; GFX10-CU:       ; %bb.0: ; %entry
 613 ; GFX10-CU-NEXT:    s_clause 0x1
 614 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
 615 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 616 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 617 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 618 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
 619 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 620 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 621 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
 622 ; GFX10-CU-NEXT:    s_endpgm
 623 ;
 624 ; SKIP-CACHE-INV-LABEL: global_system_release_store:
 625 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 626 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
 627 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 628 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
 629 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
 630 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 631 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
 632 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 633 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 634 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 635 ;
 636 ; GFX90A-NOTTGSPLIT-LABEL: global_system_release_store:
 637 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 638 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 639 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 640 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 641 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 642 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 643 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
 644 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 645 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 646 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 647 ;
 648 ; GFX90A-TGSPLIT-LABEL: global_system_release_store:
 649 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 650 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 651 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 652 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 653 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 654 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 655 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
 656 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 657 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 658 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 659     i32 %in, i32 addrspace(1)* %out) {
 660 entry:
 661   store atomic i32 %in, i32 addrspace(1)* %out release, align 4
 662   ret void
 663 }
 664
 665 define amdgpu_kernel void @global_system_seq_cst_store(
 666 ; GFX6-LABEL: global_system_seq_cst_store:
 667 ; GFX6:       ; %bb.0: ; %entry
 668 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
 669 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 670 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 671 ; GFX6-NEXT:    s_mov_b32 s2, -1
 672 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 673 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
 674 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 675 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 676 ; GFX6-NEXT:    s_endpgm
 677 ;
 678 ; GFX7-LABEL: global_system_seq_cst_store:
 679 ; GFX7:       ; %bb.0: ; %entry
 680 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
 681 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
 682 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 683 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 684 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 685 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 686 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 687 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
 688 ; GFX7-NEXT:    s_endpgm
 689 ;
 690 ; GFX10-WGP-LABEL: global_system_seq_cst_store:
 691 ; GFX10-WGP:       ; %bb.0: ; %entry
 692 ; GFX10-WGP-NEXT:    s_clause 0x1
 693 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
 694 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 695 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 696 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 697 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
 698 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 699 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 700 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
 701 ; GFX10-WGP-NEXT:    s_endpgm
 702 ;
 703 ; GFX10-CU-LABEL: global_system_seq_cst_store:
 704 ; GFX10-CU:       ; %bb.0: ; %entry
 705 ; GFX10-CU-NEXT:    s_clause 0x1
 706 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
 707 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 708 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 709 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 710 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
 711 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 712 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 713 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
 714 ; GFX10-CU-NEXT:    s_endpgm
 715 ;
 716 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_store:
 717 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 718 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
 719 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
 720 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
 721 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
 722 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 723 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
 724 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 725 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 726 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 727 ;
 728 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_store:
 729 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 730 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 731 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 732 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 733 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 734 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 735 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
 736 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 737 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 738 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 739 ;
 740 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_store:
 741 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 742 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
 743 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
 744 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 745 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 746 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 747 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
 748 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 749 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
 750 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 751     i32 %in, i32 addrspace(1)* %out) {
 752 entry:
 753   store atomic i32 %in, i32 addrspace(1)* %out seq_cst, align 4
 754   ret void
 755 }
 756
 757 define amdgpu_kernel void @global_system_monotonic_atomicrmw(
 758 ; GFX6-LABEL: global_system_monotonic_atomicrmw:
 759 ; GFX6:       ; %bb.0: ; %entry
 760 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 761 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
 762 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 763 ; GFX6-NEXT:    s_mov_b32 s2, -1
 764 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 765 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 766 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
 767 ; GFX6-NEXT:    s_endpgm
 768 ;
 769 ; GFX7-LABEL: global_system_monotonic_atomicrmw:
 770 ; GFX7:       ; %bb.0: ; %entry
 771 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 772 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
 773 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 774 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 775 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 776 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 777 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
 778 ; GFX7-NEXT:    s_endpgm
 779 ;
 780 ; GFX10-WGP-LABEL: global_system_monotonic_atomicrmw:
 781 ; GFX10-WGP:       ; %bb.0: ; %entry
 782 ; GFX10-WGP-NEXT:    s_clause 0x1
 783 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
 784 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 785 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 786 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 787 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
 788 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
 789 ; GFX10-WGP-NEXT:    s_endpgm
 790 ;
 791 ; GFX10-CU-LABEL: global_system_monotonic_atomicrmw:
 792 ; GFX10-CU:       ; %bb.0: ; %entry
 793 ; GFX10-CU-NEXT:    s_clause 0x1
 794 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
 795 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 796 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 797 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 798 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
 799 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
 800 ; GFX10-CU-NEXT:    s_endpgm
 801 ;
 802 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_atomicrmw:
 803 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 804 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 805 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
 806 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
 807 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
 808 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 809 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
 810 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
 811 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 812 ;
 813 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_atomicrmw:
 814 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 815 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 816 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
 817 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 818 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 819 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 820 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
 821 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 822 ;
 823 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_atomicrmw:
 824 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 825 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 826 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
 827 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 828 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 829 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 830 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
 831 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 832     i32 addrspace(1)* %out, i32 %in) {
 833 entry:
 834   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in monotonic
 835   ret void
 836 }
 837
 838 define amdgpu_kernel void @global_system_acquire_atomicrmw(
 839 ; GFX6-LABEL: global_system_acquire_atomicrmw:
 840 ; GFX6:       ; %bb.0: ; %entry
 841 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 842 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
 843 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 844 ; GFX6-NEXT:    s_mov_b32 s2, -1
 845 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 846 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 847 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
 848 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 849 ; GFX6-NEXT:    buffer_wbinvl1
 850 ; GFX6-NEXT:    s_endpgm
 851 ;
 852 ; GFX7-LABEL: global_system_acquire_atomicrmw:
 853 ; GFX7:       ; %bb.0: ; %entry
 854 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 855 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
 856 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 857 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 858 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 859 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 860 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
 861 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 862 ; GFX7-NEXT:    buffer_wbinvl1_vol
 863 ; GFX7-NEXT:    s_endpgm
 864 ;
 865 ; GFX10-WGP-LABEL: global_system_acquire_atomicrmw:
 866 ; GFX10-WGP:       ; %bb.0: ; %entry
 867 ; GFX10-WGP-NEXT:    s_clause 0x1
 868 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
 869 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 870 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 871 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 872 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
 873 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
 874 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 875 ; GFX10-WGP-NEXT:    buffer_gl0_inv
 876 ; GFX10-WGP-NEXT:    buffer_gl1_inv
 877 ; GFX10-WGP-NEXT:    s_endpgm
 878 ;
 879 ; GFX10-CU-LABEL: global_system_acquire_atomicrmw:
 880 ; GFX10-CU:       ; %bb.0: ; %entry
 881 ; GFX10-CU-NEXT:    s_clause 0x1
 882 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
 883 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 884 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 885 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 886 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
 887 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
 888 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 889 ; GFX10-CU-NEXT:    buffer_gl0_inv
 890 ; GFX10-CU-NEXT:    buffer_gl1_inv
 891 ; GFX10-CU-NEXT:    s_endpgm
 892 ;
 893 ; SKIP-CACHE-INV-LABEL: global_system_acquire_atomicrmw:
 894 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 895 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 896 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
 897 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
 898 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
 899 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 900 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
 901 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
 902 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
 903 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 904 ;
 905 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_atomicrmw:
 906 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
 907 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 908 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
 909 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 910 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 911 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 912 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
 913 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 914 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
 915 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
 916 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
 917 ;
 918 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_atomicrmw:
 919 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
 920 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 921 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
 922 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
 923 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
 924 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
 925 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
 926 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
 927 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
 928 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
 929 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
 930     i32 addrspace(1)* %out, i32 %in) {
 931 entry:
 932   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acquire
 933   ret void
 934 }
 935
 936 define amdgpu_kernel void @global_system_release_atomicrmw(
 937 ; GFX6-LABEL: global_system_release_atomicrmw:
 938 ; GFX6:       ; %bb.0: ; %entry
 939 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 940 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
 941 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
 942 ; GFX6-NEXT:    s_mov_b32 s2, -1
 943 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 944 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 945 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 946 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
 947 ; GFX6-NEXT:    s_endpgm
 948 ;
 949 ; GFX7-LABEL: global_system_release_atomicrmw:
 950 ; GFX7:       ; %bb.0: ; %entry
 951 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 952 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
 953 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
 954 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
 955 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
 956 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
 957 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 958 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
 959 ; GFX7-NEXT:    s_endpgm
 960 ;
 961 ; GFX10-WGP-LABEL: global_system_release_atomicrmw:
 962 ; GFX10-WGP:       ; %bb.0: ; %entry
 963 ; GFX10-WGP-NEXT:    s_clause 0x1
 964 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
 965 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 966 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
 967 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 968 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
 969 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 970 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 971 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
 972 ; GFX10-WGP-NEXT:    s_endpgm
 973 ;
 974 ; GFX10-CU-LABEL: global_system_release_atomicrmw:
 975 ; GFX10-CU:       ; %bb.0: ; %entry
 976 ; GFX10-CU-NEXT:    s_clause 0x1
 977 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
 978 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
 979 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
 980 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
 981 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
 982 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 983 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 984 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
 985 ; GFX10-CU-NEXT:    s_endpgm
 986 ;
 987 ; SKIP-CACHE-INV-LABEL: global_system_release_atomicrmw:
 988 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
 989 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
 990 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
 991 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
 992 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
 993 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
 994 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
 995 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 996 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
 997 ; SKIP-CACHE-INV-NEXT:    s_endpgm
 998 ;
 999 ; GFX90A-NOTTGSPLIT-LABEL: global_system_release_atomicrmw:
1000 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1001 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1002 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1003 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1004 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1005 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1006 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1007 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1008 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1009 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1010 ;
1011 ; GFX90A-TGSPLIT-LABEL: global_system_release_atomicrmw:
1012 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1013 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1014 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1015 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1016 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1017 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1018 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1019 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1020 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1021 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1022     i32 addrspace(1)* %out, i32 %in) {
1023 entry:
1024   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in release
1025   ret void
1026 }
1027
1028 define amdgpu_kernel void @global_system_acq_rel_atomicrmw(
1029 ; GFX6-LABEL: global_system_acq_rel_atomicrmw:
1030 ; GFX6:       ; %bb.0: ; %entry
1031 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1032 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1033 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1034 ; GFX6-NEXT:    s_mov_b32 s2, -1
1035 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1036 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1037 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1038 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1039 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
1040 ; GFX6-NEXT:    buffer_wbinvl1
1041 ; GFX6-NEXT:    s_endpgm
1042 ;
1043 ; GFX7-LABEL: global_system_acq_rel_atomicrmw:
1044 ; GFX7:       ; %bb.0: ; %entry
1045 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1046 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1047 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1048 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1049 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1050 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1051 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1052 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1053 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
1054 ; GFX7-NEXT:    buffer_wbinvl1_vol
1055 ; GFX7-NEXT:    s_endpgm
1056 ;
1057 ; GFX10-WGP-LABEL: global_system_acq_rel_atomicrmw:
1058 ; GFX10-WGP:       ; %bb.0: ; %entry
1059 ; GFX10-WGP-NEXT:    s_clause 0x1
1060 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1061 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1062 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1063 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1064 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1065 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1066 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1067 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1068 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1069 ; GFX10-WGP-NEXT:    buffer_gl0_inv
1070 ; GFX10-WGP-NEXT:    buffer_gl1_inv
1071 ; GFX10-WGP-NEXT:    s_endpgm
1072 ;
1073 ; GFX10-CU-LABEL: global_system_acq_rel_atomicrmw:
1074 ; GFX10-CU:       ; %bb.0: ; %entry
1075 ; GFX10-CU-NEXT:    s_clause 0x1
1076 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1077 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1078 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1079 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1080 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1081 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1082 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1083 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1084 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1085 ; GFX10-CU-NEXT:    buffer_gl0_inv
1086 ; GFX10-CU-NEXT:    buffer_gl1_inv
1087 ; GFX10-CU-NEXT:    s_endpgm
1088 ;
1089 ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_atomicrmw:
1090 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1091 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1092 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1093 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1094 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1095 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1096 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1097 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1098 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
1099 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1100 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1101 ;
1102 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acq_rel_atomicrmw:
1103 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1104 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1105 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1106 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1107 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1108 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1109 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1110 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1111 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1112 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1113 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1114 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1115 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1116 ;
1117 ; GFX90A-TGSPLIT-LABEL: global_system_acq_rel_atomicrmw:
1118 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1119 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1120 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1121 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1122 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1123 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1124 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1125 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1126 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1127 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1128 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1129 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1130 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1131     i32 addrspace(1)* %out, i32 %in) {
1132 entry:
1133   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acq_rel
1134   ret void
1135 }
1136
1137 define amdgpu_kernel void @global_system_seq_cst_atomicrmw(
1138 ; GFX6-LABEL: global_system_seq_cst_atomicrmw:
1139 ; GFX6:       ; %bb.0: ; %entry
1140 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1141 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1142 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1143 ; GFX6-NEXT:    s_mov_b32 s2, -1
1144 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1145 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1146 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1147 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
1148 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
1149 ; GFX6-NEXT:    buffer_wbinvl1
1150 ; GFX6-NEXT:    s_endpgm
1151 ;
1152 ; GFX7-LABEL: global_system_seq_cst_atomicrmw:
1153 ; GFX7:       ; %bb.0: ; %entry
1154 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1155 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1156 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1157 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1158 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1159 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1160 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1161 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
1162 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
1163 ; GFX7-NEXT:    buffer_wbinvl1_vol
1164 ; GFX7-NEXT:    s_endpgm
1165 ;
1166 ; GFX10-WGP-LABEL: global_system_seq_cst_atomicrmw:
1167 ; GFX10-WGP:       ; %bb.0: ; %entry
1168 ; GFX10-WGP-NEXT:    s_clause 0x1
1169 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1170 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1171 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1172 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1173 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1174 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1175 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1176 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
1177 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1178 ; GFX10-WGP-NEXT:    buffer_gl0_inv
1179 ; GFX10-WGP-NEXT:    buffer_gl1_inv
1180 ; GFX10-WGP-NEXT:    s_endpgm
1181 ;
1182 ; GFX10-CU-LABEL: global_system_seq_cst_atomicrmw:
1183 ; GFX10-CU:       ; %bb.0: ; %entry
1184 ; GFX10-CU-NEXT:    s_clause 0x1
1185 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1186 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1187 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1188 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1189 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1190 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1191 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1192 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
1193 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1194 ; GFX10-CU-NEXT:    buffer_gl0_inv
1195 ; GFX10-CU-NEXT:    buffer_gl1_inv
1196 ; GFX10-CU-NEXT:    s_endpgm
1197 ;
1198 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_atomicrmw:
1199 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1200 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1201 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1202 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1203 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1204 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1205 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1206 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1207 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
1208 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1209 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1210 ;
1211 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_atomicrmw:
1212 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1213 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1214 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1215 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1216 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1217 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1218 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1219 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1220 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1221 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1222 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1223 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1224 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1225 ;
1226 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_atomicrmw:
1227 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1228 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1229 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1230 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1231 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1232 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1233 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1234 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1235 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
1236 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1237 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1238 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1239 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1240     i32 addrspace(1)* %out, i32 %in) {
1241 entry:
1242   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
1243   ret void
1244 }
1245
1246 define amdgpu_kernel void @global_system_acquire_ret_atomicrmw(
1247 ; GFX6-LABEL: global_system_acquire_ret_atomicrmw:
1248 ; GFX6:       ; %bb.0: ; %entry
1249 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1250 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1251 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1252 ; GFX6-NEXT:    s_mov_b32 s2, -1
1253 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1254 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1255 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1256 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
1257 ; GFX6-NEXT:    buffer_wbinvl1
1258 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1259 ; GFX6-NEXT:    s_endpgm
1260 ;
1261 ; GFX7-LABEL: global_system_acquire_ret_atomicrmw:
1262 ; GFX7:       ; %bb.0: ; %entry
1263 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1264 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1265 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1266 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1267 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1268 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1269 ; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1270 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
1271 ; GFX7-NEXT:    buffer_wbinvl1_vol
1272 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
1273 ; GFX7-NEXT:    s_endpgm
1274 ;
1275 ; GFX10-WGP-LABEL: global_system_acquire_ret_atomicrmw:
1276 ; GFX10-WGP:       ; %bb.0: ; %entry
1277 ; GFX10-WGP-NEXT:    s_clause 0x1
1278 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1279 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1280 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1281 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1282 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1283 ; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1284 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1285 ; GFX10-WGP-NEXT:    buffer_gl0_inv
1286 ; GFX10-WGP-NEXT:    buffer_gl1_inv
1287 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1288 ; GFX10-WGP-NEXT:    s_endpgm
1289 ;
1290 ; GFX10-CU-LABEL: global_system_acquire_ret_atomicrmw:
1291 ; GFX10-CU:       ; %bb.0: ; %entry
1292 ; GFX10-CU-NEXT:    s_clause 0x1
1293 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1294 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1295 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1296 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1297 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1298 ; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1299 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1300 ; GFX10-CU-NEXT:    buffer_gl0_inv
1301 ; GFX10-CU-NEXT:    buffer_gl1_inv
1302 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1303 ; GFX10-CU-NEXT:    s_endpgm
1304 ;
1305 ; SKIP-CACHE-INV-LABEL: global_system_acquire_ret_atomicrmw:
1306 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1307 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1308 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1309 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1310 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1311 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1312 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1313 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
1314 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1315 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1316 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1317 ;
1318 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_ret_atomicrmw:
1319 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1320 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1321 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1322 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1323 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1324 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1325 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1326 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1327 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1328 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1329 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1330 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1331 ;
1332 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_ret_atomicrmw:
1333 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1334 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1335 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1336 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1337 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1338 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1339 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1340 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1341 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1342 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1343 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1344 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1345     i32 addrspace(1)* %out, i32 %in) {
1346 entry:
1347   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acquire
1348   store i32 %val, i32 addrspace(1)* %out, align 4
1349   ret void
1350 }
1351
1352 define amdgpu_kernel void @global_system_acq_rel_ret_atomicrmw(
1353 ; GFX6-LABEL: global_system_acq_rel_ret_atomicrmw:
1354 ; GFX6:       ; %bb.0: ; %entry
1355 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1356 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1357 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1358 ; GFX6-NEXT:    s_mov_b32 s2, -1
1359 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1360 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1361 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1362 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1363 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
1364 ; GFX6-NEXT:    buffer_wbinvl1
1365 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1366 ; GFX6-NEXT:    s_endpgm
1367 ;
1368 ; GFX7-LABEL: global_system_acq_rel_ret_atomicrmw:
1369 ; GFX7:       ; %bb.0: ; %entry
1370 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1371 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1372 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1373 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1374 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1375 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1376 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1377 ; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1378 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
1379 ; GFX7-NEXT:    buffer_wbinvl1_vol
1380 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
1381 ; GFX7-NEXT:    s_endpgm
1382 ;
1383 ; GFX10-WGP-LABEL: global_system_acq_rel_ret_atomicrmw:
1384 ; GFX10-WGP:       ; %bb.0: ; %entry
1385 ; GFX10-WGP-NEXT:    s_clause 0x1
1386 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1387 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1388 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1389 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1390 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1391 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1392 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1393 ; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1394 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1395 ; GFX10-WGP-NEXT:    buffer_gl0_inv
1396 ; GFX10-WGP-NEXT:    buffer_gl1_inv
1397 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1398 ; GFX10-WGP-NEXT:    s_endpgm
1399 ;
1400 ; GFX10-CU-LABEL: global_system_acq_rel_ret_atomicrmw:
1401 ; GFX10-CU:       ; %bb.0: ; %entry
1402 ; GFX10-CU-NEXT:    s_clause 0x1
1403 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1404 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1405 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1406 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1407 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1408 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1409 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1410 ; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1411 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1412 ; GFX10-CU-NEXT:    buffer_gl0_inv
1413 ; GFX10-CU-NEXT:    buffer_gl1_inv
1414 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1415 ; GFX10-CU-NEXT:    s_endpgm
1416 ;
1417 ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_ret_atomicrmw:
1418 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1419 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1420 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1421 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1422 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1423 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1424 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1425 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1426 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
1427 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1428 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1429 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1430 ;
1431 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acq_rel_ret_atomicrmw:
1432 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1433 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1434 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1435 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1436 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1437 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1438 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1439 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1440 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1441 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1442 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1443 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1444 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1445 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1446 ;
1447 ; GFX90A-TGSPLIT-LABEL: global_system_acq_rel_ret_atomicrmw:
1448 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1449 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1450 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1451 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1452 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1453 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1454 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1455 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1456 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1457 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1458 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1459 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1460 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1461 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1462     i32 addrspace(1)* %out, i32 %in) {
1463 entry:
1464   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in acq_rel
1465   store i32 %val, i32 addrspace(1)* %out, align 4
1466   ret void
1467 }
1468
1469 define amdgpu_kernel void @global_system_seq_cst_ret_atomicrmw(
1470 ; GFX6-LABEL: global_system_seq_cst_ret_atomicrmw:
1471 ; GFX6:       ; %bb.0: ; %entry
1472 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1473 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
1474 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1475 ; GFX6-NEXT:    s_mov_b32 s2, -1
1476 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1477 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1478 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1479 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
1480 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
1481 ; GFX6-NEXT:    buffer_wbinvl1
1482 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
1483 ; GFX6-NEXT:    s_endpgm
1484 ;
1485 ; GFX7-LABEL: global_system_seq_cst_ret_atomicrmw:
1486 ; GFX7:       ; %bb.0: ; %entry
1487 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1488 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
1489 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1490 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1491 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1492 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1493 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1494 ; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
1495 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
1496 ; GFX7-NEXT:    buffer_wbinvl1_vol
1497 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
1498 ; GFX7-NEXT:    s_endpgm
1499 ;
1500 ; GFX10-WGP-LABEL: global_system_seq_cst_ret_atomicrmw:
1501 ; GFX10-WGP:       ; %bb.0: ; %entry
1502 ; GFX10-WGP-NEXT:    s_clause 0x1
1503 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
1504 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1505 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
1506 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1507 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
1508 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1509 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1510 ; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1511 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
1512 ; GFX10-WGP-NEXT:    buffer_gl0_inv
1513 ; GFX10-WGP-NEXT:    buffer_gl1_inv
1514 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
1515 ; GFX10-WGP-NEXT:    s_endpgm
1516 ;
1517 ; GFX10-CU-LABEL: global_system_seq_cst_ret_atomicrmw:
1518 ; GFX10-CU:       ; %bb.0: ; %entry
1519 ; GFX10-CU-NEXT:    s_clause 0x1
1520 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
1521 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1522 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
1523 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1524 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
1525 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1526 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1527 ; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1528 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
1529 ; GFX10-CU-NEXT:    buffer_gl0_inv
1530 ; GFX10-CU-NEXT:    buffer_gl1_inv
1531 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
1532 ; GFX10-CU-NEXT:    s_endpgm
1533 ;
1534 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_ret_atomicrmw:
1535 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1536 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1537 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
1538 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1539 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1540 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1541 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1542 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1543 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
1544 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1545 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
1546 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1547 ;
1548 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_ret_atomicrmw:
1549 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1550 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1551 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1552 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1553 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1554 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1555 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1556 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1557 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1558 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1559 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1560 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1561 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1562 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1563 ;
1564 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_ret_atomicrmw:
1565 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1566 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1567 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
1568 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
1569 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1570 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
1571 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1572 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1573 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
1574 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1575 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1576 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1577 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
1578 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1579     i32 addrspace(1)* %out, i32 %in) {
1580 entry:
1581   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in seq_cst
1582   store i32 %val, i32 addrspace(1)* %out, align 4
1583   ret void
1584 }
1585
1586 define amdgpu_kernel void @global_system_monotonic_monotonic_cmpxchg(
1587 ; GFX6-LABEL: global_system_monotonic_monotonic_cmpxchg:
1588 ; GFX6:       ; %bb.0: ; %entry
1589 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1590 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1591 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1592 ; GFX6-NEXT:    s_mov_b32 s2, -1
1593 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1594 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1595 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1596 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1597 ; GFX6-NEXT:    s_endpgm
1598 ;
1599 ; GFX7-LABEL: global_system_monotonic_monotonic_cmpxchg:
1600 ; GFX7:       ; %bb.0: ; %entry
1601 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1602 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1603 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1604 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
1605 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1606 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1607 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1608 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1609 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1610 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1611 ; GFX7-NEXT:    s_endpgm
1612 ;
1613 ; GFX10-WGP-LABEL: global_system_monotonic_monotonic_cmpxchg:
1614 ; GFX10-WGP:       ; %bb.0: ; %entry
1615 ; GFX10-WGP-NEXT:    s_clause 0x1
1616 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1617 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1618 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1619 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1620 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1621 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1622 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1623 ; GFX10-WGP-NEXT:    s_endpgm
1624 ;
1625 ; GFX10-CU-LABEL: global_system_monotonic_monotonic_cmpxchg:
1626 ; GFX10-CU:       ; %bb.0: ; %entry
1627 ; GFX10-CU-NEXT:    s_clause 0x1
1628 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1629 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1630 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1631 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1632 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1633 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1634 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1635 ; GFX10-CU-NEXT:    s_endpgm
1636 ;
1637 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_monotonic_cmpxchg:
1638 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1639 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1640 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1641 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1642 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1643 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1644 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1645 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1646 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1647 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1648 ;
1649 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_monotonic_cmpxchg:
1650 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1651 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1652 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1653 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1654 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1655 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1656 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1657 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1658 ;
1659 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_monotonic_cmpxchg:
1660 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1661 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1662 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1663 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1664 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1665 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1666 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1667 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1668     i32 addrspace(1)* %out, i32 %in, i32 %old) {
1669 entry:
1670   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1671   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in monotonic monotonic
1672   ret void
1673 }
1674
1675 define amdgpu_kernel void @global_system_acquire_monotonic_cmpxchg(
1676 ; GFX6-LABEL: global_system_acquire_monotonic_cmpxchg:
1677 ; GFX6:       ; %bb.0: ; %entry
1678 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1679 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1680 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1681 ; GFX6-NEXT:    s_mov_b32 s2, -1
1682 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1683 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1684 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1685 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1686 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
1687 ; GFX6-NEXT:    buffer_wbinvl1
1688 ; GFX6-NEXT:    s_endpgm
1689 ;
1690 ; GFX7-LABEL: global_system_acquire_monotonic_cmpxchg:
1691 ; GFX7:       ; %bb.0: ; %entry
1692 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1693 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1694 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1695 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
1696 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1697 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1698 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1699 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1700 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1701 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1702 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
1703 ; GFX7-NEXT:    buffer_wbinvl1_vol
1704 ; GFX7-NEXT:    s_endpgm
1705 ;
1706 ; GFX10-WGP-LABEL: global_system_acquire_monotonic_cmpxchg:
1707 ; GFX10-WGP:       ; %bb.0: ; %entry
1708 ; GFX10-WGP-NEXT:    s_clause 0x1
1709 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1710 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1711 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1712 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1713 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1714 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1715 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1716 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1717 ; GFX10-WGP-NEXT:    buffer_gl0_inv
1718 ; GFX10-WGP-NEXT:    buffer_gl1_inv
1719 ; GFX10-WGP-NEXT:    s_endpgm
1720 ;
1721 ; GFX10-CU-LABEL: global_system_acquire_monotonic_cmpxchg:
1722 ; GFX10-CU:       ; %bb.0: ; %entry
1723 ; GFX10-CU-NEXT:    s_clause 0x1
1724 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1725 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1726 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1727 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1728 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1729 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1730 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1731 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1732 ; GFX10-CU-NEXT:    buffer_gl0_inv
1733 ; GFX10-CU-NEXT:    buffer_gl1_inv
1734 ; GFX10-CU-NEXT:    s_endpgm
1735 ;
1736 ; SKIP-CACHE-INV-LABEL: global_system_acquire_monotonic_cmpxchg:
1737 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1738 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1739 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1740 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1741 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1742 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1743 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1744 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1745 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1746 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1747 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1748 ;
1749 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_monotonic_cmpxchg:
1750 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1751 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1752 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1753 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1754 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1755 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1756 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1757 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1758 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1759 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1760 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1761 ;
1762 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_monotonic_cmpxchg:
1763 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1764 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1765 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1766 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1767 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1768 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1769 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1770 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1771 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1772 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1773 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1774     i32 addrspace(1)* %out, i32 %in, i32 %old) {
1775 entry:
1776   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1777   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire monotonic
1778   ret void
1779 }
1780
1781 define amdgpu_kernel void @global_system_release_monotonic_cmpxchg(
1782 ; GFX6-LABEL: global_system_release_monotonic_cmpxchg:
1783 ; GFX6:       ; %bb.0: ; %entry
1784 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1785 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1786 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1787 ; GFX6-NEXT:    s_mov_b32 s2, -1
1788 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1789 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1790 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1791 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1792 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1793 ; GFX6-NEXT:    s_endpgm
1794 ;
1795 ; GFX7-LABEL: global_system_release_monotonic_cmpxchg:
1796 ; GFX7:       ; %bb.0: ; %entry
1797 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1798 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1799 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1800 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
1801 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1802 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1803 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1804 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1805 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1806 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1807 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1808 ; GFX7-NEXT:    s_endpgm
1809 ;
1810 ; GFX10-WGP-LABEL: global_system_release_monotonic_cmpxchg:
1811 ; GFX10-WGP:       ; %bb.0: ; %entry
1812 ; GFX10-WGP-NEXT:    s_clause 0x1
1813 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1814 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1815 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1816 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1817 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1818 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1819 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1820 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1821 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1822 ; GFX10-WGP-NEXT:    s_endpgm
1823 ;
1824 ; GFX10-CU-LABEL: global_system_release_monotonic_cmpxchg:
1825 ; GFX10-CU:       ; %bb.0: ; %entry
1826 ; GFX10-CU-NEXT:    s_clause 0x1
1827 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1828 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1829 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1830 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1831 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1832 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1833 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1834 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1835 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1836 ; GFX10-CU-NEXT:    s_endpgm
1837 ;
1838 ; SKIP-CACHE-INV-LABEL: global_system_release_monotonic_cmpxchg:
1839 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1840 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1841 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1842 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1843 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1844 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1845 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1846 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1847 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1848 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1849 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1850 ;
1851 ; GFX90A-NOTTGSPLIT-LABEL: global_system_release_monotonic_cmpxchg:
1852 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1853 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1854 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1855 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1856 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1857 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1858 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1859 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1860 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1861 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1862 ;
1863 ; GFX90A-TGSPLIT-LABEL: global_system_release_monotonic_cmpxchg:
1864 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1865 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1866 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1867 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1868 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1869 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1870 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1871 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1872 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1873 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1874     i32 addrspace(1)* %out, i32 %in, i32 %old) {
1875 entry:
1876   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1877   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in release monotonic
1878   ret void
1879 }
1880
1881 define amdgpu_kernel void @global_system_acq_rel_monotonic_cmpxchg(
1882 ; GFX6-LABEL: global_system_acq_rel_monotonic_cmpxchg:
1883 ; GFX6:       ; %bb.0: ; %entry
1884 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1885 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
1886 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
1887 ; GFX6-NEXT:    s_mov_b32 s2, -1
1888 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
1889 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
1890 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
1891 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1892 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
1893 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
1894 ; GFX6-NEXT:    buffer_wbinvl1
1895 ; GFX6-NEXT:    s_endpgm
1896 ;
1897 ; GFX7-LABEL: global_system_acq_rel_monotonic_cmpxchg:
1898 ; GFX7:       ; %bb.0: ; %entry
1899 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1900 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
1901 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
1902 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
1903 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
1904 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
1905 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
1906 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
1907 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
1908 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1909 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
1910 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
1911 ; GFX7-NEXT:    buffer_wbinvl1_vol
1912 ; GFX7-NEXT:    s_endpgm
1913 ;
1914 ; GFX10-WGP-LABEL: global_system_acq_rel_monotonic_cmpxchg:
1915 ; GFX10-WGP:       ; %bb.0: ; %entry
1916 ; GFX10-WGP-NEXT:    s_clause 0x1
1917 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1918 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1919 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
1920 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
1921 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
1922 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
1923 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1924 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1925 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1926 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
1927 ; GFX10-WGP-NEXT:    buffer_gl0_inv
1928 ; GFX10-WGP-NEXT:    buffer_gl1_inv
1929 ; GFX10-WGP-NEXT:    s_endpgm
1930 ;
1931 ; GFX10-CU-LABEL: global_system_acq_rel_monotonic_cmpxchg:
1932 ; GFX10-CU:       ; %bb.0: ; %entry
1933 ; GFX10-CU-NEXT:    s_clause 0x1
1934 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
1935 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
1936 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
1937 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
1938 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
1939 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
1940 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1941 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1942 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
1943 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
1944 ; GFX10-CU-NEXT:    buffer_gl0_inv
1945 ; GFX10-CU-NEXT:    buffer_gl1_inv
1946 ; GFX10-CU-NEXT:    s_endpgm
1947 ;
1948 ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_monotonic_cmpxchg:
1949 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
1950 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
1951 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
1952 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
1953 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
1954 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
1955 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
1956 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
1957 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1958 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
1959 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
1960 ; SKIP-CACHE-INV-NEXT:    s_endpgm
1961 ;
1962 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acq_rel_monotonic_cmpxchg:
1963 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
1964 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1965 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1966 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1967 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1968 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1969 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
1970 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1971 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1972 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1973 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
1974 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
1975 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
1976 ;
1977 ; GFX90A-TGSPLIT-LABEL: global_system_acq_rel_monotonic_cmpxchg:
1978 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
1979 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
1980 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
1981 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
1982 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
1983 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
1984 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
1985 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
1986 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
1987 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
1988 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
1989 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
1990 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
1991     i32 addrspace(1)* %out, i32 %in, i32 %old) {
1992 entry:
1993   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
1994   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel monotonic
1995   ret void
1996 }
1997
1998 define amdgpu_kernel void @global_system_seq_cst_monotonic_cmpxchg(
1999 ; GFX6-LABEL: global_system_seq_cst_monotonic_cmpxchg:
2000 ; GFX6:       ; %bb.0: ; %entry
2001 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2002 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2003 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2004 ; GFX6-NEXT:    s_mov_b32 s2, -1
2005 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2006 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2007 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2008 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2009 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2010 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2011 ; GFX6-NEXT:    buffer_wbinvl1
2012 ; GFX6-NEXT:    s_endpgm
2013 ;
2014 ; GFX7-LABEL: global_system_seq_cst_monotonic_cmpxchg:
2015 ; GFX7:       ; %bb.0: ; %entry
2016 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2017 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2018 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2019 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
2020 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2021 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2022 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2023 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2024 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2025 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2026 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2027 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2028 ; GFX7-NEXT:    buffer_wbinvl1_vol
2029 ; GFX7-NEXT:    s_endpgm
2030 ;
2031 ; GFX10-WGP-LABEL: global_system_seq_cst_monotonic_cmpxchg:
2032 ; GFX10-WGP:       ; %bb.0: ; %entry
2033 ; GFX10-WGP-NEXT:    s_clause 0x1
2034 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2035 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2036 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2037 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2038 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2039 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2040 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2041 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2042 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2043 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2044 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2045 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2046 ; GFX10-WGP-NEXT:    s_endpgm
2047 ;
2048 ; GFX10-CU-LABEL: global_system_seq_cst_monotonic_cmpxchg:
2049 ; GFX10-CU:       ; %bb.0: ; %entry
2050 ; GFX10-CU-NEXT:    s_clause 0x1
2051 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2052 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2053 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2054 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2055 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2056 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2057 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2058 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2059 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2060 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2061 ; GFX10-CU-NEXT:    buffer_gl0_inv
2062 ; GFX10-CU-NEXT:    buffer_gl1_inv
2063 ; GFX10-CU-NEXT:    s_endpgm
2064 ;
2065 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_monotonic_cmpxchg:
2066 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2067 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2068 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2069 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2070 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2071 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2072 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2073 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2074 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2075 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2076 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2077 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2078 ;
2079 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_monotonic_cmpxchg:
2080 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2081 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2082 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2083 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2084 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2085 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2086 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2087 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2088 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2089 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2090 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2091 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2092 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2093 ;
2094 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_monotonic_cmpxchg:
2095 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2096 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2097 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2098 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2099 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2100 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2101 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2102 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2103 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2104 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2105 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2106 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2107 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2108     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2109 entry:
2110   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2111   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst monotonic
2112   ret void
2113 }
2114
2115 define amdgpu_kernel void @global_system_monotonic_acquire_cmpxchg(
2116 ; GFX6-LABEL: global_system_monotonic_acquire_cmpxchg:
2117 ; GFX6:       ; %bb.0: ; %entry
2118 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2119 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2120 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2121 ; GFX6-NEXT:    s_mov_b32 s2, -1
2122 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2123 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2124 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2125 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2126 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2127 ; GFX6-NEXT:    buffer_wbinvl1
2128 ; GFX6-NEXT:    s_endpgm
2129 ;
2130 ; GFX7-LABEL: global_system_monotonic_acquire_cmpxchg:
2131 ; GFX7:       ; %bb.0: ; %entry
2132 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2133 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2134 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2135 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
2136 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2137 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2138 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2139 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2140 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2141 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2142 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2143 ; GFX7-NEXT:    buffer_wbinvl1_vol
2144 ; GFX7-NEXT:    s_endpgm
2145 ;
2146 ; GFX10-WGP-LABEL: global_system_monotonic_acquire_cmpxchg:
2147 ; GFX10-WGP:       ; %bb.0: ; %entry
2148 ; GFX10-WGP-NEXT:    s_clause 0x1
2149 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2150 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2151 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2152 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2153 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2154 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2155 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2156 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2157 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2158 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2159 ; GFX10-WGP-NEXT:    s_endpgm
2160 ;
2161 ; GFX10-CU-LABEL: global_system_monotonic_acquire_cmpxchg:
2162 ; GFX10-CU:       ; %bb.0: ; %entry
2163 ; GFX10-CU-NEXT:    s_clause 0x1
2164 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2165 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2166 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2167 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2168 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2169 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2170 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2171 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2172 ; GFX10-CU-NEXT:    buffer_gl0_inv
2173 ; GFX10-CU-NEXT:    buffer_gl1_inv
2174 ; GFX10-CU-NEXT:    s_endpgm
2175 ;
2176 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_acquire_cmpxchg:
2177 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2178 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2179 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2180 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2181 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2182 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2183 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2184 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2185 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2186 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2187 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2188 ;
2189 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_acquire_cmpxchg:
2190 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2191 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2192 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2193 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2194 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2195 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2196 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2197 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2198 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2199 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2200 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2201 ;
2202 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_acquire_cmpxchg:
2203 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2204 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2205 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2206 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2207 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2208 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2209 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2210 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2211 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2212 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2213 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2214     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2215 entry:
2216   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2217   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in monotonic acquire
2218   ret void
2219 }
2220
2221 define amdgpu_kernel void @global_system_acquire_acquire_cmpxchg(
2222 ; GFX6-LABEL: global_system_acquire_acquire_cmpxchg:
2223 ; GFX6:       ; %bb.0: ; %entry
2224 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2225 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2226 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2227 ; GFX6-NEXT:    s_mov_b32 s2, -1
2228 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2229 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2230 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2231 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2232 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2233 ; GFX6-NEXT:    buffer_wbinvl1
2234 ; GFX6-NEXT:    s_endpgm
2235 ;
2236 ; GFX7-LABEL: global_system_acquire_acquire_cmpxchg:
2237 ; GFX7:       ; %bb.0: ; %entry
2238 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2239 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2240 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2241 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
2242 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2243 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2244 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2245 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2246 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2247 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2248 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2249 ; GFX7-NEXT:    buffer_wbinvl1_vol
2250 ; GFX7-NEXT:    s_endpgm
2251 ;
2252 ; GFX10-WGP-LABEL: global_system_acquire_acquire_cmpxchg:
2253 ; GFX10-WGP:       ; %bb.0: ; %entry
2254 ; GFX10-WGP-NEXT:    s_clause 0x1
2255 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2256 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2257 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2258 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2259 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2260 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2261 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2262 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2263 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2264 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2265 ; GFX10-WGP-NEXT:    s_endpgm
2266 ;
2267 ; GFX10-CU-LABEL: global_system_acquire_acquire_cmpxchg:
2268 ; GFX10-CU:       ; %bb.0: ; %entry
2269 ; GFX10-CU-NEXT:    s_clause 0x1
2270 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2271 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2272 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2273 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2274 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2275 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2276 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2277 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2278 ; GFX10-CU-NEXT:    buffer_gl0_inv
2279 ; GFX10-CU-NEXT:    buffer_gl1_inv
2280 ; GFX10-CU-NEXT:    s_endpgm
2281 ;
2282 ; SKIP-CACHE-INV-LABEL: global_system_acquire_acquire_cmpxchg:
2283 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2284 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2285 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2286 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2287 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2288 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2289 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2290 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2291 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2292 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2293 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2294 ;
2295 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_acquire_cmpxchg:
2296 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2297 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2298 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2299 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2300 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2301 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2302 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2303 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2304 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2305 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2306 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2307 ;
2308 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_acquire_cmpxchg:
2309 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2310 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2311 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2312 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2313 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2314 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2315 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2316 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2317 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2318 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2319 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2320     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2321 entry:
2322   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2323   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire acquire
2324   ret void
2325 }
2326
2327 define amdgpu_kernel void @global_system_release_acquire_cmpxchg(
2328 ; GFX6-LABEL: global_system_release_acquire_cmpxchg:
2329 ; GFX6:       ; %bb.0: ; %entry
2330 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2331 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2332 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2333 ; GFX6-NEXT:    s_mov_b32 s2, -1
2334 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2335 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2336 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2337 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2338 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2339 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2340 ; GFX6-NEXT:    buffer_wbinvl1
2341 ; GFX6-NEXT:    s_endpgm
2342 ;
2343 ; GFX7-LABEL: global_system_release_acquire_cmpxchg:
2344 ; GFX7:       ; %bb.0: ; %entry
2345 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2346 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2347 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2348 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
2349 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2350 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2351 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2352 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2353 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2354 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2355 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2356 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2357 ; GFX7-NEXT:    buffer_wbinvl1_vol
2358 ; GFX7-NEXT:    s_endpgm
2359 ;
2360 ; GFX10-WGP-LABEL: global_system_release_acquire_cmpxchg:
2361 ; GFX10-WGP:       ; %bb.0: ; %entry
2362 ; GFX10-WGP-NEXT:    s_clause 0x1
2363 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2364 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2365 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2366 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2367 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2368 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2369 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2370 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2371 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2372 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2373 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2374 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2375 ; GFX10-WGP-NEXT:    s_endpgm
2376 ;
2377 ; GFX10-CU-LABEL: global_system_release_acquire_cmpxchg:
2378 ; GFX10-CU:       ; %bb.0: ; %entry
2379 ; GFX10-CU-NEXT:    s_clause 0x1
2380 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2381 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2382 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2383 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2384 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2385 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2386 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2387 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2388 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2389 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2390 ; GFX10-CU-NEXT:    buffer_gl0_inv
2391 ; GFX10-CU-NEXT:    buffer_gl1_inv
2392 ; GFX10-CU-NEXT:    s_endpgm
2393 ;
2394 ; SKIP-CACHE-INV-LABEL: global_system_release_acquire_cmpxchg:
2395 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2396 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2397 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2398 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2399 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2400 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2401 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2402 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2403 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2404 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2405 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2406 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2407 ;
2408 ; GFX90A-NOTTGSPLIT-LABEL: global_system_release_acquire_cmpxchg:
2409 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2410 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2411 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2412 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2413 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2414 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2415 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2416 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2417 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2418 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2419 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2420 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2421 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2422 ;
2423 ; GFX90A-TGSPLIT-LABEL: global_system_release_acquire_cmpxchg:
2424 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2425 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2426 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2427 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2428 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2429 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2430 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2431 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2432 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2433 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2434 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2435 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2436 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2437     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2438 entry:
2439   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2440   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in release acquire
2441   ret void
2442 }
2443
2444 define amdgpu_kernel void @global_system_acq_rel_acquire_cmpxchg(
2445 ; GFX6-LABEL: global_system_acq_rel_acquire_cmpxchg:
2446 ; GFX6:       ; %bb.0: ; %entry
2447 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2448 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2449 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2450 ; GFX6-NEXT:    s_mov_b32 s2, -1
2451 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2452 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2453 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2454 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2455 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2456 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2457 ; GFX6-NEXT:    buffer_wbinvl1
2458 ; GFX6-NEXT:    s_endpgm
2459 ;
2460 ; GFX7-LABEL: global_system_acq_rel_acquire_cmpxchg:
2461 ; GFX7:       ; %bb.0: ; %entry
2462 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2463 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2464 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2465 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
2466 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2467 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2468 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2469 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2470 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2471 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2472 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2473 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2474 ; GFX7-NEXT:    buffer_wbinvl1_vol
2475 ; GFX7-NEXT:    s_endpgm
2476 ;
2477 ; GFX10-WGP-LABEL: global_system_acq_rel_acquire_cmpxchg:
2478 ; GFX10-WGP:       ; %bb.0: ; %entry
2479 ; GFX10-WGP-NEXT:    s_clause 0x1
2480 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2481 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2482 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2483 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2484 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2485 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2486 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2487 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2488 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2489 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2490 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2491 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2492 ; GFX10-WGP-NEXT:    s_endpgm
2493 ;
2494 ; GFX10-CU-LABEL: global_system_acq_rel_acquire_cmpxchg:
2495 ; GFX10-CU:       ; %bb.0: ; %entry
2496 ; GFX10-CU-NEXT:    s_clause 0x1
2497 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2498 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2499 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2500 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2501 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2502 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2503 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2504 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2505 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2506 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2507 ; GFX10-CU-NEXT:    buffer_gl0_inv
2508 ; GFX10-CU-NEXT:    buffer_gl1_inv
2509 ; GFX10-CU-NEXT:    s_endpgm
2510 ;
2511 ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_acquire_cmpxchg:
2512 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2513 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2514 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2515 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2516 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2517 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2518 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2519 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2520 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2521 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2522 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2523 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2524 ;
2525 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acq_rel_acquire_cmpxchg:
2526 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2527 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2528 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2529 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2530 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2531 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2532 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2533 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2534 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2535 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2536 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2537 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2538 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2539 ;
2540 ; GFX90A-TGSPLIT-LABEL: global_system_acq_rel_acquire_cmpxchg:
2541 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2542 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2543 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2544 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2545 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2546 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2547 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2548 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2549 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2550 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2551 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2552 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2553 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2554     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2555 entry:
2556   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2557   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel acquire
2558   ret void
2559 }
2560
2561 define amdgpu_kernel void @global_system_seq_cst_acquire_cmpxchg(
2562 ; GFX6-LABEL: global_system_seq_cst_acquire_cmpxchg:
2563 ; GFX6:       ; %bb.0: ; %entry
2564 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2565 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2566 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2567 ; GFX6-NEXT:    s_mov_b32 s2, -1
2568 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2569 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2570 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2571 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2572 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2573 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2574 ; GFX6-NEXT:    buffer_wbinvl1
2575 ; GFX6-NEXT:    s_endpgm
2576 ;
2577 ; GFX7-LABEL: global_system_seq_cst_acquire_cmpxchg:
2578 ; GFX7:       ; %bb.0: ; %entry
2579 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2580 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2581 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2582 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
2583 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2584 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2585 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2586 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2587 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2588 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2589 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2590 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2591 ; GFX7-NEXT:    buffer_wbinvl1_vol
2592 ; GFX7-NEXT:    s_endpgm
2593 ;
2594 ; GFX10-WGP-LABEL: global_system_seq_cst_acquire_cmpxchg:
2595 ; GFX10-WGP:       ; %bb.0: ; %entry
2596 ; GFX10-WGP-NEXT:    s_clause 0x1
2597 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2598 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2599 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2600 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2601 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2602 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2603 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2604 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2605 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2606 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2607 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2608 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2609 ; GFX10-WGP-NEXT:    s_endpgm
2610 ;
2611 ; GFX10-CU-LABEL: global_system_seq_cst_acquire_cmpxchg:
2612 ; GFX10-CU:       ; %bb.0: ; %entry
2613 ; GFX10-CU-NEXT:    s_clause 0x1
2614 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2615 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2616 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2617 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2618 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2619 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2620 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2621 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2622 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2623 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2624 ; GFX10-CU-NEXT:    buffer_gl0_inv
2625 ; GFX10-CU-NEXT:    buffer_gl1_inv
2626 ; GFX10-CU-NEXT:    s_endpgm
2627 ;
2628 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_acquire_cmpxchg:
2629 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2630 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2631 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2632 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2633 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2634 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2635 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2636 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2637 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2638 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2639 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2640 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2641 ;
2642 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_acquire_cmpxchg:
2643 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2644 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2645 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2646 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2647 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2648 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2649 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2650 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2651 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2652 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2653 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2654 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2655 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2656 ;
2657 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_acquire_cmpxchg:
2658 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2659 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2660 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2661 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2662 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2663 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2664 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2665 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2666 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2667 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2668 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2669 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2670 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2671     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2672 entry:
2673   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2674   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst acquire
2675   ret void
2676 }
2677
2678 define amdgpu_kernel void @global_system_seq_cst_seq_cst_cmpxchg(
2679 ; GFX6-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
2680 ; GFX6:       ; %bb.0: ; %entry
2681 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2682 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2683 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2684 ; GFX6-NEXT:    s_mov_b32 s2, -1
2685 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2686 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2687 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2688 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2689 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
2690 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2691 ; GFX6-NEXT:    buffer_wbinvl1
2692 ; GFX6-NEXT:    s_endpgm
2693 ;
2694 ; GFX7-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
2695 ; GFX7:       ; %bb.0: ; %entry
2696 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2697 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2698 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2699 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
2700 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
2701 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2702 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2703 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2704 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2705 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2706 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
2707 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2708 ; GFX7-NEXT:    buffer_wbinvl1_vol
2709 ; GFX7-NEXT:    s_endpgm
2710 ;
2711 ; GFX10-WGP-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
2712 ; GFX10-WGP:       ; %bb.0: ; %entry
2713 ; GFX10-WGP-NEXT:    s_clause 0x1
2714 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2715 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2716 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2717 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2718 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2719 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2720 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2721 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2722 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2723 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
2724 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2725 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2726 ; GFX10-WGP-NEXT:    s_endpgm
2727 ;
2728 ; GFX10-CU-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
2729 ; GFX10-CU:       ; %bb.0: ; %entry
2730 ; GFX10-CU-NEXT:    s_clause 0x1
2731 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2732 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2733 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2734 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2735 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2736 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2737 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2738 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2739 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
2740 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
2741 ; GFX10-CU-NEXT:    buffer_gl0_inv
2742 ; GFX10-CU-NEXT:    buffer_gl1_inv
2743 ; GFX10-CU-NEXT:    s_endpgm
2744 ;
2745 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
2746 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2747 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2748 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2749 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2750 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2751 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2752 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2753 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2754 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2755 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
2756 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2757 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2758 ;
2759 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
2760 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2761 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2762 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2763 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2764 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2765 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2766 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
2767 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2768 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2769 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2770 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2771 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2772 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2773 ;
2774 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_seq_cst_cmpxchg:
2775 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2776 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2777 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2778 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2779 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2780 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2781 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
2782 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
2783 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
2784 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2785 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
2786 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
2787 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2788     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2789 entry:
2790   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2791   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
2792   ret void
2793 }
2794
2795 define amdgpu_kernel void @global_system_monotonic_monotonic_ret_cmpxchg(
2796 ; GFX6-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
2797 ; GFX6:       ; %bb.0: ; %entry
2798 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2799 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2800 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2801 ; GFX6-NEXT:    s_mov_b32 s2, -1
2802 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2803 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2804 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2805 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2806 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2807 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2808 ; GFX6-NEXT:    s_endpgm
2809 ;
2810 ; GFX7-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
2811 ; GFX7:       ; %bb.0: ; %entry
2812 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2813 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2814 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2815 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
2816 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2817 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2818 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2819 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2820 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2821 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2822 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2823 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2824 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2825 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
2826 ; GFX7-NEXT:    s_endpgm
2827 ;
2828 ; GFX10-WGP-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
2829 ; GFX10-WGP:       ; %bb.0: ; %entry
2830 ; GFX10-WGP-NEXT:    s_clause 0x1
2831 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2832 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2833 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2834 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2835 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2836 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2837 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2838 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2839 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
2840 ; GFX10-WGP-NEXT:    s_endpgm
2841 ;
2842 ; GFX10-CU-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
2843 ; GFX10-CU:       ; %bb.0: ; %entry
2844 ; GFX10-CU-NEXT:    s_clause 0x1
2845 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2846 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2847 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2848 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2849 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2850 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2851 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2852 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2853 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
2854 ; GFX10-CU-NEXT:    s_endpgm
2855 ;
2856 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
2857 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2858 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2859 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2860 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2861 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2862 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2863 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2864 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2865 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2866 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2867 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2868 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2869 ;
2870 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
2871 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2872 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2873 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2874 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2875 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2876 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2877 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2878 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2879 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2880 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2881 ;
2882 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_monotonic_ret_cmpxchg:
2883 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2884 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2885 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2886 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2887 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2888 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2889 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2890 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2891 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2892 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
2893     i32 addrspace(1)* %out, i32 %in, i32 %old) {
2894 entry:
2895   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
2896   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in monotonic monotonic
2897   %val0 = extractvalue { i32, i1 } %val, 0
2898   store i32 %val0, i32 addrspace(1)* %out, align 4
2899   ret void
2900 }
2901
2902 define amdgpu_kernel void @global_system_acquire_monotonic_ret_cmpxchg(
2903 ; GFX6-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
2904 ; GFX6:       ; %bb.0: ; %entry
2905 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2906 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
2907 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
2908 ; GFX6-NEXT:    s_mov_b32 s2, -1
2909 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
2910 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
2911 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
2912 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
2913 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
2914 ; GFX6-NEXT:    buffer_wbinvl1
2915 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
2916 ; GFX6-NEXT:    s_endpgm
2917 ;
2918 ; GFX7-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
2919 ; GFX7:       ; %bb.0: ; %entry
2920 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2921 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
2922 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
2923 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
2924 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
2925 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
2926 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
2927 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
2928 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
2929 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
2930 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
2931 ; GFX7-NEXT:    buffer_wbinvl1_vol
2932 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
2933 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
2934 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
2935 ; GFX7-NEXT:    s_endpgm
2936 ;
2937 ; GFX10-WGP-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
2938 ; GFX10-WGP:       ; %bb.0: ; %entry
2939 ; GFX10-WGP-NEXT:    s_clause 0x1
2940 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2941 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2942 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
2943 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
2944 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
2945 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
2946 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2947 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
2948 ; GFX10-WGP-NEXT:    buffer_gl0_inv
2949 ; GFX10-WGP-NEXT:    buffer_gl1_inv
2950 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
2951 ; GFX10-WGP-NEXT:    s_endpgm
2952 ;
2953 ; GFX10-CU-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
2954 ; GFX10-CU:       ; %bb.0: ; %entry
2955 ; GFX10-CU-NEXT:    s_clause 0x1
2956 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
2957 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
2958 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
2959 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
2960 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
2961 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
2962 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
2963 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
2964 ; GFX10-CU-NEXT:    buffer_gl0_inv
2965 ; GFX10-CU-NEXT:    buffer_gl1_inv
2966 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
2967 ; GFX10-CU-NEXT:    s_endpgm
2968 ;
2969 ; SKIP-CACHE-INV-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
2970 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
2971 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
2972 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
2973 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
2974 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
2975 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
2976 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
2977 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
2978 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
2979 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
2980 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
2981 ; SKIP-CACHE-INV-NEXT:    s_endpgm
2982 ;
2983 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
2984 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
2985 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
2986 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
2987 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
2988 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
2989 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
2990 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
2991 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
2992 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
2993 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
2994 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
2995 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
2996 ;
2997 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_monotonic_ret_cmpxchg:
2998 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
2999 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3000 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3001 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3002 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3003 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3004 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3005 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3006 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3007 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3008 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3009 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3010     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3011 entry:
3012   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3013   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire monotonic
3014   %val0 = extractvalue { i32, i1 } %val, 0
3015   store i32 %val0, i32 addrspace(1)* %out, align 4
3016   ret void
3017 }
3018
3019 define amdgpu_kernel void @global_system_acq_rel_monotonic_ret_cmpxchg(
3020 ; GFX6-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
3021 ; GFX6:       ; %bb.0: ; %entry
3022 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3023 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3024 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3025 ; GFX6-NEXT:    s_mov_b32 s2, -1
3026 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3027 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3028 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3029 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3030 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3031 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3032 ; GFX6-NEXT:    buffer_wbinvl1
3033 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3034 ; GFX6-NEXT:    s_endpgm
3035 ;
3036 ; GFX7-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
3037 ; GFX7:       ; %bb.0: ; %entry
3038 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3039 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3040 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3041 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3042 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3043 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3044 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3045 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3046 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3047 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3048 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3049 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3050 ; GFX7-NEXT:    buffer_wbinvl1_vol
3051 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3052 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3053 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3054 ; GFX7-NEXT:    s_endpgm
3055 ;
3056 ; GFX10-WGP-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
3057 ; GFX10-WGP:       ; %bb.0: ; %entry
3058 ; GFX10-WGP-NEXT:    s_clause 0x1
3059 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3060 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3061 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3062 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3063 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3064 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3065 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3066 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3067 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3068 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3069 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3070 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3071 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3072 ; GFX10-WGP-NEXT:    s_endpgm
3073 ;
3074 ; GFX10-CU-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
3075 ; GFX10-CU:       ; %bb.0: ; %entry
3076 ; GFX10-CU-NEXT:    s_clause 0x1
3077 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3078 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3079 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3080 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3081 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3082 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3083 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3084 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3085 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3086 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3087 ; GFX10-CU-NEXT:    buffer_gl0_inv
3088 ; GFX10-CU-NEXT:    buffer_gl1_inv
3089 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3090 ; GFX10-CU-NEXT:    s_endpgm
3091 ;
3092 ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
3093 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3094 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3095 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3096 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3097 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3098 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3099 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3100 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3101 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3102 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3103 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3104 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3105 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3106 ;
3107 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
3108 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3109 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3110 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3111 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3112 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3113 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3114 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3115 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3116 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3117 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3118 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3119 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3120 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3121 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3122 ;
3123 ; GFX90A-TGSPLIT-LABEL: global_system_acq_rel_monotonic_ret_cmpxchg:
3124 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3125 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3126 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3127 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3128 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3129 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3130 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3131 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3132 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3133 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3134 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3135 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3136 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3137 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3138     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3139 entry:
3140   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3141   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel monotonic
3142   %val0 = extractvalue { i32, i1 } %val, 0
3143   store i32 %val0, i32 addrspace(1)* %out, align 4
3144   ret void
3145 }
3146
3147 define amdgpu_kernel void @global_system_seq_cst_monotonic_ret_cmpxchg(
3148 ; GFX6-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
3149 ; GFX6:       ; %bb.0: ; %entry
3150 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3151 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3152 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3153 ; GFX6-NEXT:    s_mov_b32 s2, -1
3154 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3155 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3156 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3157 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3158 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3159 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3160 ; GFX6-NEXT:    buffer_wbinvl1
3161 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3162 ; GFX6-NEXT:    s_endpgm
3163 ;
3164 ; GFX7-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
3165 ; GFX7:       ; %bb.0: ; %entry
3166 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3167 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3168 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3169 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3170 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3171 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3172 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3173 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3174 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3175 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3176 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3177 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3178 ; GFX7-NEXT:    buffer_wbinvl1_vol
3179 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3180 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3181 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3182 ; GFX7-NEXT:    s_endpgm
3183 ;
3184 ; GFX10-WGP-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
3185 ; GFX10-WGP:       ; %bb.0: ; %entry
3186 ; GFX10-WGP-NEXT:    s_clause 0x1
3187 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3188 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3189 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3190 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3191 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3192 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3193 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3194 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3195 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3196 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3197 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3198 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3199 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3200 ; GFX10-WGP-NEXT:    s_endpgm
3201 ;
3202 ; GFX10-CU-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
3203 ; GFX10-CU:       ; %bb.0: ; %entry
3204 ; GFX10-CU-NEXT:    s_clause 0x1
3205 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3206 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3207 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3208 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3209 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3210 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3211 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3212 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3213 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3214 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3215 ; GFX10-CU-NEXT:    buffer_gl0_inv
3216 ; GFX10-CU-NEXT:    buffer_gl1_inv
3217 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3218 ; GFX10-CU-NEXT:    s_endpgm
3219 ;
3220 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
3221 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3222 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3223 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3224 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3225 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3226 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3227 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3228 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3229 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3230 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3231 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3232 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3233 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3234 ;
3235 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
3236 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3237 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3238 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3239 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3240 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3241 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3242 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3243 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3244 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3245 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3246 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3247 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3248 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3249 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3250 ;
3251 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_monotonic_ret_cmpxchg:
3252 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3253 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3254 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3255 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3256 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3257 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3258 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3259 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3260 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3261 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3262 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3263 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3264 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3265 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3266     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3267 entry:
3268   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3269   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst monotonic
3270   %val0 = extractvalue { i32, i1 } %val, 0
3271   store i32 %val0, i32 addrspace(1)* %out, align 4
3272   ret void
3273 }
3274
3275 define amdgpu_kernel void @global_system_monotonic_acquire_ret_cmpxchg(
3276 ; GFX6-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
3277 ; GFX6:       ; %bb.0: ; %entry
3278 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3279 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3280 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3281 ; GFX6-NEXT:    s_mov_b32 s2, -1
3282 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3283 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3284 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3285 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3286 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3287 ; GFX6-NEXT:    buffer_wbinvl1
3288 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3289 ; GFX6-NEXT:    s_endpgm
3290 ;
3291 ; GFX7-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
3292 ; GFX7:       ; %bb.0: ; %entry
3293 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3294 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3295 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3296 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3297 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3298 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3299 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3300 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3301 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3302 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3303 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3304 ; GFX7-NEXT:    buffer_wbinvl1_vol
3305 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3306 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3307 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3308 ; GFX7-NEXT:    s_endpgm
3309 ;
3310 ; GFX10-WGP-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
3311 ; GFX10-WGP:       ; %bb.0: ; %entry
3312 ; GFX10-WGP-NEXT:    s_clause 0x1
3313 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3314 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3315 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3316 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3317 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3318 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3319 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3320 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3321 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3322 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3323 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3324 ; GFX10-WGP-NEXT:    s_endpgm
3325 ;
3326 ; GFX10-CU-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
3327 ; GFX10-CU:       ; %bb.0: ; %entry
3328 ; GFX10-CU-NEXT:    s_clause 0x1
3329 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3330 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3331 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3332 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3333 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3334 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3335 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3336 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3337 ; GFX10-CU-NEXT:    buffer_gl0_inv
3338 ; GFX10-CU-NEXT:    buffer_gl1_inv
3339 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3340 ; GFX10-CU-NEXT:    s_endpgm
3341 ;
3342 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
3343 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3344 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3345 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3346 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3347 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3348 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3349 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3350 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3351 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3352 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3353 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3354 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3355 ;
3356 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
3357 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3358 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3359 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3360 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3361 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3362 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3363 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3364 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3365 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3366 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3367 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3368 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3369 ;
3370 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_acquire_ret_cmpxchg:
3371 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3372 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3373 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3374 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3375 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3376 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3377 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3378 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3379 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3380 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3381 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3382 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3383     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3384 entry:
3385   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3386   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in monotonic acquire
3387   %val0 = extractvalue { i32, i1 } %val, 0
3388   store i32 %val0, i32 addrspace(1)* %out, align 4
3389   ret void
3390 }
3391
3392 define amdgpu_kernel void @global_system_acquire_acquire_ret_cmpxchg(
3393 ; GFX6-LABEL: global_system_acquire_acquire_ret_cmpxchg:
3394 ; GFX6:       ; %bb.0: ; %entry
3395 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3396 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3397 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3398 ; GFX6-NEXT:    s_mov_b32 s2, -1
3399 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3400 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3401 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3402 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3403 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3404 ; GFX6-NEXT:    buffer_wbinvl1
3405 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3406 ; GFX6-NEXT:    s_endpgm
3407 ;
3408 ; GFX7-LABEL: global_system_acquire_acquire_ret_cmpxchg:
3409 ; GFX7:       ; %bb.0: ; %entry
3410 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3411 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3412 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3413 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3414 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3415 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3416 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3417 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3418 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3419 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3420 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3421 ; GFX7-NEXT:    buffer_wbinvl1_vol
3422 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3423 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3424 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3425 ; GFX7-NEXT:    s_endpgm
3426 ;
3427 ; GFX10-WGP-LABEL: global_system_acquire_acquire_ret_cmpxchg:
3428 ; GFX10-WGP:       ; %bb.0: ; %entry
3429 ; GFX10-WGP-NEXT:    s_clause 0x1
3430 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3431 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3432 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3433 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3434 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3435 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3436 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3437 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3438 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3439 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3440 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3441 ; GFX10-WGP-NEXT:    s_endpgm
3442 ;
3443 ; GFX10-CU-LABEL: global_system_acquire_acquire_ret_cmpxchg:
3444 ; GFX10-CU:       ; %bb.0: ; %entry
3445 ; GFX10-CU-NEXT:    s_clause 0x1
3446 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3447 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3448 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3449 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3450 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3451 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3452 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3453 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3454 ; GFX10-CU-NEXT:    buffer_gl0_inv
3455 ; GFX10-CU-NEXT:    buffer_gl1_inv
3456 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3457 ; GFX10-CU-NEXT:    s_endpgm
3458 ;
3459 ; SKIP-CACHE-INV-LABEL: global_system_acquire_acquire_ret_cmpxchg:
3460 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3461 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3462 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3463 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3464 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3465 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3466 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3467 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3468 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3469 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3470 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3471 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3472 ;
3473 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_acquire_ret_cmpxchg:
3474 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3475 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3476 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3477 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3478 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3479 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3480 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3481 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3482 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3483 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3484 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3485 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3486 ;
3487 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_acquire_ret_cmpxchg:
3488 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3489 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3490 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3491 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3492 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3493 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3494 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3495 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3496 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3497 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3498 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3499 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3500     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3501 entry:
3502   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3503   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire acquire
3504   %val0 = extractvalue { i32, i1 } %val, 0
3505   store i32 %val0, i32 addrspace(1)* %out, align 4
3506   ret void
3507 }
3508
3509 define amdgpu_kernel void @global_system_release_acquire_ret_cmpxchg(
3510 ; GFX6-LABEL: global_system_release_acquire_ret_cmpxchg:
3511 ; GFX6:       ; %bb.0: ; %entry
3512 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3513 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3514 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3515 ; GFX6-NEXT:    s_mov_b32 s2, -1
3516 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3517 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3518 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3519 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3520 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3521 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3522 ; GFX6-NEXT:    buffer_wbinvl1
3523 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3524 ; GFX6-NEXT:    s_endpgm
3525 ;
3526 ; GFX7-LABEL: global_system_release_acquire_ret_cmpxchg:
3527 ; GFX7:       ; %bb.0: ; %entry
3528 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3529 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3530 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3531 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3532 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3533 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3534 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3535 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3536 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3537 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3538 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3539 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3540 ; GFX7-NEXT:    buffer_wbinvl1_vol
3541 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3542 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3543 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3544 ; GFX7-NEXT:    s_endpgm
3545 ;
3546 ; GFX10-WGP-LABEL: global_system_release_acquire_ret_cmpxchg:
3547 ; GFX10-WGP:       ; %bb.0: ; %entry
3548 ; GFX10-WGP-NEXT:    s_clause 0x1
3549 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3550 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3551 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3552 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3553 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3554 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3555 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3556 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3557 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3558 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3559 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3560 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3561 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3562 ; GFX10-WGP-NEXT:    s_endpgm
3563 ;
3564 ; GFX10-CU-LABEL: global_system_release_acquire_ret_cmpxchg:
3565 ; GFX10-CU:       ; %bb.0: ; %entry
3566 ; GFX10-CU-NEXT:    s_clause 0x1
3567 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3568 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3569 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3570 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3571 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3572 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3573 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3574 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3575 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3576 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3577 ; GFX10-CU-NEXT:    buffer_gl0_inv
3578 ; GFX10-CU-NEXT:    buffer_gl1_inv
3579 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3580 ; GFX10-CU-NEXT:    s_endpgm
3581 ;
3582 ; SKIP-CACHE-INV-LABEL: global_system_release_acquire_ret_cmpxchg:
3583 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3584 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3585 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3586 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3587 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3588 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3589 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3590 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3591 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3592 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3593 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3594 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3595 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3596 ;
3597 ; GFX90A-NOTTGSPLIT-LABEL: global_system_release_acquire_ret_cmpxchg:
3598 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3599 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3600 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3601 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3602 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3603 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3604 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3605 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3606 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3607 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3608 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3609 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3610 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3611 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3612 ;
3613 ; GFX90A-TGSPLIT-LABEL: global_system_release_acquire_ret_cmpxchg:
3614 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3615 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3616 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3617 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3618 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3619 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3620 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3621 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3622 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3623 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3624 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3625 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3626 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3627 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3628     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3629 entry:
3630   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3631   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in release acquire
3632   %val0 = extractvalue { i32, i1 } %val, 0
3633   store i32 %val0, i32 addrspace(1)* %out, align 4
3634   ret void
3635 }
3636
3637 define amdgpu_kernel void @global_system_acq_rel_acquire_ret_cmpxchg(
3638 ; GFX6-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
3639 ; GFX6:       ; %bb.0: ; %entry
3640 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3641 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3642 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3643 ; GFX6-NEXT:    s_mov_b32 s2, -1
3644 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3645 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3646 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3647 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3648 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3649 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3650 ; GFX6-NEXT:    buffer_wbinvl1
3651 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3652 ; GFX6-NEXT:    s_endpgm
3653 ;
3654 ; GFX7-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
3655 ; GFX7:       ; %bb.0: ; %entry
3656 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3657 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3658 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3659 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3660 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3661 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3662 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3663 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3664 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3665 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3666 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3667 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3668 ; GFX7-NEXT:    buffer_wbinvl1_vol
3669 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3670 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3671 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3672 ; GFX7-NEXT:    s_endpgm
3673 ;
3674 ; GFX10-WGP-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
3675 ; GFX10-WGP:       ; %bb.0: ; %entry
3676 ; GFX10-WGP-NEXT:    s_clause 0x1
3677 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3678 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3679 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3680 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3681 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3682 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3683 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3684 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3685 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3686 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3687 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3688 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3689 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3690 ; GFX10-WGP-NEXT:    s_endpgm
3691 ;
3692 ; GFX10-CU-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
3693 ; GFX10-CU:       ; %bb.0: ; %entry
3694 ; GFX10-CU-NEXT:    s_clause 0x1
3695 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3696 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3697 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3698 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3699 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3700 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3701 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3702 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3703 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3704 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3705 ; GFX10-CU-NEXT:    buffer_gl0_inv
3706 ; GFX10-CU-NEXT:    buffer_gl1_inv
3707 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3708 ; GFX10-CU-NEXT:    s_endpgm
3709 ;
3710 ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
3711 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3712 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3713 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3714 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3715 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3716 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3717 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3718 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3719 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3720 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3721 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3722 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3723 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3724 ;
3725 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
3726 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3727 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3728 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3729 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3730 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3731 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3732 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3733 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3734 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3735 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3736 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3737 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3738 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3739 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3740 ;
3741 ; GFX90A-TGSPLIT-LABEL: global_system_acq_rel_acquire_ret_cmpxchg:
3742 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3743 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3744 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3745 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3746 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3747 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3748 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3749 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3750 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3751 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3752 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3753 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3754 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3755 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3756     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3757 entry:
3758   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3759   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel acquire
3760   %val0 = extractvalue { i32, i1 } %val, 0
3761   store i32 %val0, i32 addrspace(1)* %out, align 4
3762   ret void
3763 }
3764
3765 define amdgpu_kernel void @global_system_seq_cst_acquire_ret_cmpxchg(
3766 ; GFX6-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
3767 ; GFX6:       ; %bb.0: ; %entry
3768 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3769 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3770 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3771 ; GFX6-NEXT:    s_mov_b32 s2, -1
3772 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3773 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3774 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3775 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3776 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3777 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3778 ; GFX6-NEXT:    buffer_wbinvl1
3779 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3780 ; GFX6-NEXT:    s_endpgm
3781 ;
3782 ; GFX7-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
3783 ; GFX7:       ; %bb.0: ; %entry
3784 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3785 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3786 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3787 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3788 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3789 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3790 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3791 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3792 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3793 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3794 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3795 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3796 ; GFX7-NEXT:    buffer_wbinvl1_vol
3797 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3798 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3799 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3800 ; GFX7-NEXT:    s_endpgm
3801 ;
3802 ; GFX10-WGP-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
3803 ; GFX10-WGP:       ; %bb.0: ; %entry
3804 ; GFX10-WGP-NEXT:    s_clause 0x1
3805 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3806 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3807 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3808 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3809 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3810 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3811 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3812 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3813 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3814 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3815 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3816 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3817 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3818 ; GFX10-WGP-NEXT:    s_endpgm
3819 ;
3820 ; GFX10-CU-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
3821 ; GFX10-CU:       ; %bb.0: ; %entry
3822 ; GFX10-CU-NEXT:    s_clause 0x1
3823 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3824 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3825 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3826 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3827 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3828 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3829 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3830 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3831 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3832 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3833 ; GFX10-CU-NEXT:    buffer_gl0_inv
3834 ; GFX10-CU-NEXT:    buffer_gl1_inv
3835 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3836 ; GFX10-CU-NEXT:    s_endpgm
3837 ;
3838 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
3839 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3840 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3841 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3842 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3843 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3844 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3845 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3846 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3847 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3848 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3849 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3850 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3851 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3852 ;
3853 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
3854 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3855 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3856 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3857 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3858 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3859 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3860 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3861 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3862 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3863 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3864 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3865 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3866 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3867 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3868 ;
3869 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_acquire_ret_cmpxchg:
3870 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3871 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3872 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3873 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3874 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3875 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3876 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
3877 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3878 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3879 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3880 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
3881 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
3882 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3883 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
3884     i32 addrspace(1)* %out, i32 %in, i32 %old) {
3885 entry:
3886   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
3887   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst acquire
3888   %val0 = extractvalue { i32, i1 } %val, 0
3889   store i32 %val0, i32 addrspace(1)* %out, align 4
3890   ret void
3891 }
3892
3893 define amdgpu_kernel void @global_system_monotonic_seq_cst_ret_cmpxchg(
3894 ; GFX6-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
3895 ; GFX6:       ; %bb.0: ; %entry
3896 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3897 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
3898 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
3899 ; GFX6-NEXT:    s_mov_b32 s2, -1
3900 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
3901 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
3902 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
3903 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3904 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
3905 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
3906 ; GFX6-NEXT:    buffer_wbinvl1
3907 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
3908 ; GFX6-NEXT:    s_endpgm
3909 ;
3910 ; GFX7-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
3911 ; GFX7:       ; %bb.0: ; %entry
3912 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3913 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
3914 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
3915 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
3916 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
3917 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
3918 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
3919 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
3920 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
3921 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3922 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
3923 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
3924 ; GFX7-NEXT:    buffer_wbinvl1_vol
3925 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
3926 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
3927 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
3928 ; GFX7-NEXT:    s_endpgm
3929 ;
3930 ; GFX10-WGP-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
3931 ; GFX10-WGP:       ; %bb.0: ; %entry
3932 ; GFX10-WGP-NEXT:    s_clause 0x1
3933 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3934 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3935 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
3936 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
3937 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
3938 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
3939 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3940 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
3941 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3942 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
3943 ; GFX10-WGP-NEXT:    buffer_gl0_inv
3944 ; GFX10-WGP-NEXT:    buffer_gl1_inv
3945 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
3946 ; GFX10-WGP-NEXT:    s_endpgm
3947 ;
3948 ; GFX10-CU-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
3949 ; GFX10-CU:       ; %bb.0: ; %entry
3950 ; GFX10-CU-NEXT:    s_clause 0x1
3951 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
3952 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
3953 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
3954 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
3955 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
3956 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
3957 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3958 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
3959 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
3960 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
3961 ; GFX10-CU-NEXT:    buffer_gl0_inv
3962 ; GFX10-CU-NEXT:    buffer_gl1_inv
3963 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
3964 ; GFX10-CU-NEXT:    s_endpgm
3965 ;
3966 ; SKIP-CACHE-INV-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
3967 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
3968 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
3969 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
3970 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
3971 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
3972 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
3973 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
3974 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
3975 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3976 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
3977 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
3978 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
3979 ; SKIP-CACHE-INV-NEXT:    s_endpgm
3980 ;
3981 ; GFX90A-NOTTGSPLIT-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
3982 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
3983 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
3984 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
3985 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
3986 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
3987 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
3988 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
3989 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
3990 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
3991 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
3992 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
3993 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
3994 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
3995 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
3996 ;
3997 ; GFX90A-TGSPLIT-LABEL: global_system_monotonic_seq_cst_ret_cmpxchg:
3998 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
3999 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4000 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4001 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4002 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4003 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4004 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4005 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4006 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4007 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4008 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4009 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4010 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4011 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4012     i32 addrspace(1)* %out, i32 %in, i32 %old) {
4013 entry:
4014   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4015   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in monotonic seq_cst
4016   %val0 = extractvalue { i32, i1 } %val, 0
4017   store i32 %val0, i32 addrspace(1)* %out, align 4
4018   ret void
4019 }
4020
4021 define amdgpu_kernel void @global_system_acquire_seq_cst_ret_cmpxchg(
4022 ; GFX6-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
4023 ; GFX6:       ; %bb.0: ; %entry
4024 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4025 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4026 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4027 ; GFX6-NEXT:    s_mov_b32 s2, -1
4028 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4029 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4030 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4031 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4032 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4033 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4034 ; GFX6-NEXT:    buffer_wbinvl1
4035 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4036 ; GFX6-NEXT:    s_endpgm
4037 ;
4038 ; GFX7-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
4039 ; GFX7:       ; %bb.0: ; %entry
4040 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4041 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4042 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4043 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
4044 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4045 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4046 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4047 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4048 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4049 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4050 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4051 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4052 ; GFX7-NEXT:    buffer_wbinvl1_vol
4053 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4054 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4055 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
4056 ; GFX7-NEXT:    s_endpgm
4057 ;
4058 ; GFX10-WGP-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
4059 ; GFX10-WGP:       ; %bb.0: ; %entry
4060 ; GFX10-WGP-NEXT:    s_clause 0x1
4061 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4062 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4063 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4064 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4065 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4066 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4067 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4068 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4069 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4070 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4071 ; GFX10-WGP-NEXT:    buffer_gl0_inv
4072 ; GFX10-WGP-NEXT:    buffer_gl1_inv
4073 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4074 ; GFX10-WGP-NEXT:    s_endpgm
4075 ;
4076 ; GFX10-CU-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
4077 ; GFX10-CU:       ; %bb.0: ; %entry
4078 ; GFX10-CU-NEXT:    s_clause 0x1
4079 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4080 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4081 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4082 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4083 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4084 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4085 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4086 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4087 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4088 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4089 ; GFX10-CU-NEXT:    buffer_gl0_inv
4090 ; GFX10-CU-NEXT:    buffer_gl1_inv
4091 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4092 ; GFX10-CU-NEXT:    s_endpgm
4093 ;
4094 ; SKIP-CACHE-INV-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
4095 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4096 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4097 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4098 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4099 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4100 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4101 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4102 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4103 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4104 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
4105 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4106 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4107 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4108 ;
4109 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
4110 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4111 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4112 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4113 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4114 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4115 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4116 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4117 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4118 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4119 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4120 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4121 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4122 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4123 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4124 ;
4125 ; GFX90A-TGSPLIT-LABEL: global_system_acquire_seq_cst_ret_cmpxchg:
4126 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4127 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4128 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4129 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4130 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4131 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4132 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4133 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4134 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4135 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4136 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4137 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4138 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4139 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4140     i32 addrspace(1)* %out, i32 %in, i32 %old) {
4141 entry:
4142   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4143   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acquire seq_cst
4144   %val0 = extractvalue { i32, i1 } %val, 0
4145   store i32 %val0, i32 addrspace(1)* %out, align 4
4146   ret void
4147 }
4148
4149 define amdgpu_kernel void @global_system_relese_seq_cst_ret_cmpxchg(
4150 ; GFX6-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
4151 ; GFX6:       ; %bb.0: ; %entry
4152 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4153 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4154 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4155 ; GFX6-NEXT:    s_mov_b32 s2, -1
4156 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4157 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4158 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4159 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4160 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4161 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4162 ; GFX6-NEXT:    buffer_wbinvl1
4163 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4164 ; GFX6-NEXT:    s_endpgm
4165 ;
4166 ; GFX7-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
4167 ; GFX7:       ; %bb.0: ; %entry
4168 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4169 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4170 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4171 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
4172 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4173 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4174 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4175 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4176 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4177 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4178 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4179 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4180 ; GFX7-NEXT:    buffer_wbinvl1_vol
4181 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4182 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4183 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
4184 ; GFX7-NEXT:    s_endpgm
4185 ;
4186 ; GFX10-WGP-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
4187 ; GFX10-WGP:       ; %bb.0: ; %entry
4188 ; GFX10-WGP-NEXT:    s_clause 0x1
4189 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4190 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4191 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4192 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4193 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4194 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4195 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4196 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4197 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4198 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4199 ; GFX10-WGP-NEXT:    buffer_gl0_inv
4200 ; GFX10-WGP-NEXT:    buffer_gl1_inv
4201 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4202 ; GFX10-WGP-NEXT:    s_endpgm
4203 ;
4204 ; GFX10-CU-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
4205 ; GFX10-CU:       ; %bb.0: ; %entry
4206 ; GFX10-CU-NEXT:    s_clause 0x1
4207 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4208 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4209 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4210 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4211 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4212 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4213 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4214 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4215 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4216 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4217 ; GFX10-CU-NEXT:    buffer_gl0_inv
4218 ; GFX10-CU-NEXT:    buffer_gl1_inv
4219 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4220 ; GFX10-CU-NEXT:    s_endpgm
4221 ;
4222 ; SKIP-CACHE-INV-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
4223 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4224 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4225 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4226 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4227 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4228 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4229 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4230 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4231 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4232 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
4233 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4234 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4235 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4236 ;
4237 ; GFX90A-NOTTGSPLIT-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
4238 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4239 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4240 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4241 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4242 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4243 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4244 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4245 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4246 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4247 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4248 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4249 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4250 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4251 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4252 ;
4253 ; GFX90A-TGSPLIT-LABEL: global_system_relese_seq_cst_ret_cmpxchg:
4254 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4255 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4256 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4257 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4258 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4259 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4260 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4261 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4262 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4263 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4264 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4265 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4266 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4267 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4268     i32 addrspace(1)* %out, i32 %in, i32 %old) {
4269 entry:
4270   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4271   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in release seq_cst
4272   %val0 = extractvalue { i32, i1 } %val, 0
4273   store i32 %val0, i32 addrspace(1)* %out, align 4
4274   ret void
4275 }
4276
4277 define amdgpu_kernel void @global_system_acq_rel_seq_cst_ret_cmpxchg(
4278 ; GFX6-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
4279 ; GFX6:       ; %bb.0: ; %entry
4280 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4281 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4282 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4283 ; GFX6-NEXT:    s_mov_b32 s2, -1
4284 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4285 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4286 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4287 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4288 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4289 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4290 ; GFX6-NEXT:    buffer_wbinvl1
4291 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4292 ; GFX6-NEXT:    s_endpgm
4293 ;
4294 ; GFX7-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
4295 ; GFX7:       ; %bb.0: ; %entry
4296 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4297 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4298 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4299 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
4300 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4301 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4302 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4303 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4304 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4305 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4306 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4307 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4308 ; GFX7-NEXT:    buffer_wbinvl1_vol
4309 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4310 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4311 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
4312 ; GFX7-NEXT:    s_endpgm
4313 ;
4314 ; GFX10-WGP-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
4315 ; GFX10-WGP:       ; %bb.0: ; %entry
4316 ; GFX10-WGP-NEXT:    s_clause 0x1
4317 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4318 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4319 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4320 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4321 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4322 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4323 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4324 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4325 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4326 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4327 ; GFX10-WGP-NEXT:    buffer_gl0_inv
4328 ; GFX10-WGP-NEXT:    buffer_gl1_inv
4329 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4330 ; GFX10-WGP-NEXT:    s_endpgm
4331 ;
4332 ; GFX10-CU-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
4333 ; GFX10-CU:       ; %bb.0: ; %entry
4334 ; GFX10-CU-NEXT:    s_clause 0x1
4335 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4336 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4337 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4338 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4339 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4340 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4341 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4342 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4343 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4344 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4345 ; GFX10-CU-NEXT:    buffer_gl0_inv
4346 ; GFX10-CU-NEXT:    buffer_gl1_inv
4347 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4348 ; GFX10-CU-NEXT:    s_endpgm
4349 ;
4350 ; SKIP-CACHE-INV-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
4351 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4352 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4353 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4354 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4355 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4356 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4357 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4358 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4359 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4360 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
4361 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4362 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4363 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4364 ;
4365 ; GFX90A-NOTTGSPLIT-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
4366 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4367 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4368 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4369 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4370 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4371 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4372 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4373 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4374 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4375 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4376 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4377 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4378 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4379 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4380 ;
4381 ; GFX90A-TGSPLIT-LABEL: global_system_acq_rel_seq_cst_ret_cmpxchg:
4382 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4383 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4384 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4385 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4386 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4387 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4388 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4389 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4390 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4391 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4392 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4393 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4394 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4395 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4396     i32 addrspace(1)* %out, i32 %in, i32 %old) {
4397 entry:
4398   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4399   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in acq_rel seq_cst
4400   %val0 = extractvalue { i32, i1 } %val, 0
4401   store i32 %val0, i32 addrspace(1)* %out, align 4
4402   ret void
4403 }
4404
4405 define amdgpu_kernel void @global_system_seq_cst_seq_cst_ret_cmpxchg(
4406 ; GFX6-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
4407 ; GFX6:       ; %bb.0: ; %entry
4408 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4409 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
4410 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4411 ; GFX6-NEXT:    s_mov_b32 s2, -1
4412 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4413 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
4414 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
4415 ; GFX6-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4416 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
4417 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4418 ; GFX6-NEXT:    buffer_wbinvl1
4419 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4420 ; GFX6-NEXT:    s_endpgm
4421 ;
4422 ; GFX7-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
4423 ; GFX7:       ; %bb.0: ; %entry
4424 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4425 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
4426 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4427 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
4428 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
4429 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
4430 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4431 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
4432 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4433 ; GFX7-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4434 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
4435 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4436 ; GFX7-NEXT:    buffer_wbinvl1_vol
4437 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4438 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4439 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
4440 ; GFX7-NEXT:    s_endpgm
4441 ;
4442 ; GFX10-WGP-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
4443 ; GFX10-WGP:       ; %bb.0: ; %entry
4444 ; GFX10-WGP-NEXT:    s_clause 0x1
4445 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4446 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4447 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
4448 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4449 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
4450 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
4451 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4452 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4453 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4454 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4455 ; GFX10-WGP-NEXT:    buffer_gl0_inv
4456 ; GFX10-WGP-NEXT:    buffer_gl1_inv
4457 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
4458 ; GFX10-WGP-NEXT:    s_endpgm
4459 ;
4460 ; GFX10-CU-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
4461 ; GFX10-CU:       ; %bb.0: ; %entry
4462 ; GFX10-CU-NEXT:    s_clause 0x1
4463 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4464 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
4465 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
4466 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4467 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
4468 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
4469 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4470 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4471 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
4472 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4473 ; GFX10-CU-NEXT:    buffer_gl0_inv
4474 ; GFX10-CU-NEXT:    buffer_gl1_inv
4475 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
4476 ; GFX10-CU-NEXT:    s_endpgm
4477 ;
4478 ; SKIP-CACHE-INV-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
4479 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4480 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
4481 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4482 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
4483 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
4484 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4485 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
4486 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
4487 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4488 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
4489 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4490 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4491 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4492 ;
4493 ; GFX90A-NOTTGSPLIT-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
4494 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4495 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4496 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4497 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4498 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4499 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4500 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
4501 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4502 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4503 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4504 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4505 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4506 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4507 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4508 ;
4509 ; GFX90A-TGSPLIT-LABEL: global_system_seq_cst_seq_cst_ret_cmpxchg:
4510 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4511 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
4512 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
4513 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
4514 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4515 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
4516 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
4517 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4518 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
4519 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4520 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4521 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4522 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
4523 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4524     i32 addrspace(1)* %out, i32 %in, i32 %old) {
4525 entry:
4526   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
4527   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in seq_cst seq_cst
4528   %val0 = extractvalue { i32, i1 } %val, 0
4529   store i32 %val0, i32 addrspace(1)* %out, align 4
4530   ret void
4531 }
4532
4533 define amdgpu_kernel void @global_system_one_as_unordered_load(
4534 ; GFX6-LABEL: global_system_one_as_unordered_load:
4535 ; GFX6:       ; %bb.0: ; %entry
4536 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
4537 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4538 ; GFX6-NEXT:    s_mov_b32 s2, -1
4539 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4540 ; GFX6-NEXT:    s_mov_b32 s0, s4
4541 ; GFX6-NEXT:    s_mov_b32 s1, s5
4542 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0
4543 ; GFX6-NEXT:    s_mov_b32 s4, s6
4544 ; GFX6-NEXT:    s_mov_b32 s5, s7
4545 ; GFX6-NEXT:    s_mov_b32 s6, s2
4546 ; GFX6-NEXT:    s_mov_b32 s7, s3
4547 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4548 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4549 ; GFX6-NEXT:    s_endpgm
4550 ;
4551 ; GFX7-LABEL: global_system_one_as_unordered_load:
4552 ; GFX7:       ; %bb.0: ; %entry
4553 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4554 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4555 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4556 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4557 ; GFX7-NEXT:    flat_load_dword v0, v[0:1]
4558 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4559 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4560 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4561 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
4562 ; GFX7-NEXT:    s_endpgm
4563 ;
4564 ; GFX10-WGP-LABEL: global_system_one_as_unordered_load:
4565 ; GFX10-WGP:       ; %bb.0: ; %entry
4566 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4567 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4568 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4569 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1]
4570 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4571 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4572 ; GFX10-WGP-NEXT:    s_endpgm
4573 ;
4574 ; GFX10-CU-LABEL: global_system_one_as_unordered_load:
4575 ; GFX10-CU:       ; %bb.0: ; %entry
4576 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4577 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4578 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4579 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1]
4580 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4581 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4582 ; GFX10-CU-NEXT:    s_endpgm
4583 ;
4584 ; SKIP-CACHE-INV-LABEL: global_system_one_as_unordered_load:
4585 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4586 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
4587 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4588 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4589 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4590 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
4591 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
4592 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0
4593 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
4594 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
4595 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
4596 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
4597 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4598 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4599 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4600 ;
4601 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_unordered_load:
4602 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4603 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4604 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4605 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4606 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4607 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4608 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4609 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4610 ;
4611 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_unordered_load:
4612 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4613 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4614 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4615 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4616 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1]
4617 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4618 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4619 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4620     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4621 entry:
4622   %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") unordered, align 4
4623   store i32 %val, i32 addrspace(1)* %out
4624   ret void
4625 }
4626
4627 define amdgpu_kernel void @global_system_one_as_monotonic_load(
4628 ; GFX6-LABEL: global_system_one_as_monotonic_load:
4629 ; GFX6:       ; %bb.0: ; %entry
4630 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
4631 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4632 ; GFX6-NEXT:    s_mov_b32 s2, -1
4633 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4634 ; GFX6-NEXT:    s_mov_b32 s0, s4
4635 ; GFX6-NEXT:    s_mov_b32 s1, s5
4636 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
4637 ; GFX6-NEXT:    s_mov_b32 s4, s6
4638 ; GFX6-NEXT:    s_mov_b32 s5, s7
4639 ; GFX6-NEXT:    s_mov_b32 s6, s2
4640 ; GFX6-NEXT:    s_mov_b32 s7, s3
4641 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4642 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4643 ; GFX6-NEXT:    s_endpgm
4644 ;
4645 ; GFX7-LABEL: global_system_one_as_monotonic_load:
4646 ; GFX7:       ; %bb.0: ; %entry
4647 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4648 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4649 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4650 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4651 ; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
4652 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4653 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4654 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4655 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
4656 ; GFX7-NEXT:    s_endpgm
4657 ;
4658 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_load:
4659 ; GFX10-WGP:       ; %bb.0: ; %entry
4660 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4661 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4662 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4663 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
4664 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4665 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4666 ; GFX10-WGP-NEXT:    s_endpgm
4667 ;
4668 ; GFX10-CU-LABEL: global_system_one_as_monotonic_load:
4669 ; GFX10-CU:       ; %bb.0: ; %entry
4670 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4671 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4672 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4673 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
4674 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4675 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4676 ; GFX10-CU-NEXT:    s_endpgm
4677 ;
4678 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_load:
4679 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4680 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
4681 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4682 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4683 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4684 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
4685 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
4686 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
4687 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
4688 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
4689 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
4690 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
4691 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4692 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4693 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4694 ;
4695 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_load:
4696 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4697 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4698 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4699 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4700 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
4701 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4702 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4703 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4704 ;
4705 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_load:
4706 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4707 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4708 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4709 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4710 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
4711 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4712 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4713 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4714     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4715 entry:
4716   %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") monotonic, align 4
4717   store i32 %val, i32 addrspace(1)* %out
4718   ret void
4719 }
4720
4721 define amdgpu_kernel void @global_system_one_as_acquire_load(
4722 ; GFX6-LABEL: global_system_one_as_acquire_load:
4723 ; GFX6:       ; %bb.0: ; %entry
4724 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
4725 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4726 ; GFX6-NEXT:    s_mov_b32 s2, -1
4727 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4728 ; GFX6-NEXT:    s_mov_b32 s0, s4
4729 ; GFX6-NEXT:    s_mov_b32 s1, s5
4730 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
4731 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4732 ; GFX6-NEXT:    buffer_wbinvl1
4733 ; GFX6-NEXT:    s_mov_b32 s4, s6
4734 ; GFX6-NEXT:    s_mov_b32 s5, s7
4735 ; GFX6-NEXT:    s_mov_b32 s6, s2
4736 ; GFX6-NEXT:    s_mov_b32 s7, s3
4737 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4738 ; GFX6-NEXT:    s_endpgm
4739 ;
4740 ; GFX7-LABEL: global_system_one_as_acquire_load:
4741 ; GFX7:       ; %bb.0: ; %entry
4742 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4743 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4744 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4745 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4746 ; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
4747 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4748 ; GFX7-NEXT:    buffer_wbinvl1_vol
4749 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4750 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4751 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
4752 ; GFX7-NEXT:    s_endpgm
4753 ;
4754 ; GFX10-WGP-LABEL: global_system_one_as_acquire_load:
4755 ; GFX10-WGP:       ; %bb.0: ; %entry
4756 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4757 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4758 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4759 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
4760 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4761 ; GFX10-WGP-NEXT:    buffer_gl0_inv
4762 ; GFX10-WGP-NEXT:    buffer_gl1_inv
4763 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4764 ; GFX10-WGP-NEXT:    s_endpgm
4765 ;
4766 ; GFX10-CU-LABEL: global_system_one_as_acquire_load:
4767 ; GFX10-CU:       ; %bb.0: ; %entry
4768 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4769 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4770 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4771 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
4772 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4773 ; GFX10-CU-NEXT:    buffer_gl0_inv
4774 ; GFX10-CU-NEXT:    buffer_gl1_inv
4775 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4776 ; GFX10-CU-NEXT:    s_endpgm
4777 ;
4778 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_load:
4779 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4780 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
4781 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4782 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4783 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4784 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
4785 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
4786 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
4787 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4788 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
4789 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
4790 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
4791 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
4792 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4793 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4794 ;
4795 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_load:
4796 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4797 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4798 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4799 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4800 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
4801 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4802 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4803 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4804 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4805 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4806 ;
4807 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_load:
4808 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4809 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4810 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4811 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4812 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
4813 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4814 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4815 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4816 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4817 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4818     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4819 entry:
4820   %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") acquire, align 4
4821   store i32 %val, i32 addrspace(1)* %out
4822   ret void
4823 }
4824
4825 define amdgpu_kernel void @global_system_one_as_seq_cst_load(
4826 ; GFX6-LABEL: global_system_one_as_seq_cst_load:
4827 ; GFX6:       ; %bb.0: ; %entry
4828 ; GFX6-NEXT:    s_load_dwordx4 s[4:7], s[4:5], 0x0
4829 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4830 ; GFX6-NEXT:    s_mov_b32 s2, -1
4831 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4832 ; GFX6-NEXT:    s_mov_b32 s0, s4
4833 ; GFX6-NEXT:    s_mov_b32 s1, s5
4834 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4835 ; GFX6-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
4836 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
4837 ; GFX6-NEXT:    buffer_wbinvl1
4838 ; GFX6-NEXT:    s_mov_b32 s4, s6
4839 ; GFX6-NEXT:    s_mov_b32 s5, s7
4840 ; GFX6-NEXT:    s_mov_b32 s6, s2
4841 ; GFX6-NEXT:    s_mov_b32 s7, s3
4842 ; GFX6-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4843 ; GFX6-NEXT:    s_endpgm
4844 ;
4845 ; GFX7-LABEL: global_system_one_as_seq_cst_load:
4846 ; GFX7:       ; %bb.0: ; %entry
4847 ; GFX7-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4848 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4849 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4850 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4851 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4852 ; GFX7-NEXT:    flat_load_dword v0, v[0:1] glc
4853 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
4854 ; GFX7-NEXT:    buffer_wbinvl1_vol
4855 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4856 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
4857 ; GFX7-NEXT:    flat_store_dword v[2:3], v0
4858 ; GFX7-NEXT:    s_endpgm
4859 ;
4860 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_load:
4861 ; GFX10-WGP:       ; %bb.0: ; %entry
4862 ; GFX10-WGP-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4863 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4864 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4865 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
4866 ; GFX10-WGP-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
4867 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
4868 ; GFX10-WGP-NEXT:    buffer_gl0_inv
4869 ; GFX10-WGP-NEXT:    buffer_gl1_inv
4870 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[2:3]
4871 ; GFX10-WGP-NEXT:    s_endpgm
4872 ;
4873 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_load:
4874 ; GFX10-CU:       ; %bb.0: ; %entry
4875 ; GFX10-CU-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4876 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4877 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4878 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
4879 ; GFX10-CU-NEXT:    global_load_dword v1, v0, s[0:1] glc dlc
4880 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
4881 ; GFX10-CU-NEXT:    buffer_gl0_inv
4882 ; GFX10-CU-NEXT:    buffer_gl1_inv
4883 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[2:3]
4884 ; GFX10-CU-NEXT:    s_endpgm
4885 ;
4886 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_load:
4887 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4888 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
4889 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4890 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4891 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4892 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s0, s4
4893 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s1, s5
4894 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4895 ; SKIP-CACHE-INV-NEXT:    buffer_load_dword v0, off, s[0:3], 0 glc
4896 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
4897 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s4, s6
4898 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s5, s7
4899 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, s2
4900 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, s3
4901 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
4902 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4903 ;
4904 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_load:
4905 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4906 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4907 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4908 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4909 ; GFX90A-NOTTGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
4910 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4911 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
4912 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
4913 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4914 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4915 ;
4916 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_load:
4917 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
4918 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
4919 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4920 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
4921 ; GFX90A-TGSPLIT-NEXT:    global_load_dword v1, v0, s[0:1] glc
4922 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
4923 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
4924 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
4925 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[2:3]
4926 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
4927     i32 addrspace(1)* %in, i32 addrspace(1)* %out) {
4928 entry:
4929   %val = load atomic i32, i32 addrspace(1)* %in syncscope("one-as") seq_cst, align 4
4930   store i32 %val, i32 addrspace(1)* %out
4931   ret void
4932 }
4933
4934 define amdgpu_kernel void @global_system_one_as_unordered_store(
4935 ; GFX6-LABEL: global_system_one_as_unordered_store:
4936 ; GFX6:       ; %bb.0: ; %entry
4937 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
4938 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4939 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
4940 ; GFX6-NEXT:    s_mov_b32 s2, -1
4941 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
4942 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
4943 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4944 ; GFX6-NEXT:    s_endpgm
4945 ;
4946 ; GFX7-LABEL: global_system_one_as_unordered_store:
4947 ; GFX7:       ; %bb.0: ; %entry
4948 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
4949 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
4950 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
4951 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
4952 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
4953 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
4954 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
4955 ; GFX7-NEXT:    s_endpgm
4956 ;
4957 ; GFX10-WGP-LABEL: global_system_one_as_unordered_store:
4958 ; GFX10-WGP:       ; %bb.0: ; %entry
4959 ; GFX10-WGP-NEXT:    s_clause 0x1
4960 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
4961 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4962 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
4963 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
4964 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
4965 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
4966 ; GFX10-WGP-NEXT:    s_endpgm
4967 ;
4968 ; GFX10-CU-LABEL: global_system_one_as_unordered_store:
4969 ; GFX10-CU:       ; %bb.0: ; %entry
4970 ; GFX10-CU-NEXT:    s_clause 0x1
4971 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
4972 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4973 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
4974 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
4975 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
4976 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
4977 ; GFX10-CU-NEXT:    s_endpgm
4978 ;
4979 ; SKIP-CACHE-INV-LABEL: global_system_one_as_unordered_store:
4980 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
4981 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
4982 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
4983 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
4984 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
4985 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
4986 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
4987 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
4988 ; SKIP-CACHE-INV-NEXT:    s_endpgm
4989 ;
4990 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_unordered_store:
4991 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
4992 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
4993 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
4994 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
4995 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
4996 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
4997 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
4998 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
4999 ;
5000 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_unordered_store:
5001 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5002 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5003 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5004 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5005 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5006 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5007 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5008 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5009     i32 %in, i32 addrspace(1)* %out) {
5010 entry:
5011   store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") unordered, align 4
5012   ret void
5013 }
5014
5015 define amdgpu_kernel void @global_system_one_as_monotonic_store(
5016 ; GFX6-LABEL: global_system_one_as_monotonic_store:
5017 ; GFX6:       ; %bb.0: ; %entry
5018 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
5019 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5020 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5021 ; GFX6-NEXT:    s_mov_b32 s2, -1
5022 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5023 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5024 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5025 ; GFX6-NEXT:    s_endpgm
5026 ;
5027 ; GFX7-LABEL: global_system_one_as_monotonic_store:
5028 ; GFX7:       ; %bb.0: ; %entry
5029 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5030 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5031 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5032 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5033 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5034 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5035 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
5036 ; GFX7-NEXT:    s_endpgm
5037 ;
5038 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_store:
5039 ; GFX10-WGP:       ; %bb.0: ; %entry
5040 ; GFX10-WGP-NEXT:    s_clause 0x1
5041 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5042 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5043 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5044 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5045 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5046 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5047 ; GFX10-WGP-NEXT:    s_endpgm
5048 ;
5049 ; GFX10-CU-LABEL: global_system_one_as_monotonic_store:
5050 ; GFX10-CU:       ; %bb.0: ; %entry
5051 ; GFX10-CU-NEXT:    s_clause 0x1
5052 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5053 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5054 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5055 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5056 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5057 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5058 ; GFX10-CU-NEXT:    s_endpgm
5059 ;
5060 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_store:
5061 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5062 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
5063 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5064 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5065 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5066 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5067 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5068 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5069 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5070 ;
5071 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_store:
5072 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5073 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5074 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5075 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5076 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5077 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5078 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5079 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5080 ;
5081 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_store:
5082 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5083 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5084 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5085 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5086 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5087 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5088 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5089 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5090     i32 %in, i32 addrspace(1)* %out) {
5091 entry:
5092   store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") monotonic, align 4
5093   ret void
5094 }
5095
5096 define amdgpu_kernel void @global_system_one_as_release_store(
5097 ; GFX6-LABEL: global_system_one_as_release_store:
5098 ; GFX6:       ; %bb.0: ; %entry
5099 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
5100 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5101 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5102 ; GFX6-NEXT:    s_mov_b32 s2, -1
5103 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5104 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5105 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5106 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5107 ; GFX6-NEXT:    s_endpgm
5108 ;
5109 ; GFX7-LABEL: global_system_one_as_release_store:
5110 ; GFX7:       ; %bb.0: ; %entry
5111 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5112 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5113 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5114 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5115 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5116 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5117 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5118 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
5119 ; GFX7-NEXT:    s_endpgm
5120 ;
5121 ; GFX10-WGP-LABEL: global_system_one_as_release_store:
5122 ; GFX10-WGP:       ; %bb.0: ; %entry
5123 ; GFX10-WGP-NEXT:    s_clause 0x1
5124 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5125 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5126 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5127 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5128 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5129 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5130 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5131 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5132 ; GFX10-WGP-NEXT:    s_endpgm
5133 ;
5134 ; GFX10-CU-LABEL: global_system_one_as_release_store:
5135 ; GFX10-CU:       ; %bb.0: ; %entry
5136 ; GFX10-CU-NEXT:    s_clause 0x1
5137 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5138 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5139 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5140 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5141 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5142 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5143 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5144 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5145 ; GFX10-CU-NEXT:    s_endpgm
5146 ;
5147 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_store:
5148 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5149 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
5150 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5151 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5152 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5153 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5154 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5155 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5156 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5157 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5158 ;
5159 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_store:
5160 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5161 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5162 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5163 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5164 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5165 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5166 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5167 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5168 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5169 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5170 ;
5171 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_store:
5172 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5173 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5174 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5175 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5176 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5177 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5178 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5179 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5180 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5181 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5182     i32 %in, i32 addrspace(1)* %out) {
5183 entry:
5184   store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") release, align 4
5185   ret void
5186 }
5187
5188 define amdgpu_kernel void @global_system_one_as_seq_cst_store(
5189 ; GFX6-LABEL: global_system_one_as_seq_cst_store:
5190 ; GFX6:       ; %bb.0: ; %entry
5191 ; GFX6-NEXT:    s_load_dword s6, s[4:5], 0x0
5192 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5193 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5194 ; GFX6-NEXT:    s_mov_b32 s2, -1
5195 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5196 ; GFX6-NEXT:    v_mov_b32_e32 v0, s6
5197 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5198 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5199 ; GFX6-NEXT:    s_endpgm
5200 ;
5201 ; GFX7-LABEL: global_system_one_as_seq_cst_store:
5202 ; GFX7:       ; %bb.0: ; %entry
5203 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
5204 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x2
5205 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5206 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5207 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5208 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5209 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5210 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
5211 ; GFX7-NEXT:    s_endpgm
5212 ;
5213 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_store:
5214 ; GFX10-WGP:       ; %bb.0: ; %entry
5215 ; GFX10-WGP-NEXT:    s_clause 0x1
5216 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x0
5217 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5218 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5219 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5220 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5221 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5222 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5223 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5224 ; GFX10-WGP-NEXT:    s_endpgm
5225 ;
5226 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_store:
5227 ; GFX10-CU:       ; %bb.0: ; %entry
5228 ; GFX10-CU-NEXT:    s_clause 0x1
5229 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x0
5230 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5231 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5232 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5233 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5234 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5235 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5236 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5237 ; GFX10-CU-NEXT:    s_endpgm
5238 ;
5239 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_store:
5240 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5241 ; SKIP-CACHE-INV-NEXT:    s_load_dword s4, s[0:1], 0x9
5242 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
5243 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s3, 0xf000
5244 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s2, -1
5245 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5246 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s4
5247 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5248 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5249 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5250 ;
5251 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_store:
5252 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5253 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5254 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5255 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5256 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5257 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5258 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5259 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5260 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5261 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5262 ;
5263 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_store:
5264 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5265 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x0
5266 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
5267 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5268 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5269 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5270 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5271 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5272 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5273 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5274     i32 %in, i32 addrspace(1)* %out) {
5275 entry:
5276   store atomic i32 %in, i32 addrspace(1)* %out syncscope("one-as") seq_cst, align 4
5277   ret void
5278 }
5279
5280 define amdgpu_kernel void @global_system_one_as_monotonic_atomicrmw(
5281 ; GFX6-LABEL: global_system_one_as_monotonic_atomicrmw:
5282 ; GFX6:       ; %bb.0: ; %entry
5283 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5284 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5285 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5286 ; GFX6-NEXT:    s_mov_b32 s2, -1
5287 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5288 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5289 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5290 ; GFX6-NEXT:    s_endpgm
5291 ;
5292 ; GFX7-LABEL: global_system_one_as_monotonic_atomicrmw:
5293 ; GFX7:       ; %bb.0: ; %entry
5294 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5295 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5296 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5297 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5298 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5299 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5300 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5301 ; GFX7-NEXT:    s_endpgm
5302 ;
5303 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_atomicrmw:
5304 ; GFX10-WGP:       ; %bb.0: ; %entry
5305 ; GFX10-WGP-NEXT:    s_clause 0x1
5306 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5307 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5308 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5309 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5310 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5311 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5312 ; GFX10-WGP-NEXT:    s_endpgm
5313 ;
5314 ; GFX10-CU-LABEL: global_system_one_as_monotonic_atomicrmw:
5315 ; GFX10-CU:       ; %bb.0: ; %entry
5316 ; GFX10-CU-NEXT:    s_clause 0x1
5317 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5318 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5319 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5320 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5321 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5322 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5323 ; GFX10-CU-NEXT:    s_endpgm
5324 ;
5325 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_atomicrmw:
5326 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5327 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5328 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
5329 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5330 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5331 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5332 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5333 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
5334 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5335 ;
5336 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_atomicrmw:
5337 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5338 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5339 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5340 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5341 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5342 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5343 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5344 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5345 ;
5346 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_atomicrmw:
5347 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5348 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5349 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5350 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5351 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5352 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5353 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5354 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5355     i32 addrspace(1)* %out, i32 %in) {
5356 entry:
5357   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") monotonic
5358   ret void
5359 }
5360
5361 define amdgpu_kernel void @global_system_one_as_acquire_atomicrmw(
5362 ; GFX6-LABEL: global_system_one_as_acquire_atomicrmw:
5363 ; GFX6:       ; %bb.0: ; %entry
5364 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5365 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5366 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5367 ; GFX6-NEXT:    s_mov_b32 s2, -1
5368 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5369 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5370 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5371 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5372 ; GFX6-NEXT:    buffer_wbinvl1
5373 ; GFX6-NEXT:    s_endpgm
5374 ;
5375 ; GFX7-LABEL: global_system_one_as_acquire_atomicrmw:
5376 ; GFX7:       ; %bb.0: ; %entry
5377 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5378 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5379 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5380 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5381 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5382 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5383 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5384 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5385 ; GFX7-NEXT:    buffer_wbinvl1_vol
5386 ; GFX7-NEXT:    s_endpgm
5387 ;
5388 ; GFX10-WGP-LABEL: global_system_one_as_acquire_atomicrmw:
5389 ; GFX10-WGP:       ; %bb.0: ; %entry
5390 ; GFX10-WGP-NEXT:    s_clause 0x1
5391 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5392 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5393 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5394 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5395 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5396 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5397 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5398 ; GFX10-WGP-NEXT:    buffer_gl0_inv
5399 ; GFX10-WGP-NEXT:    buffer_gl1_inv
5400 ; GFX10-WGP-NEXT:    s_endpgm
5401 ;
5402 ; GFX10-CU-LABEL: global_system_one_as_acquire_atomicrmw:
5403 ; GFX10-CU:       ; %bb.0: ; %entry
5404 ; GFX10-CU-NEXT:    s_clause 0x1
5405 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5406 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5407 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5408 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5409 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5410 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5411 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5412 ; GFX10-CU-NEXT:    buffer_gl0_inv
5413 ; GFX10-CU-NEXT:    buffer_gl1_inv
5414 ; GFX10-CU-NEXT:    s_endpgm
5415 ;
5416 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_atomicrmw:
5417 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5418 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5419 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
5420 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5421 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5422 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5423 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5424 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
5425 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5426 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5427 ;
5428 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_atomicrmw:
5429 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5430 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5431 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5432 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5433 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5434 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5435 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5436 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5437 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5438 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5439 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5440 ;
5441 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_atomicrmw:
5442 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5443 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5444 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5445 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5446 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5447 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5448 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5449 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5450 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5451 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5452 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5453     i32 addrspace(1)* %out, i32 %in) {
5454 entry:
5455   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acquire
5456   ret void
5457 }
5458
5459 define amdgpu_kernel void @global_system_one_as_release_atomicrmw(
5460 ; GFX6-LABEL: global_system_one_as_release_atomicrmw:
5461 ; GFX6:       ; %bb.0: ; %entry
5462 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5463 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5464 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5465 ; GFX6-NEXT:    s_mov_b32 s2, -1
5466 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5467 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5468 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5469 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5470 ; GFX6-NEXT:    s_endpgm
5471 ;
5472 ; GFX7-LABEL: global_system_one_as_release_atomicrmw:
5473 ; GFX7:       ; %bb.0: ; %entry
5474 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5475 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5476 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5477 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5478 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5479 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5480 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5481 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5482 ; GFX7-NEXT:    s_endpgm
5483 ;
5484 ; GFX10-WGP-LABEL: global_system_one_as_release_atomicrmw:
5485 ; GFX10-WGP:       ; %bb.0: ; %entry
5486 ; GFX10-WGP-NEXT:    s_clause 0x1
5487 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5488 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5489 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5490 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5491 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5492 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5493 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5494 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5495 ; GFX10-WGP-NEXT:    s_endpgm
5496 ;
5497 ; GFX10-CU-LABEL: global_system_one_as_release_atomicrmw:
5498 ; GFX10-CU:       ; %bb.0: ; %entry
5499 ; GFX10-CU-NEXT:    s_clause 0x1
5500 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5501 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5502 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5503 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5504 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5505 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5506 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5507 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5508 ; GFX10-CU-NEXT:    s_endpgm
5509 ;
5510 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_atomicrmw:
5511 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5512 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5513 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
5514 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5515 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5516 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5517 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5518 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5519 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
5520 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5521 ;
5522 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_atomicrmw:
5523 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5524 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5525 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5526 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5527 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5528 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5529 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5530 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5531 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5532 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5533 ;
5534 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_atomicrmw:
5535 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5536 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5537 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5538 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5539 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5540 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5541 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5542 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5543 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5544 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5545     i32 addrspace(1)* %out, i32 %in) {
5546 entry:
5547   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") release
5548   ret void
5549 }
5550
5551 define amdgpu_kernel void @global_system_one_as_acq_rel_atomicrmw(
5552 ; GFX6-LABEL: global_system_one_as_acq_rel_atomicrmw:
5553 ; GFX6:       ; %bb.0: ; %entry
5554 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5555 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5556 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5557 ; GFX6-NEXT:    s_mov_b32 s2, -1
5558 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5559 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5560 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5561 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5562 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5563 ; GFX6-NEXT:    buffer_wbinvl1
5564 ; GFX6-NEXT:    s_endpgm
5565 ;
5566 ; GFX7-LABEL: global_system_one_as_acq_rel_atomicrmw:
5567 ; GFX7:       ; %bb.0: ; %entry
5568 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5569 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5570 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5571 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5572 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5573 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5574 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5575 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5576 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5577 ; GFX7-NEXT:    buffer_wbinvl1_vol
5578 ; GFX7-NEXT:    s_endpgm
5579 ;
5580 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_atomicrmw:
5581 ; GFX10-WGP:       ; %bb.0: ; %entry
5582 ; GFX10-WGP-NEXT:    s_clause 0x1
5583 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5584 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5585 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5586 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5587 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5588 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5589 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5590 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5591 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5592 ; GFX10-WGP-NEXT:    buffer_gl0_inv
5593 ; GFX10-WGP-NEXT:    buffer_gl1_inv
5594 ; GFX10-WGP-NEXT:    s_endpgm
5595 ;
5596 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_atomicrmw:
5597 ; GFX10-CU:       ; %bb.0: ; %entry
5598 ; GFX10-CU-NEXT:    s_clause 0x1
5599 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5600 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5601 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5602 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5603 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5604 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5605 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5606 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5607 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5608 ; GFX10-CU-NEXT:    buffer_gl0_inv
5609 ; GFX10-CU-NEXT:    buffer_gl1_inv
5610 ; GFX10-CU-NEXT:    s_endpgm
5611 ;
5612 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_atomicrmw:
5613 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5614 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5615 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
5616 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5617 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5618 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5619 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5620 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5621 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
5622 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5623 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5624 ;
5625 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_atomicrmw:
5626 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5627 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5628 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5629 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5630 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5631 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5632 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5633 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5634 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5635 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5636 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5637 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5638 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5639 ;
5640 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_atomicrmw:
5641 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5642 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5643 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5644 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5645 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5646 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5647 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5648 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5649 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5650 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5651 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5652 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5653 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5654     i32 addrspace(1)* %out, i32 %in) {
5655 entry:
5656   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acq_rel
5657   ret void
5658 }
5659
5660 define amdgpu_kernel void @global_system_one_as_seq_cst_atomicrmw(
5661 ; GFX6-LABEL: global_system_one_as_seq_cst_atomicrmw:
5662 ; GFX6:       ; %bb.0: ; %entry
5663 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5664 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5665 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5666 ; GFX6-NEXT:    s_mov_b32 s2, -1
5667 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5668 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5669 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5670 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0
5671 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5672 ; GFX6-NEXT:    buffer_wbinvl1
5673 ; GFX6-NEXT:    s_endpgm
5674 ;
5675 ; GFX7-LABEL: global_system_one_as_seq_cst_atomicrmw:
5676 ; GFX7:       ; %bb.0: ; %entry
5677 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5678 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5679 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5680 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5681 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5682 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5683 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5684 ; GFX7-NEXT:    flat_atomic_swap v[0:1], v2
5685 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5686 ; GFX7-NEXT:    buffer_wbinvl1_vol
5687 ; GFX7-NEXT:    s_endpgm
5688 ;
5689 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_atomicrmw:
5690 ; GFX10-WGP:       ; %bb.0: ; %entry
5691 ; GFX10-WGP-NEXT:    s_clause 0x1
5692 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5693 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5694 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5695 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5696 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5697 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5698 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5699 ; GFX10-WGP-NEXT:    global_atomic_swap v0, v1, s[0:1]
5700 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5701 ; GFX10-WGP-NEXT:    buffer_gl0_inv
5702 ; GFX10-WGP-NEXT:    buffer_gl1_inv
5703 ; GFX10-WGP-NEXT:    s_endpgm
5704 ;
5705 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_atomicrmw:
5706 ; GFX10-CU:       ; %bb.0: ; %entry
5707 ; GFX10-CU-NEXT:    s_clause 0x1
5708 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5709 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5710 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5711 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5712 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5713 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5714 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5715 ; GFX10-CU-NEXT:    global_atomic_swap v0, v1, s[0:1]
5716 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5717 ; GFX10-CU-NEXT:    buffer_gl0_inv
5718 ; GFX10-CU-NEXT:    buffer_gl1_inv
5719 ; GFX10-CU-NEXT:    s_endpgm
5720 ;
5721 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_atomicrmw:
5722 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5723 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5724 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
5725 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5726 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5727 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5728 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5729 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5730 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0
5731 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5732 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5733 ;
5734 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_atomicrmw:
5735 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5736 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5737 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5738 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5739 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5740 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5741 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5742 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5743 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5744 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5745 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5746 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5747 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5748 ;
5749 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_atomicrmw:
5750 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5751 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5752 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5753 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5754 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5755 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5756 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5757 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5758 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v0, v1, s[0:1]
5759 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5760 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5761 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5762 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5763     i32 addrspace(1)* %out, i32 %in) {
5764 entry:
5765   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") seq_cst
5766   ret void
5767 }
5768
5769 define amdgpu_kernel void @global_system_one_as_acquire_ret_atomicrmw(
5770 ; GFX6-LABEL: global_system_one_as_acquire_ret_atomicrmw:
5771 ; GFX6:       ; %bb.0: ; %entry
5772 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5773 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5774 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5775 ; GFX6-NEXT:    s_mov_b32 s2, -1
5776 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5777 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5778 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5779 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5780 ; GFX6-NEXT:    buffer_wbinvl1
5781 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5782 ; GFX6-NEXT:    s_endpgm
5783 ;
5784 ; GFX7-LABEL: global_system_one_as_acquire_ret_atomicrmw:
5785 ; GFX7:       ; %bb.0: ; %entry
5786 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5787 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5788 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5789 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5790 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5791 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5792 ; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5793 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5794 ; GFX7-NEXT:    buffer_wbinvl1_vol
5795 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
5796 ; GFX7-NEXT:    s_endpgm
5797 ;
5798 ; GFX10-WGP-LABEL: global_system_one_as_acquire_ret_atomicrmw:
5799 ; GFX10-WGP:       ; %bb.0: ; %entry
5800 ; GFX10-WGP-NEXT:    s_clause 0x1
5801 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5802 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5803 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5804 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5805 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5806 ; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5807 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5808 ; GFX10-WGP-NEXT:    buffer_gl0_inv
5809 ; GFX10-WGP-NEXT:    buffer_gl1_inv
5810 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5811 ; GFX10-WGP-NEXT:    s_endpgm
5812 ;
5813 ; GFX10-CU-LABEL: global_system_one_as_acquire_ret_atomicrmw:
5814 ; GFX10-CU:       ; %bb.0: ; %entry
5815 ; GFX10-CU-NEXT:    s_clause 0x1
5816 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5817 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5818 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5819 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5820 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5821 ; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5822 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5823 ; GFX10-CU-NEXT:    buffer_gl0_inv
5824 ; GFX10-CU-NEXT:    buffer_gl1_inv
5825 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5826 ; GFX10-CU-NEXT:    s_endpgm
5827 ;
5828 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_ret_atomicrmw:
5829 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5830 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5831 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
5832 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5833 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5834 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5835 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5836 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
5837 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5838 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5839 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5840 ;
5841 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_ret_atomicrmw:
5842 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5843 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5844 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5845 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5846 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5847 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5848 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5849 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5850 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5851 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5852 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5853 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5854 ;
5855 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_ret_atomicrmw:
5856 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5857 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5858 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5859 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5860 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5861 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5862 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5863 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5864 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5865 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5866 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5867 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5868     i32 addrspace(1)* %out, i32 %in) {
5869 entry:
5870   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acquire
5871   store i32 %val, i32 addrspace(1)* %out, align 4
5872   ret void
5873 }
5874
5875 define amdgpu_kernel void @global_system_one_as_acq_rel_ret_atomicrmw(
5876 ; GFX6-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
5877 ; GFX6:       ; %bb.0: ; %entry
5878 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5879 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5880 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5881 ; GFX6-NEXT:    s_mov_b32 s2, -1
5882 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
5883 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
5884 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5885 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
5886 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
5887 ; GFX6-NEXT:    buffer_wbinvl1
5888 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
5889 ; GFX6-NEXT:    s_endpgm
5890 ;
5891 ; GFX7-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
5892 ; GFX7:       ; %bb.0: ; %entry
5893 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5894 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
5895 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
5896 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
5897 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
5898 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
5899 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5900 ; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
5901 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
5902 ; GFX7-NEXT:    buffer_wbinvl1_vol
5903 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
5904 ; GFX7-NEXT:    s_endpgm
5905 ;
5906 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
5907 ; GFX10-WGP:       ; %bb.0: ; %entry
5908 ; GFX10-WGP-NEXT:    s_clause 0x1
5909 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
5910 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5911 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
5912 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
5913 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
5914 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5915 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
5916 ; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5917 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
5918 ; GFX10-WGP-NEXT:    buffer_gl0_inv
5919 ; GFX10-WGP-NEXT:    buffer_gl1_inv
5920 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
5921 ; GFX10-WGP-NEXT:    s_endpgm
5922 ;
5923 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
5924 ; GFX10-CU:       ; %bb.0: ; %entry
5925 ; GFX10-CU-NEXT:    s_clause 0x1
5926 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
5927 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5928 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
5929 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
5930 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
5931 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5932 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
5933 ; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5934 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
5935 ; GFX10-CU-NEXT:    buffer_gl0_inv
5936 ; GFX10-CU-NEXT:    buffer_gl1_inv
5937 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
5938 ; GFX10-CU-NEXT:    s_endpgm
5939 ;
5940 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
5941 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
5942 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
5943 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
5944 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
5945 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
5946 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
5947 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
5948 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5949 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
5950 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
5951 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
5952 ; SKIP-CACHE-INV-NEXT:    s_endpgm
5953 ;
5954 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
5955 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
5956 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5957 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5958 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5959 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5960 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5961 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
5962 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5963 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5964 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5965 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
5966 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
5967 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5968 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
5969 ;
5970 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_ret_atomicrmw:
5971 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
5972 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5973 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
5974 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
5975 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
5976 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
5977 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
5978 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5979 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
5980 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
5981 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
5982 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
5983 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
5984 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
5985     i32 addrspace(1)* %out, i32 %in) {
5986 entry:
5987   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") acq_rel
5988   store i32 %val, i32 addrspace(1)* %out, align 4
5989   ret void
5990 }
5991
5992 define amdgpu_kernel void @global_system_one_as_seq_cst_ret_atomicrmw(
5993 ; GFX6-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
5994 ; GFX6:       ; %bb.0: ; %entry
5995 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
5996 ; GFX6-NEXT:    s_load_dword s4, s[4:5], 0x2
5997 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
5998 ; GFX6-NEXT:    s_mov_b32 s2, -1
5999 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6000 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6001 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6002 ; GFX6-NEXT:    buffer_atomic_swap v0, off, s[0:3], 0 glc
6003 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6004 ; GFX6-NEXT:    buffer_wbinvl1
6005 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
6006 ; GFX6-NEXT:    s_endpgm
6007 ;
6008 ; GFX7-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
6009 ; GFX7:       ; %bb.0: ; %entry
6010 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6011 ; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x2
6012 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6013 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6014 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6015 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6016 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6017 ; GFX7-NEXT:    flat_atomic_swap v2, v[0:1], v2 glc
6018 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6019 ; GFX7-NEXT:    buffer_wbinvl1_vol
6020 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
6021 ; GFX7-NEXT:    s_endpgm
6022 ;
6023 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
6024 ; GFX10-WGP:       ; %bb.0: ; %entry
6025 ; GFX10-WGP-NEXT:    s_clause 0x1
6026 ; GFX10-WGP-NEXT:    s_load_dword s2, s[4:5], 0x8
6027 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6028 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, 0
6029 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6030 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s2
6031 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6032 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6033 ; GFX10-WGP-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6034 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6035 ; GFX10-WGP-NEXT:    buffer_gl0_inv
6036 ; GFX10-WGP-NEXT:    buffer_gl1_inv
6037 ; GFX10-WGP-NEXT:    global_store_dword v0, v1, s[0:1]
6038 ; GFX10-WGP-NEXT:    s_endpgm
6039 ;
6040 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
6041 ; GFX10-CU:       ; %bb.0: ; %entry
6042 ; GFX10-CU-NEXT:    s_clause 0x1
6043 ; GFX10-CU-NEXT:    s_load_dword s2, s[4:5], 0x8
6044 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6045 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, 0
6046 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6047 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s2
6048 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6049 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6050 ; GFX10-CU-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6051 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6052 ; GFX10-CU-NEXT:    buffer_gl0_inv
6053 ; GFX10-CU-NEXT:    buffer_gl1_inv
6054 ; GFX10-CU-NEXT:    global_store_dword v0, v1, s[0:1]
6055 ; GFX10-CU-NEXT:    s_endpgm
6056 ;
6057 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
6058 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6059 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6060 ; SKIP-CACHE-INV-NEXT:    s_load_dword s0, s[0:1], 0xb
6061 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6062 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6063 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6064 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6065 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6066 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_swap v0, off, s[4:7], 0 glc
6067 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6068 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
6069 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6070 ;
6071 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
6072 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6073 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6074 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6075 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6076 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6077 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6078 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6079 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6080 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6081 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6082 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6083 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6084 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6085 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6086 ;
6087 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_ret_atomicrmw:
6088 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6089 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6090 ; GFX90A-TGSPLIT-NEXT:    s_load_dword s2, s[4:5], 0x8
6091 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v0, 0
6092 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6093 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v1, s2
6094 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6095 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6096 ; GFX90A-TGSPLIT-NEXT:    global_atomic_swap v1, v0, v1, s[0:1] glc
6097 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6098 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6099 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6100 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v0, v1, s[0:1]
6101 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6102     i32 addrspace(1)* %out, i32 %in) {
6103 entry:
6104   %val = atomicrmw volatile xchg i32 addrspace(1)* %out, i32 %in syncscope("one-as") seq_cst
6105   store i32 %val, i32 addrspace(1)* %out, align 4
6106   ret void
6107 }
6108
6109 define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_cmpxchg(
6110 ; GFX6-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
6111 ; GFX6:       ; %bb.0: ; %entry
6112 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6113 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6114 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6115 ; GFX6-NEXT:    s_mov_b32 s2, -1
6116 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6117 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6118 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6119 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6120 ; GFX6-NEXT:    s_endpgm
6121 ;
6122 ; GFX7-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
6123 ; GFX7:       ; %bb.0: ; %entry
6124 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6125 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6126 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6127 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6128 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6129 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6130 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6131 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6132 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6133 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6134 ; GFX7-NEXT:    s_endpgm
6135 ;
6136 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
6137 ; GFX10-WGP:       ; %bb.0: ; %entry
6138 ; GFX10-WGP-NEXT:    s_clause 0x1
6139 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6140 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6141 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6142 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6143 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6144 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6145 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6146 ; GFX10-WGP-NEXT:    s_endpgm
6147 ;
6148 ; GFX10-CU-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
6149 ; GFX10-CU:       ; %bb.0: ; %entry
6150 ; GFX10-CU-NEXT:    s_clause 0x1
6151 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6152 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6153 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6154 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6155 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6156 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6157 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6158 ; GFX10-CU-NEXT:    s_endpgm
6159 ;
6160 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
6161 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6162 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6163 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6164 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6165 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6166 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6167 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6168 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6169 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6170 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6171 ;
6172 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
6173 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6174 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6175 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6176 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6177 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6178 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6179 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6180 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6181 ;
6182 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_monotonic_cmpxchg:
6183 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6184 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6185 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6186 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6187 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6188 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6189 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6190 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6191     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6192 entry:
6193   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6194   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
6195   ret void
6196 }
6197
6198 define amdgpu_kernel void @global_system_one_as_acquire_monotonic_cmpxchg(
6199 ; GFX6-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
6200 ; GFX6:       ; %bb.0: ; %entry
6201 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6202 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6203 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6204 ; GFX6-NEXT:    s_mov_b32 s2, -1
6205 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6206 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6207 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6208 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6209 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6210 ; GFX6-NEXT:    buffer_wbinvl1
6211 ; GFX6-NEXT:    s_endpgm
6212 ;
6213 ; GFX7-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
6214 ; GFX7:       ; %bb.0: ; %entry
6215 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6216 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6217 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6218 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6219 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6220 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6221 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6222 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6223 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6224 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6225 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6226 ; GFX7-NEXT:    buffer_wbinvl1_vol
6227 ; GFX7-NEXT:    s_endpgm
6228 ;
6229 ; GFX10-WGP-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
6230 ; GFX10-WGP:       ; %bb.0: ; %entry
6231 ; GFX10-WGP-NEXT:    s_clause 0x1
6232 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6233 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6234 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6235 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6236 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6237 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6238 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6239 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6240 ; GFX10-WGP-NEXT:    buffer_gl0_inv
6241 ; GFX10-WGP-NEXT:    buffer_gl1_inv
6242 ; GFX10-WGP-NEXT:    s_endpgm
6243 ;
6244 ; GFX10-CU-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
6245 ; GFX10-CU:       ; %bb.0: ; %entry
6246 ; GFX10-CU-NEXT:    s_clause 0x1
6247 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6248 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6249 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6250 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6251 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6252 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6253 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6254 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6255 ; GFX10-CU-NEXT:    buffer_gl0_inv
6256 ; GFX10-CU-NEXT:    buffer_gl1_inv
6257 ; GFX10-CU-NEXT:    s_endpgm
6258 ;
6259 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
6260 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6261 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6262 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6263 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6264 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6265 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6266 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6267 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6268 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6269 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6270 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6271 ;
6272 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
6273 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6274 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6275 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6276 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6277 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6278 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6279 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6280 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6281 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6282 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6283 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6284 ;
6285 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_monotonic_cmpxchg:
6286 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6287 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6288 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6289 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6290 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6291 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6292 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6293 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6294 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6295 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6296 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6297     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6298 entry:
6299   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6300   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
6301   ret void
6302 }
6303
6304 define amdgpu_kernel void @global_system_one_as_release_monotonic_cmpxchg(
6305 ; GFX6-LABEL: global_system_one_as_release_monotonic_cmpxchg:
6306 ; GFX6:       ; %bb.0: ; %entry
6307 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6308 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6309 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6310 ; GFX6-NEXT:    s_mov_b32 s2, -1
6311 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6312 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6313 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6314 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6315 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6316 ; GFX6-NEXT:    s_endpgm
6317 ;
6318 ; GFX7-LABEL: global_system_one_as_release_monotonic_cmpxchg:
6319 ; GFX7:       ; %bb.0: ; %entry
6320 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6321 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6322 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6323 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6324 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6325 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6326 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6327 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6328 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6329 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6330 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6331 ; GFX7-NEXT:    s_endpgm
6332 ;
6333 ; GFX10-WGP-LABEL: global_system_one_as_release_monotonic_cmpxchg:
6334 ; GFX10-WGP:       ; %bb.0: ; %entry
6335 ; GFX10-WGP-NEXT:    s_clause 0x1
6336 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6337 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6338 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6339 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6340 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6341 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6342 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6343 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6344 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6345 ; GFX10-WGP-NEXT:    s_endpgm
6346 ;
6347 ; GFX10-CU-LABEL: global_system_one_as_release_monotonic_cmpxchg:
6348 ; GFX10-CU:       ; %bb.0: ; %entry
6349 ; GFX10-CU-NEXT:    s_clause 0x1
6350 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6351 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6352 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6353 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6354 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6355 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6356 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6357 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6358 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6359 ; GFX10-CU-NEXT:    s_endpgm
6360 ;
6361 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_monotonic_cmpxchg:
6362 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6363 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6364 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6365 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6366 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6367 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6368 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6369 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6370 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6371 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6372 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6373 ;
6374 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_monotonic_cmpxchg:
6375 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6376 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6377 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6378 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6379 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6380 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6381 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6382 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6383 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6384 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6385 ;
6386 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_monotonic_cmpxchg:
6387 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6388 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6389 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6390 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6391 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6392 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6393 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6394 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6395 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6396 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6397     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6398 entry:
6399   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6400   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
6401   ret void
6402 }
6403
6404 define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_cmpxchg(
6405 ; GFX6-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
6406 ; GFX6:       ; %bb.0: ; %entry
6407 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6408 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6409 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6410 ; GFX6-NEXT:    s_mov_b32 s2, -1
6411 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6412 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6413 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6414 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6415 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6416 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6417 ; GFX6-NEXT:    buffer_wbinvl1
6418 ; GFX6-NEXT:    s_endpgm
6419 ;
6420 ; GFX7-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
6421 ; GFX7:       ; %bb.0: ; %entry
6422 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6423 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6424 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6425 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6426 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6427 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6428 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6429 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6430 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6431 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6432 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6433 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6434 ; GFX7-NEXT:    buffer_wbinvl1_vol
6435 ; GFX7-NEXT:    s_endpgm
6436 ;
6437 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
6438 ; GFX10-WGP:       ; %bb.0: ; %entry
6439 ; GFX10-WGP-NEXT:    s_clause 0x1
6440 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6441 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6442 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6443 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6444 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6445 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6446 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6447 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6448 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6449 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6450 ; GFX10-WGP-NEXT:    buffer_gl0_inv
6451 ; GFX10-WGP-NEXT:    buffer_gl1_inv
6452 ; GFX10-WGP-NEXT:    s_endpgm
6453 ;
6454 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
6455 ; GFX10-CU:       ; %bb.0: ; %entry
6456 ; GFX10-CU-NEXT:    s_clause 0x1
6457 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6458 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6459 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6460 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6461 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6462 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6463 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6464 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6465 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6466 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6467 ; GFX10-CU-NEXT:    buffer_gl0_inv
6468 ; GFX10-CU-NEXT:    buffer_gl1_inv
6469 ; GFX10-CU-NEXT:    s_endpgm
6470 ;
6471 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
6472 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6473 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6474 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6475 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6476 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6477 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6478 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6479 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6480 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6481 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6482 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6483 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6484 ;
6485 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
6486 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6487 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6488 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6489 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6490 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6491 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6492 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6493 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6494 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6495 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6496 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6497 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6498 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6499 ;
6500 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_monotonic_cmpxchg:
6501 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6502 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6503 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6504 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6505 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6506 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6507 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6508 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6509 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6510 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6511 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6512 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6513 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6514     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6515 entry:
6516   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6517   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
6518   ret void
6519 }
6520
6521 define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_cmpxchg(
6522 ; GFX6-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
6523 ; GFX6:       ; %bb.0: ; %entry
6524 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6525 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6526 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6527 ; GFX6-NEXT:    s_mov_b32 s2, -1
6528 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6529 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6530 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6531 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6532 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6533 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6534 ; GFX6-NEXT:    buffer_wbinvl1
6535 ; GFX6-NEXT:    s_endpgm
6536 ;
6537 ; GFX7-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
6538 ; GFX7:       ; %bb.0: ; %entry
6539 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6540 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6541 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6542 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6543 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6544 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6545 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6546 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6547 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6548 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6549 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6550 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6551 ; GFX7-NEXT:    buffer_wbinvl1_vol
6552 ; GFX7-NEXT:    s_endpgm
6553 ;
6554 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
6555 ; GFX10-WGP:       ; %bb.0: ; %entry
6556 ; GFX10-WGP-NEXT:    s_clause 0x1
6557 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6558 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6559 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6560 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6561 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6562 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6563 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6564 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6565 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6566 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6567 ; GFX10-WGP-NEXT:    buffer_gl0_inv
6568 ; GFX10-WGP-NEXT:    buffer_gl1_inv
6569 ; GFX10-WGP-NEXT:    s_endpgm
6570 ;
6571 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
6572 ; GFX10-CU:       ; %bb.0: ; %entry
6573 ; GFX10-CU-NEXT:    s_clause 0x1
6574 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6575 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6576 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6577 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6578 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6579 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6580 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6581 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6582 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6583 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6584 ; GFX10-CU-NEXT:    buffer_gl0_inv
6585 ; GFX10-CU-NEXT:    buffer_gl1_inv
6586 ; GFX10-CU-NEXT:    s_endpgm
6587 ;
6588 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
6589 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6590 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6591 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6592 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6593 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6594 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6595 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6596 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6597 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6598 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6599 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6600 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6601 ;
6602 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
6603 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6604 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6605 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6606 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6607 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6608 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6609 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6610 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6611 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6612 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6613 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6614 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6615 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6616 ;
6617 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_monotonic_cmpxchg:
6618 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6619 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6620 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6621 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6622 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6623 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6624 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6625 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6626 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6627 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6628 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6629 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6630 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6631     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6632 entry:
6633   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6634   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
6635   ret void
6636 }
6637
6638 define amdgpu_kernel void @global_system_one_as_monotonic_acquire_cmpxchg(
6639 ; GFX6-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
6640 ; GFX6:       ; %bb.0: ; %entry
6641 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6642 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6643 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6644 ; GFX6-NEXT:    s_mov_b32 s2, -1
6645 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6646 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6647 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6648 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6649 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6650 ; GFX6-NEXT:    buffer_wbinvl1
6651 ; GFX6-NEXT:    s_endpgm
6652 ;
6653 ; GFX7-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
6654 ; GFX7:       ; %bb.0: ; %entry
6655 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6656 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6657 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6658 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6659 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6660 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6661 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6662 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6663 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6664 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6665 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6666 ; GFX7-NEXT:    buffer_wbinvl1_vol
6667 ; GFX7-NEXT:    s_endpgm
6668 ;
6669 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
6670 ; GFX10-WGP:       ; %bb.0: ; %entry
6671 ; GFX10-WGP-NEXT:    s_clause 0x1
6672 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6673 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6674 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6675 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6676 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6677 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6678 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6679 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6680 ; GFX10-WGP-NEXT:    buffer_gl0_inv
6681 ; GFX10-WGP-NEXT:    buffer_gl1_inv
6682 ; GFX10-WGP-NEXT:    s_endpgm
6683 ;
6684 ; GFX10-CU-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
6685 ; GFX10-CU:       ; %bb.0: ; %entry
6686 ; GFX10-CU-NEXT:    s_clause 0x1
6687 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6688 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6689 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6690 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6691 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6692 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6693 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6694 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6695 ; GFX10-CU-NEXT:    buffer_gl0_inv
6696 ; GFX10-CU-NEXT:    buffer_gl1_inv
6697 ; GFX10-CU-NEXT:    s_endpgm
6698 ;
6699 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
6700 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6701 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6702 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6703 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6704 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6705 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6706 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6707 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6708 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6709 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6710 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6711 ;
6712 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
6713 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6714 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6715 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6716 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6717 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6718 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6719 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6720 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6721 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6722 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6723 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6724 ;
6725 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_acquire_cmpxchg:
6726 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6727 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6728 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6729 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6730 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6731 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6732 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6733 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6734 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6735 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6736 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6737     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6738 entry:
6739   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6740   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
6741   ret void
6742 }
6743
6744 define amdgpu_kernel void @global_system_one_as_acquire_acquire_cmpxchg(
6745 ; GFX6-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
6746 ; GFX6:       ; %bb.0: ; %entry
6747 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6748 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6749 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6750 ; GFX6-NEXT:    s_mov_b32 s2, -1
6751 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6752 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6753 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6754 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6755 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6756 ; GFX6-NEXT:    buffer_wbinvl1
6757 ; GFX6-NEXT:    s_endpgm
6758 ;
6759 ; GFX7-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
6760 ; GFX7:       ; %bb.0: ; %entry
6761 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6762 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6763 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6764 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6765 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6766 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6767 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6768 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6769 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6770 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6771 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6772 ; GFX7-NEXT:    buffer_wbinvl1_vol
6773 ; GFX7-NEXT:    s_endpgm
6774 ;
6775 ; GFX10-WGP-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
6776 ; GFX10-WGP:       ; %bb.0: ; %entry
6777 ; GFX10-WGP-NEXT:    s_clause 0x1
6778 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6779 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6780 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6781 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6782 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6783 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6784 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6785 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6786 ; GFX10-WGP-NEXT:    buffer_gl0_inv
6787 ; GFX10-WGP-NEXT:    buffer_gl1_inv
6788 ; GFX10-WGP-NEXT:    s_endpgm
6789 ;
6790 ; GFX10-CU-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
6791 ; GFX10-CU:       ; %bb.0: ; %entry
6792 ; GFX10-CU-NEXT:    s_clause 0x1
6793 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6794 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6795 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6796 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6797 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6798 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6799 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6800 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6801 ; GFX10-CU-NEXT:    buffer_gl0_inv
6802 ; GFX10-CU-NEXT:    buffer_gl1_inv
6803 ; GFX10-CU-NEXT:    s_endpgm
6804 ;
6805 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
6806 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6807 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6808 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6809 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6810 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6811 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6812 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6813 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6814 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6815 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6816 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6817 ;
6818 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
6819 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6820 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6821 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6822 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6823 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6824 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6825 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6826 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6827 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6828 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6829 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6830 ;
6831 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_acquire_cmpxchg:
6832 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6833 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6834 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6835 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6836 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6837 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6838 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6839 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6840 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6841 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6842 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6843     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6844 entry:
6845   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6846   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
6847   ret void
6848 }
6849
6850 define amdgpu_kernel void @global_system_one_as_release_acquire_cmpxchg(
6851 ; GFX6-LABEL: global_system_one_as_release_acquire_cmpxchg:
6852 ; GFX6:       ; %bb.0: ; %entry
6853 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6854 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6855 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6856 ; GFX6-NEXT:    s_mov_b32 s2, -1
6857 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6858 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6859 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6860 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6861 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6862 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6863 ; GFX6-NEXT:    buffer_wbinvl1
6864 ; GFX6-NEXT:    s_endpgm
6865 ;
6866 ; GFX7-LABEL: global_system_one_as_release_acquire_cmpxchg:
6867 ; GFX7:       ; %bb.0: ; %entry
6868 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6869 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6870 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6871 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6872 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6873 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6874 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6875 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6876 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6877 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6878 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6879 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6880 ; GFX7-NEXT:    buffer_wbinvl1_vol
6881 ; GFX7-NEXT:    s_endpgm
6882 ;
6883 ; GFX10-WGP-LABEL: global_system_one_as_release_acquire_cmpxchg:
6884 ; GFX10-WGP:       ; %bb.0: ; %entry
6885 ; GFX10-WGP-NEXT:    s_clause 0x1
6886 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6887 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6888 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
6889 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
6890 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
6891 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
6892 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
6893 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6894 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6895 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
6896 ; GFX10-WGP-NEXT:    buffer_gl0_inv
6897 ; GFX10-WGP-NEXT:    buffer_gl1_inv
6898 ; GFX10-WGP-NEXT:    s_endpgm
6899 ;
6900 ; GFX10-CU-LABEL: global_system_one_as_release_acquire_cmpxchg:
6901 ; GFX10-CU:       ; %bb.0: ; %entry
6902 ; GFX10-CU-NEXT:    s_clause 0x1
6903 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
6904 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
6905 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
6906 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
6907 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
6908 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
6909 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
6910 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6911 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
6912 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
6913 ; GFX10-CU-NEXT:    buffer_gl0_inv
6914 ; GFX10-CU-NEXT:    buffer_gl1_inv
6915 ; GFX10-CU-NEXT:    s_endpgm
6916 ;
6917 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_acquire_cmpxchg:
6918 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
6919 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
6920 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
6921 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
6922 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
6923 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
6924 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
6925 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
6926 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6927 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
6928 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
6929 ; SKIP-CACHE-INV-NEXT:    s_endpgm
6930 ;
6931 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_acquire_cmpxchg:
6932 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
6933 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6934 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6935 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6936 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6937 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6938 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
6939 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6940 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6941 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6942 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
6943 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
6944 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
6945 ;
6946 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_acquire_cmpxchg:
6947 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
6948 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6949 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
6950 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
6951 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
6952 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
6953 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
6954 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6955 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
6956 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
6957 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
6958 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
6959 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
6960     i32 addrspace(1)* %out, i32 %in, i32 %old) {
6961 entry:
6962   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
6963   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
6964   ret void
6965 }
6966
6967 define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_cmpxchg(
6968 ; GFX6-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
6969 ; GFX6:       ; %bb.0: ; %entry
6970 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6971 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
6972 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
6973 ; GFX6-NEXT:    s_mov_b32 s2, -1
6974 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
6975 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
6976 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
6977 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6978 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
6979 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
6980 ; GFX6-NEXT:    buffer_wbinvl1
6981 ; GFX6-NEXT:    s_endpgm
6982 ;
6983 ; GFX7-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
6984 ; GFX7:       ; %bb.0: ; %entry
6985 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
6986 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
6987 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
6988 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
6989 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
6990 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
6991 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
6992 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
6993 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
6994 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6995 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
6996 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
6997 ; GFX7-NEXT:    buffer_wbinvl1_vol
6998 ; GFX7-NEXT:    s_endpgm
6999 ;
7000 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
7001 ; GFX10-WGP:       ; %bb.0: ; %entry
7002 ; GFX10-WGP-NEXT:    s_clause 0x1
7003 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7004 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7005 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7006 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7007 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7008 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7009 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7010 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7011 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7012 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7013 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7014 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7015 ; GFX10-WGP-NEXT:    s_endpgm
7016 ;
7017 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
7018 ; GFX10-CU:       ; %bb.0: ; %entry
7019 ; GFX10-CU-NEXT:    s_clause 0x1
7020 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7021 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7022 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7023 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7024 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7025 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7026 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7027 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7028 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7029 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7030 ; GFX10-CU-NEXT:    buffer_gl0_inv
7031 ; GFX10-CU-NEXT:    buffer_gl1_inv
7032 ; GFX10-CU-NEXT:    s_endpgm
7033 ;
7034 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
7035 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7036 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7037 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7038 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7039 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7040 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7041 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7042 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7043 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7044 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
7045 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7046 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7047 ;
7048 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
7049 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7050 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7051 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7052 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7053 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7054 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7055 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7056 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7057 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7058 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7059 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7060 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7061 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7062 ;
7063 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_acquire_cmpxchg:
7064 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7065 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7066 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7067 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7068 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7069 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7070 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7071 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7072 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7073 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7074 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7075 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7076 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7077     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7078 entry:
7079   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7080   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
7081   ret void
7082 }
7083
7084 define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_cmpxchg(
7085 ; GFX6-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
7086 ; GFX6:       ; %bb.0: ; %entry
7087 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7088 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7089 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7090 ; GFX6-NEXT:    s_mov_b32 s2, -1
7091 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7092 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7093 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7094 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7095 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7096 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7097 ; GFX6-NEXT:    buffer_wbinvl1
7098 ; GFX6-NEXT:    s_endpgm
7099 ;
7100 ; GFX7-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
7101 ; GFX7:       ; %bb.0: ; %entry
7102 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7103 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7104 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7105 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
7106 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7107 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7108 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7109 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7110 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7111 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7112 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7113 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7114 ; GFX7-NEXT:    buffer_wbinvl1_vol
7115 ; GFX7-NEXT:    s_endpgm
7116 ;
7117 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
7118 ; GFX10-WGP:       ; %bb.0: ; %entry
7119 ; GFX10-WGP-NEXT:    s_clause 0x1
7120 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7121 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7122 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7123 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7124 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7125 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7126 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7127 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7128 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7129 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7130 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7131 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7132 ; GFX10-WGP-NEXT:    s_endpgm
7133 ;
7134 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
7135 ; GFX10-CU:       ; %bb.0: ; %entry
7136 ; GFX10-CU-NEXT:    s_clause 0x1
7137 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7138 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7139 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7140 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7141 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7142 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7143 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7144 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7145 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7146 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7147 ; GFX10-CU-NEXT:    buffer_gl0_inv
7148 ; GFX10-CU-NEXT:    buffer_gl1_inv
7149 ; GFX10-CU-NEXT:    s_endpgm
7150 ;
7151 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
7152 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7153 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7154 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7155 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7156 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7157 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7158 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7159 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7160 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7161 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
7162 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7163 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7164 ;
7165 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
7166 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7167 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7168 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7169 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7170 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7171 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7172 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7173 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7174 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7175 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7176 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7177 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7178 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7179 ;
7180 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_acquire_cmpxchg:
7181 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7182 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7183 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7184 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7185 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7186 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7187 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7188 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7189 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7190 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7191 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7192 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7193 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7194     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7195 entry:
7196   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7197   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
7198   ret void
7199 }
7200
7201 define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_cmpxchg(
7202 ; GFX6-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
7203 ; GFX6:       ; %bb.0: ; %entry
7204 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7205 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7206 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7207 ; GFX6-NEXT:    s_mov_b32 s2, -1
7208 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7209 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7210 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7211 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7212 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7213 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7214 ; GFX6-NEXT:    buffer_wbinvl1
7215 ; GFX6-NEXT:    s_endpgm
7216 ;
7217 ; GFX7-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
7218 ; GFX7:       ; %bb.0: ; %entry
7219 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7220 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7221 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7222 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
7223 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7224 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7225 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7226 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7227 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7228 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7229 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7230 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7231 ; GFX7-NEXT:    buffer_wbinvl1_vol
7232 ; GFX7-NEXT:    s_endpgm
7233 ;
7234 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
7235 ; GFX10-WGP:       ; %bb.0: ; %entry
7236 ; GFX10-WGP-NEXT:    s_clause 0x1
7237 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7238 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7239 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7240 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7241 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7242 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7243 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7244 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7245 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7246 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7247 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7248 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7249 ; GFX10-WGP-NEXT:    s_endpgm
7250 ;
7251 ; GFX10-CU-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
7252 ; GFX10-CU:       ; %bb.0: ; %entry
7253 ; GFX10-CU-NEXT:    s_clause 0x1
7254 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7255 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7256 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7257 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7258 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7259 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7260 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7261 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7262 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7263 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7264 ; GFX10-CU-NEXT:    buffer_gl0_inv
7265 ; GFX10-CU-NEXT:    buffer_gl1_inv
7266 ; GFX10-CU-NEXT:    s_endpgm
7267 ;
7268 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
7269 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7270 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7271 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7272 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7273 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7274 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7275 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7276 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7277 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7278 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
7279 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7280 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7281 ;
7282 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
7283 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7284 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7285 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7286 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7287 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7288 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7289 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7290 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7291 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7292 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7293 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7294 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7295 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7296 ;
7297 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_seq_cst_cmpxchg:
7298 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7299 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7300 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7301 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7302 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7303 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7304 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7305 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7306 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7307 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7308 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7309 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7310 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7311     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7312 entry:
7313   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7314   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
7315   ret void
7316 }
7317
7318 define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_cmpxchg(
7319 ; GFX6-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
7320 ; GFX6:       ; %bb.0: ; %entry
7321 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7322 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7323 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7324 ; GFX6-NEXT:    s_mov_b32 s2, -1
7325 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7326 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7327 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7328 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7329 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7330 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7331 ; GFX6-NEXT:    buffer_wbinvl1
7332 ; GFX6-NEXT:    s_endpgm
7333 ;
7334 ; GFX7-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
7335 ; GFX7:       ; %bb.0: ; %entry
7336 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7337 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7338 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7339 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
7340 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7341 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7342 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7343 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7344 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7345 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7346 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7347 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7348 ; GFX7-NEXT:    buffer_wbinvl1_vol
7349 ; GFX7-NEXT:    s_endpgm
7350 ;
7351 ; GFX10-WGP-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
7352 ; GFX10-WGP:       ; %bb.0: ; %entry
7353 ; GFX10-WGP-NEXT:    s_clause 0x1
7354 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7355 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7356 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7357 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7358 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7359 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7360 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7361 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7362 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7363 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7364 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7365 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7366 ; GFX10-WGP-NEXT:    s_endpgm
7367 ;
7368 ; GFX10-CU-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
7369 ; GFX10-CU:       ; %bb.0: ; %entry
7370 ; GFX10-CU-NEXT:    s_clause 0x1
7371 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7372 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7373 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7374 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7375 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7376 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7377 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7378 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7379 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7380 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7381 ; GFX10-CU-NEXT:    buffer_gl0_inv
7382 ; GFX10-CU-NEXT:    buffer_gl1_inv
7383 ; GFX10-CU-NEXT:    s_endpgm
7384 ;
7385 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
7386 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7387 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7388 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7389 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7390 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7391 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7392 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7393 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7394 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7395 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
7396 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7397 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7398 ;
7399 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
7400 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7401 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7402 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7403 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7404 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7405 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7406 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7407 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7408 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7409 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7410 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7411 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7412 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7413 ;
7414 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_seq_cst_cmpxchg:
7415 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7416 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7417 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7418 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7419 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7420 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7421 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7422 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7423 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7424 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7425 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7426 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7427 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7428     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7429 entry:
7430   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7431   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
7432   ret void
7433 }
7434
7435 define amdgpu_kernel void @global_system_one_as_release_seq_cst_cmpxchg(
7436 ; GFX6-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
7437 ; GFX6:       ; %bb.0: ; %entry
7438 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7439 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7440 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7441 ; GFX6-NEXT:    s_mov_b32 s2, -1
7442 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7443 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7444 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7445 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7446 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7447 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7448 ; GFX6-NEXT:    buffer_wbinvl1
7449 ; GFX6-NEXT:    s_endpgm
7450 ;
7451 ; GFX7-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
7452 ; GFX7:       ; %bb.0: ; %entry
7453 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7454 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7455 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7456 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
7457 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7458 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7459 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7460 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7461 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7462 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7463 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7464 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7465 ; GFX7-NEXT:    buffer_wbinvl1_vol
7466 ; GFX7-NEXT:    s_endpgm
7467 ;
7468 ; GFX10-WGP-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
7469 ; GFX10-WGP:       ; %bb.0: ; %entry
7470 ; GFX10-WGP-NEXT:    s_clause 0x1
7471 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7472 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7473 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7474 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7475 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7476 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7477 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7478 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7479 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7480 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7481 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7482 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7483 ; GFX10-WGP-NEXT:    s_endpgm
7484 ;
7485 ; GFX10-CU-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
7486 ; GFX10-CU:       ; %bb.0: ; %entry
7487 ; GFX10-CU-NEXT:    s_clause 0x1
7488 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7489 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7490 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7491 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7492 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7493 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7494 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7495 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7496 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7497 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7498 ; GFX10-CU-NEXT:    buffer_gl0_inv
7499 ; GFX10-CU-NEXT:    buffer_gl1_inv
7500 ; GFX10-CU-NEXT:    s_endpgm
7501 ;
7502 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
7503 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7504 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7505 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7506 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7507 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7508 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7509 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7510 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7511 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7512 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
7513 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7514 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7515 ;
7516 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
7517 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7518 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7519 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7520 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7521 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7522 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7523 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7524 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7525 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7526 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7527 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7528 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7529 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7530 ;
7531 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_seq_cst_cmpxchg:
7532 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7533 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7534 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7535 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7536 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7537 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7538 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7539 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7540 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7541 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7542 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7543 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7544 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7545     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7546 entry:
7547   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7548   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
7549   ret void
7550 }
7551
7552 define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_cmpxchg(
7553 ; GFX6-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
7554 ; GFX6:       ; %bb.0: ; %entry
7555 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7556 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7557 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7558 ; GFX6-NEXT:    s_mov_b32 s2, -1
7559 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7560 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7561 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7562 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7563 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7564 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7565 ; GFX6-NEXT:    buffer_wbinvl1
7566 ; GFX6-NEXT:    s_endpgm
7567 ;
7568 ; GFX7-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
7569 ; GFX7:       ; %bb.0: ; %entry
7570 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7571 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7572 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7573 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
7574 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7575 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7576 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7577 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7578 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7579 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7580 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7581 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7582 ; GFX7-NEXT:    buffer_wbinvl1_vol
7583 ; GFX7-NEXT:    s_endpgm
7584 ;
7585 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
7586 ; GFX10-WGP:       ; %bb.0: ; %entry
7587 ; GFX10-WGP-NEXT:    s_clause 0x1
7588 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7589 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7590 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7591 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7592 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7593 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7594 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7595 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7596 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7597 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7598 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7599 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7600 ; GFX10-WGP-NEXT:    s_endpgm
7601 ;
7602 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
7603 ; GFX10-CU:       ; %bb.0: ; %entry
7604 ; GFX10-CU-NEXT:    s_clause 0x1
7605 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7606 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7607 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7608 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7609 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7610 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7611 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7612 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7613 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7614 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7615 ; GFX10-CU-NEXT:    buffer_gl0_inv
7616 ; GFX10-CU-NEXT:    buffer_gl1_inv
7617 ; GFX10-CU-NEXT:    s_endpgm
7618 ;
7619 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
7620 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7621 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7622 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7623 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7624 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7625 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7626 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7627 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7628 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7629 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
7630 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7631 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7632 ;
7633 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
7634 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7635 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7636 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7637 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7638 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7639 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7640 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7641 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7642 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7643 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7644 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7645 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7646 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7647 ;
7648 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_seq_cst_cmpxchg:
7649 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7650 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7651 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7652 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7653 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7654 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7655 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7656 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7657 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7658 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7659 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7660 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7661 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7662     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7663 entry:
7664   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7665   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
7666   ret void
7667 }
7668
7669 define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_cmpxchg(
7670 ; GFX6-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
7671 ; GFX6:       ; %bb.0: ; %entry
7672 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7673 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7674 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7675 ; GFX6-NEXT:    s_mov_b32 s2, -1
7676 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7677 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7678 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7679 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7680 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16
7681 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7682 ; GFX6-NEXT:    buffer_wbinvl1
7683 ; GFX6-NEXT:    s_endpgm
7684 ;
7685 ; GFX7-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
7686 ; GFX7:       ; %bb.0: ; %entry
7687 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7688 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7689 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7690 ; GFX7-NEXT:    s_add_u32 s0, s0, 16
7691 ; GFX7-NEXT:    s_addc_u32 s1, s1, 0
7692 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7693 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7694 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7695 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7696 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7697 ; GFX7-NEXT:    flat_atomic_cmpswap v[0:1], v[2:3]
7698 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7699 ; GFX7-NEXT:    buffer_wbinvl1_vol
7700 ; GFX7-NEXT:    s_endpgm
7701 ;
7702 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
7703 ; GFX10-WGP:       ; %bb.0: ; %entry
7704 ; GFX10-WGP-NEXT:    s_clause 0x1
7705 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7706 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7707 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7708 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7709 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7710 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7711 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7712 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7713 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7714 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
7715 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7716 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7717 ; GFX10-WGP-NEXT:    s_endpgm
7718 ;
7719 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
7720 ; GFX10-CU:       ; %bb.0: ; %entry
7721 ; GFX10-CU-NEXT:    s_clause 0x1
7722 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7723 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7724 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7725 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7726 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7727 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7728 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7729 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7730 ; GFX10-CU-NEXT:    global_atomic_cmpswap v2, v[0:1], s[2:3] offset:16
7731 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
7732 ; GFX10-CU-NEXT:    buffer_gl0_inv
7733 ; GFX10-CU-NEXT:    buffer_gl1_inv
7734 ; GFX10-CU-NEXT:    s_endpgm
7735 ;
7736 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
7737 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7738 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7739 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7740 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7741 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7742 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7743 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7744 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7745 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7746 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
7747 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7748 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7749 ;
7750 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
7751 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7752 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7753 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7754 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7755 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7756 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7757 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
7758 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7759 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7760 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7761 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7762 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7763 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7764 ;
7765 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_seq_cst_cmpxchg:
7766 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7767 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7768 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7769 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7770 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7771 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7772 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
7773 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7774 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
7775 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7776 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7777 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7778 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7779     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7780 entry:
7781   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7782   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
7783   ret void
7784 }
7785
7786 define amdgpu_kernel void @global_system_one_as_monotonic_monotonic_ret_cmpxchg(
7787 ; GFX6-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
7788 ; GFX6:       ; %bb.0: ; %entry
7789 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7790 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7791 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7792 ; GFX6-NEXT:    s_mov_b32 s2, -1
7793 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7794 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7795 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7796 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7797 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7798 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7799 ; GFX6-NEXT:    s_endpgm
7800 ;
7801 ; GFX7-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
7802 ; GFX7:       ; %bb.0: ; %entry
7803 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7804 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7805 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7806 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
7807 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7808 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7809 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7810 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7811 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7812 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7813 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7814 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7815 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7816 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
7817 ; GFX7-NEXT:    s_endpgm
7818 ;
7819 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
7820 ; GFX10-WGP:       ; %bb.0: ; %entry
7821 ; GFX10-WGP-NEXT:    s_clause 0x1
7822 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7823 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7824 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7825 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7826 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7827 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7828 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7829 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7830 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7831 ; GFX10-WGP-NEXT:    s_endpgm
7832 ;
7833 ; GFX10-CU-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
7834 ; GFX10-CU:       ; %bb.0: ; %entry
7835 ; GFX10-CU-NEXT:    s_clause 0x1
7836 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7837 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7838 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7839 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7840 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7841 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7842 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7843 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7844 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7845 ; GFX10-CU-NEXT:    s_endpgm
7846 ;
7847 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
7848 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7849 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7850 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7851 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7852 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7853 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7854 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7855 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7856 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
7857 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7858 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
7859 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7860 ;
7861 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
7862 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7863 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7864 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7865 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7866 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7867 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7868 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
7869 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7870 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
7871 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7872 ;
7873 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_monotonic_ret_cmpxchg:
7874 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7875 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7876 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7877 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7878 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7879 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7880 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
7881 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7882 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
7883 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
7884     i32 addrspace(1)* %out, i32 %in, i32 %old) {
7885 entry:
7886   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
7887   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic monotonic
7888   %val0 = extractvalue { i32, i1 } %val, 0
7889   store i32 %val0, i32 addrspace(1)* %out, align 4
7890   ret void
7891 }
7892
7893 define amdgpu_kernel void @global_system_one_as_acquire_monotonic_ret_cmpxchg(
7894 ; GFX6-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
7895 ; GFX6:       ; %bb.0: ; %entry
7896 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7897 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
7898 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
7899 ; GFX6-NEXT:    s_mov_b32 s2, -1
7900 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
7901 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
7902 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
7903 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
7904 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
7905 ; GFX6-NEXT:    buffer_wbinvl1
7906 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
7907 ; GFX6-NEXT:    s_endpgm
7908 ;
7909 ; GFX7-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
7910 ; GFX7:       ; %bb.0: ; %entry
7911 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7912 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
7913 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
7914 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
7915 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
7916 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
7917 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
7918 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
7919 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
7920 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
7921 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
7922 ; GFX7-NEXT:    buffer_wbinvl1_vol
7923 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
7924 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
7925 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
7926 ; GFX7-NEXT:    s_endpgm
7927 ;
7928 ; GFX10-WGP-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
7929 ; GFX10-WGP:       ; %bb.0: ; %entry
7930 ; GFX10-WGP-NEXT:    s_clause 0x1
7931 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7932 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7933 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
7934 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
7935 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
7936 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
7937 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7938 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
7939 ; GFX10-WGP-NEXT:    buffer_gl0_inv
7940 ; GFX10-WGP-NEXT:    buffer_gl1_inv
7941 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
7942 ; GFX10-WGP-NEXT:    s_endpgm
7943 ;
7944 ; GFX10-CU-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
7945 ; GFX10-CU:       ; %bb.0: ; %entry
7946 ; GFX10-CU-NEXT:    s_clause 0x1
7947 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
7948 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
7949 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
7950 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
7951 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
7952 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
7953 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
7954 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
7955 ; GFX10-CU-NEXT:    buffer_gl0_inv
7956 ; GFX10-CU-NEXT:    buffer_gl1_inv
7957 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
7958 ; GFX10-CU-NEXT:    s_endpgm
7959 ;
7960 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
7961 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
7962 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
7963 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
7964 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
7965 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
7966 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
7967 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
7968 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
7969 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
7970 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
7971 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
7972 ; SKIP-CACHE-INV-NEXT:    s_endpgm
7973 ;
7974 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
7975 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
7976 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7977 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7978 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7979 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7980 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7981 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
7982 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7983 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
7984 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
7985 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
7986 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
7987 ;
7988 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_monotonic_ret_cmpxchg:
7989 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
7990 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
7991 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
7992 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
7993 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
7994 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
7995 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
7996 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
7997 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
7998 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
7999 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8000 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8001     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8002 entry:
8003   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8004   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire monotonic
8005   %val0 = extractvalue { i32, i1 } %val, 0
8006   store i32 %val0, i32 addrspace(1)* %out, align 4
8007   ret void
8008 }
8009
8010 define amdgpu_kernel void @global_system_one_as_release_monotonic_ret_cmpxchg(
8011 ; GFX6-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
8012 ; GFX6:       ; %bb.0: ; %entry
8013 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8014 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8015 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8016 ; GFX6-NEXT:    s_mov_b32 s2, -1
8017 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8018 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8019 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8020 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8021 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8022 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8023 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8024 ; GFX6-NEXT:    s_endpgm
8025 ;
8026 ; GFX7-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
8027 ; GFX7:       ; %bb.0: ; %entry
8028 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8029 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8030 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8031 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8032 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8033 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8034 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8035 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8036 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8037 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8038 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8039 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8040 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8041 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8042 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8043 ; GFX7-NEXT:    s_endpgm
8044 ;
8045 ; GFX10-WGP-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
8046 ; GFX10-WGP:       ; %bb.0: ; %entry
8047 ; GFX10-WGP-NEXT:    s_clause 0x1
8048 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8049 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8050 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8051 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8052 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8053 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8054 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8055 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8056 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8057 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8058 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8059 ; GFX10-WGP-NEXT:    s_endpgm
8060 ;
8061 ; GFX10-CU-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
8062 ; GFX10-CU:       ; %bb.0: ; %entry
8063 ; GFX10-CU-NEXT:    s_clause 0x1
8064 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8065 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8066 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8067 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8068 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8069 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8070 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8071 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8072 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8073 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8074 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8075 ; GFX10-CU-NEXT:    s_endpgm
8076 ;
8077 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
8078 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8079 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8080 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8081 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8082 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8083 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8084 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8085 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8086 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8087 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8088 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8089 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8090 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8091 ;
8092 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
8093 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8094 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8095 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8096 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8097 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8098 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8099 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8100 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8101 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8102 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8103 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8104 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8105 ;
8106 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_monotonic_ret_cmpxchg:
8107 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8108 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8109 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8110 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8111 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8112 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8113 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8114 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8115 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8116 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8117 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8118 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8119     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8120 entry:
8121   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8122   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release monotonic
8123   %val0 = extractvalue { i32, i1 } %val, 0
8124   store i32 %val0, i32 addrspace(1)* %out, align 4
8125   ret void
8126 }
8127
8128 define amdgpu_kernel void @global_system_one_as_acq_rel_monotonic_ret_cmpxchg(
8129 ; GFX6-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
8130 ; GFX6:       ; %bb.0: ; %entry
8131 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8132 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8133 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8134 ; GFX6-NEXT:    s_mov_b32 s2, -1
8135 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8136 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8137 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8138 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8139 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8140 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8141 ; GFX6-NEXT:    buffer_wbinvl1
8142 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8143 ; GFX6-NEXT:    s_endpgm
8144 ;
8145 ; GFX7-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
8146 ; GFX7:       ; %bb.0: ; %entry
8147 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8148 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8149 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8150 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8151 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8152 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8153 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8154 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8155 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8156 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8157 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8158 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8159 ; GFX7-NEXT:    buffer_wbinvl1_vol
8160 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8161 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8162 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8163 ; GFX7-NEXT:    s_endpgm
8164 ;
8165 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
8166 ; GFX10-WGP:       ; %bb.0: ; %entry
8167 ; GFX10-WGP-NEXT:    s_clause 0x1
8168 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8169 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8170 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8171 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8172 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8173 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8174 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8175 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8176 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8177 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8178 ; GFX10-WGP-NEXT:    buffer_gl0_inv
8179 ; GFX10-WGP-NEXT:    buffer_gl1_inv
8180 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8181 ; GFX10-WGP-NEXT:    s_endpgm
8182 ;
8183 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
8184 ; GFX10-CU:       ; %bb.0: ; %entry
8185 ; GFX10-CU-NEXT:    s_clause 0x1
8186 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8187 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8188 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8189 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8190 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8191 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8192 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8193 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8194 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8195 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8196 ; GFX10-CU-NEXT:    buffer_gl0_inv
8197 ; GFX10-CU-NEXT:    buffer_gl1_inv
8198 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8199 ; GFX10-CU-NEXT:    s_endpgm
8200 ;
8201 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
8202 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8203 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8204 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8205 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8206 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8207 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8208 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8209 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8210 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8211 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8212 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8213 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8214 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8215 ;
8216 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
8217 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8218 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8219 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8220 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8221 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8222 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8223 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8224 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8225 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8226 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8227 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8228 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8229 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8230 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8231 ;
8232 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_monotonic_ret_cmpxchg:
8233 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8234 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8235 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8236 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8237 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8238 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8239 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8240 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8241 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8242 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8243 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8244 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8245 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8246 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8247     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8248 entry:
8249   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8250   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel monotonic
8251   %val0 = extractvalue { i32, i1 } %val, 0
8252   store i32 %val0, i32 addrspace(1)* %out, align 4
8253   ret void
8254 }
8255
8256 define amdgpu_kernel void @global_system_one_as_seq_cst_monotonic_ret_cmpxchg(
8257 ; GFX6-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
8258 ; GFX6:       ; %bb.0: ; %entry
8259 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8260 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8261 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8262 ; GFX6-NEXT:    s_mov_b32 s2, -1
8263 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8264 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8265 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8266 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8267 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8268 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8269 ; GFX6-NEXT:    buffer_wbinvl1
8270 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8271 ; GFX6-NEXT:    s_endpgm
8272 ;
8273 ; GFX7-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
8274 ; GFX7:       ; %bb.0: ; %entry
8275 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8276 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8277 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8278 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8279 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8280 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8281 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8282 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8283 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8284 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8285 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8286 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8287 ; GFX7-NEXT:    buffer_wbinvl1_vol
8288 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8289 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8290 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8291 ; GFX7-NEXT:    s_endpgm
8292 ;
8293 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
8294 ; GFX10-WGP:       ; %bb.0: ; %entry
8295 ; GFX10-WGP-NEXT:    s_clause 0x1
8296 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8297 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8298 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8299 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8300 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8301 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8302 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8303 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8304 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8305 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8306 ; GFX10-WGP-NEXT:    buffer_gl0_inv
8307 ; GFX10-WGP-NEXT:    buffer_gl1_inv
8308 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8309 ; GFX10-WGP-NEXT:    s_endpgm
8310 ;
8311 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
8312 ; GFX10-CU:       ; %bb.0: ; %entry
8313 ; GFX10-CU-NEXT:    s_clause 0x1
8314 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8315 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8316 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8317 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8318 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8319 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8320 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8321 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8322 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8323 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8324 ; GFX10-CU-NEXT:    buffer_gl0_inv
8325 ; GFX10-CU-NEXT:    buffer_gl1_inv
8326 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8327 ; GFX10-CU-NEXT:    s_endpgm
8328 ;
8329 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
8330 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8331 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8332 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8333 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8334 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8335 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8336 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8337 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8338 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8339 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8340 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8341 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8342 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8343 ;
8344 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
8345 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8346 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8347 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8348 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8349 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8350 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8351 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8352 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8353 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8354 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8355 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8356 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8357 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8358 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8359 ;
8360 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_monotonic_ret_cmpxchg:
8361 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8362 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8363 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8364 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8365 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8366 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8367 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8368 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8369 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8370 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8371 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8372 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8373 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8374 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8375     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8376 entry:
8377   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8378   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst monotonic
8379   %val0 = extractvalue { i32, i1 } %val, 0
8380   store i32 %val0, i32 addrspace(1)* %out, align 4
8381   ret void
8382 }
8383
8384 define amdgpu_kernel void @global_system_one_as_monotonic_acquire_ret_cmpxchg(
8385 ; GFX6-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
8386 ; GFX6:       ; %bb.0: ; %entry
8387 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8388 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8389 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8390 ; GFX6-NEXT:    s_mov_b32 s2, -1
8391 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8392 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8393 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8394 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8395 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8396 ; GFX6-NEXT:    buffer_wbinvl1
8397 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8398 ; GFX6-NEXT:    s_endpgm
8399 ;
8400 ; GFX7-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
8401 ; GFX7:       ; %bb.0: ; %entry
8402 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8403 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8404 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8405 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8406 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8407 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8408 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8409 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8410 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8411 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8412 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8413 ; GFX7-NEXT:    buffer_wbinvl1_vol
8414 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8415 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8416 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8417 ; GFX7-NEXT:    s_endpgm
8418 ;
8419 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
8420 ; GFX10-WGP:       ; %bb.0: ; %entry
8421 ; GFX10-WGP-NEXT:    s_clause 0x1
8422 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8423 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8424 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8425 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8426 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8427 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8428 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8429 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8430 ; GFX10-WGP-NEXT:    buffer_gl0_inv
8431 ; GFX10-WGP-NEXT:    buffer_gl1_inv
8432 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8433 ; GFX10-WGP-NEXT:    s_endpgm
8434 ;
8435 ; GFX10-CU-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
8436 ; GFX10-CU:       ; %bb.0: ; %entry
8437 ; GFX10-CU-NEXT:    s_clause 0x1
8438 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8439 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8440 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8441 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8442 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8443 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8444 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8445 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8446 ; GFX10-CU-NEXT:    buffer_gl0_inv
8447 ; GFX10-CU-NEXT:    buffer_gl1_inv
8448 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8449 ; GFX10-CU-NEXT:    s_endpgm
8450 ;
8451 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
8452 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8453 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8454 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8455 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8456 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8457 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8458 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8459 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8460 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8461 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8462 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8463 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8464 ;
8465 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
8466 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8467 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8468 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8469 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8470 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8471 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8472 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8473 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8474 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8475 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8476 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8477 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8478 ;
8479 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_acquire_ret_cmpxchg:
8480 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8481 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8482 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8483 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8484 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8485 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8486 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8487 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8488 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8489 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8490 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8491 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8492     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8493 entry:
8494   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8495   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic acquire
8496   %val0 = extractvalue { i32, i1 } %val, 0
8497   store i32 %val0, i32 addrspace(1)* %out, align 4
8498   ret void
8499 }
8500
8501 define amdgpu_kernel void @global_system_one_as_acquire_acquire_ret_cmpxchg(
8502 ; GFX6-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
8503 ; GFX6:       ; %bb.0: ; %entry
8504 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8505 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8506 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8507 ; GFX6-NEXT:    s_mov_b32 s2, -1
8508 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8509 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8510 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8511 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8512 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8513 ; GFX6-NEXT:    buffer_wbinvl1
8514 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8515 ; GFX6-NEXT:    s_endpgm
8516 ;
8517 ; GFX7-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
8518 ; GFX7:       ; %bb.0: ; %entry
8519 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8520 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8521 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8522 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8523 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8524 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8525 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8526 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8527 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8528 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8529 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8530 ; GFX7-NEXT:    buffer_wbinvl1_vol
8531 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8532 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8533 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8534 ; GFX7-NEXT:    s_endpgm
8535 ;
8536 ; GFX10-WGP-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
8537 ; GFX10-WGP:       ; %bb.0: ; %entry
8538 ; GFX10-WGP-NEXT:    s_clause 0x1
8539 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8540 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8541 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8542 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8543 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8544 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8545 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8546 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8547 ; GFX10-WGP-NEXT:    buffer_gl0_inv
8548 ; GFX10-WGP-NEXT:    buffer_gl1_inv
8549 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8550 ; GFX10-WGP-NEXT:    s_endpgm
8551 ;
8552 ; GFX10-CU-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
8553 ; GFX10-CU:       ; %bb.0: ; %entry
8554 ; GFX10-CU-NEXT:    s_clause 0x1
8555 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8556 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8557 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8558 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8559 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8560 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8561 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8562 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8563 ; GFX10-CU-NEXT:    buffer_gl0_inv
8564 ; GFX10-CU-NEXT:    buffer_gl1_inv
8565 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8566 ; GFX10-CU-NEXT:    s_endpgm
8567 ;
8568 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
8569 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8570 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8571 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8572 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8573 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8574 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8575 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8576 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8577 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8578 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8579 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8580 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8581 ;
8582 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
8583 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8584 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8585 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8586 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8587 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8588 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8589 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8590 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8591 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8592 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8593 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8594 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8595 ;
8596 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_acquire_ret_cmpxchg:
8597 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8598 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8599 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8600 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8601 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8602 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8603 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8604 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8605 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8606 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8607 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8608 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8609     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8610 entry:
8611   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8612   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire acquire
8613   %val0 = extractvalue { i32, i1 } %val, 0
8614   store i32 %val0, i32 addrspace(1)* %out, align 4
8615   ret void
8616 }
8617
8618 define amdgpu_kernel void @global_system_one_as_release_acquire_ret_cmpxchg(
8619 ; GFX6-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
8620 ; GFX6:       ; %bb.0: ; %entry
8621 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8622 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8623 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8624 ; GFX6-NEXT:    s_mov_b32 s2, -1
8625 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8626 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8627 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8628 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8629 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8630 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8631 ; GFX6-NEXT:    buffer_wbinvl1
8632 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8633 ; GFX6-NEXT:    s_endpgm
8634 ;
8635 ; GFX7-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
8636 ; GFX7:       ; %bb.0: ; %entry
8637 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8638 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8639 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8640 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8641 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8642 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8643 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8644 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8645 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8646 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8647 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8648 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8649 ; GFX7-NEXT:    buffer_wbinvl1_vol
8650 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8651 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8652 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8653 ; GFX7-NEXT:    s_endpgm
8654 ;
8655 ; GFX10-WGP-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
8656 ; GFX10-WGP:       ; %bb.0: ; %entry
8657 ; GFX10-WGP-NEXT:    s_clause 0x1
8658 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8659 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8660 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8661 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8662 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8663 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8664 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8665 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8666 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8667 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8668 ; GFX10-WGP-NEXT:    buffer_gl0_inv
8669 ; GFX10-WGP-NEXT:    buffer_gl1_inv
8670 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8671 ; GFX10-WGP-NEXT:    s_endpgm
8672 ;
8673 ; GFX10-CU-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
8674 ; GFX10-CU:       ; %bb.0: ; %entry
8675 ; GFX10-CU-NEXT:    s_clause 0x1
8676 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8677 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8678 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8679 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8680 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8681 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8682 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8683 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8684 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8685 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8686 ; GFX10-CU-NEXT:    buffer_gl0_inv
8687 ; GFX10-CU-NEXT:    buffer_gl1_inv
8688 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8689 ; GFX10-CU-NEXT:    s_endpgm
8690 ;
8691 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
8692 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8693 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8694 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8695 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8696 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8697 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8698 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8699 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8700 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8701 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8702 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8703 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8704 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8705 ;
8706 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
8707 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8708 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8709 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8710 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8711 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8712 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8713 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8714 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8715 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8716 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8717 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8718 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8719 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8720 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8721 ;
8722 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_acquire_ret_cmpxchg:
8723 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8724 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8725 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8726 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8727 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8728 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8729 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8730 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8731 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8732 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8733 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8734 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8735 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8736 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8737     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8738 entry:
8739   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8740   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release acquire
8741   %val0 = extractvalue { i32, i1 } %val, 0
8742   store i32 %val0, i32 addrspace(1)* %out, align 4
8743   ret void
8744 }
8745
8746 define amdgpu_kernel void @global_system_one_as_acq_rel_acquire_ret_cmpxchg(
8747 ; GFX6-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
8748 ; GFX6:       ; %bb.0: ; %entry
8749 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8750 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8751 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8752 ; GFX6-NEXT:    s_mov_b32 s2, -1
8753 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8754 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8755 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8756 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8757 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8758 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8759 ; GFX6-NEXT:    buffer_wbinvl1
8760 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8761 ; GFX6-NEXT:    s_endpgm
8762 ;
8763 ; GFX7-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
8764 ; GFX7:       ; %bb.0: ; %entry
8765 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8766 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8767 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8768 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8769 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8770 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8771 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8772 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8773 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8774 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8775 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8776 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8777 ; GFX7-NEXT:    buffer_wbinvl1_vol
8778 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8779 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8780 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8781 ; GFX7-NEXT:    s_endpgm
8782 ;
8783 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
8784 ; GFX10-WGP:       ; %bb.0: ; %entry
8785 ; GFX10-WGP-NEXT:    s_clause 0x1
8786 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8787 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8788 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8789 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8790 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8791 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8792 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8793 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8794 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8795 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8796 ; GFX10-WGP-NEXT:    buffer_gl0_inv
8797 ; GFX10-WGP-NEXT:    buffer_gl1_inv
8798 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8799 ; GFX10-WGP-NEXT:    s_endpgm
8800 ;
8801 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
8802 ; GFX10-CU:       ; %bb.0: ; %entry
8803 ; GFX10-CU-NEXT:    s_clause 0x1
8804 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8805 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8806 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8807 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8808 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8809 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8810 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8811 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8812 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8813 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8814 ; GFX10-CU-NEXT:    buffer_gl0_inv
8815 ; GFX10-CU-NEXT:    buffer_gl1_inv
8816 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8817 ; GFX10-CU-NEXT:    s_endpgm
8818 ;
8819 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
8820 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8821 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8822 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8823 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8824 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8825 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8826 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8827 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8828 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8829 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8830 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8831 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8832 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8833 ;
8834 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
8835 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8836 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8837 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8838 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8839 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8840 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8841 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8842 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8843 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8844 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8845 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8846 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8847 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8848 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8849 ;
8850 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_acquire_ret_cmpxchg:
8851 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8852 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8853 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8854 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8855 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8856 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8857 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8858 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8859 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8860 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8861 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8862 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8863 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8864 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8865     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8866 entry:
8867   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8868   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel acquire
8869   %val0 = extractvalue { i32, i1 } %val, 0
8870   store i32 %val0, i32 addrspace(1)* %out, align 4
8871   ret void
8872 }
8873
8874 define amdgpu_kernel void @global_system_one_as_seq_cst_acquire_ret_cmpxchg(
8875 ; GFX6-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
8876 ; GFX6:       ; %bb.0: ; %entry
8877 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8878 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
8879 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
8880 ; GFX6-NEXT:    s_mov_b32 s2, -1
8881 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
8882 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
8883 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
8884 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8885 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
8886 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
8887 ; GFX6-NEXT:    buffer_wbinvl1
8888 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
8889 ; GFX6-NEXT:    s_endpgm
8890 ;
8891 ; GFX7-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
8892 ; GFX7:       ; %bb.0: ; %entry
8893 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8894 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
8895 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
8896 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
8897 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
8898 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
8899 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
8900 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
8901 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
8902 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8903 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
8904 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
8905 ; GFX7-NEXT:    buffer_wbinvl1_vol
8906 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
8907 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
8908 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
8909 ; GFX7-NEXT:    s_endpgm
8910 ;
8911 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
8912 ; GFX10-WGP:       ; %bb.0: ; %entry
8913 ; GFX10-WGP-NEXT:    s_clause 0x1
8914 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8915 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8916 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
8917 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
8918 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
8919 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
8920 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8921 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
8922 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8923 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
8924 ; GFX10-WGP-NEXT:    buffer_gl0_inv
8925 ; GFX10-WGP-NEXT:    buffer_gl1_inv
8926 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
8927 ; GFX10-WGP-NEXT:    s_endpgm
8928 ;
8929 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
8930 ; GFX10-CU:       ; %bb.0: ; %entry
8931 ; GFX10-CU-NEXT:    s_clause 0x1
8932 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
8933 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
8934 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
8935 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
8936 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
8937 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
8938 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8939 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
8940 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
8941 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
8942 ; GFX10-CU-NEXT:    buffer_gl0_inv
8943 ; GFX10-CU-NEXT:    buffer_gl1_inv
8944 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
8945 ; GFX10-CU-NEXT:    s_endpgm
8946 ;
8947 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
8948 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
8949 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
8950 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
8951 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
8952 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
8953 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
8954 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
8955 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
8956 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8957 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
8958 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
8959 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
8960 ; SKIP-CACHE-INV-NEXT:    s_endpgm
8961 ;
8962 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
8963 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
8964 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8965 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8966 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8967 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8968 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8969 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
8970 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8971 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8972 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8973 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
8974 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
8975 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8976 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
8977 ;
8978 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_acquire_ret_cmpxchg:
8979 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
8980 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
8981 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
8982 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
8983 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
8984 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
8985 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
8986 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8987 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
8988 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
8989 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
8990 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
8991 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
8992 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
8993     i32 addrspace(1)* %out, i32 %in, i32 %old) {
8994 entry:
8995   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
8996   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst acquire
8997   %val0 = extractvalue { i32, i1 } %val, 0
8998   store i32 %val0, i32 addrspace(1)* %out, align 4
8999   ret void
9000 }
9001
9002 define amdgpu_kernel void @global_system_one_as_monotonic_seq_cst_ret_cmpxchg(
9003 ; GFX6-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
9004 ; GFX6:       ; %bb.0: ; %entry
9005 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9006 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
9007 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9008 ; GFX6-NEXT:    s_mov_b32 s2, -1
9009 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9010 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
9011 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
9012 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9013 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9014 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9015 ; GFX6-NEXT:    buffer_wbinvl1
9016 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9017 ; GFX6-NEXT:    s_endpgm
9018 ;
9019 ; GFX7-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
9020 ; GFX7:       ; %bb.0: ; %entry
9021 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9022 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9023 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9024 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
9025 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9026 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9027 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9028 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9029 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9030 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9031 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9032 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9033 ; GFX7-NEXT:    buffer_wbinvl1_vol
9034 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9035 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9036 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
9037 ; GFX7-NEXT:    s_endpgm
9038 ;
9039 ; GFX10-WGP-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
9040 ; GFX10-WGP:       ; %bb.0: ; %entry
9041 ; GFX10-WGP-NEXT:    s_clause 0x1
9042 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9043 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9044 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9045 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9046 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9047 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9048 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9049 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9050 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9051 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9052 ; GFX10-WGP-NEXT:    buffer_gl0_inv
9053 ; GFX10-WGP-NEXT:    buffer_gl1_inv
9054 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9055 ; GFX10-WGP-NEXT:    s_endpgm
9056 ;
9057 ; GFX10-CU-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
9058 ; GFX10-CU:       ; %bb.0: ; %entry
9059 ; GFX10-CU-NEXT:    s_clause 0x1
9060 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9061 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9062 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9063 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9064 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9065 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9066 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9067 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9068 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9069 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9070 ; GFX10-CU-NEXT:    buffer_gl0_inv
9071 ; GFX10-CU-NEXT:    buffer_gl1_inv
9072 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9073 ; GFX10-CU-NEXT:    s_endpgm
9074 ;
9075 ; SKIP-CACHE-INV-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
9076 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9077 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
9078 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
9079 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
9080 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
9081 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9082 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9083 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
9084 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9085 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9086 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9087 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
9088 ; SKIP-CACHE-INV-NEXT:    s_endpgm
9089 ;
9090 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
9091 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9092 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9093 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9094 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9095 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9096 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9097 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9098 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9099 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9100 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9101 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9102 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9103 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9104 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9105 ;
9106 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_monotonic_seq_cst_ret_cmpxchg:
9107 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9108 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9109 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9110 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9111 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9112 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9113 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9114 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9115 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9116 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9117 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9118 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9119 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9120 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
9121     i32 addrspace(1)* %out, i32 %in, i32 %old) {
9122 entry:
9123   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9124   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") monotonic seq_cst
9125   %val0 = extractvalue { i32, i1 } %val, 0
9126   store i32 %val0, i32 addrspace(1)* %out, align 4
9127   ret void
9128 }
9129
9130 define amdgpu_kernel void @global_system_one_as_acquire_seq_cst_ret_cmpxchg(
9131 ; GFX6-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
9132 ; GFX6:       ; %bb.0: ; %entry
9133 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9134 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
9135 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9136 ; GFX6-NEXT:    s_mov_b32 s2, -1
9137 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9138 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
9139 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
9140 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9141 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9142 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9143 ; GFX6-NEXT:    buffer_wbinvl1
9144 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9145 ; GFX6-NEXT:    s_endpgm
9146 ;
9147 ; GFX7-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
9148 ; GFX7:       ; %bb.0: ; %entry
9149 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9150 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9151 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9152 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
9153 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9154 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9155 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9156 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9157 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9158 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9159 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9160 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9161 ; GFX7-NEXT:    buffer_wbinvl1_vol
9162 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9163 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9164 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
9165 ; GFX7-NEXT:    s_endpgm
9166 ;
9167 ; GFX10-WGP-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
9168 ; GFX10-WGP:       ; %bb.0: ; %entry
9169 ; GFX10-WGP-NEXT:    s_clause 0x1
9170 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9171 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9172 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9173 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9174 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9175 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9176 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9177 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9178 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9179 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9180 ; GFX10-WGP-NEXT:    buffer_gl0_inv
9181 ; GFX10-WGP-NEXT:    buffer_gl1_inv
9182 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9183 ; GFX10-WGP-NEXT:    s_endpgm
9184 ;
9185 ; GFX10-CU-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
9186 ; GFX10-CU:       ; %bb.0: ; %entry
9187 ; GFX10-CU-NEXT:    s_clause 0x1
9188 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9189 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9190 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9191 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9192 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9193 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9194 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9195 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9196 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9197 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9198 ; GFX10-CU-NEXT:    buffer_gl0_inv
9199 ; GFX10-CU-NEXT:    buffer_gl1_inv
9200 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9201 ; GFX10-CU-NEXT:    s_endpgm
9202 ;
9203 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
9204 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9205 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
9206 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
9207 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
9208 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
9209 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9210 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9211 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
9212 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9213 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9214 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9215 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
9216 ; SKIP-CACHE-INV-NEXT:    s_endpgm
9217 ;
9218 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
9219 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9220 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9221 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9222 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9223 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9224 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9225 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9226 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9227 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9228 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9229 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9230 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9231 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9232 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9233 ;
9234 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acquire_seq_cst_ret_cmpxchg:
9235 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9236 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9237 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9238 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9239 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9240 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9241 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9242 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9243 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9244 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9245 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9246 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9247 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9248 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
9249     i32 addrspace(1)* %out, i32 %in, i32 %old) {
9250 entry:
9251   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9252   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acquire seq_cst
9253   %val0 = extractvalue { i32, i1 } %val, 0
9254   store i32 %val0, i32 addrspace(1)* %out, align 4
9255   ret void
9256 }
9257
9258 define amdgpu_kernel void @global_system_one_as_release_seq_cst_ret_cmpxchg(
9259 ; GFX6-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
9260 ; GFX6:       ; %bb.0: ; %entry
9261 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9262 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
9263 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9264 ; GFX6-NEXT:    s_mov_b32 s2, -1
9265 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9266 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
9267 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
9268 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9269 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9270 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9271 ; GFX6-NEXT:    buffer_wbinvl1
9272 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9273 ; GFX6-NEXT:    s_endpgm
9274 ;
9275 ; GFX7-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
9276 ; GFX7:       ; %bb.0: ; %entry
9277 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9278 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9279 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9280 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
9281 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9282 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9283 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9284 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9285 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9286 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9287 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9288 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9289 ; GFX7-NEXT:    buffer_wbinvl1_vol
9290 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9291 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9292 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
9293 ; GFX7-NEXT:    s_endpgm
9294 ;
9295 ; GFX10-WGP-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
9296 ; GFX10-WGP:       ; %bb.0: ; %entry
9297 ; GFX10-WGP-NEXT:    s_clause 0x1
9298 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9299 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9300 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9301 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9302 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9303 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9304 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9305 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9306 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9307 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9308 ; GFX10-WGP-NEXT:    buffer_gl0_inv
9309 ; GFX10-WGP-NEXT:    buffer_gl1_inv
9310 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9311 ; GFX10-WGP-NEXT:    s_endpgm
9312 ;
9313 ; GFX10-CU-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
9314 ; GFX10-CU:       ; %bb.0: ; %entry
9315 ; GFX10-CU-NEXT:    s_clause 0x1
9316 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9317 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9318 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9319 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9320 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9321 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9322 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9323 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9324 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9325 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9326 ; GFX10-CU-NEXT:    buffer_gl0_inv
9327 ; GFX10-CU-NEXT:    buffer_gl1_inv
9328 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9329 ; GFX10-CU-NEXT:    s_endpgm
9330 ;
9331 ; SKIP-CACHE-INV-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
9332 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9333 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
9334 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
9335 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
9336 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
9337 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9338 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9339 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
9340 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9341 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9342 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9343 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
9344 ; SKIP-CACHE-INV-NEXT:    s_endpgm
9345 ;
9346 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
9347 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9348 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9349 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9350 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9351 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9352 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9353 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9354 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9355 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9356 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9357 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9358 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9359 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9360 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9361 ;
9362 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_release_seq_cst_ret_cmpxchg:
9363 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9364 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9365 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9366 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9367 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9368 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9369 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9370 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9371 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9372 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9373 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9374 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9375 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9376 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
9377     i32 addrspace(1)* %out, i32 %in, i32 %old) {
9378 entry:
9379   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9380   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") release seq_cst
9381   %val0 = extractvalue { i32, i1 } %val, 0
9382   store i32 %val0, i32 addrspace(1)* %out, align 4
9383   ret void
9384 }
9385
9386 define amdgpu_kernel void @global_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
9387 ; GFX6-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
9388 ; GFX6:       ; %bb.0: ; %entry
9389 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9390 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
9391 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9392 ; GFX6-NEXT:    s_mov_b32 s2, -1
9393 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9394 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
9395 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
9396 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9397 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9398 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9399 ; GFX6-NEXT:    buffer_wbinvl1
9400 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9401 ; GFX6-NEXT:    s_endpgm
9402 ;
9403 ; GFX7-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
9404 ; GFX7:       ; %bb.0: ; %entry
9405 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9406 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9407 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9408 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
9409 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9410 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9411 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9412 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9413 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9414 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9415 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9416 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9417 ; GFX7-NEXT:    buffer_wbinvl1_vol
9418 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9419 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9420 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
9421 ; GFX7-NEXT:    s_endpgm
9422 ;
9423 ; GFX10-WGP-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
9424 ; GFX10-WGP:       ; %bb.0: ; %entry
9425 ; GFX10-WGP-NEXT:    s_clause 0x1
9426 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9427 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9428 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9429 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9430 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9431 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9432 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9433 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9434 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9435 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9436 ; GFX10-WGP-NEXT:    buffer_gl0_inv
9437 ; GFX10-WGP-NEXT:    buffer_gl1_inv
9438 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9439 ; GFX10-WGP-NEXT:    s_endpgm
9440 ;
9441 ; GFX10-CU-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
9442 ; GFX10-CU:       ; %bb.0: ; %entry
9443 ; GFX10-CU-NEXT:    s_clause 0x1
9444 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9445 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9446 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9447 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9448 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9449 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9450 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9451 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9452 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9453 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9454 ; GFX10-CU-NEXT:    buffer_gl0_inv
9455 ; GFX10-CU-NEXT:    buffer_gl1_inv
9456 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9457 ; GFX10-CU-NEXT:    s_endpgm
9458 ;
9459 ; SKIP-CACHE-INV-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
9460 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9461 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
9462 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
9463 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
9464 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
9465 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9466 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9467 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
9468 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9469 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9470 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9471 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
9472 ; SKIP-CACHE-INV-NEXT:    s_endpgm
9473 ;
9474 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
9475 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9476 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9477 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9478 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9479 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9480 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9481 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9482 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9483 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9484 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9485 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9486 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9487 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9488 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9489 ;
9490 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
9491 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9492 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9493 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9494 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9495 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9496 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9497 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9498 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9499 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9500 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9501 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9502 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9503 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9504 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
9505     i32 addrspace(1)* %out, i32 %in, i32 %old) {
9506 entry:
9507   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9508   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") acq_rel seq_cst
9509   %val0 = extractvalue { i32, i1 } %val, 0
9510   store i32 %val0, i32 addrspace(1)* %out, align 4
9511   ret void
9512 }
9513
9514 define amdgpu_kernel void @global_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
9515 ; GFX6-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
9516 ; GFX6:       ; %bb.0: ; %entry
9517 ; GFX6-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9518 ; GFX6-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x2
9519 ; GFX6-NEXT:    s_mov_b32 s3, 0x100f000
9520 ; GFX6-NEXT:    s_mov_b32 s2, -1
9521 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
9522 ; GFX6-NEXT:    v_mov_b32_e32 v0, s4
9523 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
9524 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9525 ; GFX6-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
9526 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
9527 ; GFX6-NEXT:    buffer_wbinvl1
9528 ; GFX6-NEXT:    buffer_store_dword v0, off, s[0:3], 0
9529 ; GFX6-NEXT:    s_endpgm
9530 ;
9531 ; GFX7-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
9532 ; GFX7:       ; %bb.0: ; %entry
9533 ; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9534 ; GFX7-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x2
9535 ; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
9536 ; GFX7-NEXT:    s_add_u32 s4, s0, 16
9537 ; GFX7-NEXT:    s_addc_u32 s5, s1, 0
9538 ; GFX7-NEXT:    v_mov_b32_e32 v0, s4
9539 ; GFX7-NEXT:    v_mov_b32_e32 v2, s2
9540 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
9541 ; GFX7-NEXT:    v_mov_b32_e32 v3, s3
9542 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9543 ; GFX7-NEXT:    flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
9544 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
9545 ; GFX7-NEXT:    buffer_wbinvl1_vol
9546 ; GFX7-NEXT:    v_mov_b32_e32 v0, s0
9547 ; GFX7-NEXT:    v_mov_b32_e32 v1, s1
9548 ; GFX7-NEXT:    flat_store_dword v[0:1], v2
9549 ; GFX7-NEXT:    s_endpgm
9550 ;
9551 ; GFX10-WGP-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
9552 ; GFX10-WGP:       ; %bb.0: ; %entry
9553 ; GFX10-WGP-NEXT:    s_clause 0x1
9554 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9555 ; GFX10-WGP-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9556 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v2, 0
9557 ; GFX10-WGP-NEXT:    s_waitcnt lgkmcnt(0)
9558 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v0, s0
9559 ; GFX10-WGP-NEXT:    v_mov_b32_e32 v1, s1
9560 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9561 ; GFX10-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
9562 ; GFX10-WGP-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9563 ; GFX10-WGP-NEXT:    s_waitcnt vmcnt(0)
9564 ; GFX10-WGP-NEXT:    buffer_gl0_inv
9565 ; GFX10-WGP-NEXT:    buffer_gl1_inv
9566 ; GFX10-WGP-NEXT:    global_store_dword v2, v0, s[2:3]
9567 ; GFX10-WGP-NEXT:    s_endpgm
9568 ;
9569 ; GFX10-CU-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
9570 ; GFX10-CU:       ; %bb.0: ; %entry
9571 ; GFX10-CU-NEXT:    s_clause 0x1
9572 ; GFX10-CU-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x8
9573 ; GFX10-CU-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x0
9574 ; GFX10-CU-NEXT:    v_mov_b32_e32 v2, 0
9575 ; GFX10-CU-NEXT:    s_waitcnt lgkmcnt(0)
9576 ; GFX10-CU-NEXT:    v_mov_b32_e32 v0, s0
9577 ; GFX10-CU-NEXT:    v_mov_b32_e32 v1, s1
9578 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9579 ; GFX10-CU-NEXT:    s_waitcnt_vscnt null, 0x0
9580 ; GFX10-CU-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[2:3] offset:16 glc
9581 ; GFX10-CU-NEXT:    s_waitcnt vmcnt(0)
9582 ; GFX10-CU-NEXT:    buffer_gl0_inv
9583 ; GFX10-CU-NEXT:    buffer_gl1_inv
9584 ; GFX10-CU-NEXT:    global_store_dword v2, v0, s[2:3]
9585 ; GFX10-CU-NEXT:    s_endpgm
9586 ;
9587 ; SKIP-CACHE-INV-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
9588 ; SKIP-CACHE-INV:       ; %bb.0: ; %entry
9589 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x9
9590 ; SKIP-CACHE-INV-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xb
9591 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s7, 0xf000
9592 ; SKIP-CACHE-INV-NEXT:    s_mov_b32 s6, -1
9593 ; SKIP-CACHE-INV-NEXT:    s_waitcnt lgkmcnt(0)
9594 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v0, s0
9595 ; SKIP-CACHE-INV-NEXT:    v_mov_b32_e32 v1, s1
9596 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9597 ; SKIP-CACHE-INV-NEXT:    buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
9598 ; SKIP-CACHE-INV-NEXT:    s_waitcnt vmcnt(0)
9599 ; SKIP-CACHE-INV-NEXT:    buffer_store_dword v0, off, s[4:7], 0
9600 ; SKIP-CACHE-INV-NEXT:    s_endpgm
9601 ;
9602 ; GFX90A-NOTTGSPLIT-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
9603 ; GFX90A-NOTTGSPLIT:       ; %bb.0: ; %entry
9604 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9605 ; GFX90A-NOTTGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9606 ; GFX90A-NOTTGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9607 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9608 ; GFX90A-NOTTGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9609 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbl2
9610 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9611 ; GFX90A-NOTTGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9612 ; GFX90A-NOTTGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9613 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_invl2
9614 ; GFX90A-NOTTGSPLIT-NEXT:    buffer_wbinvl1_vol
9615 ; GFX90A-NOTTGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9616 ; GFX90A-NOTTGSPLIT-NEXT:    s_endpgm
9617 ;
9618 ; GFX90A-TGSPLIT-LABEL: global_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
9619 ; GFX90A-TGSPLIT:       ; %bb.0: ; %entry
9620 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
9621 ; GFX90A-TGSPLIT-NEXT:    s_load_dwordx2 s[2:3], s[4:5], 0x8
9622 ; GFX90A-TGSPLIT-NEXT:    v_mov_b32_e32 v2, 0
9623 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt lgkmcnt(0)
9624 ; GFX90A-TGSPLIT-NEXT:    v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
9625 ; GFX90A-TGSPLIT-NEXT:    buffer_wbl2
9626 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9627 ; GFX90A-TGSPLIT-NEXT:    global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
9628 ; GFX90A-TGSPLIT-NEXT:    s_waitcnt vmcnt(0)
9629 ; GFX90A-TGSPLIT-NEXT:    buffer_invl2
9630 ; GFX90A-TGSPLIT-NEXT:    buffer_wbinvl1_vol
9631 ; GFX90A-TGSPLIT-NEXT:    global_store_dword v2, v0, s[0:1]
9632 ; GFX90A-TGSPLIT-NEXT:    s_endpgm
9633     i32 addrspace(1)* %out, i32 %in, i32 %old) {
9634 entry:
9635   %gep = getelementptr i32, i32 addrspace(1)* %out, i32 4
9636   %val = cmpxchg volatile i32 addrspace(1)* %gep, i32 %old, i32 %in syncscope("one-as") seq_cst seq_cst
9637   %val0 = extractvalue { i32, i1 } %val, 0
9638   store i32 %val0, i32 addrspace(1)* %out, align 4
9639   ret void
9640 }
9641