test/CodeGen/AMDGPU/memory-legalizer-atomic-rmw.ll

   1 ; RUN: llc -mtriple=amdgcn-amd- -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
   2 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX8 %s
   3
   4 ; GCN-LABEL: {{^}}system_monotonic:
   5 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
   6 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
   7 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
   8 ; GCN-NOT:   buffer_wbinvl1_vol
   9 define amdgpu_kernel void @system_monotonic(
  10     i32* %out, i32 %in) {
  11 entry:
  12   %val = atomicrmw volatile xchg i32* %out, i32 %in monotonic
  13   ret void
  14 }
  15
  16 ; GCN-LABEL: {{^}}system_acquire:
  17 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
  18 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
  19 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
  20 ; GFX8-NEXT:  buffer_wbinvl1_vol
  21 define amdgpu_kernel void @system_acquire(
  22     i32* %out, i32 %in) {
  23 entry:
  24   %val = atomicrmw volatile xchg i32* %out, i32 %in acquire
  25   ret void
  26 }
  27
  28 ; GCN-LABEL: {{^}}system_release:
  29 ; GCN:        s_waitcnt vmcnt(0){{$}}
  30 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
  31 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
  32 ; GCN-NOT:    buffer_wbinvl1_vol
  33 define amdgpu_kernel void @system_release(
  34     i32* %out, i32 %in) {
  35 entry:
  36   %val = atomicrmw volatile xchg i32* %out, i32 %in release
  37   ret void
  38 }
  39
  40 ; GCN-LABEL: {{^}}system_acq_rel:
  41 ; GCN:         s_waitcnt vmcnt(0){{$}}
  42 ; GCN-NEXT:    flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
  43 ; GCN-NEXT:    s_waitcnt vmcnt(0){{$}}
  44 ; GFX8-NEXT:   buffer_wbinvl1_vol
  45 define amdgpu_kernel void @system_acq_rel(
  46     i32* %out, i32 %in) {
  47 entry:
  48   %val = atomicrmw volatile xchg i32* %out, i32 %in acq_rel
  49   ret void
  50 }
  51
  52 ; GCN-LABEL: {{^}}system_seq_cst:
  53 ; GCN:        s_waitcnt vmcnt(0){{$}}
  54 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
  55 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
  56 ; GFX8-NEXT:  buffer_wbinvl1_vol
  57 define amdgpu_kernel void @system_seq_cst(
  58     i32* %out, i32 %in) {
  59 entry:
  60   %val = atomicrmw volatile xchg i32* %out, i32 %in seq_cst
  61   ret void
  62 }
  63
  64 ; GCN-LABEL: {{^}}singlethread_monotonic:
  65 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
  66 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
  67 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
  68 ; GCN-NOT:   buffer_wbinvl1_vol
  69 define amdgpu_kernel void @singlethread_monotonic(
  70     i32* %out, i32 %in) {
  71 entry:
  72   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") monotonic
  73   ret void
  74 }
  75
  76 ; GCN-LABEL: {{^}}singlethread_acquire:
  77 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
  78 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
  79 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
  80 ; GCN-NOT:   buffer_wbinvl1_vol
  81 define amdgpu_kernel void @singlethread_acquire(
  82     i32* %out, i32 %in) {
  83 entry:
  84   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acquire
  85   ret void
  86 }
  87
  88 ; GCN-LABEL: {{^}}singlethread_release:
  89 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
  90 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
  91 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
  92 ; GCN-NOT:   buffer_wbinvl1_vol
  93 define amdgpu_kernel void @singlethread_release(
  94     i32* %out, i32 %in) {
  95 entry:
  96   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") release
  97   ret void
  98 }
  99
 100 ; GCN-LABEL: {{^}}singlethread_acq_rel:
 101 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 102 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 103 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 104 ; GCN-NOT:   buffer_wbinvl1_vol
 105 define amdgpu_kernel void @singlethread_acq_rel(
 106     i32* %out, i32 %in) {
 107 entry:
 108   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") acq_rel
 109   ret void
 110 }
 111
 112 ; GCN-LABEL: {{^}}singlethread_seq_cst:
 113 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 114 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 115 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 116 ; GCN-NOT:   buffer_wbinvl1_vol
 117 define amdgpu_kernel void @singlethread_seq_cst(
 118     i32* %out, i32 %in) {
 119 entry:
 120   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("singlethread") seq_cst
 121   ret void
 122 }
 123
 124 ; GCN-LABEL: {{^}}agent_monotonic:
 125 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 126 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 127 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 128 ; GCN-NOT:   buffer_wbinvl1_vol
 129 define amdgpu_kernel void @agent_monotonic(
 130     i32* %out, i32 %in) {
 131 entry:
 132   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") monotonic
 133   ret void
 134 }
 135
 136 ; GCN-LABEL: {{^}}agent_acquire:
 137 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
 138 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 139 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 140 ; GFX8-NEXT:  buffer_wbinvl1_vol
 141 define amdgpu_kernel void @agent_acquire(
 142     i32* %out, i32 %in) {
 143 entry:
 144   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acquire
 145   ret void
 146 }
 147
 148 ; GCN-LABEL: {{^}}agent_release:
 149 ; GCN:        s_waitcnt vmcnt(0){{$}}
 150 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 151 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
 152 ; GCN-NOT:    buffer_wbinvl1_vol
 153 define amdgpu_kernel void @agent_release(
 154     i32* %out, i32 %in) {
 155 entry:
 156   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") release
 157   ret void
 158 }
 159
 160 ; GCN-LABEL: {{^}}agent_acq_rel:
 161 ; GCN:        s_waitcnt vmcnt(0){{$}}
 162 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 163 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 164 ; GFX8-NEXT:  buffer_wbinvl1_vol
 165 define amdgpu_kernel void @agent_acq_rel(
 166     i32* %out, i32 %in) {
 167 entry:
 168   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") acq_rel
 169   ret void
 170 }
 171
 172 ; GCN-LABEL: {{^}}agent_seq_cst:
 173 ; GCN:        s_waitcnt vmcnt(0){{$}}
 174 ; GCN-NEXT:   flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 175 ; GCN-NEXT:   s_waitcnt vmcnt(0){{$}}
 176 ; GFX8-NEXT:  buffer_wbinvl1_vol
 177 define amdgpu_kernel void @agent_seq_cst(
 178     i32* %out, i32 %in) {
 179 entry:
 180   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("agent") seq_cst
 181   ret void
 182 }
 183
 184 ; GCN-LABEL: {{^}}workgroup_monotonic:
 185 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 186 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 187 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 188 ; GCN-NOT:   buffer_wbinvl1_vol
 189 define amdgpu_kernel void @workgroup_monotonic(
 190     i32* %out, i32 %in) {
 191 entry:
 192   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") monotonic
 193   ret void
 194 }
 195
 196 ; GCN-LABEL: {{^}}workgroup_acquire:
 197 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
 198 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 199 ; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
 200 ; GFX8-NOT:   buffer_wbinvl1_vol
 201 define amdgpu_kernel void @workgroup_acquire(
 202     i32* %out, i32 %in) {
 203 entry:
 204   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acquire
 205   ret void
 206 }
 207
 208 ; GCN-LABEL: {{^}}workgroup_release:
 209 ; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
 210 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 211 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
 212 ; GCN-NOT:    buffer_wbinvl1_vol
 213 define amdgpu_kernel void @workgroup_release(
 214     i32* %out, i32 %in) {
 215 entry:
 216   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") release
 217   ret void
 218 }
 219
 220 ; GCN-LABEL: {{^}}workgroup_acq_rel:
 221 ; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
 222 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 223 ; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
 224 ; GFX8-NOT:   buffer_wbinvl1_vol
 225 define amdgpu_kernel void @workgroup_acq_rel(
 226     i32* %out, i32 %in) {
 227 entry:
 228   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") acq_rel
 229   ret void
 230 }
 231
 232 ; GCN-LABEL: {{^}}workgroup_seq_cst:
 233 ; GFX8-NOT:   s_waitcnt vmcnt(0){{$}}
 234 ; GCN:        flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 235 ; GCN-NOT:    s_waitcnt vmcnt(0){{$}}
 236 ; GFX8-NOT:   buffer_wbinvl1_vol
 237 define amdgpu_kernel void @workgroup_seq_cst(
 238     i32* %out, i32 %in) {
 239 entry:
 240   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("workgroup") seq_cst
 241   ret void
 242 }
 243
 244 ; GCN-LABEL: {{^}}wavefront_monotonic:
 245 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 246 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 247 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 248 ; GCN-NOT:   buffer_wbinvl1_vol
 249 define amdgpu_kernel void @wavefront_monotonic(
 250     i32* %out, i32 %in) {
 251 entry:
 252   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") monotonic
 253   ret void
 254 }
 255
 256 ; GCN-LABEL: {{^}}wavefront_acquire:
 257 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 258 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 259 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 260 ; GCN-NOT:   buffer_wbinvl1_vol
 261 define amdgpu_kernel void @wavefront_acquire(
 262     i32* %out, i32 %in) {
 263 entry:
 264   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acquire
 265   ret void
 266 }
 267
 268 ; GCN-LABEL: {{^}}wavefront_release:
 269 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 270 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 271 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 272 ; GCN-NOT:   buffer_wbinvl1_vol
 273 define amdgpu_kernel void @wavefront_release(
 274     i32* %out, i32 %in) {
 275 entry:
 276   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") release
 277   ret void
 278 }
 279
 280 ; GCN-LABEL: {{^}}wavefront_acq_rel:
 281 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 282 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 283 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 284 ; GCN-NOT:   buffer_wbinvl1_vol
 285 define amdgpu_kernel void @wavefront_acq_rel(
 286     i32* %out, i32 %in) {
 287 entry:
 288   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") acq_rel
 289   ret void
 290 }
 291
 292 ; GCN-LABEL: {{^}}wavefront_seq_cst:
 293 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 294 ; GCN:       flat_atomic_swap v[{{[0-9]+}}:{{[0-9]+}}], v{{[0-9]+}}{{$}}
 295 ; GCN-NOT:   s_waitcnt vmcnt(0){{$}}
 296 ; GCN-NOT:   buffer_wbinvl1_vol
 297 define amdgpu_kernel void @wavefront_seq_cst(
 298     i32* %out, i32 %in) {
 299 entry:
 300   %val = atomicrmw volatile xchg i32* %out, i32 %in syncscope("wavefront") seq_cst
 301   ret void
 302 }