lib/Target/AArch64/AArch64InstrAtomics.td

   1 //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // AArch64 Atomic operand code-gen constructs.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 //===----------------------------------
  14 // Atomic fences
  15 //===----------------------------------
  16 let AddedComplexity = 15, Size = 0 in
  17 def CompilerBarrier : Pseudo<(outs), (ins i32imm:$ordering),
  18                              [(atomic_fence imm:$ordering, 0)]>, Sched<[]>;
  19 def : Pat<(atomic_fence (i64 4), (imm)), (DMB (i32 0x9))>;
  20 def : Pat<(atomic_fence (imm), (imm)), (DMB (i32 0xb))>;
  21
  22 //===----------------------------------
  23 // Atomic loads
  24 //===----------------------------------
  25
  26 // When they're actually atomic, only one addressing mode (GPR64sp) is
  27 // supported, but when they're relaxed and anything can be used, all the
  28 // standard modes would be valid and may give efficiency gains.
  29
  30 // A atomic load operation that actually needs acquire semantics.
  31 class acquiring_load<PatFrag base>
  32   : PatFrag<(ops node:$ptr), (base node:$ptr)> {
  33   let IsAtomic = 1;
  34   let IsAtomicOrderingAcquireOrStronger = 1;
  35 }
  36
  37 // An atomic load operation that does not need either acquire or release
  38 // semantics.
  39 class relaxed_load<PatFrag base>
  40   : PatFrag<(ops node:$ptr), (base node:$ptr)> {
  41   let IsAtomic = 1;
  42   let IsAtomicOrderingAcquireOrStronger = 0;
  43 }
  44
  45 // 8-bit loads
  46 def : Pat<(acquiring_load<atomic_load_8>  GPR64sp:$ptr), (LDARB GPR64sp:$ptr)>;
  47 def : Pat<(relaxed_load<atomic_load_8> (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm,
  48                                                      ro_Wextend8:$offset)),
  49           (LDRBBroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$offset)>;
  50 def : Pat<(relaxed_load<atomic_load_8> (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm,
  51                                                      ro_Xextend8:$offset)),
  52           (LDRBBroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$offset)>;
  53 def : Pat<(relaxed_load<atomic_load_8> (am_indexed8 GPR64sp:$Rn,
  54                                                     uimm12s1:$offset)),
  55           (LDRBBui GPR64sp:$Rn, uimm12s1:$offset)>;
  56 def : Pat<(relaxed_load<atomic_load_8>
  57                (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
  58           (LDURBBi GPR64sp:$Rn, simm9:$offset)>;
  59
  60 // 16-bit loads
  61 def : Pat<(acquiring_load<atomic_load_16> GPR64sp:$ptr), (LDARH GPR64sp:$ptr)>;
  62 def : Pat<(relaxed_load<atomic_load_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
  63                                                        ro_Wextend16:$extend)),
  64           (LDRHHroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
  65 def : Pat<(relaxed_load<atomic_load_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
  66                                                        ro_Xextend16:$extend)),
  67           (LDRHHroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
  68 def : Pat<(relaxed_load<atomic_load_16> (am_indexed16 GPR64sp:$Rn,
  69                                                       uimm12s2:$offset)),
  70           (LDRHHui GPR64sp:$Rn, uimm12s2:$offset)>;
  71 def : Pat<(relaxed_load<atomic_load_16>
  72                (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
  73           (LDURHHi GPR64sp:$Rn, simm9:$offset)>;
  74
  75 // 32-bit loads
  76 def : Pat<(acquiring_load<atomic_load_32> GPR64sp:$ptr), (LDARW GPR64sp:$ptr)>;
  77 def : Pat<(relaxed_load<atomic_load_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
  78                                                        ro_Wextend32:$extend)),
  79           (LDRWroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
  80 def : Pat<(relaxed_load<atomic_load_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
  81                                                        ro_Xextend32:$extend)),
  82           (LDRWroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
  83 def : Pat<(relaxed_load<atomic_load_32> (am_indexed32 GPR64sp:$Rn,
  84                                                       uimm12s4:$offset)),
  85           (LDRWui GPR64sp:$Rn, uimm12s4:$offset)>;
  86 def : Pat<(relaxed_load<atomic_load_32>
  87                (am_unscaled32 GPR64sp:$Rn, simm9:$offset)),
  88           (LDURWi GPR64sp:$Rn, simm9:$offset)>;
  89
  90 // 64-bit loads
  91 def : Pat<(acquiring_load<atomic_load_64> GPR64sp:$ptr), (LDARX GPR64sp:$ptr)>;
  92 def : Pat<(relaxed_load<atomic_load_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
  93                                                        ro_Wextend64:$extend)),
  94           (LDRXroW GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
  95 def : Pat<(relaxed_load<atomic_load_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
  96                                                        ro_Xextend64:$extend)),
  97           (LDRXroX GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
  98 def : Pat<(relaxed_load<atomic_load_64> (am_indexed64 GPR64sp:$Rn,
  99                                                       uimm12s8:$offset)),
 100           (LDRXui GPR64sp:$Rn, uimm12s8:$offset)>;
 101 def : Pat<(relaxed_load<atomic_load_64>
 102                (am_unscaled64 GPR64sp:$Rn, simm9:$offset)),
 103           (LDURXi GPR64sp:$Rn, simm9:$offset)>;
 104
 105 //===----------------------------------
 106 // Atomic stores
 107 //===----------------------------------
 108
 109 // When they're actually atomic, only one addressing mode (GPR64sp) is
 110 // supported, but when they're relaxed and anything can be used, all the
 111 // standard modes would be valid and may give efficiency gains.
 112
 113 // A store operation that actually needs release semantics.
 114 class releasing_store<PatFrag base>
 115   : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
 116   let IsAtomic = 1;
 117   let IsAtomicOrderingReleaseOrStronger = 1;
 118 }
 119
 120 // An atomic store operation that doesn't actually need to be atomic on AArch64.
 121 class relaxed_store<PatFrag base>
 122   : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
 123   let IsAtomic = 1;
 124   let IsAtomicOrderingReleaseOrStronger = 0;
 125 }
 126
 127 // 8-bit stores
 128 def : Pat<(releasing_store<atomic_store_8> GPR64sp:$ptr, GPR32:$val),
 129           (STLRB GPR32:$val, GPR64sp:$ptr)>;
 130 def : Pat<(relaxed_store<atomic_store_8>
 131                (ro_Windexed8 GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend),
 132                GPR32:$val),
 133           (STRBBroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend8:$extend)>;
 134 def : Pat<(relaxed_store<atomic_store_8>
 135                (ro_Xindexed8 GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend),
 136                GPR32:$val),
 137           (STRBBroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend8:$extend)>;
 138 def : Pat<(relaxed_store<atomic_store_8>
 139                (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset), GPR32:$val),
 140           (STRBBui GPR32:$val, GPR64sp:$Rn, uimm12s1:$offset)>;
 141 def : Pat<(relaxed_store<atomic_store_8>
 142                (am_unscaled8 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 143           (STURBBi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 144
 145 // 16-bit stores
 146 def : Pat<(releasing_store<atomic_store_16> GPR64sp:$ptr, GPR32:$val),
 147           (STLRH GPR32:$val, GPR64sp:$ptr)>;
 148 def : Pat<(relaxed_store<atomic_store_16> (ro_Windexed16 GPR64sp:$Rn, GPR32:$Rm,
 149                                                          ro_Wextend16:$extend),
 150                                           GPR32:$val),
 151           (STRHHroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend16:$extend)>;
 152 def : Pat<(relaxed_store<atomic_store_16> (ro_Xindexed16 GPR64sp:$Rn, GPR64:$Rm,
 153                                                          ro_Xextend16:$extend),
 154                                           GPR32:$val),
 155           (STRHHroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend16:$extend)>;
 156 def : Pat<(relaxed_store<atomic_store_16>
 157               (am_indexed16 GPR64sp:$Rn, uimm12s2:$offset), GPR32:$val),
 158           (STRHHui GPR32:$val, GPR64sp:$Rn, uimm12s2:$offset)>;
 159 def : Pat<(relaxed_store<atomic_store_16>
 160                (am_unscaled16 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 161           (STURHHi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 162
 163 // 32-bit stores
 164 def : Pat<(releasing_store<atomic_store_32> GPR64sp:$ptr, GPR32:$val),
 165           (STLRW GPR32:$val, GPR64sp:$ptr)>;
 166 def : Pat<(relaxed_store<atomic_store_32> (ro_Windexed32 GPR64sp:$Rn, GPR32:$Rm,
 167                                                          ro_Wextend32:$extend),
 168                                           GPR32:$val),
 169           (STRWroW GPR32:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend32:$extend)>;
 170 def : Pat<(relaxed_store<atomic_store_32> (ro_Xindexed32 GPR64sp:$Rn, GPR64:$Rm,
 171                                                          ro_Xextend32:$extend),
 172                                           GPR32:$val),
 173           (STRWroX GPR32:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend32:$extend)>;
 174 def : Pat<(relaxed_store<atomic_store_32>
 175               (am_indexed32 GPR64sp:$Rn, uimm12s4:$offset), GPR32:$val),
 176           (STRWui GPR32:$val, GPR64sp:$Rn, uimm12s4:$offset)>;
 177 def : Pat<(relaxed_store<atomic_store_32>
 178                (am_unscaled32 GPR64sp:$Rn, simm9:$offset), GPR32:$val),
 179           (STURWi GPR32:$val, GPR64sp:$Rn, simm9:$offset)>;
 180
 181 // 64-bit stores
 182 def : Pat<(releasing_store<atomic_store_64> GPR64sp:$ptr, GPR64:$val),
 183           (STLRX GPR64:$val, GPR64sp:$ptr)>;
 184 def : Pat<(relaxed_store<atomic_store_64> (ro_Windexed64 GPR64sp:$Rn, GPR32:$Rm,
 185                                                          ro_Wextend16:$extend),
 186                                           GPR64:$val),
 187           (STRXroW GPR64:$val, GPR64sp:$Rn, GPR32:$Rm, ro_Wextend64:$extend)>;
 188 def : Pat<(relaxed_store<atomic_store_64> (ro_Xindexed64 GPR64sp:$Rn, GPR64:$Rm,
 189                                                          ro_Xextend16:$extend),
 190                                           GPR64:$val),
 191           (STRXroX GPR64:$val, GPR64sp:$Rn, GPR64:$Rm, ro_Xextend64:$extend)>;
 192 def : Pat<(relaxed_store<atomic_store_64>
 193               (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset), GPR64:$val),
 194           (STRXui GPR64:$val, GPR64sp:$Rn, uimm12s8:$offset)>;
 195 def : Pat<(relaxed_store<atomic_store_64>
 196                (am_unscaled64 GPR64sp:$Rn, simm9:$offset), GPR64:$val),
 197           (STURXi GPR64:$val, GPR64sp:$Rn, simm9:$offset)>;
 198
 199 //===----------------------------------
 200 // Low-level exclusive operations
 201 //===----------------------------------
 202
 203 // Load-exclusives.
 204
 205 def ldxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 206   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 207 }]> {
 208   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
 209 }
 210
 211 def ldxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 212   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 213 }]> {
 214   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
 215 }
 216
 217 def ldxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 218   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 219 }]> {
 220   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
 221 }
 222
 223 def ldxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldxr node:$ptr), [{
 224   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 225 }]> {
 226   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
 227 }
 228
 229 def : Pat<(ldxr_1 GPR64sp:$addr),
 230           (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
 231 def : Pat<(ldxr_2 GPR64sp:$addr),
 232           (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
 233 def : Pat<(ldxr_4 GPR64sp:$addr),
 234           (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
 235 def : Pat<(ldxr_8 GPR64sp:$addr), (LDXRX GPR64sp:$addr)>;
 236
 237 def : Pat<(and (ldxr_1 GPR64sp:$addr), 0xff),
 238           (SUBREG_TO_REG (i64 0), (LDXRB GPR64sp:$addr), sub_32)>;
 239 def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
 240           (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
 241 def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
 242           (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
 243
 244 // Load-exclusives.
 245
 246 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 247   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 248 }]> {
 249   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
 250 }
 251
 252 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 253   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 254 }]> {
 255   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
 256 }
 257
 258 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 259   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 260 }]> {
 261   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
 262 }
 263
 264 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
 265   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 266 }]> {
 267   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
 268 }
 269
 270 def : Pat<(ldaxr_1 GPR64sp:$addr),
 271           (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
 272 def : Pat<(ldaxr_2 GPR64sp:$addr),
 273           (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
 274 def : Pat<(ldaxr_4 GPR64sp:$addr),
 275           (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
 276 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;
 277
 278 def : Pat<(and (ldaxr_1 GPR64sp:$addr), 0xff),
 279           (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
 280 def : Pat<(and (ldaxr_2 GPR64sp:$addr), 0xffff),
 281           (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
 282 def : Pat<(and (ldaxr_4 GPR64sp:$addr), 0xffffffff),
 283           (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
 284
 285 // Store-exclusives.
 286
 287 def stxr_1 : PatFrag<(ops node:$val, node:$ptr),
 288                      (int_aarch64_stxr node:$val, node:$ptr), [{
 289   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 290 }]> {
 291   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
 292 }
 293
 294 def stxr_2 : PatFrag<(ops node:$val, node:$ptr),
 295                      (int_aarch64_stxr node:$val, node:$ptr), [{
 296   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 297 }]> {
 298   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
 299 }
 300
 301 def stxr_4 : PatFrag<(ops node:$val, node:$ptr),
 302                      (int_aarch64_stxr node:$val, node:$ptr), [{
 303   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 304 }]> {
 305   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
 306 }
 307
 308 def stxr_8 : PatFrag<(ops node:$val, node:$ptr),
 309                      (int_aarch64_stxr node:$val, node:$ptr), [{
 310   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 311 }]> {
 312   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
 313 }
 314
 315
 316 def : Pat<(stxr_1 GPR64:$val, GPR64sp:$addr),
 317           (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 318 def : Pat<(stxr_2 GPR64:$val, GPR64sp:$addr),
 319           (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 320 def : Pat<(stxr_4 GPR64:$val, GPR64sp:$addr),
 321           (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 322 def : Pat<(stxr_8 GPR64:$val, GPR64sp:$addr),
 323           (STXRX GPR64:$val, GPR64sp:$addr)>;
 324
 325 def : Pat<(stxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
 326           (STXRB GPR32:$val, GPR64sp:$addr)>;
 327 def : Pat<(stxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
 328           (STXRH GPR32:$val, GPR64sp:$addr)>;
 329 def : Pat<(stxr_4 (zext GPR32:$val), GPR64sp:$addr),
 330           (STXRW GPR32:$val, GPR64sp:$addr)>;
 331
 332 def : Pat<(stxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
 333           (STXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 334 def : Pat<(stxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
 335           (STXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 336 def : Pat<(stxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
 337           (STXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 338
 339 // Store-release-exclusives.
 340
 341 def stlxr_1 : PatFrag<(ops node:$val, node:$ptr),
 342                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 343   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
 344 }]> {
 345   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
 346 }
 347
 348 def stlxr_2 : PatFrag<(ops node:$val, node:$ptr),
 349                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 350   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
 351 }]> {
 352   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
 353 }
 354
 355 def stlxr_4 : PatFrag<(ops node:$val, node:$ptr),
 356                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 357   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
 358 }]> {
 359   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
 360 }
 361
 362 def stlxr_8 : PatFrag<(ops node:$val, node:$ptr),
 363                      (int_aarch64_stlxr node:$val, node:$ptr), [{
 364   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
 365 }]> {
 366   let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
 367 }
 368
 369
 370 def : Pat<(stlxr_1 GPR64:$val, GPR64sp:$addr),
 371           (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 372 def : Pat<(stlxr_2 GPR64:$val, GPR64sp:$addr),
 373           (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 374 def : Pat<(stlxr_4 GPR64:$val, GPR64sp:$addr),
 375           (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 376 def : Pat<(stlxr_8 GPR64:$val, GPR64sp:$addr),
 377           (STLXRX GPR64:$val, GPR64sp:$addr)>;
 378
 379 def : Pat<(stlxr_1 (zext (and GPR32:$val, 0xff)), GPR64sp:$addr),
 380           (STLXRB GPR32:$val, GPR64sp:$addr)>;
 381 def : Pat<(stlxr_2 (zext (and GPR32:$val, 0xffff)), GPR64sp:$addr),
 382           (STLXRH GPR32:$val, GPR64sp:$addr)>;
 383 def : Pat<(stlxr_4 (zext GPR32:$val), GPR64sp:$addr),
 384           (STLXRW GPR32:$val, GPR64sp:$addr)>;
 385
 386 def : Pat<(stlxr_1 (and GPR64:$val, 0xff), GPR64sp:$addr),
 387           (STLXRB (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 388 def : Pat<(stlxr_2 (and GPR64:$val, 0xffff), GPR64sp:$addr),
 389           (STLXRH (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 390 def : Pat<(stlxr_4 (and GPR64:$val, 0xffffffff), GPR64sp:$addr),
 391           (STLXRW (EXTRACT_SUBREG GPR64:$val, sub_32), GPR64sp:$addr)>;
 392
 393
 394 // And clear exclusive.
 395
 396 def : Pat<(int_aarch64_clrex), (CLREX 0xf)>;
 397
 398 //===----------------------------------
 399 // Atomic cmpxchg for -O0
 400 //===----------------------------------
 401
 402 // The fast register allocator used during -O0 inserts spills to cover any VRegs
 403 // live across basic block boundaries. When this happens between an LDXR and an
 404 // STXR it can clear the exclusive monitor, causing all cmpxchg attempts to
 405 // fail.
 406
 407 // Unfortunately, this means we have to have an alternative (expanded
 408 // post-regalloc) path for -O0 compilations. Fortunately this path can be
 409 // significantly more naive than the standard expansion: we conservatively
 410 // assume seq_cst, strong cmpxchg and omit clrex on failure.
 411
 412 let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch",
 413     mayLoad = 1, mayStore = 1 in {
 414 def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 415                         (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 416                  Sched<[WriteAtomic]>;
 417
 418 def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 419                          (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 420                   Sched<[WriteAtomic]>;
 421
 422 def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch),
 423                          (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>,
 424                   Sched<[WriteAtomic]>;
 425
 426 def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch),
 427                          (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>,
 428                   Sched<[WriteAtomic]>;
 429 }
 430
 431 let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch",
 432     mayLoad = 1, mayStore = 1 in
 433 def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch),
 434                           (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi,
 435                                GPR64:$newLo, GPR64:$newHi), []>,
 436                    Sched<[WriteAtomic]>;
 437
 438 // v8.1 Atomic instructions:
 439 let Predicates = [HasLSE] in {
 440   defm : LDOPregister_patterns<"LDADD", "atomic_load_add">;
 441   defm : LDOPregister_patterns<"LDSET", "atomic_load_or">;
 442   defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">;
 443   defm : LDOPregister_patterns<"LDCLR", "atomic_load_clr">;
 444   defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">;
 445   defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">;
 446   defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">;
 447   defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">;
 448   defm : LDOPregister_patterns<"SWP", "atomic_swap">;
 449   defm : CASregister_patterns<"CAS", "atomic_cmp_swap">;
 450
 451   // These two patterns are only needed for global isel, selection dag isel
 452   // converts atomic load-sub into a sub and atomic load-add, and likewise for
 453   // and -> clr.
 454   defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">;
 455   defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">;
 456 }