tools/testing/selftests/rseq/rseq-s390.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2
   3 #define RSEQ_SIG        0x53053053
   4
   5 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
   6 #define rseq_smp_rmb()  rseq_smp_mb()
   7 #define rseq_smp_wmb()  rseq_smp_mb()
   8
   9 #define rseq_smp_load_acquire(p)                                        \
  10 __extension__ ({                                                        \
  11         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  12         rseq_barrier();                                                 \
  13         ____p1;                                                         \
  14 })
  15
  16 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  17
  18 #define rseq_smp_store_release(p, v)                                    \
  19 do {                                                                    \
  20         rseq_barrier();                                                 \
  21         RSEQ_WRITE_ONCE(*p, v);                                         \
  22 } while (0)
  23
  24 #ifdef RSEQ_SKIP_FASTPATH
  25 #include "rseq-skip.h"
  26 #else /* !RSEQ_SKIP_FASTPATH */
  27
  28 #ifdef __s390x__
  29
  30 #define LONG_L                  "lg"
  31 #define LONG_S                  "stg"
  32 #define LONG_LT_R               "ltgr"
  33 #define LONG_CMP                "cg"
  34 #define LONG_CMP_R              "cgr"
  35 #define LONG_ADDI               "aghi"
  36 #define LONG_ADD_R              "agr"
  37
  38 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  39                                 start_ip, post_commit_offset, abort_ip) \
  40                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
  41                 ".balign 32\n\t"                                        \
  42                 __rseq_str(label) ":\n\t"                               \
  43                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  44                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  45                 ".popsection\n\t"
  46
  47 #elif __s390__
  48
  49 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  50                                 start_ip, post_commit_offset, abort_ip) \
  51                 ".pushsection __rseq_table, \"aw\"\n\t"                 \
  52                 ".balign 32\n\t"                                        \
  53                 __rseq_str(label) ":\n\t"                               \
  54                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  55                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
  56                 ".popsection\n\t"
  57
  58 #define LONG_L                  "l"
  59 #define LONG_S                  "st"
  60 #define LONG_LT_R               "ltr"
  61 #define LONG_CMP                "c"
  62 #define LONG_CMP_R              "cr"
  63 #define LONG_ADDI               "ahi"
  64 #define LONG_ADD_R              "ar"
  65
  66 #endif
  67
  68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  69         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  70                                 (post_commit_ip - start_ip), abort_ip)
  71
  72 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  73                 RSEQ_INJECT_ASM(1)                                      \
  74                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
  75                 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"          \
  76                 __rseq_str(label) ":\n\t"
  77
  78 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  79                 RSEQ_INJECT_ASM(2)                                      \
  80                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
  81                 "jnz " __rseq_str(label) "\n\t"
  82
  83 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
  84                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
  85                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
  86                 __rseq_str(label) ":\n\t"                               \
  87                 teardown                                                \
  88                 "j %l[" __rseq_str(abort_label) "]\n\t"                 \
  89                 ".popsection\n\t"
  90
  91 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
  92                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
  93                 __rseq_str(label) ":\n\t"                               \
  94                 teardown                                                \
  95                 "j %l[" __rseq_str(cmpfail_label) "]\n\t"               \
  96                 ".popsection\n\t"
  97
  98 static inline __attribute__((always_inline))
  99 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 100 {
 101         RSEQ_INJECT_C(9)
 102
 103         __asm__ __volatile__ goto (
 104                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 105                 /* Start rseq by storing table entry pointer into rseq_cs. */
 106                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 107                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 108                 RSEQ_INJECT_ASM(3)
 109                 LONG_CMP " %[expect], %[v]\n\t"
 110                 "jnz %l[cmpfail]\n\t"
 111                 RSEQ_INJECT_ASM(4)
 112 #ifdef RSEQ_COMPARE_TWICE
 113                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 114                 LONG_CMP " %[expect], %[v]\n\t"
 115                 "jnz %l[error2]\n\t"
 116 #endif
 117                 /* final store */
 118                 LONG_S " %[newv], %[v]\n\t"
 119                 "2:\n\t"
 120                 RSEQ_INJECT_ASM(5)
 121                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 122                 : /* gcc asm goto does not allow outputs */
 123                 : [cpu_id]              "r" (cpu),
 124                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 125                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 126                   [v]                   "m" (*v),
 127                   [expect]              "r" (expect),
 128                   [newv]                "r" (newv)
 129                   RSEQ_INJECT_INPUT
 130                 : "memory", "cc", "r0"
 131                   RSEQ_INJECT_CLOBBER
 132                 : abort, cmpfail
 133 #ifdef RSEQ_COMPARE_TWICE
 134                   , error1, error2
 135 #endif
 136         );
 137         return 0;
 138 abort:
 139         RSEQ_INJECT_FAILED
 140         return -1;
 141 cmpfail:
 142         return 1;
 143 #ifdef RSEQ_COMPARE_TWICE
 144 error1:
 145         rseq_bug("cpu_id comparison failed");
 146 error2:
 147         rseq_bug("expected value comparison failed");
 148 #endif
 149 }
 150
 151 /*
 152  * Compare @v against @expectnot. When it does _not_ match, load @v
 153  * into @load, and store the content of *@v + voffp into @v.
 154  */
 155 static inline __attribute__((always_inline))
 156 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 157                                off_t voffp, intptr_t *load, int cpu)
 158 {
 159         RSEQ_INJECT_C(9)
 160
 161         __asm__ __volatile__ goto (
 162                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 163                 /* Start rseq by storing table entry pointer into rseq_cs. */
 164                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 165                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 166                 RSEQ_INJECT_ASM(3)
 167                 LONG_L " %%r1, %[v]\n\t"
 168                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 169                 "je %l[cmpfail]\n\t"
 170                 RSEQ_INJECT_ASM(4)
 171 #ifdef RSEQ_COMPARE_TWICE
 172                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 173                 LONG_L " %%r1, %[v]\n\t"
 174                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 175                 "je %l[error2]\n\t"
 176 #endif
 177                 LONG_S " %%r1, %[load]\n\t"
 178                 LONG_ADD_R " %%r1, %[voffp]\n\t"
 179                 LONG_L " %%r1, 0(%%r1)\n\t"
 180                 /* final store */
 181                 LONG_S " %%r1, %[v]\n\t"
 182                 "2:\n\t"
 183                 RSEQ_INJECT_ASM(5)
 184                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 185                 : /* gcc asm goto does not allow outputs */
 186                 : [cpu_id]              "r" (cpu),
 187                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 188                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 189                   /* final store input */
 190                   [v]                   "m" (*v),
 191                   [expectnot]           "r" (expectnot),
 192                   [voffp]               "r" (voffp),
 193                   [load]                "m" (*load)
 194                   RSEQ_INJECT_INPUT
 195                 : "memory", "cc", "r0", "r1"
 196                   RSEQ_INJECT_CLOBBER
 197                 : abort, cmpfail
 198 #ifdef RSEQ_COMPARE_TWICE
 199                   , error1, error2
 200 #endif
 201         );
 202         return 0;
 203 abort:
 204         RSEQ_INJECT_FAILED
 205         return -1;
 206 cmpfail:
 207         return 1;
 208 #ifdef RSEQ_COMPARE_TWICE
 209 error1:
 210         rseq_bug("cpu_id comparison failed");
 211 error2:
 212         rseq_bug("expected value comparison failed");
 213 #endif
 214 }
 215
 216 static inline __attribute__((always_inline))
 217 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 218 {
 219         RSEQ_INJECT_C(9)
 220
 221         __asm__ __volatile__ goto (
 222                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 223                 /* Start rseq by storing table entry pointer into rseq_cs. */
 224                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 225                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 226                 RSEQ_INJECT_ASM(3)
 227 #ifdef RSEQ_COMPARE_TWICE
 228                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 229 #endif
 230                 LONG_L " %%r0, %[v]\n\t"
 231                 LONG_ADD_R " %%r0, %[count]\n\t"
 232                 /* final store */
 233                 LONG_S " %%r0, %[v]\n\t"
 234                 "2:\n\t"
 235                 RSEQ_INJECT_ASM(4)
 236                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 237                 : /* gcc asm goto does not allow outputs */
 238                 : [cpu_id]              "r" (cpu),
 239                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 240                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 241                   /* final store input */
 242                   [v]                   "m" (*v),
 243                   [count]               "r" (count)
 244                   RSEQ_INJECT_INPUT
 245                 : "memory", "cc", "r0"
 246                   RSEQ_INJECT_CLOBBER
 247                 : abort
 248 #ifdef RSEQ_COMPARE_TWICE
 249                   , error1
 250 #endif
 251         );
 252         return 0;
 253 abort:
 254         RSEQ_INJECT_FAILED
 255         return -1;
 256 #ifdef RSEQ_COMPARE_TWICE
 257 error1:
 258         rseq_bug("cpu_id comparison failed");
 259 #endif
 260 }
 261
 262 static inline __attribute__((always_inline))
 263 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 264                                  intptr_t *v2, intptr_t newv2,
 265                                  intptr_t newv, int cpu)
 266 {
 267         RSEQ_INJECT_C(9)
 268
 269         __asm__ __volatile__ goto (
 270                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 271                 /* Start rseq by storing table entry pointer into rseq_cs. */
 272                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 273                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 274                 RSEQ_INJECT_ASM(3)
 275                 LONG_CMP " %[expect], %[v]\n\t"
 276                 "jnz %l[cmpfail]\n\t"
 277                 RSEQ_INJECT_ASM(4)
 278 #ifdef RSEQ_COMPARE_TWICE
 279                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 280                 LONG_CMP " %[expect], %[v]\n\t"
 281                 "jnz %l[error2]\n\t"
 282 #endif
 283                 /* try store */
 284                 LONG_S " %[newv2], %[v2]\n\t"
 285                 RSEQ_INJECT_ASM(5)
 286                 /* final store */
 287                 LONG_S " %[newv], %[v]\n\t"
 288                 "2:\n\t"
 289                 RSEQ_INJECT_ASM(6)
 290                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 291                 : /* gcc asm goto does not allow outputs */
 292                 : [cpu_id]              "r" (cpu),
 293                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 294                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 295                   /* try store input */
 296                   [v2]                  "m" (*v2),
 297                   [newv2]               "r" (newv2),
 298                   /* final store input */
 299                   [v]                   "m" (*v),
 300                   [expect]              "r" (expect),
 301                   [newv]                "r" (newv)
 302                   RSEQ_INJECT_INPUT
 303                 : "memory", "cc", "r0"
 304                   RSEQ_INJECT_CLOBBER
 305                 : abort, cmpfail
 306 #ifdef RSEQ_COMPARE_TWICE
 307                   , error1, error2
 308 #endif
 309         );
 310         return 0;
 311 abort:
 312         RSEQ_INJECT_FAILED
 313         return -1;
 314 cmpfail:
 315         return 1;
 316 #ifdef RSEQ_COMPARE_TWICE
 317 error1:
 318         rseq_bug("cpu_id comparison failed");
 319 error2:
 320         rseq_bug("expected value comparison failed");
 321 #endif
 322 }
 323
 324 /* s390 is TSO. */
 325 static inline __attribute__((always_inline))
 326 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 327                                          intptr_t *v2, intptr_t newv2,
 328                                          intptr_t newv, int cpu)
 329 {
 330         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 331 }
 332
 333 static inline __attribute__((always_inline))
 334 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 335                               intptr_t *v2, intptr_t expect2,
 336                               intptr_t newv, int cpu)
 337 {
 338         RSEQ_INJECT_C(9)
 339
 340         __asm__ __volatile__ goto (
 341                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 342                 /* Start rseq by storing table entry pointer into rseq_cs. */
 343                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 344                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 345                 RSEQ_INJECT_ASM(3)
 346                 LONG_CMP " %[expect], %[v]\n\t"
 347                 "jnz %l[cmpfail]\n\t"
 348                 RSEQ_INJECT_ASM(4)
 349                 LONG_CMP " %[expect2], %[v2]\n\t"
 350                 "jnz %l[cmpfail]\n\t"
 351                 RSEQ_INJECT_ASM(5)
 352 #ifdef RSEQ_COMPARE_TWICE
 353                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 354                 LONG_CMP " %[expect], %[v]\n\t"
 355                 "jnz %l[error2]\n\t"
 356                 LONG_CMP " %[expect2], %[v2]\n\t"
 357                 "jnz %l[error3]\n\t"
 358 #endif
 359                 /* final store */
 360                 LONG_S " %[newv], %[v]\n\t"
 361                 "2:\n\t"
 362                 RSEQ_INJECT_ASM(6)
 363                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 364                 : /* gcc asm goto does not allow outputs */
 365                 : [cpu_id]              "r" (cpu),
 366                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 367                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 368                   /* cmp2 input */
 369                   [v2]                  "m" (*v2),
 370                   [expect2]             "r" (expect2),
 371                   /* final store input */
 372                   [v]                   "m" (*v),
 373                   [expect]              "r" (expect),
 374                   [newv]                "r" (newv)
 375                   RSEQ_INJECT_INPUT
 376                 : "memory", "cc", "r0"
 377                   RSEQ_INJECT_CLOBBER
 378                 : abort, cmpfail
 379 #ifdef RSEQ_COMPARE_TWICE
 380                   , error1, error2, error3
 381 #endif
 382         );
 383         return 0;
 384 abort:
 385         RSEQ_INJECT_FAILED
 386         return -1;
 387 cmpfail:
 388         return 1;
 389 #ifdef RSEQ_COMPARE_TWICE
 390 error1:
 391         rseq_bug("cpu_id comparison failed");
 392 error2:
 393         rseq_bug("1st expected value comparison failed");
 394 error3:
 395         rseq_bug("2nd expected value comparison failed");
 396 #endif
 397 }
 398
 399 static inline __attribute__((always_inline))
 400 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 401                                  void *dst, void *src, size_t len,
 402                                  intptr_t newv, int cpu)
 403 {
 404         uint64_t rseq_scratch[3];
 405
 406         RSEQ_INJECT_C(9)
 407
 408         __asm__ __volatile__ goto (
 409                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 410                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 411                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 412                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 413                 /* Start rseq by storing table entry pointer into rseq_cs. */
 414                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 415                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 416                 RSEQ_INJECT_ASM(3)
 417                 LONG_CMP " %[expect], %[v]\n\t"
 418                 "jnz 5f\n\t"
 419                 RSEQ_INJECT_ASM(4)
 420 #ifdef RSEQ_COMPARE_TWICE
 421                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 422                 LONG_CMP " %[expect], %[v]\n\t"
 423                 "jnz 7f\n\t"
 424 #endif
 425                 /* try memcpy */
 426                 LONG_LT_R " %[len], %[len]\n\t"
 427                 "jz 333f\n\t"
 428                 "222:\n\t"
 429                 "ic %%r0,0(%[src])\n\t"
 430                 "stc %%r0,0(%[dst])\n\t"
 431                 LONG_ADDI " %[src], 1\n\t"
 432                 LONG_ADDI " %[dst], 1\n\t"
 433                 LONG_ADDI " %[len], -1\n\t"
 434                 "jnz 222b\n\t"
 435                 "333:\n\t"
 436                 RSEQ_INJECT_ASM(5)
 437                 /* final store */
 438                 LONG_S " %[newv], %[v]\n\t"
 439                 "2:\n\t"
 440                 RSEQ_INJECT_ASM(6)
 441                 /* teardown */
 442                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 443                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 444                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 445                 RSEQ_ASM_DEFINE_ABORT(4,
 446                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 447                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 448                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 449                         abort)
 450                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 451                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 452                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 453                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 454                         cmpfail)
 455 #ifdef RSEQ_COMPARE_TWICE
 456                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 457                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 458                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 459                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 460                         error1)
 461                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 462                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 463                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 464                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 465                         error2)
 466 #endif
 467                 : /* gcc asm goto does not allow outputs */
 468                 : [cpu_id]              "r" (cpu),
 469                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 470                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 471                   /* final store input */
 472                   [v]                   "m" (*v),
 473                   [expect]              "r" (expect),
 474                   [newv]                "r" (newv),
 475                   /* try memcpy input */
 476                   [dst]                 "r" (dst),
 477                   [src]                 "r" (src),
 478                   [len]                 "r" (len),
 479                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 480                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 481                   [rseq_scratch2]       "m" (rseq_scratch[2])
 482                   RSEQ_INJECT_INPUT
 483                 : "memory", "cc", "r0"
 484                   RSEQ_INJECT_CLOBBER
 485                 : abort, cmpfail
 486 #ifdef RSEQ_COMPARE_TWICE
 487                   , error1, error2
 488 #endif
 489         );
 490         return 0;
 491 abort:
 492         RSEQ_INJECT_FAILED
 493         return -1;
 494 cmpfail:
 495         return 1;
 496 #ifdef RSEQ_COMPARE_TWICE
 497 error1:
 498         rseq_bug("cpu_id comparison failed");
 499 error2:
 500         rseq_bug("expected value comparison failed");
 501 #endif
 502 }
 503
 504 /* s390 is TSO. */
 505 static inline __attribute__((always_inline))
 506 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 507                                          void *dst, void *src, size_t len,
 508                                          intptr_t newv, int cpu)
 509 {
 510         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 511                                             newv, cpu);
 512 }
 513 #endif /* !RSEQ_SKIP_FASTPATH */