tools/testing/selftests/rseq/rseq-s390.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2
   3 /*
   4  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
   5  * access-register mode nor the linkage stack this instruction will always
   6  * cause a special-operation exception (the trap-enabled bit in the DUCT
   7  * is and will stay 0). The instruction pattern is
   8  *      b2 ff 0f ff     trap4   4095(%r0)
   9  */
  10 #define RSEQ_SIG        0xB2FF0FFF
  11
  12 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  13 #define rseq_smp_rmb()  rseq_smp_mb()
  14 #define rseq_smp_wmb()  rseq_smp_mb()
  15
  16 #define rseq_smp_load_acquire(p)                                        \
  17 __extension__ ({                                                        \
  18         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  19         rseq_barrier();                                                 \
  20         ____p1;                                                         \
  21 })
  22
  23 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  24
  25 #define rseq_smp_store_release(p, v)                                    \
  26 do {                                                                    \
  27         rseq_barrier();                                                 \
  28         RSEQ_WRITE_ONCE(*p, v);                                         \
  29 } while (0)
  30
  31 #ifdef RSEQ_SKIP_FASTPATH
  32 #include "rseq-skip.h"
  33 #else /* !RSEQ_SKIP_FASTPATH */
  34
  35 #ifdef __s390x__
  36
  37 #define LONG_L                  "lg"
  38 #define LONG_S                  "stg"
  39 #define LONG_LT_R               "ltgr"
  40 #define LONG_CMP                "cg"
  41 #define LONG_CMP_R              "cgr"
  42 #define LONG_ADDI               "aghi"
  43 #define LONG_ADD_R              "agr"
  44
  45 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  46                                 start_ip, post_commit_offset, abort_ip) \
  47                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  48                 ".balign 32\n\t"                                        \
  49                 __rseq_str(label) ":\n\t"                               \
  50                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  51                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  52                 ".popsection\n\t"                                       \
  53                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  54                 ".quad " __rseq_str(label) "b\n\t"                      \
  55                 ".popsection\n\t"
  56
  57 /*
  58  * Exit points of a rseq critical section consist of all instructions outside
  59  * of the critical section where a critical section can either branch to or
  60  * reach through the normal course of its execution. The abort IP and the
  61  * post-commit IP are already part of the __rseq_cs section and should not be
  62  * explicitly defined as additional exit points. Knowing all exit points is
  63  * useful to assist debuggers stepping over the critical section.
  64  */
  65 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  66                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  67                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  68                 ".popsection\n\t"
  69
  70 #elif __s390__
  71
  72 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  73                                 start_ip, post_commit_offset, abort_ip) \
  74                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  75                 ".balign 32\n\t"                                        \
  76                 __rseq_str(label) ":\n\t"                               \
  77                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  78                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
  79                 ".popsection\n\t"                                       \
  80                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  81                 ".long 0x0, " __rseq_str(label) "b\n\t"                 \
  82                 ".popsection\n\t"
  83
  84 /*
  85  * Exit points of a rseq critical section consist of all instructions outside
  86  * of the critical section where a critical section can either branch to or
  87  * reach through the normal course of its execution. The abort IP and the
  88  * post-commit IP are already part of the __rseq_cs section and should not be
  89  * explicitly defined as additional exit points. Knowing all exit points is
  90  * useful to assist debuggers stepping over the critical section.
  91  */
  92 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  93                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  94                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
  95                 ".popsection\n\t"
  96
  97 #define LONG_L                  "l"
  98 #define LONG_S                  "st"
  99 #define LONG_LT_R               "ltr"
 100 #define LONG_CMP                "c"
 101 #define LONG_CMP_R              "cr"
 102 #define LONG_ADDI               "ahi"
 103 #define LONG_ADD_R              "ar"
 104
 105 #endif
 106
 107 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 108         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 109                                 (post_commit_ip - start_ip), abort_ip)
 110
 111 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 112                 RSEQ_INJECT_ASM(1)                                      \
 113                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 114                 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"          \
 115                 __rseq_str(label) ":\n\t"
 116
 117 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 118                 RSEQ_INJECT_ASM(2)                                      \
 119                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 120                 "jnz " __rseq_str(label) "\n\t"
 121
 122 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 123                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 124                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 125                 __rseq_str(label) ":\n\t"                               \
 126                 teardown                                                \
 127                 "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 128                 ".popsection\n\t"
 129
 130 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 131                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 132                 __rseq_str(label) ":\n\t"                               \
 133                 teardown                                                \
 134                 "jg %l[" __rseq_str(cmpfail_label) "]\n\t"              \
 135                 ".popsection\n\t"
 136
 137 static inline __attribute__((always_inline))
 138 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 139 {
 140         RSEQ_INJECT_C(9)
 141
 142         __asm__ __volatile__ goto (
 143                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 144                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 145 #ifdef RSEQ_COMPARE_TWICE
 146                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 147                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 148 #endif
 149                 /* Start rseq by storing table entry pointer into rseq_cs. */
 150                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 151                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 152                 RSEQ_INJECT_ASM(3)
 153                 LONG_CMP " %[expect], %[v]\n\t"
 154                 "jnz %l[cmpfail]\n\t"
 155                 RSEQ_INJECT_ASM(4)
 156 #ifdef RSEQ_COMPARE_TWICE
 157                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 158                 LONG_CMP " %[expect], %[v]\n\t"
 159                 "jnz %l[error2]\n\t"
 160 #endif
 161                 /* final store */
 162                 LONG_S " %[newv], %[v]\n\t"
 163                 "2:\n\t"
 164                 RSEQ_INJECT_ASM(5)
 165                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 166                 : /* gcc asm goto does not allow outputs */
 167                 : [cpu_id]              "r" (cpu),
 168                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 169                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 170                   [v]                   "m" (*v),
 171                   [expect]              "r" (expect),
 172                   [newv]                "r" (newv)
 173                   RSEQ_INJECT_INPUT
 174                 : "memory", "cc", "r0"
 175                   RSEQ_INJECT_CLOBBER
 176                 : abort, cmpfail
 177 #ifdef RSEQ_COMPARE_TWICE
 178                   , error1, error2
 179 #endif
 180         );
 181         return 0;
 182 abort:
 183         RSEQ_INJECT_FAILED
 184         return -1;
 185 cmpfail:
 186         return 1;
 187 #ifdef RSEQ_COMPARE_TWICE
 188 error1:
 189         rseq_bug("cpu_id comparison failed");
 190 error2:
 191         rseq_bug("expected value comparison failed");
 192 #endif
 193 }
 194
 195 /*
 196  * Compare @v against @expectnot. When it does _not_ match, load @v
 197  * into @load, and store the content of *@v + voffp into @v.
 198  */
 199 static inline __attribute__((always_inline))
 200 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 201                                off_t voffp, intptr_t *load, int cpu)
 202 {
 203         RSEQ_INJECT_C(9)
 204
 205         __asm__ __volatile__ goto (
 206                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 207                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 208 #ifdef RSEQ_COMPARE_TWICE
 209                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 210                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 211 #endif
 212                 /* Start rseq by storing table entry pointer into rseq_cs. */
 213                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 214                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 215                 RSEQ_INJECT_ASM(3)
 216                 LONG_L " %%r1, %[v]\n\t"
 217                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 218                 "je %l[cmpfail]\n\t"
 219                 RSEQ_INJECT_ASM(4)
 220 #ifdef RSEQ_COMPARE_TWICE
 221                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 222                 LONG_L " %%r1, %[v]\n\t"
 223                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 224                 "je %l[error2]\n\t"
 225 #endif
 226                 LONG_S " %%r1, %[load]\n\t"
 227                 LONG_ADD_R " %%r1, %[voffp]\n\t"
 228                 LONG_L " %%r1, 0(%%r1)\n\t"
 229                 /* final store */
 230                 LONG_S " %%r1, %[v]\n\t"
 231                 "2:\n\t"
 232                 RSEQ_INJECT_ASM(5)
 233                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 234                 : /* gcc asm goto does not allow outputs */
 235                 : [cpu_id]              "r" (cpu),
 236                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 237                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 238                   /* final store input */
 239                   [v]                   "m" (*v),
 240                   [expectnot]           "r" (expectnot),
 241                   [voffp]               "r" (voffp),
 242                   [load]                "m" (*load)
 243                   RSEQ_INJECT_INPUT
 244                 : "memory", "cc", "r0", "r1"
 245                   RSEQ_INJECT_CLOBBER
 246                 : abort, cmpfail
 247 #ifdef RSEQ_COMPARE_TWICE
 248                   , error1, error2
 249 #endif
 250         );
 251         return 0;
 252 abort:
 253         RSEQ_INJECT_FAILED
 254         return -1;
 255 cmpfail:
 256         return 1;
 257 #ifdef RSEQ_COMPARE_TWICE
 258 error1:
 259         rseq_bug("cpu_id comparison failed");
 260 error2:
 261         rseq_bug("expected value comparison failed");
 262 #endif
 263 }
 264
 265 static inline __attribute__((always_inline))
 266 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 267 {
 268         RSEQ_INJECT_C(9)
 269
 270         __asm__ __volatile__ goto (
 271                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 272 #ifdef RSEQ_COMPARE_TWICE
 273                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 274 #endif
 275                 /* Start rseq by storing table entry pointer into rseq_cs. */
 276                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 277                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 278                 RSEQ_INJECT_ASM(3)
 279 #ifdef RSEQ_COMPARE_TWICE
 280                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 281 #endif
 282                 LONG_L " %%r0, %[v]\n\t"
 283                 LONG_ADD_R " %%r0, %[count]\n\t"
 284                 /* final store */
 285                 LONG_S " %%r0, %[v]\n\t"
 286                 "2:\n\t"
 287                 RSEQ_INJECT_ASM(4)
 288                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 289                 : /* gcc asm goto does not allow outputs */
 290                 : [cpu_id]              "r" (cpu),
 291                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 292                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 293                   /* final store input */
 294                   [v]                   "m" (*v),
 295                   [count]               "r" (count)
 296                   RSEQ_INJECT_INPUT
 297                 : "memory", "cc", "r0"
 298                   RSEQ_INJECT_CLOBBER
 299                 : abort
 300 #ifdef RSEQ_COMPARE_TWICE
 301                   , error1
 302 #endif
 303         );
 304         return 0;
 305 abort:
 306         RSEQ_INJECT_FAILED
 307         return -1;
 308 #ifdef RSEQ_COMPARE_TWICE
 309 error1:
 310         rseq_bug("cpu_id comparison failed");
 311 #endif
 312 }
 313
 314 static inline __attribute__((always_inline))
 315 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 316                                  intptr_t *v2, intptr_t newv2,
 317                                  intptr_t newv, int cpu)
 318 {
 319         RSEQ_INJECT_C(9)
 320
 321         __asm__ __volatile__ goto (
 322                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 323                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 324 #ifdef RSEQ_COMPARE_TWICE
 325                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 326                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 327 #endif
 328                 /* Start rseq by storing table entry pointer into rseq_cs. */
 329                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 330                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 331                 RSEQ_INJECT_ASM(3)
 332                 LONG_CMP " %[expect], %[v]\n\t"
 333                 "jnz %l[cmpfail]\n\t"
 334                 RSEQ_INJECT_ASM(4)
 335 #ifdef RSEQ_COMPARE_TWICE
 336                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 337                 LONG_CMP " %[expect], %[v]\n\t"
 338                 "jnz %l[error2]\n\t"
 339 #endif
 340                 /* try store */
 341                 LONG_S " %[newv2], %[v2]\n\t"
 342                 RSEQ_INJECT_ASM(5)
 343                 /* final store */
 344                 LONG_S " %[newv], %[v]\n\t"
 345                 "2:\n\t"
 346                 RSEQ_INJECT_ASM(6)
 347                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 348                 : /* gcc asm goto does not allow outputs */
 349                 : [cpu_id]              "r" (cpu),
 350                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 351                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 352                   /* try store input */
 353                   [v2]                  "m" (*v2),
 354                   [newv2]               "r" (newv2),
 355                   /* final store input */
 356                   [v]                   "m" (*v),
 357                   [expect]              "r" (expect),
 358                   [newv]                "r" (newv)
 359                   RSEQ_INJECT_INPUT
 360                 : "memory", "cc", "r0"
 361                   RSEQ_INJECT_CLOBBER
 362                 : abort, cmpfail
 363 #ifdef RSEQ_COMPARE_TWICE
 364                   , error1, error2
 365 #endif
 366         );
 367         return 0;
 368 abort:
 369         RSEQ_INJECT_FAILED
 370         return -1;
 371 cmpfail:
 372         return 1;
 373 #ifdef RSEQ_COMPARE_TWICE
 374 error1:
 375         rseq_bug("cpu_id comparison failed");
 376 error2:
 377         rseq_bug("expected value comparison failed");
 378 #endif
 379 }
 380
 381 /* s390 is TSO. */
 382 static inline __attribute__((always_inline))
 383 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 384                                          intptr_t *v2, intptr_t newv2,
 385                                          intptr_t newv, int cpu)
 386 {
 387         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 388 }
 389
 390 static inline __attribute__((always_inline))
 391 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 392                               intptr_t *v2, intptr_t expect2,
 393                               intptr_t newv, int cpu)
 394 {
 395         RSEQ_INJECT_C(9)
 396
 397         __asm__ __volatile__ goto (
 398                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 399                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 400 #ifdef RSEQ_COMPARE_TWICE
 401                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 402                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 403                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 404 #endif
 405                 /* Start rseq by storing table entry pointer into rseq_cs. */
 406                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 407                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 408                 RSEQ_INJECT_ASM(3)
 409                 LONG_CMP " %[expect], %[v]\n\t"
 410                 "jnz %l[cmpfail]\n\t"
 411                 RSEQ_INJECT_ASM(4)
 412                 LONG_CMP " %[expect2], %[v2]\n\t"
 413                 "jnz %l[cmpfail]\n\t"
 414                 RSEQ_INJECT_ASM(5)
 415 #ifdef RSEQ_COMPARE_TWICE
 416                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 417                 LONG_CMP " %[expect], %[v]\n\t"
 418                 "jnz %l[error2]\n\t"
 419                 LONG_CMP " %[expect2], %[v2]\n\t"
 420                 "jnz %l[error3]\n\t"
 421 #endif
 422                 /* final store */
 423                 LONG_S " %[newv], %[v]\n\t"
 424                 "2:\n\t"
 425                 RSEQ_INJECT_ASM(6)
 426                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 427                 : /* gcc asm goto does not allow outputs */
 428                 : [cpu_id]              "r" (cpu),
 429                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 430                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 431                   /* cmp2 input */
 432                   [v2]                  "m" (*v2),
 433                   [expect2]             "r" (expect2),
 434                   /* final store input */
 435                   [v]                   "m" (*v),
 436                   [expect]              "r" (expect),
 437                   [newv]                "r" (newv)
 438                   RSEQ_INJECT_INPUT
 439                 : "memory", "cc", "r0"
 440                   RSEQ_INJECT_CLOBBER
 441                 : abort, cmpfail
 442 #ifdef RSEQ_COMPARE_TWICE
 443                   , error1, error2, error3
 444 #endif
 445         );
 446         return 0;
 447 abort:
 448         RSEQ_INJECT_FAILED
 449         return -1;
 450 cmpfail:
 451         return 1;
 452 #ifdef RSEQ_COMPARE_TWICE
 453 error1:
 454         rseq_bug("cpu_id comparison failed");
 455 error2:
 456         rseq_bug("1st expected value comparison failed");
 457 error3:
 458         rseq_bug("2nd expected value comparison failed");
 459 #endif
 460 }
 461
 462 static inline __attribute__((always_inline))
 463 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 464                                  void *dst, void *src, size_t len,
 465                                  intptr_t newv, int cpu)
 466 {
 467         uint64_t rseq_scratch[3];
 468
 469         RSEQ_INJECT_C(9)
 470
 471         __asm__ __volatile__ goto (
 472                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 473                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 474 #ifdef RSEQ_COMPARE_TWICE
 475                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 476                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 477 #endif
 478                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 479                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 480                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 481                 /* Start rseq by storing table entry pointer into rseq_cs. */
 482                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 483                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 484                 RSEQ_INJECT_ASM(3)
 485                 LONG_CMP " %[expect], %[v]\n\t"
 486                 "jnz 5f\n\t"
 487                 RSEQ_INJECT_ASM(4)
 488 #ifdef RSEQ_COMPARE_TWICE
 489                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 490                 LONG_CMP " %[expect], %[v]\n\t"
 491                 "jnz 7f\n\t"
 492 #endif
 493                 /* try memcpy */
 494                 LONG_LT_R " %[len], %[len]\n\t"
 495                 "jz 333f\n\t"
 496                 "222:\n\t"
 497                 "ic %%r0,0(%[src])\n\t"
 498                 "stc %%r0,0(%[dst])\n\t"
 499                 LONG_ADDI " %[src], 1\n\t"
 500                 LONG_ADDI " %[dst], 1\n\t"
 501                 LONG_ADDI " %[len], -1\n\t"
 502                 "jnz 222b\n\t"
 503                 "333:\n\t"
 504                 RSEQ_INJECT_ASM(5)
 505                 /* final store */
 506                 LONG_S " %[newv], %[v]\n\t"
 507                 "2:\n\t"
 508                 RSEQ_INJECT_ASM(6)
 509                 /* teardown */
 510                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 511                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 512                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 513                 RSEQ_ASM_DEFINE_ABORT(4,
 514                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 515                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 516                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 517                         abort)
 518                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 519                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 520                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 521                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 522                         cmpfail)
 523 #ifdef RSEQ_COMPARE_TWICE
 524                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 525                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 526                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 527                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 528                         error1)
 529                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 530                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 531                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 532                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 533                         error2)
 534 #endif
 535                 : /* gcc asm goto does not allow outputs */
 536                 : [cpu_id]              "r" (cpu),
 537                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 538                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 539                   /* final store input */
 540                   [v]                   "m" (*v),
 541                   [expect]              "r" (expect),
 542                   [newv]                "r" (newv),
 543                   /* try memcpy input */
 544                   [dst]                 "r" (dst),
 545                   [src]                 "r" (src),
 546                   [len]                 "r" (len),
 547                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 548                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 549                   [rseq_scratch2]       "m" (rseq_scratch[2])
 550                   RSEQ_INJECT_INPUT
 551                 : "memory", "cc", "r0"
 552                   RSEQ_INJECT_CLOBBER
 553                 : abort, cmpfail
 554 #ifdef RSEQ_COMPARE_TWICE
 555                   , error1, error2
 556 #endif
 557         );
 558         return 0;
 559 abort:
 560         RSEQ_INJECT_FAILED
 561         return -1;
 562 cmpfail:
 563         return 1;
 564 #ifdef RSEQ_COMPARE_TWICE
 565 error1:
 566         rseq_bug("cpu_id comparison failed");
 567 error2:
 568         rseq_bug("expected value comparison failed");
 569 #endif
 570 }
 571
 572 /* s390 is TSO. */
 573 static inline __attribute__((always_inline))
 574 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 575                                          void *dst, void *src, size_t len,
 576                                          intptr_t newv, int cpu)
 577 {
 578         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 579                                             newv, cpu);
 580 }
 581 #endif /* !RSEQ_SKIP_FASTPATH */