tools/testing/selftests/rseq/rseq-ppc.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-ppc.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
   7  */
   8
   9 /*
  10  * RSEQ_SIG is used with the following trap instruction:
  11  *
  12  * powerpc-be:    0f e5 00 0b           twui   r5,11
  13  * powerpc64-le:  0b 00 e5 0f           twui   r5,11
  14  * powerpc64-be:  0f e5 00 0b           twui   r5,11
  15  */
  16
  17 #define RSEQ_SIG        0x0fe5000b
  18
  19 #define rseq_smp_mb()           __asm__ __volatile__ ("sync"    ::: "memory", "cc")
  20 #define rseq_smp_lwsync()       __asm__ __volatile__ ("lwsync"  ::: "memory", "cc")
  21 #define rseq_smp_rmb()          rseq_smp_lwsync()
  22 #define rseq_smp_wmb()          rseq_smp_lwsync()
  23
  24 #define rseq_smp_load_acquire(p)                                        \
  25 __extension__ ({                                                        \
  26         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  27         rseq_smp_lwsync();                                              \
  28         ____p1;                                                         \
  29 })
  30
  31 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_lwsync()
  32
  33 #define rseq_smp_store_release(p, v)                                    \
  34 do {                                                                    \
  35         rseq_smp_lwsync();                                              \
  36         RSEQ_WRITE_ONCE(*p, v);                                         \
  37 } while (0)
  38
  39 #ifdef RSEQ_SKIP_FASTPATH
  40 #include "rseq-skip.h"
  41 #else /* !RSEQ_SKIP_FASTPATH */
  42
  43 /*
  44  * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
  45  * better handle single-stepping through the restartable critical sections.
  46  */
  47
  48 #ifdef __PPC64__
  49
  50 #define STORE_WORD      "std "
  51 #define LOAD_WORD       "ld "
  52 #define LOADX_WORD      "ldx "
  53 #define CMP_WORD        "cmpd "
  54
  55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  56                         start_ip, post_commit_offset, abort_ip)                 \
  57                 ".pushsection __rseq_cs, \"aw\"\n\t"                            \
  58                 ".balign 32\n\t"                                                \
  59                 __rseq_str(label) ":\n\t"                                       \
  60                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
  61                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  62                 ".popsection\n\t"                                               \
  63                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"                  \
  64                 ".quad " __rseq_str(label) "b\n\t"                              \
  65                 ".popsection\n\t"
  66
  67 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
  68                 RSEQ_INJECT_ASM(1)                                              \
  69                 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"             \
  70                 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"       \
  71                 "rldicr %%r17, %%r17, 32, 31\n\t"                               \
  72                 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"        \
  73                 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"            \
  74                 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
  75                 __rseq_str(label) ":\n\t"
  76
  77 /*
  78  * Exit points of a rseq critical section consist of all instructions outside
  79  * of the critical section where a critical section can either branch to or
  80  * reach through the normal course of its execution. The abort IP and the
  81  * post-commit IP are already part of the __rseq_cs section and should not be
  82  * explicitly defined as additional exit points. Knowing all exit points is
  83  * useful to assist debuggers stepping over the critical section.
  84  */
  85 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  86                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  87                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  88                 ".popsection\n\t"
  89
  90 #else /* #ifdef __PPC64__ */
  91
  92 #define STORE_WORD      "stw "
  93 #define LOAD_WORD       "lwz "
  94 #define LOADX_WORD      "lwzx "
  95 #define CMP_WORD        "cmpw "
  96
  97 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  98                         start_ip, post_commit_offset, abort_ip)                 \
  99                 ".pushsection __rseq_cs, \"aw\"\n\t"                            \
 100                 ".balign 32\n\t"                                                \
 101                 __rseq_str(label) ":\n\t"                                       \
 102                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
 103                 /* 32-bit only supported on BE */                               \
 104                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
 105                 ".popsection\n\t"                                       \
 106                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 107                 ".long 0x0, " __rseq_str(label) "b\n\t"                 \
 108                 ".popsection\n\t"
 109
 110 /*
 111  * Exit points of a rseq critical section consist of all instructions outside
 112  * of the critical section where a critical section can either branch to or
 113  * reach through the normal course of its execution. The abort IP and the
 114  * post-commit IP are already part of the __rseq_cs section and should not be
 115  * explicitly defined as additional exit points. Knowing all exit points is
 116  * useful to assist debuggers stepping over the critical section.
 117  */
 118 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                           \
 119                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"              \
 120                 /* 32-bit only supported on BE */                               \
 121                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
 122                 ".popsection\n\t"
 123
 124 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 125                 RSEQ_INJECT_ASM(1)                                              \
 126                 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"                  \
 127                 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"           \
 128                 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
 129                 __rseq_str(label) ":\n\t"
 130
 131 #endif /* #ifdef __PPC64__ */
 132
 133 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
 134                 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 135                                         (post_commit_ip - start_ip), abort_ip)
 136
 137 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)                      \
 138                 RSEQ_INJECT_ASM(2)                                              \
 139                 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t"              \
 140                 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"                \
 141                 "bne- cr7, " __rseq_str(label) "\n\t"
 142
 143 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                               \
 144                 ".pushsection __rseq_failure, \"ax\"\n\t"                       \
 145                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                            \
 146                 __rseq_str(label) ":\n\t"                                       \
 147                 "b %l[" __rseq_str(abort_label) "]\n\t"                         \
 148                 ".popsection\n\t"
 149
 150 /*
 151  * RSEQ_ASM_OPs: asm operations for rseq
 152  *      RSEQ_ASM_OP_R_*: has hard-code registers in it
 153  *      RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
 154  */
 155 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)                                   \
 156                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
 157                 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"            \
 158                 "bne- cr7, " __rseq_str(label) "\n\t"
 159
 160 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label)                                \
 161                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
 162                 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"         \
 163                 "beq- cr7, " __rseq_str(label) "\n\t"
 164
 165 #define RSEQ_ASM_OP_STORE(value, var)                                           \
 166                 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
 167
 168 /* Load @var to r17 */
 169 #define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 170                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
 171
 172 /* Store r17 to @var */
 173 #define RSEQ_ASM_OP_R_STORE(var)                                                \
 174                 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
 175
 176 /* Add @count to r17 */
 177 #define RSEQ_ASM_OP_R_ADD(count)                                                \
 178                 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
 179
 180 /* Load (r17 + voffp) to r17 */
 181 #define RSEQ_ASM_OP_R_LOADX(voffp)                                              \
 182                 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
 183
 184 /* TODO: implement a faster memcpy. */
 185 #define RSEQ_ASM_OP_R_MEMCPY() \
 186                 "cmpdi %%r19, 0\n\t" \
 187                 "beq 333f\n\t" \
 188                 "addi %%r20, %%r20, -1\n\t" \
 189                 "addi %%r21, %%r21, -1\n\t" \
 190                 "222:\n\t" \
 191                 "lbzu %%r18, 1(%%r20)\n\t" \
 192                 "stbu %%r18, 1(%%r21)\n\t" \
 193                 "addi %%r19, %%r19, -1\n\t" \
 194                 "cmpdi %%r19, 0\n\t" \
 195                 "bne 222b\n\t" \
 196                 "333:\n\t" \
 197
 198 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 199                 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                  \
 200                 __rseq_str(post_commit_label) ":\n\t"
 201
 202 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 203                 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
 204                 __rseq_str(post_commit_label) ":\n\t"
 205
 206 static inline __attribute__((always_inline))
 207 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 208 {
 209         RSEQ_INJECT_C(9)
 210
 211         __asm__ __volatile__ goto (
 212                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 213                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 214 #ifdef RSEQ_COMPARE_TWICE
 215                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 216                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 217 #endif
 218                 /* Start rseq by storing table entry pointer into rseq_cs. */
 219                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 220                 /* cmp cpuid */
 221                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 222                 RSEQ_INJECT_ASM(3)
 223                 /* cmp @v equal to @expect */
 224                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 225                 RSEQ_INJECT_ASM(4)
 226 #ifdef RSEQ_COMPARE_TWICE
 227                 /* cmp cpuid */
 228                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 229                 /* cmp @v equal to @expect */
 230                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 231 #endif
 232                 /* final store */
 233                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 234                 RSEQ_INJECT_ASM(5)
 235                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 236                 : /* gcc asm goto does not allow outputs */
 237                 : [cpu_id]              "r" (cpu),
 238                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 239                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 240                   [v]                   "m" (*v),
 241                   [expect]              "r" (expect),
 242                   [newv]                "r" (newv)
 243                   RSEQ_INJECT_INPUT
 244                 : "memory", "cc", "r17"
 245                   RSEQ_INJECT_CLOBBER
 246                 : abort, cmpfail
 247 #ifdef RSEQ_COMPARE_TWICE
 248                   , error1, error2
 249 #endif
 250         );
 251         return 0;
 252 abort:
 253         RSEQ_INJECT_FAILED
 254         return -1;
 255 cmpfail:
 256         return 1;
 257 #ifdef RSEQ_COMPARE_TWICE
 258 error1:
 259         rseq_bug("cpu_id comparison failed");
 260 error2:
 261         rseq_bug("expected value comparison failed");
 262 #endif
 263 }
 264
 265 static inline __attribute__((always_inline))
 266 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 267                                off_t voffp, intptr_t *load, int cpu)
 268 {
 269         RSEQ_INJECT_C(9)
 270
 271         __asm__ __volatile__ goto (
 272                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 273                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 274 #ifdef RSEQ_COMPARE_TWICE
 275                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 276                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 277 #endif
 278                 /* Start rseq by storing table entry pointer into rseq_cs. */
 279                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 280                 /* cmp cpuid */
 281                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 282                 RSEQ_INJECT_ASM(3)
 283                 /* cmp @v not equal to @expectnot */
 284                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
 285                 RSEQ_INJECT_ASM(4)
 286 #ifdef RSEQ_COMPARE_TWICE
 287                 /* cmp cpuid */
 288                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 289                 /* cmp @v not equal to @expectnot */
 290                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
 291 #endif
 292                 /* load the value of @v */
 293                 RSEQ_ASM_OP_R_LOAD(v)
 294                 /* store it in @load */
 295                 RSEQ_ASM_OP_R_STORE(load)
 296                 /* dereference voffp(v) */
 297                 RSEQ_ASM_OP_R_LOADX(voffp)
 298                 /* final store the value at voffp(v) */
 299                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 300                 RSEQ_INJECT_ASM(5)
 301                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 302                 : /* gcc asm goto does not allow outputs */
 303                 : [cpu_id]              "r" (cpu),
 304                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 305                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 306                   /* final store input */
 307                   [v]                   "m" (*v),
 308                   [expectnot]           "r" (expectnot),
 309                   [voffp]               "b" (voffp),
 310                   [load]                "m" (*load)
 311                   RSEQ_INJECT_INPUT
 312                 : "memory", "cc", "r17"
 313                   RSEQ_INJECT_CLOBBER
 314                 : abort, cmpfail
 315 #ifdef RSEQ_COMPARE_TWICE
 316                   , error1, error2
 317 #endif
 318         );
 319         return 0;
 320 abort:
 321         RSEQ_INJECT_FAILED
 322         return -1;
 323 cmpfail:
 324         return 1;
 325 #ifdef RSEQ_COMPARE_TWICE
 326 error1:
 327         rseq_bug("cpu_id comparison failed");
 328 error2:
 329         rseq_bug("expected value comparison failed");
 330 #endif
 331 }
 332
 333 static inline __attribute__((always_inline))
 334 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 335 {
 336         RSEQ_INJECT_C(9)
 337
 338         __asm__ __volatile__ goto (
 339                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 340 #ifdef RSEQ_COMPARE_TWICE
 341                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 342 #endif
 343                 /* Start rseq by storing table entry pointer into rseq_cs. */
 344                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 345                 /* cmp cpuid */
 346                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 347                 RSEQ_INJECT_ASM(3)
 348 #ifdef RSEQ_COMPARE_TWICE
 349                 /* cmp cpuid */
 350                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 351 #endif
 352                 /* load the value of @v */
 353                 RSEQ_ASM_OP_R_LOAD(v)
 354                 /* add @count to it */
 355                 RSEQ_ASM_OP_R_ADD(count)
 356                 /* final store */
 357                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 358                 RSEQ_INJECT_ASM(4)
 359                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 360                 : /* gcc asm goto does not allow outputs */
 361                 : [cpu_id]              "r" (cpu),
 362                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 363                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 364                   /* final store input */
 365                   [v]                   "m" (*v),
 366                   [count]               "r" (count)
 367                   RSEQ_INJECT_INPUT
 368                 : "memory", "cc", "r17"
 369                   RSEQ_INJECT_CLOBBER
 370                 : abort
 371 #ifdef RSEQ_COMPARE_TWICE
 372                   , error1
 373 #endif
 374         );
 375         return 0;
 376 abort:
 377         RSEQ_INJECT_FAILED
 378         return -1;
 379 #ifdef RSEQ_COMPARE_TWICE
 380 error1:
 381         rseq_bug("cpu_id comparison failed");
 382 #endif
 383 }
 384
 385 static inline __attribute__((always_inline))
 386 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 387                                  intptr_t *v2, intptr_t newv2,
 388                                  intptr_t newv, int cpu)
 389 {
 390         RSEQ_INJECT_C(9)
 391
 392         __asm__ __volatile__ goto (
 393                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 394                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 395 #ifdef RSEQ_COMPARE_TWICE
 396                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 397                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 398 #endif
 399                 /* Start rseq by storing table entry pointer into rseq_cs. */
 400                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 401                 /* cmp cpuid */
 402                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 403                 RSEQ_INJECT_ASM(3)
 404                 /* cmp @v equal to @expect */
 405                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 406                 RSEQ_INJECT_ASM(4)
 407 #ifdef RSEQ_COMPARE_TWICE
 408                 /* cmp cpuid */
 409                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 410                 /* cmp @v equal to @expect */
 411                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 412 #endif
 413                 /* try store */
 414                 RSEQ_ASM_OP_STORE(newv2, v2)
 415                 RSEQ_INJECT_ASM(5)
 416                 /* final store */
 417                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 418                 RSEQ_INJECT_ASM(6)
 419                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 420                 : /* gcc asm goto does not allow outputs */
 421                 : [cpu_id]              "r" (cpu),
 422                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 423                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 424                   /* try store input */
 425                   [v2]                  "m" (*v2),
 426                   [newv2]               "r" (newv2),
 427                   /* final store input */
 428                   [v]                   "m" (*v),
 429                   [expect]              "r" (expect),
 430                   [newv]                "r" (newv)
 431                   RSEQ_INJECT_INPUT
 432                 : "memory", "cc", "r17"
 433                   RSEQ_INJECT_CLOBBER
 434                 : abort, cmpfail
 435 #ifdef RSEQ_COMPARE_TWICE
 436                   , error1, error2
 437 #endif
 438         );
 439         return 0;
 440 abort:
 441         RSEQ_INJECT_FAILED
 442         return -1;
 443 cmpfail:
 444         return 1;
 445 #ifdef RSEQ_COMPARE_TWICE
 446 error1:
 447         rseq_bug("cpu_id comparison failed");
 448 error2:
 449         rseq_bug("expected value comparison failed");
 450 #endif
 451 }
 452
 453 static inline __attribute__((always_inline))
 454 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 455                                          intptr_t *v2, intptr_t newv2,
 456                                          intptr_t newv, int cpu)
 457 {
 458         RSEQ_INJECT_C(9)
 459
 460         __asm__ __volatile__ goto (
 461                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 462                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 463 #ifdef RSEQ_COMPARE_TWICE
 464                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 465                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 466 #endif
 467                 /* Start rseq by storing table entry pointer into rseq_cs. */
 468                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 469                 /* cmp cpuid */
 470                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 471                 RSEQ_INJECT_ASM(3)
 472                 /* cmp @v equal to @expect */
 473                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 474                 RSEQ_INJECT_ASM(4)
 475 #ifdef RSEQ_COMPARE_TWICE
 476                 /* cmp cpuid */
 477                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 478                 /* cmp @v equal to @expect */
 479                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 480 #endif
 481                 /* try store */
 482                 RSEQ_ASM_OP_STORE(newv2, v2)
 483                 RSEQ_INJECT_ASM(5)
 484                 /* for 'release' */
 485                 "lwsync\n\t"
 486                 /* final store */
 487                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 488                 RSEQ_INJECT_ASM(6)
 489                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 490                 : /* gcc asm goto does not allow outputs */
 491                 : [cpu_id]              "r" (cpu),
 492                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 493                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 494                   /* try store input */
 495                   [v2]                  "m" (*v2),
 496                   [newv2]               "r" (newv2),
 497                   /* final store input */
 498                   [v]                   "m" (*v),
 499                   [expect]              "r" (expect),
 500                   [newv]                "r" (newv)
 501                   RSEQ_INJECT_INPUT
 502                 : "memory", "cc", "r17"
 503                   RSEQ_INJECT_CLOBBER
 504                 : abort, cmpfail
 505 #ifdef RSEQ_COMPARE_TWICE
 506                   , error1, error2
 507 #endif
 508         );
 509         return 0;
 510 abort:
 511         RSEQ_INJECT_FAILED
 512         return -1;
 513 cmpfail:
 514         return 1;
 515 #ifdef RSEQ_COMPARE_TWICE
 516 error1:
 517         rseq_bug("cpu_id comparison failed");
 518 error2:
 519         rseq_bug("expected value comparison failed");
 520 #endif
 521 }
 522
 523 static inline __attribute__((always_inline))
 524 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 525                               intptr_t *v2, intptr_t expect2,
 526                               intptr_t newv, int cpu)
 527 {
 528         RSEQ_INJECT_C(9)
 529
 530         __asm__ __volatile__ goto (
 531                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 532                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 533 #ifdef RSEQ_COMPARE_TWICE
 534                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 535                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 536                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 537 #endif
 538                 /* Start rseq by storing table entry pointer into rseq_cs. */
 539                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 540                 /* cmp cpuid */
 541                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 542                 RSEQ_INJECT_ASM(3)
 543                 /* cmp @v equal to @expect */
 544                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 545                 RSEQ_INJECT_ASM(4)
 546                 /* cmp @v2 equal to @expct2 */
 547                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 548                 RSEQ_INJECT_ASM(5)
 549 #ifdef RSEQ_COMPARE_TWICE
 550                 /* cmp cpuid */
 551                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 552                 /* cmp @v equal to @expect */
 553                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 554                 /* cmp @v2 equal to @expct2 */
 555                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 556 #endif
 557                 /* final store */
 558                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 559                 RSEQ_INJECT_ASM(6)
 560                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 561                 : /* gcc asm goto does not allow outputs */
 562                 : [cpu_id]              "r" (cpu),
 563                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 564                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 565                   /* cmp2 input */
 566                   [v2]                  "m" (*v2),
 567                   [expect2]             "r" (expect2),
 568                   /* final store input */
 569                   [v]                   "m" (*v),
 570                   [expect]              "r" (expect),
 571                   [newv]                "r" (newv)
 572                   RSEQ_INJECT_INPUT
 573                 : "memory", "cc", "r17"
 574                   RSEQ_INJECT_CLOBBER
 575                 : abort, cmpfail
 576 #ifdef RSEQ_COMPARE_TWICE
 577                   , error1, error2, error3
 578 #endif
 579         );
 580         return 0;
 581 abort:
 582         RSEQ_INJECT_FAILED
 583         return -1;
 584 cmpfail:
 585         return 1;
 586 #ifdef RSEQ_COMPARE_TWICE
 587 error1:
 588         rseq_bug("cpu_id comparison failed");
 589 error2:
 590         rseq_bug("1st expected value comparison failed");
 591 error3:
 592         rseq_bug("2nd expected value comparison failed");
 593 #endif
 594 }
 595
 596 static inline __attribute__((always_inline))
 597 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 598                                  void *dst, void *src, size_t len,
 599                                  intptr_t newv, int cpu)
 600 {
 601         RSEQ_INJECT_C(9)
 602
 603         __asm__ __volatile__ goto (
 604                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 605                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 606 #ifdef RSEQ_COMPARE_TWICE
 607                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 608                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 609 #endif
 610                 /* setup for mempcy */
 611                 "mr %%r19, %[len]\n\t"
 612                 "mr %%r20, %[src]\n\t"
 613                 "mr %%r21, %[dst]\n\t"
 614                 /* Start rseq by storing table entry pointer into rseq_cs. */
 615                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 616                 /* cmp cpuid */
 617                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 618                 RSEQ_INJECT_ASM(3)
 619                 /* cmp @v equal to @expect */
 620                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 621                 RSEQ_INJECT_ASM(4)
 622 #ifdef RSEQ_COMPARE_TWICE
 623                 /* cmp cpuid */
 624                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 625                 /* cmp @v equal to @expect */
 626                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 627 #endif
 628                 /* try memcpy */
 629                 RSEQ_ASM_OP_R_MEMCPY()
 630                 RSEQ_INJECT_ASM(5)
 631                 /* final store */
 632                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 633                 RSEQ_INJECT_ASM(6)
 634                 /* teardown */
 635                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 636                 : /* gcc asm goto does not allow outputs */
 637                 : [cpu_id]              "r" (cpu),
 638                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 639                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 640                   /* final store input */
 641                   [v]                   "m" (*v),
 642                   [expect]              "r" (expect),
 643                   [newv]                "r" (newv),
 644                   /* try memcpy input */
 645                   [dst]                 "r" (dst),
 646                   [src]                 "r" (src),
 647                   [len]                 "r" (len)
 648                   RSEQ_INJECT_INPUT
 649                 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 650                   RSEQ_INJECT_CLOBBER
 651                 : abort, cmpfail
 652 #ifdef RSEQ_COMPARE_TWICE
 653                   , error1, error2
 654 #endif
 655         );
 656         return 0;
 657 abort:
 658         RSEQ_INJECT_FAILED
 659         return -1;
 660 cmpfail:
 661         return 1;
 662 #ifdef RSEQ_COMPARE_TWICE
 663 error1:
 664         rseq_bug("cpu_id comparison failed");
 665 error2:
 666         rseq_bug("expected value comparison failed");
 667 #endif
 668 }
 669
 670 static inline __attribute__((always_inline))
 671 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 672                                          void *dst, void *src, size_t len,
 673                                          intptr_t newv, int cpu)
 674 {
 675         RSEQ_INJECT_C(9)
 676
 677         __asm__ __volatile__ goto (
 678                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 679                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 680 #ifdef RSEQ_COMPARE_TWICE
 681                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 682                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 683 #endif
 684                 /* setup for mempcy */
 685                 "mr %%r19, %[len]\n\t"
 686                 "mr %%r20, %[src]\n\t"
 687                 "mr %%r21, %[dst]\n\t"
 688                 /* Start rseq by storing table entry pointer into rseq_cs. */
 689                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 690                 /* cmp cpuid */
 691                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 692                 RSEQ_INJECT_ASM(3)
 693                 /* cmp @v equal to @expect */
 694                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 695                 RSEQ_INJECT_ASM(4)
 696 #ifdef RSEQ_COMPARE_TWICE
 697                 /* cmp cpuid */
 698                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 699                 /* cmp @v equal to @expect */
 700                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 701 #endif
 702                 /* try memcpy */
 703                 RSEQ_ASM_OP_R_MEMCPY()
 704                 RSEQ_INJECT_ASM(5)
 705                 /* for 'release' */
 706                 "lwsync\n\t"
 707                 /* final store */
 708                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 709                 RSEQ_INJECT_ASM(6)
 710                 /* teardown */
 711                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 712                 : /* gcc asm goto does not allow outputs */
 713                 : [cpu_id]              "r" (cpu),
 714                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 715                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 716                   /* final store input */
 717                   [v]                   "m" (*v),
 718                   [expect]              "r" (expect),
 719                   [newv]                "r" (newv),
 720                   /* try memcpy input */
 721                   [dst]                 "r" (dst),
 722                   [src]                 "r" (src),
 723                   [len]                 "r" (len)
 724                   RSEQ_INJECT_INPUT
 725                 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 726                   RSEQ_INJECT_CLOBBER
 727                 : abort, cmpfail
 728 #ifdef RSEQ_COMPARE_TWICE
 729                   , error1, error2
 730 #endif
 731         );
 732         return 0;
 733 abort:
 734         RSEQ_INJECT_FAILED
 735         return -1;
 736 cmpfail:
 737         return 1;
 738 #ifdef RSEQ_COMPARE_TWICE
 739 error1:
 740         rseq_bug("cpu_id comparison failed");
 741 error2:
 742         rseq_bug("expected value comparison failed");
 743 #endif
 744 }
 745
 746 #undef STORE_WORD
 747 #undef LOAD_WORD
 748 #undef LOADX_WORD
 749 #undef CMP_WORD
 750
 751 #endif /* !RSEQ_SKIP_FASTPATH */