tools/testing/selftests/rseq/rseq-ppc.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-ppc.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
   7  */
   8
   9 #define RSEQ_SIG        0x53053053
  10
  11 #define rseq_smp_mb()           __asm__ __volatile__ ("sync"    ::: "memory", "cc")
  12 #define rseq_smp_lwsync()       __asm__ __volatile__ ("lwsync"  ::: "memory", "cc")
  13 #define rseq_smp_rmb()          rseq_smp_lwsync()
  14 #define rseq_smp_wmb()          rseq_smp_lwsync()
  15
  16 #define rseq_smp_load_acquire(p)                                        \
  17 __extension__ ({                                                        \
  18         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  19         rseq_smp_lwsync();                                              \
  20         ____p1;                                                         \
  21 })
  22
  23 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_lwsync()
  24
  25 #define rseq_smp_store_release(p, v)                                    \
  26 do {                                                                    \
  27         rseq_smp_lwsync();                                              \
  28         RSEQ_WRITE_ONCE(*p, v);                                         \
  29 } while (0)
  30
  31 #ifdef RSEQ_SKIP_FASTPATH
  32 #include "rseq-skip.h"
  33 #else /* !RSEQ_SKIP_FASTPATH */
  34
  35 /*
  36  * The __rseq_table section can be used by debuggers to better handle
  37  * single-stepping through the restartable critical sections.
  38  */
  39
  40 #ifdef __PPC64__
  41
  42 #define STORE_WORD      "std "
  43 #define LOAD_WORD       "ld "
  44 #define LOADX_WORD      "ldx "
  45 #define CMP_WORD        "cmpd "
  46
  47 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  48                         start_ip, post_commit_offset, abort_ip)                 \
  49                 ".pushsection __rseq_table, \"aw\"\n\t"                         \
  50                 ".balign 32\n\t"                                                \
  51                 __rseq_str(label) ":\n\t"                                       \
  52                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
  53                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  54                 ".popsection\n\t"
  55
  56 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
  57                 RSEQ_INJECT_ASM(1)                                              \
  58                 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"             \
  59                 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"       \
  60                 "rldicr %%r17, %%r17, 32, 31\n\t"                               \
  61                 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"        \
  62                 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"            \
  63                 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
  64                 __rseq_str(label) ":\n\t"
  65
  66 #else /* #ifdef __PPC64__ */
  67
  68 #define STORE_WORD      "stw "
  69 #define LOAD_WORD       "lwz "
  70 #define LOADX_WORD      "lwzx "
  71 #define CMP_WORD        "cmpw "
  72
  73 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  74                         start_ip, post_commit_offset, abort_ip)                 \
  75                 ".pushsection __rseq_table, \"aw\"\n\t"                         \
  76                 ".balign 32\n\t"                                                \
  77                 __rseq_str(label) ":\n\t"                                       \
  78                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"      \
  79                 /* 32-bit only supported on BE */                               \
  80                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
  81                 ".popsection\n\t"
  82
  83 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
  84                 RSEQ_INJECT_ASM(1)                                              \
  85                 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"                  \
  86                 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"           \
  87                 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t"                     \
  88                 __rseq_str(label) ":\n\t"
  89
  90 #endif /* #ifdef __PPC64__ */
  91
  92 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
  93                 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  94                                         (post_commit_ip - start_ip), abort_ip)
  95
  96 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)                      \
  97                 RSEQ_INJECT_ASM(2)                                              \
  98                 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t"              \
  99                 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"                \
 100                 "bne- cr7, " __rseq_str(label) "\n\t"
 101
 102 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                               \
 103                 ".pushsection __rseq_failure, \"ax\"\n\t"                       \
 104                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                            \
 105                 __rseq_str(label) ":\n\t"                                       \
 106                 "b %l[" __rseq_str(abort_label) "]\n\t"                         \
 107                 ".popsection\n\t"
 108
 109 /*
 110  * RSEQ_ASM_OPs: asm operations for rseq
 111  *      RSEQ_ASM_OP_R_*: has hard-code registers in it
 112  *      RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
 113  */
 114 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)                                   \
 115                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
 116                 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"            \
 117                 "bne- cr7, " __rseq_str(label) "\n\t"
 118
 119 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label)                                \
 120                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                   \
 121                 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"         \
 122                 "beq- cr7, " __rseq_str(label) "\n\t"
 123
 124 #define RSEQ_ASM_OP_STORE(value, var)                                           \
 125                 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
 126
 127 /* Load @var to r17 */
 128 #define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 129                 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
 130
 131 /* Store r17 to @var */
 132 #define RSEQ_ASM_OP_R_STORE(var)                                                \
 133                 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
 134
 135 /* Add @count to r17 */
 136 #define RSEQ_ASM_OP_R_ADD(count)                                                \
 137                 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
 138
 139 /* Load (r17 + voffp) to r17 */
 140 #define RSEQ_ASM_OP_R_LOADX(voffp)                                              \
 141                 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
 142
 143 /* TODO: implement a faster memcpy. */
 144 #define RSEQ_ASM_OP_R_MEMCPY() \
 145                 "cmpdi %%r19, 0\n\t" \
 146                 "beq 333f\n\t" \
 147                 "addi %%r20, %%r20, -1\n\t" \
 148                 "addi %%r21, %%r21, -1\n\t" \
 149                 "222:\n\t" \
 150                 "lbzu %%r18, 1(%%r20)\n\t" \
 151                 "stbu %%r18, 1(%%r21)\n\t" \
 152                 "addi %%r19, %%r19, -1\n\t" \
 153                 "cmpdi %%r19, 0\n\t" \
 154                 "bne 222b\n\t" \
 155                 "333:\n\t" \
 156
 157 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 158                 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"                  \
 159                 __rseq_str(post_commit_label) ":\n\t"
 160
 161 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 162                 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
 163                 __rseq_str(post_commit_label) ":\n\t"
 164
 165 static inline __attribute__((always_inline))
 166 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 167 {
 168         RSEQ_INJECT_C(9)
 169
 170         __asm__ __volatile__ goto (
 171                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 172                 /* Start rseq by storing table entry pointer into rseq_cs. */
 173                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 174                 /* cmp cpuid */
 175                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 176                 RSEQ_INJECT_ASM(3)
 177                 /* cmp @v equal to @expect */
 178                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 179                 RSEQ_INJECT_ASM(4)
 180 #ifdef RSEQ_COMPARE_TWICE
 181                 /* cmp cpuid */
 182                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 183                 /* cmp @v equal to @expect */
 184                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 185 #endif
 186                 /* final store */
 187                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 188                 RSEQ_INJECT_ASM(5)
 189                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 190                 : /* gcc asm goto does not allow outputs */
 191                 : [cpu_id]              "r" (cpu),
 192                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 193                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 194                   [v]                   "m" (*v),
 195                   [expect]              "r" (expect),
 196                   [newv]                "r" (newv)
 197                   RSEQ_INJECT_INPUT
 198                 : "memory", "cc", "r17"
 199                   RSEQ_INJECT_CLOBBER
 200                 : abort, cmpfail
 201 #ifdef RSEQ_COMPARE_TWICE
 202                   , error1, error2
 203 #endif
 204         );
 205         return 0;
 206 abort:
 207         RSEQ_INJECT_FAILED
 208         return -1;
 209 cmpfail:
 210         return 1;
 211 #ifdef RSEQ_COMPARE_TWICE
 212 error1:
 213         rseq_bug("cpu_id comparison failed");
 214 error2:
 215         rseq_bug("expected value comparison failed");
 216 #endif
 217 }
 218
 219 static inline __attribute__((always_inline))
 220 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 221                                off_t voffp, intptr_t *load, int cpu)
 222 {
 223         RSEQ_INJECT_C(9)
 224
 225         __asm__ __volatile__ goto (
 226                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 227                 /* Start rseq by storing table entry pointer into rseq_cs. */
 228                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 229                 /* cmp cpuid */
 230                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 231                 RSEQ_INJECT_ASM(3)
 232                 /* cmp @v not equal to @expectnot */
 233                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
 234                 RSEQ_INJECT_ASM(4)
 235 #ifdef RSEQ_COMPARE_TWICE
 236                 /* cmp cpuid */
 237                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 238                 /* cmp @v not equal to @expectnot */
 239                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
 240 #endif
 241                 /* load the value of @v */
 242                 RSEQ_ASM_OP_R_LOAD(v)
 243                 /* store it in @load */
 244                 RSEQ_ASM_OP_R_STORE(load)
 245                 /* dereference voffp(v) */
 246                 RSEQ_ASM_OP_R_LOADX(voffp)
 247                 /* final store the value at voffp(v) */
 248                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 249                 RSEQ_INJECT_ASM(5)
 250                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 251                 : /* gcc asm goto does not allow outputs */
 252                 : [cpu_id]              "r" (cpu),
 253                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 254                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 255                   /* final store input */
 256                   [v]                   "m" (*v),
 257                   [expectnot]           "r" (expectnot),
 258                   [voffp]               "b" (voffp),
 259                   [load]                "m" (*load)
 260                   RSEQ_INJECT_INPUT
 261                 : "memory", "cc", "r17"
 262                   RSEQ_INJECT_CLOBBER
 263                 : abort, cmpfail
 264 #ifdef RSEQ_COMPARE_TWICE
 265                   , error1, error2
 266 #endif
 267         );
 268         return 0;
 269 abort:
 270         RSEQ_INJECT_FAILED
 271         return -1;
 272 cmpfail:
 273         return 1;
 274 #ifdef RSEQ_COMPARE_TWICE
 275 error1:
 276         rseq_bug("cpu_id comparison failed");
 277 error2:
 278         rseq_bug("expected value comparison failed");
 279 #endif
 280 }
 281
 282 static inline __attribute__((always_inline))
 283 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 284 {
 285         RSEQ_INJECT_C(9)
 286
 287         __asm__ __volatile__ goto (
 288                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 289                 /* Start rseq by storing table entry pointer into rseq_cs. */
 290                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 291                 /* cmp cpuid */
 292                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 293                 RSEQ_INJECT_ASM(3)
 294 #ifdef RSEQ_COMPARE_TWICE
 295                 /* cmp cpuid */
 296                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 297 #endif
 298                 /* load the value of @v */
 299                 RSEQ_ASM_OP_R_LOAD(v)
 300                 /* add @count to it */
 301                 RSEQ_ASM_OP_R_ADD(count)
 302                 /* final store */
 303                 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
 304                 RSEQ_INJECT_ASM(4)
 305                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 306                 : /* gcc asm goto does not allow outputs */
 307                 : [cpu_id]              "r" (cpu),
 308                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 309                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 310                   /* final store input */
 311                   [v]                   "m" (*v),
 312                   [count]               "r" (count)
 313                   RSEQ_INJECT_INPUT
 314                 : "memory", "cc", "r17"
 315                   RSEQ_INJECT_CLOBBER
 316                 : abort
 317 #ifdef RSEQ_COMPARE_TWICE
 318                   , error1
 319 #endif
 320         );
 321         return 0;
 322 abort:
 323         RSEQ_INJECT_FAILED
 324         return -1;
 325 #ifdef RSEQ_COMPARE_TWICE
 326 error1:
 327         rseq_bug("cpu_id comparison failed");
 328 #endif
 329 }
 330
 331 static inline __attribute__((always_inline))
 332 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 333                                  intptr_t *v2, intptr_t newv2,
 334                                  intptr_t newv, int cpu)
 335 {
 336         RSEQ_INJECT_C(9)
 337
 338         __asm__ __volatile__ goto (
 339                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 340                 /* Start rseq by storing table entry pointer into rseq_cs. */
 341                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 342                 /* cmp cpuid */
 343                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 344                 RSEQ_INJECT_ASM(3)
 345                 /* cmp @v equal to @expect */
 346                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 347                 RSEQ_INJECT_ASM(4)
 348 #ifdef RSEQ_COMPARE_TWICE
 349                 /* cmp cpuid */
 350                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 351                 /* cmp @v equal to @expect */
 352                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 353 #endif
 354                 /* try store */
 355                 RSEQ_ASM_OP_STORE(newv2, v2)
 356                 RSEQ_INJECT_ASM(5)
 357                 /* final store */
 358                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 359                 RSEQ_INJECT_ASM(6)
 360                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 361                 : /* gcc asm goto does not allow outputs */
 362                 : [cpu_id]              "r" (cpu),
 363                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 364                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 365                   /* try store input */
 366                   [v2]                  "m" (*v2),
 367                   [newv2]               "r" (newv2),
 368                   /* final store input */
 369                   [v]                   "m" (*v),
 370                   [expect]              "r" (expect),
 371                   [newv]                "r" (newv)
 372                   RSEQ_INJECT_INPUT
 373                 : "memory", "cc", "r17"
 374                   RSEQ_INJECT_CLOBBER
 375                 : abort, cmpfail
 376 #ifdef RSEQ_COMPARE_TWICE
 377                   , error1, error2
 378 #endif
 379         );
 380         return 0;
 381 abort:
 382         RSEQ_INJECT_FAILED
 383         return -1;
 384 cmpfail:
 385         return 1;
 386 #ifdef RSEQ_COMPARE_TWICE
 387 error1:
 388         rseq_bug("cpu_id comparison failed");
 389 error2:
 390         rseq_bug("expected value comparison failed");
 391 #endif
 392 }
 393
 394 static inline __attribute__((always_inline))
 395 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 396                                          intptr_t *v2, intptr_t newv2,
 397                                          intptr_t newv, int cpu)
 398 {
 399         RSEQ_INJECT_C(9)
 400
 401         __asm__ __volatile__ goto (
 402                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 403                 /* Start rseq by storing table entry pointer into rseq_cs. */
 404                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 405                 /* cmp cpuid */
 406                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 407                 RSEQ_INJECT_ASM(3)
 408                 /* cmp @v equal to @expect */
 409                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 410                 RSEQ_INJECT_ASM(4)
 411 #ifdef RSEQ_COMPARE_TWICE
 412                 /* cmp cpuid */
 413                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 414                 /* cmp @v equal to @expect */
 415                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 416 #endif
 417                 /* try store */
 418                 RSEQ_ASM_OP_STORE(newv2, v2)
 419                 RSEQ_INJECT_ASM(5)
 420                 /* for 'release' */
 421                 "lwsync\n\t"
 422                 /* final store */
 423                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 424                 RSEQ_INJECT_ASM(6)
 425                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 426                 : /* gcc asm goto does not allow outputs */
 427                 : [cpu_id]              "r" (cpu),
 428                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 429                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 430                   /* try store input */
 431                   [v2]                  "m" (*v2),
 432                   [newv2]               "r" (newv2),
 433                   /* final store input */
 434                   [v]                   "m" (*v),
 435                   [expect]              "r" (expect),
 436                   [newv]                "r" (newv)
 437                   RSEQ_INJECT_INPUT
 438                 : "memory", "cc", "r17"
 439                   RSEQ_INJECT_CLOBBER
 440                 : abort, cmpfail
 441 #ifdef RSEQ_COMPARE_TWICE
 442                   , error1, error2
 443 #endif
 444         );
 445         return 0;
 446 abort:
 447         RSEQ_INJECT_FAILED
 448         return -1;
 449 cmpfail:
 450         return 1;
 451 #ifdef RSEQ_COMPARE_TWICE
 452 error1:
 453         rseq_bug("cpu_id comparison failed");
 454 error2:
 455         rseq_bug("expected value comparison failed");
 456 #endif
 457 }
 458
 459 static inline __attribute__((always_inline))
 460 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 461                               intptr_t *v2, intptr_t expect2,
 462                               intptr_t newv, int cpu)
 463 {
 464         RSEQ_INJECT_C(9)
 465
 466         __asm__ __volatile__ goto (
 467                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 468                 /* Start rseq by storing table entry pointer into rseq_cs. */
 469                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 470                 /* cmp cpuid */
 471                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 472                 RSEQ_INJECT_ASM(3)
 473                 /* cmp @v equal to @expect */
 474                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 475                 RSEQ_INJECT_ASM(4)
 476                 /* cmp @v2 equal to @expct2 */
 477                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 478                 RSEQ_INJECT_ASM(5)
 479 #ifdef RSEQ_COMPARE_TWICE
 480                 /* cmp cpuid */
 481                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 482                 /* cmp @v equal to @expect */
 483                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 484                 /* cmp @v2 equal to @expct2 */
 485                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 486 #endif
 487                 /* final store */
 488                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 489                 RSEQ_INJECT_ASM(6)
 490                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 491                 : /* gcc asm goto does not allow outputs */
 492                 : [cpu_id]              "r" (cpu),
 493                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 494                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 495                   /* cmp2 input */
 496                   [v2]                  "m" (*v2),
 497                   [expect2]             "r" (expect2),
 498                   /* final store input */
 499                   [v]                   "m" (*v),
 500                   [expect]              "r" (expect),
 501                   [newv]                "r" (newv)
 502                   RSEQ_INJECT_INPUT
 503                 : "memory", "cc", "r17"
 504                   RSEQ_INJECT_CLOBBER
 505                 : abort, cmpfail
 506 #ifdef RSEQ_COMPARE_TWICE
 507                   , error1, error2, error3
 508 #endif
 509         );
 510         return 0;
 511 abort:
 512         RSEQ_INJECT_FAILED
 513         return -1;
 514 cmpfail:
 515         return 1;
 516 #ifdef RSEQ_COMPARE_TWICE
 517 error1:
 518         rseq_bug("cpu_id comparison failed");
 519 error2:
 520         rseq_bug("1st expected value comparison failed");
 521 error3:
 522         rseq_bug("2nd expected value comparison failed");
 523 #endif
 524 }
 525
 526 static inline __attribute__((always_inline))
 527 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 528                                  void *dst, void *src, size_t len,
 529                                  intptr_t newv, int cpu)
 530 {
 531         RSEQ_INJECT_C(9)
 532
 533         __asm__ __volatile__ goto (
 534                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 535                 /* setup for mempcy */
 536                 "mr %%r19, %[len]\n\t"
 537                 "mr %%r20, %[src]\n\t"
 538                 "mr %%r21, %[dst]\n\t"
 539                 /* Start rseq by storing table entry pointer into rseq_cs. */
 540                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 541                 /* cmp cpuid */
 542                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 543                 RSEQ_INJECT_ASM(3)
 544                 /* cmp @v equal to @expect */
 545                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 546                 RSEQ_INJECT_ASM(4)
 547 #ifdef RSEQ_COMPARE_TWICE
 548                 /* cmp cpuid */
 549                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 550                 /* cmp @v equal to @expect */
 551                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 552 #endif
 553                 /* try memcpy */
 554                 RSEQ_ASM_OP_R_MEMCPY()
 555                 RSEQ_INJECT_ASM(5)
 556                 /* final store */
 557                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 558                 RSEQ_INJECT_ASM(6)
 559                 /* teardown */
 560                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 561                 : /* gcc asm goto does not allow outputs */
 562                 : [cpu_id]              "r" (cpu),
 563                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 564                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 565                   /* final store input */
 566                   [v]                   "m" (*v),
 567                   [expect]              "r" (expect),
 568                   [newv]                "r" (newv),
 569                   /* try memcpy input */
 570                   [dst]                 "r" (dst),
 571                   [src]                 "r" (src),
 572                   [len]                 "r" (len)
 573                   RSEQ_INJECT_INPUT
 574                 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 575                   RSEQ_INJECT_CLOBBER
 576                 : abort, cmpfail
 577 #ifdef RSEQ_COMPARE_TWICE
 578                   , error1, error2
 579 #endif
 580         );
 581         return 0;
 582 abort:
 583         RSEQ_INJECT_FAILED
 584         return -1;
 585 cmpfail:
 586         return 1;
 587 #ifdef RSEQ_COMPARE_TWICE
 588 error1:
 589         rseq_bug("cpu_id comparison failed");
 590 error2:
 591         rseq_bug("expected value comparison failed");
 592 #endif
 593 }
 594
 595 static inline __attribute__((always_inline))
 596 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 597                                          void *dst, void *src, size_t len,
 598                                          intptr_t newv, int cpu)
 599 {
 600         RSEQ_INJECT_C(9)
 601
 602         __asm__ __volatile__ goto (
 603                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 604                 /* setup for mempcy */
 605                 "mr %%r19, %[len]\n\t"
 606                 "mr %%r20, %[src]\n\t"
 607                 "mr %%r21, %[dst]\n\t"
 608                 /* Start rseq by storing table entry pointer into rseq_cs. */
 609                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 610                 /* cmp cpuid */
 611                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 612                 RSEQ_INJECT_ASM(3)
 613                 /* cmp @v equal to @expect */
 614                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 615                 RSEQ_INJECT_ASM(4)
 616 #ifdef RSEQ_COMPARE_TWICE
 617                 /* cmp cpuid */
 618                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 619                 /* cmp @v equal to @expect */
 620                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 621 #endif
 622                 /* try memcpy */
 623                 RSEQ_ASM_OP_R_MEMCPY()
 624                 RSEQ_INJECT_ASM(5)
 625                 /* for 'release' */
 626                 "lwsync\n\t"
 627                 /* final store */
 628                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
 629                 RSEQ_INJECT_ASM(6)
 630                 /* teardown */
 631                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 632                 : /* gcc asm goto does not allow outputs */
 633                 : [cpu_id]              "r" (cpu),
 634                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 635                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 636                   /* final store input */
 637                   [v]                   "m" (*v),
 638                   [expect]              "r" (expect),
 639                   [newv]                "r" (newv),
 640                   /* try memcpy input */
 641                   [dst]                 "r" (dst),
 642                   [src]                 "r" (src),
 643                   [len]                 "r" (len)
 644                   RSEQ_INJECT_INPUT
 645                 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
 646                   RSEQ_INJECT_CLOBBER
 647                 : abort, cmpfail
 648 #ifdef RSEQ_COMPARE_TWICE
 649                   , error1, error2
 650 #endif
 651         );
 652         return 0;
 653 abort:
 654         RSEQ_INJECT_FAILED
 655         return -1;
 656 cmpfail:
 657         return 1;
 658 #ifdef RSEQ_COMPARE_TWICE
 659 error1:
 660         rseq_bug("cpu_id comparison failed");
 661 error2:
 662         rseq_bug("expected value comparison failed");
 663 #endif
 664 }
 665
 666 #undef STORE_WORD
 667 #undef LOAD_WORD
 668 #undef LOADX_WORD
 669 #undef CMP_WORD
 670
 671 #endif /* !RSEQ_SKIP_FASTPATH */