tools/testing/selftests/rseq/rseq-mips.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * Author: Paul Burton <paul.burton@mips.com>
   4  * (C) Copyright 2018 MIPS Tech LLC
   5  *
   6  * Based on rseq-arm.h:
   7  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   8  */
   9
  10 /*
  11  * RSEQ_SIG uses the break instruction. The instruction pattern is:
  12  *
  13  * On MIPS:
  14  *      0350000d        break     0x350
  15  *
  16  * On nanoMIPS:
  17  *      00100350        break     0x350
  18  *
  19  * On microMIPS:
  20  *      0000d407        break     0x350
  21  *
  22  * For nanoMIPS32 and microMIPS, the instruction stream is encoded as 16-bit
  23  * halfwords, so the signature halfwords need to be swapped accordingly for
  24  * little-endian.
  25  */
  26 #if defined(__nanomips__)
  27 # ifdef __MIPSEL__
  28 #  define RSEQ_SIG      0x03500010
  29 # else
  30 #  define RSEQ_SIG      0x00100350
  31 # endif
  32 #elif defined(__mips_micromips)
  33 # ifdef __MIPSEL__
  34 #  define RSEQ_SIG      0xd4070000
  35 # else
  36 #  define RSEQ_SIG      0x0000d407
  37 # endif
  38 #elif defined(__mips__)
  39 # define RSEQ_SIG       0x0350000d
  40 #else
  41 /* Unknown MIPS architecture. */
  42 #endif
  43
  44 #define rseq_smp_mb()   __asm__ __volatile__ ("sync" ::: "memory")
  45 #define rseq_smp_rmb()  rseq_smp_mb()
  46 #define rseq_smp_wmb()  rseq_smp_mb()
  47
  48 #define rseq_smp_load_acquire(p)                                        \
  49 __extension__ ({                                                        \
  50         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  51         rseq_smp_mb();                                                  \
  52         ____p1;                                                         \
  53 })
  54
  55 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  56
  57 #define rseq_smp_store_release(p, v)                                    \
  58 do {                                                                    \
  59         rseq_smp_mb();                                                  \
  60         RSEQ_WRITE_ONCE(*p, v);                                         \
  61 } while (0)
  62
  63 #ifdef RSEQ_SKIP_FASTPATH
  64 #include "rseq-skip.h"
  65 #else /* !RSEQ_SKIP_FASTPATH */
  66
  67 #if _MIPS_SZLONG == 64
  68 # define LONG                   ".dword"
  69 # define LONG_LA                "dla"
  70 # define LONG_L                 "ld"
  71 # define LONG_S                 "sd"
  72 # define LONG_ADDI              "daddiu"
  73 # define U32_U64_PAD(x)         x
  74 #elif _MIPS_SZLONG == 32
  75 # define LONG                   ".word"
  76 # define LONG_LA                "la"
  77 # define LONG_L                 "lw"
  78 # define LONG_S                 "sw"
  79 # define LONG_ADDI              "addiu"
  80 # ifdef __BIG_ENDIAN
  81 #  define U32_U64_PAD(x)        "0x0, " x
  82 # else
  83 #  define U32_U64_PAD(x)        x ", 0x0"
  84 # endif
  85 #else
  86 # error unsupported _MIPS_SZLONG
  87 #endif
  88
  89 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
  90                                 post_commit_offset, abort_ip) \
  91                 ".pushsection __rseq_cs, \"aw\"\n\t" \
  92                 ".balign 32\n\t" \
  93                 __rseq_str(label) ":\n\t"                                       \
  94                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  95                 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
  96                 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
  97                 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
  98                 ".popsection\n\t" \
  99                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
 100                 LONG " " U32_U64_PAD(__rseq_str(label) "b") "\n\t" \
 101                 ".popsection\n\t"
 102
 103 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 104         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
 105                                 (post_commit_ip - start_ip), abort_ip)
 106
 107 /*
 108  * Exit points of a rseq critical section consist of all instructions outside
 109  * of the critical section where a critical section can either branch to or
 110  * reach through the normal course of its execution. The abort IP and the
 111  * post-commit IP are already part of the __rseq_cs section and should not be
 112  * explicitly defined as additional exit points. Knowing all exit points is
 113  * useful to assist debuggers stepping over the critical section.
 114  */
 115 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
 116                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
 117                 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
 118                 LONG " " U32_U64_PAD(__rseq_str(exit_ip)) "\n\t" \
 119                 ".popsection\n\t"
 120
 121 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
 122                 RSEQ_INJECT_ASM(1) \
 123                 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
 124                 LONG_S  " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
 125                 __rseq_str(label) ":\n\t"
 126
 127 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
 128                 RSEQ_INJECT_ASM(2) \
 129                 "lw  $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
 130                 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
 131
 132 #define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
 133                                 abort_label, version, flags, \
 134                                 start_ip, post_commit_offset, abort_ip) \
 135                 ".balign 32\n\t" \
 136                 __rseq_str(table_label) ":\n\t" \
 137                 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 138                 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
 139                 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
 140                 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
 141                 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
 142                 __rseq_str(label) ":\n\t" \
 143                 teardown \
 144                 "b %l[" __rseq_str(abort_label) "]\n\t"
 145
 146 #define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
 147                               start_ip, post_commit_ip, abort_ip) \
 148         __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
 149                                 abort_label, 0x0, 0x0, start_ip, \
 150                                 (post_commit_ip - start_ip), abort_ip)
 151
 152 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
 153                 __rseq_str(label) ":\n\t" \
 154                 teardown \
 155                 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
 156
 157 #define rseq_workaround_gcc_asm_size_guess()    __asm__ __volatile__("")
 158
 159 static inline __attribute__((always_inline))
 160 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 161 {
 162         RSEQ_INJECT_C(9)
 163
 164         rseq_workaround_gcc_asm_size_guess();
 165         __asm__ __volatile__ goto (
 166                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 167                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 168 #ifdef RSEQ_COMPARE_TWICE
 169                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 170                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 171 #endif
 172                 /* Start rseq by storing table entry pointer into rseq_cs. */
 173                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 174                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 175                 RSEQ_INJECT_ASM(3)
 176                 LONG_L " $4, %[v]\n\t"
 177                 "bne $4, %[expect], %l[cmpfail]\n\t"
 178                 RSEQ_INJECT_ASM(4)
 179 #ifdef RSEQ_COMPARE_TWICE
 180                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 181                 LONG_L " $4, %[v]\n\t"
 182                 "bne $4, %[expect], %l[error2]\n\t"
 183 #endif
 184                 /* final store */
 185                 LONG_S " %[newv], %[v]\n\t"
 186                 "2:\n\t"
 187                 RSEQ_INJECT_ASM(5)
 188                 "b 5f\n\t"
 189                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 190                 "5:\n\t"
 191                 : /* gcc asm goto does not allow outputs */
 192                 : [cpu_id]              "r" (cpu),
 193                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 194                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 195                   [v]                   "m" (*v),
 196                   [expect]              "r" (expect),
 197                   [newv]                "r" (newv)
 198                   RSEQ_INJECT_INPUT
 199                 : "$4", "memory"
 200                   RSEQ_INJECT_CLOBBER
 201                 : abort, cmpfail
 202 #ifdef RSEQ_COMPARE_TWICE
 203                   , error1, error2
 204 #endif
 205         );
 206         rseq_workaround_gcc_asm_size_guess();
 207         return 0;
 208 abort:
 209         rseq_workaround_gcc_asm_size_guess();
 210         RSEQ_INJECT_FAILED
 211         return -1;
 212 cmpfail:
 213         rseq_workaround_gcc_asm_size_guess();
 214         return 1;
 215 #ifdef RSEQ_COMPARE_TWICE
 216 error1:
 217         rseq_bug("cpu_id comparison failed");
 218 error2:
 219         rseq_bug("expected value comparison failed");
 220 #endif
 221 }
 222
 223 static inline __attribute__((always_inline))
 224 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 225                                off_t voffp, intptr_t *load, int cpu)
 226 {
 227         RSEQ_INJECT_C(9)
 228
 229         rseq_workaround_gcc_asm_size_guess();
 230         __asm__ __volatile__ goto (
 231                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 232                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 233 #ifdef RSEQ_COMPARE_TWICE
 234                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 235                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 236 #endif
 237                 /* Start rseq by storing table entry pointer into rseq_cs. */
 238                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 239                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 240                 RSEQ_INJECT_ASM(3)
 241                 LONG_L " $4, %[v]\n\t"
 242                 "beq $4, %[expectnot], %l[cmpfail]\n\t"
 243                 RSEQ_INJECT_ASM(4)
 244 #ifdef RSEQ_COMPARE_TWICE
 245                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 246                 LONG_L " $4, %[v]\n\t"
 247                 "beq $4, %[expectnot], %l[error2]\n\t"
 248 #endif
 249                 LONG_S " $4, %[load]\n\t"
 250                 LONG_ADDI " $4, %[voffp]\n\t"
 251                 LONG_L " $4, 0($4)\n\t"
 252                 /* final store */
 253                 LONG_S " $4, %[v]\n\t"
 254                 "2:\n\t"
 255                 RSEQ_INJECT_ASM(5)
 256                 "b 5f\n\t"
 257                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 258                 "5:\n\t"
 259                 : /* gcc asm goto does not allow outputs */
 260                 : [cpu_id]              "r" (cpu),
 261                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 262                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 263                   /* final store input */
 264                   [v]                   "m" (*v),
 265                   [expectnot]           "r" (expectnot),
 266                   [voffp]               "Ir" (voffp),
 267                   [load]                "m" (*load)
 268                   RSEQ_INJECT_INPUT
 269                 : "$4", "memory"
 270                   RSEQ_INJECT_CLOBBER
 271                 : abort, cmpfail
 272 #ifdef RSEQ_COMPARE_TWICE
 273                   , error1, error2
 274 #endif
 275         );
 276         rseq_workaround_gcc_asm_size_guess();
 277         return 0;
 278 abort:
 279         rseq_workaround_gcc_asm_size_guess();
 280         RSEQ_INJECT_FAILED
 281         return -1;
 282 cmpfail:
 283         rseq_workaround_gcc_asm_size_guess();
 284         return 1;
 285 #ifdef RSEQ_COMPARE_TWICE
 286 error1:
 287         rseq_bug("cpu_id comparison failed");
 288 error2:
 289         rseq_bug("expected value comparison failed");
 290 #endif
 291 }
 292
 293 static inline __attribute__((always_inline))
 294 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 295 {
 296         RSEQ_INJECT_C(9)
 297
 298         rseq_workaround_gcc_asm_size_guess();
 299         __asm__ __volatile__ goto (
 300                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 301 #ifdef RSEQ_COMPARE_TWICE
 302                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 303 #endif
 304                 /* Start rseq by storing table entry pointer into rseq_cs. */
 305                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 306                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 307                 RSEQ_INJECT_ASM(3)
 308 #ifdef RSEQ_COMPARE_TWICE
 309                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 310 #endif
 311                 LONG_L " $4, %[v]\n\t"
 312                 LONG_ADDI " $4, %[count]\n\t"
 313                 /* final store */
 314                 LONG_S " $4, %[v]\n\t"
 315                 "2:\n\t"
 316                 RSEQ_INJECT_ASM(4)
 317                 "b 5f\n\t"
 318                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 319                 "5:\n\t"
 320                 : /* gcc asm goto does not allow outputs */
 321                 : [cpu_id]              "r" (cpu),
 322                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 323                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 324                   [v]                   "m" (*v),
 325                   [count]               "Ir" (count)
 326                   RSEQ_INJECT_INPUT
 327                 : "$4", "memory"
 328                   RSEQ_INJECT_CLOBBER
 329                 : abort
 330 #ifdef RSEQ_COMPARE_TWICE
 331                   , error1
 332 #endif
 333         );
 334         rseq_workaround_gcc_asm_size_guess();
 335         return 0;
 336 abort:
 337         rseq_workaround_gcc_asm_size_guess();
 338         RSEQ_INJECT_FAILED
 339         return -1;
 340 #ifdef RSEQ_COMPARE_TWICE
 341 error1:
 342         rseq_bug("cpu_id comparison failed");
 343 #endif
 344 }
 345
 346 static inline __attribute__((always_inline))
 347 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 348                                  intptr_t *v2, intptr_t newv2,
 349                                  intptr_t newv, int cpu)
 350 {
 351         RSEQ_INJECT_C(9)
 352
 353         rseq_workaround_gcc_asm_size_guess();
 354         __asm__ __volatile__ goto (
 355                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 356                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 357 #ifdef RSEQ_COMPARE_TWICE
 358                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 359                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 360 #endif
 361                 /* Start rseq by storing table entry pointer into rseq_cs. */
 362                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 363                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 364                 RSEQ_INJECT_ASM(3)
 365                 LONG_L " $4, %[v]\n\t"
 366                 "bne $4, %[expect], %l[cmpfail]\n\t"
 367                 RSEQ_INJECT_ASM(4)
 368 #ifdef RSEQ_COMPARE_TWICE
 369                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 370                 LONG_L " $4, %[v]\n\t"
 371                 "bne $4, %[expect], %l[error2]\n\t"
 372 #endif
 373                 /* try store */
 374                 LONG_S " %[newv2], %[v2]\n\t"
 375                 RSEQ_INJECT_ASM(5)
 376                 /* final store */
 377                 LONG_S " %[newv], %[v]\n\t"
 378                 "2:\n\t"
 379                 RSEQ_INJECT_ASM(6)
 380                 "b 5f\n\t"
 381                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 382                 "5:\n\t"
 383                 : /* gcc asm goto does not allow outputs */
 384                 : [cpu_id]              "r" (cpu),
 385                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 386                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 387                   /* try store input */
 388                   [v2]                  "m" (*v2),
 389                   [newv2]               "r" (newv2),
 390                   /* final store input */
 391                   [v]                   "m" (*v),
 392                   [expect]              "r" (expect),
 393                   [newv]                "r" (newv)
 394                   RSEQ_INJECT_INPUT
 395                 : "$4", "memory"
 396                   RSEQ_INJECT_CLOBBER
 397                 : abort, cmpfail
 398 #ifdef RSEQ_COMPARE_TWICE
 399                   , error1, error2
 400 #endif
 401         );
 402         rseq_workaround_gcc_asm_size_guess();
 403         return 0;
 404 abort:
 405         rseq_workaround_gcc_asm_size_guess();
 406         RSEQ_INJECT_FAILED
 407         return -1;
 408 cmpfail:
 409         rseq_workaround_gcc_asm_size_guess();
 410         return 1;
 411 #ifdef RSEQ_COMPARE_TWICE
 412 error1:
 413         rseq_bug("cpu_id comparison failed");
 414 error2:
 415         rseq_bug("expected value comparison failed");
 416 #endif
 417 }
 418
 419 static inline __attribute__((always_inline))
 420 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 421                                          intptr_t *v2, intptr_t newv2,
 422                                          intptr_t newv, int cpu)
 423 {
 424         RSEQ_INJECT_C(9)
 425
 426         rseq_workaround_gcc_asm_size_guess();
 427         __asm__ __volatile__ goto (
 428                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 429                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 430 #ifdef RSEQ_COMPARE_TWICE
 431                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 432                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 433 #endif
 434                 /* Start rseq by storing table entry pointer into rseq_cs. */
 435                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 436                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 437                 RSEQ_INJECT_ASM(3)
 438                 LONG_L " $4, %[v]\n\t"
 439                 "bne $4, %[expect], %l[cmpfail]\n\t"
 440                 RSEQ_INJECT_ASM(4)
 441 #ifdef RSEQ_COMPARE_TWICE
 442                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 443                 LONG_L " $4, %[v]\n\t"
 444                 "bne $4, %[expect], %l[error2]\n\t"
 445 #endif
 446                 /* try store */
 447                 LONG_S " %[newv2], %[v2]\n\t"
 448                 RSEQ_INJECT_ASM(5)
 449                 "sync\n\t"      /* full sync provides store-release */
 450                 /* final store */
 451                 LONG_S " %[newv], %[v]\n\t"
 452                 "2:\n\t"
 453                 RSEQ_INJECT_ASM(6)
 454                 "b 5f\n\t"
 455                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 456                 "5:\n\t"
 457                 : /* gcc asm goto does not allow outputs */
 458                 : [cpu_id]              "r" (cpu),
 459                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 460                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 461                   /* try store input */
 462                   [v2]                  "m" (*v2),
 463                   [newv2]               "r" (newv2),
 464                   /* final store input */
 465                   [v]                   "m" (*v),
 466                   [expect]              "r" (expect),
 467                   [newv]                "r" (newv)
 468                   RSEQ_INJECT_INPUT
 469                 : "$4", "memory"
 470                   RSEQ_INJECT_CLOBBER
 471                 : abort, cmpfail
 472 #ifdef RSEQ_COMPARE_TWICE
 473                   , error1, error2
 474 #endif
 475         );
 476         rseq_workaround_gcc_asm_size_guess();
 477         return 0;
 478 abort:
 479         rseq_workaround_gcc_asm_size_guess();
 480         RSEQ_INJECT_FAILED
 481         return -1;
 482 cmpfail:
 483         rseq_workaround_gcc_asm_size_guess();
 484         return 1;
 485 #ifdef RSEQ_COMPARE_TWICE
 486 error1:
 487         rseq_bug("cpu_id comparison failed");
 488 error2:
 489         rseq_bug("expected value comparison failed");
 490 #endif
 491 }
 492
 493 static inline __attribute__((always_inline))
 494 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 495                               intptr_t *v2, intptr_t expect2,
 496                               intptr_t newv, int cpu)
 497 {
 498         RSEQ_INJECT_C(9)
 499
 500         rseq_workaround_gcc_asm_size_guess();
 501         __asm__ __volatile__ goto (
 502                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 503                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 504 #ifdef RSEQ_COMPARE_TWICE
 505                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 506                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 507                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 508 #endif
 509                 /* Start rseq by storing table entry pointer into rseq_cs. */
 510                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 511                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 512                 RSEQ_INJECT_ASM(3)
 513                 LONG_L " $4, %[v]\n\t"
 514                 "bne $4, %[expect], %l[cmpfail]\n\t"
 515                 RSEQ_INJECT_ASM(4)
 516                 LONG_L " $4, %[v2]\n\t"
 517                 "bne $4, %[expect2], %l[cmpfail]\n\t"
 518                 RSEQ_INJECT_ASM(5)
 519 #ifdef RSEQ_COMPARE_TWICE
 520                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 521                 LONG_L " $4, %[v]\n\t"
 522                 "bne $4, %[expect], %l[error2]\n\t"
 523                 LONG_L " $4, %[v2]\n\t"
 524                 "bne $4, %[expect2], %l[error3]\n\t"
 525 #endif
 526                 /* final store */
 527                 LONG_S " %[newv], %[v]\n\t"
 528                 "2:\n\t"
 529                 RSEQ_INJECT_ASM(6)
 530                 "b 5f\n\t"
 531                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 532                 "5:\n\t"
 533                 : /* gcc asm goto does not allow outputs */
 534                 : [cpu_id]              "r" (cpu),
 535                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 536                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 537                   /* cmp2 input */
 538                   [v2]                  "m" (*v2),
 539                   [expect2]             "r" (expect2),
 540                   /* final store input */
 541                   [v]                   "m" (*v),
 542                   [expect]              "r" (expect),
 543                   [newv]                "r" (newv)
 544                   RSEQ_INJECT_INPUT
 545                 : "$4", "memory"
 546                   RSEQ_INJECT_CLOBBER
 547                 : abort, cmpfail
 548 #ifdef RSEQ_COMPARE_TWICE
 549                   , error1, error2, error3
 550 #endif
 551         );
 552         rseq_workaround_gcc_asm_size_guess();
 553         return 0;
 554 abort:
 555         rseq_workaround_gcc_asm_size_guess();
 556         RSEQ_INJECT_FAILED
 557         return -1;
 558 cmpfail:
 559         rseq_workaround_gcc_asm_size_guess();
 560         return 1;
 561 #ifdef RSEQ_COMPARE_TWICE
 562 error1:
 563         rseq_bug("cpu_id comparison failed");
 564 error2:
 565         rseq_bug("1st expected value comparison failed");
 566 error3:
 567         rseq_bug("2nd expected value comparison failed");
 568 #endif
 569 }
 570
 571 static inline __attribute__((always_inline))
 572 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 573                                  void *dst, void *src, size_t len,
 574                                  intptr_t newv, int cpu)
 575 {
 576         uintptr_t rseq_scratch[3];
 577
 578         RSEQ_INJECT_C(9)
 579
 580         rseq_workaround_gcc_asm_size_guess();
 581         __asm__ __volatile__ goto (
 582                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 583                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 584 #ifdef RSEQ_COMPARE_TWICE
 585                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 586                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 587 #endif
 588                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 589                 LONG_S "  %[dst], %[rseq_scratch1]\n\t"
 590                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 591                 /* Start rseq by storing table entry pointer into rseq_cs. */
 592                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 593                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 594                 RSEQ_INJECT_ASM(3)
 595                 LONG_L " $4, %[v]\n\t"
 596                 "bne $4, %[expect], 5f\n\t"
 597                 RSEQ_INJECT_ASM(4)
 598 #ifdef RSEQ_COMPARE_TWICE
 599                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 600                 LONG_L " $4, %[v]\n\t"
 601                 "bne $4, %[expect], 7f\n\t"
 602 #endif
 603                 /* try memcpy */
 604                 "beqz %[len], 333f\n\t" \
 605                 "222:\n\t" \
 606                 "lb   $4, 0(%[src])\n\t" \
 607                 "sb   $4, 0(%[dst])\n\t" \
 608                 LONG_ADDI " %[src], 1\n\t" \
 609                 LONG_ADDI " %[dst], 1\n\t" \
 610                 LONG_ADDI " %[len], -1\n\t" \
 611                 "bnez %[len], 222b\n\t" \
 612                 "333:\n\t" \
 613                 RSEQ_INJECT_ASM(5)
 614                 /* final store */
 615                 LONG_S " %[newv], %[v]\n\t"
 616                 "2:\n\t"
 617                 RSEQ_INJECT_ASM(6)
 618                 /* teardown */
 619                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 620                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 621                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 622                 "b 8f\n\t"
 623                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 624                                       /* teardown */
 625                                       LONG_L " %[len], %[rseq_scratch2]\n\t"
 626                                       LONG_L " %[dst], %[rseq_scratch1]\n\t"
 627                                       LONG_L " %[src], %[rseq_scratch0]\n\t",
 628                                       abort, 1b, 2b, 4f)
 629                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 630                                         /* teardown */
 631                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 632                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 633                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 634                                         cmpfail)
 635 #ifdef RSEQ_COMPARE_TWICE
 636                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 637                                         /* teardown */
 638                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 639                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 640                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 641                                         error1)
 642                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 643                                         /* teardown */
 644                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 645                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 646                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 647                                         error2)
 648 #endif
 649                 "8:\n\t"
 650                 : /* gcc asm goto does not allow outputs */
 651                 : [cpu_id]              "r" (cpu),
 652                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 653                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 654                   /* final store input */
 655                   [v]                   "m" (*v),
 656                   [expect]              "r" (expect),
 657                   [newv]                "r" (newv),
 658                   /* try memcpy input */
 659                   [dst]                 "r" (dst),
 660                   [src]                 "r" (src),
 661                   [len]                 "r" (len),
 662                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 663                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 664                   [rseq_scratch2]       "m" (rseq_scratch[2])
 665                   RSEQ_INJECT_INPUT
 666                 : "$4", "memory"
 667                   RSEQ_INJECT_CLOBBER
 668                 : abort, cmpfail
 669 #ifdef RSEQ_COMPARE_TWICE
 670                   , error1, error2
 671 #endif
 672         );
 673         rseq_workaround_gcc_asm_size_guess();
 674         return 0;
 675 abort:
 676         rseq_workaround_gcc_asm_size_guess();
 677         RSEQ_INJECT_FAILED
 678         return -1;
 679 cmpfail:
 680         rseq_workaround_gcc_asm_size_guess();
 681         return 1;
 682 #ifdef RSEQ_COMPARE_TWICE
 683 error1:
 684         rseq_workaround_gcc_asm_size_guess();
 685         rseq_bug("cpu_id comparison failed");
 686 error2:
 687         rseq_workaround_gcc_asm_size_guess();
 688         rseq_bug("expected value comparison failed");
 689 #endif
 690 }
 691
 692 static inline __attribute__((always_inline))
 693 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 694                                          void *dst, void *src, size_t len,
 695                                          intptr_t newv, int cpu)
 696 {
 697         uintptr_t rseq_scratch[3];
 698
 699         RSEQ_INJECT_C(9)
 700
 701         rseq_workaround_gcc_asm_size_guess();
 702         __asm__ __volatile__ goto (
 703                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 704                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 705 #ifdef RSEQ_COMPARE_TWICE
 706                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 707                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 708 #endif
 709                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 710                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 711                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 712                 /* Start rseq by storing table entry pointer into rseq_cs. */
 713                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 714                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 715                 RSEQ_INJECT_ASM(3)
 716                 LONG_L " $4, %[v]\n\t"
 717                 "bne $4, %[expect], 5f\n\t"
 718                 RSEQ_INJECT_ASM(4)
 719 #ifdef RSEQ_COMPARE_TWICE
 720                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 721                 LONG_L " $4, %[v]\n\t"
 722                 "bne $4, %[expect], 7f\n\t"
 723 #endif
 724                 /* try memcpy */
 725                 "beqz %[len], 333f\n\t" \
 726                 "222:\n\t" \
 727                 "lb   $4, 0(%[src])\n\t" \
 728                 "sb   $4, 0(%[dst])\n\t" \
 729                 LONG_ADDI " %[src], 1\n\t" \
 730                 LONG_ADDI " %[dst], 1\n\t" \
 731                 LONG_ADDI " %[len], -1\n\t" \
 732                 "bnez %[len], 222b\n\t" \
 733                 "333:\n\t" \
 734                 RSEQ_INJECT_ASM(5)
 735                 "sync\n\t"      /* full sync provides store-release */
 736                 /* final store */
 737                 LONG_S " %[newv], %[v]\n\t"
 738                 "2:\n\t"
 739                 RSEQ_INJECT_ASM(6)
 740                 /* teardown */
 741                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 742                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 743                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 744                 "b 8f\n\t"
 745                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 746                                       /* teardown */
 747                                       LONG_L " %[len], %[rseq_scratch2]\n\t"
 748                                       LONG_L " %[dst], %[rseq_scratch1]\n\t"
 749                                       LONG_L " %[src], %[rseq_scratch0]\n\t",
 750                                       abort, 1b, 2b, 4f)
 751                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 752                                         /* teardown */
 753                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 754                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 755                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 756                                         cmpfail)
 757 #ifdef RSEQ_COMPARE_TWICE
 758                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 759                                         /* teardown */
 760                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 761                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 762                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 763                                         error1)
 764                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 765                                         /* teardown */
 766                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 767                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 768                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 769                                         error2)
 770 #endif
 771                 "8:\n\t"
 772                 : /* gcc asm goto does not allow outputs */
 773                 : [cpu_id]              "r" (cpu),
 774                   [current_cpu_id]      "m" (__rseq_abi.cpu_id),
 775                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 776                   /* final store input */
 777                   [v]                   "m" (*v),
 778                   [expect]              "r" (expect),
 779                   [newv]                "r" (newv),
 780                   /* try memcpy input */
 781                   [dst]                 "r" (dst),
 782                   [src]                 "r" (src),
 783                   [len]                 "r" (len),
 784                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 785                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 786                   [rseq_scratch2]       "m" (rseq_scratch[2])
 787                   RSEQ_INJECT_INPUT
 788                 : "$4", "memory"
 789                   RSEQ_INJECT_CLOBBER
 790                 : abort, cmpfail
 791 #ifdef RSEQ_COMPARE_TWICE
 792                   , error1, error2
 793 #endif
 794         );
 795         rseq_workaround_gcc_asm_size_guess();
 796         return 0;
 797 abort:
 798         rseq_workaround_gcc_asm_size_guess();
 799         RSEQ_INJECT_FAILED
 800         return -1;
 801 cmpfail:
 802         rseq_workaround_gcc_asm_size_guess();
 803         return 1;
 804 #ifdef RSEQ_COMPARE_TWICE
 805 error1:
 806         rseq_workaround_gcc_asm_size_guess();
 807         rseq_bug("cpu_id comparison failed");
 808 error2:
 809         rseq_workaround_gcc_asm_size_guess();
 810         rseq_bug("expected value comparison failed");
 811 #endif
 812 }
 813
 814 #endif /* !RSEQ_SKIP_FASTPATH */