tools/testing/selftests/rseq/rseq-arm64.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-arm64.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
   7  */
   8
   9 /*
  10  * aarch64 -mbig-endian generates mixed endianness code vs data:
  11  * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
  12  * matches code endianness.
  13  */
  14 #define RSEQ_SIG_CODE   0xd428bc00      /* BRK #0x45E0.  */
  15
  16 #ifdef __AARCH64EB__
  17 #define RSEQ_SIG_DATA   0x00bc28d4      /* BRK #0x45E0.  */
  18 #else
  19 #define RSEQ_SIG_DATA   RSEQ_SIG_CODE
  20 #endif
  21
  22 #define RSEQ_SIG        RSEQ_SIG_DATA
  23
  24 #define rseq_smp_mb()   __asm__ __volatile__ ("dmb ish" ::: "memory")
  25 #define rseq_smp_rmb()  __asm__ __volatile__ ("dmb ishld" ::: "memory")
  26 #define rseq_smp_wmb()  __asm__ __volatile__ ("dmb ishst" ::: "memory")
  27
  28 #define rseq_smp_load_acquire(p)                                                \
  29 __extension__ ({                                                                \
  30         __typeof(*p) ____p1;                                                    \
  31         switch (sizeof(*p)) {                                                   \
  32         case 1:                                                                 \
  33                 asm volatile ("ldarb %w0, %1"                                   \
  34                         : "=r" (*(__u8 *)p)                                     \
  35                         : "Q" (*p) : "memory");                                 \
  36                 break;                                                          \
  37         case 2:                                                                 \
  38                 asm volatile ("ldarh %w0, %1"                                   \
  39                         : "=r" (*(__u16 *)p)                                    \
  40                         : "Q" (*p) : "memory");                                 \
  41                 break;                                                          \
  42         case 4:                                                                 \
  43                 asm volatile ("ldar %w0, %1"                                    \
  44                         : "=r" (*(__u32 *)p)                                    \
  45                         : "Q" (*p) : "memory");                                 \
  46                 break;                                                          \
  47         case 8:                                                                 \
  48                 asm volatile ("ldar %0, %1"                                     \
  49                         : "=r" (*(__u64 *)p)                                    \
  50                         : "Q" (*p) : "memory");                                 \
  51                 break;                                                          \
  52         }                                                                       \
  53         ____p1;                                                                 \
  54 })
  55
  56 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  57
  58 #define rseq_smp_store_release(p, v)                                            \
  59 do {                                                                            \
  60         switch (sizeof(*p)) {                                                   \
  61         case 1:                                                                 \
  62                 asm volatile ("stlrb %w1, %0"                                   \
  63                                 : "=Q" (*p)                                     \
  64                                 : "r" ((__u8)v)                                 \
  65                                 : "memory");                                    \
  66                 break;                                                          \
  67         case 2:                                                                 \
  68                 asm volatile ("stlrh %w1, %0"                                   \
  69                                 : "=Q" (*p)                                     \
  70                                 : "r" ((__u16)v)                                \
  71                                 : "memory");                                    \
  72                 break;                                                          \
  73         case 4:                                                                 \
  74                 asm volatile ("stlr %w1, %0"                                    \
  75                                 : "=Q" (*p)                                     \
  76                                 : "r" ((__u32)v)                                \
  77                                 : "memory");                                    \
  78                 break;                                                          \
  79         case 8:                                                                 \
  80                 asm volatile ("stlr %1, %0"                                     \
  81                                 : "=Q" (*p)                                     \
  82                                 : "r" ((__u64)v)                                \
  83                                 : "memory");                                    \
  84                 break;                                                          \
  85         }                                                                       \
  86 } while (0)
  87
  88 #ifdef RSEQ_SKIP_FASTPATH
  89 #include "rseq-skip.h"
  90 #else /* !RSEQ_SKIP_FASTPATH */
  91
  92 #define RSEQ_ASM_TMP_REG32      "w15"
  93 #define RSEQ_ASM_TMP_REG        "x15"
  94 #define RSEQ_ASM_TMP_REG_2      "x14"
  95
  96 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,                \
  97                                 post_commit_offset, abort_ip)                   \
  98         "       .pushsection    __rseq_cs, \"aw\"\n"                            \
  99         "       .balign 32\n"                                                   \
 100         __rseq_str(label) ":\n"                                                 \
 101         "       .long   " __rseq_str(version) ", " __rseq_str(flags) "\n"       \
 102         "       .quad   " __rseq_str(start_ip) ", "                             \
 103                           __rseq_str(post_commit_offset) ", "                   \
 104                           __rseq_str(abort_ip) "\n"                             \
 105         "       .popsection\n\t"                                                \
 106         "       .pushsection __rseq_cs_ptr_array, \"aw\"\n"                             \
 107         "       .quad " __rseq_str(label) "b\n"                                 \
 108         "       .popsection\n"
 109
 110 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)        \
 111         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,                      \
 112                                 (post_commit_ip - start_ip), abort_ip)
 113
 114 /*
 115  * Exit points of a rseq critical section consist of all instructions outside
 116  * of the critical section where a critical section can either branch to or
 117  * reach through the normal course of its execution. The abort IP and the
 118  * post-commit IP are already part of the __rseq_cs section and should not be
 119  * explicitly defined as additional exit points. Knowing all exit points is
 120  * useful to assist debuggers stepping over the critical section.
 121  */
 122 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                           \
 123         "       .pushsection __rseq_exit_point_array, \"aw\"\n"                 \
 124         "       .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n"      \
 125         "       .popsection\n"
 126
 127 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                        \
 128         RSEQ_INJECT_ASM(1)                                                      \
 129         "       adrp    " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"       \
 130         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 131                         ", :lo12:" __rseq_str(cs_label) "\n"                    \
 132         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"     \
 133         __rseq_str(label) ":\n"
 134
 135 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label)                               \
 136         "       b       222f\n"                                                 \
 137         "       .inst   "       __rseq_str(RSEQ_SIG_CODE) "\n"                  \
 138         __rseq_str(label) ":\n"                                                 \
 139         "       b       %l[" __rseq_str(abort_label) "]\n"                      \
 140         "222:\n"
 141
 142 #define RSEQ_ASM_OP_STORE(value, var)                                           \
 143         "       str     %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 144
 145 #define RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 146         "       stlr    %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
 147
 148 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)                  \
 149         RSEQ_ASM_OP_STORE(value, var)                                           \
 150         __rseq_str(post_commit_label) ":\n"
 151
 152 #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)          \
 153         RSEQ_ASM_OP_STORE_RELEASE(value, var)                                   \
 154         __rseq_str(post_commit_label) ":\n"
 155
 156 #define RSEQ_ASM_OP_CMPEQ(var, expect, label)                                   \
 157         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 158         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 159                         ", %[" __rseq_str(expect) "]\n"                         \
 160         "       cbnz    " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 161
 162 #define RSEQ_ASM_OP_CMPEQ32(var, expect, label)                                 \
 163         "       ldr     " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"       \
 164         "       sub     " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32            \
 165                         ", %w[" __rseq_str(expect) "]\n"                        \
 166         "       cbnz    " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
 167
 168 #define RSEQ_ASM_OP_CMPNE(var, expect, label)                                   \
 169         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 170         "       sub     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 171                         ", %[" __rseq_str(expect) "]\n"                         \
 172         "       cbz     " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
 173
 174 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)                      \
 175         RSEQ_INJECT_ASM(2)                                                      \
 176         RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
 177
 178 #define RSEQ_ASM_OP_R_LOAD(var)                                                 \
 179         "       ldr     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 180
 181 #define RSEQ_ASM_OP_R_STORE(var)                                                \
 182         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
 183
 184 #define RSEQ_ASM_OP_R_LOAD_OFF(offset)                                          \
 185         "       ldr     " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG               \
 186                         ", %[" __rseq_str(offset) "]]\n"
 187
 188 #define RSEQ_ASM_OP_R_ADD(count)                                                \
 189         "       add     " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG                \
 190                         ", %[" __rseq_str(count) "]\n"
 191
 192 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)                       \
 193         "       str     " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"         \
 194         __rseq_str(post_commit_label) ":\n"
 195
 196 #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)                                 \
 197         "       cbz     %[" __rseq_str(len) "], 333f\n"                         \
 198         "       mov     " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"       \
 199         "222:   sub     " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"   \
 200         "       ldrb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"        \
 201                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 202         "       strb    " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"        \
 203                         ", " RSEQ_ASM_TMP_REG_2 "]\n"                           \
 204         "       cbnz    " RSEQ_ASM_TMP_REG_2 ", 222b\n"                         \
 205         "333:\n"
 206
 207 static inline __attribute__((always_inline))
 208 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 209 {
 210         RSEQ_INJECT_C(9)
 211
 212         __asm__ __volatile__ goto (
 213                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 214                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 215 #ifdef RSEQ_COMPARE_TWICE
 216                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 217                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 218 #endif
 219                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 220                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 221                 RSEQ_INJECT_ASM(3)
 222                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 223                 RSEQ_INJECT_ASM(4)
 224 #ifdef RSEQ_COMPARE_TWICE
 225                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 226                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 227 #endif
 228                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 229                 RSEQ_INJECT_ASM(5)
 230                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 231                 : /* gcc asm goto does not allow outputs */
 232                 : [cpu_id]              "r" (cpu),
 233                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 234                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 235                   [v]                   "Qo" (*v),
 236                   [expect]              "r" (expect),
 237                   [newv]                "r" (newv)
 238                   RSEQ_INJECT_INPUT
 239                 : "memory", RSEQ_ASM_TMP_REG
 240                 : abort, cmpfail
 241 #ifdef RSEQ_COMPARE_TWICE
 242                   , error1, error2
 243 #endif
 244         );
 245
 246         return 0;
 247 abort:
 248         RSEQ_INJECT_FAILED
 249         return -1;
 250 cmpfail:
 251         return 1;
 252 #ifdef RSEQ_COMPARE_TWICE
 253 error1:
 254         rseq_bug("cpu_id comparison failed");
 255 error2:
 256         rseq_bug("expected value comparison failed");
 257 #endif
 258 }
 259
 260 static inline __attribute__((always_inline))
 261 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 262                                off_t voffp, intptr_t *load, int cpu)
 263 {
 264         RSEQ_INJECT_C(9)
 265
 266         __asm__ __volatile__ goto (
 267                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 268                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 269 #ifdef RSEQ_COMPARE_TWICE
 270                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 271                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 272 #endif
 273                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 274                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 275                 RSEQ_INJECT_ASM(3)
 276                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
 277                 RSEQ_INJECT_ASM(4)
 278 #ifdef RSEQ_COMPARE_TWICE
 279                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 280                 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
 281 #endif
 282                 RSEQ_ASM_OP_R_LOAD(v)
 283                 RSEQ_ASM_OP_R_STORE(load)
 284                 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
 285                 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 286                 RSEQ_INJECT_ASM(5)
 287                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 288                 : /* gcc asm goto does not allow outputs */
 289                 : [cpu_id]              "r" (cpu),
 290                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 291                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 292                   [v]                   "Qo" (*v),
 293                   [expectnot]           "r" (expectnot),
 294                   [load]                "Qo" (*load),
 295                   [voffp]               "r" (voffp)
 296                   RSEQ_INJECT_INPUT
 297                 : "memory", RSEQ_ASM_TMP_REG
 298                 : abort, cmpfail
 299 #ifdef RSEQ_COMPARE_TWICE
 300                   , error1, error2
 301 #endif
 302         );
 303         return 0;
 304 abort:
 305         RSEQ_INJECT_FAILED
 306         return -1;
 307 cmpfail:
 308         return 1;
 309 #ifdef RSEQ_COMPARE_TWICE
 310 error1:
 311         rseq_bug("cpu_id comparison failed");
 312 error2:
 313         rseq_bug("expected value comparison failed");
 314 #endif
 315 }
 316
 317 static inline __attribute__((always_inline))
 318 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 319 {
 320         RSEQ_INJECT_C(9)
 321
 322         __asm__ __volatile__ goto (
 323                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 324 #ifdef RSEQ_COMPARE_TWICE
 325                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 326 #endif
 327                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 328                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 329                 RSEQ_INJECT_ASM(3)
 330 #ifdef RSEQ_COMPARE_TWICE
 331                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 332 #endif
 333                 RSEQ_ASM_OP_R_LOAD(v)
 334                 RSEQ_ASM_OP_R_ADD(count)
 335                 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
 336                 RSEQ_INJECT_ASM(4)
 337                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 338                 : /* gcc asm goto does not allow outputs */
 339                 : [cpu_id]              "r" (cpu),
 340                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 341                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 342                   [v]                   "Qo" (*v),
 343                   [count]               "r" (count)
 344                   RSEQ_INJECT_INPUT
 345                 : "memory", RSEQ_ASM_TMP_REG
 346                 : abort
 347 #ifdef RSEQ_COMPARE_TWICE
 348                   , error1
 349 #endif
 350         );
 351         return 0;
 352 abort:
 353         RSEQ_INJECT_FAILED
 354         return -1;
 355 #ifdef RSEQ_COMPARE_TWICE
 356 error1:
 357         rseq_bug("cpu_id comparison failed");
 358 #endif
 359 }
 360
 361 static inline __attribute__((always_inline))
 362 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 363                                  intptr_t *v2, intptr_t newv2,
 364                                  intptr_t newv, int cpu)
 365 {
 366         RSEQ_INJECT_C(9)
 367
 368         __asm__ __volatile__ goto (
 369                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 370                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 371 #ifdef RSEQ_COMPARE_TWICE
 372                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 373                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 374 #endif
 375                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 376                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 377                 RSEQ_INJECT_ASM(3)
 378                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 379                 RSEQ_INJECT_ASM(4)
 380 #ifdef RSEQ_COMPARE_TWICE
 381                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 382                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 383 #endif
 384                 RSEQ_ASM_OP_STORE(newv2, v2)
 385                 RSEQ_INJECT_ASM(5)
 386                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 387                 RSEQ_INJECT_ASM(6)
 388                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 389                 : /* gcc asm goto does not allow outputs */
 390                 : [cpu_id]              "r" (cpu),
 391                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 392                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 393                   [expect]              "r" (expect),
 394                   [v]                   "Qo" (*v),
 395                   [newv]                "r" (newv),
 396                   [v2]                  "Qo" (*v2),
 397                   [newv2]               "r" (newv2)
 398                   RSEQ_INJECT_INPUT
 399                 : "memory", RSEQ_ASM_TMP_REG
 400                 : abort, cmpfail
 401 #ifdef RSEQ_COMPARE_TWICE
 402                   , error1, error2
 403 #endif
 404         );
 405
 406         return 0;
 407 abort:
 408         RSEQ_INJECT_FAILED
 409         return -1;
 410 cmpfail:
 411         return 1;
 412 #ifdef RSEQ_COMPARE_TWICE
 413 error1:
 414         rseq_bug("cpu_id comparison failed");
 415 error2:
 416         rseq_bug("expected value comparison failed");
 417 #endif
 418 }
 419
 420 static inline __attribute__((always_inline))
 421 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 422                                          intptr_t *v2, intptr_t newv2,
 423                                          intptr_t newv, int cpu)
 424 {
 425         RSEQ_INJECT_C(9)
 426
 427         __asm__ __volatile__ goto (
 428                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 429                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 430 #ifdef RSEQ_COMPARE_TWICE
 431                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 432                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 433 #endif
 434                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 435                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 436                 RSEQ_INJECT_ASM(3)
 437                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 438                 RSEQ_INJECT_ASM(4)
 439 #ifdef RSEQ_COMPARE_TWICE
 440                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 441                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 442 #endif
 443                 RSEQ_ASM_OP_STORE(newv2, v2)
 444                 RSEQ_INJECT_ASM(5)
 445                 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 446                 RSEQ_INJECT_ASM(6)
 447                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 448                 : /* gcc asm goto does not allow outputs */
 449                 : [cpu_id]              "r" (cpu),
 450                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 451                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 452                   [expect]              "r" (expect),
 453                   [v]                   "Qo" (*v),
 454                   [newv]                "r" (newv),
 455                   [v2]                  "Qo" (*v2),
 456                   [newv2]               "r" (newv2)
 457                   RSEQ_INJECT_INPUT
 458                 : "memory", RSEQ_ASM_TMP_REG
 459                 : abort, cmpfail
 460 #ifdef RSEQ_COMPARE_TWICE
 461                   , error1, error2
 462 #endif
 463         );
 464
 465         return 0;
 466 abort:
 467         RSEQ_INJECT_FAILED
 468         return -1;
 469 cmpfail:
 470         return 1;
 471 #ifdef RSEQ_COMPARE_TWICE
 472 error1:
 473         rseq_bug("cpu_id comparison failed");
 474 error2:
 475         rseq_bug("expected value comparison failed");
 476 #endif
 477 }
 478
 479 static inline __attribute__((always_inline))
 480 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 481                               intptr_t *v2, intptr_t expect2,
 482                               intptr_t newv, int cpu)
 483 {
 484         RSEQ_INJECT_C(9)
 485
 486         __asm__ __volatile__ goto (
 487                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 488                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 489 #ifdef RSEQ_COMPARE_TWICE
 490                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 491                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 492                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
 493 #endif
 494                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 495                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 496                 RSEQ_INJECT_ASM(3)
 497                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 498                 RSEQ_INJECT_ASM(4)
 499                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
 500                 RSEQ_INJECT_ASM(5)
 501 #ifdef RSEQ_COMPARE_TWICE
 502                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 503                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 504                 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
 505 #endif
 506                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 507                 RSEQ_INJECT_ASM(6)
 508                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 509                 : /* gcc asm goto does not allow outputs */
 510                 : [cpu_id]              "r" (cpu),
 511                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 512                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 513                   [v]                   "Qo" (*v),
 514                   [expect]              "r" (expect),
 515                   [v2]                  "Qo" (*v2),
 516                   [expect2]             "r" (expect2),
 517                   [newv]                "r" (newv)
 518                   RSEQ_INJECT_INPUT
 519                 : "memory", RSEQ_ASM_TMP_REG
 520                 : abort, cmpfail
 521 #ifdef RSEQ_COMPARE_TWICE
 522                   , error1, error2, error3
 523 #endif
 524         );
 525
 526         return 0;
 527 abort:
 528         RSEQ_INJECT_FAILED
 529         return -1;
 530 cmpfail:
 531         return 1;
 532 #ifdef RSEQ_COMPARE_TWICE
 533 error1:
 534         rseq_bug("cpu_id comparison failed");
 535 error2:
 536         rseq_bug("expected value comparison failed");
 537 error3:
 538         rseq_bug("2nd expected value comparison failed");
 539 #endif
 540 }
 541
 542 static inline __attribute__((always_inline))
 543 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 544                                  void *dst, void *src, size_t len,
 545                                  intptr_t newv, int cpu)
 546 {
 547         RSEQ_INJECT_C(9)
 548
 549         __asm__ __volatile__ goto (
 550                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 551                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 552 #ifdef RSEQ_COMPARE_TWICE
 553                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 554                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 555 #endif
 556                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 557                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 558                 RSEQ_INJECT_ASM(3)
 559                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 560                 RSEQ_INJECT_ASM(4)
 561 #ifdef RSEQ_COMPARE_TWICE
 562                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 563                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 564 #endif
 565                 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 566                 RSEQ_INJECT_ASM(5)
 567                 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
 568                 RSEQ_INJECT_ASM(6)
 569                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 570                 : /* gcc asm goto does not allow outputs */
 571                 : [cpu_id]              "r" (cpu),
 572                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 573                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 574                   [expect]              "r" (expect),
 575                   [v]                   "Qo" (*v),
 576                   [newv]                "r" (newv),
 577                   [dst]                 "r" (dst),
 578                   [src]                 "r" (src),
 579                   [len]                 "r" (len)
 580                   RSEQ_INJECT_INPUT
 581                 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
 582                 : abort, cmpfail
 583 #ifdef RSEQ_COMPARE_TWICE
 584                   , error1, error2
 585 #endif
 586         );
 587
 588         return 0;
 589 abort:
 590         RSEQ_INJECT_FAILED
 591         return -1;
 592 cmpfail:
 593         return 1;
 594 #ifdef RSEQ_COMPARE_TWICE
 595 error1:
 596         rseq_bug("cpu_id comparison failed");
 597 error2:
 598         rseq_bug("expected value comparison failed");
 599 #endif
 600 }
 601
 602 static inline __attribute__((always_inline))
 603 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 604                                          void *dst, void *src, size_t len,
 605                                          intptr_t newv, int cpu)
 606 {
 607         RSEQ_INJECT_C(9)
 608
 609         __asm__ __volatile__ goto (
 610                 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
 611                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
 612 #ifdef RSEQ_COMPARE_TWICE
 613                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
 614                 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
 615 #endif
 616                 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
 617                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 618                 RSEQ_INJECT_ASM(3)
 619                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
 620                 RSEQ_INJECT_ASM(4)
 621 #ifdef RSEQ_COMPARE_TWICE
 622                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 623                 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
 624 #endif
 625                 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
 626                 RSEQ_INJECT_ASM(5)
 627                 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
 628                 RSEQ_INJECT_ASM(6)
 629                 RSEQ_ASM_DEFINE_ABORT(4, abort)
 630                 : /* gcc asm goto does not allow outputs */
 631                 : [cpu_id]              "r" (cpu),
 632                   [current_cpu_id]      "Qo" (__rseq_abi.cpu_id),
 633                   [rseq_cs]             "m" (__rseq_abi.rseq_cs),
 634                   [expect]              "r" (expect),
 635                   [v]                   "Qo" (*v),
 636                   [newv]                "r" (newv),
 637                   [dst]                 "r" (dst),
 638                   [src]                 "r" (src),
 639                   [len]                 "r" (len)
 640                   RSEQ_INJECT_INPUT
 641                 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
 642                 : abort, cmpfail
 643 #ifdef RSEQ_COMPARE_TWICE
 644                   , error1, error2
 645 #endif
 646         );
 647
 648         return 0;
 649 abort:
 650         RSEQ_INJECT_FAILED
 651         return -1;
 652 cmpfail:
 653         return 1;
 654 #ifdef RSEQ_COMPARE_TWICE
 655 error1:
 656         rseq_bug("cpu_id comparison failed");
 657 error2:
 658         rseq_bug("expected value comparison failed");
 659 #endif
 660 }
 661
 662 #endif /* !RSEQ_SKIP_FASTPATH */