tools/testing/selftests/rseq/rseq-x86.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-x86.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 #include <stdint.h>
   9
  10 /*
  11  * RSEQ_SIG is used with the following reserved undefined instructions, which
  12  * trap in user-space:
  13  *
  14  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
  15  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
  16  */
  17 #define RSEQ_SIG        0x53053053
  18
  19 /*
  20  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
  21  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
  22  * address through a "r" input operand.
  23  */
  24
  25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
  26 #define RSEQ_CPU_ID_OFFSET      4
  27 #define RSEQ_CS_OFFSET          8
  28
  29 #ifdef __x86_64__
  30
  31 #define rseq_smp_mb()   \
  32         __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  33 #define rseq_smp_rmb()  rseq_barrier()
  34 #define rseq_smp_wmb()  rseq_barrier()
  35
  36 #define rseq_smp_load_acquire(p)                                        \
  37 __extension__ ({                                                        \
  38         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  39         rseq_barrier();                                                 \
  40         ____p1;                                                         \
  41 })
  42
  43 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  44
  45 #define rseq_smp_store_release(p, v)                                    \
  46 do {                                                                    \
  47         rseq_barrier();                                                 \
  48         RSEQ_WRITE_ONCE(*p, v);                                         \
  49 } while (0)
  50
  51 #ifdef RSEQ_SKIP_FASTPATH
  52 #include "rseq-skip.h"
  53 #else /* !RSEQ_SKIP_FASTPATH */
  54
  55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  56                                 start_ip, post_commit_offset, abort_ip) \
  57                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  58                 ".balign 32\n\t"                                        \
  59                 __rseq_str(label) ":\n\t"                               \
  60                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  61                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  62                 ".popsection\n\t"                                       \
  63                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  64                 ".quad " __rseq_str(label) "b\n\t"                      \
  65                 ".popsection\n\t"
  66
  67
  68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  69         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  70                                 (post_commit_ip - start_ip), abort_ip)
  71
  72 /*
  73  * Exit points of a rseq critical section consist of all instructions outside
  74  * of the critical section where a critical section can either branch to or
  75  * reach through the normal course of its execution. The abort IP and the
  76  * post-commit IP are already part of the __rseq_cs section and should not be
  77  * explicitly defined as additional exit points. Knowing all exit points is
  78  * useful to assist debuggers stepping over the critical section.
  79  */
  80 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  81                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  82                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  83                 ".popsection\n\t"
  84
  85 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  86                 RSEQ_INJECT_ASM(1)                                      \
  87                 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
  88                 "movq %%rax, " __rseq_str(rseq_cs) "\n\t"               \
  89                 __rseq_str(label) ":\n\t"
  90
  91 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  92                 RSEQ_INJECT_ASM(2)                                      \
  93                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
  94                 "jnz " __rseq_str(label) "\n\t"
  95
  96 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
  97                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
  98                 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
  99                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 100                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 101                 __rseq_str(label) ":\n\t"                               \
 102                 teardown                                                \
 103                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 104                 ".popsection\n\t"
 105
 106 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 107                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 108                 __rseq_str(label) ":\n\t"                               \
 109                 teardown                                                \
 110                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 111                 ".popsection\n\t"
 112
 113 static inline __attribute__((always_inline))
 114 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 115 {
 116         RSEQ_INJECT_C(9)
 117
 118         __asm__ __volatile__ goto (
 119                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 120                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 121 #ifdef RSEQ_COMPARE_TWICE
 122                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 123                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 124 #endif
 125                 /* Start rseq by storing table entry pointer into rseq_cs. */
 126                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 127                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 128                 RSEQ_INJECT_ASM(3)
 129                 "cmpq %[v], %[expect]\n\t"
 130                 "jnz %l[cmpfail]\n\t"
 131                 RSEQ_INJECT_ASM(4)
 132 #ifdef RSEQ_COMPARE_TWICE
 133                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 134                 "cmpq %[v], %[expect]\n\t"
 135                 "jnz %l[error2]\n\t"
 136 #endif
 137                 /* final store */
 138                 "movq %[newv], %[v]\n\t"
 139                 "2:\n\t"
 140                 RSEQ_INJECT_ASM(5)
 141                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 142                 : /* gcc asm goto does not allow outputs */
 143                 : [cpu_id]              "r" (cpu),
 144                   [rseq_abi]            "r" (&__rseq_abi),
 145                   [v]                   "m" (*v),
 146                   [expect]              "r" (expect),
 147                   [newv]                "r" (newv)
 148                 : "memory", "cc", "rax"
 149                   RSEQ_INJECT_CLOBBER
 150                 : abort, cmpfail
 151 #ifdef RSEQ_COMPARE_TWICE
 152                   , error1, error2
 153 #endif
 154         );
 155         return 0;
 156 abort:
 157         RSEQ_INJECT_FAILED
 158         return -1;
 159 cmpfail:
 160         return 1;
 161 #ifdef RSEQ_COMPARE_TWICE
 162 error1:
 163         rseq_bug("cpu_id comparison failed");
 164 error2:
 165         rseq_bug("expected value comparison failed");
 166 #endif
 167 }
 168
 169 /*
 170  * Compare @v against @expectnot. When it does _not_ match, load @v
 171  * into @load, and store the content of *@v + voffp into @v.
 172  */
 173 static inline __attribute__((always_inline))
 174 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 175                                off_t voffp, intptr_t *load, int cpu)
 176 {
 177         RSEQ_INJECT_C(9)
 178
 179         __asm__ __volatile__ goto (
 180                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 181                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 182 #ifdef RSEQ_COMPARE_TWICE
 183                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 184                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 185 #endif
 186                 /* Start rseq by storing table entry pointer into rseq_cs. */
 187                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 188                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 189                 RSEQ_INJECT_ASM(3)
 190                 "movq %[v], %%rbx\n\t"
 191                 "cmpq %%rbx, %[expectnot]\n\t"
 192                 "je %l[cmpfail]\n\t"
 193                 RSEQ_INJECT_ASM(4)
 194 #ifdef RSEQ_COMPARE_TWICE
 195                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 196                 "movq %[v], %%rbx\n\t"
 197                 "cmpq %%rbx, %[expectnot]\n\t"
 198                 "je %l[error2]\n\t"
 199 #endif
 200                 "movq %%rbx, %[load]\n\t"
 201                 "addq %[voffp], %%rbx\n\t"
 202                 "movq (%%rbx), %%rbx\n\t"
 203                 /* final store */
 204                 "movq %%rbx, %[v]\n\t"
 205                 "2:\n\t"
 206                 RSEQ_INJECT_ASM(5)
 207                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 208                 : /* gcc asm goto does not allow outputs */
 209                 : [cpu_id]              "r" (cpu),
 210                   [rseq_abi]            "r" (&__rseq_abi),
 211                   /* final store input */
 212                   [v]                   "m" (*v),
 213                   [expectnot]           "r" (expectnot),
 214                   [voffp]               "er" (voffp),
 215                   [load]                "m" (*load)
 216                 : "memory", "cc", "rax", "rbx"
 217                   RSEQ_INJECT_CLOBBER
 218                 : abort, cmpfail
 219 #ifdef RSEQ_COMPARE_TWICE
 220                   , error1, error2
 221 #endif
 222         );
 223         return 0;
 224 abort:
 225         RSEQ_INJECT_FAILED
 226         return -1;
 227 cmpfail:
 228         return 1;
 229 #ifdef RSEQ_COMPARE_TWICE
 230 error1:
 231         rseq_bug("cpu_id comparison failed");
 232 error2:
 233         rseq_bug("expected value comparison failed");
 234 #endif
 235 }
 236
 237 static inline __attribute__((always_inline))
 238 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 239 {
 240         RSEQ_INJECT_C(9)
 241
 242         __asm__ __volatile__ goto (
 243                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 244 #ifdef RSEQ_COMPARE_TWICE
 245                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 246 #endif
 247                 /* Start rseq by storing table entry pointer into rseq_cs. */
 248                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 249                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 250                 RSEQ_INJECT_ASM(3)
 251 #ifdef RSEQ_COMPARE_TWICE
 252                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 253 #endif
 254                 /* final store */
 255                 "addq %[count], %[v]\n\t"
 256                 "2:\n\t"
 257                 RSEQ_INJECT_ASM(4)
 258                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 259                 : /* gcc asm goto does not allow outputs */
 260                 : [cpu_id]              "r" (cpu),
 261                   [rseq_abi]            "r" (&__rseq_abi),
 262                   /* final store input */
 263                   [v]                   "m" (*v),
 264                   [count]               "er" (count)
 265                 : "memory", "cc", "rax"
 266                   RSEQ_INJECT_CLOBBER
 267                 : abort
 268 #ifdef RSEQ_COMPARE_TWICE
 269                   , error1
 270 #endif
 271         );
 272         return 0;
 273 abort:
 274         RSEQ_INJECT_FAILED
 275         return -1;
 276 #ifdef RSEQ_COMPARE_TWICE
 277 error1:
 278         rseq_bug("cpu_id comparison failed");
 279 #endif
 280 }
 281
 282 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
 283
 284 /*
 285  *   pval = *(ptr+off)
 286  *  *pval += inc;
 287  */
 288 static inline __attribute__((always_inline))
 289 int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
 290 {
 291         RSEQ_INJECT_C(9)
 292
 293         __asm__ __volatile__ goto (
 294                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 295 #ifdef RSEQ_COMPARE_TWICE
 296                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 297 #endif
 298                 /* Start rseq by storing table entry pointer into rseq_cs. */
 299                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 300                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 301                 RSEQ_INJECT_ASM(3)
 302 #ifdef RSEQ_COMPARE_TWICE
 303                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 304 #endif
 305                 /* get p+v */
 306                 "movq %[ptr], %%rbx\n\t"
 307                 "addq %[off], %%rbx\n\t"
 308                 /* get pv */
 309                 "movq (%%rbx), %%rcx\n\t"
 310                 /* *pv += inc */
 311                 "addq %[inc], (%%rcx)\n\t"
 312                 "2:\n\t"
 313                 RSEQ_INJECT_ASM(4)
 314                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 315                 : /* gcc asm goto does not allow outputs */
 316                 : [cpu_id]              "r" (cpu),
 317                   [rseq_abi]            "r" (&__rseq_abi),
 318                   /* final store input */
 319                   [ptr]                 "m" (*ptr),
 320                   [off]                 "er" (off),
 321                   [inc]                 "er" (inc)
 322                 : "memory", "cc", "rax", "rbx", "rcx"
 323                   RSEQ_INJECT_CLOBBER
 324                 : abort
 325 #ifdef RSEQ_COMPARE_TWICE
 326                   , error1
 327 #endif
 328         );
 329         return 0;
 330 abort:
 331         RSEQ_INJECT_FAILED
 332         return -1;
 333 #ifdef RSEQ_COMPARE_TWICE
 334 error1:
 335         rseq_bug("cpu_id comparison failed");
 336 #endif
 337 }
 338
 339 static inline __attribute__((always_inline))
 340 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 341                                  intptr_t *v2, intptr_t newv2,
 342                                  intptr_t newv, int cpu)
 343 {
 344         RSEQ_INJECT_C(9)
 345
 346         __asm__ __volatile__ goto (
 347                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 348                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 349 #ifdef RSEQ_COMPARE_TWICE
 350                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 351                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 352 #endif
 353                 /* Start rseq by storing table entry pointer into rseq_cs. */
 354                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 355                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 356                 RSEQ_INJECT_ASM(3)
 357                 "cmpq %[v], %[expect]\n\t"
 358                 "jnz %l[cmpfail]\n\t"
 359                 RSEQ_INJECT_ASM(4)
 360 #ifdef RSEQ_COMPARE_TWICE
 361                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 362                 "cmpq %[v], %[expect]\n\t"
 363                 "jnz %l[error2]\n\t"
 364 #endif
 365                 /* try store */
 366                 "movq %[newv2], %[v2]\n\t"
 367                 RSEQ_INJECT_ASM(5)
 368                 /* final store */
 369                 "movq %[newv], %[v]\n\t"
 370                 "2:\n\t"
 371                 RSEQ_INJECT_ASM(6)
 372                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 373                 : /* gcc asm goto does not allow outputs */
 374                 : [cpu_id]              "r" (cpu),
 375                   [rseq_abi]            "r" (&__rseq_abi),
 376                   /* try store input */
 377                   [v2]                  "m" (*v2),
 378                   [newv2]               "r" (newv2),
 379                   /* final store input */
 380                   [v]                   "m" (*v),
 381                   [expect]              "r" (expect),
 382                   [newv]                "r" (newv)
 383                 : "memory", "cc", "rax"
 384                   RSEQ_INJECT_CLOBBER
 385                 : abort, cmpfail
 386 #ifdef RSEQ_COMPARE_TWICE
 387                   , error1, error2
 388 #endif
 389         );
 390         return 0;
 391 abort:
 392         RSEQ_INJECT_FAILED
 393         return -1;
 394 cmpfail:
 395         return 1;
 396 #ifdef RSEQ_COMPARE_TWICE
 397 error1:
 398         rseq_bug("cpu_id comparison failed");
 399 error2:
 400         rseq_bug("expected value comparison failed");
 401 #endif
 402 }
 403
 404 /* x86-64 is TSO. */
 405 static inline __attribute__((always_inline))
 406 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 407                                          intptr_t *v2, intptr_t newv2,
 408                                          intptr_t newv, int cpu)
 409 {
 410         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 411 }
 412
 413 static inline __attribute__((always_inline))
 414 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 415                               intptr_t *v2, intptr_t expect2,
 416                               intptr_t newv, int cpu)
 417 {
 418         RSEQ_INJECT_C(9)
 419
 420         __asm__ __volatile__ goto (
 421                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 422                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 423 #ifdef RSEQ_COMPARE_TWICE
 424                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 425                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 426                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 427 #endif
 428                 /* Start rseq by storing table entry pointer into rseq_cs. */
 429                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 430                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 431                 RSEQ_INJECT_ASM(3)
 432                 "cmpq %[v], %[expect]\n\t"
 433                 "jnz %l[cmpfail]\n\t"
 434                 RSEQ_INJECT_ASM(4)
 435                 "cmpq %[v2], %[expect2]\n\t"
 436                 "jnz %l[cmpfail]\n\t"
 437                 RSEQ_INJECT_ASM(5)
 438 #ifdef RSEQ_COMPARE_TWICE
 439                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 440                 "cmpq %[v], %[expect]\n\t"
 441                 "jnz %l[error2]\n\t"
 442                 "cmpq %[v2], %[expect2]\n\t"
 443                 "jnz %l[error3]\n\t"
 444 #endif
 445                 /* final store */
 446                 "movq %[newv], %[v]\n\t"
 447                 "2:\n\t"
 448                 RSEQ_INJECT_ASM(6)
 449                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 450                 : /* gcc asm goto does not allow outputs */
 451                 : [cpu_id]              "r" (cpu),
 452                   [rseq_abi]            "r" (&__rseq_abi),
 453                   /* cmp2 input */
 454                   [v2]                  "m" (*v2),
 455                   [expect2]             "r" (expect2),
 456                   /* final store input */
 457                   [v]                   "m" (*v),
 458                   [expect]              "r" (expect),
 459                   [newv]                "r" (newv)
 460                 : "memory", "cc", "rax"
 461                   RSEQ_INJECT_CLOBBER
 462                 : abort, cmpfail
 463 #ifdef RSEQ_COMPARE_TWICE
 464                   , error1, error2, error3
 465 #endif
 466         );
 467         return 0;
 468 abort:
 469         RSEQ_INJECT_FAILED
 470         return -1;
 471 cmpfail:
 472         return 1;
 473 #ifdef RSEQ_COMPARE_TWICE
 474 error1:
 475         rseq_bug("cpu_id comparison failed");
 476 error2:
 477         rseq_bug("1st expected value comparison failed");
 478 error3:
 479         rseq_bug("2nd expected value comparison failed");
 480 #endif
 481 }
 482
 483 static inline __attribute__((always_inline))
 484 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 485                                  void *dst, void *src, size_t len,
 486                                  intptr_t newv, int cpu)
 487 {
 488         uint64_t rseq_scratch[3];
 489
 490         RSEQ_INJECT_C(9)
 491
 492         __asm__ __volatile__ goto (
 493                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 494                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 495 #ifdef RSEQ_COMPARE_TWICE
 496                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 497                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 498 #endif
 499                 "movq %[src], %[rseq_scratch0]\n\t"
 500                 "movq %[dst], %[rseq_scratch1]\n\t"
 501                 "movq %[len], %[rseq_scratch2]\n\t"
 502                 /* Start rseq by storing table entry pointer into rseq_cs. */
 503                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 504                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 505                 RSEQ_INJECT_ASM(3)
 506                 "cmpq %[v], %[expect]\n\t"
 507                 "jnz 5f\n\t"
 508                 RSEQ_INJECT_ASM(4)
 509 #ifdef RSEQ_COMPARE_TWICE
 510                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
 511                 "cmpq %[v], %[expect]\n\t"
 512                 "jnz 7f\n\t"
 513 #endif
 514                 /* try memcpy */
 515                 "test %[len], %[len]\n\t" \
 516                 "jz 333f\n\t" \
 517                 "222:\n\t" \
 518                 "movb (%[src]), %%al\n\t" \
 519                 "movb %%al, (%[dst])\n\t" \
 520                 "inc %[src]\n\t" \
 521                 "inc %[dst]\n\t" \
 522                 "dec %[len]\n\t" \
 523                 "jnz 222b\n\t" \
 524                 "333:\n\t" \
 525                 RSEQ_INJECT_ASM(5)
 526                 /* final store */
 527                 "movq %[newv], %[v]\n\t"
 528                 "2:\n\t"
 529                 RSEQ_INJECT_ASM(6)
 530                 /* teardown */
 531                 "movq %[rseq_scratch2], %[len]\n\t"
 532                 "movq %[rseq_scratch1], %[dst]\n\t"
 533                 "movq %[rseq_scratch0], %[src]\n\t"
 534                 RSEQ_ASM_DEFINE_ABORT(4,
 535                         "movq %[rseq_scratch2], %[len]\n\t"
 536                         "movq %[rseq_scratch1], %[dst]\n\t"
 537                         "movq %[rseq_scratch0], %[src]\n\t",
 538                         abort)
 539                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 540                         "movq %[rseq_scratch2], %[len]\n\t"
 541                         "movq %[rseq_scratch1], %[dst]\n\t"
 542                         "movq %[rseq_scratch0], %[src]\n\t",
 543                         cmpfail)
 544 #ifdef RSEQ_COMPARE_TWICE
 545                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 546                         "movq %[rseq_scratch2], %[len]\n\t"
 547                         "movq %[rseq_scratch1], %[dst]\n\t"
 548                         "movq %[rseq_scratch0], %[src]\n\t",
 549                         error1)
 550                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 551                         "movq %[rseq_scratch2], %[len]\n\t"
 552                         "movq %[rseq_scratch1], %[dst]\n\t"
 553                         "movq %[rseq_scratch0], %[src]\n\t",
 554                         error2)
 555 #endif
 556                 : /* gcc asm goto does not allow outputs */
 557                 : [cpu_id]              "r" (cpu),
 558                   [rseq_abi]            "r" (&__rseq_abi),
 559                   /* final store input */
 560                   [v]                   "m" (*v),
 561                   [expect]              "r" (expect),
 562                   [newv]                "r" (newv),
 563                   /* try memcpy input */
 564                   [dst]                 "r" (dst),
 565                   [src]                 "r" (src),
 566                   [len]                 "r" (len),
 567                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 568                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 569                   [rseq_scratch2]       "m" (rseq_scratch[2])
 570                 : "memory", "cc", "rax"
 571                   RSEQ_INJECT_CLOBBER
 572                 : abort, cmpfail
 573 #ifdef RSEQ_COMPARE_TWICE
 574                   , error1, error2
 575 #endif
 576         );
 577         return 0;
 578 abort:
 579         RSEQ_INJECT_FAILED
 580         return -1;
 581 cmpfail:
 582         return 1;
 583 #ifdef RSEQ_COMPARE_TWICE
 584 error1:
 585         rseq_bug("cpu_id comparison failed");
 586 error2:
 587         rseq_bug("expected value comparison failed");
 588 #endif
 589 }
 590
 591 /* x86-64 is TSO. */
 592 static inline __attribute__((always_inline))
 593 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 594                                          void *dst, void *src, size_t len,
 595                                          intptr_t newv, int cpu)
 596 {
 597         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 598                                             newv, cpu);
 599 }
 600
 601 #endif /* !RSEQ_SKIP_FASTPATH */
 602
 603 #elif __i386__
 604
 605 #define rseq_smp_mb()   \
 606         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 607 #define rseq_smp_rmb()  \
 608         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 609 #define rseq_smp_wmb()  \
 610         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 611
 612 #define rseq_smp_load_acquire(p)                                        \
 613 __extension__ ({                                                        \
 614         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 615         rseq_smp_mb();                                                  \
 616         ____p1;                                                         \
 617 })
 618
 619 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 620
 621 #define rseq_smp_store_release(p, v)                                    \
 622 do {                                                                    \
 623         rseq_smp_mb();                                                  \
 624         RSEQ_WRITE_ONCE(*p, v);                                         \
 625 } while (0)
 626
 627 #ifdef RSEQ_SKIP_FASTPATH
 628 #include "rseq-skip.h"
 629 #else /* !RSEQ_SKIP_FASTPATH */
 630
 631 /*
 632  * Use eax as scratch register and take memory operands as input to
 633  * lessen register pressure. Especially needed when compiling in O0.
 634  */
 635 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 636                                 start_ip, post_commit_offset, abort_ip) \
 637                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 638                 ".balign 32\n\t"                                        \
 639                 __rseq_str(label) ":\n\t"                               \
 640                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 641                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 642                 ".popsection\n\t"                                       \
 643                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 644                 ".long " __rseq_str(label) "b, 0x0\n\t"                 \
 645                 ".popsection\n\t"
 646
 647 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 648         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 649                                 (post_commit_ip - start_ip), abort_ip)
 650
 651 /*
 652  * Exit points of a rseq critical section consist of all instructions outside
 653  * of the critical section where a critical section can either branch to or
 654  * reach through the normal course of its execution. The abort IP and the
 655  * post-commit IP are already part of the __rseq_cs section and should not be
 656  * explicitly defined as additional exit points. Knowing all exit points is
 657  * useful to assist debuggers stepping over the critical section.
 658  */
 659 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 660                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 661                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
 662                 ".popsection\n\t"
 663
 664 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 665                 RSEQ_INJECT_ASM(1)                                      \
 666                 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
 667                 __rseq_str(label) ":\n\t"
 668
 669 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 670                 RSEQ_INJECT_ASM(2)                                      \
 671                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
 672                 "jnz " __rseq_str(label) "\n\t"
 673
 674 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 675                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 676                 /* Disassembler-friendly signature: ud1 <sig>,%edi. */  \
 677                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 678                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 679                 __rseq_str(label) ":\n\t"                               \
 680                 teardown                                                \
 681                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 682                 ".popsection\n\t"
 683
 684 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 685                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 686                 __rseq_str(label) ":\n\t"                               \
 687                 teardown                                                \
 688                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 689                 ".popsection\n\t"
 690
 691 static inline __attribute__((always_inline))
 692 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 693 {
 694         RSEQ_INJECT_C(9)
 695
 696         __asm__ __volatile__ goto (
 697                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 698                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 699 #ifdef RSEQ_COMPARE_TWICE
 700                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 701                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 702 #endif
 703                 /* Start rseq by storing table entry pointer into rseq_cs. */
 704                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 705                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 706                 RSEQ_INJECT_ASM(3)
 707                 "cmpl %[v], %[expect]\n\t"
 708                 "jnz %l[cmpfail]\n\t"
 709                 RSEQ_INJECT_ASM(4)
 710 #ifdef RSEQ_COMPARE_TWICE
 711                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 712                 "cmpl %[v], %[expect]\n\t"
 713                 "jnz %l[error2]\n\t"
 714 #endif
 715                 /* final store */
 716                 "movl %[newv], %[v]\n\t"
 717                 "2:\n\t"
 718                 RSEQ_INJECT_ASM(5)
 719                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 720                 : /* gcc asm goto does not allow outputs */
 721                 : [cpu_id]              "r" (cpu),
 722                   [rseq_abi]            "r" (&__rseq_abi),
 723                   [v]                   "m" (*v),
 724                   [expect]              "r" (expect),
 725                   [newv]                "r" (newv)
 726                 : "memory", "cc", "eax"
 727                   RSEQ_INJECT_CLOBBER
 728                 : abort, cmpfail
 729 #ifdef RSEQ_COMPARE_TWICE
 730                   , error1, error2
 731 #endif
 732         );
 733         return 0;
 734 abort:
 735         RSEQ_INJECT_FAILED
 736         return -1;
 737 cmpfail:
 738         return 1;
 739 #ifdef RSEQ_COMPARE_TWICE
 740 error1:
 741         rseq_bug("cpu_id comparison failed");
 742 error2:
 743         rseq_bug("expected value comparison failed");
 744 #endif
 745 }
 746
 747 /*
 748  * Compare @v against @expectnot. When it does _not_ match, load @v
 749  * into @load, and store the content of *@v + voffp into @v.
 750  */
 751 static inline __attribute__((always_inline))
 752 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 753                                off_t voffp, intptr_t *load, int cpu)
 754 {
 755         RSEQ_INJECT_C(9)
 756
 757         __asm__ __volatile__ goto (
 758                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 759                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 760 #ifdef RSEQ_COMPARE_TWICE
 761                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 762                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 763 #endif
 764                 /* Start rseq by storing table entry pointer into rseq_cs. */
 765                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 766                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 767                 RSEQ_INJECT_ASM(3)
 768                 "movl %[v], %%ebx\n\t"
 769                 "cmpl %%ebx, %[expectnot]\n\t"
 770                 "je %l[cmpfail]\n\t"
 771                 RSEQ_INJECT_ASM(4)
 772 #ifdef RSEQ_COMPARE_TWICE
 773                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 774                 "movl %[v], %%ebx\n\t"
 775                 "cmpl %%ebx, %[expectnot]\n\t"
 776                 "je %l[error2]\n\t"
 777 #endif
 778                 "movl %%ebx, %[load]\n\t"
 779                 "addl %[voffp], %%ebx\n\t"
 780                 "movl (%%ebx), %%ebx\n\t"
 781                 /* final store */
 782                 "movl %%ebx, %[v]\n\t"
 783                 "2:\n\t"
 784                 RSEQ_INJECT_ASM(5)
 785                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 786                 : /* gcc asm goto does not allow outputs */
 787                 : [cpu_id]              "r" (cpu),
 788                   [rseq_abi]            "r" (&__rseq_abi),
 789                   /* final store input */
 790                   [v]                   "m" (*v),
 791                   [expectnot]           "r" (expectnot),
 792                   [voffp]               "ir" (voffp),
 793                   [load]                "m" (*load)
 794                 : "memory", "cc", "eax", "ebx"
 795                   RSEQ_INJECT_CLOBBER
 796                 : abort, cmpfail
 797 #ifdef RSEQ_COMPARE_TWICE
 798                   , error1, error2
 799 #endif
 800         );
 801         return 0;
 802 abort:
 803         RSEQ_INJECT_FAILED
 804         return -1;
 805 cmpfail:
 806         return 1;
 807 #ifdef RSEQ_COMPARE_TWICE
 808 error1:
 809         rseq_bug("cpu_id comparison failed");
 810 error2:
 811         rseq_bug("expected value comparison failed");
 812 #endif
 813 }
 814
 815 static inline __attribute__((always_inline))
 816 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 817 {
 818         RSEQ_INJECT_C(9)
 819
 820         __asm__ __volatile__ goto (
 821                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 822 #ifdef RSEQ_COMPARE_TWICE
 823                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 824 #endif
 825                 /* Start rseq by storing table entry pointer into rseq_cs. */
 826                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 827                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 828                 RSEQ_INJECT_ASM(3)
 829 #ifdef RSEQ_COMPARE_TWICE
 830                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 831 #endif
 832                 /* final store */
 833                 "addl %[count], %[v]\n\t"
 834                 "2:\n\t"
 835                 RSEQ_INJECT_ASM(4)
 836                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 837                 : /* gcc asm goto does not allow outputs */
 838                 : [cpu_id]              "r" (cpu),
 839                   [rseq_abi]            "r" (&__rseq_abi),
 840                   /* final store input */
 841                   [v]                   "m" (*v),
 842                   [count]               "ir" (count)
 843                 : "memory", "cc", "eax"
 844                   RSEQ_INJECT_CLOBBER
 845                 : abort
 846 #ifdef RSEQ_COMPARE_TWICE
 847                   , error1
 848 #endif
 849         );
 850         return 0;
 851 abort:
 852         RSEQ_INJECT_FAILED
 853         return -1;
 854 #ifdef RSEQ_COMPARE_TWICE
 855 error1:
 856         rseq_bug("cpu_id comparison failed");
 857 #endif
 858 }
 859
 860 static inline __attribute__((always_inline))
 861 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 862                                  intptr_t *v2, intptr_t newv2,
 863                                  intptr_t newv, int cpu)
 864 {
 865         RSEQ_INJECT_C(9)
 866
 867         __asm__ __volatile__ goto (
 868                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 869                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 870 #ifdef RSEQ_COMPARE_TWICE
 871                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 872                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 873 #endif
 874                 /* Start rseq by storing table entry pointer into rseq_cs. */
 875                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 876                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 877                 RSEQ_INJECT_ASM(3)
 878                 "cmpl %[v], %[expect]\n\t"
 879                 "jnz %l[cmpfail]\n\t"
 880                 RSEQ_INJECT_ASM(4)
 881 #ifdef RSEQ_COMPARE_TWICE
 882                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 883                 "cmpl %[v], %[expect]\n\t"
 884                 "jnz %l[error2]\n\t"
 885 #endif
 886                 /* try store */
 887                 "movl %[newv2], %%eax\n\t"
 888                 "movl %%eax, %[v2]\n\t"
 889                 RSEQ_INJECT_ASM(5)
 890                 /* final store */
 891                 "movl %[newv], %[v]\n\t"
 892                 "2:\n\t"
 893                 RSEQ_INJECT_ASM(6)
 894                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 895                 : /* gcc asm goto does not allow outputs */
 896                 : [cpu_id]              "r" (cpu),
 897                   [rseq_abi]            "r" (&__rseq_abi),
 898                   /* try store input */
 899                   [v2]                  "m" (*v2),
 900                   [newv2]               "m" (newv2),
 901                   /* final store input */
 902                   [v]                   "m" (*v),
 903                   [expect]              "r" (expect),
 904                   [newv]                "r" (newv)
 905                 : "memory", "cc", "eax"
 906                   RSEQ_INJECT_CLOBBER
 907                 : abort, cmpfail
 908 #ifdef RSEQ_COMPARE_TWICE
 909                   , error1, error2
 910 #endif
 911         );
 912         return 0;
 913 abort:
 914         RSEQ_INJECT_FAILED
 915         return -1;
 916 cmpfail:
 917         return 1;
 918 #ifdef RSEQ_COMPARE_TWICE
 919 error1:
 920         rseq_bug("cpu_id comparison failed");
 921 error2:
 922         rseq_bug("expected value comparison failed");
 923 #endif
 924 }
 925
 926 static inline __attribute__((always_inline))
 927 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 928                                          intptr_t *v2, intptr_t newv2,
 929                                          intptr_t newv, int cpu)
 930 {
 931         RSEQ_INJECT_C(9)
 932
 933         __asm__ __volatile__ goto (
 934                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 935                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 936 #ifdef RSEQ_COMPARE_TWICE
 937                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 938                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 939 #endif
 940                 /* Start rseq by storing table entry pointer into rseq_cs. */
 941                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 942                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 943                 RSEQ_INJECT_ASM(3)
 944                 "movl %[expect], %%eax\n\t"
 945                 "cmpl %[v], %%eax\n\t"
 946                 "jnz %l[cmpfail]\n\t"
 947                 RSEQ_INJECT_ASM(4)
 948 #ifdef RSEQ_COMPARE_TWICE
 949                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 950                 "movl %[expect], %%eax\n\t"
 951                 "cmpl %[v], %%eax\n\t"
 952                 "jnz %l[error2]\n\t"
 953 #endif
 954                 /* try store */
 955                 "movl %[newv2], %[v2]\n\t"
 956                 RSEQ_INJECT_ASM(5)
 957                 "lock; addl $0,-128(%%esp)\n\t"
 958                 /* final store */
 959                 "movl %[newv], %[v]\n\t"
 960                 "2:\n\t"
 961                 RSEQ_INJECT_ASM(6)
 962                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 963                 : /* gcc asm goto does not allow outputs */
 964                 : [cpu_id]              "r" (cpu),
 965                   [rseq_abi]            "r" (&__rseq_abi),
 966                   /* try store input */
 967                   [v2]                  "m" (*v2),
 968                   [newv2]               "r" (newv2),
 969                   /* final store input */
 970                   [v]                   "m" (*v),
 971                   [expect]              "m" (expect),
 972                   [newv]                "r" (newv)
 973                 : "memory", "cc", "eax"
 974                   RSEQ_INJECT_CLOBBER
 975                 : abort, cmpfail
 976 #ifdef RSEQ_COMPARE_TWICE
 977                   , error1, error2
 978 #endif
 979         );
 980         return 0;
 981 abort:
 982         RSEQ_INJECT_FAILED
 983         return -1;
 984 cmpfail:
 985         return 1;
 986 #ifdef RSEQ_COMPARE_TWICE
 987 error1:
 988         rseq_bug("cpu_id comparison failed");
 989 error2:
 990         rseq_bug("expected value comparison failed");
 991 #endif
 992
 993 }
 994
 995 static inline __attribute__((always_inline))
 996 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 997                               intptr_t *v2, intptr_t expect2,
 998                               intptr_t newv, int cpu)
 999 {
1000         RSEQ_INJECT_C(9)
1001
1002         __asm__ __volatile__ goto (
1003                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1004                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1005 #ifdef RSEQ_COMPARE_TWICE
1006                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1007                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1008                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1009 #endif
1010                 /* Start rseq by storing table entry pointer into rseq_cs. */
1011                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1012                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1013                 RSEQ_INJECT_ASM(3)
1014                 "cmpl %[v], %[expect]\n\t"
1015                 "jnz %l[cmpfail]\n\t"
1016                 RSEQ_INJECT_ASM(4)
1017                 "cmpl %[expect2], %[v2]\n\t"
1018                 "jnz %l[cmpfail]\n\t"
1019                 RSEQ_INJECT_ASM(5)
1020 #ifdef RSEQ_COMPARE_TWICE
1021                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1022                 "cmpl %[v], %[expect]\n\t"
1023                 "jnz %l[error2]\n\t"
1024                 "cmpl %[expect2], %[v2]\n\t"
1025                 "jnz %l[error3]\n\t"
1026 #endif
1027                 "movl %[newv], %%eax\n\t"
1028                 /* final store */
1029                 "movl %%eax, %[v]\n\t"
1030                 "2:\n\t"
1031                 RSEQ_INJECT_ASM(6)
1032                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1033                 : /* gcc asm goto does not allow outputs */
1034                 : [cpu_id]              "r" (cpu),
1035                   [rseq_abi]            "r" (&__rseq_abi),
1036                   /* cmp2 input */
1037                   [v2]                  "m" (*v2),
1038                   [expect2]             "r" (expect2),
1039                   /* final store input */
1040                   [v]                   "m" (*v),
1041                   [expect]              "r" (expect),
1042                   [newv]                "m" (newv)
1043                 : "memory", "cc", "eax"
1044                   RSEQ_INJECT_CLOBBER
1045                 : abort, cmpfail
1046 #ifdef RSEQ_COMPARE_TWICE
1047                   , error1, error2, error3
1048 #endif
1049         );
1050         return 0;
1051 abort:
1052         RSEQ_INJECT_FAILED
1053         return -1;
1054 cmpfail:
1055         return 1;
1056 #ifdef RSEQ_COMPARE_TWICE
1057 error1:
1058         rseq_bug("cpu_id comparison failed");
1059 error2:
1060         rseq_bug("1st expected value comparison failed");
1061 error3:
1062         rseq_bug("2nd expected value comparison failed");
1063 #endif
1064 }
1065
1066 /* TODO: implement a faster memcpy. */
1067 static inline __attribute__((always_inline))
1068 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1069                                  void *dst, void *src, size_t len,
1070                                  intptr_t newv, int cpu)
1071 {
1072         uint32_t rseq_scratch[3];
1073
1074         RSEQ_INJECT_C(9)
1075
1076         __asm__ __volatile__ goto (
1077                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1078                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1079 #ifdef RSEQ_COMPARE_TWICE
1080                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1081                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1082 #endif
1083                 "movl %[src], %[rseq_scratch0]\n\t"
1084                 "movl %[dst], %[rseq_scratch1]\n\t"
1085                 "movl %[len], %[rseq_scratch2]\n\t"
1086                 /* Start rseq by storing table entry pointer into rseq_cs. */
1087                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1088                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1089                 RSEQ_INJECT_ASM(3)
1090                 "movl %[expect], %%eax\n\t"
1091                 "cmpl %%eax, %[v]\n\t"
1092                 "jnz 5f\n\t"
1093                 RSEQ_INJECT_ASM(4)
1094 #ifdef RSEQ_COMPARE_TWICE
1095                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1096                 "movl %[expect], %%eax\n\t"
1097                 "cmpl %%eax, %[v]\n\t"
1098                 "jnz 7f\n\t"
1099 #endif
1100                 /* try memcpy */
1101                 "test %[len], %[len]\n\t" \
1102                 "jz 333f\n\t" \
1103                 "222:\n\t" \
1104                 "movb (%[src]), %%al\n\t" \
1105                 "movb %%al, (%[dst])\n\t" \
1106                 "inc %[src]\n\t" \
1107                 "inc %[dst]\n\t" \
1108                 "dec %[len]\n\t" \
1109                 "jnz 222b\n\t" \
1110                 "333:\n\t" \
1111                 RSEQ_INJECT_ASM(5)
1112                 "movl %[newv], %%eax\n\t"
1113                 /* final store */
1114                 "movl %%eax, %[v]\n\t"
1115                 "2:\n\t"
1116                 RSEQ_INJECT_ASM(6)
1117                 /* teardown */
1118                 "movl %[rseq_scratch2], %[len]\n\t"
1119                 "movl %[rseq_scratch1], %[dst]\n\t"
1120                 "movl %[rseq_scratch0], %[src]\n\t"
1121                 RSEQ_ASM_DEFINE_ABORT(4,
1122                         "movl %[rseq_scratch2], %[len]\n\t"
1123                         "movl %[rseq_scratch1], %[dst]\n\t"
1124                         "movl %[rseq_scratch0], %[src]\n\t",
1125                         abort)
1126                 RSEQ_ASM_DEFINE_CMPFAIL(5,
1127                         "movl %[rseq_scratch2], %[len]\n\t"
1128                         "movl %[rseq_scratch1], %[dst]\n\t"
1129                         "movl %[rseq_scratch0], %[src]\n\t",
1130                         cmpfail)
1131 #ifdef RSEQ_COMPARE_TWICE
1132                 RSEQ_ASM_DEFINE_CMPFAIL(6,
1133                         "movl %[rseq_scratch2], %[len]\n\t"
1134                         "movl %[rseq_scratch1], %[dst]\n\t"
1135                         "movl %[rseq_scratch0], %[src]\n\t",
1136                         error1)
1137                 RSEQ_ASM_DEFINE_CMPFAIL(7,
1138                         "movl %[rseq_scratch2], %[len]\n\t"
1139                         "movl %[rseq_scratch1], %[dst]\n\t"
1140                         "movl %[rseq_scratch0], %[src]\n\t",
1141                         error2)
1142 #endif
1143                 : /* gcc asm goto does not allow outputs */
1144                 : [cpu_id]              "r" (cpu),
1145                   [rseq_abi]            "r" (&__rseq_abi),
1146                   /* final store input */
1147                   [v]                   "m" (*v),
1148                   [expect]              "m" (expect),
1149                   [newv]                "m" (newv),
1150                   /* try memcpy input */
1151                   [dst]                 "r" (dst),
1152                   [src]                 "r" (src),
1153                   [len]                 "r" (len),
1154                   [rseq_scratch0]       "m" (rseq_scratch[0]),
1155                   [rseq_scratch1]       "m" (rseq_scratch[1]),
1156                   [rseq_scratch2]       "m" (rseq_scratch[2])
1157                 : "memory", "cc", "eax"
1158                   RSEQ_INJECT_CLOBBER
1159                 : abort, cmpfail
1160 #ifdef RSEQ_COMPARE_TWICE
1161                   , error1, error2
1162 #endif
1163         );
1164         return 0;
1165 abort:
1166         RSEQ_INJECT_FAILED
1167         return -1;
1168 cmpfail:
1169         return 1;
1170 #ifdef RSEQ_COMPARE_TWICE
1171 error1:
1172         rseq_bug("cpu_id comparison failed");
1173 error2:
1174         rseq_bug("expected value comparison failed");
1175 #endif
1176 }
1177
1178 /* TODO: implement a faster memcpy. */
1179 static inline __attribute__((always_inline))
1180 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1181                                          void *dst, void *src, size_t len,
1182                                          intptr_t newv, int cpu)
1183 {
1184         uint32_t rseq_scratch[3];
1185
1186         RSEQ_INJECT_C(9)
1187
1188         __asm__ __volatile__ goto (
1189                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1190                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1191 #ifdef RSEQ_COMPARE_TWICE
1192                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1193                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1194 #endif
1195                 "movl %[src], %[rseq_scratch0]\n\t"
1196                 "movl %[dst], %[rseq_scratch1]\n\t"
1197                 "movl %[len], %[rseq_scratch2]\n\t"
1198                 /* Start rseq by storing table entry pointer into rseq_cs. */
1199                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1200                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1201                 RSEQ_INJECT_ASM(3)
1202                 "movl %[expect], %%eax\n\t"
1203                 "cmpl %%eax, %[v]\n\t"
1204                 "jnz 5f\n\t"
1205                 RSEQ_INJECT_ASM(4)
1206 #ifdef RSEQ_COMPARE_TWICE
1207                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1208                 "movl %[expect], %%eax\n\t"
1209                 "cmpl %%eax, %[v]\n\t"
1210                 "jnz 7f\n\t"
1211 #endif
1212                 /* try memcpy */
1213                 "test %[len], %[len]\n\t" \
1214                 "jz 333f\n\t" \
1215                 "222:\n\t" \
1216                 "movb (%[src]), %%al\n\t" \
1217                 "movb %%al, (%[dst])\n\t" \
1218                 "inc %[src]\n\t" \
1219                 "inc %[dst]\n\t" \
1220                 "dec %[len]\n\t" \
1221                 "jnz 222b\n\t" \
1222                 "333:\n\t" \
1223                 RSEQ_INJECT_ASM(5)
1224                 "lock; addl $0,-128(%%esp)\n\t"
1225                 "movl %[newv], %%eax\n\t"
1226                 /* final store */
1227                 "movl %%eax, %[v]\n\t"
1228                 "2:\n\t"
1229                 RSEQ_INJECT_ASM(6)
1230                 /* teardown */
1231                 "movl %[rseq_scratch2], %[len]\n\t"
1232                 "movl %[rseq_scratch1], %[dst]\n\t"
1233                 "movl %[rseq_scratch0], %[src]\n\t"
1234                 RSEQ_ASM_DEFINE_ABORT(4,
1235                         "movl %[rseq_scratch2], %[len]\n\t"
1236                         "movl %[rseq_scratch1], %[dst]\n\t"
1237                         "movl %[rseq_scratch0], %[src]\n\t",
1238                         abort)
1239                 RSEQ_ASM_DEFINE_CMPFAIL(5,
1240                         "movl %[rseq_scratch2], %[len]\n\t"
1241                         "movl %[rseq_scratch1], %[dst]\n\t"
1242                         "movl %[rseq_scratch0], %[src]\n\t",
1243                         cmpfail)
1244 #ifdef RSEQ_COMPARE_TWICE
1245                 RSEQ_ASM_DEFINE_CMPFAIL(6,
1246                         "movl %[rseq_scratch2], %[len]\n\t"
1247                         "movl %[rseq_scratch1], %[dst]\n\t"
1248                         "movl %[rseq_scratch0], %[src]\n\t",
1249                         error1)
1250                 RSEQ_ASM_DEFINE_CMPFAIL(7,
1251                         "movl %[rseq_scratch2], %[len]\n\t"
1252                         "movl %[rseq_scratch1], %[dst]\n\t"
1253                         "movl %[rseq_scratch0], %[src]\n\t",
1254                         error2)
1255 #endif
1256                 : /* gcc asm goto does not allow outputs */
1257                 : [cpu_id]              "r" (cpu),
1258                   [rseq_abi]            "r" (&__rseq_abi),
1259                   /* final store input */
1260                   [v]                   "m" (*v),
1261                   [expect]              "m" (expect),
1262                   [newv]                "m" (newv),
1263                   /* try memcpy input */
1264                   [dst]                 "r" (dst),
1265                   [src]                 "r" (src),
1266                   [len]                 "r" (len),
1267                   [rseq_scratch0]       "m" (rseq_scratch[0]),
1268                   [rseq_scratch1]       "m" (rseq_scratch[1]),
1269                   [rseq_scratch2]       "m" (rseq_scratch[2])
1270                 : "memory", "cc", "eax"
1271                   RSEQ_INJECT_CLOBBER
1272                 : abort, cmpfail
1273 #ifdef RSEQ_COMPARE_TWICE
1274                   , error1, error2
1275 #endif
1276         );
1277         return 0;
1278 abort:
1279         RSEQ_INJECT_FAILED
1280         return -1;
1281 cmpfail:
1282         return 1;
1283 #ifdef RSEQ_COMPARE_TWICE
1284 error1:
1285         rseq_bug("cpu_id comparison failed");
1286 error2:
1287         rseq_bug("expected value comparison failed");
1288 #endif
1289 }
1290
1291 #endif /* !RSEQ_SKIP_FASTPATH */
1292
1293 #endif