tools/testing/selftests/rseq/rseq-x86.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-x86.h
   4  *
   5  * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 #include <stdint.h>
   9
  10 /*
  11  * RSEQ_SIG is used with the following reserved undefined instructions, which
  12  * trap in user-space:
  13  *
  14  * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
  15  * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
  16  */
  17 #define RSEQ_SIG        0x53053053
  18
  19 /*
  20  * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
  21  * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
  22  * address through a "r" input operand.
  23  */
  24
  25 /* Offset of cpu_id and rseq_cs fields in struct rseq. */
  26 #define RSEQ_CPU_ID_OFFSET      4
  27 #define RSEQ_CS_OFFSET          8
  28
  29 #ifdef __x86_64__
  30
  31 #define rseq_smp_mb()   \
  32         __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
  33 #define rseq_smp_rmb()  rseq_barrier()
  34 #define rseq_smp_wmb()  rseq_barrier()
  35
  36 #define rseq_smp_load_acquire(p)                                        \
  37 __extension__ ({                                                        \
  38         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  39         rseq_barrier();                                                 \
  40         ____p1;                                                         \
  41 })
  42
  43 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  44
  45 #define rseq_smp_store_release(p, v)                                    \
  46 do {                                                                    \
  47         rseq_barrier();                                                 \
  48         RSEQ_WRITE_ONCE(*p, v);                                         \
  49 } while (0)
  50
  51 #ifdef RSEQ_SKIP_FASTPATH
  52 #include "rseq-skip.h"
  53 #else /* !RSEQ_SKIP_FASTPATH */
  54
  55 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  56                                 start_ip, post_commit_offset, abort_ip) \
  57                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  58                 ".balign 32\n\t"                                        \
  59                 __rseq_str(label) ":\n\t"                               \
  60                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  61                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  62                 ".popsection\n\t"                                       \
  63                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  64                 ".quad " __rseq_str(label) "b\n\t"                      \
  65                 ".popsection\n\t"
  66
  67
  68 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
  69         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
  70                                 (post_commit_ip - start_ip), abort_ip)
  71
  72 /*
  73  * Exit points of a rseq critical section consist of all instructions outside
  74  * of the critical section where a critical section can either branch to or
  75  * reach through the normal course of its execution. The abort IP and the
  76  * post-commit IP are already part of the __rseq_cs section and should not be
  77  * explicitly defined as additional exit points. Knowing all exit points is
  78  * useful to assist debuggers stepping over the critical section.
  79  */
  80 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  81                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  82                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  83                 ".popsection\n\t"
  84
  85 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
  86                 RSEQ_INJECT_ASM(1)                                      \
  87                 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"       \
  88                 "movq %%rax, " __rseq_str(rseq_cs) "\n\t"               \
  89                 __rseq_str(label) ":\n\t"
  90
  91 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
  92                 RSEQ_INJECT_ASM(2)                                      \
  93                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
  94                 "jnz " __rseq_str(label) "\n\t"
  95
  96 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
  97                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
  98                 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
  99                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 100                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 101                 __rseq_str(label) ":\n\t"                               \
 102                 teardown                                                \
 103                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 104                 ".popsection\n\t"
 105
 106 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 107                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 108                 __rseq_str(label) ":\n\t"                               \
 109                 teardown                                                \
 110                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 111                 ".popsection\n\t"
 112
 113 static inline __attribute__((always_inline))
 114 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 115 {
 116         RSEQ_INJECT_C(9)
 117
 118         __asm__ __volatile__ goto (
 119                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 120                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 121 #ifdef RSEQ_COMPARE_TWICE
 122                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 123                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 124 #endif
 125                 /* Start rseq by storing table entry pointer into rseq_cs. */
 126                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 127                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 128                 RSEQ_INJECT_ASM(3)
 129                 "cmpq %[v], %[expect]\n\t"
 130                 "jnz %l[cmpfail]\n\t"
 131                 RSEQ_INJECT_ASM(4)
 132 #ifdef RSEQ_COMPARE_TWICE
 133                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 134                 "cmpq %[v], %[expect]\n\t"
 135                 "jnz %l[error2]\n\t"
 136 #endif
 137                 /* final store */
 138                 "movq %[newv], %[v]\n\t"
 139                 "2:\n\t"
 140                 RSEQ_INJECT_ASM(5)
 141                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 142                 : /* gcc asm goto does not allow outputs */
 143                 : [cpu_id]              "r" (cpu),
 144                   [rseq_abi]            "r" (&__rseq_abi),
 145                   [v]                   "m" (*v),
 146                   [expect]              "r" (expect),
 147                   [newv]                "r" (newv)
 148                 : "memory", "cc", "rax"
 149                   RSEQ_INJECT_CLOBBER
 150                 : abort, cmpfail
 151 #ifdef RSEQ_COMPARE_TWICE
 152                   , error1, error2
 153 #endif
 154         );
 155         return 0;
 156 abort:
 157         RSEQ_INJECT_FAILED
 158         return -1;
 159 cmpfail:
 160         return 1;
 161 #ifdef RSEQ_COMPARE_TWICE
 162 error1:
 163         rseq_bug("cpu_id comparison failed");
 164 error2:
 165         rseq_bug("expected value comparison failed");
 166 #endif
 167 }
 168
 169 /*
 170  * Compare @v against @expectnot. When it does _not_ match, load @v
 171  * into @load, and store the content of *@v + voffp into @v.
 172  */
 173 static inline __attribute__((always_inline))
 174 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 175                                off_t voffp, intptr_t *load, int cpu)
 176 {
 177         RSEQ_INJECT_C(9)
 178
 179         __asm__ __volatile__ goto (
 180                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 181                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 182 #ifdef RSEQ_COMPARE_TWICE
 183                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 184                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 185 #endif
 186                 /* Start rseq by storing table entry pointer into rseq_cs. */
 187                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 188                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 189                 RSEQ_INJECT_ASM(3)
 190                 "movq %[v], %%rbx\n\t"
 191                 "cmpq %%rbx, %[expectnot]\n\t"
 192                 "je %l[cmpfail]\n\t"
 193                 RSEQ_INJECT_ASM(4)
 194 #ifdef RSEQ_COMPARE_TWICE
 195                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 196                 "movq %[v], %%rbx\n\t"
 197                 "cmpq %%rbx, %[expectnot]\n\t"
 198                 "je %l[error2]\n\t"
 199 #endif
 200                 "movq %%rbx, %[load]\n\t"
 201                 "addq %[voffp], %%rbx\n\t"
 202                 "movq (%%rbx), %%rbx\n\t"
 203                 /* final store */
 204                 "movq %%rbx, %[v]\n\t"
 205                 "2:\n\t"
 206                 RSEQ_INJECT_ASM(5)
 207                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 208                 : /* gcc asm goto does not allow outputs */
 209                 : [cpu_id]              "r" (cpu),
 210                   [rseq_abi]            "r" (&__rseq_abi),
 211                   /* final store input */
 212                   [v]                   "m" (*v),
 213                   [expectnot]           "r" (expectnot),
 214                   [voffp]               "er" (voffp),
 215                   [load]                "m" (*load)
 216                 : "memory", "cc", "rax", "rbx"
 217                   RSEQ_INJECT_CLOBBER
 218                 : abort, cmpfail
 219 #ifdef RSEQ_COMPARE_TWICE
 220                   , error1, error2
 221 #endif
 222         );
 223         return 0;
 224 abort:
 225         RSEQ_INJECT_FAILED
 226         return -1;
 227 cmpfail:
 228         return 1;
 229 #ifdef RSEQ_COMPARE_TWICE
 230 error1:
 231         rseq_bug("cpu_id comparison failed");
 232 error2:
 233         rseq_bug("expected value comparison failed");
 234 #endif
 235 }
 236
 237 static inline __attribute__((always_inline))
 238 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 239 {
 240         RSEQ_INJECT_C(9)
 241
 242         __asm__ __volatile__ goto (
 243                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 244 #ifdef RSEQ_COMPARE_TWICE
 245                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 246 #endif
 247                 /* Start rseq by storing table entry pointer into rseq_cs. */
 248                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 249                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 250                 RSEQ_INJECT_ASM(3)
 251 #ifdef RSEQ_COMPARE_TWICE
 252                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 253 #endif
 254                 /* final store */
 255                 "addq %[count], %[v]\n\t"
 256                 "2:\n\t"
 257                 RSEQ_INJECT_ASM(4)
 258                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 259                 : /* gcc asm goto does not allow outputs */
 260                 : [cpu_id]              "r" (cpu),
 261                   [rseq_abi]            "r" (&__rseq_abi),
 262                   /* final store input */
 263                   [v]                   "m" (*v),
 264                   [count]               "er" (count)
 265                 : "memory", "cc", "rax"
 266                   RSEQ_INJECT_CLOBBER
 267                 : abort
 268 #ifdef RSEQ_COMPARE_TWICE
 269                   , error1
 270 #endif
 271         );
 272         return 0;
 273 abort:
 274         RSEQ_INJECT_FAILED
 275         return -1;
 276 #ifdef RSEQ_COMPARE_TWICE
 277 error1:
 278         rseq_bug("cpu_id comparison failed");
 279 #endif
 280 }
 281
 282 static inline __attribute__((always_inline))
 283 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 284                                  intptr_t *v2, intptr_t newv2,
 285                                  intptr_t newv, int cpu)
 286 {
 287         RSEQ_INJECT_C(9)
 288
 289         __asm__ __volatile__ goto (
 290                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 291                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 292 #ifdef RSEQ_COMPARE_TWICE
 293                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 294                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 295 #endif
 296                 /* Start rseq by storing table entry pointer into rseq_cs. */
 297                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 298                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 299                 RSEQ_INJECT_ASM(3)
 300                 "cmpq %[v], %[expect]\n\t"
 301                 "jnz %l[cmpfail]\n\t"
 302                 RSEQ_INJECT_ASM(4)
 303 #ifdef RSEQ_COMPARE_TWICE
 304                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 305                 "cmpq %[v], %[expect]\n\t"
 306                 "jnz %l[error2]\n\t"
 307 #endif
 308                 /* try store */
 309                 "movq %[newv2], %[v2]\n\t"
 310                 RSEQ_INJECT_ASM(5)
 311                 /* final store */
 312                 "movq %[newv], %[v]\n\t"
 313                 "2:\n\t"
 314                 RSEQ_INJECT_ASM(6)
 315                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 316                 : /* gcc asm goto does not allow outputs */
 317                 : [cpu_id]              "r" (cpu),
 318                   [rseq_abi]            "r" (&__rseq_abi),
 319                   /* try store input */
 320                   [v2]                  "m" (*v2),
 321                   [newv2]               "r" (newv2),
 322                   /* final store input */
 323                   [v]                   "m" (*v),
 324                   [expect]              "r" (expect),
 325                   [newv]                "r" (newv)
 326                 : "memory", "cc", "rax"
 327                   RSEQ_INJECT_CLOBBER
 328                 : abort, cmpfail
 329 #ifdef RSEQ_COMPARE_TWICE
 330                   , error1, error2
 331 #endif
 332         );
 333         return 0;
 334 abort:
 335         RSEQ_INJECT_FAILED
 336         return -1;
 337 cmpfail:
 338         return 1;
 339 #ifdef RSEQ_COMPARE_TWICE
 340 error1:
 341         rseq_bug("cpu_id comparison failed");
 342 error2:
 343         rseq_bug("expected value comparison failed");
 344 #endif
 345 }
 346
 347 /* x86-64 is TSO. */
 348 static inline __attribute__((always_inline))
 349 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 350                                          intptr_t *v2, intptr_t newv2,
 351                                          intptr_t newv, int cpu)
 352 {
 353         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 354 }
 355
 356 static inline __attribute__((always_inline))
 357 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 358                               intptr_t *v2, intptr_t expect2,
 359                               intptr_t newv, int cpu)
 360 {
 361         RSEQ_INJECT_C(9)
 362
 363         __asm__ __volatile__ goto (
 364                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 365                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 366 #ifdef RSEQ_COMPARE_TWICE
 367                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 368                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 369                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 370 #endif
 371                 /* Start rseq by storing table entry pointer into rseq_cs. */
 372                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 373                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 374                 RSEQ_INJECT_ASM(3)
 375                 "cmpq %[v], %[expect]\n\t"
 376                 "jnz %l[cmpfail]\n\t"
 377                 RSEQ_INJECT_ASM(4)
 378                 "cmpq %[v2], %[expect2]\n\t"
 379                 "jnz %l[cmpfail]\n\t"
 380                 RSEQ_INJECT_ASM(5)
 381 #ifdef RSEQ_COMPARE_TWICE
 382                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 383                 "cmpq %[v], %[expect]\n\t"
 384                 "jnz %l[error2]\n\t"
 385                 "cmpq %[v2], %[expect2]\n\t"
 386                 "jnz %l[error3]\n\t"
 387 #endif
 388                 /* final store */
 389                 "movq %[newv], %[v]\n\t"
 390                 "2:\n\t"
 391                 RSEQ_INJECT_ASM(6)
 392                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 393                 : /* gcc asm goto does not allow outputs */
 394                 : [cpu_id]              "r" (cpu),
 395                   [rseq_abi]            "r" (&__rseq_abi),
 396                   /* cmp2 input */
 397                   [v2]                  "m" (*v2),
 398                   [expect2]             "r" (expect2),
 399                   /* final store input */
 400                   [v]                   "m" (*v),
 401                   [expect]              "r" (expect),
 402                   [newv]                "r" (newv)
 403                 : "memory", "cc", "rax"
 404                   RSEQ_INJECT_CLOBBER
 405                 : abort, cmpfail
 406 #ifdef RSEQ_COMPARE_TWICE
 407                   , error1, error2, error3
 408 #endif
 409         );
 410         return 0;
 411 abort:
 412         RSEQ_INJECT_FAILED
 413         return -1;
 414 cmpfail:
 415         return 1;
 416 #ifdef RSEQ_COMPARE_TWICE
 417 error1:
 418         rseq_bug("cpu_id comparison failed");
 419 error2:
 420         rseq_bug("1st expected value comparison failed");
 421 error3:
 422         rseq_bug("2nd expected value comparison failed");
 423 #endif
 424 }
 425
 426 static inline __attribute__((always_inline))
 427 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 428                                  void *dst, void *src, size_t len,
 429                                  intptr_t newv, int cpu)
 430 {
 431         uint64_t rseq_scratch[3];
 432
 433         RSEQ_INJECT_C(9)
 434
 435         __asm__ __volatile__ goto (
 436                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 437                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 438 #ifdef RSEQ_COMPARE_TWICE
 439                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 440                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 441 #endif
 442                 "movq %[src], %[rseq_scratch0]\n\t"
 443                 "movq %[dst], %[rseq_scratch1]\n\t"
 444                 "movq %[len], %[rseq_scratch2]\n\t"
 445                 /* Start rseq by storing table entry pointer into rseq_cs. */
 446                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 447                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 448                 RSEQ_INJECT_ASM(3)
 449                 "cmpq %[v], %[expect]\n\t"
 450                 "jnz 5f\n\t"
 451                 RSEQ_INJECT_ASM(4)
 452 #ifdef RSEQ_COMPARE_TWICE
 453                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
 454                 "cmpq %[v], %[expect]\n\t"
 455                 "jnz 7f\n\t"
 456 #endif
 457                 /* try memcpy */
 458                 "test %[len], %[len]\n\t" \
 459                 "jz 333f\n\t" \
 460                 "222:\n\t" \
 461                 "movb (%[src]), %%al\n\t" \
 462                 "movb %%al, (%[dst])\n\t" \
 463                 "inc %[src]\n\t" \
 464                 "inc %[dst]\n\t" \
 465                 "dec %[len]\n\t" \
 466                 "jnz 222b\n\t" \
 467                 "333:\n\t" \
 468                 RSEQ_INJECT_ASM(5)
 469                 /* final store */
 470                 "movq %[newv], %[v]\n\t"
 471                 "2:\n\t"
 472                 RSEQ_INJECT_ASM(6)
 473                 /* teardown */
 474                 "movq %[rseq_scratch2], %[len]\n\t"
 475                 "movq %[rseq_scratch1], %[dst]\n\t"
 476                 "movq %[rseq_scratch0], %[src]\n\t"
 477                 RSEQ_ASM_DEFINE_ABORT(4,
 478                         "movq %[rseq_scratch2], %[len]\n\t"
 479                         "movq %[rseq_scratch1], %[dst]\n\t"
 480                         "movq %[rseq_scratch0], %[src]\n\t",
 481                         abort)
 482                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 483                         "movq %[rseq_scratch2], %[len]\n\t"
 484                         "movq %[rseq_scratch1], %[dst]\n\t"
 485                         "movq %[rseq_scratch0], %[src]\n\t",
 486                         cmpfail)
 487 #ifdef RSEQ_COMPARE_TWICE
 488                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 489                         "movq %[rseq_scratch2], %[len]\n\t"
 490                         "movq %[rseq_scratch1], %[dst]\n\t"
 491                         "movq %[rseq_scratch0], %[src]\n\t",
 492                         error1)
 493                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 494                         "movq %[rseq_scratch2], %[len]\n\t"
 495                         "movq %[rseq_scratch1], %[dst]\n\t"
 496                         "movq %[rseq_scratch0], %[src]\n\t",
 497                         error2)
 498 #endif
 499                 : /* gcc asm goto does not allow outputs */
 500                 : [cpu_id]              "r" (cpu),
 501                   [rseq_abi]            "r" (&__rseq_abi),
 502                   /* final store input */
 503                   [v]                   "m" (*v),
 504                   [expect]              "r" (expect),
 505                   [newv]                "r" (newv),
 506                   /* try memcpy input */
 507                   [dst]                 "r" (dst),
 508                   [src]                 "r" (src),
 509                   [len]                 "r" (len),
 510                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 511                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 512                   [rseq_scratch2]       "m" (rseq_scratch[2])
 513                 : "memory", "cc", "rax"
 514                   RSEQ_INJECT_CLOBBER
 515                 : abort, cmpfail
 516 #ifdef RSEQ_COMPARE_TWICE
 517                   , error1, error2
 518 #endif
 519         );
 520         return 0;
 521 abort:
 522         RSEQ_INJECT_FAILED
 523         return -1;
 524 cmpfail:
 525         return 1;
 526 #ifdef RSEQ_COMPARE_TWICE
 527 error1:
 528         rseq_bug("cpu_id comparison failed");
 529 error2:
 530         rseq_bug("expected value comparison failed");
 531 #endif
 532 }
 533
 534 /* x86-64 is TSO. */
 535 static inline __attribute__((always_inline))
 536 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 537                                          void *dst, void *src, size_t len,
 538                                          intptr_t newv, int cpu)
 539 {
 540         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 541                                             newv, cpu);
 542 }
 543
 544 #endif /* !RSEQ_SKIP_FASTPATH */
 545
 546 #elif __i386__
 547
 548 #define rseq_smp_mb()   \
 549         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 550 #define rseq_smp_rmb()  \
 551         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 552 #define rseq_smp_wmb()  \
 553         __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 554
 555 #define rseq_smp_load_acquire(p)                                        \
 556 __extension__ ({                                                        \
 557         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
 558         rseq_smp_mb();                                                  \
 559         ____p1;                                                         \
 560 })
 561
 562 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
 563
 564 #define rseq_smp_store_release(p, v)                                    \
 565 do {                                                                    \
 566         rseq_smp_mb();                                                  \
 567         RSEQ_WRITE_ONCE(*p, v);                                         \
 568 } while (0)
 569
 570 #ifdef RSEQ_SKIP_FASTPATH
 571 #include "rseq-skip.h"
 572 #else /* !RSEQ_SKIP_FASTPATH */
 573
 574 /*
 575  * Use eax as scratch register and take memory operands as input to
 576  * lessen register pressure. Especially needed when compiling in O0.
 577  */
 578 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 579                                 start_ip, post_commit_offset, abort_ip) \
 580                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 581                 ".balign 32\n\t"                                        \
 582                 __rseq_str(label) ":\n\t"                               \
 583                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 584                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
 585                 ".popsection\n\t"                                       \
 586                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 587                 ".long " __rseq_str(label) "b, 0x0\n\t"                 \
 588                 ".popsection\n\t"
 589
 590 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 591         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 592                                 (post_commit_ip - start_ip), abort_ip)
 593
 594 /*
 595  * Exit points of a rseq critical section consist of all instructions outside
 596  * of the critical section where a critical section can either branch to or
 597  * reach through the normal course of its execution. The abort IP and the
 598  * post-commit IP are already part of the __rseq_cs section and should not be
 599  * explicitly defined as additional exit points. Knowing all exit points is
 600  * useful to assist debuggers stepping over the critical section.
 601  */
 602 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 603                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 604                 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
 605                 ".popsection\n\t"
 606
 607 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 608                 RSEQ_INJECT_ASM(1)                                      \
 609                 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"   \
 610                 __rseq_str(label) ":\n\t"
 611
 612 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 613                 RSEQ_INJECT_ASM(2)                                      \
 614                 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
 615                 "jnz " __rseq_str(label) "\n\t"
 616
 617 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 618                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 619                 /* Disassembler-friendly signature: ud1 <sig>,%edi. */  \
 620                 ".byte 0x0f, 0xb9, 0x3d\n\t"                            \
 621                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 622                 __rseq_str(label) ":\n\t"                               \
 623                 teardown                                                \
 624                 "jmp %l[" __rseq_str(abort_label) "]\n\t"               \
 625                 ".popsection\n\t"
 626
 627 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 628                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 629                 __rseq_str(label) ":\n\t"                               \
 630                 teardown                                                \
 631                 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t"             \
 632                 ".popsection\n\t"
 633
 634 static inline __attribute__((always_inline))
 635 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 636 {
 637         RSEQ_INJECT_C(9)
 638
 639         __asm__ __volatile__ goto (
 640                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 641                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 642 #ifdef RSEQ_COMPARE_TWICE
 643                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 644                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 645 #endif
 646                 /* Start rseq by storing table entry pointer into rseq_cs. */
 647                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 648                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 649                 RSEQ_INJECT_ASM(3)
 650                 "cmpl %[v], %[expect]\n\t"
 651                 "jnz %l[cmpfail]\n\t"
 652                 RSEQ_INJECT_ASM(4)
 653 #ifdef RSEQ_COMPARE_TWICE
 654                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 655                 "cmpl %[v], %[expect]\n\t"
 656                 "jnz %l[error2]\n\t"
 657 #endif
 658                 /* final store */
 659                 "movl %[newv], %[v]\n\t"
 660                 "2:\n\t"
 661                 RSEQ_INJECT_ASM(5)
 662                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 663                 : /* gcc asm goto does not allow outputs */
 664                 : [cpu_id]              "r" (cpu),
 665                   [rseq_abi]            "r" (&__rseq_abi),
 666                   [v]                   "m" (*v),
 667                   [expect]              "r" (expect),
 668                   [newv]                "r" (newv)
 669                 : "memory", "cc", "eax"
 670                   RSEQ_INJECT_CLOBBER
 671                 : abort, cmpfail
 672 #ifdef RSEQ_COMPARE_TWICE
 673                   , error1, error2
 674 #endif
 675         );
 676         return 0;
 677 abort:
 678         RSEQ_INJECT_FAILED
 679         return -1;
 680 cmpfail:
 681         return 1;
 682 #ifdef RSEQ_COMPARE_TWICE
 683 error1:
 684         rseq_bug("cpu_id comparison failed");
 685 error2:
 686         rseq_bug("expected value comparison failed");
 687 #endif
 688 }
 689
 690 /*
 691  * Compare @v against @expectnot. When it does _not_ match, load @v
 692  * into @load, and store the content of *@v + voffp into @v.
 693  */
 694 static inline __attribute__((always_inline))
 695 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 696                                off_t voffp, intptr_t *load, int cpu)
 697 {
 698         RSEQ_INJECT_C(9)
 699
 700         __asm__ __volatile__ goto (
 701                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 702                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 703 #ifdef RSEQ_COMPARE_TWICE
 704                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 705                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 706 #endif
 707                 /* Start rseq by storing table entry pointer into rseq_cs. */
 708                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 709                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 710                 RSEQ_INJECT_ASM(3)
 711                 "movl %[v], %%ebx\n\t"
 712                 "cmpl %%ebx, %[expectnot]\n\t"
 713                 "je %l[cmpfail]\n\t"
 714                 RSEQ_INJECT_ASM(4)
 715 #ifdef RSEQ_COMPARE_TWICE
 716                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 717                 "movl %[v], %%ebx\n\t"
 718                 "cmpl %%ebx, %[expectnot]\n\t"
 719                 "je %l[error2]\n\t"
 720 #endif
 721                 "movl %%ebx, %[load]\n\t"
 722                 "addl %[voffp], %%ebx\n\t"
 723                 "movl (%%ebx), %%ebx\n\t"
 724                 /* final store */
 725                 "movl %%ebx, %[v]\n\t"
 726                 "2:\n\t"
 727                 RSEQ_INJECT_ASM(5)
 728                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 729                 : /* gcc asm goto does not allow outputs */
 730                 : [cpu_id]              "r" (cpu),
 731                   [rseq_abi]            "r" (&__rseq_abi),
 732                   /* final store input */
 733                   [v]                   "m" (*v),
 734                   [expectnot]           "r" (expectnot),
 735                   [voffp]               "ir" (voffp),
 736                   [load]                "m" (*load)
 737                 : "memory", "cc", "eax", "ebx"
 738                   RSEQ_INJECT_CLOBBER
 739                 : abort, cmpfail
 740 #ifdef RSEQ_COMPARE_TWICE
 741                   , error1, error2
 742 #endif
 743         );
 744         return 0;
 745 abort:
 746         RSEQ_INJECT_FAILED
 747         return -1;
 748 cmpfail:
 749         return 1;
 750 #ifdef RSEQ_COMPARE_TWICE
 751 error1:
 752         rseq_bug("cpu_id comparison failed");
 753 error2:
 754         rseq_bug("expected value comparison failed");
 755 #endif
 756 }
 757
 758 static inline __attribute__((always_inline))
 759 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 760 {
 761         RSEQ_INJECT_C(9)
 762
 763         __asm__ __volatile__ goto (
 764                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 765 #ifdef RSEQ_COMPARE_TWICE
 766                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 767 #endif
 768                 /* Start rseq by storing table entry pointer into rseq_cs. */
 769                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 770                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 771                 RSEQ_INJECT_ASM(3)
 772 #ifdef RSEQ_COMPARE_TWICE
 773                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 774 #endif
 775                 /* final store */
 776                 "addl %[count], %[v]\n\t"
 777                 "2:\n\t"
 778                 RSEQ_INJECT_ASM(4)
 779                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 780                 : /* gcc asm goto does not allow outputs */
 781                 : [cpu_id]              "r" (cpu),
 782                   [rseq_abi]            "r" (&__rseq_abi),
 783                   /* final store input */
 784                   [v]                   "m" (*v),
 785                   [count]               "ir" (count)
 786                 : "memory", "cc", "eax"
 787                   RSEQ_INJECT_CLOBBER
 788                 : abort
 789 #ifdef RSEQ_COMPARE_TWICE
 790                   , error1
 791 #endif
 792         );
 793         return 0;
 794 abort:
 795         RSEQ_INJECT_FAILED
 796         return -1;
 797 #ifdef RSEQ_COMPARE_TWICE
 798 error1:
 799         rseq_bug("cpu_id comparison failed");
 800 #endif
 801 }
 802
 803 static inline __attribute__((always_inline))
 804 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 805                                  intptr_t *v2, intptr_t newv2,
 806                                  intptr_t newv, int cpu)
 807 {
 808         RSEQ_INJECT_C(9)
 809
 810         __asm__ __volatile__ goto (
 811                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 812                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 813 #ifdef RSEQ_COMPARE_TWICE
 814                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 815                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 816 #endif
 817                 /* Start rseq by storing table entry pointer into rseq_cs. */
 818                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 819                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 820                 RSEQ_INJECT_ASM(3)
 821                 "cmpl %[v], %[expect]\n\t"
 822                 "jnz %l[cmpfail]\n\t"
 823                 RSEQ_INJECT_ASM(4)
 824 #ifdef RSEQ_COMPARE_TWICE
 825                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 826                 "cmpl %[v], %[expect]\n\t"
 827                 "jnz %l[error2]\n\t"
 828 #endif
 829                 /* try store */
 830                 "movl %[newv2], %%eax\n\t"
 831                 "movl %%eax, %[v2]\n\t"
 832                 RSEQ_INJECT_ASM(5)
 833                 /* final store */
 834                 "movl %[newv], %[v]\n\t"
 835                 "2:\n\t"
 836                 RSEQ_INJECT_ASM(6)
 837                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 838                 : /* gcc asm goto does not allow outputs */
 839                 : [cpu_id]              "r" (cpu),
 840                   [rseq_abi]            "r" (&__rseq_abi),
 841                   /* try store input */
 842                   [v2]                  "m" (*v2),
 843                   [newv2]               "m" (newv2),
 844                   /* final store input */
 845                   [v]                   "m" (*v),
 846                   [expect]              "r" (expect),
 847                   [newv]                "r" (newv)
 848                 : "memory", "cc", "eax"
 849                   RSEQ_INJECT_CLOBBER
 850                 : abort, cmpfail
 851 #ifdef RSEQ_COMPARE_TWICE
 852                   , error1, error2
 853 #endif
 854         );
 855         return 0;
 856 abort:
 857         RSEQ_INJECT_FAILED
 858         return -1;
 859 cmpfail:
 860         return 1;
 861 #ifdef RSEQ_COMPARE_TWICE
 862 error1:
 863         rseq_bug("cpu_id comparison failed");
 864 error2:
 865         rseq_bug("expected value comparison failed");
 866 #endif
 867 }
 868
 869 static inline __attribute__((always_inline))
 870 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 871                                          intptr_t *v2, intptr_t newv2,
 872                                          intptr_t newv, int cpu)
 873 {
 874         RSEQ_INJECT_C(9)
 875
 876         __asm__ __volatile__ goto (
 877                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 878                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 879 #ifdef RSEQ_COMPARE_TWICE
 880                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 881                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 882 #endif
 883                 /* Start rseq by storing table entry pointer into rseq_cs. */
 884                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 885                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 886                 RSEQ_INJECT_ASM(3)
 887                 "movl %[expect], %%eax\n\t"
 888                 "cmpl %[v], %%eax\n\t"
 889                 "jnz %l[cmpfail]\n\t"
 890                 RSEQ_INJECT_ASM(4)
 891 #ifdef RSEQ_COMPARE_TWICE
 892                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 893                 "movl %[expect], %%eax\n\t"
 894                 "cmpl %[v], %%eax\n\t"
 895                 "jnz %l[error2]\n\t"
 896 #endif
 897                 /* try store */
 898                 "movl %[newv2], %[v2]\n\t"
 899                 RSEQ_INJECT_ASM(5)
 900                 "lock; addl $0,-128(%%esp)\n\t"
 901                 /* final store */
 902                 "movl %[newv], %[v]\n\t"
 903                 "2:\n\t"
 904                 RSEQ_INJECT_ASM(6)
 905                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 906                 : /* gcc asm goto does not allow outputs */
 907                 : [cpu_id]              "r" (cpu),
 908                   [rseq_abi]            "r" (&__rseq_abi),
 909                   /* try store input */
 910                   [v2]                  "m" (*v2),
 911                   [newv2]               "r" (newv2),
 912                   /* final store input */
 913                   [v]                   "m" (*v),
 914                   [expect]              "m" (expect),
 915                   [newv]                "r" (newv)
 916                 : "memory", "cc", "eax"
 917                   RSEQ_INJECT_CLOBBER
 918                 : abort, cmpfail
 919 #ifdef RSEQ_COMPARE_TWICE
 920                   , error1, error2
 921 #endif
 922         );
 923         return 0;
 924 abort:
 925         RSEQ_INJECT_FAILED
 926         return -1;
 927 cmpfail:
 928         return 1;
 929 #ifdef RSEQ_COMPARE_TWICE
 930 error1:
 931         rseq_bug("cpu_id comparison failed");
 932 error2:
 933         rseq_bug("expected value comparison failed");
 934 #endif
 935
 936 }
 937
 938 static inline __attribute__((always_inline))
 939 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 940                               intptr_t *v2, intptr_t expect2,
 941                               intptr_t newv, int cpu)
 942 {
 943         RSEQ_INJECT_C(9)
 944
 945         __asm__ __volatile__ goto (
 946                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 947                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 948 #ifdef RSEQ_COMPARE_TWICE
 949                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 950                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 951                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 952 #endif
 953                 /* Start rseq by storing table entry pointer into rseq_cs. */
 954                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
 955                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
 956                 RSEQ_INJECT_ASM(3)
 957                 "cmpl %[v], %[expect]\n\t"
 958                 "jnz %l[cmpfail]\n\t"
 959                 RSEQ_INJECT_ASM(4)
 960                 "cmpl %[expect2], %[v2]\n\t"
 961                 "jnz %l[cmpfail]\n\t"
 962                 RSEQ_INJECT_ASM(5)
 963 #ifdef RSEQ_COMPARE_TWICE
 964                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
 965                 "cmpl %[v], %[expect]\n\t"
 966                 "jnz %l[error2]\n\t"
 967                 "cmpl %[expect2], %[v2]\n\t"
 968                 "jnz %l[error3]\n\t"
 969 #endif
 970                 "movl %[newv], %%eax\n\t"
 971                 /* final store */
 972                 "movl %%eax, %[v]\n\t"
 973                 "2:\n\t"
 974                 RSEQ_INJECT_ASM(6)
 975                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 976                 : /* gcc asm goto does not allow outputs */
 977                 : [cpu_id]              "r" (cpu),
 978                   [rseq_abi]            "r" (&__rseq_abi),
 979                   /* cmp2 input */
 980                   [v2]                  "m" (*v2),
 981                   [expect2]             "r" (expect2),
 982                   /* final store input */
 983                   [v]                   "m" (*v),
 984                   [expect]              "r" (expect),
 985                   [newv]                "m" (newv)
 986                 : "memory", "cc", "eax"
 987                   RSEQ_INJECT_CLOBBER
 988                 : abort, cmpfail
 989 #ifdef RSEQ_COMPARE_TWICE
 990                   , error1, error2, error3
 991 #endif
 992         );
 993         return 0;
 994 abort:
 995         RSEQ_INJECT_FAILED
 996         return -1;
 997 cmpfail:
 998         return 1;
 999 #ifdef RSEQ_COMPARE_TWICE
1000 error1:
1001         rseq_bug("cpu_id comparison failed");
1002 error2:
1003         rseq_bug("1st expected value comparison failed");
1004 error3:
1005         rseq_bug("2nd expected value comparison failed");
1006 #endif
1007 }
1008
1009 /* TODO: implement a faster memcpy. */
1010 static inline __attribute__((always_inline))
1011 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1012                                  void *dst, void *src, size_t len,
1013                                  intptr_t newv, int cpu)
1014 {
1015         uint32_t rseq_scratch[3];
1016
1017         RSEQ_INJECT_C(9)
1018
1019         __asm__ __volatile__ goto (
1020                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1021                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1022 #ifdef RSEQ_COMPARE_TWICE
1023                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1024                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1025 #endif
1026                 "movl %[src], %[rseq_scratch0]\n\t"
1027                 "movl %[dst], %[rseq_scratch1]\n\t"
1028                 "movl %[len], %[rseq_scratch2]\n\t"
1029                 /* Start rseq by storing table entry pointer into rseq_cs. */
1030                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1031                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1032                 RSEQ_INJECT_ASM(3)
1033                 "movl %[expect], %%eax\n\t"
1034                 "cmpl %%eax, %[v]\n\t"
1035                 "jnz 5f\n\t"
1036                 RSEQ_INJECT_ASM(4)
1037 #ifdef RSEQ_COMPARE_TWICE
1038                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1039                 "movl %[expect], %%eax\n\t"
1040                 "cmpl %%eax, %[v]\n\t"
1041                 "jnz 7f\n\t"
1042 #endif
1043                 /* try memcpy */
1044                 "test %[len], %[len]\n\t" \
1045                 "jz 333f\n\t" \
1046                 "222:\n\t" \
1047                 "movb (%[src]), %%al\n\t" \
1048                 "movb %%al, (%[dst])\n\t" \
1049                 "inc %[src]\n\t" \
1050                 "inc %[dst]\n\t" \
1051                 "dec %[len]\n\t" \
1052                 "jnz 222b\n\t" \
1053                 "333:\n\t" \
1054                 RSEQ_INJECT_ASM(5)
1055                 "movl %[newv], %%eax\n\t"
1056                 /* final store */
1057                 "movl %%eax, %[v]\n\t"
1058                 "2:\n\t"
1059                 RSEQ_INJECT_ASM(6)
1060                 /* teardown */
1061                 "movl %[rseq_scratch2], %[len]\n\t"
1062                 "movl %[rseq_scratch1], %[dst]\n\t"
1063                 "movl %[rseq_scratch0], %[src]\n\t"
1064                 RSEQ_ASM_DEFINE_ABORT(4,
1065                         "movl %[rseq_scratch2], %[len]\n\t"
1066                         "movl %[rseq_scratch1], %[dst]\n\t"
1067                         "movl %[rseq_scratch0], %[src]\n\t",
1068                         abort)
1069                 RSEQ_ASM_DEFINE_CMPFAIL(5,
1070                         "movl %[rseq_scratch2], %[len]\n\t"
1071                         "movl %[rseq_scratch1], %[dst]\n\t"
1072                         "movl %[rseq_scratch0], %[src]\n\t",
1073                         cmpfail)
1074 #ifdef RSEQ_COMPARE_TWICE
1075                 RSEQ_ASM_DEFINE_CMPFAIL(6,
1076                         "movl %[rseq_scratch2], %[len]\n\t"
1077                         "movl %[rseq_scratch1], %[dst]\n\t"
1078                         "movl %[rseq_scratch0], %[src]\n\t",
1079                         error1)
1080                 RSEQ_ASM_DEFINE_CMPFAIL(7,
1081                         "movl %[rseq_scratch2], %[len]\n\t"
1082                         "movl %[rseq_scratch1], %[dst]\n\t"
1083                         "movl %[rseq_scratch0], %[src]\n\t",
1084                         error2)
1085 #endif
1086                 : /* gcc asm goto does not allow outputs */
1087                 : [cpu_id]              "r" (cpu),
1088                   [rseq_abi]            "r" (&__rseq_abi),
1089                   /* final store input */
1090                   [v]                   "m" (*v),
1091                   [expect]              "m" (expect),
1092                   [newv]                "m" (newv),
1093                   /* try memcpy input */
1094                   [dst]                 "r" (dst),
1095                   [src]                 "r" (src),
1096                   [len]                 "r" (len),
1097                   [rseq_scratch0]       "m" (rseq_scratch[0]),
1098                   [rseq_scratch1]       "m" (rseq_scratch[1]),
1099                   [rseq_scratch2]       "m" (rseq_scratch[2])
1100                 : "memory", "cc", "eax"
1101                   RSEQ_INJECT_CLOBBER
1102                 : abort, cmpfail
1103 #ifdef RSEQ_COMPARE_TWICE
1104                   , error1, error2
1105 #endif
1106         );
1107         return 0;
1108 abort:
1109         RSEQ_INJECT_FAILED
1110         return -1;
1111 cmpfail:
1112         return 1;
1113 #ifdef RSEQ_COMPARE_TWICE
1114 error1:
1115         rseq_bug("cpu_id comparison failed");
1116 error2:
1117         rseq_bug("expected value comparison failed");
1118 #endif
1119 }
1120
1121 /* TODO: implement a faster memcpy. */
1122 static inline __attribute__((always_inline))
1123 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1124                                          void *dst, void *src, size_t len,
1125                                          intptr_t newv, int cpu)
1126 {
1127         uint32_t rseq_scratch[3];
1128
1129         RSEQ_INJECT_C(9)
1130
1131         __asm__ __volatile__ goto (
1132                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1133                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1134 #ifdef RSEQ_COMPARE_TWICE
1135                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1136                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1137 #endif
1138                 "movl %[src], %[rseq_scratch0]\n\t"
1139                 "movl %[dst], %[rseq_scratch1]\n\t"
1140                 "movl %[len], %[rseq_scratch2]\n\t"
1141                 /* Start rseq by storing table entry pointer into rseq_cs. */
1142                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1143                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1144                 RSEQ_INJECT_ASM(3)
1145                 "movl %[expect], %%eax\n\t"
1146                 "cmpl %%eax, %[v]\n\t"
1147                 "jnz 5f\n\t"
1148                 RSEQ_INJECT_ASM(4)
1149 #ifdef RSEQ_COMPARE_TWICE
1150                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1151                 "movl %[expect], %%eax\n\t"
1152                 "cmpl %%eax, %[v]\n\t"
1153                 "jnz 7f\n\t"
1154 #endif
1155                 /* try memcpy */
1156                 "test %[len], %[len]\n\t" \
1157                 "jz 333f\n\t" \
1158                 "222:\n\t" \
1159                 "movb (%[src]), %%al\n\t" \
1160                 "movb %%al, (%[dst])\n\t" \
1161                 "inc %[src]\n\t" \
1162                 "inc %[dst]\n\t" \
1163                 "dec %[len]\n\t" \
1164                 "jnz 222b\n\t" \
1165                 "333:\n\t" \
1166                 RSEQ_INJECT_ASM(5)
1167                 "lock; addl $0,-128(%%esp)\n\t"
1168                 "movl %[newv], %%eax\n\t"
1169                 /* final store */
1170                 "movl %%eax, %[v]\n\t"
1171                 "2:\n\t"
1172                 RSEQ_INJECT_ASM(6)
1173                 /* teardown */
1174                 "movl %[rseq_scratch2], %[len]\n\t"
1175                 "movl %[rseq_scratch1], %[dst]\n\t"
1176                 "movl %[rseq_scratch0], %[src]\n\t"
1177                 RSEQ_ASM_DEFINE_ABORT(4,
1178                         "movl %[rseq_scratch2], %[len]\n\t"
1179                         "movl %[rseq_scratch1], %[dst]\n\t"
1180                         "movl %[rseq_scratch0], %[src]\n\t",
1181                         abort)
1182                 RSEQ_ASM_DEFINE_CMPFAIL(5,
1183                         "movl %[rseq_scratch2], %[len]\n\t"
1184                         "movl %[rseq_scratch1], %[dst]\n\t"
1185                         "movl %[rseq_scratch0], %[src]\n\t",
1186                         cmpfail)
1187 #ifdef RSEQ_COMPARE_TWICE
1188                 RSEQ_ASM_DEFINE_CMPFAIL(6,
1189                         "movl %[rseq_scratch2], %[len]\n\t"
1190                         "movl %[rseq_scratch1], %[dst]\n\t"
1191                         "movl %[rseq_scratch0], %[src]\n\t",
1192                         error1)
1193                 RSEQ_ASM_DEFINE_CMPFAIL(7,
1194                         "movl %[rseq_scratch2], %[len]\n\t"
1195                         "movl %[rseq_scratch1], %[dst]\n\t"
1196                         "movl %[rseq_scratch0], %[src]\n\t",
1197                         error2)
1198 #endif
1199                 : /* gcc asm goto does not allow outputs */
1200                 : [cpu_id]              "r" (cpu),
1201                   [rseq_abi]            "r" (&__rseq_abi),
1202                   /* final store input */
1203                   [v]                   "m" (*v),
1204                   [expect]              "m" (expect),
1205                   [newv]                "m" (newv),
1206                   /* try memcpy input */
1207                   [dst]                 "r" (dst),
1208                   [src]                 "r" (src),
1209                   [len]                 "r" (len),
1210                   [rseq_scratch0]       "m" (rseq_scratch[0]),
1211                   [rseq_scratch1]       "m" (rseq_scratch[1]),
1212                   [rseq_scratch2]       "m" (rseq_scratch[2])
1213                 : "memory", "cc", "eax"
1214                   RSEQ_INJECT_CLOBBER
1215                 : abort, cmpfail
1216 #ifdef RSEQ_COMPARE_TWICE
1217                   , error1, error2
1218 #endif
1219         );
1220         return 0;
1221 abort:
1222         RSEQ_INJECT_FAILED
1223         return -1;
1224 cmpfail:
1225         return 1;
1226 #ifdef RSEQ_COMPARE_TWICE
1227 error1:
1228         rseq_bug("cpu_id comparison failed");
1229 error2:
1230         rseq_bug("expected value comparison failed");
1231 #endif
1232 }
1233
1234 #endif /* !RSEQ_SKIP_FASTPATH */
1235
1236 #endif