tools/testing/selftests/rseq/rseq-x86-bits.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * rseq-x86-bits.h
   4  *
   5  * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 #include "rseq-bits-template.h"
   9
  10 #ifdef __x86_64__
  11
  12 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
  13         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
  14
  15 static inline __attribute__((always_inline))
  16 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  17 {
  18         RSEQ_INJECT_C(9)
  19
  20         __asm__ __volatile__ goto (
  21                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  22                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  23 #ifdef RSEQ_COMPARE_TWICE
  24                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  25                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  26 #endif
  27                 /* Start rseq by storing table entry pointer into rseq_cs. */
  28                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
  29                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
  30                 RSEQ_INJECT_ASM(3)
  31                 "cmpq %[v], %[expect]\n\t"
  32                 "jnz %l[cmpfail]\n\t"
  33                 RSEQ_INJECT_ASM(4)
  34 #ifdef RSEQ_COMPARE_TWICE
  35                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
  36                 "cmpq %[v], %[expect]\n\t"
  37                 "jnz %l[error2]\n\t"
  38 #endif
  39                 /* final store */
  40                 "movq %[newv], %[v]\n\t"
  41                 "2:\n\t"
  42                 RSEQ_INJECT_ASM(5)
  43                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
  44                 : /* gcc asm goto does not allow outputs */
  45                 : [cpu_id]              "r" (cpu),
  46                   [rseq_offset]         "r" (rseq_offset),
  47                   [v]                   "m" (*v),
  48                   [expect]              "r" (expect),
  49                   [newv]                "r" (newv)
  50                 : "memory", "cc", "rax"
  51                   RSEQ_INJECT_CLOBBER
  52                 : abort, cmpfail
  53 #ifdef RSEQ_COMPARE_TWICE
  54                   , error1, error2
  55 #endif
  56         );
  57         rseq_after_asm_goto();
  58         return 0;
  59 abort:
  60         rseq_after_asm_goto();
  61         RSEQ_INJECT_FAILED
  62         return -1;
  63 cmpfail:
  64         rseq_after_asm_goto();
  65         return 1;
  66 #ifdef RSEQ_COMPARE_TWICE
  67 error1:
  68         rseq_after_asm_goto();
  69         rseq_bug("cpu_id comparison failed");
  70 error2:
  71         rseq_after_asm_goto();
  72         rseq_bug("expected value comparison failed");
  73 #endif
  74 }
  75
  76 /*
  77  * Compare @v against @expectnot. When it does _not_ match, load @v
  78  * into @load, and store the content of *@v + voffp into @v.
  79  */
  80 static inline __attribute__((always_inline))
  81 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  82                                long voffp, intptr_t *load, int cpu)
  83 {
  84         RSEQ_INJECT_C(9)
  85
  86         __asm__ __volatile__ goto (
  87                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  88                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  89 #ifdef RSEQ_COMPARE_TWICE
  90                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  91                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  92 #endif
  93                 /* Start rseq by storing table entry pointer into rseq_cs. */
  94                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
  95                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
  96                 RSEQ_INJECT_ASM(3)
  97                 "movq %[v], %%rbx\n\t"
  98                 "cmpq %%rbx, %[expectnot]\n\t"
  99                 "je %l[cmpfail]\n\t"
 100                 RSEQ_INJECT_ASM(4)
 101 #ifdef RSEQ_COMPARE_TWICE
 102                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 103                 "movq %[v], %%rbx\n\t"
 104                 "cmpq %%rbx, %[expectnot]\n\t"
 105                 "je %l[error2]\n\t"
 106 #endif
 107                 "movq %%rbx, %[load]\n\t"
 108                 "addq %[voffp], %%rbx\n\t"
 109                 "movq (%%rbx), %%rbx\n\t"
 110                 /* final store */
 111                 "movq %%rbx, %[v]\n\t"
 112                 "2:\n\t"
 113                 RSEQ_INJECT_ASM(5)
 114                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 115                 : /* gcc asm goto does not allow outputs */
 116                 : [cpu_id]              "r" (cpu),
 117                   [rseq_offset]         "r" (rseq_offset),
 118                   /* final store input */
 119                   [v]                   "m" (*v),
 120                   [expectnot]           "r" (expectnot),
 121                   [voffp]               "er" (voffp),
 122                   [load]                "m" (*load)
 123                 : "memory", "cc", "rax", "rbx"
 124                   RSEQ_INJECT_CLOBBER
 125                 : abort, cmpfail
 126 #ifdef RSEQ_COMPARE_TWICE
 127                   , error1, error2
 128 #endif
 129         );
 130         rseq_after_asm_goto();
 131         return 0;
 132 abort:
 133         rseq_after_asm_goto();
 134         RSEQ_INJECT_FAILED
 135         return -1;
 136 cmpfail:
 137         rseq_after_asm_goto();
 138         return 1;
 139 #ifdef RSEQ_COMPARE_TWICE
 140 error1:
 141         rseq_after_asm_goto();
 142         rseq_bug("cpu_id comparison failed");
 143 error2:
 144         rseq_after_asm_goto();
 145         rseq_bug("expected value comparison failed");
 146 #endif
 147 }
 148
 149 static inline __attribute__((always_inline))
 150 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 151 {
 152         RSEQ_INJECT_C(9)
 153
 154         __asm__ __volatile__ goto (
 155                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 156 #ifdef RSEQ_COMPARE_TWICE
 157                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 158 #endif
 159                 /* Start rseq by storing table entry pointer into rseq_cs. */
 160                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 161                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 162                 RSEQ_INJECT_ASM(3)
 163 #ifdef RSEQ_COMPARE_TWICE
 164                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 165 #endif
 166                 /* final store */
 167                 "addq %[count], %[v]\n\t"
 168                 "2:\n\t"
 169                 RSEQ_INJECT_ASM(4)
 170                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 171                 : /* gcc asm goto does not allow outputs */
 172                 : [cpu_id]              "r" (cpu),
 173                   [rseq_offset]         "r" (rseq_offset),
 174                   /* final store input */
 175                   [v]                   "m" (*v),
 176                   [count]               "er" (count)
 177                 : "memory", "cc", "rax"
 178                   RSEQ_INJECT_CLOBBER
 179                 : abort
 180 #ifdef RSEQ_COMPARE_TWICE
 181                   , error1
 182 #endif
 183         );
 184         rseq_after_asm_goto();
 185         return 0;
 186 abort:
 187         rseq_after_asm_goto();
 188         RSEQ_INJECT_FAILED
 189         return -1;
 190 #ifdef RSEQ_COMPARE_TWICE
 191 error1:
 192         rseq_after_asm_goto();
 193         rseq_bug("cpu_id comparison failed");
 194 #endif
 195 }
 196
 197 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
 198
 199 /*
 200  *   pval = *(ptr+off)
 201  *  *pval += inc;
 202  */
 203 static inline __attribute__((always_inline))
 204 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, intptr_t inc, int cpu)
 205 {
 206         RSEQ_INJECT_C(9)
 207
 208         __asm__ __volatile__ goto (
 209                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 210 #ifdef RSEQ_COMPARE_TWICE
 211                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 212 #endif
 213                 /* Start rseq by storing table entry pointer into rseq_cs. */
 214                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 215                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 216                 RSEQ_INJECT_ASM(3)
 217 #ifdef RSEQ_COMPARE_TWICE
 218                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 219 #endif
 220                 /* get p+v */
 221                 "movq %[ptr], %%rbx\n\t"
 222                 "addq %[off], %%rbx\n\t"
 223                 /* get pv */
 224                 "movq (%%rbx), %%rcx\n\t"
 225                 /* *pv += inc */
 226                 "addq %[inc], (%%rcx)\n\t"
 227                 "2:\n\t"
 228                 RSEQ_INJECT_ASM(4)
 229                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 230                 : /* gcc asm goto does not allow outputs */
 231                 : [cpu_id]              "r" (cpu),
 232                   [rseq_offset]         "r" (rseq_offset),
 233                   /* final store input */
 234                   [ptr]                 "m" (*ptr),
 235                   [off]                 "er" (off),
 236                   [inc]                 "er" (inc)
 237                 : "memory", "cc", "rax", "rbx", "rcx"
 238                   RSEQ_INJECT_CLOBBER
 239                 : abort
 240 #ifdef RSEQ_COMPARE_TWICE
 241                   , error1
 242 #endif
 243         );
 244         return 0;
 245 abort:
 246         RSEQ_INJECT_FAILED
 247         return -1;
 248 #ifdef RSEQ_COMPARE_TWICE
 249 error1:
 250         rseq_bug("cpu_id comparison failed");
 251 #endif
 252 }
 253
 254 static inline __attribute__((always_inline))
 255 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 256                               intptr_t *v2, intptr_t expect2,
 257                               intptr_t newv, int cpu)
 258 {
 259         RSEQ_INJECT_C(9)
 260
 261         __asm__ __volatile__ goto (
 262                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 263                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 264 #ifdef RSEQ_COMPARE_TWICE
 265                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 266                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 267                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 268 #endif
 269                 /* Start rseq by storing table entry pointer into rseq_cs. */
 270                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 271                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 272                 RSEQ_INJECT_ASM(3)
 273                 "cmpq %[v], %[expect]\n\t"
 274                 "jnz %l[cmpfail]\n\t"
 275                 RSEQ_INJECT_ASM(4)
 276                 "cmpq %[v2], %[expect2]\n\t"
 277                 "jnz %l[cmpfail]\n\t"
 278                 RSEQ_INJECT_ASM(5)
 279 #ifdef RSEQ_COMPARE_TWICE
 280                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 281                 "cmpq %[v], %[expect]\n\t"
 282                 "jnz %l[error2]\n\t"
 283                 "cmpq %[v2], %[expect2]\n\t"
 284                 "jnz %l[error3]\n\t"
 285 #endif
 286                 /* final store */
 287                 "movq %[newv], %[v]\n\t"
 288                 "2:\n\t"
 289                 RSEQ_INJECT_ASM(6)
 290                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 291                 : /* gcc asm goto does not allow outputs */
 292                 : [cpu_id]              "r" (cpu),
 293                   [rseq_offset]         "r" (rseq_offset),
 294                   /* cmp2 input */
 295                   [v2]                  "m" (*v2),
 296                   [expect2]             "r" (expect2),
 297                   /* final store input */
 298                   [v]                   "m" (*v),
 299                   [expect]              "r" (expect),
 300                   [newv]                "r" (newv)
 301                 : "memory", "cc", "rax"
 302                   RSEQ_INJECT_CLOBBER
 303                 : abort, cmpfail
 304 #ifdef RSEQ_COMPARE_TWICE
 305                   , error1, error2, error3
 306 #endif
 307         );
 308         rseq_after_asm_goto();
 309         return 0;
 310 abort:
 311         rseq_after_asm_goto();
 312         RSEQ_INJECT_FAILED
 313         return -1;
 314 cmpfail:
 315         rseq_after_asm_goto();
 316         return 1;
 317 #ifdef RSEQ_COMPARE_TWICE
 318 error1:
 319         rseq_after_asm_goto();
 320         rseq_bug("cpu_id comparison failed");
 321 error2:
 322         rseq_after_asm_goto();
 323         rseq_bug("1st expected value comparison failed");
 324 error3:
 325         rseq_after_asm_goto();
 326         rseq_bug("2nd expected value comparison failed");
 327 #endif
 328 }
 329
 330 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 331         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 332
 333 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 334         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 335
 336 static inline __attribute__((always_inline))
 337 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 338                                  intptr_t *v2, intptr_t newv2,
 339                                  intptr_t newv, int cpu)
 340 {
 341         RSEQ_INJECT_C(9)
 342
 343         __asm__ __volatile__ goto (
 344                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 345                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 346 #ifdef RSEQ_COMPARE_TWICE
 347                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 348                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 349 #endif
 350                 /* Start rseq by storing table entry pointer into rseq_cs. */
 351                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 352                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 353                 RSEQ_INJECT_ASM(3)
 354                 "cmpq %[v], %[expect]\n\t"
 355                 "jnz %l[cmpfail]\n\t"
 356                 RSEQ_INJECT_ASM(4)
 357 #ifdef RSEQ_COMPARE_TWICE
 358                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 359                 "cmpq %[v], %[expect]\n\t"
 360                 "jnz %l[error2]\n\t"
 361 #endif
 362                 /* try store */
 363                 "movq %[newv2], %[v2]\n\t"
 364                 RSEQ_INJECT_ASM(5)
 365                 /* final store */
 366                 "movq %[newv], %[v]\n\t"
 367                 "2:\n\t"
 368                 RSEQ_INJECT_ASM(6)
 369                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 370                 : /* gcc asm goto does not allow outputs */
 371                 : [cpu_id]              "r" (cpu),
 372                   [rseq_offset]         "r" (rseq_offset),
 373                   /* try store input */
 374                   [v2]                  "m" (*v2),
 375                   [newv2]               "r" (newv2),
 376                   /* final store input */
 377                   [v]                   "m" (*v),
 378                   [expect]              "r" (expect),
 379                   [newv]                "r" (newv)
 380                 : "memory", "cc", "rax"
 381                   RSEQ_INJECT_CLOBBER
 382                 : abort, cmpfail
 383 #ifdef RSEQ_COMPARE_TWICE
 384                   , error1, error2
 385 #endif
 386         );
 387         rseq_after_asm_goto();
 388         return 0;
 389 abort:
 390         rseq_after_asm_goto();
 391         RSEQ_INJECT_FAILED
 392         return -1;
 393 cmpfail:
 394         rseq_after_asm_goto();
 395         return 1;
 396 #ifdef RSEQ_COMPARE_TWICE
 397 error1:
 398         rseq_after_asm_goto();
 399         rseq_bug("cpu_id comparison failed");
 400 error2:
 401         rseq_after_asm_goto();
 402         rseq_bug("expected value comparison failed");
 403 #endif
 404 }
 405
 406 static inline __attribute__((always_inline))
 407 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 408                                  void *dst, void *src, size_t len,
 409                                  intptr_t newv, int cpu)
 410 {
 411         uint64_t rseq_scratch[3];
 412
 413         RSEQ_INJECT_C(9)
 414
 415         __asm__ __volatile__ goto (
 416                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 417                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 418 #ifdef RSEQ_COMPARE_TWICE
 419                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 420                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 421 #endif
 422                 "movq %[src], %[rseq_scratch0]\n\t"
 423                 "movq %[dst], %[rseq_scratch1]\n\t"
 424                 "movq %[len], %[rseq_scratch2]\n\t"
 425                 /* Start rseq by storing table entry pointer into rseq_cs. */
 426                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 427                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 428                 RSEQ_INJECT_ASM(3)
 429                 "cmpq %[v], %[expect]\n\t"
 430                 "jnz 5f\n\t"
 431                 RSEQ_INJECT_ASM(4)
 432 #ifdef RSEQ_COMPARE_TWICE
 433                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
 434                 "cmpq %[v], %[expect]\n\t"
 435                 "jnz 7f\n\t"
 436 #endif
 437                 /* try memcpy */
 438                 "test %[len], %[len]\n\t" \
 439                 "jz 333f\n\t" \
 440                 "222:\n\t" \
 441                 "movb (%[src]), %%al\n\t" \
 442                 "movb %%al, (%[dst])\n\t" \
 443                 "inc %[src]\n\t" \
 444                 "inc %[dst]\n\t" \
 445                 "dec %[len]\n\t" \
 446                 "jnz 222b\n\t" \
 447                 "333:\n\t" \
 448                 RSEQ_INJECT_ASM(5)
 449                 /* final store */
 450                 "movq %[newv], %[v]\n\t"
 451                 "2:\n\t"
 452                 RSEQ_INJECT_ASM(6)
 453                 /* teardown */
 454                 "movq %[rseq_scratch2], %[len]\n\t"
 455                 "movq %[rseq_scratch1], %[dst]\n\t"
 456                 "movq %[rseq_scratch0], %[src]\n\t"
 457                 RSEQ_ASM_DEFINE_ABORT(4,
 458                         "movq %[rseq_scratch2], %[len]\n\t"
 459                         "movq %[rseq_scratch1], %[dst]\n\t"
 460                         "movq %[rseq_scratch0], %[src]\n\t",
 461                         abort)
 462                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 463                         "movq %[rseq_scratch2], %[len]\n\t"
 464                         "movq %[rseq_scratch1], %[dst]\n\t"
 465                         "movq %[rseq_scratch0], %[src]\n\t",
 466                         cmpfail)
 467 #ifdef RSEQ_COMPARE_TWICE
 468                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 469                         "movq %[rseq_scratch2], %[len]\n\t"
 470                         "movq %[rseq_scratch1], %[dst]\n\t"
 471                         "movq %[rseq_scratch0], %[src]\n\t",
 472                         error1)
 473                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 474                         "movq %[rseq_scratch2], %[len]\n\t"
 475                         "movq %[rseq_scratch1], %[dst]\n\t"
 476                         "movq %[rseq_scratch0], %[src]\n\t",
 477                         error2)
 478 #endif
 479                 : /* gcc asm goto does not allow outputs */
 480                 : [cpu_id]              "r" (cpu),
 481                   [rseq_offset]         "r" (rseq_offset),
 482                   /* final store input */
 483                   [v]                   "m" (*v),
 484                   [expect]              "r" (expect),
 485                   [newv]                "r" (newv),
 486                   /* try memcpy input */
 487                   [dst]                 "r" (dst),
 488                   [src]                 "r" (src),
 489                   [len]                 "r" (len),
 490                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 491                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 492                   [rseq_scratch2]       "m" (rseq_scratch[2])
 493                 : "memory", "cc", "rax"
 494                   RSEQ_INJECT_CLOBBER
 495                 : abort, cmpfail
 496 #ifdef RSEQ_COMPARE_TWICE
 497                   , error1, error2
 498 #endif
 499         );
 500         rseq_after_asm_goto();
 501         return 0;
 502 abort:
 503         rseq_after_asm_goto();
 504         RSEQ_INJECT_FAILED
 505         return -1;
 506 cmpfail:
 507         rseq_after_asm_goto();
 508         return 1;
 509 #ifdef RSEQ_COMPARE_TWICE
 510 error1:
 511         rseq_after_asm_goto();
 512         rseq_bug("cpu_id comparison failed");
 513 error2:
 514         rseq_after_asm_goto();
 515         rseq_bug("expected value comparison failed");
 516 #endif
 517 }
 518
 519 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 520         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 521
 522 #elif defined(__i386__)
 523
 524 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
 525         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 526
 527 static inline __attribute__((always_inline))
 528 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 529 {
 530         RSEQ_INJECT_C(9)
 531
 532         __asm__ __volatile__ goto (
 533                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 534                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 535 #ifdef RSEQ_COMPARE_TWICE
 536                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 537                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 538 #endif
 539                 /* Start rseq by storing table entry pointer into rseq_cs. */
 540                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 541                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 542                 RSEQ_INJECT_ASM(3)
 543                 "cmpl %[v], %[expect]\n\t"
 544                 "jnz %l[cmpfail]\n\t"
 545                 RSEQ_INJECT_ASM(4)
 546 #ifdef RSEQ_COMPARE_TWICE
 547                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 548                 "cmpl %[v], %[expect]\n\t"
 549                 "jnz %l[error2]\n\t"
 550 #endif
 551                 /* final store */
 552                 "movl %[newv], %[v]\n\t"
 553                 "2:\n\t"
 554                 RSEQ_INJECT_ASM(5)
 555                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 556                 : /* gcc asm goto does not allow outputs */
 557                 : [cpu_id]              "r" (cpu),
 558                   [rseq_offset]         "r" (rseq_offset),
 559                   [v]                   "m" (*v),
 560                   [expect]              "r" (expect),
 561                   [newv]                "r" (newv)
 562                 : "memory", "cc", "eax"
 563                   RSEQ_INJECT_CLOBBER
 564                 : abort, cmpfail
 565 #ifdef RSEQ_COMPARE_TWICE
 566                   , error1, error2
 567 #endif
 568         );
 569         rseq_after_asm_goto();
 570         return 0;
 571 abort:
 572         rseq_after_asm_goto();
 573         RSEQ_INJECT_FAILED
 574         return -1;
 575 cmpfail:
 576         rseq_after_asm_goto();
 577         return 1;
 578 #ifdef RSEQ_COMPARE_TWICE
 579 error1:
 580         rseq_after_asm_goto();
 581         rseq_bug("cpu_id comparison failed");
 582 error2:
 583         rseq_after_asm_goto();
 584         rseq_bug("expected value comparison failed");
 585 #endif
 586 }
 587
 588 /*
 589  * Compare @v against @expectnot. When it does _not_ match, load @v
 590  * into @load, and store the content of *@v + voffp into @v.
 591  */
 592 static inline __attribute__((always_inline))
 593 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
 594                                long voffp, intptr_t *load, int cpu)
 595 {
 596         RSEQ_INJECT_C(9)
 597
 598         __asm__ __volatile__ goto (
 599                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 600                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 601 #ifdef RSEQ_COMPARE_TWICE
 602                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 603                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 604 #endif
 605                 /* Start rseq by storing table entry pointer into rseq_cs. */
 606                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 607                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 608                 RSEQ_INJECT_ASM(3)
 609                 "movl %[v], %%ebx\n\t"
 610                 "cmpl %%ebx, %[expectnot]\n\t"
 611                 "je %l[cmpfail]\n\t"
 612                 RSEQ_INJECT_ASM(4)
 613 #ifdef RSEQ_COMPARE_TWICE
 614                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 615                 "movl %[v], %%ebx\n\t"
 616                 "cmpl %%ebx, %[expectnot]\n\t"
 617                 "je %l[error2]\n\t"
 618 #endif
 619                 "movl %%ebx, %[load]\n\t"
 620                 "addl %[voffp], %%ebx\n\t"
 621                 "movl (%%ebx), %%ebx\n\t"
 622                 /* final store */
 623                 "movl %%ebx, %[v]\n\t"
 624                 "2:\n\t"
 625                 RSEQ_INJECT_ASM(5)
 626                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 627                 : /* gcc asm goto does not allow outputs */
 628                 : [cpu_id]              "r" (cpu),
 629                   [rseq_offset]         "r" (rseq_offset),
 630                   /* final store input */
 631                   [v]                   "m" (*v),
 632                   [expectnot]           "r" (expectnot),
 633                   [voffp]               "ir" (voffp),
 634                   [load]                "m" (*load)
 635                 : "memory", "cc", "eax", "ebx"
 636                   RSEQ_INJECT_CLOBBER
 637                 : abort, cmpfail
 638 #ifdef RSEQ_COMPARE_TWICE
 639                   , error1, error2
 640 #endif
 641         );
 642         rseq_after_asm_goto();
 643         return 0;
 644 abort:
 645         rseq_after_asm_goto();
 646         RSEQ_INJECT_FAILED
 647         return -1;
 648 cmpfail:
 649         rseq_after_asm_goto();
 650         return 1;
 651 #ifdef RSEQ_COMPARE_TWICE
 652 error1:
 653         rseq_after_asm_goto();
 654         rseq_bug("cpu_id comparison failed");
 655 error2:
 656         rseq_after_asm_goto();
 657         rseq_bug("expected value comparison failed");
 658 #endif
 659 }
 660
 661 static inline __attribute__((always_inline))
 662 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 663 {
 664         RSEQ_INJECT_C(9)
 665
 666         __asm__ __volatile__ goto (
 667                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 668 #ifdef RSEQ_COMPARE_TWICE
 669                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 670 #endif
 671                 /* Start rseq by storing table entry pointer into rseq_cs. */
 672                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 673                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 674                 RSEQ_INJECT_ASM(3)
 675 #ifdef RSEQ_COMPARE_TWICE
 676                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 677 #endif
 678                 /* final store */
 679                 "addl %[count], %[v]\n\t"
 680                 "2:\n\t"
 681                 RSEQ_INJECT_ASM(4)
 682                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 683                 : /* gcc asm goto does not allow outputs */
 684                 : [cpu_id]              "r" (cpu),
 685                   [rseq_offset]         "r" (rseq_offset),
 686                   /* final store input */
 687                   [v]                   "m" (*v),
 688                   [count]               "ir" (count)
 689                 : "memory", "cc", "eax"
 690                   RSEQ_INJECT_CLOBBER
 691                 : abort
 692 #ifdef RSEQ_COMPARE_TWICE
 693                   , error1
 694 #endif
 695         );
 696         rseq_after_asm_goto();
 697         return 0;
 698 abort:
 699         rseq_after_asm_goto();
 700         RSEQ_INJECT_FAILED
 701         return -1;
 702 #ifdef RSEQ_COMPARE_TWICE
 703 error1:
 704         rseq_after_asm_goto();
 705         rseq_bug("cpu_id comparison failed");
 706 #endif
 707 }
 708
 709 static inline __attribute__((always_inline))
 710 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 711                               intptr_t *v2, intptr_t expect2,
 712                               intptr_t newv, int cpu)
 713 {
 714         RSEQ_INJECT_C(9)
 715
 716         __asm__ __volatile__ goto (
 717                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 718                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 719 #ifdef RSEQ_COMPARE_TWICE
 720                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 721                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 722                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 723 #endif
 724                 /* Start rseq by storing table entry pointer into rseq_cs. */
 725                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 726                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 727                 RSEQ_INJECT_ASM(3)
 728                 "cmpl %[v], %[expect]\n\t"
 729                 "jnz %l[cmpfail]\n\t"
 730                 RSEQ_INJECT_ASM(4)
 731                 "cmpl %[expect2], %[v2]\n\t"
 732                 "jnz %l[cmpfail]\n\t"
 733                 RSEQ_INJECT_ASM(5)
 734 #ifdef RSEQ_COMPARE_TWICE
 735                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 736                 "cmpl %[v], %[expect]\n\t"
 737                 "jnz %l[error2]\n\t"
 738                 "cmpl %[expect2], %[v2]\n\t"
 739                 "jnz %l[error3]\n\t"
 740 #endif
 741                 "movl %[newv], %%eax\n\t"
 742                 /* final store */
 743                 "movl %%eax, %[v]\n\t"
 744                 "2:\n\t"
 745                 RSEQ_INJECT_ASM(6)
 746                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 747                 : /* gcc asm goto does not allow outputs */
 748                 : [cpu_id]              "r" (cpu),
 749                   [rseq_offset]         "r" (rseq_offset),
 750                   /* cmp2 input */
 751                   [v2]                  "m" (*v2),
 752                   [expect2]             "r" (expect2),
 753                   /* final store input */
 754                   [v]                   "m" (*v),
 755                   [expect]              "r" (expect),
 756                   [newv]                "m" (newv)
 757                 : "memory", "cc", "eax"
 758                   RSEQ_INJECT_CLOBBER
 759                 : abort, cmpfail
 760 #ifdef RSEQ_COMPARE_TWICE
 761                   , error1, error2, error3
 762 #endif
 763         );
 764         rseq_after_asm_goto();
 765         return 0;
 766 abort:
 767         rseq_after_asm_goto();
 768         RSEQ_INJECT_FAILED
 769         return -1;
 770 cmpfail:
 771         rseq_after_asm_goto();
 772         return 1;
 773 #ifdef RSEQ_COMPARE_TWICE
 774 error1:
 775         rseq_after_asm_goto();
 776         rseq_bug("cpu_id comparison failed");
 777 error2:
 778         rseq_after_asm_goto();
 779         rseq_bug("1st expected value comparison failed");
 780 error3:
 781         rseq_after_asm_goto();
 782         rseq_bug("2nd expected value comparison failed");
 783 #endif
 784 }
 785
 786 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 787         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 788
 789 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 790         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 791
 792 static inline __attribute__((always_inline))
 793 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 794                                          intptr_t *v2, intptr_t newv2,
 795                                          intptr_t newv, int cpu)
 796 {
 797         RSEQ_INJECT_C(9)
 798
 799         __asm__ __volatile__ goto (
 800                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 801                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 802 #ifdef RSEQ_COMPARE_TWICE
 803                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 804                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 805 #endif
 806                 /* Start rseq by storing table entry pointer into rseq_cs. */
 807                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 808                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 809                 RSEQ_INJECT_ASM(3)
 810                 "movl %[expect], %%eax\n\t"
 811                 "cmpl %[v], %%eax\n\t"
 812                 "jnz %l[cmpfail]\n\t"
 813                 RSEQ_INJECT_ASM(4)
 814 #ifdef RSEQ_COMPARE_TWICE
 815                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 816                 "movl %[expect], %%eax\n\t"
 817                 "cmpl %[v], %%eax\n\t"
 818                 "jnz %l[error2]\n\t"
 819 #endif
 820                 /* try store */
 821                 "movl %[newv2], %[v2]\n\t"
 822                 RSEQ_INJECT_ASM(5)
 823 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 824                 "lock; addl $0,-128(%%esp)\n\t"
 825 #endif
 826                 /* final store */
 827                 "movl %[newv], %[v]\n\t"
 828                 "2:\n\t"
 829                 RSEQ_INJECT_ASM(6)
 830                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 831                 : /* gcc asm goto does not allow outputs */
 832                 : [cpu_id]              "r" (cpu),
 833                   [rseq_offset]         "r" (rseq_offset),
 834                   /* try store input */
 835                   [v2]                  "m" (*v2),
 836                   [newv2]               "r" (newv2),
 837                   /* final store input */
 838                   [v]                   "m" (*v),
 839                   [expect]              "m" (expect),
 840                   [newv]                "r" (newv)
 841                 : "memory", "cc", "eax"
 842                   RSEQ_INJECT_CLOBBER
 843                 : abort, cmpfail
 844 #ifdef RSEQ_COMPARE_TWICE
 845                   , error1, error2
 846 #endif
 847         );
 848         rseq_after_asm_goto();
 849         return 0;
 850 abort:
 851         rseq_after_asm_goto();
 852         RSEQ_INJECT_FAILED
 853         return -1;
 854 cmpfail:
 855         rseq_after_asm_goto();
 856         return 1;
 857 #ifdef RSEQ_COMPARE_TWICE
 858 error1:
 859         rseq_after_asm_goto();
 860         rseq_bug("cpu_id comparison failed");
 861 error2:
 862         rseq_after_asm_goto();
 863         rseq_bug("expected value comparison failed");
 864 #endif
 865
 866 }
 867
 868 /* TODO: implement a faster memcpy. */
 869 static inline __attribute__((always_inline))
 870 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 871                                          void *dst, void *src, size_t len,
 872                                          intptr_t newv, int cpu)
 873 {
 874         uint32_t rseq_scratch[3];
 875
 876         RSEQ_INJECT_C(9)
 877
 878         __asm__ __volatile__ goto (
 879                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 880                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 881 #ifdef RSEQ_COMPARE_TWICE
 882                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 883                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 884 #endif
 885                 "movl %[src], %[rseq_scratch0]\n\t"
 886                 "movl %[dst], %[rseq_scratch1]\n\t"
 887                 "movl %[len], %[rseq_scratch2]\n\t"
 888                 /* Start rseq by storing table entry pointer into rseq_cs. */
 889                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
 890                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f)
 891                 RSEQ_INJECT_ASM(3)
 892                 "movl %[expect], %%eax\n\t"
 893                 "cmpl %%eax, %[v]\n\t"
 894                 "jnz 5f\n\t"
 895                 RSEQ_INJECT_ASM(4)
 896 #ifdef RSEQ_COMPARE_TWICE
 897                 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f)
 898                 "movl %[expect], %%eax\n\t"
 899                 "cmpl %%eax, %[v]\n\t"
 900                 "jnz 7f\n\t"
 901 #endif
 902                 /* try memcpy */
 903                 "test %[len], %[len]\n\t" \
 904                 "jz 333f\n\t" \
 905                 "222:\n\t" \
 906                 "movb (%[src]), %%al\n\t" \
 907                 "movb %%al, (%[dst])\n\t" \
 908                 "inc %[src]\n\t" \
 909                 "inc %[dst]\n\t" \
 910                 "dec %[len]\n\t" \
 911                 "jnz 222b\n\t" \
 912                 "333:\n\t" \
 913                 RSEQ_INJECT_ASM(5)
 914 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 915                 "lock; addl $0,-128(%%esp)\n\t"
 916 #endif
 917                 "movl %[newv], %%eax\n\t"
 918                 /* final store */
 919                 "movl %%eax, %[v]\n\t"
 920                 "2:\n\t"
 921                 RSEQ_INJECT_ASM(6)
 922                 /* teardown */
 923                 "movl %[rseq_scratch2], %[len]\n\t"
 924                 "movl %[rseq_scratch1], %[dst]\n\t"
 925                 "movl %[rseq_scratch0], %[src]\n\t"
 926                 RSEQ_ASM_DEFINE_ABORT(4,
 927                         "movl %[rseq_scratch2], %[len]\n\t"
 928                         "movl %[rseq_scratch1], %[dst]\n\t"
 929                         "movl %[rseq_scratch0], %[src]\n\t",
 930                         abort)
 931                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 932                         "movl %[rseq_scratch2], %[len]\n\t"
 933                         "movl %[rseq_scratch1], %[dst]\n\t"
 934                         "movl %[rseq_scratch0], %[src]\n\t",
 935                         cmpfail)
 936 #ifdef RSEQ_COMPARE_TWICE
 937                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 938                         "movl %[rseq_scratch2], %[len]\n\t"
 939                         "movl %[rseq_scratch1], %[dst]\n\t"
 940                         "movl %[rseq_scratch0], %[src]\n\t",
 941                         error1)
 942                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 943                         "movl %[rseq_scratch2], %[len]\n\t"
 944                         "movl %[rseq_scratch1], %[dst]\n\t"
 945                         "movl %[rseq_scratch0], %[src]\n\t",
 946                         error2)
 947 #endif
 948                 : /* gcc asm goto does not allow outputs */
 949                 : [cpu_id]              "r" (cpu),
 950                   [rseq_offset]         "r" (rseq_offset),
 951                   /* final store input */
 952                   [v]                   "m" (*v),
 953                   [expect]              "m" (expect),
 954                   [newv]                "m" (newv),
 955                   /* try memcpy input */
 956                   [dst]                 "r" (dst),
 957                   [src]                 "r" (src),
 958                   [len]                 "r" (len),
 959                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 960                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 961                   [rseq_scratch2]       "m" (rseq_scratch[2])
 962                 : "memory", "cc", "eax"
 963                   RSEQ_INJECT_CLOBBER
 964                 : abort, cmpfail
 965 #ifdef RSEQ_COMPARE_TWICE
 966                   , error1, error2
 967 #endif
 968         );
 969         rseq_after_asm_goto();
 970         return 0;
 971 abort:
 972         rseq_after_asm_goto();
 973         RSEQ_INJECT_FAILED
 974         return -1;
 975 cmpfail:
 976         rseq_after_asm_goto();
 977         return 1;
 978 #ifdef RSEQ_COMPARE_TWICE
 979 error1:
 980         rseq_after_asm_goto();
 981         rseq_bug("cpu_id comparison failed");
 982 error2:
 983         rseq_after_asm_goto();
 984         rseq_bug("expected value comparison failed");
 985 #endif
 986 }
 987
 988 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 989         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 990
 991 #endif
 992
 993 #include "rseq-bits-reset.h"