tools/testing/selftests/rseq/rseq-mips-bits.h

   1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
   2 /*
   3  * Author: Paul Burton <paul.burton@mips.com>
   4  * (C) Copyright 2018 MIPS Tech LLC
   5  * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   6  */
   7
   8 #include "rseq-bits-template.h"
   9
  10 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
  11         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
  12
  13 static inline __attribute__((always_inline))
  14 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  15 {
  16         RSEQ_INJECT_C(9)
  17
  18         __asm__ __volatile__ goto (
  19                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
  20                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  21 #ifdef RSEQ_COMPARE_TWICE
  22                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  23                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  24 #endif
  25                 /* Start rseq by storing table entry pointer into rseq_cs. */
  26                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
  27                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  28                 RSEQ_INJECT_ASM(3)
  29                 LONG_L " $4, %[v]\n\t"
  30                 "bne $4, %[expect], %l[cmpfail]\n\t"
  31                 RSEQ_INJECT_ASM(4)
  32 #ifdef RSEQ_COMPARE_TWICE
  33                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  34                 LONG_L " $4, %[v]\n\t"
  35                 "bne $4, %[expect], %l[error2]\n\t"
  36 #endif
  37                 /* final store */
  38                 LONG_S " %[newv], %[v]\n\t"
  39                 "2:\n\t"
  40                 RSEQ_INJECT_ASM(5)
  41                 "b 5f\n\t"
  42                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
  43                 "5:\n\t"
  44                 : /* gcc asm goto does not allow outputs */
  45                 : [cpu_id]              "r" (cpu),
  46                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  47                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  48                   [v]                   "m" (*v),
  49                   [expect]              "r" (expect),
  50                   [newv]                "r" (newv)
  51                   RSEQ_INJECT_INPUT
  52                 : "$4", "memory"
  53                   RSEQ_INJECT_CLOBBER
  54                 : abort, cmpfail
  55 #ifdef RSEQ_COMPARE_TWICE
  56                   , error1, error2
  57 #endif
  58         );
  59         return 0;
  60 abort:
  61         RSEQ_INJECT_FAILED
  62         return -1;
  63 cmpfail:
  64         return 1;
  65 #ifdef RSEQ_COMPARE_TWICE
  66 error1:
  67         rseq_bug("cpu_id comparison failed");
  68 error2:
  69         rseq_bug("expected value comparison failed");
  70 #endif
  71 }
  72
  73 static inline __attribute__((always_inline))
  74 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  75                                long voffp, intptr_t *load, int cpu)
  76 {
  77         RSEQ_INJECT_C(9)
  78
  79         __asm__ __volatile__ goto (
  80                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
  81                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  82 #ifdef RSEQ_COMPARE_TWICE
  83                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  84                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  85 #endif
  86                 /* Start rseq by storing table entry pointer into rseq_cs. */
  87                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
  88                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
  89                 RSEQ_INJECT_ASM(3)
  90                 LONG_L " $4, %[v]\n\t"
  91                 "beq $4, %[expectnot], %l[cmpfail]\n\t"
  92                 RSEQ_INJECT_ASM(4)
  93 #ifdef RSEQ_COMPARE_TWICE
  94                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  95                 LONG_L " $4, %[v]\n\t"
  96                 "beq $4, %[expectnot], %l[error2]\n\t"
  97 #endif
  98                 LONG_S " $4, %[load]\n\t"
  99                 LONG_ADDI " $4, %[voffp]\n\t"
 100                 LONG_L " $4, 0($4)\n\t"
 101                 /* final store */
 102                 LONG_S " $4, %[v]\n\t"
 103                 "2:\n\t"
 104                 RSEQ_INJECT_ASM(5)
 105                 "b 5f\n\t"
 106                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 107                 "5:\n\t"
 108                 : /* gcc asm goto does not allow outputs */
 109                 : [cpu_id]              "r" (cpu),
 110                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 111                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 112                   /* final store input */
 113                   [v]                   "m" (*v),
 114                   [expectnot]           "r" (expectnot),
 115                   [voffp]               "Ir" (voffp),
 116                   [load]                "m" (*load)
 117                   RSEQ_INJECT_INPUT
 118                 : "$4", "memory"
 119                   RSEQ_INJECT_CLOBBER
 120                 : abort, cmpfail
 121 #ifdef RSEQ_COMPARE_TWICE
 122                   , error1, error2
 123 #endif
 124         );
 125         return 0;
 126 abort:
 127         RSEQ_INJECT_FAILED
 128         return -1;
 129 cmpfail:
 130         return 1;
 131 #ifdef RSEQ_COMPARE_TWICE
 132 error1:
 133         rseq_bug("cpu_id comparison failed");
 134 error2:
 135         rseq_bug("expected value comparison failed");
 136 #endif
 137 }
 138
 139 static inline __attribute__((always_inline))
 140 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 141 {
 142         RSEQ_INJECT_C(9)
 143
 144         __asm__ __volatile__ goto (
 145                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 146 #ifdef RSEQ_COMPARE_TWICE
 147                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 148 #endif
 149                 /* Start rseq by storing table entry pointer into rseq_cs. */
 150                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 151                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 152                 RSEQ_INJECT_ASM(3)
 153 #ifdef RSEQ_COMPARE_TWICE
 154                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 155 #endif
 156                 LONG_L " $4, %[v]\n\t"
 157                 LONG_ADDI " $4, %[count]\n\t"
 158                 /* final store */
 159                 LONG_S " $4, %[v]\n\t"
 160                 "2:\n\t"
 161                 RSEQ_INJECT_ASM(4)
 162                 "b 5f\n\t"
 163                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 164                 "5:\n\t"
 165                 : /* gcc asm goto does not allow outputs */
 166                 : [cpu_id]              "r" (cpu),
 167                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 168                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 169                   [v]                   "m" (*v),
 170                   [count]               "Ir" (count)
 171                   RSEQ_INJECT_INPUT
 172                 : "$4", "memory"
 173                   RSEQ_INJECT_CLOBBER
 174                 : abort
 175 #ifdef RSEQ_COMPARE_TWICE
 176                   , error1
 177 #endif
 178         );
 179         return 0;
 180 abort:
 181         RSEQ_INJECT_FAILED
 182         return -1;
 183 #ifdef RSEQ_COMPARE_TWICE
 184 error1:
 185         rseq_bug("cpu_id comparison failed");
 186 #endif
 187 }
 188
 189 static inline __attribute__((always_inline))
 190 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 191                               intptr_t *v2, intptr_t expect2,
 192                               intptr_t newv, int cpu)
 193 {
 194         RSEQ_INJECT_C(9)
 195
 196         __asm__ __volatile__ goto (
 197                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 198                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 199 #ifdef RSEQ_COMPARE_TWICE
 200                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 201                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 202                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 203 #endif
 204                 /* Start rseq by storing table entry pointer into rseq_cs. */
 205                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 206                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 207                 RSEQ_INJECT_ASM(3)
 208                 LONG_L " $4, %[v]\n\t"
 209                 "bne $4, %[expect], %l[cmpfail]\n\t"
 210                 RSEQ_INJECT_ASM(4)
 211                 LONG_L " $4, %[v2]\n\t"
 212                 "bne $4, %[expect2], %l[cmpfail]\n\t"
 213                 RSEQ_INJECT_ASM(5)
 214 #ifdef RSEQ_COMPARE_TWICE
 215                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 216                 LONG_L " $4, %[v]\n\t"
 217                 "bne $4, %[expect], %l[error2]\n\t"
 218                 LONG_L " $4, %[v2]\n\t"
 219                 "bne $4, %[expect2], %l[error3]\n\t"
 220 #endif
 221                 /* final store */
 222                 LONG_S " %[newv], %[v]\n\t"
 223                 "2:\n\t"
 224                 RSEQ_INJECT_ASM(6)
 225                 "b 5f\n\t"
 226                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 227                 "5:\n\t"
 228                 : /* gcc asm goto does not allow outputs */
 229                 : [cpu_id]              "r" (cpu),
 230                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 231                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 232                   /* cmp2 input */
 233                   [v2]                  "m" (*v2),
 234                   [expect2]             "r" (expect2),
 235                   /* final store input */
 236                   [v]                   "m" (*v),
 237                   [expect]              "r" (expect),
 238                   [newv]                "r" (newv)
 239                   RSEQ_INJECT_INPUT
 240                 : "$4", "memory"
 241                   RSEQ_INJECT_CLOBBER
 242                 : abort, cmpfail
 243 #ifdef RSEQ_COMPARE_TWICE
 244                   , error1, error2, error3
 245 #endif
 246         );
 247         return 0;
 248 abort:
 249         RSEQ_INJECT_FAILED
 250         return -1;
 251 cmpfail:
 252         return 1;
 253 #ifdef RSEQ_COMPARE_TWICE
 254 error1:
 255         rseq_bug("cpu_id comparison failed");
 256 error2:
 257         rseq_bug("1st expected value comparison failed");
 258 error3:
 259         rseq_bug("2nd expected value comparison failed");
 260 #endif
 261 }
 262
 263 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 264         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 265
 266 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 267         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 268
 269 static inline __attribute__((always_inline))
 270 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 271                                  intptr_t *v2, intptr_t newv2,
 272                                  intptr_t newv, int cpu)
 273 {
 274         RSEQ_INJECT_C(9)
 275
 276         __asm__ __volatile__ goto (
 277                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 278                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 279 #ifdef RSEQ_COMPARE_TWICE
 280                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 281                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 282 #endif
 283                 /* Start rseq by storing table entry pointer into rseq_cs. */
 284                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 285                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 286                 RSEQ_INJECT_ASM(3)
 287                 LONG_L " $4, %[v]\n\t"
 288                 "bne $4, %[expect], %l[cmpfail]\n\t"
 289                 RSEQ_INJECT_ASM(4)
 290 #ifdef RSEQ_COMPARE_TWICE
 291                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 292                 LONG_L " $4, %[v]\n\t"
 293                 "bne $4, %[expect], %l[error2]\n\t"
 294 #endif
 295                 /* try store */
 296                 LONG_S " %[newv2], %[v2]\n\t"
 297                 RSEQ_INJECT_ASM(5)
 298 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 299                 "sync\n\t"      /* full sync provides store-release */
 300 #endif
 301                 /* final store */
 302                 LONG_S " %[newv], %[v]\n\t"
 303                 "2:\n\t"
 304                 RSEQ_INJECT_ASM(6)
 305                 "b 5f\n\t"
 306                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 307                 "5:\n\t"
 308                 : /* gcc asm goto does not allow outputs */
 309                 : [cpu_id]              "r" (cpu),
 310                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 311                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 312                   /* try store input */
 313                   [v2]                  "m" (*v2),
 314                   [newv2]               "r" (newv2),
 315                   /* final store input */
 316                   [v]                   "m" (*v),
 317                   [expect]              "r" (expect),
 318                   [newv]                "r" (newv)
 319                   RSEQ_INJECT_INPUT
 320                 : "$4", "memory"
 321                   RSEQ_INJECT_CLOBBER
 322                 : abort, cmpfail
 323 #ifdef RSEQ_COMPARE_TWICE
 324                   , error1, error2
 325 #endif
 326         );
 327         return 0;
 328 abort:
 329         RSEQ_INJECT_FAILED
 330         return -1;
 331 cmpfail:
 332         return 1;
 333 #ifdef RSEQ_COMPARE_TWICE
 334 error1:
 335         rseq_bug("cpu_id comparison failed");
 336 error2:
 337         rseq_bug("expected value comparison failed");
 338 #endif
 339 }
 340
 341 static inline __attribute__((always_inline))
 342 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 343                                  void *dst, void *src, size_t len,
 344                                  intptr_t newv, int cpu)
 345 {
 346         uintptr_t rseq_scratch[3];
 347
 348         RSEQ_INJECT_C(9)
 349
 350         __asm__ __volatile__ goto (
 351                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 352                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 353 #ifdef RSEQ_COMPARE_TWICE
 354                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 355                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 356 #endif
 357                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 358                 LONG_S "  %[dst], %[rseq_scratch1]\n\t"
 359                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 360                 /* Start rseq by storing table entry pointer into rseq_cs. */
 361                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 362                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 363                 RSEQ_INJECT_ASM(3)
 364                 LONG_L " $4, %[v]\n\t"
 365                 "bne $4, %[expect], 5f\n\t"
 366                 RSEQ_INJECT_ASM(4)
 367 #ifdef RSEQ_COMPARE_TWICE
 368                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 369                 LONG_L " $4, %[v]\n\t"
 370                 "bne $4, %[expect], 7f\n\t"
 371 #endif
 372                 /* try memcpy */
 373                 "beqz %[len], 333f\n\t" \
 374                 "222:\n\t" \
 375                 "lb   $4, 0(%[src])\n\t" \
 376                 "sb   $4, 0(%[dst])\n\t" \
 377                 LONG_ADDI " %[src], 1\n\t" \
 378                 LONG_ADDI " %[dst], 1\n\t" \
 379                 LONG_ADDI " %[len], -1\n\t" \
 380                 "bnez %[len], 222b\n\t" \
 381                 "333:\n\t" \
 382                 RSEQ_INJECT_ASM(5)
 383 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 384                 "sync\n\t"      /* full sync provides store-release */
 385 #endif
 386                 /* final store */
 387                 LONG_S " %[newv], %[v]\n\t"
 388                 "2:\n\t"
 389                 RSEQ_INJECT_ASM(6)
 390                 /* teardown */
 391                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 392                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 393                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 394                 "b 8f\n\t"
 395                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 396                                       /* teardown */
 397                                       LONG_L " %[len], %[rseq_scratch2]\n\t"
 398                                       LONG_L " %[dst], %[rseq_scratch1]\n\t"
 399                                       LONG_L " %[src], %[rseq_scratch0]\n\t",
 400                                       abort, 1b, 2b, 4f)
 401                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 402                                         /* teardown */
 403                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 404                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 405                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 406                                         cmpfail)
 407 #ifdef RSEQ_COMPARE_TWICE
 408                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 409                                         /* teardown */
 410                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 411                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 412                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 413                                         error1)
 414                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 415                                         /* teardown */
 416                                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 417                                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 418                                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 419                                         error2)
 420 #endif
 421                 "8:\n\t"
 422                 : /* gcc asm goto does not allow outputs */
 423                 : [cpu_id]              "r" (cpu),
 424                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 425                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 426                   /* final store input */
 427                   [v]                   "m" (*v),
 428                   [expect]              "r" (expect),
 429                   [newv]                "r" (newv),
 430                   /* try memcpy input */
 431                   [dst]                 "r" (dst),
 432                   [src]                 "r" (src),
 433                   [len]                 "r" (len),
 434                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 435                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 436                   [rseq_scratch2]       "m" (rseq_scratch[2])
 437                   RSEQ_INJECT_INPUT
 438                 : "$4", "memory"
 439                   RSEQ_INJECT_CLOBBER
 440                 : abort, cmpfail
 441 #ifdef RSEQ_COMPARE_TWICE
 442                   , error1, error2
 443 #endif
 444         );
 445         return 0;
 446 abort:
 447         RSEQ_INJECT_FAILED
 448         return -1;
 449 cmpfail:
 450         return 1;
 451 #ifdef RSEQ_COMPARE_TWICE
 452 error1:
 453         rseq_bug("cpu_id comparison failed");
 454 error2:
 455         rseq_bug("expected value comparison failed");
 456 #endif
 457 }
 458
 459 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 460         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 461
 462 #include "rseq-bits-reset.h"