asm.c

   1 /*
   2  * Copyright (C) 2024 Mikulas Patocka
   3  *
   4  * This file is part of Ajla.
   5  *
   6  * Ajla is free software: you can redistribute it and/or modify it under the
   7  * terms of the GNU General Public License as published by the Free Software
   8  * Foundation, either version 3 of the License, or (at your option) any later
   9  * version.
  10  *
  11  * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
  12  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  13  * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * Ajla. If not, see <https://www.gnu.org/licenses/>.
  17  */
  18
  19 #include "ajla.h"
  20
  21 #include "str.h"
  22 #include "os_util.h"
  23 #include "arithm-b.h"
  24 #include "arithm-i.h"
  25 #include "arithm-r.h"
  26
  27 #include "asm.h"
  28
  29 #ifdef DEBUG_CRASH_HANDLER
  30 void *u_data_trap_lookup(void *ptr);
  31 void *c_data_trap_lookup(void *ptr);
  32 static void dump_registers(int sig, ucontext_t *uc)
  33 {
  34         int i;
  35 #if defined(ARCH_IA64)
  36         debug("%s at %lx", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.sc_ip);
  37         for (i = 0; i < 32; i += 2) {
  38                 debug("gr_%02x = %016lx gr_%02x = %016lx", i, uc->uc_mcontext.sc_gr[i], i + 1, uc->uc_mcontext.sc_gr[i + 1]);
  39         }
  40         /*for (i = 0; i < 32; i += 2) {
  41                 debug("gr_%02x = %016lx gr_%02x = %016lx", 32 + i, ptr[i - 16], 32 + i + 1, ptr[i + 1 - 16]);
  42         }*/
  43         for (i = 0; i < 8; i += 2) {
  44                 debug("br_%02x = %016lx br_%02x = %016lx", i, uc->uc_mcontext.sc_br[i], i + 1, uc->uc_mcontext.sc_br[i + 1]);
  45         }
  46 #endif
  47 #if defined(ARCH_MIPS)
  48         debug("%s at %llx", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.pc);
  49         for (i = 0; i < 32; i++)
  50                 debug("gpr_%d = %llx", i, uc->uc_mcontext.gregs[i]);
  51         call(data_trap_lookup)(num_to_ptr(uc->uc_mcontext.pc));
  52 #endif
  53 #if defined(ARCH_POWER)
  54         debug("%s at %lx", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.regs->nip);
  55         for (i = 0; i < 32; i++)
  56                 debug("gpr_%d = %lx", i, uc->uc_mcontext.regs->gpr[i]);
  57 #endif
  58 #if defined(ARCH_S390)
  59         debug("%s at %lx", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.psw.addr);
  60         for (i = 0; i < 16; i++)
  61                 debug("gpr_%d = %lx", i, uc->uc_mcontext.gregs[i]);
  62 #endif
  63 #if defined(__SH4__)
  64         debug("%s at %x", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.pc);
  65         for (i = 0; i < 16; i++)
  66                 debug("gpr_%d = %x", i, uc->uc_mcontext.gregs[i]);
  67         debug("pr = %x", uc->uc_mcontext.pr);
  68         debug("sr = %x", uc->uc_mcontext.sr);
  69         debug("gbr = %x", uc->uc_mcontext.gbr);
  70         debug("mach = %x", uc->uc_mcontext.mach);
  71         debug("macl = %x", uc->uc_mcontext.macl);
  72 #endif
  73 }
  74 static void crash(int sig, siginfo_t attr_unused *siginfo, void *ucontext)
  75 {
  76         dump_registers(sig, ucontext);
  77         _exit(127);
  78 }
  79 #endif
  80
  81 static const char * const cpu_feature_names[] = {
  82 #define ASM_INC_NAMES
  83 #include "asm.inc"
  84 #undef ASM_INC_NAMES
  85         NULL
  86 };
  87
  88 cpu_feature_mask_t cpu_feature_flags = 0;
  89 static bool detection_failed;
  90
  91
  92
  93 #ifdef ARCH_ALPHA
  94
  95 static uint32_t amask = 0;
  96 static void alpha_read_amask(void)
  97 {
  98         char *c;
  99         size_t cs;
 100         uint64_t (*fn)(void);
 101         uint64_t res;
 102         str_init(&c, &cs);
 103         str_add_hex(&c, &cs, "ffff1f20200ce0470180fa6b");
 104         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 105         res = fn();
 106         os_code_unmap(fn, cs);
 107         amask = ~res;
 108 }
 109
 110 #endif
 111
 112
 113
 114 #ifdef ARCH_ARM
 115
 116 #ifdef HAVE_SYS_AUXV_H
 117 #include <sys/auxv.h>
 118 #endif
 119
 120 static uint32_t elf_hwcap;
 121 static unsigned arm_arch;
 122
 123 static char *proc_get_field(const char *data, const char *field)
 124 {
 125 again:
 126         if (!strncmp(data, field, strlen(field))) {
 127                 const char *colon, *newline;
 128                 colon = strchr(data, ':');
 129                 newline = strchr(data, '\n');
 130                 if (!newline)
 131                         newline = strchr(data, 0);
 132                 if (!colon || colon > newline)
 133                         return NULL;
 134                 colon++;
 135                 while (*colon == ' ' || *colon == '\t')
 136                         colon++;
 137                 return str_dup(colon, newline - colon, NULL);
 138         }
 139         data = strchr(data, '\n');
 140         if (!data)
 141                 return NULL;
 142         data++;
 143         goto again;
 144 }
 145
 146 static void arm_read_caps(void)
 147 {
 148         ajla_error_t sink;
 149         char *data;
 150         char *arch;
 151         size_t i, l;
 152         unsigned long c = 0;
 153         elf_hwcap = c;
 154         arm_arch = ARM_VERSION;
 155 #if defined(HAVE_GETAUXVAL) && defined(AT_PLATFORM) && !defined(UNUSUAL)
 156         c = getauxval(AT_PLATFORM);
 157         if (c) {
 158                 const char *p = (const char *)c;
 159                 if (!strcmp(p, "aarch64")) {
 160                         arm_arch = 8;
 161                         goto got_arch;
 162                 }
 163                 if (p[0] != 'v')
 164                         goto no_aux_platform;
 165                 if (p[1] < '1' || p[1] > '9')
 166                         goto no_aux_platform;
 167                 arm_arch = atoi(&p[1]);
 168                 goto got_arch;
 169         }
 170         no_aux_platform:
 171 #endif
 172         if (!os_read_file("/proc/cpuinfo", &data, &l, &sink))
 173                 array_init(char, &data, &l);
 174         array_add(char, &data, &l, 0);
 175         arch = proc_get_field(data, "CPU architecture");
 176         if (arch) {
 177                 int a = atoi(arch);
 178                 if (a == 7) {
 179                         char *proc = proc_get_field(data, "Processor");
 180                         if (!proc) proc = proc_get_field(data, "model name");
 181                         if (proc) {
 182                                 if (strstr(proc, "(v6l)"))
 183                                         a = 6;
 184                                 mem_free(proc);
 185                         }
 186                 }
 187                 mem_free(arch);
 188                 if (a > 0) {
 189                         arm_arch = a;
 190                         goto got_arch_free;
 191                 }
 192         }
 193 #ifdef ARM_VERSION_UNKNOWN
 194         detection_failed = true;
 195 #endif
 196 got_arch_free:
 197         mem_free(data);
 198         goto got_arch;  /* avoid warning */
 199 got_arch:
 200 #if defined(HAVE_GETAUXVAL) && defined(AT_HWCAP) && !defined(UNUSUAL)
 201         c = getauxval(AT_HWCAP);
 202         if (c) {
 203                 elf_hwcap = c;
 204                 goto got_hwcap;
 205         }
 206 #endif
 207         if (!os_read_file("/proc/self/auxv", &data, &l, &sink))
 208                 array_init(char, &data, &l);
 209         for (i = 0; i + sizeof(unsigned long) * 2 > i && i + sizeof(unsigned long) * 2 <= l; i += sizeof(unsigned long) * 2) {
 210                 unsigned long tag = *(unsigned long *)(data + i);
 211                 unsigned long value = *(unsigned long *)(data + i + sizeof(unsigned long));
 212                 if (tag == 16) {
 213                         elf_hwcap = value;
 214                         goto got_hwcap_free;
 215                 }
 216         }
 217
 218         detection_failed = true;
 219 got_hwcap_free:
 220         mem_free(data);
 221         goto got_hwcap; /* avoid warning */
 222 got_hwcap:
 223         ;
 224         /*debug("arm arch %u, caps %x", arm_arch, elf_hwcap);*/
 225 }
 226
 227 #endif
 228
 229
 230 #ifdef ARCH_IA64
 231
 232 static uint64_t cpuid_4 = 0;
 233 static void ia64_read_cpuid(void)
 234 {
 235         void * volatile desc[2];
 236         char *c;
 237         size_t cs;
 238         str_init(&c, &cs);
 239         str_add_hex(&c, &cs, "0a4000401704000000020000000004001d000000010000000002008008008400");
 240         desc[0] = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 241         desc[1] = NULL;
 242         cpuid_4 = ((uint64_t (*)(uint64_t))desc)(4);
 243         os_code_unmap(desc[0], cs);
 244 }
 245
 246 #endif
 247
 248
 249 #ifdef ARCH_LOONGARCH64
 250
 251 static uint32_t cpucfg_1 = 0;
 252 static void loongarch_read_cpucfg(void)
 253 {
 254         char *c;
 255         size_t cs;
 256         uint64_t (*fn)(uint64_t);
 257         str_init(&c, &cs);
 258         str_add_hex(&c, &cs, "846c00002000004c");
 259         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 260         cpucfg_1 = fn(1);
 261         os_code_unmap(fn, cs);
 262 }
 263
 264 #endif
 265
 266
 267
 268 #ifdef ARCH_PARISC32
 269
 270 #include <unistd.h>
 271 static bool parisc_detect_20(void)
 272 {
 273 #if defined(__hpux) && defined(_SC_KERNEL_BITS)
 274         return sysconf(_SC_KERNEL_BITS) == 64;
 275 #else
 276         os_utsname_t un;
 277         os_get_uname(&un);
 278         return !strcasecmp(un.machine, "parisc64");
 279 #endif
 280 }
 281
 282 #endif
 283
 284
 285
 286 #ifdef ARCH_POWER
 287
 288 static void sigill(int attr_unused sig, siginfo_t attr_unused *siginfo, void *ucontext)
 289 {
 290         ucontext_t *uc = ucontext;
 291         uc->uc_mcontext.regs->nip += 4;
 292         uc->uc_mcontext.regs->gpr[3] = 0;
 293 }
 294
 295 static bool trap_insn(const char *hex)
 296 {
 297         char *c;
 298         size_t cs;
 299         size_t attr_unused i;
 300         int (*fn)(void);
 301         int ret;
 302         if (unlikely(!OS_SUPPORTS_TRAPS)) {
 303                 detection_failed = true;
 304                 return false;
 305         }
 306         str_init(&c, &cs);
 307         str_add_hex(&c, &cs, "38600001");
 308         str_add_hex(&c, &cs, hex);
 309         str_add_hex(&c, &cs, "4e800020");
 310 #if defined(C_LITTLE_ENDIAN)
 311         for (i = 0; i < cs; i += 4) {
 312                 char t;
 313                 t = c[i]; c[i] = c[i + 3]; c[i + 3] = t;
 314                 t = c[i + 1]; c[i + 1] = c[i + 2]; c[i + 2] = t;
 315         }
 316 #endif
 317         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 318 #ifdef _CALL_AIXDESC
 319         {
 320                 volatile uintptr_t desc[3];
 321                 desc[0] = ptr_to_num(fn);
 322                 desc[1] = 0;
 323                 desc[2] = 0;
 324                 ret = ((int (*)(void))desc)();
 325         }
 326 #else
 327         ret = fn();
 328 #endif
 329         os_code_unmap(fn, cs);
 330         /*debug("trap: %d", ret);*/
 331         return ret;
 332 }
 333
 334 #endif
 335
 336
 337
 338 #ifdef ARCH_RISCV64
 339
 340 static void sigill(int attr_unused sig, siginfo_t attr_unused *siginfo, void *ucontext)
 341 {
 342         ucontext_t *uc = ucontext;
 343         uc->uc_mcontext.__gregs[REG_PC] += 4;
 344         uc->uc_mcontext.__gregs[REG_A0] = 0;
 345 }
 346
 347 static bool trap_insn(const char *hex)
 348 {
 349         char *c;
 350         size_t cs;
 351         int (*fn)(void);
 352         int ret;
 353         if (unlikely(!OS_SUPPORTS_TRAPS)) {
 354                 detection_failed = true;
 355                 return false;
 356         }
 357         str_init(&c, &cs);
 358         str_add_hex(&c, &cs, "0545");
 359         str_add_hex(&c, &cs, hex);
 360         str_add_hex(&c, &cs, "8280");
 361         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 362         ret = fn();
 363         os_code_unmap(fn, cs);
 364         /*debug("trap: %d", ret);*/
 365         return ret;
 366 }
 367
 368 #endif
 369
 370
 371
 372 #ifdef ARCH_S390
 373
 374 static uint64_t s390_facilities[4];
 375
 376 static void s390_sigill(int attr_unused sig, siginfo_t attr_unused *siginfo, void *ucontext)
 377 {
 378         ucontext_t *uc = ucontext;
 379         uc->uc_mcontext.psw.addr += 4;
 380         detection_failed = true;
 381 }
 382
 383 static void s390_stfle(void)
 384 {
 385         char *c;
 386         size_t cs;
 387         void (*fn)(uint64_t *);
 388         memset(&s390_facilities, 0, sizeof s390_facilities);
 389         if (unlikely(!OS_SUPPORTS_TRAPS)) {
 390                 detection_failed = true;
 391                 return;
 392         }
 393         str_init(&c, &cs);
 394         str_add_hex(&c, &cs, "a7090003b2b0200007fe");
 395         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 396         os_signal_trap(SIGILL, s390_sigill);
 397         fn(s390_facilities);
 398         os_signal_untrap(SIGILL);
 399         os_code_unmap(fn, cs);
 400 #if 0
 401         debug("facilities0: %016llx", (unsigned long long)s390_facilities[0]);
 402         debug("facilities1: %016llx", (unsigned long long)s390_facilities[1]);
 403         debug("facilities2: %016llx", (unsigned long long)s390_facilities[2]);
 404         debug("facilities3: %016llx", (unsigned long long)s390_facilities[3]);
 405 #endif
 406 }
 407
 408 static bool test_facility(unsigned f)
 409 {
 410         return (s390_facilities[f >> 6] >> (~f & 63)) & 1;
 411 }
 412
 413 #endif
 414
 415
 416 #ifdef ARCH_RISCV64
 417
 418 static struct {
 419         int64_t key;
 420         uint64_t value;
 421 } riscv_hwp[2];
 422
 423 static void riscv64_syscall(void)
 424 {
 425         int r;
 426         memset(riscv_hwp, 0, sizeof riscv_hwp);
 427         riscv_hwp[0].key = 4;
 428         riscv_hwp[1].key = 5;
 429 #ifdef HAVE_SYSCALL
 430         EINTR_LOOP(r, syscall(258, riscv_hwp, n_array_elements(riscv_hwp), 0, NULL, 0));
 431 #else
 432         r = -1;
 433 #endif
 434         if (r == -1) {
 435                 riscv_hwp[0].key = -1;
 436                 riscv_hwp[1].key = -1;
 437         }
 438 }
 439
 440 #endif
 441
 442
 443 #ifdef ARCH_SPARC32
 444
 445 static bool sparc_detect_9(void)
 446 {
 447         os_utsname_t un;
 448         os_get_uname(&un);
 449         return !strcasecmp(un.machine, "sparc64");
 450 }
 451
 452 #endif
 453
 454
 455
 456 #ifdef ARCH_X86
 457
 458 #if !defined(ARCH_X86_32) || defined(__i686__) || defined(__athlon__) || defined(__SSE__)
 459 #define test_eflags_bits()      do { } while (0)
 460 #define eflags_bits             ((1U << 18) | (1U << 21))
 461 #else
 462 static uint32_t eflags_bits = 0;
 463 static void test_eflags_bits(void)
 464 {
 465         char *c;
 466         size_t cs;
 467         uint32_t (*fn)(void);
 468         sig_state_t set;
 469         str_init(&c, &cs);
 470         str_add_hex(&c, &cs, "b8000024009c330424509d9c583304249dc3");
 471         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 472         os_block_signals(&set);
 473         eflags_bits = fn();
 474         os_unblock_signals(&set);
 475         os_code_unmap(fn, cs);
 476 }
 477 #endif
 478
 479 static uint32_t cpuid_0[4];
 480 static uint32_t cpuid_1[4];
 481 static uint32_t cpuid_7[4];
 482 static uint32_t cpuid_8_0[4];
 483 static uint32_t cpuid_8_1[4];
 484
 485 static void do_cpuid(void)
 486 {
 487         char *c;
 488         size_t cs;
 489         void (*cpuid)(uint32_t level, uint32_t sublevel, uint32_t *result);
 490
 491         memset(cpuid_0, 0, sizeof cpuid_0);
 492         memset(cpuid_1, 0, sizeof cpuid_1);
 493         memset(cpuid_7, 0, sizeof cpuid_7);
 494         memset(cpuid_8_0, 0, sizeof cpuid_8_0);
 495         memset(cpuid_8_0, 1, sizeof cpuid_8_1);
 496         if (unlikely(!(eflags_bits & (1U << 21))))
 497                 return;
 498
 499         str_init(&c, &cs);
 500 #if defined(ARCH_X86_32)
 501         str_add_hex(&c, &cs, "53568b44240c8b4c24100fa28b7424148906895e04894e0889560c5e5bc3");
 502 #elif defined(ARCH_X86_64) && defined(ARCH_X86_WIN_ABI)
 503         str_add_hex(&c, &cs, "5389c889d10fa241890041895804418948084189500c5bc3");
 504 #elif defined(ARCH_X86_64)
 505         str_add_hex(&c, &cs, "5389f889f14889d60fa28906895e04894e0889560c5bc3");
 506 #elif defined(ARCH_X86_X32)
 507         str_add_hex(&c, &cs, "5389f889f189d60fa28906895e04894e0889560c5bc3");
 508 #else
 509         unknown arch
 510 #endif
 511         cpuid = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 512
 513         cpuid(0, 0, cpuid_0);
 514         if (likely(cpuid_0[0] >= 1))
 515                 cpuid(1, 0, cpuid_1);
 516         if (likely(cpuid_0[0] >= 7))
 517                 cpuid(7, 0, cpuid_7);
 518         cpuid(0x80000000, 0, cpuid_8_0);
 519         if (likely((cpuid_8_0[0] & 0xffff0000) == 0x80000000)) {
 520                 if (likely(cpuid_8_0[0] >= 0x80000001))
 521                         cpuid(0x80000001, 0, cpuid_8_1);
 522         }
 523
 524         os_code_unmap(cpuid, cs);
 525 }
 526
 527 static bool test_fxsave(void)
 528 {
 529 #if defined(ARCH_X86_32)
 530         bool supported;
 531         unsigned char space[1024 + 15] = "";    /* avoid warning */
 532         unsigned char *mem = space + (-ptr_to_num(space) & 15);
 533
 534         char *c;
 535         size_t cs;
 536         void (*fn)(unsigned char *ptr);
 537
 538         mem[160] = 0;
 539         mem[160 + 512] = 0;
 540
 541         str_init(&c, &cs);
 542         str_add_hex(&c, &cs, "8b4424040fae00fe80a00000000fae080fae8000020000c3");
 543         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 544         fn(mem);
 545         os_code_unmap(fn, cs);
 546         supported = mem[160] == mem[160 + 512];
 547         return supported;
 548 #else
 549         return true;
 550 #endif
 551 }
 552
 553 static bool test_xcr0(unsigned mask)
 554 {
 555         char *c;
 556         size_t cs;
 557         uint32_t (*fn)(void);
 558         uint32_t res;
 559         str_init(&c, &cs);
 560         str_add_hex(&c, &cs, "31c90f01d0c3");
 561         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 562         res = fn();
 563         os_code_unmap(fn, cs);
 564         return (res & mask) == mask;
 565 }
 566
 567 #endif
 568
 569
 570
 571 attr_noinline void asm_setup_thread(void)
 572 {
 573 #if defined(INLINE_ASM_GCC_X86)
 574 #if 0
 575         {
 576                 unsigned short fpcw;
 577                 unsigned mxcsr;
 578                 __asm__ volatile("fstcw %0" : "=m"(fpcw));
 579                 __asm__ volatile("stmxcsr %0" : "=m"(mxcsr));
 580                 debug("fpcw: %x, mxcsr: %x", fpcw, mxcsr);
 581         }
 582 #endif
 583         {
 584                 unsigned short fpcw = 0x37f;
 585                 __asm__ volatile("fldcw %0" : : "m"(fpcw));
 586         }
 587 #if defined(HAVE_X86_ASSEMBLER_SSE)
 588         if (likely(cpu_test_feature(CPU_FEATURE_sse))) {
 589                 unsigned mxcsr = 0x1f80;
 590                 __asm__ volatile("ldmxcsr %0" : : "m"(mxcsr));
 591         }
 592 #endif
 593 #endif
 594 #if defined(INLINE_ASM_GCC_ARM)
 595         __asm__ volatile (ARM_ASM_PREFIX "vmsr fpscr, %0" : : "r"(0));
 596 #endif
 597 #if defined(INLINE_ASM_GCC_ARM64)
 598         __asm__ volatile (ARM_ASM_PREFIX "msr fpcr, %0" : : "r"(0UL));
 599 #endif
 600 }
 601
 602
 603 static const struct {
 604         code_t orig;
 605         code_t alt;
 606         cpu_feature_mask_t mask;
 607 } code_alttable[] = {
 608 #define EMIT_ALTTABLE(orig, alt, flags) { orig, alt, flags },
 609 #include "ipret.inc"
 610         { 0, 0, 0 }
 611 };
 612 #define code_alttable_n         (n_array_elements(code_alttable) - 1)
 613
 614 code_t code_alt(code_t code)
 615 {
 616         code_t ret;
 617         size_t s;
 618         binary_search(size_t, code_alttable_n, s, false, code_alttable[s].orig < code, break);
 619         ret = code;
 620         for (; s < code_alttable_n + uzero && code_alttable[s].orig == code; s++) {
 621                 if ((cpu_feature_flags & code_alttable[s].mask) == code_alttable[s].mask)
 622                         ret = code_alttable[s].alt;
 623         }
 624         /*if (code != ret) debug("code alt: %x -> %x", code, ret);*/
 625         return ret;
 626 }
 627
 628 #ifdef DEBUG_BIST
 629 static attr_noinline void verify_alttable(void)
 630 {
 631         int i;
 632         /*debug("Alttable size: %x", (unsigned)code_alttable_n);*/
 633         for (i = 0; i < (int)code_alttable_n - 1; i++) {
 634                 /*debug("Alttable: %x -> %x", code_alttable[i].orig, code_alttable[i].alt);*/
 635                 if (unlikely(code_alttable[i].orig > code_alttable[i + 1].orig))
 636                         internal(file_line, "verify_alttable: code_alttable is not sorted: 0x%x > 0x%x @ %d", (unsigned)code_alttable[i].orig, (unsigned)code_alttable[i + 1].orig, i);
 637                 /*code_alt(code_alttable[i].orig);*/
 638         }
 639 }
 640 #else
 641 #define verify_alttable()               do { } while (0)
 642 #endif
 643
 644
 645 void asm_init(void)
 646 {
 647         bool trap_sigill = false;
 648         uint32_t missing_features;
 649
 650         verify_alttable();
 651
 652         detection_failed = false;
 653 #ifdef ARCH_ALPHA
 654         alpha_read_amask();
 655 #endif
 656 #ifdef ARCH_ARM
 657         arm_read_caps();
 658 #endif
 659 #ifdef ARCH_IA64
 660         ia64_read_cpuid();
 661 #endif
 662 #ifdef ARCH_LOONGARCH64
 663         loongarch_read_cpucfg();
 664 #endif
 665 #ifdef ARCH_POWER
 666         trap_sigill = true;
 667 #endif
 668 #ifdef ARCH_S390
 669         s390_stfle();
 670 #endif
 671 #ifdef ARCH_RISCV64
 672         riscv64_syscall();
 673         if (riscv_hwp[0].key < 0)
 674                 trap_sigill = true;
 675 #endif
 676 #ifdef ARCH_X86
 677         test_eflags_bits();
 678         do_cpuid();
 679 #endif
 680         if (trap_sigill) {
 681 #if defined(ARCH_POWER) || defined(ARCH_RISCV64)
 682                 os_signal_trap(SIGILL, sigill);
 683 #endif
 684 }
 685 #define ASM_INC_DYNAMIC
 686 #include "asm.inc"
 687 #undef ASM_INC_DYNAMIC
 688         if (trap_sigill) {
 689 #if defined(ARCH_POWER) || defined(ARCH_RISCV64)
 690                 os_signal_untrap(SIGILL);
 691 #endif
 692         }
 693         if (unlikely(detection_failed))
 694                 cpu_feature_flags |= cpu_feature_static_flags;
 695         missing_features = cpu_feature_static_flags & ~cpu_feature_flags;
 696         if (unlikely(missing_features != 0)) {
 697                 int first;
 698                 int f;
 699                 char *error;
 700                 size_t error_l;
 701                 str_init(&error, &error_l);
 702                 str_add_string(&error, &error_l, "CPU doesn't have the following features: ");
 703                 for (first = 1, f = 0; missing_features; missing_features >>= 1, f++) {
 704                         if (missing_features & 1) {
 705                                 if (!first)
 706                                         str_add_string(&error, &error_l, ", ");
 707                                 first = 0;
 708                                 str_add_string(&error, &error_l, cpu_feature_names[f]);
 709                         }
 710                 }
 711                 str_finish(&error, &error_l);
 712                 fatal("%s", error);
 713         }
 714 #ifdef DEBUG_ENV
 715         {
 716                 char *str = getenv("CPU_FLAGS");
 717                 if (str && *str) {
 718                         char *e;
 719                         cpu_feature_flags = strtoul(str, &e, 16);
 720                         if (unlikely(*e))
 721                                 fatal("invalid CPU_FLAGS");
 722                 }
 723         }
 724 #endif
 725 #ifdef DEBUG_ENV
 726         if (getenv("PRINT_FLAGS")) {
 727                 debug("static flags: %x", cpu_feature_static_flags);
 728                 debug("dynamic flags: %x", cpu_feature_flags);
 729         }
 730 #endif
 731         asm_setup_thread();
 732
 733 #ifdef DEBUG_CRASH_HANDLER
 734         os_signal_trap(SIGSEGV, crash);
 735         os_signal_trap(SIGBUS, crash);
 736         os_signal_trap(SIGILL, crash);
 737 #endif
 738 }
 739
 740 void asm_done(void)
 741 {
 742 #ifdef DEBUG_CRASH_HANDLER
 743         os_signal_untrap(SIGSEGV);
 744         os_signal_untrap(SIGBUS);
 745         os_signal_untrap(SIGILL);
 746 #endif
 747 }