asm.c

   1 /*
   2  * Copyright (C) 2024 Mikulas Patocka
   3  *
   4  * This file is part of Ajla.
   5  *
   6  * Ajla is free software: you can redistribute it and/or modify it under the
   7  * terms of the GNU General Public License as published by the Free Software
   8  * Foundation, either version 3 of the License, or (at your option) any later
   9  * version.
  10  *
  11  * Ajla is distributed in the hope that it will be useful, but WITHOUT ANY
  12  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  13  * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along with
  16  * Ajla. If not, see <https://www.gnu.org/licenses/>.
  17  */
  18
  19 #include "ajla.h"
  20
  21 #include "str.h"
  22 #include "os_util.h"
  23 #include "arithm-b.h"
  24 #include "arithm-i.h"
  25 #include "arithm-r.h"
  26
  27 #include "asm.h"
  28
  29 #ifdef DEBUG_CRASH_HANDLER
  30 void *u_data_trap_lookup(void *ptr);
  31 void *c_data_trap_lookup(void *ptr);
  32 static void dump_registers(int sig, ucontext_t *uc)
  33 {
  34         int i;
  35 #if defined(ARCH_IA64)
  36         debug("%s at %lx", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.sc_ip);
  37         for (i = 0; i < 32; i += 2) {
  38                 debug("gr_%02x = %016lx gr_%02x = %016lx", i, uc->uc_mcontext.sc_gr[i], i + 1, uc->uc_mcontext.sc_gr[i + 1]);
  39         }
  40         /*for (i = 0; i < 32; i += 2) {
  41                 debug("gr_%02x = %016lx gr_%02x = %016lx", 32 + i, ptr[i - 16], 32 + i + 1, ptr[i + 1 - 16]);
  42         }*/
  43         for (i = 0; i < 8; i += 2) {
  44                 debug("br_%02x = %016lx br_%02x = %016lx", i, uc->uc_mcontext.sc_br[i], i + 1, uc->uc_mcontext.sc_br[i + 1]);
  45         }
  46 #endif
  47 #if defined(ARCH_MIPS)
  48         debug("%s at %llx", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.pc);
  49         for (i = 0; i < 32; i++)
  50                 debug("gpr_%d = %llx", i, uc->uc_mcontext.gregs[i]);
  51         call(data_trap_lookup)(num_to_ptr(uc->uc_mcontext.pc));
  52 #endif
  53 #if defined(ARCH_POWER)
  54         debug("%s at %lx", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.regs->nip);
  55         for (i = 0; i < 32; i++)
  56                 debug("gpr_%d = %lx", i, uc->uc_mcontext.regs->gpr[i]);
  57 #endif
  58 #if defined(__SH4__)
  59         debug("%s at %x", sig == SIGSEGV ? "sigsegv" : sig == SIGBUS ? "sigbus" : "sigill", uc->uc_mcontext.pc);
  60         for (i = 0; i < 16; i++)
  61                 debug("gpr_%d = %x", i, uc->uc_mcontext.gregs[i]);
  62         debug("pr = %x", uc->uc_mcontext.pr);
  63         debug("sr = %x", uc->uc_mcontext.sr);
  64         debug("gbr = %x", uc->uc_mcontext.gbr);
  65         debug("mach = %x", uc->uc_mcontext.mach);
  66         debug("macl = %x", uc->uc_mcontext.macl);
  67 #endif
  68 }
  69 static void crash(int sig, siginfo_t attr_unused *siginfo, void *ucontext)
  70 {
  71         dump_registers(sig, ucontext);
  72         _exit(127);
  73 }
  74 #endif
  75
  76 static const char * const cpu_feature_names[] = {
  77 #define ASM_INC_NAMES
  78 #include "asm.inc"
  79 #undef ASM_INC_NAMES
  80         NULL
  81 };
  82
  83 cpu_feature_mask_t cpu_feature_flags = 0;
  84 static bool detection_failed;
  85
  86
  87
  88 #ifdef ARCH_ALPHA
  89
  90 static uint32_t amask = 0;
  91 static void alpha_read_amask(void)
  92 {
  93         char *c;
  94         size_t cs;
  95         uint64_t (*fn)(void);
  96         uint64_t res;
  97         str_init(&c, &cs);
  98         str_add_hex(&c, &cs, "ffff1f20200ce0470180fa6b");
  99         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 100         res = fn();
 101         os_code_unmap(fn, cs);
 102         amask = ~res;
 103 }
 104
 105 #endif
 106
 107
 108
 109 #ifdef ARCH_ARM
 110
 111 #ifdef HAVE_SYS_AUXV_H
 112 #include <sys/auxv.h>
 113 #endif
 114
 115 static uint32_t elf_hwcap;
 116 static unsigned arm_arch;
 117
 118 static char *proc_get_field(const char *data, const char *field)
 119 {
 120 again:
 121         if (!strncmp(data, field, strlen(field))) {
 122                 const char *colon, *newline;
 123                 colon = strchr(data, ':');
 124                 newline = strchr(data, '\n');
 125                 if (!newline)
 126                         newline = strchr(data, 0);
 127                 if (!colon || colon > newline)
 128                         return NULL;
 129                 colon++;
 130                 while (*colon == ' ' || *colon == '\t')
 131                         colon++;
 132                 return str_dup(colon, newline - colon, NULL);
 133         }
 134         data = strchr(data, '\n');
 135         if (!data)
 136                 return NULL;
 137         data++;
 138         goto again;
 139 }
 140
 141 static void arm_read_caps(void)
 142 {
 143         ajla_error_t sink;
 144         char *data;
 145         char *arch;
 146         size_t i, l;
 147         unsigned long c = 0;
 148         elf_hwcap = c;
 149         arm_arch = ARM_VERSION;
 150 #if defined(HAVE_GETAUXVAL) && defined(AT_PLATFORM) && !defined(UNUSUAL)
 151         c = getauxval(AT_PLATFORM);
 152         if (c) {
 153                 const char *p = (const char *)c;
 154                 if (!strcmp(p, "aarch64")) {
 155                         arm_arch = 8;
 156                         goto got_arch;
 157                 }
 158                 if (p[0] != 'v')
 159                         goto no_aux_platform;
 160                 if (p[1] < '1' || p[1] > '9')
 161                         goto no_aux_platform;
 162                 arm_arch = atoi(&p[1]);
 163                 goto got_arch;
 164         }
 165         no_aux_platform:
 166 #endif
 167         if (!os_read_file("/proc/cpuinfo", &data, &l, &sink))
 168                 array_init(char, &data, &l);
 169         array_add(char, &data, &l, 0);
 170         arch = proc_get_field(data, "CPU architecture");
 171         if (arch) {
 172                 int a = atoi(arch);
 173                 if (a == 7) {
 174                         char *proc = proc_get_field(data, "Processor");
 175                         if (!proc) proc = proc_get_field(data, "model name");
 176                         if (proc) {
 177                                 if (strstr(proc, "(v6l)"))
 178                                         a = 6;
 179                                 mem_free(proc);
 180                         }
 181                 }
 182                 mem_free(arch);
 183                 if (a > 0) {
 184                         arm_arch = a;
 185                         goto got_arch_free;
 186                 }
 187         }
 188 #ifdef ARM_VERSION_UNKNOWN
 189         detection_failed = true;
 190 #endif
 191 got_arch_free:
 192         mem_free(data);
 193         goto got_arch;  /* avoid warning */
 194 got_arch:
 195 #if defined(HAVE_GETAUXVAL) && defined(AT_HWCAP) && !defined(UNUSUAL)
 196         c = getauxval(AT_HWCAP);
 197         if (c) {
 198                 elf_hwcap = c;
 199                 goto got_hwcap;
 200         }
 201 #endif
 202         if (!os_read_file("/proc/self/auxv", &data, &l, &sink))
 203                 array_init(char, &data, &l);
 204         for (i = 0; i + sizeof(unsigned long) * 2 > i && i + sizeof(unsigned long) * 2 <= l; i += sizeof(unsigned long) * 2) {
 205                 unsigned long tag = *(unsigned long *)(data + i);
 206                 unsigned long value = *(unsigned long *)(data + i + sizeof(unsigned long));
 207                 if (tag == 16) {
 208                         elf_hwcap = value;
 209                         goto got_hwcap_free;
 210                 }
 211         }
 212
 213         detection_failed = true;
 214 got_hwcap_free:
 215         mem_free(data);
 216         goto got_hwcap; /* avoid warning */
 217 got_hwcap:
 218         ;
 219         /*debug("arm arch %u, caps %x", arm_arch, elf_hwcap);*/
 220 }
 221
 222 #endif
 223
 224
 225 #ifdef ARCH_IA64
 226
 227 static uint64_t cpuid_4 = 0;
 228 static void ia64_read_cpuid(void)
 229 {
 230         void * volatile desc[2];
 231         char *c;
 232         size_t cs;
 233         str_init(&c, &cs);
 234         str_add_hex(&c, &cs, "0a4000401704000000020000000004001d000000010000000002008008008400");
 235         desc[0] = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 236         desc[1] = NULL;
 237         cpuid_4 = ((uint64_t (*)(uint64_t))desc)(4);
 238         os_code_unmap(desc[0], cs);
 239 }
 240
 241 #endif
 242
 243
 244 #ifdef ARCH_LOONGARCH64
 245
 246 static uint32_t cpucfg_1 = 0;
 247 static void loongarch_read_cpucfg(void)
 248 {
 249         char *c;
 250         size_t cs;
 251         uint64_t (*fn)(uint64_t);
 252         str_init(&c, &cs);
 253         str_add_hex(&c, &cs, "846c00002000004c");
 254         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 255         cpucfg_1 = fn(1);
 256         os_code_unmap(fn, cs);
 257 }
 258
 259 #endif
 260
 261
 262
 263 #ifdef ARCH_PARISC32
 264
 265 #include <unistd.h>
 266 static bool parisc_detect_20(void)
 267 {
 268 #if defined(__hpux) && defined(_SC_KERNEL_BITS)
 269         return sysconf(_SC_KERNEL_BITS) == 64;
 270 #else
 271         os_utsname_t un;
 272         os_get_uname(&un);
 273         return !strcasecmp(un.machine, "parisc64");
 274 #endif
 275 }
 276
 277 #endif
 278
 279
 280
 281 #ifdef ARCH_POWER
 282
 283 static void sigill(int attr_unused sig, siginfo_t attr_unused *siginfo, void *ucontext)
 284 {
 285         ucontext_t *uc = ucontext;
 286         uc->uc_mcontext.regs->nip += 4;
 287         uc->uc_mcontext.regs->gpr[3] = 0;
 288 }
 289
 290 static bool trap_insn(const char *hex)
 291 {
 292         char *c;
 293         size_t cs;
 294         size_t attr_unused i;
 295         int (*fn)(void);
 296         int ret;
 297         if (unlikely(!OS_SUPPORTS_TRAPS)) {
 298                 detection_failed = true;
 299                 return false;
 300         }
 301         str_init(&c, &cs);
 302         str_add_hex(&c, &cs, "38600001");
 303         str_add_hex(&c, &cs, hex);
 304         str_add_hex(&c, &cs, "4e800020");
 305 #if defined(C_LITTLE_ENDIAN)
 306         for (i = 0; i < cs; i += 4) {
 307                 char t;
 308                 t = c[i]; c[i] = c[i + 3]; c[i + 3] = t;
 309                 t = c[i + 1]; c[i + 1] = c[i + 2]; c[i + 2] = t;
 310         }
 311 #endif
 312         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 313 #ifdef _CALL_AIXDESC
 314         {
 315                 volatile uintptr_t desc[3];
 316                 desc[0] = ptr_to_num(fn);
 317                 desc[1] = 0;
 318                 desc[2] = 0;
 319                 ret = ((int (*)(void))desc)();
 320         }
 321 #else
 322         ret = fn();
 323 #endif
 324         os_code_unmap(fn, cs);
 325         /*debug("trap: %d", ret);*/
 326         return ret;
 327 }
 328
 329 #endif
 330
 331
 332
 333 #ifdef ARCH_RISCV64
 334
 335 static void sigill(int attr_unused sig, siginfo_t attr_unused *siginfo, void *ucontext)
 336 {
 337         ucontext_t *uc = ucontext;
 338         uc->uc_mcontext.__gregs[REG_PC] += 4;
 339         uc->uc_mcontext.__gregs[REG_A0] = 0;
 340 }
 341
 342 static bool trap_insn(const char *hex)
 343 {
 344         char *c;
 345         size_t cs;
 346         int (*fn)(void);
 347         int ret;
 348         if (unlikely(!OS_SUPPORTS_TRAPS)) {
 349                 detection_failed = true;
 350                 return false;
 351         }
 352         str_init(&c, &cs);
 353         str_add_hex(&c, &cs, "0545");
 354         str_add_hex(&c, &cs, hex);
 355         str_add_hex(&c, &cs, "8280");
 356         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 357         ret = fn();
 358         os_code_unmap(fn, cs);
 359         /*debug("trap: %d", ret);*/
 360         return ret;
 361 }
 362
 363 #endif
 364
 365
 366
 367 #ifdef ARCH_S390
 368
 369 static uint64_t s390_facilities[4];
 370
 371 static void s390_sigill(int attr_unused sig, siginfo_t attr_unused *siginfo, void *ucontext)
 372 {
 373         ucontext_t *uc = ucontext;
 374         uc->uc_mcontext.psw.addr += 4;
 375         detection_failed = true;
 376 }
 377
 378 static void s390_stfle(void)
 379 {
 380         char *c;
 381         size_t cs;
 382         void (*fn)(uint64_t *);
 383         memset(&s390_facilities, 0, sizeof s390_facilities);
 384         if (unlikely(!OS_SUPPORTS_TRAPS)) {
 385                 detection_failed = true;
 386                 return;
 387         }
 388         str_init(&c, &cs);
 389         str_add_hex(&c, &cs, "a7090003b2b0200007fe");
 390         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 391         os_signal_trap(SIGILL, s390_sigill);
 392         fn(s390_facilities);
 393         os_signal_untrap(SIGILL);
 394         os_code_unmap(fn, cs);
 395 #if 0
 396         debug("facilities0: %016llx", (unsigned long long)s390_facilities[0]);
 397         debug("facilities1: %016llx", (unsigned long long)s390_facilities[1]);
 398         debug("facilities2: %016llx", (unsigned long long)s390_facilities[2]);
 399         debug("facilities3: %016llx", (unsigned long long)s390_facilities[3]);
 400 #endif
 401 }
 402
 403 static bool test_facility(unsigned f)
 404 {
 405         return (s390_facilities[f >> 6] >> (~f & 63)) & 1;
 406 }
 407
 408 #endif
 409
 410
 411 #ifdef ARCH_RISCV64
 412
 413 static struct {
 414         int64_t key;
 415         uint64_t value;
 416 } riscv_hwp[2];
 417
 418 static void riscv64_syscall(void)
 419 {
 420         int r;
 421         memset(riscv_hwp, 0, sizeof riscv_hwp);
 422         riscv_hwp[0].key = 4;
 423         riscv_hwp[1].key = 5;
 424 #ifdef HAVE_SYSCALL
 425         EINTR_LOOP(r, syscall(258, riscv_hwp, n_array_elements(riscv_hwp), 0, NULL, 0));
 426 #else
 427         r = -1;
 428 #endif
 429         if (r == -1) {
 430                 riscv_hwp[0].key = -1;
 431                 riscv_hwp[1].key = -1;
 432         }
 433 }
 434
 435 #endif
 436
 437
 438 #ifdef ARCH_SPARC32
 439
 440 static bool sparc_detect_9(void)
 441 {
 442         os_utsname_t un;
 443         os_get_uname(&un);
 444         return !strcasecmp(un.machine, "sparc64");
 445 }
 446
 447 #endif
 448
 449
 450
 451 #ifdef ARCH_X86
 452
 453 #if !defined(ARCH_X86_32) || defined(__i686__) || defined(__athlon__) || defined(__SSE__)
 454 #define test_eflags_bits()      do { } while (0)
 455 #define eflags_bits             ((1U << 18) | (1U << 21))
 456 #else
 457 static uint32_t eflags_bits = 0;
 458 static void test_eflags_bits(void)
 459 {
 460         char *c;
 461         size_t cs;
 462         uint32_t (*fn)(void);
 463         sig_state_t set;
 464         str_init(&c, &cs);
 465         str_add_hex(&c, &cs, "b8000024009c330424509d9c583304249dc3");
 466         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 467         os_block_signals(&set);
 468         eflags_bits = fn();
 469         os_unblock_signals(&set);
 470         os_code_unmap(fn, cs);
 471 }
 472 #endif
 473
 474 static uint32_t cpuid_0[4];
 475 static uint32_t cpuid_1[4];
 476 static uint32_t cpuid_7[4];
 477 static uint32_t cpuid_8_0[4];
 478 static uint32_t cpuid_8_1[4];
 479
 480 static void do_cpuid(void)
 481 {
 482         char *c;
 483         size_t cs;
 484         void (*cpuid)(uint32_t level, uint32_t sublevel, uint32_t *result);
 485
 486         memset(cpuid_0, 0, sizeof cpuid_0);
 487         memset(cpuid_1, 0, sizeof cpuid_1);
 488         memset(cpuid_7, 0, sizeof cpuid_7);
 489         memset(cpuid_8_0, 0, sizeof cpuid_8_0);
 490         memset(cpuid_8_0, 1, sizeof cpuid_8_1);
 491         if (unlikely(!(eflags_bits & (1U << 21))))
 492                 return;
 493
 494         str_init(&c, &cs);
 495 #if defined(ARCH_X86_32)
 496         str_add_hex(&c, &cs, "53568b44240c8b4c24100fa28b7424148906895e04894e0889560c5e5bc3");
 497 #elif defined(ARCH_X86_64) && defined(ARCH_X86_WIN_ABI)
 498         str_add_hex(&c, &cs, "5389c889d10fa241890041895804418948084189500c5bc3");
 499 #elif defined(ARCH_X86_64)
 500         str_add_hex(&c, &cs, "5389f889f14889d60fa28906895e04894e0889560c5bc3");
 501 #elif defined(ARCH_X86_X32)
 502         str_add_hex(&c, &cs, "5389f889f189d60fa28906895e04894e0889560c5bc3");
 503 #else
 504         unknown arch
 505 #endif
 506         cpuid = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 507
 508         cpuid(0, 0, cpuid_0);
 509         if (likely(cpuid_0[0] >= 1))
 510                 cpuid(1, 0, cpuid_1);
 511         if (likely(cpuid_0[0] >= 7))
 512                 cpuid(7, 0, cpuid_7);
 513         cpuid(0x80000000, 0, cpuid_8_0);
 514         if (likely((cpuid_8_0[0] & 0xffff0000) == 0x80000000)) {
 515                 if (likely(cpuid_8_0[0] >= 0x80000001))
 516                         cpuid(0x80000001, 0, cpuid_8_1);
 517         }
 518
 519         os_code_unmap(cpuid, cs);
 520 }
 521
 522 static bool test_fxsave(void)
 523 {
 524 #if defined(ARCH_X86_32)
 525         bool supported;
 526         unsigned char space[1024 + 15] = "";    /* avoid warning */
 527         unsigned char *mem = space + (-ptr_to_num(space) & 15);
 528
 529         char *c;
 530         size_t cs;
 531         void (*fn)(unsigned char *ptr);
 532
 533         mem[160] = 0;
 534         mem[160 + 512] = 0;
 535
 536         str_init(&c, &cs);
 537         str_add_hex(&c, &cs, "8b4424040fae00fe80a00000000fae080fae8000020000c3");
 538         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 539         fn(mem);
 540         os_code_unmap(fn, cs);
 541         supported = mem[160] == mem[160 + 512];
 542         return supported;
 543 #else
 544         return true;
 545 #endif
 546 }
 547
 548 static bool test_xcr0(unsigned mask)
 549 {
 550         char *c;
 551         size_t cs;
 552         uint32_t (*fn)(void);
 553         uint32_t res;
 554         str_init(&c, &cs);
 555         str_add_hex(&c, &cs, "31c90f01d0c3");
 556         fn = os_code_map(cast_ptr(uint8_t *, c), cs, NULL);
 557         res = fn();
 558         os_code_unmap(fn, cs);
 559         return (res & mask) == mask;
 560 }
 561
 562 #endif
 563
 564
 565
 566 attr_noinline void asm_setup_thread(void)
 567 {
 568 #if defined(INLINE_ASM_GCC_X86)
 569 #if 0
 570         {
 571                 unsigned short fpcw;
 572                 unsigned mxcsr;
 573                 __asm__ volatile("fstcw %0" : "=m"(fpcw));
 574                 __asm__ volatile("stmxcsr %0" : "=m"(mxcsr));
 575                 debug("fpcw: %x, mxcsr: %x", fpcw, mxcsr);
 576         }
 577 #endif
 578         {
 579                 unsigned short fpcw = 0x37f;
 580                 __asm__ volatile("fldcw %0" : : "m"(fpcw));
 581         }
 582 #if defined(HAVE_X86_ASSEMBLER_SSE)
 583         if (likely(cpu_test_feature(CPU_FEATURE_sse))) {
 584                 unsigned mxcsr = 0x1f80;
 585                 __asm__ volatile("ldmxcsr %0" : : "m"(mxcsr));
 586         }
 587 #endif
 588 #endif
 589 #if defined(INLINE_ASM_GCC_ARM)
 590         __asm__ volatile (ARM_ASM_PREFIX "vmsr fpscr, %0" : : "r"(0));
 591 #endif
 592 #if defined(INLINE_ASM_GCC_ARM64)
 593         __asm__ volatile (ARM_ASM_PREFIX "msr fpcr, %0" : : "r"(0UL));
 594 #endif
 595 }
 596
 597
 598 static const struct {
 599         code_t orig;
 600         code_t alt;
 601         cpu_feature_mask_t mask;
 602 } code_alttable[] = {
 603 #define EMIT_ALTTABLE(orig, alt, flags) { orig, alt, flags },
 604 #include "ipret.inc"
 605         { 0, 0, 0 }
 606 };
 607 #define code_alttable_n         (n_array_elements(code_alttable) - 1)
 608
 609 code_t code_alt(code_t code)
 610 {
 611         code_t ret;
 612         size_t s;
 613         binary_search(size_t, code_alttable_n, s, false, code_alttable[s].orig < code, break);
 614         ret = code;
 615         for (; s < code_alttable_n + uzero && code_alttable[s].orig == code; s++) {
 616                 if ((cpu_feature_flags & code_alttable[s].mask) == code_alttable[s].mask)
 617                         ret = code_alttable[s].alt;
 618         }
 619         /*if (code != ret) debug("code alt: %x -> %x", code, ret);*/
 620         return ret;
 621 }
 622
 623 #ifdef DEBUG_BIST
 624 static attr_noinline void verify_alttable(void)
 625 {
 626         int i;
 627         /*debug("Alttable size: %x", (unsigned)code_alttable_n);*/
 628         for (i = 0; i < (int)code_alttable_n - 1; i++) {
 629                 /*debug("Alttable: %x -> %x", code_alttable[i].orig, code_alttable[i].alt);*/
 630                 if (unlikely(code_alttable[i].orig > code_alttable[i + 1].orig))
 631                         internal(file_line, "verify_alttable: code_alttable is not sorted: 0x%x > 0x%x @ %d", (unsigned)code_alttable[i].orig, (unsigned)code_alttable[i + 1].orig, i);
 632                 /*code_alt(code_alttable[i].orig);*/
 633         }
 634 }
 635 #else
 636 #define verify_alttable()               do { } while (0)
 637 #endif
 638
 639
 640 void asm_init(void)
 641 {
 642         bool trap_sigill = false;
 643         uint32_t missing_features;
 644
 645         verify_alttable();
 646
 647         detection_failed = false;
 648 #ifdef ARCH_ALPHA
 649         alpha_read_amask();
 650 #endif
 651 #ifdef ARCH_ARM
 652         arm_read_caps();
 653 #endif
 654 #ifdef ARCH_IA64
 655         ia64_read_cpuid();
 656 #endif
 657 #ifdef ARCH_LOONGARCH64
 658         loongarch_read_cpucfg();
 659 #endif
 660 #ifdef ARCH_POWER
 661         trap_sigill = true;
 662 #endif
 663 #ifdef ARCH_S390
 664         s390_stfle();
 665 #endif
 666 #ifdef ARCH_RISCV64
 667         riscv64_syscall();
 668         if (riscv_hwp[0].key < 0)
 669                 trap_sigill = true;
 670 #endif
 671 #ifdef ARCH_X86
 672         test_eflags_bits();
 673         do_cpuid();
 674 #endif
 675         if (trap_sigill) {
 676 #if defined(ARCH_POWER) || defined(ARCH_RISCV64)
 677                 os_signal_trap(SIGILL, sigill);
 678 #endif
 679 }
 680 #define ASM_INC_DYNAMIC
 681 #include "asm.inc"
 682 #undef ASM_INC_DYNAMIC
 683         if (trap_sigill) {
 684 #if defined(ARCH_POWER) || defined(ARCH_RISCV64)
 685                 os_signal_untrap(SIGILL);
 686 #endif
 687         }
 688         if (unlikely(detection_failed))
 689                 cpu_feature_flags |= cpu_feature_static_flags;
 690         missing_features = cpu_feature_static_flags & ~cpu_feature_flags;
 691         if (unlikely(missing_features != 0)) {
 692                 int first;
 693                 int f;
 694                 char *error;
 695                 size_t error_l;
 696                 str_init(&error, &error_l);
 697                 str_add_string(&error, &error_l, "CPU doesn't have the following features: ");
 698                 for (first = 1, f = 0; missing_features; missing_features >>= 1, f++) {
 699                         if (missing_features & 1) {
 700                                 if (!first)
 701                                         str_add_string(&error, &error_l, ", ");
 702                                 first = 0;
 703                                 str_add_string(&error, &error_l, cpu_feature_names[f]);
 704                         }
 705                 }
 706                 str_finish(&error, &error_l);
 707                 fatal("%s", error);
 708         }
 709 #ifdef DEBUG_ENV
 710         {
 711                 char *str = getenv("CPU_FLAGS");
 712                 if (str && *str) {
 713                         char *e;
 714                         cpu_feature_flags = strtoul(str, &e, 16);
 715                         if (unlikely(*e))
 716                                 fatal("invalid CPU_FLAGS");
 717                 }
 718         }
 719 #endif
 720 #ifdef DEBUG_ENV
 721         if (getenv("PRINT_FLAGS")) {
 722                 debug("static flags: %x", cpu_feature_static_flags);
 723                 debug("dynamic flags: %x", cpu_feature_flags);
 724         }
 725 #endif
 726         asm_setup_thread();
 727
 728 #ifdef DEBUG_CRASH_HANDLER
 729         os_signal_trap(SIGSEGV, crash);
 730         os_signal_trap(SIGBUS, crash);
 731         os_signal_trap(SIGILL, crash);
 732 #endif
 733 }
 734
 735 void asm_done(void)
 736 {
 737 #ifdef DEBUG_CRASH_HANDLER
 738         os_signal_untrap(SIGSEGV);
 739         os_signal_untrap(SIGBUS);
 740         os_signal_untrap(SIGILL);
 741 #endif
 742 }