compiler-rt/lib/builtins/cpu_model.c

   1 //===-- cpu_model.c - Support for __cpu_model builtin  ------------*- C -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 //  This file is based on LLVM's lib/Support/Host.cpp.
  10 //  It implements the operating system Host concept and builtin
  11 //  __cpu_model for the compiler_rt library for x86 and
  12 //  __aarch64_have_lse_atomics, __aarch64_cpu_features for AArch64.
  13 //
  14 //===----------------------------------------------------------------------===//
  15
  16 #ifndef __has_attribute
  17 #define __has_attribute(attr) 0
  18 #endif
  19
  20 #if __has_attribute(constructor)
  21 #if __GNUC__ >= 9
  22 // Ordinarily init priorities below 101 are disallowed as they are reserved for the
  23 // implementation. However, we are the implementation, so silence the diagnostic,
  24 // since it doesn't apply to us.
  25 #pragma GCC diagnostic ignored "-Wprio-ctor-dtor"
  26 #endif
  27 // We're choosing init priority 90 to force our constructors to run before any
  28 // constructors in the end user application (starting at priority 101). This value
  29 // matches the libgcc choice for the same functions.
  30 #define CONSTRUCTOR_ATTRIBUTE __attribute__((constructor(90)))
  31 #else
  32 // FIXME: For MSVC, we should make a function pointer global in .CRT$X?? so that
  33 // this runs during initialization.
  34 #define CONSTRUCTOR_ATTRIBUTE
  35 #endif
  36
  37 #if (defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) ||           \
  38      defined(_M_X64)) &&                                                       \
  39     (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER))
  40
  41 #include <assert.h>
  42
  43 #define bool int
  44 #define true 1
  45 #define false 0
  46
  47 #ifdef _MSC_VER
  48 #include <intrin.h>
  49 #endif
  50
  51 enum VendorSignatures {
  52   SIG_INTEL = 0x756e6547, // Genu
  53   SIG_AMD = 0x68747541,   // Auth
  54 };
  55
  56 enum ProcessorVendors {
  57   VENDOR_INTEL = 1,
  58   VENDOR_AMD,
  59   VENDOR_OTHER,
  60   VENDOR_MAX
  61 };
  62
  63 enum ProcessorTypes {
  64   INTEL_BONNELL = 1,
  65   INTEL_CORE2,
  66   INTEL_COREI7,
  67   AMDFAM10H,
  68   AMDFAM15H,
  69   INTEL_SILVERMONT,
  70   INTEL_KNL,
  71   AMD_BTVER1,
  72   AMD_BTVER2,
  73   AMDFAM17H,
  74   INTEL_KNM,
  75   INTEL_GOLDMONT,
  76   INTEL_GOLDMONT_PLUS,
  77   INTEL_TREMONT,
  78   AMDFAM19H,
  79   ZHAOXIN_FAM7H,
  80   INTEL_SIERRAFOREST,
  81   INTEL_GRANDRIDGE,
  82   CPU_TYPE_MAX
  83 };
  84
  85 enum ProcessorSubtypes {
  86   INTEL_COREI7_NEHALEM = 1,
  87   INTEL_COREI7_WESTMERE,
  88   INTEL_COREI7_SANDYBRIDGE,
  89   AMDFAM10H_BARCELONA,
  90   AMDFAM10H_SHANGHAI,
  91   AMDFAM10H_ISTANBUL,
  92   AMDFAM15H_BDVER1,
  93   AMDFAM15H_BDVER2,
  94   AMDFAM15H_BDVER3,
  95   AMDFAM15H_BDVER4,
  96   AMDFAM17H_ZNVER1,
  97   INTEL_COREI7_IVYBRIDGE,
  98   INTEL_COREI7_HASWELL,
  99   INTEL_COREI7_BROADWELL,
 100   INTEL_COREI7_SKYLAKE,
 101   INTEL_COREI7_SKYLAKE_AVX512,
 102   INTEL_COREI7_CANNONLAKE,
 103   INTEL_COREI7_ICELAKE_CLIENT,
 104   INTEL_COREI7_ICELAKE_SERVER,
 105   AMDFAM17H_ZNVER2,
 106   INTEL_COREI7_CASCADELAKE,
 107   INTEL_COREI7_TIGERLAKE,
 108   INTEL_COREI7_COOPERLAKE,
 109   INTEL_COREI7_SAPPHIRERAPIDS,
 110   INTEL_COREI7_ALDERLAKE,
 111   AMDFAM19H_ZNVER3,
 112   INTEL_COREI7_ROCKETLAKE,
 113   ZHAOXIN_FAM7H_LUJIAZUI,
 114   AMDFAM19H_ZNVER4,
 115   INTEL_COREI7_GRANITERAPIDS,
 116   CPU_SUBTYPE_MAX
 117 };
 118
 119 enum ProcessorFeatures {
 120   FEATURE_CMOV = 0,
 121   FEATURE_MMX,
 122   FEATURE_POPCNT,
 123   FEATURE_SSE,
 124   FEATURE_SSE2,
 125   FEATURE_SSE3,
 126   FEATURE_SSSE3,
 127   FEATURE_SSE4_1,
 128   FEATURE_SSE4_2,
 129   FEATURE_AVX,
 130   FEATURE_AVX2,
 131   FEATURE_SSE4_A,
 132   FEATURE_FMA4,
 133   FEATURE_XOP,
 134   FEATURE_FMA,
 135   FEATURE_AVX512F,
 136   FEATURE_BMI,
 137   FEATURE_BMI2,
 138   FEATURE_AES,
 139   FEATURE_PCLMUL,
 140   FEATURE_AVX512VL,
 141   FEATURE_AVX512BW,
 142   FEATURE_AVX512DQ,
 143   FEATURE_AVX512CD,
 144   FEATURE_AVX512ER,
 145   FEATURE_AVX512PF,
 146   FEATURE_AVX512VBMI,
 147   FEATURE_AVX512IFMA,
 148   FEATURE_AVX5124VNNIW,
 149   FEATURE_AVX5124FMAPS,
 150   FEATURE_AVX512VPOPCNTDQ,
 151   FEATURE_AVX512VBMI2,
 152   FEATURE_GFNI,
 153   FEATURE_VPCLMULQDQ,
 154   FEATURE_AVX512VNNI,
 155   FEATURE_AVX512BITALG,
 156   FEATURE_AVX512BF16,
 157   FEATURE_AVX512VP2INTERSECT,
 158   CPU_FEATURE_MAX
 159 };
 160
 161 // The check below for i386 was copied from clang's cpuid.h (__get_cpuid_max).
 162 // Check motivated by bug reports for OpenSSL crashing on CPUs without CPUID
 163 // support. Consequently, for i386, the presence of CPUID is checked first
 164 // via the corresponding eflags bit.
 165 static bool isCpuIdSupported(void) {
 166 #if defined(__GNUC__) || defined(__clang__)
 167 #if defined(__i386__)
 168   int __cpuid_supported;
 169   __asm__("  pushfl\n"
 170           "  popl   %%eax\n"
 171           "  movl   %%eax,%%ecx\n"
 172           "  xorl   $0x00200000,%%eax\n"
 173           "  pushl  %%eax\n"
 174           "  popfl\n"
 175           "  pushfl\n"
 176           "  popl   %%eax\n"
 177           "  movl   $0,%0\n"
 178           "  cmpl   %%eax,%%ecx\n"
 179           "  je     1f\n"
 180           "  movl   $1,%0\n"
 181           "1:"
 182           : "=r"(__cpuid_supported)
 183           :
 184           : "eax", "ecx");
 185   if (!__cpuid_supported)
 186     return false;
 187 #endif
 188   return true;
 189 #endif
 190   return true;
 191 }
 192
 193 // This code is copied from lib/Support/Host.cpp.
 194 // Changes to either file should be mirrored in the other.
 195
 196 /// getX86CpuIDAndInfo - Execute the specified cpuid and return the 4 values in
 197 /// the specified arguments.  If we can't run cpuid on the host, return true.
 198 static bool getX86CpuIDAndInfo(unsigned value, unsigned *rEAX, unsigned *rEBX,
 199                                unsigned *rECX, unsigned *rEDX) {
 200 #if defined(__GNUC__) || defined(__clang__)
 201 #if defined(__x86_64__)
 202   // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually.
 203   // FIXME: should we save this for Clang?
 204   __asm__("movq\t%%rbx, %%rsi\n\t"
 205           "cpuid\n\t"
 206           "xchgq\t%%rbx, %%rsi\n\t"
 207           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
 208           : "a"(value));
 209   return false;
 210 #elif defined(__i386__)
 211   __asm__("movl\t%%ebx, %%esi\n\t"
 212           "cpuid\n\t"
 213           "xchgl\t%%ebx, %%esi\n\t"
 214           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
 215           : "a"(value));
 216   return false;
 217 #else
 218   return true;
 219 #endif
 220 #elif defined(_MSC_VER)
 221   // The MSVC intrinsic is portable across x86 and x64.
 222   int registers[4];
 223   __cpuid(registers, value);
 224   *rEAX = registers[0];
 225   *rEBX = registers[1];
 226   *rECX = registers[2];
 227   *rEDX = registers[3];
 228   return false;
 229 #else
 230   return true;
 231 #endif
 232 }
 233
 234 /// getX86CpuIDAndInfoEx - Execute the specified cpuid with subleaf and return
 235 /// the 4 values in the specified arguments.  If we can't run cpuid on the host,
 236 /// return true.
 237 static bool getX86CpuIDAndInfoEx(unsigned value, unsigned subleaf,
 238                                  unsigned *rEAX, unsigned *rEBX, unsigned *rECX,
 239                                  unsigned *rEDX) {
 240 #if defined(__GNUC__) || defined(__clang__)
 241 #if defined(__x86_64__)
 242   // gcc doesn't know cpuid would clobber ebx/rbx. Preserve it manually.
 243   // FIXME: should we save this for Clang?
 244   __asm__("movq\t%%rbx, %%rsi\n\t"
 245           "cpuid\n\t"
 246           "xchgq\t%%rbx, %%rsi\n\t"
 247           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
 248           : "a"(value), "c"(subleaf));
 249   return false;
 250 #elif defined(__i386__)
 251   __asm__("movl\t%%ebx, %%esi\n\t"
 252           "cpuid\n\t"
 253           "xchgl\t%%ebx, %%esi\n\t"
 254           : "=a"(*rEAX), "=S"(*rEBX), "=c"(*rECX), "=d"(*rEDX)
 255           : "a"(value), "c"(subleaf));
 256   return false;
 257 #else
 258   return true;
 259 #endif
 260 #elif defined(_MSC_VER)
 261   int registers[4];
 262   __cpuidex(registers, value, subleaf);
 263   *rEAX = registers[0];
 264   *rEBX = registers[1];
 265   *rECX = registers[2];
 266   *rEDX = registers[3];
 267   return false;
 268 #else
 269   return true;
 270 #endif
 271 }
 272
 273 // Read control register 0 (XCR0). Used to detect features such as AVX.
 274 static bool getX86XCR0(unsigned *rEAX, unsigned *rEDX) {
 275 #if defined(__GNUC__) || defined(__clang__)
 276   // Check xgetbv; this uses a .byte sequence instead of the instruction
 277   // directly because older assemblers do not include support for xgetbv and
 278   // there is no easy way to conditionally compile based on the assembler used.
 279   __asm__(".byte 0x0f, 0x01, 0xd0" : "=a"(*rEAX), "=d"(*rEDX) : "c"(0));
 280   return false;
 281 #elif defined(_MSC_FULL_VER) && defined(_XCR_XFEATURE_ENABLED_MASK)
 282   unsigned long long Result = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
 283   *rEAX = Result;
 284   *rEDX = Result >> 32;
 285   return false;
 286 #else
 287   return true;
 288 #endif
 289 }
 290
 291 static void detectX86FamilyModel(unsigned EAX, unsigned *Family,
 292                                  unsigned *Model) {
 293   *Family = (EAX >> 8) & 0xf; // Bits 8 - 11
 294   *Model = (EAX >> 4) & 0xf;  // Bits 4 - 7
 295   if (*Family == 6 || *Family == 0xf) {
 296     if (*Family == 0xf)
 297       // Examine extended family ID if family ID is F.
 298       *Family += (EAX >> 20) & 0xff; // Bits 20 - 27
 299     // Examine extended model ID if family ID is 6 or F.
 300     *Model += ((EAX >> 16) & 0xf) << 4; // Bits 16 - 19
 301   }
 302 }
 303
 304 static const char *
 305 getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model,
 306                                 const unsigned *Features,
 307                                 unsigned *Type, unsigned *Subtype) {
 308 #define testFeature(F)                                                         \
 309   (Features[F / 32] & (1 << (F % 32))) != 0
 310
 311   // We select CPU strings to match the code in Host.cpp, but we don't use them
 312   // in compiler-rt.
 313   const char *CPU = 0;
 314
 315   switch (Family) {
 316   case 6:
 317     switch (Model) {
 318     case 0x0f: // Intel Core 2 Duo processor, Intel Core 2 Duo mobile
 319                // processor, Intel Core 2 Quad processor, Intel Core 2 Quad
 320                // mobile processor, Intel Core 2 Extreme processor, Intel
 321                // Pentium Dual-Core processor, Intel Xeon processor, model
 322                // 0Fh. All processors are manufactured using the 65 nm process.
 323     case 0x16: // Intel Celeron processor model 16h. All processors are
 324                // manufactured using the 65 nm process
 325       CPU = "core2";
 326       *Type = INTEL_CORE2;
 327       break;
 328     case 0x17: // Intel Core 2 Extreme processor, Intel Xeon processor, model
 329                // 17h. All processors are manufactured using the 45 nm process.
 330                //
 331                // 45nm: Penryn , Wolfdale, Yorkfield (XE)
 332     case 0x1d: // Intel Xeon processor MP. All processors are manufactured using
 333                // the 45 nm process.
 334       CPU = "penryn";
 335       *Type = INTEL_CORE2;
 336       break;
 337     case 0x1a: // Intel Core i7 processor and Intel Xeon processor. All
 338                // processors are manufactured using the 45 nm process.
 339     case 0x1e: // Intel(R) Core(TM) i7 CPU         870  @ 2.93GHz.
 340                // As found in a Summer 2010 model iMac.
 341     case 0x1f:
 342     case 0x2e:              // Nehalem EX
 343       CPU = "nehalem";
 344       *Type = INTEL_COREI7;
 345       *Subtype = INTEL_COREI7_NEHALEM;
 346       break;
 347     case 0x25: // Intel Core i7, laptop version.
 348     case 0x2c: // Intel Core i7 processor and Intel Xeon processor. All
 349                // processors are manufactured using the 32 nm process.
 350     case 0x2f: // Westmere EX
 351       CPU = "westmere";
 352       *Type = INTEL_COREI7;
 353       *Subtype = INTEL_COREI7_WESTMERE;
 354       break;
 355     case 0x2a: // Intel Core i7 processor. All processors are manufactured
 356                // using the 32 nm process.
 357     case 0x2d:
 358       CPU = "sandybridge";
 359       *Type = INTEL_COREI7;
 360       *Subtype = INTEL_COREI7_SANDYBRIDGE;
 361       break;
 362     case 0x3a:
 363     case 0x3e:              // Ivy Bridge EP
 364       CPU = "ivybridge";
 365       *Type = INTEL_COREI7;
 366       *Subtype = INTEL_COREI7_IVYBRIDGE;
 367       break;
 368
 369     // Haswell:
 370     case 0x3c:
 371     case 0x3f:
 372     case 0x45:
 373     case 0x46:
 374       CPU = "haswell";
 375       *Type = INTEL_COREI7;
 376       *Subtype = INTEL_COREI7_HASWELL;
 377       break;
 378
 379     // Broadwell:
 380     case 0x3d:
 381     case 0x47:
 382     case 0x4f:
 383     case 0x56:
 384       CPU = "broadwell";
 385       *Type = INTEL_COREI7;
 386       *Subtype = INTEL_COREI7_BROADWELL;
 387       break;
 388
 389     // Skylake:
 390     case 0x4e:              // Skylake mobile
 391     case 0x5e:              // Skylake desktop
 392     case 0x8e:              // Kaby Lake mobile
 393     case 0x9e:              // Kaby Lake desktop
 394     case 0xa5:              // Comet Lake-H/S
 395     case 0xa6:              // Comet Lake-U
 396       CPU = "skylake";
 397       *Type = INTEL_COREI7;
 398       *Subtype = INTEL_COREI7_SKYLAKE;
 399       break;
 400
 401     // Rocketlake:
 402     case 0xa7:
 403       CPU = "rocketlake";
 404       *Type = INTEL_COREI7;
 405       *Subtype = INTEL_COREI7_ROCKETLAKE;
 406       break;
 407
 408     // Skylake Xeon:
 409     case 0x55:
 410       *Type = INTEL_COREI7;
 411       if (testFeature(FEATURE_AVX512BF16)) {
 412         CPU = "cooperlake";
 413         *Subtype = INTEL_COREI7_COOPERLAKE;
 414       } else if (testFeature(FEATURE_AVX512VNNI)) {
 415         CPU = "cascadelake";
 416         *Subtype = INTEL_COREI7_CASCADELAKE;
 417       } else {
 418         CPU = "skylake-avx512";
 419         *Subtype = INTEL_COREI7_SKYLAKE_AVX512;
 420       }
 421       break;
 422
 423     // Cannonlake:
 424     case 0x66:
 425       CPU = "cannonlake";
 426       *Type = INTEL_COREI7;
 427       *Subtype = INTEL_COREI7_CANNONLAKE;
 428       break;
 429
 430     // Icelake:
 431     case 0x7d:
 432     case 0x7e:
 433       CPU = "icelake-client";
 434       *Type = INTEL_COREI7;
 435       *Subtype = INTEL_COREI7_ICELAKE_CLIENT;
 436       break;
 437
 438     // Tigerlake:
 439     case 0x8c:
 440     case 0x8d:
 441       CPU = "tigerlake";
 442       *Type = INTEL_COREI7;
 443       *Subtype = INTEL_COREI7_TIGERLAKE;
 444       break;
 445
 446     // Alderlake:
 447     case 0x97:
 448     case 0x9a:
 449     // Raptorlake:
 450     case 0xb7:
 451     // Meteorlake:
 452     case 0xaa:
 453     case 0xac:
 454       CPU = "alderlake";
 455       *Type = INTEL_COREI7;
 456       *Subtype = INTEL_COREI7_ALDERLAKE;
 457       break;
 458
 459     // Icelake Xeon:
 460     case 0x6a:
 461     case 0x6c:
 462       CPU = "icelake-server";
 463       *Type = INTEL_COREI7;
 464       *Subtype = INTEL_COREI7_ICELAKE_SERVER;
 465       break;
 466
 467     // Emerald Rapids:
 468     case 0xcf:
 469     // Sapphire Rapids:
 470     case 0x8f:
 471       CPU = "sapphirerapids";
 472       *Type = INTEL_COREI7;
 473       *Subtype = INTEL_COREI7_SAPPHIRERAPIDS;
 474       break;
 475
 476     // Granite Rapids:
 477     case 0xae:
 478     case 0xad:
 479       CPU = "graniterapids";
 480       *Type = INTEL_COREI7;
 481       *Subtype = INTEL_COREI7_GRANITERAPIDS;
 482       break;
 483
 484     case 0x1c: // Most 45 nm Intel Atom processors
 485     case 0x26: // 45 nm Atom Lincroft
 486     case 0x27: // 32 nm Atom Medfield
 487     case 0x35: // 32 nm Atom Midview
 488     case 0x36: // 32 nm Atom Midview
 489       CPU = "bonnell";
 490       *Type = INTEL_BONNELL;
 491       break;
 492
 493     // Atom Silvermont codes from the Intel software optimization guide.
 494     case 0x37:
 495     case 0x4a:
 496     case 0x4d:
 497     case 0x5a:
 498     case 0x5d:
 499     case 0x4c: // really airmont
 500       CPU = "silvermont";
 501       *Type = INTEL_SILVERMONT;
 502       break;
 503     // Goldmont:
 504     case 0x5c: // Apollo Lake
 505     case 0x5f: // Denverton
 506       CPU = "goldmont";
 507       *Type = INTEL_GOLDMONT;
 508       break; // "goldmont"
 509     case 0x7a:
 510       CPU = "goldmont-plus";
 511       *Type = INTEL_GOLDMONT_PLUS;
 512       break;
 513     case 0x86:
 514       CPU = "tremont";
 515       *Type = INTEL_TREMONT;
 516       break;
 517
 518     // Sierraforest:
 519     case 0xaf:
 520       CPU = "sierraforest";
 521       *Type = INTEL_SIERRAFOREST;
 522       break;
 523
 524     // Grandridge:
 525     case 0xb6:
 526       CPU = "grandridge";
 527       *Type = INTEL_GRANDRIDGE;
 528       break;
 529
 530     case 0x57:
 531       CPU = "knl";
 532       *Type = INTEL_KNL;
 533       break;
 534
 535     case 0x85:
 536       CPU = "knm";
 537       *Type = INTEL_KNM;
 538       break;
 539
 540     default: // Unknown family 6 CPU.
 541       break;
 542     }
 543     break;
 544   default:
 545     break; // Unknown.
 546   }
 547
 548   return CPU;
 549 }
 550
 551 static const char *
 552 getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model,
 553                               const unsigned *Features,
 554                               unsigned *Type, unsigned *Subtype) {
 555   // We select CPU strings to match the code in Host.cpp, but we don't use them
 556   // in compiler-rt.
 557   const char *CPU = 0;
 558
 559   switch (Family) {
 560   case 16:
 561     CPU = "amdfam10";
 562     *Type = AMDFAM10H;
 563     switch (Model) {
 564     case 2:
 565       *Subtype = AMDFAM10H_BARCELONA;
 566       break;
 567     case 4:
 568       *Subtype = AMDFAM10H_SHANGHAI;
 569       break;
 570     case 8:
 571       *Subtype = AMDFAM10H_ISTANBUL;
 572       break;
 573     }
 574     break;
 575   case 20:
 576     CPU = "btver1";
 577     *Type = AMD_BTVER1;
 578     break;
 579   case 21:
 580     CPU = "bdver1";
 581     *Type = AMDFAM15H;
 582     if (Model >= 0x60 && Model <= 0x7f) {
 583       CPU = "bdver4";
 584       *Subtype = AMDFAM15H_BDVER4;
 585       break; // 60h-7Fh: Excavator
 586     }
 587     if (Model >= 0x30 && Model <= 0x3f) {
 588       CPU = "bdver3";
 589       *Subtype = AMDFAM15H_BDVER3;
 590       break; // 30h-3Fh: Steamroller
 591     }
 592     if ((Model >= 0x10 && Model <= 0x1f) || Model == 0x02) {
 593       CPU = "bdver2";
 594       *Subtype = AMDFAM15H_BDVER2;
 595       break; // 02h, 10h-1Fh: Piledriver
 596     }
 597     if (Model <= 0x0f) {
 598       *Subtype = AMDFAM15H_BDVER1;
 599       break; // 00h-0Fh: Bulldozer
 600     }
 601     break;
 602   case 22:
 603     CPU = "btver2";
 604     *Type = AMD_BTVER2;
 605     break;
 606   case 23:
 607     CPU = "znver1";
 608     *Type = AMDFAM17H;
 609     if ((Model >= 0x30 && Model <= 0x3f) || Model == 0x71) {
 610       CPU = "znver2";
 611       *Subtype = AMDFAM17H_ZNVER2;
 612       break; // 30h-3fh, 71h: Zen2
 613     }
 614     if (Model <= 0x0f) {
 615       *Subtype = AMDFAM17H_ZNVER1;
 616       break; // 00h-0Fh: Zen1
 617     }
 618     break;
 619   case 25:
 620     CPU = "znver3";
 621     *Type = AMDFAM19H;
 622     if (Model <= 0x0f || (Model >= 0x20 && Model <= 0x5f)) {
 623       // Family 19h Models 00h-0Fh - Zen3
 624       // Family 19h Models 20h-2Fh - Zen3
 625       // Family 19h Models 30h-3Fh - Zen3
 626       // Family 19h Models 40h-4Fh - Zen3+
 627       // Family 19h Models 50h-5Fh - Zen3+
 628       *Subtype = AMDFAM19H_ZNVER3;
 629       break;
 630     }
 631     if ((Model >= 0x10 && Model <= 0x1f) ||
 632         (Model >= 0x60 && Model <= 0x74) ||
 633         (Model >= 0x78 && Model <= 0x7b) ||
 634         (Model >= 0xA0 && Model <= 0xAf)) {
 635       CPU = "znver4";
 636       *Subtype = AMDFAM19H_ZNVER4;
 637       break; //  "znver4"
 638     }
 639     break;
 640   default:
 641     break; // Unknown AMD CPU.
 642   }
 643
 644   return CPU;
 645 }
 646
 647 static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf,
 648                                  unsigned *Features) {
 649   unsigned EAX = 0, EBX = 0;
 650
 651 #define setFeature(F)                                                          \
 652   Features[F / 32] |= 1U << (F % 32)
 653
 654   if ((EDX >> 15) & 1)
 655     setFeature(FEATURE_CMOV);
 656   if ((EDX >> 23) & 1)
 657     setFeature(FEATURE_MMX);
 658   if ((EDX >> 25) & 1)
 659     setFeature(FEATURE_SSE);
 660   if ((EDX >> 26) & 1)
 661     setFeature(FEATURE_SSE2);
 662
 663   if ((ECX >> 0) & 1)
 664     setFeature(FEATURE_SSE3);
 665   if ((ECX >> 1) & 1)
 666     setFeature(FEATURE_PCLMUL);
 667   if ((ECX >> 9) & 1)
 668     setFeature(FEATURE_SSSE3);
 669   if ((ECX >> 12) & 1)
 670     setFeature(FEATURE_FMA);
 671   if ((ECX >> 19) & 1)
 672     setFeature(FEATURE_SSE4_1);
 673   if ((ECX >> 20) & 1)
 674     setFeature(FEATURE_SSE4_2);
 675   if ((ECX >> 23) & 1)
 676     setFeature(FEATURE_POPCNT);
 677   if ((ECX >> 25) & 1)
 678     setFeature(FEATURE_AES);
 679
 680   // If CPUID indicates support for XSAVE, XRESTORE and AVX, and XGETBV
 681   // indicates that the AVX registers will be saved and restored on context
 682   // switch, then we have full AVX support.
 683   const unsigned AVXBits = (1 << 27) | (1 << 28);
 684   bool HasAVX = ((ECX & AVXBits) == AVXBits) && !getX86XCR0(&EAX, &EDX) &&
 685                 ((EAX & 0x6) == 0x6);
 686 #if defined(__APPLE__)
 687   // Darwin lazily saves the AVX512 context on first use: trust that the OS will
 688   // save the AVX512 context if we use AVX512 instructions, even the bit is not
 689   // set right now.
 690   bool HasAVX512Save = true;
 691 #else
 692   // AVX512 requires additional context to be saved by the OS.
 693   bool HasAVX512Save = HasAVX && ((EAX & 0xe0) == 0xe0);
 694 #endif
 695
 696   if (HasAVX)
 697     setFeature(FEATURE_AVX);
 698
 699   bool HasLeaf7 =
 700       MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x0, &EAX, &EBX, &ECX, &EDX);
 701
 702   if (HasLeaf7 && ((EBX >> 3) & 1))
 703     setFeature(FEATURE_BMI);
 704   if (HasLeaf7 && ((EBX >> 5) & 1) && HasAVX)
 705     setFeature(FEATURE_AVX2);
 706   if (HasLeaf7 && ((EBX >> 8) & 1))
 707     setFeature(FEATURE_BMI2);
 708   if (HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save)
 709     setFeature(FEATURE_AVX512F);
 710   if (HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save)
 711     setFeature(FEATURE_AVX512DQ);
 712   if (HasLeaf7 && ((EBX >> 21) & 1) && HasAVX512Save)
 713     setFeature(FEATURE_AVX512IFMA);
 714   if (HasLeaf7 && ((EBX >> 26) & 1) && HasAVX512Save)
 715     setFeature(FEATURE_AVX512PF);
 716   if (HasLeaf7 && ((EBX >> 27) & 1) && HasAVX512Save)
 717     setFeature(FEATURE_AVX512ER);
 718   if (HasLeaf7 && ((EBX >> 28) & 1) && HasAVX512Save)
 719     setFeature(FEATURE_AVX512CD);
 720   if (HasLeaf7 && ((EBX >> 30) & 1) && HasAVX512Save)
 721     setFeature(FEATURE_AVX512BW);
 722   if (HasLeaf7 && ((EBX >> 31) & 1) && HasAVX512Save)
 723     setFeature(FEATURE_AVX512VL);
 724
 725   if (HasLeaf7 && ((ECX >> 1) & 1) && HasAVX512Save)
 726     setFeature(FEATURE_AVX512VBMI);
 727   if (HasLeaf7 && ((ECX >> 6) & 1) && HasAVX512Save)
 728     setFeature(FEATURE_AVX512VBMI2);
 729   if (HasLeaf7 && ((ECX >> 8) & 1))
 730     setFeature(FEATURE_GFNI);
 731   if (HasLeaf7 && ((ECX >> 10) & 1) && HasAVX)
 732     setFeature(FEATURE_VPCLMULQDQ);
 733   if (HasLeaf7 && ((ECX >> 11) & 1) && HasAVX512Save)
 734     setFeature(FEATURE_AVX512VNNI);
 735   if (HasLeaf7 && ((ECX >> 12) & 1) && HasAVX512Save)
 736     setFeature(FEATURE_AVX512BITALG);
 737   if (HasLeaf7 && ((ECX >> 14) & 1) && HasAVX512Save)
 738     setFeature(FEATURE_AVX512VPOPCNTDQ);
 739
 740   if (HasLeaf7 && ((EDX >> 2) & 1) && HasAVX512Save)
 741     setFeature(FEATURE_AVX5124VNNIW);
 742   if (HasLeaf7 && ((EDX >> 3) & 1) && HasAVX512Save)
 743     setFeature(FEATURE_AVX5124FMAPS);
 744   if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save)
 745     setFeature(FEATURE_AVX512VP2INTERSECT);
 746
 747   bool HasLeaf7Subleaf1 =
 748       MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX);
 749   if (HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save)
 750     setFeature(FEATURE_AVX512BF16);
 751
 752   unsigned MaxExtLevel;
 753   getX86CpuIDAndInfo(0x80000000, &MaxExtLevel, &EBX, &ECX, &EDX);
 754
 755   bool HasExtLeaf1 = MaxExtLevel >= 0x80000001 &&
 756                      !getX86CpuIDAndInfo(0x80000001, &EAX, &EBX, &ECX, &EDX);
 757   if (HasExtLeaf1 && ((ECX >> 6) & 1))
 758     setFeature(FEATURE_SSE4_A);
 759   if (HasExtLeaf1 && ((ECX >> 11) & 1))
 760     setFeature(FEATURE_XOP);
 761   if (HasExtLeaf1 && ((ECX >> 16) & 1))
 762     setFeature(FEATURE_FMA4);
 763 #undef setFeature
 764 }
 765
 766 #ifndef _WIN32
 767 __attribute__((visibility("hidden")))
 768 #endif
 769 int __cpu_indicator_init(void) CONSTRUCTOR_ATTRIBUTE;
 770
 771 #ifndef _WIN32
 772 __attribute__((visibility("hidden")))
 773 #endif
 774 struct __processor_model {
 775   unsigned int __cpu_vendor;
 776   unsigned int __cpu_type;
 777   unsigned int __cpu_subtype;
 778   unsigned int __cpu_features[1];
 779 } __cpu_model = {0, 0, 0, {0}};
 780
 781 #ifndef _WIN32
 782 __attribute__((visibility("hidden")))
 783 #endif
 784 unsigned int __cpu_features2 = 0;
 785
 786 // A constructor function that is sets __cpu_model and __cpu_features2 with
 787 // the right values.  This needs to run only once.  This constructor is
 788 // given the highest priority and it should run before constructors without
 789 // the priority set.  However, it still runs after ifunc initializers and
 790 // needs to be called explicitly there.
 791
 792 int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) {
 793   unsigned EAX, EBX, ECX, EDX;
 794   unsigned MaxLeaf = 5;
 795   unsigned Vendor;
 796   unsigned Model, Family;
 797   unsigned Features[(CPU_FEATURE_MAX + 31) / 32] = {0};
 798
 799   // This function needs to run just once.
 800   if (__cpu_model.__cpu_vendor)
 801     return 0;
 802
 803   if (!isCpuIdSupported() ||
 804       getX86CpuIDAndInfo(0, &MaxLeaf, &Vendor, &ECX, &EDX) || MaxLeaf < 1) {
 805     __cpu_model.__cpu_vendor = VENDOR_OTHER;
 806     return -1;
 807   }
 808
 809   getX86CpuIDAndInfo(1, &EAX, &EBX, &ECX, &EDX);
 810   detectX86FamilyModel(EAX, &Family, &Model);
 811
 812   // Find available features.
 813   getAvailableFeatures(ECX, EDX, MaxLeaf, &Features[0]);
 814
 815   assert((sizeof(Features)/sizeof(Features[0])) == 2);
 816   __cpu_model.__cpu_features[0] = Features[0];
 817   __cpu_features2 = Features[1];
 818
 819   if (Vendor == SIG_INTEL) {
 820     // Get CPU type.
 821     getIntelProcessorTypeAndSubtype(Family, Model, &Features[0],
 822                                     &(__cpu_model.__cpu_type),
 823                                     &(__cpu_model.__cpu_subtype));
 824     __cpu_model.__cpu_vendor = VENDOR_INTEL;
 825   } else if (Vendor == SIG_AMD) {
 826     // Get CPU type.
 827     getAMDProcessorTypeAndSubtype(Family, Model, &Features[0],
 828                                   &(__cpu_model.__cpu_type),
 829                                   &(__cpu_model.__cpu_subtype));
 830     __cpu_model.__cpu_vendor = VENDOR_AMD;
 831   } else
 832     __cpu_model.__cpu_vendor = VENDOR_OTHER;
 833
 834   assert(__cpu_model.__cpu_vendor < VENDOR_MAX);
 835   assert(__cpu_model.__cpu_type < CPU_TYPE_MAX);
 836   assert(__cpu_model.__cpu_subtype < CPU_SUBTYPE_MAX);
 837
 838   return 0;
 839 }
 840 #elif defined(__aarch64__)
 841
 842 #ifndef AT_HWCAP
 843 #define AT_HWCAP 16
 844 #endif
 845 #ifndef HWCAP_CPUID
 846 #define HWCAP_CPUID (1 << 11)
 847 #endif
 848 #ifndef HWCAP_FP
 849 #define HWCAP_FP (1 << 0)
 850 #endif
 851 #ifndef HWCAP_ASIMD
 852 #define HWCAP_ASIMD (1 << 1)
 853 #endif
 854 #ifndef HWCAP_AES
 855 #define HWCAP_AES (1 << 3)
 856 #endif
 857 #ifndef HWCAP_PMULL
 858 #define HWCAP_PMULL (1 << 4)
 859 #endif
 860 #ifndef HWCAP_SHA1
 861 #define HWCAP_SHA1 (1 << 5)
 862 #endif
 863 #ifndef HWCAP_SHA2
 864 #define HWCAP_SHA2 (1 << 6)
 865 #endif
 866 #ifndef HWCAP_ATOMICS
 867 #define HWCAP_ATOMICS (1 << 8)
 868 #endif
 869 #ifndef HWCAP_FPHP
 870 #define HWCAP_FPHP (1 << 9)
 871 #endif
 872 #ifndef HWCAP_ASIMDHP
 873 #define HWCAP_ASIMDHP (1 << 10)
 874 #endif
 875 #ifndef HWCAP_ASIMDRDM
 876 #define HWCAP_ASIMDRDM (1 << 12)
 877 #endif
 878 #ifndef HWCAP_JSCVT
 879 #define HWCAP_JSCVT (1 << 13)
 880 #endif
 881 #ifndef HWCAP_FCMA
 882 #define HWCAP_FCMA (1 << 14)
 883 #endif
 884 #ifndef HWCAP_LRCPC
 885 #define HWCAP_LRCPC (1 << 15)
 886 #endif
 887 #ifndef HWCAP_DCPOP
 888 #define HWCAP_DCPOP (1 << 16)
 889 #endif
 890 #ifndef HWCAP_SHA3
 891 #define HWCAP_SHA3 (1 << 17)
 892 #endif
 893 #ifndef HWCAP_SM3
 894 #define HWCAP_SM3 (1 << 18)
 895 #endif
 896 #ifndef HWCAP_SM4
 897 #define HWCAP_SM4 (1 << 19)
 898 #endif
 899 #ifndef HWCAP_ASIMDDP
 900 #define HWCAP_ASIMDDP (1 << 20)
 901 #endif
 902 #ifndef HWCAP_SHA512
 903 #define HWCAP_SHA512 (1 << 21)
 904 #endif
 905 #ifndef HWCAP_SVE
 906 #define HWCAP_SVE (1 << 22)
 907 #endif
 908 #ifndef HWCAP_ASIMDFHM
 909 #define HWCAP_ASIMDFHM (1 << 23)
 910 #endif
 911 #ifndef HWCAP_DIT
 912 #define HWCAP_DIT (1 << 24)
 913 #endif
 914 #ifndef HWCAP_ILRCPC
 915 #define HWCAP_ILRCPC (1 << 26)
 916 #endif
 917 #ifndef HWCAP_FLAGM
 918 #define HWCAP_FLAGM (1 << 27)
 919 #endif
 920 #ifndef HWCAP_SSBS
 921 #define HWCAP_SSBS (1 << 28)
 922 #endif
 923 #ifndef HWCAP_SB
 924 #define HWCAP_SB (1 << 29)
 925 #endif
 926
 927 #ifndef HWCAP2_DCPODP
 928 #define HWCAP2_DCPODP (1 << 0)
 929 #endif
 930 #ifndef HWCAP2_SVE2
 931 #define HWCAP2_SVE2 (1 << 1)
 932 #endif
 933 #ifndef HWCAP2_SVEAES
 934 #define HWCAP2_SVEAES (1 << 2)
 935 #endif
 936 #ifndef HWCAP2_SVEPMULL
 937 #define HWCAP2_SVEPMULL (1 << 3)
 938 #endif
 939 #ifndef HWCAP2_SVEBITPERM
 940 #define HWCAP2_SVEBITPERM (1 << 4)
 941 #endif
 942 #ifndef HWCAP2_SVESHA3
 943 #define HWCAP2_SVESHA3 (1 << 5)
 944 #endif
 945 #ifndef HWCAP2_SVESM4
 946 #define HWCAP2_SVESM4 (1 << 6)
 947 #endif
 948 #ifndef HWCAP2_FLAGM2
 949 #define HWCAP2_FLAGM2 (1 << 7)
 950 #endif
 951 #ifndef HWCAP2_FRINT
 952 #define HWCAP2_FRINT (1 << 8)
 953 #endif
 954 #ifndef HWCAP2_SVEI8MM
 955 #define HWCAP2_SVEI8MM (1 << 9)
 956 #endif
 957 #ifndef HWCAP2_SVEF32MM
 958 #define HWCAP2_SVEF32MM (1 << 10)
 959 #endif
 960 #ifndef HWCAP2_SVEF64MM
 961 #define HWCAP2_SVEF64MM (1 << 11)
 962 #endif
 963 #ifndef HWCAP2_SVEBF16
 964 #define HWCAP2_SVEBF16 (1 << 12)
 965 #endif
 966 #ifndef HWCAP2_I8MM
 967 #define HWCAP2_I8MM (1 << 13)
 968 #endif
 969 #ifndef HWCAP2_BF16
 970 #define HWCAP2_BF16 (1 << 14)
 971 #endif
 972 #ifndef HWCAP2_DGH
 973 #define HWCAP2_DGH (1 << 15)
 974 #endif
 975 #ifndef HWCAP2_RNG
 976 #define HWCAP2_RNG (1 << 16)
 977 #endif
 978 #ifndef HWCAP2_BTI
 979 #define HWCAP2_BTI (1 << 17)
 980 #endif
 981 #ifndef HWCAP2_MTE
 982 #define HWCAP2_MTE (1 << 18)
 983 #endif
 984 #ifndef HWCAP2_RPRES
 985 #define HWCAP2_RPRES (1 << 21)
 986 #endif
 987 #ifndef HWCAP2_MTE3
 988 #define HWCAP2_MTE3 (1 << 22)
 989 #endif
 990 #ifndef HWCAP2_SME
 991 #define HWCAP2_SME (1 << 23)
 992 #endif
 993 #ifndef HWCAP2_SME_I16I64
 994 #define HWCAP2_SME_I16I64 (1 << 24)
 995 #endif
 996 #ifndef HWCAP2_SME_F64F64
 997 #define HWCAP2_SME_F64F64 (1 << 25)
 998 #endif
 999 #ifndef HWCAP2_WFXT
1000 #define HWCAP2_WFXT (1UL << 31)
1001 #endif
1002 #ifndef HWCAP2_EBF16
1003 #define HWCAP2_EBF16 (1UL << 32)
1004 #endif
1005 #ifndef HWCAP2_SVE_EBF16
1006 #define HWCAP2_SVE_EBF16 (1UL << 33)
1007 #endif
1008
1009 // LSE support detection for out-of-line atomics
1010 // using HWCAP and Auxiliary vector
1011 _Bool __aarch64_have_lse_atomics
1012     __attribute__((visibility("hidden"), nocommon));
1013
1014 #if defined(__has_include)
1015 #if __has_include(<sys/auxv.h>)
1016 #include <sys/auxv.h>
1017 #if __has_include(<asm/hwcap.h>)
1018 #include <asm/hwcap.h>
1019
1020 #if defined(__ANDROID__)
1021 #include <string.h>
1022 #include <sys/system_properties.h>
1023 #elif defined(__Fuchsia__)
1024 #include <zircon/features.h>
1025 #include <zircon/syscalls.h>
1026 #endif
1027
1028 // Detect Exynos 9810 CPU
1029 #define IF_EXYNOS9810                                                          \
1030   char arch[PROP_VALUE_MAX];                                                   \
1031   if (__system_property_get("ro.arch", arch) > 0 &&                            \
1032       strncmp(arch, "exynos9810", sizeof("exynos9810") - 1) == 0)
1033
1034 static void CONSTRUCTOR_ATTRIBUTE init_have_lse_atomics(void) {
1035 #if defined(__FreeBSD__)
1036   unsigned long hwcap;
1037   int result = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap);
1038   __aarch64_have_lse_atomics = result == 0 && (hwcap & HWCAP_ATOMICS) != 0;
1039 #elif defined(__Fuchsia__)
1040   // This ensures the vDSO is a direct link-time dependency of anything that
1041   // needs this initializer code.
1042 #pragma comment(lib, "zircon")
1043   uint32_t features;
1044   zx_status_t status = _zx_system_get_features(ZX_FEATURE_KIND_CPU, &features);
1045   __aarch64_have_lse_atomics =
1046       status == ZX_OK && (features & ZX_ARM64_FEATURE_ISA_ATOMICS) != 0;
1047 #else
1048   unsigned long hwcap = getauxval(AT_HWCAP);
1049   _Bool result = (hwcap & HWCAP_ATOMICS) != 0;
1050 #if defined(__ANDROID__)
1051   if (result) {
1052     // Some cores in the Exynos 9810 CPU are ARMv8.2 and others are ARMv8.0;
1053     // only the former support LSE atomics.  However, the kernel in the
1054     // initial Android 8.0 release of Galaxy S9/S9+ devices incorrectly
1055     // reported the feature as being supported.
1056     //
1057     // The kernel appears to have been corrected to mark it unsupported as of
1058     // the Android 9.0 release on those devices, and this issue has not been
1059     // observed anywhere else. Thus, this workaround may be removed if
1060     // compiler-rt ever drops support for Android 8.0.
1061     IF_EXYNOS9810 result = false;
1062   }
1063 #endif // defined(__ANDROID__)
1064   __aarch64_have_lse_atomics = result;
1065 #endif // defined(__FreeBSD__)
1066 }
1067
1068 #if !defined(DISABLE_AARCH64_FMV)
1069 // CPUFeatures must correspond to the same AArch64 features in
1070 // AArch64TargetParser.h
1071 enum CPUFeatures {
1072   FEAT_RNG,
1073   FEAT_FLAGM,
1074   FEAT_FLAGM2,
1075   FEAT_FP16FML,
1076   FEAT_DOTPROD,
1077   FEAT_SM4,
1078   FEAT_RDM,
1079   FEAT_LSE,
1080   FEAT_FP,
1081   FEAT_SIMD,
1082   FEAT_CRC,
1083   FEAT_SHA1,
1084   FEAT_SHA2,
1085   FEAT_SHA3,
1086   FEAT_AES,
1087   FEAT_PMULL,
1088   FEAT_FP16,
1089   FEAT_DIT,
1090   FEAT_DPB,
1091   FEAT_DPB2,
1092   FEAT_JSCVT,
1093   FEAT_FCMA,
1094   FEAT_RCPC,
1095   FEAT_RCPC2,
1096   FEAT_FRINTTS,
1097   FEAT_DGH,
1098   FEAT_I8MM,
1099   FEAT_BF16,
1100   FEAT_EBF16,
1101   FEAT_RPRES,
1102   FEAT_SVE,
1103   FEAT_SVE_BF16,
1104   FEAT_SVE_EBF16,
1105   FEAT_SVE_I8MM,
1106   FEAT_SVE_F32MM,
1107   FEAT_SVE_F64MM,
1108   FEAT_SVE2,
1109   FEAT_SVE_AES,
1110   FEAT_SVE_PMULL128,
1111   FEAT_SVE_BITPERM,
1112   FEAT_SVE_SHA3,
1113   FEAT_SVE_SM4,
1114   FEAT_SME,
1115   FEAT_MEMTAG,
1116   FEAT_MEMTAG2,
1117   FEAT_MEMTAG3,
1118   FEAT_SB,
1119   FEAT_PREDRES,
1120   FEAT_SSBS,
1121   FEAT_SSBS2,
1122   FEAT_BTI,
1123   FEAT_LS64,
1124   FEAT_LS64_V,
1125   FEAT_LS64_ACCDATA,
1126   FEAT_WFXT,
1127   FEAT_SME_F64,
1128   FEAT_SME_I64,
1129   FEAT_SME2,
1130   FEAT_MAX
1131 };
1132
1133 // Architecture features used
1134 // in Function Multi Versioning
1135 struct {
1136   unsigned long long features;
1137   // As features grows new fields could be added
1138 } __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
1139
1140 void init_cpu_features_resolver(unsigned long hwcap, unsigned long hwcap2) {
1141 #define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F
1142 #define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr))
1143 #define extractBits(val, start, number)                                        \
1144   (val & ((1ULL << number) - 1ULL) << start) >> start
1145   if (hwcap & HWCAP_CRC32)
1146     setCPUFeature(FEAT_CRC);
1147   if (hwcap & HWCAP_PMULL)
1148     setCPUFeature(FEAT_PMULL);
1149   if (hwcap & HWCAP_FLAGM)
1150     setCPUFeature(FEAT_FLAGM);
1151   if (hwcap2 & HWCAP2_FLAGM2) {
1152     setCPUFeature(FEAT_FLAGM);
1153     setCPUFeature(FEAT_FLAGM2);
1154   }
1155   if (hwcap & HWCAP_SM3 && hwcap & HWCAP_SM4)
1156     setCPUFeature(FEAT_SM4);
1157   if (hwcap & HWCAP_ASIMDDP)
1158     setCPUFeature(FEAT_DOTPROD);
1159   if (hwcap & HWCAP_ASIMDFHM)
1160     setCPUFeature(FEAT_FP16FML);
1161   if (hwcap & HWCAP_FPHP) {
1162     setCPUFeature(FEAT_FP16);
1163     setCPUFeature(FEAT_FP);
1164   }
1165   if (hwcap & HWCAP_DIT)
1166     setCPUFeature(FEAT_DIT);
1167   if (hwcap & HWCAP_ASIMDRDM)
1168     setCPUFeature(FEAT_RDM);
1169   if (hwcap & HWCAP_ILRCPC)
1170     setCPUFeature(FEAT_RCPC2);
1171   if (hwcap & HWCAP_AES)
1172     setCPUFeature(FEAT_AES);
1173   if (hwcap & HWCAP_SHA1)
1174     setCPUFeature(FEAT_SHA1);
1175   if (hwcap & HWCAP_SHA2)
1176     setCPUFeature(FEAT_SHA2);
1177   if (hwcap & HWCAP_JSCVT)
1178     setCPUFeature(FEAT_JSCVT);
1179   if (hwcap & HWCAP_FCMA)
1180     setCPUFeature(FEAT_FCMA);
1181   if (hwcap & HWCAP_SB)
1182     setCPUFeature(FEAT_SB);
1183   if (hwcap & HWCAP_SSBS)
1184     setCPUFeature(FEAT_SSBS2);
1185   if (hwcap2 & HWCAP2_MTE) {
1186     setCPUFeature(FEAT_MEMTAG);
1187     setCPUFeature(FEAT_MEMTAG2);
1188   }
1189   if (hwcap2 & HWCAP2_MTE3) {
1190     setCPUFeature(FEAT_MEMTAG);
1191     setCPUFeature(FEAT_MEMTAG2);
1192     setCPUFeature(FEAT_MEMTAG3);
1193   }
1194   if (hwcap2 & HWCAP2_SVEAES)
1195     setCPUFeature(FEAT_SVE_AES);
1196   if (hwcap2 & HWCAP2_SVEPMULL) {
1197     setCPUFeature(FEAT_SVE_AES);
1198     setCPUFeature(FEAT_SVE_PMULL128);
1199   }
1200   if (hwcap2 & HWCAP2_SVEBITPERM)
1201     setCPUFeature(FEAT_SVE_BITPERM);
1202   if (hwcap2 & HWCAP2_SVESHA3)
1203     setCPUFeature(FEAT_SVE_SHA3);
1204   if (hwcap2 & HWCAP2_SVESM4)
1205     setCPUFeature(FEAT_SVE_SM4);
1206   if (hwcap2 & HWCAP2_DCPODP)
1207     setCPUFeature(FEAT_DPB2);
1208   if (hwcap & HWCAP_ATOMICS)
1209     setCPUFeature(FEAT_LSE);
1210   if (hwcap2 & HWCAP2_RNG)
1211     setCPUFeature(FEAT_RNG);
1212   if (hwcap2 & HWCAP2_I8MM)
1213     setCPUFeature(FEAT_I8MM);
1214   if (hwcap2 & HWCAP2_EBF16)
1215     setCPUFeature(FEAT_EBF16);
1216   if (hwcap2 & HWCAP2_SVE_EBF16)
1217     setCPUFeature(FEAT_SVE_EBF16);
1218   if (hwcap2 & HWCAP2_DGH)
1219     setCPUFeature(FEAT_DGH);
1220   if (hwcap2 & HWCAP2_FRINT)
1221     setCPUFeature(FEAT_FRINTTS);
1222   if (hwcap2 & HWCAP2_SVEI8MM)
1223     setCPUFeature(FEAT_SVE_I8MM);
1224   if (hwcap2 & HWCAP2_SVEF32MM)
1225     setCPUFeature(FEAT_SVE_F32MM);
1226   if (hwcap2 & HWCAP2_SVEF64MM)
1227     setCPUFeature(FEAT_SVE_F64MM);
1228   if (hwcap2 & HWCAP2_BTI)
1229     setCPUFeature(FEAT_BTI);
1230   if (hwcap2 & HWCAP2_RPRES)
1231     setCPUFeature(FEAT_RPRES);
1232   if (hwcap2 & HWCAP2_WFXT)
1233     setCPUFeature(FEAT_WFXT);
1234   if (hwcap2 & HWCAP2_SME)
1235     setCPUFeature(FEAT_SME);
1236   if (hwcap2 & HWCAP2_SME_I16I64)
1237     setCPUFeature(FEAT_SME_I64);
1238   if (hwcap2 & HWCAP2_SME_F64F64)
1239     setCPUFeature(FEAT_SME_F64);
1240   if (hwcap & HWCAP_CPUID) {
1241     unsigned long ftr;
1242     getCPUFeature(ID_AA64PFR1_EL1, ftr);
1243     // ID_AA64PFR1_EL1.MTE >= 0b0001
1244     if (extractBits(ftr, 8, 4) >= 0x1)
1245       setCPUFeature(FEAT_MEMTAG);
1246     // ID_AA64PFR1_EL1.SSBS == 0b0001
1247     if (extractBits(ftr, 4, 4) == 0x1)
1248       setCPUFeature(FEAT_SSBS);
1249     // ID_AA64PFR1_EL1.SME == 0b0010
1250     if (extractBits(ftr, 24, 4) == 0x2)
1251       setCPUFeature(FEAT_SME2);
1252     getCPUFeature(ID_AA64PFR0_EL1, ftr);
1253     // ID_AA64PFR0_EL1.FP != 0b1111
1254     if (extractBits(ftr, 16, 4) != 0xF) {
1255       setCPUFeature(FEAT_FP);
1256       // ID_AA64PFR0_EL1.AdvSIMD has the same value as ID_AA64PFR0_EL1.FP
1257       setCPUFeature(FEAT_SIMD);
1258     }
1259     // ID_AA64PFR0_EL1.SVE != 0b0000
1260     if (extractBits(ftr, 32, 4) != 0x0) {
1261       // get ID_AA64ZFR0_EL1, that name supported
1262       // if sve enabled only
1263       getCPUFeature(S3_0_C0_C4_4, ftr);
1264       // ID_AA64ZFR0_EL1.SVEver == 0b0000
1265       if (extractBits(ftr, 0, 4) == 0x0)
1266         setCPUFeature(FEAT_SVE);
1267       // ID_AA64ZFR0_EL1.SVEver == 0b0001
1268       if (extractBits(ftr, 0, 4) == 0x1)
1269         setCPUFeature(FEAT_SVE2);
1270       // ID_AA64ZFR0_EL1.BF16 != 0b0000
1271       if (extractBits(ftr, 20, 4) != 0x0)
1272         setCPUFeature(FEAT_SVE_BF16);
1273     }
1274     getCPUFeature(ID_AA64ISAR0_EL1, ftr);
1275     // ID_AA64ISAR0_EL1.SHA3 != 0b0000
1276     if (extractBits(ftr, 32, 4) != 0x0)
1277       setCPUFeature(FEAT_SHA3);
1278     getCPUFeature(ID_AA64ISAR1_EL1, ftr);
1279     // ID_AA64ISAR1_EL1.DPB >= 0b0001
1280     if (extractBits(ftr, 0, 4) >= 0x1)
1281       setCPUFeature(FEAT_DPB);
1282     // ID_AA64ISAR1_EL1.LRCPC != 0b0000
1283     if (extractBits(ftr, 20, 4) != 0x0)
1284       setCPUFeature(FEAT_RCPC);
1285     // ID_AA64ISAR1_EL1.SPECRES == 0b0001
1286     if (extractBits(ftr, 40, 4) == 0x2)
1287       setCPUFeature(FEAT_PREDRES);
1288     // ID_AA64ISAR1_EL1.BF16 != 0b0000
1289     if (extractBits(ftr, 44, 4) != 0x0)
1290       setCPUFeature(FEAT_BF16);
1291     // ID_AA64ISAR1_EL1.LS64 >= 0b0001
1292     if (extractBits(ftr, 60, 4) >= 0x1)
1293       setCPUFeature(FEAT_LS64);
1294     // ID_AA64ISAR1_EL1.LS64 >= 0b0010
1295     if (extractBits(ftr, 60, 4) >= 0x2)
1296       setCPUFeature(FEAT_LS64_V);
1297     // ID_AA64ISAR1_EL1.LS64 >= 0b0011
1298     if (extractBits(ftr, 60, 4) >= 0x3)
1299       setCPUFeature(FEAT_LS64_ACCDATA);
1300   } else {
1301     // Set some features in case of no CPUID support
1302     if (hwcap & (HWCAP_FP | HWCAP_FPHP)) {
1303       setCPUFeature(FEAT_FP);
1304       // FP and AdvSIMD fields have the same value
1305       setCPUFeature(FEAT_SIMD);
1306     }
1307     if (hwcap & HWCAP_DCPOP || hwcap2 & HWCAP2_DCPODP)
1308       setCPUFeature(FEAT_DPB);
1309     if (hwcap & HWCAP_LRCPC || hwcap & HWCAP_ILRCPC)
1310       setCPUFeature(FEAT_RCPC);
1311     if (hwcap2 & HWCAP2_BF16 || hwcap2 & HWCAP2_EBF16)
1312       setCPUFeature(FEAT_BF16);
1313     if (hwcap2 & HWCAP2_SVEBF16)
1314       setCPUFeature(FEAT_SVE_BF16);
1315     if (hwcap2 & HWCAP2_SVE2 && hwcap & HWCAP_SVE)
1316       setCPUFeature(FEAT_SVE2);
1317     if (hwcap & HWCAP_SHA3)
1318       setCPUFeature(FEAT_SHA3);
1319   }
1320 }
1321
1322 void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) {
1323   unsigned long hwcap;
1324   unsigned long hwcap2;
1325   // CPU features already initialized.
1326   if (__aarch64_cpu_features.features)
1327     return;
1328   setCPUFeature(FEAT_MAX);
1329 #if defined(__FreeBSD__)
1330   int res = 0;
1331   res = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap);
1332   res |= elf_aux_info(AT_HWCAP2, &hwcap2, sizeof hwcap2);
1333   if (res)
1334     return;
1335 #else
1336 #if defined(__ANDROID__)
1337   // Don't set any CPU features,
1338   // detection could be wrong on Exynos 9810.
1339   IF_EXYNOS9810 return;
1340 #endif // defined(__ANDROID__)
1341   hwcap = getauxval(AT_HWCAP);
1342   hwcap2 = getauxval(AT_HWCAP2);
1343 #endif // defined(__FreeBSD__)
1344   init_cpu_features_resolver(hwcap, hwcap2);
1345 #undef extractBits
1346 #undef getCPUFeature
1347 #undef setCPUFeature
1348 #undef IF_EXYNOS9810
1349 }
1350 #endif // !defined(DISABLE_AARCH64_FMV)
1351 #endif // defined(__has_include)
1352 #endif // __has_include(<sys/auxv.h>)
1353 #endif // __has_include(<asm/hwcap.h>)
1354 #endif // defined(__aarch64__)