src/system/kernel/arch/x86/arch_cpu.cpp

   1 /*
   2  * Copyright 2002-2010, Axel Dörfler, axeld@pinc-software.de.
   3  * Copyright 2013, Paweł Dziepak, pdziepak@quarnos.org.
   4  * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
   5  * Distributed under the terms of the MIT License.
   6  *
   7  * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved.
   8  * Distributed under the terms of the NewOS License.
   9  */
  10
  11
  12 #include <cpu.h>
  13
  14 #include <string.h>
  15 #include <stdlib.h>
  16 #include <stdio.h>
  17
  18 #include <algorithm>
  19
  20 #include <ACPI.h>
  21
  22 #include <boot_device.h>
  23 #include <commpage.h>
  24 #include <debug.h>
  25 #include <elf.h>
  26 #include <smp.h>
  27 #include <util/BitUtils.h>
  28 #include <vm/vm.h>
  29 #include <vm/vm_types.h>
  30 #include <vm/VMAddressSpace.h>
  31
  32 #include <arch_system_info.h>
  33 #include <arch/x86/apic.h>
  34 #include <boot/kernel_args.h>
  35
  36 #include "paging/X86PagingStructures.h"
  37 #include "paging/X86VMTranslationMap.h"
  38
  39
  40 #define DUMP_FEATURE_STRING 1
  41 #define DUMP_CPU_TOPOLOGY       1
  42
  43
  44 /* cpu vendor info */
  45 struct cpu_vendor_info {
  46         const char *vendor;
  47         const char *ident_string[2];
  48 };
  49
  50 static const struct cpu_vendor_info vendor_info[VENDOR_NUM] = {
  51         { "Intel", { "GenuineIntel" } },
  52         { "AMD", { "AuthenticAMD" } },
  53         { "Cyrix", { "CyrixInstead" } },
  54         { "UMC", { "UMC UMC UMC" } },
  55         { "NexGen", { "NexGenDriven" } },
  56         { "Centaur", { "CentaurHauls" } },
  57         { "Rise", { "RiseRiseRise" } },
  58         { "Transmeta", { "GenuineTMx86", "TransmetaCPU" } },
  59         { "NSC", { "Geode by NSC" } },
  60 };
  61
  62 #define K8_SMIONCMPHALT                 (1ULL << 27)
  63 #define K8_C1EONCMPHALT                 (1ULL << 28)
  64
  65 #define K8_CMPHALT                              (K8_SMIONCMPHALT | K8_C1EONCMPHALT)
  66
  67 struct set_mtrr_parameter {
  68         int32   index;
  69         uint64  base;
  70         uint64  length;
  71         uint8   type;
  72 };
  73
  74 struct set_mtrrs_parameter {
  75         const x86_mtrr_info*    infos;
  76         uint32                                  count;
  77         uint8                                   defaultType;
  78 };
  79
  80
  81 extern "C" void x86_reboot(void);
  82         // from arch.S
  83
  84 void (*gCpuIdleFunc)(void);
  85 #ifndef __x86_64__
  86 void (*gX86SwapFPUFunc)(void* oldState, const void* newState) = x86_noop_swap;
  87 bool gHasSSE = false;
  88 #endif
  89
  90 static uint32 sCpuRendezvous;
  91 static uint32 sCpuRendezvous2;
  92 static uint32 sCpuRendezvous3;
  93 static vint32 sTSCSyncRendezvous;
  94
  95 /* Some specials for the double fault handler */
  96 static uint8* sDoubleFaultStacks;
  97 static const size_t kDoubleFaultStackSize = 4096;       // size per CPU
  98
  99 static x86_cpu_module_info* sCpuModule;
 100
 101
 102 /* CPU topology information */
 103 static uint32 (*sGetCPUTopologyID)(int currentCPU);
 104 static uint32 sHierarchyMask[CPU_TOPOLOGY_LEVELS];
 105 static uint32 sHierarchyShift[CPU_TOPOLOGY_LEVELS];
 106
 107 /* Cache topology information */
 108 static uint32 sCacheSharingMask[CPU_MAX_CACHE_LEVEL];
 109
 110
 111 static status_t
 112 acpi_shutdown(bool rebootSystem)
 113 {
 114         if (debug_debugger_running() || !are_interrupts_enabled())
 115                 return B_ERROR;
 116
 117         acpi_module_info* acpi;
 118         if (get_module(B_ACPI_MODULE_NAME, (module_info**)&acpi) != B_OK)
 119                 return B_NOT_SUPPORTED;
 120
 121         status_t status;
 122         if (rebootSystem) {
 123                 status = acpi->reboot();
 124         } else {
 125                 // Make sure we run on the boot CPU (apparently needed for some ACPI
 126                 // implementations)
 127                 _user_set_cpu_enabled(0, true);
 128                 for (int32 cpu = 1; cpu < smp_get_num_cpus(); cpu++) {
 129                         _user_set_cpu_enabled(cpu, false);
 130                 }
 131                 // TODO: must not be called from the idle thread!
 132                 thread_yield();
 133
 134                 status = acpi->prepare_sleep_state(ACPI_POWER_STATE_OFF, NULL, 0);
 135                 if (status == B_OK) {
 136                         //cpu_status state = disable_interrupts();
 137                         status = acpi->enter_sleep_state(ACPI_POWER_STATE_OFF);
 138                         //restore_interrupts(state);
 139                 }
 140         }
 141
 142         put_module(B_ACPI_MODULE_NAME);
 143         return status;
 144 }
 145
 146
 147 /*!     Disable CPU caches, and invalidate them. */
 148 static void
 149 disable_caches()
 150 {
 151         x86_write_cr0((x86_read_cr0() | CR0_CACHE_DISABLE)
 152                 & ~CR0_NOT_WRITE_THROUGH);
 153         wbinvd();
 154         arch_cpu_global_TLB_invalidate();
 155 }
 156
 157
 158 /*!     Invalidate CPU caches, and enable them. */
 159 static void
 160 enable_caches()
 161 {
 162         wbinvd();
 163         arch_cpu_global_TLB_invalidate();
 164         x86_write_cr0(x86_read_cr0()
 165                 & ~(CR0_CACHE_DISABLE | CR0_NOT_WRITE_THROUGH));
 166 }
 167
 168
 169 static void
 170 set_mtrr(void* _parameter, int cpu)
 171 {
 172         struct set_mtrr_parameter* parameter
 173                 = (struct set_mtrr_parameter*)_parameter;
 174
 175         // wait until all CPUs have arrived here
 176         smp_cpu_rendezvous(&sCpuRendezvous);
 177
 178         // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
 179         // that initiated the call_all_cpus() from doing that again and clearing
 180         // sCpuRendezvous2 before the last CPU has actually left the loop in
 181         // smp_cpu_rendezvous();
 182         if (cpu == 0)
 183                 atomic_set((int32*)&sCpuRendezvous3, 0);
 184
 185         disable_caches();
 186
 187         sCpuModule->set_mtrr(parameter->index, parameter->base, parameter->length,
 188                 parameter->type);
 189
 190         enable_caches();
 191
 192         // wait until all CPUs have arrived here
 193         smp_cpu_rendezvous(&sCpuRendezvous2);
 194         smp_cpu_rendezvous(&sCpuRendezvous3);
 195 }
 196
 197
 198 static void
 199 set_mtrrs(void* _parameter, int cpu)
 200 {
 201         set_mtrrs_parameter* parameter = (set_mtrrs_parameter*)_parameter;
 202
 203         // wait until all CPUs have arrived here
 204         smp_cpu_rendezvous(&sCpuRendezvous);
 205
 206         // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
 207         // that initiated the call_all_cpus() from doing that again and clearing
 208         // sCpuRendezvous2 before the last CPU has actually left the loop in
 209         // smp_cpu_rendezvous();
 210         if (cpu == 0)
 211                 atomic_set((int32*)&sCpuRendezvous3, 0);
 212
 213         disable_caches();
 214
 215         sCpuModule->set_mtrrs(parameter->defaultType, parameter->infos,
 216                 parameter->count);
 217
 218         enable_caches();
 219
 220         // wait until all CPUs have arrived here
 221         smp_cpu_rendezvous(&sCpuRendezvous2);
 222         smp_cpu_rendezvous(&sCpuRendezvous3);
 223 }
 224
 225
 226 static void
 227 init_mtrrs(void* _unused, int cpu)
 228 {
 229         // wait until all CPUs have arrived here
 230         smp_cpu_rendezvous(&sCpuRendezvous);
 231
 232         // One CPU has to reset sCpuRendezvous3 -- it is needed to prevent the CPU
 233         // that initiated the call_all_cpus() from doing that again and clearing
 234         // sCpuRendezvous2 before the last CPU has actually left the loop in
 235         // smp_cpu_rendezvous();
 236         if (cpu == 0)
 237                 atomic_set((int32*)&sCpuRendezvous3, 0);
 238
 239         disable_caches();
 240
 241         sCpuModule->init_mtrrs();
 242
 243         enable_caches();
 244
 245         // wait until all CPUs have arrived here
 246         smp_cpu_rendezvous(&sCpuRendezvous2);
 247         smp_cpu_rendezvous(&sCpuRendezvous3);
 248 }
 249
 250
 251 uint32
 252 x86_count_mtrrs(void)
 253 {
 254         if (sCpuModule == NULL)
 255                 return 0;
 256
 257         return sCpuModule->count_mtrrs();
 258 }
 259
 260
 261 void
 262 x86_set_mtrr(uint32 index, uint64 base, uint64 length, uint8 type)
 263 {
 264         struct set_mtrr_parameter parameter;
 265         parameter.index = index;
 266         parameter.base = base;
 267         parameter.length = length;
 268         parameter.type = type;
 269
 270         sCpuRendezvous = sCpuRendezvous2 = 0;
 271         call_all_cpus(&set_mtrr, &parameter);
 272 }
 273
 274
 275 status_t
 276 x86_get_mtrr(uint32 index, uint64* _base, uint64* _length, uint8* _type)
 277 {
 278         // the MTRRs are identical on all CPUs, so it doesn't matter
 279         // on which CPU this runs
 280         return sCpuModule->get_mtrr(index, _base, _length, _type);
 281 }
 282
 283
 284 void
 285 x86_set_mtrrs(uint8 defaultType, const x86_mtrr_info* infos, uint32 count)
 286 {
 287         if (sCpuModule == NULL)
 288                 return;
 289
 290         struct set_mtrrs_parameter parameter;
 291         parameter.defaultType = defaultType;
 292         parameter.infos = infos;
 293         parameter.count = count;
 294
 295         sCpuRendezvous = sCpuRendezvous2 = 0;
 296         call_all_cpus(&set_mtrrs, &parameter);
 297 }
 298
 299
 300 void
 301 x86_init_fpu(void)
 302 {
 303         // All x86_64 CPUs support SSE, don't need to bother checking for it.
 304 #ifndef __x86_64__
 305         if (!x86_check_feature(IA32_FEATURE_FPU, FEATURE_COMMON)) {
 306                 // No FPU... time to install one in your 386?
 307                 dprintf("%s: Warning: CPU has no reported FPU.\n", __func__);
 308                 gX86SwapFPUFunc = x86_noop_swap;
 309                 return;
 310         }
 311
 312         if (!x86_check_feature(IA32_FEATURE_SSE, FEATURE_COMMON)
 313                 || !x86_check_feature(IA32_FEATURE_FXSR, FEATURE_COMMON)) {
 314                 dprintf("%s: CPU has no SSE... just enabling FPU.\n", __func__);
 315                 // we don't have proper SSE support, just enable FPU
 316                 x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
 317                 gX86SwapFPUFunc = x86_fnsave_swap;
 318                 return;
 319         }
 320 #endif
 321
 322         dprintf("%s: CPU has SSE... enabling FXSR and XMM.\n", __func__);
 323 #ifndef __x86_64__
 324         // enable OS support for SSE
 325         x86_write_cr4(x86_read_cr4() | CR4_OS_FXSR | CR4_OS_XMM_EXCEPTION);
 326         x86_write_cr0(x86_read_cr0() & ~(CR0_FPU_EMULATION | CR0_MONITOR_FPU));
 327
 328         gX86SwapFPUFunc = x86_fxsave_swap;
 329         gHasSSE = true;
 330 #endif
 331 }
 332
 333
 334 #if DUMP_FEATURE_STRING
 335 static void
 336 dump_feature_string(int currentCPU, cpu_ent* cpu)
 337 {
 338         char features[384];
 339         features[0] = 0;
 340
 341         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FPU)
 342                 strlcat(features, "fpu ", sizeof(features));
 343         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_VME)
 344                 strlcat(features, "vme ", sizeof(features));
 345         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DE)
 346                 strlcat(features, "de ", sizeof(features));
 347         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE)
 348                 strlcat(features, "pse ", sizeof(features));
 349         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TSC)
 350                 strlcat(features, "tsc ", sizeof(features));
 351         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MSR)
 352                 strlcat(features, "msr ", sizeof(features));
 353         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAE)
 354                 strlcat(features, "pae ", sizeof(features));
 355         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCE)
 356                 strlcat(features, "mce ", sizeof(features));
 357         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CX8)
 358                 strlcat(features, "cx8 ", sizeof(features));
 359         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_APIC)
 360                 strlcat(features, "apic ", sizeof(features));
 361         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SEP)
 362                 strlcat(features, "sep ", sizeof(features));
 363         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MTRR)
 364                 strlcat(features, "mtrr ", sizeof(features));
 365         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PGE)
 366                 strlcat(features, "pge ", sizeof(features));
 367         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MCA)
 368                 strlcat(features, "mca ", sizeof(features));
 369         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CMOV)
 370                 strlcat(features, "cmov ", sizeof(features));
 371         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PAT)
 372                 strlcat(features, "pat ", sizeof(features));
 373         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSE36)
 374                 strlcat(features, "pse36 ", sizeof(features));
 375         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PSN)
 376                 strlcat(features, "psn ", sizeof(features));
 377         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_CLFSH)
 378                 strlcat(features, "clfsh ", sizeof(features));
 379         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_DS)
 380                 strlcat(features, "ds ", sizeof(features));
 381         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_ACPI)
 382                 strlcat(features, "acpi ", sizeof(features));
 383         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_MMX)
 384                 strlcat(features, "mmx ", sizeof(features));
 385         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_FXSR)
 386                 strlcat(features, "fxsr ", sizeof(features));
 387         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE)
 388                 strlcat(features, "sse ", sizeof(features));
 389         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SSE2)
 390                 strlcat(features, "sse2 ", sizeof(features));
 391         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_SS)
 392                 strlcat(features, "ss ", sizeof(features));
 393         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_HTT)
 394                 strlcat(features, "htt ", sizeof(features));
 395         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_TM)
 396                 strlcat(features, "tm ", sizeof(features));
 397         if (cpu->arch.feature[FEATURE_COMMON] & IA32_FEATURE_PBE)
 398                 strlcat(features, "pbe ", sizeof(features));
 399         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE3)
 400                 strlcat(features, "sse3 ", sizeof(features));
 401         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCLMULQDQ)
 402                 strlcat(features, "pclmulqdq ", sizeof(features));
 403         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DTES64)
 404                 strlcat(features, "dtes64 ", sizeof(features));
 405         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MONITOR)
 406                 strlcat(features, "monitor ", sizeof(features));
 407         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DSCPL)
 408                 strlcat(features, "dscpl ", sizeof(features));
 409         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_VMX)
 410                 strlcat(features, "vmx ", sizeof(features));
 411         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SMX)
 412                 strlcat(features, "smx ", sizeof(features));
 413         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_EST)
 414                 strlcat(features, "est ", sizeof(features));
 415         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TM2)
 416                 strlcat(features, "tm2 ", sizeof(features));
 417         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSSE3)
 418                 strlcat(features, "ssse3 ", sizeof(features));
 419         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CNXTID)
 420                 strlcat(features, "cnxtid ", sizeof(features));
 421         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_FMA)
 422                 strlcat(features, "fma ", sizeof(features));
 423         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_CX16)
 424                 strlcat(features, "cx16 ", sizeof(features));
 425         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XTPR)
 426                 strlcat(features, "xtpr ", sizeof(features));
 427         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PDCM)
 428                 strlcat(features, "pdcm ", sizeof(features));
 429         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_PCID)
 430                 strlcat(features, "pcid ", sizeof(features));
 431         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_DCA)
 432                 strlcat(features, "dca ", sizeof(features));
 433         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_1)
 434                 strlcat(features, "sse4_1 ", sizeof(features));
 435         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_SSE4_2)
 436                 strlcat(features, "sse4_2 ", sizeof(features));
 437         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_X2APIC)
 438                 strlcat(features, "x2apic ", sizeof(features));
 439         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_MOVBE)
 440                 strlcat(features, "movbe ", sizeof(features));
 441         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_POPCNT)
 442                 strlcat(features, "popcnt ", sizeof(features));
 443         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_TSCDEADLINE)
 444                 strlcat(features, "tscdeadline ", sizeof(features));
 445         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AES)
 446                 strlcat(features, "aes ", sizeof(features));
 447         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_XSAVE)
 448                 strlcat(features, "xsave ", sizeof(features));
 449         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_OSXSAVE)
 450                 strlcat(features, "osxsave ", sizeof(features));
 451         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_AVX)
 452                 strlcat(features, "avx ", sizeof(features));
 453         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_F16C)
 454                 strlcat(features, "f16c ", sizeof(features));
 455         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_RDRND)
 456                 strlcat(features, "rdrnd ", sizeof(features));
 457         if (cpu->arch.feature[FEATURE_EXT] & IA32_FEATURE_EXT_HYPERVISOR)
 458                 strlcat(features, "hypervisor ", sizeof(features));
 459         if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_SYSCALL)
 460                 strlcat(features, "syscall ", sizeof(features));
 461         if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_NX)
 462                 strlcat(features, "nx ", sizeof(features));
 463         if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_MMXEXT)
 464                 strlcat(features, "mmxext ", sizeof(features));
 465         if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_FFXSR)
 466                 strlcat(features, "ffxsr ", sizeof(features));
 467         if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_LONG)
 468                 strlcat(features, "long ", sizeof(features));
 469         if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOWEXT)
 470                 strlcat(features, "3dnowext ", sizeof(features));
 471         if (cpu->arch.feature[FEATURE_EXT_AMD] & IA32_FEATURE_AMD_EXT_3DNOW)
 472                 strlcat(features, "3dnow ", sizeof(features));
 473         if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_DTS)
 474                 strlcat(features, "dts ", sizeof(features));
 475         if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ITB)
 476                 strlcat(features, "itb ", sizeof(features));
 477         if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ARAT)
 478                 strlcat(features, "arat ", sizeof(features));
 479         if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PLN)
 480                 strlcat(features, "pln ", sizeof(features));
 481         if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_ECMD)
 482                 strlcat(features, "ecmd ", sizeof(features));
 483         if (cpu->arch.feature[FEATURE_6_EAX] & IA32_FEATURE_PTM)
 484                 strlcat(features, "ptm ", sizeof(features));
 485         if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_APERFMPERF)
 486                 strlcat(features, "aperfmperf ", sizeof(features));
 487         if (cpu->arch.feature[FEATURE_6_ECX] & IA32_FEATURE_EPB)
 488                 strlcat(features, "epb ", sizeof(features));
 489
 490         dprintf("CPU %d: features: %s\n", currentCPU, features);
 491 }
 492 #endif  // DUMP_FEATURE_STRING
 493
 494
 495 static void
 496 compute_cpu_hierarchy_masks(int maxLogicalID, int maxCoreID)
 497 {
 498         ASSERT(maxLogicalID >= maxCoreID);
 499         const int kMaxSMTID = maxLogicalID / maxCoreID;
 500
 501         sHierarchyMask[CPU_TOPOLOGY_SMT] = kMaxSMTID - 1;
 502         sHierarchyShift[CPU_TOPOLOGY_SMT] = 0;
 503
 504         sHierarchyMask[CPU_TOPOLOGY_CORE] = (maxCoreID - 1) * kMaxSMTID;
 505         sHierarchyShift[CPU_TOPOLOGY_CORE]
 506                 = count_set_bits(sHierarchyMask[CPU_TOPOLOGY_SMT]);
 507
 508         const uint32 kSinglePackageMask = sHierarchyMask[CPU_TOPOLOGY_SMT]
 509                 | sHierarchyMask[CPU_TOPOLOGY_CORE];
 510         sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~kSinglePackageMask;
 511         sHierarchyShift[CPU_TOPOLOGY_PACKAGE] = count_set_bits(kSinglePackageMask);
 512 }
 513
 514
 515 static uint32
 516 get_cpu_legacy_initial_apic_id(int /* currentCPU */)
 517 {
 518         cpuid_info cpuid;
 519         get_current_cpuid(&cpuid, 1, 0);
 520         return cpuid.regs.ebx >> 24;
 521 }
 522
 523
 524 static inline status_t
 525 detect_amd_cpu_topology(uint32 maxBasicLeaf, uint32 maxExtendedLeaf)
 526 {
 527         sGetCPUTopologyID = get_cpu_legacy_initial_apic_id;
 528
 529         cpuid_info cpuid;
 530         get_current_cpuid(&cpuid, 1, 0);
 531         int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff);
 532
 533         int maxCoreID = 1;
 534         if (maxExtendedLeaf >= 0x80000008) {
 535                 get_current_cpuid(&cpuid, 0x80000008, 0);
 536                 maxCoreID = (cpuid.regs.ecx >> 12) & 0xf;
 537                 if (maxCoreID != 0)
 538                         maxCoreID = 1 << maxCoreID;
 539                 else
 540                         maxCoreID = next_power_of_2((cpuid.regs.edx & 0xf) + 1);
 541         }
 542
 543         if (maxExtendedLeaf >= 0x80000001) {
 544                 get_current_cpuid(&cpuid, 0x80000001, 0);
 545                 if (x86_check_feature(IA32_FEATURE_AMD_EXT_CMPLEGACY,
 546                                 FEATURE_EXT_AMD_ECX))
 547                         maxCoreID = maxLogicalID;
 548         }
 549
 550         compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID);
 551
 552         return B_OK;
 553 }
 554
 555
 556 static void
 557 detect_amd_cache_topology(uint32 maxExtendedLeaf)
 558 {
 559         if (!x86_check_feature(IA32_FEATURE_AMD_EXT_TOPOLOGY, FEATURE_EXT_AMD_ECX))
 560                 return;
 561
 562         if (maxExtendedLeaf < 0x8000001d)
 563                 return;
 564
 565         uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL];
 566         int maxCacheLevel = 0;
 567
 568         int currentLevel = 0;
 569         int cacheType;
 570         do {
 571                 cpuid_info cpuid;
 572                 get_current_cpuid(&cpuid, 0x8000001d, currentLevel);
 573
 574                 cacheType = cpuid.regs.eax & 0x1f;
 575                 if (cacheType == 0)
 576                         break;
 577
 578                 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7;
 579                 int coresCount = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1);
 580                 hierarchyLevels[cacheLevel - 1]
 581                         = coresCount * (sHierarchyMask[CPU_TOPOLOGY_SMT] + 1);
 582                 maxCacheLevel = std::max(maxCacheLevel, cacheLevel);
 583
 584                 currentLevel++;
 585         } while (true);
 586
 587         for (int i = 0; i < maxCacheLevel; i++)
 588                 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1);
 589         gCPUCacheLevelCount = maxCacheLevel;
 590 }
 591
 592
 593 static uint32
 594 get_intel_cpu_initial_x2apic_id(int /* currentCPU */)
 595 {
 596         cpuid_info cpuid;
 597         get_current_cpuid(&cpuid, 11, 0);
 598         return cpuid.regs.edx;
 599 }
 600
 601
 602 static inline status_t
 603 detect_intel_cpu_topology_x2apic(uint32 maxBasicLeaf)
 604 {
 605         if (maxBasicLeaf < 11)
 606                 return B_UNSUPPORTED;
 607
 608         uint8 hierarchyLevels[CPU_TOPOLOGY_LEVELS] = { 0 };
 609
 610         int currentLevel = 0;
 611         int levelType;
 612         unsigned int levelsSet = 0;
 613
 614         do {
 615                 cpuid_info cpuid;
 616                 get_current_cpuid(&cpuid, 11, currentLevel);
 617                 if (currentLevel == 0 && cpuid.regs.ebx == 0)
 618                         return B_UNSUPPORTED;
 619
 620                 levelType = (cpuid.regs.ecx >> 8) & 0xff;
 621                 int levelValue = cpuid.regs.eax & 0x1f;
 622
 623                 switch (levelType) {
 624                         case 1: // SMT
 625                                 hierarchyLevels[CPU_TOPOLOGY_SMT] = levelValue;
 626                                 levelsSet |= 1;
 627                                 break;
 628                         case 2: // core
 629                                 hierarchyLevels[CPU_TOPOLOGY_CORE] = levelValue;
 630                                 levelsSet |= 2;
 631                                 break;
 632                 }
 633
 634                 currentLevel++;
 635         } while (levelType != 0 && levelsSet != 3);
 636
 637         sGetCPUTopologyID = get_intel_cpu_initial_x2apic_id;
 638
 639         for (int i = 1; i < CPU_TOPOLOGY_LEVELS; i++) {
 640                 if ((levelsSet & (1u << i)) != 0)
 641                         continue;
 642                 hierarchyLevels[i] = hierarchyLevels[i - 1];
 643         }
 644
 645         for (int i = 0; i < CPU_TOPOLOGY_LEVELS; i++) {
 646                 uint32 mask = ~uint32(0);
 647                 if (i < CPU_TOPOLOGY_LEVELS - 1)
 648                         mask = (1u << hierarchyLevels[i]) - 1;
 649                 if (i > 0)
 650                         mask &= ~sHierarchyMask[i - 1];
 651                 sHierarchyMask[i] = mask;
 652                 sHierarchyShift[i] = i > 0 ? hierarchyLevels[i - 1] : 0;
 653         }
 654
 655         return B_OK;
 656 }
 657
 658
 659 static inline status_t
 660 detect_intel_cpu_topology_legacy(uint32 maxBasicLeaf)
 661 {
 662         sGetCPUTopologyID = get_cpu_legacy_initial_apic_id;
 663
 664         cpuid_info cpuid;
 665
 666         get_current_cpuid(&cpuid, 1, 0);
 667         int maxLogicalID = next_power_of_2((cpuid.regs.ebx >> 16) & 0xff);
 668
 669         int maxCoreID = 1;
 670         if (maxBasicLeaf >= 4) {
 671                 get_current_cpuid(&cpuid, 4, 0);
 672                 maxCoreID = next_power_of_2((cpuid.regs.eax >> 26) + 1);
 673         }
 674
 675         compute_cpu_hierarchy_masks(maxLogicalID, maxCoreID);
 676
 677         return B_OK;
 678 }
 679
 680
 681 static void
 682 detect_intel_cache_topology(uint32 maxBasicLeaf)
 683 {
 684         if (maxBasicLeaf < 4)
 685                 return;
 686
 687         uint8 hierarchyLevels[CPU_MAX_CACHE_LEVEL];
 688         int maxCacheLevel = 0;
 689
 690         int currentLevel = 0;
 691         int cacheType;
 692         do {
 693                 cpuid_info cpuid;
 694                 get_current_cpuid(&cpuid, 4, currentLevel);
 695
 696                 cacheType = cpuid.regs.eax & 0x1f;
 697                 if (cacheType == 0)
 698                         break;
 699
 700                 int cacheLevel = (cpuid.regs.eax >> 5) & 0x7;
 701                 hierarchyLevels[cacheLevel - 1]
 702                         = next_power_of_2(((cpuid.regs.eax >> 14) & 0x3f) + 1);
 703                 maxCacheLevel = std::max(maxCacheLevel, cacheLevel);
 704
 705                 currentLevel++;
 706         } while (true);
 707
 708         for (int i = 0; i < maxCacheLevel; i++)
 709                 sCacheSharingMask[i] = ~uint32(hierarchyLevels[i] - 1);
 710
 711         gCPUCacheLevelCount = maxCacheLevel;
 712 }
 713
 714
 715 static uint32
 716 get_simple_cpu_topology_id(int currentCPU)
 717 {
 718         return currentCPU;
 719 }
 720
 721
 722 static inline int
 723 get_topology_level_id(uint32 id, cpu_topology_level level)
 724 {
 725         ASSERT(level < CPU_TOPOLOGY_LEVELS);
 726         return (id & sHierarchyMask[level]) >> sHierarchyShift[level];
 727 }
 728
 729
 730 static void
 731 detect_cpu_topology(int currentCPU, cpu_ent* cpu, uint32 maxBasicLeaf,
 732         uint32 maxExtendedLeaf)
 733 {
 734         if (currentCPU == 0) {
 735                 memset(sCacheSharingMask, 0xff, sizeof(sCacheSharingMask));
 736
 737                 status_t result = B_UNSUPPORTED;
 738                 if (x86_check_feature(IA32_FEATURE_HTT, FEATURE_COMMON)) {
 739                         if (cpu->arch.vendor == VENDOR_AMD) {
 740                                 result = detect_amd_cpu_topology(maxBasicLeaf, maxExtendedLeaf);
 741
 742                                 if (result == B_OK)
 743                                         detect_amd_cache_topology(maxExtendedLeaf);
 744                         }
 745
 746                         if (cpu->arch.vendor == VENDOR_INTEL) {
 747                                 result = detect_intel_cpu_topology_x2apic(maxBasicLeaf);
 748                                 if (result != B_OK)
 749                                         result = detect_intel_cpu_topology_legacy(maxBasicLeaf);
 750
 751                                 if (result == B_OK)
 752                                         detect_intel_cache_topology(maxBasicLeaf);
 753                         }
 754                 }
 755
 756                 if (result != B_OK) {
 757                         dprintf("No CPU topology information available.\n");
 758
 759                         sGetCPUTopologyID = get_simple_cpu_topology_id;
 760
 761                         sHierarchyMask[CPU_TOPOLOGY_PACKAGE] = ~uint32(0);
 762                 }
 763         }
 764
 765         ASSERT(sGetCPUTopologyID != NULL);
 766         int topologyID = sGetCPUTopologyID(currentCPU);
 767         cpu->topology_id[CPU_TOPOLOGY_SMT]
 768                 = get_topology_level_id(topologyID, CPU_TOPOLOGY_SMT);
 769         cpu->topology_id[CPU_TOPOLOGY_CORE]
 770                 = get_topology_level_id(topologyID, CPU_TOPOLOGY_CORE);
 771         cpu->topology_id[CPU_TOPOLOGY_PACKAGE]
 772                 = get_topology_level_id(topologyID, CPU_TOPOLOGY_PACKAGE);
 773
 774         unsigned int i;
 775         for (i = 0; i < gCPUCacheLevelCount; i++)
 776                 cpu->cache_id[i] = topologyID & sCacheSharingMask[i];
 777         for (; i < CPU_MAX_CACHE_LEVEL; i++)
 778                 cpu->cache_id[i] = -1;
 779
 780 #if DUMP_CPU_TOPOLOGY
 781         dprintf("CPU %d: apic id %d, package %d, core %d, smt %d\n", currentCPU,
 782                 topologyID, cpu->topology_id[CPU_TOPOLOGY_PACKAGE],
 783                 cpu->topology_id[CPU_TOPOLOGY_CORE],
 784                 cpu->topology_id[CPU_TOPOLOGY_SMT]);
 785
 786         if (gCPUCacheLevelCount > 0) {
 787                 char cacheLevels[256];
 788                 unsigned int offset = 0;
 789                 for (i = 0; i < gCPUCacheLevelCount; i++) {
 790                         offset += snprintf(cacheLevels + offset,
 791                                         sizeof(cacheLevels) - offset,
 792                                         " L%d id %d%s", i + 1, cpu->cache_id[i],
 793                                         i < gCPUCacheLevelCount - 1 ? "," : "");
 794
 795                         if (offset >= sizeof(cacheLevels))
 796                                 break;
 797                 }
 798
 799                 dprintf("CPU %d: cache sharing:%s\n", currentCPU, cacheLevels);
 800         }
 801 #endif
 802 }
 803
 804
 805 static void
 806 detect_cpu(int currentCPU)
 807 {
 808         cpu_ent* cpu = get_cpu_struct();
 809         char vendorString[17];
 810         cpuid_info cpuid;
 811
 812         // clear out the cpu info data
 813         cpu->arch.vendor = VENDOR_UNKNOWN;
 814         cpu->arch.vendor_name = "UNKNOWN VENDOR";
 815         cpu->arch.feature[FEATURE_COMMON] = 0;
 816         cpu->arch.feature[FEATURE_EXT] = 0;
 817         cpu->arch.feature[FEATURE_EXT_AMD] = 0;
 818         cpu->arch.model_name[0] = 0;
 819
 820         // print some fun data
 821         get_current_cpuid(&cpuid, 0, 0);
 822         uint32 maxBasicLeaf = cpuid.eax_0.max_eax;
 823
 824         // build the vendor string
 825         memset(vendorString, 0, sizeof(vendorString));
 826         memcpy(vendorString, cpuid.eax_0.vendor_id, sizeof(cpuid.eax_0.vendor_id));
 827
 828         // get the family, model, stepping
 829         get_current_cpuid(&cpuid, 1, 0);
 830         cpu->arch.type = cpuid.eax_1.type;
 831         cpu->arch.family = cpuid.eax_1.family;
 832         cpu->arch.extended_family = cpuid.eax_1.extended_family;
 833         cpu->arch.model = cpuid.eax_1.model;
 834         cpu->arch.extended_model = cpuid.eax_1.extended_model;
 835         cpu->arch.stepping = cpuid.eax_1.stepping;
 836         dprintf("CPU %d: type %d family %d extended_family %d model %d "
 837                 "extended_model %d stepping %d, string '%s'\n",
 838                 currentCPU, cpu->arch.type, cpu->arch.family,
 839                 cpu->arch.extended_family, cpu->arch.model,
 840                 cpu->arch.extended_model, cpu->arch.stepping, vendorString);
 841
 842         // figure out what vendor we have here
 843
 844         for (int32 i = 0; i < VENDOR_NUM; i++) {
 845                 if (vendor_info[i].ident_string[0]
 846                         && !strcmp(vendorString, vendor_info[i].ident_string[0])) {
 847                         cpu->arch.vendor = (x86_vendors)i;
 848                         cpu->arch.vendor_name = vendor_info[i].vendor;
 849                         break;
 850                 }
 851                 if (vendor_info[i].ident_string[1]
 852                         && !strcmp(vendorString, vendor_info[i].ident_string[1])) {
 853                         cpu->arch.vendor = (x86_vendors)i;
 854                         cpu->arch.vendor_name = vendor_info[i].vendor;
 855                         break;
 856                 }
 857         }
 858
 859         // see if we can get the model name
 860         get_current_cpuid(&cpuid, 0x80000000, 0);
 861         uint32 maxExtendedLeaf = cpuid.eax_0.max_eax;
 862         if (maxExtendedLeaf >= 0x80000004) {
 863                 // build the model string (need to swap ecx/edx data before copying)
 864                 unsigned int temp;
 865                 memset(cpu->arch.model_name, 0, sizeof(cpu->arch.model_name));
 866
 867                 get_current_cpuid(&cpuid, 0x80000002, 0);
 868                 temp = cpuid.regs.edx;
 869                 cpuid.regs.edx = cpuid.regs.ecx;
 870                 cpuid.regs.ecx = temp;
 871                 memcpy(cpu->arch.model_name, cpuid.as_chars, sizeof(cpuid.as_chars));
 872
 873                 get_current_cpuid(&cpuid, 0x80000003, 0);
 874                 temp = cpuid.regs.edx;
 875                 cpuid.regs.edx = cpuid.regs.ecx;
 876                 cpuid.regs.ecx = temp;
 877                 memcpy(cpu->arch.model_name + 16, cpuid.as_chars,
 878                         sizeof(cpuid.as_chars));
 879
 880                 get_current_cpuid(&cpuid, 0x80000004, 0);
 881                 temp = cpuid.regs.edx;
 882                 cpuid.regs.edx = cpuid.regs.ecx;
 883                 cpuid.regs.ecx = temp;
 884                 memcpy(cpu->arch.model_name + 32, cpuid.as_chars,
 885                         sizeof(cpuid.as_chars));
 886
 887                 // some cpus return a right-justified string
 888                 int32 i = 0;
 889                 while (cpu->arch.model_name[i] == ' ')
 890                         i++;
 891                 if (i > 0) {
 892                         memmove(cpu->arch.model_name, &cpu->arch.model_name[i],
 893                                 strlen(&cpu->arch.model_name[i]) + 1);
 894                 }
 895
 896                 dprintf("CPU %d: vendor '%s' model name '%s'\n",
 897                         currentCPU, cpu->arch.vendor_name, cpu->arch.model_name);
 898         } else {
 899                 strlcpy(cpu->arch.model_name, "unknown", sizeof(cpu->arch.model_name));
 900         }
 901
 902         // load feature bits
 903         get_current_cpuid(&cpuid, 1, 0);
 904         cpu->arch.feature[FEATURE_COMMON] = cpuid.eax_1.features; // edx
 905         cpu->arch.feature[FEATURE_EXT] = cpuid.eax_1.extended_features; // ecx
 906
 907         if (maxExtendedLeaf >= 0x80000001) {
 908                 get_current_cpuid(&cpuid, 0x80000001, 0);
 909                 if (cpu->arch.vendor == VENDOR_AMD)
 910                         cpu->arch.feature[FEATURE_EXT_AMD_ECX] = cpuid.regs.ecx; // ecx
 911                 cpu->arch.feature[FEATURE_EXT_AMD] = cpuid.regs.edx; // edx
 912                 if (cpu->arch.vendor != VENDOR_AMD)
 913                         cpu->arch.feature[FEATURE_EXT_AMD] &= IA32_FEATURES_INTEL_EXT;
 914         }
 915
 916         if (maxBasicLeaf >= 5) {
 917                 get_current_cpuid(&cpuid, 5, 0);
 918                 cpu->arch.feature[FEATURE_5_ECX] = cpuid.regs.ecx;
 919         }
 920
 921         if (maxBasicLeaf >= 6) {
 922                 get_current_cpuid(&cpuid, 6, 0);
 923                 cpu->arch.feature[FEATURE_6_EAX] = cpuid.regs.eax;
 924                 cpu->arch.feature[FEATURE_6_ECX] = cpuid.regs.ecx;
 925         }
 926
 927         if (maxExtendedLeaf >= 0x80000007) {
 928                 get_current_cpuid(&cpuid, 0x80000007, 0);
 929                 cpu->arch.feature[FEATURE_EXT_7_EDX] = cpuid.regs.edx;
 930         }
 931
 932         detect_cpu_topology(currentCPU, cpu, maxBasicLeaf, maxExtendedLeaf);
 933
 934 #if DUMP_FEATURE_STRING
 935         dump_feature_string(currentCPU, cpu);
 936 #endif
 937 }
 938
 939
 940 bool
 941 x86_check_feature(uint32 feature, enum x86_feature_type type)
 942 {
 943         cpu_ent* cpu = get_cpu_struct();
 944
 945 #if 0
 946         int i;
 947         dprintf("x86_check_feature: feature 0x%x, type %d\n", feature, type);
 948         for (i = 0; i < FEATURE_NUM; i++) {
 949                 dprintf("features %d: 0x%x\n", i, cpu->arch.feature[i]);
 950         }
 951 #endif
 952
 953         return (cpu->arch.feature[type] & feature) != 0;
 954 }
 955
 956
 957 void*
 958 x86_get_double_fault_stack(int32 cpu, size_t* _size)
 959 {
 960         *_size = kDoubleFaultStackSize;
 961         return sDoubleFaultStacks + kDoubleFaultStackSize * cpu;
 962 }
 963
 964
 965 /*!     Returns the index of the current CPU. Can only be called from the double
 966         fault handler.
 967 */
 968 int32
 969 x86_double_fault_get_cpu(void)
 970 {
 971         addr_t stack = x86_get_stack_frame();
 972         return (stack - (addr_t)sDoubleFaultStacks) / kDoubleFaultStackSize;
 973 }
 974
 975
 976 //      #pragma mark -
 977
 978
 979 status_t
 980 arch_cpu_preboot_init_percpu(kernel_args* args, int cpu)
 981 {
 982         // On SMP system we want to synchronize the CPUs' TSCs, so system_time()
 983         // will return consistent values.
 984         if (smp_get_num_cpus() > 1) {
 985                 // let the first CPU prepare the rendezvous point
 986                 if (cpu == 0)
 987                         sTSCSyncRendezvous = smp_get_num_cpus() - 1;
 988
 989                 // One CPU after the other will drop out of this loop and be caught by
 990                 // the loop below, until the last CPU (0) gets there. Save for +/- a few
 991                 // cycles the CPUs should pass the second loop at the same time.
 992                 while (sTSCSyncRendezvous != cpu) {
 993                 }
 994
 995                 sTSCSyncRendezvous = cpu - 1;
 996
 997                 while (sTSCSyncRendezvous != -1) {
 998                 }
 999
1000                 // reset TSC to 0
1001                 x86_write_msr(IA32_MSR_TSC, 0);
1002         }
1003
1004         x86_descriptors_preboot_init_percpu(args, cpu);
1005
1006         return B_OK;
1007 }
1008
1009
1010 static void
1011 halt_idle(void)
1012 {
1013         asm("hlt");
1014 }
1015
1016
1017 static void
1018 amdc1e_noarat_idle(void)
1019 {
1020         uint64 msr = x86_read_msr(K8_MSR_IPM);
1021         if (msr & K8_CMPHALT)
1022                 x86_write_msr(K8_MSR_IPM, msr & ~K8_CMPHALT);
1023         halt_idle();
1024 }
1025
1026
1027 static bool
1028 detect_amdc1e_noarat()
1029 {
1030         cpu_ent* cpu = get_cpu_struct();
1031
1032         if (cpu->arch.vendor != VENDOR_AMD)
1033                 return false;
1034
1035         // Family 0x12 and higher processors support ARAT
1036         // Family lower than 0xf processors doesn't support C1E
1037         // Family 0xf with model <= 0x40 procssors doesn't support C1E
1038         uint32 family = cpu->arch.family + cpu->arch.extended_family;
1039         uint32 model = (cpu->arch.extended_model << 4) | cpu->arch.model;
1040         return (family < 0x12 && family > 0xf) || (family == 0xf && model > 0x40);
1041 }
1042
1043
1044 status_t
1045 arch_cpu_init_percpu(kernel_args* args, int cpu)
1046 {
1047         detect_cpu(cpu);
1048
1049         if (!gCpuIdleFunc) {
1050                 if (detect_amdc1e_noarat())
1051                         gCpuIdleFunc = amdc1e_noarat_idle;
1052                 else
1053                         gCpuIdleFunc = halt_idle;
1054         }
1055
1056         return B_OK;
1057 }
1058
1059
1060 status_t
1061 arch_cpu_init(kernel_args* args)
1062 {
1063         // init the TSC -> system_time() conversion factors
1064
1065         uint32 conversionFactor = args->arch_args.system_time_cv_factor;
1066         uint64 conversionFactorNsecs = (uint64)conversionFactor * 1000;
1067
1068 #ifdef __x86_64__
1069         // The x86_64 system_time() implementation uses 64-bit multiplication and
1070         // therefore shifting is not necessary for low frequencies (it's also not
1071         // too likely that there'll be any x86_64 CPUs clocked under 1GHz).
1072         __x86_setup_system_time((uint64)conversionFactor << 32,
1073                 conversionFactorNsecs);
1074 #else
1075         if (conversionFactorNsecs >> 32 != 0) {
1076                 // the TSC frequency is < 1 GHz, which forces us to shift the factor
1077                 __x86_setup_system_time(conversionFactor, conversionFactorNsecs >> 16,
1078                         true);
1079         } else {
1080                 // the TSC frequency is >= 1 GHz
1081                 __x86_setup_system_time(conversionFactor, conversionFactorNsecs, false);
1082         }
1083 #endif
1084
1085         // Initialize descriptor tables.
1086         x86_descriptors_init(args);
1087
1088         return B_OK;
1089 }
1090
1091
1092 status_t
1093 arch_cpu_init_post_vm(kernel_args* args)
1094 {
1095         uint32 i;
1096
1097         // allocate an area for the double fault stacks
1098         virtual_address_restrictions virtualRestrictions = {};
1099         virtualRestrictions.address_specification = B_ANY_KERNEL_ADDRESS;
1100         physical_address_restrictions physicalRestrictions = {};
1101         create_area_etc(B_SYSTEM_TEAM, "double fault stacks",
1102                 kDoubleFaultStackSize * smp_get_num_cpus(), B_FULL_LOCK,
1103                 B_KERNEL_READ_AREA | B_KERNEL_WRITE_AREA, CREATE_AREA_DONT_WAIT, 0,
1104                 &virtualRestrictions, &physicalRestrictions,
1105                 (void**)&sDoubleFaultStacks);
1106
1107         X86PagingStructures* kernelPagingStructures
1108                 = static_cast<X86VMTranslationMap*>(
1109                         VMAddressSpace::Kernel()->TranslationMap())->PagingStructures();
1110
1111         // Set active translation map on each CPU.
1112         for (i = 0; i < args->num_cpus; i++) {
1113                 gCPU[i].arch.active_paging_structures = kernelPagingStructures;
1114                 kernelPagingStructures->AddReference();
1115         }
1116
1117         if (!apic_available())
1118                 x86_init_fpu();
1119         // else fpu gets set up in smp code
1120
1121         return B_OK;
1122 }
1123
1124
1125 status_t
1126 arch_cpu_init_post_modules(kernel_args* args)
1127 {
1128         // initialize CPU module
1129
1130         void* cookie = open_module_list("cpu");
1131
1132         while (true) {
1133                 char name[B_FILE_NAME_LENGTH];
1134                 size_t nameLength = sizeof(name);
1135
1136                 if (read_next_module_name(cookie, name, &nameLength) != B_OK
1137                         || get_module(name, (module_info**)&sCpuModule) == B_OK)
1138                         break;
1139         }
1140
1141         close_module_list(cookie);
1142
1143         // initialize MTRRs if available
1144         if (x86_count_mtrrs() > 0) {
1145                 sCpuRendezvous = sCpuRendezvous2 = 0;
1146                 call_all_cpus(&init_mtrrs, NULL);
1147         }
1148
1149         size_t threadExitLen = (addr_t)x86_end_userspace_thread_exit
1150                 - (addr_t)x86_userspace_thread_exit;
1151         addr_t threadExitPosition = fill_commpage_entry(
1152                 COMMPAGE_ENTRY_X86_THREAD_EXIT, (const void*)x86_userspace_thread_exit,
1153                 threadExitLen);
1154
1155         // add the functions to the commpage image
1156         image_id image = get_commpage_image();
1157
1158         elf_add_memory_image_symbol(image, "commpage_thread_exit",
1159                 threadExitPosition, threadExitLen, B_SYMBOL_TYPE_TEXT);
1160
1161         return B_OK;
1162 }
1163
1164
1165 void
1166 arch_cpu_user_TLB_invalidate(void)
1167 {
1168         x86_write_cr3(x86_read_cr3());
1169 }
1170
1171
1172 void
1173 arch_cpu_global_TLB_invalidate(void)
1174 {
1175         uint32 flags = x86_read_cr4();
1176
1177         if (flags & IA32_CR4_GLOBAL_PAGES) {
1178                 // disable and reenable the global pages to flush all TLBs regardless
1179                 // of the global page bit
1180                 x86_write_cr4(flags & ~IA32_CR4_GLOBAL_PAGES);
1181                 x86_write_cr4(flags | IA32_CR4_GLOBAL_PAGES);
1182         } else {
1183                 cpu_status state = disable_interrupts();
1184                 arch_cpu_user_TLB_invalidate();
1185                 restore_interrupts(state);
1186         }
1187 }
1188
1189
1190 void
1191 arch_cpu_invalidate_TLB_range(addr_t start, addr_t end)
1192 {
1193         int32 num_pages = end / B_PAGE_SIZE - start / B_PAGE_SIZE;
1194         while (num_pages-- >= 0) {
1195                 invalidate_TLB(start);
1196                 start += B_PAGE_SIZE;
1197         }
1198 }
1199
1200
1201 void
1202 arch_cpu_invalidate_TLB_list(addr_t pages[], int num_pages)
1203 {
1204         int i;
1205         for (i = 0; i < num_pages; i++) {
1206                 invalidate_TLB(pages[i]);
1207         }
1208 }
1209
1210
1211 status_t
1212 arch_cpu_shutdown(bool rebootSystem)
1213 {
1214         if (acpi_shutdown(rebootSystem) == B_OK)
1215                 return B_OK;
1216
1217         if (!rebootSystem) {
1218 #ifndef __x86_64__
1219                 return apm_shutdown();
1220 #else
1221                 return B_NOT_SUPPORTED;
1222 #endif
1223         }
1224
1225         cpu_status state = disable_interrupts();
1226
1227         // try to reset the system using the keyboard controller
1228         out8(0xfe, 0x64);
1229
1230         // Give some time to the controller to do its job (0.5s)
1231         snooze(500000);
1232
1233         // if that didn't help, try it this way
1234         x86_reboot();
1235
1236         restore_interrupts(state);
1237         return B_ERROR;
1238 }
1239
1240
1241 void
1242 arch_cpu_sync_icache(void* address, size_t length)
1243 {
1244         // instruction cache is always consistent on x86
1245 }
1246