arch/all-pc/kernel/apic_ia32.c

   1 /*
   2     Copyright © 1995-2018, The AROS Development Team. All rights reserved.
   3     $Id$
   4
   5     Desc: Intel IA-32 APIC driver.
   6 */
   7
   8 #include <aros/macros.h>
   9 #include <asm/cpu.h>
  10 #include <asm/io.h>
  11 #include <exec/types.h>
  12
  13 #define __KERNEL_NOLIBBASE__
  14 #include <proto/kernel.h>
  15 #include <proto/exec.h>
  16 #include <proto/acpica.h>
  17
  18 #include <acpica/acnames.h>
  19 #include <acpica/accommon.h>
  20
  21 #include <inttypes.h>
  22
  23 #include "kernel_base.h"
  24 #include "kernel_intern.h"
  25 #include "kernel_objects.h"
  26 #include "kernel_debug.h"
  27 #include "kernel_syscall.h"
  28 #include "kernel_timer.h"
  29 #include "kernel_ipi.h"
  30
  31 #include "kernel_interrupts.h"
  32
  33 #include "acpi.h"
  34 #include "apic_ia32.h"
  35
  36 #define D(x)
  37 #define DINT(x)
  38 #define DWAKE(x)        /* Badly interferes with AP startup */
  39 #define DID(x)          /* Badly interferes with everything */
  40 /* #define DEBUG_WAIT */
  41
  42 /*
  43  * On i386 platform we need to support various quirks of old APICs.
  44  * x86-64 is free of that crap.
  45  */
  46 #ifdef __i386__
  47 #define CONFIG_LEGACY
  48 #endif
  49
  50 extern int core_APICErrorHandle(struct ExceptionContext *, void *, void *);
  51 extern int core_APICSpuriousHandle(struct ExceptionContext *, void *, void *);
  52 extern int APICHeartbeatServer(struct ExceptionContext *regs, struct KernelBase *KernelBase, struct ExecBase *SysBase);
  53
  54 /* APIC Interrupt Controller Functions ... ***************************/
  55
  56 struct APICInt_Private
  57 {
  58
  59 };
  60
  61 icid_t APICInt_Register(struct KernelBase *KernelBase)
  62 {
  63     DINT(bug("[Kernel:APIC-IA32] %s()\n", __func__));
  64
  65     return (icid_t)APICInt_IntrController.ic_Node.ln_Type;
  66 }
  67
  68 BOOL APICInt_Init(struct KernelBase *KernelBase, icid_t instanceCount)
  69 {
  70     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
  71     struct ACPIData *acpiData  = kernPlatD->kb_ACPI;
  72     struct APICData *apicPrivate = kernPlatD->kb_APIC;
  73     APTR ssp;
  74     int irq, count = 0;
  75
  76     DINT(bug("[Kernel:APIC-IA32] %s(%d)\n", __func__, instanceCount));
  77
  78     /* It's not fatal to fail on these IRQs */
  79     if ((ssp = SuperState()) != NULL)
  80     {
  81         /* Set up the APIC IRQs for CPU #0 */
  82         for (irq = (APIC_IRQ_BASE - X86_CPU_EXCEPT_COUNT); irq < HW_IRQ_COUNT; irq++)
  83         {
  84             if (!krnInitInterrupt(KernelBase, irq, APICInt_IntrController.ic_Node.ln_Type, 0))
  85             {
  86                 D(bug("[Kernel:APIC-IA32] %s: failed to obtain IRQ %d\n", __func__, irq);)
  87             }
  88             else
  89             {
  90                 /* Don't enable the vector yet */
  91                 if (!core_SetIDTGate((apicidt_t *)apicPrivate->cores[0].cpu_IDT, HW_IRQ_BASE + irq, (uintptr_t)IntrDefaultGates[HW_IRQ_BASE + irq], FALSE))
  92                 {
  93                     bug("[Kernel:APIC-IA32] %s: failed to set IRQ %d's Vector gate\n", __func__, irq);
  94                 }
  95                 else
  96                     count++;
  97             }
  98         }
  99         UserState(ssp);
 100     }
 101
 102     /*
 103      * If we have at least 32 APIC interrupts available (the
 104      * most a single MSI device will request) then report that
 105      * we can use MSI
 106      */
 107     if ((count > 32) && (acpiData->acpi_fadt))
 108     {
 109         ACPI_TABLE_FADT *fadt = (ACPI_TABLE_FADT *)acpiData->acpi_fadt;
 110
 111         if ((!(fadt->BootFlags & ACPI_FADT_NO_MSI)) &&
 112             (!(kernPlatD->kb_PDFlags & PLATFORMF_HAVEMSI)))
 113         {
 114             // TODO: Register the MSI interrupt controller..
 115             kernPlatD->kb_PDFlags |= PLATFORMF_HAVEMSI;
 116             bug("[Kernel:APIC-IA32] MSI Interrupts enabled\n");
 117         }
 118     }
 119     return TRUE;
 120 }
 121
 122 BOOL APICInt_DisableIRQ(APTR icPrivate, icid_t icInstance, icid_t intNum)
 123 {
 124     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
 125     struct APICData *apicPrivate = kernPlatD->kb_APIC;
 126     apicid_t cpuNum = KrnGetCPUNumber();
 127     apicidt_t *IGATES;
 128     APTR ssp = NULL;
 129     BOOL retVal = FALSE;
 130
 131     DINT(bug("[Kernel:APIC-IA32.%03u] %s(#$%02X)\n", cpuNum, __func__, intNum));
 132
 133     IGATES = (apicidt_t *)apicPrivate->cores[cpuNum].cpu_IDT;
 134
 135     if ((KrnIsSuper()) || ((ssp = SuperState()) != NULL))
 136     {
 137         IGATES[HW_IRQ_BASE + intNum].p = 0;
 138         retVal = TRUE;
 139
 140         if (ssp)
 141             UserState(ssp);
 142     }
 143
 144     return retVal;
 145 }
 146
 147 BOOL APICInt_EnableIRQ(APTR icPrivate, icid_t icInstance, icid_t intNum)
 148 {
 149     struct PlatformData *kernPlatD = (struct PlatformData *)KernelBase->kb_PlatformData;
 150     struct APICData *apicPrivate = kernPlatD->kb_APIC;
 151     apicid_t cpuNum = KrnGetCPUNumber();
 152     apicidt_t *IGATES;
 153     APTR ssp = NULL;
 154     BOOL retVal = FALSE;
 155
 156     DINT(bug("[Kernel:APIC-IA32.%03u] %s(#$%02X)\n", cpuNum, __func__, intNum));
 157
 158     IGATES = (apicidt_t *)apicPrivate->cores[cpuNum].cpu_IDT;
 159
 160     if ((KrnIsSuper()) || ((ssp = SuperState()) != NULL))
 161     {
 162         IGATES[HW_IRQ_BASE + intNum].p = 1;
 163         retVal = TRUE;
 164
 165         if (ssp)
 166             UserState(ssp);
 167     }
 168
 169     return retVal;
 170 }
 171
 172 BOOL APICInt_AckIntr(APTR icPrivate, icid_t icInstance, icid_t intNum)
 173 {
 174     IPTR apic_base;
 175
 176     DINT(bug("[Kernel:APIC-IA32] %s(%03u #$%02X)\n", __func__, icInstance, intNum));
 177
 178     /* Write zero to EOI of APIC */
 179     apic_base = core_APIC_GetBase();
 180
 181     APIC_REG(apic_base, APIC_EOI) = 0;
 182
 183     return TRUE;
 184 }
 185
 186 struct IntrController APICInt_IntrController =
 187 {
 188     {
 189         .ln_Name = "x86 Local APIC",
 190         .ln_Pri = -50
 191     },
 192     0,
 193     AROS_MAKE_ID('A','P','I','C'),
 194     0,
 195     NULL,
 196     APICInt_Register,
 197     APICInt_Init,
 198     APICInt_EnableIRQ,
 199     APICInt_DisableIRQ,
 200     APICInt_AckIntr
 201 };
 202
 203 /* APIC IPI Related Functions ... ***************************/
 204
 205 static ULONG DoIPI(IPTR __APICBase, ULONG target, ULONG cmd)
 206 {
 207     ULONG ipisend_timeout, status_ipisend;
 208
 209     D(
 210         apicid_t cpuNum = KrnGetCPUNumber();
 211         bug("[Kernel:APIC-IA32.%03u] %s: Command 0x%08X to target %03u\n", cpuNum, __func__, cmd, target);
 212     )
 213
 214     /*
 215      * Send the IPI.
 216      * First we write target APIC ID into high command register.
 217      * Writing to the low register triggers the IPI itself.
 218      */
 219     APIC_REG(__APICBase, APIC_ICRH) = target << 24;
 220     APIC_REG(__APICBase, APIC_ICRL) = cmd;
 221
 222     D(bug("[Kernel:APIC-IA32.%03u] %s: Waiting for IPI to complete ", cpuNum, __func__));
 223
 224     for (ipisend_timeout = 1000; ipisend_timeout > 0; ipisend_timeout--)
 225     {
 226         pit_udelay(100);
 227 #ifdef DEBUG_WAIT
 228         if ((ipisend_timeout % 100) == 0)
 229         {
 230             bug(".");
 231         }
 232 #endif
 233         status_ipisend = APIC_REG(__APICBase, APIC_ICRL) & ICR_DS;
 234         /* Delivery status resets to 0 when delivery is done */
 235         if (status_ipisend == 0)
 236             break;
 237     }
 238     D(bug("\n"));
 239     D(bug("[Kernel:APIC-IA32.%03u] %s: ... left wait loop (status = 0x%08X)\n", cpuNum, __func__, status_ipisend));
 240
 241     return status_ipisend;
 242 }
 243
 244 /**********************************************************
 245                         Driver functions
 246  **********************************************************/
 247
 248 void core_APIC_Init(struct APICData *apic, apicid_t cpuNum)
 249 {
 250     IPTR __APICBase = apic->lapicBase;
 251     ULONG apic_ver = APIC_REG(__APICBase, APIC_VERSION);
 252     ULONG maxlvt = APIC_LVT(apic_ver), calibrated = 0;
 253     LONG lapic_initial, lapic_final;
 254     UQUAD tsc_initial, tsc_final;
 255     UQUAD calibrated_tsc = 0;
 256     WORD pit_final;
 257     icintrid_t coreICInstID;
 258
 259 #ifdef CONFIG_LEGACY
 260     /* 82489DX doesn't report no. of LVT entries. */
 261     if (!APIC_INTEGRATED(apic_ver))
 262         maxlvt = 2;
 263 #endif
 264
 265     if ((coreICInstID = krnAddInterruptController(KernelBase, &APICInt_IntrController)) != (icintrid_t)-1)
 266     {
 267         APTR ssp = NULL;
 268         int i;
 269
 270         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC IC ID #%d:%d\n", cpuNum, __func__, ICINTR_ICID(coreICInstID), ICINTR_INST(coreICInstID)));
 271
 272         /*
 273          * NB: - BSP calls us in user mode, but AP's call us from supervisor
 274          */
 275         if ((KrnIsSuper()) || ((ssp = SuperState()) != NULL))
 276         {
 277             /* Obtain/set the critical IRQs and Vectors */
 278             for (i = 0; i < X86_CPU_EXCEPT_COUNT; i++)
 279             {
 280                 if (!core_SetIDTGate((apicidt_t *)apic->cores[cpuNum].cpu_IDT, i, (uintptr_t)IntrDefaultGates[i], TRUE))
 281                 {
 282                     krnPanic(NULL, "Failed to set CPU Exception Vector\n"
 283                                    "Vector #$%02X\n", i);
 284                 }
 285             }
 286             for (i = X86_CPU_EXCEPT_COUNT; i < APIC_EXCEPT_TOP; i++)
 287             {
 288                 if (i == APIC_EXCEPT_SYSCALL)
 289                     continue;
 290
 291                 if (!core_SetIDTGate((apicidt_t *)apic->cores[cpuNum].cpu_IDT, APIC_CPU_EXCEPT_TO_VECTOR(i), (uintptr_t)IntrDefaultGates[APIC_CPU_EXCEPT_TO_VECTOR(i)], TRUE))
 292                 {
 293                     krnPanic(NULL, "Failed to set APIC Exception Vector\n"
 294                                     "Vector #$%02X\n", i);
 295                 }
 296             }
 297             D(bug("[Kernel:APIC-IA32.%03u] %s: APIC Exception Vectors configured\n", cpuNum, __func__));
 298
 299             if (cpuNum == 0)
 300             {
 301                 KrnAddExceptionHandler(APIC_EXCEPT_ERROR, core_APICErrorHandle, KernelBase, NULL);
 302                 KrnAddExceptionHandler(APIC_EXCEPT_SPURIOUS, core_APICSpuriousHandle, KernelBase, NULL);
 303
 304                 D(bug("[Kernel:APIC-IA32.%03u] %s: APIC Error Exception handler (exception #$%02X) installed\n", cpuNum, __func__, APIC_EXCEPT_ERROR));
 305
 306                 for (i = APIC_EXCEPT_IPI_NOP; i <= APIC_EXCEPT_IPI_CAUSE; i++)
 307                 {
 308                     KrnAddExceptionHandler(i, core_IPIHandle, (void *)((intptr_t)i - APIC_EXCEPT_IPI_NOP), KernelBase);
 309                 }
 310                 D(bug("[Kernel:APIC-IA32.%03u] %s: APIC IPI Vectors configured\n", cpuNum, __func__));
 311             }
 312
 313             if (ssp)
 314                 UserState(ssp);
 315         }
 316         else
 317         {
 318             krnPanic(NULL, "Failed to configure APIC\n"
 319                                "APIC #%03e ID %03u\n", cpuNum, apic->cores[cpuNum].cpu_LocalID);
 320         }
 321
 322         /* Use flat interrupt model with logical destination ID = 1 */
 323         APIC_REG(__APICBase, APIC_DFR) = DFR_FLAT;
 324         APIC_REG(__APICBase, APIC_LDR) = 1 << LDR_ID_SHIFT;
 325
 326         /* Set Task Priority to 'accept all interrupts' */
 327         APIC_REG(__APICBase, APIC_TPR) = 0;
 328
 329         /* Set spurious IRQ vector to 0xFF. APIC enabled, focus check disabled. */
 330         APIC_REG(__APICBase, APIC_SVR) = SVR_ASE|APIC_CPU_EXCEPT_TO_VECTOR(APIC_EXCEPT_SPURIOUS);
 331
 332         /*
 333          * Set LINT0 to external and LINT1 to NMI.
 334          * These are common defaults and they are going to be overridden by ACPI tables.
 335          *
 336          * On all other LAPICs mask LINT0 and use some fake vector (0xff in this case),
 337          * otherwise LAPIC may throw an error.
 338          */
 339         if (cpuNum == 0)
 340             APIC_REG(__APICBase, APIC_LINT0_VEC) = LVT_MT_EXT;
 341         else
 342             APIC_REG(__APICBase, APIC_LINT0_VEC) = LVT_MASK | 0xff;
 343
 344         APIC_REG(__APICBase, APIC_LINT1_VEC) = LVT_MT_NMI;
 345
 346 #ifdef CONFIG_LEGACY
 347         /* Due to the Pentium erratum 3AP. */
 348         if (maxlvt > 3)
 349              APIC_REG(__APICBase, APIC_ESR) = 0;
 350 #endif
 351
 352         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC ESR before enabling vector: %08x\n", cpuNum, __func__, APIC_REG(__APICBase, APIC_ESR)));
 353
 354         /* Set APIC error interrupt to fixed vector interrupt "APIC_IRQ_ERROR",  on APIC error */
 355         APIC_REG(__APICBase, APIC_ERROR_VEC) = APIC_CPU_EXCEPT_TO_VECTOR(APIC_EXCEPT_ERROR);
 356
 357         /* spec says clear errors after enabling vector. */
 358         if (maxlvt > 3)
 359              APIC_REG(__APICBase, APIC_ESR) = 0;
 360
 361         D(bug("[Kernel:APIC-IA32.%03u] %s: APIC ESR after enabling vector: %08x\n", cpuNum, __func__, APIC_REG(__APICBase, APIC_ESR)));
 362
 363         /*
 364          * Now the tricky thing - calibrate LAPIC timer frequency.
 365          * In fact we could simply query CPU's clock frequency, but... x86 sucks. There's no
 366          * unified way to get it on whatever CPU. Intel has own way, AMD has own way... Etc... Which, additionally,
 367          * varies between CPU generations.
 368          *
 369          * The idea behind the calibration is to run the timer once, and see how many ticks
 370          * pass in some defined period of time. Then calculate a proportion.
 371          * We use 8253 PIT as our reference.
 372          * This calibration algorithm is based on NetBSD one.
 373          */
 374
 375         /* Set the timer to one-shot mode, no interrupt, 1:1 divisor */
 376         APIC_REG(__APICBase, APIC_TIMER_VEC) = LVT_MASK | APIC_CPU_EXCEPT_TO_VECTOR(APIC_EXCEPT_HEARTBEAT);
 377         APIC_REG(__APICBase, APIC_TIMER_DIV) = TIMER_DIV_1;
 378         APIC_REG(__APICBase, APIC_TIMER_ICR) = 0x80000000;      /* Just some very large value */
 379
 380         /*
 381          * Now wait for 11931 PIT ticks, which is equal to 10 milliseconds.
 382          * We don't use pit_udelay() here, because for improved accuracy we need to sample LAPIC timer counter twice,
 383          * before and after our actual delay (PIT setup also takes up some time, so LAPIC will count away from its
 384          * initial value).  We run it 10 times to make up for cache setup discrepancies.
 385          */
 386         for (i = 0; i < 10; i ++)
 387         {
 388             pit_start(11931);
 389             lapic_initial = (LONG)APIC_REG(__APICBase, APIC_TIMER_CCR);
 390             tsc_initial = RDTSC();
 391
 392             pit_final   = pit_wait(11931);
 393
 394             tsc_final = RDTSC();
 395             lapic_final = (LONG)APIC_REG(__APICBase, APIC_TIMER_CCR);
 396
 397             calibrated += (((QUAD)(lapic_initial - lapic_final) * 11931LL)/(11931LL - (QUAD)pit_final)) ;
 398             calibrated_tsc += ((tsc_final - tsc_initial) * 11931LL) / (11931LL - (QUAD)pit_final);
 399         }
 400         apic->cores[cpuNum].cpu_TimerFreq = 10 * calibrated;
 401         apic->cores[cpuNum].cpu_TSCFreq = 10 * calibrated_tsc;
 402         D(bug("[Kernel:APIC-IA32.%03u] %s: LAPIC frequency should be %u Hz (%u MHz)\n", cpuNum, __func__, apic->cores[cpuNum].cpu_TimerFreq, (apic->cores[cpuNum].cpu_TimerFreq + 500000) / 1000000));
 403         D(bug("[Kernel:APIC-IA32.%03u] %s: TSC frequency should be %u kHz (%u MHz)\n", cpuNum, __func__, (ULONG)((apic->cores[cpuNum].cpu_TSCFreq + 500)/1000), (ULONG)((apic->cores[cpuNum].cpu_TSCFreq + 500000) / 1000000)));
 404         /*
 405          * Once APIC timer has been calibrated -:
 406          * # Set it to run at its full frequency.
 407          * # Enable the heartbeat vector and use a suitable rate,
 408          *   otherwise set to reload every second and disable it.
 409          */
 410         if (cpuNum == 0)
 411         {
 412             KrnAddExceptionHandler(APIC_EXCEPT_HEARTBEAT, APICHeartbeatServer, KernelBase, SysBase);
 413
 414             apic->flags |= APF_TIMER;
 415         }
 416
 417         APIC_REG(__APICBase, APIC_TIMER_DIV) = TIMER_DIV_1;
 418
 419         if ((apic->flags & APF_TIMER) &&
 420             ((KrnIsSuper()) || ((ssp = SuperState()) != NULL)))
 421         {
 422 #if defined(__AROSEXEC_SMP__)
 423             tls_t *apicTLS = apic->cores[cpuNum].cpu_TLS;
 424             struct X86SchedulerPrivate *schedData = apicTLS->ScheduleData;
 425             D(bug("[Kernel:APIC-IA32.%03u] %s: tls @ 0x%p, scheduling data @ 0x%p\n", cpuNum, __func__, apicTLS, schedData);)
 426 #endif
 427
 428             apic->cores[cpuNum].cpu_LAPICTick = 0;
 429             D(bug("[Kernel:APIC-IA32.%03u] %s: heartbeat Exception Vector #$%02X (%d) set\n", cpuNum, __func__, APIC_EXCEPT_HEARTBEAT, APIC_CPU_EXCEPT_TO_VECTOR(APIC_EXCEPT_HEARTBEAT));)
 430
 431             if (ssp)
 432                 UserState(ssp);
 433
 434 #if defined(__AROSEXEC_SMP__)
 435             // TODO: Adjust based on the amount of work the APIC can do at its given frequency.
 436             schedData->Granularity = 1;
 437             schedData->Quantum = 5;
 438             APIC_REG(__APICBase, APIC_TIMER_ICR) = (apic->cores[cpuNum].cpu_TimerFreq);
 439 #else
 440             APIC_REG(__APICBase, APIC_TIMER_ICR) = (apic->cores[cpuNum].cpu_TimerFreq + 25) / 50;
 441 #endif
 442             APIC_REG(__APICBase, APIC_TIMER_VEC) = APIC_CPU_EXCEPT_TO_VECTOR(APIC_EXCEPT_HEARTBEAT); // | LVT_TMM_PERIOD;
 443         }
 444         else
 445         {
 446             APIC_REG(__APICBase, APIC_TIMER_ICR) = apic->cores[cpuNum].cpu_TimerFreq;
 447             APIC_REG(__APICBase, APIC_TIMER_VEC) = LVT_MASK | LVT_TMM_PERIOD | APIC_CPU_EXCEPT_TO_VECTOR(APIC_EXCEPT_HEARTBEAT);
 448         }
 449     }
 450 }
 451
 452 apicid_t core_APIC_GetID(IPTR _APICBase)
 453 {
 454     apicid_t _apic_id;
 455
 456     /* The actual ID is in 8 most significant bits */
 457     _apic_id = APIC_REG(_APICBase, APIC_ID) >> APIC_ID_SHIFT;
 458     DID(bug("[Kernel:APIC-IA32] %s: %03u\n", __func__, _apic_id));
 459
 460     return _apic_id;
 461 }
 462
 463 ULONG core_APIC_Wake(APTR wake_apicstartrip, apicid_t wake_apicid, IPTR __APICBase)
 464 {
 465     ULONG status_ipisend, status_ipirecv;
 466     ULONG start_count;
 467 #ifdef CONFIG_LEGACY
 468     ULONG apic_ver = APIC_REG(__APICBase, APIC_VERSION);
 469 #endif
 470     D(
 471         apicid_t cpuNo = KrnGetCPUNumber();
 472
 473         bug("[Kernel:APIC-IA32.%03u] %s(%03u @ %p)\n", cpuNo, __func__, wake_apicid, wake_apicstartrip);
 474         bug("[Kernel:APIC-IA32.%03u] %s: Base @ %p\n", cpuNo, __func__, __APICBase);
 475     )
 476 #ifdef CONFIG_LEGACY
 477     /*
 478      * Check if we have old 82489DX discrete APIC (version & 0xF0 == 0).
 479      * This APIC needs different startup procedure. It doesn't support STARTUP IPI
 480      * because old CPUs didn't have INIT signal. They jump to BIOS ROM boot code
 481      * immediately after INIT IPI. In order to run the bootstrap, a BIOS warm reset
 482      * magic has to be used there.
 483      */
 484     if (!APIC_INTEGRATED(apic_ver))
 485     {
 486         /*
 487          * BIOS warm reset magic, part one.
 488          * Write real-mode bootstrap routine address to 40:67 (real-mode address) location.
 489          * This is standard feature of IBM PC AT BIOS. If a warm reset condition is detected,
 490          * the BIOS jumps to the given address.
 491          */
 492         D(bug("[Kernel:APIC-IA32.%03u] %s: Setting BIOS vector for trampoline @ %p ..\n", cpuNo, __func__, wake_apicstartrip));
 493         *((volatile unsigned short *)0x469) = (IPTR)wake_apicstartrip >> 4;
 494         *((volatile unsigned short *)0x467) = 0; /* Actually wake_apicstartrip & 0x0F, it's page-aligned. */
 495
 496         /*
 497          * BIOS warm reset magic, part two.
 498          * This writes 0x0A into CMOS RAM, location 0x0F. This signals a warm reset condition to BIOS,
 499          * making part one work.
 500          */
 501         D(bug("[Kernel:APIC-IA32.%03u] %s: Setting warm reset code ..\n", cpuNo, __func__));
 502         outb(0xf, 0x70);
 503         outb(0xa, 0x71);
 504     }
 505 #endif
 506
 507     /* Flush TLB (we are supervisor here) */
 508     wrcr(cr3, rdcr(cr3));
 509
 510     /* First we send the INIT command (reset the core). Vector must be zero for this. */
 511     status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_INT_LEVELTRIG | ICR_INT_ASSERT | ICR_DM_INIT);
 512     if (status_ipisend)
 513     {
 514         D(bug("[Kernel:APIC-IA32.%03u] %s: Error asserting INIT\n", cpuNo, __func__));
 515         return status_ipisend;
 516     }
 517
 518     /* Deassert INIT after a small delay */
 519     pit_udelay(10 * 1000);
 520
 521     /* Deassert INIT */
 522     status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_INT_LEVELTRIG | ICR_DM_INIT);
 523     if (status_ipisend)
 524     {
 525         D(bug("[Kernel:APIC-IA32.%03u] %s: Error deasserting INIT\n", cpuNo, __func__));
 526         return status_ipisend;
 527     }
 528
 529     /* memory barrier */
 530     asm volatile("mfence":::"memory");
 531
 532 #ifdef CONFIG_LEGACY
 533     /* If it's 82489DX, we are done. */
 534     if (!APIC_INTEGRATED(apic_ver))
 535     {
 536         DWAKE(bug("[Kernel:APIC-IA32.%03u] %s: 82489DX detected, wakeup done\n", cpuNo, __func__));
 537         return 0;
 538     }
 539 #endif
 540
 541     /*
 542      * Perform IPI STARTUP loop.
 543      * According to official Intel specification, this must be done twice.
 544      * It's not explained why. ;-)
 545      */
 546     for (start_count = 1; start_count <= 2; start_count++)
 547     {
 548         D(bug("[Kernel:APIC-IA32.%03u] %s: Attempting STARTUP .. %u\n", cpuNo, __func__, start_count));
 549
 550         /* Clear any pending error condition */
 551         APIC_REG(__APICBase, APIC_ESR) = 0;
 552
 553         /*
 554          * Send STARTUP IPI.
 555          * The processor starts up at CS = (vector << 16) and IP = 0.
 556          */
 557         status_ipisend = DoIPI(__APICBase, wake_apicid, ICR_DM_STARTUP | ((IPTR)wake_apicstartrip >> 12));
 558
 559         /* Allow the target APIC to accept the IPI */
 560         pit_udelay(200);
 561
 562 #ifdef CONFIG_LEGACY
 563         /* Pentium erratum 3AP quirk */
 564         if (APIC_LVT(apic_ver) > 3)
 565             APIC_REG(__APICBase, APIC_ESR) = 0;
 566 #endif
 567
 568         status_ipirecv = APIC_REG(__APICBase, APIC_ESR) & 0xEF;
 569
 570         /*
 571          * EXPERIMENTAL:
 572          * On my machine (macmini 3,1, as OS X system profiler says), the core starts up from first
 573          * attempt. The second attempt ends up in error (according to the documentation, the STARTUP
 574          * can be accepted only once, while the core in RESET or INIT state, and first STARTUP, if
 575          * successful, brings the core out of this state).
 576          * Here we try to detect this condition. If the core accepted STARTUP, we suggest that it has
 577          * started up, and break the loop.
 578          * A topic at osdev.org forum (http://forum.osdev.org/viewtopic.php?f=1&t=23018)
 579          * also tells about some problems with double STARTUP. According to it, the second STARTUP can
 580          * manage to re-run the core from the given address, leaving it in 64-bit mode, causing it to crash.
 581          *
 582          * If startup problems pops up (the core doesn't respond and AROS halts at "Launching APIC no X" stage),
 583          * the following two variations of this algorithm can be tried:
 584          * a) Always send STARTUP twice, but signal error condition only if both attempts failed.
 585          * b) Send first STARTUP, abort on error. Allow second attempt to fail and ignore its result.
 586          *
 587          *                                                              Sonic <pavel_fedin@mail.ru>
 588          */
 589         if (!status_ipisend && !status_ipirecv)
 590             break;
 591     }
 592
 593     DWAKE(bug("[Kernel:APIC-IA32.%03u] %s: STARTUP run status 0x%08X, error 0x%08X\n", cpuNo, __func__, status_ipisend, status_ipirecv));
 594
 595     /*
 596      * We return nonzero on error.
 597      * Actually least significant byte of this value holds ESR value, and 12th bit
 598      * holds delivery status flag from DoIPI() routine. It will be '1' if we got
 599      * stuck at sending phase.
 600      */
 601     return (status_ipisend | status_ipirecv);
 602 }