8354 sync regcomp(3C) with upstream (fix make catalog)

[unleashed/tickless.git] / usr / src / uts / sun4u / os / mach_startup.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <sys/machsystm.h>
#include <sys/archsystm.h>
#include <sys/vm.h>
#include <sys/cpu.h>
#include <sys/cpupart.h>
#include <sys/cmt.h>
#include <sys/bitset.h>
#include <sys/reboot.h>
#include <sys/kdi.h>
#include <sys/bootconf.h>
#include <sys/memlist_plat.h>
#include <sys/memlist_impl.h>
#include <sys/prom_plat.h>
#include <sys/prom_isa.h>
#include <sys/autoconf.h>
#include <sys/intreg.h>
#include <sys/ivintr.h>
#include <sys/fpu/fpusystm.h>
#include <sys/iommutsb.h>
#include <vm/vm_dep.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/seg_map.h>
#include <vm/seg_kp.h>
#include <sys/sysconf.h>
#include <vm/hat_sfmmu.h>
#include <sys/kobj.h>
#include <sys/sun4asi.h>
#include <sys/clconf.h>
#include <sys/platform_module.h>
#include <sys/panic.h>
#include <sys/cpu_sgnblk_defs.h>
#include <sys/clock.h>
#include <sys/fpras_impl.h>
#include <sys/prom_debug.h>
#include <sys/traptrace.h>
#include <sys/memnode.h>
#include <sys/mem_cage.h>

/*
 * fpRAS implementation structures.
 */
struct fpras_chkfn *fpras_chkfnaddrs[FPRAS_NCOPYOPS];
struct fpras_chkfngrp *fpras_chkfngrps;
struct fpras_chkfngrp *fpras_chkfngrps_base;
int fpras_frequency = -1;
int64_t fpras_interval = -1;

/*
 * Increase unix symbol table size as a work around for 6828121
 */
int alloc_mem_bermuda_triangle;

/*
 * Halt idling cpus optimization
 *
 * This optimation is only enabled in platforms that have
 * the CPU halt support. The cpu_halt_cpu() support is provided
 * in the cpu module and it is referenced here with a pragma weak.
 * The presence of this routine automatically enable the halt idling
 * cpus functionality if the global switch enable_halt_idle_cpus
 * is set (default is set).
 *
 */
#pragma weak    cpu_halt_cpu
extern void     cpu_halt_cpu();

/*
 * Defines for the idle_state_transition DTrace probe
 *
 * The probe fires when the CPU undergoes an idle state change (e.g. halting)
 * The agument passed is the state to which the CPU is transitioning.
 *
 * The states are defined here.
 */
#define IDLE_STATE_NORMAL 0
#define IDLE_STATE_HALTED 1

int             enable_halt_idle_cpus = 1; /* global switch */

uint_t cp_haltset_fanout = 3;

void
setup_trap_table(void)
{
        intr_init(CPU);                 /* init interrupt request free list */
        setwstate(WSTATE_KERN);
        prom_set_traptable(&trap_table);
}

void
mach_fpras()
{
        if (fpras_implemented && !fpras_disable) {
                int i;
                struct fpras_chkfngrp *fcgp;
                size_t chkfngrpsallocsz;

                /*
                 * Note that we size off of NCPU and setup for
                 * all those possibilities regardless of whether
                 * the cpu id is present or not.  We do this so that
                 * we don't have any construction or destruction
                 * activity to perform at DR time, and it's not
                 * costly in memory.  We require block alignment.
                 */
                chkfngrpsallocsz = NCPU * sizeof (struct fpras_chkfngrp);
                fpras_chkfngrps_base = kmem_alloc(chkfngrpsallocsz, KM_SLEEP);
                if (IS_P2ALIGNED((uintptr_t)fpras_chkfngrps_base, 64)) {
                        fpras_chkfngrps = fpras_chkfngrps_base;
                } else {
                        kmem_free(fpras_chkfngrps_base, chkfngrpsallocsz);
                        chkfngrpsallocsz += 64;
                        fpras_chkfngrps_base = kmem_alloc(chkfngrpsallocsz,
                            KM_SLEEP);
                        fpras_chkfngrps = (struct fpras_chkfngrp *)
                            P2ROUNDUP((uintptr_t)fpras_chkfngrps_base, 64);
                }

                /*
                 * Copy our check function into place for each copy operation
                 * and each cpu id.
                 */
                fcgp = &fpras_chkfngrps[0];
                for (i = 0; i < FPRAS_NCOPYOPS; ++i)
                        bcopy((void *)fpras_chkfn_type1, &fcgp->fpras_fn[i],
                            sizeof (struct fpras_chkfn));
                for (i = 1; i < NCPU; ++i)
                        *(&fpras_chkfngrps[i]) = *fcgp;

                /*
                 * At definition fpras_frequency is set to -1, and it will
                 * still have that value unless changed in /etc/system (not
                 * strictly supported, but not preventable).  The following
                 * both sets the default and sanity checks anything from
                 * /etc/system.
                 */
                if (fpras_frequency < 0)
                        fpras_frequency = FPRAS_DEFAULT_FREQUENCY;

                /*
                 * Now calculate fpras_interval.  When fpras_interval
                 * becomes non-negative fpras checks will commence
                 * (copies before this point in boot will bypass fpras).
                 * Our stores of instructions must be visible; no need
                 * to flush as they're never been executed before.
                 */
                membar_producer();
                fpras_interval = (fpras_frequency == 0) ?
                    0 : sys_tick_freq / fpras_frequency;
        }
}

void
mach_hw_copy_limit(void)
{
        if (!fpu_exists) {
                use_hw_bcopy = 0;
                hw_copy_limit_1 = 0;
                hw_copy_limit_2 = 0;
                hw_copy_limit_4 = 0;
                hw_copy_limit_8 = 0;
                use_hw_bzero = 0;
        }
}

void
load_tod_module()
{
        /*
         * Load tod driver module for the tod part found on this system.
         * Recompute the cpu frequency/delays based on tod as tod part
         * tends to keep time more accurately.
         */
        if (tod_module_name == NULL || modload("tod", tod_module_name) == -1)
                halt("Can't load tod module");
}

void
mach_memscrub(void)
{
        /*
         * Startup memory scrubber, if not running fpu emulation code.
         */

#ifndef _HW_MEMSCRUB_SUPPORT
        if (fpu_exists) {
                if (memscrub_init()) {
                        cmn_err(CE_WARN,
                            "Memory scrubber failed to initialize");
                }
        }
#endif /* _HW_MEMSCRUB_SUPPORT */
}

/*
 * Halt the present CPU until awoken via an interrupt.
 * This routine should only be invoked if cpu_halt_cpu()
 * exists and is supported, see mach_cpu_halt_idle()
 */
void
cpu_halt(void)
{
        cpu_t *cpup = CPU;
        processorid_t cpu_sid = cpup->cpu_seqid;
        cpupart_t *cp = cpup->cpu_part;
        int hset_update = 1;
        volatile int *p = &cpup->cpu_disp->disp_nrunnable;
        uint_t s;

        /*
         * If this CPU is online then we should notate our halting
         * by adding ourselves to the partition's halted CPU
         * bitset. This allows other CPUs to find/awaken us when
         * work becomes available.
         */
        if (CPU->cpu_flags & CPU_OFFLINE)
                hset_update = 0;

        /*
         * Add ourselves to the partition's halted CPUs bitset
         * and set our HALTED flag, if necessary.
         *
         * When a thread becomes runnable, it is placed on the queue
         * and then the halted cpu bitset is checked to determine who
         * (if anyone) should be awoken. We therefore need to first
         * add ourselves to the halted bitset, and then check if there
         * is any work available.  The order is important to prevent a race
         * that can lead to work languishing on a run queue somewhere while
         * this CPU remains halted.
         *
         * Either the producing CPU will see we're halted and will awaken us,
         * or this CPU will see the work available in disp_anywork()
         */
        if (hset_update) {
                cpup->cpu_disp_flags |= CPU_DISP_HALTED;
                membar_producer();
                bitset_atomic_add(&cp->cp_haltset, cpu_sid);
        }

        /*
         * Check to make sure there's really nothing to do.
         * Work destined for this CPU may become available after
         * this check. We'll be notified through the clearing of our
         * bit in the halted CPU bitset, and a poke.
         */
        if (disp_anywork()) {
                if (hset_update) {
                        cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
                        bitset_atomic_del(&cp->cp_haltset, cpu_sid);
                }
                return;
        }

        /*
         * We're on our way to being halted.  Wait until something becomes
         * runnable locally or we are awaken (i.e. removed from the halt set).
         * Note that the call to hv_cpu_yield() can return even if we have
         * nothing to do.
         *
         * Disable interrupts now, so that we'll awaken immediately
         * after halting if someone tries to poke us between now and
         * the time we actually halt.
         *
         * We check for the presence of our bit after disabling interrupts.
         * If it's cleared, we'll return. If the bit is cleared after
         * we check then the poke will pop us out of the halted state.
         * Also, if the offlined CPU has been brought back on-line, then
         * we return as well.
         *
         * The ordering of the poke and the clearing of the bit by cpu_wakeup
         * is important.
         * cpu_wakeup() must clear, then poke.
         * cpu_halt() must disable interrupts, then check for the bit.
         *
         * The check for anything locally runnable is here for performance
         * and isn't needed for correctness. disp_nrunnable ought to be
         * in our cache still, so it's inexpensive to check, and if there
         * is anything runnable we won't have to wait for the poke.
         *
         * Any interrupt will awaken the cpu from halt. Looping here
         * will filter spurious interrupts that wake us up, but don't
         * represent a need for us to head back out to idle().  This
         * will enable the idle loop to be more efficient and sleep in
         * the processor pipeline for a larger percent of the time,
         * which returns useful cycles to the peer hardware strand
         * that shares the pipeline.
         */
        s = disable_vec_intr();
        while (*p == 0 &&
            ((hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid)) ||
            (!hset_update && (CPU->cpu_flags & CPU_OFFLINE)))) {

                DTRACE_PROBE1(idle__state__transition,
                    uint_t, IDLE_STATE_HALTED);
                (void) cpu_halt_cpu();
                DTRACE_PROBE1(idle__state__transition,
                    uint_t, IDLE_STATE_NORMAL);

                enable_vec_intr(s);
                s = disable_vec_intr();
        }

        /*
         * We're no longer halted
         */
        enable_vec_intr(s);
        if (hset_update) {
                cpup->cpu_disp_flags &= ~CPU_DISP_HALTED;
                bitset_atomic_del(&cp->cp_haltset, cpu_sid);
        }
}

/*
 * If "cpu" is halted, then wake it up clearing its halted bit in advance.
 * Otherwise, see if other CPUs in the cpu partition are halted and need to
 * be woken up so that they can steal the thread we placed on this CPU.
 * This function is only used on MP systems.
 * This function should only be invoked if cpu_halt_cpu()
 * exists and is supported, see mach_cpu_halt_idle()
 */
static void
cpu_wakeup(cpu_t *cpu, int bound)
{
        uint_t          cpu_found;
        processorid_t   cpu_sid;
        cpupart_t       *cp;

        cp = cpu->cpu_part;
        cpu_sid = cpu->cpu_seqid;
        if (bitset_in_set(&cp->cp_haltset, cpu_sid)) {
                /*
                 * Clear the halted bit for that CPU since it will be
                 * poked in a moment.
                 */
                bitset_atomic_del(&cp->cp_haltset, cpu_sid);
                /*
                 * We may find the current CPU present in the halted cpu bitset
                 * if we're in the context of an interrupt that occurred
                 * before we had a chance to clear our bit in cpu_halt().
                 * Poking ourself is obviously unnecessary, since if
                 * we're here, we're not halted.
                 */
                if (cpu != CPU)
                        poke_cpu(cpu->cpu_id);
                return;
        } else {
                /*
                 * This cpu isn't halted, but it's idle or undergoing a
                 * context switch. No need to awaken anyone else.
                 */
                if (cpu->cpu_thread == cpu->cpu_idle_thread ||
                    cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL)
                        return;
        }

        /*
         * No need to wake up other CPUs if this is for a bound thread.
         */
        if (bound)
                return;

        /*
         * The CPU specified for wakeup isn't currently halted, so check
         * to see if there are any other halted CPUs in the partition,
         * and if there are then awaken one.
         *
         * If possible, try to select a CPU close to the target, since this
         * will likely trigger a migration.
         */
        do {
                cpu_found = bitset_find(&cp->cp_haltset);
                if (cpu_found == (uint_t)-1)
                        return;
        } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0);

        if (cpu_found != CPU->cpu_seqid)
                poke_cpu(cpu_seq[cpu_found]->cpu_id);
}

void
mach_cpu_halt_idle(void)
{
        if (enable_halt_idle_cpus) {
                if (&cpu_halt_cpu) {
                        idle_cpu = cpu_halt;
                        disp_enq_thread = cpu_wakeup;
                }
        }
}

/*ARGSUSED*/
int
cpu_intrq_setup(struct cpu *cp)
{
        /* Interrupt mondo queues not applicable to sun4u */
        return (0);
}

/*ARGSUSED*/
void
cpu_intrq_cleanup(struct cpu *cp)
{
        /* Interrupt mondo queues not applicable to sun4u */
}

/*ARGSUSED*/
void
cpu_intrq_register(struct cpu *cp)
{
        /* Interrupt/error queues not applicable to sun4u */
}

/*ARGSUSED*/
void
mach_htraptrace_setup(int cpuid)
{
        /* Setup hypervisor traptrace buffer, not applicable to sun4u */
}

/*ARGSUSED*/
void
mach_htraptrace_configure(int cpuid)
{
        /* enable/ disable hypervisor traptracing, not applicable to sun4u */
}

/*ARGSUSED*/
void
mach_htraptrace_cleanup(int cpuid)
{
        /* cleanup hypervisor traptrace buffer, not applicable to sun4u */
}

void
mach_descrip_startup_init(void)
{
        /*
         * Only for sun4v.
         * Initialize Machine description framework during startup.
         */
}
void
mach_descrip_startup_fini(void)
{
        /*
         * Only for sun4v.
         * Clean up Machine Description framework during startup.
         */
}

void
mach_descrip_init(void)
{
        /*
         * Only for sun4v.
         * Initialize Machine description framework.
         */
}

void
hsvc_setup(void)
{
        /* Setup hypervisor services, not applicable to sun4u */
}

void
load_mach_drivers(void)
{
        /* Currently no machine class (sun4u) specific drivers to load */
}

/*
 * Return true if the machine we're running on is a Positron.
 * (Positron is an unsupported developers platform.)
 */
int
iam_positron(void)
{
        char model[32];
        const char proto_model[] = "SUNW,501-2732";
        pnode_t root = prom_rootnode();

        if (prom_getproplen(root, "model") != sizeof (proto_model))
                return (0);

        (void) prom_getprop(root, "model", model);
        if (strcmp(model, proto_model) == 0)
                return (1);
        return (0);
}

/*
 * Find a physically contiguous area of twice the largest ecache size
 * to be used while doing displacement flush of ecaches.
 */
uint64_t
ecache_flush_address(void)
{
        struct memlist *pmem;
        uint64_t flush_size;
        uint64_t ret_val;

        flush_size = ecache_size * 2;
        for (pmem = phys_install; pmem; pmem = pmem->ml_next) {
                ret_val = P2ROUNDUP(pmem->ml_address, ecache_size);
                if (ret_val + flush_size <= pmem->ml_address + pmem->ml_size)
                        return (ret_val);
        }
        return ((uint64_t)-1);
}

/*
 * Called with the memlist lock held to say that phys_install has
 * changed.
 */
void
phys_install_has_changed(void)
{
        /*
         * Get the new address into a temporary just in case panicking
         * involves use of ecache_flushaddr.
         */
        uint64_t new_addr;

        new_addr = ecache_flush_address();
        if (new_addr == (uint64_t)-1) {
                cmn_err(CE_PANIC,
                    "ecache_flush_address(): failed, ecache_size=%x",
                    ecache_size);
                /*NOTREACHED*/
        }
        ecache_flushaddr = new_addr;
        membar_producer();
}