arch/powerpc/platforms/powernv/smp.c

   1 /*
   2  * SMP support for PowerNV machines.
   3  *
   4  * Copyright 2011 IBM Corp.
   5  *
   6  * This program is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU General Public License
   8  * as published by the Free Software Foundation; either version
   9  * 2 of the License, or (at your option) any later version.
  10  */
  11
  12 #include <linux/kernel.h>
  13 #include <linux/module.h>
  14 #include <linux/sched.h>
  15 #include <linux/sched/hotplug.h>
  16 #include <linux/smp.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/delay.h>
  19 #include <linux/init.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/cpu.h>
  22
  23 #include <asm/irq.h>
  24 #include <asm/smp.h>
  25 #include <asm/paca.h>
  26 #include <asm/machdep.h>
  27 #include <asm/cputable.h>
  28 #include <asm/firmware.h>
  29 #include <asm/vdso_datapage.h>
  30 #include <asm/cputhreads.h>
  31 #include <asm/xics.h>
  32 #include <asm/xive.h>
  33 #include <asm/opal.h>
  34 #include <asm/runlatch.h>
  35 #include <asm/code-patching.h>
  36 #include <asm/dbell.h>
  37 #include <asm/kvm_ppc.h>
  38 #include <asm/ppc-opcode.h>
  39 #include <asm/cpuidle.h>
  40 #include <asm/kexec.h>
  41 #include <asm/reg.h>
  42
  43 #include "powernv.h"
  44
  45 #ifdef DEBUG
  46 #include <asm/udbg.h>
  47 #define DBG(fmt...) udbg_printf(fmt)
  48 #else
  49 #define DBG(fmt...)
  50 #endif
  51
  52 static void pnv_smp_setup_cpu(int cpu)
  53 {
  54         /*
  55          * P9 workaround for CI vector load (see traps.c),
  56          * enable the corresponding HMI interrupt
  57          */
  58         if (pvr_version_is(PVR_POWER9))
  59                 mtspr(SPRN_HMEER, mfspr(SPRN_HMEER) | PPC_BIT(17));
  60
  61         if (xive_enabled())
  62                 xive_smp_setup_cpu();
  63         else if (cpu != boot_cpuid)
  64                 xics_setup_cpu();
  65 }
  66
  67 static int pnv_smp_kick_cpu(int nr)
  68 {
  69         unsigned int pcpu;
  70         unsigned long start_here =
  71                         __pa(ppc_function_entry(generic_secondary_smp_init));
  72         long rc;
  73         uint8_t status;
  74
  75         if (nr < 0 || nr >= nr_cpu_ids)
  76                 return -EINVAL;
  77
  78         pcpu = get_hard_smp_processor_id(nr);
  79         /*
  80          * If we already started or OPAL is not supported, we just
  81          * kick the CPU via the PACA
  82          */
  83         if (paca[nr].cpu_start || !firmware_has_feature(FW_FEATURE_OPAL))
  84                 goto kick;
  85
  86         /*
  87          * At this point, the CPU can either be spinning on the way in
  88          * from kexec or be inside OPAL waiting to be started for the
  89          * first time. OPAL v3 allows us to query OPAL to know if it
  90          * has the CPUs, so we do that
  91          */
  92         rc = opal_query_cpu_status(pcpu, &status);
  93         if (rc != OPAL_SUCCESS) {
  94                 pr_warn("OPAL Error %ld querying CPU %d state\n", rc, nr);
  95                 return -ENODEV;
  96         }
  97
  98         /*
  99          * Already started, just kick it, probably coming from
 100          * kexec and spinning
 101          */
 102         if (status == OPAL_THREAD_STARTED)
 103                 goto kick;
 104
 105         /*
 106          * Available/inactive, let's kick it
 107          */
 108         if (status == OPAL_THREAD_INACTIVE) {
 109                 pr_devel("OPAL: Starting CPU %d (HW 0x%x)...\n", nr, pcpu);
 110                 rc = opal_start_cpu(pcpu, start_here);
 111                 if (rc != OPAL_SUCCESS) {
 112                         pr_warn("OPAL Error %ld starting CPU %d\n", rc, nr);
 113                         return -ENODEV;
 114                 }
 115         } else {
 116                 /*
 117                  * An unavailable CPU (or any other unknown status)
 118                  * shouldn't be started. It should also
 119                  * not be in the possible map but currently it can
 120                  * happen
 121                  */
 122                 pr_devel("OPAL: CPU %d (HW 0x%x) is unavailable"
 123                          " (status %d)...\n", nr, pcpu, status);
 124                 return -ENODEV;
 125         }
 126
 127 kick:
 128         return smp_generic_kick_cpu(nr);
 129 }
 130
 131 #ifdef CONFIG_HOTPLUG_CPU
 132
 133 static int pnv_smp_cpu_disable(void)
 134 {
 135         int cpu = smp_processor_id();
 136
 137         /* This is identical to pSeries... might consolidate by
 138          * moving migrate_irqs_away to a ppc_md with default to
 139          * the generic fixup_irqs. --BenH.
 140          */
 141         set_cpu_online(cpu, false);
 142         vdso_data->processorCount--;
 143         if (cpu == boot_cpuid)
 144                 boot_cpuid = cpumask_any(cpu_online_mask);
 145         if (xive_enabled())
 146                 xive_smp_disable_cpu();
 147         else
 148                 xics_migrate_irqs_away();
 149         return 0;
 150 }
 151
 152 static void pnv_smp_cpu_kill_self(void)
 153 {
 154         unsigned int cpu;
 155         unsigned long srr1, wmask;
 156
 157         /* Standard hot unplug procedure */
 158         /*
 159          * This hard disables local interurpts, ensuring we have no lazy
 160          * irqs pending.
 161          */
 162         WARN_ON(irqs_disabled());
 163         hard_irq_disable();
 164         WARN_ON(lazy_irq_pending());
 165
 166         idle_task_exit();
 167         current->active_mm = NULL; /* for sanity */
 168         cpu = smp_processor_id();
 169         DBG("CPU%d offline\n", cpu);
 170         generic_set_cpu_dead(cpu);
 171         smp_wmb();
 172
 173         wmask = SRR1_WAKEMASK;
 174         if (cpu_has_feature(CPU_FTR_ARCH_207S))
 175                 wmask = SRR1_WAKEMASK_P8;
 176
 177         while (!generic_check_cpu_restart(cpu)) {
 178                 /*
 179                  * Clear IPI flag, since we don't handle IPIs while
 180                  * offline, except for those when changing micro-threading
 181                  * mode, which are handled explicitly below, and those
 182                  * for coming online, which are handled via
 183                  * generic_check_cpu_restart() calls.
 184                  */
 185                 kvmppc_set_host_ipi(cpu, 0);
 186
 187                 srr1 = pnv_cpu_offline(cpu);
 188
 189                 WARN_ON(lazy_irq_pending());
 190
 191                 /*
 192                  * If the SRR1 value indicates that we woke up due to
 193                  * an external interrupt, then clear the interrupt.
 194                  * We clear the interrupt before checking for the
 195                  * reason, so as to avoid a race where we wake up for
 196                  * some other reason, find nothing and clear the interrupt
 197                  * just as some other cpu is sending us an interrupt.
 198                  * If we returned from power7_nap as a result of
 199                  * having finished executing in a KVM guest, then srr1
 200                  * contains 0.
 201                  */
 202                 if (((srr1 & wmask) == SRR1_WAKEEE) ||
 203                     ((srr1 & wmask) == SRR1_WAKEHVI)) {
 204                         if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 205                                 if (xive_enabled())
 206                                         xive_flush_interrupt();
 207                                 else
 208                                         icp_opal_flush_interrupt();
 209                         } else
 210                                 icp_native_flush_interrupt();
 211                 } else if ((srr1 & wmask) == SRR1_WAKEHDBELL) {
 212                         unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER);
 213                         asm volatile(PPC_MSGCLR(%0) : : "r" (msg));
 214                 } else if ((srr1 & wmask) == SRR1_WAKERESET) {
 215                         irq_set_pending_from_srr1(srr1);
 216                         /* Does not return */
 217                 }
 218
 219                 smp_mb();
 220
 221                 /*
 222                  * For kdump kernels, we process the ipi and jump to
 223                  * crash_ipi_callback
 224                  */
 225                 if (kdump_in_progress()) {
 226                         /*
 227                          * If we got to this point, we've not used
 228                          * NMI's, otherwise we would have gone
 229                          * via the SRR1_WAKERESET path. We are
 230                          * using regular IPI's for waking up offline
 231                          * threads.
 232                          */
 233                         struct pt_regs regs;
 234
 235                         ppc_save_regs(&regs);
 236                         crash_ipi_callback(&regs);
 237                         /* Does not return */
 238                 }
 239
 240                 if (cpu_core_split_required())
 241                         continue;
 242
 243                 if (srr1 && !generic_check_cpu_restart(cpu))
 244                         DBG("CPU%d Unexpected exit while offline srr1=%lx!\n",
 245                                         cpu, srr1);
 246
 247         }
 248
 249         DBG("CPU%d coming online...\n", cpu);
 250 }
 251
 252 #endif /* CONFIG_HOTPLUG_CPU */
 253
 254 static int pnv_cpu_bootable(unsigned int nr)
 255 {
 256         /*
 257          * Starting with POWER8, the subcore logic relies on all threads of a
 258          * core being booted so that they can participate in split mode
 259          * switches. So on those machines we ignore the smt_enabled_at_boot
 260          * setting (smt-enabled on the kernel command line).
 261          */
 262         if (cpu_has_feature(CPU_FTR_ARCH_207S))
 263                 return 1;
 264
 265         return smp_generic_cpu_bootable(nr);
 266 }
 267
 268 static int pnv_smp_prepare_cpu(int cpu)
 269 {
 270         if (xive_enabled())
 271                 return xive_smp_prepare_cpu(cpu);
 272         return 0;
 273 }
 274
 275 /* Cause IPI as setup by the interrupt controller (xics or xive) */
 276 static void (*ic_cause_ipi)(int cpu);
 277
 278 static void pnv_cause_ipi(int cpu)
 279 {
 280         if (doorbell_try_core_ipi(cpu))
 281                 return;
 282
 283         ic_cause_ipi(cpu);
 284 }
 285
 286 static void pnv_p9_dd1_cause_ipi(int cpu)
 287 {
 288         int this_cpu = get_cpu();
 289
 290         /*
 291          * POWER9 DD1 has a global addressed msgsnd, but for now we restrict
 292          * IPIs to same core, because it requires additional synchronization
 293          * for inter-core doorbells which we do not implement.
 294          */
 295         if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu)))
 296                 doorbell_global_ipi(cpu);
 297         else
 298                 ic_cause_ipi(cpu);
 299
 300         put_cpu();
 301 }
 302
 303 static void __init pnv_smp_probe(void)
 304 {
 305         if (xive_enabled())
 306                 xive_smp_probe();
 307         else
 308                 xics_smp_probe();
 309
 310         if (cpu_has_feature(CPU_FTR_DBELL)) {
 311                 ic_cause_ipi = smp_ops->cause_ipi;
 312                 WARN_ON(!ic_cause_ipi);
 313
 314                 if (cpu_has_feature(CPU_FTR_ARCH_300)) {
 315                         if (cpu_has_feature(CPU_FTR_POWER9_DD1))
 316                                 smp_ops->cause_ipi = pnv_p9_dd1_cause_ipi;
 317                         else
 318                                 smp_ops->cause_ipi = doorbell_global_ipi;
 319                 } else {
 320                         smp_ops->cause_ipi = pnv_cause_ipi;
 321                 }
 322         }
 323 }
 324
 325 static int pnv_system_reset_exception(struct pt_regs *regs)
 326 {
 327         if (smp_handle_nmi_ipi(regs))
 328                 return 1;
 329         return 0;
 330 }
 331
 332 static int pnv_cause_nmi_ipi(int cpu)
 333 {
 334         int64_t rc;
 335
 336         if (cpu >= 0) {
 337                 rc = opal_signal_system_reset(get_hard_smp_processor_id(cpu));
 338                 if (rc != OPAL_SUCCESS)
 339                         return 0;
 340                 return 1;
 341
 342         } else if (cpu == NMI_IPI_ALL_OTHERS) {
 343                 bool success = true;
 344                 int c;
 345
 346
 347                 /*
 348                  * We do not use broadcasts (yet), because it's not clear
 349                  * exactly what semantics Linux wants or the firmware should
 350                  * provide.
 351                  */
 352                 for_each_online_cpu(c) {
 353                         if (c == smp_processor_id())
 354                                 continue;
 355
 356                         rc = opal_signal_system_reset(
 357                                                 get_hard_smp_processor_id(c));
 358                         if (rc != OPAL_SUCCESS)
 359                                 success = false;
 360                 }
 361                 if (success)
 362                         return 1;
 363
 364                 /*
 365                  * Caller will fall back to doorbells, which may pick
 366                  * up the remainders.
 367                  */
 368         }
 369
 370         return 0;
 371 }
 372
 373 static struct smp_ops_t pnv_smp_ops = {
 374         .message_pass   = NULL, /* Use smp_muxed_ipi_message_pass */
 375         .cause_ipi      = NULL, /* Filled at runtime by pnv_smp_probe() */
 376         .cause_nmi_ipi  = NULL,
 377         .probe          = pnv_smp_probe,
 378         .prepare_cpu    = pnv_smp_prepare_cpu,
 379         .kick_cpu       = pnv_smp_kick_cpu,
 380         .setup_cpu      = pnv_smp_setup_cpu,
 381         .cpu_bootable   = pnv_cpu_bootable,
 382 #ifdef CONFIG_HOTPLUG_CPU
 383         .cpu_disable    = pnv_smp_cpu_disable,
 384         .cpu_die        = generic_cpu_die,
 385 #endif /* CONFIG_HOTPLUG_CPU */
 386 };
 387
 388 /* This is called very early during platform setup_arch */
 389 void __init pnv_smp_init(void)
 390 {
 391         if (opal_check_token(OPAL_SIGNAL_SYSTEM_RESET)) {
 392                 ppc_md.system_reset_exception = pnv_system_reset_exception;
 393                 pnv_smp_ops.cause_nmi_ipi = pnv_cause_nmi_ipi;
 394         }
 395         smp_ops = &pnv_smp_ops;
 396
 397 #ifdef CONFIG_HOTPLUG_CPU
 398         ppc_md.cpu_die  = pnv_smp_cpu_kill_self;
 399 #ifdef CONFIG_KEXEC_CORE
 400         crash_wake_offline = 1;
 401 #endif
 402 #endif
 403 }