arch/powerpc/platforms/85xx/smp.c

   1 /*
   2  * Author: Andy Fleming <afleming@freescale.com>
   3  *         Kumar Gala <galak@kernel.crashing.org>
   4  *
   5  * Copyright 2006-2008, 2011-2012, 2015 Freescale Semiconductor Inc.
   6  *
   7  * This program is free software; you can redistribute  it and/or modify it
   8  * under  the terms of  the GNU General  Public License as published by the
   9  * Free Software Foundation;  either version 2 of the  License, or (at your
  10  * option) any later version.
  11  */
  12
  13 #include <linux/stddef.h>
  14 #include <linux/kernel.h>
  15 #include <linux/init.h>
  16 #include <linux/delay.h>
  17 #include <linux/of.h>
  18 #include <linux/kexec.h>
  19 #include <linux/highmem.h>
  20 #include <linux/cpu.h>
  21 #include <linux/fsl/guts.h>
  22
  23 #include <asm/machdep.h>
  24 #include <asm/pgtable.h>
  25 #include <asm/page.h>
  26 #include <asm/mpic.h>
  27 #include <asm/cacheflush.h>
  28 #include <asm/dbell.h>
  29 #include <asm/code-patching.h>
  30 #include <asm/cputhreads.h>
  31 #include <asm/fsl_pm.h>
  32
  33 #include <sysdev/fsl_soc.h>
  34 #include <sysdev/mpic.h>
  35 #include "smp.h"
  36
  37 struct epapr_spin_table {
  38         u32     addr_h;
  39         u32     addr_l;
  40         u32     r3_h;
  41         u32     r3_l;
  42         u32     reserved;
  43         u32     pir;
  44 };
  45
  46 #ifdef CONFIG_HOTPLUG_CPU
  47 static u64 timebase;
  48 static int tb_req;
  49 static int tb_valid;
  50
  51 static void mpc85xx_give_timebase(void)
  52 {
  53         unsigned long flags;
  54
  55         local_irq_save(flags);
  56         hard_irq_disable();
  57
  58         while (!tb_req)
  59                 barrier();
  60         tb_req = 0;
  61
  62         qoriq_pm_ops->freeze_time_base(true);
  63 #ifdef CONFIG_PPC64
  64         /*
  65          * e5500/e6500 have a workaround for erratum A-006958 in place
  66          * that will reread the timebase until TBL is non-zero.
  67          * That would be a bad thing when the timebase is frozen.
  68          *
  69          * Thus, we read it manually, and instead of checking that
  70          * TBL is non-zero, we ensure that TB does not change.  We don't
  71          * do that for the main mftb implementation, because it requires
  72          * a scratch register
  73          */
  74         {
  75                 u64 prev;
  76
  77                 asm volatile("mfspr %0, %1" : "=r" (timebase) :
  78                              "i" (SPRN_TBRL));
  79
  80                 do {
  81                         prev = timebase;
  82                         asm volatile("mfspr %0, %1" : "=r" (timebase) :
  83                                      "i" (SPRN_TBRL));
  84                 } while (prev != timebase);
  85         }
  86 #else
  87         timebase = get_tb();
  88 #endif
  89         mb();
  90         tb_valid = 1;
  91
  92         while (tb_valid)
  93                 barrier();
  94
  95         qoriq_pm_ops->freeze_time_base(false);
  96
  97         local_irq_restore(flags);
  98 }
  99
 100 static void mpc85xx_take_timebase(void)
 101 {
 102         unsigned long flags;
 103
 104         local_irq_save(flags);
 105         hard_irq_disable();
 106
 107         tb_req = 1;
 108         while (!tb_valid)
 109                 barrier();
 110
 111         set_tb(timebase >> 32, timebase & 0xffffffff);
 112         isync();
 113         tb_valid = 0;
 114
 115         local_irq_restore(flags);
 116 }
 117
 118 static void smp_85xx_mach_cpu_die(void)
 119 {
 120         unsigned int cpu = smp_processor_id();
 121
 122         local_irq_disable();
 123         hard_irq_disable();
 124         /* mask all irqs to prevent cpu wakeup */
 125         qoriq_pm_ops->irq_mask(cpu);
 126
 127         idle_task_exit();
 128
 129         mtspr(SPRN_TCR, 0);
 130         mtspr(SPRN_TSR, mfspr(SPRN_TSR));
 131
 132         generic_set_cpu_dead(cpu);
 133
 134         cur_cpu_spec->cpu_down_flush();
 135
 136         qoriq_pm_ops->cpu_die(cpu);
 137
 138         while (1)
 139                 ;
 140 }
 141
 142 static void qoriq_cpu_kill(unsigned int cpu)
 143 {
 144         int i;
 145
 146         for (i = 0; i < 500; i++) {
 147                 if (is_cpu_dead(cpu)) {
 148 #ifdef CONFIG_PPC64
 149                         paca[cpu].cpu_start = 0;
 150 #endif
 151                         return;
 152                 }
 153                 msleep(20);
 154         }
 155         pr_err("CPU%d didn't die...\n", cpu);
 156 }
 157 #endif
 158
 159 /*
 160  * To keep it compatible with old boot program which uses
 161  * cache-inhibit spin table, we need to flush the cache
 162  * before accessing spin table to invalidate any staled data.
 163  * We also need to flush the cache after writing to spin
 164  * table to push data out.
 165  */
 166 static inline void flush_spin_table(void *spin_table)
 167 {
 168         flush_dcache_range((ulong)spin_table,
 169                 (ulong)spin_table + sizeof(struct epapr_spin_table));
 170 }
 171
 172 static inline u32 read_spin_table_addr_l(void *spin_table)
 173 {
 174         flush_dcache_range((ulong)spin_table,
 175                 (ulong)spin_table + sizeof(struct epapr_spin_table));
 176         return in_be32(&((struct epapr_spin_table *)spin_table)->addr_l);
 177 }
 178
 179 #ifdef CONFIG_PPC64
 180 static void wake_hw_thread(void *info)
 181 {
 182         void fsl_secondary_thread_init(void);
 183         unsigned long inia;
 184         int cpu = *(const int *)info;
 185
 186         inia = *(unsigned long *)fsl_secondary_thread_init;
 187         book3e_start_thread(cpu_thread_in_core(cpu), inia);
 188 }
 189 #endif
 190
 191 static int smp_85xx_start_cpu(int cpu)
 192 {
 193         int ret = 0;
 194         struct device_node *np;
 195         const u64 *cpu_rel_addr;
 196         unsigned long flags;
 197         int ioremappable;
 198         int hw_cpu = get_hard_smp_processor_id(cpu);
 199         struct epapr_spin_table __iomem *spin_table;
 200
 201         np = of_get_cpu_node(cpu, NULL);
 202         cpu_rel_addr = of_get_property(np, "cpu-release-addr", NULL);
 203         if (!cpu_rel_addr) {
 204                 pr_err("No cpu-release-addr for cpu %d\n", cpu);
 205                 return -ENOENT;
 206         }
 207
 208         /*
 209          * A secondary core could be in a spinloop in the bootpage
 210          * (0xfffff000), somewhere in highmem, or somewhere in lowmem.
 211          * The bootpage and highmem can be accessed via ioremap(), but
 212          * we need to directly access the spinloop if its in lowmem.
 213          */
 214         ioremappable = *cpu_rel_addr > virt_to_phys(high_memory);
 215
 216         /* Map the spin table */
 217         if (ioremappable)
 218                 spin_table = ioremap_prot(*cpu_rel_addr,
 219                         sizeof(struct epapr_spin_table), _PAGE_COHERENT);
 220         else
 221                 spin_table = phys_to_virt(*cpu_rel_addr);
 222
 223         local_irq_save(flags);
 224         hard_irq_disable();
 225
 226         if (qoriq_pm_ops)
 227                 qoriq_pm_ops->cpu_up_prepare(cpu);
 228
 229         /* if cpu is not spinning, reset it */
 230         if (read_spin_table_addr_l(spin_table) != 1) {
 231                 /*
 232                  * We don't set the BPTR register here since it already points
 233                  * to the boot page properly.
 234                  */
 235                 mpic_reset_core(cpu);
 236
 237                 /*
 238                  * wait until core is ready...
 239                  * We need to invalidate the stale data, in case the boot
 240                  * loader uses a cache-inhibited spin table.
 241                  */
 242                 if (!spin_event_timeout(
 243                                 read_spin_table_addr_l(spin_table) == 1,
 244                                 10000, 100)) {
 245                         pr_err("timeout waiting for cpu %d to reset\n",
 246                                 hw_cpu);
 247                         ret = -EAGAIN;
 248                         goto err;
 249                 }
 250         }
 251
 252         flush_spin_table(spin_table);
 253         out_be32(&spin_table->pir, hw_cpu);
 254 #ifdef CONFIG_PPC64
 255         out_be64((u64 *)(&spin_table->addr_h),
 256                 __pa(ppc_function_entry(generic_secondary_smp_init)));
 257 #else
 258         out_be32(&spin_table->addr_l, __pa(__early_start));
 259 #endif
 260         flush_spin_table(spin_table);
 261 err:
 262         local_irq_restore(flags);
 263
 264         if (ioremappable)
 265                 iounmap(spin_table);
 266
 267         return ret;
 268 }
 269
 270 static int smp_85xx_kick_cpu(int nr)
 271 {
 272         int ret = 0;
 273 #ifdef CONFIG_PPC64
 274         int primary = nr;
 275 #endif
 276
 277         WARN_ON(nr < 0 || nr >= num_possible_cpus());
 278
 279         pr_debug("kick CPU #%d\n", nr);
 280
 281 #ifdef CONFIG_PPC64
 282         if (threads_per_core == 2) {
 283                 if (WARN_ON_ONCE(!cpu_has_feature(CPU_FTR_SMT)))
 284                         return -ENOENT;
 285
 286                 booting_thread_hwid = cpu_thread_in_core(nr);
 287                 primary = cpu_first_thread_sibling(nr);
 288
 289                 if (qoriq_pm_ops)
 290                         qoriq_pm_ops->cpu_up_prepare(nr);
 291
 292                 /*
 293                  * If either thread in the core is online, use it to start
 294                  * the other.
 295                  */
 296                 if (cpu_online(primary)) {
 297                         smp_call_function_single(primary,
 298                                         wake_hw_thread, &nr, 1);
 299                         goto done;
 300                 } else if (cpu_online(primary + 1)) {
 301                         smp_call_function_single(primary + 1,
 302                                         wake_hw_thread, &nr, 1);
 303                         goto done;
 304                 }
 305
 306                 /*
 307                  * If getting here, it means both threads in the core are
 308                  * offline. So start the primary thread, then it will start
 309                  * the thread specified in booting_thread_hwid, the one
 310                  * corresponding to nr.
 311                  */
 312
 313         } else if (threads_per_core == 1) {
 314                 /*
 315                  * If one core has only one thread, set booting_thread_hwid to
 316                  * an invalid value.
 317                  */
 318                 booting_thread_hwid = INVALID_THREAD_HWID;
 319
 320         } else if (threads_per_core > 2) {
 321                 pr_err("Do not support more than 2 threads per CPU.");
 322                 return -EINVAL;
 323         }
 324
 325         ret = smp_85xx_start_cpu(primary);
 326         if (ret)
 327                 return ret;
 328
 329 done:
 330         paca[nr].cpu_start = 1;
 331         generic_set_cpu_up(nr);
 332
 333         return ret;
 334 #else
 335         ret = smp_85xx_start_cpu(nr);
 336         if (ret)
 337                 return ret;
 338
 339         generic_set_cpu_up(nr);
 340
 341         return ret;
 342 #endif
 343 }
 344
 345 struct smp_ops_t smp_85xx_ops = {
 346         .kick_cpu = smp_85xx_kick_cpu,
 347         .cpu_bootable = smp_generic_cpu_bootable,
 348 #ifdef CONFIG_HOTPLUG_CPU
 349         .cpu_disable    = generic_cpu_disable,
 350         .cpu_die        = generic_cpu_die,
 351 #endif
 352 #if defined(CONFIG_KEXEC) && !defined(CONFIG_PPC64)
 353         .give_timebase  = smp_generic_give_timebase,
 354         .take_timebase  = smp_generic_take_timebase,
 355 #endif
 356 };
 357
 358 #ifdef CONFIG_KEXEC
 359 #ifdef CONFIG_PPC32
 360 atomic_t kexec_down_cpus = ATOMIC_INIT(0);
 361
 362 void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary)
 363 {
 364         local_irq_disable();
 365
 366         if (secondary) {
 367                 cur_cpu_spec->cpu_down_flush();
 368                 atomic_inc(&kexec_down_cpus);
 369                 /* loop forever */
 370                 while (1);
 371         }
 372 }
 373
 374 static void mpc85xx_smp_kexec_down(void *arg)
 375 {
 376         if (ppc_md.kexec_cpu_down)
 377                 ppc_md.kexec_cpu_down(0,1);
 378 }
 379 #else
 380 void mpc85xx_smp_kexec_cpu_down(int crash_shutdown, int secondary)
 381 {
 382         int cpu = smp_processor_id();
 383         int sibling = cpu_last_thread_sibling(cpu);
 384         bool notified = false;
 385         int disable_cpu;
 386         int disable_threadbit = 0;
 387         long start = mftb();
 388         long now;
 389
 390         local_irq_disable();
 391         hard_irq_disable();
 392         mpic_teardown_this_cpu(secondary);
 393
 394         if (cpu == crashing_cpu && cpu_thread_in_core(cpu) != 0) {
 395                 /*
 396                  * We enter the crash kernel on whatever cpu crashed,
 397                  * even if it's a secondary thread.  If that's the case,
 398                  * disable the corresponding primary thread.
 399                  */
 400                 disable_threadbit = 1;
 401                 disable_cpu = cpu_first_thread_sibling(cpu);
 402         } else if (sibling != crashing_cpu &&
 403                    cpu_thread_in_core(cpu) == 0 &&
 404                    cpu_thread_in_core(sibling) != 0) {
 405                 disable_threadbit = 2;
 406                 disable_cpu = sibling;
 407         }
 408
 409         if (disable_threadbit) {
 410                 while (paca[disable_cpu].kexec_state < KEXEC_STATE_REAL_MODE) {
 411                         barrier();
 412                         now = mftb();
 413                         if (!notified && now - start > 1000000) {
 414                                 pr_info("%s/%d: waiting for cpu %d to enter KEXEC_STATE_REAL_MODE (%d)\n",
 415                                         __func__, smp_processor_id(),
 416                                         disable_cpu,
 417                                         paca[disable_cpu].kexec_state);
 418                                 notified = true;
 419                         }
 420                 }
 421
 422                 if (notified) {
 423                         pr_info("%s: cpu %d done waiting\n",
 424                                 __func__, disable_cpu);
 425                 }
 426
 427                 mtspr(SPRN_TENC, disable_threadbit);
 428                 while (mfspr(SPRN_TENSR) & disable_threadbit)
 429                         cpu_relax();
 430         }
 431 }
 432 #endif
 433
 434 static void mpc85xx_smp_machine_kexec(struct kimage *image)
 435 {
 436 #ifdef CONFIG_PPC32
 437         int timeout = INT_MAX;
 438         int i, num_cpus = num_present_cpus();
 439
 440         if (image->type == KEXEC_TYPE_DEFAULT)
 441                 smp_call_function(mpc85xx_smp_kexec_down, NULL, 0);
 442
 443         while ( (atomic_read(&kexec_down_cpus) != (num_cpus - 1)) &&
 444                 ( timeout > 0 ) )
 445         {
 446                 timeout--;
 447         }
 448
 449         if ( !timeout )
 450                 printk(KERN_ERR "Unable to bring down secondary cpu(s)");
 451
 452         for_each_online_cpu(i)
 453         {
 454                 if ( i == smp_processor_id() ) continue;
 455                 mpic_reset_core(i);
 456         }
 457 #endif
 458
 459         default_machine_kexec(image);
 460 }
 461 #endif /* CONFIG_KEXEC */
 462
 463 static void smp_85xx_basic_setup(int cpu_nr)
 464 {
 465         if (cpu_has_feature(CPU_FTR_DBELL))
 466                 doorbell_setup_this_cpu();
 467 }
 468
 469 static void smp_85xx_setup_cpu(int cpu_nr)
 470 {
 471         mpic_setup_this_cpu();
 472         smp_85xx_basic_setup(cpu_nr);
 473 }
 474
 475 void __init mpc85xx_smp_init(void)
 476 {
 477         struct device_node *np;
 478
 479
 480         np = of_find_node_by_type(NULL, "open-pic");
 481         if (np) {
 482                 smp_85xx_ops.probe = smp_mpic_probe;
 483                 smp_85xx_ops.setup_cpu = smp_85xx_setup_cpu;
 484                 smp_85xx_ops.message_pass = smp_mpic_message_pass;
 485         } else
 486                 smp_85xx_ops.setup_cpu = smp_85xx_basic_setup;
 487
 488         if (cpu_has_feature(CPU_FTR_DBELL)) {
 489                 /*
 490                  * If left NULL, .message_pass defaults to
 491                  * smp_muxed_ipi_message_pass
 492                  */
 493                 smp_85xx_ops.message_pass = NULL;
 494                 smp_85xx_ops.cause_ipi = doorbell_cause_ipi;
 495                 smp_85xx_ops.probe = NULL;
 496         }
 497
 498 #ifdef CONFIG_HOTPLUG_CPU
 499 #ifdef CONFIG_FSL_CORENET_RCPM
 500         fsl_rcpm_init();
 501 #endif
 502
 503 #ifdef CONFIG_FSL_PMC
 504         mpc85xx_setup_pmc();
 505 #endif
 506         if (qoriq_pm_ops) {
 507                 smp_85xx_ops.give_timebase = mpc85xx_give_timebase;
 508                 smp_85xx_ops.take_timebase = mpc85xx_take_timebase;
 509                 ppc_md.cpu_die = smp_85xx_mach_cpu_die;
 510                 smp_85xx_ops.cpu_die = qoriq_cpu_kill;
 511         }
 512 #endif
 513         smp_ops = &smp_85xx_ops;
 514
 515 #ifdef CONFIG_KEXEC
 516         ppc_md.kexec_cpu_down = mpc85xx_smp_kexec_cpu_down;
 517         ppc_md.machine_kexec = mpc85xx_smp_machine_kexec;
 518 #endif
 519 }