arch/mips/kernel/smp-cps.c

   1 /*
   2  * Copyright (C) 2013 Imagination Technologies
   3  * Author: Paul Burton <paul.burton@imgtec.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License as published by the
   7  * Free Software Foundation;  either version 2 of the  License, or (at your
   8  * option) any later version.
   9  */
  10
  11 #include <linux/io.h>
  12 #include <linux/sched.h>
  13 #include <linux/slab.h>
  14 #include <linux/smp.h>
  15 #include <linux/types.h>
  16
  17 #include <asm/bcache.h>
  18 #include <asm/gic.h>
  19 #include <asm/mips-cm.h>
  20 #include <asm/mips-cpc.h>
  21 #include <asm/mips_mt.h>
  22 #include <asm/mipsregs.h>
  23 #include <asm/pm-cps.h>
  24 #include <asm/r4kcache.h>
  25 #include <asm/smp-cps.h>
  26 #include <asm/time.h>
  27 #include <asm/uasm.h>
  28
  29 static DECLARE_BITMAP(core_power, NR_CPUS);
  30
  31 struct core_boot_config *mips_cps_core_bootcfg;
  32
  33 static unsigned core_vpe_count(unsigned core)
  34 {
  35         unsigned cfg;
  36
  37         if (!config_enabled(CONFIG_MIPS_MT_SMP) || !cpu_has_mipsmt)
  38                 return 1;
  39
  40         write_gcr_cl_other(core << CM_GCR_Cx_OTHER_CORENUM_SHF);
  41         cfg = read_gcr_co_config() & CM_GCR_Cx_CONFIG_PVPE_MSK;
  42         return (cfg >> CM_GCR_Cx_CONFIG_PVPE_SHF) + 1;
  43 }
  44
  45 static void __init cps_smp_setup(void)
  46 {
  47         unsigned int ncores, nvpes, core_vpes;
  48         int c, v;
  49
  50         /* Detect & record VPE topology */
  51         ncores = mips_cm_numcores();
  52         pr_info("VPE topology ");
  53         for (c = nvpes = 0; c < ncores; c++) {
  54                 core_vpes = core_vpe_count(c);
  55                 pr_cont("%c%u", c ? ',' : '{', core_vpes);
  56
  57                 /* Use the number of VPEs in core 0 for smp_num_siblings */
  58                 if (!c)
  59                         smp_num_siblings = core_vpes;
  60
  61                 for (v = 0; v < min_t(int, core_vpes, NR_CPUS - nvpes); v++) {
  62                         cpu_data[nvpes + v].core = c;
  63 #ifdef CONFIG_MIPS_MT_SMP
  64                         cpu_data[nvpes + v].vpe_id = v;
  65 #endif
  66                 }
  67
  68                 nvpes += core_vpes;
  69         }
  70         pr_cont("} total %u\n", nvpes);
  71
  72         /* Indicate present CPUs (CPU being synonymous with VPE) */
  73         for (v = 0; v < min_t(unsigned, nvpes, NR_CPUS); v++) {
  74                 set_cpu_possible(v, true);
  75                 set_cpu_present(v, true);
  76                 __cpu_number_map[v] = v;
  77                 __cpu_logical_map[v] = v;
  78         }
  79
  80         /* Set a coherent default CCA (CWB) */
  81         change_c0_config(CONF_CM_CMASK, 0x5);
  82
  83         /* Core 0 is powered up (we're running on it) */
  84         bitmap_set(core_power, 0, 1);
  85
  86         /* Initialise core 0 */
  87         mips_cps_core_init();
  88
  89         /* Make core 0 coherent with everything */
  90         write_gcr_cl_coherence(0xff);
  91 }
  92
  93 static void __init cps_prepare_cpus(unsigned int max_cpus)
  94 {
  95         unsigned ncores, core_vpes, c, cca;
  96         bool cca_unsuitable;
  97         u32 *entry_code;
  98
  99         mips_mt_set_cpuoptions();
 100
 101         /* Detect whether the CCA is unsuited to multi-core SMP */
 102         cca = read_c0_config() & CONF_CM_CMASK;
 103         switch (cca) {
 104         case 0x4: /* CWBE */
 105         case 0x5: /* CWB */
 106                 /* The CCA is coherent, multi-core is fine */
 107                 cca_unsuitable = false;
 108                 break;
 109
 110         default:
 111                 /* CCA is not coherent, multi-core is not usable */
 112                 cca_unsuitable = true;
 113         }
 114
 115         /* Warn the user if the CCA prevents multi-core */
 116         ncores = mips_cm_numcores();
 117         if (cca_unsuitable && ncores > 1) {
 118                 pr_warn("Using only one core due to unsuitable CCA 0x%x\n",
 119                         cca);
 120
 121                 for_each_present_cpu(c) {
 122                         if (cpu_data[c].core)
 123                                 set_cpu_present(c, false);
 124                 }
 125         }
 126
 127         /*
 128          * Patch the start of mips_cps_core_entry to provide:
 129          *
 130          * v0 = CM base address
 131          * s0 = kseg0 CCA
 132          */
 133         entry_code = (u32 *)&mips_cps_core_entry;
 134         UASM_i_LA(&entry_code, 3, (long)mips_cm_base);
 135         uasm_i_addiu(&entry_code, 16, 0, cca);
 136         blast_dcache_range((unsigned long)&mips_cps_core_entry,
 137                            (unsigned long)entry_code);
 138         bc_wback_inv((unsigned long)&mips_cps_core_entry,
 139                      (void *)entry_code - (void *)&mips_cps_core_entry);
 140         __sync();
 141
 142         /* Allocate core boot configuration structs */
 143         mips_cps_core_bootcfg = kcalloc(ncores, sizeof(*mips_cps_core_bootcfg),
 144                                         GFP_KERNEL);
 145         if (!mips_cps_core_bootcfg) {
 146                 pr_err("Failed to allocate boot config for %u cores\n", ncores);
 147                 goto err_out;
 148         }
 149
 150         /* Allocate VPE boot configuration structs */
 151         for (c = 0; c < ncores; c++) {
 152                 core_vpes = core_vpe_count(c);
 153                 mips_cps_core_bootcfg[c].vpe_config = kcalloc(core_vpes,
 154                                 sizeof(*mips_cps_core_bootcfg[c].vpe_config),
 155                                 GFP_KERNEL);
 156                 if (!mips_cps_core_bootcfg[c].vpe_config) {
 157                         pr_err("Failed to allocate %u VPE boot configs\n",
 158                                core_vpes);
 159                         goto err_out;
 160                 }
 161         }
 162
 163         /* Mark this CPU as booted */
 164         atomic_set(&mips_cps_core_bootcfg[current_cpu_data.core].vpe_mask,
 165                    1 << cpu_vpe_id(&current_cpu_data));
 166
 167         return;
 168 err_out:
 169         /* Clean up allocations */
 170         if (mips_cps_core_bootcfg) {
 171                 for (c = 0; c < ncores; c++)
 172                         kfree(mips_cps_core_bootcfg[c].vpe_config);
 173                 kfree(mips_cps_core_bootcfg);
 174                 mips_cps_core_bootcfg = NULL;
 175         }
 176
 177         /* Effectively disable SMP by declaring CPUs not present */
 178         for_each_possible_cpu(c) {
 179                 if (c == 0)
 180                         continue;
 181                 set_cpu_present(c, false);
 182         }
 183 }
 184
 185 static void boot_core(unsigned core)
 186 {
 187         u32 access;
 188
 189         /* Select the appropriate core */
 190         write_gcr_cl_other(core << CM_GCR_Cx_OTHER_CORENUM_SHF);
 191
 192         /* Set its reset vector */
 193         write_gcr_co_reset_base(CKSEG1ADDR((unsigned long)mips_cps_core_entry));
 194
 195         /* Ensure its coherency is disabled */
 196         write_gcr_co_coherence(0);
 197
 198         /* Ensure the core can access the GCRs */
 199         access = read_gcr_access();
 200         access |= 1 << (CM_GCR_ACCESS_ACCESSEN_SHF + core);
 201         write_gcr_access(access);
 202
 203         if (mips_cpc_present()) {
 204                 /* Reset the core */
 205                 mips_cpc_lock_other(core);
 206                 write_cpc_co_cmd(CPC_Cx_CMD_RESET);
 207                 mips_cpc_unlock_other();
 208         } else {
 209                 /* Take the core out of reset */
 210                 write_gcr_co_reset_release(0);
 211         }
 212
 213         /* The core is now powered up */
 214         bitmap_set(core_power, core, 1);
 215 }
 216
 217 static void remote_vpe_boot(void *dummy)
 218 {
 219         mips_cps_boot_vpes();
 220 }
 221
 222 static void cps_boot_secondary(int cpu, struct task_struct *idle)
 223 {
 224         unsigned core = cpu_data[cpu].core;
 225         unsigned vpe_id = cpu_vpe_id(&cpu_data[cpu]);
 226         struct core_boot_config *core_cfg = &mips_cps_core_bootcfg[core];
 227         struct vpe_boot_config *vpe_cfg = &core_cfg->vpe_config[vpe_id];
 228         unsigned int remote;
 229         int err;
 230
 231         vpe_cfg->pc = (unsigned long)&smp_bootstrap;
 232         vpe_cfg->sp = __KSTK_TOS(idle);
 233         vpe_cfg->gp = (unsigned long)task_thread_info(idle);
 234
 235         atomic_or(1 << cpu_vpe_id(&cpu_data[cpu]), &core_cfg->vpe_mask);
 236
 237         preempt_disable();
 238
 239         if (!test_bit(core, core_power)) {
 240                 /* Boot a VPE on a powered down core */
 241                 boot_core(core);
 242                 goto out;
 243         }
 244
 245         if (core != current_cpu_data.core) {
 246                 /* Boot a VPE on another powered up core */
 247                 for (remote = 0; remote < NR_CPUS; remote++) {
 248                         if (cpu_data[remote].core != core)
 249                                 continue;
 250                         if (cpu_online(remote))
 251                                 break;
 252                 }
 253                 BUG_ON(remote >= NR_CPUS);
 254
 255                 err = smp_call_function_single(remote, remote_vpe_boot,
 256                                                NULL, 1);
 257                 if (err)
 258                         panic("Failed to call remote CPU\n");
 259                 goto out;
 260         }
 261
 262         BUG_ON(!cpu_has_mipsmt);
 263
 264         /* Boot a VPE on this core */
 265         mips_cps_boot_vpes();
 266 out:
 267         preempt_enable();
 268 }
 269
 270 static void cps_init_secondary(void)
 271 {
 272         /* Disable MT - we only want to run 1 TC per VPE */
 273         if (cpu_has_mipsmt)
 274                 dmt();
 275
 276         change_c0_status(ST0_IM, STATUSF_IP3 | STATUSF_IP4 |
 277                                  STATUSF_IP6 | STATUSF_IP7);
 278 }
 279
 280 static void cps_smp_finish(void)
 281 {
 282         write_c0_compare(read_c0_count() + (8 * mips_hpt_frequency / HZ));
 283
 284 #ifdef CONFIG_MIPS_MT_FPAFF
 285         /* If we have an FPU, enroll ourselves in the FPU-full mask */
 286         if (cpu_has_fpu)
 287                 cpu_set(smp_processor_id(), mt_fpu_cpumask);
 288 #endif /* CONFIG_MIPS_MT_FPAFF */
 289
 290         local_irq_enable();
 291 }
 292
 293 #ifdef CONFIG_HOTPLUG_CPU
 294
 295 static int cps_cpu_disable(void)
 296 {
 297         unsigned cpu = smp_processor_id();
 298         struct core_boot_config *core_cfg;
 299
 300         if (!cpu)
 301                 return -EBUSY;
 302
 303         if (!cps_pm_support_state(CPS_PM_POWER_GATED))
 304                 return -EINVAL;
 305
 306         core_cfg = &mips_cps_core_bootcfg[current_cpu_data.core];
 307         atomic_sub(1 << cpu_vpe_id(&current_cpu_data), &core_cfg->vpe_mask);
 308         smp_mb__after_atomic();
 309         set_cpu_online(cpu, false);
 310         cpu_clear(cpu, cpu_callin_map);
 311
 312         return 0;
 313 }
 314
 315 static DECLARE_COMPLETION(cpu_death_chosen);
 316 static unsigned cpu_death_sibling;
 317 static enum {
 318         CPU_DEATH_HALT,
 319         CPU_DEATH_POWER,
 320 } cpu_death;
 321
 322 void play_dead(void)
 323 {
 324         unsigned cpu, core;
 325
 326         local_irq_disable();
 327         idle_task_exit();
 328         cpu = smp_processor_id();
 329         cpu_death = CPU_DEATH_POWER;
 330
 331         if (cpu_has_mipsmt) {
 332                 core = cpu_data[cpu].core;
 333
 334                 /* Look for another online VPE within the core */
 335                 for_each_online_cpu(cpu_death_sibling) {
 336                         if (cpu_data[cpu_death_sibling].core != core)
 337                                 continue;
 338
 339                         /*
 340                          * There is an online VPE within the core. Just halt
 341                          * this TC and leave the core alone.
 342                          */
 343                         cpu_death = CPU_DEATH_HALT;
 344                         break;
 345                 }
 346         }
 347
 348         /* This CPU has chosen its way out */
 349         complete(&cpu_death_chosen);
 350
 351         if (cpu_death == CPU_DEATH_HALT) {
 352                 /* Halt this TC */
 353                 write_c0_tchalt(TCHALT_H);
 354                 instruction_hazard();
 355         } else {
 356                 /* Power down the core */
 357                 cps_pm_enter_state(CPS_PM_POWER_GATED);
 358         }
 359
 360         /* This should never be reached */
 361         panic("Failed to offline CPU %u", cpu);
 362 }
 363
 364 static void wait_for_sibling_halt(void *ptr_cpu)
 365 {
 366         unsigned cpu = (unsigned)ptr_cpu;
 367         unsigned vpe_id = cpu_vpe_id(&cpu_data[cpu]);
 368         unsigned halted;
 369         unsigned long flags;
 370
 371         do {
 372                 local_irq_save(flags);
 373                 settc(vpe_id);
 374                 halted = read_tc_c0_tchalt();
 375                 local_irq_restore(flags);
 376         } while (!(halted & TCHALT_H));
 377 }
 378
 379 static void cps_cpu_die(unsigned int cpu)
 380 {
 381         unsigned core = cpu_data[cpu].core;
 382         unsigned stat;
 383         int err;
 384
 385         /* Wait for the cpu to choose its way out */
 386         if (!wait_for_completion_timeout(&cpu_death_chosen,
 387                                          msecs_to_jiffies(5000))) {
 388                 pr_err("CPU%u: didn't offline\n", cpu);
 389                 return;
 390         }
 391
 392         /*
 393          * Now wait for the CPU to actually offline. Without doing this that
 394          * offlining may race with one or more of:
 395          *
 396          *   - Onlining the CPU again.
 397          *   - Powering down the core if another VPE within it is offlined.
 398          *   - A sibling VPE entering a non-coherent state.
 399          *
 400          * In the non-MT halt case (ie. infinite loop) the CPU is doing nothing
 401          * with which we could race, so do nothing.
 402          */
 403         if (cpu_death == CPU_DEATH_POWER) {
 404                 /*
 405                  * Wait for the core to enter a powered down or clock gated
 406                  * state, the latter happening when a JTAG probe is connected
 407                  * in which case the CPC will refuse to power down the core.
 408                  */
 409                 do {
 410                         mips_cpc_lock_other(core);
 411                         stat = read_cpc_co_stat_conf();
 412                         stat &= CPC_Cx_STAT_CONF_SEQSTATE_MSK;
 413                         mips_cpc_unlock_other();
 414                 } while (stat != CPC_Cx_STAT_CONF_SEQSTATE_D0 &&
 415                          stat != CPC_Cx_STAT_CONF_SEQSTATE_D2 &&
 416                          stat != CPC_Cx_STAT_CONF_SEQSTATE_U2);
 417
 418                 /* Indicate the core is powered off */
 419                 bitmap_clear(core_power, core, 1);
 420         } else if (cpu_has_mipsmt) {
 421                 /*
 422                  * Have a CPU with access to the offlined CPUs registers wait
 423                  * for its TC to halt.
 424                  */
 425                 err = smp_call_function_single(cpu_death_sibling,
 426                                                wait_for_sibling_halt,
 427                                                (void *)cpu, 1);
 428                 if (err)
 429                         panic("Failed to call remote sibling CPU\n");
 430         }
 431 }
 432
 433 #endif /* CONFIG_HOTPLUG_CPU */
 434
 435 static struct plat_smp_ops cps_smp_ops = {
 436         .smp_setup              = cps_smp_setup,
 437         .prepare_cpus           = cps_prepare_cpus,
 438         .boot_secondary         = cps_boot_secondary,
 439         .init_secondary         = cps_init_secondary,
 440         .smp_finish             = cps_smp_finish,
 441         .send_ipi_single        = gic_send_ipi_single,
 442         .send_ipi_mask          = gic_send_ipi_mask,
 443 #ifdef CONFIG_HOTPLUG_CPU
 444         .cpu_disable            = cps_cpu_disable,
 445         .cpu_die                = cps_cpu_die,
 446 #endif
 447 };
 448
 449 bool mips_cps_smp_in_use(void)
 450 {
 451         extern struct plat_smp_ops *mp_ops;
 452         return mp_ops == &cps_smp_ops;
 453 }
 454
 455 int register_cps_smp_ops(void)
 456 {
 457         if (!mips_cm_present()) {
 458                 pr_warn("MIPS CPS SMP unable to proceed without a CM\n");
 459                 return -ENODEV;
 460         }
 461
 462         /* check we have a GIC - we need one for IPIs */
 463         if (!(read_gcr_gic_status() & CM_GCR_GIC_STATUS_EX_MSK)) {
 464                 pr_warn("MIPS CPS SMP unable to proceed without a GIC\n");
 465                 return -ENODEV;
 466         }
 467
 468         register_smp_ops(&cps_smp_ops);
 469         return 0;
 470 }