arch/xtensa/kernel/smp.c

   1 /*
   2  * Xtensa SMP support functions.
   3  *
   4  * This file is subject to the terms and conditions of the GNU General Public
   5  * License.  See the file "COPYING" in the main directory of this archive
   6  * for more details.
   7  *
   8  * Copyright (C) 2008 - 2013 Tensilica Inc.
   9  *
  10  * Chris Zankel <chris@zankel.net>
  11  * Joe Taylor <joe@tensilica.com>
  12  * Pete Delaney <piet@tensilica.com
  13  */
  14
  15 #include <linux/cpu.h>
  16 #include <linux/cpumask.h>
  17 #include <linux/delay.h>
  18 #include <linux/init.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/irqdomain.h>
  21 #include <linux/irq.h>
  22 #include <linux/kdebug.h>
  23 #include <linux/module.h>
  24 #include <linux/sched/mm.h>
  25 #include <linux/sched/hotplug.h>
  26 #include <linux/sched/task_stack.h>
  27 #include <linux/reboot.h>
  28 #include <linux/seq_file.h>
  29 #include <linux/smp.h>
  30 #include <linux/thread_info.h>
  31
  32 #include <asm/cacheflush.h>
  33 #include <asm/kdebug.h>
  34 #include <asm/mmu_context.h>
  35 #include <asm/mxregs.h>
  36 #include <asm/platform.h>
  37 #include <asm/tlbflush.h>
  38 #include <asm/traps.h>
  39
  40 #ifdef CONFIG_SMP
  41 # if XCHAL_HAVE_S32C1I == 0
  42 #  error "The S32C1I option is required for SMP."
  43 # endif
  44 #endif
  45
  46 static void system_invalidate_dcache_range(unsigned long start,
  47                 unsigned long size);
  48 static void system_flush_invalidate_dcache_range(unsigned long start,
  49                 unsigned long size);
  50
  51 /* IPI (Inter Process Interrupt) */
  52
  53 #define IPI_IRQ 0
  54
  55 static irqreturn_t ipi_interrupt(int irq, void *dev_id);
  56 static struct irqaction ipi_irqaction = {
  57         .handler =      ipi_interrupt,
  58         .flags =        IRQF_PERCPU,
  59         .name =         "ipi",
  60 };
  61
  62 void ipi_init(void)
  63 {
  64         unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
  65         setup_irq(irq, &ipi_irqaction);
  66 }
  67
  68 static inline unsigned int get_core_count(void)
  69 {
  70         /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
  71         unsigned int syscfgid = get_er(SYSCFGID);
  72         return ((syscfgid >> 18) & 0xf) + 1;
  73 }
  74
  75 static inline int get_core_id(void)
  76 {
  77         /* Bits 0...18 of SYSCFGID contain the core id  */
  78         unsigned int core_id = get_er(SYSCFGID);
  79         return core_id & 0x3fff;
  80 }
  81
  82 void __init smp_prepare_cpus(unsigned int max_cpus)
  83 {
  84         unsigned i;
  85
  86         for (i = 0; i < max_cpus; ++i)
  87                 set_cpu_present(i, true);
  88 }
  89
  90 void __init smp_init_cpus(void)
  91 {
  92         unsigned i;
  93         unsigned int ncpus = get_core_count();
  94         unsigned int core_id = get_core_id();
  95
  96         pr_info("%s: Core Count = %d\n", __func__, ncpus);
  97         pr_info("%s: Core Id = %d\n", __func__, core_id);
  98
  99         for (i = 0; i < ncpus; ++i)
 100                 set_cpu_possible(i, true);
 101 }
 102
 103 void __init smp_prepare_boot_cpu(void)
 104 {
 105         unsigned int cpu = smp_processor_id();
 106         BUG_ON(cpu != 0);
 107         cpu_asid_cache(cpu) = ASID_USER_FIRST;
 108 }
 109
 110 void __init smp_cpus_done(unsigned int max_cpus)
 111 {
 112 }
 113
 114 static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
 115 static DECLARE_COMPLETION(cpu_running);
 116
 117 void secondary_start_kernel(void)
 118 {
 119         struct mm_struct *mm = &init_mm;
 120         unsigned int cpu = smp_processor_id();
 121
 122         init_mmu();
 123
 124 #ifdef CONFIG_DEBUG_KERNEL
 125         if (boot_secondary_processors == 0) {
 126                 pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
 127                         __func__, boot_secondary_processors, cpu);
 128                 for (;;)
 129                         __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
 130         }
 131
 132         pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
 133                 __func__, boot_secondary_processors, cpu);
 134 #endif
 135         /* Init EXCSAVE1 */
 136
 137         secondary_trap_init();
 138
 139         /* All kernel threads share the same mm context. */
 140
 141         mmget(mm);
 142         mmgrab(mm);
 143         current->active_mm = mm;
 144         cpumask_set_cpu(cpu, mm_cpumask(mm));
 145         enter_lazy_tlb(mm, current);
 146
 147         preempt_disable();
 148         trace_hardirqs_off();
 149
 150         calibrate_delay();
 151
 152         notify_cpu_starting(cpu);
 153
 154         secondary_init_irq();
 155         local_timer_setup(cpu);
 156
 157         set_cpu_online(cpu, true);
 158
 159         local_irq_enable();
 160
 161         complete(&cpu_running);
 162
 163         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 164 }
 165
 166 static void mx_cpu_start(void *p)
 167 {
 168         unsigned cpu = (unsigned)p;
 169         unsigned long run_stall_mask = get_er(MPSCORE);
 170
 171         set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
 172         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 173                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 174 }
 175
 176 static void mx_cpu_stop(void *p)
 177 {
 178         unsigned cpu = (unsigned)p;
 179         unsigned long run_stall_mask = get_er(MPSCORE);
 180
 181         set_er(run_stall_mask | (1u << cpu), MPSCORE);
 182         pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
 183                         __func__, cpu, run_stall_mask, get_er(MPSCORE));
 184 }
 185
 186 #ifdef CONFIG_HOTPLUG_CPU
 187 unsigned long cpu_start_id __cacheline_aligned;
 188 #endif
 189 unsigned long cpu_start_ccount;
 190
 191 static int boot_secondary(unsigned int cpu, struct task_struct *ts)
 192 {
 193         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 194         unsigned long ccount;
 195         int i;
 196
 197 #ifdef CONFIG_HOTPLUG_CPU
 198         cpu_start_id = cpu;
 199         system_flush_invalidate_dcache_range(
 200                         (unsigned long)&cpu_start_id, sizeof(cpu_start_id));
 201 #endif
 202         smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
 203
 204         for (i = 0; i < 2; ++i) {
 205                 do
 206                         ccount = get_ccount();
 207                 while (!ccount);
 208
 209                 cpu_start_ccount = ccount;
 210
 211                 while (time_before(jiffies, timeout)) {
 212                         mb();
 213                         if (!cpu_start_ccount)
 214                                 break;
 215                 }
 216
 217                 if (cpu_start_ccount) {
 218                         smp_call_function_single(0, mx_cpu_stop,
 219                                         (void *)cpu, 1);
 220                         cpu_start_ccount = 0;
 221                         return -EIO;
 222                 }
 223         }
 224         return 0;
 225 }
 226
 227 int __cpu_up(unsigned int cpu, struct task_struct *idle)
 228 {
 229         int ret = 0;
 230
 231         if (cpu_asid_cache(cpu) == 0)
 232                 cpu_asid_cache(cpu) = ASID_USER_FIRST;
 233
 234         start_info.stack = (unsigned long)task_pt_regs(idle);
 235         wmb();
 236
 237         pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
 238                         __func__, cpu, idle, start_info.stack);
 239
 240         ret = boot_secondary(cpu, idle);
 241         if (ret == 0) {
 242                 wait_for_completion_timeout(&cpu_running,
 243                                 msecs_to_jiffies(1000));
 244                 if (!cpu_online(cpu))
 245                         ret = -EIO;
 246         }
 247
 248         if (ret)
 249                 pr_err("CPU %u failed to boot\n", cpu);
 250
 251         return ret;
 252 }
 253
 254 #ifdef CONFIG_HOTPLUG_CPU
 255
 256 /*
 257  * __cpu_disable runs on the processor to be shutdown.
 258  */
 259 int __cpu_disable(void)
 260 {
 261         unsigned int cpu = smp_processor_id();
 262
 263         /*
 264          * Take this CPU offline.  Once we clear this, we can't return,
 265          * and we must not schedule until we're ready to give up the cpu.
 266          */
 267         set_cpu_online(cpu, false);
 268
 269         /*
 270          * OK - migrate IRQs away from this CPU
 271          */
 272         migrate_irqs();
 273
 274         /*
 275          * Flush user cache and TLB mappings, and then remove this CPU
 276          * from the vm mask set of all processes.
 277          */
 278         local_flush_cache_all();
 279         local_flush_tlb_all();
 280         invalidate_page_directory();
 281
 282         clear_tasks_mm_cpumask(cpu);
 283
 284         return 0;
 285 }
 286
 287 static void platform_cpu_kill(unsigned int cpu)
 288 {
 289         smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
 290 }
 291
 292 /*
 293  * called on the thread which is asking for a CPU to be shutdown -
 294  * waits until shutdown has completed, or it is timed out.
 295  */
 296 void __cpu_die(unsigned int cpu)
 297 {
 298         unsigned long timeout = jiffies + msecs_to_jiffies(1000);
 299         while (time_before(jiffies, timeout)) {
 300                 system_invalidate_dcache_range((unsigned long)&cpu_start_id,
 301                                 sizeof(cpu_start_id));
 302                 if (cpu_start_id == -cpu) {
 303                         platform_cpu_kill(cpu);
 304                         return;
 305                 }
 306         }
 307         pr_err("CPU%u: unable to kill\n", cpu);
 308 }
 309
 310 void arch_cpu_idle_dead(void)
 311 {
 312         cpu_die();
 313 }
 314 /*
 315  * Called from the idle thread for the CPU which has been shutdown.
 316  *
 317  * Note that we disable IRQs here, but do not re-enable them
 318  * before returning to the caller. This is also the behaviour
 319  * of the other hotplug-cpu capable cores, so presumably coming
 320  * out of idle fixes this.
 321  */
 322 void __ref cpu_die(void)
 323 {
 324         idle_task_exit();
 325         local_irq_disable();
 326         __asm__ __volatile__(
 327                         "       movi    a2, cpu_restart\n"
 328                         "       jx      a2\n");
 329 }
 330
 331 #endif /* CONFIG_HOTPLUG_CPU */
 332
 333 enum ipi_msg_type {
 334         IPI_RESCHEDULE = 0,
 335         IPI_CALL_FUNC,
 336         IPI_CPU_STOP,
 337         IPI_MAX
 338 };
 339
 340 static const struct {
 341         const char *short_text;
 342         const char *long_text;
 343 } ipi_text[] = {
 344         { .short_text = "RES", .long_text = "Rescheduling interrupts" },
 345         { .short_text = "CAL", .long_text = "Function call interrupts" },
 346         { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
 347 };
 348
 349 struct ipi_data {
 350         unsigned long ipi_count[IPI_MAX];
 351 };
 352
 353 static DEFINE_PER_CPU(struct ipi_data, ipi_data);
 354
 355 static void send_ipi_message(const struct cpumask *callmask,
 356                 enum ipi_msg_type msg_id)
 357 {
 358         int index;
 359         unsigned long mask = 0;
 360
 361         for_each_cpu(index, callmask)
 362                 if (index != smp_processor_id())
 363                         mask |= 1 << index;
 364
 365         set_er(mask, MIPISET(msg_id));
 366 }
 367
 368 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 369 {
 370         send_ipi_message(mask, IPI_CALL_FUNC);
 371 }
 372
 373 void arch_send_call_function_single_ipi(int cpu)
 374 {
 375         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
 376 }
 377
 378 void smp_send_reschedule(int cpu)
 379 {
 380         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
 381 }
 382
 383 void smp_send_stop(void)
 384 {
 385         struct cpumask targets;
 386
 387         cpumask_copy(&targets, cpu_online_mask);
 388         cpumask_clear_cpu(smp_processor_id(), &targets);
 389         send_ipi_message(&targets, IPI_CPU_STOP);
 390 }
 391
 392 static void ipi_cpu_stop(unsigned int cpu)
 393 {
 394         set_cpu_online(cpu, false);
 395         machine_halt();
 396 }
 397
 398 irqreturn_t ipi_interrupt(int irq, void *dev_id)
 399 {
 400         unsigned int cpu = smp_processor_id();
 401         struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
 402         unsigned int msg;
 403         unsigned i;
 404
 405         msg = get_er(MIPICAUSE(cpu));
 406         for (i = 0; i < IPI_MAX; i++)
 407                 if (msg & (1 << i)) {
 408                         set_er(1 << i, MIPICAUSE(cpu));
 409                         ++ipi->ipi_count[i];
 410                 }
 411
 412         if (msg & (1 << IPI_RESCHEDULE))
 413                 scheduler_ipi();
 414         if (msg & (1 << IPI_CALL_FUNC))
 415                 generic_smp_call_function_interrupt();
 416         if (msg & (1 << IPI_CPU_STOP))
 417                 ipi_cpu_stop(cpu);
 418
 419         return IRQ_HANDLED;
 420 }
 421
 422 void show_ipi_list(struct seq_file *p, int prec)
 423 {
 424         unsigned int cpu;
 425         unsigned i;
 426
 427         for (i = 0; i < IPI_MAX; ++i) {
 428                 seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
 429                 for_each_online_cpu(cpu)
 430                         seq_printf(p, " %10lu",
 431                                         per_cpu(ipi_data, cpu).ipi_count[i]);
 432                 seq_printf(p, "   %s\n", ipi_text[i].long_text);
 433         }
 434 }
 435
 436 int setup_profiling_timer(unsigned int multiplier)
 437 {
 438         pr_debug("setup_profiling_timer %d\n", multiplier);
 439         return 0;
 440 }
 441
 442 /* TLB flush functions */
 443
 444 struct flush_data {
 445         struct vm_area_struct *vma;
 446         unsigned long addr1;
 447         unsigned long addr2;
 448 };
 449
 450 static void ipi_flush_tlb_all(void *arg)
 451 {
 452         local_flush_tlb_all();
 453 }
 454
 455 void flush_tlb_all(void)
 456 {
 457         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 458 }
 459
 460 static void ipi_flush_tlb_mm(void *arg)
 461 {
 462         local_flush_tlb_mm(arg);
 463 }
 464
 465 void flush_tlb_mm(struct mm_struct *mm)
 466 {
 467         on_each_cpu(ipi_flush_tlb_mm, mm, 1);
 468 }
 469
 470 static void ipi_flush_tlb_page(void *arg)
 471 {
 472         struct flush_data *fd = arg;
 473         local_flush_tlb_page(fd->vma, fd->addr1);
 474 }
 475
 476 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 477 {
 478         struct flush_data fd = {
 479                 .vma = vma,
 480                 .addr1 = addr,
 481         };
 482         on_each_cpu(ipi_flush_tlb_page, &fd, 1);
 483 }
 484
 485 static void ipi_flush_tlb_range(void *arg)
 486 {
 487         struct flush_data *fd = arg;
 488         local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
 489 }
 490
 491 void flush_tlb_range(struct vm_area_struct *vma,
 492                      unsigned long start, unsigned long end)
 493 {
 494         struct flush_data fd = {
 495                 .vma = vma,
 496                 .addr1 = start,
 497                 .addr2 = end,
 498         };
 499         on_each_cpu(ipi_flush_tlb_range, &fd, 1);
 500 }
 501
 502 static void ipi_flush_tlb_kernel_range(void *arg)
 503 {
 504         struct flush_data *fd = arg;
 505         local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
 506 }
 507
 508 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
 509 {
 510         struct flush_data fd = {
 511                 .addr1 = start,
 512                 .addr2 = end,
 513         };
 514         on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
 515 }
 516
 517 /* Cache flush functions */
 518
 519 static void ipi_flush_cache_all(void *arg)
 520 {
 521         local_flush_cache_all();
 522 }
 523
 524 void flush_cache_all(void)
 525 {
 526         on_each_cpu(ipi_flush_cache_all, NULL, 1);
 527 }
 528
 529 static void ipi_flush_cache_page(void *arg)
 530 {
 531         struct flush_data *fd = arg;
 532         local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
 533 }
 534
 535 void flush_cache_page(struct vm_area_struct *vma,
 536                      unsigned long address, unsigned long pfn)
 537 {
 538         struct flush_data fd = {
 539                 .vma = vma,
 540                 .addr1 = address,
 541                 .addr2 = pfn,
 542         };
 543         on_each_cpu(ipi_flush_cache_page, &fd, 1);
 544 }
 545
 546 static void ipi_flush_cache_range(void *arg)
 547 {
 548         struct flush_data *fd = arg;
 549         local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
 550 }
 551
 552 void flush_cache_range(struct vm_area_struct *vma,
 553                      unsigned long start, unsigned long end)
 554 {
 555         struct flush_data fd = {
 556                 .vma = vma,
 557                 .addr1 = start,
 558                 .addr2 = end,
 559         };
 560         on_each_cpu(ipi_flush_cache_range, &fd, 1);
 561 }
 562
 563 static void ipi_flush_icache_range(void *arg)
 564 {
 565         struct flush_data *fd = arg;
 566         local_flush_icache_range(fd->addr1, fd->addr2);
 567 }
 568
 569 void flush_icache_range(unsigned long start, unsigned long end)
 570 {
 571         struct flush_data fd = {
 572                 .addr1 = start,
 573                 .addr2 = end,
 574         };
 575         on_each_cpu(ipi_flush_icache_range, &fd, 1);
 576 }
 577 EXPORT_SYMBOL(flush_icache_range);
 578
 579 /* ------------------------------------------------------------------------- */
 580
 581 static void ipi_invalidate_dcache_range(void *arg)
 582 {
 583         struct flush_data *fd = arg;
 584         __invalidate_dcache_range(fd->addr1, fd->addr2);
 585 }
 586
 587 static void system_invalidate_dcache_range(unsigned long start,
 588                 unsigned long size)
 589 {
 590         struct flush_data fd = {
 591                 .addr1 = start,
 592                 .addr2 = size,
 593         };
 594         on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
 595 }
 596
 597 static void ipi_flush_invalidate_dcache_range(void *arg)
 598 {
 599         struct flush_data *fd = arg;
 600         __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
 601 }
 602
 603 static void system_flush_invalidate_dcache_range(unsigned long start,
 604                 unsigned long size)
 605 {
 606         struct flush_data fd = {
 607                 .addr1 = start,
 608                 .addr2 = size,
 609         };
 610         on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
 611 }