arch/riscv/kernel/smp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * SMP initialisation and IPI support
   4  * Based on arch/arm64/kernel/smp.c
   5  *
   6  * Copyright (C) 2012 ARM Ltd.
   7  * Copyright (C) 2015 Regents of the University of California
   8  * Copyright (C) 2017 SiFive
   9  */
  10
  11 #include <linux/cpu.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/module.h>
  14 #include <linux/profile.h>
  15 #include <linux/smp.h>
  16 #include <linux/sched.h>
  17 #include <linux/seq_file.h>
  18 #include <linux/delay.h>
  19 #include <linux/irq_work.h>
  20
  21 #include <asm/sbi.h>
  22 #include <asm/tlbflush.h>
  23 #include <asm/cacheflush.h>
  24
  25 enum ipi_message_type {
  26         IPI_RESCHEDULE,
  27         IPI_CALL_FUNC,
  28         IPI_CPU_STOP,
  29         IPI_IRQ_WORK,
  30         IPI_MAX
  31 };
  32
  33 unsigned long __cpuid_to_hartid_map[NR_CPUS] = {
  34         [0 ... NR_CPUS-1] = INVALID_HARTID
  35 };
  36
  37 void __init smp_setup_processor_id(void)
  38 {
  39         cpuid_to_hartid_map(0) = boot_cpu_hartid;
  40 }
  41
  42 /* A collection of single bit ipi messages.  */
  43 static struct {
  44         unsigned long stats[IPI_MAX] ____cacheline_aligned;
  45         unsigned long bits ____cacheline_aligned;
  46 } ipi_data[NR_CPUS] __cacheline_aligned;
  47
  48 int riscv_hartid_to_cpuid(int hartid)
  49 {
  50         int i;
  51
  52         for (i = 0; i < NR_CPUS; i++)
  53                 if (cpuid_to_hartid_map(i) == hartid)
  54                         return i;
  55
  56         pr_err("Couldn't find cpu id for hartid [%d]\n", hartid);
  57         return i;
  58 }
  59
  60 void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out)
  61 {
  62         int cpu;
  63
  64         cpumask_clear(out);
  65         for_each_cpu(cpu, in)
  66                 cpumask_set_cpu(cpuid_to_hartid_map(cpu), out);
  67 }
  68 EXPORT_SYMBOL_GPL(riscv_cpuid_to_hartid_mask);
  69
  70 bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
  71 {
  72         return phys_id == cpuid_to_hartid_map(cpu);
  73 }
  74
  75 /* Unsupported */
  76 int setup_profiling_timer(unsigned int multiplier)
  77 {
  78         return -EINVAL;
  79 }
  80
  81 static void ipi_stop(void)
  82 {
  83         set_cpu_online(smp_processor_id(), false);
  84         while (1)
  85                 wait_for_interrupt();
  86 }
  87
  88 static struct riscv_ipi_ops *ipi_ops;
  89
  90 void riscv_set_ipi_ops(struct riscv_ipi_ops *ops)
  91 {
  92         ipi_ops = ops;
  93 }
  94 EXPORT_SYMBOL_GPL(riscv_set_ipi_ops);
  95
  96 void riscv_clear_ipi(void)
  97 {
  98         if (ipi_ops && ipi_ops->ipi_clear)
  99                 ipi_ops->ipi_clear();
 100
 101         csr_clear(CSR_IP, IE_SIE);
 102 }
 103 EXPORT_SYMBOL_GPL(riscv_clear_ipi);
 104
 105 static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op)
 106 {
 107         int cpu;
 108
 109         smp_mb__before_atomic();
 110         for_each_cpu(cpu, mask)
 111                 set_bit(op, &ipi_data[cpu].bits);
 112         smp_mb__after_atomic();
 113
 114         if (ipi_ops && ipi_ops->ipi_inject)
 115                 ipi_ops->ipi_inject(mask);
 116         else
 117                 pr_warn("SMP: IPI inject method not available\n");
 118 }
 119
 120 static void send_ipi_single(int cpu, enum ipi_message_type op)
 121 {
 122         smp_mb__before_atomic();
 123         set_bit(op, &ipi_data[cpu].bits);
 124         smp_mb__after_atomic();
 125
 126         if (ipi_ops && ipi_ops->ipi_inject)
 127                 ipi_ops->ipi_inject(cpumask_of(cpu));
 128         else
 129                 pr_warn("SMP: IPI inject method not available\n");
 130 }
 131
 132 #ifdef CONFIG_IRQ_WORK
 133 void arch_irq_work_raise(void)
 134 {
 135         send_ipi_single(smp_processor_id(), IPI_IRQ_WORK);
 136 }
 137 #endif
 138
 139 void handle_IPI(struct pt_regs *regs)
 140 {
 141         struct pt_regs *old_regs = set_irq_regs(regs);
 142         unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits;
 143         unsigned long *stats = ipi_data[smp_processor_id()].stats;
 144
 145         irq_enter();
 146
 147         riscv_clear_ipi();
 148
 149         while (true) {
 150                 unsigned long ops;
 151
 152                 /* Order bit clearing and data access. */
 153                 mb();
 154
 155                 ops = xchg(pending_ipis, 0);
 156                 if (ops == 0)
 157                         goto done;
 158
 159                 if (ops & (1 << IPI_RESCHEDULE)) {
 160                         stats[IPI_RESCHEDULE]++;
 161                         scheduler_ipi();
 162                 }
 163
 164                 if (ops & (1 << IPI_CALL_FUNC)) {
 165                         stats[IPI_CALL_FUNC]++;
 166                         generic_smp_call_function_interrupt();
 167                 }
 168
 169                 if (ops & (1 << IPI_CPU_STOP)) {
 170                         stats[IPI_CPU_STOP]++;
 171                         ipi_stop();
 172                 }
 173
 174                 if (ops & (1 << IPI_IRQ_WORK)) {
 175                         stats[IPI_IRQ_WORK]++;
 176                         irq_work_run();
 177                 }
 178
 179                 BUG_ON((ops >> IPI_MAX) != 0);
 180
 181                 /* Order data access and bit testing. */
 182                 mb();
 183         }
 184
 185 done:
 186         irq_exit();
 187         set_irq_regs(old_regs);
 188 }
 189
 190 static const char * const ipi_names[] = {
 191         [IPI_RESCHEDULE]        = "Rescheduling interrupts",
 192         [IPI_CALL_FUNC]         = "Function call interrupts",
 193         [IPI_CPU_STOP]          = "CPU stop interrupts",
 194         [IPI_IRQ_WORK]          = "IRQ work interrupts",
 195 };
 196
 197 void show_ipi_stats(struct seq_file *p, int prec)
 198 {
 199         unsigned int cpu, i;
 200
 201         for (i = 0; i < IPI_MAX; i++) {
 202                 seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
 203                            prec >= 4 ? " " : "");
 204                 for_each_online_cpu(cpu)
 205                         seq_printf(p, "%10lu ", ipi_data[cpu].stats[i]);
 206                 seq_printf(p, " %s\n", ipi_names[i]);
 207         }
 208 }
 209
 210 void arch_send_call_function_ipi_mask(struct cpumask *mask)
 211 {
 212         send_ipi_mask(mask, IPI_CALL_FUNC);
 213 }
 214
 215 void arch_send_call_function_single_ipi(int cpu)
 216 {
 217         send_ipi_single(cpu, IPI_CALL_FUNC);
 218 }
 219
 220 void smp_send_stop(void)
 221 {
 222         unsigned long timeout;
 223
 224         if (num_online_cpus() > 1) {
 225                 cpumask_t mask;
 226
 227                 cpumask_copy(&mask, cpu_online_mask);
 228                 cpumask_clear_cpu(smp_processor_id(), &mask);
 229
 230                 if (system_state <= SYSTEM_RUNNING)
 231                         pr_crit("SMP: stopping secondary CPUs\n");
 232                 send_ipi_mask(&mask, IPI_CPU_STOP);
 233         }
 234
 235         /* Wait up to one second for other CPUs to stop */
 236         timeout = USEC_PER_SEC;
 237         while (num_online_cpus() > 1 && timeout--)
 238                 udelay(1);
 239
 240         if (num_online_cpus() > 1)
 241                 pr_warn("SMP: failed to stop secondary CPUs %*pbl\n",
 242                            cpumask_pr_args(cpu_online_mask));
 243 }
 244
 245 void smp_send_reschedule(int cpu)
 246 {
 247         send_ipi_single(cpu, IPI_RESCHEDULE);
 248 }
 249 EXPORT_SYMBOL_GPL(smp_send_reschedule);