arch/riscv/kernel/paravirt.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2023 Ventana Micro Systems Inc.
   4  */
   5
   6 #define pr_fmt(fmt) "riscv-pv: " fmt
   7
   8 #include <linux/cpuhotplug.h>
   9 #include <linux/compiler.h>
  10 #include <linux/errno.h>
  11 #include <linux/init.h>
  12 #include <linux/jump_label.h>
  13 #include <linux/kconfig.h>
  14 #include <linux/kernel.h>
  15 #include <linux/percpu-defs.h>
  16 #include <linux/printk.h>
  17 #include <linux/static_call.h>
  18 #include <linux/types.h>
  19
  20 #include <asm/barrier.h>
  21 #include <asm/page.h>
  22 #include <asm/paravirt.h>
  23 #include <asm/sbi.h>
  24
  25 struct static_key paravirt_steal_enabled;
  26 struct static_key paravirt_steal_rq_enabled;
  27
  28 static u64 native_steal_clock(int cpu)
  29 {
  30         return 0;
  31 }
  32
  33 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
  34
  35 static bool steal_acc = true;
  36 static int __init parse_no_stealacc(char *arg)
  37 {
  38         steal_acc = false;
  39         return 0;
  40 }
  41
  42 early_param("no-steal-acc", parse_no_stealacc);
  43
  44 static DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64);
  45
  46 static bool __init has_pv_steal_clock(void)
  47 {
  48         if (sbi_spec_version >= sbi_mk_version(2, 0) &&
  49             sbi_probe_extension(SBI_EXT_STA) > 0) {
  50                 pr_info("SBI STA extension detected\n");
  51                 return true;
  52         }
  53
  54         return false;
  55 }
  56
  57 static int sbi_sta_steal_time_set_shmem(unsigned long lo, unsigned long hi,
  58                                         unsigned long flags)
  59 {
  60         struct sbiret ret;
  61
  62         ret = sbi_ecall(SBI_EXT_STA, SBI_EXT_STA_STEAL_TIME_SET_SHMEM,
  63                         lo, hi, flags, 0, 0, 0);
  64         if (ret.error) {
  65                 if (lo == SBI_SHMEM_DISABLE && hi == SBI_SHMEM_DISABLE)
  66                         pr_warn("Failed to disable steal-time shmem");
  67                 else
  68                         pr_warn("Failed to set steal-time shmem");
  69                 return sbi_err_map_linux_errno(ret.error);
  70         }
  71
  72         return 0;
  73 }
  74
  75 static int pv_time_cpu_online(unsigned int cpu)
  76 {
  77         struct sbi_sta_struct *st = this_cpu_ptr(&steal_time);
  78         phys_addr_t pa = __pa(st);
  79         unsigned long lo = (unsigned long)pa;
  80         unsigned long hi = IS_ENABLED(CONFIG_32BIT) ? upper_32_bits((u64)pa) : 0;
  81
  82         return sbi_sta_steal_time_set_shmem(lo, hi, 0);
  83 }
  84
  85 static int pv_time_cpu_down_prepare(unsigned int cpu)
  86 {
  87         return sbi_sta_steal_time_set_shmem(SBI_SHMEM_DISABLE,
  88                                             SBI_SHMEM_DISABLE, 0);
  89 }
  90
  91 static u64 pv_time_steal_clock(int cpu)
  92 {
  93         struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu);
  94         __le32 sequence;
  95         __le64 steal;
  96
  97         /*
  98          * Check the sequence field before and after reading the steal
  99          * field. Repeat the read if it is different or odd.
 100          */
 101         do {
 102                 sequence = READ_ONCE(st->sequence);
 103                 virt_rmb();
 104                 steal = READ_ONCE(st->steal);
 105                 virt_rmb();
 106         } while ((le32_to_cpu(sequence) & 1) ||
 107                  sequence != READ_ONCE(st->sequence));
 108
 109         return le64_to_cpu(steal);
 110 }
 111
 112 int __init pv_time_init(void)
 113 {
 114         int ret;
 115
 116         if (!has_pv_steal_clock())
 117                 return 0;
 118
 119         ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
 120                                 "riscv/pv_time:online",
 121                                 pv_time_cpu_online,
 122                                 pv_time_cpu_down_prepare);
 123         if (ret < 0)
 124                 return ret;
 125
 126         static_call_update(pv_steal_clock, pv_time_steal_clock);
 127
 128         static_key_slow_inc(&paravirt_steal_enabled);
 129         if (steal_acc)
 130                 static_key_slow_inc(&paravirt_steal_rq_enabled);
 131
 132         pr_info("Computing paravirt steal-time\n");
 133
 134         return 0;
 135 }