arch/x86/lib/msr-smp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/export.h>
   3 #include <linux/preempt.h>
   4 #include <linux/smp.h>
   5 #include <linux/completion.h>
   6 #include <asm/msr.h>
   7
   8 static void __rdmsr_on_cpu(void *info)
   9 {
  10         struct msr_info *rv = info;
  11         struct msr *reg;
  12         int this_cpu = raw_smp_processor_id();
  13
  14         if (rv->msrs)
  15                 reg = per_cpu_ptr(rv->msrs, this_cpu);
  16         else
  17                 reg = &rv->reg;
  18
  19         rdmsr(rv->msr_no, reg->l, reg->h);
  20 }
  21
  22 static void __wrmsr_on_cpu(void *info)
  23 {
  24         struct msr_info *rv = info;
  25         struct msr *reg;
  26         int this_cpu = raw_smp_processor_id();
  27
  28         if (rv->msrs)
  29                 reg = per_cpu_ptr(rv->msrs, this_cpu);
  30         else
  31                 reg = &rv->reg;
  32
  33         wrmsr(rv->msr_no, reg->l, reg->h);
  34 }
  35
  36 int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
  37 {
  38         int err;
  39         struct msr_info rv;
  40
  41         memset(&rv, 0, sizeof(rv));
  42
  43         rv.msr_no = msr_no;
  44         err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
  45         *l = rv.reg.l;
  46         *h = rv.reg.h;
  47
  48         return err;
  49 }
  50 EXPORT_SYMBOL(rdmsr_on_cpu);
  51
  52 int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
  53 {
  54         int err;
  55         struct msr_info rv;
  56
  57         memset(&rv, 0, sizeof(rv));
  58
  59         rv.msr_no = msr_no;
  60         err = smp_call_function_single(cpu, __rdmsr_on_cpu, &rv, 1);
  61         *q = rv.reg.q;
  62
  63         return err;
  64 }
  65 EXPORT_SYMBOL(rdmsrl_on_cpu);
  66
  67 int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
  68 {
  69         int err;
  70         struct msr_info rv;
  71
  72         memset(&rv, 0, sizeof(rv));
  73
  74         rv.msr_no = msr_no;
  75         rv.reg.l = l;
  76         rv.reg.h = h;
  77         err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
  78
  79         return err;
  80 }
  81 EXPORT_SYMBOL(wrmsr_on_cpu);
  82
  83 int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
  84 {
  85         int err;
  86         struct msr_info rv;
  87
  88         memset(&rv, 0, sizeof(rv));
  89
  90         rv.msr_no = msr_no;
  91         rv.reg.q = q;
  92
  93         err = smp_call_function_single(cpu, __wrmsr_on_cpu, &rv, 1);
  94
  95         return err;
  96 }
  97 EXPORT_SYMBOL(wrmsrl_on_cpu);
  98
  99 static void __rwmsr_on_cpus(const struct cpumask *mask, u32 msr_no,
 100                             struct msr *msrs,
 101                             void (*msr_func) (void *info))
 102 {
 103         struct msr_info rv;
 104         int this_cpu;
 105
 106         memset(&rv, 0, sizeof(rv));
 107
 108         rv.msrs   = msrs;
 109         rv.msr_no = msr_no;
 110
 111         this_cpu = get_cpu();
 112
 113         if (cpumask_test_cpu(this_cpu, mask))
 114                 msr_func(&rv);
 115
 116         smp_call_function_many(mask, msr_func, &rv, 1);
 117         put_cpu();
 118 }
 119
 120 /* rdmsr on a bunch of CPUs
 121  *
 122  * @mask:       which CPUs
 123  * @msr_no:     which MSR
 124  * @msrs:       array of MSR values
 125  *
 126  */
 127 void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs)
 128 {
 129         __rwmsr_on_cpus(mask, msr_no, msrs, __rdmsr_on_cpu);
 130 }
 131 EXPORT_SYMBOL(rdmsr_on_cpus);
 132
 133 /*
 134  * wrmsr on a bunch of CPUs
 135  *
 136  * @mask:       which CPUs
 137  * @msr_no:     which MSR
 138  * @msrs:       array of MSR values
 139  *
 140  */
 141 void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs)
 142 {
 143         __rwmsr_on_cpus(mask, msr_no, msrs, __wrmsr_on_cpu);
 144 }
 145 EXPORT_SYMBOL(wrmsr_on_cpus);
 146
 147 struct msr_info_completion {
 148         struct msr_info         msr;
 149         struct completion       done;
 150 };
 151
 152 /* These "safe" variants are slower and should be used when the target MSR
 153    may not actually exist. */
 154 static void __rdmsr_safe_on_cpu(void *info)
 155 {
 156         struct msr_info_completion *rv = info;
 157
 158         rv->msr.err = rdmsr_safe(rv->msr.msr_no, &rv->msr.reg.l, &rv->msr.reg.h);
 159         complete(&rv->done);
 160 }
 161
 162 static void __wrmsr_safe_on_cpu(void *info)
 163 {
 164         struct msr_info *rv = info;
 165
 166         rv->err = wrmsr_safe(rv->msr_no, rv->reg.l, rv->reg.h);
 167 }
 168
 169 int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
 170 {
 171         struct msr_info_completion rv;
 172         call_single_data_t csd;
 173         int err;
 174
 175         INIT_CSD(&csd, __rdmsr_safe_on_cpu, &rv);
 176
 177         memset(&rv, 0, sizeof(rv));
 178         init_completion(&rv.done);
 179         rv.msr.msr_no = msr_no;
 180
 181         err = smp_call_function_single_async(cpu, &csd);
 182         if (!err) {
 183                 wait_for_completion(&rv.done);
 184                 err = rv.msr.err;
 185         }
 186         *l = rv.msr.reg.l;
 187         *h = rv.msr.reg.h;
 188
 189         return err;
 190 }
 191 EXPORT_SYMBOL(rdmsr_safe_on_cpu);
 192
 193 int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
 194 {
 195         int err;
 196         struct msr_info rv;
 197
 198         memset(&rv, 0, sizeof(rv));
 199
 200         rv.msr_no = msr_no;
 201         rv.reg.l = l;
 202         rv.reg.h = h;
 203         err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
 204
 205         return err ? err : rv.err;
 206 }
 207 EXPORT_SYMBOL(wrmsr_safe_on_cpu);
 208
 209 int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
 210 {
 211         int err;
 212         struct msr_info rv;
 213
 214         memset(&rv, 0, sizeof(rv));
 215
 216         rv.msr_no = msr_no;
 217         rv.reg.q = q;
 218
 219         err = smp_call_function_single(cpu, __wrmsr_safe_on_cpu, &rv, 1);
 220
 221         return err ? err : rv.err;
 222 }
 223 EXPORT_SYMBOL(wrmsrl_safe_on_cpu);
 224
 225 int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
 226 {
 227         u32 low, high;
 228         int err;
 229
 230         err = rdmsr_safe_on_cpu(cpu, msr_no, &low, &high);
 231         *q = (u64)high << 32 | low;
 232
 233         return err;
 234 }
 235 EXPORT_SYMBOL(rdmsrl_safe_on_cpu);
 236
 237 /*
 238  * These variants are significantly slower, but allows control over
 239  * the entire 32-bit GPR set.
 240  */
 241 static void __rdmsr_safe_regs_on_cpu(void *info)
 242 {
 243         struct msr_regs_info *rv = info;
 244
 245         rv->err = rdmsr_safe_regs(rv->regs);
 246 }
 247
 248 static void __wrmsr_safe_regs_on_cpu(void *info)
 249 {
 250         struct msr_regs_info *rv = info;
 251
 252         rv->err = wrmsr_safe_regs(rv->regs);
 253 }
 254
 255 int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs)
 256 {
 257         int err;
 258         struct msr_regs_info rv;
 259
 260         rv.regs   = regs;
 261         rv.err    = -EIO;
 262         err = smp_call_function_single(cpu, __rdmsr_safe_regs_on_cpu, &rv, 1);
 263
 264         return err ? err : rv.err;
 265 }
 266 EXPORT_SYMBOL(rdmsr_safe_regs_on_cpu);
 267
 268 int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 *regs)
 269 {
 270         int err;
 271         struct msr_regs_info rv;
 272
 273         rv.regs = regs;
 274         rv.err  = -EIO;
 275         err = smp_call_function_single(cpu, __wrmsr_safe_regs_on_cpu, &rv, 1);
 276
 277         return err ? err : rv.err;
 278 }
 279 EXPORT_SYMBOL(wrmsr_safe_regs_on_cpu);