arch/riscv/kernel/kernel_mode_vector.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2012 ARM Ltd.
   4  * Author: Catalin Marinas <catalin.marinas@arm.com>
   5  * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
   6  * Copyright (C) 2021 SiFive
   7  */
   8 #include <linux/compiler.h>
   9 #include <linux/irqflags.h>
  10 #include <linux/percpu.h>
  11 #include <linux/preempt.h>
  12 #include <linux/types.h>
  13
  14 #include <asm/vector.h>
  15 #include <asm/switch_to.h>
  16 #include <asm/simd.h>
  17 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
  18 #include <asm/asm-prototypes.h>
  19 #endif
  20
  21 static inline void riscv_v_flags_set(u32 flags)
  22 {
  23         WRITE_ONCE(current->thread.riscv_v_flags, flags);
  24 }
  25
  26 static inline void riscv_v_start(u32 flags)
  27 {
  28         int orig;
  29
  30         orig = riscv_v_flags();
  31         BUG_ON((orig & flags) != 0);
  32         riscv_v_flags_set(orig | flags);
  33         barrier();
  34 }
  35
  36 static inline void riscv_v_stop(u32 flags)
  37 {
  38         int orig;
  39
  40         barrier();
  41         orig = riscv_v_flags();
  42         BUG_ON((orig & flags) == 0);
  43         riscv_v_flags_set(orig & ~flags);
  44 }
  45
  46 /*
  47  * Claim ownership of the CPU vector context for use by the calling context.
  48  *
  49  * The caller may freely manipulate the vector context metadata until
  50  * put_cpu_vector_context() is called.
  51  */
  52 void get_cpu_vector_context(void)
  53 {
  54         /*
  55          * disable softirqs so it is impossible for softirqs to nest
  56          * get_cpu_vector_context() when kernel is actively using Vector.
  57          */
  58         if (!IS_ENABLED(CONFIG_PREEMPT_RT))
  59                 local_bh_disable();
  60         else
  61                 preempt_disable();
  62
  63         riscv_v_start(RISCV_KERNEL_MODE_V);
  64 }
  65
  66 /*
  67  * Release the CPU vector context.
  68  *
  69  * Must be called from a context in which get_cpu_vector_context() was
  70  * previously called, with no call to put_cpu_vector_context() in the
  71  * meantime.
  72  */
  73 void put_cpu_vector_context(void)
  74 {
  75         riscv_v_stop(RISCV_KERNEL_MODE_V);
  76
  77         if (!IS_ENABLED(CONFIG_PREEMPT_RT))
  78                 local_bh_enable();
  79         else
  80                 preempt_enable();
  81 }
  82
  83 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
  84 static __always_inline u32 *riscv_v_flags_ptr(void)
  85 {
  86         return &current->thread.riscv_v_flags;
  87 }
  88
  89 static inline void riscv_preempt_v_set_dirty(void)
  90 {
  91         *riscv_v_flags_ptr() |= RISCV_PREEMPT_V_DIRTY;
  92 }
  93
  94 static inline void riscv_preempt_v_reset_flags(void)
  95 {
  96         *riscv_v_flags_ptr() &= ~(RISCV_PREEMPT_V_DIRTY | RISCV_PREEMPT_V_NEED_RESTORE);
  97 }
  98
  99 static inline void riscv_v_ctx_depth_inc(void)
 100 {
 101         *riscv_v_flags_ptr() += RISCV_V_CTX_UNIT_DEPTH;
 102 }
 103
 104 static inline void riscv_v_ctx_depth_dec(void)
 105 {
 106         *riscv_v_flags_ptr() -= RISCV_V_CTX_UNIT_DEPTH;
 107 }
 108
 109 static inline u32 riscv_v_ctx_get_depth(void)
 110 {
 111         return *riscv_v_flags_ptr() & RISCV_V_CTX_DEPTH_MASK;
 112 }
 113
 114 static int riscv_v_stop_kernel_context(void)
 115 {
 116         if (riscv_v_ctx_get_depth() != 0 || !riscv_preempt_v_started(current))
 117                 return 1;
 118
 119         riscv_preempt_v_clear_dirty(current);
 120         riscv_v_stop(RISCV_PREEMPT_V);
 121         return 0;
 122 }
 123
 124 static int riscv_v_start_kernel_context(bool *is_nested)
 125 {
 126         struct __riscv_v_ext_state *kvstate, *uvstate;
 127
 128         kvstate = &current->thread.kernel_vstate;
 129         if (!kvstate->datap)
 130                 return -ENOENT;
 131
 132         if (riscv_preempt_v_started(current)) {
 133                 WARN_ON(riscv_v_ctx_get_depth() == 0);
 134                 *is_nested = true;
 135                 get_cpu_vector_context();
 136                 if (riscv_preempt_v_dirty(current)) {
 137                         __riscv_v_vstate_save(kvstate, kvstate->datap);
 138                         riscv_preempt_v_clear_dirty(current);
 139                 }
 140                 riscv_preempt_v_set_restore(current);
 141                 return 0;
 142         }
 143
 144         /* Transfer the ownership of V from user to kernel, then save */
 145         riscv_v_start(RISCV_PREEMPT_V | RISCV_PREEMPT_V_DIRTY);
 146         if ((task_pt_regs(current)->status & SR_VS) == SR_VS_DIRTY) {
 147                 uvstate = &current->thread.vstate;
 148                 __riscv_v_vstate_save(uvstate, uvstate->datap);
 149         }
 150         riscv_preempt_v_clear_dirty(current);
 151         return 0;
 152 }
 153
 154 /* low-level V context handling code, called with irq disabled */
 155 asmlinkage void riscv_v_context_nesting_start(struct pt_regs *regs)
 156 {
 157         int depth;
 158
 159         if (!riscv_preempt_v_started(current))
 160                 return;
 161
 162         depth = riscv_v_ctx_get_depth();
 163         if (depth == 0 && (regs->status & SR_VS) == SR_VS_DIRTY)
 164                 riscv_preempt_v_set_dirty();
 165
 166         riscv_v_ctx_depth_inc();
 167 }
 168
 169 asmlinkage void riscv_v_context_nesting_end(struct pt_regs *regs)
 170 {
 171         struct __riscv_v_ext_state *vstate = &current->thread.kernel_vstate;
 172         u32 depth;
 173
 174         WARN_ON(!irqs_disabled());
 175
 176         if (!riscv_preempt_v_started(current))
 177                 return;
 178
 179         riscv_v_ctx_depth_dec();
 180         depth = riscv_v_ctx_get_depth();
 181         if (depth == 0) {
 182                 if (riscv_preempt_v_restore(current)) {
 183                         __riscv_v_vstate_restore(vstate, vstate->datap);
 184                         __riscv_v_vstate_clean(regs);
 185                         riscv_preempt_v_reset_flags();
 186                 }
 187         }
 188 }
 189 #else
 190 #define riscv_v_start_kernel_context(nested)    (-ENOENT)
 191 #define riscv_v_stop_kernel_context()           (-ENOENT)
 192 #endif /* CONFIG_RISCV_ISA_V_PREEMPTIVE */
 193
 194 /*
 195  * kernel_vector_begin(): obtain the CPU vector registers for use by the calling
 196  * context
 197  *
 198  * Must not be called unless may_use_simd() returns true.
 199  * Task context in the vector registers is saved back to memory as necessary.
 200  *
 201  * A matching call to kernel_vector_end() must be made before returning from the
 202  * calling context.
 203  *
 204  * The caller may freely use the vector registers until kernel_vector_end() is
 205  * called.
 206  */
 207 void kernel_vector_begin(void)
 208 {
 209         bool nested = false;
 210
 211         if (WARN_ON(!has_vector()))
 212                 return;
 213
 214         BUG_ON(!may_use_simd());
 215
 216         if (riscv_v_start_kernel_context(&nested)) {
 217                 get_cpu_vector_context();
 218                 riscv_v_vstate_save(&current->thread.vstate, task_pt_regs(current));
 219         }
 220
 221         if (!nested)
 222                 riscv_v_vstate_set_restore(current, task_pt_regs(current));
 223
 224         riscv_v_enable();
 225 }
 226 EXPORT_SYMBOL_GPL(kernel_vector_begin);
 227
 228 /*
 229  * kernel_vector_end(): give the CPU vector registers back to the current task
 230  *
 231  * Must be called from a context in which kernel_vector_begin() was previously
 232  * called, with no call to kernel_vector_end() in the meantime.
 233  *
 234  * The caller must not use the vector registers after this function is called,
 235  * unless kernel_vector_begin() is called again in the meantime.
 236  */
 237 void kernel_vector_end(void)
 238 {
 239         if (WARN_ON(!has_vector()))
 240                 return;
 241
 242         riscv_v_disable();
 243
 244         if (riscv_v_stop_kernel_context())
 245                 put_cpu_vector_context();
 246 }
 247 EXPORT_SYMBOL_GPL(kernel_vector_end);