arch/riscv/kernel/vector.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright (C) 2023 SiFive
   4  * Author: Andy Chiu <andy.chiu@sifive.com>
   5  */
   6 #include <linux/export.h>
   7 #include <linux/sched/signal.h>
   8 #include <linux/types.h>
   9 #include <linux/slab.h>
  10 #include <linux/sched.h>
  11 #include <linux/uaccess.h>
  12 #include <linux/prctl.h>
  13
  14 #include <asm/thread_info.h>
  15 #include <asm/processor.h>
  16 #include <asm/insn.h>
  17 #include <asm/vector.h>
  18 #include <asm/csr.h>
  19 #include <asm/elf.h>
  20 #include <asm/ptrace.h>
  21 #include <asm/bug.h>
  22
  23 static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE);
  24 static struct kmem_cache *riscv_v_user_cachep;
  25 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
  26 static struct kmem_cache *riscv_v_kernel_cachep;
  27 #endif
  28
  29 unsigned long riscv_v_vsize __read_mostly;
  30 EXPORT_SYMBOL_GPL(riscv_v_vsize);
  31
  32 int riscv_v_setup_vsize(void)
  33 {
  34         unsigned long this_vsize;
  35
  36         /* There are 32 vector registers with vlenb length. */
  37         riscv_v_enable();
  38         this_vsize = csr_read(CSR_VLENB) * 32;
  39         riscv_v_disable();
  40
  41         if (!riscv_v_vsize) {
  42                 riscv_v_vsize = this_vsize;
  43                 return 0;
  44         }
  45
  46         if (riscv_v_vsize != this_vsize) {
  47                 WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems");
  48                 return -EOPNOTSUPP;
  49         }
  50
  51         return 0;
  52 }
  53
  54 void __init riscv_v_setup_ctx_cache(void)
  55 {
  56         if (!has_vector())
  57                 return;
  58
  59         riscv_v_user_cachep = kmem_cache_create_usercopy("riscv_vector_ctx",
  60                                                          riscv_v_vsize, 16, SLAB_PANIC,
  61                                                          0, riscv_v_vsize, NULL);
  62 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
  63         riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx",
  64                                                   riscv_v_vsize, 16,
  65                                                   SLAB_PANIC, NULL);
  66 #endif
  67 }
  68
  69 static bool insn_is_vector(u32 insn_buf)
  70 {
  71         u32 opcode = insn_buf & __INSN_OPCODE_MASK;
  72         u32 width, csr;
  73
  74         /*
  75          * All V-related instructions, including CSR operations are 4-Byte. So,
  76          * do not handle if the instruction length is not 4-Byte.
  77          */
  78         if (unlikely(GET_INSN_LENGTH(insn_buf) != 4))
  79                 return false;
  80
  81         switch (opcode) {
  82         case RVV_OPCODE_VECTOR:
  83                 return true;
  84         case RVV_OPCODE_VL:
  85         case RVV_OPCODE_VS:
  86                 width = RVV_EXRACT_VL_VS_WIDTH(insn_buf);
  87                 if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 ||
  88                     width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64)
  89                         return true;
  90
  91                 break;
  92         case RVG_OPCODE_SYSTEM:
  93                 csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf);
  94                 if ((csr >= CSR_VSTART && csr <= CSR_VCSR) ||
  95                     (csr >= CSR_VL && csr <= CSR_VLENB))
  96                         return true;
  97         }
  98
  99         return false;
 100 }
 101
 102 static int riscv_v_thread_zalloc(struct kmem_cache *cache,
 103                                  struct __riscv_v_ext_state *ctx)
 104 {
 105         void *datap;
 106
 107         datap = kmem_cache_zalloc(cache, GFP_KERNEL);
 108         if (!datap)
 109                 return -ENOMEM;
 110
 111         ctx->datap = datap;
 112         memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap));
 113         return 0;
 114 }
 115
 116 void riscv_v_thread_alloc(struct task_struct *tsk)
 117 {
 118 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
 119         riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate);
 120 #endif
 121 }
 122
 123 void riscv_v_thread_free(struct task_struct *tsk)
 124 {
 125         if (tsk->thread.vstate.datap)
 126                 kmem_cache_free(riscv_v_user_cachep, tsk->thread.vstate.datap);
 127 #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE
 128         if (tsk->thread.kernel_vstate.datap)
 129                 kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap);
 130 #endif
 131 }
 132
 133 #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK)
 134 #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2)
 135 #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK)
 136 #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT))
 137 static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk)
 138 {
 139         return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl);
 140 }
 141
 142 static inline int riscv_v_ctrl_get_next(struct task_struct *tsk)
 143 {
 144         return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl);
 145 }
 146
 147 static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk)
 148 {
 149         return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl);
 150 }
 151
 152 static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt,
 153                                     bool inherit)
 154 {
 155         unsigned long ctrl;
 156
 157         ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK;
 158         ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt);
 159         if (inherit)
 160                 ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT;
 161         tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK;
 162         tsk->thread.vstate_ctrl |= ctrl;
 163 }
 164
 165 bool riscv_v_vstate_ctrl_user_allowed(void)
 166 {
 167         return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON;
 168 }
 169 EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed);
 170
 171 bool riscv_v_first_use_handler(struct pt_regs *regs)
 172 {
 173         u32 __user *epc = (u32 __user *)regs->epc;
 174         u32 insn = (u32)regs->badaddr;
 175
 176         if (!has_vector())
 177                 return false;
 178
 179         /* Do not handle if V is not supported, or disabled */
 180         if (!riscv_v_vstate_ctrl_user_allowed())
 181                 return false;
 182
 183         /* If V has been enabled then it is not the first-use trap */
 184         if (riscv_v_vstate_query(regs))
 185                 return false;
 186
 187         /* Get the instruction */
 188         if (!insn) {
 189                 if (__get_user(insn, epc))
 190                         return false;
 191         }
 192
 193         /* Filter out non-V instructions */
 194         if (!insn_is_vector(insn))
 195                 return false;
 196
 197         /* Sanity check. datap should be null by the time of the first-use trap */
 198         WARN_ON(current->thread.vstate.datap);
 199
 200         /*
 201          * Now we sure that this is a V instruction. And it executes in the
 202          * context where VS has been off. So, try to allocate the user's V
 203          * context and resume execution.
 204          */
 205         if (riscv_v_thread_zalloc(riscv_v_user_cachep, &current->thread.vstate)) {
 206                 force_sig(SIGBUS);
 207                 return true;
 208         }
 209         riscv_v_vstate_on(regs);
 210         riscv_v_vstate_set_restore(current, regs);
 211         return true;
 212 }
 213
 214 void riscv_v_vstate_ctrl_init(struct task_struct *tsk)
 215 {
 216         bool inherit;
 217         int cur, next;
 218
 219         if (!has_vector())
 220                 return;
 221
 222         next = riscv_v_ctrl_get_next(tsk);
 223         if (!next) {
 224                 if (READ_ONCE(riscv_v_implicit_uacc))
 225                         cur = PR_RISCV_V_VSTATE_CTRL_ON;
 226                 else
 227                         cur = PR_RISCV_V_VSTATE_CTRL_OFF;
 228         } else {
 229                 cur = next;
 230         }
 231         /* Clear next mask if inherit-bit is not set */
 232         inherit = riscv_v_ctrl_test_inherit(tsk);
 233         if (!inherit)
 234                 next = PR_RISCV_V_VSTATE_CTRL_DEFAULT;
 235
 236         riscv_v_ctrl_set(tsk, cur, next, inherit);
 237 }
 238
 239 long riscv_v_vstate_ctrl_get_current(void)
 240 {
 241         if (!has_vector())
 242                 return -EINVAL;
 243
 244         return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK;
 245 }
 246
 247 long riscv_v_vstate_ctrl_set_current(unsigned long arg)
 248 {
 249         bool inherit;
 250         int cur, next;
 251
 252         if (!has_vector())
 253                 return -EINVAL;
 254
 255         if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK)
 256                 return -EINVAL;
 257
 258         cur = VSTATE_CTRL_GET_CUR(arg);
 259         switch (cur) {
 260         case PR_RISCV_V_VSTATE_CTRL_OFF:
 261                 /* Do not allow user to turn off V if current is not off */
 262                 if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF)
 263                         return -EPERM;
 264
 265                 break;
 266         case PR_RISCV_V_VSTATE_CTRL_ON:
 267                 break;
 268         case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
 269                 cur = riscv_v_ctrl_get_cur(current);
 270                 break;
 271         default:
 272                 return -EINVAL;
 273         }
 274
 275         next = VSTATE_CTRL_GET_NEXT(arg);
 276         inherit = VSTATE_CTRL_GET_INHERIT(arg);
 277         switch (next) {
 278         case PR_RISCV_V_VSTATE_CTRL_DEFAULT:
 279         case PR_RISCV_V_VSTATE_CTRL_OFF:
 280         case PR_RISCV_V_VSTATE_CTRL_ON:
 281                 riscv_v_ctrl_set(current, cur, next, inherit);
 282                 return 0;
 283         }
 284
 285         return -EINVAL;
 286 }
 287
 288 #ifdef CONFIG_SYSCTL
 289
 290 static struct ctl_table riscv_v_default_vstate_table[] = {
 291         {
 292                 .procname       = "riscv_v_default_allow",
 293                 .data           = &riscv_v_implicit_uacc,
 294                 .maxlen         = sizeof(riscv_v_implicit_uacc),
 295                 .mode           = 0644,
 296                 .proc_handler   = proc_dobool,
 297         },
 298 };
 299
 300 static int __init riscv_v_sysctl_init(void)
 301 {
 302         if (has_vector())
 303                 if (!register_sysctl("abi", riscv_v_default_vstate_table))
 304                         return -EINVAL;
 305         return 0;
 306 }
 307
 308 #else /* ! CONFIG_SYSCTL */
 309 static int __init riscv_v_sysctl_init(void) { return 0; }
 310 #endif /* ! CONFIG_SYSCTL */
 311
 312 static int riscv_v_init(void)
 313 {
 314         return riscv_v_sysctl_init();
 315 }
 316 core_initcall(riscv_v_init);