arch/arm64/include/asm/fpsimd.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * Copyright (C) 2012 ARM Ltd.
   4  */
   5 #ifndef __ASM_FP_H
   6 #define __ASM_FP_H
   7
   8 #include <asm/errno.h>
   9 #include <asm/ptrace.h>
  10 #include <asm/processor.h>
  11 #include <asm/sigcontext.h>
  12 #include <asm/sysreg.h>
  13
  14 #ifndef __ASSEMBLY__
  15
  16 #include <linux/bitmap.h>
  17 #include <linux/build_bug.h>
  18 #include <linux/bug.h>
  19 #include <linux/cache.h>
  20 #include <linux/init.h>
  21 #include <linux/stddef.h>
  22 #include <linux/types.h>
  23
  24 /* Masks for extracting the FPSR and FPCR from the FPSCR */
  25 #define VFP_FPSCR_STAT_MASK     0xf800009f
  26 #define VFP_FPSCR_CTRL_MASK     0x07f79f00
  27 /*
  28  * The VFP state has 32x64-bit registers and a single 32-bit
  29  * control/status register.
  30  */
  31 #define VFP_STATE_SIZE          ((32 * 8) + 4)
  32
  33 static inline unsigned long cpacr_save_enable_kernel_sve(void)
  34 {
  35         unsigned long old = read_sysreg(cpacr_el1);
  36         unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN;
  37
  38         write_sysreg(old | set, cpacr_el1);
  39         isb();
  40         return old;
  41 }
  42
  43 static inline unsigned long cpacr_save_enable_kernel_sme(void)
  44 {
  45         unsigned long old = read_sysreg(cpacr_el1);
  46         unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN;
  47
  48         write_sysreg(old | set, cpacr_el1);
  49         isb();
  50         return old;
  51 }
  52
  53 static inline void cpacr_restore(unsigned long cpacr)
  54 {
  55         write_sysreg(cpacr, cpacr_el1);
  56         isb();
  57 }
  58
  59 /*
  60  * When we defined the maximum SVE vector length we defined the ABI so
  61  * that the maximum vector length included all the reserved for future
  62  * expansion bits in ZCR rather than those just currently defined by
  63  * the architecture.  Using this length to allocate worst size buffers
  64  * results in excessively large allocations, and this effect is even
  65  * more pronounced for SME due to ZA.  Define more suitable VLs for
  66  * these situations.
  67  */
  68 #define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
  69 #define SME_VQ_MAX      ((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)
  70
  71 struct task_struct;
  72
  73 extern void fpsimd_save_state(struct user_fpsimd_state *state);
  74 extern void fpsimd_load_state(struct user_fpsimd_state *state);
  75
  76 extern void fpsimd_thread_switch(struct task_struct *next);
  77 extern void fpsimd_flush_thread(void);
  78
  79 extern void fpsimd_signal_preserve_current_state(void);
  80 extern void fpsimd_preserve_current_state(void);
  81 extern void fpsimd_restore_current_state(void);
  82 extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
  83 extern void fpsimd_kvm_prepare(void);
  84
  85 struct cpu_fp_state {
  86         struct user_fpsimd_state *st;
  87         void *sve_state;
  88         void *sme_state;
  89         u64 *svcr;
  90         u64 *fpmr;
  91         unsigned int sve_vl;
  92         unsigned int sme_vl;
  93         enum fp_type *fp_type;
  94         enum fp_type to_save;
  95 };
  96
  97 extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
  98
  99 extern void fpsimd_flush_task_state(struct task_struct *target);
 100 extern void fpsimd_save_and_flush_cpu_state(void);
 101
 102 static inline bool thread_sm_enabled(struct thread_struct *thread)
 103 {
 104         return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
 105 }
 106
 107 static inline bool thread_za_enabled(struct thread_struct *thread)
 108 {
 109         return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
 110 }
 111
 112 /* Maximum VL that SVE/SME VL-agnostic software can transparently support */
 113 #define VL_ARCH_MAX 0x100
 114
 115 /* Offset of FFR in the SVE register dump */
 116 static inline size_t sve_ffr_offset(int vl)
 117 {
 118         return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
 119 }
 120
 121 static inline void *sve_pffr(struct thread_struct *thread)
 122 {
 123         unsigned int vl;
 124
 125         if (system_supports_sme() && thread_sm_enabled(thread))
 126                 vl = thread_get_sme_vl(thread);
 127         else
 128                 vl = thread_get_sve_vl(thread);
 129
 130         return (char *)thread->sve_state + sve_ffr_offset(vl);
 131 }
 132
 133 static inline void *thread_zt_state(struct thread_struct *thread)
 134 {
 135         /* The ZT register state is stored immediately after the ZA state */
 136         unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
 137         return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
 138 }
 139
 140 extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
 141 extern void sve_load_state(void const *state, u32 const *pfpsr,
 142                            int restore_ffr);
 143 extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
 144 extern unsigned int sve_get_vl(void);
 145 extern void sve_set_vq(unsigned long vq_minus_1);
 146 extern void sme_set_vq(unsigned long vq_minus_1);
 147 extern void sme_save_state(void *state, int zt);
 148 extern void sme_load_state(void const *state, int zt);
 149
 150 struct arm64_cpu_capabilities;
 151 extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
 152 extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
 153 extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
 154 extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
 155 extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
 156 extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused);
 157
 158 /*
 159  * Helpers to translate bit indices in sve_vq_map to VQ values (and
 160  * vice versa).  This allows find_next_bit() to be used to find the
 161  * _maximum_ VQ not exceeding a certain value.
 162  */
 163 static inline unsigned int __vq_to_bit(unsigned int vq)
 164 {
 165         return SVE_VQ_MAX - vq;
 166 }
 167
 168 static inline unsigned int __bit_to_vq(unsigned int bit)
 169 {
 170         return SVE_VQ_MAX - bit;
 171 }
 172
 173
 174 struct vl_info {
 175         enum vec_type type;
 176         const char *name;               /* For display purposes */
 177
 178         /* Minimum supported vector length across all CPUs */
 179         int min_vl;
 180
 181         /* Maximum supported vector length across all CPUs */
 182         int max_vl;
 183         int max_virtualisable_vl;
 184
 185         /*
 186          * Set of available vector lengths,
 187          * where length vq encoded as bit __vq_to_bit(vq):
 188          */
 189         DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
 190
 191         /* Set of vector lengths present on at least one cpu: */
 192         DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
 193 };
 194
 195 #ifdef CONFIG_ARM64_SVE
 196
 197 extern void sve_alloc(struct task_struct *task, bool flush);
 198 extern void fpsimd_release_task(struct task_struct *task);
 199 extern void fpsimd_sync_to_sve(struct task_struct *task);
 200 extern void fpsimd_force_sync_to_sve(struct task_struct *task);
 201 extern void sve_sync_to_fpsimd(struct task_struct *task);
 202 extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
 203
 204 extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
 205                                  unsigned long vl, unsigned long flags);
 206
 207 extern int sve_set_current_vl(unsigned long arg);
 208 extern int sve_get_current_vl(void);
 209
 210 static inline void sve_user_disable(void)
 211 {
 212         sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
 213 }
 214
 215 static inline void sve_user_enable(void)
 216 {
 217         sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
 218 }
 219
 220 #define sve_cond_update_zcr_vq(val, reg)                \
 221         do {                                            \
 222                 u64 __zcr = read_sysreg_s((reg));       \
 223                 u64 __new = __zcr & ~ZCR_ELx_LEN_MASK;  \
 224                 __new |= (val) & ZCR_ELx_LEN_MASK;      \
 225                 if (__zcr != __new)                     \
 226                         write_sysreg_s(__new, (reg));   \
 227         } while (0)
 228
 229 /*
 230  * Probing and setup functions.
 231  * Calls to these functions must be serialised with one another.
 232  */
 233 enum vec_type;
 234
 235 extern void __init vec_init_vq_map(enum vec_type type);
 236 extern void vec_update_vq_map(enum vec_type type);
 237 extern int vec_verify_vq_map(enum vec_type type);
 238 extern void __init sve_setup(void);
 239
 240 extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
 241
 242 static inline void write_vl(enum vec_type type, u64 val)
 243 {
 244         u64 tmp;
 245
 246         switch (type) {
 247 #ifdef CONFIG_ARM64_SVE
 248         case ARM64_VEC_SVE:
 249                 tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
 250                 write_sysreg_s(tmp | val, SYS_ZCR_EL1);
 251                 break;
 252 #endif
 253 #ifdef CONFIG_ARM64_SME
 254         case ARM64_VEC_SME:
 255                 tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
 256                 write_sysreg_s(tmp | val, SYS_SMCR_EL1);
 257                 break;
 258 #endif
 259         default:
 260                 WARN_ON_ONCE(1);
 261                 break;
 262         }
 263 }
 264
 265 static inline int vec_max_vl(enum vec_type type)
 266 {
 267         return vl_info[type].max_vl;
 268 }
 269
 270 static inline int vec_max_virtualisable_vl(enum vec_type type)
 271 {
 272         return vl_info[type].max_virtualisable_vl;
 273 }
 274
 275 static inline int sve_max_vl(void)
 276 {
 277         return vec_max_vl(ARM64_VEC_SVE);
 278 }
 279
 280 static inline int sve_max_virtualisable_vl(void)
 281 {
 282         return vec_max_virtualisable_vl(ARM64_VEC_SVE);
 283 }
 284
 285 /* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
 286 static inline bool vq_available(enum vec_type type, unsigned int vq)
 287 {
 288         return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
 289 }
 290
 291 static inline bool sve_vq_available(unsigned int vq)
 292 {
 293         return vq_available(ARM64_VEC_SVE, vq);
 294 }
 295
 296 size_t sve_state_size(struct task_struct const *task);
 297
 298 #else /* ! CONFIG_ARM64_SVE */
 299
 300 static inline void sve_alloc(struct task_struct *task, bool flush) { }
 301 static inline void fpsimd_release_task(struct task_struct *task) { }
 302 static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
 303 static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
 304
 305 static inline int sve_max_virtualisable_vl(void)
 306 {
 307         return 0;
 308 }
 309
 310 static inline int sve_set_current_vl(unsigned long arg)
 311 {
 312         return -EINVAL;
 313 }
 314
 315 static inline int sve_get_current_vl(void)
 316 {
 317         return -EINVAL;
 318 }
 319
 320 static inline int sve_max_vl(void)
 321 {
 322         return -EINVAL;
 323 }
 324
 325 static inline bool sve_vq_available(unsigned int vq) { return false; }
 326
 327 static inline void sve_user_disable(void) { BUILD_BUG(); }
 328 static inline void sve_user_enable(void) { BUILD_BUG(); }
 329
 330 #define sve_cond_update_zcr_vq(val, reg) do { } while (0)
 331
 332 static inline void vec_init_vq_map(enum vec_type t) { }
 333 static inline void vec_update_vq_map(enum vec_type t) { }
 334 static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
 335 static inline void sve_setup(void) { }
 336
 337 static inline size_t sve_state_size(struct task_struct const *task)
 338 {
 339         return 0;
 340 }
 341
 342 #endif /* ! CONFIG_ARM64_SVE */
 343
 344 #ifdef CONFIG_ARM64_SME
 345
 346 static inline void sme_user_disable(void)
 347 {
 348         sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
 349 }
 350
 351 static inline void sme_user_enable(void)
 352 {
 353         sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
 354 }
 355
 356 static inline void sme_smstart_sm(void)
 357 {
 358         asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
 359 }
 360
 361 static inline void sme_smstop_sm(void)
 362 {
 363         asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
 364 }
 365
 366 static inline void sme_smstop(void)
 367 {
 368         asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
 369 }
 370
 371 extern void __init sme_setup(void);
 372
 373 static inline int sme_max_vl(void)
 374 {
 375         return vec_max_vl(ARM64_VEC_SME);
 376 }
 377
 378 static inline int sme_max_virtualisable_vl(void)
 379 {
 380         return vec_max_virtualisable_vl(ARM64_VEC_SME);
 381 }
 382
 383 extern void sme_alloc(struct task_struct *task, bool flush);
 384 extern unsigned int sme_get_vl(void);
 385 extern int sme_set_current_vl(unsigned long arg);
 386 extern int sme_get_current_vl(void);
 387 extern void sme_suspend_exit(void);
 388
 389 /*
 390  * Return how many bytes of memory are required to store the full SME
 391  * specific state for task, given task's currently configured vector
 392  * length.
 393  */
 394 static inline size_t sme_state_size(struct task_struct const *task)
 395 {
 396         unsigned int vl = task_get_sme_vl(task);
 397         size_t size;
 398
 399         size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
 400
 401         if (system_supports_sme2())
 402                 size += ZT_SIG_REG_SIZE;
 403
 404         return size;
 405 }
 406
 407 #else
 408
 409 static inline void sme_user_disable(void) { BUILD_BUG(); }
 410 static inline void sme_user_enable(void) { BUILD_BUG(); }
 411
 412 static inline void sme_smstart_sm(void) { }
 413 static inline void sme_smstop_sm(void) { }
 414 static inline void sme_smstop(void) { }
 415
 416 static inline void sme_alloc(struct task_struct *task, bool flush) { }
 417 static inline void sme_setup(void) { }
 418 static inline unsigned int sme_get_vl(void) { return 0; }
 419 static inline int sme_max_vl(void) { return 0; }
 420 static inline int sme_max_virtualisable_vl(void) { return 0; }
 421 static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
 422 static inline int sme_get_current_vl(void) { return -EINVAL; }
 423 static inline void sme_suspend_exit(void) { }
 424
 425 static inline size_t sme_state_size(struct task_struct const *task)
 426 {
 427         return 0;
 428 }
 429
 430 #endif /* ! CONFIG_ARM64_SME */
 431
 432 /* For use by EFI runtime services calls only */
 433 extern void __efi_fpsimd_begin(void);
 434 extern void __efi_fpsimd_end(void);
 435
 436 #endif
 437
 438 #endif