arch/riscv/kernel/traps_misaligned.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2020 Western Digital Corporation or its affiliates.
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/init.h>
   7 #include <linux/mm.h>
   8 #include <linux/module.h>
   9 #include <linux/perf_event.h>
  10 #include <linux/irq.h>
  11 #include <linux/stringify.h>
  12
  13 #include <asm/processor.h>
  14 #include <asm/ptrace.h>
  15 #include <asm/csr.h>
  16 #include <asm/entry-common.h>
  17 #include <asm/hwprobe.h>
  18 #include <asm/cpufeature.h>
  19 #include <asm/vector.h>
  20
  21 #define INSN_MATCH_LB                   0x3
  22 #define INSN_MASK_LB                    0x707f
  23 #define INSN_MATCH_LH                   0x1003
  24 #define INSN_MASK_LH                    0x707f
  25 #define INSN_MATCH_LW                   0x2003
  26 #define INSN_MASK_LW                    0x707f
  27 #define INSN_MATCH_LD                   0x3003
  28 #define INSN_MASK_LD                    0x707f
  29 #define INSN_MATCH_LBU                  0x4003
  30 #define INSN_MASK_LBU                   0x707f
  31 #define INSN_MATCH_LHU                  0x5003
  32 #define INSN_MASK_LHU                   0x707f
  33 #define INSN_MATCH_LWU                  0x6003
  34 #define INSN_MASK_LWU                   0x707f
  35 #define INSN_MATCH_SB                   0x23
  36 #define INSN_MASK_SB                    0x707f
  37 #define INSN_MATCH_SH                   0x1023
  38 #define INSN_MASK_SH                    0x707f
  39 #define INSN_MATCH_SW                   0x2023
  40 #define INSN_MASK_SW                    0x707f
  41 #define INSN_MATCH_SD                   0x3023
  42 #define INSN_MASK_SD                    0x707f
  43
  44 #define INSN_MATCH_FLW                  0x2007
  45 #define INSN_MASK_FLW                   0x707f
  46 #define INSN_MATCH_FLD                  0x3007
  47 #define INSN_MASK_FLD                   0x707f
  48 #define INSN_MATCH_FLQ                  0x4007
  49 #define INSN_MASK_FLQ                   0x707f
  50 #define INSN_MATCH_FSW                  0x2027
  51 #define INSN_MASK_FSW                   0x707f
  52 #define INSN_MATCH_FSD                  0x3027
  53 #define INSN_MASK_FSD                   0x707f
  54 #define INSN_MATCH_FSQ                  0x4027
  55 #define INSN_MASK_FSQ                   0x707f
  56
  57 #define INSN_MATCH_C_LD                 0x6000
  58 #define INSN_MASK_C_LD                  0xe003
  59 #define INSN_MATCH_C_SD                 0xe000
  60 #define INSN_MASK_C_SD                  0xe003
  61 #define INSN_MATCH_C_LW                 0x4000
  62 #define INSN_MASK_C_LW                  0xe003
  63 #define INSN_MATCH_C_SW                 0xc000
  64 #define INSN_MASK_C_SW                  0xe003
  65 #define INSN_MATCH_C_LDSP               0x6002
  66 #define INSN_MASK_C_LDSP                0xe003
  67 #define INSN_MATCH_C_SDSP               0xe002
  68 #define INSN_MASK_C_SDSP                0xe003
  69 #define INSN_MATCH_C_LWSP               0x4002
  70 #define INSN_MASK_C_LWSP                0xe003
  71 #define INSN_MATCH_C_SWSP               0xc002
  72 #define INSN_MASK_C_SWSP                0xe003
  73
  74 #define INSN_MATCH_C_FLD                0x2000
  75 #define INSN_MASK_C_FLD                 0xe003
  76 #define INSN_MATCH_C_FLW                0x6000
  77 #define INSN_MASK_C_FLW                 0xe003
  78 #define INSN_MATCH_C_FSD                0xa000
  79 #define INSN_MASK_C_FSD                 0xe003
  80 #define INSN_MATCH_C_FSW                0xe000
  81 #define INSN_MASK_C_FSW                 0xe003
  82 #define INSN_MATCH_C_FLDSP              0x2002
  83 #define INSN_MASK_C_FLDSP               0xe003
  84 #define INSN_MATCH_C_FSDSP              0xa002
  85 #define INSN_MASK_C_FSDSP               0xe003
  86 #define INSN_MATCH_C_FLWSP              0x6002
  87 #define INSN_MASK_C_FLWSP               0xe003
  88 #define INSN_MATCH_C_FSWSP              0xe002
  89 #define INSN_MASK_C_FSWSP               0xe003
  90
  91 #define INSN_LEN(insn)                  ((((insn) & 0x3) < 0x3) ? 2 : 4)
  92
  93 #if defined(CONFIG_64BIT)
  94 #define LOG_REGBYTES                    3
  95 #define XLEN                            64
  96 #else
  97 #define LOG_REGBYTES                    2
  98 #define XLEN                            32
  99 #endif
 100 #define REGBYTES                        (1 << LOG_REGBYTES)
 101 #define XLEN_MINUS_16                   ((XLEN) - 16)
 102
 103 #define SH_RD                           7
 104 #define SH_RS1                          15
 105 #define SH_RS2                          20
 106 #define SH_RS2C                         2
 107
 108 #define RV_X(x, s, n)                   (((x) >> (s)) & ((1 << (n)) - 1))
 109 #define RVC_LW_IMM(x)                   ((RV_X(x, 6, 1) << 2) | \
 110                                          (RV_X(x, 10, 3) << 3) | \
 111                                          (RV_X(x, 5, 1) << 6))
 112 #define RVC_LD_IMM(x)                   ((RV_X(x, 10, 3) << 3) | \
 113                                          (RV_X(x, 5, 2) << 6))
 114 #define RVC_LWSP_IMM(x)                 ((RV_X(x, 4, 3) << 2) | \
 115                                          (RV_X(x, 12, 1) << 5) | \
 116                                          (RV_X(x, 2, 2) << 6))
 117 #define RVC_LDSP_IMM(x)                 ((RV_X(x, 5, 2) << 3) | \
 118                                          (RV_X(x, 12, 1) << 5) | \
 119                                          (RV_X(x, 2, 3) << 6))
 120 #define RVC_SWSP_IMM(x)                 ((RV_X(x, 9, 4) << 2) | \
 121                                          (RV_X(x, 7, 2) << 6))
 122 #define RVC_SDSP_IMM(x)                 ((RV_X(x, 10, 3) << 3) | \
 123                                          (RV_X(x, 7, 3) << 6))
 124 #define RVC_RS1S(insn)                  (8 + RV_X(insn, SH_RD, 3))
 125 #define RVC_RS2S(insn)                  (8 + RV_X(insn, SH_RS2C, 3))
 126 #define RVC_RS2(insn)                   RV_X(insn, SH_RS2C, 5)
 127
 128 #define SHIFT_RIGHT(x, y)               \
 129         ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
 130
 131 #define REG_MASK                        \
 132         ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
 133
 134 #define REG_OFFSET(insn, pos)           \
 135         (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
 136
 137 #define REG_PTR(insn, pos, regs)        \
 138         (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
 139
 140 #define GET_RS1(insn, regs)             (*REG_PTR(insn, SH_RS1, regs))
 141 #define GET_RS2(insn, regs)             (*REG_PTR(insn, SH_RS2, regs))
 142 #define GET_RS1S(insn, regs)            (*REG_PTR(RVC_RS1S(insn), 0, regs))
 143 #define GET_RS2S(insn, regs)            (*REG_PTR(RVC_RS2S(insn), 0, regs))
 144 #define GET_RS2C(insn, regs)            (*REG_PTR(insn, SH_RS2C, regs))
 145 #define GET_SP(regs)                    (*REG_PTR(2, 0, regs))
 146 #define SET_RD(insn, regs, val)         (*REG_PTR(insn, SH_RD, regs) = (val))
 147 #define IMM_I(insn)                     ((s32)(insn) >> 20)
 148 #define IMM_S(insn)                     (((s32)(insn) >> 25 << 5) | \
 149                                          (s32)(((insn) >> 7) & 0x1f))
 150 #define MASK_FUNCT3                     0x7000
 151
 152 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
 153 #define GET_RM(insn) (((insn) >> 12) & 7)
 154 #define PRECISION_S 0
 155 #define PRECISION_D 1
 156
 157 #ifdef CONFIG_FPU
 158
 159 #define FP_GET_RD(insn)         (insn >> 7 & 0x1F)
 160
 161 extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
 162
 163 static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
 164                       unsigned long val)
 165 {
 166         unsigned long fp_reg = FP_GET_RD(insn);
 167
 168         put_f32_reg(fp_reg, val);
 169         regs->status |= SR_FS_DIRTY;
 170
 171         return 0;
 172 }
 173
 174 extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
 175
 176 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
 177 {
 178         unsigned long fp_reg = FP_GET_RD(insn);
 179         unsigned long value;
 180
 181 #if __riscv_xlen == 32
 182         value = (unsigned long) &val;
 183 #else
 184         value = val;
 185 #endif
 186         put_f64_reg(fp_reg, value);
 187         regs->status |= SR_FS_DIRTY;
 188
 189         return 0;
 190 }
 191
 192 #if __riscv_xlen == 32
 193 extern void get_f64_reg(unsigned long fp_reg, u64 *value);
 194
 195 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
 196                       struct pt_regs *regs)
 197 {
 198         unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
 199         u64 val;
 200
 201         get_f64_reg(fp_reg, &val);
 202         regs->status |= SR_FS_DIRTY;
 203
 204         return val;
 205 }
 206 #else
 207
 208 extern unsigned long get_f64_reg(unsigned long fp_reg);
 209
 210 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
 211                                 struct pt_regs *regs)
 212 {
 213         unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
 214         unsigned long val;
 215
 216         val = get_f64_reg(fp_reg);
 217         regs->status |= SR_FS_DIRTY;
 218
 219         return val;
 220 }
 221
 222 #endif
 223
 224 extern unsigned long get_f32_reg(unsigned long fp_reg);
 225
 226 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
 227                                 struct pt_regs *regs)
 228 {
 229         unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
 230         unsigned long val;
 231
 232         val = get_f32_reg(fp_reg);
 233         regs->status |= SR_FS_DIRTY;
 234
 235         return val;
 236 }
 237
 238 #else /* CONFIG_FPU */
 239 static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
 240                        unsigned long val) {}
 241
 242 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
 243
 244 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
 245                                 struct pt_regs *regs)
 246 {
 247         return 0;
 248 }
 249
 250 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
 251                                 struct pt_regs *regs)
 252 {
 253         return 0;
 254 }
 255
 256 #endif
 257
 258 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
 259 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
 260 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
 261
 262 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
 263 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
 264 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
 265
 266 #define __read_insn(regs, insn, insn_addr, type)        \
 267 ({                                                      \
 268         int __ret;                                      \
 269                                                         \
 270         if (user_mode(regs)) {                          \
 271                 __ret = __get_user(insn, (type __user *) insn_addr); \
 272         } else {                                        \
 273                 insn = *(type *)insn_addr;              \
 274                 __ret = 0;                              \
 275         }                                               \
 276                                                         \
 277         __ret;                                          \
 278 })
 279
 280 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
 281 {
 282         ulong insn = 0;
 283
 284         if (epc & 0x2) {
 285                 ulong tmp = 0;
 286
 287                 if (__read_insn(regs, insn, epc, u16))
 288                         return -EFAULT;
 289                 /* __get_user() uses regular "lw" which sign extend the loaded
 290                  * value make sure to clear higher order bits in case we "or" it
 291                  * below with the upper 16 bits half.
 292                  */
 293                 insn &= GENMASK(15, 0);
 294                 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
 295                         *r_insn = insn;
 296                         return 0;
 297                 }
 298                 epc += sizeof(u16);
 299                 if (__read_insn(regs, tmp, epc, u16))
 300                         return -EFAULT;
 301                 *r_insn = (tmp << 16) | insn;
 302
 303                 return 0;
 304         } else {
 305                 if (__read_insn(regs, insn, epc, u32))
 306                         return -EFAULT;
 307                 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
 308                         *r_insn = insn;
 309                         return 0;
 310                 }
 311                 insn &= GENMASK(15, 0);
 312                 *r_insn = insn;
 313
 314                 return 0;
 315         }
 316 }
 317
 318 union reg_data {
 319         u8 data_bytes[8];
 320         ulong data_ulong;
 321         u64 data_u64;
 322 };
 323
 324 /* sysctl hooks */
 325 int unaligned_enabled __read_mostly = 1;        /* Enabled by default */
 326
 327 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
 328 static int handle_vector_misaligned_load(struct pt_regs *regs)
 329 {
 330         unsigned long epc = regs->epc;
 331         unsigned long insn;
 332
 333         if (get_insn(regs, epc, &insn))
 334                 return -1;
 335
 336         /* Only return 0 when in check_vector_unaligned_access_emulated */
 337         if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
 338                 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
 339                 regs->epc = epc + INSN_LEN(insn);
 340                 return 0;
 341         }
 342
 343         /* If vector instruction we don't emulate it yet */
 344         regs->epc = epc;
 345         return -1;
 346 }
 347 #else
 348 static int handle_vector_misaligned_load(struct pt_regs *regs)
 349 {
 350         return -1;
 351 }
 352 #endif
 353
 354 static int handle_scalar_misaligned_load(struct pt_regs *regs)
 355 {
 356         union reg_data val;
 357         unsigned long epc = regs->epc;
 358         unsigned long insn;
 359         unsigned long addr = regs->badaddr;
 360         int fp = 0, shift = 0, len = 0;
 361
 362         perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
 363
 364 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
 365         *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
 366 #endif
 367
 368         if (!unaligned_enabled)
 369                 return -1;
 370
 371         if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
 372                 return -1;
 373
 374         if (get_insn(regs, epc, &insn))
 375                 return -1;
 376
 377         regs->epc = 0;
 378
 379         if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
 380                 len = 4;
 381                 shift = 8 * (sizeof(unsigned long) - len);
 382 #if defined(CONFIG_64BIT)
 383         } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
 384                 len = 8;
 385                 shift = 8 * (sizeof(unsigned long) - len);
 386         } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
 387                 len = 4;
 388 #endif
 389         } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
 390                 fp = 1;
 391                 len = 8;
 392         } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
 393                 fp = 1;
 394                 len = 4;
 395         } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
 396                 len = 2;
 397                 shift = 8 * (sizeof(unsigned long) - len);
 398         } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
 399                 len = 2;
 400 #if defined(CONFIG_64BIT)
 401         } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
 402                 len = 8;
 403                 shift = 8 * (sizeof(unsigned long) - len);
 404                 insn = RVC_RS2S(insn) << SH_RD;
 405         } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
 406                    ((insn >> SH_RD) & 0x1f)) {
 407                 len = 8;
 408                 shift = 8 * (sizeof(unsigned long) - len);
 409 #endif
 410         } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
 411                 len = 4;
 412                 shift = 8 * (sizeof(unsigned long) - len);
 413                 insn = RVC_RS2S(insn) << SH_RD;
 414         } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
 415                    ((insn >> SH_RD) & 0x1f)) {
 416                 len = 4;
 417                 shift = 8 * (sizeof(unsigned long) - len);
 418         } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
 419                 fp = 1;
 420                 len = 8;
 421                 insn = RVC_RS2S(insn) << SH_RD;
 422         } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
 423                 fp = 1;
 424                 len = 8;
 425 #if defined(CONFIG_32BIT)
 426         } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
 427                 fp = 1;
 428                 len = 4;
 429                 insn = RVC_RS2S(insn) << SH_RD;
 430         } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
 431                 fp = 1;
 432                 len = 4;
 433 #endif
 434         } else {
 435                 regs->epc = epc;
 436                 return -1;
 437         }
 438
 439         if (!IS_ENABLED(CONFIG_FPU) && fp)
 440                 return -EOPNOTSUPP;
 441
 442         val.data_u64 = 0;
 443         if (user_mode(regs)) {
 444                 if (copy_from_user(&val, (u8 __user *)addr, len))
 445                         return -1;
 446         } else {
 447                 memcpy(&val, (u8 *)addr, len);
 448         }
 449
 450         if (!fp)
 451                 SET_RD(insn, regs, val.data_ulong << shift >> shift);
 452         else if (len == 8)
 453                 set_f64_rd(insn, regs, val.data_u64);
 454         else
 455                 set_f32_rd(insn, regs, val.data_ulong);
 456
 457         regs->epc = epc + INSN_LEN(insn);
 458
 459         return 0;
 460 }
 461
 462 static int handle_scalar_misaligned_store(struct pt_regs *regs)
 463 {
 464         union reg_data val;
 465         unsigned long epc = regs->epc;
 466         unsigned long insn;
 467         unsigned long addr = regs->badaddr;
 468         int len = 0, fp = 0;
 469
 470         perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
 471
 472         if (!unaligned_enabled)
 473                 return -1;
 474
 475         if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
 476                 return -1;
 477
 478         if (get_insn(regs, epc, &insn))
 479                 return -1;
 480
 481         regs->epc = 0;
 482
 483         val.data_ulong = GET_RS2(insn, regs);
 484
 485         if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
 486                 len = 4;
 487 #if defined(CONFIG_64BIT)
 488         } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
 489                 len = 8;
 490 #endif
 491         } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
 492                 fp = 1;
 493                 len = 8;
 494                 val.data_u64 = GET_F64_RS2(insn, regs);
 495         } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
 496                 fp = 1;
 497                 len = 4;
 498                 val.data_ulong = GET_F32_RS2(insn, regs);
 499         } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
 500                 len = 2;
 501 #if defined(CONFIG_64BIT)
 502         } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
 503                 len = 8;
 504                 val.data_ulong = GET_RS2S(insn, regs);
 505         } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
 506                 len = 8;
 507                 val.data_ulong = GET_RS2C(insn, regs);
 508 #endif
 509         } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
 510                 len = 4;
 511                 val.data_ulong = GET_RS2S(insn, regs);
 512         } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
 513                 len = 4;
 514                 val.data_ulong = GET_RS2C(insn, regs);
 515         } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
 516                 fp = 1;
 517                 len = 8;
 518                 val.data_u64 = GET_F64_RS2S(insn, regs);
 519         } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
 520                 fp = 1;
 521                 len = 8;
 522                 val.data_u64 = GET_F64_RS2C(insn, regs);
 523 #if !defined(CONFIG_64BIT)
 524         } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
 525                 fp = 1;
 526                 len = 4;
 527                 val.data_ulong = GET_F32_RS2S(insn, regs);
 528         } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
 529                 fp = 1;
 530                 len = 4;
 531                 val.data_ulong = GET_F32_RS2C(insn, regs);
 532 #endif
 533         } else {
 534                 regs->epc = epc;
 535                 return -1;
 536         }
 537
 538         if (!IS_ENABLED(CONFIG_FPU) && fp)
 539                 return -EOPNOTSUPP;
 540
 541         if (user_mode(regs)) {
 542                 if (copy_to_user((u8 __user *)addr, &val, len))
 543                         return -1;
 544         } else {
 545                 memcpy((u8 *)addr, &val, len);
 546         }
 547
 548         regs->epc = epc + INSN_LEN(insn);
 549
 550         return 0;
 551 }
 552
 553 int handle_misaligned_load(struct pt_regs *regs)
 554 {
 555         unsigned long epc = regs->epc;
 556         unsigned long insn;
 557
 558         if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
 559                 if (get_insn(regs, epc, &insn))
 560                         return -1;
 561
 562                 if (insn_is_vector(insn))
 563                         return handle_vector_misaligned_load(regs);
 564         }
 565
 566         if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
 567                 return handle_scalar_misaligned_load(regs);
 568
 569         return -1;
 570 }
 571
 572 int handle_misaligned_store(struct pt_regs *regs)
 573 {
 574         if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
 575                 return handle_scalar_misaligned_store(regs);
 576
 577         return -1;
 578 }
 579
 580 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
 581 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
 582 {
 583         long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
 584         unsigned long tmp_var;
 585
 586         *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
 587
 588         kernel_vector_begin();
 589         /*
 590          * In pre-13.0.0 versions of GCC, vector registers cannot appear in
 591          * the clobber list. This inline asm clobbers v0, but since we do not
 592          * currently build the kernel with V enabled, the v0 clobber arg is not
 593          * needed (as the compiler will not emit vector code itself). If the kernel
 594          * is changed to build with V enabled, the clobber arg will need to be
 595          * added here.
 596          */
 597         __asm__ __volatile__ (
 598                 ".balign 4\n\t"
 599                 ".option push\n\t"
 600                 ".option arch, +zve32x\n\t"
 601                 "       vsetivli zero, 1, e16, m1, ta, ma\n\t"  // Vectors of 16b
 602                 "       vle16.v v0, (%[ptr])\n\t"               // Load bytes
 603                 ".option pop\n\t"
 604                 : : [ptr] "r" ((u8 *)&tmp_var + 1));
 605         kernel_vector_end();
 606 }
 607
 608 bool check_vector_unaligned_access_emulated_all_cpus(void)
 609 {
 610         int cpu;
 611
 612         if (!has_vector()) {
 613                 for_each_online_cpu(cpu)
 614                         per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
 615                 return false;
 616         }
 617
 618         schedule_on_each_cpu(check_vector_unaligned_access_emulated);
 619
 620         for_each_online_cpu(cpu)
 621                 if (per_cpu(vector_misaligned_access, cpu)
 622                     == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
 623                         return false;
 624
 625         return true;
 626 }
 627 #else
 628 bool check_vector_unaligned_access_emulated_all_cpus(void)
 629 {
 630         return false;
 631 }
 632 #endif
 633
 634 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED
 635
 636 static bool unaligned_ctl __read_mostly;
 637
 638 void check_unaligned_access_emulated(struct work_struct *work __always_unused)
 639 {
 640         int cpu = smp_processor_id();
 641         long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
 642         unsigned long tmp_var, tmp_val;
 643
 644         *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
 645
 646         __asm__ __volatile__ (
 647                 "       "REG_L" %[tmp], 1(%[ptr])\n"
 648                 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
 649
 650         /*
 651          * If unaligned_ctl is already set, this means that we detected that all
 652          * CPUS uses emulated misaligned access at boot time. If that changed
 653          * when hotplugging the new cpu, this is something we don't handle.
 654          */
 655         if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
 656                 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
 657                 while (true)
 658                         cpu_relax();
 659         }
 660 }
 661
 662 bool check_unaligned_access_emulated_all_cpus(void)
 663 {
 664         int cpu;
 665
 666         /*
 667          * We can only support PR_UNALIGN controls if all CPUs have misaligned
 668          * accesses emulated since tasks requesting such control can run on any
 669          * CPU.
 670          */
 671         schedule_on_each_cpu(check_unaligned_access_emulated);
 672
 673         for_each_online_cpu(cpu)
 674                 if (per_cpu(misaligned_access_speed, cpu)
 675                     != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
 676                         return false;
 677
 678         unaligned_ctl = true;
 679         return true;
 680 }
 681
 682 bool unaligned_ctl_available(void)
 683 {
 684         return unaligned_ctl;
 685 }
 686 #else
 687 bool check_unaligned_access_emulated_all_cpus(void)
 688 {
 689         return false;
 690 }
 691 #endif