arch/powerpc/kernel/align.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* align.c - handle alignment exceptions for the Power PC.
   3  *
   4  * Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au>
   5  * Copyright (c) 1998-1999 TiVo, Inc.
   6  *   PowerPC 403GCX modifications.
   7  * Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu>
   8  *   PowerPC 403GCX/405GP modifications.
   9  * Copyright (c) 2001-2002 PPC64 team, IBM Corp
  10  *   64-bit and Power4 support
  11  * Copyright (c) 2005 Benjamin Herrenschmidt, IBM Corp
  12  *                    <benh@kernel.crashing.org>
  13  *   Merge ppc32 and ppc64 implementations
  14  */
  15
  16 #include <linux/kernel.h>
  17 #include <linux/mm.h>
  18 #include <asm/processor.h>
  19 #include <linux/uaccess.h>
  20 #include <asm/cache.h>
  21 #include <asm/cputable.h>
  22 #include <asm/emulated_ops.h>
  23 #include <asm/switch_to.h>
  24 #include <asm/disassemble.h>
  25 #include <asm/cpu_has_feature.h>
  26 #include <asm/sstep.h>
  27 #include <asm/inst.h>
  28
  29 struct aligninfo {
  30         unsigned char len;
  31         unsigned char flags;
  32 };
  33
  34
  35 #define INVALID { 0, 0 }
  36
  37 /* Bits in the flags field */
  38 #define LD      0       /* load */
  39 #define ST      1       /* store */
  40 #define SE      2       /* sign-extend value, or FP ld/st as word */
  41 #define SW      0x20    /* byte swap */
  42 #define E4      0x40    /* SPE endianness is word */
  43 #define E8      0x80    /* SPE endianness is double word */
  44
  45 #ifdef CONFIG_SPE
  46
  47 static struct aligninfo spe_aligninfo[32] = {
  48         { 8, LD+E8 },           /* 0 00 00: evldd[x] */
  49         { 8, LD+E4 },           /* 0 00 01: evldw[x] */
  50         { 8, LD },              /* 0 00 10: evldh[x] */
  51         INVALID,                /* 0 00 11 */
  52         { 2, LD },              /* 0 01 00: evlhhesplat[x] */
  53         INVALID,                /* 0 01 01 */
  54         { 2, LD },              /* 0 01 10: evlhhousplat[x] */
  55         { 2, LD+SE },           /* 0 01 11: evlhhossplat[x] */
  56         { 4, LD },              /* 0 10 00: evlwhe[x] */
  57         INVALID,                /* 0 10 01 */
  58         { 4, LD },              /* 0 10 10: evlwhou[x] */
  59         { 4, LD+SE },           /* 0 10 11: evlwhos[x] */
  60         { 4, LD+E4 },           /* 0 11 00: evlwwsplat[x] */
  61         INVALID,                /* 0 11 01 */
  62         { 4, LD },              /* 0 11 10: evlwhsplat[x] */
  63         INVALID,                /* 0 11 11 */
  64
  65         { 8, ST+E8 },           /* 1 00 00: evstdd[x] */
  66         { 8, ST+E4 },           /* 1 00 01: evstdw[x] */
  67         { 8, ST },              /* 1 00 10: evstdh[x] */
  68         INVALID,                /* 1 00 11 */
  69         INVALID,                /* 1 01 00 */
  70         INVALID,                /* 1 01 01 */
  71         INVALID,                /* 1 01 10 */
  72         INVALID,                /* 1 01 11 */
  73         { 4, ST },              /* 1 10 00: evstwhe[x] */
  74         INVALID,                /* 1 10 01 */
  75         { 4, ST },              /* 1 10 10: evstwho[x] */
  76         INVALID,                /* 1 10 11 */
  77         { 4, ST+E4 },           /* 1 11 00: evstwwe[x] */
  78         INVALID,                /* 1 11 01 */
  79         { 4, ST+E4 },           /* 1 11 10: evstwwo[x] */
  80         INVALID,                /* 1 11 11 */
  81 };
  82
  83 #define EVLDD           0x00
  84 #define EVLDW           0x01
  85 #define EVLDH           0x02
  86 #define EVLHHESPLAT     0x04
  87 #define EVLHHOUSPLAT    0x06
  88 #define EVLHHOSSPLAT    0x07
  89 #define EVLWHE          0x08
  90 #define EVLWHOU         0x0A
  91 #define EVLWHOS         0x0B
  92 #define EVLWWSPLAT      0x0C
  93 #define EVLWHSPLAT      0x0E
  94 #define EVSTDD          0x10
  95 #define EVSTDW          0x11
  96 #define EVSTDH          0x12
  97 #define EVSTWHE         0x18
  98 #define EVSTWHO         0x1A
  99 #define EVSTWWE         0x1C
 100 #define EVSTWWO         0x1E
 101
 102 /*
 103  * Emulate SPE loads and stores.
 104  * Only Book-E has these instructions, and it does true little-endian,
 105  * so we don't need the address swizzling.
 106  */
 107 static int emulate_spe(struct pt_regs *regs, unsigned int reg,
 108                        struct ppc_inst ppc_instr)
 109 {
 110         int ret;
 111         union {
 112                 u64 ll;
 113                 u32 w[2];
 114                 u16 h[4];
 115                 u8 v[8];
 116         } data, temp;
 117         unsigned char __user *p, *addr;
 118         unsigned long *evr = &current->thread.evr[reg];
 119         unsigned int nb, flags, instr;
 120
 121         instr = ppc_inst_val(ppc_instr);
 122         instr = (instr >> 1) & 0x1f;
 123
 124         /* DAR has the operand effective address */
 125         addr = (unsigned char __user *)regs->dar;
 126
 127         nb = spe_aligninfo[instr].len;
 128         flags = spe_aligninfo[instr].flags;
 129
 130         /* Verify the address of the operand */
 131         if (unlikely(user_mode(regs) &&
 132                      !access_ok(addr, nb)))
 133                 return -EFAULT;
 134
 135         /* userland only */
 136         if (unlikely(!user_mode(regs)))
 137                 return 0;
 138
 139         flush_spe_to_thread(current);
 140
 141         /* If we are loading, get the data from user space, else
 142          * get it from register values
 143          */
 144         if (flags & ST) {
 145                 data.ll = 0;
 146                 switch (instr) {
 147                 case EVSTDD:
 148                 case EVSTDW:
 149                 case EVSTDH:
 150                         data.w[0] = *evr;
 151                         data.w[1] = regs->gpr[reg];
 152                         break;
 153                 case EVSTWHE:
 154                         data.h[2] = *evr >> 16;
 155                         data.h[3] = regs->gpr[reg] >> 16;
 156                         break;
 157                 case EVSTWHO:
 158                         data.h[2] = *evr & 0xffff;
 159                         data.h[3] = regs->gpr[reg] & 0xffff;
 160                         break;
 161                 case EVSTWWE:
 162                         data.w[1] = *evr;
 163                         break;
 164                 case EVSTWWO:
 165                         data.w[1] = regs->gpr[reg];
 166                         break;
 167                 default:
 168                         return -EINVAL;
 169                 }
 170         } else {
 171                 temp.ll = data.ll = 0;
 172                 ret = 0;
 173                 p = addr;
 174
 175                 switch (nb) {
 176                 case 8:
 177                         ret |= __get_user_inatomic(temp.v[0], p++);
 178                         ret |= __get_user_inatomic(temp.v[1], p++);
 179                         ret |= __get_user_inatomic(temp.v[2], p++);
 180                         ret |= __get_user_inatomic(temp.v[3], p++);
 181                         fallthrough;
 182                 case 4:
 183                         ret |= __get_user_inatomic(temp.v[4], p++);
 184                         ret |= __get_user_inatomic(temp.v[5], p++);
 185                         fallthrough;
 186                 case 2:
 187                         ret |= __get_user_inatomic(temp.v[6], p++);
 188                         ret |= __get_user_inatomic(temp.v[7], p++);
 189                         if (unlikely(ret))
 190                                 return -EFAULT;
 191                 }
 192
 193                 switch (instr) {
 194                 case EVLDD:
 195                 case EVLDW:
 196                 case EVLDH:
 197                         data.ll = temp.ll;
 198                         break;
 199                 case EVLHHESPLAT:
 200                         data.h[0] = temp.h[3];
 201                         data.h[2] = temp.h[3];
 202                         break;
 203                 case EVLHHOUSPLAT:
 204                 case EVLHHOSSPLAT:
 205                         data.h[1] = temp.h[3];
 206                         data.h[3] = temp.h[3];
 207                         break;
 208                 case EVLWHE:
 209                         data.h[0] = temp.h[2];
 210                         data.h[2] = temp.h[3];
 211                         break;
 212                 case EVLWHOU:
 213                 case EVLWHOS:
 214                         data.h[1] = temp.h[2];
 215                         data.h[3] = temp.h[3];
 216                         break;
 217                 case EVLWWSPLAT:
 218                         data.w[0] = temp.w[1];
 219                         data.w[1] = temp.w[1];
 220                         break;
 221                 case EVLWHSPLAT:
 222                         data.h[0] = temp.h[2];
 223                         data.h[1] = temp.h[2];
 224                         data.h[2] = temp.h[3];
 225                         data.h[3] = temp.h[3];
 226                         break;
 227                 default:
 228                         return -EINVAL;
 229                 }
 230         }
 231
 232         if (flags & SW) {
 233                 switch (flags & 0xf0) {
 234                 case E8:
 235                         data.ll = swab64(data.ll);
 236                         break;
 237                 case E4:
 238                         data.w[0] = swab32(data.w[0]);
 239                         data.w[1] = swab32(data.w[1]);
 240                         break;
 241                 /* Its half word endian */
 242                 default:
 243                         data.h[0] = swab16(data.h[0]);
 244                         data.h[1] = swab16(data.h[1]);
 245                         data.h[2] = swab16(data.h[2]);
 246                         data.h[3] = swab16(data.h[3]);
 247                         break;
 248                 }
 249         }
 250
 251         if (flags & SE) {
 252                 data.w[0] = (s16)data.h[1];
 253                 data.w[1] = (s16)data.h[3];
 254         }
 255
 256         /* Store result to memory or update registers */
 257         if (flags & ST) {
 258                 ret = 0;
 259                 p = addr;
 260                 switch (nb) {
 261                 case 8:
 262                         ret |= __put_user_inatomic(data.v[0], p++);
 263                         ret |= __put_user_inatomic(data.v[1], p++);
 264                         ret |= __put_user_inatomic(data.v[2], p++);
 265                         ret |= __put_user_inatomic(data.v[3], p++);
 266                         fallthrough;
 267                 case 4:
 268                         ret |= __put_user_inatomic(data.v[4], p++);
 269                         ret |= __put_user_inatomic(data.v[5], p++);
 270                         fallthrough;
 271                 case 2:
 272                         ret |= __put_user_inatomic(data.v[6], p++);
 273                         ret |= __put_user_inatomic(data.v[7], p++);
 274                 }
 275                 if (unlikely(ret))
 276                         return -EFAULT;
 277         } else {
 278                 *evr = data.w[0];
 279                 regs->gpr[reg] = data.w[1];
 280         }
 281
 282         return 1;
 283 }
 284 #endif /* CONFIG_SPE */
 285
 286 /*
 287  * Called on alignment exception. Attempts to fixup
 288  *
 289  * Return 1 on success
 290  * Return 0 if unable to handle the interrupt
 291  * Return -EFAULT if data address is bad
 292  * Other negative return values indicate that the instruction can't
 293  * be emulated, and the process should be given a SIGBUS.
 294  */
 295
 296 int fix_alignment(struct pt_regs *regs)
 297 {
 298         struct ppc_inst instr;
 299         struct instruction_op op;
 300         int r, type;
 301
 302         /*
 303          * We require a complete register set, if not, then our assembly
 304          * is broken
 305          */
 306         CHECK_FULL_REGS(regs);
 307
 308         if (unlikely(__get_user_instr(instr, (void __user *)regs->nip)))
 309                 return -EFAULT;
 310         if ((regs->msr & MSR_LE) != (MSR_KERNEL & MSR_LE)) {
 311                 /* We don't handle PPC little-endian any more... */
 312                 if (cpu_has_feature(CPU_FTR_PPC_LE))
 313                         return -EIO;
 314                 instr = ppc_inst_swab(instr);
 315         }
 316
 317 #ifdef CONFIG_SPE
 318         if (ppc_inst_primary_opcode(instr) == 0x4) {
 319                 int reg = (ppc_inst_val(instr) >> 21) & 0x1f;
 320                 PPC_WARN_ALIGNMENT(spe, regs);
 321                 return emulate_spe(regs, reg, instr);
 322         }
 323 #endif
 324
 325
 326         /*
 327          * ISA 3.0 (such as P9) copy, copy_first, paste and paste_last alignment
 328          * check.
 329          *
 330          * Send a SIGBUS to the process that caused the fault.
 331          *
 332          * We do not emulate these because paste may contain additional metadata
 333          * when pasting to a co-processor. Furthermore, paste_last is the
 334          * synchronisation point for preceding copy/paste sequences.
 335          */
 336         if ((ppc_inst_val(instr) & 0xfc0006fe) == (PPC_INST_COPY & 0xfc0006fe))
 337                 return -EIO;
 338
 339         r = analyse_instr(&op, regs, instr);
 340         if (r < 0)
 341                 return -EINVAL;
 342
 343         type = GETTYPE(op.type);
 344         if (!OP_IS_LOAD_STORE(type)) {
 345                 if (op.type != CACHEOP + DCBZ)
 346                         return -EINVAL;
 347                 PPC_WARN_ALIGNMENT(dcbz, regs);
 348                 r = emulate_dcbz(op.ea, regs);
 349         } else {
 350                 if (type == LARX || type == STCX)
 351                         return -EIO;
 352                 PPC_WARN_ALIGNMENT(unaligned, regs);
 353                 r = emulate_loadstore(regs, &op);
 354         }
 355
 356         if (!r)
 357                 return 1;
 358         return r;
 359 }