2 * unaligned.c: Unaligned load/store trap handling with special
3 * cases for the kernel to do them more quickly.
5 * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net)
6 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
10 #include <linux/jiffies.h>
11 #include <linux/kernel.h>
12 #include <linux/sched.h>
14 #include <linux/module.h>
16 #include <asm/ptrace.h>
17 #include <asm/pstate.h>
18 #include <asm/processor.h>
19 #include <asm/uaccess.h>
20 #include <linux/smp.h>
21 #include <linux/bitops.h>
22 #include <linux/perf_event.h>
23 #include <linux/ratelimit.h>
24 #include <asm/fpumacro.h>
25 #include <asm/cacheflush.h>
28 load
, /* ld, ldd, ldh, ldsh */
29 store
, /* st, std, sth, stsh */
30 both
, /* Swap, ldstub, cas, ... */
36 static inline enum direction
decode_direction(unsigned int insn
)
38 unsigned long tmp
= (insn
>> 21) & 1;
43 switch ((insn
>>19)&0xf) {
52 /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */
53 static inline int decode_access_size(struct pt_regs
*regs
, unsigned int insn
)
57 tmp
= ((insn
>> 19) & 0xf);
58 if (tmp
== 11 || tmp
== 14) /* ldx/stx */
64 return 16; /* ldd/std - Although it is actually 8 */
68 printk("Impossible unaligned trap. insn=%08x\n", insn
);
69 die_if_kernel("Byte sized unaligned access?!?!", regs
);
71 /* GCC should never warn that control reaches the end
72 * of this function without returning a value because
73 * die_if_kernel() is marked with attribute 'noreturn'.
74 * Alas, some versions do...
81 static inline int decode_asi(unsigned int insn
, struct pt_regs
*regs
)
83 if (insn
& 0x800000) {
85 return (unsigned char)(regs
->tstate
>> 24); /* %asi */
87 return (unsigned char)(insn
>> 5); /* imm_asi */
92 /* 0x400000 = signed, 0 = unsigned */
93 static inline int decode_signedness(unsigned int insn
)
95 return (insn
& 0x400000);
98 static inline void maybe_flush_windows(unsigned int rs1
, unsigned int rs2
,
99 unsigned int rd
, int from_kernel
)
101 if (rs2
>= 16 || rs1
>= 16 || rd
>= 16) {
102 if (from_kernel
!= 0)
103 __asm__
__volatile__("flushw");
109 static inline long sign_extend_imm13(long imm
)
111 return imm
<< 51 >> 51;
114 static unsigned long fetch_reg(unsigned int reg
, struct pt_regs
*regs
)
116 unsigned long value
, fp
;
119 return (!reg
? 0 : regs
->u_regs
[reg
]);
121 fp
= regs
->u_regs
[UREG_FP
];
123 if (regs
->tstate
& TSTATE_PRIV
) {
124 struct reg_window
*win
;
125 win
= (struct reg_window
*)(fp
+ STACK_BIAS
);
126 value
= win
->locals
[reg
- 16];
127 } else if (!test_thread_64bit_stack(fp
)) {
128 struct reg_window32 __user
*win32
;
129 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)fp
));
130 get_user(value
, &win32
->locals
[reg
- 16]);
132 struct reg_window __user
*win
;
133 win
= (struct reg_window __user
*)(fp
+ STACK_BIAS
);
134 get_user(value
, &win
->locals
[reg
- 16]);
139 static unsigned long *fetch_reg_addr(unsigned int reg
, struct pt_regs
*regs
)
144 return ®s
->u_regs
[reg
];
146 fp
= regs
->u_regs
[UREG_FP
];
148 if (regs
->tstate
& TSTATE_PRIV
) {
149 struct reg_window
*win
;
150 win
= (struct reg_window
*)(fp
+ STACK_BIAS
);
151 return &win
->locals
[reg
- 16];
152 } else if (!test_thread_64bit_stack(fp
)) {
153 struct reg_window32
*win32
;
154 win32
= (struct reg_window32
*)((unsigned long)((u32
)fp
));
155 return (unsigned long *)&win32
->locals
[reg
- 16];
157 struct reg_window
*win
;
158 win
= (struct reg_window
*)(fp
+ STACK_BIAS
);
159 return &win
->locals
[reg
- 16];
163 unsigned long compute_effective_address(struct pt_regs
*regs
,
164 unsigned int insn
, unsigned int rd
)
166 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
167 unsigned int rs1
= (insn
>> 14) & 0x1f;
168 unsigned int rs2
= insn
& 0x1f;
172 maybe_flush_windows(rs1
, 0, rd
, from_kernel
);
173 addr
= (fetch_reg(rs1
, regs
) + sign_extend_imm13(insn
));
175 maybe_flush_windows(rs1
, rs2
, rd
, from_kernel
);
176 addr
= (fetch_reg(rs1
, regs
) + fetch_reg(rs2
, regs
));
179 if (!from_kernel
&& test_thread_flag(TIF_32BIT
))
185 /* This is just to make gcc think die_if_kernel does return... */
186 static void __used
unaligned_panic(char *str
, struct pt_regs
*regs
)
188 die_if_kernel(str
, regs
);
191 extern int do_int_load(unsigned long *dest_reg
, int size
,
192 unsigned long *saddr
, int is_signed
, int asi
);
194 extern int __do_int_store(unsigned long *dst_addr
, int size
,
195 unsigned long src_val
, int asi
);
197 static inline int do_int_store(int reg_num
, int size
, unsigned long *dst_addr
,
198 struct pt_regs
*regs
, int asi
, int orig_asi
)
200 unsigned long zero
= 0;
201 unsigned long *src_val_p
= &zero
;
202 unsigned long src_val
;
206 zero
= (((long)(reg_num
?
207 (unsigned)fetch_reg(reg_num
, regs
) : 0)) << 32) |
208 (unsigned)fetch_reg(reg_num
+ 1, regs
);
209 } else if (reg_num
) {
210 src_val_p
= fetch_reg_addr(reg_num
, regs
);
212 src_val
= *src_val_p
;
213 if (unlikely(asi
!= orig_asi
)) {
216 src_val
= swab16(src_val
);
219 src_val
= swab32(src_val
);
222 src_val
= swab64(src_val
);
230 return __do_int_store(dst_addr
, size
, src_val
, asi
);
233 static inline void advance(struct pt_regs
*regs
)
235 regs
->tpc
= regs
->tnpc
;
237 if (test_thread_flag(TIF_32BIT
)) {
238 regs
->tpc
&= 0xffffffff;
239 regs
->tnpc
&= 0xffffffff;
243 static inline int floating_point_load_or_store_p(unsigned int insn
)
245 return (insn
>> 24) & 1;
248 static inline int ok_for_kernel(unsigned int insn
)
250 return !floating_point_load_or_store_p(insn
);
253 static void kernel_mna_trap_fault(int fixup_tstate_asi
)
255 struct pt_regs
*regs
= current_thread_info()->kern_una_regs
;
256 unsigned int insn
= current_thread_info()->kern_una_insn
;
257 const struct exception_table_entry
*entry
;
259 entry
= search_exception_tables(regs
->tpc
);
261 unsigned long address
;
263 address
= compute_effective_address(regs
, insn
,
264 ((insn
>> 25) & 0x1f));
265 if (address
< PAGE_SIZE
) {
266 printk(KERN_ALERT
"Unable to handle kernel NULL "
267 "pointer dereference in mna handler");
269 printk(KERN_ALERT
"Unable to handle kernel paging "
270 "request in mna handler");
271 printk(KERN_ALERT
" at virtual address %016lx\n",address
);
272 printk(KERN_ALERT
"current->{active_,}mm->context = %016lx\n",
273 (current
->mm
? CTX_HWBITS(current
->mm
->context
) :
274 CTX_HWBITS(current
->active_mm
->context
)));
275 printk(KERN_ALERT
"current->{active_,}mm->pgd = %016lx\n",
276 (current
->mm
? (unsigned long) current
->mm
->pgd
:
277 (unsigned long) current
->active_mm
->pgd
));
278 die_if_kernel("Oops", regs
);
281 regs
->tpc
= entry
->fixup
;
282 regs
->tnpc
= regs
->tpc
+ 4;
284 if (fixup_tstate_asi
) {
285 regs
->tstate
&= ~TSTATE_ASI
;
286 regs
->tstate
|= (ASI_AIUS
<< 24UL);
290 static void log_unaligned(struct pt_regs
*regs
)
292 static DEFINE_RATELIMIT_STATE(ratelimit
, 5 * HZ
, 5);
294 if (__ratelimit(&ratelimit
)) {
295 printk("Kernel unaligned access at TPC[%lx] %pS\n",
296 regs
->tpc
, (void *) regs
->tpc
);
300 asmlinkage
void kernel_unaligned_trap(struct pt_regs
*regs
, unsigned int insn
)
302 enum direction dir
= decode_direction(insn
);
303 int size
= decode_access_size(regs
, insn
);
306 current_thread_info()->kern_una_regs
= regs
;
307 current_thread_info()->kern_una_insn
= insn
;
309 orig_asi
= asi
= decode_asi(insn
, regs
);
311 /* If this is a {get,put}_user() on an unaligned userspace pointer,
312 * just signal a fault and do not log the event.
314 if (asi
== ASI_AIUS
) {
315 kernel_mna_trap_fault(0);
321 if (!ok_for_kernel(insn
) || dir
== both
) {
322 printk("Unsupported unaligned load/store trap for kernel "
323 "at <%016lx>.\n", regs
->tpc
);
324 unaligned_panic("Kernel does fpu/atomic "
325 "unaligned load/store.", regs
);
327 kernel_mna_trap_fault(0);
329 unsigned long addr
, *reg_addr
;
332 addr
= compute_effective_address(regs
, insn
,
333 ((insn
>> 25) & 0x1f));
334 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, regs
, addr
);
348 reg_addr
= fetch_reg_addr(((insn
>>25)&0x1f), regs
);
349 err
= do_int_load(reg_addr
, size
,
350 (unsigned long *) addr
,
351 decode_signedness(insn
), asi
);
352 if (likely(!err
) && unlikely(asi
!= orig_asi
)) {
353 unsigned long val_in
= *reg_addr
;
356 val_in
= swab16(val_in
);
359 val_in
= swab32(val_in
);
362 val_in
= swab64(val_in
);
374 err
= do_int_store(((insn
>>25)&0x1f), size
,
375 (unsigned long *) addr
, regs
,
380 panic("Impossible kernel unaligned trap.");
384 kernel_mna_trap_fault(1);
390 int handle_popc(u32 insn
, struct pt_regs
*regs
)
392 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
393 int ret
, rd
= ((insn
>> 25) & 0x1f);
396 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, 0);
398 maybe_flush_windows(0, 0, rd
, from_kernel
);
399 value
= sign_extend_imm13(insn
);
401 maybe_flush_windows(0, insn
& 0x1f, rd
, from_kernel
);
402 value
= fetch_reg(insn
& 0x1f, regs
);
404 ret
= hweight64(value
);
407 regs
->u_regs
[rd
] = ret
;
409 unsigned long fp
= regs
->u_regs
[UREG_FP
];
411 if (!test_thread_64bit_stack(fp
)) {
412 struct reg_window32 __user
*win32
;
413 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)fp
));
414 put_user(ret
, &win32
->locals
[rd
- 16]);
416 struct reg_window __user
*win
;
417 win
= (struct reg_window __user
*)(fp
+ STACK_BIAS
);
418 put_user(ret
, &win
->locals
[rd
- 16]);
425 extern void do_fpother(struct pt_regs
*regs
);
426 extern void do_privact(struct pt_regs
*regs
);
427 extern void spitfire_data_access_exception(struct pt_regs
*regs
,
430 extern void sun4v_data_access_exception(struct pt_regs
*regs
,
432 unsigned long type_ctx
);
434 int handle_ldf_stq(u32 insn
, struct pt_regs
*regs
)
436 unsigned long addr
= compute_effective_address(regs
, insn
, 0);
437 int freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
438 struct fpustate
*f
= FPUSTATE
;
439 int asi
= decode_asi(insn
, regs
);
440 int flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
442 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, 0);
444 save_and_clear_fpu();
445 current_thread_info()->xfsr
[0] &= ~0x1c000;
447 current_thread_info()->xfsr
[0] |= (6 << 14) /* invalid_fp_register */;
451 if (insn
& 0x200000) {
453 u64 first
= 0, second
= 0;
455 if (current_thread_info()->fpsaved
[0] & flag
) {
456 first
= *(u64
*)&f
->regs
[freg
];
457 second
= *(u64
*)&f
->regs
[freg
+2];
469 /* Need to convert endians */
470 u64 tmp
= __swab64p(&first
);
472 first
= __swab64p(&second
);
477 if (tlb_type
== hypervisor
)
478 sun4v_data_access_exception(regs
, addr
, 0);
480 spitfire_data_access_exception(regs
, 0, addr
);
483 if (put_user (first
>> 32, (u32 __user
*)addr
) ||
484 __put_user ((u32
)first
, (u32 __user
*)(addr
+ 4)) ||
485 __put_user (second
>> 32, (u32 __user
*)(addr
+ 8)) ||
486 __put_user ((u32
)second
, (u32 __user
*)(addr
+ 12))) {
487 if (tlb_type
== hypervisor
)
488 sun4v_data_access_exception(regs
, addr
, 0);
490 spitfire_data_access_exception(regs
, 0, addr
);
494 /* LDF, LDDF, LDQF */
495 u32 data
[4] __attribute__ ((aligned(8)));
502 } else if (asi
> ASI_SNFL
) {
503 if (tlb_type
== hypervisor
)
504 sun4v_data_access_exception(regs
, addr
, 0);
506 spitfire_data_access_exception(regs
, 0, addr
);
509 switch (insn
& 0x180000) {
510 case 0x000000: size
= 1; break;
511 case 0x100000: size
= 4; break;
512 default: size
= 2; break;
514 for (i
= 0; i
< size
; i
++)
517 err
= get_user (data
[0], (u32 __user
*) addr
);
519 for (i
= 1; i
< size
; i
++)
520 err
|= __get_user (data
[i
], (u32 __user
*)(addr
+ 4*i
));
522 if (err
&& !(asi
& 0x2 /* NF */)) {
523 if (tlb_type
== hypervisor
)
524 sun4v_data_access_exception(regs
, addr
, 0);
526 spitfire_data_access_exception(regs
, 0, addr
);
529 if (asi
& 0x8) /* Little */ {
533 case 1: data
[0] = le32_to_cpup(data
+ 0); break;
534 default:*(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 0));
536 case 4: tmp
= le64_to_cpup((u64
*)(data
+ 0));
537 *(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 2));
538 *(u64
*)(data
+ 2) = tmp
;
542 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
543 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
544 current_thread_info()->gsr
[0] = 0;
546 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
548 memset(f
->regs
, 0, 32*sizeof(u32
));
550 memset(f
->regs
+32, 0, 32*sizeof(u32
));
552 memcpy(f
->regs
+ freg
, data
, size
* 4);
553 current_thread_info()->fpsaved
[0] |= flag
;
559 void handle_ld_nf(u32 insn
, struct pt_regs
*regs
)
561 int rd
= ((insn
>> 25) & 0x1f);
562 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
565 perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS
, 1, regs
, 0);
567 maybe_flush_windows(0, 0, rd
, from_kernel
);
568 reg
= fetch_reg_addr(rd
, regs
);
569 if (from_kernel
|| rd
< 16) {
571 if ((insn
& 0x780000) == 0x180000)
573 } else if (!test_thread_64bit_stack(regs
->u_regs
[UREG_FP
])) {
574 put_user(0, (int __user
*) reg
);
575 if ((insn
& 0x780000) == 0x180000)
576 put_user(0, ((int __user
*) reg
) + 1);
578 put_user(0, (unsigned long __user
*) reg
);
579 if ((insn
& 0x780000) == 0x180000)
580 put_user(0, (unsigned long __user
*) reg
+ 1);
585 void handle_lddfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
587 unsigned long pc
= regs
->tpc
;
588 unsigned long tstate
= regs
->tstate
;
593 struct fpustate
*f
= FPUSTATE
;
595 if (tstate
& TSTATE_PRIV
)
596 die_if_kernel("lddfmna from kernel", regs
);
597 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, regs
, sfar
);
598 if (test_thread_flag(TIF_32BIT
))
600 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
601 int asi
= decode_asi(insn
, regs
);
605 if ((asi
> ASI_SNFL
) ||
609 err
= get_user(first
, (u32 __user
*)sfar
);
611 err
= get_user(second
, (u32 __user
*)(sfar
+ 4));
617 save_and_clear_fpu();
618 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
619 value
= (((u64
)first
) << 32) | second
;
620 if (asi
& 0x8) /* Little */
621 value
= __swab64p(&value
);
622 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
623 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
624 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
625 current_thread_info()->gsr
[0] = 0;
627 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
629 memset(f
->regs
, 0, 32*sizeof(u32
));
631 memset(f
->regs
+32, 0, 32*sizeof(u32
));
633 *(u64
*)(f
->regs
+ freg
) = value
;
634 current_thread_info()->fpsaved
[0] |= flag
;
637 if (tlb_type
== hypervisor
)
638 sun4v_data_access_exception(regs
, sfar
, sfsr
);
640 spitfire_data_access_exception(regs
, sfsr
, sfar
);
646 void handle_stdfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
648 unsigned long pc
= regs
->tpc
;
649 unsigned long tstate
= regs
->tstate
;
654 struct fpustate
*f
= FPUSTATE
;
656 if (tstate
& TSTATE_PRIV
)
657 die_if_kernel("stdfmna from kernel", regs
);
658 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS
, 1, regs
, sfar
);
659 if (test_thread_flag(TIF_32BIT
))
661 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
662 int asi
= decode_asi(insn
, regs
);
663 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
665 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
666 if ((asi
> ASI_SNFL
) ||
669 save_and_clear_fpu();
670 if (current_thread_info()->fpsaved
[0] & flag
)
671 value
= *(u64
*)&f
->regs
[freg
];
677 value
= __swab64p(&value
); break;
680 if (put_user (value
>> 32, (u32 __user
*) sfar
) ||
681 __put_user ((u32
)value
, (u32 __user
*)(sfar
+ 4)))
685 if (tlb_type
== hypervisor
)
686 sun4v_data_access_exception(regs
, sfar
, sfsr
);
688 spitfire_data_access_exception(regs
, sfsr
, sfar
);