1 /* $Id: unaligned.c,v 1.24 2002/02/09 19:49:31 davem Exp $
2 * unaligned.c: Unaligned load/store trap handling with special
3 * cases for the kernel to do them more quickly.
5 * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu)
6 * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
10 #include <linux/kernel.h>
11 #include <linux/sched.h>
13 #include <linux/module.h>
15 #include <asm/ptrace.h>
16 #include <asm/pstate.h>
17 #include <asm/processor.h>
18 #include <asm/system.h>
19 #include <asm/uaccess.h>
20 #include <linux/smp.h>
21 #include <linux/smp_lock.h>
22 #include <linux/bitops.h>
23 #include <asm/fpumacro.h>
25 /* #define DEBUG_MNA */
28 load
, /* ld, ldd, ldh, ldsh */
29 store
, /* st, std, sth, stsh */
30 both
, /* Swap, ldstub, cas, ... */
37 static char *dirstrings
[] = {
38 "load", "store", "both", "fpload", "fpstore", "invalid"
42 static inline enum direction
decode_direction(unsigned int insn
)
44 unsigned long tmp
= (insn
>> 21) & 1;
49 switch ((insn
>>19)&0xf) {
58 /* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */
59 static inline int decode_access_size(unsigned int insn
)
63 tmp
= ((insn
>> 19) & 0xf);
64 if (tmp
== 11 || tmp
== 14) /* ldx/stx */
70 return 16; /* ldd/std - Although it is actually 8 */
74 printk("Impossible unaligned trap. insn=%08x\n", insn
);
75 die_if_kernel("Byte sized unaligned access?!?!", current_thread_info()->kregs
);
77 /* GCC should never warn that control reaches the end
78 * of this function without returning a value because
79 * die_if_kernel() is marked with attribute 'noreturn'.
80 * Alas, some versions do...
87 static inline int decode_asi(unsigned int insn
, struct pt_regs
*regs
)
89 if (insn
& 0x800000) {
91 return (unsigned char)(regs
->tstate
>> 24); /* %asi */
93 return (unsigned char)(insn
>> 5); /* imm_asi */
98 /* 0x400000 = signed, 0 = unsigned */
99 static inline int decode_signedness(unsigned int insn
)
101 return (insn
& 0x400000);
104 static inline void maybe_flush_windows(unsigned int rs1
, unsigned int rs2
,
105 unsigned int rd
, int from_kernel
)
107 if (rs2
>= 16 || rs1
>= 16 || rd
>= 16) {
108 if (from_kernel
!= 0)
109 __asm__
__volatile__("flushw");
115 static inline long sign_extend_imm13(long imm
)
117 return imm
<< 51 >> 51;
120 static unsigned long fetch_reg(unsigned int reg
, struct pt_regs
*regs
)
125 return (!reg
? 0 : regs
->u_regs
[reg
]);
126 if (regs
->tstate
& TSTATE_PRIV
) {
127 struct reg_window
*win
;
128 win
= (struct reg_window
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
129 value
= win
->locals
[reg
- 16];
130 } else if (test_thread_flag(TIF_32BIT
)) {
131 struct reg_window32 __user
*win32
;
132 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)regs
->u_regs
[UREG_FP
]));
133 get_user(value
, &win32
->locals
[reg
- 16]);
135 struct reg_window __user
*win
;
136 win
= (struct reg_window __user
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
137 get_user(value
, &win
->locals
[reg
- 16]);
142 static unsigned long *fetch_reg_addr(unsigned int reg
, struct pt_regs
*regs
)
145 return ®s
->u_regs
[reg
];
146 if (regs
->tstate
& TSTATE_PRIV
) {
147 struct reg_window
*win
;
148 win
= (struct reg_window
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
149 return &win
->locals
[reg
- 16];
150 } else if (test_thread_flag(TIF_32BIT
)) {
151 struct reg_window32
*win32
;
152 win32
= (struct reg_window32
*)((unsigned long)((u32
)regs
->u_regs
[UREG_FP
]));
153 return (unsigned long *)&win32
->locals
[reg
- 16];
155 struct reg_window
*win
;
156 win
= (struct reg_window
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
157 return &win
->locals
[reg
- 16];
161 unsigned long compute_effective_address(struct pt_regs
*regs
,
162 unsigned int insn
, unsigned int rd
)
164 unsigned int rs1
= (insn
>> 14) & 0x1f;
165 unsigned int rs2
= insn
& 0x1f;
166 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
169 maybe_flush_windows(rs1
, 0, rd
, from_kernel
);
170 return (fetch_reg(rs1
, regs
) + sign_extend_imm13(insn
));
172 maybe_flush_windows(rs1
, rs2
, rd
, from_kernel
);
173 return (fetch_reg(rs1
, regs
) + fetch_reg(rs2
, regs
));
177 /* This is just to make gcc think die_if_kernel does return... */
178 static void __attribute_used__
unaligned_panic(char *str
, struct pt_regs
*regs
)
180 die_if_kernel(str
, regs
);
183 extern void do_int_load(unsigned long *dest_reg
, int size
,
184 unsigned long *saddr
, int is_signed
, int asi
);
186 extern void __do_int_store(unsigned long *dst_addr
, int size
,
187 unsigned long *src_val
, int asi
);
189 static inline void do_int_store(int reg_num
, int size
, unsigned long *dst_addr
,
190 struct pt_regs
*regs
, int asi
)
192 unsigned long zero
= 0;
193 unsigned long *src_val
= &zero
;
197 zero
= (((long)(reg_num
?
198 (unsigned)fetch_reg(reg_num
, regs
) : 0)) << 32) |
199 (unsigned)fetch_reg(reg_num
+ 1, regs
);
200 } else if (reg_num
) {
201 src_val
= fetch_reg_addr(reg_num
, regs
);
203 __do_int_store(dst_addr
, size
, src_val
, asi
);
206 static inline void advance(struct pt_regs
*regs
)
208 regs
->tpc
= regs
->tnpc
;
210 if (test_thread_flag(TIF_32BIT
)) {
211 regs
->tpc
&= 0xffffffff;
212 regs
->tnpc
&= 0xffffffff;
216 static inline int floating_point_load_or_store_p(unsigned int insn
)
218 return (insn
>> 24) & 1;
221 static inline int ok_for_kernel(unsigned int insn
)
223 return !floating_point_load_or_store_p(insn
);
226 void kernel_mna_trap_fault(void)
228 struct pt_regs
*regs
= current_thread_info()->kern_una_regs
;
229 unsigned int insn
= current_thread_info()->kern_una_insn
;
230 unsigned long g2
= regs
->u_regs
[UREG_G2
];
231 unsigned long fixup
= search_extables_range(regs
->tpc
, &g2
);
234 unsigned long address
;
236 address
= compute_effective_address(regs
, insn
,
237 ((insn
>> 25) & 0x1f));
238 if (address
< PAGE_SIZE
) {
239 printk(KERN_ALERT
"Unable to handle kernel NULL "
240 "pointer dereference in mna handler");
242 printk(KERN_ALERT
"Unable to handle kernel paging "
243 "request in mna handler");
244 printk(KERN_ALERT
" at virtual address %016lx\n",address
);
245 printk(KERN_ALERT
"current->{active_,}mm->context = %016lx\n",
246 (current
->mm
? CTX_HWBITS(current
->mm
->context
) :
247 CTX_HWBITS(current
->active_mm
->context
)));
248 printk(KERN_ALERT
"current->{active_,}mm->pgd = %016lx\n",
249 (current
->mm
? (unsigned long) current
->mm
->pgd
:
250 (unsigned long) current
->active_mm
->pgd
));
251 die_if_kernel("Oops", regs
);
255 regs
->tnpc
= regs
->tpc
+ 4;
256 regs
->u_regs
[UREG_G2
] = g2
;
258 regs
->tstate
&= ~TSTATE_ASI
;
259 regs
->tstate
|= (ASI_AIUS
<< 24UL);
262 asmlinkage
void kernel_unaligned_trap(struct pt_regs
*regs
, unsigned int insn
, unsigned long sfar
, unsigned long sfsr
)
264 enum direction dir
= decode_direction(insn
);
265 int size
= decode_access_size(insn
);
267 current_thread_info()->kern_una_regs
= regs
;
268 current_thread_info()->kern_una_insn
= insn
;
270 if (!ok_for_kernel(insn
) || dir
== both
) {
271 printk("Unsupported unaligned load/store trap for kernel "
272 "at <%016lx>.\n", regs
->tpc
);
273 unaligned_panic("Kernel does fpu/atomic "
274 "unaligned load/store.", regs
);
276 kernel_mna_trap_fault();
280 addr
= compute_effective_address(regs
, insn
,
281 ((insn
>> 25) & 0x1f));
283 printk("KMNA: pc=%016lx [dir=%s addr=%016lx size=%d] "
285 regs
->tpc
, dirstrings
[dir
], addr
, size
,
286 regs
->u_regs
[UREG_RETPC
]);
290 do_int_load(fetch_reg_addr(((insn
>>25)&0x1f), regs
),
291 size
, (unsigned long *) addr
,
292 decode_signedness(insn
),
293 decode_asi(insn
, regs
));
297 do_int_store(((insn
>>25)&0x1f), size
,
298 (unsigned long *) addr
, regs
,
299 decode_asi(insn
, regs
));
303 panic("Impossible kernel unaligned trap.");
310 static char popc_helper
[] = {
311 0, 1, 1, 2, 1, 2, 2, 3,
312 1, 2, 2, 3, 2, 3, 3, 4,
315 int handle_popc(u32 insn
, struct pt_regs
*regs
)
318 int ret
, i
, rd
= ((insn
>> 25) & 0x1f);
319 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
322 maybe_flush_windows(0, 0, rd
, from_kernel
);
323 value
= sign_extend_imm13(insn
);
325 maybe_flush_windows(0, insn
& 0x1f, rd
, from_kernel
);
326 value
= fetch_reg(insn
& 0x1f, regs
);
328 for (ret
= 0, i
= 0; i
< 16; i
++) {
329 ret
+= popc_helper
[value
& 0xf];
334 regs
->u_regs
[rd
] = ret
;
336 if (test_thread_flag(TIF_32BIT
)) {
337 struct reg_window32 __user
*win32
;
338 win32
= (struct reg_window32 __user
*)((unsigned long)((u32
)regs
->u_regs
[UREG_FP
]));
339 put_user(ret
, &win32
->locals
[rd
- 16]);
341 struct reg_window __user
*win
;
342 win
= (struct reg_window __user
*)(regs
->u_regs
[UREG_FP
] + STACK_BIAS
);
343 put_user(ret
, &win
->locals
[rd
- 16]);
350 extern void do_fpother(struct pt_regs
*regs
);
351 extern void do_privact(struct pt_regs
*regs
);
352 extern void data_access_exception(struct pt_regs
*regs
,
356 int handle_ldf_stq(u32 insn
, struct pt_regs
*regs
)
358 unsigned long addr
= compute_effective_address(regs
, insn
, 0);
359 int freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
360 struct fpustate
*f
= FPUSTATE
;
361 int asi
= decode_asi(insn
, regs
);
362 int flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
364 save_and_clear_fpu();
365 current_thread_info()->xfsr
[0] &= ~0x1c000;
367 current_thread_info()->xfsr
[0] |= (6 << 14) /* invalid_fp_register */;
371 if (insn
& 0x200000) {
373 u64 first
= 0, second
= 0;
375 if (current_thread_info()->fpsaved
[0] & flag
) {
376 first
= *(u64
*)&f
->regs
[freg
];
377 second
= *(u64
*)&f
->regs
[freg
+2];
389 /* Need to convert endians */
390 u64 tmp
= __swab64p(&first
);
392 first
= __swab64p(&second
);
397 data_access_exception(regs
, 0, addr
);
400 if (put_user (first
>> 32, (u32 __user
*)addr
) ||
401 __put_user ((u32
)first
, (u32 __user
*)(addr
+ 4)) ||
402 __put_user (second
>> 32, (u32 __user
*)(addr
+ 8)) ||
403 __put_user ((u32
)second
, (u32 __user
*)(addr
+ 12))) {
404 data_access_exception(regs
, 0, addr
);
408 /* LDF, LDDF, LDQF */
409 u32 data
[4] __attribute__ ((aligned(8)));
416 } else if (asi
> ASI_SNFL
) {
417 data_access_exception(regs
, 0, addr
);
420 switch (insn
& 0x180000) {
421 case 0x000000: size
= 1; break;
422 case 0x100000: size
= 4; break;
423 default: size
= 2; break;
425 for (i
= 0; i
< size
; i
++)
428 err
= get_user (data
[0], (u32 __user
*) addr
);
430 for (i
= 1; i
< size
; i
++)
431 err
|= __get_user (data
[i
], (u32 __user
*)(addr
+ 4*i
));
433 if (err
&& !(asi
& 0x2 /* NF */)) {
434 data_access_exception(regs
, 0, addr
);
437 if (asi
& 0x8) /* Little */ {
441 case 1: data
[0] = le32_to_cpup(data
+ 0); break;
442 default:*(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 0));
444 case 4: tmp
= le64_to_cpup((u64
*)(data
+ 0));
445 *(u64
*)(data
+ 0) = le64_to_cpup((u64
*)(data
+ 2));
446 *(u64
*)(data
+ 2) = tmp
;
450 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
451 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
452 current_thread_info()->gsr
[0] = 0;
454 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
456 memset(f
->regs
, 0, 32*sizeof(u32
));
458 memset(f
->regs
+32, 0, 32*sizeof(u32
));
460 memcpy(f
->regs
+ freg
, data
, size
* 4);
461 current_thread_info()->fpsaved
[0] |= flag
;
467 void handle_ld_nf(u32 insn
, struct pt_regs
*regs
)
469 int rd
= ((insn
>> 25) & 0x1f);
470 int from_kernel
= (regs
->tstate
& TSTATE_PRIV
) != 0;
473 maybe_flush_windows(0, 0, rd
, from_kernel
);
474 reg
= fetch_reg_addr(rd
, regs
);
475 if (from_kernel
|| rd
< 16) {
477 if ((insn
& 0x780000) == 0x180000)
479 } else if (test_thread_flag(TIF_32BIT
)) {
480 put_user(0, (int __user
*) reg
);
481 if ((insn
& 0x780000) == 0x180000)
482 put_user(0, ((int __user
*) reg
) + 1);
484 put_user(0, (unsigned long __user
*) reg
);
485 if ((insn
& 0x780000) == 0x180000)
486 put_user(0, (unsigned long __user
*) reg
+ 1);
491 void handle_lddfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
493 unsigned long pc
= regs
->tpc
;
494 unsigned long tstate
= regs
->tstate
;
500 struct fpustate
*f
= FPUSTATE
;
502 if (tstate
& TSTATE_PRIV
)
503 die_if_kernel("lddfmna from kernel", regs
);
504 if (test_thread_flag(TIF_32BIT
))
506 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
508 if ((asi
> ASI_SNFL
) ||
511 if (get_user(first
, (u32 __user
*)sfar
) ||
512 get_user(second
, (u32 __user
*)(sfar
+ 4))) {
513 if (asi
& 0x2) /* NF */ {
514 first
= 0; second
= 0;
518 save_and_clear_fpu();
519 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
520 value
= (((u64
)first
) << 32) | second
;
521 if (asi
& 0x8) /* Little */
522 value
= __swab64p(&value
);
523 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
524 if (!(current_thread_info()->fpsaved
[0] & FPRS_FEF
)) {
525 current_thread_info()->fpsaved
[0] = FPRS_FEF
;
526 current_thread_info()->gsr
[0] = 0;
528 if (!(current_thread_info()->fpsaved
[0] & flag
)) {
530 memset(f
->regs
, 0, 32*sizeof(u32
));
532 memset(f
->regs
+32, 0, 32*sizeof(u32
));
534 *(u64
*)(f
->regs
+ freg
) = value
;
535 current_thread_info()->fpsaved
[0] |= flag
;
537 daex
: data_access_exception(regs
, sfsr
, sfar
);
544 void handle_stdfmna(struct pt_regs
*regs
, unsigned long sfar
, unsigned long sfsr
)
546 unsigned long pc
= regs
->tpc
;
547 unsigned long tstate
= regs
->tstate
;
552 struct fpustate
*f
= FPUSTATE
;
554 if (tstate
& TSTATE_PRIV
)
555 die_if_kernel("stdfmna from kernel", regs
);
556 if (test_thread_flag(TIF_32BIT
))
558 if (get_user(insn
, (u32 __user
*) pc
) != -EFAULT
) {
559 freg
= ((insn
>> 25) & 0x1e) | ((insn
>> 20) & 0x20);
562 flag
= (freg
< 32) ? FPRS_DL
: FPRS_DU
;
563 if ((asi
> ASI_SNFL
) ||
566 save_and_clear_fpu();
567 if (current_thread_info()->fpsaved
[0] & flag
)
568 value
= *(u64
*)&f
->regs
[freg
];
574 value
= __swab64p(&value
); break;
577 if (put_user (value
>> 32, (u32 __user
*) sfar
) ||
578 __put_user ((u32
)value
, (u32 __user
*)(sfar
+ 4)))
581 daex
: data_access_exception(regs
, sfsr
, sfar
);