| blob | dfd35dcc1cb553cb9105f5f824250a3b388e40fb |
1 /*
2 * arch/xtensa/kernel/entry.S
3 *
4 * Low-level exception handling
5 *
6 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file "COPYING" in the main directory of this archive
8 * for more details.
9 *
10 * Copyright (C) 2004-2007 by Tensilica Inc.
11 *
12 * Chris Zankel <chris@zankel.net>
13 *
14 */
16 #include <linux/linkage.h>
17 #include <asm/asm-offsets.h>
18 #include <asm/processor.h>
19 #include <asm/thread_info.h>
20 #include <asm/uaccess.h>
21 #include <asm/unistd.h>
22 #include <asm/ptrace.h>
23 #include <asm/current.h>
24 #include <asm/pgtable.h>
25 #include <asm/page.h>
26 #include <asm/signal.h>
27 #include <asm/tlbflush.h>
28 #include <asm/variant/tie-asm.h>
30 /* Unimplemented features. */
32 #undef KERNEL_STACK_OVERFLOW_CHECK
33 #undef PREEMPTIBLE_KERNEL
34 #undef ALLOCA_EXCEPTION_IN_IRAM
36 /* Not well tested.
37 *
38 * - fast_coprocessor
39 */
41 /*
42 * Macro to find first bit set in WINDOWBASE from the left + 1
43 *
44 * 100....0 -> 1
45 * 010....0 -> 2
46 * 000....1 -> WSBITS
47 */
49 .macro ffs_ws bit mask
51 #if XCHAL_HAVE_NSA
52 nsau \bit, \mask # 32-WSBITS ... 31 (32 iff 0)
53 addi \bit, \bit, WSBITS - 32 + 1 # uppest bit set -> return 1
54 #else
55 movi \bit, WSBITS
56 #if WSBITS > 16
57 _bltui \mask, 0x10000, 99f
58 addi \bit, \bit, -16
59 extui \mask, \mask, 16, 16
60 #endif
61 #if WSBITS > 8
62 99: _bltui \mask, 0x100, 99f
63 addi \bit, \bit, -8
64 srli \mask, \mask, 8
65 #endif
66 99: _bltui \mask, 0x10, 99f
67 addi \bit, \bit, -4
68 srli \mask, \mask, 4
69 99: _bltui \mask, 0x4, 99f
70 addi \bit, \bit, -2
71 srli \mask, \mask, 2
72 99: _bltui \mask, 0x2, 99f
73 addi \bit, \bit, -1
74 99:
76 #endif
77 .endm
79 /* ----------------- DEFAULT FIRST LEVEL EXCEPTION HANDLERS ----------------- */
81 /*
82 * First-level exception handler for user exceptions.
83 * Save some special registers, extra states and all registers in the AR
84 * register file that were in use in the user task, and jump to the common
85 * exception code.
86 * We save SAR (used to calculate WMASK), and WB and WS (we don't have to
87 * save them for kernel exceptions).
88 *
89 * Entry condition for user_exception:
90 *
91 * a0: trashed, original value saved on stack (PT_AREG0)
92 * a1: a1
93 * a2: new stack pointer, original value in depc
94 * a3: dispatch table
95 * depc: a2, original value saved on stack (PT_DEPC)
96 * excsave1: a3
97 *
98 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
99 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
100 *
101 * Entry condition for _user_exception:
102 *
103 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
104 * excsave has been restored, and
105 * stack pointer (a1) has been set.
106 *
107 * Note: _user_exception might be at an odd adress. Don't use call0..call12
108 */
110 ENTRY(user_exception)
112 /* Save a2, a3, and depc, restore excsave_1 and set SP. */
114 xsr a3, EXCSAVE_1
115 rsr a0, DEPC
116 s32i a1, a2, PT_AREG1
117 s32i a0, a2, PT_AREG2
118 s32i a3, a2, PT_AREG3
119 mov a1, a2
121 .globl _user_exception
122 _user_exception:
124 /* Save SAR and turn off single stepping */
126 movi a2, 0
127 rsr a3, SAR
128 xsr a2, ICOUNTLEVEL
129 s32i a3, a1, PT_SAR
130 s32i a2, a1, PT_ICOUNTLEVEL
132 /* Rotate ws so that the current windowbase is at bit0. */
133 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
135 rsr a2, WINDOWBASE
136 rsr a3, WINDOWSTART
137 ssr a2
138 s32i a2, a1, PT_WINDOWBASE
139 s32i a3, a1, PT_WINDOWSTART
140 slli a2, a3, 32-WSBITS
141 src a2, a3, a2
142 srli a2, a2, 32-WSBITS
143 s32i a2, a1, PT_WMASK # needed for restoring registers
145 /* Save only live registers. */
147 _bbsi.l a2, 1, 1f
148 s32i a4, a1, PT_AREG4
149 s32i a5, a1, PT_AREG5
150 s32i a6, a1, PT_AREG6
151 s32i a7, a1, PT_AREG7
152 _bbsi.l a2, 2, 1f
153 s32i a8, a1, PT_AREG8
154 s32i a9, a1, PT_AREG9
155 s32i a10, a1, PT_AREG10
156 s32i a11, a1, PT_AREG11
157 _bbsi.l a2, 3, 1f
158 s32i a12, a1, PT_AREG12
159 s32i a13, a1, PT_AREG13
160 s32i a14, a1, PT_AREG14
161 s32i a15, a1, PT_AREG15
162 _bnei a2, 1, 1f # only one valid frame?
164 /* Only one valid frame, skip saving regs. */
166 j 2f
168 /* Save the remaining registers.
169 * We have to save all registers up to the first '1' from
170 * the right, except the current frame (bit 0).
171 * Assume a2 is: 001001000110001
172 * All register frames starting from the top field to the marked '1'
173 * must be saved.
174 */
176 1: addi a3, a2, -1 # eliminate '1' in bit 0: yyyyxxww0
177 neg a3, a3 # yyyyxxww0 -> YYYYXXWW1+1
178 and a3, a3, a2 # max. only one bit is set
180 /* Find number of frames to save */
182 ffs_ws a0, a3 # number of frames to the '1' from left
184 /* Store information into WMASK:
185 * bits 0..3: xxx1 masked lower 4 bits of the rotated windowstart,
186 * bits 4...: number of valid 4-register frames
187 */
189 slli a3, a0, 4 # number of frames to save in bits 8..4
190 extui a2, a2, 0, 4 # mask for the first 16 registers
191 or a2, a3, a2
192 s32i a2, a1, PT_WMASK # needed when we restore the reg-file
194 /* Save 4 registers at a time */
196 1: rotw -1
197 s32i a0, a5, PT_AREG_END - 16
198 s32i a1, a5, PT_AREG_END - 12
199 s32i a2, a5, PT_AREG_END - 8
200 s32i a3, a5, PT_AREG_END - 4
201 addi a0, a4, -1
202 addi a1, a5, -16
203 _bnez a0, 1b
205 /* WINDOWBASE still in SAR! */
207 rsr a2, SAR # original WINDOWBASE
208 movi a3, 1
209 ssl a2
210 sll a3, a3
211 wsr a3, WINDOWSTART # set corresponding WINDOWSTART bit
212 wsr a2, WINDOWBASE # and WINDOWSTART
213 rsync
215 /* We are back to the original stack pointer (a1) */
217 2: /* Now, jump to the common exception handler. */
219 j common_exception
222 /*
223 * First-level exit handler for kernel exceptions
224 * Save special registers and the live window frame.
225 * Note: Even though we changes the stack pointer, we don't have to do a
226 * MOVSP here, as we do that when we return from the exception.
227 * (See comment in the kernel exception exit code)
228 *
229 * Entry condition for kernel_exception:
230 *
231 * a0: trashed, original value saved on stack (PT_AREG0)
232 * a1: a1
233 * a2: new stack pointer, original in DEPC
234 * a3: dispatch table
235 * depc: a2, original value saved on stack (PT_DEPC)
236 * excsave_1: a3
237 *
238 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
239 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
240 *
241 * Entry condition for _kernel_exception:
242 *
243 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
244 * excsave has been restored, and
245 * stack pointer (a1) has been set.
246 *
247 * Note: _kernel_exception might be at an odd adress. Don't use call0..call12
248 */
250 ENTRY(kernel_exception)
252 /* Save a0, a2, a3, DEPC and set SP. */
254 xsr a3, EXCSAVE_1 # restore a3, excsave_1
255 rsr a0, DEPC # get a2
256 s32i a1, a2, PT_AREG1
257 s32i a0, a2, PT_AREG2
258 s32i a3, a2, PT_AREG3
259 mov a1, a2
261 .globl _kernel_exception
262 _kernel_exception:
264 /* Save SAR and turn off single stepping */
266 movi a2, 0
267 rsr a3, SAR
268 xsr a2, ICOUNTLEVEL
269 s32i a3, a1, PT_SAR
270 s32i a2, a1, PT_ICOUNTLEVEL
272 /* Rotate ws so that the current windowbase is at bit0. */
273 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
275 rsr a2, WINDOWBASE # don't need to save these, we only
276 rsr a3, WINDOWSTART # need shifted windowstart: windowmask
277 ssr a2
278 slli a2, a3, 32-WSBITS
279 src a2, a3, a2
280 srli a2, a2, 32-WSBITS
281 s32i a2, a1, PT_WMASK # needed for kernel_exception_exit
283 /* Save only the live window-frame */
285 _bbsi.l a2, 1, 1f
286 s32i a4, a1, PT_AREG4
287 s32i a5, a1, PT_AREG5
288 s32i a6, a1, PT_AREG6
289 s32i a7, a1, PT_AREG7
290 _bbsi.l a2, 2, 1f
291 s32i a8, a1, PT_AREG8
292 s32i a9, a1, PT_AREG9
293 s32i a10, a1, PT_AREG10
294 s32i a11, a1, PT_AREG11
295 _bbsi.l a2, 3, 1f
296 s32i a12, a1, PT_AREG12
297 s32i a13, a1, PT_AREG13
298 s32i a14, a1, PT_AREG14
299 s32i a15, a1, PT_AREG15
301 1:
303 #ifdef KERNEL_STACK_OVERFLOW_CHECK
305 /* Stack overflow check, for debugging */
306 extui a2, a1, TASK_SIZE_BITS,XX
307 movi a3, SIZE??
308 _bge a2, a3, out_of_stack_panic
310 #endif
312 /*
313 * This is the common exception handler.
314 * We get here from the user exception handler or simply by falling through
315 * from the kernel exception handler.
316 * Save the remaining special registers, switch to kernel mode, and jump
317 * to the second-level exception handler.
318 *
319 */
321 common_exception:
323 /* Save some registers, disable loops and clear the syscall flag. */
325 rsr a2, DEBUGCAUSE
326 rsr a3, EPC_1
327 s32i a2, a1, PT_DEBUGCAUSE
328 s32i a3, a1, PT_PC
330 movi a2, -1
331 rsr a3, EXCVADDR
332 s32i a2, a1, PT_SYSCALL
333 movi a2, 0
334 s32i a3, a1, PT_EXCVADDR
335 xsr a2, LCOUNT
336 s32i a2, a1, PT_LCOUNT
338 /* It is now save to restore the EXC_TABLE_FIXUP variable. */
340 rsr a0, EXCCAUSE
341 movi a3, 0
342 rsr a2, EXCSAVE_1
343 s32i a0, a1, PT_EXCCAUSE
344 s32i a3, a2, EXC_TABLE_FIXUP
346 /* All unrecoverable states are saved on stack, now, and a1 is valid,
347 * so we can allow exceptions and interrupts (*) again.
348 * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X)
349 *
350 * (*) We only allow interrupts if PS.INTLEVEL was not set to 1 before
351 * (interrupts disabled) and if this exception is not an interrupt.
352 */
354 rsr a3, PS
355 addi a0, a0, -4
356 movi a2, 1
357 extui a3, a3, 0, 1 # a3 = PS.INTLEVEL[0]
358 moveqz a3, a2, a0 # a3 = 1 iff interrupt exception
359 movi a2, 1 << PS_WOE_BIT
360 or a3, a3, a2
361 rsr a0, EXCCAUSE
362 xsr a3, PS
364 s32i a3, a1, PT_PS # save ps
366 /* Save LBEG, LEND */
368 rsr a2, LBEG
369 rsr a3, LEND
370 s32i a2, a1, PT_LBEG
371 s32i a3, a1, PT_LEND
373 /* Save optional registers. */
375 save_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
377 /* Go to second-level dispatcher. Set up parameters to pass to the
378 * exception handler and call the exception handler.
379 */
381 movi a4, exc_table
382 mov a6, a1 # pass stack frame
383 mov a7, a0 # pass EXCCAUSE
384 addx4 a4, a0, a4
385 l32i a4, a4, EXC_TABLE_DEFAULT # load handler
387 /* Call the second-level handler */
389 callx4 a4
391 /* Jump here for exception exit */
393 common_exception_return:
395 /* Jump if we are returning from kernel exceptions. */
397 1: l32i a3, a1, PT_PS
398 _bbci.l a3, PS_UM_BIT, 4f
400 /* Specific to a user exception exit:
401 * We need to check some flags for signal handling and rescheduling,
402 * and have to restore WB and WS, extra states, and all registers
403 * in the register file that were in use in the user task.
404 * Note that we don't disable interrupts here.
405 */
407 GET_THREAD_INFO(a2,a1)
408 l32i a4, a2, TI_FLAGS
410 _bbsi.l a4, TIF_NEED_RESCHED, 3f
411 _bbci.l a4, TIF_SIGPENDING, 4f
413 l32i a4, a1, PT_DEPC
414 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
416 /* Call do_signal() */
418 movi a4, do_signal # int do_signal(struct pt_regs*, sigset_t*)
419 mov a6, a1
420 movi a7, 0
421 callx4 a4
422 j 1b
424 3: /* Reschedule */
426 movi a4, schedule # void schedule (void)
427 callx4 a4
428 j 1b
430 4: /* Restore optional registers. */
432 load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
434 wsr a3, PS /* disable interrupts */
436 _bbci.l a3, PS_UM_BIT, kernel_exception_exit
438 user_exception_exit:
440 /* Restore the state of the task and return from the exception. */
442 /* Switch to the user thread WINDOWBASE. Save SP temporarily in DEPC */
444 l32i a2, a1, PT_WINDOWBASE
445 l32i a3, a1, PT_WINDOWSTART
446 wsr a1, DEPC # use DEPC as temp storage
447 wsr a3, WINDOWSTART # restore WINDOWSTART
448 ssr a2 # preserve user's WB in the SAR
449 wsr a2, WINDOWBASE # switch to user's saved WB
450 rsync
451 rsr a1, DEPC # restore stack pointer
452 l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9)
453 rotw -1 # we restore a4..a7
454 _bltui a6, 16, 1f # only have to restore current window?
456 /* The working registers are a0 and a3. We are restoring to
457 * a4..a7. Be careful not to destroy what we have just restored.
458 * Note: wmask has the format YYYYM:
459 * Y: number of registers saved in groups of 4
460 * M: 4 bit mask of first 16 registers
461 */
463 mov a2, a6
464 mov a3, a5
466 2: rotw -1 # a0..a3 become a4..a7
467 addi a3, a7, -4*4 # next iteration
468 addi a2, a6, -16 # decrementing Y in WMASK
469 l32i a4, a3, PT_AREG_END + 0
470 l32i a5, a3, PT_AREG_END + 4
471 l32i a6, a3, PT_AREG_END + 8
472 l32i a7, a3, PT_AREG_END + 12
473 _bgeui a2, 16, 2b
475 /* Clear unrestored registers (don't leak anything to user-land */
477 1: rsr a0, WINDOWBASE
478 rsr a3, SAR
479 sub a3, a0, a3
480 beqz a3, 2f
481 extui a3, a3, 0, WBBITS
483 1: rotw -1
484 addi a3, a7, -1
485 movi a4, 0
486 movi a5, 0
487 movi a6, 0
488 movi a7, 0
489 bgei a3, 1, 1b
491 /* We are back were we were when we started.
492 * Note: a2 still contains WMASK (if we've returned to the original
493 * frame where we had loaded a2), or at least the lower 4 bits
494 * (if we have restored WSBITS-1 frames).
495 */
497 2: j common_exception_exit
499 /* This is the kernel exception exit.
500 * We avoided to do a MOVSP when we entered the exception, but we
501 * have to do it here.
502 */
504 kernel_exception_exit:
506 #ifdef PREEMPTIBLE_KERNEL
508 #ifdef CONFIG_PREEMPT
510 /*
511 * Note: We've just returned from a call4, so we have
512 * at least 4 addt'l regs.
513 */
515 /* Check current_thread_info->preempt_count */
517 GET_THREAD_INFO(a2)
518 l32i a3, a2, TI_PREEMPT
519 bnez a3, 1f
521 l32i a2, a2, TI_FLAGS
523 1:
525 #endif
527 #endif
529 /* Check if we have to do a movsp.
530 *
531 * We only have to do a movsp if the previous window-frame has
532 * been spilled to the *temporary* exception stack instead of the
533 * task's stack. This is the case if the corresponding bit in
534 * WINDOWSTART for the previous window-frame was set before
535 * (not spilled) but is zero now (spilled).
536 * If this bit is zero, all other bits except the one for the
537 * current window frame are also zero. So, we can use a simple test:
538 * 'and' WINDOWSTART and WINDOWSTART-1:
539 *
540 * (XXXXXX1[0]* - 1) AND XXXXXX1[0]* = XXXXXX0[0]*
541 *
542 * The result is zero only if one bit was set.
543 *
544 * (Note: We might have gone through several task switches before
545 * we come back to the current task, so WINDOWBASE might be
546 * different from the time the exception occurred.)
547 */
549 /* Test WINDOWSTART before and after the exception.
550 * We actually have WMASK, so we only have to test if it is 1 or not.
551 */
553 l32i a2, a1, PT_WMASK
554 _beqi a2, 1, common_exception_exit # Spilled before exception,jump
556 /* Test WINDOWSTART now. If spilled, do the movsp */
558 rsr a3, WINDOWSTART
559 addi a0, a3, -1
560 and a3, a3, a0
561 _bnez a3, common_exception_exit
563 /* Do a movsp (we returned from a call4, so we have at least a0..a7) */
565 addi a0, a1, -16
566 l32i a3, a0, 0
567 l32i a4, a0, 4
568 s32i a3, a1, PT_SIZE+0
569 s32i a4, a1, PT_SIZE+4
570 l32i a3, a0, 8
571 l32i a4, a0, 12
572 s32i a3, a1, PT_SIZE+8
573 s32i a4, a1, PT_SIZE+12
575 /* Common exception exit.
576 * We restore the special register and the current window frame, and
577 * return from the exception.
578 *
579 * Note: We expect a2 to hold PT_WMASK
580 */
582 common_exception_exit:
584 /* Restore address registers. */
586 _bbsi.l a2, 1, 1f
587 l32i a4, a1, PT_AREG4
588 l32i a5, a1, PT_AREG5
589 l32i a6, a1, PT_AREG6
590 l32i a7, a1, PT_AREG7
591 _bbsi.l a2, 2, 1f
592 l32i a8, a1, PT_AREG8
593 l32i a9, a1, PT_AREG9
594 l32i a10, a1, PT_AREG10
595 l32i a11, a1, PT_AREG11
596 _bbsi.l a2, 3, 1f
597 l32i a12, a1, PT_AREG12
598 l32i a13, a1, PT_AREG13
599 l32i a14, a1, PT_AREG14
600 l32i a15, a1, PT_AREG15
602 /* Restore PC, SAR */
604 1: l32i a2, a1, PT_PC
605 l32i a3, a1, PT_SAR
606 wsr a2, EPC_1
607 wsr a3, SAR
609 /* Restore LBEG, LEND, LCOUNT */
611 l32i a2, a1, PT_LBEG
612 l32i a3, a1, PT_LEND
613 wsr a2, LBEG
614 l32i a2, a1, PT_LCOUNT
615 wsr a3, LEND
616 wsr a2, LCOUNT
618 /* We control single stepping through the ICOUNTLEVEL register. */
620 l32i a2, a1, PT_ICOUNTLEVEL
621 movi a3, -2
622 wsr a2, ICOUNTLEVEL
623 wsr a3, ICOUNT
625 /* Check if it was double exception. */
627 l32i a0, a1, PT_DEPC
628 l32i a3, a1, PT_AREG3
629 l32i a2, a1, PT_AREG2
630 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
632 /* Restore a0...a3 and return */
634 l32i a0, a1, PT_AREG0
635 l32i a1, a1, PT_AREG1
636 rfe
638 1: wsr a0, DEPC
639 l32i a0, a1, PT_AREG0
640 l32i a1, a1, PT_AREG1
641 rfde
643 /*
644 * Debug exception handler.
645 *
646 * Currently, we don't support KGDB, so only user application can be debugged.
647 *
648 * When we get here, a0 is trashed and saved to excsave[debuglevel]
649 */
651 ENTRY(debug_exception)
653 rsr a0, EPS + XCHAL_DEBUGLEVEL
654 bbsi.l a0, PS_EXCM_BIT, 1f # exception mode
656 /* Set EPC_1 and EXCCAUSE */
658 wsr a2, DEPC # save a2 temporarily
659 rsr a2, EPC + XCHAL_DEBUGLEVEL
660 wsr a2, EPC_1
662 movi a2, EXCCAUSE_MAPPED_DEBUG
663 wsr a2, EXCCAUSE
665 /* Restore PS to the value before the debug exc but with PS.EXCM set.*/
667 movi a2, 1 << PS_EXCM_BIT
668 or a2, a0, a2
669 movi a0, debug_exception # restore a3, debug jump vector
670 wsr a2, PS
671 xsr a0, EXCSAVE + XCHAL_DEBUGLEVEL
673 /* Switch to kernel/user stack, restore jump vector, and save a0 */
675 bbsi.l a2, PS_UM_BIT, 2f # jump if user mode
677 addi a2, a1, -16-PT_SIZE # assume kernel stack
678 s32i a0, a2, PT_AREG0
679 movi a0, 0
680 s32i a1, a2, PT_AREG1
681 s32i a0, a2, PT_DEPC # mark it as a regular exception
682 xsr a0, DEPC
683 s32i a3, a2, PT_AREG3
684 s32i a0, a2, PT_AREG2
685 mov a1, a2
686 j _kernel_exception
688 2: rsr a2, EXCSAVE_1
689 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer
690 s32i a0, a2, PT_AREG0
691 movi a0, 0
692 s32i a1, a2, PT_AREG1
693 s32i a0, a2, PT_DEPC
694 xsr a0, DEPC
695 s32i a3, a2, PT_AREG3
696 s32i a0, a2, PT_AREG2
697 mov a1, a2
698 j _user_exception
700 /* Debug exception while in exception mode. */
701 1: j 1b // FIXME!!
704 /*
705 * We get here in case of an unrecoverable exception.
706 * The only thing we can do is to be nice and print a panic message.
707 * We only produce a single stack frame for panic, so ???
708 *
709 *
710 * Entry conditions:
711 *
712 * - a0 contains the caller address; original value saved in excsave1.
713 * - the original a0 contains a valid return address (backtrace) or 0.
714 * - a2 contains a valid stackpointer
715 *
716 * Notes:
717 *
718 * - If the stack pointer could be invalid, the caller has to setup a
719 * dummy stack pointer (e.g. the stack of the init_task)
720 *
721 * - If the return address could be invalid, the caller has to set it
722 * to 0, so the backtrace would stop.
723 *
724 */
725 .align 4
726 unrecoverable_text:
727 .ascii "Unrecoverable error in exception handler\0"
729 ENTRY(unrecoverable_exception)
731 movi a0, 1
732 movi a1, 0
734 wsr a0, WINDOWSTART
735 wsr a1, WINDOWBASE
736 rsync
738 movi a1, (1 << PS_WOE_BIT) | 1
739 wsr a1, PS
740 rsync
742 movi a1, init_task
743 movi a0, 0
744 addi a1, a1, PT_REGS_OFFSET
746 movi a4, panic
747 movi a6, unrecoverable_text
749 callx4 a4
751 1: j 1b
754 /* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */
756 /*
757 * Fast-handler for alloca exceptions
758 *
759 * The ALLOCA handler is entered when user code executes the MOVSP
760 * instruction and the caller's frame is not in the register file.
761 * In this case, the caller frame's a0..a3 are on the stack just
762 * below sp (a1), and this handler moves them.
763 *
764 * For "MOVSP <ar>,<as>" without destination register a1, this routine
765 * simply moves the value from <as> to <ar> without moving the save area.
766 *
767 * Entry condition:
768 *
769 * a0: trashed, original value saved on stack (PT_AREG0)
770 * a1: a1
771 * a2: new stack pointer, original in DEPC
772 * a3: dispatch table
773 * depc: a2, original value saved on stack (PT_DEPC)
774 * excsave_1: a3
775 *
776 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
777 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
778 */
780 #if XCHAL_HAVE_BE
781 #define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 4, 4
782 #define _EXTUI_MOVSP_DST(ar) extui ar, ar, 0, 4
783 #else
784 #define _EXTUI_MOVSP_SRC(ar) extui ar, ar, 0, 4
785 #define _EXTUI_MOVSP_DST(ar) extui ar, ar, 4, 4
786 #endif
788 ENTRY(fast_alloca)
790 /* We shouldn't be in a double exception. */
792 l32i a0, a2, PT_DEPC
793 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, .Lunhandled_double
795 rsr a0, DEPC # get a2
796 s32i a4, a2, PT_AREG4 # save a4 and
797 s32i a0, a2, PT_AREG2 # a2 to stack
799 /* Exit critical section. */
801 movi a0, 0
802 s32i a0, a3, EXC_TABLE_FIXUP
804 /* Restore a3, excsave_1 */
806 xsr a3, EXCSAVE_1 # make sure excsave_1 is valid for dbl.
807 rsr a4, EPC_1 # get exception address
808 s32i a3, a2, PT_AREG3 # save a3 to stack
810 #ifdef ALLOCA_EXCEPTION_IN_IRAM
811 #error iram not supported
812 #else
813 /* Note: l8ui not allowed in IRAM/IROM!! */
814 l8ui a0, a4, 1 # read as(src) from MOVSP instruction
815 #endif
816 movi a3, .Lmovsp_src
817 _EXTUI_MOVSP_SRC(a0) # extract source register number
818 addx8 a3, a0, a3
819 jx a3
821 .Lunhandled_double:
822 wsr a0, EXCSAVE_1
823 movi a0, unrecoverable_exception
824 callx0 a0
826 .align 8
827 .Lmovsp_src:
828 l32i a3, a2, PT_AREG0; _j 1f; .align 8
829 mov a3, a1; _j 1f; .align 8
830 l32i a3, a2, PT_AREG2; _j 1f; .align 8
831 l32i a3, a2, PT_AREG3; _j 1f; .align 8
832 l32i a3, a2, PT_AREG4; _j 1f; .align 8
833 mov a3, a5; _j 1f; .align 8
834 mov a3, a6; _j 1f; .align 8
835 mov a3, a7; _j 1f; .align 8
836 mov a3, a8; _j 1f; .align 8
837 mov a3, a9; _j 1f; .align 8
838 mov a3, a10; _j 1f; .align 8
839 mov a3, a11; _j 1f; .align 8
840 mov a3, a12; _j 1f; .align 8
841 mov a3, a13; _j 1f; .align 8
842 mov a3, a14; _j 1f; .align 8
843 mov a3, a15; _j 1f; .align 8
845 1:
847 #ifdef ALLOCA_EXCEPTION_IN_IRAM
848 #error iram not supported
849 #else
850 l8ui a0, a4, 0 # read ar(dst) from MOVSP instruction
851 #endif
852 addi a4, a4, 3 # step over movsp
853 _EXTUI_MOVSP_DST(a0) # extract destination register
854 wsr a4, EPC_1 # save new epc_1
856 _bnei a0, 1, 1f # no 'movsp a1, ax': jump
858 /* Move the save area. This implies the use of the L32E
859 * and S32E instructions, because this move must be done with
860 * the user's PS.RING privilege levels, not with ring 0
861 * (kernel's) privileges currently active with PS.EXCM
862 * set. Note that we have stil registered a fixup routine with the
863 * double exception vector in case a double exception occurs.
864 */
866 /* a0,a4:avail a1:old user stack a2:exc. stack a3:new user stack. */
868 l32e a0, a1, -16
869 l32e a4, a1, -12
870 s32e a0, a3, -16
871 s32e a4, a3, -12
872 l32e a0, a1, -8
873 l32e a4, a1, -4
874 s32e a0, a3, -8
875 s32e a4, a3, -4
877 /* Restore stack-pointer and all the other saved registers. */
879 mov a1, a3
881 l32i a4, a2, PT_AREG4
882 l32i a3, a2, PT_AREG3
883 l32i a0, a2, PT_AREG0
884 l32i a2, a2, PT_AREG2
885 rfe
887 /* MOVSP <at>,<as> was invoked with <at> != a1.
888 * Because the stack pointer is not being modified,
889 * we should be able to just modify the pointer
890 * without moving any save area.
891 * The processor only traps these occurrences if the
892 * caller window isn't live, so unfortunately we can't
893 * use this as an alternate trap mechanism.
894 * So we just do the move. This requires that we
895 * resolve the destination register, not just the source,
896 * so there's some extra work.
897 * (PERHAPS NOT REALLY NEEDED, BUT CLEANER...)
898 */
900 /* a0 dst-reg, a1 user-stack, a2 stack, a3 value of src reg. */
902 1: movi a4, .Lmovsp_dst
903 addx8 a4, a0, a4
904 jx a4
906 .align 8
907 .Lmovsp_dst:
908 s32i a3, a2, PT_AREG0; _j 1f; .align 8
909 mov a1, a3; _j 1f; .align 8
910 s32i a3, a2, PT_AREG2; _j 1f; .align 8
911 s32i a3, a2, PT_AREG3; _j 1f; .align 8
912 s32i a3, a2, PT_AREG4; _j 1f; .align 8
913 mov a5, a3; _j 1f; .align 8
914 mov a6, a3; _j 1f; .align 8
915 mov a7, a3; _j 1f; .align 8
916 mov a8, a3; _j 1f; .align 8
917 mov a9, a3; _j 1f; .align 8
918 mov a10, a3; _j 1f; .align 8
919 mov a11, a3; _j 1f; .align 8
920 mov a12, a3; _j 1f; .align 8
921 mov a13, a3; _j 1f; .align 8
922 mov a14, a3; _j 1f; .align 8
923 mov a15, a3; _j 1f; .align 8
925 1: l32i a4, a2, PT_AREG4
926 l32i a3, a2, PT_AREG3
927 l32i a0, a2, PT_AREG0
928 l32i a2, a2, PT_AREG2
929 rfe
932 /*
933 * fast system calls.
934 *
935 * WARNING: The kernel doesn't save the entire user context before
936 * handling a fast system call. These functions are small and short,
937 * usually offering some functionality not available to user tasks.
938 *
939 * BE CAREFUL TO PRESERVE THE USER'S CONTEXT.
940 *
941 * Entry condition:
942 *
943 * a0: trashed, original value saved on stack (PT_AREG0)
944 * a1: a1
945 * a2: new stack pointer, original in DEPC
946 * a3: dispatch table
947 * depc: a2, original value saved on stack (PT_DEPC)
948 * excsave_1: a3
949 */
951 ENTRY(fast_syscall_kernel)
953 /* Skip syscall. */
955 rsr a0, EPC_1
956 addi a0, a0, 3
957 wsr a0, EPC_1
959 l32i a0, a2, PT_DEPC
960 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
962 rsr a0, DEPC # get syscall-nr
963 _beqz a0, fast_syscall_spill_registers
964 _beqi a0, __NR_xtensa, fast_syscall_xtensa
966 j kernel_exception
968 ENTRY(fast_syscall_user)
970 /* Skip syscall. */
972 rsr a0, EPC_1
973 addi a0, a0, 3
974 wsr a0, EPC_1
976 l32i a0, a2, PT_DEPC
977 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
979 rsr a0, DEPC # get syscall-nr
980 _beqz a0, fast_syscall_spill_registers
981 _beqi a0, __NR_xtensa, fast_syscall_xtensa
983 j user_exception
985 ENTRY(fast_syscall_unrecoverable)
987 /* Restore all states. */
989 l32i a0, a2, PT_AREG0 # restore a0
990 xsr a2, DEPC # restore a2, depc
991 rsr a3, EXCSAVE_1
993 wsr a0, EXCSAVE_1
994 movi a0, unrecoverable_exception
995 callx0 a0
999 /*
1000 * sysxtensa syscall handler
1001 *
1002 * int sysxtensa (SYS_XTENSA_ATOMIC_SET, ptr, val, unused);
1003 * int sysxtensa (SYS_XTENSA_ATOMIC_ADD, ptr, val, unused);
1004 * int sysxtensa (SYS_XTENSA_ATOMIC_EXG_ADD, ptr, val, unused);
1005 * int sysxtensa (SYS_XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval);
1006 * a2 a6 a3 a4 a5
1007 *
1008 * Entry condition:
1009 *
1010 * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0)
1011 * a1: a1
1012 * a2: new stack pointer, original in a0 and DEPC
1013 * a3: dispatch table, original in excsave_1
1014 * a4..a15: unchanged
1015 * depc: a2, original value saved on stack (PT_DEPC)
1016 * excsave_1: a3
1017 *
1018 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1019 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1020 *
1021 * Note: we don't have to save a2; a2 holds the return value
1022 *
1023 * We use the two macros TRY and CATCH:
1024 *
1025 * TRY adds an entry to the __ex_table fixup table for the immediately
1026 * following instruction.
1027 *
1028 * CATCH catches any exception that occurred at one of the preceeding TRY
1029 * statements and continues from there
1030 *
1031 * Usage TRY l32i a0, a1, 0
1032 * <other code>
1033 * done: rfe
1034 * CATCH <set return code>
1035 * j done
1036 */
1038 #define TRY \
1039 .section __ex_table, "a"; \
1040 .word 66f, 67f; \
1041 .text; \
1042 66:
1044 #define CATCH \
1045 67:
1047 ENTRY(fast_syscall_xtensa)
1049 xsr a3, EXCSAVE_1 # restore a3, excsave1
1051 s32i a7, a2, PT_AREG7 # we need an additional register
1052 movi a7, 4 # sizeof(unsigned int)
1053 access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
1055 addi a6, a6, -1 # assuming SYS_XTENSA_ATOMIC_SET = 1
1056 _bgeui a6, SYS_XTENSA_COUNT - 1, .Lill
1057 _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP - 1, .Lnswp
1059 /* Fall through for ATOMIC_CMP_SWP. */
1061 .Lswp: /* Atomic compare and swap */
1063 TRY l32i a0, a3, 0 # read old value
1064 bne a0, a4, 1f # same as old value? jump
1065 TRY s32i a5, a3, 0 # different, modify value
1066 l32i a7, a2, PT_AREG7 # restore a7
1067 l32i a0, a2, PT_AREG0 # restore a0
1068 movi a2, 1 # and return 1
1069 addi a6, a6, 1 # restore a6 (really necessary?)
1070 rfe
1072 1: l32i a7, a2, PT_AREG7 # restore a7
1073 l32i a0, a2, PT_AREG0 # restore a0
1074 movi a2, 0 # return 0 (note that we cannot set
1075 addi a6, a6, 1 # restore a6 (really necessary?)
1076 rfe
1078 .Lnswp: /* Atomic set, add, and exg_add. */
1080 TRY l32i a7, a3, 0 # orig
1081 add a0, a4, a7 # + arg
1082 moveqz a0, a4, a6 # set
1083 TRY s32i a0, a3, 0 # write new value
1085 mov a0, a2
1086 mov a2, a7
1087 l32i a7, a0, PT_AREG7 # restore a7
1088 l32i a0, a0, PT_AREG0 # restore a0
1089 addi a6, a6, 1 # restore a6 (really necessary?)
1090 rfe
1092 CATCH
1093 .Leac: l32i a7, a2, PT_AREG7 # restore a7
1094 l32i a0, a2, PT_AREG0 # restore a0
1095 movi a2, -EFAULT
1096 rfe
1098 .Lill: l32i a7, a2, PT_AREG0 # restore a7
1099 l32i a0, a2, PT_AREG0 # restore a0
1100 movi a2, -EINVAL
1101 rfe
1106 /* fast_syscall_spill_registers.
1107 *
1108 * Entry condition:
1109 *
1110 * a0: trashed, original value saved on stack (PT_AREG0)
1111 * a1: a1
1112 * a2: new stack pointer, original in DEPC
1113 * a3: dispatch table
1114 * depc: a2, original value saved on stack (PT_DEPC)
1115 * excsave_1: a3
1116 *
1117 * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
1118 */
1120 ENTRY(fast_syscall_spill_registers)
1122 /* Register a FIXUP handler (pass current wb as a parameter) */
1124 movi a0, fast_syscall_spill_registers_fixup
1125 s32i a0, a3, EXC_TABLE_FIXUP
1126 rsr a0, WINDOWBASE
1127 s32i a0, a3, EXC_TABLE_PARAM
1129 /* Save a3 and SAR on stack. */
1131 rsr a0, SAR
1132 xsr a3, EXCSAVE_1 # restore a3 and excsave_1
1133 s32i a3, a2, PT_AREG3
1134 s32i a4, a2, PT_AREG4
1135 s32i a0, a2, PT_AREG5 # store SAR to PT_AREG5
1137 /* The spill routine might clobber a7, a11, and a15. */
1139 s32i a7, a2, PT_AREG7
1140 s32i a11, a2, PT_AREG11
1141 s32i a15, a2, PT_AREG15
1143 call0 _spill_registers # destroys a3, a4, and SAR
1145 /* Advance PC, restore registers and SAR, and return from exception. */
1147 l32i a3, a2, PT_AREG5
1148 l32i a4, a2, PT_AREG4
1149 l32i a0, a2, PT_AREG0
1150 wsr a3, SAR
1151 l32i a3, a2, PT_AREG3
1153 /* Restore clobbered registers. */
1155 l32i a7, a2, PT_AREG7
1156 l32i a11, a2, PT_AREG11
1157 l32i a15, a2, PT_AREG15
1159 movi a2, 0
1160 rfe
1162 /* Fixup handler.
1163 *
1164 * We get here if the spill routine causes an exception, e.g. tlb miss.
1165 * We basically restore WINDOWBASE and WINDOWSTART to the condition when
1166 * we entered the spill routine and jump to the user exception handler.
1167 *
1168 * a0: value of depc, original value in depc
1169 * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE
1170 * a3: exctable, original value in excsave1
1171 */
1173 fast_syscall_spill_registers_fixup:
1175 rsr a2, WINDOWBASE # get current windowbase (a2 is saved)
1176 xsr a0, DEPC # restore depc and a0
1177 ssl a2 # set shift (32 - WB)
1179 /* We need to make sure the current registers (a0-a3) are preserved.
1180 * To do this, we simply set the bit for the current window frame
1181 * in WS, so that the exception handlers save them to the task stack.
1182 */
1184 rsr a3, EXCSAVE_1 # get spill-mask
1185 slli a2, a3, 1 # shift left by one
1187 slli a3, a2, 32-WSBITS
1188 src a2, a2, a3 # a1 = xxwww1yyxxxwww1yy......
1189 wsr a2, WINDOWSTART # set corrected windowstart
1191 movi a3, exc_table
1192 l32i a2, a3, EXC_TABLE_DOUBLE_SAVE # restore a2
1193 l32i a3, a3, EXC_TABLE_PARAM # original WB (in user task)
1195 /* Return to the original (user task) WINDOWBASE.
1196 * We leave the following frame behind:
1197 * a0, a1, a2 same
1198 * a3: trashed (saved in excsave_1)
1199 * depc: depc (we have to return to that address)
1200 * excsave_1: a3
1201 */
1203 wsr a3, WINDOWBASE
1204 rsync
1206 /* We are now in the original frame when we entered _spill_registers:
1207 * a0: return address
1208 * a1: used, stack pointer
1209 * a2: kernel stack pointer
1210 * a3: available, saved in EXCSAVE_1
1211 * depc: exception address
1212 * excsave: a3
1213 * Note: This frame might be the same as above.
1214 */
1216 /* Setup stack pointer. */
1218 addi a2, a2, -PT_USER_SIZE
1219 s32i a0, a2, PT_AREG0
1221 /* Make sure we return to this fixup handler. */
1223 movi a3, fast_syscall_spill_registers_fixup_return
1224 s32i a3, a2, PT_DEPC # setup depc
1226 /* Jump to the exception handler. */
1228 movi a3, exc_table
1229 rsr a0, EXCCAUSE
1230 addx4 a0, a0, a3 # find entry in table
1231 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
1232 jx a0
1234 fast_syscall_spill_registers_fixup_return:
1236 /* When we return here, all registers have been restored (a2: DEPC) */
1238 wsr a2, DEPC # exception address
1240 /* Restore fixup handler. */
1242 xsr a3, EXCSAVE_1
1243 movi a2, fast_syscall_spill_registers_fixup
1244 s32i a2, a3, EXC_TABLE_FIXUP
1245 rsr a2, WINDOWBASE
1246 s32i a2, a3, EXC_TABLE_PARAM
1247 l32i a2, a3, EXC_TABLE_KSTK
1249 /* Load WB at the time the exception occurred. */
1251 rsr a3, SAR # WB is still in SAR
1252 neg a3, a3
1253 wsr a3, WINDOWBASE
1254 rsync
1256 /* Restore a3 and return. */
1258 movi a3, exc_table
1259 xsr a3, EXCSAVE_1
1261 rfde
1264 /*
1265 * spill all registers.
1266 *
1267 * This is not a real function. The following conditions must be met:
1268 *
1269 * - must be called with call0.
1270 * - uses a3, a4 and SAR.
1271 * - the last 'valid' register of each frame are clobbered.
1272 * - the caller must have registered a fixup handler
1273 * (or be inside a critical section)
1274 * - PS_EXCM must be set (PS_WOE cleared?)
1275 */
1277 ENTRY(_spill_registers)
1279 /*
1280 * Rotate ws so that the current windowbase is at bit 0.
1281 * Assume ws = xxxwww1yy (www1 current window frame).
1282 * Rotate ws right so that a4 = yyxxxwww1.
1283 */
1285 rsr a4, WINDOWBASE
1286 rsr a3, WINDOWSTART # a3 = xxxwww1yy
1287 ssr a4 # holds WB
1288 slli a4, a3, WSBITS
1289 or a3, a3, a4 # a3 = xxxwww1yyxxxwww1yy
1290 srl a3, a3 # a3 = 00xxxwww1yyxxxwww1
1292 /* We are done if there are no more than the current register frame. */
1294 extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww
1295 movi a4, (1 << (WSBITS-1))
1296 _beqz a3, .Lnospill # only one active frame? jump
1298 /* We want 1 at the top, so that we return to the current windowbase */
1300 or a3, a3, a4 # 1yyxxxwww
1302 /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */
1304 wsr a3, WINDOWSTART # save shifted windowstart
1305 neg a4, a3
1306 and a3, a4, a3 # first bit set from right: 000010000
1308 ffs_ws a4, a3 # a4: shifts to skip empty frames
1309 movi a3, WSBITS
1310 sub a4, a3, a4 # WSBITS-a4:number of 0-bits from right
1311 ssr a4 # save in SAR for later.
1313 rsr a3, WINDOWBASE
1314 add a3, a3, a4
1315 wsr a3, WINDOWBASE
1316 rsync
1318 rsr a3, WINDOWSTART
1319 srl a3, a3 # shift windowstart
1321 /* WB is now just one frame below the oldest frame in the register
1322 window. WS is shifted so the oldest frame is in bit 0, thus, WB
1323 and WS differ by one 4-register frame. */
1325 /* Save frames. Depending what call was used (call4, call8, call12),
1326 * we have to save 4,8. or 12 registers.
1327 */
1329 _bbsi.l a3, 1, .Lc4
1330 _bbsi.l a3, 2, .Lc8
1332 /* Special case: we have a call12-frame starting at a4. */
1334 _bbci.l a3, 3, .Lc12 # bit 3 shouldn't be zero! (Jump to Lc12 first)
1336 s32e a4, a1, -16 # a1 is valid with an empty spill area
1337 l32e a4, a5, -12
1338 s32e a8, a4, -48
1339 mov a8, a4
1340 l32e a4, a1, -16
1341 j .Lc12c
1343 .Lnospill:
1344 ret
1346 .Lloop: _bbsi.l a3, 1, .Lc4
1347 _bbci.l a3, 2, .Lc12
1349 .Lc8: s32e a4, a13, -16
1350 l32e a4, a5, -12
1351 s32e a8, a4, -32
1352 s32e a5, a13, -12
1353 s32e a6, a13, -8
1354 s32e a7, a13, -4
1355 s32e a9, a4, -28
1356 s32e a10, a4, -24
1357 s32e a11, a4, -20
1359 srli a11, a3, 2 # shift windowbase by 2
1360 rotw 2
1361 _bnei a3, 1, .Lloop
1363 .Lexit: /* Done. Do the final rotation, set WS, and return. */
1365 rotw 1
1366 rsr a3, WINDOWBASE
1367 ssl a3
1368 movi a3, 1
1369 sll a3, a3
1370 wsr a3, WINDOWSTART
1371 ret
1373 .Lc4: s32e a4, a9, -16
1374 s32e a5, a9, -12
1375 s32e a6, a9, -8
1376 s32e a7, a9, -4
1378 srli a7, a3, 1
1379 rotw 1
1380 _bnei a3, 1, .Lloop
1381 j .Lexit
1383 .Lc12: _bbci.l a3, 3, .Linvalid_mask # bit 2 shouldn't be zero!
1385 /* 12-register frame (call12) */
1387 l32e a2, a5, -12
1388 s32e a8, a2, -48
1389 mov a8, a2
1391 .Lc12c: s32e a9, a8, -44
1392 s32e a10, a8, -40
1393 s32e a11, a8, -36
1394 s32e a12, a8, -32
1395 s32e a13, a8, -28
1396 s32e a14, a8, -24
1397 s32e a15, a8, -20
1398 srli a15, a3, 3
1400 /* The stack pointer for a4..a7 is out of reach, so we rotate the
1401 * window, grab the stackpointer, and rotate back.
1402 * Alternatively, we could also use the following approach, but that
1403 * makes the fixup routine much more complicated:
1404 * rotw 1
1405 * s32e a0, a13, -16
1406 * ...
1407 * rotw 2
1408 */
1410 rotw 1
1411 mov a5, a13
1412 rotw -1
1414 s32e a4, a9, -16
1415 s32e a5, a9, -12
1416 s32e a6, a9, -8
1417 s32e a7, a9, -4
1419 rotw 3
1421 _beqi a3, 1, .Lexit
1422 j .Lloop
1424 .Linvalid_mask:
1426 /* We get here because of an unrecoverable error in the window
1427 * registers. If we are in user space, we kill the application,
1428 * however, this condition is unrecoverable in kernel space.
1429 */
1431 rsr a0, PS
1432 _bbci.l a0, PS_UM_BIT, 1f
1434 /* User space: Setup a dummy frame and kill application.
1435 * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer.
1436 */
1438 movi a0, 1
1439 movi a1, 0
1441 wsr a0, WINDOWSTART
1442 wsr a1, WINDOWBASE
1443 rsync
1445 movi a0, 0
1447 movi a3, exc_table
1448 l32i a1, a3, EXC_TABLE_KSTK
1449 wsr a3, EXCSAVE_1
1451 movi a4, (1 << PS_WOE_BIT) | 1
1452 wsr a4, PS
1453 rsync
1455 movi a6, SIGSEGV
1456 movi a4, do_exit
1457 callx4 a4
1459 1: /* Kernel space: PANIC! */
1461 wsr a0, EXCSAVE_1
1462 movi a0, unrecoverable_exception
1463 callx0 a0 # should not return
1464 1: j 1b
1466 /*
1467 * We should never get here. Bail out!
1468 */
1470 ENTRY(fast_second_level_miss_double_kernel)
1472 1: movi a0, unrecoverable_exception
1473 callx0 a0 # should not return
1474 1: j 1b
1476 /* First-level entry handler for user, kernel, and double 2nd-level
1477 * TLB miss exceptions. Note that for now, user and kernel miss
1478 * exceptions share the same entry point and are handled identically.
1479 *
1480 * An old, less-efficient C version of this function used to exist.
1481 * We include it below, interleaved as comments, for reference.
1482 *
1483 * Entry condition:
1484 *
1485 * a0: trashed, original value saved on stack (PT_AREG0)
1486 * a1: a1
1487 * a2: new stack pointer, original in DEPC
1488 * a3: dispatch table
1489 * depc: a2, original value saved on stack (PT_DEPC)
1490 * excsave_1: a3
1491 *
1492 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1493 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1494 */
1496 ENTRY(fast_second_level_miss)
1498 /* Save a1. Note: we don't expect a double exception. */
1500 s32i a1, a2, PT_AREG1
1502 /* We need to map the page of PTEs for the user task. Find
1503 * the pointer to that page. Also, it's possible for tsk->mm
1504 * to be NULL while tsk->active_mm is nonzero if we faulted on
1505 * a vmalloc address. In that rare case, we must use
1506 * active_mm instead to avoid a fault in this handler. See
1507 *
1508 * http://mail.nl.linux.org/linux-mm/2002-08/msg00258.html
1509 * (or search Internet on "mm vs. active_mm")
1510 *
1511 * if (!mm)
1512 * mm = tsk->active_mm;
1513 * pgd = pgd_offset (mm, regs->excvaddr);
1514 * pmd = pmd_offset (pgd, regs->excvaddr);
1515 * pmdval = *pmd;
1516 */
1518 GET_CURRENT(a1,a2)
1519 l32i a0, a1, TASK_MM # tsk->mm
1520 beqz a0, 9f
1523 /* We deliberately destroy a3 that holds the exception table. */
1525 8: rsr a3, EXCVADDR # fault address
1526 _PGD_OFFSET(a0, a3, a1)
1527 l32i a0, a0, 0 # read pmdval
1528 beqz a0, 2f
1530 /* Read ptevaddr and convert to top of page-table page.
1531 *
1532 * vpnval = read_ptevaddr_register() & PAGE_MASK;
1533 * vpnval += DTLB_WAY_PGTABLE;
1534 * pteval = mk_pte (virt_to_page(pmd_val(pmdval)), PAGE_KERNEL);
1535 * write_dtlb_entry (pteval, vpnval);
1536 *
1537 * The messy computation for 'pteval' above really simplifies
1538 * into the following:
1539 *
1540 * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY
1541 */
1543 movi a1, -PAGE_OFFSET
1544 add a0, a0, a1 # pmdval - PAGE_OFFSET
1545 extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK
1546 xor a0, a0, a1
1548 movi a1, _PAGE_DIRECTORY
1549 or a0, a0, a1 # ... | PAGE_DIRECTORY
1551 /*
1552 * We utilize all three wired-ways (7-9) to hold pmd translations.
1553 * Memory regions are mapped to the DTLBs according to bits 28 and 29.
1554 * This allows to map the three most common regions to three different
1555 * DTLBs:
1556 * 0,1 -> way 7 program (0040.0000) and virtual (c000.0000)
1557 * 2 -> way 8 shared libaries (2000.0000)
1558 * 3 -> way 0 stack (3000.0000)
1559 */
1561 extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3
1562 rsr a1, PTEVADDR
1563 addx2 a3, a3, a3 # -> 0,3,6,9
1564 srli a1, a1, PAGE_SHIFT
1565 extui a3, a3, 2, 2 # -> 0,0,1,2
1566 slli a1, a1, PAGE_SHIFT # ptevaddr & PAGE_MASK
1567 addi a3, a3, DTLB_WAY_PGD
1568 add a1, a1, a3 # ... + way_number
1570 3: wdtlb a0, a1
1571 dsync
1573 /* Exit critical section. */
1575 4: movi a3, exc_table # restore a3
1576 movi a0, 0
1577 s32i a0, a3, EXC_TABLE_FIXUP
1579 /* Restore the working registers, and return. */
1581 l32i a0, a2, PT_AREG0
1582 l32i a1, a2, PT_AREG1
1583 l32i a2, a2, PT_DEPC
1584 xsr a3, EXCSAVE_1
1586 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1588 /* Restore excsave1 and return. */
1590 rsr a2, DEPC
1591 rfe
1593 /* Return from double exception. */
1595 1: xsr a2, DEPC
1596 esync
1597 rfde
1599 9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1600 j 8b
1602 #if (DCACHE_WAY_SIZE > PAGE_SIZE)
1604 2: /* Special case for cache aliasing.
1605 * We (should) only get here if a clear_user_page, copy_user_page
1606 * or the aliased cache flush functions got preemptively interrupted
1607 * by another task. Re-establish temporary mapping to the
1608 * TLBTEMP_BASE areas.
1609 */
1611 /* We shouldn't be in a double exception */
1613 l32i a0, a2, PT_DEPC
1614 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f
1616 /* Make sure the exception originated in the special functions */
1618 movi a0, __tlbtemp_mapping_start
1619 rsr a3, EPC_1
1620 bltu a3, a0, 2f
1621 movi a0, __tlbtemp_mapping_end
1622 bgeu a3, a0, 2f
1624 /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */
1626 movi a3, TLBTEMP_BASE_1
1627 rsr a0, EXCVADDR
1628 bltu a0, a3, 2f
1630 addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT))
1631 bgeu a1, a3, 2f
1633 /* Check if we have to restore an ITLB mapping. */
1635 movi a1, __tlbtemp_mapping_itlb
1636 rsr a3, EPC_1
1637 sub a3, a3, a1
1639 /* Calculate VPN */
1641 movi a1, PAGE_MASK
1642 and a1, a1, a0
1644 /* Jump for ITLB entry */
1646 bgez a3, 1f
1648 /* We can use up to two TLBTEMP areas, one for src and one for dst. */
1650 extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1
1651 add a1, a3, a1
1653 /* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */
1655 mov a0, a6
1656 movnez a0, a7, a3
1657 j 3b
1659 /* ITLB entry. We only use dst in a6. */
1661 1: witlb a6, a1
1662 isync
1663 j 4b
1666 #endif // DCACHE_WAY_SIZE > PAGE_SIZE
1669 2: /* Invalid PGD, default exception handling */
1671 movi a3, exc_table
1672 rsr a1, DEPC
1673 xsr a3, EXCSAVE_1
1674 s32i a1, a2, PT_AREG2
1675 s32i a3, a2, PT_AREG3
1676 mov a1, a2
1678 rsr a2, PS
1679 bbsi.l a2, PS_UM_BIT, 1f
1680 j _kernel_exception
1681 1: j _user_exception
1684 /*
1685 * StoreProhibitedException
1686 *
1687 * Update the pte and invalidate the itlb mapping for this pte.
1688 *
1689 * Entry condition:
1690 *
1691 * a0: trashed, original value saved on stack (PT_AREG0)
1692 * a1: a1
1693 * a2: new stack pointer, original in DEPC
1694 * a3: dispatch table
1695 * depc: a2, original value saved on stack (PT_DEPC)
1696 * excsave_1: a3
1697 *
1698 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1699 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1700 */
1702 ENTRY(fast_store_prohibited)
1704 /* Save a1 and a4. */
1706 s32i a1, a2, PT_AREG1
1707 s32i a4, a2, PT_AREG4
1709 GET_CURRENT(a1,a2)
1710 l32i a0, a1, TASK_MM # tsk->mm
1711 beqz a0, 9f
1713 8: rsr a1, EXCVADDR # fault address
1714 _PGD_OFFSET(a0, a1, a4)
1715 l32i a0, a0, 0
1716 beqz a0, 2f
1718 /* Note that we assume _PAGE_WRITABLE_BIT is only set if pte is valid.*/
1720 _PTE_OFFSET(a0, a1, a4)
1721 l32i a4, a0, 0 # read pteval
1722 bbci.l a4, _PAGE_WRITABLE_BIT, 2f
1724 movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
1725 or a4, a4, a1
1726 rsr a1, EXCVADDR
1727 s32i a4, a0, 0
1729 /* We need to flush the cache if we have page coloring. */
1730 #if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
1731 dhwb a0, 0
1732 #endif
1733 pdtlb a0, a1
1734 wdtlb a4, a0
1736 /* Exit critical section. */
1738 movi a0, 0
1739 s32i a0, a3, EXC_TABLE_FIXUP
1741 /* Restore the working registers, and return. */
1743 l32i a4, a2, PT_AREG4
1744 l32i a1, a2, PT_AREG1
1745 l32i a0, a2, PT_AREG0
1746 l32i a2, a2, PT_DEPC
1748 /* Restore excsave1 and a3. */
1750 xsr a3, EXCSAVE_1
1751 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1753 rsr a2, DEPC
1754 rfe
1756 /* Double exception. Restore FIXUP handler and return. */
1758 1: xsr a2, DEPC
1759 esync
1760 rfde
1762 9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1763 j 8b
1765 2: /* If there was a problem, handle fault in C */
1767 rsr a4, DEPC # still holds a2
1768 xsr a3, EXCSAVE_1
1769 s32i a4, a2, PT_AREG2
1770 s32i a3, a2, PT_AREG3
1771 l32i a4, a2, PT_AREG4
1772 mov a1, a2
1774 rsr a2, PS
1775 bbsi.l a2, PS_UM_BIT, 1f
1776 j _kernel_exception
1777 1: j _user_exception
1780 /*
1781 * System Calls.
1782 *
1783 * void system_call (struct pt_regs* regs, int exccause)
1784 * a2 a3
1785 */
1787 ENTRY(system_call)
1788 entry a1, 32
1790 /* regs->syscall = regs->areg[2] */
1792 l32i a3, a2, PT_AREG2
1793 mov a6, a2
1794 movi a4, do_syscall_trace_enter
1795 s32i a3, a2, PT_SYSCALL
1796 callx4 a4
1798 /* syscall = sys_call_table[syscall_nr] */
1800 movi a4, sys_call_table;
1801 movi a5, __NR_syscall_count
1802 movi a6, -ENOSYS
1803 bgeu a3, a5, 1f
1805 addx4 a4, a3, a4
1806 l32i a4, a4, 0
1807 movi a5, sys_ni_syscall;
1808 beq a4, a5, 1f
1810 /* Load args: arg0 - arg5 are passed via regs. */
1812 l32i a6, a2, PT_AREG6
1813 l32i a7, a2, PT_AREG3
1814 l32i a8, a2, PT_AREG4
1815 l32i a9, a2, PT_AREG5
1816 l32i a10, a2, PT_AREG8
1817 l32i a11, a2, PT_AREG9
1819 /* Pass one additional argument to the syscall: pt_regs (on stack) */
1820 s32i a2, a1, 0
1822 callx4 a4
1824 1: /* regs->areg[2] = return_value */
1826 s32i a6, a2, PT_AREG2
1827 movi a4, do_syscall_trace_leave
1828 mov a6, a2
1829 callx4 a4
1830 retw
1833 /*
1834 * Create a kernel thread
1835 *
1836 * int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
1837 * a2 a2 a3 a4
1838 */
1840 ENTRY(kernel_thread)
1841 entry a1, 16
1843 mov a5, a2 # preserve fn over syscall
1844 mov a7, a3 # preserve args over syscall
1846 movi a3, _CLONE_VM | _CLONE_UNTRACED
1847 movi a2, __NR_clone
1848 or a6, a4, a3 # arg0: flags
1849 mov a3, a1 # arg1: sp
1850 syscall
1852 beq a3, a1, 1f # branch if parent
1853 mov a6, a7 # args
1854 callx4 a5 # fn(args)
1856 movi a2, __NR_exit
1857 syscall # return value of fn(args) still in a6
1859 1: retw
1861 /*
1862 * Do a system call from kernel instead of calling sys_execve, so we end up
1863 * with proper pt_regs.
1864 *
1865 * int kernel_execve(const char *fname, char *const argv[], charg *const envp[])
1866 * a2 a2 a3 a4
1867 */
1869 ENTRY(kernel_execve)
1870 entry a1, 16
1871 mov a6, a2 # arg0 is in a6
1872 movi a2, __NR_execve
1873 syscall
1875 retw
1877 /*
1878 * Task switch.
1879 *
1880 * struct task* _switch_to (struct task* prev, struct task* next)
1881 * a2 a2 a3
1882 */
1884 ENTRY(_switch_to)
1886 entry a1, 16
1888 mov a12, a2 # preserve 'prev' (a2)
1889 mov a13, a3 # and 'next' (a3)
1891 l32i a4, a2, TASK_THREAD_INFO
1892 l32i a5, a3, TASK_THREAD_INFO
1894 save_xtregs_user a4 a6 a8 a9 a10 a11 THREAD_XTREGS_USER
1896 s32i a0, a12, THREAD_RA # save return address
1897 s32i a1, a12, THREAD_SP # save stack pointer
1899 /* Disable ints while we manipulate the stack pointer. */
1901 movi a14, (1 << PS_EXCM_BIT) | LOCKLEVEL
1902 xsr a14, PS
1903 rsr a3, EXCSAVE_1
1904 rsync
1905 s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */
1907 /* Switch CPENABLE */
1909 #if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS)
1910 l32i a3, a5, THREAD_CPENABLE
1911 xsr a3, CPENABLE
1912 s32i a3, a4, THREAD_CPENABLE
1913 #endif
1915 /* Flush register file. */
1917 call0 _spill_registers # destroys a3, a4, and SAR
1919 /* Set kernel stack (and leave critical section)
1920 * Note: It's save to set it here. The stack will not be overwritten
1921 * because the kernel stack will only be loaded again after
1922 * we return from kernel space.
1923 */
1925 rsr a3, EXCSAVE_1 # exc_table
1926 movi a6, 0
1927 addi a7, a5, PT_REGS_OFFSET
1928 s32i a6, a3, EXC_TABLE_FIXUP
1929 s32i a7, a3, EXC_TABLE_KSTK
1931 /* restore context of the task that 'next' addresses */
1933 l32i a0, a13, THREAD_RA # restore return address
1934 l32i a1, a13, THREAD_SP # restore stack pointer
1936 load_xtregs_user a5 a6 a8 a9 a10 a11 THREAD_XTREGS_USER
1938 wsr a14, PS
1939 mov a2, a12 # return 'prev'
1940 rsync
1942 retw
1945 ENTRY(ret_from_fork)
1947 /* void schedule_tail (struct task_struct *prev)
1948 * Note: prev is still in a6 (return value from fake call4 frame)
1949 */
1950 movi a4, schedule_tail
1951 callx4 a4
1953 movi a4, do_syscall_trace_leave
1954 mov a6, a1
1955 callx4 a4
1957 j common_exception_return