| blob | a2a902140c4e50575e30dbf365edd247ead02f72 |
1 /*
2 * arch/xtensa/kernel/entry.S
3 *
4 * Low-level exception handling
5 *
6 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file "COPYING" in the main directory of this archive
8 * for more details.
9 *
10 * Copyright (C) 2004 - 2008 by Tensilica Inc.
11 *
12 * Chris Zankel <chris@zankel.net>
13 *
14 */
16 #include <linux/linkage.h>
17 #include <asm/asm-offsets.h>
18 #include <asm/processor.h>
19 #include <asm/coprocessor.h>
20 #include <asm/thread_info.h>
21 #include <asm/uaccess.h>
22 #include <asm/unistd.h>
23 #include <asm/ptrace.h>
24 #include <asm/current.h>
25 #include <asm/pgtable.h>
26 #include <asm/page.h>
27 #include <asm/signal.h>
28 #include <asm/tlbflush.h>
29 #include <variant/tie-asm.h>
31 /* Unimplemented features. */
33 #undef KERNEL_STACK_OVERFLOW_CHECK
35 /* Not well tested.
36 *
37 * - fast_coprocessor
38 */
40 /*
41 * Macro to find first bit set in WINDOWBASE from the left + 1
42 *
43 * 100....0 -> 1
44 * 010....0 -> 2
45 * 000....1 -> WSBITS
46 */
48 .macro ffs_ws bit mask
50 #if XCHAL_HAVE_NSA
51 nsau \bit, \mask # 32-WSBITS ... 31 (32 iff 0)
52 addi \bit, \bit, WSBITS - 32 + 1 # uppest bit set -> return 1
53 #else
54 movi \bit, WSBITS
55 #if WSBITS > 16
56 _bltui \mask, 0x10000, 99f
57 addi \bit, \bit, -16
58 extui \mask, \mask, 16, 16
59 #endif
60 #if WSBITS > 8
61 99: _bltui \mask, 0x100, 99f
62 addi \bit, \bit, -8
63 srli \mask, \mask, 8
64 #endif
65 99: _bltui \mask, 0x10, 99f
66 addi \bit, \bit, -4
67 srli \mask, \mask, 4
68 99: _bltui \mask, 0x4, 99f
69 addi \bit, \bit, -2
70 srli \mask, \mask, 2
71 99: _bltui \mask, 0x2, 99f
72 addi \bit, \bit, -1
73 99:
75 #endif
76 .endm
78 /* ----------------- DEFAULT FIRST LEVEL EXCEPTION HANDLERS ----------------- */
80 /*
81 * First-level exception handler for user exceptions.
82 * Save some special registers, extra states and all registers in the AR
83 * register file that were in use in the user task, and jump to the common
84 * exception code.
85 * We save SAR (used to calculate WMASK), and WB and WS (we don't have to
86 * save them for kernel exceptions).
87 *
88 * Entry condition for user_exception:
89 *
90 * a0: trashed, original value saved on stack (PT_AREG0)
91 * a1: a1
92 * a2: new stack pointer, original value in depc
93 * a3: a3
94 * depc: a2, original value saved on stack (PT_DEPC)
95 * excsave1: dispatch table
96 *
97 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
98 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
99 *
100 * Entry condition for _user_exception:
101 *
102 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
103 * excsave has been restored, and
104 * stack pointer (a1) has been set.
105 *
106 * Note: _user_exception might be at an odd address. Don't use call0..call12
107 */
109 ENTRY(user_exception)
111 /* Save a1, a2, a3, and set SP. */
113 rsr a0, depc
114 s32i a1, a2, PT_AREG1
115 s32i a0, a2, PT_AREG2
116 s32i a3, a2, PT_AREG3
117 mov a1, a2
119 .globl _user_exception
120 _user_exception:
122 /* Save SAR and turn off single stepping */
124 movi a2, 0
125 rsr a3, sar
126 xsr a2, icountlevel
127 s32i a3, a1, PT_SAR
128 s32i a2, a1, PT_ICOUNTLEVEL
130 #if XCHAL_HAVE_THREADPTR
131 rur a2, threadptr
132 s32i a2, a1, PT_THREADPTR
133 #endif
135 /* Rotate ws so that the current windowbase is at bit0. */
136 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
138 rsr a2, windowbase
139 rsr a3, windowstart
140 ssr a2
141 s32i a2, a1, PT_WINDOWBASE
142 s32i a3, a1, PT_WINDOWSTART
143 slli a2, a3, 32-WSBITS
144 src a2, a3, a2
145 srli a2, a2, 32-WSBITS
146 s32i a2, a1, PT_WMASK # needed for restoring registers
148 /* Save only live registers. */
150 _bbsi.l a2, 1, 1f
151 s32i a4, a1, PT_AREG4
152 s32i a5, a1, PT_AREG5
153 s32i a6, a1, PT_AREG6
154 s32i a7, a1, PT_AREG7
155 _bbsi.l a2, 2, 1f
156 s32i a8, a1, PT_AREG8
157 s32i a9, a1, PT_AREG9
158 s32i a10, a1, PT_AREG10
159 s32i a11, a1, PT_AREG11
160 _bbsi.l a2, 3, 1f
161 s32i a12, a1, PT_AREG12
162 s32i a13, a1, PT_AREG13
163 s32i a14, a1, PT_AREG14
164 s32i a15, a1, PT_AREG15
165 _bnei a2, 1, 1f # only one valid frame?
167 /* Only one valid frame, skip saving regs. */
169 j 2f
171 /* Save the remaining registers.
172 * We have to save all registers up to the first '1' from
173 * the right, except the current frame (bit 0).
174 * Assume a2 is: 001001000110001
175 * All register frames starting from the top field to the marked '1'
176 * must be saved.
177 */
179 1: addi a3, a2, -1 # eliminate '1' in bit 0: yyyyxxww0
180 neg a3, a3 # yyyyxxww0 -> YYYYXXWW1+1
181 and a3, a3, a2 # max. only one bit is set
183 /* Find number of frames to save */
185 ffs_ws a0, a3 # number of frames to the '1' from left
187 /* Store information into WMASK:
188 * bits 0..3: xxx1 masked lower 4 bits of the rotated windowstart,
189 * bits 4...: number of valid 4-register frames
190 */
192 slli a3, a0, 4 # number of frames to save in bits 8..4
193 extui a2, a2, 0, 4 # mask for the first 16 registers
194 or a2, a3, a2
195 s32i a2, a1, PT_WMASK # needed when we restore the reg-file
197 /* Save 4 registers at a time */
199 1: rotw -1
200 s32i a0, a5, PT_AREG_END - 16
201 s32i a1, a5, PT_AREG_END - 12
202 s32i a2, a5, PT_AREG_END - 8
203 s32i a3, a5, PT_AREG_END - 4
204 addi a0, a4, -1
205 addi a1, a5, -16
206 _bnez a0, 1b
208 /* WINDOWBASE still in SAR! */
210 rsr a2, sar # original WINDOWBASE
211 movi a3, 1
212 ssl a2
213 sll a3, a3
214 wsr a3, windowstart # set corresponding WINDOWSTART bit
215 wsr a2, windowbase # and WINDOWSTART
216 rsync
218 /* We are back to the original stack pointer (a1) */
220 2: /* Now, jump to the common exception handler. */
222 j common_exception
224 ENDPROC(user_exception)
226 /*
227 * First-level exit handler for kernel exceptions
228 * Save special registers and the live window frame.
229 * Note: Even though we changes the stack pointer, we don't have to do a
230 * MOVSP here, as we do that when we return from the exception.
231 * (See comment in the kernel exception exit code)
232 *
233 * Entry condition for kernel_exception:
234 *
235 * a0: trashed, original value saved on stack (PT_AREG0)
236 * a1: a1
237 * a2: new stack pointer, original in DEPC
238 * a3: a3
239 * depc: a2, original value saved on stack (PT_DEPC)
240 * excsave_1: dispatch table
241 *
242 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
243 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
244 *
245 * Entry condition for _kernel_exception:
246 *
247 * a0-a3 and depc have been saved to PT_AREG0...PT_AREG3 and PT_DEPC
248 * excsave has been restored, and
249 * stack pointer (a1) has been set.
250 *
251 * Note: _kernel_exception might be at an odd address. Don't use call0..call12
252 */
254 ENTRY(kernel_exception)
256 /* Save a1, a2, a3, and set SP. */
258 rsr a0, depc # get a2
259 s32i a1, a2, PT_AREG1
260 s32i a0, a2, PT_AREG2
261 s32i a3, a2, PT_AREG3
262 mov a1, a2
264 .globl _kernel_exception
265 _kernel_exception:
267 /* Save SAR and turn off single stepping */
269 movi a2, 0
270 rsr a3, sar
271 xsr a2, icountlevel
272 s32i a3, a1, PT_SAR
273 s32i a2, a1, PT_ICOUNTLEVEL
275 /* Rotate ws so that the current windowbase is at bit0. */
276 /* Assume ws = xxwww1yyyy. Rotate ws right, so that a2 = yyyyxxwww1 */
278 rsr a2, windowbase # don't need to save these, we only
279 rsr a3, windowstart # need shifted windowstart: windowmask
280 ssr a2
281 slli a2, a3, 32-WSBITS
282 src a2, a3, a2
283 srli a2, a2, 32-WSBITS
284 s32i a2, a1, PT_WMASK # needed for kernel_exception_exit
286 /* Save only the live window-frame */
288 _bbsi.l a2, 1, 1f
289 s32i a4, a1, PT_AREG4
290 s32i a5, a1, PT_AREG5
291 s32i a6, a1, PT_AREG6
292 s32i a7, a1, PT_AREG7
293 _bbsi.l a2, 2, 1f
294 s32i a8, a1, PT_AREG8
295 s32i a9, a1, PT_AREG9
296 s32i a10, a1, PT_AREG10
297 s32i a11, a1, PT_AREG11
298 _bbsi.l a2, 3, 1f
299 s32i a12, a1, PT_AREG12
300 s32i a13, a1, PT_AREG13
301 s32i a14, a1, PT_AREG14
302 s32i a15, a1, PT_AREG15
304 1:
306 #ifdef KERNEL_STACK_OVERFLOW_CHECK
308 /* Stack overflow check, for debugging */
309 extui a2, a1, TASK_SIZE_BITS,XX
310 movi a3, SIZE??
311 _bge a2, a3, out_of_stack_panic
313 #endif
315 /*
316 * This is the common exception handler.
317 * We get here from the user exception handler or simply by falling through
318 * from the kernel exception handler.
319 * Save the remaining special registers, switch to kernel mode, and jump
320 * to the second-level exception handler.
321 *
322 */
324 common_exception:
326 /* Save some registers, disable loops and clear the syscall flag. */
328 rsr a2, debugcause
329 rsr a3, epc1
330 s32i a2, a1, PT_DEBUGCAUSE
331 s32i a3, a1, PT_PC
333 movi a2, -1
334 rsr a3, excvaddr
335 s32i a2, a1, PT_SYSCALL
336 movi a2, 0
337 s32i a3, a1, PT_EXCVADDR
338 xsr a2, lcount
339 s32i a2, a1, PT_LCOUNT
341 /* It is now save to restore the EXC_TABLE_FIXUP variable. */
343 rsr a0, exccause
344 movi a3, 0
345 rsr a2, excsave1
346 s32i a0, a1, PT_EXCCAUSE
347 s32i a3, a2, EXC_TABLE_FIXUP
349 /* All unrecoverable states are saved on stack, now, and a1 is valid,
350 * so we can allow exceptions and interrupts (*) again.
351 * Set PS(EXCM = 0, UM = 0, RING = 0, OWB = 0, WOE = 1, INTLEVEL = X)
352 *
353 * (*) We only allow interrupts if they were previously enabled and
354 * we're not handling an IRQ
355 */
357 rsr a3, ps
358 addi a0, a0, -EXCCAUSE_LEVEL1_INTERRUPT
359 movi a2, LOCKLEVEL
360 extui a3, a3, PS_INTLEVEL_SHIFT, PS_INTLEVEL_WIDTH
361 # a3 = PS.INTLEVEL
362 moveqz a3, a2, a0 # a3 = LOCKLEVEL iff interrupt
363 movi a2, 1 << PS_WOE_BIT
364 or a3, a3, a2
365 rsr a0, exccause
366 xsr a3, ps
368 s32i a3, a1, PT_PS # save ps
370 /* Save lbeg, lend */
372 rsr a2, lbeg
373 rsr a3, lend
374 s32i a2, a1, PT_LBEG
375 s32i a3, a1, PT_LEND
377 /* Save SCOMPARE1 */
379 #if XCHAL_HAVE_S32C1I
380 rsr a2, scompare1
381 s32i a2, a1, PT_SCOMPARE1
382 #endif
384 /* Save optional registers. */
386 save_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
388 #ifdef CONFIG_TRACE_IRQFLAGS
389 l32i a4, a1, PT_DEPC
390 /* Double exception means we came here with an exception
391 * while PS.EXCM was set, i.e. interrupts disabled.
392 */
393 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
394 l32i a4, a1, PT_EXCCAUSE
395 bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
396 /* We came here with an interrupt means interrupts were enabled
397 * and we've just disabled them.
398 */
399 movi a4, trace_hardirqs_off
400 callx4 a4
401 1:
402 #endif
404 /* Go to second-level dispatcher. Set up parameters to pass to the
405 * exception handler and call the exception handler.
406 */
408 rsr a4, excsave1
409 mov a6, a1 # pass stack frame
410 mov a7, a0 # pass EXCCAUSE
411 addx4 a4, a0, a4
412 l32i a4, a4, EXC_TABLE_DEFAULT # load handler
414 /* Call the second-level handler */
416 callx4 a4
418 /* Jump here for exception exit */
419 .global common_exception_return
420 common_exception_return:
422 1:
423 rsil a2, LOCKLEVEL
425 /* Jump if we are returning from kernel exceptions. */
427 l32i a3, a1, PT_PS
428 GET_THREAD_INFO(a2, a1)
429 l32i a4, a2, TI_FLAGS
430 _bbci.l a3, PS_UM_BIT, 6f
432 /* Specific to a user exception exit:
433 * We need to check some flags for signal handling and rescheduling,
434 * and have to restore WB and WS, extra states, and all registers
435 * in the register file that were in use in the user task.
436 * Note that we don't disable interrupts here.
437 */
439 _bbsi.l a4, TIF_NEED_RESCHED, 3f
440 _bbsi.l a4, TIF_NOTIFY_RESUME, 2f
441 _bbci.l a4, TIF_SIGPENDING, 5f
443 2: l32i a4, a1, PT_DEPC
444 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
446 /* Call do_signal() */
448 rsil a2, 0
449 movi a4, do_notify_resume # int do_notify_resume(struct pt_regs*)
450 mov a6, a1
451 callx4 a4
452 j 1b
454 3: /* Reschedule */
456 rsil a2, 0
457 movi a4, schedule # void schedule (void)
458 callx4 a4
459 j 1b
461 #ifdef CONFIG_PREEMPT
462 6:
463 _bbci.l a4, TIF_NEED_RESCHED, 4f
465 /* Check current_thread_info->preempt_count */
467 l32i a4, a2, TI_PRE_COUNT
468 bnez a4, 4f
469 movi a4, preempt_schedule_irq
470 callx4 a4
471 j 1b
472 #endif
474 5:
475 #ifdef CONFIG_DEBUG_TLB_SANITY
476 l32i a4, a1, PT_DEPC
477 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 4f
478 movi a4, check_tlb_sanity
479 callx4 a4
480 #endif
481 6:
482 4:
483 #ifdef CONFIG_TRACE_IRQFLAGS
484 l32i a4, a1, PT_DEPC
485 /* Double exception means we came here with an exception
486 * while PS.EXCM was set, i.e. interrupts disabled.
487 */
488 bgeui a4, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
489 l32i a4, a1, PT_EXCCAUSE
490 bnei a4, EXCCAUSE_LEVEL1_INTERRUPT, 1f
491 /* We came here with an interrupt means interrupts were enabled
492 * and we'll reenable them on return.
493 */
494 movi a4, trace_hardirqs_on
495 callx4 a4
496 1:
497 #endif
498 /* Restore optional registers. */
500 load_xtregs_opt a1 a2 a4 a5 a6 a7 PT_XTREGS_OPT
502 /* Restore SCOMPARE1 */
504 #if XCHAL_HAVE_S32C1I
505 l32i a2, a1, PT_SCOMPARE1
506 wsr a2, scompare1
507 #endif
508 wsr a3, ps /* disable interrupts */
510 _bbci.l a3, PS_UM_BIT, kernel_exception_exit
512 user_exception_exit:
514 /* Restore the state of the task and return from the exception. */
516 /* Switch to the user thread WINDOWBASE. Save SP temporarily in DEPC */
518 l32i a2, a1, PT_WINDOWBASE
519 l32i a3, a1, PT_WINDOWSTART
520 wsr a1, depc # use DEPC as temp storage
521 wsr a3, windowstart # restore WINDOWSTART
522 ssr a2 # preserve user's WB in the SAR
523 wsr a2, windowbase # switch to user's saved WB
524 rsync
525 rsr a1, depc # restore stack pointer
526 l32i a2, a1, PT_WMASK # register frames saved (in bits 4...9)
527 rotw -1 # we restore a4..a7
528 _bltui a6, 16, 1f # only have to restore current window?
530 /* The working registers are a0 and a3. We are restoring to
531 * a4..a7. Be careful not to destroy what we have just restored.
532 * Note: wmask has the format YYYYM:
533 * Y: number of registers saved in groups of 4
534 * M: 4 bit mask of first 16 registers
535 */
537 mov a2, a6
538 mov a3, a5
540 2: rotw -1 # a0..a3 become a4..a7
541 addi a3, a7, -4*4 # next iteration
542 addi a2, a6, -16 # decrementing Y in WMASK
543 l32i a4, a3, PT_AREG_END + 0
544 l32i a5, a3, PT_AREG_END + 4
545 l32i a6, a3, PT_AREG_END + 8
546 l32i a7, a3, PT_AREG_END + 12
547 _bgeui a2, 16, 2b
549 /* Clear unrestored registers (don't leak anything to user-land */
551 1: rsr a0, windowbase
552 rsr a3, sar
553 sub a3, a0, a3
554 beqz a3, 2f
555 extui a3, a3, 0, WBBITS
557 1: rotw -1
558 addi a3, a7, -1
559 movi a4, 0
560 movi a5, 0
561 movi a6, 0
562 movi a7, 0
563 bgei a3, 1, 1b
565 /* We are back were we were when we started.
566 * Note: a2 still contains WMASK (if we've returned to the original
567 * frame where we had loaded a2), or at least the lower 4 bits
568 * (if we have restored WSBITS-1 frames).
569 */
571 2:
572 #if XCHAL_HAVE_THREADPTR
573 l32i a3, a1, PT_THREADPTR
574 wur a3, threadptr
575 #endif
577 j common_exception_exit
579 /* This is the kernel exception exit.
580 * We avoided to do a MOVSP when we entered the exception, but we
581 * have to do it here.
582 */
584 kernel_exception_exit:
586 /* Check if we have to do a movsp.
587 *
588 * We only have to do a movsp if the previous window-frame has
589 * been spilled to the *temporary* exception stack instead of the
590 * task's stack. This is the case if the corresponding bit in
591 * WINDOWSTART for the previous window-frame was set before
592 * (not spilled) but is zero now (spilled).
593 * If this bit is zero, all other bits except the one for the
594 * current window frame are also zero. So, we can use a simple test:
595 * 'and' WINDOWSTART and WINDOWSTART-1:
596 *
597 * (XXXXXX1[0]* - 1) AND XXXXXX1[0]* = XXXXXX0[0]*
598 *
599 * The result is zero only if one bit was set.
600 *
601 * (Note: We might have gone through several task switches before
602 * we come back to the current task, so WINDOWBASE might be
603 * different from the time the exception occurred.)
604 */
606 /* Test WINDOWSTART before and after the exception.
607 * We actually have WMASK, so we only have to test if it is 1 or not.
608 */
610 l32i a2, a1, PT_WMASK
611 _beqi a2, 1, common_exception_exit # Spilled before exception,jump
613 /* Test WINDOWSTART now. If spilled, do the movsp */
615 rsr a3, windowstart
616 addi a0, a3, -1
617 and a3, a3, a0
618 _bnez a3, common_exception_exit
620 /* Do a movsp (we returned from a call4, so we have at least a0..a7) */
622 addi a0, a1, -16
623 l32i a3, a0, 0
624 l32i a4, a0, 4
625 s32i a3, a1, PT_SIZE+0
626 s32i a4, a1, PT_SIZE+4
627 l32i a3, a0, 8
628 l32i a4, a0, 12
629 s32i a3, a1, PT_SIZE+8
630 s32i a4, a1, PT_SIZE+12
632 /* Common exception exit.
633 * We restore the special register and the current window frame, and
634 * return from the exception.
635 *
636 * Note: We expect a2 to hold PT_WMASK
637 */
639 common_exception_exit:
641 /* Restore address registers. */
643 _bbsi.l a2, 1, 1f
644 l32i a4, a1, PT_AREG4
645 l32i a5, a1, PT_AREG5
646 l32i a6, a1, PT_AREG6
647 l32i a7, a1, PT_AREG7
648 _bbsi.l a2, 2, 1f
649 l32i a8, a1, PT_AREG8
650 l32i a9, a1, PT_AREG9
651 l32i a10, a1, PT_AREG10
652 l32i a11, a1, PT_AREG11
653 _bbsi.l a2, 3, 1f
654 l32i a12, a1, PT_AREG12
655 l32i a13, a1, PT_AREG13
656 l32i a14, a1, PT_AREG14
657 l32i a15, a1, PT_AREG15
659 /* Restore PC, SAR */
661 1: l32i a2, a1, PT_PC
662 l32i a3, a1, PT_SAR
663 wsr a2, epc1
664 wsr a3, sar
666 /* Restore LBEG, LEND, LCOUNT */
668 l32i a2, a1, PT_LBEG
669 l32i a3, a1, PT_LEND
670 wsr a2, lbeg
671 l32i a2, a1, PT_LCOUNT
672 wsr a3, lend
673 wsr a2, lcount
675 /* We control single stepping through the ICOUNTLEVEL register. */
677 l32i a2, a1, PT_ICOUNTLEVEL
678 movi a3, -2
679 wsr a2, icountlevel
680 wsr a3, icount
682 /* Check if it was double exception. */
684 l32i a0, a1, PT_DEPC
685 l32i a3, a1, PT_AREG3
686 l32i a2, a1, PT_AREG2
687 _bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
689 /* Restore a0...a3 and return */
691 l32i a0, a1, PT_AREG0
692 l32i a1, a1, PT_AREG1
693 rfe
695 1: wsr a0, depc
696 l32i a0, a1, PT_AREG0
697 l32i a1, a1, PT_AREG1
698 rfde
700 ENDPROC(kernel_exception)
702 /*
703 * Debug exception handler.
704 *
705 * Currently, we don't support KGDB, so only user application can be debugged.
706 *
707 * When we get here, a0 is trashed and saved to excsave[debuglevel]
708 */
710 ENTRY(debug_exception)
712 rsr a0, SREG_EPS + XCHAL_DEBUGLEVEL
713 bbsi.l a0, PS_EXCM_BIT, 1f # exception mode
715 /* Set EPC1 and EXCCAUSE */
717 wsr a2, depc # save a2 temporarily
718 rsr a2, SREG_EPC + XCHAL_DEBUGLEVEL
719 wsr a2, epc1
721 movi a2, EXCCAUSE_MAPPED_DEBUG
722 wsr a2, exccause
724 /* Restore PS to the value before the debug exc but with PS.EXCM set.*/
726 movi a2, 1 << PS_EXCM_BIT
727 or a2, a0, a2
728 movi a0, debug_exception # restore a3, debug jump vector
729 wsr a2, ps
730 xsr a0, SREG_EXCSAVE + XCHAL_DEBUGLEVEL
732 /* Switch to kernel/user stack, restore jump vector, and save a0 */
734 bbsi.l a2, PS_UM_BIT, 2f # jump if user mode
736 addi a2, a1, -16-PT_SIZE # assume kernel stack
737 s32i a0, a2, PT_AREG0
738 movi a0, 0
739 s32i a1, a2, PT_AREG1
740 s32i a0, a2, PT_DEPC # mark it as a regular exception
741 xsr a0, depc
742 s32i a3, a2, PT_AREG3
743 s32i a0, a2, PT_AREG2
744 mov a1, a2
745 j _kernel_exception
747 2: rsr a2, excsave1
748 l32i a2, a2, EXC_TABLE_KSTK # load kernel stack pointer
749 s32i a0, a2, PT_AREG0
750 movi a0, 0
751 s32i a1, a2, PT_AREG1
752 s32i a0, a2, PT_DEPC
753 xsr a0, depc
754 s32i a3, a2, PT_AREG3
755 s32i a0, a2, PT_AREG2
756 mov a1, a2
757 j _user_exception
759 /* Debug exception while in exception mode. */
760 1: j 1b // FIXME!!
762 ENDPROC(debug_exception)
764 /*
765 * We get here in case of an unrecoverable exception.
766 * The only thing we can do is to be nice and print a panic message.
767 * We only produce a single stack frame for panic, so ???
768 *
769 *
770 * Entry conditions:
771 *
772 * - a0 contains the caller address; original value saved in excsave1.
773 * - the original a0 contains a valid return address (backtrace) or 0.
774 * - a2 contains a valid stackpointer
775 *
776 * Notes:
777 *
778 * - If the stack pointer could be invalid, the caller has to setup a
779 * dummy stack pointer (e.g. the stack of the init_task)
780 *
781 * - If the return address could be invalid, the caller has to set it
782 * to 0, so the backtrace would stop.
783 *
784 */
785 .align 4
786 unrecoverable_text:
787 .ascii "Unrecoverable error in exception handler\0"
789 ENTRY(unrecoverable_exception)
791 movi a0, 1
792 movi a1, 0
794 wsr a0, windowstart
795 wsr a1, windowbase
796 rsync
798 movi a1, (1 << PS_WOE_BIT) | LOCKLEVEL
799 wsr a1, ps
800 rsync
802 movi a1, init_task
803 movi a0, 0
804 addi a1, a1, PT_REGS_OFFSET
806 movi a4, panic
807 movi a6, unrecoverable_text
809 callx4 a4
811 1: j 1b
813 ENDPROC(unrecoverable_exception)
815 /* -------------------------- FAST EXCEPTION HANDLERS ----------------------- */
817 /*
818 * Fast-handler for alloca exceptions
819 *
820 * The ALLOCA handler is entered when user code executes the MOVSP
821 * instruction and the caller's frame is not in the register file.
822 *
823 * This algorithm was taken from the Ross Morley's RTOS Porting Layer:
824 *
825 * /home/ross/rtos/porting/XtensaRTOS-PortingLayer-20090507/xtensa_vectors.S
826 *
827 * It leverages the existing window spill/fill routines and their support for
828 * double exceptions. The 'movsp' instruction will only cause an exception if
829 * the next window needs to be loaded. In fact this ALLOCA exception may be
830 * replaced at some point by changing the hardware to do a underflow exception
831 * of the proper size instead.
832 *
833 * This algorithm simply backs out the register changes started by the user
834 * excpetion handler, makes it appear that we have started a window underflow
835 * by rotating the window back and then setting the old window base (OWB) in
836 * the 'ps' register with the rolled back window base. The 'movsp' instruction
837 * will be re-executed and this time since the next window frames is in the
838 * active AR registers it won't cause an exception.
839 *
840 * If the WindowUnderflow code gets a TLB miss the page will get mapped
841 * the the partial windeowUnderflow will be handeled in the double exception
842 * handler.
843 *
844 * Entry condition:
845 *
846 * a0: trashed, original value saved on stack (PT_AREG0)
847 * a1: a1
848 * a2: new stack pointer, original in DEPC
849 * a3: a3
850 * depc: a2, original value saved on stack (PT_DEPC)
851 * excsave_1: dispatch table
852 *
853 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
854 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
855 */
857 ENTRY(fast_alloca)
858 rsr a0, windowbase
859 rotw -1
860 rsr a2, ps
861 extui a3, a2, PS_OWB_SHIFT, PS_OWB_WIDTH
862 xor a3, a3, a4
863 l32i a4, a6, PT_AREG0
864 l32i a1, a6, PT_DEPC
865 rsr a6, depc
866 wsr a1, depc
867 slli a3, a3, PS_OWB_SHIFT
868 xor a2, a2, a3
869 wsr a2, ps
870 rsync
872 _bbci.l a4, 31, 4f
873 rotw -1
874 _bbci.l a8, 30, 8f
875 rotw -1
876 j _WindowUnderflow12
877 8: j _WindowUnderflow8
878 4: j _WindowUnderflow4
879 ENDPROC(fast_alloca)
881 /*
882 * fast system calls.
883 *
884 * WARNING: The kernel doesn't save the entire user context before
885 * handling a fast system call. These functions are small and short,
886 * usually offering some functionality not available to user tasks.
887 *
888 * BE CAREFUL TO PRESERVE THE USER'S CONTEXT.
889 *
890 * Entry condition:
891 *
892 * a0: trashed, original value saved on stack (PT_AREG0)
893 * a1: a1
894 * a2: new stack pointer, original in DEPC
895 * a3: a3
896 * depc: a2, original value saved on stack (PT_DEPC)
897 * excsave_1: dispatch table
898 */
900 ENTRY(fast_syscall_kernel)
902 /* Skip syscall. */
904 rsr a0, epc1
905 addi a0, a0, 3
906 wsr a0, epc1
908 l32i a0, a2, PT_DEPC
909 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
911 rsr a0, depc # get syscall-nr
912 _beqz a0, fast_syscall_spill_registers
913 _beqi a0, __NR_xtensa, fast_syscall_xtensa
915 j kernel_exception
917 ENDPROC(fast_syscall_kernel)
919 ENTRY(fast_syscall_user)
921 /* Skip syscall. */
923 rsr a0, epc1
924 addi a0, a0, 3
925 wsr a0, epc1
927 l32i a0, a2, PT_DEPC
928 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, fast_syscall_unrecoverable
930 rsr a0, depc # get syscall-nr
931 _beqz a0, fast_syscall_spill_registers
932 _beqi a0, __NR_xtensa, fast_syscall_xtensa
934 j user_exception
936 ENDPROC(fast_syscall_user)
938 ENTRY(fast_syscall_unrecoverable)
940 /* Restore all states. */
942 l32i a0, a2, PT_AREG0 # restore a0
943 xsr a2, depc # restore a2, depc
945 wsr a0, excsave1
946 movi a0, unrecoverable_exception
947 callx0 a0
949 ENDPROC(fast_syscall_unrecoverable)
951 /*
952 * sysxtensa syscall handler
953 *
954 * int sysxtensa (SYS_XTENSA_ATOMIC_SET, ptr, val, unused);
955 * int sysxtensa (SYS_XTENSA_ATOMIC_ADD, ptr, val, unused);
956 * int sysxtensa (SYS_XTENSA_ATOMIC_EXG_ADD, ptr, val, unused);
957 * int sysxtensa (SYS_XTENSA_ATOMIC_CMP_SWP, ptr, oldval, newval);
958 * a2 a6 a3 a4 a5
959 *
960 * Entry condition:
961 *
962 * a0: a2 (syscall-nr), original value saved on stack (PT_AREG0)
963 * a1: a1
964 * a2: new stack pointer, original in a0 and DEPC
965 * a3: a3
966 * a4..a15: unchanged
967 * depc: a2, original value saved on stack (PT_DEPC)
968 * excsave_1: dispatch table
969 *
970 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
971 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
972 *
973 * Note: we don't have to save a2; a2 holds the return value
974 *
975 * We use the two macros TRY and CATCH:
976 *
977 * TRY adds an entry to the __ex_table fixup table for the immediately
978 * following instruction.
979 *
980 * CATCH catches any exception that occurred at one of the preceding TRY
981 * statements and continues from there
982 *
983 * Usage TRY l32i a0, a1, 0
984 * <other code>
985 * done: rfe
986 * CATCH <set return code>
987 * j done
988 */
990 #ifdef CONFIG_FAST_SYSCALL_XTENSA
992 #define TRY \
993 .section __ex_table, "a"; \
994 .word 66f, 67f; \
995 .text; \
996 66:
998 #define CATCH \
999 67:
1001 ENTRY(fast_syscall_xtensa)
1003 s32i a7, a2, PT_AREG7 # we need an additional register
1004 movi a7, 4 # sizeof(unsigned int)
1005 access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
1007 _bgeui a6, SYS_XTENSA_COUNT, .Lill
1008 _bnei a6, SYS_XTENSA_ATOMIC_CMP_SWP, .Lnswp
1010 /* Fall through for ATOMIC_CMP_SWP. */
1012 .Lswp: /* Atomic compare and swap */
1014 TRY l32i a0, a3, 0 # read old value
1015 bne a0, a4, 1f # same as old value? jump
1016 TRY s32i a5, a3, 0 # different, modify value
1017 l32i a7, a2, PT_AREG7 # restore a7
1018 l32i a0, a2, PT_AREG0 # restore a0
1019 movi a2, 1 # and return 1
1020 rfe
1022 1: l32i a7, a2, PT_AREG7 # restore a7
1023 l32i a0, a2, PT_AREG0 # restore a0
1024 movi a2, 0 # return 0 (note that we cannot set
1025 rfe
1027 .Lnswp: /* Atomic set, add, and exg_add. */
1029 TRY l32i a7, a3, 0 # orig
1030 addi a6, a6, -SYS_XTENSA_ATOMIC_SET
1031 add a0, a4, a7 # + arg
1032 moveqz a0, a4, a6 # set
1033 addi a6, a6, SYS_XTENSA_ATOMIC_SET
1034 TRY s32i a0, a3, 0 # write new value
1036 mov a0, a2
1037 mov a2, a7
1038 l32i a7, a0, PT_AREG7 # restore a7
1039 l32i a0, a0, PT_AREG0 # restore a0
1040 rfe
1042 CATCH
1043 .Leac: l32i a7, a2, PT_AREG7 # restore a7
1044 l32i a0, a2, PT_AREG0 # restore a0
1045 movi a2, -EFAULT
1046 rfe
1048 .Lill: l32i a7, a2, PT_AREG7 # restore a7
1049 l32i a0, a2, PT_AREG0 # restore a0
1050 movi a2, -EINVAL
1051 rfe
1053 ENDPROC(fast_syscall_xtensa)
1055 #else /* CONFIG_FAST_SYSCALL_XTENSA */
1057 ENTRY(fast_syscall_xtensa)
1059 l32i a0, a2, PT_AREG0 # restore a0
1060 movi a2, -ENOSYS
1061 rfe
1063 ENDPROC(fast_syscall_xtensa)
1065 #endif /* CONFIG_FAST_SYSCALL_XTENSA */
1068 /* fast_syscall_spill_registers.
1069 *
1070 * Entry condition:
1071 *
1072 * a0: trashed, original value saved on stack (PT_AREG0)
1073 * a1: a1
1074 * a2: new stack pointer, original in DEPC
1075 * a3: a3
1076 * depc: a2, original value saved on stack (PT_DEPC)
1077 * excsave_1: dispatch table
1078 *
1079 * Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
1080 */
1082 #ifdef CONFIG_FAST_SYSCALL_SPILL_REGISTERS
1084 ENTRY(fast_syscall_spill_registers)
1086 /* Register a FIXUP handler (pass current wb as a parameter) */
1088 xsr a3, excsave1
1089 movi a0, fast_syscall_spill_registers_fixup
1090 s32i a0, a3, EXC_TABLE_FIXUP
1091 rsr a0, windowbase
1092 s32i a0, a3, EXC_TABLE_PARAM
1093 xsr a3, excsave1 # restore a3 and excsave_1
1095 /* Save a3, a4 and SAR on stack. */
1097 rsr a0, sar
1098 s32i a3, a2, PT_AREG3
1099 s32i a0, a2, PT_SAR
1101 /* The spill routine might clobber a4, a7, a8, a11, a12, and a15. */
1103 s32i a4, a2, PT_AREG4
1104 s32i a7, a2, PT_AREG7
1105 s32i a8, a2, PT_AREG8
1106 s32i a11, a2, PT_AREG11
1107 s32i a12, a2, PT_AREG12
1108 s32i a15, a2, PT_AREG15
1110 /*
1111 * Rotate ws so that the current windowbase is at bit 0.
1112 * Assume ws = xxxwww1yy (www1 current window frame).
1113 * Rotate ws right so that a4 = yyxxxwww1.
1114 */
1116 rsr a0, windowbase
1117 rsr a3, windowstart # a3 = xxxwww1yy
1118 ssr a0 # holds WB
1119 slli a0, a3, WSBITS
1120 or a3, a3, a0 # a3 = xxxwww1yyxxxwww1yy
1121 srl a3, a3 # a3 = 00xxxwww1yyxxxwww1
1123 /* We are done if there are no more than the current register frame. */
1125 extui a3, a3, 1, WSBITS-1 # a3 = 0yyxxxwww
1126 movi a0, (1 << (WSBITS-1))
1127 _beqz a3, .Lnospill # only one active frame? jump
1129 /* We want 1 at the top, so that we return to the current windowbase */
1131 or a3, a3, a0 # 1yyxxxwww
1133 /* Skip empty frames - get 'oldest' WINDOWSTART-bit. */
1135 wsr a3, windowstart # save shifted windowstart
1136 neg a0, a3
1137 and a3, a0, a3 # first bit set from right: 000010000
1139 ffs_ws a0, a3 # a0: shifts to skip empty frames
1140 movi a3, WSBITS
1141 sub a0, a3, a0 # WSBITS-a0:number of 0-bits from right
1142 ssr a0 # save in SAR for later.
1144 rsr a3, windowbase
1145 add a3, a3, a0
1146 wsr a3, windowbase
1147 rsync
1149 rsr a3, windowstart
1150 srl a3, a3 # shift windowstart
1152 /* WB is now just one frame below the oldest frame in the register
1153 window. WS is shifted so the oldest frame is in bit 0, thus, WB
1154 and WS differ by one 4-register frame. */
1156 /* Save frames. Depending what call was used (call4, call8, call12),
1157 * we have to save 4,8. or 12 registers.
1158 */
1161 .Lloop: _bbsi.l a3, 1, .Lc4
1162 _bbci.l a3, 2, .Lc12
1164 .Lc8: s32e a4, a13, -16
1165 l32e a4, a5, -12
1166 s32e a8, a4, -32
1167 s32e a5, a13, -12
1168 s32e a6, a13, -8
1169 s32e a7, a13, -4
1170 s32e a9, a4, -28
1171 s32e a10, a4, -24
1172 s32e a11, a4, -20
1173 srli a11, a3, 2 # shift windowbase by 2
1174 rotw 2
1175 _bnei a3, 1, .Lloop
1176 j .Lexit
1178 .Lc4: s32e a4, a9, -16
1179 s32e a5, a9, -12
1180 s32e a6, a9, -8
1181 s32e a7, a9, -4
1183 srli a7, a3, 1
1184 rotw 1
1185 _bnei a3, 1, .Lloop
1186 j .Lexit
1188 .Lc12: _bbci.l a3, 3, .Linvalid_mask # bit 2 shouldn't be zero!
1190 /* 12-register frame (call12) */
1192 l32e a0, a5, -12
1193 s32e a8, a0, -48
1194 mov a8, a0
1196 s32e a9, a8, -44
1197 s32e a10, a8, -40
1198 s32e a11, a8, -36
1199 s32e a12, a8, -32
1200 s32e a13, a8, -28
1201 s32e a14, a8, -24
1202 s32e a15, a8, -20
1203 srli a15, a3, 3
1205 /* The stack pointer for a4..a7 is out of reach, so we rotate the
1206 * window, grab the stackpointer, and rotate back.
1207 * Alternatively, we could also use the following approach, but that
1208 * makes the fixup routine much more complicated:
1209 * rotw 1
1210 * s32e a0, a13, -16
1211 * ...
1212 * rotw 2
1213 */
1215 rotw 1
1216 mov a4, a13
1217 rotw -1
1219 s32e a4, a8, -16
1220 s32e a5, a8, -12
1221 s32e a6, a8, -8
1222 s32e a7, a8, -4
1224 rotw 3
1226 _beqi a3, 1, .Lexit
1227 j .Lloop
1229 .Lexit:
1231 /* Done. Do the final rotation and set WS */
1233 rotw 1
1234 rsr a3, windowbase
1235 ssl a3
1236 movi a3, 1
1237 sll a3, a3
1238 wsr a3, windowstart
1239 .Lnospill:
1241 /* Advance PC, restore registers and SAR, and return from exception. */
1243 l32i a3, a2, PT_SAR
1244 l32i a0, a2, PT_AREG0
1245 wsr a3, sar
1246 l32i a3, a2, PT_AREG3
1248 /* Restore clobbered registers. */
1250 l32i a4, a2, PT_AREG4
1251 l32i a7, a2, PT_AREG7
1252 l32i a8, a2, PT_AREG8
1253 l32i a11, a2, PT_AREG11
1254 l32i a12, a2, PT_AREG12
1255 l32i a15, a2, PT_AREG15
1257 movi a2, 0
1258 rfe
1260 .Linvalid_mask:
1262 /* We get here because of an unrecoverable error in the window
1263 * registers, so set up a dummy frame and kill the user application.
1264 * Note: We assume EXC_TABLE_KSTK contains a valid stack pointer.
1265 */
1267 movi a0, 1
1268 movi a1, 0
1270 wsr a0, windowstart
1271 wsr a1, windowbase
1272 rsync
1274 movi a0, 0
1276 rsr a3, excsave1
1277 l32i a1, a3, EXC_TABLE_KSTK
1279 movi a4, (1 << PS_WOE_BIT) | LOCKLEVEL
1280 wsr a4, ps
1281 rsync
1283 movi a6, SIGSEGV
1284 movi a4, do_exit
1285 callx4 a4
1287 /* shouldn't return, so panic */
1289 wsr a0, excsave1
1290 movi a0, unrecoverable_exception
1291 callx0 a0 # should not return
1292 1: j 1b
1295 ENDPROC(fast_syscall_spill_registers)
1297 /* Fixup handler.
1298 *
1299 * We get here if the spill routine causes an exception, e.g. tlb miss.
1300 * We basically restore WINDOWBASE and WINDOWSTART to the condition when
1301 * we entered the spill routine and jump to the user exception handler.
1302 *
1303 * Note that we only need to restore the bits in windowstart that have not
1304 * been spilled yet by the _spill_register routine. Luckily, a3 contains a
1305 * rotated windowstart with only those bits set for frames that haven't been
1306 * spilled yet. Because a3 is rotated such that bit 0 represents the register
1307 * frame for the current windowbase - 1, we need to rotate a3 left by the
1308 * value of the current windowbase + 1 and move it to windowstart.
1309 *
1310 * a0: value of depc, original value in depc
1311 * a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE
1312 * a3: exctable, original value in excsave1
1313 */
1315 ENTRY(fast_syscall_spill_registers_fixup)
1317 rsr a2, windowbase # get current windowbase (a2 is saved)
1318 xsr a0, depc # restore depc and a0
1319 ssl a2 # set shift (32 - WB)
1321 /* We need to make sure the current registers (a0-a3) are preserved.
1322 * To do this, we simply set the bit for the current window frame
1323 * in WS, so that the exception handlers save them to the task stack.
1324 *
1325 * Note: we use a3 to set the windowbase, so we take a special care
1326 * of it, saving it in the original _spill_registers frame across
1327 * the exception handler call.
1328 */
1330 xsr a3, excsave1 # get spill-mask
1331 slli a3, a3, 1 # shift left by one
1332 addi a3, a3, 1 # set the bit for the current window frame
1334 slli a2, a3, 32-WSBITS
1335 src a2, a3, a2 # a2 = xxwww1yyxxxwww1yy......
1336 wsr a2, windowstart # set corrected windowstart
1338 srli a3, a3, 1
1339 rsr a2, excsave1
1340 l32i a2, a2, EXC_TABLE_DOUBLE_SAVE # restore a2
1341 xsr a2, excsave1
1342 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE # save a3
1343 l32i a3, a2, EXC_TABLE_PARAM # original WB (in user task)
1344 xsr a2, excsave1
1346 /* Return to the original (user task) WINDOWBASE.
1347 * We leave the following frame behind:
1348 * a0, a1, a2 same
1349 * a3: trashed (saved in EXC_TABLE_DOUBLE_SAVE)
1350 * depc: depc (we have to return to that address)
1351 * excsave_1: exctable
1352 */
1354 wsr a3, windowbase
1355 rsync
1357 /* We are now in the original frame when we entered _spill_registers:
1358 * a0: return address
1359 * a1: used, stack pointer
1360 * a2: kernel stack pointer
1361 * a3: available
1362 * depc: exception address
1363 * excsave: exctable
1364 * Note: This frame might be the same as above.
1365 */
1367 /* Setup stack pointer. */
1369 addi a2, a2, -PT_USER_SIZE
1370 s32i a0, a2, PT_AREG0
1372 /* Make sure we return to this fixup handler. */
1374 movi a3, fast_syscall_spill_registers_fixup_return
1375 s32i a3, a2, PT_DEPC # setup depc
1377 /* Jump to the exception handler. */
1379 rsr a3, excsave1
1380 rsr a0, exccause
1381 addx4 a0, a0, a3 # find entry in table
1382 l32i a0, a0, EXC_TABLE_FAST_USER # load handler
1383 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
1384 jx a0
1386 ENDPROC(fast_syscall_spill_registers_fixup)
1388 ENTRY(fast_syscall_spill_registers_fixup_return)
1390 /* When we return here, all registers have been restored (a2: DEPC) */
1392 wsr a2, depc # exception address
1394 /* Restore fixup handler. */
1396 rsr a2, excsave1
1397 s32i a3, a2, EXC_TABLE_DOUBLE_SAVE
1398 movi a3, fast_syscall_spill_registers_fixup
1399 s32i a3, a2, EXC_TABLE_FIXUP
1400 rsr a3, windowbase
1401 s32i a3, a2, EXC_TABLE_PARAM
1402 l32i a2, a2, EXC_TABLE_KSTK
1404 /* Load WB at the time the exception occurred. */
1406 rsr a3, sar # WB is still in SAR
1407 neg a3, a3
1408 wsr a3, windowbase
1409 rsync
1411 rsr a3, excsave1
1412 l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
1414 rfde
1416 ENDPROC(fast_syscall_spill_registers_fixup_return)
1418 #else /* CONFIG_FAST_SYSCALL_SPILL_REGISTERS */
1420 ENTRY(fast_syscall_spill_registers)
1422 l32i a0, a2, PT_AREG0 # restore a0
1423 movi a2, -ENOSYS
1424 rfe
1426 ENDPROC(fast_syscall_spill_registers)
1428 #endif /* CONFIG_FAST_SYSCALL_SPILL_REGISTERS */
1430 #ifdef CONFIG_MMU
1431 /*
1432 * We should never get here. Bail out!
1433 */
1435 ENTRY(fast_second_level_miss_double_kernel)
1437 1: movi a0, unrecoverable_exception
1438 callx0 a0 # should not return
1439 1: j 1b
1441 ENDPROC(fast_second_level_miss_double_kernel)
1443 /* First-level entry handler for user, kernel, and double 2nd-level
1444 * TLB miss exceptions. Note that for now, user and kernel miss
1445 * exceptions share the same entry point and are handled identically.
1446 *
1447 * An old, less-efficient C version of this function used to exist.
1448 * We include it below, interleaved as comments, for reference.
1449 *
1450 * Entry condition:
1451 *
1452 * a0: trashed, original value saved on stack (PT_AREG0)
1453 * a1: a1
1454 * a2: new stack pointer, original in DEPC
1455 * a3: a3
1456 * depc: a2, original value saved on stack (PT_DEPC)
1457 * excsave_1: dispatch table
1458 *
1459 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1460 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1461 */
1463 ENTRY(fast_second_level_miss)
1465 /* Save a1 and a3. Note: we don't expect a double exception. */
1467 s32i a1, a2, PT_AREG1
1468 s32i a3, a2, PT_AREG3
1470 /* We need to map the page of PTEs for the user task. Find
1471 * the pointer to that page. Also, it's possible for tsk->mm
1472 * to be NULL while tsk->active_mm is nonzero if we faulted on
1473 * a vmalloc address. In that rare case, we must use
1474 * active_mm instead to avoid a fault in this handler. See
1475 *
1476 * http://mail.nl.linux.org/linux-mm/2002-08/msg00258.html
1477 * (or search Internet on "mm vs. active_mm")
1478 *
1479 * if (!mm)
1480 * mm = tsk->active_mm;
1481 * pgd = pgd_offset (mm, regs->excvaddr);
1482 * pmd = pmd_offset (pgd, regs->excvaddr);
1483 * pmdval = *pmd;
1484 */
1486 GET_CURRENT(a1,a2)
1487 l32i a0, a1, TASK_MM # tsk->mm
1488 beqz a0, 9f
1490 8: rsr a3, excvaddr # fault address
1491 _PGD_OFFSET(a0, a3, a1)
1492 l32i a0, a0, 0 # read pmdval
1493 beqz a0, 2f
1495 /* Read ptevaddr and convert to top of page-table page.
1496 *
1497 * vpnval = read_ptevaddr_register() & PAGE_MASK;
1498 * vpnval += DTLB_WAY_PGTABLE;
1499 * pteval = mk_pte (virt_to_page(pmd_val(pmdval)), PAGE_KERNEL);
1500 * write_dtlb_entry (pteval, vpnval);
1501 *
1502 * The messy computation for 'pteval' above really simplifies
1503 * into the following:
1504 *
1505 * pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY
1506 */
1508 movi a1, (-PAGE_OFFSET) & 0xffffffff
1509 add a0, a0, a1 # pmdval - PAGE_OFFSET
1510 extui a1, a0, 0, PAGE_SHIFT # ... & PAGE_MASK
1511 xor a0, a0, a1
1513 movi a1, _PAGE_DIRECTORY
1514 or a0, a0, a1 # ... | PAGE_DIRECTORY
1516 /*
1517 * We utilize all three wired-ways (7-9) to hold pmd translations.
1518 * Memory regions are mapped to the DTLBs according to bits 28 and 29.
1519 * This allows to map the three most common regions to three different
1520 * DTLBs:
1521 * 0,1 -> way 7 program (0040.0000) and virtual (c000.0000)
1522 * 2 -> way 8 shared libaries (2000.0000)
1523 * 3 -> way 0 stack (3000.0000)
1524 */
1526 extui a3, a3, 28, 2 # addr. bit 28 and 29 0,1,2,3
1527 rsr a1, ptevaddr
1528 addx2 a3, a3, a3 # -> 0,3,6,9
1529 srli a1, a1, PAGE_SHIFT
1530 extui a3, a3, 2, 2 # -> 0,0,1,2
1531 slli a1, a1, PAGE_SHIFT # ptevaddr & PAGE_MASK
1532 addi a3, a3, DTLB_WAY_PGD
1533 add a1, a1, a3 # ... + way_number
1535 3: wdtlb a0, a1
1536 dsync
1538 /* Exit critical section. */
1540 4: rsr a3, excsave1
1541 movi a0, 0
1542 s32i a0, a3, EXC_TABLE_FIXUP
1544 /* Restore the working registers, and return. */
1546 l32i a0, a2, PT_AREG0
1547 l32i a1, a2, PT_AREG1
1548 l32i a3, a2, PT_AREG3
1549 l32i a2, a2, PT_DEPC
1551 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1553 /* Restore excsave1 and return. */
1555 rsr a2, depc
1556 rfe
1558 /* Return from double exception. */
1560 1: xsr a2, depc
1561 esync
1562 rfde
1564 9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1565 j 8b
1567 #if (DCACHE_WAY_SIZE > PAGE_SIZE)
1569 2: /* Special case for cache aliasing.
1570 * We (should) only get here if a clear_user_page, copy_user_page
1571 * or the aliased cache flush functions got preemptively interrupted
1572 * by another task. Re-establish temporary mapping to the
1573 * TLBTEMP_BASE areas.
1574 */
1576 /* We shouldn't be in a double exception */
1578 l32i a0, a2, PT_DEPC
1579 bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f
1581 /* Make sure the exception originated in the special functions */
1583 movi a0, __tlbtemp_mapping_start
1584 rsr a3, epc1
1585 bltu a3, a0, 2f
1586 movi a0, __tlbtemp_mapping_end
1587 bgeu a3, a0, 2f
1589 /* Check if excvaddr was in one of the TLBTEMP_BASE areas. */
1591 movi a3, TLBTEMP_BASE_1
1592 rsr a0, excvaddr
1593 bltu a0, a3, 2f
1595 addi a1, a0, -TLBTEMP_SIZE
1596 bgeu a1, a3, 2f
1598 /* Check if we have to restore an ITLB mapping. */
1600 movi a1, __tlbtemp_mapping_itlb
1601 rsr a3, epc1
1602 sub a3, a3, a1
1604 /* Calculate VPN */
1606 movi a1, PAGE_MASK
1607 and a1, a1, a0
1609 /* Jump for ITLB entry */
1611 bgez a3, 1f
1613 /* We can use up to two TLBTEMP areas, one for src and one for dst. */
1615 extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1
1616 add a1, a3, a1
1618 /* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */
1620 mov a0, a6
1621 movnez a0, a7, a3
1622 j 3b
1624 /* ITLB entry. We only use dst in a6. */
1626 1: witlb a6, a1
1627 isync
1628 j 4b
1631 #endif // DCACHE_WAY_SIZE > PAGE_SIZE
1634 2: /* Invalid PGD, default exception handling */
1636 rsr a1, depc
1637 s32i a1, a2, PT_AREG2
1638 mov a1, a2
1640 rsr a2, ps
1641 bbsi.l a2, PS_UM_BIT, 1f
1642 j _kernel_exception
1643 1: j _user_exception
1645 ENDPROC(fast_second_level_miss)
1647 /*
1648 * StoreProhibitedException
1649 *
1650 * Update the pte and invalidate the itlb mapping for this pte.
1651 *
1652 * Entry condition:
1653 *
1654 * a0: trashed, original value saved on stack (PT_AREG0)
1655 * a1: a1
1656 * a2: new stack pointer, original in DEPC
1657 * a3: a3
1658 * depc: a2, original value saved on stack (PT_DEPC)
1659 * excsave_1: dispatch table
1660 *
1661 * PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
1662 * < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
1663 */
1665 ENTRY(fast_store_prohibited)
1667 /* Save a1 and a3. */
1669 s32i a1, a2, PT_AREG1
1670 s32i a3, a2, PT_AREG3
1672 GET_CURRENT(a1,a2)
1673 l32i a0, a1, TASK_MM # tsk->mm
1674 beqz a0, 9f
1676 8: rsr a1, excvaddr # fault address
1677 _PGD_OFFSET(a0, a1, a3)
1678 l32i a0, a0, 0
1679 beqz a0, 2f
1681 /*
1682 * Note that we test _PAGE_WRITABLE_BIT only if PTE is present
1683 * and is not PAGE_NONE. See pgtable.h for possible PTE layouts.
1684 */
1686 _PTE_OFFSET(a0, a1, a3)
1687 l32i a3, a0, 0 # read pteval
1688 movi a1, _PAGE_CA_INVALID
1689 ball a3, a1, 2f
1690 bbci.l a3, _PAGE_WRITABLE_BIT, 2f
1692 movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
1693 or a3, a3, a1
1694 rsr a1, excvaddr
1695 s32i a3, a0, 0
1697 /* We need to flush the cache if we have page coloring. */
1698 #if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
1699 dhwb a0, 0
1700 #endif
1701 pdtlb a0, a1
1702 wdtlb a3, a0
1704 /* Exit critical section. */
1706 movi a0, 0
1707 rsr a3, excsave1
1708 s32i a0, a3, EXC_TABLE_FIXUP
1710 /* Restore the working registers, and return. */
1712 l32i a3, a2, PT_AREG3
1713 l32i a1, a2, PT_AREG1
1714 l32i a0, a2, PT_AREG0
1715 l32i a2, a2, PT_DEPC
1717 bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
1719 rsr a2, depc
1720 rfe
1722 /* Double exception. Restore FIXUP handler and return. */
1724 1: xsr a2, depc
1725 esync
1726 rfde
1728 9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
1729 j 8b
1731 2: /* If there was a problem, handle fault in C */
1733 rsr a3, depc # still holds a2
1734 s32i a3, a2, PT_AREG2
1735 mov a1, a2
1737 rsr a2, ps
1738 bbsi.l a2, PS_UM_BIT, 1f
1739 j _kernel_exception
1740 1: j _user_exception
1742 ENDPROC(fast_store_prohibited)
1744 #endif /* CONFIG_MMU */
1746 /*
1747 * System Calls.
1748 *
1749 * void system_call (struct pt_regs* regs, int exccause)
1750 * a2 a3
1751 */
1753 ENTRY(system_call)
1755 entry a1, 32
1757 /* regs->syscall = regs->areg[2] */
1759 l32i a3, a2, PT_AREG2
1760 mov a6, a2
1761 movi a4, do_syscall_trace_enter
1762 s32i a3, a2, PT_SYSCALL
1763 callx4 a4
1765 /* syscall = sys_call_table[syscall_nr] */
1767 movi a4, sys_call_table;
1768 movi a5, __NR_syscall_count
1769 movi a6, -ENOSYS
1770 bgeu a3, a5, 1f
1772 addx4 a4, a3, a4
1773 l32i a4, a4, 0
1774 movi a5, sys_ni_syscall;
1775 beq a4, a5, 1f
1777 /* Load args: arg0 - arg5 are passed via regs. */
1779 l32i a6, a2, PT_AREG6
1780 l32i a7, a2, PT_AREG3
1781 l32i a8, a2, PT_AREG4
1782 l32i a9, a2, PT_AREG5
1783 l32i a10, a2, PT_AREG8
1784 l32i a11, a2, PT_AREG9
1786 /* Pass one additional argument to the syscall: pt_regs (on stack) */
1787 s32i a2, a1, 0
1789 callx4 a4
1791 1: /* regs->areg[2] = return_value */
1793 s32i a6, a2, PT_AREG2
1794 movi a4, do_syscall_trace_leave
1795 mov a6, a2
1796 callx4 a4
1797 retw
1799 ENDPROC(system_call)
1801 /*
1802 * Spill live registers on the kernel stack macro.
1803 *
1804 * Entry condition: ps.woe is set, ps.excm is cleared
1805 * Exit condition: windowstart has single bit set
1806 * May clobber: a12, a13
1807 */
1808 .macro spill_registers_kernel
1810 #if XCHAL_NUM_AREGS > 16
1811 call12 1f
1812 _j 2f
1813 retw
1814 .align 4
1815 1:
1816 _entry a1, 48
1817 addi a12, a0, 3
1818 #if XCHAL_NUM_AREGS > 32
1819 .rept (XCHAL_NUM_AREGS - 32) / 12
1820 _entry a1, 48
1821 mov a12, a0
1822 .endr
1823 #endif
1824 _entry a1, 16
1825 #if XCHAL_NUM_AREGS % 12 == 0
1826 mov a8, a8
1827 #elif XCHAL_NUM_AREGS % 12 == 4
1828 mov a12, a12
1829 #elif XCHAL_NUM_AREGS % 12 == 8
1830 mov a4, a4
1831 #endif
1832 retw
1833 2:
1834 #else
1835 mov a12, a12
1836 #endif
1837 .endm
1839 /*
1840 * Task switch.
1841 *
1842 * struct task* _switch_to (struct task* prev, struct task* next)
1843 * a2 a2 a3
1844 */
1846 ENTRY(_switch_to)
1848 entry a1, 48
1850 mov a11, a3 # and 'next' (a3)
1852 l32i a4, a2, TASK_THREAD_INFO
1853 l32i a5, a3, TASK_THREAD_INFO
1855 save_xtregs_user a4 a6 a8 a9 a12 a13 THREAD_XTREGS_USER
1857 #if THREAD_RA > 1020 || THREAD_SP > 1020
1858 addi a10, a2, TASK_THREAD
1859 s32i a0, a10, THREAD_RA - TASK_THREAD # save return address
1860 s32i a1, a10, THREAD_SP - TASK_THREAD # save stack pointer
1861 #else
1862 s32i a0, a2, THREAD_RA # save return address
1863 s32i a1, a2, THREAD_SP # save stack pointer
1864 #endif
1866 /* Disable ints while we manipulate the stack pointer. */
1868 rsil a14, LOCKLEVEL
1869 rsr a3, excsave1
1870 rsync
1871 s32i a3, a3, EXC_TABLE_FIXUP /* enter critical section */
1873 /* Switch CPENABLE */
1875 #if (XTENSA_HAVE_COPROCESSORS || XTENSA_HAVE_IO_PORTS)
1876 l32i a3, a5, THREAD_CPENABLE
1877 xsr a3, cpenable
1878 s32i a3, a4, THREAD_CPENABLE
1879 #endif
1881 /* Flush register file. */
1883 spill_registers_kernel
1885 /* Set kernel stack (and leave critical section)
1886 * Note: It's save to set it here. The stack will not be overwritten
1887 * because the kernel stack will only be loaded again after
1888 * we return from kernel space.
1889 */
1891 rsr a3, excsave1 # exc_table
1892 movi a6, 0
1893 addi a7, a5, PT_REGS_OFFSET
1894 s32i a6, a3, EXC_TABLE_FIXUP
1895 s32i a7, a3, EXC_TABLE_KSTK
1897 /* restore context of the task 'next' */
1899 l32i a0, a11, THREAD_RA # restore return address
1900 l32i a1, a11, THREAD_SP # restore stack pointer
1902 load_xtregs_user a5 a6 a8 a9 a12 a13 THREAD_XTREGS_USER
1904 wsr a14, ps
1905 rsync
1907 retw
1909 ENDPROC(_switch_to)
1911 ENTRY(ret_from_fork)
1913 /* void schedule_tail (struct task_struct *prev)
1914 * Note: prev is still in a6 (return value from fake call4 frame)
1915 */
1916 movi a4, schedule_tail
1917 callx4 a4
1919 movi a4, do_syscall_trace_leave
1920 mov a6, a1
1921 callx4 a4
1923 j common_exception_return
1925 ENDPROC(ret_from_fork)
1927 /*
1928 * Kernel thread creation helper
1929 * On entry, set up by copy_thread: a2 = thread_fn, a3 = thread_fn arg
1930 * left from _switch_to: a6 = prev
1931 */
1932 ENTRY(ret_from_kernel_thread)
1934 call4 schedule_tail
1935 mov a6, a3
1936 callx4 a2
1937 j common_exception_return
1939 ENDPROC(ret_from_kernel_thread)