| blob | 13d4826ab2a1d3610504bbf30ff6625507244ad2 |
1 /*
2 * Copyright (C) 2012 Regents of the University of California
3 * Copyright (C) 2017 SiFive
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation, version 2.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 */
15 #include <linux/init.h>
16 #include <linux/linkage.h>
18 #include <asm/asm.h>
19 #include <asm/csr.h>
20 #include <asm/unistd.h>
21 #include <asm/thread_info.h>
22 #include <asm/asm-offsets.h>
24 .text
25 .altmacro
27 /*
28 * Prepares to enter a system call or exception by saving all registers to the
29 * stack.
30 */
31 .macro SAVE_ALL
32 LOCAL _restore_kernel_tpsp
33 LOCAL _save_context
35 /*
36 * If coming from userspace, preserve the user thread pointer and load
37 * the kernel thread pointer. If we came from the kernel, sscratch
38 * will contain 0, and we should continue on the current TP.
39 */
40 csrrw tp, sscratch, tp
41 bnez tp, _save_context
43 _restore_kernel_tpsp:
44 csrr tp, sscratch
45 REG_S sp, TASK_TI_KERNEL_SP(tp)
46 _save_context:
47 REG_S sp, TASK_TI_USER_SP(tp)
48 REG_L sp, TASK_TI_KERNEL_SP(tp)
49 addi sp, sp, -(PT_SIZE_ON_STACK)
50 REG_S x1, PT_RA(sp)
51 REG_S x3, PT_GP(sp)
52 REG_S x5, PT_T0(sp)
53 REG_S x6, PT_T1(sp)
54 REG_S x7, PT_T2(sp)
55 REG_S x8, PT_S0(sp)
56 REG_S x9, PT_S1(sp)
57 REG_S x10, PT_A0(sp)
58 REG_S x11, PT_A1(sp)
59 REG_S x12, PT_A2(sp)
60 REG_S x13, PT_A3(sp)
61 REG_S x14, PT_A4(sp)
62 REG_S x15, PT_A5(sp)
63 REG_S x16, PT_A6(sp)
64 REG_S x17, PT_A7(sp)
65 REG_S x18, PT_S2(sp)
66 REG_S x19, PT_S3(sp)
67 REG_S x20, PT_S4(sp)
68 REG_S x21, PT_S5(sp)
69 REG_S x22, PT_S6(sp)
70 REG_S x23, PT_S7(sp)
71 REG_S x24, PT_S8(sp)
72 REG_S x25, PT_S9(sp)
73 REG_S x26, PT_S10(sp)
74 REG_S x27, PT_S11(sp)
75 REG_S x28, PT_T3(sp)
76 REG_S x29, PT_T4(sp)
77 REG_S x30, PT_T5(sp)
78 REG_S x31, PT_T6(sp)
80 /*
81 * Disable user-mode memory access as it should only be set in the
82 * actual user copy routines.
83 *
84 * Disable the FPU to detect illegal usage of floating point in kernel
85 * space.
86 */
87 li t0, SR_SUM | SR_FS
89 REG_L s0, TASK_TI_USER_SP(tp)
90 csrrc s1, sstatus, t0
91 csrr s2, sepc
92 csrr s3, sbadaddr
93 csrr s4, scause
94 csrr s5, sscratch
95 REG_S s0, PT_SP(sp)
96 REG_S s1, PT_SSTATUS(sp)
97 REG_S s2, PT_SEPC(sp)
98 REG_S s3, PT_SBADADDR(sp)
99 REG_S s4, PT_SCAUSE(sp)
100 REG_S s5, PT_TP(sp)
101 .endm
103 /*
104 * Prepares to return from a system call or exception by restoring all
105 * registers from the stack.
106 */
107 .macro RESTORE_ALL
108 REG_L a0, PT_SSTATUS(sp)
109 REG_L a2, PT_SEPC(sp)
110 csrw sstatus, a0
111 csrw sepc, a2
113 REG_L x1, PT_RA(sp)
114 REG_L x3, PT_GP(sp)
115 REG_L x4, PT_TP(sp)
116 REG_L x5, PT_T0(sp)
117 REG_L x6, PT_T1(sp)
118 REG_L x7, PT_T2(sp)
119 REG_L x8, PT_S0(sp)
120 REG_L x9, PT_S1(sp)
121 REG_L x10, PT_A0(sp)
122 REG_L x11, PT_A1(sp)
123 REG_L x12, PT_A2(sp)
124 REG_L x13, PT_A3(sp)
125 REG_L x14, PT_A4(sp)
126 REG_L x15, PT_A5(sp)
127 REG_L x16, PT_A6(sp)
128 REG_L x17, PT_A7(sp)
129 REG_L x18, PT_S2(sp)
130 REG_L x19, PT_S3(sp)
131 REG_L x20, PT_S4(sp)
132 REG_L x21, PT_S5(sp)
133 REG_L x22, PT_S6(sp)
134 REG_L x23, PT_S7(sp)
135 REG_L x24, PT_S8(sp)
136 REG_L x25, PT_S9(sp)
137 REG_L x26, PT_S10(sp)
138 REG_L x27, PT_S11(sp)
139 REG_L x28, PT_T3(sp)
140 REG_L x29, PT_T4(sp)
141 REG_L x30, PT_T5(sp)
142 REG_L x31, PT_T6(sp)
144 REG_L x2, PT_SP(sp)
145 .endm
147 ENTRY(handle_exception)
148 SAVE_ALL
150 /*
151 * Set sscratch register to 0, so that if a recursive exception
152 * occurs, the exception vector knows it came from the kernel
153 */
154 csrw sscratch, x0
156 /* Load the global pointer */
157 .option push
158 .option norelax
159 la gp, __global_pointer$
160 .option pop
162 la ra, ret_from_exception
163 /*
164 * MSB of cause differentiates between
165 * interrupts and exceptions
166 */
167 bge s4, zero, 1f
169 /* Handle interrupts */
170 move a0, sp /* pt_regs */
171 tail do_IRQ
172 1:
173 /* Exceptions run with interrupts enabled */
174 csrs sstatus, SR_SIE
176 /* Handle syscalls */
177 li t0, EXC_SYSCALL
178 beq s4, t0, handle_syscall
180 /* Handle other exceptions */
181 slli t0, s4, RISCV_LGPTR
182 la t1, excp_vect_table
183 la t2, excp_vect_table_end
184 move a0, sp /* pt_regs */
185 add t0, t1, t0
186 /* Check if exception code lies within bounds */
187 bgeu t0, t2, 1f
188 REG_L t0, 0(t0)
189 jr t0
190 1:
191 tail do_trap_unknown
193 handle_syscall:
194 /* save the initial A0 value (needed in signal handlers) */
195 REG_S a0, PT_ORIG_A0(sp)
196 /*
197 * Advance SEPC to avoid executing the original
198 * scall instruction on sret
199 */
200 addi s2, s2, 0x4
201 REG_S s2, PT_SEPC(sp)
202 /* Trace syscalls, but only if requested by the user. */
203 REG_L t0, TASK_TI_FLAGS(tp)
204 andi t0, t0, _TIF_SYSCALL_TRACE
205 bnez t0, handle_syscall_trace_enter
206 check_syscall_nr:
207 /* Check to make sure we don't jump to a bogus syscall number. */
208 li t0, __NR_syscalls
209 la s0, sys_ni_syscall
210 /* Syscall number held in a7 */
211 bgeu a7, t0, 1f
212 la s0, sys_call_table
213 slli t0, a7, RISCV_LGPTR
214 add s0, s0, t0
215 REG_L s0, 0(s0)
216 1:
217 jalr s0
219 ret_from_syscall:
220 /* Set user a0 to kernel a0 */
221 REG_S a0, PT_A0(sp)
222 /* Trace syscalls, but only if requested by the user. */
223 REG_L t0, TASK_TI_FLAGS(tp)
224 andi t0, t0, _TIF_SYSCALL_TRACE
225 bnez t0, handle_syscall_trace_exit
227 ret_from_exception:
228 REG_L s0, PT_SSTATUS(sp)
229 csrc sstatus, SR_SIE
230 andi s0, s0, SR_SPP
231 bnez s0, restore_all
233 resume_userspace:
234 /* Interrupts must be disabled here so flags are checked atomically */
235 REG_L s0, TASK_TI_FLAGS(tp) /* current_thread_info->flags */
236 andi s1, s0, _TIF_WORK_MASK
237 bnez s1, work_pending
239 /* Save unwound kernel stack pointer in thread_info */
240 addi s0, sp, PT_SIZE_ON_STACK
241 REG_S s0, TASK_TI_KERNEL_SP(tp)
243 /*
244 * Save TP into sscratch, so we can find the kernel data structures
245 * again.
246 */
247 csrw sscratch, tp
249 restore_all:
250 RESTORE_ALL
251 sret
253 work_pending:
254 /* Enter slow path for supplementary processing */
255 la ra, ret_from_exception
256 andi s1, s0, _TIF_NEED_RESCHED
257 bnez s1, work_resched
258 work_notifysig:
259 /* Handle pending signals and notify-resume requests */
260 csrs sstatus, SR_SIE /* Enable interrupts for do_notify_resume() */
261 move a0, sp /* pt_regs */
262 move a1, s0 /* current_thread_info->flags */
263 tail do_notify_resume
264 work_resched:
265 tail schedule
267 /* Slow paths for ptrace. */
268 handle_syscall_trace_enter:
269 move a0, sp
270 call do_syscall_trace_enter
271 REG_L a0, PT_A0(sp)
272 REG_L a1, PT_A1(sp)
273 REG_L a2, PT_A2(sp)
274 REG_L a3, PT_A3(sp)
275 REG_L a4, PT_A4(sp)
276 REG_L a5, PT_A5(sp)
277 REG_L a6, PT_A6(sp)
278 REG_L a7, PT_A7(sp)
279 j check_syscall_nr
280 handle_syscall_trace_exit:
281 move a0, sp
282 call do_syscall_trace_exit
283 j ret_from_exception
285 END(handle_exception)
287 ENTRY(ret_from_fork)
288 la ra, ret_from_exception
289 tail schedule_tail
290 ENDPROC(ret_from_fork)
292 ENTRY(ret_from_kernel_thread)
293 call schedule_tail
294 /* Call fn(arg) */
295 la ra, ret_from_exception
296 move a0, s1
297 jr s0
298 ENDPROC(ret_from_kernel_thread)
301 /*
302 * Integer register context switch
303 * The callee-saved registers must be saved and restored.
304 *
305 * a0: previous task_struct (must be preserved across the switch)
306 * a1: next task_struct
307 *
308 * The value of a0 and a1 must be preserved by this function, as that's how
309 * arguments are passed to schedule_tail.
310 */
311 ENTRY(__switch_to)
312 /* Save context into prev->thread */
313 li a4, TASK_THREAD_RA
314 add a3, a0, a4
315 add a4, a1, a4
316 REG_S ra, TASK_THREAD_RA_RA(a3)
317 REG_S sp, TASK_THREAD_SP_RA(a3)
318 REG_S s0, TASK_THREAD_S0_RA(a3)
319 REG_S s1, TASK_THREAD_S1_RA(a3)
320 REG_S s2, TASK_THREAD_S2_RA(a3)
321 REG_S s3, TASK_THREAD_S3_RA(a3)
322 REG_S s4, TASK_THREAD_S4_RA(a3)
323 REG_S s5, TASK_THREAD_S5_RA(a3)
324 REG_S s6, TASK_THREAD_S6_RA(a3)
325 REG_S s7, TASK_THREAD_S7_RA(a3)
326 REG_S s8, TASK_THREAD_S8_RA(a3)
327 REG_S s9, TASK_THREAD_S9_RA(a3)
328 REG_S s10, TASK_THREAD_S10_RA(a3)
329 REG_S s11, TASK_THREAD_S11_RA(a3)
330 /* Restore context from next->thread */
331 REG_L ra, TASK_THREAD_RA_RA(a4)
332 REG_L sp, TASK_THREAD_SP_RA(a4)
333 REG_L s0, TASK_THREAD_S0_RA(a4)
334 REG_L s1, TASK_THREAD_S1_RA(a4)
335 REG_L s2, TASK_THREAD_S2_RA(a4)
336 REG_L s3, TASK_THREAD_S3_RA(a4)
337 REG_L s4, TASK_THREAD_S4_RA(a4)
338 REG_L s5, TASK_THREAD_S5_RA(a4)
339 REG_L s6, TASK_THREAD_S6_RA(a4)
340 REG_L s7, TASK_THREAD_S7_RA(a4)
341 REG_L s8, TASK_THREAD_S8_RA(a4)
342 REG_L s9, TASK_THREAD_S9_RA(a4)
343 REG_L s10, TASK_THREAD_S10_RA(a4)
344 REG_L s11, TASK_THREAD_S11_RA(a4)
345 /* Swap the CPU entry around. */
346 lw a3, TASK_TI_CPU(a0)
347 lw a4, TASK_TI_CPU(a1)
348 sw a3, TASK_TI_CPU(a1)
349 sw a4, TASK_TI_CPU(a0)
350 #if TASK_TI != 0
351 #error "TASK_TI != 0: tp will contain a 'struct thread_info', not a 'struct task_struct' so get_current() won't work."
352 addi tp, a1, TASK_TI
353 #else
354 move tp, a1
355 #endif
356 ret
357 ENDPROC(__switch_to)
359 .section ".rodata"
360 /* Exception vector table */
361 ENTRY(excp_vect_table)
362 RISCV_PTR do_trap_insn_misaligned
363 RISCV_PTR do_trap_insn_fault
364 RISCV_PTR do_trap_insn_illegal
365 RISCV_PTR do_trap_break
366 RISCV_PTR do_trap_load_misaligned
367 RISCV_PTR do_trap_load_fault
368 RISCV_PTR do_trap_store_misaligned
369 RISCV_PTR do_trap_store_fault
370 RISCV_PTR do_trap_ecall_u /* system call, gets intercepted */
371 RISCV_PTR do_trap_ecall_s
372 RISCV_PTR do_trap_unknown
373 RISCV_PTR do_trap_ecall_m
374 RISCV_PTR do_page_fault /* instruction page fault */
375 RISCV_PTR do_page_fault /* load page fault */
376 RISCV_PTR do_trap_unknown
377 RISCV_PTR do_page_fault /* store page fault */
378 excp_vect_table_end:
379 END(excp_vect_table)