| blob | 5d7d9ee90cdfd33ad6cfad9904b9b45048c7aa6c |
1 /*
2 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
3 * Distributed under the terms of the MIT License.
4 */
7 #include <asm_defs.h>
9 #include <thread_types.h>
11 #include <arch/x86/descriptors.h>
12 #include <arch/x86/arch_cpu.h>
13 #include <arch/x86/arch_kernel.h>
15 #include "asm_offsets.h"
16 #include "syscall_numbers.h"
17 #include "syscall_table.h"
20 // Push the remainder of the interrupt frame onto the stack.
21 #define PUSH_IFRAME_BOTTOM(iframeType) \
22 push %rax; /* orig_rax */ \
23 push %rax; \
24 push %rbx; \
25 push %rcx; \
26 push %rdx; \
27 push %rdi; \
28 push %rsi; \
29 push %rbp; \
30 push %r8; \
31 push %r9; \
32 push %r10; \
33 push %r11; \
34 push %r12; \
35 push %r13; \
36 push %r14; \
37 push %r15; \
38 pushq $0; \
39 push $iframeType;
42 // Restore the interrupt frame.
43 #define RESTORE_IFRAME() \
44 add $16, %rsp; \
45 pop %r15; \
46 pop %r14; \
47 pop %r13; \
48 pop %r12; \
49 pop %r11; \
50 pop %r10; \
51 pop %r9; \
52 pop %r8; \
53 pop %rbp; \
54 pop %rsi; \
55 pop %rdi; \
56 pop %rdx; \
57 pop %rcx; \
58 pop %rbx; \
59 pop %rax; \
60 addq $24, %rsp;
63 // The macros below require R12 to contain the current thread pointer. R12 is
64 // callee-save so will be preserved through all function calls and only needs
65 // to be obtained once. R13 is used to store the system call start time, will
66 // also be preserved.
68 #define LOCK_THREAD_TIME() \
69 leaq THREAD_time_lock(%r12), %rdi; \
70 call acquire_spinlock;
72 #define UNLOCK_THREAD_TIME() \
73 leaq THREAD_time_lock(%r12), %rdi; \
74 call release_spinlock; \
76 #define UPDATE_THREAD_USER_TIME() \
77 LOCK_THREAD_TIME() \
78 \
79 call system_time; \
80 \
81 /* Preserve system_time for post syscall debug */ \
82 movq %rax, %r13; \
83 \
84 /* thread->user_time += now - thread->last_time; */ \
85 subq THREAD_last_time(%r12), %rax; \
86 addq %rax, THREAD_user_time(%r12); \
87 \
88 /* thread->last_time = now; */ \
89 movq %r13, THREAD_last_time(%r12); \
90 \
91 /* thread->in_kernel = true; */ \
92 movb $1, THREAD_in_kernel(%r12); \
93 \
94 UNLOCK_THREAD_TIME()
96 #define UPDATE_THREAD_KERNEL_TIME() \
97 LOCK_THREAD_TIME() \
98 \
99 call system_time; \
100 movq %rax, %r13; \
101 \
102 /* thread->kernel_time += now - thread->last_time; */ \
103 subq THREAD_last_time(%r12), %rax; \
104 addq %rax, THREAD_kernel_time(%r12); \
105 \
106 /* thread->last_time = now; */ \
107 movq %r13, THREAD_last_time(%r12); \
108 \
109 /* thread->in_kernel = false; */ \
110 movb $0, THREAD_in_kernel(%r12); \
111 \
112 UNLOCK_THREAD_TIME()
114 #define STOP_USER_DEBUGGING() \
115 testl $(THREAD_FLAGS_BREAKPOINTS_INSTALLED \
116 | THREAD_FLAGS_SINGLE_STEP), THREAD_flags(%r12); \
117 jz 1f; \
118 call x86_exit_user_debug_at_kernel_entry; \
119 1:
121 #define CLEAR_FPU_STATE() \
122 pxor %xmm0, %xmm0; \
123 pxor %xmm1, %xmm1; \
124 pxor %xmm2, %xmm2; \
125 pxor %xmm3, %xmm3; \
126 pxor %xmm4, %xmm4; \
127 pxor %xmm5, %xmm5; \
128 pxor %xmm6, %xmm6; \
129 pxor %xmm7, %xmm7; \
130 pxor %xmm8, %xmm8; \
131 pxor %xmm9, %xmm9; \
132 pxor %xmm10, %xmm10; \
133 pxor %xmm11, %xmm11; \
134 pxor %xmm12, %xmm12; \
135 pxor %xmm13, %xmm13; \
136 pxor %xmm14, %xmm14; \
137 pxor %xmm15, %xmm15
139 // The following code defines the interrupt service routines for all 256
140 // interrupts. It creates a block of handlers, each 16 bytes, that the IDT
141 // initialization code just loops through.
143 // Interrupt with no error code, pushes a 0 error code.
144 #define DEFINE_ISR(nr) \
145 .align 16; \
146 push $0; \
147 push $nr; \
148 jmp int_bottom;
150 // Interrupt with an error code.
151 #define DEFINE_ISR_E(nr) \
152 .align 16; \
153 push $nr; \
154 jmp int_bottom;
156 // Array of interrupt service routines.
157 .align 16
158 SYMBOL(isr_array):
159 // Exceptions (0-19) and reserved interrupts (20-31).
160 DEFINE_ISR(0)
161 DEFINE_ISR(1)
162 DEFINE_ISR(2)
163 DEFINE_ISR(3)
164 DEFINE_ISR(4)
165 DEFINE_ISR(5)
166 DEFINE_ISR(6)
167 DEFINE_ISR(7)
168 DEFINE_ISR_E(8)
169 DEFINE_ISR(9)
170 DEFINE_ISR_E(10)
171 DEFINE_ISR_E(11)
172 DEFINE_ISR_E(12)
173 DEFINE_ISR_E(13)
174 DEFINE_ISR_E(14)
175 DEFINE_ISR(15)
176 DEFINE_ISR(16)
177 DEFINE_ISR_E(17)
178 DEFINE_ISR(18)
179 DEFINE_ISR(19)
180 DEFINE_ISR(20)
181 DEFINE_ISR(21)
182 DEFINE_ISR(22)
183 DEFINE_ISR(23)
184 DEFINE_ISR(24)
185 DEFINE_ISR(25)
186 DEFINE_ISR(26)
187 DEFINE_ISR(27)
188 DEFINE_ISR(28)
189 DEFINE_ISR(29)
190 DEFINE_ISR(30)
191 DEFINE_ISR(31)
193 // User-defined ISRs (32-255) - none take an error code.
194 .Lintr = 32
195 .rept 224
196 DEFINE_ISR(.Lintr)
197 .Lintr = .Lintr+1
198 .endr
201 // Common interrupt handling code.
202 STATIC_FUNCTION(int_bottom):
203 // Coming from user-mode requires special handling.
204 testl $3, 24(%rsp)
205 jnz int_bottom_user
207 // Push the rest of the interrupt frame to the stack.
208 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER)
210 cld
212 // Frame pointer is the iframe.
213 movq %rsp, %rbp
215 // Set the RF (resume flag) in RFLAGS. This prevents an instruction
216 // breakpoint on the instruction we're returning to to trigger a debug
217 // exception.
218 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp)
220 subq $512, %rsp
221 andq $~15, %rsp
222 fxsaveq (%rsp)
224 // Call the interrupt handler.
225 movq %rbp, %rdi
226 movq IFRAME_vector(%rbp), %rax
227 call *gInterruptHandlerTable(, %rax, 8)
229 fxrstorq (%rsp)
230 movq %rbp, %rsp
232 // Restore the saved registers.
233 RESTORE_IFRAME()
235 iretq
236 FUNCTION_END(int_bottom)
239 // Handler for an interrupt that occurred in user-mode.
240 STATIC_FUNCTION(int_bottom_user):
241 // Load the kerrnel GS segment base.
242 swapgs
244 // Push the rest of the interrupt frame to the stack.
245 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER)
246 cld
248 // Frame pointer is the iframe.
249 movq %rsp, %rbp
251 subq $512, %rsp
252 andq $~15, %rsp
253 fxsaveq (%rsp)
254 movq %rsp, IFRAME_fpu(%rbp)
256 // Set the RF (resume flag) in RFLAGS. This prevents an instruction
257 // breakpoint on the instruction we're returning to to trigger a debug
258 // exception.
259 orq $X86_EFLAGS_RESUME, IFRAME_flags(%rbp)
261 // Get thread pointer.
262 movq %gs:0, %r12
264 STOP_USER_DEBUGGING()
265 UPDATE_THREAD_USER_TIME()
267 // Call the interrupt handler.
268 movq %rbp, %rdi
269 movq IFRAME_vector(%rbp), %rax
270 call *gInterruptHandlerTable(, %rax, 8)
272 // If there are no signals pending or we're not debugging, we can avoid
273 // most of the work here, just need to update the kernel time.
274 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
275 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
276 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
277 , THREAD_flags(%r12)
278 jnz .Lkernel_exit_work
280 cli
282 UPDATE_THREAD_KERNEL_TIME()
284 fxrstorq (%rsp)
285 movq %rbp, %rsp
287 // Restore the saved registers.
288 RESTORE_IFRAME()
290 // Restore the previous GS base and return.
291 swapgs
292 iretq
294 .Lkernel_exit_work:
295 // Slow path for return to userland.
297 // Do we need to handle signals?
298 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
299 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
300 , THREAD_flags(%r12)
301 jnz .Lkernel_exit_handle_signals
302 cli
303 call thread_at_kernel_exit_no_signals
305 .Lkernel_exit_work_done:
306 // Install breakpoints, if defined.
307 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12)
308 jz 1f
309 movq %rbp, %rdi
310 call x86_init_user_debug_at_kernel_exit
311 1:
312 fxrstorq (%rsp)
313 movq %rbp, %rsp
315 // Restore the saved registers.
316 RESTORE_IFRAME()
318 // Restore the previous GS base and return.
319 swapgs
320 iretq
322 .Lkernel_exit_handle_signals:
323 // thread_at_kernel_exit requires interrupts to be enabled, it will disable
324 // them after.
325 sti
326 call thread_at_kernel_exit
327 jmp .Lkernel_exit_work_done
328 FUNCTION_END(int_bottom_user)
331 // SYSCALL entry point.
332 FUNCTION(x86_64_syscall_entry):
333 // Upon entry, RSP still points at the user stack. Load the kernel GS
334 // segment base address, which points at the current thread's arch_thread
335 // structure. This contains our kernel stack pointer and a temporary
336 // scratch space to store the user stack pointer in before we can push it
337 // to the stack.
338 swapgs
339 movq %rsp, %gs:ARCH_THREAD_user_rsp
340 movq %gs:ARCH_THREAD_syscall_rsp, %rsp
342 // Set up an iframe on the stack (R11 = saved RFLAGS, RCX = saved RIP).
343 push $USER_DATA_SELECTOR // ss
344 push %gs:ARCH_THREAD_user_rsp // rsp
345 push %r11 // flags
346 push $USER_CODE_SELECTOR // cs
347 push %rcx // ip
348 push $0 // error_code
349 push $99 // vector
350 PUSH_IFRAME_BOTTOM(IFRAME_TYPE_SYSCALL)
352 cld
354 // Frame pointer is the iframe.
355 movq %rsp, %rbp
356 andq $~15, %rsp
358 // Preserve call number (R14 is callee-save), get thread pointer.
359 movq %rax, %r14
360 movq %gs:0, %r12
362 STOP_USER_DEBUGGING()
363 UPDATE_THREAD_USER_TIME()
365 // No longer need interrupts disabled.
366 sti
368 // Check whether the syscall number is valid.
369 cmpq $SYSCALL_COUNT, %r14
370 jae .Lsyscall_return
372 // Get the system call table entry. Note I'm hardcoding the shift because
373 // sizeof(syscall_info) is 16 and scale factors of 16 aren't supported,
374 // so can't just do leaq kSyscallInfos(, %rax, SYSCALL_INFO_sizeof).
375 movq %r14, %rax
376 shlq $4, %rax
377 leaq kSyscallInfos(, %rax, 1), %rax
379 // Check the number of call arguments, greater than 6 (6 * 8 = 48) requires
380 // a stack copy.
381 movq SYSCALL_INFO_parameter_size(%rax), %rcx
382 cmpq $48, %rcx
383 ja .Lsyscall_stack_args
385 .Lperform_syscall:
386 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12)
387 jnz .Lpre_syscall_debug
389 .Lpre_syscall_debug_done:
390 // Restore the arguments from the iframe. UPDATE_THREAD_USER_TIME() makes
391 // 2 function calls which means they may have been overwritten. Note that
392 // argument 4 is in R10 on the frame rather than RCX as RCX is used by
393 // SYSCALL.
394 movq IFRAME_di(%rbp), %rdi
395 movq IFRAME_si(%rbp), %rsi
396 movq IFRAME_dx(%rbp), %rdx
397 movq IFRAME_r10(%rbp), %rcx
398 movq IFRAME_r8(%rbp), %r8
399 movq IFRAME_r9(%rbp), %r9
401 // TODO: pre-syscall tracing
403 // Call the function and save its return value.
404 call *SYSCALL_INFO_function(%rax)
405 movq %rax, IFRAME_ax(%rbp)
407 // TODO: post-syscall tracing
409 .Lsyscall_return:
410 // Restore the original stack pointer and return.
411 movq %rbp, %rsp
413 // Clear the restarted flag.
414 testl $THREAD_FLAGS_SYSCALL_RESTARTED, THREAD_flags(%r12)
415 jz 2f
416 1:
417 movl THREAD_flags(%r12), %eax
418 movl %eax, %edx
419 andl $~THREAD_FLAGS_SYSCALL_RESTARTED, %edx
420 lock
421 cmpxchgl %edx, THREAD_flags(%r12)
422 jnz 1b
423 2:
424 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
425 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
426 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP | THREAD_FLAGS_RESTART_SYSCALL) \
427 , THREAD_flags(%r12)
428 jnz .Lpost_syscall_work
430 cli
432 UPDATE_THREAD_KERNEL_TIME()
434 // If we've just restored a signal frame, use the IRET path.
435 cmpq $SYSCALL_RESTORE_SIGNAL_FRAME, %r14
436 je .Lrestore_fpu
438 CLEAR_FPU_STATE()
440 // Restore the iframe and RCX/R11 for SYSRET.
441 RESTORE_IFRAME()
442 pop %rcx
443 addq $8, %rsp
444 pop %r11
445 pop %rsp
447 // Restore previous GS base and return.
448 swapgs
449 sysretq
451 .Lpre_syscall_debug:
452 // user_debug_pre_syscall expects a pointer to a block of arguments, need
453 // to push the register arguments onto the stack.
454 push IFRAME_r9(%rbp)
455 push IFRAME_r8(%rbp)
456 push IFRAME_r10(%rbp)
457 push IFRAME_dx(%rbp)
458 push IFRAME_si(%rbp)
459 push IFRAME_di(%rbp)
460 movq %r14, %rdi // syscall number
461 movq %rsp, %rsi
462 push %rax
463 call user_debug_pre_syscall
464 pop %rax
465 addq $48, %rsp
466 jmp .Lpre_syscall_debug_done
468 .Lpost_syscall_work:
469 testl $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12)
470 jz 1f
472 // Post-syscall debugging. Same as above, need a block of arguments.
473 push IFRAME_r9(%rbp)
474 push IFRAME_r8(%rbp)
475 push IFRAME_r10(%rbp)
476 push IFRAME_dx(%rbp)
477 push IFRAME_si(%rbp)
478 push IFRAME_di(%rbp)
479 movq %r14, %rdi // syscall number
480 movq %rsp, %rsi
481 movq IFRAME_ax(%rbp), %rdx // return value
482 movq %r13, %rcx // start time, preserved earlier
483 call user_debug_post_syscall
484 addq $48, %rsp
485 1:
486 // Do we need to handle signals?
487 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
488 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
489 , THREAD_flags(%r12)
490 jnz .Lpost_syscall_handle_signals
491 cli
492 call thread_at_kernel_exit_no_signals
494 .Lpost_syscall_work_done:
495 // Handle syscall restarting.
496 testl $THREAD_FLAGS_RESTART_SYSCALL, THREAD_flags(%r12)
497 jz 1f
498 movq %rsp, %rdi
499 call x86_restart_syscall
500 1:
501 // Install breakpoints, if defined.
502 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12)
503 jz 1f
504 movq %rbp, %rdi
505 call x86_init_user_debug_at_kernel_exit
506 1:
507 // On this return path it is possible that the frame has been modified,
508 // for example to execute a signal handler. In this case it is safer to
509 // return via IRET.
510 CLEAR_FPU_STATE()
511 jmp .Liret
513 .Lrestore_fpu:
514 movq IFRAME_fpu(%rbp), %rax
515 fxrstorq (%rax)
516 .Liret:
517 // Restore the saved registers.
518 RESTORE_IFRAME()
520 // Restore the previous GS base and return.
521 swapgs
522 iretq
524 .Lpost_syscall_handle_signals:
525 call thread_at_kernel_exit
526 jmp .Lpost_syscall_work_done
528 .Lsyscall_stack_args:
529 // Some arguments are on the stack, work out what we need to copy. 6
530 // arguments (48 bytes) are already in registers.
531 // RAX = syscall table entry address, RCX = argument size.
532 subq $48, %rcx
534 // Get the address to copy from.
535 movq IFRAME_user_sp(%rbp), %rsi
536 addq $8, %rsi
537 movabs $(USER_BASE + USER_SIZE), %rdx
538 cmp %rdx, %rsi
539 jae .Lbad_syscall_args
541 // Make space on the stack.
542 subq %rcx, %rsp
543 andq $~15, %rsp
544 movq %rsp, %rdi
546 // Set a fault handler.
547 movq $.Lbad_syscall_args, THREAD_fault_handler(%r12)
549 // Copy them by quadwords.
550 shrq $3, %rcx
551 rep
552 movsq
553 movq $0, THREAD_fault_handler(%r12)
555 // Perform the call.
556 jmp .Lperform_syscall
558 .Lbad_syscall_args:
559 movq $0, THREAD_fault_handler(%r12)
560 movq %rbp, %rsp
561 jmp .Lsyscall_return
562 FUNCTION_END(x86_64_syscall_entry)
565 /*! \fn void x86_return_to_userland(iframe* frame)
566 \brief Returns to the userland environment given by \a frame.
568 Before returning to userland all potentially necessary kernel exit work is
569 done.
571 \a frame must point to a location somewhere on the caller's stack (e.g. a
572 local variable).
573 The function must be called with interrupts disabled.
575 \param frame The iframe defining the userland environment.
576 */
577 FUNCTION(x86_return_to_userland):
578 movq %rdi, %rbp
579 movq %rbp, %rsp
581 // Perform kernel exit work.
582 movq %gs:0, %r12
583 testl $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
584 | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
585 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
586 , THREAD_flags(%r12)
587 jnz .Luserland_return_work
589 // update the thread's kernel time and return
590 UPDATE_THREAD_KERNEL_TIME()
592 // Restore the frame and return.
593 RESTORE_IFRAME()
594 swapgs
595 iretq
596 .Luserland_return_work:
597 // Slow path for return to userland.
599 // Do we need to handle signals?
600 testl $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
601 | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
602 , THREAD_flags(%r12)
603 jnz .Luserland_return_handle_signals
604 cli
605 call thread_at_kernel_exit_no_signals
607 .Luserland_return_work_done:
608 // Install breakpoints, if defined.
609 testl $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12)
610 jz 1f
611 movq %rbp, %rdi
612 call x86_init_user_debug_at_kernel_exit
613 1:
614 // Restore the saved registers.
615 RESTORE_IFRAME()
617 // Restore the previous GS base and return.
618 swapgs
619 iretq
620 .Luserland_return_handle_signals:
621 // thread_at_kernel_exit requires interrupts to be enabled, it will disable
622 // them after.
623 sti
624 call thread_at_kernel_exit
625 jmp .Luserland_return_work_done
626 FUNCTION_END(x86_return_to_userland)