| blob | aa8d8939abc1692df22e7000c226095a44167678 |
1 /*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 *
8 * $Id$
9 */
11 /*
12 * entry.S contains the system-call and fault low-level handling routines.
13 *
14 * NOTE: This code handles signal-recognition, which happens every time
15 * after an interrupt and after each system call.
16 *
17 * Normal syscalls and interrupts don't save a full stack frame, this is
18 * only done for syscall tracing, signals or fork/exec et.al.
19 *
20 * A note on terminology:
21 * - top of stack: Architecture defined interrupt frame from SS to RIP
22 * at the top of the kernel process stack.
23 * - partial stack frame: partially saved registers upto R11.
24 * - full stack frame: Like partial stack frame, but all register saved.
25 *
26 * TODO:
27 * - schedule it carefully for the final hardware.
28 */
30 #define ASSEMBLY 1
31 #include <linux/linkage.h>
32 #include <asm/segment.h>
33 #include <asm/smp.h>
34 #include <asm/cache.h>
35 #include <asm/errno.h>
36 #include <asm/dwarf2.h>
37 #include <asm/calling.h>
38 #include <asm/asm-offsets.h>
39 #include <asm/msr.h>
40 #include <asm/unistd.h>
41 #include <asm/thread_info.h>
42 #include <asm/hw_irq.h>
43 #include <asm/page.h>
44 #include <asm/irqflags.h>
46 .code64
48 #ifndef CONFIG_PREEMPT
49 #define retint_kernel retint_restore_args
50 #endif
53 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
54 #ifdef CONFIG_TRACE_IRQFLAGS
55 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
56 jnc 1f
57 TRACE_IRQS_ON
58 1:
59 #endif
60 .endm
62 /*
63 * C code is not supposed to know about undefined top of stack. Every time
64 * a C function with an pt_regs argument is called from the SYSCALL based
65 * fast path FIXUP_TOP_OF_STACK is needed.
66 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
67 * manipulation.
68 */
70 /* %rsp:at FRAMEEND */
71 .macro FIXUP_TOP_OF_STACK tmp
72 movq %gs:pda_oldrsp,\tmp
73 movq \tmp,RSP(%rsp)
74 movq $__USER_DS,SS(%rsp)
75 movq $__USER_CS,CS(%rsp)
76 movq $-1,RCX(%rsp)
77 movq R11(%rsp),\tmp /* get eflags */
78 movq \tmp,EFLAGS(%rsp)
79 .endm
81 .macro RESTORE_TOP_OF_STACK tmp,offset=0
82 movq RSP-\offset(%rsp),\tmp
83 movq \tmp,%gs:pda_oldrsp
84 movq EFLAGS-\offset(%rsp),\tmp
85 movq \tmp,R11-\offset(%rsp)
86 .endm
88 .macro FAKE_STACK_FRAME child_rip
89 /* push in order ss, rsp, eflags, cs, rip */
90 xorl %eax, %eax
91 pushq %rax /* ss */
92 CFI_ADJUST_CFA_OFFSET 8
93 /*CFI_REL_OFFSET ss,0*/
94 pushq %rax /* rsp */
95 CFI_ADJUST_CFA_OFFSET 8
96 CFI_REL_OFFSET rsp,0
97 pushq $(1<<9) /* eflags - interrupts on */
98 CFI_ADJUST_CFA_OFFSET 8
99 /*CFI_REL_OFFSET rflags,0*/
100 pushq $__KERNEL_CS /* cs */
101 CFI_ADJUST_CFA_OFFSET 8
102 /*CFI_REL_OFFSET cs,0*/
103 pushq \child_rip /* rip */
104 CFI_ADJUST_CFA_OFFSET 8
105 CFI_REL_OFFSET rip,0
106 pushq %rax /* orig rax */
107 CFI_ADJUST_CFA_OFFSET 8
108 .endm
110 .macro UNFAKE_STACK_FRAME
111 addq $8*6, %rsp
112 CFI_ADJUST_CFA_OFFSET -(6*8)
113 .endm
115 .macro CFI_DEFAULT_STACK start=1
116 .if \start
117 CFI_STARTPROC simple
118 CFI_DEF_CFA rsp,SS+8
119 .else
120 CFI_DEF_CFA_OFFSET SS+8
121 .endif
122 CFI_REL_OFFSET r15,R15
123 CFI_REL_OFFSET r14,R14
124 CFI_REL_OFFSET r13,R13
125 CFI_REL_OFFSET r12,R12
126 CFI_REL_OFFSET rbp,RBP
127 CFI_REL_OFFSET rbx,RBX
128 CFI_REL_OFFSET r11,R11
129 CFI_REL_OFFSET r10,R10
130 CFI_REL_OFFSET r9,R9
131 CFI_REL_OFFSET r8,R8
132 CFI_REL_OFFSET rax,RAX
133 CFI_REL_OFFSET rcx,RCX
134 CFI_REL_OFFSET rdx,RDX
135 CFI_REL_OFFSET rsi,RSI
136 CFI_REL_OFFSET rdi,RDI
137 CFI_REL_OFFSET rip,RIP
138 /*CFI_REL_OFFSET cs,CS*/
139 /*CFI_REL_OFFSET rflags,EFLAGS*/
140 CFI_REL_OFFSET rsp,RSP
141 /*CFI_REL_OFFSET ss,SS*/
142 .endm
143 /*
144 * A newly forked process directly context switches into this.
145 */
146 /* rdi: prev */
147 ENTRY(ret_from_fork)
148 CFI_DEFAULT_STACK
149 call schedule_tail
150 GET_THREAD_INFO(%rcx)
151 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
152 jnz rff_trace
153 rff_action:
154 RESTORE_REST
155 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
156 je int_ret_from_sys_call
157 testl $_TIF_IA32,threadinfo_flags(%rcx)
158 jnz int_ret_from_sys_call
159 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
160 jmp ret_from_sys_call
161 rff_trace:
162 movq %rsp,%rdi
163 call syscall_trace_leave
164 GET_THREAD_INFO(%rcx)
165 jmp rff_action
166 CFI_ENDPROC
167 END(ret_from_fork)
169 /*
170 * System call entry. Upto 6 arguments in registers are supported.
171 *
172 * SYSCALL does not save anything on the stack and does not change the
173 * stack pointer.
174 */
176 /*
177 * Register setup:
178 * rax system call number
179 * rdi arg0
180 * rcx return address for syscall/sysret, C arg3
181 * rsi arg1
182 * rdx arg2
183 * r10 arg3 (--> moved to rcx for C)
184 * r8 arg4
185 * r9 arg5
186 * r11 eflags for syscall/sysret, temporary for C
187 * r12-r15,rbp,rbx saved by C code, not touched.
188 *
189 * Interrupts are off on entry.
190 * Only called from user space.
191 *
192 * XXX if we had a free scratch register we could save the RSP into the stack frame
193 * and report it properly in ps. Unfortunately we haven't.
194 *
195 * When user can change the frames always force IRET. That is because
196 * it deals with uncanonical addresses better. SYSRET has trouble
197 * with them due to bugs in both AMD and Intel CPUs.
198 */
200 ENTRY(system_call)
201 CFI_STARTPROC simple
202 CFI_DEF_CFA rsp,PDA_STACKOFFSET
203 CFI_REGISTER rip,rcx
204 /*CFI_REGISTER rflags,r11*/
205 swapgs
206 movq %rsp,%gs:pda_oldrsp
207 movq %gs:pda_kernelstack,%rsp
208 /*
209 * No need to follow this irqs off/on section - it's straight
210 * and short:
211 */
212 sti
213 SAVE_ARGS 8,1
214 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
215 movq %rcx,RIP-ARGOFFSET(%rsp)
216 CFI_REL_OFFSET rip,RIP-ARGOFFSET
217 GET_THREAD_INFO(%rcx)
218 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
219 CFI_REMEMBER_STATE
220 jnz tracesys
221 cmpq $__NR_syscall_max,%rax
222 ja badsys
223 movq %r10,%rcx
224 call *sys_call_table(,%rax,8) # XXX: rip relative
225 movq %rax,RAX-ARGOFFSET(%rsp)
226 /*
227 * Syscall return path ending with SYSRET (fast path)
228 * Has incomplete stack frame and undefined top of stack.
229 */
230 .globl ret_from_sys_call
231 ret_from_sys_call:
232 movl $_TIF_ALLWORK_MASK,%edi
233 /* edi: flagmask */
234 sysret_check:
235 GET_THREAD_INFO(%rcx)
236 cli
237 TRACE_IRQS_OFF
238 movl threadinfo_flags(%rcx),%edx
239 andl %edi,%edx
240 CFI_REMEMBER_STATE
241 jnz sysret_careful
242 /*
243 * sysretq will re-enable interrupts:
244 */
245 TRACE_IRQS_ON
246 movq RIP-ARGOFFSET(%rsp),%rcx
247 CFI_REGISTER rip,rcx
248 RESTORE_ARGS 0,-ARG_SKIP,1
249 /*CFI_REGISTER rflags,r11*/
250 movq %gs:pda_oldrsp,%rsp
251 swapgs
252 sysretq
254 /* Handle reschedules */
255 /* edx: work, edi: workmask */
256 sysret_careful:
257 CFI_RESTORE_STATE
258 bt $TIF_NEED_RESCHED,%edx
259 jnc sysret_signal
260 TRACE_IRQS_ON
261 sti
262 pushq %rdi
263 CFI_ADJUST_CFA_OFFSET 8
264 call schedule
265 popq %rdi
266 CFI_ADJUST_CFA_OFFSET -8
267 jmp sysret_check
269 /* Handle a signal */
270 sysret_signal:
271 TRACE_IRQS_ON
272 sti
273 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
274 jz 1f
276 /* Really a signal */
277 /* edx: work flags (arg3) */
278 leaq do_notify_resume(%rip),%rax
279 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
280 xorl %esi,%esi # oldset -> arg2
281 call ptregscall_common
282 1: movl $_TIF_NEED_RESCHED,%edi
283 /* Use IRET because user could have changed frame. This
284 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
285 cli
286 TRACE_IRQS_OFF
287 jmp int_with_check
289 badsys:
290 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
291 jmp ret_from_sys_call
293 /* Do syscall tracing */
294 tracesys:
295 CFI_RESTORE_STATE
296 SAVE_REST
297 movq $-ENOSYS,RAX(%rsp)
298 FIXUP_TOP_OF_STACK %rdi
299 movq %rsp,%rdi
300 call syscall_trace_enter
301 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
302 RESTORE_REST
303 cmpq $__NR_syscall_max,%rax
304 ja 1f
305 movq %r10,%rcx /* fixup for C */
306 call *sys_call_table(,%rax,8)
307 1: movq %rax,RAX-ARGOFFSET(%rsp)
308 /* Use IRET because user could have changed frame */
309 jmp int_ret_from_sys_call
310 CFI_ENDPROC
311 END(system_call)
313 /*
314 * Syscall return path ending with IRET.
315 * Has correct top of stack, but partial stack frame.
316 */
317 ENTRY(int_ret_from_sys_call)
318 CFI_STARTPROC simple
319 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
320 /*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
321 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
322 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
323 /*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
324 CFI_REL_OFFSET rip,RIP-ARGOFFSET
325 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
326 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
327 CFI_REL_OFFSET rax,RAX-ARGOFFSET
328 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
329 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
330 CFI_REL_OFFSET r8,R8-ARGOFFSET
331 CFI_REL_OFFSET r9,R9-ARGOFFSET
332 CFI_REL_OFFSET r10,R10-ARGOFFSET
333 CFI_REL_OFFSET r11,R11-ARGOFFSET
334 cli
335 TRACE_IRQS_OFF
336 testl $3,CS-ARGOFFSET(%rsp)
337 je retint_restore_args
338 movl $_TIF_ALLWORK_MASK,%edi
339 /* edi: mask to check */
340 int_with_check:
341 GET_THREAD_INFO(%rcx)
342 movl threadinfo_flags(%rcx),%edx
343 andl %edi,%edx
344 jnz int_careful
345 andl $~TS_COMPAT,threadinfo_status(%rcx)
346 jmp retint_swapgs
348 /* Either reschedule or signal or syscall exit tracking needed. */
349 /* First do a reschedule test. */
350 /* edx: work, edi: workmask */
351 int_careful:
352 bt $TIF_NEED_RESCHED,%edx
353 jnc int_very_careful
354 TRACE_IRQS_ON
355 sti
356 pushq %rdi
357 CFI_ADJUST_CFA_OFFSET 8
358 call schedule
359 popq %rdi
360 CFI_ADJUST_CFA_OFFSET -8
361 cli
362 TRACE_IRQS_OFF
363 jmp int_with_check
365 /* handle signals and tracing -- both require a full stack frame */
366 int_very_careful:
367 TRACE_IRQS_ON
368 sti
369 SAVE_REST
370 /* Check for syscall exit trace */
371 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
372 jz int_signal
373 pushq %rdi
374 CFI_ADJUST_CFA_OFFSET 8
375 leaq 8(%rsp),%rdi # &ptregs -> arg1
376 call syscall_trace_leave
377 popq %rdi
378 CFI_ADJUST_CFA_OFFSET -8
379 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
380 cli
381 TRACE_IRQS_OFF
382 jmp int_restore_rest
384 int_signal:
385 testl $(_TIF_NOTIFY_RESUME|_TIF_SIGPENDING|_TIF_SINGLESTEP),%edx
386 jz 1f
387 movq %rsp,%rdi # &ptregs -> arg1
388 xorl %esi,%esi # oldset -> arg2
389 call do_notify_resume
390 1: movl $_TIF_NEED_RESCHED,%edi
391 int_restore_rest:
392 RESTORE_REST
393 cli
394 TRACE_IRQS_OFF
395 jmp int_with_check
396 CFI_ENDPROC
397 END(int_ret_from_sys_call)
399 /*
400 * Certain special system calls that need to save a complete full stack frame.
401 */
403 .macro PTREGSCALL label,func,arg
404 .globl \label
405 \label:
406 leaq \func(%rip),%rax
407 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
408 jmp ptregscall_common
409 END(\label)
410 .endm
412 CFI_STARTPROC
414 PTREGSCALL stub_clone, sys_clone, %r8
415 PTREGSCALL stub_fork, sys_fork, %rdi
416 PTREGSCALL stub_vfork, sys_vfork, %rdi
417 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
418 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
419 PTREGSCALL stub_iopl, sys_iopl, %rsi
421 ENTRY(ptregscall_common)
422 popq %r11
423 CFI_ADJUST_CFA_OFFSET -8
424 CFI_REGISTER rip, r11
425 SAVE_REST
426 movq %r11, %r15
427 CFI_REGISTER rip, r15
428 FIXUP_TOP_OF_STACK %r11
429 call *%rax
430 RESTORE_TOP_OF_STACK %r11
431 movq %r15, %r11
432 CFI_REGISTER rip, r11
433 RESTORE_REST
434 pushq %r11
435 CFI_ADJUST_CFA_OFFSET 8
436 CFI_REL_OFFSET rip, 0
437 ret
438 CFI_ENDPROC
439 END(ptregscall_common)
441 ENTRY(stub_execve)
442 CFI_STARTPROC
443 popq %r11
444 CFI_ADJUST_CFA_OFFSET -8
445 CFI_REGISTER rip, r11
446 SAVE_REST
447 FIXUP_TOP_OF_STACK %r11
448 call sys_execve
449 RESTORE_TOP_OF_STACK %r11
450 movq %rax,RAX(%rsp)
451 RESTORE_REST
452 jmp int_ret_from_sys_call
453 CFI_ENDPROC
454 END(stub_execve)
456 /*
457 * sigreturn is special because it needs to restore all registers on return.
458 * This cannot be done with SYSRET, so use the IRET return path instead.
459 */
460 ENTRY(stub_rt_sigreturn)
461 CFI_STARTPROC
462 addq $8, %rsp
463 CFI_ADJUST_CFA_OFFSET -8
464 SAVE_REST
465 movq %rsp,%rdi
466 FIXUP_TOP_OF_STACK %r11
467 call sys_rt_sigreturn
468 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
469 RESTORE_REST
470 jmp int_ret_from_sys_call
471 CFI_ENDPROC
472 END(stub_rt_sigreturn)
474 /*
475 * initial frame state for interrupts and exceptions
476 */
477 .macro _frame ref
478 CFI_STARTPROC simple
479 CFI_DEF_CFA rsp,SS+8-\ref
480 /*CFI_REL_OFFSET ss,SS-\ref*/
481 CFI_REL_OFFSET rsp,RSP-\ref
482 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
483 /*CFI_REL_OFFSET cs,CS-\ref*/
484 CFI_REL_OFFSET rip,RIP-\ref
485 .endm
487 /* initial frame state for interrupts (and exceptions without error code) */
488 #define INTR_FRAME _frame RIP
489 /* initial frame state for exceptions with error code (and interrupts with
490 vector already pushed) */
491 #define XCPT_FRAME _frame ORIG_RAX
493 /*
494 * Interrupt entry/exit.
495 *
496 * Interrupt entry points save only callee clobbered registers in fast path.
497 *
498 * Entry runs with interrupts off.
499 */
501 /* 0(%rsp): interrupt number */
502 .macro interrupt func
503 cld
504 SAVE_ARGS
505 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
506 pushq %rbp
507 CFI_ADJUST_CFA_OFFSET 8
508 CFI_REL_OFFSET rbp, 0
509 movq %rsp,%rbp
510 CFI_DEF_CFA_REGISTER rbp
511 testl $3,CS(%rdi)
512 je 1f
513 swapgs
514 1: incl %gs:pda_irqcount # RED-PEN should check preempt count
515 cmoveq %gs:pda_irqstackptr,%rsp
516 push %rbp # backlink for old unwinder
517 /*
518 * We entered an interrupt context - irqs are off:
519 */
520 TRACE_IRQS_OFF
521 call \func
522 .endm
524 ENTRY(common_interrupt)
525 XCPT_FRAME
526 interrupt do_IRQ
527 /* 0(%rsp): oldrsp-ARGOFFSET */
528 ret_from_intr:
529 cli
530 TRACE_IRQS_OFF
531 decl %gs:pda_irqcount
532 leaveq
533 CFI_DEF_CFA_REGISTER rsp
534 CFI_ADJUST_CFA_OFFSET -8
535 exit_intr:
536 GET_THREAD_INFO(%rcx)
537 testl $3,CS-ARGOFFSET(%rsp)
538 je retint_kernel
540 /* Interrupt came from user space */
541 /*
542 * Has a correct top of stack, but a partial stack frame
543 * %rcx: thread info. Interrupts off.
544 */
545 retint_with_reschedule:
546 movl $_TIF_WORK_MASK,%edi
547 retint_check:
548 movl threadinfo_flags(%rcx),%edx
549 andl %edi,%edx
550 CFI_REMEMBER_STATE
551 jnz retint_careful
552 retint_swapgs:
553 /*
554 * The iretq could re-enable interrupts:
555 */
556 cli
557 TRACE_IRQS_IRETQ
558 swapgs
559 jmp restore_args
561 retint_restore_args:
562 cli
563 /*
564 * The iretq could re-enable interrupts:
565 */
566 TRACE_IRQS_IRETQ
567 restore_args:
568 RESTORE_ARGS 0,8,0
569 iret_label:
570 iretq
572 .section __ex_table,"a"
573 .quad iret_label,bad_iret
574 .previous
575 .section .fixup,"ax"
576 /* force a signal here? this matches i386 behaviour */
577 /* running with kernel gs */
578 bad_iret:
579 movq $11,%rdi /* SIGSEGV */
580 TRACE_IRQS_ON
581 sti
582 jmp do_exit
583 .previous
585 /* edi: workmask, edx: work */
586 retint_careful:
587 CFI_RESTORE_STATE
588 bt $TIF_NEED_RESCHED,%edx
589 jnc retint_signal
590 TRACE_IRQS_ON
591 sti
592 pushq %rdi
593 CFI_ADJUST_CFA_OFFSET 8
594 call schedule
595 popq %rdi
596 CFI_ADJUST_CFA_OFFSET -8
597 GET_THREAD_INFO(%rcx)
598 cli
599 TRACE_IRQS_OFF
600 jmp retint_check
602 retint_signal:
603 testl $(_TIF_SIGPENDING|_TIF_NOTIFY_RESUME|_TIF_SINGLESTEP),%edx
604 jz retint_swapgs
605 TRACE_IRQS_ON
606 sti
607 SAVE_REST
608 movq $-1,ORIG_RAX(%rsp)
609 xorl %esi,%esi # oldset
610 movq %rsp,%rdi # &pt_regs
611 call do_notify_resume
612 RESTORE_REST
613 cli
614 TRACE_IRQS_OFF
615 movl $_TIF_NEED_RESCHED,%edi
616 GET_THREAD_INFO(%rcx)
617 jmp retint_check
619 #ifdef CONFIG_PREEMPT
620 /* Returning to kernel space. Check if we need preemption */
621 /* rcx: threadinfo. interrupts off. */
622 .p2align
623 retint_kernel:
624 cmpl $0,threadinfo_preempt_count(%rcx)
625 jnz retint_restore_args
626 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
627 jnc retint_restore_args
628 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
629 jnc retint_restore_args
630 call preempt_schedule_irq
631 jmp exit_intr
632 #endif
634 CFI_ENDPROC
635 END(common_interrupt)
637 /*
638 * APIC interrupts.
639 */
640 .macro apicinterrupt num,func
641 INTR_FRAME
642 pushq $~(\num)
643 CFI_ADJUST_CFA_OFFSET 8
644 interrupt \func
645 jmp ret_from_intr
646 CFI_ENDPROC
647 .endm
649 ENTRY(thermal_interrupt)
650 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
651 END(thermal_interrupt)
653 ENTRY(threshold_interrupt)
654 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
655 END(threshold_interrupt)
657 #ifdef CONFIG_SMP
658 ENTRY(reschedule_interrupt)
659 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
660 END(reschedule_interrupt)
662 .macro INVALIDATE_ENTRY num
663 ENTRY(invalidate_interrupt\num)
664 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
665 END(invalidate_interrupt\num)
666 .endm
668 INVALIDATE_ENTRY 0
669 INVALIDATE_ENTRY 1
670 INVALIDATE_ENTRY 2
671 INVALIDATE_ENTRY 3
672 INVALIDATE_ENTRY 4
673 INVALIDATE_ENTRY 5
674 INVALIDATE_ENTRY 6
675 INVALIDATE_ENTRY 7
677 ENTRY(call_function_interrupt)
678 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
679 END(call_function_interrupt)
680 #endif
682 #ifdef CONFIG_X86_LOCAL_APIC
683 ENTRY(apic_timer_interrupt)
684 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
685 END(apic_timer_interrupt)
687 ENTRY(error_interrupt)
688 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
689 END(error_interrupt)
691 ENTRY(spurious_interrupt)
692 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
693 END(spurious_interrupt)
694 #endif
696 /*
697 * Exception entry points.
698 */
699 .macro zeroentry sym
700 INTR_FRAME
701 pushq $0 /* push error code/oldrax */
702 CFI_ADJUST_CFA_OFFSET 8
703 pushq %rax /* push real oldrax to the rdi slot */
704 CFI_ADJUST_CFA_OFFSET 8
705 leaq \sym(%rip),%rax
706 jmp error_entry
707 CFI_ENDPROC
708 .endm
710 .macro errorentry sym
711 XCPT_FRAME
712 pushq %rax
713 CFI_ADJUST_CFA_OFFSET 8
714 leaq \sym(%rip),%rax
715 jmp error_entry
716 CFI_ENDPROC
717 .endm
719 /* error code is on the stack already */
720 /* handle NMI like exceptions that can happen everywhere */
721 .macro paranoidentry sym, ist=0, irqtrace=1
722 SAVE_ALL
723 cld
724 movl $1,%ebx
725 movl $MSR_GS_BASE,%ecx
726 rdmsr
727 testl %edx,%edx
728 js 1f
729 swapgs
730 xorl %ebx,%ebx
731 1:
732 .if \ist
733 movq %gs:pda_data_offset, %rbp
734 .endif
735 movq %rsp,%rdi
736 movq ORIG_RAX(%rsp),%rsi
737 movq $-1,ORIG_RAX(%rsp)
738 .if \ist
739 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
740 .endif
741 call \sym
742 .if \ist
743 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
744 .endif
745 cli
746 .if \irqtrace
747 TRACE_IRQS_OFF
748 .endif
749 .endm
751 /*
752 * "Paranoid" exit path from exception stack.
753 * Paranoid because this is used by NMIs and cannot take
754 * any kernel state for granted.
755 * We don't do kernel preemption checks here, because only
756 * NMI should be common and it does not enable IRQs and
757 * cannot get reschedule ticks.
758 *
759 * "trace" is 0 for the NMI handler only, because irq-tracing
760 * is fundamentally NMI-unsafe. (we cannot change the soft and
761 * hard flags at once, atomically)
762 */
763 .macro paranoidexit trace=1
764 /* ebx: no swapgs flag */
765 paranoid_exit\trace:
766 testl %ebx,%ebx /* swapgs needed? */
767 jnz paranoid_restore\trace
768 testl $3,CS(%rsp)
769 jnz paranoid_userspace\trace
770 paranoid_swapgs\trace:
771 TRACE_IRQS_IRETQ 0
772 swapgs
773 paranoid_restore\trace:
774 RESTORE_ALL 8
775 iretq
776 paranoid_userspace\trace:
777 GET_THREAD_INFO(%rcx)
778 movl threadinfo_flags(%rcx),%ebx
779 andl $_TIF_WORK_MASK,%ebx
780 jz paranoid_swapgs\trace
781 movq %rsp,%rdi /* &pt_regs */
782 call sync_regs
783 movq %rax,%rsp /* switch stack for scheduling */
784 testl $_TIF_NEED_RESCHED,%ebx
785 jnz paranoid_schedule\trace
786 movl %ebx,%edx /* arg3: thread flags */
787 .if \trace
788 TRACE_IRQS_ON
789 .endif
790 sti
791 xorl %esi,%esi /* arg2: oldset */
792 movq %rsp,%rdi /* arg1: &pt_regs */
793 call do_notify_resume
794 cli
795 .if \trace
796 TRACE_IRQS_OFF
797 .endif
798 jmp paranoid_userspace\trace
799 paranoid_schedule\trace:
800 .if \trace
801 TRACE_IRQS_ON
802 .endif
803 sti
804 call schedule
805 cli
806 .if \trace
807 TRACE_IRQS_OFF
808 .endif
809 jmp paranoid_userspace\trace
810 CFI_ENDPROC
811 .endm
813 /*
814 * Exception entry point. This expects an error code/orig_rax on the stack
815 * and the exception handler in %rax.
816 */
817 ENTRY(error_entry)
818 _frame RDI
819 /* rdi slot contains rax, oldrax contains error code */
820 cld
821 subq $14*8,%rsp
822 CFI_ADJUST_CFA_OFFSET (14*8)
823 movq %rsi,13*8(%rsp)
824 CFI_REL_OFFSET rsi,RSI
825 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
826 movq %rdx,12*8(%rsp)
827 CFI_REL_OFFSET rdx,RDX
828 movq %rcx,11*8(%rsp)
829 CFI_REL_OFFSET rcx,RCX
830 movq %rsi,10*8(%rsp) /* store rax */
831 CFI_REL_OFFSET rax,RAX
832 movq %r8, 9*8(%rsp)
833 CFI_REL_OFFSET r8,R8
834 movq %r9, 8*8(%rsp)
835 CFI_REL_OFFSET r9,R9
836 movq %r10,7*8(%rsp)
837 CFI_REL_OFFSET r10,R10
838 movq %r11,6*8(%rsp)
839 CFI_REL_OFFSET r11,R11
840 movq %rbx,5*8(%rsp)
841 CFI_REL_OFFSET rbx,RBX
842 movq %rbp,4*8(%rsp)
843 CFI_REL_OFFSET rbp,RBP
844 movq %r12,3*8(%rsp)
845 CFI_REL_OFFSET r12,R12
846 movq %r13,2*8(%rsp)
847 CFI_REL_OFFSET r13,R13
848 movq %r14,1*8(%rsp)
849 CFI_REL_OFFSET r14,R14
850 movq %r15,(%rsp)
851 CFI_REL_OFFSET r15,R15
852 xorl %ebx,%ebx
853 testl $3,CS(%rsp)
854 je error_kernelspace
855 error_swapgs:
856 swapgs
857 error_sti:
858 movq %rdi,RDI(%rsp)
859 movq %rsp,%rdi
860 movq ORIG_RAX(%rsp),%rsi /* get error code */
861 movq $-1,ORIG_RAX(%rsp)
862 call *%rax
863 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
864 error_exit:
865 movl %ebx,%eax
866 RESTORE_REST
867 cli
868 TRACE_IRQS_OFF
869 GET_THREAD_INFO(%rcx)
870 testl %eax,%eax
871 jne retint_kernel
872 movl threadinfo_flags(%rcx),%edx
873 movl $_TIF_WORK_MASK,%edi
874 andl %edi,%edx
875 jnz retint_careful
876 /*
877 * The iret might restore flags:
878 */
879 TRACE_IRQS_IRETQ
880 swapgs
881 RESTORE_ARGS 0,8,0
882 jmp iret_label
883 CFI_ENDPROC
885 error_kernelspace:
886 incl %ebx
887 /* There are two places in the kernel that can potentially fault with
888 usergs. Handle them here. The exception handlers after
889 iret run with kernel gs again, so don't set the user space flag.
890 B stepping K8s sometimes report an truncated RIP for IRET
891 exceptions returning to compat mode. Check for these here too. */
892 leaq iret_label(%rip),%rbp
893 cmpq %rbp,RIP(%rsp)
894 je error_swapgs
895 movl %ebp,%ebp /* zero extend */
896 cmpq %rbp,RIP(%rsp)
897 je error_swapgs
898 cmpq $gs_change,RIP(%rsp)
899 je error_swapgs
900 jmp error_sti
901 END(error_entry)
903 /* Reload gs selector with exception handling */
904 /* edi: new selector */
905 ENTRY(load_gs_index)
906 CFI_STARTPROC
907 pushf
908 CFI_ADJUST_CFA_OFFSET 8
909 cli
910 swapgs
911 gs_change:
912 movl %edi,%gs
913 2: mfence /* workaround */
914 swapgs
915 popf
916 CFI_ADJUST_CFA_OFFSET -8
917 ret
918 CFI_ENDPROC
919 ENDPROC(load_gs_index)
921 .section __ex_table,"a"
922 .align 8
923 .quad gs_change,bad_gs
924 .previous
925 .section .fixup,"ax"
926 /* running with kernelgs */
927 bad_gs:
928 swapgs /* switch back to user gs */
929 xorl %eax,%eax
930 movl %eax,%gs
931 jmp 2b
932 .previous
934 /*
935 * Create a kernel thread.
936 *
937 * C extern interface:
938 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
939 *
940 * asm input arguments:
941 * rdi: fn, rsi: arg, rdx: flags
942 */
943 ENTRY(kernel_thread)
944 CFI_STARTPROC
945 FAKE_STACK_FRAME $child_rip
946 SAVE_ALL
948 # rdi: flags, rsi: usp, rdx: will be &pt_regs
949 movq %rdx,%rdi
950 orq kernel_thread_flags(%rip),%rdi
951 movq $-1, %rsi
952 movq %rsp, %rdx
954 xorl %r8d,%r8d
955 xorl %r9d,%r9d
957 # clone now
958 call do_fork
959 movq %rax,RAX(%rsp)
960 xorl %edi,%edi
962 /*
963 * It isn't worth to check for reschedule here,
964 * so internally to the x86_64 port you can rely on kernel_thread()
965 * not to reschedule the child before returning, this avoids the need
966 * of hacks for example to fork off the per-CPU idle tasks.
967 * [Hopefully no generic code relies on the reschedule -AK]
968 */
969 RESTORE_ALL
970 UNFAKE_STACK_FRAME
971 ret
972 CFI_ENDPROC
973 ENDPROC(kernel_thread)
975 child_rip:
976 pushq $0 # fake return address
977 CFI_STARTPROC
978 /*
979 * Here we are in the child and the registers are set as they were
980 * at kernel_thread() invocation in the parent.
981 */
982 movq %rdi, %rax
983 movq %rsi, %rdi
984 call *%rax
985 # exit
986 xorl %edi, %edi
987 call do_exit
988 CFI_ENDPROC
989 ENDPROC(child_rip)
991 /*
992 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
993 *
994 * C extern interface:
995 * extern long execve(char *name, char **argv, char **envp)
996 *
997 * asm input arguments:
998 * rdi: name, rsi: argv, rdx: envp
999 *
1000 * We want to fallback into:
1001 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
1002 *
1003 * do_sys_execve asm fallback arguments:
1004 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
1005 */
1006 ENTRY(execve)
1007 CFI_STARTPROC
1008 FAKE_STACK_FRAME $0
1009 SAVE_ALL
1010 call sys_execve
1011 movq %rax, RAX(%rsp)
1012 RESTORE_REST
1013 testq %rax,%rax
1014 je int_ret_from_sys_call
1015 RESTORE_ARGS
1016 UNFAKE_STACK_FRAME
1017 ret
1018 CFI_ENDPROC
1019 ENDPROC(execve)
1021 KPROBE_ENTRY(page_fault)
1022 errorentry do_page_fault
1023 END(page_fault)
1024 .previous .text
1026 ENTRY(coprocessor_error)
1027 zeroentry do_coprocessor_error
1028 END(coprocessor_error)
1030 ENTRY(simd_coprocessor_error)
1031 zeroentry do_simd_coprocessor_error
1032 END(simd_coprocessor_error)
1034 ENTRY(device_not_available)
1035 zeroentry math_state_restore
1036 END(device_not_available)
1038 /* runs on exception stack */
1039 KPROBE_ENTRY(debug)
1040 INTR_FRAME
1041 pushq $0
1042 CFI_ADJUST_CFA_OFFSET 8
1043 paranoidentry do_debug, DEBUG_STACK
1044 paranoidexit
1045 END(debug)
1046 .previous .text
1048 /* runs on exception stack */
1049 KPROBE_ENTRY(nmi)
1050 INTR_FRAME
1051 pushq $-1
1052 CFI_ADJUST_CFA_OFFSET 8
1053 paranoidentry do_nmi, 0, 0
1054 #ifdef CONFIG_TRACE_IRQFLAGS
1055 paranoidexit 0
1056 #else
1057 jmp paranoid_exit1
1058 CFI_ENDPROC
1059 #endif
1060 END(nmi)
1061 .previous .text
1063 KPROBE_ENTRY(int3)
1064 INTR_FRAME
1065 pushq $0
1066 CFI_ADJUST_CFA_OFFSET 8
1067 paranoidentry do_int3, DEBUG_STACK
1068 jmp paranoid_exit1
1069 CFI_ENDPROC
1070 END(int3)
1071 .previous .text
1073 ENTRY(overflow)
1074 zeroentry do_overflow
1075 END(overflow)
1077 ENTRY(bounds)
1078 zeroentry do_bounds
1079 END(bounds)
1081 ENTRY(invalid_op)
1082 zeroentry do_invalid_op
1083 END(invalid_op)
1085 ENTRY(coprocessor_segment_overrun)
1086 zeroentry do_coprocessor_segment_overrun
1087 END(coprocessor_segment_overrun)
1089 ENTRY(reserved)
1090 zeroentry do_reserved
1091 END(reserved)
1093 /* runs on exception stack */
1094 ENTRY(double_fault)
1095 XCPT_FRAME
1096 paranoidentry do_double_fault
1097 jmp paranoid_exit1
1098 CFI_ENDPROC
1099 END(double_fault)
1101 ENTRY(invalid_TSS)
1102 errorentry do_invalid_TSS
1103 END(invalid_TSS)
1105 ENTRY(segment_not_present)
1106 errorentry do_segment_not_present
1107 END(segment_not_present)
1109 /* runs on exception stack */
1110 ENTRY(stack_segment)
1111 XCPT_FRAME
1112 paranoidentry do_stack_segment
1113 jmp paranoid_exit1
1114 CFI_ENDPROC
1115 END(stack_segment)
1117 KPROBE_ENTRY(general_protection)
1118 errorentry do_general_protection
1119 END(general_protection)
1120 .previous .text
1122 ENTRY(alignment_check)
1123 errorentry do_alignment_check
1124 END(alignment_check)
1126 ENTRY(divide_error)
1127 zeroentry do_divide_error
1128 END(divide_error)
1130 ENTRY(spurious_interrupt_bug)
1131 zeroentry do_spurious_interrupt_bug
1132 END(spurious_interrupt_bug)
1134 #ifdef CONFIG_X86_MCE
1135 /* runs on exception stack */
1136 ENTRY(machine_check)
1137 INTR_FRAME
1138 pushq $0
1139 CFI_ADJUST_CFA_OFFSET 8
1140 paranoidentry do_machine_check
1141 jmp paranoid_exit1
1142 CFI_ENDPROC
1143 END(machine_check)
1144 #endif
1146 /* Call softirq on interrupt stack. Interrupts are off. */
1147 ENTRY(call_softirq)
1148 CFI_STARTPROC
1149 push %rbp
1150 CFI_ADJUST_CFA_OFFSET 8
1151 CFI_REL_OFFSET rbp,0
1152 mov %rsp,%rbp
1153 CFI_DEF_CFA_REGISTER rbp
1154 incl %gs:pda_irqcount
1155 cmove %gs:pda_irqstackptr,%rsp
1156 push %rbp # backlink for old unwinder
1157 call __do_softirq
1158 leaveq
1159 CFI_DEF_CFA_REGISTER rsp
1160 CFI_ADJUST_CFA_OFFSET -8
1161 decl %gs:pda_irqcount
1162 ret
1163 CFI_ENDPROC
1164 ENDPROC(call_softirq)
1166 #ifdef CONFIG_STACK_UNWIND
1167 ENTRY(arch_unwind_init_running)
1168 CFI_STARTPROC
1169 movq %r15, R15(%rdi)
1170 movq %r14, R14(%rdi)
1171 xchgq %rsi, %rdx
1172 movq %r13, R13(%rdi)
1173 movq %r12, R12(%rdi)
1174 xorl %eax, %eax
1175 movq %rbp, RBP(%rdi)
1176 movq %rbx, RBX(%rdi)
1177 movq (%rsp), %rcx
1178 movq %rax, R11(%rdi)
1179 movq %rax, R10(%rdi)
1180 movq %rax, R9(%rdi)
1181 movq %rax, R8(%rdi)
1182 movq %rax, RAX(%rdi)
1183 movq %rax, RCX(%rdi)
1184 movq %rax, RDX(%rdi)
1185 movq %rax, RSI(%rdi)
1186 movq %rax, RDI(%rdi)
1187 movq %rax, ORIG_RAX(%rdi)
1188 movq %rcx, RIP(%rdi)
1189 leaq 8(%rsp), %rcx
1190 movq $__KERNEL_CS, CS(%rdi)
1191 movq %rax, EFLAGS(%rdi)
1192 movq %rcx, RSP(%rdi)
1193 movq $__KERNEL_DS, SS(%rdi)
1194 jmpq *%rdx
1195 CFI_ENDPROC
1196 ENDPROC(arch_unwind_init_running)
1197 #endif