| blob | 2ad9a1bc6a73fafd6ea8e34b9117877eb2e736dd |
1 /*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 */
9 /*
10 * entry.S contains the system-call and fault low-level handling routines.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
14 *
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
17 *
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
23 *
24 * Some macro usage:
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
38 */
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
47 #include <asm/msr.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
51 #include <asm/page.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
55 .code64
57 #ifndef CONFIG_PREEMPT
58 #define retint_kernel retint_restore_args
59 #endif
61 #ifdef CONFIG_PARAVIRT
62 ENTRY(native_irq_enable_syscall_ret)
63 movq %gs:pda_oldrsp,%rsp
64 swapgs
65 sysretq
66 #endif /* CONFIG_PARAVIRT */
69 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
70 #ifdef CONFIG_TRACE_IRQFLAGS
71 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
72 jnc 1f
73 TRACE_IRQS_ON
74 1:
75 #endif
76 .endm
78 /*
79 * C code is not supposed to know about undefined top of stack. Every time
80 * a C function with an pt_regs argument is called from the SYSCALL based
81 * fast path FIXUP_TOP_OF_STACK is needed.
82 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
83 * manipulation.
84 */
86 /* %rsp:at FRAMEEND */
87 .macro FIXUP_TOP_OF_STACK tmp
88 movq %gs:pda_oldrsp,\tmp
89 movq \tmp,RSP(%rsp)
90 movq $__USER_DS,SS(%rsp)
91 movq $__USER_CS,CS(%rsp)
92 movq $-1,RCX(%rsp)
93 movq R11(%rsp),\tmp /* get eflags */
94 movq \tmp,EFLAGS(%rsp)
95 .endm
97 .macro RESTORE_TOP_OF_STACK tmp,offset=0
98 movq RSP-\offset(%rsp),\tmp
99 movq \tmp,%gs:pda_oldrsp
100 movq EFLAGS-\offset(%rsp),\tmp
101 movq \tmp,R11-\offset(%rsp)
102 .endm
104 .macro FAKE_STACK_FRAME child_rip
105 /* push in order ss, rsp, eflags, cs, rip */
106 xorl %eax, %eax
107 pushq %rax /* ss */
108 CFI_ADJUST_CFA_OFFSET 8
109 /*CFI_REL_OFFSET ss,0*/
110 pushq %rax /* rsp */
111 CFI_ADJUST_CFA_OFFSET 8
112 CFI_REL_OFFSET rsp,0
113 pushq $(1<<9) /* eflags - interrupts on */
114 CFI_ADJUST_CFA_OFFSET 8
115 /*CFI_REL_OFFSET rflags,0*/
116 pushq $__KERNEL_CS /* cs */
117 CFI_ADJUST_CFA_OFFSET 8
118 /*CFI_REL_OFFSET cs,0*/
119 pushq \child_rip /* rip */
120 CFI_ADJUST_CFA_OFFSET 8
121 CFI_REL_OFFSET rip,0
122 pushq %rax /* orig rax */
123 CFI_ADJUST_CFA_OFFSET 8
124 .endm
126 .macro UNFAKE_STACK_FRAME
127 addq $8*6, %rsp
128 CFI_ADJUST_CFA_OFFSET -(6*8)
129 .endm
131 .macro CFI_DEFAULT_STACK start=1
132 .if \start
133 CFI_STARTPROC simple
134 CFI_SIGNAL_FRAME
135 CFI_DEF_CFA rsp,SS+8
136 .else
137 CFI_DEF_CFA_OFFSET SS+8
138 .endif
139 CFI_REL_OFFSET r15,R15
140 CFI_REL_OFFSET r14,R14
141 CFI_REL_OFFSET r13,R13
142 CFI_REL_OFFSET r12,R12
143 CFI_REL_OFFSET rbp,RBP
144 CFI_REL_OFFSET rbx,RBX
145 CFI_REL_OFFSET r11,R11
146 CFI_REL_OFFSET r10,R10
147 CFI_REL_OFFSET r9,R9
148 CFI_REL_OFFSET r8,R8
149 CFI_REL_OFFSET rax,RAX
150 CFI_REL_OFFSET rcx,RCX
151 CFI_REL_OFFSET rdx,RDX
152 CFI_REL_OFFSET rsi,RSI
153 CFI_REL_OFFSET rdi,RDI
154 CFI_REL_OFFSET rip,RIP
155 /*CFI_REL_OFFSET cs,CS*/
156 /*CFI_REL_OFFSET rflags,EFLAGS*/
157 CFI_REL_OFFSET rsp,RSP
158 /*CFI_REL_OFFSET ss,SS*/
159 .endm
160 /*
161 * A newly forked process directly context switches into this.
162 */
163 /* rdi: prev */
164 ENTRY(ret_from_fork)
165 CFI_DEFAULT_STACK
166 push kernel_eflags(%rip)
167 CFI_ADJUST_CFA_OFFSET 4
168 popf # reset kernel eflags
169 CFI_ADJUST_CFA_OFFSET -4
170 call schedule_tail
171 GET_THREAD_INFO(%rcx)
172 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
173 jnz rff_trace
174 rff_action:
175 RESTORE_REST
176 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
177 je int_ret_from_sys_call
178 testl $_TIF_IA32,threadinfo_flags(%rcx)
179 jnz int_ret_from_sys_call
180 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
181 jmp ret_from_sys_call
182 rff_trace:
183 movq %rsp,%rdi
184 call syscall_trace_leave
185 GET_THREAD_INFO(%rcx)
186 jmp rff_action
187 CFI_ENDPROC
188 END(ret_from_fork)
190 /*
191 * System call entry. Upto 6 arguments in registers are supported.
192 *
193 * SYSCALL does not save anything on the stack and does not change the
194 * stack pointer.
195 */
197 /*
198 * Register setup:
199 * rax system call number
200 * rdi arg0
201 * rcx return address for syscall/sysret, C arg3
202 * rsi arg1
203 * rdx arg2
204 * r10 arg3 (--> moved to rcx for C)
205 * r8 arg4
206 * r9 arg5
207 * r11 eflags for syscall/sysret, temporary for C
208 * r12-r15,rbp,rbx saved by C code, not touched.
209 *
210 * Interrupts are off on entry.
211 * Only called from user space.
212 *
213 * XXX if we had a free scratch register we could save the RSP into the stack frame
214 * and report it properly in ps. Unfortunately we haven't.
215 *
216 * When user can change the frames always force IRET. That is because
217 * it deals with uncanonical addresses better. SYSRET has trouble
218 * with them due to bugs in both AMD and Intel CPUs.
219 */
221 ENTRY(system_call)
222 CFI_STARTPROC simple
223 CFI_SIGNAL_FRAME
224 CFI_DEF_CFA rsp,PDA_STACKOFFSET
225 CFI_REGISTER rip,rcx
226 /*CFI_REGISTER rflags,r11*/
227 SWAPGS_UNSAFE_STACK
228 /*
229 * A hypervisor implementation might want to use a label
230 * after the swapgs, so that it can do the swapgs
231 * for the guest and jump here on syscall.
232 */
233 ENTRY(system_call_after_swapgs)
235 movq %rsp,%gs:pda_oldrsp
236 movq %gs:pda_kernelstack,%rsp
237 /*
238 * No need to follow this irqs off/on section - it's straight
239 * and short:
240 */
241 ENABLE_INTERRUPTS(CLBR_NONE)
242 SAVE_ARGS 8,1
243 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
244 movq %rcx,RIP-ARGOFFSET(%rsp)
245 CFI_REL_OFFSET rip,RIP-ARGOFFSET
246 GET_THREAD_INFO(%rcx)
247 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
248 jnz tracesys
249 cmpq $__NR_syscall_max,%rax
250 ja badsys
251 movq %r10,%rcx
252 call *sys_call_table(,%rax,8) # XXX: rip relative
253 movq %rax,RAX-ARGOFFSET(%rsp)
254 /*
255 * Syscall return path ending with SYSRET (fast path)
256 * Has incomplete stack frame and undefined top of stack.
257 */
258 ret_from_sys_call:
259 movl $_TIF_ALLWORK_MASK,%edi
260 /* edi: flagmask */
261 sysret_check:
262 LOCKDEP_SYS_EXIT
263 GET_THREAD_INFO(%rcx)
264 DISABLE_INTERRUPTS(CLBR_NONE)
265 TRACE_IRQS_OFF
266 movl threadinfo_flags(%rcx),%edx
267 andl %edi,%edx
268 jnz sysret_careful
269 CFI_REMEMBER_STATE
270 /*
271 * sysretq will re-enable interrupts:
272 */
273 TRACE_IRQS_ON
274 movq RIP-ARGOFFSET(%rsp),%rcx
275 CFI_REGISTER rip,rcx
276 RESTORE_ARGS 0,-ARG_SKIP,1
277 /*CFI_REGISTER rflags,r11*/
278 ENABLE_INTERRUPTS_SYSCALL_RET
280 CFI_RESTORE_STATE
281 /* Handle reschedules */
282 /* edx: work, edi: workmask */
283 sysret_careful:
284 bt $TIF_NEED_RESCHED,%edx
285 jnc sysret_signal
286 TRACE_IRQS_ON
287 ENABLE_INTERRUPTS(CLBR_NONE)
288 pushq %rdi
289 CFI_ADJUST_CFA_OFFSET 8
290 call schedule
291 popq %rdi
292 CFI_ADJUST_CFA_OFFSET -8
293 jmp sysret_check
295 /* Handle a signal */
296 sysret_signal:
297 TRACE_IRQS_ON
298 ENABLE_INTERRUPTS(CLBR_NONE)
299 testl $_TIF_DO_NOTIFY_MASK,%edx
300 jz 1f
302 /* Really a signal */
303 /* edx: work flags (arg3) */
304 leaq do_notify_resume(%rip),%rax
305 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
306 xorl %esi,%esi # oldset -> arg2
307 call ptregscall_common
308 1: movl $_TIF_NEED_RESCHED,%edi
309 /* Use IRET because user could have changed frame. This
310 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
311 DISABLE_INTERRUPTS(CLBR_NONE)
312 TRACE_IRQS_OFF
313 jmp int_with_check
315 badsys:
316 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
317 jmp ret_from_sys_call
319 /* Do syscall tracing */
320 tracesys:
321 SAVE_REST
322 movq $-ENOSYS,RAX(%rsp)
323 FIXUP_TOP_OF_STACK %rdi
324 movq %rsp,%rdi
325 call syscall_trace_enter
326 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
327 RESTORE_REST
328 cmpq $__NR_syscall_max,%rax
329 movq $-ENOSYS,%rcx
330 cmova %rcx,%rax
331 ja 1f
332 movq %r10,%rcx /* fixup for C */
333 call *sys_call_table(,%rax,8)
334 1: movq %rax,RAX-ARGOFFSET(%rsp)
335 /* Use IRET because user could have changed frame */
337 /*
338 * Syscall return path ending with IRET.
339 * Has correct top of stack, but partial stack frame.
340 */
341 .globl int_ret_from_sys_call
342 int_ret_from_sys_call:
343 DISABLE_INTERRUPTS(CLBR_NONE)
344 TRACE_IRQS_OFF
345 testl $3,CS-ARGOFFSET(%rsp)
346 je retint_restore_args
347 movl $_TIF_ALLWORK_MASK,%edi
348 /* edi: mask to check */
349 int_with_check:
350 LOCKDEP_SYS_EXIT_IRQ
351 GET_THREAD_INFO(%rcx)
352 movl threadinfo_flags(%rcx),%edx
353 andl %edi,%edx
354 jnz int_careful
355 andl $~TS_COMPAT,threadinfo_status(%rcx)
356 jmp retint_swapgs
358 /* Either reschedule or signal or syscall exit tracking needed. */
359 /* First do a reschedule test. */
360 /* edx: work, edi: workmask */
361 int_careful:
362 bt $TIF_NEED_RESCHED,%edx
363 jnc int_very_careful
364 TRACE_IRQS_ON
365 ENABLE_INTERRUPTS(CLBR_NONE)
366 pushq %rdi
367 CFI_ADJUST_CFA_OFFSET 8
368 call schedule
369 popq %rdi
370 CFI_ADJUST_CFA_OFFSET -8
371 DISABLE_INTERRUPTS(CLBR_NONE)
372 TRACE_IRQS_OFF
373 jmp int_with_check
375 /* handle signals and tracing -- both require a full stack frame */
376 int_very_careful:
377 TRACE_IRQS_ON
378 ENABLE_INTERRUPTS(CLBR_NONE)
379 SAVE_REST
380 /* Check for syscall exit trace */
381 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
382 jz int_signal
383 pushq %rdi
384 CFI_ADJUST_CFA_OFFSET 8
385 leaq 8(%rsp),%rdi # &ptregs -> arg1
386 call syscall_trace_leave
387 popq %rdi
388 CFI_ADJUST_CFA_OFFSET -8
389 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
390 jmp int_restore_rest
392 int_signal:
393 testl $_TIF_DO_NOTIFY_MASK,%edx
394 jz 1f
395 movq %rsp,%rdi # &ptregs -> arg1
396 xorl %esi,%esi # oldset -> arg2
397 call do_notify_resume
398 1: movl $_TIF_NEED_RESCHED,%edi
399 int_restore_rest:
400 RESTORE_REST
401 DISABLE_INTERRUPTS(CLBR_NONE)
402 TRACE_IRQS_OFF
403 jmp int_with_check
404 CFI_ENDPROC
405 END(system_call)
407 /*
408 * Certain special system calls that need to save a complete full stack frame.
409 */
411 .macro PTREGSCALL label,func,arg
412 .globl \label
413 \label:
414 leaq \func(%rip),%rax
415 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
416 jmp ptregscall_common
417 END(\label)
418 .endm
420 CFI_STARTPROC
422 PTREGSCALL stub_clone, sys_clone, %r8
423 PTREGSCALL stub_fork, sys_fork, %rdi
424 PTREGSCALL stub_vfork, sys_vfork, %rdi
425 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
426 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
427 PTREGSCALL stub_iopl, sys_iopl, %rsi
429 ENTRY(ptregscall_common)
430 popq %r11
431 CFI_ADJUST_CFA_OFFSET -8
432 CFI_REGISTER rip, r11
433 SAVE_REST
434 movq %r11, %r15
435 CFI_REGISTER rip, r15
436 FIXUP_TOP_OF_STACK %r11
437 call *%rax
438 RESTORE_TOP_OF_STACK %r11
439 movq %r15, %r11
440 CFI_REGISTER rip, r11
441 RESTORE_REST
442 pushq %r11
443 CFI_ADJUST_CFA_OFFSET 8
444 CFI_REL_OFFSET rip, 0
445 ret
446 CFI_ENDPROC
447 END(ptregscall_common)
449 ENTRY(stub_execve)
450 CFI_STARTPROC
451 popq %r11
452 CFI_ADJUST_CFA_OFFSET -8
453 CFI_REGISTER rip, r11
454 SAVE_REST
455 FIXUP_TOP_OF_STACK %r11
456 call sys_execve
457 RESTORE_TOP_OF_STACK %r11
458 movq %rax,RAX(%rsp)
459 RESTORE_REST
460 jmp int_ret_from_sys_call
461 CFI_ENDPROC
462 END(stub_execve)
464 /*
465 * sigreturn is special because it needs to restore all registers on return.
466 * This cannot be done with SYSRET, so use the IRET return path instead.
467 */
468 ENTRY(stub_rt_sigreturn)
469 CFI_STARTPROC
470 addq $8, %rsp
471 CFI_ADJUST_CFA_OFFSET -8
472 SAVE_REST
473 movq %rsp,%rdi
474 FIXUP_TOP_OF_STACK %r11
475 call sys_rt_sigreturn
476 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
477 RESTORE_REST
478 jmp int_ret_from_sys_call
479 CFI_ENDPROC
480 END(stub_rt_sigreturn)
482 /*
483 * initial frame state for interrupts and exceptions
484 */
485 .macro _frame ref
486 CFI_STARTPROC simple
487 CFI_SIGNAL_FRAME
488 CFI_DEF_CFA rsp,SS+8-\ref
489 /*CFI_REL_OFFSET ss,SS-\ref*/
490 CFI_REL_OFFSET rsp,RSP-\ref
491 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
492 /*CFI_REL_OFFSET cs,CS-\ref*/
493 CFI_REL_OFFSET rip,RIP-\ref
494 .endm
496 /* initial frame state for interrupts (and exceptions without error code) */
497 #define INTR_FRAME _frame RIP
498 /* initial frame state for exceptions with error code (and interrupts with
499 vector already pushed) */
500 #define XCPT_FRAME _frame ORIG_RAX
502 /*
503 * Interrupt entry/exit.
504 *
505 * Interrupt entry points save only callee clobbered registers in fast path.
506 *
507 * Entry runs with interrupts off.
508 */
510 /* 0(%rsp): interrupt number */
511 .macro interrupt func
512 cld
513 SAVE_ARGS
514 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
515 pushq %rbp
516 CFI_ADJUST_CFA_OFFSET 8
517 CFI_REL_OFFSET rbp, 0
518 movq %rsp,%rbp
519 CFI_DEF_CFA_REGISTER rbp
520 testl $3,CS(%rdi)
521 je 1f
522 SWAPGS
523 /* irqcount is used to check if a CPU is already on an interrupt
524 stack or not. While this is essentially redundant with preempt_count
525 it is a little cheaper to use a separate counter in the PDA
526 (short of moving irq_enter into assembly, which would be too
527 much work) */
528 1: incl %gs:pda_irqcount
529 cmoveq %gs:pda_irqstackptr,%rsp
530 push %rbp # backlink for old unwinder
531 /*
532 * We entered an interrupt context - irqs are off:
533 */
534 TRACE_IRQS_OFF
535 call \func
536 .endm
538 ENTRY(common_interrupt)
539 XCPT_FRAME
540 interrupt do_IRQ
541 /* 0(%rsp): oldrsp-ARGOFFSET */
542 ret_from_intr:
543 DISABLE_INTERRUPTS(CLBR_NONE)
544 TRACE_IRQS_OFF
545 decl %gs:pda_irqcount
546 leaveq
547 CFI_DEF_CFA_REGISTER rsp
548 CFI_ADJUST_CFA_OFFSET -8
549 exit_intr:
550 GET_THREAD_INFO(%rcx)
551 testl $3,CS-ARGOFFSET(%rsp)
552 je retint_kernel
554 /* Interrupt came from user space */
555 /*
556 * Has a correct top of stack, but a partial stack frame
557 * %rcx: thread info. Interrupts off.
558 */
559 retint_with_reschedule:
560 movl $_TIF_WORK_MASK,%edi
561 retint_check:
562 LOCKDEP_SYS_EXIT_IRQ
563 movl threadinfo_flags(%rcx),%edx
564 andl %edi,%edx
565 CFI_REMEMBER_STATE
566 jnz retint_careful
568 retint_swapgs: /* return to user-space */
569 /*
570 * The iretq could re-enable interrupts:
571 */
572 DISABLE_INTERRUPTS(CLBR_ANY)
573 TRACE_IRQS_IRETQ
574 SWAPGS
575 jmp restore_args
577 retint_restore_args: /* return to kernel space */
578 DISABLE_INTERRUPTS(CLBR_ANY)
579 /*
580 * The iretq could re-enable interrupts:
581 */
582 TRACE_IRQS_IRETQ
583 restore_args:
584 RESTORE_ARGS 0,8,0
586 irq_return:
587 INTERRUPT_RETURN
589 .section __ex_table, "a"
590 .quad irq_return, bad_iret
591 .previous
593 #ifdef CONFIG_PARAVIRT
594 ENTRY(native_iret)
595 iretq
597 .section __ex_table,"a"
598 .quad native_iret, bad_iret
599 .previous
600 #endif
602 .section .fixup,"ax"
603 bad_iret:
604 /*
605 * The iret traps when the %cs or %ss being restored is bogus.
606 * We've lost the original trap vector and error code.
607 * #GPF is the most likely one to get for an invalid selector.
608 * So pretend we completed the iret and took the #GPF in user mode.
609 *
610 * We are now running with the kernel GS after exception recovery.
611 * But error_entry expects us to have user GS to match the user %cs,
612 * so swap back.
613 */
614 pushq $0
616 SWAPGS
617 jmp general_protection
619 .previous
621 /* edi: workmask, edx: work */
622 retint_careful:
623 CFI_RESTORE_STATE
624 bt $TIF_NEED_RESCHED,%edx
625 jnc retint_signal
626 TRACE_IRQS_ON
627 ENABLE_INTERRUPTS(CLBR_NONE)
628 pushq %rdi
629 CFI_ADJUST_CFA_OFFSET 8
630 call schedule
631 popq %rdi
632 CFI_ADJUST_CFA_OFFSET -8
633 GET_THREAD_INFO(%rcx)
634 DISABLE_INTERRUPTS(CLBR_NONE)
635 TRACE_IRQS_OFF
636 jmp retint_check
638 retint_signal:
639 testl $_TIF_DO_NOTIFY_MASK,%edx
640 jz retint_swapgs
641 TRACE_IRQS_ON
642 ENABLE_INTERRUPTS(CLBR_NONE)
643 SAVE_REST
644 movq $-1,ORIG_RAX(%rsp)
645 xorl %esi,%esi # oldset
646 movq %rsp,%rdi # &pt_regs
647 call do_notify_resume
648 RESTORE_REST
649 DISABLE_INTERRUPTS(CLBR_NONE)
650 TRACE_IRQS_OFF
651 movl $_TIF_NEED_RESCHED,%edi
652 GET_THREAD_INFO(%rcx)
653 jmp retint_check
655 #ifdef CONFIG_PREEMPT
656 /* Returning to kernel space. Check if we need preemption */
657 /* rcx: threadinfo. interrupts off. */
658 ENTRY(retint_kernel)
659 cmpl $0,threadinfo_preempt_count(%rcx)
660 jnz retint_restore_args
661 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
662 jnc retint_restore_args
663 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
664 jnc retint_restore_args
665 call preempt_schedule_irq
666 jmp exit_intr
667 #endif
669 CFI_ENDPROC
670 END(common_interrupt)
672 /*
673 * APIC interrupts.
674 */
675 .macro apicinterrupt num,func
676 INTR_FRAME
677 pushq $~(\num)
678 CFI_ADJUST_CFA_OFFSET 8
679 interrupt \func
680 jmp ret_from_intr
681 CFI_ENDPROC
682 .endm
684 ENTRY(thermal_interrupt)
685 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
686 END(thermal_interrupt)
688 ENTRY(threshold_interrupt)
689 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
690 END(threshold_interrupt)
692 #ifdef CONFIG_SMP
693 ENTRY(reschedule_interrupt)
694 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
695 END(reschedule_interrupt)
697 .macro INVALIDATE_ENTRY num
698 ENTRY(invalidate_interrupt\num)
699 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
700 END(invalidate_interrupt\num)
701 .endm
703 INVALIDATE_ENTRY 0
704 INVALIDATE_ENTRY 1
705 INVALIDATE_ENTRY 2
706 INVALIDATE_ENTRY 3
707 INVALIDATE_ENTRY 4
708 INVALIDATE_ENTRY 5
709 INVALIDATE_ENTRY 6
710 INVALIDATE_ENTRY 7
712 ENTRY(call_function_interrupt)
713 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
714 END(call_function_interrupt)
715 ENTRY(irq_move_cleanup_interrupt)
716 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
717 END(irq_move_cleanup_interrupt)
718 #endif
720 ENTRY(apic_timer_interrupt)
721 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
722 END(apic_timer_interrupt)
724 ENTRY(error_interrupt)
725 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
726 END(error_interrupt)
728 ENTRY(spurious_interrupt)
729 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
730 END(spurious_interrupt)
732 /*
733 * Exception entry points.
734 */
735 .macro zeroentry sym
736 INTR_FRAME
737 pushq $0 /* push error code/oldrax */
738 CFI_ADJUST_CFA_OFFSET 8
739 pushq %rax /* push real oldrax to the rdi slot */
740 CFI_ADJUST_CFA_OFFSET 8
741 CFI_REL_OFFSET rax,0
742 leaq \sym(%rip),%rax
743 jmp error_entry
744 CFI_ENDPROC
745 .endm
747 .macro errorentry sym
748 XCPT_FRAME
749 pushq %rax
750 CFI_ADJUST_CFA_OFFSET 8
751 CFI_REL_OFFSET rax,0
752 leaq \sym(%rip),%rax
753 jmp error_entry
754 CFI_ENDPROC
755 .endm
757 /* error code is on the stack already */
758 /* handle NMI like exceptions that can happen everywhere */
759 .macro paranoidentry sym, ist=0, irqtrace=1
760 SAVE_ALL
761 cld
762 movl $1,%ebx
763 movl $MSR_GS_BASE,%ecx
764 rdmsr
765 testl %edx,%edx
766 js 1f
767 SWAPGS
768 xorl %ebx,%ebx
769 1:
770 .if \ist
771 movq %gs:pda_data_offset, %rbp
772 .endif
773 movq %rsp,%rdi
774 movq ORIG_RAX(%rsp),%rsi
775 movq $-1,ORIG_RAX(%rsp)
776 .if \ist
777 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
778 .endif
779 call \sym
780 .if \ist
781 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
782 .endif
783 DISABLE_INTERRUPTS(CLBR_NONE)
784 .if \irqtrace
785 TRACE_IRQS_OFF
786 .endif
787 .endm
789 /*
790 * "Paranoid" exit path from exception stack.
791 * Paranoid because this is used by NMIs and cannot take
792 * any kernel state for granted.
793 * We don't do kernel preemption checks here, because only
794 * NMI should be common and it does not enable IRQs and
795 * cannot get reschedule ticks.
796 *
797 * "trace" is 0 for the NMI handler only, because irq-tracing
798 * is fundamentally NMI-unsafe. (we cannot change the soft and
799 * hard flags at once, atomically)
800 */
801 .macro paranoidexit trace=1
802 /* ebx: no swapgs flag */
803 paranoid_exit\trace:
804 testl %ebx,%ebx /* swapgs needed? */
805 jnz paranoid_restore\trace
806 testl $3,CS(%rsp)
807 jnz paranoid_userspace\trace
808 paranoid_swapgs\trace:
809 .if \trace
810 TRACE_IRQS_IRETQ 0
811 .endif
812 SWAPGS_UNSAFE_STACK
813 paranoid_restore\trace:
814 RESTORE_ALL 8
815 jmp irq_return
816 paranoid_userspace\trace:
817 GET_THREAD_INFO(%rcx)
818 movl threadinfo_flags(%rcx),%ebx
819 andl $_TIF_WORK_MASK,%ebx
820 jz paranoid_swapgs\trace
821 movq %rsp,%rdi /* &pt_regs */
822 call sync_regs
823 movq %rax,%rsp /* switch stack for scheduling */
824 testl $_TIF_NEED_RESCHED,%ebx
825 jnz paranoid_schedule\trace
826 movl %ebx,%edx /* arg3: thread flags */
827 .if \trace
828 TRACE_IRQS_ON
829 .endif
830 ENABLE_INTERRUPTS(CLBR_NONE)
831 xorl %esi,%esi /* arg2: oldset */
832 movq %rsp,%rdi /* arg1: &pt_regs */
833 call do_notify_resume
834 DISABLE_INTERRUPTS(CLBR_NONE)
835 .if \trace
836 TRACE_IRQS_OFF
837 .endif
838 jmp paranoid_userspace\trace
839 paranoid_schedule\trace:
840 .if \trace
841 TRACE_IRQS_ON
842 .endif
843 ENABLE_INTERRUPTS(CLBR_ANY)
844 call schedule
845 DISABLE_INTERRUPTS(CLBR_ANY)
846 .if \trace
847 TRACE_IRQS_OFF
848 .endif
849 jmp paranoid_userspace\trace
850 CFI_ENDPROC
851 .endm
853 /*
854 * Exception entry point. This expects an error code/orig_rax on the stack
855 * and the exception handler in %rax.
856 */
857 KPROBE_ENTRY(error_entry)
858 _frame RDI
859 CFI_REL_OFFSET rax,0
860 /* rdi slot contains rax, oldrax contains error code */
861 cld
862 subq $14*8,%rsp
863 CFI_ADJUST_CFA_OFFSET (14*8)
864 movq %rsi,13*8(%rsp)
865 CFI_REL_OFFSET rsi,RSI
866 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
867 CFI_REGISTER rax,rsi
868 movq %rdx,12*8(%rsp)
869 CFI_REL_OFFSET rdx,RDX
870 movq %rcx,11*8(%rsp)
871 CFI_REL_OFFSET rcx,RCX
872 movq %rsi,10*8(%rsp) /* store rax */
873 CFI_REL_OFFSET rax,RAX
874 movq %r8, 9*8(%rsp)
875 CFI_REL_OFFSET r8,R8
876 movq %r9, 8*8(%rsp)
877 CFI_REL_OFFSET r9,R9
878 movq %r10,7*8(%rsp)
879 CFI_REL_OFFSET r10,R10
880 movq %r11,6*8(%rsp)
881 CFI_REL_OFFSET r11,R11
882 movq %rbx,5*8(%rsp)
883 CFI_REL_OFFSET rbx,RBX
884 movq %rbp,4*8(%rsp)
885 CFI_REL_OFFSET rbp,RBP
886 movq %r12,3*8(%rsp)
887 CFI_REL_OFFSET r12,R12
888 movq %r13,2*8(%rsp)
889 CFI_REL_OFFSET r13,R13
890 movq %r14,1*8(%rsp)
891 CFI_REL_OFFSET r14,R14
892 movq %r15,(%rsp)
893 CFI_REL_OFFSET r15,R15
894 xorl %ebx,%ebx
895 testl $3,CS(%rsp)
896 je error_kernelspace
897 error_swapgs:
898 SWAPGS
899 error_sti:
900 movq %rdi,RDI(%rsp)
901 CFI_REL_OFFSET rdi,RDI
902 movq %rsp,%rdi
903 movq ORIG_RAX(%rsp),%rsi /* get error code */
904 movq $-1,ORIG_RAX(%rsp)
905 call *%rax
906 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
907 error_exit:
908 movl %ebx,%eax
909 RESTORE_REST
910 DISABLE_INTERRUPTS(CLBR_NONE)
911 TRACE_IRQS_OFF
912 GET_THREAD_INFO(%rcx)
913 testl %eax,%eax
914 jne retint_kernel
915 LOCKDEP_SYS_EXIT_IRQ
916 movl threadinfo_flags(%rcx),%edx
917 movl $_TIF_WORK_MASK,%edi
918 andl %edi,%edx
919 jnz retint_careful
920 jmp retint_swapgs
921 CFI_ENDPROC
923 error_kernelspace:
924 incl %ebx
925 /* There are two places in the kernel that can potentially fault with
926 usergs. Handle them here. The exception handlers after
927 iret run with kernel gs again, so don't set the user space flag.
928 B stepping K8s sometimes report an truncated RIP for IRET
929 exceptions returning to compat mode. Check for these here too. */
930 leaq irq_return(%rip),%rbp
931 cmpq %rbp,RIP(%rsp)
932 je error_swapgs
933 movl %ebp,%ebp /* zero extend */
934 cmpq %rbp,RIP(%rsp)
935 je error_swapgs
936 cmpq $gs_change,RIP(%rsp)
937 je error_swapgs
938 jmp error_sti
939 KPROBE_END(error_entry)
941 /* Reload gs selector with exception handling */
942 /* edi: new selector */
943 ENTRY(load_gs_index)
944 CFI_STARTPROC
945 pushf
946 CFI_ADJUST_CFA_OFFSET 8
947 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
948 SWAPGS
949 gs_change:
950 movl %edi,%gs
951 2: mfence /* workaround */
952 SWAPGS
953 popf
954 CFI_ADJUST_CFA_OFFSET -8
955 ret
956 CFI_ENDPROC
957 ENDPROC(load_gs_index)
959 .section __ex_table,"a"
960 .align 8
961 .quad gs_change,bad_gs
962 .previous
963 .section .fixup,"ax"
964 /* running with kernelgs */
965 bad_gs:
966 SWAPGS /* switch back to user gs */
967 xorl %eax,%eax
968 movl %eax,%gs
969 jmp 2b
970 .previous
972 /*
973 * Create a kernel thread.
974 *
975 * C extern interface:
976 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
977 *
978 * asm input arguments:
979 * rdi: fn, rsi: arg, rdx: flags
980 */
981 ENTRY(kernel_thread)
982 CFI_STARTPROC
983 FAKE_STACK_FRAME $child_rip
984 SAVE_ALL
986 # rdi: flags, rsi: usp, rdx: will be &pt_regs
987 movq %rdx,%rdi
988 orq kernel_thread_flags(%rip),%rdi
989 movq $-1, %rsi
990 movq %rsp, %rdx
992 xorl %r8d,%r8d
993 xorl %r9d,%r9d
995 # clone now
996 call do_fork
997 movq %rax,RAX(%rsp)
998 xorl %edi,%edi
1000 /*
1001 * It isn't worth to check for reschedule here,
1002 * so internally to the x86_64 port you can rely on kernel_thread()
1003 * not to reschedule the child before returning, this avoids the need
1004 * of hacks for example to fork off the per-CPU idle tasks.
1005 * [Hopefully no generic code relies on the reschedule -AK]
1006 */
1007 RESTORE_ALL
1008 UNFAKE_STACK_FRAME
1009 ret
1010 CFI_ENDPROC
1011 ENDPROC(kernel_thread)
1013 child_rip:
1014 pushq $0 # fake return address
1015 CFI_STARTPROC
1016 /*
1017 * Here we are in the child and the registers are set as they were
1018 * at kernel_thread() invocation in the parent.
1019 */
1020 movq %rdi, %rax
1021 movq %rsi, %rdi
1022 call *%rax
1023 # exit
1024 mov %eax, %edi
1025 call do_exit
1026 CFI_ENDPROC
1027 ENDPROC(child_rip)
1029 /*
1030 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1031 *
1032 * C extern interface:
1033 * extern long execve(char *name, char **argv, char **envp)
1034 *
1035 * asm input arguments:
1036 * rdi: name, rsi: argv, rdx: envp
1037 *
1038 * We want to fallback into:
1039 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs regs)
1040 *
1041 * do_sys_execve asm fallback arguments:
1042 * rdi: name, rsi: argv, rdx: envp, fake frame on the stack
1043 */
1044 ENTRY(kernel_execve)
1045 CFI_STARTPROC
1046 FAKE_STACK_FRAME $0
1047 SAVE_ALL
1048 call sys_execve
1049 movq %rax, RAX(%rsp)
1050 RESTORE_REST
1051 testq %rax,%rax
1052 je int_ret_from_sys_call
1053 RESTORE_ARGS
1054 UNFAKE_STACK_FRAME
1055 ret
1056 CFI_ENDPROC
1057 ENDPROC(kernel_execve)
1059 KPROBE_ENTRY(page_fault)
1060 errorentry do_page_fault
1061 KPROBE_END(page_fault)
1063 ENTRY(coprocessor_error)
1064 zeroentry do_coprocessor_error
1065 END(coprocessor_error)
1067 ENTRY(simd_coprocessor_error)
1068 zeroentry do_simd_coprocessor_error
1069 END(simd_coprocessor_error)
1071 ENTRY(device_not_available)
1072 zeroentry math_state_restore
1073 END(device_not_available)
1075 /* runs on exception stack */
1076 KPROBE_ENTRY(debug)
1077 INTR_FRAME
1078 pushq $0
1079 CFI_ADJUST_CFA_OFFSET 8
1080 paranoidentry do_debug, DEBUG_STACK
1081 paranoidexit
1082 KPROBE_END(debug)
1084 /* runs on exception stack */
1085 KPROBE_ENTRY(nmi)
1086 INTR_FRAME
1087 pushq $-1
1088 CFI_ADJUST_CFA_OFFSET 8
1089 paranoidentry do_nmi, 0, 0
1090 #ifdef CONFIG_TRACE_IRQFLAGS
1091 paranoidexit 0
1092 #else
1093 jmp paranoid_exit1
1094 CFI_ENDPROC
1095 #endif
1096 KPROBE_END(nmi)
1098 KPROBE_ENTRY(int3)
1099 INTR_FRAME
1100 pushq $0
1101 CFI_ADJUST_CFA_OFFSET 8
1102 paranoidentry do_int3, DEBUG_STACK
1103 jmp paranoid_exit1
1104 CFI_ENDPROC
1105 KPROBE_END(int3)
1107 ENTRY(overflow)
1108 zeroentry do_overflow
1109 END(overflow)
1111 ENTRY(bounds)
1112 zeroentry do_bounds
1113 END(bounds)
1115 ENTRY(invalid_op)
1116 zeroentry do_invalid_op
1117 END(invalid_op)
1119 ENTRY(coprocessor_segment_overrun)
1120 zeroentry do_coprocessor_segment_overrun
1121 END(coprocessor_segment_overrun)
1123 ENTRY(reserved)
1124 zeroentry do_reserved
1125 END(reserved)
1127 /* runs on exception stack */
1128 ENTRY(double_fault)
1129 XCPT_FRAME
1130 paranoidentry do_double_fault
1131 jmp paranoid_exit1
1132 CFI_ENDPROC
1133 END(double_fault)
1135 ENTRY(invalid_TSS)
1136 errorentry do_invalid_TSS
1137 END(invalid_TSS)
1139 ENTRY(segment_not_present)
1140 errorentry do_segment_not_present
1141 END(segment_not_present)
1143 /* runs on exception stack */
1144 ENTRY(stack_segment)
1145 XCPT_FRAME
1146 paranoidentry do_stack_segment
1147 jmp paranoid_exit1
1148 CFI_ENDPROC
1149 END(stack_segment)
1151 KPROBE_ENTRY(general_protection)
1152 errorentry do_general_protection
1153 KPROBE_END(general_protection)
1155 ENTRY(alignment_check)
1156 errorentry do_alignment_check
1157 END(alignment_check)
1159 ENTRY(divide_error)
1160 zeroentry do_divide_error
1161 END(divide_error)
1163 ENTRY(spurious_interrupt_bug)
1164 zeroentry do_spurious_interrupt_bug
1165 END(spurious_interrupt_bug)
1167 #ifdef CONFIG_X86_MCE
1168 /* runs on exception stack */
1169 ENTRY(machine_check)
1170 INTR_FRAME
1171 pushq $0
1172 CFI_ADJUST_CFA_OFFSET 8
1173 paranoidentry do_machine_check
1174 jmp paranoid_exit1
1175 CFI_ENDPROC
1176 END(machine_check)
1177 #endif
1179 /* Call softirq on interrupt stack. Interrupts are off. */
1180 ENTRY(call_softirq)
1181 CFI_STARTPROC
1182 push %rbp
1183 CFI_ADJUST_CFA_OFFSET 8
1184 CFI_REL_OFFSET rbp,0
1185 mov %rsp,%rbp
1186 CFI_DEF_CFA_REGISTER rbp
1187 incl %gs:pda_irqcount
1188 cmove %gs:pda_irqstackptr,%rsp
1189 push %rbp # backlink for old unwinder
1190 call __do_softirq
1191 leaveq
1192 CFI_DEF_CFA_REGISTER rsp
1193 CFI_ADJUST_CFA_OFFSET -8
1194 decl %gs:pda_irqcount
1195 ret
1196 CFI_ENDPROC
1197 ENDPROC(call_softirq)
1199 KPROBE_ENTRY(ignore_sysret)
1200 CFI_STARTPROC
1201 mov $-ENOSYS,%eax
1202 sysret
1203 CFI_ENDPROC
1204 ENDPROC(ignore_sysret)