| blob | 556a8df522a7adcf9583164be4cad60e29b00a97 |
1 /*
2 * linux/arch/x86_64/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
6 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
7 */
9 /*
10 * entry.S contains the system-call and fault low-level handling routines.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after an interrupt and after each system call.
14 *
15 * Normal syscalls and interrupts don't save a full stack frame, this is
16 * only done for syscall tracing, signals or fork/exec et.al.
17 *
18 * A note on terminology:
19 * - top of stack: Architecture defined interrupt frame from SS to RIP
20 * at the top of the kernel process stack.
21 * - partial stack frame: partially saved registers upto R11.
22 * - full stack frame: Like partial stack frame, but all register saved.
23 *
24 * Some macro usage:
25 * - CFI macros are used to generate dwarf2 unwind information for better
26 * backtraces. They don't change any code.
27 * - SAVE_ALL/RESTORE_ALL - Save/restore all registers
28 * - SAVE_ARGS/RESTORE_ARGS - Save/restore registers that C functions modify.
29 * There are unfortunately lots of special cases where some registers
30 * not touched. The macro is a big mess that should be cleaned up.
31 * - SAVE_REST/RESTORE_REST - Handle the registers not saved by SAVE_ARGS.
32 * Gives a full stack frame.
33 * - ENTRY/END Define functions in the symbol table.
34 * - FIXUP_TOP_OF_STACK/RESTORE_TOP_OF_STACK - Fix up the hardware stack
35 * frame that is otherwise undefined after a SYSCALL
36 * - TRACE_IRQ_* - Trace hard interrupt state for lock debugging.
37 * - errorentry/paranoidentry/zeroentry - Define exception entry points.
38 */
40 #include <linux/linkage.h>
41 #include <asm/segment.h>
42 #include <asm/cache.h>
43 #include <asm/errno.h>
44 #include <asm/dwarf2.h>
45 #include <asm/calling.h>
46 #include <asm/asm-offsets.h>
47 #include <asm/msr.h>
48 #include <asm/unistd.h>
49 #include <asm/thread_info.h>
50 #include <asm/hw_irq.h>
51 #include <asm/page.h>
52 #include <asm/irqflags.h>
53 #include <asm/paravirt.h>
55 .code64
57 #ifndef CONFIG_PREEMPT
58 #define retint_kernel retint_restore_args
59 #endif
61 #ifdef CONFIG_PARAVIRT
62 ENTRY(native_irq_enable_syscall_ret)
63 movq %gs:pda_oldrsp,%rsp
64 swapgs
65 sysretq
66 #endif /* CONFIG_PARAVIRT */
69 .macro TRACE_IRQS_IRETQ offset=ARGOFFSET
70 #ifdef CONFIG_TRACE_IRQFLAGS
71 bt $9,EFLAGS-\offset(%rsp) /* interrupts off? */
72 jnc 1f
73 TRACE_IRQS_ON
74 1:
75 #endif
76 .endm
78 /*
79 * C code is not supposed to know about undefined top of stack. Every time
80 * a C function with an pt_regs argument is called from the SYSCALL based
81 * fast path FIXUP_TOP_OF_STACK is needed.
82 * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs
83 * manipulation.
84 */
86 /* %rsp:at FRAMEEND */
87 .macro FIXUP_TOP_OF_STACK tmp
88 movq %gs:pda_oldrsp,\tmp
89 movq \tmp,RSP(%rsp)
90 movq $__USER_DS,SS(%rsp)
91 movq $__USER_CS,CS(%rsp)
92 movq $-1,RCX(%rsp)
93 movq R11(%rsp),\tmp /* get eflags */
94 movq \tmp,EFLAGS(%rsp)
95 .endm
97 .macro RESTORE_TOP_OF_STACK tmp,offset=0
98 movq RSP-\offset(%rsp),\tmp
99 movq \tmp,%gs:pda_oldrsp
100 movq EFLAGS-\offset(%rsp),\tmp
101 movq \tmp,R11-\offset(%rsp)
102 .endm
104 .macro FAKE_STACK_FRAME child_rip
105 /* push in order ss, rsp, eflags, cs, rip */
106 xorl %eax, %eax
107 pushq %rax /* ss */
108 CFI_ADJUST_CFA_OFFSET 8
109 /*CFI_REL_OFFSET ss,0*/
110 pushq %rax /* rsp */
111 CFI_ADJUST_CFA_OFFSET 8
112 CFI_REL_OFFSET rsp,0
113 pushq $(1<<9) /* eflags - interrupts on */
114 CFI_ADJUST_CFA_OFFSET 8
115 /*CFI_REL_OFFSET rflags,0*/
116 pushq $__KERNEL_CS /* cs */
117 CFI_ADJUST_CFA_OFFSET 8
118 /*CFI_REL_OFFSET cs,0*/
119 pushq \child_rip /* rip */
120 CFI_ADJUST_CFA_OFFSET 8
121 CFI_REL_OFFSET rip,0
122 pushq %rax /* orig rax */
123 CFI_ADJUST_CFA_OFFSET 8
124 .endm
126 .macro UNFAKE_STACK_FRAME
127 addq $8*6, %rsp
128 CFI_ADJUST_CFA_OFFSET -(6*8)
129 .endm
131 .macro CFI_DEFAULT_STACK start=1
132 .if \start
133 CFI_STARTPROC simple
134 CFI_SIGNAL_FRAME
135 CFI_DEF_CFA rsp,SS+8
136 .else
137 CFI_DEF_CFA_OFFSET SS+8
138 .endif
139 CFI_REL_OFFSET r15,R15
140 CFI_REL_OFFSET r14,R14
141 CFI_REL_OFFSET r13,R13
142 CFI_REL_OFFSET r12,R12
143 CFI_REL_OFFSET rbp,RBP
144 CFI_REL_OFFSET rbx,RBX
145 CFI_REL_OFFSET r11,R11
146 CFI_REL_OFFSET r10,R10
147 CFI_REL_OFFSET r9,R9
148 CFI_REL_OFFSET r8,R8
149 CFI_REL_OFFSET rax,RAX
150 CFI_REL_OFFSET rcx,RCX
151 CFI_REL_OFFSET rdx,RDX
152 CFI_REL_OFFSET rsi,RSI
153 CFI_REL_OFFSET rdi,RDI
154 CFI_REL_OFFSET rip,RIP
155 /*CFI_REL_OFFSET cs,CS*/
156 /*CFI_REL_OFFSET rflags,EFLAGS*/
157 CFI_REL_OFFSET rsp,RSP
158 /*CFI_REL_OFFSET ss,SS*/
159 .endm
160 /*
161 * A newly forked process directly context switches into this.
162 */
163 /* rdi: prev */
164 ENTRY(ret_from_fork)
165 CFI_DEFAULT_STACK
166 push kernel_eflags(%rip)
167 CFI_ADJUST_CFA_OFFSET 4
168 popf # reset kernel eflags
169 CFI_ADJUST_CFA_OFFSET -4
170 call schedule_tail
171 GET_THREAD_INFO(%rcx)
172 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),threadinfo_flags(%rcx)
173 jnz rff_trace
174 rff_action:
175 RESTORE_REST
176 testl $3,CS-ARGOFFSET(%rsp) # from kernel_thread?
177 je int_ret_from_sys_call
178 testl $_TIF_IA32,threadinfo_flags(%rcx)
179 jnz int_ret_from_sys_call
180 RESTORE_TOP_OF_STACK %rdi,ARGOFFSET
181 jmp ret_from_sys_call
182 rff_trace:
183 movq %rsp,%rdi
184 call syscall_trace_leave
185 GET_THREAD_INFO(%rcx)
186 jmp rff_action
187 CFI_ENDPROC
188 END(ret_from_fork)
190 /*
191 * System call entry. Upto 6 arguments in registers are supported.
192 *
193 * SYSCALL does not save anything on the stack and does not change the
194 * stack pointer.
195 */
197 /*
198 * Register setup:
199 * rax system call number
200 * rdi arg0
201 * rcx return address for syscall/sysret, C arg3
202 * rsi arg1
203 * rdx arg2
204 * r10 arg3 (--> moved to rcx for C)
205 * r8 arg4
206 * r9 arg5
207 * r11 eflags for syscall/sysret, temporary for C
208 * r12-r15,rbp,rbx saved by C code, not touched.
209 *
210 * Interrupts are off on entry.
211 * Only called from user space.
212 *
213 * XXX if we had a free scratch register we could save the RSP into the stack frame
214 * and report it properly in ps. Unfortunately we haven't.
215 *
216 * When user can change the frames always force IRET. That is because
217 * it deals with uncanonical addresses better. SYSRET has trouble
218 * with them due to bugs in both AMD and Intel CPUs.
219 */
221 ENTRY(system_call)
222 CFI_STARTPROC simple
223 CFI_SIGNAL_FRAME
224 CFI_DEF_CFA rsp,PDA_STACKOFFSET
225 CFI_REGISTER rip,rcx
226 /*CFI_REGISTER rflags,r11*/
227 SWAPGS_UNSAFE_STACK
228 /*
229 * A hypervisor implementation might want to use a label
230 * after the swapgs, so that it can do the swapgs
231 * for the guest and jump here on syscall.
232 */
233 ENTRY(system_call_after_swapgs)
235 movq %rsp,%gs:pda_oldrsp
236 movq %gs:pda_kernelstack,%rsp
237 /*
238 * No need to follow this irqs off/on section - it's straight
239 * and short:
240 */
241 ENABLE_INTERRUPTS(CLBR_NONE)
242 SAVE_ARGS 8,1
243 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
244 movq %rcx,RIP-ARGOFFSET(%rsp)
245 CFI_REL_OFFSET rip,RIP-ARGOFFSET
246 GET_THREAD_INFO(%rcx)
247 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
248 jnz tracesys
249 cmpq $__NR_syscall_max,%rax
250 ja badsys
251 movq %r10,%rcx
252 call *sys_call_table(,%rax,8) # XXX: rip relative
253 movq %rax,RAX-ARGOFFSET(%rsp)
254 /*
255 * Syscall return path ending with SYSRET (fast path)
256 * Has incomplete stack frame and undefined top of stack.
257 */
258 ret_from_sys_call:
259 movl $_TIF_ALLWORK_MASK,%edi
260 /* edi: flagmask */
261 sysret_check:
262 LOCKDEP_SYS_EXIT
263 GET_THREAD_INFO(%rcx)
264 DISABLE_INTERRUPTS(CLBR_NONE)
265 TRACE_IRQS_OFF
266 movl threadinfo_flags(%rcx),%edx
267 andl %edi,%edx
268 jnz sysret_careful
269 CFI_REMEMBER_STATE
270 /*
271 * sysretq will re-enable interrupts:
272 */
273 TRACE_IRQS_ON
274 movq RIP-ARGOFFSET(%rsp),%rcx
275 CFI_REGISTER rip,rcx
276 RESTORE_ARGS 0,-ARG_SKIP,1
277 /*CFI_REGISTER rflags,r11*/
278 ENABLE_INTERRUPTS_SYSCALL_RET
280 CFI_RESTORE_STATE
281 /* Handle reschedules */
282 /* edx: work, edi: workmask */
283 sysret_careful:
284 bt $TIF_NEED_RESCHED,%edx
285 jnc sysret_signal
286 TRACE_IRQS_ON
287 ENABLE_INTERRUPTS(CLBR_NONE)
288 pushq %rdi
289 CFI_ADJUST_CFA_OFFSET 8
290 call schedule
291 popq %rdi
292 CFI_ADJUST_CFA_OFFSET -8
293 jmp sysret_check
295 /* Handle a signal */
296 sysret_signal:
297 TRACE_IRQS_ON
298 ENABLE_INTERRUPTS(CLBR_NONE)
299 testl $_TIF_DO_NOTIFY_MASK,%edx
300 jz 1f
302 /* Really a signal */
303 /* edx: work flags (arg3) */
304 leaq do_notify_resume(%rip),%rax
305 leaq -ARGOFFSET(%rsp),%rdi # &pt_regs -> arg1
306 xorl %esi,%esi # oldset -> arg2
307 call ptregscall_common
308 1: movl $_TIF_NEED_RESCHED,%edi
309 /* Use IRET because user could have changed frame. This
310 works because ptregscall_common has called FIXUP_TOP_OF_STACK. */
311 DISABLE_INTERRUPTS(CLBR_NONE)
312 TRACE_IRQS_OFF
313 jmp int_with_check
315 badsys:
316 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
317 jmp ret_from_sys_call
319 /* Do syscall tracing */
320 tracesys:
321 SAVE_REST
322 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
323 FIXUP_TOP_OF_STACK %rdi
324 movq %rsp,%rdi
325 call syscall_trace_enter
326 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
327 RESTORE_REST
328 cmpq $__NR_syscall_max,%rax
329 ja int_ret_from_sys_call /* RAX(%rsp) set to -ENOSYS above */
330 movq %r10,%rcx /* fixup for C */
331 call *sys_call_table(,%rax,8)
332 movq %rax,RAX-ARGOFFSET(%rsp)
333 /* Use IRET because user could have changed frame */
335 /*
336 * Syscall return path ending with IRET.
337 * Has correct top of stack, but partial stack frame.
338 */
339 .globl int_ret_from_sys_call
340 int_ret_from_sys_call:
341 DISABLE_INTERRUPTS(CLBR_NONE)
342 TRACE_IRQS_OFF
343 testl $3,CS-ARGOFFSET(%rsp)
344 je retint_restore_args
345 movl $_TIF_ALLWORK_MASK,%edi
346 /* edi: mask to check */
347 int_with_check:
348 LOCKDEP_SYS_EXIT_IRQ
349 GET_THREAD_INFO(%rcx)
350 movl threadinfo_flags(%rcx),%edx
351 andl %edi,%edx
352 jnz int_careful
353 andl $~TS_COMPAT,threadinfo_status(%rcx)
354 jmp retint_swapgs
356 /* Either reschedule or signal or syscall exit tracking needed. */
357 /* First do a reschedule test. */
358 /* edx: work, edi: workmask */
359 int_careful:
360 bt $TIF_NEED_RESCHED,%edx
361 jnc int_very_careful
362 TRACE_IRQS_ON
363 ENABLE_INTERRUPTS(CLBR_NONE)
364 pushq %rdi
365 CFI_ADJUST_CFA_OFFSET 8
366 call schedule
367 popq %rdi
368 CFI_ADJUST_CFA_OFFSET -8
369 DISABLE_INTERRUPTS(CLBR_NONE)
370 TRACE_IRQS_OFF
371 jmp int_with_check
373 /* handle signals and tracing -- both require a full stack frame */
374 int_very_careful:
375 TRACE_IRQS_ON
376 ENABLE_INTERRUPTS(CLBR_NONE)
377 SAVE_REST
378 /* Check for syscall exit trace */
379 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
380 jz int_signal
381 pushq %rdi
382 CFI_ADJUST_CFA_OFFSET 8
383 leaq 8(%rsp),%rdi # &ptregs -> arg1
384 call syscall_trace_leave
385 popq %rdi
386 CFI_ADJUST_CFA_OFFSET -8
387 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
388 jmp int_restore_rest
390 int_signal:
391 testl $_TIF_DO_NOTIFY_MASK,%edx
392 jz 1f
393 movq %rsp,%rdi # &ptregs -> arg1
394 xorl %esi,%esi # oldset -> arg2
395 call do_notify_resume
396 1: movl $_TIF_NEED_RESCHED,%edi
397 int_restore_rest:
398 RESTORE_REST
399 DISABLE_INTERRUPTS(CLBR_NONE)
400 TRACE_IRQS_OFF
401 jmp int_with_check
402 CFI_ENDPROC
403 END(system_call)
405 /*
406 * Certain special system calls that need to save a complete full stack frame.
407 */
409 .macro PTREGSCALL label,func,arg
410 .globl \label
411 \label:
412 leaq \func(%rip),%rax
413 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
414 jmp ptregscall_common
415 END(\label)
416 .endm
418 CFI_STARTPROC
420 PTREGSCALL stub_clone, sys_clone, %r8
421 PTREGSCALL stub_fork, sys_fork, %rdi
422 PTREGSCALL stub_vfork, sys_vfork, %rdi
423 PTREGSCALL stub_rt_sigsuspend, sys_rt_sigsuspend, %rdx
424 PTREGSCALL stub_sigaltstack, sys_sigaltstack, %rdx
425 PTREGSCALL stub_iopl, sys_iopl, %rsi
427 ENTRY(ptregscall_common)
428 popq %r11
429 CFI_ADJUST_CFA_OFFSET -8
430 CFI_REGISTER rip, r11
431 SAVE_REST
432 movq %r11, %r15
433 CFI_REGISTER rip, r15
434 FIXUP_TOP_OF_STACK %r11
435 call *%rax
436 RESTORE_TOP_OF_STACK %r11
437 movq %r15, %r11
438 CFI_REGISTER rip, r11
439 RESTORE_REST
440 pushq %r11
441 CFI_ADJUST_CFA_OFFSET 8
442 CFI_REL_OFFSET rip, 0
443 ret
444 CFI_ENDPROC
445 END(ptregscall_common)
447 ENTRY(stub_execve)
448 CFI_STARTPROC
449 popq %r11
450 CFI_ADJUST_CFA_OFFSET -8
451 CFI_REGISTER rip, r11
452 SAVE_REST
453 FIXUP_TOP_OF_STACK %r11
454 movq %rsp, %rcx
455 call sys_execve
456 RESTORE_TOP_OF_STACK %r11
457 movq %rax,RAX(%rsp)
458 RESTORE_REST
459 jmp int_ret_from_sys_call
460 CFI_ENDPROC
461 END(stub_execve)
463 /*
464 * sigreturn is special because it needs to restore all registers on return.
465 * This cannot be done with SYSRET, so use the IRET return path instead.
466 */
467 ENTRY(stub_rt_sigreturn)
468 CFI_STARTPROC
469 addq $8, %rsp
470 CFI_ADJUST_CFA_OFFSET -8
471 SAVE_REST
472 movq %rsp,%rdi
473 FIXUP_TOP_OF_STACK %r11
474 call sys_rt_sigreturn
475 movq %rax,RAX(%rsp) # fixme, this could be done at the higher layer
476 RESTORE_REST
477 jmp int_ret_from_sys_call
478 CFI_ENDPROC
479 END(stub_rt_sigreturn)
481 /*
482 * initial frame state for interrupts and exceptions
483 */
484 .macro _frame ref
485 CFI_STARTPROC simple
486 CFI_SIGNAL_FRAME
487 CFI_DEF_CFA rsp,SS+8-\ref
488 /*CFI_REL_OFFSET ss,SS-\ref*/
489 CFI_REL_OFFSET rsp,RSP-\ref
490 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
491 /*CFI_REL_OFFSET cs,CS-\ref*/
492 CFI_REL_OFFSET rip,RIP-\ref
493 .endm
495 /* initial frame state for interrupts (and exceptions without error code) */
496 #define INTR_FRAME _frame RIP
497 /* initial frame state for exceptions with error code (and interrupts with
498 vector already pushed) */
499 #define XCPT_FRAME _frame ORIG_RAX
501 /*
502 * Interrupt entry/exit.
503 *
504 * Interrupt entry points save only callee clobbered registers in fast path.
505 *
506 * Entry runs with interrupts off.
507 */
509 /* 0(%rsp): interrupt number */
510 .macro interrupt func
511 cld
512 SAVE_ARGS
513 leaq -ARGOFFSET(%rsp),%rdi # arg1 for handler
514 pushq %rbp
515 CFI_ADJUST_CFA_OFFSET 8
516 CFI_REL_OFFSET rbp, 0
517 movq %rsp,%rbp
518 CFI_DEF_CFA_REGISTER rbp
519 testl $3,CS(%rdi)
520 je 1f
521 SWAPGS
522 /* irqcount is used to check if a CPU is already on an interrupt
523 stack or not. While this is essentially redundant with preempt_count
524 it is a little cheaper to use a separate counter in the PDA
525 (short of moving irq_enter into assembly, which would be too
526 much work) */
527 1: incl %gs:pda_irqcount
528 cmoveq %gs:pda_irqstackptr,%rsp
529 push %rbp # backlink for old unwinder
530 /*
531 * We entered an interrupt context - irqs are off:
532 */
533 TRACE_IRQS_OFF
534 call \func
535 .endm
537 ENTRY(common_interrupt)
538 XCPT_FRAME
539 interrupt do_IRQ
540 /* 0(%rsp): oldrsp-ARGOFFSET */
541 ret_from_intr:
542 DISABLE_INTERRUPTS(CLBR_NONE)
543 TRACE_IRQS_OFF
544 decl %gs:pda_irqcount
545 leaveq
546 CFI_DEF_CFA_REGISTER rsp
547 CFI_ADJUST_CFA_OFFSET -8
548 exit_intr:
549 GET_THREAD_INFO(%rcx)
550 testl $3,CS-ARGOFFSET(%rsp)
551 je retint_kernel
553 /* Interrupt came from user space */
554 /*
555 * Has a correct top of stack, but a partial stack frame
556 * %rcx: thread info. Interrupts off.
557 */
558 retint_with_reschedule:
559 movl $_TIF_WORK_MASK,%edi
560 retint_check:
561 LOCKDEP_SYS_EXIT_IRQ
562 movl threadinfo_flags(%rcx),%edx
563 andl %edi,%edx
564 CFI_REMEMBER_STATE
565 jnz retint_careful
567 retint_swapgs: /* return to user-space */
568 /*
569 * The iretq could re-enable interrupts:
570 */
571 DISABLE_INTERRUPTS(CLBR_ANY)
572 TRACE_IRQS_IRETQ
573 SWAPGS
574 jmp restore_args
576 retint_restore_args: /* return to kernel space */
577 DISABLE_INTERRUPTS(CLBR_ANY)
578 /*
579 * The iretq could re-enable interrupts:
580 */
581 TRACE_IRQS_IRETQ
582 restore_args:
583 RESTORE_ARGS 0,8,0
585 irq_return:
586 INTERRUPT_RETURN
588 .section __ex_table, "a"
589 .quad irq_return, bad_iret
590 .previous
592 #ifdef CONFIG_PARAVIRT
593 ENTRY(native_iret)
594 iretq
596 .section __ex_table,"a"
597 .quad native_iret, bad_iret
598 .previous
599 #endif
601 .section .fixup,"ax"
602 bad_iret:
603 /*
604 * The iret traps when the %cs or %ss being restored is bogus.
605 * We've lost the original trap vector and error code.
606 * #GPF is the most likely one to get for an invalid selector.
607 * So pretend we completed the iret and took the #GPF in user mode.
608 *
609 * We are now running with the kernel GS after exception recovery.
610 * But error_entry expects us to have user GS to match the user %cs,
611 * so swap back.
612 */
613 pushq $0
615 SWAPGS
616 jmp general_protection
618 .previous
620 /* edi: workmask, edx: work */
621 retint_careful:
622 CFI_RESTORE_STATE
623 bt $TIF_NEED_RESCHED,%edx
624 jnc retint_signal
625 TRACE_IRQS_ON
626 ENABLE_INTERRUPTS(CLBR_NONE)
627 pushq %rdi
628 CFI_ADJUST_CFA_OFFSET 8
629 call schedule
630 popq %rdi
631 CFI_ADJUST_CFA_OFFSET -8
632 GET_THREAD_INFO(%rcx)
633 DISABLE_INTERRUPTS(CLBR_NONE)
634 TRACE_IRQS_OFF
635 jmp retint_check
637 retint_signal:
638 testl $_TIF_DO_NOTIFY_MASK,%edx
639 jz retint_swapgs
640 TRACE_IRQS_ON
641 ENABLE_INTERRUPTS(CLBR_NONE)
642 SAVE_REST
643 movq $-1,ORIG_RAX(%rsp)
644 xorl %esi,%esi # oldset
645 movq %rsp,%rdi # &pt_regs
646 call do_notify_resume
647 RESTORE_REST
648 DISABLE_INTERRUPTS(CLBR_NONE)
649 TRACE_IRQS_OFF
650 movl $_TIF_NEED_RESCHED,%edi
651 GET_THREAD_INFO(%rcx)
652 jmp retint_check
654 #ifdef CONFIG_PREEMPT
655 /* Returning to kernel space. Check if we need preemption */
656 /* rcx: threadinfo. interrupts off. */
657 ENTRY(retint_kernel)
658 cmpl $0,threadinfo_preempt_count(%rcx)
659 jnz retint_restore_args
660 bt $TIF_NEED_RESCHED,threadinfo_flags(%rcx)
661 jnc retint_restore_args
662 bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
663 jnc retint_restore_args
664 call preempt_schedule_irq
665 jmp exit_intr
666 #endif
668 CFI_ENDPROC
669 END(common_interrupt)
671 /*
672 * APIC interrupts.
673 */
674 .macro apicinterrupt num,func
675 INTR_FRAME
676 pushq $~(\num)
677 CFI_ADJUST_CFA_OFFSET 8
678 interrupt \func
679 jmp ret_from_intr
680 CFI_ENDPROC
681 .endm
683 ENTRY(thermal_interrupt)
684 apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
685 END(thermal_interrupt)
687 ENTRY(threshold_interrupt)
688 apicinterrupt THRESHOLD_APIC_VECTOR,mce_threshold_interrupt
689 END(threshold_interrupt)
691 #ifdef CONFIG_SMP
692 ENTRY(reschedule_interrupt)
693 apicinterrupt RESCHEDULE_VECTOR,smp_reschedule_interrupt
694 END(reschedule_interrupt)
696 .macro INVALIDATE_ENTRY num
697 ENTRY(invalidate_interrupt\num)
698 apicinterrupt INVALIDATE_TLB_VECTOR_START+\num,smp_invalidate_interrupt
699 END(invalidate_interrupt\num)
700 .endm
702 INVALIDATE_ENTRY 0
703 INVALIDATE_ENTRY 1
704 INVALIDATE_ENTRY 2
705 INVALIDATE_ENTRY 3
706 INVALIDATE_ENTRY 4
707 INVALIDATE_ENTRY 5
708 INVALIDATE_ENTRY 6
709 INVALIDATE_ENTRY 7
711 ENTRY(call_function_interrupt)
712 apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
713 END(call_function_interrupt)
714 ENTRY(irq_move_cleanup_interrupt)
715 apicinterrupt IRQ_MOVE_CLEANUP_VECTOR,smp_irq_move_cleanup_interrupt
716 END(irq_move_cleanup_interrupt)
717 #endif
719 ENTRY(apic_timer_interrupt)
720 apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt
721 END(apic_timer_interrupt)
723 ENTRY(error_interrupt)
724 apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt
725 END(error_interrupt)
727 ENTRY(spurious_interrupt)
728 apicinterrupt SPURIOUS_APIC_VECTOR,smp_spurious_interrupt
729 END(spurious_interrupt)
731 /*
732 * Exception entry points.
733 */
734 .macro zeroentry sym
735 INTR_FRAME
736 pushq $0 /* push error code/oldrax */
737 CFI_ADJUST_CFA_OFFSET 8
738 pushq %rax /* push real oldrax to the rdi slot */
739 CFI_ADJUST_CFA_OFFSET 8
740 CFI_REL_OFFSET rax,0
741 leaq \sym(%rip),%rax
742 jmp error_entry
743 CFI_ENDPROC
744 .endm
746 .macro errorentry sym
747 XCPT_FRAME
748 pushq %rax
749 CFI_ADJUST_CFA_OFFSET 8
750 CFI_REL_OFFSET rax,0
751 leaq \sym(%rip),%rax
752 jmp error_entry
753 CFI_ENDPROC
754 .endm
756 /* error code is on the stack already */
757 /* handle NMI like exceptions that can happen everywhere */
758 .macro paranoidentry sym, ist=0, irqtrace=1
759 SAVE_ALL
760 cld
761 movl $1,%ebx
762 movl $MSR_GS_BASE,%ecx
763 rdmsr
764 testl %edx,%edx
765 js 1f
766 SWAPGS
767 xorl %ebx,%ebx
768 1:
769 .if \ist
770 movq %gs:pda_data_offset, %rbp
771 .endif
772 movq %rsp,%rdi
773 movq ORIG_RAX(%rsp),%rsi
774 movq $-1,ORIG_RAX(%rsp)
775 .if \ist
776 subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
777 .endif
778 call \sym
779 .if \ist
780 addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
781 .endif
782 DISABLE_INTERRUPTS(CLBR_NONE)
783 .if \irqtrace
784 TRACE_IRQS_OFF
785 .endif
786 .endm
788 /*
789 * "Paranoid" exit path from exception stack.
790 * Paranoid because this is used by NMIs and cannot take
791 * any kernel state for granted.
792 * We don't do kernel preemption checks here, because only
793 * NMI should be common and it does not enable IRQs and
794 * cannot get reschedule ticks.
795 *
796 * "trace" is 0 for the NMI handler only, because irq-tracing
797 * is fundamentally NMI-unsafe. (we cannot change the soft and
798 * hard flags at once, atomically)
799 */
800 .macro paranoidexit trace=1
801 /* ebx: no swapgs flag */
802 paranoid_exit\trace:
803 testl %ebx,%ebx /* swapgs needed? */
804 jnz paranoid_restore\trace
805 testl $3,CS(%rsp)
806 jnz paranoid_userspace\trace
807 paranoid_swapgs\trace:
808 .if \trace
809 TRACE_IRQS_IRETQ 0
810 .endif
811 SWAPGS_UNSAFE_STACK
812 paranoid_restore\trace:
813 RESTORE_ALL 8
814 jmp irq_return
815 paranoid_userspace\trace:
816 GET_THREAD_INFO(%rcx)
817 movl threadinfo_flags(%rcx),%ebx
818 andl $_TIF_WORK_MASK,%ebx
819 jz paranoid_swapgs\trace
820 movq %rsp,%rdi /* &pt_regs */
821 call sync_regs
822 movq %rax,%rsp /* switch stack for scheduling */
823 testl $_TIF_NEED_RESCHED,%ebx
824 jnz paranoid_schedule\trace
825 movl %ebx,%edx /* arg3: thread flags */
826 .if \trace
827 TRACE_IRQS_ON
828 .endif
829 ENABLE_INTERRUPTS(CLBR_NONE)
830 xorl %esi,%esi /* arg2: oldset */
831 movq %rsp,%rdi /* arg1: &pt_regs */
832 call do_notify_resume
833 DISABLE_INTERRUPTS(CLBR_NONE)
834 .if \trace
835 TRACE_IRQS_OFF
836 .endif
837 jmp paranoid_userspace\trace
838 paranoid_schedule\trace:
839 .if \trace
840 TRACE_IRQS_ON
841 .endif
842 ENABLE_INTERRUPTS(CLBR_ANY)
843 call schedule
844 DISABLE_INTERRUPTS(CLBR_ANY)
845 .if \trace
846 TRACE_IRQS_OFF
847 .endif
848 jmp paranoid_userspace\trace
849 CFI_ENDPROC
850 .endm
852 /*
853 * Exception entry point. This expects an error code/orig_rax on the stack
854 * and the exception handler in %rax.
855 */
856 KPROBE_ENTRY(error_entry)
857 _frame RDI
858 CFI_REL_OFFSET rax,0
859 /* rdi slot contains rax, oldrax contains error code */
860 cld
861 subq $14*8,%rsp
862 CFI_ADJUST_CFA_OFFSET (14*8)
863 movq %rsi,13*8(%rsp)
864 CFI_REL_OFFSET rsi,RSI
865 movq 14*8(%rsp),%rsi /* load rax from rdi slot */
866 CFI_REGISTER rax,rsi
867 movq %rdx,12*8(%rsp)
868 CFI_REL_OFFSET rdx,RDX
869 movq %rcx,11*8(%rsp)
870 CFI_REL_OFFSET rcx,RCX
871 movq %rsi,10*8(%rsp) /* store rax */
872 CFI_REL_OFFSET rax,RAX
873 movq %r8, 9*8(%rsp)
874 CFI_REL_OFFSET r8,R8
875 movq %r9, 8*8(%rsp)
876 CFI_REL_OFFSET r9,R9
877 movq %r10,7*8(%rsp)
878 CFI_REL_OFFSET r10,R10
879 movq %r11,6*8(%rsp)
880 CFI_REL_OFFSET r11,R11
881 movq %rbx,5*8(%rsp)
882 CFI_REL_OFFSET rbx,RBX
883 movq %rbp,4*8(%rsp)
884 CFI_REL_OFFSET rbp,RBP
885 movq %r12,3*8(%rsp)
886 CFI_REL_OFFSET r12,R12
887 movq %r13,2*8(%rsp)
888 CFI_REL_OFFSET r13,R13
889 movq %r14,1*8(%rsp)
890 CFI_REL_OFFSET r14,R14
891 movq %r15,(%rsp)
892 CFI_REL_OFFSET r15,R15
893 xorl %ebx,%ebx
894 testl $3,CS(%rsp)
895 je error_kernelspace
896 error_swapgs:
897 SWAPGS
898 error_sti:
899 movq %rdi,RDI(%rsp)
900 CFI_REL_OFFSET rdi,RDI
901 movq %rsp,%rdi
902 movq ORIG_RAX(%rsp),%rsi /* get error code */
903 movq $-1,ORIG_RAX(%rsp)
904 call *%rax
905 /* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
906 error_exit:
907 movl %ebx,%eax
908 RESTORE_REST
909 DISABLE_INTERRUPTS(CLBR_NONE)
910 TRACE_IRQS_OFF
911 GET_THREAD_INFO(%rcx)
912 testl %eax,%eax
913 jne retint_kernel
914 LOCKDEP_SYS_EXIT_IRQ
915 movl threadinfo_flags(%rcx),%edx
916 movl $_TIF_WORK_MASK,%edi
917 andl %edi,%edx
918 jnz retint_careful
919 jmp retint_swapgs
920 CFI_ENDPROC
922 error_kernelspace:
923 incl %ebx
924 /* There are two places in the kernel that can potentially fault with
925 usergs. Handle them here. The exception handlers after
926 iret run with kernel gs again, so don't set the user space flag.
927 B stepping K8s sometimes report an truncated RIP for IRET
928 exceptions returning to compat mode. Check for these here too. */
929 leaq irq_return(%rip),%rbp
930 cmpq %rbp,RIP(%rsp)
931 je error_swapgs
932 movl %ebp,%ebp /* zero extend */
933 cmpq %rbp,RIP(%rsp)
934 je error_swapgs
935 cmpq $gs_change,RIP(%rsp)
936 je error_swapgs
937 jmp error_sti
938 KPROBE_END(error_entry)
940 /* Reload gs selector with exception handling */
941 /* edi: new selector */
942 ENTRY(load_gs_index)
943 CFI_STARTPROC
944 pushf
945 CFI_ADJUST_CFA_OFFSET 8
946 DISABLE_INTERRUPTS(CLBR_ANY | ~(CLBR_RDI))
947 SWAPGS
948 gs_change:
949 movl %edi,%gs
950 2: mfence /* workaround */
951 SWAPGS
952 popf
953 CFI_ADJUST_CFA_OFFSET -8
954 ret
955 CFI_ENDPROC
956 ENDPROC(load_gs_index)
958 .section __ex_table,"a"
959 .align 8
960 .quad gs_change,bad_gs
961 .previous
962 .section .fixup,"ax"
963 /* running with kernelgs */
964 bad_gs:
965 SWAPGS /* switch back to user gs */
966 xorl %eax,%eax
967 movl %eax,%gs
968 jmp 2b
969 .previous
971 /*
972 * Create a kernel thread.
973 *
974 * C extern interface:
975 * extern long kernel_thread(int (*fn)(void *), void * arg, unsigned long flags)
976 *
977 * asm input arguments:
978 * rdi: fn, rsi: arg, rdx: flags
979 */
980 ENTRY(kernel_thread)
981 CFI_STARTPROC
982 FAKE_STACK_FRAME $child_rip
983 SAVE_ALL
985 # rdi: flags, rsi: usp, rdx: will be &pt_regs
986 movq %rdx,%rdi
987 orq kernel_thread_flags(%rip),%rdi
988 movq $-1, %rsi
989 movq %rsp, %rdx
991 xorl %r8d,%r8d
992 xorl %r9d,%r9d
994 # clone now
995 call do_fork
996 movq %rax,RAX(%rsp)
997 xorl %edi,%edi
999 /*
1000 * It isn't worth to check for reschedule here,
1001 * so internally to the x86_64 port you can rely on kernel_thread()
1002 * not to reschedule the child before returning, this avoids the need
1003 * of hacks for example to fork off the per-CPU idle tasks.
1004 * [Hopefully no generic code relies on the reschedule -AK]
1005 */
1006 RESTORE_ALL
1007 UNFAKE_STACK_FRAME
1008 ret
1009 CFI_ENDPROC
1010 ENDPROC(kernel_thread)
1012 child_rip:
1013 pushq $0 # fake return address
1014 CFI_STARTPROC
1015 /*
1016 * Here we are in the child and the registers are set as they were
1017 * at kernel_thread() invocation in the parent.
1018 */
1019 movq %rdi, %rax
1020 movq %rsi, %rdi
1021 call *%rax
1022 # exit
1023 mov %eax, %edi
1024 call do_exit
1025 CFI_ENDPROC
1026 ENDPROC(child_rip)
1028 /*
1029 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1030 *
1031 * C extern interface:
1032 * extern long execve(char *name, char **argv, char **envp)
1033 *
1034 * asm input arguments:
1035 * rdi: name, rsi: argv, rdx: envp
1036 *
1037 * We want to fallback into:
1038 * extern long sys_execve(char *name, char **argv,char **envp, struct pt_regs *regs)
1039 *
1040 * do_sys_execve asm fallback arguments:
1041 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1042 */
1043 ENTRY(kernel_execve)
1044 CFI_STARTPROC
1045 FAKE_STACK_FRAME $0
1046 SAVE_ALL
1047 movq %rsp,%rcx
1048 call sys_execve
1049 movq %rax, RAX(%rsp)
1050 RESTORE_REST
1051 testq %rax,%rax
1052 je int_ret_from_sys_call
1053 RESTORE_ARGS
1054 UNFAKE_STACK_FRAME
1055 ret
1056 CFI_ENDPROC
1057 ENDPROC(kernel_execve)
1059 KPROBE_ENTRY(page_fault)
1060 errorentry do_page_fault
1061 KPROBE_END(page_fault)
1063 ENTRY(coprocessor_error)
1064 zeroentry do_coprocessor_error
1065 END(coprocessor_error)
1067 ENTRY(simd_coprocessor_error)
1068 zeroentry do_simd_coprocessor_error
1069 END(simd_coprocessor_error)
1071 ENTRY(device_not_available)
1072 zeroentry math_state_restore
1073 END(device_not_available)
1075 /* runs on exception stack */
1076 KPROBE_ENTRY(debug)
1077 INTR_FRAME
1078 pushq $0
1079 CFI_ADJUST_CFA_OFFSET 8
1080 paranoidentry do_debug, DEBUG_STACK
1081 paranoidexit
1082 KPROBE_END(debug)
1084 /* runs on exception stack */
1085 KPROBE_ENTRY(nmi)
1086 INTR_FRAME
1087 pushq $-1
1088 CFI_ADJUST_CFA_OFFSET 8
1089 paranoidentry do_nmi, 0, 0
1090 #ifdef CONFIG_TRACE_IRQFLAGS
1091 paranoidexit 0
1092 #else
1093 jmp paranoid_exit1
1094 CFI_ENDPROC
1095 #endif
1096 KPROBE_END(nmi)
1098 KPROBE_ENTRY(int3)
1099 INTR_FRAME
1100 pushq $0
1101 CFI_ADJUST_CFA_OFFSET 8
1102 paranoidentry do_int3, DEBUG_STACK
1103 jmp paranoid_exit1
1104 CFI_ENDPROC
1105 KPROBE_END(int3)
1107 ENTRY(overflow)
1108 zeroentry do_overflow
1109 END(overflow)
1111 ENTRY(bounds)
1112 zeroentry do_bounds
1113 END(bounds)
1115 ENTRY(invalid_op)
1116 zeroentry do_invalid_op
1117 END(invalid_op)
1119 ENTRY(coprocessor_segment_overrun)
1120 zeroentry do_coprocessor_segment_overrun
1121 END(coprocessor_segment_overrun)
1123 ENTRY(reserved)
1124 zeroentry do_reserved
1125 END(reserved)
1127 /* runs on exception stack */
1128 ENTRY(double_fault)
1129 XCPT_FRAME
1130 paranoidentry do_double_fault
1131 jmp paranoid_exit1
1132 CFI_ENDPROC
1133 END(double_fault)
1135 ENTRY(invalid_TSS)
1136 errorentry do_invalid_TSS
1137 END(invalid_TSS)
1139 ENTRY(segment_not_present)
1140 errorentry do_segment_not_present
1141 END(segment_not_present)
1143 /* runs on exception stack */
1144 ENTRY(stack_segment)
1145 XCPT_FRAME
1146 paranoidentry do_stack_segment
1147 jmp paranoid_exit1
1148 CFI_ENDPROC
1149 END(stack_segment)
1151 KPROBE_ENTRY(general_protection)
1152 errorentry do_general_protection
1153 KPROBE_END(general_protection)
1155 ENTRY(alignment_check)
1156 errorentry do_alignment_check
1157 END(alignment_check)
1159 ENTRY(divide_error)
1160 zeroentry do_divide_error
1161 END(divide_error)
1163 ENTRY(spurious_interrupt_bug)
1164 zeroentry do_spurious_interrupt_bug
1165 END(spurious_interrupt_bug)
1167 #ifdef CONFIG_X86_MCE
1168 /* runs on exception stack */
1169 ENTRY(machine_check)
1170 INTR_FRAME
1171 pushq $0
1172 CFI_ADJUST_CFA_OFFSET 8
1173 paranoidentry do_machine_check
1174 jmp paranoid_exit1
1175 CFI_ENDPROC
1176 END(machine_check)
1177 #endif
1179 /* Call softirq on interrupt stack. Interrupts are off. */
1180 ENTRY(call_softirq)
1181 CFI_STARTPROC
1182 push %rbp
1183 CFI_ADJUST_CFA_OFFSET 8
1184 CFI_REL_OFFSET rbp,0
1185 mov %rsp,%rbp
1186 CFI_DEF_CFA_REGISTER rbp
1187 incl %gs:pda_irqcount
1188 cmove %gs:pda_irqstackptr,%rsp
1189 push %rbp # backlink for old unwinder
1190 call __do_softirq
1191 leaveq
1192 CFI_DEF_CFA_REGISTER rsp
1193 CFI_ADJUST_CFA_OFFSET -8
1194 decl %gs:pda_irqcount
1195 ret
1196 CFI_ENDPROC
1197 ENDPROC(call_softirq)
1199 KPROBE_ENTRY(ignore_sysret)
1200 CFI_STARTPROC
1201 mov $-ENOSYS,%eax
1202 sysret
1203 CFI_ENDPROC
1204 ENDPROC(ignore_sysret)