| blob | c19974a493784e1cc40f75452a96179f1010625a |
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 1991,1992 Linus Torvalds
4 *
5 * entry_32.S contains the system-call and low-level fault and trap handling routines.
6 *
7 * Stack layout while running C code:
8 * ptrace needs to have all registers on the stack.
9 * If the order here is changed, it needs to be
10 * updated in fork.c:copy_process(), signal.c:do_signal(),
11 * ptrace.c and ptrace.h
12 *
13 * 0(%esp) - %ebx
14 * 4(%esp) - %ecx
15 * 8(%esp) - %edx
16 * C(%esp) - %esi
17 * 10(%esp) - %edi
18 * 14(%esp) - %ebp
19 * 18(%esp) - %eax
20 * 1C(%esp) - %ds
21 * 20(%esp) - %es
22 * 24(%esp) - %fs
23 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
24 * 2C(%esp) - orig_eax
25 * 30(%esp) - %eip
26 * 34(%esp) - %cs
27 * 38(%esp) - %eflags
28 * 3C(%esp) - %oldesp
29 * 40(%esp) - %oldss
30 */
32 #include <linux/linkage.h>
33 #include <linux/err.h>
34 #include <asm/thread_info.h>
35 #include <asm/irqflags.h>
36 #include <asm/errno.h>
37 #include <asm/segment.h>
38 #include <asm/smp.h>
39 #include <asm/percpu.h>
40 #include <asm/processor-flags.h>
41 #include <asm/irq_vectors.h>
42 #include <asm/cpufeatures.h>
43 #include <asm/alternative-asm.h>
44 #include <asm/asm.h>
45 #include <asm/smap.h>
46 #include <asm/frame.h>
47 #include <asm/nospec-branch.h>
49 .section .entry.text, "ax"
51 /*
52 * We use macros for low-level operations which need to be overridden
53 * for paravirtualization. The following will never clobber any registers:
54 * INTERRUPT_RETURN (aka. "iret")
55 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
56 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
57 *
58 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
59 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
60 * Allowing a register to be clobbered can shrink the paravirt replacement
61 * enough to patch inline, increasing performance.
62 */
64 #ifdef CONFIG_PREEMPT
65 # define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
66 #else
67 # define preempt_stop(clobbers)
68 # define resume_kernel restore_all
69 #endif
71 .macro TRACE_IRQS_IRET
72 #ifdef CONFIG_TRACE_IRQFLAGS
73 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off?
74 jz 1f
75 TRACE_IRQS_ON
76 1:
77 #endif
78 .endm
80 /*
81 * User gs save/restore
82 *
83 * %gs is used for userland TLS and kernel only uses it for stack
84 * canary which is required to be at %gs:20 by gcc. Read the comment
85 * at the top of stackprotector.h for more info.
86 *
87 * Local labels 98 and 99 are used.
88 */
89 #ifdef CONFIG_X86_32_LAZY_GS
91 /* unfortunately push/pop can't be no-op */
92 .macro PUSH_GS
93 pushl $0
94 .endm
95 .macro POP_GS pop=0
96 addl $(4 + \pop), %esp
97 .endm
98 .macro POP_GS_EX
99 .endm
101 /* all the rest are no-op */
102 .macro PTGS_TO_GS
103 .endm
104 .macro PTGS_TO_GS_EX
105 .endm
106 .macro GS_TO_REG reg
107 .endm
108 .macro REG_TO_PTGS reg
109 .endm
110 .macro SET_KERNEL_GS reg
111 .endm
113 #else /* CONFIG_X86_32_LAZY_GS */
115 .macro PUSH_GS
116 pushl %gs
117 .endm
119 .macro POP_GS pop=0
120 98: popl %gs
121 .if \pop <> 0
122 add $\pop, %esp
123 .endif
124 .endm
125 .macro POP_GS_EX
126 .pushsection .fixup, "ax"
127 99: movl $0, (%esp)
128 jmp 98b
129 .popsection
130 _ASM_EXTABLE(98b, 99b)
131 .endm
133 .macro PTGS_TO_GS
134 98: mov PT_GS(%esp), %gs
135 .endm
136 .macro PTGS_TO_GS_EX
137 .pushsection .fixup, "ax"
138 99: movl $0, PT_GS(%esp)
139 jmp 98b
140 .popsection
141 _ASM_EXTABLE(98b, 99b)
142 .endm
144 .macro GS_TO_REG reg
145 movl %gs, \reg
146 .endm
147 .macro REG_TO_PTGS reg
148 movl \reg, PT_GS(%esp)
149 .endm
150 .macro SET_KERNEL_GS reg
151 movl $(__KERNEL_STACK_CANARY), \reg
152 movl \reg, %gs
153 .endm
155 #endif /* CONFIG_X86_32_LAZY_GS */
157 .macro SAVE_ALL pt_regs_ax=%eax
158 cld
159 PUSH_GS
160 pushl %fs
161 pushl %es
162 pushl %ds
163 pushl \pt_regs_ax
164 pushl %ebp
165 pushl %edi
166 pushl %esi
167 pushl %edx
168 pushl %ecx
169 pushl %ebx
170 movl $(__USER_DS), %edx
171 movl %edx, %ds
172 movl %edx, %es
173 movl $(__KERNEL_PERCPU), %edx
174 movl %edx, %fs
175 SET_KERNEL_GS %edx
176 .endm
178 /*
179 * This is a sneaky trick to help the unwinder find pt_regs on the stack. The
180 * frame pointer is replaced with an encoded pointer to pt_regs. The encoding
181 * is just clearing the MSB, which makes it an invalid stack address and is also
182 * a signal to the unwinder that it's a pt_regs pointer in disguise.
183 *
184 * NOTE: This macro must be used *after* SAVE_ALL because it corrupts the
185 * original rbp.
186 */
187 .macro ENCODE_FRAME_POINTER
188 #ifdef CONFIG_FRAME_POINTER
189 mov %esp, %ebp
190 andl $0x7fffffff, %ebp
191 #endif
192 .endm
194 .macro RESTORE_INT_REGS
195 popl %ebx
196 popl %ecx
197 popl %edx
198 popl %esi
199 popl %edi
200 popl %ebp
201 popl %eax
202 .endm
204 .macro RESTORE_REGS pop=0
205 RESTORE_INT_REGS
206 1: popl %ds
207 2: popl %es
208 3: popl %fs
209 POP_GS \pop
210 .pushsection .fixup, "ax"
211 4: movl $0, (%esp)
212 jmp 1b
213 5: movl $0, (%esp)
214 jmp 2b
215 6: movl $0, (%esp)
216 jmp 3b
217 .popsection
218 _ASM_EXTABLE(1b, 4b)
219 _ASM_EXTABLE(2b, 5b)
220 _ASM_EXTABLE(3b, 6b)
221 POP_GS_EX
222 .endm
224 /*
225 * %eax: prev task
226 * %edx: next task
227 */
228 ENTRY(__switch_to_asm)
229 /*
230 * Save callee-saved registers
231 * This must match the order in struct inactive_task_frame
232 */
233 pushl %ebp
234 pushl %ebx
235 pushl %edi
236 pushl %esi
237 pushfl
239 /* switch stack */
240 movl %esp, TASK_threadsp(%eax)
241 movl TASK_threadsp(%edx), %esp
243 #ifdef CONFIG_CC_STACKPROTECTOR
244 movl TASK_stack_canary(%edx), %ebx
245 movl %ebx, PER_CPU_VAR(stack_canary)+stack_canary_offset
246 #endif
248 #ifdef CONFIG_RETPOLINE
249 /*
250 * When switching from a shallower to a deeper call stack
251 * the RSB may either underflow or use entries populated
252 * with userspace addresses. On CPUs where those concerns
253 * exist, overwrite the RSB with entries which capture
254 * speculative execution to prevent attack.
255 */
256 FILL_RETURN_BUFFER %ebx, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
257 #endif
259 /* restore callee-saved registers */
260 popfl
261 popl %esi
262 popl %edi
263 popl %ebx
264 popl %ebp
266 jmp __switch_to
267 END(__switch_to_asm)
269 /*
270 * The unwinder expects the last frame on the stack to always be at the same
271 * offset from the end of the page, which allows it to validate the stack.
272 * Calling schedule_tail() directly would break that convention because its an
273 * asmlinkage function so its argument has to be pushed on the stack. This
274 * wrapper creates a proper "end of stack" frame header before the call.
275 */
276 ENTRY(schedule_tail_wrapper)
277 FRAME_BEGIN
279 pushl %eax
280 call schedule_tail
281 popl %eax
283 FRAME_END
284 ret
285 ENDPROC(schedule_tail_wrapper)
286 /*
287 * A newly forked process directly context switches into this address.
288 *
289 * eax: prev task we switched from
290 * ebx: kernel thread func (NULL for user thread)
291 * edi: kernel thread arg
292 */
293 ENTRY(ret_from_fork)
294 call schedule_tail_wrapper
296 testl %ebx, %ebx
297 jnz 1f /* kernel threads are uncommon */
299 2:
300 /* When we fork, we trace the syscall return in the child, too. */
301 movl %esp, %eax
302 call syscall_return_slowpath
303 jmp restore_all
305 /* kernel thread */
306 1: movl %edi, %eax
307 CALL_NOSPEC %ebx
308 /*
309 * A kernel thread is allowed to return here after successfully
310 * calling do_execve(). Exit to userspace to complete the execve()
311 * syscall.
312 */
313 movl $0, PT_EAX(%esp)
314 jmp 2b
315 END(ret_from_fork)
317 /*
318 * Return to user mode is not as complex as all this looks,
319 * but we want the default path for a system call return to
320 * go as quickly as possible which is why some of this is
321 * less clear than it otherwise should be.
322 */
324 # userspace resumption stub bypassing syscall exit tracing
325 ALIGN
326 ret_from_exception:
327 preempt_stop(CLBR_ANY)
328 ret_from_intr:
329 #ifdef CONFIG_VM86
330 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
331 movb PT_CS(%esp), %al
332 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
333 #else
334 /*
335 * We can be coming here from child spawned by kernel_thread().
336 */
337 movl PT_CS(%esp), %eax
338 andl $SEGMENT_RPL_MASK, %eax
339 #endif
340 cmpl $USER_RPL, %eax
341 jb resume_kernel # not returning to v8086 or userspace
343 ENTRY(resume_userspace)
344 DISABLE_INTERRUPTS(CLBR_ANY)
345 TRACE_IRQS_OFF
346 movl %esp, %eax
347 call prepare_exit_to_usermode
348 jmp restore_all
349 END(ret_from_exception)
351 #ifdef CONFIG_PREEMPT
352 ENTRY(resume_kernel)
353 DISABLE_INTERRUPTS(CLBR_ANY)
354 .Lneed_resched:
355 cmpl $0, PER_CPU_VAR(__preempt_count)
356 jnz restore_all
357 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
358 jz restore_all
359 call preempt_schedule_irq
360 jmp .Lneed_resched
361 END(resume_kernel)
362 #endif
364 GLOBAL(__begin_SYSENTER_singlestep_region)
365 /*
366 * All code from here through __end_SYSENTER_singlestep_region is subject
367 * to being single-stepped if a user program sets TF and executes SYSENTER.
368 * There is absolutely nothing that we can do to prevent this from happening
369 * (thanks Intel!). To keep our handling of this situation as simple as
370 * possible, we handle TF just like AC and NT, except that our #DB handler
371 * will ignore all of the single-step traps generated in this range.
372 */
374 #ifdef CONFIG_XEN
375 /*
376 * Xen doesn't set %esp to be precisely what the normal SYSENTER
377 * entry point expects, so fix it up before using the normal path.
378 */
379 ENTRY(xen_sysenter_target)
380 addl $5*4, %esp /* remove xen-provided frame */
381 jmp .Lsysenter_past_esp
382 #endif
384 /*
385 * 32-bit SYSENTER entry.
386 *
387 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
388 * if X86_FEATURE_SEP is available. This is the preferred system call
389 * entry on 32-bit systems.
390 *
391 * The SYSENTER instruction, in principle, should *only* occur in the
392 * vDSO. In practice, a small number of Android devices were shipped
393 * with a copy of Bionic that inlined a SYSENTER instruction. This
394 * never happened in any of Google's Bionic versions -- it only happened
395 * in a narrow range of Intel-provided versions.
396 *
397 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
398 * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
399 * SYSENTER does not save anything on the stack,
400 * and does not save old EIP (!!!), ESP, or EFLAGS.
401 *
402 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
403 * user and/or vm86 state), we explicitly disable the SYSENTER
404 * instruction in vm86 mode by reprogramming the MSRs.
405 *
406 * Arguments:
407 * eax system call number
408 * ebx arg1
409 * ecx arg2
410 * edx arg3
411 * esi arg4
412 * edi arg5
413 * ebp user stack
414 * 0(%ebp) arg6
415 */
416 ENTRY(entry_SYSENTER_32)
417 movl TSS_sysenter_sp0(%esp), %esp
418 .Lsysenter_past_esp:
419 pushl $__USER_DS /* pt_regs->ss */
420 pushl %ebp /* pt_regs->sp (stashed in bp) */
421 pushfl /* pt_regs->flags (except IF = 0) */
422 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */
423 pushl $__USER_CS /* pt_regs->cs */
424 pushl $0 /* pt_regs->ip = 0 (placeholder) */
425 pushl %eax /* pt_regs->orig_ax */
426 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
428 /*
429 * SYSENTER doesn't filter flags, so we need to clear NT, AC
430 * and TF ourselves. To save a few cycles, we can check whether
431 * either was set instead of doing an unconditional popfq.
432 * This needs to happen before enabling interrupts so that
433 * we don't get preempted with NT set.
434 *
435 * If TF is set, we will single-step all the way to here -- do_debug
436 * will ignore all the traps. (Yes, this is slow, but so is
437 * single-stepping in general. This allows us to avoid having
438 * a more complicated code to handle the case where a user program
439 * forces us to single-step through the SYSENTER entry code.)
440 *
441 * NB.: .Lsysenter_fix_flags is a label with the code under it moved
442 * out-of-line as an optimization: NT is unlikely to be set in the
443 * majority of the cases and instead of polluting the I$ unnecessarily,
444 * we're keeping that code behind a branch which will predict as
445 * not-taken and therefore its instructions won't be fetched.
446 */
447 testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
448 jnz .Lsysenter_fix_flags
449 .Lsysenter_flags_fixed:
451 /*
452 * User mode is traced as though IRQs are on, and SYSENTER
453 * turned them off.
454 */
455 TRACE_IRQS_OFF
457 movl %esp, %eax
458 call do_fast_syscall_32
459 /* XEN PV guests always use IRET path */
460 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
461 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
463 /* Opportunistic SYSEXIT */
464 TRACE_IRQS_ON /* User mode traces as IRQs on. */
465 movl PT_EIP(%esp), %edx /* pt_regs->ip */
466 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
467 1: mov PT_FS(%esp), %fs
468 PTGS_TO_GS
469 popl %ebx /* pt_regs->bx */
470 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
471 popl %esi /* pt_regs->si */
472 popl %edi /* pt_regs->di */
473 popl %ebp /* pt_regs->bp */
474 popl %eax /* pt_regs->ax */
476 /*
477 * Restore all flags except IF. (We restore IF separately because
478 * STI gives a one-instruction window in which we won't be interrupted,
479 * whereas POPF does not.)
480 */
481 addl $PT_EFLAGS-PT_DS, %esp /* point esp at pt_regs->flags */
482 btr $X86_EFLAGS_IF_BIT, (%esp)
483 popfl
485 /*
486 * Return back to the vDSO, which will pop ecx and edx.
487 * Don't bother with DS and ES (they already contain __USER_DS).
488 */
489 sti
490 sysexit
492 .pushsection .fixup, "ax"
493 2: movl $0, PT_FS(%esp)
494 jmp 1b
495 .popsection
496 _ASM_EXTABLE(1b, 2b)
497 PTGS_TO_GS_EX
499 .Lsysenter_fix_flags:
500 pushl $X86_EFLAGS_FIXED
501 popfl
502 jmp .Lsysenter_flags_fixed
503 GLOBAL(__end_SYSENTER_singlestep_region)
504 ENDPROC(entry_SYSENTER_32)
506 /*
507 * 32-bit legacy system call entry.
508 *
509 * 32-bit x86 Linux system calls traditionally used the INT $0x80
510 * instruction. INT $0x80 lands here.
511 *
512 * This entry point can be used by any 32-bit perform system calls.
513 * Instances of INT $0x80 can be found inline in various programs and
514 * libraries. It is also used by the vDSO's __kernel_vsyscall
515 * fallback for hardware that doesn't support a faster entry method.
516 * Restarted 32-bit system calls also fall back to INT $0x80
517 * regardless of what instruction was originally used to do the system
518 * call. (64-bit programs can use INT $0x80 as well, but they can
519 * only run on 64-bit kernels and therefore land in
520 * entry_INT80_compat.)
521 *
522 * This is considered a slow path. It is not used by most libc
523 * implementations on modern hardware except during process startup.
524 *
525 * Arguments:
526 * eax system call number
527 * ebx arg1
528 * ecx arg2
529 * edx arg3
530 * esi arg4
531 * edi arg5
532 * ebp arg6
533 */
534 ENTRY(entry_INT80_32)
535 ASM_CLAC
536 pushl %eax /* pt_regs->orig_ax */
537 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
539 /*
540 * User mode is traced as though IRQs are on, and the interrupt gate
541 * turned them off.
542 */
543 TRACE_IRQS_OFF
545 movl %esp, %eax
546 call do_int80_syscall_32
547 .Lsyscall_32_done:
549 restore_all:
550 TRACE_IRQS_IRET
551 .Lrestore_all_notrace:
552 #ifdef CONFIG_X86_ESPFIX32
553 ALTERNATIVE "jmp .Lrestore_nocheck", "", X86_BUG_ESPFIX
555 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
556 /*
557 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
558 * are returning to the kernel.
559 * See comments in process.c:copy_thread() for details.
560 */
561 movb PT_OLDSS(%esp), %ah
562 movb PT_CS(%esp), %al
563 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
564 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
565 je .Lldt_ss # returning to user-space with LDT SS
566 #endif
567 .Lrestore_nocheck:
568 RESTORE_REGS 4 # skip orig_eax/error_code
569 .Lirq_return:
570 INTERRUPT_RETURN
572 .section .fixup, "ax"
573 ENTRY(iret_exc )
574 pushl $0 # no error code
575 pushl $do_iret_error
576 jmp common_exception
577 .previous
578 _ASM_EXTABLE(.Lirq_return, iret_exc)
580 #ifdef CONFIG_X86_ESPFIX32
581 .Lldt_ss:
582 /*
583 * Setup and switch to ESPFIX stack
584 *
585 * We're returning to userspace with a 16 bit stack. The CPU will not
586 * restore the high word of ESP for us on executing iret... This is an
587 * "official" bug of all the x86-compatible CPUs, which we can work
588 * around to make dosemu and wine happy. We do this by preloading the
589 * high word of ESP with the high word of the userspace ESP while
590 * compensating for the offset by changing to the ESPFIX segment with
591 * a base address that matches for the difference.
592 */
593 #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
594 mov %esp, %edx /* load kernel esp */
595 mov PT_OLDESP(%esp), %eax /* load userspace esp */
596 mov %dx, %ax /* eax: new kernel esp */
597 sub %eax, %edx /* offset (low word is 0) */
598 shr $16, %edx
599 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
600 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
601 pushl $__ESPFIX_SS
602 pushl %eax /* new kernel esp */
603 /*
604 * Disable interrupts, but do not irqtrace this section: we
605 * will soon execute iret and the tracer was already set to
606 * the irqstate after the IRET:
607 */
608 DISABLE_INTERRUPTS(CLBR_ANY)
609 lss (%esp), %esp /* switch to espfix segment */
610 jmp .Lrestore_nocheck
611 #endif
612 ENDPROC(entry_INT80_32)
614 .macro FIXUP_ESPFIX_STACK
615 /*
616 * Switch back for ESPFIX stack to the normal zerobased stack
617 *
618 * We can't call C functions using the ESPFIX stack. This code reads
619 * the high word of the segment base from the GDT and swiches to the
620 * normal stack and adjusts ESP with the matching offset.
621 */
622 #ifdef CONFIG_X86_ESPFIX32
623 /* fixup the stack */
624 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
625 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
626 shl $16, %eax
627 addl %esp, %eax /* the adjusted stack pointer */
628 pushl $__KERNEL_DS
629 pushl %eax
630 lss (%esp), %esp /* switch to the normal stack segment */
631 #endif
632 .endm
633 .macro UNWIND_ESPFIX_STACK
634 #ifdef CONFIG_X86_ESPFIX32
635 movl %ss, %eax
636 /* see if on espfix stack */
637 cmpw $__ESPFIX_SS, %ax
638 jne 27f
639 movl $__KERNEL_DS, %eax
640 movl %eax, %ds
641 movl %eax, %es
642 /* switch to normal stack */
643 FIXUP_ESPFIX_STACK
644 27:
645 #endif
646 .endm
648 /*
649 * Build the entry stubs with some assembler magic.
650 * We pack 1 stub into every 8-byte block.
651 */
652 .align 8
653 ENTRY(irq_entries_start)
654 vector=FIRST_EXTERNAL_VECTOR
655 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
656 pushl $(~vector+0x80) /* Note: always in signed byte range */
657 vector=vector+1
658 jmp common_interrupt
659 .align 8
660 .endr
661 END(irq_entries_start)
663 /*
664 * the CPU automatically disables interrupts when executing an IRQ vector,
665 * so IRQ-flags tracing has to follow that:
666 */
667 .p2align CONFIG_X86_L1_CACHE_SHIFT
668 common_interrupt:
669 ASM_CLAC
670 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */
671 SAVE_ALL
672 ENCODE_FRAME_POINTER
673 TRACE_IRQS_OFF
674 movl %esp, %eax
675 call do_IRQ
676 jmp ret_from_intr
677 ENDPROC(common_interrupt)
679 #define BUILD_INTERRUPT3(name, nr, fn) \
680 ENTRY(name) \
681 ASM_CLAC; \
682 pushl $~(nr); \
683 SAVE_ALL; \
684 ENCODE_FRAME_POINTER; \
685 TRACE_IRQS_OFF \
686 movl %esp, %eax; \
687 call fn; \
688 jmp ret_from_intr; \
689 ENDPROC(name)
691 #define BUILD_INTERRUPT(name, nr) \
692 BUILD_INTERRUPT3(name, nr, smp_##name); \
694 /* The include is where all of the SMP etc. interrupts come from */
695 #include <asm/entry_arch.h>
697 ENTRY(coprocessor_error)
698 ASM_CLAC
699 pushl $0
700 pushl $do_coprocessor_error
701 jmp common_exception
702 END(coprocessor_error)
704 ENTRY(simd_coprocessor_error)
705 ASM_CLAC
706 pushl $0
707 #ifdef CONFIG_X86_INVD_BUG
708 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
709 ALTERNATIVE "pushl $do_general_protection", \
710 "pushl $do_simd_coprocessor_error", \
711 X86_FEATURE_XMM
712 #else
713 pushl $do_simd_coprocessor_error
714 #endif
715 jmp common_exception
716 END(simd_coprocessor_error)
718 ENTRY(device_not_available)
719 ASM_CLAC
720 pushl $-1 # mark this as an int
721 pushl $do_device_not_available
722 jmp common_exception
723 END(device_not_available)
725 #ifdef CONFIG_PARAVIRT
726 ENTRY(native_iret)
727 iret
728 _ASM_EXTABLE(native_iret, iret_exc)
729 END(native_iret)
730 #endif
732 ENTRY(overflow)
733 ASM_CLAC
734 pushl $0
735 pushl $do_overflow
736 jmp common_exception
737 END(overflow)
739 ENTRY(bounds)
740 ASM_CLAC
741 pushl $0
742 pushl $do_bounds
743 jmp common_exception
744 END(bounds)
746 ENTRY(invalid_op)
747 ASM_CLAC
748 pushl $0
749 pushl $do_invalid_op
750 jmp common_exception
751 END(invalid_op)
753 ENTRY(coprocessor_segment_overrun)
754 ASM_CLAC
755 pushl $0
756 pushl $do_coprocessor_segment_overrun
757 jmp common_exception
758 END(coprocessor_segment_overrun)
760 ENTRY(invalid_TSS)
761 ASM_CLAC
762 pushl $do_invalid_TSS
763 jmp common_exception
764 END(invalid_TSS)
766 ENTRY(segment_not_present)
767 ASM_CLAC
768 pushl $do_segment_not_present
769 jmp common_exception
770 END(segment_not_present)
772 ENTRY(stack_segment)
773 ASM_CLAC
774 pushl $do_stack_segment
775 jmp common_exception
776 END(stack_segment)
778 ENTRY(alignment_check)
779 ASM_CLAC
780 pushl $do_alignment_check
781 jmp common_exception
782 END(alignment_check)
784 ENTRY(divide_error)
785 ASM_CLAC
786 pushl $0 # no error code
787 pushl $do_divide_error
788 jmp common_exception
789 END(divide_error)
791 #ifdef CONFIG_X86_MCE
792 ENTRY(machine_check)
793 ASM_CLAC
794 pushl $0
795 pushl machine_check_vector
796 jmp common_exception
797 END(machine_check)
798 #endif
800 ENTRY(spurious_interrupt_bug)
801 ASM_CLAC
802 pushl $0
803 pushl $do_spurious_interrupt_bug
804 jmp common_exception
805 END(spurious_interrupt_bug)
807 #ifdef CONFIG_XEN
808 ENTRY(xen_hypervisor_callback)
809 pushl $-1 /* orig_ax = -1 => not a system call */
810 SAVE_ALL
811 ENCODE_FRAME_POINTER
812 TRACE_IRQS_OFF
814 /*
815 * Check to see if we got the event in the critical
816 * region in xen_iret_direct, after we've reenabled
817 * events and checked for pending events. This simulates
818 * iret instruction's behaviour where it delivers a
819 * pending interrupt when enabling interrupts:
820 */
821 movl PT_EIP(%esp), %eax
822 cmpl $xen_iret_start_crit, %eax
823 jb 1f
824 cmpl $xen_iret_end_crit, %eax
825 jae 1f
827 jmp xen_iret_crit_fixup
829 ENTRY(xen_do_upcall)
830 1: mov %esp, %eax
831 call xen_evtchn_do_upcall
832 #ifndef CONFIG_PREEMPT
833 call xen_maybe_preempt_hcall
834 #endif
835 jmp ret_from_intr
836 ENDPROC(xen_hypervisor_callback)
838 /*
839 * Hypervisor uses this for application faults while it executes.
840 * We get here for two reasons:
841 * 1. Fault while reloading DS, ES, FS or GS
842 * 2. Fault while executing IRET
843 * Category 1 we fix up by reattempting the load, and zeroing the segment
844 * register if the load fails.
845 * Category 2 we fix up by jumping to do_iret_error. We cannot use the
846 * normal Linux return path in this case because if we use the IRET hypercall
847 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
848 * We distinguish between categories by maintaining a status value in EAX.
849 */
850 ENTRY(xen_failsafe_callback)
851 pushl %eax
852 movl $1, %eax
853 1: mov 4(%esp), %ds
854 2: mov 8(%esp), %es
855 3: mov 12(%esp), %fs
856 4: mov 16(%esp), %gs
857 /* EAX == 0 => Category 1 (Bad segment)
858 EAX != 0 => Category 2 (Bad IRET) */
859 testl %eax, %eax
860 popl %eax
861 lea 16(%esp), %esp
862 jz 5f
863 jmp iret_exc
864 5: pushl $-1 /* orig_ax = -1 => not a system call */
865 SAVE_ALL
866 ENCODE_FRAME_POINTER
867 jmp ret_from_exception
869 .section .fixup, "ax"
870 6: xorl %eax, %eax
871 movl %eax, 4(%esp)
872 jmp 1b
873 7: xorl %eax, %eax
874 movl %eax, 8(%esp)
875 jmp 2b
876 8: xorl %eax, %eax
877 movl %eax, 12(%esp)
878 jmp 3b
879 9: xorl %eax, %eax
880 movl %eax, 16(%esp)
881 jmp 4b
882 .previous
883 _ASM_EXTABLE(1b, 6b)
884 _ASM_EXTABLE(2b, 7b)
885 _ASM_EXTABLE(3b, 8b)
886 _ASM_EXTABLE(4b, 9b)
887 ENDPROC(xen_failsafe_callback)
889 BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
890 xen_evtchn_do_upcall)
892 #endif /* CONFIG_XEN */
894 #if IS_ENABLED(CONFIG_HYPERV)
896 BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
897 hyperv_vector_handler)
899 #endif /* CONFIG_HYPERV */
901 ENTRY(page_fault)
902 ASM_CLAC
903 pushl $do_page_fault
904 ALIGN
905 jmp common_exception
906 END(page_fault)
908 common_exception:
909 /* the function address is in %gs's slot on the stack */
910 pushl %fs
911 pushl %es
912 pushl %ds
913 pushl %eax
914 pushl %ebp
915 pushl %edi
916 pushl %esi
917 pushl %edx
918 pushl %ecx
919 pushl %ebx
920 ENCODE_FRAME_POINTER
921 cld
922 movl $(__KERNEL_PERCPU), %ecx
923 movl %ecx, %fs
924 UNWIND_ESPFIX_STACK
925 GS_TO_REG %ecx
926 movl PT_GS(%esp), %edi # get the function address
927 movl PT_ORIG_EAX(%esp), %edx # get the error code
928 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
929 REG_TO_PTGS %ecx
930 SET_KERNEL_GS %ecx
931 movl $(__USER_DS), %ecx
932 movl %ecx, %ds
933 movl %ecx, %es
934 TRACE_IRQS_OFF
935 movl %esp, %eax # pt_regs pointer
936 CALL_NOSPEC %edi
937 jmp ret_from_exception
938 END(common_exception)
940 ENTRY(debug)
941 /*
942 * #DB can happen at the first instruction of
943 * entry_SYSENTER_32 or in Xen's SYSENTER prologue. If this
944 * happens, then we will be running on a very small stack. We
945 * need to detect this condition and switch to the thread
946 * stack before calling any C code at all.
947 *
948 * If you edit this code, keep in mind that NMIs can happen in here.
949 */
950 ASM_CLAC
951 pushl $-1 # mark this as an int
952 SAVE_ALL
953 ENCODE_FRAME_POINTER
954 xorl %edx, %edx # error code 0
955 movl %esp, %eax # pt_regs pointer
957 /* Are we currently on the SYSENTER stack? */
958 movl PER_CPU_VAR(cpu_entry_area), %ecx
959 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
960 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */
961 cmpl $SIZEOF_entry_stack, %ecx
962 jb .Ldebug_from_sysenter_stack
964 TRACE_IRQS_OFF
965 call do_debug
966 jmp ret_from_exception
968 .Ldebug_from_sysenter_stack:
969 /* We're on the SYSENTER stack. Switch off. */
970 movl %esp, %ebx
971 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
972 TRACE_IRQS_OFF
973 call do_debug
974 movl %ebx, %esp
975 jmp ret_from_exception
976 END(debug)
978 /*
979 * NMI is doubly nasty. It can happen on the first instruction of
980 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
981 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
982 * switched stacks. We handle both conditions by simply checking whether we
983 * interrupted kernel code running on the SYSENTER stack.
984 */
985 ENTRY(nmi)
986 ASM_CLAC
987 #ifdef CONFIG_X86_ESPFIX32
988 pushl %eax
989 movl %ss, %eax
990 cmpw $__ESPFIX_SS, %ax
991 popl %eax
992 je .Lnmi_espfix_stack
993 #endif
995 pushl %eax # pt_regs->orig_ax
996 SAVE_ALL
997 ENCODE_FRAME_POINTER
998 xorl %edx, %edx # zero error code
999 movl %esp, %eax # pt_regs pointer
1001 /* Are we currently on the SYSENTER stack? */
1002 movl PER_CPU_VAR(cpu_entry_area), %ecx
1003 addl $CPU_ENTRY_AREA_entry_stack + SIZEOF_entry_stack, %ecx
1004 subl %eax, %ecx /* ecx = (end of entry_stack) - esp */
1005 cmpl $SIZEOF_entry_stack, %ecx
1006 jb .Lnmi_from_sysenter_stack
1008 /* Not on SYSENTER stack. */
1009 call do_nmi
1010 jmp .Lrestore_all_notrace
1012 .Lnmi_from_sysenter_stack:
1013 /*
1014 * We're on the SYSENTER stack. Switch off. No one (not even debug)
1015 * is using the thread stack right now, so it's safe for us to use it.
1016 */
1017 movl %esp, %ebx
1018 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
1019 call do_nmi
1020 movl %ebx, %esp
1021 jmp .Lrestore_all_notrace
1023 #ifdef CONFIG_X86_ESPFIX32
1024 .Lnmi_espfix_stack:
1025 /*
1026 * create the pointer to lss back
1027 */
1028 pushl %ss
1029 pushl %esp
1030 addl $4, (%esp)
1031 /* copy the iret frame of 12 bytes */
1032 .rept 3
1033 pushl 16(%esp)
1034 .endr
1035 pushl %eax
1036 SAVE_ALL
1037 ENCODE_FRAME_POINTER
1038 FIXUP_ESPFIX_STACK # %eax == %esp
1039 xorl %edx, %edx # zero error code
1040 call do_nmi
1041 RESTORE_REGS
1042 lss 12+4(%esp), %esp # back to espfix stack
1043 jmp .Lirq_return
1044 #endif
1045 END(nmi)
1047 ENTRY(int3)
1048 ASM_CLAC
1049 pushl $-1 # mark this as an int
1050 SAVE_ALL
1051 ENCODE_FRAME_POINTER
1052 TRACE_IRQS_OFF
1053 xorl %edx, %edx # zero error code
1054 movl %esp, %eax # pt_regs pointer
1055 call do_int3
1056 jmp ret_from_exception
1057 END(int3)
1059 ENTRY(general_protection)
1060 ASM_CLAC
1061 pushl $do_general_protection
1062 jmp common_exception
1063 END(general_protection)
1065 #ifdef CONFIG_KVM_GUEST
1066 ENTRY(async_page_fault)
1067 ASM_CLAC
1068 pushl $do_async_page_fault
1069 jmp common_exception
1070 END(async_page_fault)
1071 #endif
1073 ENTRY(rewind_stack_do_exit)
1074 /* Prevent any naive code from trying to unwind to our caller. */
1075 xorl %ebp, %ebp
1077 movl PER_CPU_VAR(cpu_current_top_of_stack), %esi
1078 leal -TOP_OF_KERNEL_STACK_PADDING-PTREGS_SIZE(%esi), %esp
1080 call do_exit
1081 1: jmp 1b
1082 END(rewind_stack_do_exit)