| blob | 10868aa734dc07e9a438bcdd3d4c7b4aae0106d3 |
1 /*
2 * Copyright (C) 1991,1992 Linus Torvalds
3 *
4 * entry_32.S contains the system-call and low-level fault and trap handling routines.
5 *
6 * Stack layout while running C code:
7 * ptrace needs to have all registers on the stack.
8 * If the order here is changed, it needs to be
9 * updated in fork.c:copy_process(), signal.c:do_signal(),
10 * ptrace.c and ptrace.h
11 *
12 * 0(%esp) - %ebx
13 * 4(%esp) - %ecx
14 * 8(%esp) - %edx
15 * C(%esp) - %esi
16 * 10(%esp) - %edi
17 * 14(%esp) - %ebp
18 * 18(%esp) - %eax
19 * 1C(%esp) - %ds
20 * 20(%esp) - %es
21 * 24(%esp) - %fs
22 * 28(%esp) - %gs saved iff !CONFIG_X86_32_LAZY_GS
23 * 2C(%esp) - orig_eax
24 * 30(%esp) - %eip
25 * 34(%esp) - %cs
26 * 38(%esp) - %eflags
27 * 3C(%esp) - %oldesp
28 * 40(%esp) - %oldss
29 */
31 #include <linux/linkage.h>
32 #include <linux/err.h>
33 #include <asm/thread_info.h>
34 #include <asm/irqflags.h>
35 #include <asm/errno.h>
36 #include <asm/segment.h>
37 #include <asm/smp.h>
38 #include <asm/page_types.h>
39 #include <asm/percpu.h>
40 #include <asm/processor-flags.h>
41 #include <asm/ftrace.h>
42 #include <asm/irq_vectors.h>
43 #include <asm/cpufeatures.h>
44 #include <asm/alternative-asm.h>
45 #include <asm/asm.h>
46 #include <asm/smap.h>
48 .section .entry.text, "ax"
50 /*
51 * We use macros for low-level operations which need to be overridden
52 * for paravirtualization. The following will never clobber any registers:
53 * INTERRUPT_RETURN (aka. "iret")
54 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
55 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
56 *
57 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
58 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
59 * Allowing a register to be clobbered can shrink the paravirt replacement
60 * enough to patch inline, increasing performance.
61 */
63 #ifdef CONFIG_PREEMPT
64 # define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
65 #else
66 # define preempt_stop(clobbers)
67 # define resume_kernel restore_all
68 #endif
70 .macro TRACE_IRQS_IRET
71 #ifdef CONFIG_TRACE_IRQFLAGS
72 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off?
73 jz 1f
74 TRACE_IRQS_ON
75 1:
76 #endif
77 .endm
79 /*
80 * User gs save/restore
81 *
82 * %gs is used for userland TLS and kernel only uses it for stack
83 * canary which is required to be at %gs:20 by gcc. Read the comment
84 * at the top of stackprotector.h for more info.
85 *
86 * Local labels 98 and 99 are used.
87 */
88 #ifdef CONFIG_X86_32_LAZY_GS
90 /* unfortunately push/pop can't be no-op */
91 .macro PUSH_GS
92 pushl $0
93 .endm
94 .macro POP_GS pop=0
95 addl $(4 + \pop), %esp
96 .endm
97 .macro POP_GS_EX
98 .endm
100 /* all the rest are no-op */
101 .macro PTGS_TO_GS
102 .endm
103 .macro PTGS_TO_GS_EX
104 .endm
105 .macro GS_TO_REG reg
106 .endm
107 .macro REG_TO_PTGS reg
108 .endm
109 .macro SET_KERNEL_GS reg
110 .endm
112 #else /* CONFIG_X86_32_LAZY_GS */
114 .macro PUSH_GS
115 pushl %gs
116 .endm
118 .macro POP_GS pop=0
119 98: popl %gs
120 .if \pop <> 0
121 add $\pop, %esp
122 .endif
123 .endm
124 .macro POP_GS_EX
125 .pushsection .fixup, "ax"
126 99: movl $0, (%esp)
127 jmp 98b
128 .popsection
129 _ASM_EXTABLE(98b, 99b)
130 .endm
132 .macro PTGS_TO_GS
133 98: mov PT_GS(%esp), %gs
134 .endm
135 .macro PTGS_TO_GS_EX
136 .pushsection .fixup, "ax"
137 99: movl $0, PT_GS(%esp)
138 jmp 98b
139 .popsection
140 _ASM_EXTABLE(98b, 99b)
141 .endm
143 .macro GS_TO_REG reg
144 movl %gs, \reg
145 .endm
146 .macro REG_TO_PTGS reg
147 movl \reg, PT_GS(%esp)
148 .endm
149 .macro SET_KERNEL_GS reg
150 movl $(__KERNEL_STACK_CANARY), \reg
151 movl \reg, %gs
152 .endm
154 #endif /* CONFIG_X86_32_LAZY_GS */
156 .macro SAVE_ALL pt_regs_ax=%eax
157 cld
158 PUSH_GS
159 pushl %fs
160 pushl %es
161 pushl %ds
162 pushl \pt_regs_ax
163 pushl %ebp
164 pushl %edi
165 pushl %esi
166 pushl %edx
167 pushl %ecx
168 pushl %ebx
169 movl $(__USER_DS), %edx
170 movl %edx, %ds
171 movl %edx, %es
172 movl $(__KERNEL_PERCPU), %edx
173 movl %edx, %fs
174 SET_KERNEL_GS %edx
175 .endm
177 .macro RESTORE_INT_REGS
178 popl %ebx
179 popl %ecx
180 popl %edx
181 popl %esi
182 popl %edi
183 popl %ebp
184 popl %eax
185 .endm
187 .macro RESTORE_REGS pop=0
188 RESTORE_INT_REGS
189 1: popl %ds
190 2: popl %es
191 3: popl %fs
192 POP_GS \pop
193 .pushsection .fixup, "ax"
194 4: movl $0, (%esp)
195 jmp 1b
196 5: movl $0, (%esp)
197 jmp 2b
198 6: movl $0, (%esp)
199 jmp 3b
200 .popsection
201 _ASM_EXTABLE(1b, 4b)
202 _ASM_EXTABLE(2b, 5b)
203 _ASM_EXTABLE(3b, 6b)
204 POP_GS_EX
205 .endm
207 ENTRY(ret_from_fork)
208 pushl %eax
209 call schedule_tail
210 GET_THREAD_INFO(%ebp)
211 popl %eax
212 pushl $0x0202 # Reset kernel eflags
213 popfl
215 /* When we fork, we trace the syscall return in the child, too. */
216 movl %esp, %eax
217 call syscall_return_slowpath
218 jmp restore_all
219 END(ret_from_fork)
221 ENTRY(ret_from_kernel_thread)
222 pushl %eax
223 call schedule_tail
224 GET_THREAD_INFO(%ebp)
225 popl %eax
226 pushl $0x0202 # Reset kernel eflags
227 popfl
228 movl PT_EBP(%esp), %eax
229 call *PT_EBX(%esp)
230 movl $0, PT_EAX(%esp)
232 /*
233 * Kernel threads return to userspace as if returning from a syscall.
234 * We should check whether anything actually uses this path and, if so,
235 * consider switching it over to ret_from_fork.
236 */
237 movl %esp, %eax
238 call syscall_return_slowpath
239 jmp restore_all
240 ENDPROC(ret_from_kernel_thread)
242 /*
243 * Return to user mode is not as complex as all this looks,
244 * but we want the default path for a system call return to
245 * go as quickly as possible which is why some of this is
246 * less clear than it otherwise should be.
247 */
249 # userspace resumption stub bypassing syscall exit tracing
250 ALIGN
251 ret_from_exception:
252 preempt_stop(CLBR_ANY)
253 ret_from_intr:
254 GET_THREAD_INFO(%ebp)
255 #ifdef CONFIG_VM86
256 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
257 movb PT_CS(%esp), %al
258 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
259 #else
260 /*
261 * We can be coming here from child spawned by kernel_thread().
262 */
263 movl PT_CS(%esp), %eax
264 andl $SEGMENT_RPL_MASK, %eax
265 #endif
266 cmpl $USER_RPL, %eax
267 jb resume_kernel # not returning to v8086 or userspace
269 ENTRY(resume_userspace)
270 DISABLE_INTERRUPTS(CLBR_ANY)
271 TRACE_IRQS_OFF
272 movl %esp, %eax
273 call prepare_exit_to_usermode
274 jmp restore_all
275 END(ret_from_exception)
277 #ifdef CONFIG_PREEMPT
278 ENTRY(resume_kernel)
279 DISABLE_INTERRUPTS(CLBR_ANY)
280 need_resched:
281 cmpl $0, PER_CPU_VAR(__preempt_count)
282 jnz restore_all
283 testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ?
284 jz restore_all
285 call preempt_schedule_irq
286 jmp need_resched
287 END(resume_kernel)
288 #endif
290 GLOBAL(__begin_SYSENTER_singlestep_region)
291 /*
292 * All code from here through __end_SYSENTER_singlestep_region is subject
293 * to being single-stepped if a user program sets TF and executes SYSENTER.
294 * There is absolutely nothing that we can do to prevent this from happening
295 * (thanks Intel!). To keep our handling of this situation as simple as
296 * possible, we handle TF just like AC and NT, except that our #DB handler
297 * will ignore all of the single-step traps generated in this range.
298 */
300 #ifdef CONFIG_XEN
301 /*
302 * Xen doesn't set %esp to be precisely what the normal SYSENTER
303 * entry point expects, so fix it up before using the normal path.
304 */
305 ENTRY(xen_sysenter_target)
306 addl $5*4, %esp /* remove xen-provided frame */
307 jmp sysenter_past_esp
308 #endif
310 /*
311 * 32-bit SYSENTER entry.
312 *
313 * 32-bit system calls through the vDSO's __kernel_vsyscall enter here
314 * if X86_FEATURE_SEP is available. This is the preferred system call
315 * entry on 32-bit systems.
316 *
317 * The SYSENTER instruction, in principle, should *only* occur in the
318 * vDSO. In practice, a small number of Android devices were shipped
319 * with a copy of Bionic that inlined a SYSENTER instruction. This
320 * never happened in any of Google's Bionic versions -- it only happened
321 * in a narrow range of Intel-provided versions.
322 *
323 * SYSENTER loads SS, ESP, CS, and EIP from previously programmed MSRs.
324 * IF and VM in RFLAGS are cleared (IOW: interrupts are off).
325 * SYSENTER does not save anything on the stack,
326 * and does not save old EIP (!!!), ESP, or EFLAGS.
327 *
328 * To avoid losing track of EFLAGS.VM (and thus potentially corrupting
329 * user and/or vm86 state), we explicitly disable the SYSENTER
330 * instruction in vm86 mode by reprogramming the MSRs.
331 *
332 * Arguments:
333 * eax system call number
334 * ebx arg1
335 * ecx arg2
336 * edx arg3
337 * esi arg4
338 * edi arg5
339 * ebp user stack
340 * 0(%ebp) arg6
341 */
342 ENTRY(entry_SYSENTER_32)
343 movl TSS_sysenter_sp0(%esp), %esp
344 sysenter_past_esp:
345 pushl $__USER_DS /* pt_regs->ss */
346 pushl %ebp /* pt_regs->sp (stashed in bp) */
347 pushfl /* pt_regs->flags (except IF = 0) */
348 orl $X86_EFLAGS_IF, (%esp) /* Fix IF */
349 pushl $__USER_CS /* pt_regs->cs */
350 pushl $0 /* pt_regs->ip = 0 (placeholder) */
351 pushl %eax /* pt_regs->orig_ax */
352 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
354 /*
355 * SYSENTER doesn't filter flags, so we need to clear NT, AC
356 * and TF ourselves. To save a few cycles, we can check whether
357 * either was set instead of doing an unconditional popfq.
358 * This needs to happen before enabling interrupts so that
359 * we don't get preempted with NT set.
360 *
361 * If TF is set, we will single-step all the way to here -- do_debug
362 * will ignore all the traps. (Yes, this is slow, but so is
363 * single-stepping in general. This allows us to avoid having
364 * a more complicated code to handle the case where a user program
365 * forces us to single-step through the SYSENTER entry code.)
366 *
367 * NB.: .Lsysenter_fix_flags is a label with the code under it moved
368 * out-of-line as an optimization: NT is unlikely to be set in the
369 * majority of the cases and instead of polluting the I$ unnecessarily,
370 * we're keeping that code behind a branch which will predict as
371 * not-taken and therefore its instructions won't be fetched.
372 */
373 testl $X86_EFLAGS_NT|X86_EFLAGS_AC|X86_EFLAGS_TF, PT_EFLAGS(%esp)
374 jnz .Lsysenter_fix_flags
375 .Lsysenter_flags_fixed:
377 /*
378 * User mode is traced as though IRQs are on, and SYSENTER
379 * turned them off.
380 */
381 TRACE_IRQS_OFF
383 movl %esp, %eax
384 call do_fast_syscall_32
385 /* XEN PV guests always use IRET path */
386 ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \
387 "jmp .Lsyscall_32_done", X86_FEATURE_XENPV
389 /* Opportunistic SYSEXIT */
390 TRACE_IRQS_ON /* User mode traces as IRQs on. */
391 movl PT_EIP(%esp), %edx /* pt_regs->ip */
392 movl PT_OLDESP(%esp), %ecx /* pt_regs->sp */
393 1: mov PT_FS(%esp), %fs
394 PTGS_TO_GS
395 popl %ebx /* pt_regs->bx */
396 addl $2*4, %esp /* skip pt_regs->cx and pt_regs->dx */
397 popl %esi /* pt_regs->si */
398 popl %edi /* pt_regs->di */
399 popl %ebp /* pt_regs->bp */
400 popl %eax /* pt_regs->ax */
402 /*
403 * Restore all flags except IF. (We restore IF separately because
404 * STI gives a one-instruction window in which we won't be interrupted,
405 * whereas POPF does not.)
406 */
407 addl $PT_EFLAGS-PT_DS, %esp /* point esp at pt_regs->flags */
408 btr $X86_EFLAGS_IF_BIT, (%esp)
409 popfl
411 /*
412 * Return back to the vDSO, which will pop ecx and edx.
413 * Don't bother with DS and ES (they already contain __USER_DS).
414 */
415 sti
416 sysexit
418 .pushsection .fixup, "ax"
419 2: movl $0, PT_FS(%esp)
420 jmp 1b
421 .popsection
422 _ASM_EXTABLE(1b, 2b)
423 PTGS_TO_GS_EX
425 .Lsysenter_fix_flags:
426 pushl $X86_EFLAGS_FIXED
427 popfl
428 jmp .Lsysenter_flags_fixed
429 GLOBAL(__end_SYSENTER_singlestep_region)
430 ENDPROC(entry_SYSENTER_32)
432 /*
433 * 32-bit legacy system call entry.
434 *
435 * 32-bit x86 Linux system calls traditionally used the INT $0x80
436 * instruction. INT $0x80 lands here.
437 *
438 * This entry point can be used by any 32-bit perform system calls.
439 * Instances of INT $0x80 can be found inline in various programs and
440 * libraries. It is also used by the vDSO's __kernel_vsyscall
441 * fallback for hardware that doesn't support a faster entry method.
442 * Restarted 32-bit system calls also fall back to INT $0x80
443 * regardless of what instruction was originally used to do the system
444 * call. (64-bit programs can use INT $0x80 as well, but they can
445 * only run on 64-bit kernels and therefore land in
446 * entry_INT80_compat.)
447 *
448 * This is considered a slow path. It is not used by most libc
449 * implementations on modern hardware except during process startup.
450 *
451 * Arguments:
452 * eax system call number
453 * ebx arg1
454 * ecx arg2
455 * edx arg3
456 * esi arg4
457 * edi arg5
458 * ebp arg6
459 */
460 ENTRY(entry_INT80_32)
461 ASM_CLAC
462 pushl %eax /* pt_regs->orig_ax */
463 SAVE_ALL pt_regs_ax=$-ENOSYS /* save rest */
465 /*
466 * User mode is traced as though IRQs are on, and the interrupt gate
467 * turned them off.
468 */
469 TRACE_IRQS_OFF
471 movl %esp, %eax
472 call do_int80_syscall_32
473 .Lsyscall_32_done:
475 restore_all:
476 TRACE_IRQS_IRET
477 restore_all_notrace:
478 #ifdef CONFIG_X86_ESPFIX32
479 ALTERNATIVE "jmp restore_nocheck", "", X86_BUG_ESPFIX
481 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
482 /*
483 * Warning: PT_OLDSS(%esp) contains the wrong/random values if we
484 * are returning to the kernel.
485 * See comments in process.c:copy_thread() for details.
486 */
487 movb PT_OLDSS(%esp), %ah
488 movb PT_CS(%esp), %al
489 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
490 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
491 je ldt_ss # returning to user-space with LDT SS
492 #endif
493 restore_nocheck:
494 RESTORE_REGS 4 # skip orig_eax/error_code
495 irq_return:
496 INTERRUPT_RETURN
497 .section .fixup, "ax"
498 ENTRY(iret_exc )
499 pushl $0 # no error code
500 pushl $do_iret_error
501 jmp error_code
502 .previous
503 _ASM_EXTABLE(irq_return, iret_exc)
505 #ifdef CONFIG_X86_ESPFIX32
506 ldt_ss:
507 /*
508 * Setup and switch to ESPFIX stack
509 *
510 * We're returning to userspace with a 16 bit stack. The CPU will not
511 * restore the high word of ESP for us on executing iret... This is an
512 * "official" bug of all the x86-compatible CPUs, which we can work
513 * around to make dosemu and wine happy. We do this by preloading the
514 * high word of ESP with the high word of the userspace ESP while
515 * compensating for the offset by changing to the ESPFIX segment with
516 * a base address that matches for the difference.
517 */
518 #define GDT_ESPFIX_SS PER_CPU_VAR(gdt_page) + (GDT_ENTRY_ESPFIX_SS * 8)
519 mov %esp, %edx /* load kernel esp */
520 mov PT_OLDESP(%esp), %eax /* load userspace esp */
521 mov %dx, %ax /* eax: new kernel esp */
522 sub %eax, %edx /* offset (low word is 0) */
523 shr $16, %edx
524 mov %dl, GDT_ESPFIX_SS + 4 /* bits 16..23 */
525 mov %dh, GDT_ESPFIX_SS + 7 /* bits 24..31 */
526 pushl $__ESPFIX_SS
527 pushl %eax /* new kernel esp */
528 /*
529 * Disable interrupts, but do not irqtrace this section: we
530 * will soon execute iret and the tracer was already set to
531 * the irqstate after the IRET:
532 */
533 DISABLE_INTERRUPTS(CLBR_EAX)
534 lss (%esp), %esp /* switch to espfix segment */
535 jmp restore_nocheck
536 #endif
537 ENDPROC(entry_INT80_32)
539 .macro FIXUP_ESPFIX_STACK
540 /*
541 * Switch back for ESPFIX stack to the normal zerobased stack
542 *
543 * We can't call C functions using the ESPFIX stack. This code reads
544 * the high word of the segment base from the GDT and swiches to the
545 * normal stack and adjusts ESP with the matching offset.
546 */
547 #ifdef CONFIG_X86_ESPFIX32
548 /* fixup the stack */
549 mov GDT_ESPFIX_SS + 4, %al /* bits 16..23 */
550 mov GDT_ESPFIX_SS + 7, %ah /* bits 24..31 */
551 shl $16, %eax
552 addl %esp, %eax /* the adjusted stack pointer */
553 pushl $__KERNEL_DS
554 pushl %eax
555 lss (%esp), %esp /* switch to the normal stack segment */
556 #endif
557 .endm
558 .macro UNWIND_ESPFIX_STACK
559 #ifdef CONFIG_X86_ESPFIX32
560 movl %ss, %eax
561 /* see if on espfix stack */
562 cmpw $__ESPFIX_SS, %ax
563 jne 27f
564 movl $__KERNEL_DS, %eax
565 movl %eax, %ds
566 movl %eax, %es
567 /* switch to normal stack */
568 FIXUP_ESPFIX_STACK
569 27:
570 #endif
571 .endm
573 /*
574 * Build the entry stubs with some assembler magic.
575 * We pack 1 stub into every 8-byte block.
576 */
577 .align 8
578 ENTRY(irq_entries_start)
579 vector=FIRST_EXTERNAL_VECTOR
580 .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR)
581 pushl $(~vector+0x80) /* Note: always in signed byte range */
582 vector=vector+1
583 jmp common_interrupt
584 .align 8
585 .endr
586 END(irq_entries_start)
588 /*
589 * the CPU automatically disables interrupts when executing an IRQ vector,
590 * so IRQ-flags tracing has to follow that:
591 */
592 .p2align CONFIG_X86_L1_CACHE_SHIFT
593 common_interrupt:
594 ASM_CLAC
595 addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */
596 SAVE_ALL
597 TRACE_IRQS_OFF
598 movl %esp, %eax
599 call do_IRQ
600 jmp ret_from_intr
601 ENDPROC(common_interrupt)
603 #define BUILD_INTERRUPT3(name, nr, fn) \
604 ENTRY(name) \
605 ASM_CLAC; \
606 pushl $~(nr); \
607 SAVE_ALL; \
608 TRACE_IRQS_OFF \
609 movl %esp, %eax; \
610 call fn; \
611 jmp ret_from_intr; \
612 ENDPROC(name)
615 #ifdef CONFIG_TRACING
616 # define TRACE_BUILD_INTERRUPT(name, nr) BUILD_INTERRUPT3(trace_##name, nr, smp_trace_##name)
617 #else
618 # define TRACE_BUILD_INTERRUPT(name, nr)
619 #endif
621 #define BUILD_INTERRUPT(name, nr) \
622 BUILD_INTERRUPT3(name, nr, smp_##name); \
623 TRACE_BUILD_INTERRUPT(name, nr)
625 /* The include is where all of the SMP etc. interrupts come from */
626 #include <asm/entry_arch.h>
628 ENTRY(coprocessor_error)
629 ASM_CLAC
630 pushl $0
631 pushl $do_coprocessor_error
632 jmp error_code
633 END(coprocessor_error)
635 ENTRY(simd_coprocessor_error)
636 ASM_CLAC
637 pushl $0
638 #ifdef CONFIG_X86_INVD_BUG
639 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
640 ALTERNATIVE "pushl $do_general_protection", \
641 "pushl $do_simd_coprocessor_error", \
642 X86_FEATURE_XMM
643 #else
644 pushl $do_simd_coprocessor_error
645 #endif
646 jmp error_code
647 END(simd_coprocessor_error)
649 ENTRY(device_not_available)
650 ASM_CLAC
651 pushl $-1 # mark this as an int
652 pushl $do_device_not_available
653 jmp error_code
654 END(device_not_available)
656 #ifdef CONFIG_PARAVIRT
657 ENTRY(native_iret)
658 iret
659 _ASM_EXTABLE(native_iret, iret_exc)
660 END(native_iret)
661 #endif
663 ENTRY(overflow)
664 ASM_CLAC
665 pushl $0
666 pushl $do_overflow
667 jmp error_code
668 END(overflow)
670 ENTRY(bounds)
671 ASM_CLAC
672 pushl $0
673 pushl $do_bounds
674 jmp error_code
675 END(bounds)
677 ENTRY(invalid_op)
678 ASM_CLAC
679 pushl $0
680 pushl $do_invalid_op
681 jmp error_code
682 END(invalid_op)
684 ENTRY(coprocessor_segment_overrun)
685 ASM_CLAC
686 pushl $0
687 pushl $do_coprocessor_segment_overrun
688 jmp error_code
689 END(coprocessor_segment_overrun)
691 ENTRY(invalid_TSS)
692 ASM_CLAC
693 pushl $do_invalid_TSS
694 jmp error_code
695 END(invalid_TSS)
697 ENTRY(segment_not_present)
698 ASM_CLAC
699 pushl $do_segment_not_present
700 jmp error_code
701 END(segment_not_present)
703 ENTRY(stack_segment)
704 ASM_CLAC
705 pushl $do_stack_segment
706 jmp error_code
707 END(stack_segment)
709 ENTRY(alignment_check)
710 ASM_CLAC
711 pushl $do_alignment_check
712 jmp error_code
713 END(alignment_check)
715 ENTRY(divide_error)
716 ASM_CLAC
717 pushl $0 # no error code
718 pushl $do_divide_error
719 jmp error_code
720 END(divide_error)
722 #ifdef CONFIG_X86_MCE
723 ENTRY(machine_check)
724 ASM_CLAC
725 pushl $0
726 pushl machine_check_vector
727 jmp error_code
728 END(machine_check)
729 #endif
731 ENTRY(spurious_interrupt_bug)
732 ASM_CLAC
733 pushl $0
734 pushl $do_spurious_interrupt_bug
735 jmp error_code
736 END(spurious_interrupt_bug)
738 #ifdef CONFIG_XEN
739 ENTRY(xen_hypervisor_callback)
740 pushl $-1 /* orig_ax = -1 => not a system call */
741 SAVE_ALL
742 TRACE_IRQS_OFF
744 /*
745 * Check to see if we got the event in the critical
746 * region in xen_iret_direct, after we've reenabled
747 * events and checked for pending events. This simulates
748 * iret instruction's behaviour where it delivers a
749 * pending interrupt when enabling interrupts:
750 */
751 movl PT_EIP(%esp), %eax
752 cmpl $xen_iret_start_crit, %eax
753 jb 1f
754 cmpl $xen_iret_end_crit, %eax
755 jae 1f
757 jmp xen_iret_crit_fixup
759 ENTRY(xen_do_upcall)
760 1: mov %esp, %eax
761 call xen_evtchn_do_upcall
762 #ifndef CONFIG_PREEMPT
763 call xen_maybe_preempt_hcall
764 #endif
765 jmp ret_from_intr
766 ENDPROC(xen_hypervisor_callback)
768 /*
769 * Hypervisor uses this for application faults while it executes.
770 * We get here for two reasons:
771 * 1. Fault while reloading DS, ES, FS or GS
772 * 2. Fault while executing IRET
773 * Category 1 we fix up by reattempting the load, and zeroing the segment
774 * register if the load fails.
775 * Category 2 we fix up by jumping to do_iret_error. We cannot use the
776 * normal Linux return path in this case because if we use the IRET hypercall
777 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
778 * We distinguish between categories by maintaining a status value in EAX.
779 */
780 ENTRY(xen_failsafe_callback)
781 pushl %eax
782 movl $1, %eax
783 1: mov 4(%esp), %ds
784 2: mov 8(%esp), %es
785 3: mov 12(%esp), %fs
786 4: mov 16(%esp), %gs
787 /* EAX == 0 => Category 1 (Bad segment)
788 EAX != 0 => Category 2 (Bad IRET) */
789 testl %eax, %eax
790 popl %eax
791 lea 16(%esp), %esp
792 jz 5f
793 jmp iret_exc
794 5: pushl $-1 /* orig_ax = -1 => not a system call */
795 SAVE_ALL
796 jmp ret_from_exception
798 .section .fixup, "ax"
799 6: xorl %eax, %eax
800 movl %eax, 4(%esp)
801 jmp 1b
802 7: xorl %eax, %eax
803 movl %eax, 8(%esp)
804 jmp 2b
805 8: xorl %eax, %eax
806 movl %eax, 12(%esp)
807 jmp 3b
808 9: xorl %eax, %eax
809 movl %eax, 16(%esp)
810 jmp 4b
811 .previous
812 _ASM_EXTABLE(1b, 6b)
813 _ASM_EXTABLE(2b, 7b)
814 _ASM_EXTABLE(3b, 8b)
815 _ASM_EXTABLE(4b, 9b)
816 ENDPROC(xen_failsafe_callback)
818 BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
819 xen_evtchn_do_upcall)
821 #endif /* CONFIG_XEN */
823 #if IS_ENABLED(CONFIG_HYPERV)
825 BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR,
826 hyperv_vector_handler)
828 #endif /* CONFIG_HYPERV */
830 #ifdef CONFIG_FUNCTION_TRACER
831 #ifdef CONFIG_DYNAMIC_FTRACE
833 ENTRY(mcount)
834 ret
835 END(mcount)
837 ENTRY(ftrace_caller)
838 pushl %eax
839 pushl %ecx
840 pushl %edx
841 pushl $0 /* Pass NULL as regs pointer */
842 movl 4*4(%esp), %eax
843 movl 0x4(%ebp), %edx
844 movl function_trace_op, %ecx
845 subl $MCOUNT_INSN_SIZE, %eax
847 .globl ftrace_call
848 ftrace_call:
849 call ftrace_stub
851 addl $4, %esp /* skip NULL pointer */
852 popl %edx
853 popl %ecx
854 popl %eax
855 ftrace_ret:
856 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
857 .globl ftrace_graph_call
858 ftrace_graph_call:
859 jmp ftrace_stub
860 #endif
862 .globl ftrace_stub
863 ftrace_stub:
864 ret
865 END(ftrace_caller)
867 ENTRY(ftrace_regs_caller)
868 pushf /* push flags before compare (in cs location) */
870 /*
871 * i386 does not save SS and ESP when coming from kernel.
872 * Instead, to get sp, ®s->sp is used (see ptrace.h).
873 * Unfortunately, that means eflags must be at the same location
874 * as the current return ip is. We move the return ip into the
875 * ip location, and move flags into the return ip location.
876 */
877 pushl 4(%esp) /* save return ip into ip slot */
879 pushl $0 /* Load 0 into orig_ax */
880 pushl %gs
881 pushl %fs
882 pushl %es
883 pushl %ds
884 pushl %eax
885 pushl %ebp
886 pushl %edi
887 pushl %esi
888 pushl %edx
889 pushl %ecx
890 pushl %ebx
892 movl 13*4(%esp), %eax /* Get the saved flags */
893 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */
894 /* clobbering return ip */
895 movl $__KERNEL_CS, 13*4(%esp)
897 movl 12*4(%esp), %eax /* Load ip (1st parameter) */
898 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
899 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */
900 movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
901 pushl %esp /* Save pt_regs as 4th parameter */
903 GLOBAL(ftrace_regs_call)
904 call ftrace_stub
906 addl $4, %esp /* Skip pt_regs */
907 movl 14*4(%esp), %eax /* Move flags back into cs */
908 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */
909 movl 12*4(%esp), %eax /* Get return ip from regs->ip */
910 movl %eax, 14*4(%esp) /* Put return ip back for ret */
912 popl %ebx
913 popl %ecx
914 popl %edx
915 popl %esi
916 popl %edi
917 popl %ebp
918 popl %eax
919 popl %ds
920 popl %es
921 popl %fs
922 popl %gs
923 addl $8, %esp /* Skip orig_ax and ip */
924 popf /* Pop flags at end (no addl to corrupt flags) */
925 jmp ftrace_ret
927 popf
928 jmp ftrace_stub
929 #else /* ! CONFIG_DYNAMIC_FTRACE */
931 ENTRY(mcount)
932 cmpl $__PAGE_OFFSET, %esp
933 jb ftrace_stub /* Paging not enabled yet? */
935 cmpl $ftrace_stub, ftrace_trace_function
936 jnz trace
937 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
938 cmpl $ftrace_stub, ftrace_graph_return
939 jnz ftrace_graph_caller
941 cmpl $ftrace_graph_entry_stub, ftrace_graph_entry
942 jnz ftrace_graph_caller
943 #endif
944 .globl ftrace_stub
945 ftrace_stub:
946 ret
948 /* taken from glibc */
949 trace:
950 pushl %eax
951 pushl %ecx
952 pushl %edx
953 movl 0xc(%esp), %eax
954 movl 0x4(%ebp), %edx
955 subl $MCOUNT_INSN_SIZE, %eax
957 call *ftrace_trace_function
959 popl %edx
960 popl %ecx
961 popl %eax
962 jmp ftrace_stub
963 END(mcount)
964 #endif /* CONFIG_DYNAMIC_FTRACE */
965 #endif /* CONFIG_FUNCTION_TRACER */
967 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
968 ENTRY(ftrace_graph_caller)
969 pushl %eax
970 pushl %ecx
971 pushl %edx
972 movl 0xc(%esp), %eax
973 lea 0x4(%ebp), %edx
974 movl (%ebp), %ecx
975 subl $MCOUNT_INSN_SIZE, %eax
976 call prepare_ftrace_return
977 popl %edx
978 popl %ecx
979 popl %eax
980 ret
981 END(ftrace_graph_caller)
983 .globl return_to_handler
984 return_to_handler:
985 pushl %eax
986 pushl %edx
987 movl %ebp, %eax
988 call ftrace_return_to_handler
989 movl %eax, %ecx
990 popl %edx
991 popl %eax
992 jmp *%ecx
993 #endif
995 #ifdef CONFIG_TRACING
996 ENTRY(trace_page_fault)
997 ASM_CLAC
998 pushl $trace_do_page_fault
999 jmp error_code
1000 END(trace_page_fault)
1001 #endif
1003 ENTRY(page_fault)
1004 ASM_CLAC
1005 pushl $do_page_fault
1006 ALIGN
1007 error_code:
1008 /* the function address is in %gs's slot on the stack */
1009 pushl %fs
1010 pushl %es
1011 pushl %ds
1012 pushl %eax
1013 pushl %ebp
1014 pushl %edi
1015 pushl %esi
1016 pushl %edx
1017 pushl %ecx
1018 pushl %ebx
1019 cld
1020 movl $(__KERNEL_PERCPU), %ecx
1021 movl %ecx, %fs
1022 UNWIND_ESPFIX_STACK
1023 GS_TO_REG %ecx
1024 movl PT_GS(%esp), %edi # get the function address
1025 movl PT_ORIG_EAX(%esp), %edx # get the error code
1026 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
1027 REG_TO_PTGS %ecx
1028 SET_KERNEL_GS %ecx
1029 movl $(__USER_DS), %ecx
1030 movl %ecx, %ds
1031 movl %ecx, %es
1032 TRACE_IRQS_OFF
1033 movl %esp, %eax # pt_regs pointer
1034 call *%edi
1035 jmp ret_from_exception
1036 END(page_fault)
1038 ENTRY(debug)
1039 /*
1040 * #DB can happen at the first instruction of
1041 * entry_SYSENTER_32 or in Xen's SYSENTER prologue. If this
1042 * happens, then we will be running on a very small stack. We
1043 * need to detect this condition and switch to the thread
1044 * stack before calling any C code at all.
1045 *
1046 * If you edit this code, keep in mind that NMIs can happen in here.
1047 */
1048 ASM_CLAC
1049 pushl $-1 # mark this as an int
1050 SAVE_ALL
1051 xorl %edx, %edx # error code 0
1052 movl %esp, %eax # pt_regs pointer
1054 /* Are we currently on the SYSENTER stack? */
1055 PER_CPU(cpu_tss + CPU_TSS_SYSENTER_stack + SIZEOF_SYSENTER_stack, %ecx)
1056 subl %eax, %ecx /* ecx = (end of SYSENTER_stack) - esp */
1057 cmpl $SIZEOF_SYSENTER_stack, %ecx
1058 jb .Ldebug_from_sysenter_stack
1060 TRACE_IRQS_OFF
1061 call do_debug
1062 jmp ret_from_exception
1064 .Ldebug_from_sysenter_stack:
1065 /* We're on the SYSENTER stack. Switch off. */
1066 movl %esp, %ebp
1067 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
1068 TRACE_IRQS_OFF
1069 call do_debug
1070 movl %ebp, %esp
1071 jmp ret_from_exception
1072 END(debug)
1074 /*
1075 * NMI is doubly nasty. It can happen on the first instruction of
1076 * entry_SYSENTER_32 (just like #DB), but it can also interrupt the beginning
1077 * of the #DB handler even if that #DB in turn hit before entry_SYSENTER_32
1078 * switched stacks. We handle both conditions by simply checking whether we
1079 * interrupted kernel code running on the SYSENTER stack.
1080 */
1081 ENTRY(nmi)
1082 ASM_CLAC
1083 #ifdef CONFIG_X86_ESPFIX32
1084 pushl %eax
1085 movl %ss, %eax
1086 cmpw $__ESPFIX_SS, %ax
1087 popl %eax
1088 je nmi_espfix_stack
1089 #endif
1091 pushl %eax # pt_regs->orig_ax
1092 SAVE_ALL
1093 xorl %edx, %edx # zero error code
1094 movl %esp, %eax # pt_regs pointer
1096 /* Are we currently on the SYSENTER stack? */
1097 PER_CPU(cpu_tss + CPU_TSS_SYSENTER_stack + SIZEOF_SYSENTER_stack, %ecx)
1098 subl %eax, %ecx /* ecx = (end of SYSENTER_stack) - esp */
1099 cmpl $SIZEOF_SYSENTER_stack, %ecx
1100 jb .Lnmi_from_sysenter_stack
1102 /* Not on SYSENTER stack. */
1103 call do_nmi
1104 jmp restore_all_notrace
1106 .Lnmi_from_sysenter_stack:
1107 /*
1108 * We're on the SYSENTER stack. Switch off. No one (not even debug)
1109 * is using the thread stack right now, so it's safe for us to use it.
1110 */
1111 movl %esp, %ebp
1112 movl PER_CPU_VAR(cpu_current_top_of_stack), %esp
1113 call do_nmi
1114 movl %ebp, %esp
1115 jmp restore_all_notrace
1117 #ifdef CONFIG_X86_ESPFIX32
1118 nmi_espfix_stack:
1119 /*
1120 * create the pointer to lss back
1121 */
1122 pushl %ss
1123 pushl %esp
1124 addl $4, (%esp)
1125 /* copy the iret frame of 12 bytes */
1126 .rept 3
1127 pushl 16(%esp)
1128 .endr
1129 pushl %eax
1130 SAVE_ALL
1131 FIXUP_ESPFIX_STACK # %eax == %esp
1132 xorl %edx, %edx # zero error code
1133 call do_nmi
1134 RESTORE_REGS
1135 lss 12+4(%esp), %esp # back to espfix stack
1136 jmp irq_return
1137 #endif
1138 END(nmi)
1140 ENTRY(int3)
1141 ASM_CLAC
1142 pushl $-1 # mark this as an int
1143 SAVE_ALL
1144 TRACE_IRQS_OFF
1145 xorl %edx, %edx # zero error code
1146 movl %esp, %eax # pt_regs pointer
1147 call do_int3
1148 jmp ret_from_exception
1149 END(int3)
1151 ENTRY(general_protection)
1152 pushl $do_general_protection
1153 jmp error_code
1154 END(general_protection)
1156 #ifdef CONFIG_KVM_GUEST
1157 ENTRY(async_page_fault)
1158 ASM_CLAC
1159 pushl $do_async_page_fault
1160 jmp error_code
1161 END(async_page_fault)
1162 #endif