| blob | 5e47683fc63ae736e25d30e83aef093965cdc17d |
1 /*
2 * linux/arch/i386/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
7 /*
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
14 *
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
16 * on a 486.
17 *
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
23 *
24 * 0(%esp) - %ebx
25 * 4(%esp) - %ecx
26 * 8(%esp) - %edx
27 * C(%esp) - %esi
28 * 10(%esp) - %edi
29 * 14(%esp) - %ebp
30 * 18(%esp) - %eax
31 * 1C(%esp) - %ds
32 * 20(%esp) - %es
33 * 24(%esp) - %gs
34 * 28(%esp) - orig_eax
35 * 2C(%esp) - %eip
36 * 30(%esp) - %cs
37 * 34(%esp) - %eflags
38 * 38(%esp) - %oldesp
39 * 3C(%esp) - %oldss
40 *
41 * "current" is in register %ebx during any slow entries.
42 */
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/irqflags.h>
47 #include <asm/errno.h>
48 #include <asm/segment.h>
49 #include <asm/smp.h>
50 #include <asm/page.h>
51 #include <asm/desc.h>
52 #include <asm/percpu.h>
53 #include <asm/dwarf2.h>
54 #include "irq_vectors.h"
56 /*
57 * We use macros for low-level operations which need to be overridden
58 * for paravirtualization. The following will never clobber any registers:
59 * INTERRUPT_RETURN (aka. "iret")
60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61 * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit").
62 *
63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
65 * Allowing a register to be clobbered can shrink the paravirt replacement
66 * enough to patch inline, increasing performance.
67 */
69 #define nr_syscalls ((syscall_table_size)/4)
71 CF_MASK = 0x00000001
72 TF_MASK = 0x00000100
73 IF_MASK = 0x00000200
74 DF_MASK = 0x00000400
75 NT_MASK = 0x00004000
76 VM_MASK = 0x00020000
78 #ifdef CONFIG_PREEMPT
79 #define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
80 #else
81 #define preempt_stop(clobbers)
82 #define resume_kernel restore_nocheck
83 #endif
85 .macro TRACE_IRQS_IRET
86 #ifdef CONFIG_TRACE_IRQFLAGS
87 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off?
88 jz 1f
89 TRACE_IRQS_ON
90 1:
91 #endif
92 .endm
94 #ifdef CONFIG_VM86
95 #define resume_userspace_sig check_userspace
96 #else
97 #define resume_userspace_sig resume_userspace
98 #endif
100 #define SAVE_ALL \
101 cld; \
102 pushl %gs; \
103 CFI_ADJUST_CFA_OFFSET 4;\
104 /*CFI_REL_OFFSET gs, 0;*/\
105 pushl %es; \
106 CFI_ADJUST_CFA_OFFSET 4;\
107 /*CFI_REL_OFFSET es, 0;*/\
108 pushl %ds; \
109 CFI_ADJUST_CFA_OFFSET 4;\
110 /*CFI_REL_OFFSET ds, 0;*/\
111 pushl %eax; \
112 CFI_ADJUST_CFA_OFFSET 4;\
113 CFI_REL_OFFSET eax, 0;\
114 pushl %ebp; \
115 CFI_ADJUST_CFA_OFFSET 4;\
116 CFI_REL_OFFSET ebp, 0;\
117 pushl %edi; \
118 CFI_ADJUST_CFA_OFFSET 4;\
119 CFI_REL_OFFSET edi, 0;\
120 pushl %esi; \
121 CFI_ADJUST_CFA_OFFSET 4;\
122 CFI_REL_OFFSET esi, 0;\
123 pushl %edx; \
124 CFI_ADJUST_CFA_OFFSET 4;\
125 CFI_REL_OFFSET edx, 0;\
126 pushl %ecx; \
127 CFI_ADJUST_CFA_OFFSET 4;\
128 CFI_REL_OFFSET ecx, 0;\
129 pushl %ebx; \
130 CFI_ADJUST_CFA_OFFSET 4;\
131 CFI_REL_OFFSET ebx, 0;\
132 movl $(__USER_DS), %edx; \
133 movl %edx, %ds; \
134 movl %edx, %es; \
135 movl $(__KERNEL_PDA), %edx; \
136 movl %edx, %gs
138 #define RESTORE_INT_REGS \
139 popl %ebx; \
140 CFI_ADJUST_CFA_OFFSET -4;\
141 CFI_RESTORE ebx;\
142 popl %ecx; \
143 CFI_ADJUST_CFA_OFFSET -4;\
144 CFI_RESTORE ecx;\
145 popl %edx; \
146 CFI_ADJUST_CFA_OFFSET -4;\
147 CFI_RESTORE edx;\
148 popl %esi; \
149 CFI_ADJUST_CFA_OFFSET -4;\
150 CFI_RESTORE esi;\
151 popl %edi; \
152 CFI_ADJUST_CFA_OFFSET -4;\
153 CFI_RESTORE edi;\
154 popl %ebp; \
155 CFI_ADJUST_CFA_OFFSET -4;\
156 CFI_RESTORE ebp;\
157 popl %eax; \
158 CFI_ADJUST_CFA_OFFSET -4;\
159 CFI_RESTORE eax
161 #define RESTORE_REGS \
162 RESTORE_INT_REGS; \
163 1: popl %ds; \
164 CFI_ADJUST_CFA_OFFSET -4;\
165 /*CFI_RESTORE ds;*/\
166 2: popl %es; \
167 CFI_ADJUST_CFA_OFFSET -4;\
168 /*CFI_RESTORE es;*/\
169 3: popl %gs; \
170 CFI_ADJUST_CFA_OFFSET -4;\
171 /*CFI_RESTORE gs;*/\
172 .pushsection .fixup,"ax"; \
173 4: movl $0,(%esp); \
174 jmp 1b; \
175 5: movl $0,(%esp); \
176 jmp 2b; \
177 6: movl $0,(%esp); \
178 jmp 3b; \
179 .section __ex_table,"a";\
180 .align 4; \
181 .long 1b,4b; \
182 .long 2b,5b; \
183 .long 3b,6b; \
184 .popsection
186 #define RING0_INT_FRAME \
187 CFI_STARTPROC simple;\
188 CFI_SIGNAL_FRAME;\
189 CFI_DEF_CFA esp, 3*4;\
190 /*CFI_OFFSET cs, -2*4;*/\
191 CFI_OFFSET eip, -3*4
193 #define RING0_EC_FRAME \
194 CFI_STARTPROC simple;\
195 CFI_SIGNAL_FRAME;\
196 CFI_DEF_CFA esp, 4*4;\
197 /*CFI_OFFSET cs, -2*4;*/\
198 CFI_OFFSET eip, -3*4
200 #define RING0_PTREGS_FRAME \
201 CFI_STARTPROC simple;\
202 CFI_SIGNAL_FRAME;\
203 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
204 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
205 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
206 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
207 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
208 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
209 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
210 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
211 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
212 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
213 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
214 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
216 ENTRY(ret_from_fork)
217 CFI_STARTPROC
218 pushl %eax
219 CFI_ADJUST_CFA_OFFSET 4
220 call schedule_tail
221 GET_THREAD_INFO(%ebp)
222 popl %eax
223 CFI_ADJUST_CFA_OFFSET -4
224 pushl $0x0202 # Reset kernel eflags
225 CFI_ADJUST_CFA_OFFSET 4
226 popfl
227 CFI_ADJUST_CFA_OFFSET -4
228 jmp syscall_exit
229 CFI_ENDPROC
231 /*
232 * Return to user mode is not as complex as all this looks,
233 * but we want the default path for a system call return to
234 * go as quickly as possible which is why some of this is
235 * less clear than it otherwise should be.
236 */
238 # userspace resumption stub bypassing syscall exit tracing
239 ALIGN
240 RING0_PTREGS_FRAME
241 ret_from_exception:
242 preempt_stop(CLBR_ANY)
243 ret_from_intr:
244 GET_THREAD_INFO(%ebp)
245 check_userspace:
246 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
247 movb PT_CS(%esp), %al
248 andl $(VM_MASK | SEGMENT_RPL_MASK), %eax
249 cmpl $USER_RPL, %eax
250 jb resume_kernel # not returning to v8086 or userspace
252 ENTRY(resume_userspace)
253 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
254 # setting need_resched or sigpending
255 # between sampling and the iret
256 movl TI_flags(%ebp), %ecx
257 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
258 # int/exception return?
259 jne work_pending
260 jmp restore_all
262 #ifdef CONFIG_PREEMPT
263 ENTRY(resume_kernel)
264 DISABLE_INTERRUPTS(CLBR_ANY)
265 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
266 jnz restore_nocheck
267 need_resched:
268 movl TI_flags(%ebp), %ecx # need_resched set ?
269 testb $_TIF_NEED_RESCHED, %cl
270 jz restore_all
271 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off (exception path) ?
272 jz restore_all
273 call preempt_schedule_irq
274 jmp need_resched
275 #endif
276 CFI_ENDPROC
278 /* SYSENTER_RETURN points to after the "sysenter" instruction in
279 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
281 # sysenter call handler stub
282 ENTRY(sysenter_entry)
283 CFI_STARTPROC simple
284 CFI_SIGNAL_FRAME
285 CFI_DEF_CFA esp, 0
286 CFI_REGISTER esp, ebp
287 movl TSS_sysenter_esp0(%esp),%esp
288 sysenter_past_esp:
289 /*
290 * No need to follow this irqs on/off section: the syscall
291 * disabled irqs and here we enable it straight after entry:
292 */
293 ENABLE_INTERRUPTS(CLBR_NONE)
294 pushl $(__USER_DS)
295 CFI_ADJUST_CFA_OFFSET 4
296 /*CFI_REL_OFFSET ss, 0*/
297 pushl %ebp
298 CFI_ADJUST_CFA_OFFSET 4
299 CFI_REL_OFFSET esp, 0
300 pushfl
301 CFI_ADJUST_CFA_OFFSET 4
302 pushl $(__USER_CS)
303 CFI_ADJUST_CFA_OFFSET 4
304 /*CFI_REL_OFFSET cs, 0*/
305 #ifndef CONFIG_COMPAT_VDSO
306 /*
307 * Push current_thread_info()->sysenter_return to the stack.
308 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
309 * pushed above; +8 corresponds to copy_thread's esp0 setting.
310 */
311 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
312 #else
313 pushl $SYSENTER_RETURN
314 #endif
315 CFI_ADJUST_CFA_OFFSET 4
316 CFI_REL_OFFSET eip, 0
318 /*
319 * Load the potential sixth argument from user stack.
320 * Careful about security.
321 */
322 cmpl $__PAGE_OFFSET-3,%ebp
323 jae syscall_fault
324 1: movl (%ebp),%ebp
325 .section __ex_table,"a"
326 .align 4
327 .long 1b,syscall_fault
328 .previous
330 pushl %eax
331 CFI_ADJUST_CFA_OFFSET 4
332 SAVE_ALL
333 GET_THREAD_INFO(%ebp)
335 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
336 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
337 jnz syscall_trace_entry
338 cmpl $(nr_syscalls), %eax
339 jae syscall_badsys
340 call *sys_call_table(,%eax,4)
341 movl %eax,PT_EAX(%esp)
342 DISABLE_INTERRUPTS(CLBR_ECX|CLBR_EDX)
343 TRACE_IRQS_OFF
344 movl TI_flags(%ebp), %ecx
345 testw $_TIF_ALLWORK_MASK, %cx
346 jne syscall_exit_work
347 /* if something modifies registers it must also disable sysexit */
348 movl PT_EIP(%esp), %edx
349 movl PT_OLDESP(%esp), %ecx
350 xorl %ebp,%ebp
351 TRACE_IRQS_ON
352 1: mov PT_GS(%esp), %gs
353 ENABLE_INTERRUPTS_SYSEXIT
354 CFI_ENDPROC
355 .pushsection .fixup,"ax"
356 2: movl $0,PT_GS(%esp)
357 jmp 1b
358 .section __ex_table,"a"
359 .align 4
360 .long 1b,2b
361 .popsection
363 # system call handler stub
364 ENTRY(system_call)
365 RING0_INT_FRAME # can't unwind into user space anyway
366 pushl %eax # save orig_eax
367 CFI_ADJUST_CFA_OFFSET 4
368 SAVE_ALL
369 GET_THREAD_INFO(%ebp)
370 testl $TF_MASK,PT_EFLAGS(%esp)
371 jz no_singlestep
372 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
373 no_singlestep:
374 # system call tracing in operation / emulation
375 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
376 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
377 jnz syscall_trace_entry
378 cmpl $(nr_syscalls), %eax
379 jae syscall_badsys
380 syscall_call:
381 call *sys_call_table(,%eax,4)
382 movl %eax,PT_EAX(%esp) # store the return value
383 syscall_exit:
384 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
385 # setting need_resched or sigpending
386 # between sampling and the iret
387 TRACE_IRQS_OFF
388 movl TI_flags(%ebp), %ecx
389 testw $_TIF_ALLWORK_MASK, %cx # current->work
390 jne syscall_exit_work
392 restore_all:
393 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
394 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
395 # are returning to the kernel.
396 # See comments in process.c:copy_thread() for details.
397 movb PT_OLDSS(%esp), %ah
398 movb PT_CS(%esp), %al
399 andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
400 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
401 CFI_REMEMBER_STATE
402 je ldt_ss # returning to user-space with LDT SS
403 restore_nocheck:
404 TRACE_IRQS_IRET
405 restore_nocheck_notrace:
406 RESTORE_REGS
407 addl $4, %esp # skip orig_eax/error_code
408 CFI_ADJUST_CFA_OFFSET -4
409 1: INTERRUPT_RETURN
410 .section .fixup,"ax"
411 iret_exc:
412 TRACE_IRQS_ON
413 ENABLE_INTERRUPTS(CLBR_NONE)
414 pushl $0 # no error code
415 pushl $do_iret_error
416 jmp error_code
417 .previous
418 .section __ex_table,"a"
419 .align 4
420 .long 1b,iret_exc
421 .previous
423 CFI_RESTORE_STATE
424 ldt_ss:
425 larl PT_OLDSS(%esp), %eax
426 jnz restore_nocheck
427 testl $0x00400000, %eax # returning to 32bit stack?
428 jnz restore_nocheck # allright, normal return
430 #ifdef CONFIG_PARAVIRT
431 /*
432 * The kernel can't run on a non-flat stack if paravirt mode
433 * is active. Rather than try to fixup the high bits of
434 * ESP, bypass this code entirely. This may break DOSemu
435 * and/or Wine support in a paravirt VM, although the option
436 * is still available to implement the setting of the high
437 * 16-bits in the INTERRUPT_RETURN paravirt-op.
438 */
439 cmpl $0, paravirt_ops+PARAVIRT_enabled
440 jne restore_nocheck
441 #endif
443 /* If returning to userspace with 16bit stack,
444 * try to fix the higher word of ESP, as the CPU
445 * won't restore it.
446 * This is an "official" bug of all the x86-compatible
447 * CPUs, which we can try to work around to make
448 * dosemu and wine happy. */
449 movl PT_OLDESP(%esp), %eax
450 movl %esp, %edx
451 call patch_espfix_desc
452 pushl $__ESPFIX_SS
453 CFI_ADJUST_CFA_OFFSET 4
454 pushl %eax
455 CFI_ADJUST_CFA_OFFSET 4
456 DISABLE_INTERRUPTS(CLBR_EAX)
457 TRACE_IRQS_OFF
458 lss (%esp), %esp
459 CFI_ADJUST_CFA_OFFSET -8
460 jmp restore_nocheck
461 CFI_ENDPROC
463 # perform work that needs to be done immediately before resumption
464 ALIGN
465 RING0_PTREGS_FRAME # can't unwind into user space anyway
466 work_pending:
467 testb $_TIF_NEED_RESCHED, %cl
468 jz work_notifysig
469 work_resched:
470 call schedule
471 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
472 # setting need_resched or sigpending
473 # between sampling and the iret
474 TRACE_IRQS_OFF
475 movl TI_flags(%ebp), %ecx
476 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
477 # than syscall tracing?
478 jz restore_all
479 testb $_TIF_NEED_RESCHED, %cl
480 jnz work_resched
482 work_notifysig: # deal with pending signals and
483 # notify-resume requests
484 #ifdef CONFIG_VM86
485 testl $VM_MASK, PT_EFLAGS(%esp)
486 movl %esp, %eax
487 jne work_notifysig_v86 # returning to kernel-space or
488 # vm86-space
489 xorl %edx, %edx
490 call do_notify_resume
491 jmp resume_userspace_sig
493 ALIGN
494 work_notifysig_v86:
495 pushl %ecx # save ti_flags for do_notify_resume
496 CFI_ADJUST_CFA_OFFSET 4
497 call save_v86_state # %eax contains pt_regs pointer
498 popl %ecx
499 CFI_ADJUST_CFA_OFFSET -4
500 movl %eax, %esp
501 #else
502 movl %esp, %eax
503 #endif
504 xorl %edx, %edx
505 call do_notify_resume
506 jmp resume_userspace_sig
508 # perform syscall exit tracing
509 ALIGN
510 syscall_trace_entry:
511 movl $-ENOSYS,PT_EAX(%esp)
512 movl %esp, %eax
513 xorl %edx,%edx
514 call do_syscall_trace
515 cmpl $0, %eax
516 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
517 # so must skip actual syscall
518 movl PT_ORIG_EAX(%esp), %eax
519 cmpl $(nr_syscalls), %eax
520 jnae syscall_call
521 jmp syscall_exit
523 # perform syscall exit tracing
524 ALIGN
525 syscall_exit_work:
526 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
527 jz work_pending
528 TRACE_IRQS_ON
529 ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call
530 # schedule() instead
531 movl %esp, %eax
532 movl $1, %edx
533 call do_syscall_trace
534 jmp resume_userspace
535 CFI_ENDPROC
537 RING0_INT_FRAME # can't unwind into user space anyway
538 syscall_fault:
539 pushl %eax # save orig_eax
540 CFI_ADJUST_CFA_OFFSET 4
541 SAVE_ALL
542 GET_THREAD_INFO(%ebp)
543 movl $-EFAULT,PT_EAX(%esp)
544 jmp resume_userspace
546 syscall_badsys:
547 movl $-ENOSYS,PT_EAX(%esp)
548 jmp resume_userspace
549 CFI_ENDPROC
551 #define FIXUP_ESPFIX_STACK \
552 /* since we are on a wrong stack, we cant make it a C code :( */ \
553 movl %gs:PDA_cpu, %ebx; \
554 PER_CPU(cpu_gdt_descr, %ebx); \
555 movl GDS_address(%ebx), %ebx; \
556 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
557 addl %esp, %eax; \
558 pushl $__KERNEL_DS; \
559 CFI_ADJUST_CFA_OFFSET 4; \
560 pushl %eax; \
561 CFI_ADJUST_CFA_OFFSET 4; \
562 lss (%esp), %esp; \
563 CFI_ADJUST_CFA_OFFSET -8;
564 #define UNWIND_ESPFIX_STACK \
565 movl %ss, %eax; \
566 /* see if on espfix stack */ \
567 cmpw $__ESPFIX_SS, %ax; \
568 jne 27f; \
569 movl $__KERNEL_DS, %eax; \
570 movl %eax, %ds; \
571 movl %eax, %es; \
572 /* switch to normal stack */ \
573 FIXUP_ESPFIX_STACK; \
574 27:;
576 /*
577 * Build the entry stubs and pointer table with
578 * some assembler magic.
579 */
580 .data
581 ENTRY(interrupt)
582 .text
584 vector=0
585 ENTRY(irq_entries_start)
586 RING0_INT_FRAME
587 .rept NR_IRQS
588 ALIGN
589 .if vector
590 CFI_ADJUST_CFA_OFFSET -4
591 .endif
592 1: pushl $~(vector)
593 CFI_ADJUST_CFA_OFFSET 4
594 jmp common_interrupt
595 .data
596 .long 1b
597 .text
598 vector=vector+1
599 .endr
601 /*
602 * the CPU automatically disables interrupts when executing an IRQ vector,
603 * so IRQ-flags tracing has to follow that:
604 */
605 ALIGN
606 common_interrupt:
607 SAVE_ALL
608 TRACE_IRQS_OFF
609 movl %esp,%eax
610 call do_IRQ
611 jmp ret_from_intr
612 CFI_ENDPROC
614 #define BUILD_INTERRUPT(name, nr) \
615 ENTRY(name) \
616 RING0_INT_FRAME; \
617 pushl $~(nr); \
618 CFI_ADJUST_CFA_OFFSET 4; \
619 SAVE_ALL; \
620 TRACE_IRQS_OFF \
621 movl %esp,%eax; \
622 call smp_/**/name; \
623 jmp ret_from_intr; \
624 CFI_ENDPROC
626 /* The include is where all of the SMP etc. interrupts come from */
627 #include "entry_arch.h"
629 KPROBE_ENTRY(page_fault)
630 RING0_EC_FRAME
631 pushl $do_page_fault
632 CFI_ADJUST_CFA_OFFSET 4
633 ALIGN
634 error_code:
635 /* the function address is in %gs's slot on the stack */
636 pushl %es
637 CFI_ADJUST_CFA_OFFSET 4
638 /*CFI_REL_OFFSET es, 0*/
639 pushl %ds
640 CFI_ADJUST_CFA_OFFSET 4
641 /*CFI_REL_OFFSET ds, 0*/
642 pushl %eax
643 CFI_ADJUST_CFA_OFFSET 4
644 CFI_REL_OFFSET eax, 0
645 pushl %ebp
646 CFI_ADJUST_CFA_OFFSET 4
647 CFI_REL_OFFSET ebp, 0
648 pushl %edi
649 CFI_ADJUST_CFA_OFFSET 4
650 CFI_REL_OFFSET edi, 0
651 pushl %esi
652 CFI_ADJUST_CFA_OFFSET 4
653 CFI_REL_OFFSET esi, 0
654 pushl %edx
655 CFI_ADJUST_CFA_OFFSET 4
656 CFI_REL_OFFSET edx, 0
657 pushl %ecx
658 CFI_ADJUST_CFA_OFFSET 4
659 CFI_REL_OFFSET ecx, 0
660 pushl %ebx
661 CFI_ADJUST_CFA_OFFSET 4
662 CFI_REL_OFFSET ebx, 0
663 cld
664 pushl %gs
665 CFI_ADJUST_CFA_OFFSET 4
666 /*CFI_REL_OFFSET gs, 0*/
667 movl $(__KERNEL_PDA), %ecx
668 movl %ecx, %gs
669 UNWIND_ESPFIX_STACK
670 popl %ecx
671 CFI_ADJUST_CFA_OFFSET -4
672 /*CFI_REGISTER es, ecx*/
673 movl PT_GS(%esp), %edi # get the function address
674 movl PT_ORIG_EAX(%esp), %edx # get the error code
675 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
676 mov %ecx, PT_GS(%esp)
677 /*CFI_REL_OFFSET gs, ES*/
678 movl $(__USER_DS), %ecx
679 movl %ecx, %ds
680 movl %ecx, %es
681 movl %esp,%eax # pt_regs pointer
682 call *%edi
683 jmp ret_from_exception
684 CFI_ENDPROC
685 KPROBE_END(page_fault)
687 ENTRY(coprocessor_error)
688 RING0_INT_FRAME
689 pushl $0
690 CFI_ADJUST_CFA_OFFSET 4
691 pushl $do_coprocessor_error
692 CFI_ADJUST_CFA_OFFSET 4
693 jmp error_code
694 CFI_ENDPROC
696 ENTRY(simd_coprocessor_error)
697 RING0_INT_FRAME
698 pushl $0
699 CFI_ADJUST_CFA_OFFSET 4
700 pushl $do_simd_coprocessor_error
701 CFI_ADJUST_CFA_OFFSET 4
702 jmp error_code
703 CFI_ENDPROC
705 ENTRY(device_not_available)
706 RING0_INT_FRAME
707 pushl $-1 # mark this as an int
708 CFI_ADJUST_CFA_OFFSET 4
709 SAVE_ALL
710 GET_CR0_INTO_EAX
711 testl $0x4, %eax # EM (math emulation bit)
712 jne device_not_available_emulate
713 preempt_stop(CLBR_ANY)
714 call math_state_restore
715 jmp ret_from_exception
716 device_not_available_emulate:
717 pushl $0 # temporary storage for ORIG_EIP
718 CFI_ADJUST_CFA_OFFSET 4
719 call math_emulate
720 addl $4, %esp
721 CFI_ADJUST_CFA_OFFSET -4
722 jmp ret_from_exception
723 CFI_ENDPROC
725 /*
726 * Debug traps and NMI can happen at the one SYSENTER instruction
727 * that sets up the real kernel stack. Check here, since we can't
728 * allow the wrong stack to be used.
729 *
730 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have
731 * already pushed 3 words if it hits on the sysenter instruction:
732 * eflags, cs and eip.
733 *
734 * We just load the right stack, and push the three (known) values
735 * by hand onto the new stack - while updating the return eip past
736 * the instruction that would have done it for sysenter.
737 */
738 #define FIX_STACK(offset, ok, label) \
739 cmpw $__KERNEL_CS,4(%esp); \
740 jne ok; \
741 label: \
742 movl TSS_sysenter_esp0+offset(%esp),%esp; \
743 CFI_DEF_CFA esp, 0; \
744 CFI_UNDEFINED eip; \
745 pushfl; \
746 CFI_ADJUST_CFA_OFFSET 4; \
747 pushl $__KERNEL_CS; \
748 CFI_ADJUST_CFA_OFFSET 4; \
749 pushl $sysenter_past_esp; \
750 CFI_ADJUST_CFA_OFFSET 4; \
751 CFI_REL_OFFSET eip, 0
753 KPROBE_ENTRY(debug)
754 RING0_INT_FRAME
755 cmpl $sysenter_entry,(%esp)
756 jne debug_stack_correct
757 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
758 debug_stack_correct:
759 pushl $-1 # mark this as an int
760 CFI_ADJUST_CFA_OFFSET 4
761 SAVE_ALL
762 xorl %edx,%edx # error code 0
763 movl %esp,%eax # pt_regs pointer
764 call do_debug
765 jmp ret_from_exception
766 CFI_ENDPROC
767 KPROBE_END(debug)
769 /*
770 * NMI is doubly nasty. It can happen _while_ we're handling
771 * a debug fault, and the debug fault hasn't yet been able to
772 * clear up the stack. So we first check whether we got an
773 * NMI on the sysenter entry path, but after that we need to
774 * check whether we got an NMI on the debug path where the debug
775 * fault happened on the sysenter path.
776 */
777 KPROBE_ENTRY(nmi)
778 RING0_INT_FRAME
779 pushl %eax
780 CFI_ADJUST_CFA_OFFSET 4
781 movl %ss, %eax
782 cmpw $__ESPFIX_SS, %ax
783 popl %eax
784 CFI_ADJUST_CFA_OFFSET -4
785 je nmi_espfix_stack
786 cmpl $sysenter_entry,(%esp)
787 je nmi_stack_fixup
788 pushl %eax
789 CFI_ADJUST_CFA_OFFSET 4
790 movl %esp,%eax
791 /* Do not access memory above the end of our stack page,
792 * it might not exist.
793 */
794 andl $(THREAD_SIZE-1),%eax
795 cmpl $(THREAD_SIZE-20),%eax
796 popl %eax
797 CFI_ADJUST_CFA_OFFSET -4
798 jae nmi_stack_correct
799 cmpl $sysenter_entry,12(%esp)
800 je nmi_debug_stack_check
801 nmi_stack_correct:
802 /* We have a RING0_INT_FRAME here */
803 pushl %eax
804 CFI_ADJUST_CFA_OFFSET 4
805 SAVE_ALL
806 xorl %edx,%edx # zero error code
807 movl %esp,%eax # pt_regs pointer
808 call do_nmi
809 jmp restore_nocheck_notrace
810 CFI_ENDPROC
812 nmi_stack_fixup:
813 RING0_INT_FRAME
814 FIX_STACK(12,nmi_stack_correct, 1)
815 jmp nmi_stack_correct
817 nmi_debug_stack_check:
818 /* We have a RING0_INT_FRAME here */
819 cmpw $__KERNEL_CS,16(%esp)
820 jne nmi_stack_correct
821 cmpl $debug,(%esp)
822 jb nmi_stack_correct
823 cmpl $debug_esp_fix_insn,(%esp)
824 ja nmi_stack_correct
825 FIX_STACK(24,nmi_stack_correct, 1)
826 jmp nmi_stack_correct
828 nmi_espfix_stack:
829 /* We have a RING0_INT_FRAME here.
830 *
831 * create the pointer to lss back
832 */
833 pushl %ss
834 CFI_ADJUST_CFA_OFFSET 4
835 pushl %esp
836 CFI_ADJUST_CFA_OFFSET 4
837 addw $4, (%esp)
838 /* copy the iret frame of 12 bytes */
839 .rept 3
840 pushl 16(%esp)
841 CFI_ADJUST_CFA_OFFSET 4
842 .endr
843 pushl %eax
844 CFI_ADJUST_CFA_OFFSET 4
845 SAVE_ALL
846 FIXUP_ESPFIX_STACK # %eax == %esp
847 xorl %edx,%edx # zero error code
848 call do_nmi
849 RESTORE_REGS
850 lss 12+4(%esp), %esp # back to espfix stack
851 CFI_ADJUST_CFA_OFFSET -24
852 1: INTERRUPT_RETURN
853 CFI_ENDPROC
854 .section __ex_table,"a"
855 .align 4
856 .long 1b,iret_exc
857 .previous
858 KPROBE_END(nmi)
860 #ifdef CONFIG_PARAVIRT
861 ENTRY(native_iret)
862 1: iret
863 .section __ex_table,"a"
864 .align 4
865 .long 1b,iret_exc
866 .previous
868 ENTRY(native_irq_enable_sysexit)
869 sti
870 sysexit
871 #endif
873 KPROBE_ENTRY(int3)
874 RING0_INT_FRAME
875 pushl $-1 # mark this as an int
876 CFI_ADJUST_CFA_OFFSET 4
877 SAVE_ALL
878 xorl %edx,%edx # zero error code
879 movl %esp,%eax # pt_regs pointer
880 call do_int3
881 jmp ret_from_exception
882 CFI_ENDPROC
883 KPROBE_END(int3)
885 ENTRY(overflow)
886 RING0_INT_FRAME
887 pushl $0
888 CFI_ADJUST_CFA_OFFSET 4
889 pushl $do_overflow
890 CFI_ADJUST_CFA_OFFSET 4
891 jmp error_code
892 CFI_ENDPROC
894 ENTRY(bounds)
895 RING0_INT_FRAME
896 pushl $0
897 CFI_ADJUST_CFA_OFFSET 4
898 pushl $do_bounds
899 CFI_ADJUST_CFA_OFFSET 4
900 jmp error_code
901 CFI_ENDPROC
903 ENTRY(invalid_op)
904 RING0_INT_FRAME
905 pushl $0
906 CFI_ADJUST_CFA_OFFSET 4
907 pushl $do_invalid_op
908 CFI_ADJUST_CFA_OFFSET 4
909 jmp error_code
910 CFI_ENDPROC
912 ENTRY(coprocessor_segment_overrun)
913 RING0_INT_FRAME
914 pushl $0
915 CFI_ADJUST_CFA_OFFSET 4
916 pushl $do_coprocessor_segment_overrun
917 CFI_ADJUST_CFA_OFFSET 4
918 jmp error_code
919 CFI_ENDPROC
921 ENTRY(invalid_TSS)
922 RING0_EC_FRAME
923 pushl $do_invalid_TSS
924 CFI_ADJUST_CFA_OFFSET 4
925 jmp error_code
926 CFI_ENDPROC
928 ENTRY(segment_not_present)
929 RING0_EC_FRAME
930 pushl $do_segment_not_present
931 CFI_ADJUST_CFA_OFFSET 4
932 jmp error_code
933 CFI_ENDPROC
935 ENTRY(stack_segment)
936 RING0_EC_FRAME
937 pushl $do_stack_segment
938 CFI_ADJUST_CFA_OFFSET 4
939 jmp error_code
940 CFI_ENDPROC
942 KPROBE_ENTRY(general_protection)
943 RING0_EC_FRAME
944 pushl $do_general_protection
945 CFI_ADJUST_CFA_OFFSET 4
946 jmp error_code
947 CFI_ENDPROC
948 KPROBE_END(general_protection)
950 ENTRY(alignment_check)
951 RING0_EC_FRAME
952 pushl $do_alignment_check
953 CFI_ADJUST_CFA_OFFSET 4
954 jmp error_code
955 CFI_ENDPROC
957 ENTRY(divide_error)
958 RING0_INT_FRAME
959 pushl $0 # no error code
960 CFI_ADJUST_CFA_OFFSET 4
961 pushl $do_divide_error
962 CFI_ADJUST_CFA_OFFSET 4
963 jmp error_code
964 CFI_ENDPROC
966 #ifdef CONFIG_X86_MCE
967 ENTRY(machine_check)
968 RING0_INT_FRAME
969 pushl $0
970 CFI_ADJUST_CFA_OFFSET 4
971 pushl machine_check_vector
972 CFI_ADJUST_CFA_OFFSET 4
973 jmp error_code
974 CFI_ENDPROC
975 #endif
977 ENTRY(spurious_interrupt_bug)
978 RING0_INT_FRAME
979 pushl $0
980 CFI_ADJUST_CFA_OFFSET 4
981 pushl $do_spurious_interrupt_bug
982 CFI_ADJUST_CFA_OFFSET 4
983 jmp error_code
984 CFI_ENDPROC
986 ENTRY(kernel_thread_helper)
987 pushl $0 # fake return address for unwinder
988 CFI_STARTPROC
989 movl %edx,%eax
990 push %edx
991 CFI_ADJUST_CFA_OFFSET 4
992 call *%ebx
993 push %eax
994 CFI_ADJUST_CFA_OFFSET 4
995 call do_exit
996 CFI_ENDPROC
997 ENDPROC(kernel_thread_helper)
999 .section .rodata,"a"
1000 #include "syscall_table.S"
1002 syscall_table_size=(.-sys_call_table)