| blob | 4b87c32b639f50b08d92021027b40de1307871fb |
1 /*
2 * linux/arch/i386/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
7 /*
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
14 *
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
16 * on a 486.
17 *
18 * Stack layout in 'syscall_exit':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
23 *
24 * 0(%esp) - %ebx
25 * 4(%esp) - %ecx
26 * 8(%esp) - %edx
27 * C(%esp) - %esi
28 * 10(%esp) - %edi
29 * 14(%esp) - %ebp
30 * 18(%esp) - %eax
31 * 1C(%esp) - %ds
32 * 20(%esp) - %es
33 * 24(%esp) - %fs
34 * 28(%esp) - orig_eax
35 * 2C(%esp) - %eip
36 * 30(%esp) - %cs
37 * 34(%esp) - %eflags
38 * 38(%esp) - %oldesp
39 * 3C(%esp) - %oldss
40 *
41 * "current" is in register %ebx during any slow entries.
42 */
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/irqflags.h>
47 #include <asm/errno.h>
48 #include <asm/segment.h>
49 #include <asm/smp.h>
50 #include <asm/page.h>
51 #include <asm/desc.h>
52 #include <asm/percpu.h>
53 #include <asm/dwarf2.h>
54 #include "irq_vectors.h"
56 /*
57 * We use macros for low-level operations which need to be overridden
58 * for paravirtualization. The following will never clobber any registers:
59 * INTERRUPT_RETURN (aka. "iret")
60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61 * ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit").
62 *
63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
65 * Allowing a register to be clobbered can shrink the paravirt replacement
66 * enough to patch inline, increasing performance.
67 */
69 #define nr_syscalls ((syscall_table_size)/4)
71 CF_MASK = 0x00000001
72 TF_MASK = 0x00000100
73 IF_MASK = 0x00000200
74 DF_MASK = 0x00000400
75 NT_MASK = 0x00004000
76 VM_MASK = 0x00020000
78 #ifdef CONFIG_PREEMPT
79 #define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
80 #else
81 #define preempt_stop(clobbers)
82 #define resume_kernel restore_nocheck
83 #endif
85 .macro TRACE_IRQS_IRET
86 #ifdef CONFIG_TRACE_IRQFLAGS
87 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off?
88 jz 1f
89 TRACE_IRQS_ON
90 1:
91 #endif
92 .endm
94 #ifdef CONFIG_VM86
95 #define resume_userspace_sig check_userspace
96 #else
97 #define resume_userspace_sig resume_userspace
98 #endif
100 #define SAVE_ALL \
101 cld; \
102 pushl %fs; \
103 CFI_ADJUST_CFA_OFFSET 4;\
104 /*CFI_REL_OFFSET fs, 0;*/\
105 pushl %es; \
106 CFI_ADJUST_CFA_OFFSET 4;\
107 /*CFI_REL_OFFSET es, 0;*/\
108 pushl %ds; \
109 CFI_ADJUST_CFA_OFFSET 4;\
110 /*CFI_REL_OFFSET ds, 0;*/\
111 pushl %eax; \
112 CFI_ADJUST_CFA_OFFSET 4;\
113 CFI_REL_OFFSET eax, 0;\
114 pushl %ebp; \
115 CFI_ADJUST_CFA_OFFSET 4;\
116 CFI_REL_OFFSET ebp, 0;\
117 pushl %edi; \
118 CFI_ADJUST_CFA_OFFSET 4;\
119 CFI_REL_OFFSET edi, 0;\
120 pushl %esi; \
121 CFI_ADJUST_CFA_OFFSET 4;\
122 CFI_REL_OFFSET esi, 0;\
123 pushl %edx; \
124 CFI_ADJUST_CFA_OFFSET 4;\
125 CFI_REL_OFFSET edx, 0;\
126 pushl %ecx; \
127 CFI_ADJUST_CFA_OFFSET 4;\
128 CFI_REL_OFFSET ecx, 0;\
129 pushl %ebx; \
130 CFI_ADJUST_CFA_OFFSET 4;\
131 CFI_REL_OFFSET ebx, 0;\
132 movl $(__USER_DS), %edx; \
133 movl %edx, %ds; \
134 movl %edx, %es; \
135 movl $(__KERNEL_PERCPU), %edx; \
136 movl %edx, %fs
138 #define RESTORE_INT_REGS \
139 popl %ebx; \
140 CFI_ADJUST_CFA_OFFSET -4;\
141 CFI_RESTORE ebx;\
142 popl %ecx; \
143 CFI_ADJUST_CFA_OFFSET -4;\
144 CFI_RESTORE ecx;\
145 popl %edx; \
146 CFI_ADJUST_CFA_OFFSET -4;\
147 CFI_RESTORE edx;\
148 popl %esi; \
149 CFI_ADJUST_CFA_OFFSET -4;\
150 CFI_RESTORE esi;\
151 popl %edi; \
152 CFI_ADJUST_CFA_OFFSET -4;\
153 CFI_RESTORE edi;\
154 popl %ebp; \
155 CFI_ADJUST_CFA_OFFSET -4;\
156 CFI_RESTORE ebp;\
157 popl %eax; \
158 CFI_ADJUST_CFA_OFFSET -4;\
159 CFI_RESTORE eax
161 #define RESTORE_REGS \
162 RESTORE_INT_REGS; \
163 1: popl %ds; \
164 CFI_ADJUST_CFA_OFFSET -4;\
165 /*CFI_RESTORE ds;*/\
166 2: popl %es; \
167 CFI_ADJUST_CFA_OFFSET -4;\
168 /*CFI_RESTORE es;*/\
169 3: popl %fs; \
170 CFI_ADJUST_CFA_OFFSET -4;\
171 /*CFI_RESTORE fs;*/\
172 .pushsection .fixup,"ax"; \
173 4: movl $0,(%esp); \
174 jmp 1b; \
175 5: movl $0,(%esp); \
176 jmp 2b; \
177 6: movl $0,(%esp); \
178 jmp 3b; \
179 .section __ex_table,"a";\
180 .align 4; \
181 .long 1b,4b; \
182 .long 2b,5b; \
183 .long 3b,6b; \
184 .popsection
186 #define RING0_INT_FRAME \
187 CFI_STARTPROC simple;\
188 CFI_SIGNAL_FRAME;\
189 CFI_DEF_CFA esp, 3*4;\
190 /*CFI_OFFSET cs, -2*4;*/\
191 CFI_OFFSET eip, -3*4
193 #define RING0_EC_FRAME \
194 CFI_STARTPROC simple;\
195 CFI_SIGNAL_FRAME;\
196 CFI_DEF_CFA esp, 4*4;\
197 /*CFI_OFFSET cs, -2*4;*/\
198 CFI_OFFSET eip, -3*4
200 #define RING0_PTREGS_FRAME \
201 CFI_STARTPROC simple;\
202 CFI_SIGNAL_FRAME;\
203 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
204 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
205 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
206 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
207 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
208 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
209 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
210 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
211 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
212 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
213 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
214 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
216 ENTRY(ret_from_fork)
217 CFI_STARTPROC
218 pushl %eax
219 CFI_ADJUST_CFA_OFFSET 4
220 call schedule_tail
221 GET_THREAD_INFO(%ebp)
222 popl %eax
223 CFI_ADJUST_CFA_OFFSET -4
224 pushl $0x0202 # Reset kernel eflags
225 CFI_ADJUST_CFA_OFFSET 4
226 popfl
227 CFI_ADJUST_CFA_OFFSET -4
228 jmp syscall_exit
229 CFI_ENDPROC
230 END(ret_from_fork)
232 /*
233 * Return to user mode is not as complex as all this looks,
234 * but we want the default path for a system call return to
235 * go as quickly as possible which is why some of this is
236 * less clear than it otherwise should be.
237 */
239 # userspace resumption stub bypassing syscall exit tracing
240 ALIGN
241 RING0_PTREGS_FRAME
242 ret_from_exception:
243 preempt_stop(CLBR_ANY)
244 ret_from_intr:
245 GET_THREAD_INFO(%ebp)
246 check_userspace:
247 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
248 movb PT_CS(%esp), %al
249 andl $(VM_MASK | SEGMENT_RPL_MASK), %eax
250 cmpl $USER_RPL, %eax
251 jb resume_kernel # not returning to v8086 or userspace
253 ENTRY(resume_userspace)
254 LOCKDEP_SYS_EXIT
255 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
256 # setting need_resched or sigpending
257 # between sampling and the iret
258 movl TI_flags(%ebp), %ecx
259 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
260 # int/exception return?
261 jne work_pending
262 jmp restore_all
263 END(ret_from_exception)
265 #ifdef CONFIG_PREEMPT
266 ENTRY(resume_kernel)
267 DISABLE_INTERRUPTS(CLBR_ANY)
268 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
269 jnz restore_nocheck
270 need_resched:
271 movl TI_flags(%ebp), %ecx # need_resched set ?
272 testb $_TIF_NEED_RESCHED, %cl
273 jz restore_all
274 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off (exception path) ?
275 jz restore_all
276 call preempt_schedule_irq
277 jmp need_resched
278 END(resume_kernel)
279 #endif
280 CFI_ENDPROC
282 /* SYSENTER_RETURN points to after the "sysenter" instruction in
283 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
285 # sysenter call handler stub
286 ENTRY(ia32_sysenter_target)
287 CFI_STARTPROC simple
288 CFI_SIGNAL_FRAME
289 CFI_DEF_CFA esp, 0
290 CFI_REGISTER esp, ebp
291 movl TSS_sysenter_sp0(%esp),%esp
292 sysenter_past_esp:
293 /*
294 * No need to follow this irqs on/off section: the syscall
295 * disabled irqs and here we enable it straight after entry:
296 */
297 ENABLE_INTERRUPTS(CLBR_NONE)
298 pushl $(__USER_DS)
299 CFI_ADJUST_CFA_OFFSET 4
300 /*CFI_REL_OFFSET ss, 0*/
301 pushl %ebp
302 CFI_ADJUST_CFA_OFFSET 4
303 CFI_REL_OFFSET esp, 0
304 pushfl
305 CFI_ADJUST_CFA_OFFSET 4
306 pushl $(__USER_CS)
307 CFI_ADJUST_CFA_OFFSET 4
308 /*CFI_REL_OFFSET cs, 0*/
309 /*
310 * Push current_thread_info()->sysenter_return to the stack.
311 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
312 * pushed above; +8 corresponds to copy_thread's esp0 setting.
313 */
314 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
315 CFI_ADJUST_CFA_OFFSET 4
316 CFI_REL_OFFSET eip, 0
318 /*
319 * Load the potential sixth argument from user stack.
320 * Careful about security.
321 */
322 cmpl $__PAGE_OFFSET-3,%ebp
323 jae syscall_fault
324 1: movl (%ebp),%ebp
325 .section __ex_table,"a"
326 .align 4
327 .long 1b,syscall_fault
328 .previous
330 pushl %eax
331 CFI_ADJUST_CFA_OFFSET 4
332 SAVE_ALL
333 GET_THREAD_INFO(%ebp)
335 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
336 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
337 jnz syscall_trace_entry
338 cmpl $(nr_syscalls), %eax
339 jae syscall_badsys
340 call *sys_call_table(,%eax,4)
341 movl %eax,PT_EAX(%esp)
342 LOCKDEP_SYS_EXIT
343 DISABLE_INTERRUPTS(CLBR_ANY)
344 TRACE_IRQS_OFF
345 movl TI_flags(%ebp), %ecx
346 testw $_TIF_ALLWORK_MASK, %cx
347 jne syscall_exit_work
348 /* if something modifies registers it must also disable sysexit */
349 movl PT_EIP(%esp), %edx
350 movl PT_OLDESP(%esp), %ecx
351 xorl %ebp,%ebp
352 TRACE_IRQS_ON
353 1: mov PT_FS(%esp), %fs
354 ENABLE_INTERRUPTS_SYSCALL_RET
355 CFI_ENDPROC
356 .pushsection .fixup,"ax"
357 2: movl $0,PT_FS(%esp)
358 jmp 1b
359 .section __ex_table,"a"
360 .align 4
361 .long 1b,2b
362 .popsection
363 ENDPROC(ia32_sysenter_target)
365 # system call handler stub
366 ENTRY(system_call)
367 RING0_INT_FRAME # can't unwind into user space anyway
368 pushl %eax # save orig_eax
369 CFI_ADJUST_CFA_OFFSET 4
370 SAVE_ALL
371 GET_THREAD_INFO(%ebp)
372 # system call tracing in operation / emulation
373 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
374 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
375 jnz syscall_trace_entry
376 cmpl $(nr_syscalls), %eax
377 jae syscall_badsys
378 syscall_call:
379 call *sys_call_table(,%eax,4)
380 movl %eax,PT_EAX(%esp) # store the return value
381 syscall_exit:
382 LOCKDEP_SYS_EXIT
383 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
384 # setting need_resched or sigpending
385 # between sampling and the iret
386 TRACE_IRQS_OFF
387 testl $TF_MASK,PT_EFLAGS(%esp) # If tracing set singlestep flag on exit
388 jz no_singlestep
389 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
390 no_singlestep:
391 movl TI_flags(%ebp), %ecx
392 testw $_TIF_ALLWORK_MASK, %cx # current->work
393 jne syscall_exit_work
395 restore_all:
396 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
397 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
398 # are returning to the kernel.
399 # See comments in process.c:copy_thread() for details.
400 movb PT_OLDSS(%esp), %ah
401 movb PT_CS(%esp), %al
402 andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
403 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
404 CFI_REMEMBER_STATE
405 je ldt_ss # returning to user-space with LDT SS
406 restore_nocheck:
407 TRACE_IRQS_IRET
408 restore_nocheck_notrace:
409 RESTORE_REGS
410 addl $4, %esp # skip orig_eax/error_code
411 CFI_ADJUST_CFA_OFFSET -4
412 irq_return:
413 INTERRUPT_RETURN
414 .section .fixup,"ax"
415 iret_exc:
416 pushl $0 # no error code
417 pushl $do_iret_error
418 jmp error_code
419 .previous
420 .section __ex_table,"a"
421 .align 4
422 .long irq_return,iret_exc
423 .previous
425 CFI_RESTORE_STATE
426 ldt_ss:
427 larl PT_OLDSS(%esp), %eax
428 jnz restore_nocheck
429 testl $0x00400000, %eax # returning to 32bit stack?
430 jnz restore_nocheck # allright, normal return
432 #ifdef CONFIG_PARAVIRT
433 /*
434 * The kernel can't run on a non-flat stack if paravirt mode
435 * is active. Rather than try to fixup the high bits of
436 * ESP, bypass this code entirely. This may break DOSemu
437 * and/or Wine support in a paravirt VM, although the option
438 * is still available to implement the setting of the high
439 * 16-bits in the INTERRUPT_RETURN paravirt-op.
440 */
441 cmpl $0, pv_info+PARAVIRT_enabled
442 jne restore_nocheck
443 #endif
445 /* If returning to userspace with 16bit stack,
446 * try to fix the higher word of ESP, as the CPU
447 * won't restore it.
448 * This is an "official" bug of all the x86-compatible
449 * CPUs, which we can try to work around to make
450 * dosemu and wine happy. */
451 movl PT_OLDESP(%esp), %eax
452 movl %esp, %edx
453 call patch_espfix_desc
454 pushl $__ESPFIX_SS
455 CFI_ADJUST_CFA_OFFSET 4
456 pushl %eax
457 CFI_ADJUST_CFA_OFFSET 4
458 DISABLE_INTERRUPTS(CLBR_EAX)
459 TRACE_IRQS_OFF
460 lss (%esp), %esp
461 CFI_ADJUST_CFA_OFFSET -8
462 jmp restore_nocheck
463 CFI_ENDPROC
464 ENDPROC(system_call)
466 # perform work that needs to be done immediately before resumption
467 ALIGN
468 RING0_PTREGS_FRAME # can't unwind into user space anyway
469 work_pending:
470 testb $_TIF_NEED_RESCHED, %cl
471 jz work_notifysig
472 work_resched:
473 call schedule
474 LOCKDEP_SYS_EXIT
475 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
476 # setting need_resched or sigpending
477 # between sampling and the iret
478 TRACE_IRQS_OFF
479 movl TI_flags(%ebp), %ecx
480 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
481 # than syscall tracing?
482 jz restore_all
483 testb $_TIF_NEED_RESCHED, %cl
484 jnz work_resched
486 work_notifysig: # deal with pending signals and
487 # notify-resume requests
488 #ifdef CONFIG_VM86
489 testl $VM_MASK, PT_EFLAGS(%esp)
490 movl %esp, %eax
491 jne work_notifysig_v86 # returning to kernel-space or
492 # vm86-space
493 xorl %edx, %edx
494 call do_notify_resume
495 jmp resume_userspace_sig
497 ALIGN
498 work_notifysig_v86:
499 pushl %ecx # save ti_flags for do_notify_resume
500 CFI_ADJUST_CFA_OFFSET 4
501 call save_v86_state # %eax contains pt_regs pointer
502 popl %ecx
503 CFI_ADJUST_CFA_OFFSET -4
504 movl %eax, %esp
505 #else
506 movl %esp, %eax
507 #endif
508 xorl %edx, %edx
509 call do_notify_resume
510 jmp resume_userspace_sig
511 END(work_pending)
513 # perform syscall exit tracing
514 ALIGN
515 syscall_trace_entry:
516 movl $-ENOSYS,PT_EAX(%esp)
517 movl %esp, %eax
518 xorl %edx,%edx
519 call do_syscall_trace
520 cmpl $0, %eax
521 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
522 # so must skip actual syscall
523 movl PT_ORIG_EAX(%esp), %eax
524 cmpl $(nr_syscalls), %eax
525 jnae syscall_call
526 jmp syscall_exit
527 END(syscall_trace_entry)
529 # perform syscall exit tracing
530 ALIGN
531 syscall_exit_work:
532 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
533 jz work_pending
534 TRACE_IRQS_ON
535 ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call
536 # schedule() instead
537 movl %esp, %eax
538 movl $1, %edx
539 call do_syscall_trace
540 jmp resume_userspace
541 END(syscall_exit_work)
542 CFI_ENDPROC
544 RING0_INT_FRAME # can't unwind into user space anyway
545 syscall_fault:
546 pushl %eax # save orig_eax
547 CFI_ADJUST_CFA_OFFSET 4
548 SAVE_ALL
549 GET_THREAD_INFO(%ebp)
550 movl $-EFAULT,PT_EAX(%esp)
551 jmp resume_userspace
552 END(syscall_fault)
554 syscall_badsys:
555 movl $-ENOSYS,PT_EAX(%esp)
556 jmp resume_userspace
557 END(syscall_badsys)
558 CFI_ENDPROC
560 #define FIXUP_ESPFIX_STACK \
561 /* since we are on a wrong stack, we cant make it a C code :( */ \
562 PER_CPU(gdt_page, %ebx); \
563 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
564 addl %esp, %eax; \
565 pushl $__KERNEL_DS; \
566 CFI_ADJUST_CFA_OFFSET 4; \
567 pushl %eax; \
568 CFI_ADJUST_CFA_OFFSET 4; \
569 lss (%esp), %esp; \
570 CFI_ADJUST_CFA_OFFSET -8;
571 #define UNWIND_ESPFIX_STACK \
572 movl %ss, %eax; \
573 /* see if on espfix stack */ \
574 cmpw $__ESPFIX_SS, %ax; \
575 jne 27f; \
576 movl $__KERNEL_DS, %eax; \
577 movl %eax, %ds; \
578 movl %eax, %es; \
579 /* switch to normal stack */ \
580 FIXUP_ESPFIX_STACK; \
581 27:;
583 /*
584 * Build the entry stubs and pointer table with
585 * some assembler magic.
586 */
587 .section .rodata,"a"
588 ENTRY(interrupt)
589 .text
591 ENTRY(irq_entries_start)
592 RING0_INT_FRAME
593 vector=0
594 .rept NR_IRQS
595 ALIGN
596 .if vector
597 CFI_ADJUST_CFA_OFFSET -4
598 .endif
599 1: pushl $~(vector)
600 CFI_ADJUST_CFA_OFFSET 4
601 jmp common_interrupt
602 .previous
603 .long 1b
604 .text
605 vector=vector+1
606 .endr
607 END(irq_entries_start)
609 .previous
610 END(interrupt)
611 .previous
613 /*
614 * the CPU automatically disables interrupts when executing an IRQ vector,
615 * so IRQ-flags tracing has to follow that:
616 */
617 ALIGN
618 common_interrupt:
619 SAVE_ALL
620 TRACE_IRQS_OFF
621 movl %esp,%eax
622 call do_IRQ
623 jmp ret_from_intr
624 ENDPROC(common_interrupt)
625 CFI_ENDPROC
627 #define BUILD_INTERRUPT(name, nr) \
628 ENTRY(name) \
629 RING0_INT_FRAME; \
630 pushl $~(nr); \
631 CFI_ADJUST_CFA_OFFSET 4; \
632 SAVE_ALL; \
633 TRACE_IRQS_OFF \
634 movl %esp,%eax; \
635 call smp_##name; \
636 jmp ret_from_intr; \
637 CFI_ENDPROC; \
638 ENDPROC(name)
640 /* The include is where all of the SMP etc. interrupts come from */
641 #include "entry_arch.h"
643 KPROBE_ENTRY(page_fault)
644 RING0_EC_FRAME
645 pushl $do_page_fault
646 CFI_ADJUST_CFA_OFFSET 4
647 ALIGN
648 error_code:
649 /* the function address is in %fs's slot on the stack */
650 pushl %es
651 CFI_ADJUST_CFA_OFFSET 4
652 /*CFI_REL_OFFSET es, 0*/
653 pushl %ds
654 CFI_ADJUST_CFA_OFFSET 4
655 /*CFI_REL_OFFSET ds, 0*/
656 pushl %eax
657 CFI_ADJUST_CFA_OFFSET 4
658 CFI_REL_OFFSET eax, 0
659 pushl %ebp
660 CFI_ADJUST_CFA_OFFSET 4
661 CFI_REL_OFFSET ebp, 0
662 pushl %edi
663 CFI_ADJUST_CFA_OFFSET 4
664 CFI_REL_OFFSET edi, 0
665 pushl %esi
666 CFI_ADJUST_CFA_OFFSET 4
667 CFI_REL_OFFSET esi, 0
668 pushl %edx
669 CFI_ADJUST_CFA_OFFSET 4
670 CFI_REL_OFFSET edx, 0
671 pushl %ecx
672 CFI_ADJUST_CFA_OFFSET 4
673 CFI_REL_OFFSET ecx, 0
674 pushl %ebx
675 CFI_ADJUST_CFA_OFFSET 4
676 CFI_REL_OFFSET ebx, 0
677 cld
678 pushl %fs
679 CFI_ADJUST_CFA_OFFSET 4
680 /*CFI_REL_OFFSET fs, 0*/
681 movl $(__KERNEL_PERCPU), %ecx
682 movl %ecx, %fs
683 UNWIND_ESPFIX_STACK
684 popl %ecx
685 CFI_ADJUST_CFA_OFFSET -4
686 /*CFI_REGISTER es, ecx*/
687 movl PT_FS(%esp), %edi # get the function address
688 movl PT_ORIG_EAX(%esp), %edx # get the error code
689 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
690 mov %ecx, PT_FS(%esp)
691 /*CFI_REL_OFFSET fs, ES*/
692 movl $(__USER_DS), %ecx
693 movl %ecx, %ds
694 movl %ecx, %es
695 movl %esp,%eax # pt_regs pointer
696 call *%edi
697 jmp ret_from_exception
698 CFI_ENDPROC
699 KPROBE_END(page_fault)
701 ENTRY(coprocessor_error)
702 RING0_INT_FRAME
703 pushl $0
704 CFI_ADJUST_CFA_OFFSET 4
705 pushl $do_coprocessor_error
706 CFI_ADJUST_CFA_OFFSET 4
707 jmp error_code
708 CFI_ENDPROC
709 END(coprocessor_error)
711 ENTRY(simd_coprocessor_error)
712 RING0_INT_FRAME
713 pushl $0
714 CFI_ADJUST_CFA_OFFSET 4
715 pushl $do_simd_coprocessor_error
716 CFI_ADJUST_CFA_OFFSET 4
717 jmp error_code
718 CFI_ENDPROC
719 END(simd_coprocessor_error)
721 ENTRY(device_not_available)
722 RING0_INT_FRAME
723 pushl $-1 # mark this as an int
724 CFI_ADJUST_CFA_OFFSET 4
725 SAVE_ALL
726 GET_CR0_INTO_EAX
727 testl $0x4, %eax # EM (math emulation bit)
728 jne device_not_available_emulate
729 preempt_stop(CLBR_ANY)
730 call math_state_restore
731 jmp ret_from_exception
732 device_not_available_emulate:
733 pushl $0 # temporary storage for ORIG_EIP
734 CFI_ADJUST_CFA_OFFSET 4
735 call math_emulate
736 addl $4, %esp
737 CFI_ADJUST_CFA_OFFSET -4
738 jmp ret_from_exception
739 CFI_ENDPROC
740 END(device_not_available)
742 /*
743 * Debug traps and NMI can happen at the one SYSENTER instruction
744 * that sets up the real kernel stack. Check here, since we can't
745 * allow the wrong stack to be used.
746 *
747 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
748 * already pushed 3 words if it hits on the sysenter instruction:
749 * eflags, cs and eip.
750 *
751 * We just load the right stack, and push the three (known) values
752 * by hand onto the new stack - while updating the return eip past
753 * the instruction that would have done it for sysenter.
754 */
755 #define FIX_STACK(offset, ok, label) \
756 cmpw $__KERNEL_CS,4(%esp); \
757 jne ok; \
758 label: \
759 movl TSS_sysenter_sp0+offset(%esp),%esp; \
760 CFI_DEF_CFA esp, 0; \
761 CFI_UNDEFINED eip; \
762 pushfl; \
763 CFI_ADJUST_CFA_OFFSET 4; \
764 pushl $__KERNEL_CS; \
765 CFI_ADJUST_CFA_OFFSET 4; \
766 pushl $sysenter_past_esp; \
767 CFI_ADJUST_CFA_OFFSET 4; \
768 CFI_REL_OFFSET eip, 0
770 KPROBE_ENTRY(debug)
771 RING0_INT_FRAME
772 cmpl $ia32_sysenter_target,(%esp)
773 jne debug_stack_correct
774 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
775 debug_stack_correct:
776 pushl $-1 # mark this as an int
777 CFI_ADJUST_CFA_OFFSET 4
778 SAVE_ALL
779 xorl %edx,%edx # error code 0
780 movl %esp,%eax # pt_regs pointer
781 call do_debug
782 jmp ret_from_exception
783 CFI_ENDPROC
784 KPROBE_END(debug)
786 /*
787 * NMI is doubly nasty. It can happen _while_ we're handling
788 * a debug fault, and the debug fault hasn't yet been able to
789 * clear up the stack. So we first check whether we got an
790 * NMI on the sysenter entry path, but after that we need to
791 * check whether we got an NMI on the debug path where the debug
792 * fault happened on the sysenter path.
793 */
794 KPROBE_ENTRY(nmi)
795 RING0_INT_FRAME
796 pushl %eax
797 CFI_ADJUST_CFA_OFFSET 4
798 movl %ss, %eax
799 cmpw $__ESPFIX_SS, %ax
800 popl %eax
801 CFI_ADJUST_CFA_OFFSET -4
802 je nmi_espfix_stack
803 cmpl $ia32_sysenter_target,(%esp)
804 je nmi_stack_fixup
805 pushl %eax
806 CFI_ADJUST_CFA_OFFSET 4
807 movl %esp,%eax
808 /* Do not access memory above the end of our stack page,
809 * it might not exist.
810 */
811 andl $(THREAD_SIZE-1),%eax
812 cmpl $(THREAD_SIZE-20),%eax
813 popl %eax
814 CFI_ADJUST_CFA_OFFSET -4
815 jae nmi_stack_correct
816 cmpl $ia32_sysenter_target,12(%esp)
817 je nmi_debug_stack_check
818 nmi_stack_correct:
819 /* We have a RING0_INT_FRAME here */
820 pushl %eax
821 CFI_ADJUST_CFA_OFFSET 4
822 SAVE_ALL
823 xorl %edx,%edx # zero error code
824 movl %esp,%eax # pt_regs pointer
825 call do_nmi
826 jmp restore_nocheck_notrace
827 CFI_ENDPROC
829 nmi_stack_fixup:
830 RING0_INT_FRAME
831 FIX_STACK(12,nmi_stack_correct, 1)
832 jmp nmi_stack_correct
834 nmi_debug_stack_check:
835 /* We have a RING0_INT_FRAME here */
836 cmpw $__KERNEL_CS,16(%esp)
837 jne nmi_stack_correct
838 cmpl $debug,(%esp)
839 jb nmi_stack_correct
840 cmpl $debug_esp_fix_insn,(%esp)
841 ja nmi_stack_correct
842 FIX_STACK(24,nmi_stack_correct, 1)
843 jmp nmi_stack_correct
845 nmi_espfix_stack:
846 /* We have a RING0_INT_FRAME here.
847 *
848 * create the pointer to lss back
849 */
850 pushl %ss
851 CFI_ADJUST_CFA_OFFSET 4
852 pushl %esp
853 CFI_ADJUST_CFA_OFFSET 4
854 addw $4, (%esp)
855 /* copy the iret frame of 12 bytes */
856 .rept 3
857 pushl 16(%esp)
858 CFI_ADJUST_CFA_OFFSET 4
859 .endr
860 pushl %eax
861 CFI_ADJUST_CFA_OFFSET 4
862 SAVE_ALL
863 FIXUP_ESPFIX_STACK # %eax == %esp
864 xorl %edx,%edx # zero error code
865 call do_nmi
866 RESTORE_REGS
867 lss 12+4(%esp), %esp # back to espfix stack
868 CFI_ADJUST_CFA_OFFSET -24
869 jmp irq_return
870 CFI_ENDPROC
871 KPROBE_END(nmi)
873 #ifdef CONFIG_PARAVIRT
874 ENTRY(native_iret)
875 iret
876 .section __ex_table,"a"
877 .align 4
878 .long native_iret, iret_exc
879 .previous
880 END(native_iret)
882 ENTRY(native_irq_enable_syscall_ret)
883 sti
884 sysexit
885 END(native_irq_enable_syscall_ret)
886 #endif
888 KPROBE_ENTRY(int3)
889 RING0_INT_FRAME
890 pushl $-1 # mark this as an int
891 CFI_ADJUST_CFA_OFFSET 4
892 SAVE_ALL
893 xorl %edx,%edx # zero error code
894 movl %esp,%eax # pt_regs pointer
895 call do_int3
896 jmp ret_from_exception
897 CFI_ENDPROC
898 KPROBE_END(int3)
900 ENTRY(overflow)
901 RING0_INT_FRAME
902 pushl $0
903 CFI_ADJUST_CFA_OFFSET 4
904 pushl $do_overflow
905 CFI_ADJUST_CFA_OFFSET 4
906 jmp error_code
907 CFI_ENDPROC
908 END(overflow)
910 ENTRY(bounds)
911 RING0_INT_FRAME
912 pushl $0
913 CFI_ADJUST_CFA_OFFSET 4
914 pushl $do_bounds
915 CFI_ADJUST_CFA_OFFSET 4
916 jmp error_code
917 CFI_ENDPROC
918 END(bounds)
920 ENTRY(invalid_op)
921 RING0_INT_FRAME
922 pushl $0
923 CFI_ADJUST_CFA_OFFSET 4
924 pushl $do_invalid_op
925 CFI_ADJUST_CFA_OFFSET 4
926 jmp error_code
927 CFI_ENDPROC
928 END(invalid_op)
930 ENTRY(coprocessor_segment_overrun)
931 RING0_INT_FRAME
932 pushl $0
933 CFI_ADJUST_CFA_OFFSET 4
934 pushl $do_coprocessor_segment_overrun
935 CFI_ADJUST_CFA_OFFSET 4
936 jmp error_code
937 CFI_ENDPROC
938 END(coprocessor_segment_overrun)
940 ENTRY(invalid_TSS)
941 RING0_EC_FRAME
942 pushl $do_invalid_TSS
943 CFI_ADJUST_CFA_OFFSET 4
944 jmp error_code
945 CFI_ENDPROC
946 END(invalid_TSS)
948 ENTRY(segment_not_present)
949 RING0_EC_FRAME
950 pushl $do_segment_not_present
951 CFI_ADJUST_CFA_OFFSET 4
952 jmp error_code
953 CFI_ENDPROC
954 END(segment_not_present)
956 ENTRY(stack_segment)
957 RING0_EC_FRAME
958 pushl $do_stack_segment
959 CFI_ADJUST_CFA_OFFSET 4
960 jmp error_code
961 CFI_ENDPROC
962 END(stack_segment)
964 KPROBE_ENTRY(general_protection)
965 RING0_EC_FRAME
966 pushl $do_general_protection
967 CFI_ADJUST_CFA_OFFSET 4
968 jmp error_code
969 CFI_ENDPROC
970 KPROBE_END(general_protection)
972 ENTRY(alignment_check)
973 RING0_EC_FRAME
974 pushl $do_alignment_check
975 CFI_ADJUST_CFA_OFFSET 4
976 jmp error_code
977 CFI_ENDPROC
978 END(alignment_check)
980 ENTRY(divide_error)
981 RING0_INT_FRAME
982 pushl $0 # no error code
983 CFI_ADJUST_CFA_OFFSET 4
984 pushl $do_divide_error
985 CFI_ADJUST_CFA_OFFSET 4
986 jmp error_code
987 CFI_ENDPROC
988 END(divide_error)
990 #ifdef CONFIG_X86_MCE
991 ENTRY(machine_check)
992 RING0_INT_FRAME
993 pushl $0
994 CFI_ADJUST_CFA_OFFSET 4
995 pushl machine_check_vector
996 CFI_ADJUST_CFA_OFFSET 4
997 jmp error_code
998 CFI_ENDPROC
999 END(machine_check)
1000 #endif
1002 ENTRY(spurious_interrupt_bug)
1003 RING0_INT_FRAME
1004 pushl $0
1005 CFI_ADJUST_CFA_OFFSET 4
1006 pushl $do_spurious_interrupt_bug
1007 CFI_ADJUST_CFA_OFFSET 4
1008 jmp error_code
1009 CFI_ENDPROC
1010 END(spurious_interrupt_bug)
1012 ENTRY(kernel_thread_helper)
1013 pushl $0 # fake return address for unwinder
1014 CFI_STARTPROC
1015 movl %edx,%eax
1016 push %edx
1017 CFI_ADJUST_CFA_OFFSET 4
1018 call *%ebx
1019 push %eax
1020 CFI_ADJUST_CFA_OFFSET 4
1021 call do_exit
1022 CFI_ENDPROC
1023 ENDPROC(kernel_thread_helper)
1025 #ifdef CONFIG_XEN
1026 ENTRY(xen_hypervisor_callback)
1027 CFI_STARTPROC
1028 pushl $0
1029 CFI_ADJUST_CFA_OFFSET 4
1030 SAVE_ALL
1031 TRACE_IRQS_OFF
1033 /* Check to see if we got the event in the critical
1034 region in xen_iret_direct, after we've reenabled
1035 events and checked for pending events. This simulates
1036 iret instruction's behaviour where it delivers a
1037 pending interrupt when enabling interrupts. */
1038 movl PT_EIP(%esp),%eax
1039 cmpl $xen_iret_start_crit,%eax
1040 jb 1f
1041 cmpl $xen_iret_end_crit,%eax
1042 jae 1f
1044 call xen_iret_crit_fixup
1046 1: mov %esp, %eax
1047 call xen_evtchn_do_upcall
1048 jmp ret_from_intr
1049 CFI_ENDPROC
1050 ENDPROC(xen_hypervisor_callback)
1052 # Hypervisor uses this for application faults while it executes.
1053 # We get here for two reasons:
1054 # 1. Fault while reloading DS, ES, FS or GS
1055 # 2. Fault while executing IRET
1056 # Category 1 we fix up by reattempting the load, and zeroing the segment
1057 # register if the load fails.
1058 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
1059 # normal Linux return path in this case because if we use the IRET hypercall
1060 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1061 # We distinguish between categories by maintaining a status value in EAX.
1062 ENTRY(xen_failsafe_callback)
1063 CFI_STARTPROC
1064 pushl %eax
1065 CFI_ADJUST_CFA_OFFSET 4
1066 movl $1,%eax
1067 1: mov 4(%esp),%ds
1068 2: mov 8(%esp),%es
1069 3: mov 12(%esp),%fs
1070 4: mov 16(%esp),%gs
1071 testl %eax,%eax
1072 popl %eax
1073 CFI_ADJUST_CFA_OFFSET -4
1074 lea 16(%esp),%esp
1075 CFI_ADJUST_CFA_OFFSET -16
1076 jz 5f
1077 addl $16,%esp
1078 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET)
1079 5: pushl $0 # EAX == 0 => Category 1 (Bad segment)
1080 CFI_ADJUST_CFA_OFFSET 4
1081 SAVE_ALL
1082 jmp ret_from_exception
1083 CFI_ENDPROC
1085 .section .fixup,"ax"
1086 6: xorl %eax,%eax
1087 movl %eax,4(%esp)
1088 jmp 1b
1089 7: xorl %eax,%eax
1090 movl %eax,8(%esp)
1091 jmp 2b
1092 8: xorl %eax,%eax
1093 movl %eax,12(%esp)
1094 jmp 3b
1095 9: xorl %eax,%eax
1096 movl %eax,16(%esp)
1097 jmp 4b
1098 .previous
1099 .section __ex_table,"a"
1100 .align 4
1101 .long 1b,6b
1102 .long 2b,7b
1103 .long 3b,8b
1104 .long 4b,9b
1105 .previous
1106 ENDPROC(xen_failsafe_callback)
1108 #endif /* CONFIG_XEN */
1110 .section .rodata,"a"
1111 #include "syscall_table_32.S"
1113 syscall_table_size=(.-sys_call_table)