| blob | d92b4846234bb0023e56e400032e4717f70c1c92 |
1 /*
2 * linux/arch/i386/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
7 /*
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
14 *
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
16 * on a 486.
17 *
18 * Stack layout in 'syscall_exit':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
23 *
24 * 0(%esp) - %ebx
25 * 4(%esp) - %ecx
26 * 8(%esp) - %edx
27 * C(%esp) - %esi
28 * 10(%esp) - %edi
29 * 14(%esp) - %ebp
30 * 18(%esp) - %eax
31 * 1C(%esp) - %ds
32 * 20(%esp) - %es
33 * 24(%esp) - %fs
34 * 28(%esp) - orig_eax
35 * 2C(%esp) - %eip
36 * 30(%esp) - %cs
37 * 34(%esp) - %eflags
38 * 38(%esp) - %oldesp
39 * 3C(%esp) - %oldss
40 *
41 * "current" is in register %ebx during any slow entries.
42 */
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/irqflags.h>
47 #include <asm/errno.h>
48 #include <asm/segment.h>
49 #include <asm/smp.h>
50 #include <asm/page.h>
51 #include <asm/desc.h>
52 #include <asm/percpu.h>
53 #include <asm/dwarf2.h>
54 #include "irq_vectors.h"
56 /*
57 * We use macros for low-level operations which need to be overridden
58 * for paravirtualization. The following will never clobber any registers:
59 * INTERRUPT_RETURN (aka. "iret")
60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61 * ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit").
62 *
63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
65 * Allowing a register to be clobbered can shrink the paravirt replacement
66 * enough to patch inline, increasing performance.
67 */
69 #define nr_syscalls ((syscall_table_size)/4)
71 CF_MASK = 0x00000001
72 TF_MASK = 0x00000100
73 IF_MASK = 0x00000200
74 DF_MASK = 0x00000400
75 NT_MASK = 0x00004000
76 VM_MASK = 0x00020000
78 #ifdef CONFIG_PREEMPT
79 #define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
80 #else
81 #define preempt_stop(clobbers)
82 #define resume_kernel restore_nocheck
83 #endif
85 .macro TRACE_IRQS_IRET
86 #ifdef CONFIG_TRACE_IRQFLAGS
87 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off?
88 jz 1f
89 TRACE_IRQS_ON
90 1:
91 #endif
92 .endm
94 #ifdef CONFIG_VM86
95 #define resume_userspace_sig check_userspace
96 #else
97 #define resume_userspace_sig resume_userspace
98 #endif
100 #define SAVE_ALL \
101 cld; \
102 pushl %fs; \
103 CFI_ADJUST_CFA_OFFSET 4;\
104 /*CFI_REL_OFFSET fs, 0;*/\
105 pushl %es; \
106 CFI_ADJUST_CFA_OFFSET 4;\
107 /*CFI_REL_OFFSET es, 0;*/\
108 pushl %ds; \
109 CFI_ADJUST_CFA_OFFSET 4;\
110 /*CFI_REL_OFFSET ds, 0;*/\
111 pushl %eax; \
112 CFI_ADJUST_CFA_OFFSET 4;\
113 CFI_REL_OFFSET eax, 0;\
114 pushl %ebp; \
115 CFI_ADJUST_CFA_OFFSET 4;\
116 CFI_REL_OFFSET ebp, 0;\
117 pushl %edi; \
118 CFI_ADJUST_CFA_OFFSET 4;\
119 CFI_REL_OFFSET edi, 0;\
120 pushl %esi; \
121 CFI_ADJUST_CFA_OFFSET 4;\
122 CFI_REL_OFFSET esi, 0;\
123 pushl %edx; \
124 CFI_ADJUST_CFA_OFFSET 4;\
125 CFI_REL_OFFSET edx, 0;\
126 pushl %ecx; \
127 CFI_ADJUST_CFA_OFFSET 4;\
128 CFI_REL_OFFSET ecx, 0;\
129 pushl %ebx; \
130 CFI_ADJUST_CFA_OFFSET 4;\
131 CFI_REL_OFFSET ebx, 0;\
132 movl $(__USER_DS), %edx; \
133 movl %edx, %ds; \
134 movl %edx, %es; \
135 movl $(__KERNEL_PERCPU), %edx; \
136 movl %edx, %fs
138 #define RESTORE_INT_REGS \
139 popl %ebx; \
140 CFI_ADJUST_CFA_OFFSET -4;\
141 CFI_RESTORE ebx;\
142 popl %ecx; \
143 CFI_ADJUST_CFA_OFFSET -4;\
144 CFI_RESTORE ecx;\
145 popl %edx; \
146 CFI_ADJUST_CFA_OFFSET -4;\
147 CFI_RESTORE edx;\
148 popl %esi; \
149 CFI_ADJUST_CFA_OFFSET -4;\
150 CFI_RESTORE esi;\
151 popl %edi; \
152 CFI_ADJUST_CFA_OFFSET -4;\
153 CFI_RESTORE edi;\
154 popl %ebp; \
155 CFI_ADJUST_CFA_OFFSET -4;\
156 CFI_RESTORE ebp;\
157 popl %eax; \
158 CFI_ADJUST_CFA_OFFSET -4;\
159 CFI_RESTORE eax
161 #define RESTORE_REGS \
162 RESTORE_INT_REGS; \
163 1: popl %ds; \
164 CFI_ADJUST_CFA_OFFSET -4;\
165 /*CFI_RESTORE ds;*/\
166 2: popl %es; \
167 CFI_ADJUST_CFA_OFFSET -4;\
168 /*CFI_RESTORE es;*/\
169 3: popl %fs; \
170 CFI_ADJUST_CFA_OFFSET -4;\
171 /*CFI_RESTORE fs;*/\
172 .pushsection .fixup,"ax"; \
173 4: movl $0,(%esp); \
174 jmp 1b; \
175 5: movl $0,(%esp); \
176 jmp 2b; \
177 6: movl $0,(%esp); \
178 jmp 3b; \
179 .section __ex_table,"a";\
180 .align 4; \
181 .long 1b,4b; \
182 .long 2b,5b; \
183 .long 3b,6b; \
184 .popsection
186 #define RING0_INT_FRAME \
187 CFI_STARTPROC simple;\
188 CFI_SIGNAL_FRAME;\
189 CFI_DEF_CFA esp, 3*4;\
190 /*CFI_OFFSET cs, -2*4;*/\
191 CFI_OFFSET eip, -3*4
193 #define RING0_EC_FRAME \
194 CFI_STARTPROC simple;\
195 CFI_SIGNAL_FRAME;\
196 CFI_DEF_CFA esp, 4*4;\
197 /*CFI_OFFSET cs, -2*4;*/\
198 CFI_OFFSET eip, -3*4
200 #define RING0_PTREGS_FRAME \
201 CFI_STARTPROC simple;\
202 CFI_SIGNAL_FRAME;\
203 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
204 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
205 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
206 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
207 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
208 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
209 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
210 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
211 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
212 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
213 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
214 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
216 ENTRY(ret_from_fork)
217 CFI_STARTPROC
218 pushl %eax
219 CFI_ADJUST_CFA_OFFSET 4
220 call schedule_tail
221 GET_THREAD_INFO(%ebp)
222 popl %eax
223 CFI_ADJUST_CFA_OFFSET -4
224 pushl $0x0202 # Reset kernel eflags
225 CFI_ADJUST_CFA_OFFSET 4
226 popfl
227 CFI_ADJUST_CFA_OFFSET -4
228 jmp syscall_exit
229 CFI_ENDPROC
230 END(ret_from_fork)
232 /*
233 * Return to user mode is not as complex as all this looks,
234 * but we want the default path for a system call return to
235 * go as quickly as possible which is why some of this is
236 * less clear than it otherwise should be.
237 */
239 # userspace resumption stub bypassing syscall exit tracing
240 ALIGN
241 RING0_PTREGS_FRAME
242 ret_from_exception:
243 preempt_stop(CLBR_ANY)
244 ret_from_intr:
245 GET_THREAD_INFO(%ebp)
246 check_userspace:
247 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
248 movb PT_CS(%esp), %al
249 andl $(VM_MASK | SEGMENT_RPL_MASK), %eax
250 cmpl $USER_RPL, %eax
251 jb resume_kernel # not returning to v8086 or userspace
253 ENTRY(resume_userspace)
254 LOCKDEP_SYS_EXIT
255 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
256 # setting need_resched or sigpending
257 # between sampling and the iret
258 movl TI_flags(%ebp), %ecx
259 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
260 # int/exception return?
261 jne work_pending
262 jmp restore_all
263 END(ret_from_exception)
265 #ifdef CONFIG_PREEMPT
266 ENTRY(resume_kernel)
267 DISABLE_INTERRUPTS(CLBR_ANY)
268 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
269 jnz restore_nocheck
270 need_resched:
271 movl TI_flags(%ebp), %ecx # need_resched set ?
272 testb $_TIF_NEED_RESCHED, %cl
273 jz restore_all
274 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off (exception path) ?
275 jz restore_all
276 call preempt_schedule_irq
277 jmp need_resched
278 END(resume_kernel)
279 #endif
280 CFI_ENDPROC
282 /* SYSENTER_RETURN points to after the "sysenter" instruction in
283 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
285 # sysenter call handler stub
286 ENTRY(ia32_sysenter_target)
287 CFI_STARTPROC simple
288 CFI_SIGNAL_FRAME
289 CFI_DEF_CFA esp, 0
290 CFI_REGISTER esp, ebp
291 movl TSS_sysenter_sp0(%esp),%esp
292 sysenter_past_esp:
293 /*
294 * No need to follow this irqs on/off section: the syscall
295 * disabled irqs and here we enable it straight after entry:
296 */
297 ENABLE_INTERRUPTS(CLBR_NONE)
298 pushl $(__USER_DS)
299 CFI_ADJUST_CFA_OFFSET 4
300 /*CFI_REL_OFFSET ss, 0*/
301 pushl %ebp
302 CFI_ADJUST_CFA_OFFSET 4
303 CFI_REL_OFFSET esp, 0
304 pushfl
305 CFI_ADJUST_CFA_OFFSET 4
306 pushl $(__USER_CS)
307 CFI_ADJUST_CFA_OFFSET 4
308 /*CFI_REL_OFFSET cs, 0*/
309 /*
310 * Push current_thread_info()->sysenter_return to the stack.
311 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
312 * pushed above; +8 corresponds to copy_thread's esp0 setting.
313 */
314 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
315 CFI_ADJUST_CFA_OFFSET 4
316 CFI_REL_OFFSET eip, 0
318 /*
319 * Load the potential sixth argument from user stack.
320 * Careful about security.
321 */
322 cmpl $__PAGE_OFFSET-3,%ebp
323 jae syscall_fault
324 1: movl (%ebp),%ebp
325 .section __ex_table,"a"
326 .align 4
327 .long 1b,syscall_fault
328 .previous
330 pushl %eax
331 CFI_ADJUST_CFA_OFFSET 4
332 SAVE_ALL
333 GET_THREAD_INFO(%ebp)
335 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
336 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
337 jnz syscall_trace_entry
338 cmpl $(nr_syscalls), %eax
339 jae syscall_badsys
340 call *sys_call_table(,%eax,4)
341 movl %eax,PT_EAX(%esp)
342 LOCKDEP_SYS_EXIT
343 DISABLE_INTERRUPTS(CLBR_ANY)
344 TRACE_IRQS_OFF
345 movl TI_flags(%ebp), %ecx
346 testw $_TIF_ALLWORK_MASK, %cx
347 jne syscall_exit_work
348 /* if something modifies registers it must also disable sysexit */
349 movl PT_EIP(%esp), %edx
350 movl PT_OLDESP(%esp), %ecx
351 xorl %ebp,%ebp
352 TRACE_IRQS_ON
353 1: mov PT_FS(%esp), %fs
354 ENABLE_INTERRUPTS_SYSCALL_RET
355 CFI_ENDPROC
356 .pushsection .fixup,"ax"
357 2: movl $0,PT_FS(%esp)
358 jmp 1b
359 .section __ex_table,"a"
360 .align 4
361 .long 1b,2b
362 .popsection
363 ENDPROC(ia32_sysenter_target)
365 # system call handler stub
366 ENTRY(system_call)
367 RING0_INT_FRAME # can't unwind into user space anyway
368 pushl %eax # save orig_eax
369 CFI_ADJUST_CFA_OFFSET 4
370 SAVE_ALL
371 GET_THREAD_INFO(%ebp)
372 # system call tracing in operation / emulation
373 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
374 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
375 jnz syscall_trace_entry
376 cmpl $(nr_syscalls), %eax
377 jae syscall_badsys
378 syscall_call:
379 call *sys_call_table(,%eax,4)
380 movl %eax,PT_EAX(%esp) # store the return value
381 syscall_exit:
382 LOCKDEP_SYS_EXIT
383 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
384 # setting need_resched or sigpending
385 # between sampling and the iret
386 TRACE_IRQS_OFF
387 testl $TF_MASK,PT_EFLAGS(%esp) # If tracing set singlestep flag on exit
388 jz no_singlestep
389 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
390 no_singlestep:
391 movl TI_flags(%ebp), %ecx
392 testw $_TIF_ALLWORK_MASK, %cx # current->work
393 jne syscall_exit_work
395 restore_all:
396 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
397 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
398 # are returning to the kernel.
399 # See comments in process.c:copy_thread() for details.
400 movb PT_OLDSS(%esp), %ah
401 movb PT_CS(%esp), %al
402 andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
403 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
404 CFI_REMEMBER_STATE
405 je ldt_ss # returning to user-space with LDT SS
406 restore_nocheck:
407 TRACE_IRQS_IRET
408 restore_nocheck_notrace:
409 RESTORE_REGS
410 addl $4, %esp # skip orig_eax/error_code
411 CFI_ADJUST_CFA_OFFSET -4
412 <<<<<<< HEAD:arch/x86/kernel/entry_32.S
413 ENTRY(irq_return)
414 =======
415 irq_return:
416 >>>>>>> 264e3e889d86e552b4191d69bb60f4f3b383135a:arch/x86/kernel/entry_32.S
417 INTERRUPT_RETURN
418 .section .fixup,"ax"
419 iret_exc:
420 pushl $0 # no error code
421 pushl $do_iret_error
422 jmp error_code
423 .previous
424 .section __ex_table,"a"
425 .align 4
426 .long irq_return,iret_exc
427 .previous
429 CFI_RESTORE_STATE
430 ldt_ss:
431 larl PT_OLDSS(%esp), %eax
432 jnz restore_nocheck
433 testl $0x00400000, %eax # returning to 32bit stack?
434 jnz restore_nocheck # allright, normal return
436 #ifdef CONFIG_PARAVIRT
437 /*
438 * The kernel can't run on a non-flat stack if paravirt mode
439 * is active. Rather than try to fixup the high bits of
440 * ESP, bypass this code entirely. This may break DOSemu
441 * and/or Wine support in a paravirt VM, although the option
442 * is still available to implement the setting of the high
443 * 16-bits in the INTERRUPT_RETURN paravirt-op.
444 */
445 cmpl $0, pv_info+PARAVIRT_enabled
446 jne restore_nocheck
447 #endif
449 /* If returning to userspace with 16bit stack,
450 * try to fix the higher word of ESP, as the CPU
451 * won't restore it.
452 * This is an "official" bug of all the x86-compatible
453 * CPUs, which we can try to work around to make
454 * dosemu and wine happy. */
455 movl PT_OLDESP(%esp), %eax
456 movl %esp, %edx
457 call patch_espfix_desc
458 pushl $__ESPFIX_SS
459 CFI_ADJUST_CFA_OFFSET 4
460 pushl %eax
461 CFI_ADJUST_CFA_OFFSET 4
462 DISABLE_INTERRUPTS(CLBR_EAX)
463 TRACE_IRQS_OFF
464 lss (%esp), %esp
465 CFI_ADJUST_CFA_OFFSET -8
466 jmp restore_nocheck
467 CFI_ENDPROC
468 ENDPROC(system_call)
470 # perform work that needs to be done immediately before resumption
471 ALIGN
472 RING0_PTREGS_FRAME # can't unwind into user space anyway
473 work_pending:
474 testb $_TIF_NEED_RESCHED, %cl
475 jz work_notifysig
476 work_resched:
477 call schedule
478 LOCKDEP_SYS_EXIT
479 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
480 # setting need_resched or sigpending
481 # between sampling and the iret
482 TRACE_IRQS_OFF
483 movl TI_flags(%ebp), %ecx
484 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
485 # than syscall tracing?
486 jz restore_all
487 testb $_TIF_NEED_RESCHED, %cl
488 jnz work_resched
490 work_notifysig: # deal with pending signals and
491 # notify-resume requests
492 #ifdef CONFIG_VM86
493 testl $VM_MASK, PT_EFLAGS(%esp)
494 movl %esp, %eax
495 jne work_notifysig_v86 # returning to kernel-space or
496 # vm86-space
497 xorl %edx, %edx
498 call do_notify_resume
499 jmp resume_userspace_sig
501 ALIGN
502 work_notifysig_v86:
503 pushl %ecx # save ti_flags for do_notify_resume
504 CFI_ADJUST_CFA_OFFSET 4
505 call save_v86_state # %eax contains pt_regs pointer
506 popl %ecx
507 CFI_ADJUST_CFA_OFFSET -4
508 movl %eax, %esp
509 #else
510 movl %esp, %eax
511 #endif
512 xorl %edx, %edx
513 call do_notify_resume
514 jmp resume_userspace_sig
515 END(work_pending)
517 # perform syscall exit tracing
518 ALIGN
519 syscall_trace_entry:
520 movl $-ENOSYS,PT_EAX(%esp)
521 movl %esp, %eax
522 xorl %edx,%edx
523 call do_syscall_trace
524 cmpl $0, %eax
525 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
526 # so must skip actual syscall
527 movl PT_ORIG_EAX(%esp), %eax
528 cmpl $(nr_syscalls), %eax
529 jnae syscall_call
530 jmp syscall_exit
531 END(syscall_trace_entry)
533 # perform syscall exit tracing
534 ALIGN
535 syscall_exit_work:
536 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
537 jz work_pending
538 TRACE_IRQS_ON
539 ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call
540 # schedule() instead
541 movl %esp, %eax
542 movl $1, %edx
543 call do_syscall_trace
544 jmp resume_userspace
545 END(syscall_exit_work)
546 CFI_ENDPROC
548 RING0_INT_FRAME # can't unwind into user space anyway
549 syscall_fault:
550 pushl %eax # save orig_eax
551 CFI_ADJUST_CFA_OFFSET 4
552 SAVE_ALL
553 GET_THREAD_INFO(%ebp)
554 movl $-EFAULT,PT_EAX(%esp)
555 jmp resume_userspace
556 END(syscall_fault)
558 syscall_badsys:
559 movl $-ENOSYS,PT_EAX(%esp)
560 jmp resume_userspace
561 END(syscall_badsys)
562 CFI_ENDPROC
564 #define FIXUP_ESPFIX_STACK \
565 /* since we are on a wrong stack, we cant make it a C code :( */ \
566 PER_CPU(gdt_page, %ebx); \
567 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
568 addl %esp, %eax; \
569 pushl $__KERNEL_DS; \
570 CFI_ADJUST_CFA_OFFSET 4; \
571 pushl %eax; \
572 CFI_ADJUST_CFA_OFFSET 4; \
573 lss (%esp), %esp; \
574 CFI_ADJUST_CFA_OFFSET -8;
575 #define UNWIND_ESPFIX_STACK \
576 movl %ss, %eax; \
577 /* see if on espfix stack */ \
578 cmpw $__ESPFIX_SS, %ax; \
579 jne 27f; \
580 movl $__KERNEL_DS, %eax; \
581 movl %eax, %ds; \
582 movl %eax, %es; \
583 /* switch to normal stack */ \
584 FIXUP_ESPFIX_STACK; \
585 27:;
587 /*
588 * Build the entry stubs and pointer table with
589 * some assembler magic.
590 */
591 .section .rodata,"a"
592 ENTRY(interrupt)
593 .text
595 ENTRY(irq_entries_start)
596 RING0_INT_FRAME
597 vector=0
598 .rept NR_IRQS
599 ALIGN
600 .if vector
601 CFI_ADJUST_CFA_OFFSET -4
602 .endif
603 1: pushl $~(vector)
604 CFI_ADJUST_CFA_OFFSET 4
605 jmp common_interrupt
606 .previous
607 .long 1b
608 .text
609 vector=vector+1
610 .endr
611 END(irq_entries_start)
613 .previous
614 END(interrupt)
615 .previous
617 /*
618 * the CPU automatically disables interrupts when executing an IRQ vector,
619 * so IRQ-flags tracing has to follow that:
620 */
621 ALIGN
622 common_interrupt:
623 SAVE_ALL
624 TRACE_IRQS_OFF
625 movl %esp,%eax
626 call do_IRQ
627 jmp ret_from_intr
628 ENDPROC(common_interrupt)
629 CFI_ENDPROC
631 #define BUILD_INTERRUPT(name, nr) \
632 ENTRY(name) \
633 RING0_INT_FRAME; \
634 pushl $~(nr); \
635 CFI_ADJUST_CFA_OFFSET 4; \
636 SAVE_ALL; \
637 TRACE_IRQS_OFF \
638 movl %esp,%eax; \
639 call smp_##name; \
640 jmp ret_from_intr; \
641 CFI_ENDPROC; \
642 ENDPROC(name)
644 /* The include is where all of the SMP etc. interrupts come from */
645 #include "entry_arch.h"
647 KPROBE_ENTRY(page_fault)
648 RING0_EC_FRAME
649 pushl $do_page_fault
650 CFI_ADJUST_CFA_OFFSET 4
651 ALIGN
652 error_code:
653 /* the function address is in %fs's slot on the stack */
654 pushl %es
655 CFI_ADJUST_CFA_OFFSET 4
656 /*CFI_REL_OFFSET es, 0*/
657 pushl %ds
658 CFI_ADJUST_CFA_OFFSET 4
659 /*CFI_REL_OFFSET ds, 0*/
660 pushl %eax
661 CFI_ADJUST_CFA_OFFSET 4
662 CFI_REL_OFFSET eax, 0
663 pushl %ebp
664 CFI_ADJUST_CFA_OFFSET 4
665 CFI_REL_OFFSET ebp, 0
666 pushl %edi
667 CFI_ADJUST_CFA_OFFSET 4
668 CFI_REL_OFFSET edi, 0
669 pushl %esi
670 CFI_ADJUST_CFA_OFFSET 4
671 CFI_REL_OFFSET esi, 0
672 pushl %edx
673 CFI_ADJUST_CFA_OFFSET 4
674 CFI_REL_OFFSET edx, 0
675 pushl %ecx
676 CFI_ADJUST_CFA_OFFSET 4
677 CFI_REL_OFFSET ecx, 0
678 pushl %ebx
679 CFI_ADJUST_CFA_OFFSET 4
680 CFI_REL_OFFSET ebx, 0
681 cld
682 pushl %fs
683 CFI_ADJUST_CFA_OFFSET 4
684 /*CFI_REL_OFFSET fs, 0*/
685 movl $(__KERNEL_PERCPU), %ecx
686 movl %ecx, %fs
687 UNWIND_ESPFIX_STACK
688 popl %ecx
689 CFI_ADJUST_CFA_OFFSET -4
690 /*CFI_REGISTER es, ecx*/
691 movl PT_FS(%esp), %edi # get the function address
692 movl PT_ORIG_EAX(%esp), %edx # get the error code
693 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
694 mov %ecx, PT_FS(%esp)
695 /*CFI_REL_OFFSET fs, ES*/
696 movl $(__USER_DS), %ecx
697 movl %ecx, %ds
698 movl %ecx, %es
699 movl %esp,%eax # pt_regs pointer
700 call *%edi
701 jmp ret_from_exception
702 CFI_ENDPROC
703 KPROBE_END(page_fault)
705 ENTRY(coprocessor_error)
706 RING0_INT_FRAME
707 pushl $0
708 CFI_ADJUST_CFA_OFFSET 4
709 pushl $do_coprocessor_error
710 CFI_ADJUST_CFA_OFFSET 4
711 jmp error_code
712 CFI_ENDPROC
713 END(coprocessor_error)
715 ENTRY(simd_coprocessor_error)
716 RING0_INT_FRAME
717 pushl $0
718 CFI_ADJUST_CFA_OFFSET 4
719 pushl $do_simd_coprocessor_error
720 CFI_ADJUST_CFA_OFFSET 4
721 jmp error_code
722 CFI_ENDPROC
723 END(simd_coprocessor_error)
725 ENTRY(device_not_available)
726 RING0_INT_FRAME
727 pushl $-1 # mark this as an int
728 CFI_ADJUST_CFA_OFFSET 4
729 SAVE_ALL
730 GET_CR0_INTO_EAX
731 testl $0x4, %eax # EM (math emulation bit)
732 jne device_not_available_emulate
733 preempt_stop(CLBR_ANY)
734 call math_state_restore
735 jmp ret_from_exception
736 device_not_available_emulate:
737 pushl $0 # temporary storage for ORIG_EIP
738 CFI_ADJUST_CFA_OFFSET 4
739 call math_emulate
740 addl $4, %esp
741 CFI_ADJUST_CFA_OFFSET -4
742 jmp ret_from_exception
743 CFI_ENDPROC
744 END(device_not_available)
746 /*
747 * Debug traps and NMI can happen at the one SYSENTER instruction
748 * that sets up the real kernel stack. Check here, since we can't
749 * allow the wrong stack to be used.
750 *
751 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
752 * already pushed 3 words if it hits on the sysenter instruction:
753 * eflags, cs and eip.
754 *
755 * We just load the right stack, and push the three (known) values
756 * by hand onto the new stack - while updating the return eip past
757 * the instruction that would have done it for sysenter.
758 */
759 #define FIX_STACK(offset, ok, label) \
760 cmpw $__KERNEL_CS,4(%esp); \
761 jne ok; \
762 label: \
763 movl TSS_sysenter_sp0+offset(%esp),%esp; \
764 CFI_DEF_CFA esp, 0; \
765 CFI_UNDEFINED eip; \
766 pushfl; \
767 CFI_ADJUST_CFA_OFFSET 4; \
768 pushl $__KERNEL_CS; \
769 CFI_ADJUST_CFA_OFFSET 4; \
770 pushl $sysenter_past_esp; \
771 CFI_ADJUST_CFA_OFFSET 4; \
772 CFI_REL_OFFSET eip, 0
774 KPROBE_ENTRY(debug)
775 RING0_INT_FRAME
776 cmpl $ia32_sysenter_target,(%esp)
777 jne debug_stack_correct
778 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
779 debug_stack_correct:
780 pushl $-1 # mark this as an int
781 CFI_ADJUST_CFA_OFFSET 4
782 SAVE_ALL
783 xorl %edx,%edx # error code 0
784 movl %esp,%eax # pt_regs pointer
785 call do_debug
786 jmp ret_from_exception
787 CFI_ENDPROC
788 KPROBE_END(debug)
790 /*
791 * NMI is doubly nasty. It can happen _while_ we're handling
792 * a debug fault, and the debug fault hasn't yet been able to
793 * clear up the stack. So we first check whether we got an
794 * NMI on the sysenter entry path, but after that we need to
795 * check whether we got an NMI on the debug path where the debug
796 * fault happened on the sysenter path.
797 */
798 KPROBE_ENTRY(nmi)
799 RING0_INT_FRAME
800 pushl %eax
801 CFI_ADJUST_CFA_OFFSET 4
802 movl %ss, %eax
803 cmpw $__ESPFIX_SS, %ax
804 popl %eax
805 CFI_ADJUST_CFA_OFFSET -4
806 je nmi_espfix_stack
807 cmpl $ia32_sysenter_target,(%esp)
808 je nmi_stack_fixup
809 pushl %eax
810 CFI_ADJUST_CFA_OFFSET 4
811 movl %esp,%eax
812 /* Do not access memory above the end of our stack page,
813 * it might not exist.
814 */
815 andl $(THREAD_SIZE-1),%eax
816 cmpl $(THREAD_SIZE-20),%eax
817 popl %eax
818 CFI_ADJUST_CFA_OFFSET -4
819 jae nmi_stack_correct
820 cmpl $ia32_sysenter_target,12(%esp)
821 je nmi_debug_stack_check
822 nmi_stack_correct:
823 /* We have a RING0_INT_FRAME here */
824 pushl %eax
825 CFI_ADJUST_CFA_OFFSET 4
826 SAVE_ALL
827 xorl %edx,%edx # zero error code
828 movl %esp,%eax # pt_regs pointer
829 call do_nmi
830 jmp restore_nocheck_notrace
831 CFI_ENDPROC
833 nmi_stack_fixup:
834 RING0_INT_FRAME
835 FIX_STACK(12,nmi_stack_correct, 1)
836 jmp nmi_stack_correct
838 nmi_debug_stack_check:
839 /* We have a RING0_INT_FRAME here */
840 cmpw $__KERNEL_CS,16(%esp)
841 jne nmi_stack_correct
842 cmpl $debug,(%esp)
843 jb nmi_stack_correct
844 cmpl $debug_esp_fix_insn,(%esp)
845 ja nmi_stack_correct
846 FIX_STACK(24,nmi_stack_correct, 1)
847 jmp nmi_stack_correct
849 nmi_espfix_stack:
850 /* We have a RING0_INT_FRAME here.
851 *
852 * create the pointer to lss back
853 */
854 pushl %ss
855 CFI_ADJUST_CFA_OFFSET 4
856 pushl %esp
857 CFI_ADJUST_CFA_OFFSET 4
858 addw $4, (%esp)
859 /* copy the iret frame of 12 bytes */
860 .rept 3
861 pushl 16(%esp)
862 CFI_ADJUST_CFA_OFFSET 4
863 .endr
864 pushl %eax
865 CFI_ADJUST_CFA_OFFSET 4
866 SAVE_ALL
867 FIXUP_ESPFIX_STACK # %eax == %esp
868 xorl %edx,%edx # zero error code
869 call do_nmi
870 RESTORE_REGS
871 lss 12+4(%esp), %esp # back to espfix stack
872 CFI_ADJUST_CFA_OFFSET -24
873 jmp irq_return
874 CFI_ENDPROC
875 KPROBE_END(nmi)
877 #ifdef CONFIG_PARAVIRT
878 ENTRY(native_iret)
879 iret
880 .section __ex_table,"a"
881 .align 4
882 .long native_iret, iret_exc
883 .previous
884 END(native_iret)
886 ENTRY(native_irq_enable_syscall_ret)
887 sti
888 sysexit
889 END(native_irq_enable_syscall_ret)
890 #endif
892 KPROBE_ENTRY(int3)
893 RING0_INT_FRAME
894 pushl $-1 # mark this as an int
895 CFI_ADJUST_CFA_OFFSET 4
896 SAVE_ALL
897 xorl %edx,%edx # zero error code
898 movl %esp,%eax # pt_regs pointer
899 call do_int3
900 jmp ret_from_exception
901 CFI_ENDPROC
902 KPROBE_END(int3)
904 ENTRY(overflow)
905 RING0_INT_FRAME
906 pushl $0
907 CFI_ADJUST_CFA_OFFSET 4
908 pushl $do_overflow
909 CFI_ADJUST_CFA_OFFSET 4
910 jmp error_code
911 CFI_ENDPROC
912 END(overflow)
914 ENTRY(bounds)
915 RING0_INT_FRAME
916 pushl $0
917 CFI_ADJUST_CFA_OFFSET 4
918 pushl $do_bounds
919 CFI_ADJUST_CFA_OFFSET 4
920 jmp error_code
921 CFI_ENDPROC
922 END(bounds)
924 ENTRY(invalid_op)
925 RING0_INT_FRAME
926 pushl $0
927 CFI_ADJUST_CFA_OFFSET 4
928 pushl $do_invalid_op
929 CFI_ADJUST_CFA_OFFSET 4
930 jmp error_code
931 CFI_ENDPROC
932 END(invalid_op)
934 ENTRY(coprocessor_segment_overrun)
935 RING0_INT_FRAME
936 pushl $0
937 CFI_ADJUST_CFA_OFFSET 4
938 pushl $do_coprocessor_segment_overrun
939 CFI_ADJUST_CFA_OFFSET 4
940 jmp error_code
941 CFI_ENDPROC
942 END(coprocessor_segment_overrun)
944 ENTRY(invalid_TSS)
945 RING0_EC_FRAME
946 pushl $do_invalid_TSS
947 CFI_ADJUST_CFA_OFFSET 4
948 jmp error_code
949 CFI_ENDPROC
950 END(invalid_TSS)
952 ENTRY(segment_not_present)
953 RING0_EC_FRAME
954 pushl $do_segment_not_present
955 CFI_ADJUST_CFA_OFFSET 4
956 jmp error_code
957 CFI_ENDPROC
958 END(segment_not_present)
960 ENTRY(stack_segment)
961 RING0_EC_FRAME
962 pushl $do_stack_segment
963 CFI_ADJUST_CFA_OFFSET 4
964 jmp error_code
965 CFI_ENDPROC
966 END(stack_segment)
968 KPROBE_ENTRY(general_protection)
969 RING0_EC_FRAME
970 pushl $do_general_protection
971 CFI_ADJUST_CFA_OFFSET 4
972 jmp error_code
973 CFI_ENDPROC
974 KPROBE_END(general_protection)
976 ENTRY(alignment_check)
977 RING0_EC_FRAME
978 pushl $do_alignment_check
979 CFI_ADJUST_CFA_OFFSET 4
980 jmp error_code
981 CFI_ENDPROC
982 END(alignment_check)
984 ENTRY(divide_error)
985 RING0_INT_FRAME
986 pushl $0 # no error code
987 CFI_ADJUST_CFA_OFFSET 4
988 pushl $do_divide_error
989 CFI_ADJUST_CFA_OFFSET 4
990 jmp error_code
991 CFI_ENDPROC
992 END(divide_error)
994 #ifdef CONFIG_X86_MCE
995 ENTRY(machine_check)
996 RING0_INT_FRAME
997 pushl $0
998 CFI_ADJUST_CFA_OFFSET 4
999 pushl machine_check_vector
1000 CFI_ADJUST_CFA_OFFSET 4
1001 jmp error_code
1002 CFI_ENDPROC
1003 END(machine_check)
1004 #endif
1006 ENTRY(spurious_interrupt_bug)
1007 RING0_INT_FRAME
1008 pushl $0
1009 CFI_ADJUST_CFA_OFFSET 4
1010 pushl $do_spurious_interrupt_bug
1011 CFI_ADJUST_CFA_OFFSET 4
1012 jmp error_code
1013 CFI_ENDPROC
1014 END(spurious_interrupt_bug)
1016 ENTRY(kernel_thread_helper)
1017 pushl $0 # fake return address for unwinder
1018 CFI_STARTPROC
1019 movl %edx,%eax
1020 push %edx
1021 CFI_ADJUST_CFA_OFFSET 4
1022 call *%ebx
1023 push %eax
1024 CFI_ADJUST_CFA_OFFSET 4
1025 call do_exit
1026 CFI_ENDPROC
1027 ENDPROC(kernel_thread_helper)
1029 #ifdef CONFIG_XEN
1030 ENTRY(xen_hypervisor_callback)
1031 CFI_STARTPROC
1032 pushl $0
1033 CFI_ADJUST_CFA_OFFSET 4
1034 SAVE_ALL
1035 TRACE_IRQS_OFF
1037 /* Check to see if we got the event in the critical
1038 region in xen_iret_direct, after we've reenabled
1039 events and checked for pending events. This simulates
1040 iret instruction's behaviour where it delivers a
1041 pending interrupt when enabling interrupts. */
1042 movl PT_EIP(%esp),%eax
1043 cmpl $xen_iret_start_crit,%eax
1044 jb 1f
1045 cmpl $xen_iret_end_crit,%eax
1046 jae 1f
1048 call xen_iret_crit_fixup
1050 1: mov %esp, %eax
1051 call xen_evtchn_do_upcall
1052 jmp ret_from_intr
1053 CFI_ENDPROC
1054 ENDPROC(xen_hypervisor_callback)
1056 # Hypervisor uses this for application faults while it executes.
1057 # We get here for two reasons:
1058 # 1. Fault while reloading DS, ES, FS or GS
1059 # 2. Fault while executing IRET
1060 # Category 1 we fix up by reattempting the load, and zeroing the segment
1061 # register if the load fails.
1062 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
1063 # normal Linux return path in this case because if we use the IRET hypercall
1064 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1065 # We distinguish between categories by maintaining a status value in EAX.
1066 ENTRY(xen_failsafe_callback)
1067 CFI_STARTPROC
1068 pushl %eax
1069 CFI_ADJUST_CFA_OFFSET 4
1070 movl $1,%eax
1071 1: mov 4(%esp),%ds
1072 2: mov 8(%esp),%es
1073 3: mov 12(%esp),%fs
1074 4: mov 16(%esp),%gs
1075 testl %eax,%eax
1076 popl %eax
1077 CFI_ADJUST_CFA_OFFSET -4
1078 lea 16(%esp),%esp
1079 CFI_ADJUST_CFA_OFFSET -16
1080 jz 5f
1081 addl $16,%esp
1082 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET)
1083 5: pushl $0 # EAX == 0 => Category 1 (Bad segment)
1084 CFI_ADJUST_CFA_OFFSET 4
1085 SAVE_ALL
1086 jmp ret_from_exception
1087 CFI_ENDPROC
1089 .section .fixup,"ax"
1090 6: xorl %eax,%eax
1091 movl %eax,4(%esp)
1092 jmp 1b
1093 7: xorl %eax,%eax
1094 movl %eax,8(%esp)
1095 jmp 2b
1096 8: xorl %eax,%eax
1097 movl %eax,12(%esp)
1098 jmp 3b
1099 9: xorl %eax,%eax
1100 movl %eax,16(%esp)
1101 jmp 4b
1102 .previous
1103 .section __ex_table,"a"
1104 .align 4
1105 .long 1b,6b
1106 .long 2b,7b
1107 .long 3b,8b
1108 .long 4b,9b
1109 .previous
1110 ENDPROC(xen_failsafe_callback)
1112 #endif /* CONFIG_XEN */
1114 .section .rodata,"a"
1115 #include "syscall_table_32.S"
1117 syscall_table_size=(.-sys_call_table)