| blob | c778e4fa55a2eacc79d34954c2fdcef1367d5732 |
1 /*
2 *
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 */
6 /*
7 * entry.S contains the system-call and fault low-level handling routines.
8 * This also contains the timer-interrupt handler, as well as all interrupts
9 * and faults that can result in a task-switch.
10 *
11 * NOTE: This code handles signal-recognition, which happens every time
12 * after a timer-interrupt and after each system call.
13 *
14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
15 * on a 486.
16 *
17 * Stack layout in 'syscall_exit':
18 * ptrace needs to have all regs on the stack.
19 * if the order here is changed, it needs to be
20 * updated in fork.c:copy_process, signal.c:do_signal,
21 * ptrace.c and ptrace.h
22 *
23 * 0(%esp) - %ebx
24 * 4(%esp) - %ecx
25 * 8(%esp) - %edx
26 * C(%esp) - %esi
27 * 10(%esp) - %edi
28 * 14(%esp) - %ebp
29 * 18(%esp) - %eax
30 * 1C(%esp) - %ds
31 * 20(%esp) - %es
32 * 24(%esp) - %fs
33 * 28(%esp) - orig_eax
34 * 2C(%esp) - %eip
35 * 30(%esp) - %cs
36 * 34(%esp) - %eflags
37 * 38(%esp) - %oldesp
38 * 3C(%esp) - %oldss
39 *
40 * "current" is in register %ebx during any slow entries.
41 */
43 #include <linux/linkage.h>
44 #include <asm/thread_info.h>
45 #include <asm/irqflags.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
48 #include <asm/smp.h>
49 #include <asm/page.h>
50 #include <asm/desc.h>
51 #include <asm/percpu.h>
52 #include <asm/dwarf2.h>
53 #include <asm/processor-flags.h>
54 #include "irq_vectors.h"
56 /*
57 * We use macros for low-level operations which need to be overridden
58 * for paravirtualization. The following will never clobber any registers:
59 * INTERRUPT_RETURN (aka. "iret")
60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61 * ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit").
62 *
63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
65 * Allowing a register to be clobbered can shrink the paravirt replacement
66 * enough to patch inline, increasing performance.
67 */
69 #define nr_syscalls ((syscall_table_size)/4)
71 #ifdef CONFIG_PREEMPT
72 #define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
73 #else
74 #define preempt_stop(clobbers)
75 #define resume_kernel restore_nocheck
76 #endif
78 .macro TRACE_IRQS_IRET
79 #ifdef CONFIG_TRACE_IRQFLAGS
80 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off?
81 jz 1f
82 TRACE_IRQS_ON
83 1:
84 #endif
85 .endm
87 #ifdef CONFIG_VM86
88 #define resume_userspace_sig check_userspace
89 #else
90 #define resume_userspace_sig resume_userspace
91 #endif
93 #define SAVE_ALL \
94 cld; \
95 pushl %fs; \
96 CFI_ADJUST_CFA_OFFSET 4;\
97 /*CFI_REL_OFFSET fs, 0;*/\
98 pushl %es; \
99 CFI_ADJUST_CFA_OFFSET 4;\
100 /*CFI_REL_OFFSET es, 0;*/\
101 pushl %ds; \
102 CFI_ADJUST_CFA_OFFSET 4;\
103 /*CFI_REL_OFFSET ds, 0;*/\
104 pushl %eax; \
105 CFI_ADJUST_CFA_OFFSET 4;\
106 CFI_REL_OFFSET eax, 0;\
107 pushl %ebp; \
108 CFI_ADJUST_CFA_OFFSET 4;\
109 CFI_REL_OFFSET ebp, 0;\
110 pushl %edi; \
111 CFI_ADJUST_CFA_OFFSET 4;\
112 CFI_REL_OFFSET edi, 0;\
113 pushl %esi; \
114 CFI_ADJUST_CFA_OFFSET 4;\
115 CFI_REL_OFFSET esi, 0;\
116 pushl %edx; \
117 CFI_ADJUST_CFA_OFFSET 4;\
118 CFI_REL_OFFSET edx, 0;\
119 pushl %ecx; \
120 CFI_ADJUST_CFA_OFFSET 4;\
121 CFI_REL_OFFSET ecx, 0;\
122 pushl %ebx; \
123 CFI_ADJUST_CFA_OFFSET 4;\
124 CFI_REL_OFFSET ebx, 0;\
125 movl $(__USER_DS), %edx; \
126 movl %edx, %ds; \
127 movl %edx, %es; \
128 movl $(__KERNEL_PERCPU), %edx; \
129 movl %edx, %fs
131 #define RESTORE_INT_REGS \
132 popl %ebx; \
133 CFI_ADJUST_CFA_OFFSET -4;\
134 CFI_RESTORE ebx;\
135 popl %ecx; \
136 CFI_ADJUST_CFA_OFFSET -4;\
137 CFI_RESTORE ecx;\
138 popl %edx; \
139 CFI_ADJUST_CFA_OFFSET -4;\
140 CFI_RESTORE edx;\
141 popl %esi; \
142 CFI_ADJUST_CFA_OFFSET -4;\
143 CFI_RESTORE esi;\
144 popl %edi; \
145 CFI_ADJUST_CFA_OFFSET -4;\
146 CFI_RESTORE edi;\
147 popl %ebp; \
148 CFI_ADJUST_CFA_OFFSET -4;\
149 CFI_RESTORE ebp;\
150 popl %eax; \
151 CFI_ADJUST_CFA_OFFSET -4;\
152 CFI_RESTORE eax
154 #define RESTORE_REGS \
155 RESTORE_INT_REGS; \
156 1: popl %ds; \
157 CFI_ADJUST_CFA_OFFSET -4;\
158 /*CFI_RESTORE ds;*/\
159 2: popl %es; \
160 CFI_ADJUST_CFA_OFFSET -4;\
161 /*CFI_RESTORE es;*/\
162 3: popl %fs; \
163 CFI_ADJUST_CFA_OFFSET -4;\
164 /*CFI_RESTORE fs;*/\
165 .pushsection .fixup,"ax"; \
166 4: movl $0,(%esp); \
167 jmp 1b; \
168 5: movl $0,(%esp); \
169 jmp 2b; \
170 6: movl $0,(%esp); \
171 jmp 3b; \
172 .section __ex_table,"a";\
173 .align 4; \
174 .long 1b,4b; \
175 .long 2b,5b; \
176 .long 3b,6b; \
177 .popsection
179 #define RING0_INT_FRAME \
180 CFI_STARTPROC simple;\
181 CFI_SIGNAL_FRAME;\
182 CFI_DEF_CFA esp, 3*4;\
183 /*CFI_OFFSET cs, -2*4;*/\
184 CFI_OFFSET eip, -3*4
186 #define RING0_EC_FRAME \
187 CFI_STARTPROC simple;\
188 CFI_SIGNAL_FRAME;\
189 CFI_DEF_CFA esp, 4*4;\
190 /*CFI_OFFSET cs, -2*4;*/\
191 CFI_OFFSET eip, -3*4
193 #define RING0_PTREGS_FRAME \
194 CFI_STARTPROC simple;\
195 CFI_SIGNAL_FRAME;\
196 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
197 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
198 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
199 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
200 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
201 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
202 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
203 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
204 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
205 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
206 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
207 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
209 ENTRY(ret_from_fork)
210 CFI_STARTPROC
211 pushl %eax
212 CFI_ADJUST_CFA_OFFSET 4
213 call schedule_tail
214 GET_THREAD_INFO(%ebp)
215 popl %eax
216 CFI_ADJUST_CFA_OFFSET -4
217 pushl $0x0202 # Reset kernel eflags
218 CFI_ADJUST_CFA_OFFSET 4
219 popfl
220 CFI_ADJUST_CFA_OFFSET -4
221 jmp syscall_exit
222 CFI_ENDPROC
223 END(ret_from_fork)
225 /*
226 * Return to user mode is not as complex as all this looks,
227 * but we want the default path for a system call return to
228 * go as quickly as possible which is why some of this is
229 * less clear than it otherwise should be.
230 */
232 # userspace resumption stub bypassing syscall exit tracing
233 ALIGN
234 RING0_PTREGS_FRAME
235 ret_from_exception:
236 preempt_stop(CLBR_ANY)
237 ret_from_intr:
238 GET_THREAD_INFO(%ebp)
239 check_userspace:
240 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
241 movb PT_CS(%esp), %al
242 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
243 cmpl $USER_RPL, %eax
244 jb resume_kernel # not returning to v8086 or userspace
246 ENTRY(resume_userspace)
247 LOCKDEP_SYS_EXIT
248 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
249 # setting need_resched or sigpending
250 # between sampling and the iret
251 TRACE_IRQS_OFF
252 movl TI_flags(%ebp), %ecx
253 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
254 # int/exception return?
255 jne work_pending
256 jmp restore_all
257 END(ret_from_exception)
259 #ifdef CONFIG_PREEMPT
260 ENTRY(resume_kernel)
261 DISABLE_INTERRUPTS(CLBR_ANY)
262 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
263 jnz restore_nocheck
264 need_resched:
265 movl TI_flags(%ebp), %ecx # need_resched set ?
266 testb $_TIF_NEED_RESCHED, %cl
267 jz restore_all
268 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ?
269 jz restore_all
270 call preempt_schedule_irq
271 jmp need_resched
272 END(resume_kernel)
273 #endif
274 CFI_ENDPROC
276 /* SYSENTER_RETURN points to after the "sysenter" instruction in
277 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
279 # sysenter call handler stub
280 ENTRY(ia32_sysenter_target)
281 CFI_STARTPROC simple
282 CFI_SIGNAL_FRAME
283 CFI_DEF_CFA esp, 0
284 CFI_REGISTER esp, ebp
285 movl TSS_sysenter_sp0(%esp),%esp
286 sysenter_past_esp:
287 /*
288 * Interrupts are disabled here, but we can't trace it until
289 * enough kernel state to call TRACE_IRQS_OFF can be called - but
290 * we immediately enable interrupts at that point anyway.
291 */
292 pushl $(__USER_DS)
293 CFI_ADJUST_CFA_OFFSET 4
294 /*CFI_REL_OFFSET ss, 0*/
295 pushl %ebp
296 CFI_ADJUST_CFA_OFFSET 4
297 CFI_REL_OFFSET esp, 0
298 pushfl
299 orl $X86_EFLAGS_IF, (%esp)
300 CFI_ADJUST_CFA_OFFSET 4
301 pushl $(__USER_CS)
302 CFI_ADJUST_CFA_OFFSET 4
303 /*CFI_REL_OFFSET cs, 0*/
304 /*
305 * Push current_thread_info()->sysenter_return to the stack.
306 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
307 * pushed above; +8 corresponds to copy_thread's esp0 setting.
308 */
309 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
310 CFI_ADJUST_CFA_OFFSET 4
311 CFI_REL_OFFSET eip, 0
313 pushl %eax
314 CFI_ADJUST_CFA_OFFSET 4
315 SAVE_ALL
316 ENABLE_INTERRUPTS(CLBR_NONE)
318 /*
319 * Load the potential sixth argument from user stack.
320 * Careful about security.
321 */
322 cmpl $__PAGE_OFFSET-3,%ebp
323 jae syscall_fault
324 1: movl (%ebp),%ebp
325 movl %ebp,PT_EBP(%esp)
326 .section __ex_table,"a"
327 .align 4
328 .long 1b,syscall_fault
329 .previous
331 GET_THREAD_INFO(%ebp)
333 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
334 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
335 jnz syscall_trace_entry
336 cmpl $(nr_syscalls), %eax
337 jae syscall_badsys
338 call *sys_call_table(,%eax,4)
339 movl %eax,PT_EAX(%esp)
340 LOCKDEP_SYS_EXIT
341 DISABLE_INTERRUPTS(CLBR_ANY)
342 TRACE_IRQS_OFF
343 movl TI_flags(%ebp), %ecx
344 testw $_TIF_ALLWORK_MASK, %cx
345 jne syscall_exit_work
346 /* if something modifies registers it must also disable sysexit */
347 movl PT_EIP(%esp), %edx
348 movl PT_OLDESP(%esp), %ecx
349 xorl %ebp,%ebp
350 TRACE_IRQS_ON
351 1: mov PT_FS(%esp), %fs
352 ENABLE_INTERRUPTS_SYSCALL_RET
353 CFI_ENDPROC
354 .pushsection .fixup,"ax"
355 2: movl $0,PT_FS(%esp)
356 jmp 1b
357 .section __ex_table,"a"
358 .align 4
359 .long 1b,2b
360 .popsection
361 ENDPROC(ia32_sysenter_target)
363 # system call handler stub
364 ENTRY(system_call)
365 RING0_INT_FRAME # can't unwind into user space anyway
366 pushl %eax # save orig_eax
367 CFI_ADJUST_CFA_OFFSET 4
368 SAVE_ALL
369 GET_THREAD_INFO(%ebp)
370 # system call tracing in operation / emulation
371 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
372 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
373 jnz syscall_trace_entry
374 cmpl $(nr_syscalls), %eax
375 jae syscall_badsys
376 syscall_call:
377 call *sys_call_table(,%eax,4)
378 movl %eax,PT_EAX(%esp) # store the return value
379 syscall_exit:
380 LOCKDEP_SYS_EXIT
381 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
382 # setting need_resched or sigpending
383 # between sampling and the iret
384 TRACE_IRQS_OFF
385 testl $X86_EFLAGS_TF,PT_EFLAGS(%esp) # If tracing set singlestep flag on exit
386 jz no_singlestep
387 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
388 no_singlestep:
389 movl TI_flags(%ebp), %ecx
390 testw $_TIF_ALLWORK_MASK, %cx # current->work
391 jne syscall_exit_work
393 restore_all:
394 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
395 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
396 # are returning to the kernel.
397 # See comments in process.c:copy_thread() for details.
398 movb PT_OLDSS(%esp), %ah
399 movb PT_CS(%esp), %al
400 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
401 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
402 CFI_REMEMBER_STATE
403 je ldt_ss # returning to user-space with LDT SS
404 restore_nocheck:
405 TRACE_IRQS_IRET
406 restore_nocheck_notrace:
407 RESTORE_REGS
408 addl $4, %esp # skip orig_eax/error_code
409 CFI_ADJUST_CFA_OFFSET -4
410 irq_return:
411 INTERRUPT_RETURN
412 .section .fixup,"ax"
413 ENTRY(iret_exc)
414 pushl $0 # no error code
415 pushl $do_iret_error
416 jmp error_code
417 .previous
418 .section __ex_table,"a"
419 .align 4
420 .long irq_return,iret_exc
421 .previous
423 CFI_RESTORE_STATE
424 ldt_ss:
425 larl PT_OLDSS(%esp), %eax
426 jnz restore_nocheck
427 testl $0x00400000, %eax # returning to 32bit stack?
428 jnz restore_nocheck # allright, normal return
430 #ifdef CONFIG_PARAVIRT
431 /*
432 * The kernel can't run on a non-flat stack if paravirt mode
433 * is active. Rather than try to fixup the high bits of
434 * ESP, bypass this code entirely. This may break DOSemu
435 * and/or Wine support in a paravirt VM, although the option
436 * is still available to implement the setting of the high
437 * 16-bits in the INTERRUPT_RETURN paravirt-op.
438 */
439 cmpl $0, pv_info+PARAVIRT_enabled
440 jne restore_nocheck
441 #endif
443 /* If returning to userspace with 16bit stack,
444 * try to fix the higher word of ESP, as the CPU
445 * won't restore it.
446 * This is an "official" bug of all the x86-compatible
447 * CPUs, which we can try to work around to make
448 * dosemu and wine happy. */
449 movl PT_OLDESP(%esp), %eax
450 movl %esp, %edx
451 call patch_espfix_desc
452 pushl $__ESPFIX_SS
453 CFI_ADJUST_CFA_OFFSET 4
454 pushl %eax
455 CFI_ADJUST_CFA_OFFSET 4
456 DISABLE_INTERRUPTS(CLBR_EAX)
457 TRACE_IRQS_OFF
458 lss (%esp), %esp
459 CFI_ADJUST_CFA_OFFSET -8
460 jmp restore_nocheck
461 CFI_ENDPROC
462 ENDPROC(system_call)
464 # perform work that needs to be done immediately before resumption
465 ALIGN
466 RING0_PTREGS_FRAME # can't unwind into user space anyway
467 work_pending:
468 testb $_TIF_NEED_RESCHED, %cl
469 jz work_notifysig
470 work_resched:
471 call schedule
472 LOCKDEP_SYS_EXIT
473 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
474 # setting need_resched or sigpending
475 # between sampling and the iret
476 TRACE_IRQS_OFF
477 movl TI_flags(%ebp), %ecx
478 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
479 # than syscall tracing?
480 jz restore_all
481 testb $_TIF_NEED_RESCHED, %cl
482 jnz work_resched
484 work_notifysig: # deal with pending signals and
485 # notify-resume requests
486 #ifdef CONFIG_VM86
487 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
488 movl %esp, %eax
489 jne work_notifysig_v86 # returning to kernel-space or
490 # vm86-space
491 xorl %edx, %edx
492 call do_notify_resume
493 jmp resume_userspace_sig
495 ALIGN
496 work_notifysig_v86:
497 pushl %ecx # save ti_flags for do_notify_resume
498 CFI_ADJUST_CFA_OFFSET 4
499 call save_v86_state # %eax contains pt_regs pointer
500 popl %ecx
501 CFI_ADJUST_CFA_OFFSET -4
502 movl %eax, %esp
503 #else
504 movl %esp, %eax
505 #endif
506 xorl %edx, %edx
507 call do_notify_resume
508 jmp resume_userspace_sig
509 END(work_pending)
511 # perform syscall exit tracing
512 ALIGN
513 syscall_trace_entry:
514 movl $-ENOSYS,PT_EAX(%esp)
515 movl %esp, %eax
516 xorl %edx,%edx
517 call do_syscall_trace
518 cmpl $0, %eax
519 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
520 # so must skip actual syscall
521 movl PT_ORIG_EAX(%esp), %eax
522 cmpl $(nr_syscalls), %eax
523 jnae syscall_call
524 jmp syscall_exit
525 END(syscall_trace_entry)
527 # perform syscall exit tracing
528 ALIGN
529 syscall_exit_work:
530 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
531 jz work_pending
532 TRACE_IRQS_ON
533 ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call
534 # schedule() instead
535 movl %esp, %eax
536 movl $1, %edx
537 call do_syscall_trace
538 jmp resume_userspace
539 END(syscall_exit_work)
540 CFI_ENDPROC
542 RING0_INT_FRAME # can't unwind into user space anyway
543 syscall_fault:
544 GET_THREAD_INFO(%ebp)
545 movl $-EFAULT,PT_EAX(%esp)
546 jmp resume_userspace
547 END(syscall_fault)
549 syscall_badsys:
550 movl $-ENOSYS,PT_EAX(%esp)
551 jmp resume_userspace
552 END(syscall_badsys)
553 CFI_ENDPROC
555 #define FIXUP_ESPFIX_STACK \
556 /* since we are on a wrong stack, we cant make it a C code :( */ \
557 PER_CPU(gdt_page, %ebx); \
558 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
559 addl %esp, %eax; \
560 pushl $__KERNEL_DS; \
561 CFI_ADJUST_CFA_OFFSET 4; \
562 pushl %eax; \
563 CFI_ADJUST_CFA_OFFSET 4; \
564 lss (%esp), %esp; \
565 CFI_ADJUST_CFA_OFFSET -8;
566 #define UNWIND_ESPFIX_STACK \
567 movl %ss, %eax; \
568 /* see if on espfix stack */ \
569 cmpw $__ESPFIX_SS, %ax; \
570 jne 27f; \
571 movl $__KERNEL_DS, %eax; \
572 movl %eax, %ds; \
573 movl %eax, %es; \
574 /* switch to normal stack */ \
575 FIXUP_ESPFIX_STACK; \
576 27:;
578 /*
579 * Build the entry stubs and pointer table with
580 * some assembler magic.
581 */
582 .section .rodata,"a"
583 ENTRY(interrupt)
584 .text
586 ENTRY(irq_entries_start)
587 RING0_INT_FRAME
588 vector=0
589 .rept NR_IRQS
590 ALIGN
591 .if vector
592 CFI_ADJUST_CFA_OFFSET -4
593 .endif
594 1: pushl $~(vector)
595 CFI_ADJUST_CFA_OFFSET 4
596 jmp common_interrupt
597 .previous
598 .long 1b
599 .text
600 vector=vector+1
601 .endr
602 END(irq_entries_start)
604 .previous
605 END(interrupt)
606 .previous
608 /*
609 * the CPU automatically disables interrupts when executing an IRQ vector,
610 * so IRQ-flags tracing has to follow that:
611 */
612 ALIGN
613 common_interrupt:
614 SAVE_ALL
615 TRACE_IRQS_OFF
616 movl %esp,%eax
617 call do_IRQ
618 jmp ret_from_intr
619 ENDPROC(common_interrupt)
620 CFI_ENDPROC
622 #define BUILD_INTERRUPT(name, nr) \
623 ENTRY(name) \
624 RING0_INT_FRAME; \
625 pushl $~(nr); \
626 CFI_ADJUST_CFA_OFFSET 4; \
627 SAVE_ALL; \
628 TRACE_IRQS_OFF \
629 movl %esp,%eax; \
630 call smp_##name; \
631 jmp ret_from_intr; \
632 CFI_ENDPROC; \
633 ENDPROC(name)
635 /* The include is where all of the SMP etc. interrupts come from */
636 #include "entry_arch.h"
638 KPROBE_ENTRY(page_fault)
639 RING0_EC_FRAME
640 pushl $do_page_fault
641 CFI_ADJUST_CFA_OFFSET 4
642 ALIGN
643 error_code:
644 /* the function address is in %fs's slot on the stack */
645 pushl %es
646 CFI_ADJUST_CFA_OFFSET 4
647 /*CFI_REL_OFFSET es, 0*/
648 pushl %ds
649 CFI_ADJUST_CFA_OFFSET 4
650 /*CFI_REL_OFFSET ds, 0*/
651 pushl %eax
652 CFI_ADJUST_CFA_OFFSET 4
653 CFI_REL_OFFSET eax, 0
654 pushl %ebp
655 CFI_ADJUST_CFA_OFFSET 4
656 CFI_REL_OFFSET ebp, 0
657 pushl %edi
658 CFI_ADJUST_CFA_OFFSET 4
659 CFI_REL_OFFSET edi, 0
660 pushl %esi
661 CFI_ADJUST_CFA_OFFSET 4
662 CFI_REL_OFFSET esi, 0
663 pushl %edx
664 CFI_ADJUST_CFA_OFFSET 4
665 CFI_REL_OFFSET edx, 0
666 pushl %ecx
667 CFI_ADJUST_CFA_OFFSET 4
668 CFI_REL_OFFSET ecx, 0
669 pushl %ebx
670 CFI_ADJUST_CFA_OFFSET 4
671 CFI_REL_OFFSET ebx, 0
672 cld
673 pushl %fs
674 CFI_ADJUST_CFA_OFFSET 4
675 /*CFI_REL_OFFSET fs, 0*/
676 movl $(__KERNEL_PERCPU), %ecx
677 movl %ecx, %fs
678 UNWIND_ESPFIX_STACK
679 popl %ecx
680 CFI_ADJUST_CFA_OFFSET -4
681 /*CFI_REGISTER es, ecx*/
682 movl PT_FS(%esp), %edi # get the function address
683 movl PT_ORIG_EAX(%esp), %edx # get the error code
684 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
685 mov %ecx, PT_FS(%esp)
686 /*CFI_REL_OFFSET fs, ES*/
687 movl $(__USER_DS), %ecx
688 movl %ecx, %ds
689 movl %ecx, %es
690 movl %esp,%eax # pt_regs pointer
691 call *%edi
692 jmp ret_from_exception
693 CFI_ENDPROC
694 KPROBE_END(page_fault)
696 ENTRY(coprocessor_error)
697 RING0_INT_FRAME
698 pushl $0
699 CFI_ADJUST_CFA_OFFSET 4
700 pushl $do_coprocessor_error
701 CFI_ADJUST_CFA_OFFSET 4
702 jmp error_code
703 CFI_ENDPROC
704 END(coprocessor_error)
706 ENTRY(simd_coprocessor_error)
707 RING0_INT_FRAME
708 pushl $0
709 CFI_ADJUST_CFA_OFFSET 4
710 pushl $do_simd_coprocessor_error
711 CFI_ADJUST_CFA_OFFSET 4
712 jmp error_code
713 CFI_ENDPROC
714 END(simd_coprocessor_error)
716 ENTRY(device_not_available)
717 RING0_INT_FRAME
718 pushl $-1 # mark this as an int
719 CFI_ADJUST_CFA_OFFSET 4
720 SAVE_ALL
721 GET_CR0_INTO_EAX
722 testl $0x4, %eax # EM (math emulation bit)
723 jne device_not_available_emulate
724 preempt_stop(CLBR_ANY)
725 call math_state_restore
726 jmp ret_from_exception
727 device_not_available_emulate:
728 pushl $0 # temporary storage for ORIG_EIP
729 CFI_ADJUST_CFA_OFFSET 4
730 call math_emulate
731 addl $4, %esp
732 CFI_ADJUST_CFA_OFFSET -4
733 jmp ret_from_exception
734 CFI_ENDPROC
735 END(device_not_available)
737 /*
738 * Debug traps and NMI can happen at the one SYSENTER instruction
739 * that sets up the real kernel stack. Check here, since we can't
740 * allow the wrong stack to be used.
741 *
742 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
743 * already pushed 3 words if it hits on the sysenter instruction:
744 * eflags, cs and eip.
745 *
746 * We just load the right stack, and push the three (known) values
747 * by hand onto the new stack - while updating the return eip past
748 * the instruction that would have done it for sysenter.
749 */
750 #define FIX_STACK(offset, ok, label) \
751 cmpw $__KERNEL_CS,4(%esp); \
752 jne ok; \
753 label: \
754 movl TSS_sysenter_sp0+offset(%esp),%esp; \
755 CFI_DEF_CFA esp, 0; \
756 CFI_UNDEFINED eip; \
757 pushfl; \
758 CFI_ADJUST_CFA_OFFSET 4; \
759 pushl $__KERNEL_CS; \
760 CFI_ADJUST_CFA_OFFSET 4; \
761 pushl $sysenter_past_esp; \
762 CFI_ADJUST_CFA_OFFSET 4; \
763 CFI_REL_OFFSET eip, 0
765 KPROBE_ENTRY(debug)
766 RING0_INT_FRAME
767 cmpl $ia32_sysenter_target,(%esp)
768 jne debug_stack_correct
769 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
770 debug_stack_correct:
771 pushl $-1 # mark this as an int
772 CFI_ADJUST_CFA_OFFSET 4
773 SAVE_ALL
774 xorl %edx,%edx # error code 0
775 movl %esp,%eax # pt_regs pointer
776 call do_debug
777 jmp ret_from_exception
778 CFI_ENDPROC
779 KPROBE_END(debug)
781 /*
782 * NMI is doubly nasty. It can happen _while_ we're handling
783 * a debug fault, and the debug fault hasn't yet been able to
784 * clear up the stack. So we first check whether we got an
785 * NMI on the sysenter entry path, but after that we need to
786 * check whether we got an NMI on the debug path where the debug
787 * fault happened on the sysenter path.
788 */
789 KPROBE_ENTRY(nmi)
790 RING0_INT_FRAME
791 pushl %eax
792 CFI_ADJUST_CFA_OFFSET 4
793 movl %ss, %eax
794 cmpw $__ESPFIX_SS, %ax
795 popl %eax
796 CFI_ADJUST_CFA_OFFSET -4
797 je nmi_espfix_stack
798 cmpl $ia32_sysenter_target,(%esp)
799 je nmi_stack_fixup
800 pushl %eax
801 CFI_ADJUST_CFA_OFFSET 4
802 movl %esp,%eax
803 /* Do not access memory above the end of our stack page,
804 * it might not exist.
805 */
806 andl $(THREAD_SIZE-1),%eax
807 cmpl $(THREAD_SIZE-20),%eax
808 popl %eax
809 CFI_ADJUST_CFA_OFFSET -4
810 jae nmi_stack_correct
811 cmpl $ia32_sysenter_target,12(%esp)
812 je nmi_debug_stack_check
813 nmi_stack_correct:
814 /* We have a RING0_INT_FRAME here */
815 pushl %eax
816 CFI_ADJUST_CFA_OFFSET 4
817 SAVE_ALL
818 xorl %edx,%edx # zero error code
819 movl %esp,%eax # pt_regs pointer
820 call do_nmi
821 jmp restore_nocheck_notrace
822 CFI_ENDPROC
824 nmi_stack_fixup:
825 RING0_INT_FRAME
826 FIX_STACK(12,nmi_stack_correct, 1)
827 jmp nmi_stack_correct
829 nmi_debug_stack_check:
830 /* We have a RING0_INT_FRAME here */
831 cmpw $__KERNEL_CS,16(%esp)
832 jne nmi_stack_correct
833 cmpl $debug,(%esp)
834 jb nmi_stack_correct
835 cmpl $debug_esp_fix_insn,(%esp)
836 ja nmi_stack_correct
837 FIX_STACK(24,nmi_stack_correct, 1)
838 jmp nmi_stack_correct
840 nmi_espfix_stack:
841 /* We have a RING0_INT_FRAME here.
842 *
843 * create the pointer to lss back
844 */
845 pushl %ss
846 CFI_ADJUST_CFA_OFFSET 4
847 pushl %esp
848 CFI_ADJUST_CFA_OFFSET 4
849 addw $4, (%esp)
850 /* copy the iret frame of 12 bytes */
851 .rept 3
852 pushl 16(%esp)
853 CFI_ADJUST_CFA_OFFSET 4
854 .endr
855 pushl %eax
856 CFI_ADJUST_CFA_OFFSET 4
857 SAVE_ALL
858 FIXUP_ESPFIX_STACK # %eax == %esp
859 xorl %edx,%edx # zero error code
860 call do_nmi
861 RESTORE_REGS
862 lss 12+4(%esp), %esp # back to espfix stack
863 CFI_ADJUST_CFA_OFFSET -24
864 jmp irq_return
865 CFI_ENDPROC
866 KPROBE_END(nmi)
868 #ifdef CONFIG_PARAVIRT
869 ENTRY(native_iret)
870 iret
871 .section __ex_table,"a"
872 .align 4
873 .long native_iret, iret_exc
874 .previous
875 END(native_iret)
877 ENTRY(native_irq_enable_syscall_ret)
878 sti
879 sysexit
880 END(native_irq_enable_syscall_ret)
881 #endif
883 KPROBE_ENTRY(int3)
884 RING0_INT_FRAME
885 pushl $-1 # mark this as an int
886 CFI_ADJUST_CFA_OFFSET 4
887 SAVE_ALL
888 xorl %edx,%edx # zero error code
889 movl %esp,%eax # pt_regs pointer
890 call do_int3
891 jmp ret_from_exception
892 CFI_ENDPROC
893 KPROBE_END(int3)
895 ENTRY(overflow)
896 RING0_INT_FRAME
897 pushl $0
898 CFI_ADJUST_CFA_OFFSET 4
899 pushl $do_overflow
900 CFI_ADJUST_CFA_OFFSET 4
901 jmp error_code
902 CFI_ENDPROC
903 END(overflow)
905 ENTRY(bounds)
906 RING0_INT_FRAME
907 pushl $0
908 CFI_ADJUST_CFA_OFFSET 4
909 pushl $do_bounds
910 CFI_ADJUST_CFA_OFFSET 4
911 jmp error_code
912 CFI_ENDPROC
913 END(bounds)
915 ENTRY(invalid_op)
916 RING0_INT_FRAME
917 pushl $0
918 CFI_ADJUST_CFA_OFFSET 4
919 pushl $do_invalid_op
920 CFI_ADJUST_CFA_OFFSET 4
921 jmp error_code
922 CFI_ENDPROC
923 END(invalid_op)
925 ENTRY(coprocessor_segment_overrun)
926 RING0_INT_FRAME
927 pushl $0
928 CFI_ADJUST_CFA_OFFSET 4
929 pushl $do_coprocessor_segment_overrun
930 CFI_ADJUST_CFA_OFFSET 4
931 jmp error_code
932 CFI_ENDPROC
933 END(coprocessor_segment_overrun)
935 ENTRY(invalid_TSS)
936 RING0_EC_FRAME
937 pushl $do_invalid_TSS
938 CFI_ADJUST_CFA_OFFSET 4
939 jmp error_code
940 CFI_ENDPROC
941 END(invalid_TSS)
943 ENTRY(segment_not_present)
944 RING0_EC_FRAME
945 pushl $do_segment_not_present
946 CFI_ADJUST_CFA_OFFSET 4
947 jmp error_code
948 CFI_ENDPROC
949 END(segment_not_present)
951 ENTRY(stack_segment)
952 RING0_EC_FRAME
953 pushl $do_stack_segment
954 CFI_ADJUST_CFA_OFFSET 4
955 jmp error_code
956 CFI_ENDPROC
957 END(stack_segment)
959 KPROBE_ENTRY(general_protection)
960 RING0_EC_FRAME
961 pushl $do_general_protection
962 CFI_ADJUST_CFA_OFFSET 4
963 jmp error_code
964 CFI_ENDPROC
965 KPROBE_END(general_protection)
967 ENTRY(alignment_check)
968 RING0_EC_FRAME
969 pushl $do_alignment_check
970 CFI_ADJUST_CFA_OFFSET 4
971 jmp error_code
972 CFI_ENDPROC
973 END(alignment_check)
975 ENTRY(divide_error)
976 RING0_INT_FRAME
977 pushl $0 # no error code
978 CFI_ADJUST_CFA_OFFSET 4
979 pushl $do_divide_error
980 CFI_ADJUST_CFA_OFFSET 4
981 jmp error_code
982 CFI_ENDPROC
983 END(divide_error)
985 #ifdef CONFIG_X86_MCE
986 ENTRY(machine_check)
987 RING0_INT_FRAME
988 pushl $0
989 CFI_ADJUST_CFA_OFFSET 4
990 pushl machine_check_vector
991 CFI_ADJUST_CFA_OFFSET 4
992 jmp error_code
993 CFI_ENDPROC
994 END(machine_check)
995 #endif
997 ENTRY(spurious_interrupt_bug)
998 RING0_INT_FRAME
999 pushl $0
1000 CFI_ADJUST_CFA_OFFSET 4
1001 pushl $do_spurious_interrupt_bug
1002 CFI_ADJUST_CFA_OFFSET 4
1003 jmp error_code
1004 CFI_ENDPROC
1005 END(spurious_interrupt_bug)
1007 ENTRY(kernel_thread_helper)
1008 pushl $0 # fake return address for unwinder
1009 CFI_STARTPROC
1010 movl %edx,%eax
1011 push %edx
1012 CFI_ADJUST_CFA_OFFSET 4
1013 call *%ebx
1014 push %eax
1015 CFI_ADJUST_CFA_OFFSET 4
1016 call do_exit
1017 CFI_ENDPROC
1018 ENDPROC(kernel_thread_helper)
1020 #ifdef CONFIG_XEN
1021 /* Xen doesn't set %esp to be precisely what the normal sysenter
1022 entrypoint expects, so fix it up before using the normal path. */
1023 ENTRY(xen_sysenter_target)
1024 RING0_INT_FRAME
1025 addl $5*4, %esp /* remove xen-provided frame */
1026 jmp sysenter_past_esp
1028 ENTRY(xen_hypervisor_callback)
1029 CFI_STARTPROC
1030 pushl $0
1031 CFI_ADJUST_CFA_OFFSET 4
1032 SAVE_ALL
1033 TRACE_IRQS_OFF
1035 /* Check to see if we got the event in the critical
1036 region in xen_iret_direct, after we've reenabled
1037 events and checked for pending events. This simulates
1038 iret instruction's behaviour where it delivers a
1039 pending interrupt when enabling interrupts. */
1040 movl PT_EIP(%esp),%eax
1041 cmpl $xen_iret_start_crit,%eax
1042 jb 1f
1043 cmpl $xen_iret_end_crit,%eax
1044 jae 1f
1046 jmp xen_iret_crit_fixup
1048 ENTRY(xen_do_upcall)
1049 1: mov %esp, %eax
1050 call xen_evtchn_do_upcall
1051 jmp ret_from_intr
1052 CFI_ENDPROC
1053 ENDPROC(xen_hypervisor_callback)
1055 # Hypervisor uses this for application faults while it executes.
1056 # We get here for two reasons:
1057 # 1. Fault while reloading DS, ES, FS or GS
1058 # 2. Fault while executing IRET
1059 # Category 1 we fix up by reattempting the load, and zeroing the segment
1060 # register if the load fails.
1061 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
1062 # normal Linux return path in this case because if we use the IRET hypercall
1063 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1064 # We distinguish between categories by maintaining a status value in EAX.
1065 ENTRY(xen_failsafe_callback)
1066 CFI_STARTPROC
1067 pushl %eax
1068 CFI_ADJUST_CFA_OFFSET 4
1069 movl $1,%eax
1070 1: mov 4(%esp),%ds
1071 2: mov 8(%esp),%es
1072 3: mov 12(%esp),%fs
1073 4: mov 16(%esp),%gs
1074 testl %eax,%eax
1075 popl %eax
1076 CFI_ADJUST_CFA_OFFSET -4
1077 lea 16(%esp),%esp
1078 CFI_ADJUST_CFA_OFFSET -16
1079 jz 5f
1080 addl $16,%esp
1081 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET)
1082 5: pushl $0 # EAX == 0 => Category 1 (Bad segment)
1083 CFI_ADJUST_CFA_OFFSET 4
1084 SAVE_ALL
1085 jmp ret_from_exception
1086 CFI_ENDPROC
1088 .section .fixup,"ax"
1089 6: xorl %eax,%eax
1090 movl %eax,4(%esp)
1091 jmp 1b
1092 7: xorl %eax,%eax
1093 movl %eax,8(%esp)
1094 jmp 2b
1095 8: xorl %eax,%eax
1096 movl %eax,12(%esp)
1097 jmp 3b
1098 9: xorl %eax,%eax
1099 movl %eax,16(%esp)
1100 jmp 4b
1101 .previous
1102 .section __ex_table,"a"
1103 .align 4
1104 .long 1b,6b
1105 .long 2b,7b
1106 .long 3b,8b
1107 .long 4b,9b
1108 .previous
1109 ENDPROC(xen_failsafe_callback)
1111 #endif /* CONFIG_XEN */
1113 .section .rodata,"a"
1114 #include "syscall_table_32.S"
1116 syscall_table_size=(.-sys_call_table)