| blob | 2a609dc3271c3dc2008ad740133f314e5ed177b6 |
1 /*
2 *
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 */
6 /*
7 * entry.S contains the system-call and fault low-level handling routines.
8 * This also contains the timer-interrupt handler, as well as all interrupts
9 * and faults that can result in a task-switch.
10 *
11 * NOTE: This code handles signal-recognition, which happens every time
12 * after a timer-interrupt and after each system call.
13 *
14 * I changed all the .align's to 4 (16 byte alignment), as that's faster
15 * on a 486.
16 *
17 * Stack layout in 'syscall_exit':
18 * ptrace needs to have all regs on the stack.
19 * if the order here is changed, it needs to be
20 * updated in fork.c:copy_process, signal.c:do_signal,
21 * ptrace.c and ptrace.h
22 *
23 * 0(%esp) - %ebx
24 * 4(%esp) - %ecx
25 * 8(%esp) - %edx
26 * C(%esp) - %esi
27 * 10(%esp) - %edi
28 * 14(%esp) - %ebp
29 * 18(%esp) - %eax
30 * 1C(%esp) - %ds
31 * 20(%esp) - %es
32 * 24(%esp) - %fs
33 * 28(%esp) - orig_eax
34 * 2C(%esp) - %eip
35 * 30(%esp) - %cs
36 * 34(%esp) - %eflags
37 * 38(%esp) - %oldesp
38 * 3C(%esp) - %oldss
39 *
40 * "current" is in register %ebx during any slow entries.
41 */
43 #include <linux/linkage.h>
44 #include <asm/thread_info.h>
45 #include <asm/irqflags.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
48 #include <asm/smp.h>
49 #include <asm/page.h>
50 #include <asm/desc.h>
51 #include <asm/percpu.h>
52 #include <asm/dwarf2.h>
53 #include <asm/processor-flags.h>
54 #include "irq_vectors.h"
56 /*
57 * We use macros for low-level operations which need to be overridden
58 * for paravirtualization. The following will never clobber any registers:
59 * INTERRUPT_RETURN (aka. "iret")
60 * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax")
61 * ENABLE_INTERRUPTS_SYSCALL_RET (aka "sti; sysexit").
62 *
63 * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must
64 * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY).
65 * Allowing a register to be clobbered can shrink the paravirt replacement
66 * enough to patch inline, increasing performance.
67 */
69 #define nr_syscalls ((syscall_table_size)/4)
71 #ifdef CONFIG_PREEMPT
72 #define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
73 #else
74 #define preempt_stop(clobbers)
75 #define resume_kernel restore_nocheck
76 #endif
78 .macro TRACE_IRQS_IRET
79 #ifdef CONFIG_TRACE_IRQFLAGS
80 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off?
81 jz 1f
82 TRACE_IRQS_ON
83 1:
84 #endif
85 .endm
87 #ifdef CONFIG_VM86
88 #define resume_userspace_sig check_userspace
89 #else
90 #define resume_userspace_sig resume_userspace
91 #endif
93 #define SAVE_ALL \
94 cld; \
95 pushl %fs; \
96 CFI_ADJUST_CFA_OFFSET 4;\
97 /*CFI_REL_OFFSET fs, 0;*/\
98 pushl %es; \
99 CFI_ADJUST_CFA_OFFSET 4;\
100 /*CFI_REL_OFFSET es, 0;*/\
101 pushl %ds; \
102 CFI_ADJUST_CFA_OFFSET 4;\
103 /*CFI_REL_OFFSET ds, 0;*/\
104 pushl %eax; \
105 CFI_ADJUST_CFA_OFFSET 4;\
106 CFI_REL_OFFSET eax, 0;\
107 pushl %ebp; \
108 CFI_ADJUST_CFA_OFFSET 4;\
109 CFI_REL_OFFSET ebp, 0;\
110 pushl %edi; \
111 CFI_ADJUST_CFA_OFFSET 4;\
112 CFI_REL_OFFSET edi, 0;\
113 pushl %esi; \
114 CFI_ADJUST_CFA_OFFSET 4;\
115 CFI_REL_OFFSET esi, 0;\
116 pushl %edx; \
117 CFI_ADJUST_CFA_OFFSET 4;\
118 CFI_REL_OFFSET edx, 0;\
119 pushl %ecx; \
120 CFI_ADJUST_CFA_OFFSET 4;\
121 CFI_REL_OFFSET ecx, 0;\
122 pushl %ebx; \
123 CFI_ADJUST_CFA_OFFSET 4;\
124 CFI_REL_OFFSET ebx, 0;\
125 movl $(__USER_DS), %edx; \
126 movl %edx, %ds; \
127 movl %edx, %es; \
128 movl $(__KERNEL_PERCPU), %edx; \
129 movl %edx, %fs
131 #define RESTORE_INT_REGS \
132 popl %ebx; \
133 CFI_ADJUST_CFA_OFFSET -4;\
134 CFI_RESTORE ebx;\
135 popl %ecx; \
136 CFI_ADJUST_CFA_OFFSET -4;\
137 CFI_RESTORE ecx;\
138 popl %edx; \
139 CFI_ADJUST_CFA_OFFSET -4;\
140 CFI_RESTORE edx;\
141 popl %esi; \
142 CFI_ADJUST_CFA_OFFSET -4;\
143 CFI_RESTORE esi;\
144 popl %edi; \
145 CFI_ADJUST_CFA_OFFSET -4;\
146 CFI_RESTORE edi;\
147 popl %ebp; \
148 CFI_ADJUST_CFA_OFFSET -4;\
149 CFI_RESTORE ebp;\
150 popl %eax; \
151 CFI_ADJUST_CFA_OFFSET -4;\
152 CFI_RESTORE eax
154 #define RESTORE_REGS \
155 RESTORE_INT_REGS; \
156 1: popl %ds; \
157 CFI_ADJUST_CFA_OFFSET -4;\
158 /*CFI_RESTORE ds;*/\
159 2: popl %es; \
160 CFI_ADJUST_CFA_OFFSET -4;\
161 /*CFI_RESTORE es;*/\
162 3: popl %fs; \
163 CFI_ADJUST_CFA_OFFSET -4;\
164 /*CFI_RESTORE fs;*/\
165 .pushsection .fixup,"ax"; \
166 4: movl $0,(%esp); \
167 jmp 1b; \
168 5: movl $0,(%esp); \
169 jmp 2b; \
170 6: movl $0,(%esp); \
171 jmp 3b; \
172 .section __ex_table,"a";\
173 .align 4; \
174 .long 1b,4b; \
175 .long 2b,5b; \
176 .long 3b,6b; \
177 .popsection
179 #define RING0_INT_FRAME \
180 CFI_STARTPROC simple;\
181 CFI_SIGNAL_FRAME;\
182 CFI_DEF_CFA esp, 3*4;\
183 /*CFI_OFFSET cs, -2*4;*/\
184 CFI_OFFSET eip, -3*4
186 #define RING0_EC_FRAME \
187 CFI_STARTPROC simple;\
188 CFI_SIGNAL_FRAME;\
189 CFI_DEF_CFA esp, 4*4;\
190 /*CFI_OFFSET cs, -2*4;*/\
191 CFI_OFFSET eip, -3*4
193 #define RING0_PTREGS_FRAME \
194 CFI_STARTPROC simple;\
195 CFI_SIGNAL_FRAME;\
196 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
197 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
198 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
199 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
200 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
201 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
202 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
203 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
204 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
205 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
206 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
207 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
209 ENTRY(ret_from_fork)
210 CFI_STARTPROC
211 pushl %eax
212 CFI_ADJUST_CFA_OFFSET 4
213 call schedule_tail
214 GET_THREAD_INFO(%ebp)
215 popl %eax
216 CFI_ADJUST_CFA_OFFSET -4
217 pushl $0x0202 # Reset kernel eflags
218 CFI_ADJUST_CFA_OFFSET 4
219 popfl
220 CFI_ADJUST_CFA_OFFSET -4
221 jmp syscall_exit
222 CFI_ENDPROC
223 END(ret_from_fork)
225 /*
226 * Return to user mode is not as complex as all this looks,
227 * but we want the default path for a system call return to
228 * go as quickly as possible which is why some of this is
229 * less clear than it otherwise should be.
230 */
232 # userspace resumption stub bypassing syscall exit tracing
233 ALIGN
234 RING0_PTREGS_FRAME
235 ret_from_exception:
236 preempt_stop(CLBR_ANY)
237 ret_from_intr:
238 GET_THREAD_INFO(%ebp)
239 check_userspace:
240 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
241 movb PT_CS(%esp), %al
242 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
243 cmpl $USER_RPL, %eax
244 jb resume_kernel # not returning to v8086 or userspace
246 ENTRY(resume_userspace)
247 LOCKDEP_SYS_EXIT
248 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
249 # setting need_resched or sigpending
250 # between sampling and the iret
251 movl TI_flags(%ebp), %ecx
252 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
253 # int/exception return?
254 jne work_pending
255 jmp restore_all
256 END(ret_from_exception)
258 #ifdef CONFIG_PREEMPT
259 ENTRY(resume_kernel)
260 DISABLE_INTERRUPTS(CLBR_ANY)
261 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
262 jnz restore_nocheck
263 need_resched:
264 movl TI_flags(%ebp), %ecx # need_resched set ?
265 testb $_TIF_NEED_RESCHED, %cl
266 jz restore_all
267 testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ?
268 jz restore_all
269 call preempt_schedule_irq
270 jmp need_resched
271 END(resume_kernel)
272 #endif
273 CFI_ENDPROC
275 /* SYSENTER_RETURN points to after the "sysenter" instruction in
276 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
278 # sysenter call handler stub
279 ENTRY(ia32_sysenter_target)
280 CFI_STARTPROC simple
281 CFI_SIGNAL_FRAME
282 CFI_DEF_CFA esp, 0
283 CFI_REGISTER esp, ebp
284 movl TSS_sysenter_sp0(%esp),%esp
285 sysenter_past_esp:
286 /*
287 * Interrupts are disabled here, but we can't trace it until
288 * enough kernel state to call TRACE_IRQS_OFF can be called - but
289 * we immediately enable interrupts at that point anyway.
290 */
291 pushl $(__USER_DS)
292 CFI_ADJUST_CFA_OFFSET 4
293 /*CFI_REL_OFFSET ss, 0*/
294 pushl %ebp
295 CFI_ADJUST_CFA_OFFSET 4
296 CFI_REL_OFFSET esp, 0
297 pushfl
298 orl $X86_EFLAGS_IF, (%esp)
299 CFI_ADJUST_CFA_OFFSET 4
300 pushl $(__USER_CS)
301 CFI_ADJUST_CFA_OFFSET 4
302 /*CFI_REL_OFFSET cs, 0*/
303 /*
304 * Push current_thread_info()->sysenter_return to the stack.
305 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
306 * pushed above; +8 corresponds to copy_thread's esp0 setting.
307 */
308 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
309 CFI_ADJUST_CFA_OFFSET 4
310 CFI_REL_OFFSET eip, 0
312 pushl %eax
313 CFI_ADJUST_CFA_OFFSET 4
314 SAVE_ALL
315 ENABLE_INTERRUPTS(CLBR_NONE)
317 /*
318 * Load the potential sixth argument from user stack.
319 * Careful about security.
320 */
321 cmpl $__PAGE_OFFSET-3,%ebp
322 jae syscall_fault
323 1: movl (%ebp),%ebp
324 movl %ebp,PT_EBP(%esp)
325 .section __ex_table,"a"
326 .align 4
327 .long 1b,syscall_fault
328 .previous
330 GET_THREAD_INFO(%ebp)
332 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
333 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
334 jnz syscall_trace_entry
335 cmpl $(nr_syscalls), %eax
336 jae syscall_badsys
337 call *sys_call_table(,%eax,4)
338 movl %eax,PT_EAX(%esp)
339 LOCKDEP_SYS_EXIT
340 DISABLE_INTERRUPTS(CLBR_ANY)
341 TRACE_IRQS_OFF
342 movl TI_flags(%ebp), %ecx
343 testw $_TIF_ALLWORK_MASK, %cx
344 jne syscall_exit_work
345 /* if something modifies registers it must also disable sysexit */
346 movl PT_EIP(%esp), %edx
347 movl PT_OLDESP(%esp), %ecx
348 xorl %ebp,%ebp
349 TRACE_IRQS_ON
350 1: mov PT_FS(%esp), %fs
351 ENABLE_INTERRUPTS_SYSCALL_RET
352 CFI_ENDPROC
353 .pushsection .fixup,"ax"
354 2: movl $0,PT_FS(%esp)
355 jmp 1b
356 .section __ex_table,"a"
357 .align 4
358 .long 1b,2b
359 .popsection
360 ENDPROC(ia32_sysenter_target)
362 # system call handler stub
363 ENTRY(system_call)
364 RING0_INT_FRAME # can't unwind into user space anyway
365 pushl %eax # save orig_eax
366 CFI_ADJUST_CFA_OFFSET 4
367 SAVE_ALL
368 GET_THREAD_INFO(%ebp)
369 # system call tracing in operation / emulation
370 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
371 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
372 jnz syscall_trace_entry
373 cmpl $(nr_syscalls), %eax
374 jae syscall_badsys
375 syscall_call:
376 call *sys_call_table(,%eax,4)
377 movl %eax,PT_EAX(%esp) # store the return value
378 syscall_exit:
379 LOCKDEP_SYS_EXIT
380 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
381 # setting need_resched or sigpending
382 # between sampling and the iret
383 TRACE_IRQS_OFF
384 testl $X86_EFLAGS_TF,PT_EFLAGS(%esp) # If tracing set singlestep flag on exit
385 jz no_singlestep
386 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
387 no_singlestep:
388 movl TI_flags(%ebp), %ecx
389 testw $_TIF_ALLWORK_MASK, %cx # current->work
390 jne syscall_exit_work
392 restore_all:
393 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
394 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
395 # are returning to the kernel.
396 # See comments in process.c:copy_thread() for details.
397 movb PT_OLDSS(%esp), %ah
398 movb PT_CS(%esp), %al
399 andl $(X86_EFLAGS_VM | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
400 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
401 CFI_REMEMBER_STATE
402 je ldt_ss # returning to user-space with LDT SS
403 restore_nocheck:
404 TRACE_IRQS_IRET
405 restore_nocheck_notrace:
406 RESTORE_REGS
407 addl $4, %esp # skip orig_eax/error_code
408 CFI_ADJUST_CFA_OFFSET -4
409 irq_return:
410 INTERRUPT_RETURN
411 .section .fixup,"ax"
412 ENTRY(iret_exc)
413 pushl $0 # no error code
414 pushl $do_iret_error
415 jmp error_code
416 .previous
417 .section __ex_table,"a"
418 .align 4
419 .long irq_return,iret_exc
420 .previous
422 CFI_RESTORE_STATE
423 ldt_ss:
424 larl PT_OLDSS(%esp), %eax
425 jnz restore_nocheck
426 testl $0x00400000, %eax # returning to 32bit stack?
427 jnz restore_nocheck # allright, normal return
429 #ifdef CONFIG_PARAVIRT
430 /*
431 * The kernel can't run on a non-flat stack if paravirt mode
432 * is active. Rather than try to fixup the high bits of
433 * ESP, bypass this code entirely. This may break DOSemu
434 * and/or Wine support in a paravirt VM, although the option
435 * is still available to implement the setting of the high
436 * 16-bits in the INTERRUPT_RETURN paravirt-op.
437 */
438 cmpl $0, pv_info+PARAVIRT_enabled
439 jne restore_nocheck
440 #endif
442 /* If returning to userspace with 16bit stack,
443 * try to fix the higher word of ESP, as the CPU
444 * won't restore it.
445 * This is an "official" bug of all the x86-compatible
446 * CPUs, which we can try to work around to make
447 * dosemu and wine happy. */
448 movl PT_OLDESP(%esp), %eax
449 movl %esp, %edx
450 call patch_espfix_desc
451 pushl $__ESPFIX_SS
452 CFI_ADJUST_CFA_OFFSET 4
453 pushl %eax
454 CFI_ADJUST_CFA_OFFSET 4
455 DISABLE_INTERRUPTS(CLBR_EAX)
456 TRACE_IRQS_OFF
457 lss (%esp), %esp
458 CFI_ADJUST_CFA_OFFSET -8
459 jmp restore_nocheck
460 CFI_ENDPROC
461 ENDPROC(system_call)
463 # perform work that needs to be done immediately before resumption
464 ALIGN
465 RING0_PTREGS_FRAME # can't unwind into user space anyway
466 work_pending:
467 testb $_TIF_NEED_RESCHED, %cl
468 jz work_notifysig
469 work_resched:
470 call schedule
471 LOCKDEP_SYS_EXIT
472 DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
473 # setting need_resched or sigpending
474 # between sampling and the iret
475 TRACE_IRQS_OFF
476 movl TI_flags(%ebp), %ecx
477 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
478 # than syscall tracing?
479 jz restore_all
480 testb $_TIF_NEED_RESCHED, %cl
481 jnz work_resched
483 work_notifysig: # deal with pending signals and
484 # notify-resume requests
485 #ifdef CONFIG_VM86
486 testl $X86_EFLAGS_VM, PT_EFLAGS(%esp)
487 movl %esp, %eax
488 jne work_notifysig_v86 # returning to kernel-space or
489 # vm86-space
490 xorl %edx, %edx
491 call do_notify_resume
492 jmp resume_userspace_sig
494 ALIGN
495 work_notifysig_v86:
496 pushl %ecx # save ti_flags for do_notify_resume
497 CFI_ADJUST_CFA_OFFSET 4
498 call save_v86_state # %eax contains pt_regs pointer
499 popl %ecx
500 CFI_ADJUST_CFA_OFFSET -4
501 movl %eax, %esp
502 #else
503 movl %esp, %eax
504 #endif
505 xorl %edx, %edx
506 call do_notify_resume
507 jmp resume_userspace_sig
508 END(work_pending)
510 # perform syscall exit tracing
511 ALIGN
512 syscall_trace_entry:
513 movl $-ENOSYS,PT_EAX(%esp)
514 movl %esp, %eax
515 xorl %edx,%edx
516 call do_syscall_trace
517 cmpl $0, %eax
518 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
519 # so must skip actual syscall
520 movl PT_ORIG_EAX(%esp), %eax
521 cmpl $(nr_syscalls), %eax
522 jnae syscall_call
523 jmp syscall_exit
524 END(syscall_trace_entry)
526 # perform syscall exit tracing
527 ALIGN
528 syscall_exit_work:
529 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
530 jz work_pending
531 TRACE_IRQS_ON
532 ENABLE_INTERRUPTS(CLBR_ANY) # could let do_syscall_trace() call
533 # schedule() instead
534 movl %esp, %eax
535 movl $1, %edx
536 call do_syscall_trace
537 jmp resume_userspace
538 END(syscall_exit_work)
539 CFI_ENDPROC
541 RING0_INT_FRAME # can't unwind into user space anyway
542 syscall_fault:
543 GET_THREAD_INFO(%ebp)
544 movl $-EFAULT,PT_EAX(%esp)
545 jmp resume_userspace
546 END(syscall_fault)
548 syscall_badsys:
549 movl $-ENOSYS,PT_EAX(%esp)
550 jmp resume_userspace
551 END(syscall_badsys)
552 CFI_ENDPROC
554 #define FIXUP_ESPFIX_STACK \
555 /* since we are on a wrong stack, we cant make it a C code :( */ \
556 PER_CPU(gdt_page, %ebx); \
557 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
558 addl %esp, %eax; \
559 pushl $__KERNEL_DS; \
560 CFI_ADJUST_CFA_OFFSET 4; \
561 pushl %eax; \
562 CFI_ADJUST_CFA_OFFSET 4; \
563 lss (%esp), %esp; \
564 CFI_ADJUST_CFA_OFFSET -8;
565 #define UNWIND_ESPFIX_STACK \
566 movl %ss, %eax; \
567 /* see if on espfix stack */ \
568 cmpw $__ESPFIX_SS, %ax; \
569 jne 27f; \
570 movl $__KERNEL_DS, %eax; \
571 movl %eax, %ds; \
572 movl %eax, %es; \
573 /* switch to normal stack */ \
574 FIXUP_ESPFIX_STACK; \
575 27:;
577 /*
578 * Build the entry stubs and pointer table with
579 * some assembler magic.
580 */
581 .section .rodata,"a"
582 ENTRY(interrupt)
583 .text
585 ENTRY(irq_entries_start)
586 RING0_INT_FRAME
587 vector=0
588 .rept NR_IRQS
589 ALIGN
590 .if vector
591 CFI_ADJUST_CFA_OFFSET -4
592 .endif
593 1: pushl $~(vector)
594 CFI_ADJUST_CFA_OFFSET 4
595 jmp common_interrupt
596 .previous
597 .long 1b
598 .text
599 vector=vector+1
600 .endr
601 END(irq_entries_start)
603 .previous
604 END(interrupt)
605 .previous
607 /*
608 * the CPU automatically disables interrupts when executing an IRQ vector,
609 * so IRQ-flags tracing has to follow that:
610 */
611 ALIGN
612 common_interrupt:
613 SAVE_ALL
614 TRACE_IRQS_OFF
615 movl %esp,%eax
616 call do_IRQ
617 jmp ret_from_intr
618 ENDPROC(common_interrupt)
619 CFI_ENDPROC
621 #define BUILD_INTERRUPT(name, nr) \
622 ENTRY(name) \
623 RING0_INT_FRAME; \
624 pushl $~(nr); \
625 CFI_ADJUST_CFA_OFFSET 4; \
626 SAVE_ALL; \
627 TRACE_IRQS_OFF \
628 movl %esp,%eax; \
629 call smp_##name; \
630 jmp ret_from_intr; \
631 CFI_ENDPROC; \
632 ENDPROC(name)
634 /* The include is where all of the SMP etc. interrupts come from */
635 #include "entry_arch.h"
637 KPROBE_ENTRY(page_fault)
638 RING0_EC_FRAME
639 pushl $do_page_fault
640 CFI_ADJUST_CFA_OFFSET 4
641 ALIGN
642 error_code:
643 /* the function address is in %fs's slot on the stack */
644 pushl %es
645 CFI_ADJUST_CFA_OFFSET 4
646 /*CFI_REL_OFFSET es, 0*/
647 pushl %ds
648 CFI_ADJUST_CFA_OFFSET 4
649 /*CFI_REL_OFFSET ds, 0*/
650 pushl %eax
651 CFI_ADJUST_CFA_OFFSET 4
652 CFI_REL_OFFSET eax, 0
653 pushl %ebp
654 CFI_ADJUST_CFA_OFFSET 4
655 CFI_REL_OFFSET ebp, 0
656 pushl %edi
657 CFI_ADJUST_CFA_OFFSET 4
658 CFI_REL_OFFSET edi, 0
659 pushl %esi
660 CFI_ADJUST_CFA_OFFSET 4
661 CFI_REL_OFFSET esi, 0
662 pushl %edx
663 CFI_ADJUST_CFA_OFFSET 4
664 CFI_REL_OFFSET edx, 0
665 pushl %ecx
666 CFI_ADJUST_CFA_OFFSET 4
667 CFI_REL_OFFSET ecx, 0
668 pushl %ebx
669 CFI_ADJUST_CFA_OFFSET 4
670 CFI_REL_OFFSET ebx, 0
671 cld
672 pushl %fs
673 CFI_ADJUST_CFA_OFFSET 4
674 /*CFI_REL_OFFSET fs, 0*/
675 movl $(__KERNEL_PERCPU), %ecx
676 movl %ecx, %fs
677 UNWIND_ESPFIX_STACK
678 popl %ecx
679 CFI_ADJUST_CFA_OFFSET -4
680 /*CFI_REGISTER es, ecx*/
681 movl PT_FS(%esp), %edi # get the function address
682 movl PT_ORIG_EAX(%esp), %edx # get the error code
683 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
684 mov %ecx, PT_FS(%esp)
685 /*CFI_REL_OFFSET fs, ES*/
686 movl $(__USER_DS), %ecx
687 movl %ecx, %ds
688 movl %ecx, %es
689 movl %esp,%eax # pt_regs pointer
690 call *%edi
691 jmp ret_from_exception
692 CFI_ENDPROC
693 KPROBE_END(page_fault)
695 ENTRY(coprocessor_error)
696 RING0_INT_FRAME
697 pushl $0
698 CFI_ADJUST_CFA_OFFSET 4
699 pushl $do_coprocessor_error
700 CFI_ADJUST_CFA_OFFSET 4
701 jmp error_code
702 CFI_ENDPROC
703 END(coprocessor_error)
705 ENTRY(simd_coprocessor_error)
706 RING0_INT_FRAME
707 pushl $0
708 CFI_ADJUST_CFA_OFFSET 4
709 pushl $do_simd_coprocessor_error
710 CFI_ADJUST_CFA_OFFSET 4
711 jmp error_code
712 CFI_ENDPROC
713 END(simd_coprocessor_error)
715 ENTRY(device_not_available)
716 RING0_INT_FRAME
717 pushl $-1 # mark this as an int
718 CFI_ADJUST_CFA_OFFSET 4
719 SAVE_ALL
720 GET_CR0_INTO_EAX
721 testl $0x4, %eax # EM (math emulation bit)
722 jne device_not_available_emulate
723 preempt_stop(CLBR_ANY)
724 call math_state_restore
725 jmp ret_from_exception
726 device_not_available_emulate:
727 pushl $0 # temporary storage for ORIG_EIP
728 CFI_ADJUST_CFA_OFFSET 4
729 call math_emulate
730 addl $4, %esp
731 CFI_ADJUST_CFA_OFFSET -4
732 jmp ret_from_exception
733 CFI_ENDPROC
734 END(device_not_available)
736 /*
737 * Debug traps and NMI can happen at the one SYSENTER instruction
738 * that sets up the real kernel stack. Check here, since we can't
739 * allow the wrong stack to be used.
740 *
741 * "TSS_sysenter_sp0+12" is because the NMI/debug handler will have
742 * already pushed 3 words if it hits on the sysenter instruction:
743 * eflags, cs and eip.
744 *
745 * We just load the right stack, and push the three (known) values
746 * by hand onto the new stack - while updating the return eip past
747 * the instruction that would have done it for sysenter.
748 */
749 #define FIX_STACK(offset, ok, label) \
750 cmpw $__KERNEL_CS,4(%esp); \
751 jne ok; \
752 label: \
753 movl TSS_sysenter_sp0+offset(%esp),%esp; \
754 CFI_DEF_CFA esp, 0; \
755 CFI_UNDEFINED eip; \
756 pushfl; \
757 CFI_ADJUST_CFA_OFFSET 4; \
758 pushl $__KERNEL_CS; \
759 CFI_ADJUST_CFA_OFFSET 4; \
760 pushl $sysenter_past_esp; \
761 CFI_ADJUST_CFA_OFFSET 4; \
762 CFI_REL_OFFSET eip, 0
764 KPROBE_ENTRY(debug)
765 RING0_INT_FRAME
766 cmpl $ia32_sysenter_target,(%esp)
767 jne debug_stack_correct
768 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
769 debug_stack_correct:
770 pushl $-1 # mark this as an int
771 CFI_ADJUST_CFA_OFFSET 4
772 SAVE_ALL
773 xorl %edx,%edx # error code 0
774 movl %esp,%eax # pt_regs pointer
775 call do_debug
776 jmp ret_from_exception
777 CFI_ENDPROC
778 KPROBE_END(debug)
780 /*
781 * NMI is doubly nasty. It can happen _while_ we're handling
782 * a debug fault, and the debug fault hasn't yet been able to
783 * clear up the stack. So we first check whether we got an
784 * NMI on the sysenter entry path, but after that we need to
785 * check whether we got an NMI on the debug path where the debug
786 * fault happened on the sysenter path.
787 */
788 KPROBE_ENTRY(nmi)
789 RING0_INT_FRAME
790 pushl %eax
791 CFI_ADJUST_CFA_OFFSET 4
792 movl %ss, %eax
793 cmpw $__ESPFIX_SS, %ax
794 popl %eax
795 CFI_ADJUST_CFA_OFFSET -4
796 je nmi_espfix_stack
797 cmpl $ia32_sysenter_target,(%esp)
798 je nmi_stack_fixup
799 pushl %eax
800 CFI_ADJUST_CFA_OFFSET 4
801 movl %esp,%eax
802 /* Do not access memory above the end of our stack page,
803 * it might not exist.
804 */
805 andl $(THREAD_SIZE-1),%eax
806 cmpl $(THREAD_SIZE-20),%eax
807 popl %eax
808 CFI_ADJUST_CFA_OFFSET -4
809 jae nmi_stack_correct
810 cmpl $ia32_sysenter_target,12(%esp)
811 je nmi_debug_stack_check
812 nmi_stack_correct:
813 /* We have a RING0_INT_FRAME here */
814 pushl %eax
815 CFI_ADJUST_CFA_OFFSET 4
816 SAVE_ALL
817 xorl %edx,%edx # zero error code
818 movl %esp,%eax # pt_regs pointer
819 call do_nmi
820 jmp restore_nocheck_notrace
821 CFI_ENDPROC
823 nmi_stack_fixup:
824 RING0_INT_FRAME
825 FIX_STACK(12,nmi_stack_correct, 1)
826 jmp nmi_stack_correct
828 nmi_debug_stack_check:
829 /* We have a RING0_INT_FRAME here */
830 cmpw $__KERNEL_CS,16(%esp)
831 jne nmi_stack_correct
832 cmpl $debug,(%esp)
833 jb nmi_stack_correct
834 cmpl $debug_esp_fix_insn,(%esp)
835 ja nmi_stack_correct
836 FIX_STACK(24,nmi_stack_correct, 1)
837 jmp nmi_stack_correct
839 nmi_espfix_stack:
840 /* We have a RING0_INT_FRAME here.
841 *
842 * create the pointer to lss back
843 */
844 pushl %ss
845 CFI_ADJUST_CFA_OFFSET 4
846 pushl %esp
847 CFI_ADJUST_CFA_OFFSET 4
848 addw $4, (%esp)
849 /* copy the iret frame of 12 bytes */
850 .rept 3
851 pushl 16(%esp)
852 CFI_ADJUST_CFA_OFFSET 4
853 .endr
854 pushl %eax
855 CFI_ADJUST_CFA_OFFSET 4
856 SAVE_ALL
857 FIXUP_ESPFIX_STACK # %eax == %esp
858 xorl %edx,%edx # zero error code
859 call do_nmi
860 RESTORE_REGS
861 lss 12+4(%esp), %esp # back to espfix stack
862 CFI_ADJUST_CFA_OFFSET -24
863 jmp irq_return
864 CFI_ENDPROC
865 KPROBE_END(nmi)
867 #ifdef CONFIG_PARAVIRT
868 ENTRY(native_iret)
869 iret
870 .section __ex_table,"a"
871 .align 4
872 .long native_iret, iret_exc
873 .previous
874 END(native_iret)
876 ENTRY(native_irq_enable_syscall_ret)
877 sti
878 sysexit
879 END(native_irq_enable_syscall_ret)
880 #endif
882 KPROBE_ENTRY(int3)
883 RING0_INT_FRAME
884 pushl $-1 # mark this as an int
885 CFI_ADJUST_CFA_OFFSET 4
886 SAVE_ALL
887 xorl %edx,%edx # zero error code
888 movl %esp,%eax # pt_regs pointer
889 call do_int3
890 jmp ret_from_exception
891 CFI_ENDPROC
892 KPROBE_END(int3)
894 ENTRY(overflow)
895 RING0_INT_FRAME
896 pushl $0
897 CFI_ADJUST_CFA_OFFSET 4
898 pushl $do_overflow
899 CFI_ADJUST_CFA_OFFSET 4
900 jmp error_code
901 CFI_ENDPROC
902 END(overflow)
904 ENTRY(bounds)
905 RING0_INT_FRAME
906 pushl $0
907 CFI_ADJUST_CFA_OFFSET 4
908 pushl $do_bounds
909 CFI_ADJUST_CFA_OFFSET 4
910 jmp error_code
911 CFI_ENDPROC
912 END(bounds)
914 ENTRY(invalid_op)
915 RING0_INT_FRAME
916 pushl $0
917 CFI_ADJUST_CFA_OFFSET 4
918 pushl $do_invalid_op
919 CFI_ADJUST_CFA_OFFSET 4
920 jmp error_code
921 CFI_ENDPROC
922 END(invalid_op)
924 ENTRY(coprocessor_segment_overrun)
925 RING0_INT_FRAME
926 pushl $0
927 CFI_ADJUST_CFA_OFFSET 4
928 pushl $do_coprocessor_segment_overrun
929 CFI_ADJUST_CFA_OFFSET 4
930 jmp error_code
931 CFI_ENDPROC
932 END(coprocessor_segment_overrun)
934 ENTRY(invalid_TSS)
935 RING0_EC_FRAME
936 pushl $do_invalid_TSS
937 CFI_ADJUST_CFA_OFFSET 4
938 jmp error_code
939 CFI_ENDPROC
940 END(invalid_TSS)
942 ENTRY(segment_not_present)
943 RING0_EC_FRAME
944 pushl $do_segment_not_present
945 CFI_ADJUST_CFA_OFFSET 4
946 jmp error_code
947 CFI_ENDPROC
948 END(segment_not_present)
950 ENTRY(stack_segment)
951 RING0_EC_FRAME
952 pushl $do_stack_segment
953 CFI_ADJUST_CFA_OFFSET 4
954 jmp error_code
955 CFI_ENDPROC
956 END(stack_segment)
958 KPROBE_ENTRY(general_protection)
959 RING0_EC_FRAME
960 pushl $do_general_protection
961 CFI_ADJUST_CFA_OFFSET 4
962 jmp error_code
963 CFI_ENDPROC
964 KPROBE_END(general_protection)
966 ENTRY(alignment_check)
967 RING0_EC_FRAME
968 pushl $do_alignment_check
969 CFI_ADJUST_CFA_OFFSET 4
970 jmp error_code
971 CFI_ENDPROC
972 END(alignment_check)
974 ENTRY(divide_error)
975 RING0_INT_FRAME
976 pushl $0 # no error code
977 CFI_ADJUST_CFA_OFFSET 4
978 pushl $do_divide_error
979 CFI_ADJUST_CFA_OFFSET 4
980 jmp error_code
981 CFI_ENDPROC
982 END(divide_error)
984 #ifdef CONFIG_X86_MCE
985 ENTRY(machine_check)
986 RING0_INT_FRAME
987 pushl $0
988 CFI_ADJUST_CFA_OFFSET 4
989 pushl machine_check_vector
990 CFI_ADJUST_CFA_OFFSET 4
991 jmp error_code
992 CFI_ENDPROC
993 END(machine_check)
994 #endif
996 ENTRY(spurious_interrupt_bug)
997 RING0_INT_FRAME
998 pushl $0
999 CFI_ADJUST_CFA_OFFSET 4
1000 pushl $do_spurious_interrupt_bug
1001 CFI_ADJUST_CFA_OFFSET 4
1002 jmp error_code
1003 CFI_ENDPROC
1004 END(spurious_interrupt_bug)
1006 ENTRY(kernel_thread_helper)
1007 pushl $0 # fake return address for unwinder
1008 CFI_STARTPROC
1009 movl %edx,%eax
1010 push %edx
1011 CFI_ADJUST_CFA_OFFSET 4
1012 call *%ebx
1013 push %eax
1014 CFI_ADJUST_CFA_OFFSET 4
1015 call do_exit
1016 CFI_ENDPROC
1017 ENDPROC(kernel_thread_helper)
1019 #ifdef CONFIG_XEN
1020 /* Xen doesn't set %esp to be precisely what the normal sysenter
1021 entrypoint expects, so fix it up before using the normal path. */
1022 ENTRY(xen_sysenter_target)
1023 RING0_INT_FRAME
1024 addl $5*4, %esp /* remove xen-provided frame */
1025 jmp sysenter_past_esp
1027 ENTRY(xen_hypervisor_callback)
1028 CFI_STARTPROC
1029 pushl $0
1030 CFI_ADJUST_CFA_OFFSET 4
1031 SAVE_ALL
1032 TRACE_IRQS_OFF
1034 /* Check to see if we got the event in the critical
1035 region in xen_iret_direct, after we've reenabled
1036 events and checked for pending events. This simulates
1037 iret instruction's behaviour where it delivers a
1038 pending interrupt when enabling interrupts. */
1039 movl PT_EIP(%esp),%eax
1040 cmpl $xen_iret_start_crit,%eax
1041 jb 1f
1042 cmpl $xen_iret_end_crit,%eax
1043 jae 1f
1045 jmp xen_iret_crit_fixup
1047 ENTRY(xen_do_upcall)
1048 1: mov %esp, %eax
1049 call xen_evtchn_do_upcall
1050 jmp ret_from_intr
1051 CFI_ENDPROC
1052 ENDPROC(xen_hypervisor_callback)
1054 # Hypervisor uses this for application faults while it executes.
1055 # We get here for two reasons:
1056 # 1. Fault while reloading DS, ES, FS or GS
1057 # 2. Fault while executing IRET
1058 # Category 1 we fix up by reattempting the load, and zeroing the segment
1059 # register if the load fails.
1060 # Category 2 we fix up by jumping to do_iret_error. We cannot use the
1061 # normal Linux return path in this case because if we use the IRET hypercall
1062 # to pop the stack frame we end up in an infinite loop of failsafe callbacks.
1063 # We distinguish between categories by maintaining a status value in EAX.
1064 ENTRY(xen_failsafe_callback)
1065 CFI_STARTPROC
1066 pushl %eax
1067 CFI_ADJUST_CFA_OFFSET 4
1068 movl $1,%eax
1069 1: mov 4(%esp),%ds
1070 2: mov 8(%esp),%es
1071 3: mov 12(%esp),%fs
1072 4: mov 16(%esp),%gs
1073 testl %eax,%eax
1074 popl %eax
1075 CFI_ADJUST_CFA_OFFSET -4
1076 lea 16(%esp),%esp
1077 CFI_ADJUST_CFA_OFFSET -16
1078 jz 5f
1079 addl $16,%esp
1080 jmp iret_exc # EAX != 0 => Category 2 (Bad IRET)
1081 5: pushl $0 # EAX == 0 => Category 1 (Bad segment)
1082 CFI_ADJUST_CFA_OFFSET 4
1083 SAVE_ALL
1084 jmp ret_from_exception
1085 CFI_ENDPROC
1087 .section .fixup,"ax"
1088 6: xorl %eax,%eax
1089 movl %eax,4(%esp)
1090 jmp 1b
1091 7: xorl %eax,%eax
1092 movl %eax,8(%esp)
1093 jmp 2b
1094 8: xorl %eax,%eax
1095 movl %eax,12(%esp)
1096 jmp 3b
1097 9: xorl %eax,%eax
1098 movl %eax,16(%esp)
1099 jmp 4b
1100 .previous
1101 .section __ex_table,"a"
1102 .align 4
1103 .long 1b,6b
1104 .long 2b,7b
1105 .long 3b,8b
1106 .long 4b,9b
1107 .previous
1108 ENDPROC(xen_failsafe_callback)
1110 #endif /* CONFIG_XEN */
1112 .section .rodata,"a"
1113 #include "syscall_table_32.S"
1115 syscall_table_size=(.-sys_call_table)