| blob | fbdb933251b643b8e726feefe8e350205dbd7396 |
1 /*
2 * linux/arch/i386/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
7 /*
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
14 *
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
16 * on a 486.
17 *
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
23 *
24 * 0(%esp) - %ebx
25 * 4(%esp) - %ecx
26 * 8(%esp) - %edx
27 * C(%esp) - %esi
28 * 10(%esp) - %edi
29 * 14(%esp) - %ebp
30 * 18(%esp) - %eax
31 * 1C(%esp) - %ds
32 * 20(%esp) - %es
33 * 24(%esp) - orig_eax
34 * 28(%esp) - %eip
35 * 2C(%esp) - %cs
36 * 30(%esp) - %eflags
37 * 34(%esp) - %oldesp
38 * 38(%esp) - %oldss
39 *
40 * "current" is in register %ebx during any slow entries.
41 */
43 #include <linux/config.h>
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/errno.h>
47 #include <asm/segment.h>
48 #include <asm/smp.h>
49 #include <asm/page.h>
50 #include <asm/desc.h>
51 #include <asm/dwarf2.h>
52 #include "irq_vectors.h"
54 #define nr_syscalls ((syscall_table_size)/4)
56 EBX = 0x00
57 ECX = 0x04
58 EDX = 0x08
59 ESI = 0x0C
60 EDI = 0x10
61 EBP = 0x14
62 EAX = 0x18
63 DS = 0x1C
64 ES = 0x20
65 ORIG_EAX = 0x24
66 EIP = 0x28
67 CS = 0x2C
68 EFLAGS = 0x30
69 OLDESP = 0x34
70 OLDSS = 0x38
72 CF_MASK = 0x00000001
73 TF_MASK = 0x00000100
74 IF_MASK = 0x00000200
75 DF_MASK = 0x00000400
76 NT_MASK = 0x00004000
77 VM_MASK = 0x00020000
79 #ifdef CONFIG_PREEMPT
80 #define preempt_stop cli
81 #else
82 #define preempt_stop
83 #define resume_kernel restore_nocheck
84 #endif
86 #ifdef CONFIG_VM86
87 #define resume_userspace_sig check_userspace
88 #else
89 #define resume_userspace_sig resume_userspace
90 #endif
92 #define SAVE_ALL \
93 cld; \
94 pushl %es; \
95 CFI_ADJUST_CFA_OFFSET 4;\
96 /*CFI_REL_OFFSET es, 0;*/\
97 pushl %ds; \
98 CFI_ADJUST_CFA_OFFSET 4;\
99 /*CFI_REL_OFFSET ds, 0;*/\
100 pushl %eax; \
101 CFI_ADJUST_CFA_OFFSET 4;\
102 CFI_REL_OFFSET eax, 0;\
103 pushl %ebp; \
104 CFI_ADJUST_CFA_OFFSET 4;\
105 CFI_REL_OFFSET ebp, 0;\
106 pushl %edi; \
107 CFI_ADJUST_CFA_OFFSET 4;\
108 CFI_REL_OFFSET edi, 0;\
109 pushl %esi; \
110 CFI_ADJUST_CFA_OFFSET 4;\
111 CFI_REL_OFFSET esi, 0;\
112 pushl %edx; \
113 CFI_ADJUST_CFA_OFFSET 4;\
114 CFI_REL_OFFSET edx, 0;\
115 pushl %ecx; \
116 CFI_ADJUST_CFA_OFFSET 4;\
117 CFI_REL_OFFSET ecx, 0;\
118 pushl %ebx; \
119 CFI_ADJUST_CFA_OFFSET 4;\
120 CFI_REL_OFFSET ebx, 0;\
121 movl $(__USER_DS), %edx; \
122 movl %edx, %ds; \
123 movl %edx, %es;
125 #define RESTORE_INT_REGS \
126 popl %ebx; \
127 CFI_ADJUST_CFA_OFFSET -4;\
128 CFI_RESTORE ebx;\
129 popl %ecx; \
130 CFI_ADJUST_CFA_OFFSET -4;\
131 CFI_RESTORE ecx;\
132 popl %edx; \
133 CFI_ADJUST_CFA_OFFSET -4;\
134 CFI_RESTORE edx;\
135 popl %esi; \
136 CFI_ADJUST_CFA_OFFSET -4;\
137 CFI_RESTORE esi;\
138 popl %edi; \
139 CFI_ADJUST_CFA_OFFSET -4;\
140 CFI_RESTORE edi;\
141 popl %ebp; \
142 CFI_ADJUST_CFA_OFFSET -4;\
143 CFI_RESTORE ebp;\
144 popl %eax; \
145 CFI_ADJUST_CFA_OFFSET -4;\
146 CFI_RESTORE eax
148 #define RESTORE_REGS \
149 RESTORE_INT_REGS; \
150 1: popl %ds; \
151 CFI_ADJUST_CFA_OFFSET -4;\
152 /*CFI_RESTORE ds;*/\
153 2: popl %es; \
154 CFI_ADJUST_CFA_OFFSET -4;\
155 /*CFI_RESTORE es;*/\
156 .section .fixup,"ax"; \
157 3: movl $0,(%esp); \
158 jmp 1b; \
159 4: movl $0,(%esp); \
160 jmp 2b; \
161 .previous; \
162 .section __ex_table,"a";\
163 .align 4; \
164 .long 1b,3b; \
165 .long 2b,4b; \
166 .previous
168 #define RING0_INT_FRAME \
169 CFI_STARTPROC simple;\
170 CFI_DEF_CFA esp, 3*4;\
171 /*CFI_OFFSET cs, -2*4;*/\
172 CFI_OFFSET eip, -3*4
174 #define RING0_EC_FRAME \
175 CFI_STARTPROC simple;\
176 CFI_DEF_CFA esp, 4*4;\
177 /*CFI_OFFSET cs, -2*4;*/\
178 CFI_OFFSET eip, -3*4
180 #define RING0_PTREGS_FRAME \
181 CFI_STARTPROC simple;\
182 CFI_DEF_CFA esp, OLDESP-EBX;\
183 /*CFI_OFFSET cs, CS-OLDESP;*/\
184 CFI_OFFSET eip, EIP-OLDESP;\
185 /*CFI_OFFSET es, ES-OLDESP;*/\
186 /*CFI_OFFSET ds, DS-OLDESP;*/\
187 CFI_OFFSET eax, EAX-OLDESP;\
188 CFI_OFFSET ebp, EBP-OLDESP;\
189 CFI_OFFSET edi, EDI-OLDESP;\
190 CFI_OFFSET esi, ESI-OLDESP;\
191 CFI_OFFSET edx, EDX-OLDESP;\
192 CFI_OFFSET ecx, ECX-OLDESP;\
193 CFI_OFFSET ebx, EBX-OLDESP
195 ENTRY(ret_from_fork)
196 CFI_STARTPROC
197 pushl %eax
198 CFI_ADJUST_CFA_OFFSET -4
199 call schedule_tail
200 GET_THREAD_INFO(%ebp)
201 popl %eax
202 CFI_ADJUST_CFA_OFFSET -4
203 jmp syscall_exit
204 CFI_ENDPROC
206 /*
207 * Return to user mode is not as complex as all this looks,
208 * but we want the default path for a system call return to
209 * go as quickly as possible which is why some of this is
210 * less clear than it otherwise should be.
211 */
213 # userspace resumption stub bypassing syscall exit tracing
214 ALIGN
215 RING0_PTREGS_FRAME
216 ret_from_exception:
217 preempt_stop
218 ret_from_intr:
219 GET_THREAD_INFO(%ebp)
220 check_userspace:
221 movl EFLAGS(%esp), %eax # mix EFLAGS and CS
222 movb CS(%esp), %al
223 testl $(VM_MASK | 3), %eax
224 jz resume_kernel
225 ENTRY(resume_userspace)
226 cli # make sure we don't miss an interrupt
227 # setting need_resched or sigpending
228 # between sampling and the iret
229 movl TI_flags(%ebp), %ecx
230 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
231 # int/exception return?
232 jne work_pending
233 jmp restore_all
235 #ifdef CONFIG_PREEMPT
236 ENTRY(resume_kernel)
237 cli
238 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
239 jnz restore_nocheck
240 need_resched:
241 movl TI_flags(%ebp), %ecx # need_resched set ?
242 testb $_TIF_NEED_RESCHED, %cl
243 jz restore_all
244 testl $IF_MASK,EFLAGS(%esp) # interrupts off (exception path) ?
245 jz restore_all
246 call preempt_schedule_irq
247 jmp need_resched
248 #endif
249 CFI_ENDPROC
251 /* SYSENTER_RETURN points to after the "sysenter" instruction in
252 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
254 # sysenter call handler stub
255 ENTRY(sysenter_entry)
256 CFI_STARTPROC simple
257 CFI_DEF_CFA esp, 0
258 CFI_REGISTER esp, ebp
259 movl TSS_sysenter_esp0(%esp),%esp
260 sysenter_past_esp:
261 sti
262 pushl $(__USER_DS)
263 CFI_ADJUST_CFA_OFFSET 4
264 /*CFI_REL_OFFSET ss, 0*/
265 pushl %ebp
266 CFI_ADJUST_CFA_OFFSET 4
267 CFI_REL_OFFSET esp, 0
268 pushfl
269 CFI_ADJUST_CFA_OFFSET 4
270 pushl $(__USER_CS)
271 CFI_ADJUST_CFA_OFFSET 4
272 /*CFI_REL_OFFSET cs, 0*/
273 /*
274 * Push current_thread_info()->sysenter_return to the stack.
275 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
276 * pushed above; +8 corresponds to copy_thread's esp0 setting.
277 */
278 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
279 CFI_ADJUST_CFA_OFFSET 4
280 CFI_REL_OFFSET eip, 0
282 /*
283 * Load the potential sixth argument from user stack.
284 * Careful about security.
285 */
286 cmpl $__PAGE_OFFSET-3,%ebp
287 jae syscall_fault
288 1: movl (%ebp),%ebp
289 .section __ex_table,"a"
290 .align 4
291 .long 1b,syscall_fault
292 .previous
294 pushl %eax
295 CFI_ADJUST_CFA_OFFSET 4
296 SAVE_ALL
297 GET_THREAD_INFO(%ebp)
299 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
300 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
301 jnz syscall_trace_entry
302 cmpl $(nr_syscalls), %eax
303 jae syscall_badsys
304 call *sys_call_table(,%eax,4)
305 movl %eax,EAX(%esp)
306 cli
307 movl TI_flags(%ebp), %ecx
308 testw $_TIF_ALLWORK_MASK, %cx
309 jne syscall_exit_work
310 /* if something modifies registers it must also disable sysexit */
311 movl EIP(%esp), %edx
312 movl OLDESP(%esp), %ecx
313 xorl %ebp,%ebp
314 sti
315 sysexit
316 CFI_ENDPROC
319 # system call handler stub
320 ENTRY(system_call)
321 RING0_INT_FRAME # can't unwind into user space anyway
322 pushl %eax # save orig_eax
323 CFI_ADJUST_CFA_OFFSET 4
324 SAVE_ALL
325 GET_THREAD_INFO(%ebp)
326 testl $TF_MASK,EFLAGS(%esp)
327 jz no_singlestep
328 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
329 no_singlestep:
330 # system call tracing in operation / emulation
331 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
332 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
333 jnz syscall_trace_entry
334 cmpl $(nr_syscalls), %eax
335 jae syscall_badsys
336 syscall_call:
337 call *sys_call_table(,%eax,4)
338 movl %eax,EAX(%esp) # store the return value
339 syscall_exit:
340 cli # make sure we don't miss an interrupt
341 # setting need_resched or sigpending
342 # between sampling and the iret
343 movl TI_flags(%ebp), %ecx
344 testw $_TIF_ALLWORK_MASK, %cx # current->work
345 jne syscall_exit_work
347 restore_all:
348 movl EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
349 # Warning: OLDSS(%esp) contains the wrong/random values if we
350 # are returning to the kernel.
351 # See comments in process.c:copy_thread() for details.
352 movb OLDSS(%esp), %ah
353 movb CS(%esp), %al
354 andl $(VM_MASK | (4 << 8) | 3), %eax
355 cmpl $((4 << 8) | 3), %eax
356 CFI_REMEMBER_STATE
357 je ldt_ss # returning to user-space with LDT SS
358 restore_nocheck:
359 RESTORE_REGS
360 addl $4, %esp
361 CFI_ADJUST_CFA_OFFSET -4
362 1: iret
363 .section .fixup,"ax"
364 iret_exc:
365 sti
366 pushl $0 # no error code
367 pushl $do_iret_error
368 jmp error_code
369 .previous
370 .section __ex_table,"a"
371 .align 4
372 .long 1b,iret_exc
373 .previous
375 CFI_RESTORE_STATE
376 ldt_ss:
377 larl OLDSS(%esp), %eax
378 jnz restore_nocheck
379 testl $0x00400000, %eax # returning to 32bit stack?
380 jnz restore_nocheck # allright, normal return
381 /* If returning to userspace with 16bit stack,
382 * try to fix the higher word of ESP, as the CPU
383 * won't restore it.
384 * This is an "official" bug of all the x86-compatible
385 * CPUs, which we can try to work around to make
386 * dosemu and wine happy. */
387 subl $8, %esp # reserve space for switch16 pointer
388 CFI_ADJUST_CFA_OFFSET 8
389 cli
390 movl %esp, %eax
391 /* Set up the 16bit stack frame with switch32 pointer on top,
392 * and a switch16 pointer on top of the current frame. */
393 call setup_x86_bogus_stack
394 CFI_ADJUST_CFA_OFFSET -8 # frame has moved
395 RESTORE_REGS
396 lss 20+4(%esp), %esp # switch to 16bit stack
397 1: iret
398 .section __ex_table,"a"
399 .align 4
400 .long 1b,iret_exc
401 .previous
402 CFI_ENDPROC
404 # perform work that needs to be done immediately before resumption
405 ALIGN
406 RING0_PTREGS_FRAME # can't unwind into user space anyway
407 work_pending:
408 testb $_TIF_NEED_RESCHED, %cl
409 jz work_notifysig
410 work_resched:
411 call schedule
412 cli # make sure we don't miss an interrupt
413 # setting need_resched or sigpending
414 # between sampling and the iret
415 movl TI_flags(%ebp), %ecx
416 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
417 # than syscall tracing?
418 jz restore_all
419 testb $_TIF_NEED_RESCHED, %cl
420 jnz work_resched
422 work_notifysig: # deal with pending signals and
423 # notify-resume requests
424 testl $VM_MASK, EFLAGS(%esp)
425 movl %esp, %eax
426 jne work_notifysig_v86 # returning to kernel-space or
427 # vm86-space
428 xorl %edx, %edx
429 call do_notify_resume
430 jmp resume_userspace_sig
432 ALIGN
433 work_notifysig_v86:
434 #ifdef CONFIG_VM86
435 pushl %ecx # save ti_flags for do_notify_resume
436 CFI_ADJUST_CFA_OFFSET 4
437 call save_v86_state # %eax contains pt_regs pointer
438 popl %ecx
439 CFI_ADJUST_CFA_OFFSET -4
440 movl %eax, %esp
441 xorl %edx, %edx
442 call do_notify_resume
443 jmp resume_userspace_sig
444 #endif
446 # perform syscall exit tracing
447 ALIGN
448 syscall_trace_entry:
449 movl $-ENOSYS,EAX(%esp)
450 movl %esp, %eax
451 xorl %edx,%edx
452 call do_syscall_trace
453 cmpl $0, %eax
454 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
455 # so must skip actual syscall
456 movl ORIG_EAX(%esp), %eax
457 cmpl $(nr_syscalls), %eax
458 jnae syscall_call
459 jmp syscall_exit
461 # perform syscall exit tracing
462 ALIGN
463 syscall_exit_work:
464 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
465 jz work_pending
466 sti # could let do_syscall_trace() call
467 # schedule() instead
468 movl %esp, %eax
469 movl $1, %edx
470 call do_syscall_trace
471 jmp resume_userspace
472 CFI_ENDPROC
474 RING0_INT_FRAME # can't unwind into user space anyway
475 syscall_fault:
476 pushl %eax # save orig_eax
477 CFI_ADJUST_CFA_OFFSET 4
478 SAVE_ALL
479 GET_THREAD_INFO(%ebp)
480 movl $-EFAULT,EAX(%esp)
481 jmp resume_userspace
483 syscall_badsys:
484 movl $-ENOSYS,EAX(%esp)
485 jmp resume_userspace
486 CFI_ENDPROC
488 #define FIXUP_ESPFIX_STACK \
489 movl %esp, %eax; \
490 /* switch to 32bit stack using the pointer on top of 16bit stack */ \
491 lss %ss:CPU_16BIT_STACK_SIZE-8, %esp; \
492 /* copy data from 16bit stack to 32bit stack */ \
493 call fixup_x86_bogus_stack; \
494 /* put ESP to the proper location */ \
495 movl %eax, %esp;
496 #define UNWIND_ESPFIX_STACK \
497 pushl %eax; \
498 CFI_ADJUST_CFA_OFFSET 4; \
499 movl %ss, %eax; \
500 /* see if on 16bit stack */ \
501 cmpw $__ESPFIX_SS, %ax; \
502 je 28f; \
503 27: popl %eax; \
504 CFI_ADJUST_CFA_OFFSET -4; \
505 .section .fixup,"ax"; \
506 28: movl $__KERNEL_DS, %eax; \
507 movl %eax, %ds; \
508 movl %eax, %es; \
509 /* switch to 32bit stack */ \
510 FIXUP_ESPFIX_STACK; \
511 jmp 27b; \
512 .previous
514 /*
515 * Build the entry stubs and pointer table with
516 * some assembler magic.
517 */
518 .data
519 ENTRY(interrupt)
520 .text
522 vector=0
523 ENTRY(irq_entries_start)
524 RING0_INT_FRAME
525 .rept NR_IRQS
526 ALIGN
527 .if vector
528 CFI_ADJUST_CFA_OFFSET -4
529 .endif
530 1: pushl $~(vector)
531 CFI_ADJUST_CFA_OFFSET 4
532 jmp common_interrupt
533 .data
534 .long 1b
535 .text
536 vector=vector+1
537 .endr
539 ALIGN
540 common_interrupt:
541 SAVE_ALL
542 movl %esp,%eax
543 call do_IRQ
544 jmp ret_from_intr
545 CFI_ENDPROC
547 #define BUILD_INTERRUPT(name, nr) \
548 ENTRY(name) \
549 RING0_INT_FRAME; \
550 pushl $~(nr); \
551 CFI_ADJUST_CFA_OFFSET 4; \
552 SAVE_ALL; \
553 movl %esp,%eax; \
554 call smp_/**/name; \
555 jmp ret_from_intr; \
556 CFI_ENDPROC
558 /* The include is where all of the SMP etc. interrupts come from */
559 #include "entry_arch.h"
561 ENTRY(divide_error)
562 RING0_INT_FRAME
563 pushl $0 # no error code
564 CFI_ADJUST_CFA_OFFSET 4
565 pushl $do_divide_error
566 CFI_ADJUST_CFA_OFFSET 4
567 ALIGN
568 error_code:
569 pushl %ds
570 CFI_ADJUST_CFA_OFFSET 4
571 /*CFI_REL_OFFSET ds, 0*/
572 pushl %eax
573 CFI_ADJUST_CFA_OFFSET 4
574 CFI_REL_OFFSET eax, 0
575 xorl %eax, %eax
576 pushl %ebp
577 CFI_ADJUST_CFA_OFFSET 4
578 CFI_REL_OFFSET ebp, 0
579 pushl %edi
580 CFI_ADJUST_CFA_OFFSET 4
581 CFI_REL_OFFSET edi, 0
582 pushl %esi
583 CFI_ADJUST_CFA_OFFSET 4
584 CFI_REL_OFFSET esi, 0
585 pushl %edx
586 CFI_ADJUST_CFA_OFFSET 4
587 CFI_REL_OFFSET edx, 0
588 decl %eax # eax = -1
589 pushl %ecx
590 CFI_ADJUST_CFA_OFFSET 4
591 CFI_REL_OFFSET ecx, 0
592 pushl %ebx
593 CFI_ADJUST_CFA_OFFSET 4
594 CFI_REL_OFFSET ebx, 0
595 cld
596 pushl %es
597 CFI_ADJUST_CFA_OFFSET 4
598 /*CFI_REL_OFFSET es, 0*/
599 UNWIND_ESPFIX_STACK
600 popl %ecx
601 CFI_ADJUST_CFA_OFFSET -4
602 /*CFI_REGISTER es, ecx*/
603 movl ES(%esp), %edi # get the function address
604 movl ORIG_EAX(%esp), %edx # get the error code
605 movl %eax, ORIG_EAX(%esp)
606 movl %ecx, ES(%esp)
607 /*CFI_REL_OFFSET es, ES*/
608 movl $(__USER_DS), %ecx
609 movl %ecx, %ds
610 movl %ecx, %es
611 movl %esp,%eax # pt_regs pointer
612 call *%edi
613 jmp ret_from_exception
614 CFI_ENDPROC
616 ENTRY(coprocessor_error)
617 RING0_INT_FRAME
618 pushl $0
619 CFI_ADJUST_CFA_OFFSET 4
620 pushl $do_coprocessor_error
621 CFI_ADJUST_CFA_OFFSET 4
622 jmp error_code
623 CFI_ENDPROC
625 ENTRY(simd_coprocessor_error)
626 RING0_INT_FRAME
627 pushl $0
628 CFI_ADJUST_CFA_OFFSET 4
629 pushl $do_simd_coprocessor_error
630 CFI_ADJUST_CFA_OFFSET 4
631 jmp error_code
632 CFI_ENDPROC
634 ENTRY(device_not_available)
635 RING0_INT_FRAME
636 pushl $-1 # mark this as an int
637 CFI_ADJUST_CFA_OFFSET 4
638 SAVE_ALL
639 movl %cr0, %eax
640 testl $0x4, %eax # EM (math emulation bit)
641 jne device_not_available_emulate
642 preempt_stop
643 call math_state_restore
644 jmp ret_from_exception
645 device_not_available_emulate:
646 pushl $0 # temporary storage for ORIG_EIP
647 CFI_ADJUST_CFA_OFFSET 4
648 call math_emulate
649 addl $4, %esp
650 CFI_ADJUST_CFA_OFFSET -4
651 jmp ret_from_exception
652 CFI_ENDPROC
654 /*
655 * Debug traps and NMI can happen at the one SYSENTER instruction
656 * that sets up the real kernel stack. Check here, since we can't
657 * allow the wrong stack to be used.
658 *
659 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have
660 * already pushed 3 words if it hits on the sysenter instruction:
661 * eflags, cs and eip.
662 *
663 * We just load the right stack, and push the three (known) values
664 * by hand onto the new stack - while updating the return eip past
665 * the instruction that would have done it for sysenter.
666 */
667 #define FIX_STACK(offset, ok, label) \
668 cmpw $__KERNEL_CS,4(%esp); \
669 jne ok; \
670 label: \
671 movl TSS_sysenter_esp0+offset(%esp),%esp; \
672 pushfl; \
673 pushl $__KERNEL_CS; \
674 pushl $sysenter_past_esp
676 KPROBE_ENTRY(debug)
677 RING0_INT_FRAME
678 cmpl $sysenter_entry,(%esp)
679 jne debug_stack_correct
680 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
681 debug_stack_correct:
682 pushl $-1 # mark this as an int
683 CFI_ADJUST_CFA_OFFSET 4
684 SAVE_ALL
685 xorl %edx,%edx # error code 0
686 movl %esp,%eax # pt_regs pointer
687 call do_debug
688 jmp ret_from_exception
689 CFI_ENDPROC
690 .previous .text
691 /*
692 * NMI is doubly nasty. It can happen _while_ we're handling
693 * a debug fault, and the debug fault hasn't yet been able to
694 * clear up the stack. So we first check whether we got an
695 * NMI on the sysenter entry path, but after that we need to
696 * check whether we got an NMI on the debug path where the debug
697 * fault happened on the sysenter path.
698 */
699 ENTRY(nmi)
700 RING0_INT_FRAME
701 pushl %eax
702 CFI_ADJUST_CFA_OFFSET 4
703 movl %ss, %eax
704 cmpw $__ESPFIX_SS, %ax
705 popl %eax
706 CFI_ADJUST_CFA_OFFSET -4
707 je nmi_16bit_stack
708 cmpl $sysenter_entry,(%esp)
709 je nmi_stack_fixup
710 pushl %eax
711 CFI_ADJUST_CFA_OFFSET 4
712 movl %esp,%eax
713 /* Do not access memory above the end of our stack page,
714 * it might not exist.
715 */
716 andl $(THREAD_SIZE-1),%eax
717 cmpl $(THREAD_SIZE-20),%eax
718 popl %eax
719 CFI_ADJUST_CFA_OFFSET -4
720 jae nmi_stack_correct
721 cmpl $sysenter_entry,12(%esp)
722 je nmi_debug_stack_check
723 nmi_stack_correct:
724 pushl %eax
725 CFI_ADJUST_CFA_OFFSET 4
726 SAVE_ALL
727 xorl %edx,%edx # zero error code
728 movl %esp,%eax # pt_regs pointer
729 call do_nmi
730 jmp restore_all
731 CFI_ENDPROC
733 nmi_stack_fixup:
734 FIX_STACK(12,nmi_stack_correct, 1)
735 jmp nmi_stack_correct
736 nmi_debug_stack_check:
737 cmpw $__KERNEL_CS,16(%esp)
738 jne nmi_stack_correct
739 cmpl $debug,(%esp)
740 jb nmi_stack_correct
741 cmpl $debug_esp_fix_insn,(%esp)
742 ja nmi_stack_correct
743 FIX_STACK(24,nmi_stack_correct, 1)
744 jmp nmi_stack_correct
746 nmi_16bit_stack:
747 RING0_INT_FRAME
748 /* create the pointer to lss back */
749 pushl %ss
750 CFI_ADJUST_CFA_OFFSET 4
751 pushl %esp
752 CFI_ADJUST_CFA_OFFSET 4
753 movzwl %sp, %esp
754 addw $4, (%esp)
755 /* copy the iret frame of 12 bytes */
756 .rept 3
757 pushl 16(%esp)
758 CFI_ADJUST_CFA_OFFSET 4
759 .endr
760 pushl %eax
761 CFI_ADJUST_CFA_OFFSET 4
762 SAVE_ALL
763 FIXUP_ESPFIX_STACK # %eax == %esp
764 CFI_ADJUST_CFA_OFFSET -20 # the frame has now moved
765 xorl %edx,%edx # zero error code
766 call do_nmi
767 RESTORE_REGS
768 lss 12+4(%esp), %esp # back to 16bit stack
769 1: iret
770 CFI_ENDPROC
771 .section __ex_table,"a"
772 .align 4
773 .long 1b,iret_exc
774 .previous
776 KPROBE_ENTRY(int3)
777 RING0_INT_FRAME
778 pushl $-1 # mark this as an int
779 CFI_ADJUST_CFA_OFFSET 4
780 SAVE_ALL
781 xorl %edx,%edx # zero error code
782 movl %esp,%eax # pt_regs pointer
783 call do_int3
784 jmp ret_from_exception
785 CFI_ENDPROC
786 .previous .text
788 ENTRY(overflow)
789 RING0_INT_FRAME
790 pushl $0
791 CFI_ADJUST_CFA_OFFSET 4
792 pushl $do_overflow
793 CFI_ADJUST_CFA_OFFSET 4
794 jmp error_code
795 CFI_ENDPROC
797 ENTRY(bounds)
798 RING0_INT_FRAME
799 pushl $0
800 CFI_ADJUST_CFA_OFFSET 4
801 pushl $do_bounds
802 CFI_ADJUST_CFA_OFFSET 4
803 jmp error_code
804 CFI_ENDPROC
806 ENTRY(invalid_op)
807 RING0_INT_FRAME
808 pushl $0
809 CFI_ADJUST_CFA_OFFSET 4
810 pushl $do_invalid_op
811 CFI_ADJUST_CFA_OFFSET 4
812 jmp error_code
813 CFI_ENDPROC
815 ENTRY(coprocessor_segment_overrun)
816 RING0_INT_FRAME
817 pushl $0
818 CFI_ADJUST_CFA_OFFSET 4
819 pushl $do_coprocessor_segment_overrun
820 CFI_ADJUST_CFA_OFFSET 4
821 jmp error_code
822 CFI_ENDPROC
824 ENTRY(invalid_TSS)
825 RING0_EC_FRAME
826 pushl $do_invalid_TSS
827 CFI_ADJUST_CFA_OFFSET 4
828 jmp error_code
829 CFI_ENDPROC
831 ENTRY(segment_not_present)
832 RING0_EC_FRAME
833 pushl $do_segment_not_present
834 CFI_ADJUST_CFA_OFFSET 4
835 jmp error_code
836 CFI_ENDPROC
838 ENTRY(stack_segment)
839 RING0_EC_FRAME
840 pushl $do_stack_segment
841 CFI_ADJUST_CFA_OFFSET 4
842 jmp error_code
843 CFI_ENDPROC
845 KPROBE_ENTRY(general_protection)
846 RING0_EC_FRAME
847 pushl $do_general_protection
848 CFI_ADJUST_CFA_OFFSET 4
849 jmp error_code
850 CFI_ENDPROC
851 .previous .text
853 ENTRY(alignment_check)
854 RING0_EC_FRAME
855 pushl $do_alignment_check
856 CFI_ADJUST_CFA_OFFSET 4
857 jmp error_code
858 CFI_ENDPROC
860 KPROBE_ENTRY(page_fault)
861 RING0_EC_FRAME
862 pushl $do_page_fault
863 CFI_ADJUST_CFA_OFFSET 4
864 jmp error_code
865 CFI_ENDPROC
866 .previous .text
868 #ifdef CONFIG_X86_MCE
869 ENTRY(machine_check)
870 RING0_INT_FRAME
871 pushl $0
872 CFI_ADJUST_CFA_OFFSET 4
873 pushl machine_check_vector
874 CFI_ADJUST_CFA_OFFSET 4
875 jmp error_code
876 CFI_ENDPROC
877 #endif
879 ENTRY(spurious_interrupt_bug)
880 RING0_INT_FRAME
881 pushl $0
882 CFI_ADJUST_CFA_OFFSET 4
883 pushl $do_spurious_interrupt_bug
884 CFI_ADJUST_CFA_OFFSET 4
885 jmp error_code
886 CFI_ENDPROC
888 #ifdef CONFIG_STACK_UNWIND
889 ENTRY(arch_unwind_init_running)
890 CFI_STARTPROC
891 movl 4(%esp), %edx
892 movl (%esp), %ecx
893 leal 4(%esp), %eax
894 movl %ebx, EBX(%edx)
895 xorl %ebx, %ebx
896 movl %ebx, ECX(%edx)
897 movl %ebx, EDX(%edx)
898 movl %esi, ESI(%edx)
899 movl %edi, EDI(%edx)
900 movl %ebp, EBP(%edx)
901 movl %ebx, EAX(%edx)
902 movl $__USER_DS, DS(%edx)
903 movl $__USER_DS, ES(%edx)
904 movl %ebx, ORIG_EAX(%edx)
905 movl %ecx, EIP(%edx)
906 movl 12(%esp), %ecx
907 movl $__KERNEL_CS, CS(%edx)
908 movl %ebx, EFLAGS(%edx)
909 movl %eax, OLDESP(%edx)
910 movl 8(%esp), %eax
911 movl %ecx, 8(%esp)
912 movl EBX(%edx), %ebx
913 movl $__KERNEL_DS, OLDSS(%edx)
914 jmpl *%eax
915 CFI_ENDPROC
916 ENDPROC(arch_unwind_init_running)
917 #endif
919 .section .rodata,"a"
920 #include "syscall_table.S"
922 syscall_table_size=(.-sys_call_table)