2 * Compatibility mode system call entry point for x86-64.
4 * Copyright 2000-2002 Andi Kleen, SuSE Labs.
7 #include <asm/dwarf2.h>
8 #include <asm/calling.h>
9 #include <asm/asm-offsets.h>
10 #include <asm/current.h>
11 #include <asm/errno.h>
12 #include <asm/ia32_unistd.h>
13 #include <asm/thread_info.h>
14 #include <asm/segment.h>
15 #include <asm/irqflags.h>
18 #include <linux/linkage.h>
19 #include <linux/err.h>
21 /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
22 #include <linux/elf-em.h>
23 #define AUDIT_ARCH_I386 (EM_386|__AUDIT_ARCH_LE)
24 #define __AUDIT_ARCH_LE 0x40000000
26 #ifndef CONFIG_AUDITSYSCALL
27 #define sysexit_audit ia32_ret_from_sys_call
28 #define sysretl_audit ia32_ret_from_sys_call
31 .section .entry.text, "ax"
33 .macro IA32_ARG_FIXUP noebp=0
41 movl %edx,%edx /* zero extension */
45 .macro CLEAR_RREGS offset=0, _r9=rax
47 movq %rax,\offset+R11(%rsp)
48 movq %rax,\offset+R10(%rsp)
49 movq %\_r9,\offset+R9(%rsp)
50 movq %rax,\offset+R8(%rsp)
54 * Reload arg registers from stack in case ptrace changed them.
55 * We don't reload %eax because syscall_trace_enter() returned
56 * the %rax value we should see. Instead, we just truncate that
57 * value to 32 bits again as we did on entry from user mode.
58 * If it's a new value set by user_regset during entry tracing,
59 * this matches the normal truncation of the user-mode value.
60 * If it's -1 to make us punt the syscall, then (u32)-1 is still
61 * an appropriately invalid value.
63 .macro LOAD_ARGS32 offset, _r9=0
65 movl \offset+16(%rsp),%r9d
67 movl \offset+40(%rsp),%ecx
68 movl \offset+48(%rsp),%edx
69 movl \offset+56(%rsp),%esi
70 movl \offset+64(%rsp),%edi
71 movl %eax,%eax /* zero extension */
74 .macro CFI_STARTPROC32 simple
86 #ifdef CONFIG_PARAVIRT
87 ENTRY(native_usergs_sysret32)
90 ENDPROC(native_usergs_sysret32)
92 ENTRY(native_irq_enable_sysexit)
96 ENDPROC(native_irq_enable_sysexit)
100 * 32bit SYSENTER instruction entry.
103 * %eax System call number.
114 * This is purely a fast path. For anything complicated we use the int 0x80
115 * path below. Set up a complete hardware stack frame to share code
116 * with the int 0x80 path.
118 ENTRY(ia32_sysenter_target)
119 CFI_STARTPROC32 simple
124 movq PER_CPU_VAR(kernel_stack), %rsp
125 addq $(KERNEL_STACK_OFFSET),%rsp
127 * No need to follow this irqs on/off section: the syscall
128 * disabled irqs, here we enable it straight after entry:
130 ENABLE_INTERRUPTS(CLBR_NONE)
131 movl %ebp,%ebp /* zero extension */
132 pushq_cfi $__USER32_DS
133 /*CFI_REL_OFFSET ss,0*/
137 /*CFI_REL_OFFSET rflags,0*/
138 movl TI_sysenter_return+THREAD_INFO(%rsp,3*8-KERNEL_STACK_OFFSET),%r10d
140 pushq_cfi $__USER32_CS
141 /*CFI_REL_OFFSET cs,0*/
148 /* no need to do an access_ok check here because rbp has been
149 32bit zero extended */
152 _ASM_EXTABLE(1b,ia32_badarg)
154 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
155 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
157 jnz sysenter_tracesys
158 cmpq $(IA32_NR_syscalls-1),%rax
163 call *ia32_sys_call_table(,%rax,8)
164 movq %rax,RAX-ARGOFFSET(%rsp)
165 DISABLE_INTERRUPTS(CLBR_NONE)
167 testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
169 sysexit_from_sys_call:
170 andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
171 /* clear IF, that popfq doesn't enable interrupts early */
172 andl $~0x200,EFLAGS-R11(%rsp)
173 movl RIP-R11(%rsp),%edx /* User %eip */
175 RESTORE_ARGS 0,24,0,0,0,0
181 /*CFI_RESTORE rflags*/
182 popq_cfi %rcx /* User %esp */
185 ENABLE_INTERRUPTS_SYSEXIT32
187 #ifdef CONFIG_AUDITSYSCALL
188 .macro auditsys_entry_common
189 movl %esi,%r9d /* 6th arg: 4th syscall arg */
190 movl %edx,%r8d /* 5th arg: 3rd syscall arg */
191 /* (already in %ecx) 4th arg: 2nd syscall arg */
192 movl %ebx,%edx /* 3rd arg: 1st syscall arg */
193 movl %eax,%esi /* 2nd arg: syscall number */
194 movl $AUDIT_ARCH_I386,%edi /* 1st arg: audit arch */
195 call __audit_syscall_entry
196 movl RAX-ARGOFFSET(%rsp),%eax /* reload syscall number */
197 cmpq $(IA32_NR_syscalls-1),%rax
199 movl %ebx,%edi /* reload 1st syscall arg */
200 movl RCX-ARGOFFSET(%rsp),%esi /* reload 2nd syscall arg */
201 movl RDX-ARGOFFSET(%rsp),%edx /* reload 3rd syscall arg */
202 movl RSI-ARGOFFSET(%rsp),%ecx /* reload 4th syscall arg */
203 movl RDI-ARGOFFSET(%rsp),%r8d /* reload 5th syscall arg */
206 .macro auditsys_exit exit
207 testl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
208 jnz ia32_ret_from_sys_call
210 ENABLE_INTERRUPTS(CLBR_NONE)
211 movl %eax,%esi /* second arg, syscall return value */
212 cmpl $-MAX_ERRNO,%eax /* is it an error ? */
214 movslq %eax, %rsi /* if error sign extend to 64 bits */
215 1: setbe %al /* 1 if error, 0 if not */
216 movzbl %al,%edi /* zero-extend that into %edi */
217 call __audit_syscall_exit
218 movq RAX-ARGOFFSET(%rsp),%rax /* reload syscall return value */
219 movl $(_TIF_ALLWORK_MASK & ~_TIF_SYSCALL_AUDIT),%edi
220 DISABLE_INTERRUPTS(CLBR_NONE)
222 testl %edi,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
224 CLEAR_RREGS -ARGOFFSET
230 auditsys_entry_common
231 movl %ebp,%r9d /* reload 6th syscall arg */
232 jmp sysenter_dispatch
235 auditsys_exit sysexit_from_sys_call
239 #ifdef CONFIG_AUDITSYSCALL
240 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
245 movq $-ENOSYS,RAX(%rsp)/* ptrace can change this for a bad syscall */
246 movq %rsp,%rdi /* &pt_regs -> arg1 */
247 call syscall_trace_enter
248 LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */
250 cmpq $(IA32_NR_syscalls-1),%rax
251 ja int_ret_from_sys_call /* sysenter_tracesys has set RAX(%rsp) */
254 ENDPROC(ia32_sysenter_target)
257 * 32bit SYSCALL instruction entry.
260 * %eax System call number.
266 * %ebp Arg2 [note: not saved in the stack frame, should not be touched]
272 * This is purely a fast path. For anything complicated we use the int 0x80
273 * path below. Set up a complete hardware stack frame to share code
274 * with the int 0x80 path.
276 ENTRY(ia32_cstar_target)
277 CFI_STARTPROC32 simple
279 CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET
281 /*CFI_REGISTER rflags,r11*/
285 movq PER_CPU_VAR(kernel_stack),%rsp
287 * No need to follow this irqs on/off section: the syscall
288 * disabled irqs and here we enable it straight after entry:
290 ENABLE_INTERRUPTS(CLBR_NONE)
292 movl %eax,%eax /* zero extension */
293 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
294 movq %rcx,RIP-ARGOFFSET(%rsp)
295 CFI_REL_OFFSET rip,RIP-ARGOFFSET
296 movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */
298 movq $__USER32_CS,CS-ARGOFFSET(%rsp)
299 movq $__USER32_DS,SS-ARGOFFSET(%rsp)
300 movq %r11,EFLAGS-ARGOFFSET(%rsp)
301 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
302 movq %r8,RSP-ARGOFFSET(%rsp)
303 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
304 /* no need to do an access_ok check here because r8 has been
305 32bit zero extended */
306 /* hardware stack frame is complete now */
309 _ASM_EXTABLE(1b,ia32_badarg)
311 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
312 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
315 cmpq $IA32_NR_syscalls-1,%rax
320 call *ia32_sys_call_table(,%rax,8)
321 movq %rax,RAX-ARGOFFSET(%rsp)
322 DISABLE_INTERRUPTS(CLBR_NONE)
324 testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
326 sysretl_from_sys_call:
327 andl $~TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
328 RESTORE_ARGS 0,-ARG_SKIP,0,0,0
329 movl RIP-ARGOFFSET(%rsp),%ecx
331 movl EFLAGS-ARGOFFSET(%rsp),%r11d
332 /*CFI_REGISTER rflags,r11*/
337 movl RSP-ARGOFFSET(%rsp),%esp
341 #ifdef CONFIG_AUDITSYSCALL
344 movl %r9d,R9-ARGOFFSET(%rsp) /* register to be clobbered by call */
345 auditsys_entry_common
346 movl R9-ARGOFFSET(%rsp),%r9d /* reload 6th syscall arg */
350 auditsys_exit sysretl_from_sys_call
354 #ifdef CONFIG_AUDITSYSCALL
355 testl $(_TIF_WORK_SYSCALL_ENTRY & ~_TIF_SYSCALL_AUDIT),TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
361 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
362 movq %rsp,%rdi /* &pt_regs -> arg1 */
363 call syscall_trace_enter
364 LOAD_ARGS32 ARGOFFSET, 1 /* reload args from stack in case ptrace changed it */
367 cmpq $(IA32_NR_syscalls-1),%rax
368 ja int_ret_from_sys_call /* cstar_tracesys has set RAX(%rsp) */
370 END(ia32_cstar_target)
379 * Emulated IA32 system calls via int 0x80.
382 * %eax System call number.
388 * %ebp Arg6 [note: not saved in the stack frame, should not be touched]
391 * Uses the same stack frame as the x86-64 version.
392 * All registers except %eax must be saved (but ptrace may violate that)
393 * Arguments are zero extended. For system calls that want sign extension and
394 * take long arguments a wrapper is needed. Most calls can just be called
396 * Assumes it is only called from user space and entered with interrupts off.
400 CFI_STARTPROC32 simple
402 CFI_DEF_CFA rsp,SS+8-RIP
403 /*CFI_REL_OFFSET ss,SS-RIP*/
404 CFI_REL_OFFSET rsp,RSP-RIP
405 /*CFI_REL_OFFSET rflags,EFLAGS-RIP*/
406 /*CFI_REL_OFFSET cs,CS-RIP*/
407 CFI_REL_OFFSET rip,RIP-RIP
408 PARAVIRT_ADJUST_EXCEPTION_FRAME
411 * No need to follow this irqs on/off section: the syscall
412 * disabled irqs and here we enable it straight after entry:
414 ENABLE_INTERRUPTS(CLBR_NONE)
418 /* note the registers are not zero extended to the sf.
419 this could be a problem. */
421 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
422 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
424 cmpq $(IA32_NR_syscalls-1),%rax
428 call *ia32_sys_call_table(,%rax,8) # xxx: rip relative
430 movq %rax,RAX-ARGOFFSET(%rsp)
431 ia32_ret_from_sys_call:
432 CLEAR_RREGS -ARGOFFSET
433 jmp int_ret_from_sys_call
438 movq $-ENOSYS,RAX(%rsp) /* ptrace can change this for a bad syscall */
439 movq %rsp,%rdi /* &pt_regs -> arg1 */
440 call syscall_trace_enter
441 LOAD_ARGS32 ARGOFFSET /* reload args from stack in case ptrace changed it */
443 cmpq $(IA32_NR_syscalls-1),%rax
444 ja int_ret_from_sys_call /* ia32_tracesys has set RAX(%rsp) */
449 movq $0,ORIG_RAX-ARGOFFSET(%rsp)
455 .macro PTREGSCALL label, func, arg
458 leaq \func(%rip),%rax
459 leaq -ARGOFFSET+8(%rsp),\arg /* 8 for return address */
460 jmp ia32_ptregs_common
465 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
466 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
467 PTREGSCALL stub32_execve, compat_sys_execve, %rcx
468 PTREGSCALL stub32_fork, sys_fork, %rdi
469 PTREGSCALL stub32_vfork, sys_vfork, %rdi
470 PTREGSCALL stub32_iopl, sys_iopl, %rsi
474 leaq sys_clone(%rip),%rax
476 jmp ia32_ptregs_common
482 CFI_STARTPROC32 simple
484 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
485 CFI_REL_OFFSET rax,RAX-ARGOFFSET
486 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
487 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
488 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
489 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
490 CFI_REL_OFFSET rip,RIP-ARGOFFSET
491 /* CFI_REL_OFFSET cs,CS-ARGOFFSET*/
492 /* CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
493 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
494 /* CFI_REL_OFFSET ss,SS-ARGOFFSET*/
498 jmp ia32_sysret /* misbalances the return cache */
500 END(ia32_ptregs_common)