2 * Asm versions of Xen pv-ops, suitable for either direct use or
3 * inlining. The inline versions are the same as the direct-use
4 * versions, with the pre- and post-amble chopped off.
6 * This code is encoded for size rather than absolute efficiency, with
7 * a view to being able to inline as much as possible.
9 * We only bother with direct forms (ie, vcpu in pda) of the
10 * operations here; the indirect forms are better handled in C, since
11 * they're generally too large to inline anyway.
14 #include <asm/errno.h>
15 #include <asm/percpu.h>
16 #include <asm/processor-flags.h>
17 #include <asm/segment.h>
19 #include <xen/interface/xen.h>
23 ENTRY(xen_adjust_exception_frame)
28 hypercall_iret = hypercall_page + __HYPERVISOR_iret * 32
36 * rip <-- standard iret frame
41 * r11 }<-- pushed by hypercall page
51 * sysexit is not used for 64-bit processes, so it's only ever used to
52 * return to 32-bit compat userspace.
64 RELOC(xen_sysexit, 1b+1)
68 * We're already on the usermode stack at this point, but
69 * still with the kernel gs, so we can easily switch back
71 movq %rsp, PER_CPU_VAR(rsp_scratch)
72 movq PER_CPU_VAR(kernel_stack), %rsp
75 pushq PER_CPU_VAR(rsp_scratch)
80 pushq $VGCF_in_syscall
82 ENDPATCH(xen_sysret64)
83 RELOC(xen_sysret64, 1b+1)
87 * We're already on the usermode stack at this point, but
88 * still with the kernel gs, so we can easily switch back
90 movq %rsp, PER_CPU_VAR(rsp_scratch)
91 movq PER_CPU_VAR(kernel_stack), %rsp
94 pushq PER_CPU_VAR(rsp_scratch)
100 1: jmp hypercall_iret
101 ENDPATCH(xen_sysret32)
102 RELOC(xen_sysret32, 1b+1)
105 * Xen handles syscall callbacks much like ordinary exceptions, which
109 * - an iret-like stack frame on the stack (including rcx and r11):
118 * In all the entrypoints, we undo all that to make it look like a
119 * CPU-generated syscall/sysenter and jump to the normal entrypoint.
122 .macro undo_xen_syscall
128 /* Normal 64-bit system call target */
129 ENTRY(xen_syscall_target)
131 jmp system_call_after_swapgs
132 ENDPROC(xen_syscall_target)
134 #ifdef CONFIG_IA32_EMULATION
136 /* 32-bit compat syscall target */
137 ENTRY(xen_syscall32_target)
139 jmp ia32_cstar_target
140 ENDPROC(xen_syscall32_target)
142 /* 32-bit compat sysenter target */
143 ENTRY(xen_sysenter_target)
145 jmp ia32_sysenter_target
146 ENDPROC(xen_sysenter_target)
148 #else /* !CONFIG_IA32_EMULATION */
150 ENTRY(xen_syscall32_target)
151 ENTRY(xen_sysenter_target)
152 lea 16(%rsp), %rsp /* strip %rcx, %r11 */
156 ENDPROC(xen_syscall32_target)
157 ENDPROC(xen_sysenter_target)
159 #endif /* CONFIG_IA32_EMULATION */