| blob | cad08703c4ad774f2e473f0dcc71b53cdb579a68 |
1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * linux/arch/x86_64/entry.S
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 * Copyright (C) 2000, 2001, 2002 Andi Kleen SuSE Labs
7 * Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
8 *
9 * entry.S contains the system-call and fault low-level handling routines.
10 *
11 * Some of this is documented in Documentation/x86/entry_64.rst
12 *
13 * A note on terminology:
14 * - iret frame: Architecture defined interrupt frame from SS to RIP
15 * at the top of the kernel process stack.
16 *
17 * Some macro usage:
18 * - SYM_FUNC_START/END:Define functions in the symbol table.
19 * - idtentry: Define exception entry points.
20 */
21 #include <linux/linkage.h>
22 #include <asm/segment.h>
23 #include <asm/cache.h>
24 #include <asm/errno.h>
25 #include <asm/asm-offsets.h>
26 #include <asm/msr.h>
27 #include <asm/unistd.h>
28 #include <asm/thread_info.h>
29 #include <asm/hw_irq.h>
30 #include <asm/page_types.h>
31 #include <asm/irqflags.h>
32 #include <asm/paravirt.h>
33 #include <asm/percpu.h>
34 #include <asm/asm.h>
35 #include <asm/smap.h>
36 #include <asm/pgtable_types.h>
37 #include <asm/export.h>
38 #include <asm/frame.h>
39 #include <asm/trapnr.h>
40 #include <asm/nospec-branch.h>
41 #include <asm/fsgsbase.h>
42 #include <linux/err.h>
44 #include "calling.h"
46 .code64
47 .section .entry.text, "ax"
49 #ifdef CONFIG_PARAVIRT_XXL
50 SYM_CODE_START(native_usergs_sysret64)
51 UNWIND_HINT_EMPTY
52 swapgs
53 sysretq
54 SYM_CODE_END(native_usergs_sysret64)
55 #endif /* CONFIG_PARAVIRT_XXL */
57 /*
58 * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers.
59 *
60 * This is the only entry point used for 64-bit system calls. The
61 * hardware interface is reasonably well designed and the register to
62 * argument mapping Linux uses fits well with the registers that are
63 * available when SYSCALL is used.
64 *
65 * SYSCALL instructions can be found inlined in libc implementations as
66 * well as some other programs and libraries. There are also a handful
67 * of SYSCALL instructions in the vDSO used, for example, as a
68 * clock_gettimeofday fallback.
69 *
70 * 64-bit SYSCALL saves rip to rcx, clears rflags.RF, then saves rflags to r11,
71 * then loads new ss, cs, and rip from previously programmed MSRs.
72 * rflags gets masked by a value from another MSR (so CLD and CLAC
73 * are not needed). SYSCALL does not save anything on the stack
74 * and does not change rsp.
75 *
76 * Registers on entry:
77 * rax system call number
78 * rcx return address
79 * r11 saved rflags (note: r11 is callee-clobbered register in C ABI)
80 * rdi arg0
81 * rsi arg1
82 * rdx arg2
83 * r10 arg3 (needs to be moved to rcx to conform to C ABI)
84 * r8 arg4
85 * r9 arg5
86 * (note: r12-r15, rbp, rbx are callee-preserved in C ABI)
87 *
88 * Only called from user space.
89 *
90 * When user can change pt_regs->foo always force IRET. That is because
91 * it deals with uncanonical addresses better. SYSRET has trouble
92 * with them due to bugs in both AMD and Intel CPUs.
93 */
95 SYM_CODE_START(entry_SYSCALL_64)
96 UNWIND_HINT_EMPTY
98 swapgs
99 /* tss.sp2 is scratch space. */
100 movq %rsp, PER_CPU_VAR(cpu_tss_rw + TSS_sp2)
101 SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp
102 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
104 SYM_INNER_LABEL(entry_SYSCALL_64_safe_stack, SYM_L_GLOBAL)
106 /* Construct struct pt_regs on stack */
107 pushq $__USER_DS /* pt_regs->ss */
108 pushq PER_CPU_VAR(cpu_tss_rw + TSS_sp2) /* pt_regs->sp */
109 pushq %r11 /* pt_regs->flags */
110 pushq $__USER_CS /* pt_regs->cs */
111 pushq %rcx /* pt_regs->ip */
112 SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL)
113 pushq %rax /* pt_regs->orig_ax */
115 PUSH_AND_CLEAR_REGS rax=$-ENOSYS
117 /* IRQs are off. */
118 movq %rax, %rdi
119 movq %rsp, %rsi
120 call do_syscall_64 /* returns with IRQs disabled */
122 /*
123 * Try to use SYSRET instead of IRET if we're returning to
124 * a completely clean 64-bit userspace context. If we're not,
125 * go to the slow exit path.
126 */
127 movq RCX(%rsp), %rcx
128 movq RIP(%rsp), %r11
130 cmpq %rcx, %r11 /* SYSRET requires RCX == RIP */
131 jne swapgs_restore_regs_and_return_to_usermode
133 /*
134 * On Intel CPUs, SYSRET with non-canonical RCX/RIP will #GP
135 * in kernel space. This essentially lets the user take over
136 * the kernel, since userspace controls RSP.
137 *
138 * If width of "canonical tail" ever becomes variable, this will need
139 * to be updated to remain correct on both old and new CPUs.
140 *
141 * Change top bits to match most significant bit (47th or 56th bit
142 * depending on paging mode) in the address.
143 */
144 #ifdef CONFIG_X86_5LEVEL
145 ALTERNATIVE "shl $(64 - 48), %rcx; sar $(64 - 48), %rcx", \
146 "shl $(64 - 57), %rcx; sar $(64 - 57), %rcx", X86_FEATURE_LA57
147 #else
148 shl $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
149 sar $(64 - (__VIRTUAL_MASK_SHIFT+1)), %rcx
150 #endif
152 /* If this changed %rcx, it was not canonical */
153 cmpq %rcx, %r11
154 jne swapgs_restore_regs_and_return_to_usermode
156 cmpq $__USER_CS, CS(%rsp) /* CS must match SYSRET */
157 jne swapgs_restore_regs_and_return_to_usermode
159 movq R11(%rsp), %r11
160 cmpq %r11, EFLAGS(%rsp) /* R11 == RFLAGS */
161 jne swapgs_restore_regs_and_return_to_usermode
163 /*
164 * SYSCALL clears RF when it saves RFLAGS in R11 and SYSRET cannot
165 * restore RF properly. If the slowpath sets it for whatever reason, we
166 * need to restore it correctly.
167 *
168 * SYSRET can restore TF, but unlike IRET, restoring TF results in a
169 * trap from userspace immediately after SYSRET. This would cause an
170 * infinite loop whenever #DB happens with register state that satisfies
171 * the opportunistic SYSRET conditions. For example, single-stepping
172 * this user code:
173 *
174 * movq $stuck_here, %rcx
175 * pushfq
176 * popq %r11
177 * stuck_here:
178 *
179 * would never get past 'stuck_here'.
180 */
181 testq $(X86_EFLAGS_RF|X86_EFLAGS_TF), %r11
182 jnz swapgs_restore_regs_and_return_to_usermode
184 /* nothing to check for RSP */
186 cmpq $__USER_DS, SS(%rsp) /* SS must match SYSRET */
187 jne swapgs_restore_regs_and_return_to_usermode
189 /*
190 * We win! This label is here just for ease of understanding
191 * perf profiles. Nothing jumps here.
192 */
193 syscall_return_via_sysret:
194 /* rcx and r11 are already restored (see code above) */
195 POP_REGS pop_rdi=0 skip_r11rcx=1
197 /*
198 * Now all regs are restored except RSP and RDI.
199 * Save old stack pointer and switch to trampoline stack.
200 */
201 movq %rsp, %rdi
202 movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
203 UNWIND_HINT_EMPTY
205 pushq RSP-RDI(%rdi) /* RSP */
206 pushq (%rdi) /* RDI */
208 /*
209 * We are on the trampoline stack. All regs except RDI are live.
210 * We can do future final exit work right here.
211 */
212 STACKLEAK_ERASE_NOCLOBBER
214 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
216 popq %rdi
217 popq %rsp
218 USERGS_SYSRET64
219 SYM_CODE_END(entry_SYSCALL_64)
221 /*
222 * %rdi: prev task
223 * %rsi: next task
224 */
225 .pushsection .text, "ax"
226 SYM_FUNC_START(__switch_to_asm)
227 /*
228 * Save callee-saved registers
229 * This must match the order in inactive_task_frame
230 */
231 pushq %rbp
232 pushq %rbx
233 pushq %r12
234 pushq %r13
235 pushq %r14
236 pushq %r15
238 /* switch stack */
239 movq %rsp, TASK_threadsp(%rdi)
240 movq TASK_threadsp(%rsi), %rsp
242 #ifdef CONFIG_STACKPROTECTOR
243 movq TASK_stack_canary(%rsi), %rbx
244 movq %rbx, PER_CPU_VAR(fixed_percpu_data) + stack_canary_offset
245 #endif
247 #ifdef CONFIG_RETPOLINE
248 /*
249 * When switching from a shallower to a deeper call stack
250 * the RSB may either underflow or use entries populated
251 * with userspace addresses. On CPUs where those concerns
252 * exist, overwrite the RSB with entries which capture
253 * speculative execution to prevent attack.
254 */
255 FILL_RETURN_BUFFER %r12, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_CTXSW
256 #endif
258 /* restore callee-saved registers */
259 popq %r15
260 popq %r14
261 popq %r13
262 popq %r12
263 popq %rbx
264 popq %rbp
266 jmp __switch_to
267 SYM_FUNC_END(__switch_to_asm)
268 .popsection
270 /*
271 * A newly forked process directly context switches into this address.
272 *
273 * rax: prev task we switched from
274 * rbx: kernel thread func (NULL for user thread)
275 * r12: kernel thread arg
276 */
277 .pushsection .text, "ax"
278 SYM_CODE_START(ret_from_fork)
279 UNWIND_HINT_EMPTY
280 movq %rax, %rdi
281 call schedule_tail /* rdi: 'prev' task parameter */
283 testq %rbx, %rbx /* from kernel_thread? */
284 jnz 1f /* kernel threads are uncommon */
286 2:
287 UNWIND_HINT_REGS
288 movq %rsp, %rdi
289 call syscall_exit_to_user_mode /* returns with IRQs disabled */
290 jmp swapgs_restore_regs_and_return_to_usermode
292 1:
293 /* kernel thread */
294 UNWIND_HINT_EMPTY
295 movq %r12, %rdi
296 CALL_NOSPEC rbx
297 /*
298 * A kernel thread is allowed to return here after successfully
299 * calling kernel_execve(). Exit to userspace to complete the execve()
300 * syscall.
301 */
302 movq $0, RAX(%rsp)
303 jmp 2b
304 SYM_CODE_END(ret_from_fork)
305 .popsection
307 .macro DEBUG_ENTRY_ASSERT_IRQS_OFF
308 #ifdef CONFIG_DEBUG_ENTRY
309 pushq %rax
310 SAVE_FLAGS(CLBR_RAX)
311 testl $X86_EFLAGS_IF, %eax
312 jz .Lokay_\@
313 ud2
314 .Lokay_\@:
315 popq %rax
316 #endif
317 .endm
319 /**
320 * idtentry_body - Macro to emit code calling the C function
321 * @cfunc: C function to be called
322 * @has_error_code: Hardware pushed error code on stack
323 */
324 .macro idtentry_body cfunc has_error_code:req
326 call error_entry
327 UNWIND_HINT_REGS
329 movq %rsp, %rdi /* pt_regs pointer into 1st argument*/
331 .if \has_error_code == 1
332 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
333 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
334 .endif
336 call \cfunc
338 jmp error_return
339 .endm
341 /**
342 * idtentry - Macro to generate entry stubs for simple IDT entries
343 * @vector: Vector number
344 * @asmsym: ASM symbol for the entry point
345 * @cfunc: C function to be called
346 * @has_error_code: Hardware pushed error code on stack
347 *
348 * The macro emits code to set up the kernel context for straight forward
349 * and simple IDT entries. No IST stack, no paranoid entry checks.
350 */
351 .macro idtentry vector asmsym cfunc has_error_code:req
352 SYM_CODE_START(\asmsym)
353 UNWIND_HINT_IRET_REGS offset=\has_error_code*8
354 ASM_CLAC
356 .if \has_error_code == 0
357 pushq $-1 /* ORIG_RAX: no syscall to restart */
358 .endif
360 .if \vector == X86_TRAP_BP
361 /*
362 * If coming from kernel space, create a 6-word gap to allow the
363 * int3 handler to emulate a call instruction.
364 */
365 testb $3, CS-ORIG_RAX(%rsp)
366 jnz .Lfrom_usermode_no_gap_\@
367 .rept 6
368 pushq 5*8(%rsp)
369 .endr
370 UNWIND_HINT_IRET_REGS offset=8
371 .Lfrom_usermode_no_gap_\@:
372 .endif
374 idtentry_body \cfunc \has_error_code
376 _ASM_NOKPROBE(\asmsym)
377 SYM_CODE_END(\asmsym)
378 .endm
380 /*
381 * Interrupt entry/exit.
382 *
383 + The interrupt stubs push (vector) onto the stack, which is the error_code
384 * position of idtentry exceptions, and jump to one of the two idtentry points
385 * (common/spurious).
386 *
387 * common_interrupt is a hotpath, align it to a cache line
388 */
389 .macro idtentry_irq vector cfunc
390 .p2align CONFIG_X86_L1_CACHE_SHIFT
391 idtentry \vector asm_\cfunc \cfunc has_error_code=1
392 .endm
394 /*
395 * System vectors which invoke their handlers directly and are not
396 * going through the regular common device interrupt handling code.
397 */
398 .macro idtentry_sysvec vector cfunc
399 idtentry \vector asm_\cfunc \cfunc has_error_code=0
400 .endm
402 /**
403 * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB
404 * @vector: Vector number
405 * @asmsym: ASM symbol for the entry point
406 * @cfunc: C function to be called
407 *
408 * The macro emits code to set up the kernel context for #MC and #DB
409 *
410 * If the entry comes from user space it uses the normal entry path
411 * including the return to user space work and preemption checks on
412 * exit.
413 *
414 * If hits in kernel mode then it needs to go through the paranoid
415 * entry as the exception can hit any random state. No preemption
416 * check on exit to keep the paranoid path simple.
417 */
418 .macro idtentry_mce_db vector asmsym cfunc
419 SYM_CODE_START(\asmsym)
420 UNWIND_HINT_IRET_REGS
421 ASM_CLAC
423 pushq $-1 /* ORIG_RAX: no syscall to restart */
425 /*
426 * If the entry is from userspace, switch stacks and treat it as
427 * a normal entry.
428 */
429 testb $3, CS-ORIG_RAX(%rsp)
430 jnz .Lfrom_usermode_switch_stack_\@
432 /* paranoid_entry returns GS information for paranoid_exit in EBX. */
433 call paranoid_entry
435 UNWIND_HINT_REGS
437 movq %rsp, %rdi /* pt_regs pointer */
439 call \cfunc
441 jmp paranoid_exit
443 /* Switch to the regular task stack and use the noist entry point */
444 .Lfrom_usermode_switch_stack_\@:
445 idtentry_body noist_\cfunc, has_error_code=0
447 _ASM_NOKPROBE(\asmsym)
448 SYM_CODE_END(\asmsym)
449 .endm
451 #ifdef CONFIG_AMD_MEM_ENCRYPT
452 /**
453 * idtentry_vc - Macro to generate entry stub for #VC
454 * @vector: Vector number
455 * @asmsym: ASM symbol for the entry point
456 * @cfunc: C function to be called
457 *
458 * The macro emits code to set up the kernel context for #VC. The #VC handler
459 * runs on an IST stack and needs to be able to cause nested #VC exceptions.
460 *
461 * To make this work the #VC entry code tries its best to pretend it doesn't use
462 * an IST stack by switching to the task stack if coming from user-space (which
463 * includes early SYSCALL entry path) or back to the stack in the IRET frame if
464 * entered from kernel-mode.
465 *
466 * If entered from kernel-mode the return stack is validated first, and if it is
467 * not safe to use (e.g. because it points to the entry stack) the #VC handler
468 * will switch to a fall-back stack (VC2) and call a special handler function.
469 *
470 * The macro is only used for one vector, but it is planned to be extended in
471 * the future for the #HV exception.
472 */
473 .macro idtentry_vc vector asmsym cfunc
474 SYM_CODE_START(\asmsym)
475 UNWIND_HINT_IRET_REGS
476 ASM_CLAC
478 /*
479 * If the entry is from userspace, switch stacks and treat it as
480 * a normal entry.
481 */
482 testb $3, CS-ORIG_RAX(%rsp)
483 jnz .Lfrom_usermode_switch_stack_\@
485 /*
486 * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX.
487 * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS
488 */
489 call paranoid_entry
491 UNWIND_HINT_REGS
493 /*
494 * Switch off the IST stack to make it free for nested exceptions. The
495 * vc_switch_off_ist() function will switch back to the interrupted
496 * stack if it is safe to do so. If not it switches to the VC fall-back
497 * stack.
498 */
499 movq %rsp, %rdi /* pt_regs pointer */
500 call vc_switch_off_ist
501 movq %rax, %rsp /* Switch to new stack */
503 UNWIND_HINT_REGS
505 /* Update pt_regs */
506 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
507 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
509 movq %rsp, %rdi /* pt_regs pointer */
511 call \cfunc
513 /*
514 * No need to switch back to the IST stack. The current stack is either
515 * identical to the stack in the IRET frame or the VC fall-back stack,
516 * so it is definitly mapped even with PTI enabled.
517 */
518 jmp paranoid_exit
520 /* Switch to the regular task stack */
521 .Lfrom_usermode_switch_stack_\@:
522 idtentry_body safe_stack_\cfunc, has_error_code=1
524 _ASM_NOKPROBE(\asmsym)
525 SYM_CODE_END(\asmsym)
526 .endm
527 #endif
529 /*
530 * Double fault entry. Straight paranoid. No checks from which context
531 * this comes because for the espfix induced #DF this would do the wrong
532 * thing.
533 */
534 .macro idtentry_df vector asmsym cfunc
535 SYM_CODE_START(\asmsym)
536 UNWIND_HINT_IRET_REGS offset=8
537 ASM_CLAC
539 /* paranoid_entry returns GS information for paranoid_exit in EBX. */
540 call paranoid_entry
541 UNWIND_HINT_REGS
543 movq %rsp, %rdi /* pt_regs pointer into first argument */
544 movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/
545 movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */
546 call \cfunc
548 jmp paranoid_exit
550 _ASM_NOKPROBE(\asmsym)
551 SYM_CODE_END(\asmsym)
552 .endm
554 /*
555 * Include the defines which emit the idt entries which are shared
556 * shared between 32 and 64 bit and emit the __irqentry_text_* markers
557 * so the stacktrace boundary checks work.
558 */
559 .align 16
560 .globl __irqentry_text_start
561 __irqentry_text_start:
563 #include <asm/idtentry.h>
565 .align 16
566 .globl __irqentry_text_end
567 __irqentry_text_end:
569 SYM_CODE_START_LOCAL(common_interrupt_return)
570 SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
571 #ifdef CONFIG_DEBUG_ENTRY
572 /* Assert that pt_regs indicates user mode. */
573 testb $3, CS(%rsp)
574 jnz 1f
575 ud2
576 1:
577 #endif
578 POP_REGS pop_rdi=0
580 /*
581 * The stack is now user RDI, orig_ax, RIP, CS, EFLAGS, RSP, SS.
582 * Save old stack pointer and switch to trampoline stack.
583 */
584 movq %rsp, %rdi
585 movq PER_CPU_VAR(cpu_tss_rw + TSS_sp0), %rsp
586 UNWIND_HINT_EMPTY
588 /* Copy the IRET frame to the trampoline stack. */
589 pushq 6*8(%rdi) /* SS */
590 pushq 5*8(%rdi) /* RSP */
591 pushq 4*8(%rdi) /* EFLAGS */
592 pushq 3*8(%rdi) /* CS */
593 pushq 2*8(%rdi) /* RIP */
595 /* Push user RDI on the trampoline stack. */
596 pushq (%rdi)
598 /*
599 * We are on the trampoline stack. All regs except RDI are live.
600 * We can do future final exit work right here.
601 */
602 STACKLEAK_ERASE_NOCLOBBER
604 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
606 /* Restore RDI. */
607 popq %rdi
608 SWAPGS
609 INTERRUPT_RETURN
612 SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
613 #ifdef CONFIG_DEBUG_ENTRY
614 /* Assert that pt_regs indicates kernel mode. */
615 testb $3, CS(%rsp)
616 jz 1f
617 ud2
618 1:
619 #endif
620 POP_REGS
621 addq $8, %rsp /* skip regs->orig_ax */
622 /*
623 * ARCH_HAS_MEMBARRIER_SYNC_CORE rely on IRET core serialization
624 * when returning from IPI handler.
625 */
626 INTERRUPT_RETURN
628 SYM_INNER_LABEL_ALIGN(native_iret, SYM_L_GLOBAL)
629 UNWIND_HINT_IRET_REGS
630 /*
631 * Are we returning to a stack segment from the LDT? Note: in
632 * 64-bit mode SS:RSP on the exception stack is always valid.
633 */
634 #ifdef CONFIG_X86_ESPFIX64
635 testb $4, (SS-RIP)(%rsp)
636 jnz native_irq_return_ldt
637 #endif
639 SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL)
640 /*
641 * This may fault. Non-paranoid faults on return to userspace are
642 * handled by fixup_bad_iret. These include #SS, #GP, and #NP.
643 * Double-faults due to espfix64 are handled in exc_double_fault.
644 * Other faults here are fatal.
645 */
646 iretq
648 #ifdef CONFIG_X86_ESPFIX64
649 native_irq_return_ldt:
650 /*
651 * We are running with user GSBASE. All GPRs contain their user
652 * values. We have a percpu ESPFIX stack that is eight slots
653 * long (see ESPFIX_STACK_SIZE). espfix_waddr points to the bottom
654 * of the ESPFIX stack.
655 *
656 * We clobber RAX and RDI in this code. We stash RDI on the
657 * normal stack and RAX on the ESPFIX stack.
658 *
659 * The ESPFIX stack layout we set up looks like this:
660 *
661 * --- top of ESPFIX stack ---
662 * SS
663 * RSP
664 * RFLAGS
665 * CS
666 * RIP <-- RSP points here when we're done
667 * RAX <-- espfix_waddr points here
668 * --- bottom of ESPFIX stack ---
669 */
671 pushq %rdi /* Stash user RDI */
672 SWAPGS /* to kernel GS */
673 SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi /* to kernel CR3 */
675 movq PER_CPU_VAR(espfix_waddr), %rdi
676 movq %rax, (0*8)(%rdi) /* user RAX */
677 movq (1*8)(%rsp), %rax /* user RIP */
678 movq %rax, (1*8)(%rdi)
679 movq (2*8)(%rsp), %rax /* user CS */
680 movq %rax, (2*8)(%rdi)
681 movq (3*8)(%rsp), %rax /* user RFLAGS */
682 movq %rax, (3*8)(%rdi)
683 movq (5*8)(%rsp), %rax /* user SS */
684 movq %rax, (5*8)(%rdi)
685 movq (4*8)(%rsp), %rax /* user RSP */
686 movq %rax, (4*8)(%rdi)
687 /* Now RAX == RSP. */
689 andl $0xffff0000, %eax /* RAX = (RSP & 0xffff0000) */
691 /*
692 * espfix_stack[31:16] == 0. The page tables are set up such that
693 * (espfix_stack | (X & 0xffff0000)) points to a read-only alias of
694 * espfix_waddr for any X. That is, there are 65536 RO aliases of
695 * the same page. Set up RSP so that RSP[31:16] contains the
696 * respective 16 bits of the /userspace/ RSP and RSP nonetheless
697 * still points to an RO alias of the ESPFIX stack.
698 */
699 orq PER_CPU_VAR(espfix_stack), %rax
701 SWITCH_TO_USER_CR3_STACK scratch_reg=%rdi
702 SWAPGS /* to user GS */
703 popq %rdi /* Restore user RDI */
705 movq %rax, %rsp
706 UNWIND_HINT_IRET_REGS offset=8
708 /*
709 * At this point, we cannot write to the stack any more, but we can
710 * still read.
711 */
712 popq %rax /* Restore user RAX */
714 /*
715 * RSP now points to an ordinary IRET frame, except that the page
716 * is read-only and RSP[31:16] are preloaded with the userspace
717 * values. We can now IRET back to userspace.
718 */
719 jmp native_irq_return_iret
720 #endif
721 SYM_CODE_END(common_interrupt_return)
722 _ASM_NOKPROBE(common_interrupt_return)
724 /*
725 * Reload gs selector with exception handling
726 * edi: new selector
727 *
728 * Is in entry.text as it shouldn't be instrumented.
729 */
730 SYM_FUNC_START(asm_load_gs_index)
731 FRAME_BEGIN
732 swapgs
733 .Lgs_change:
734 movl %edi, %gs
735 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE
736 swapgs
737 FRAME_END
738 ret
739 SYM_FUNC_END(asm_load_gs_index)
740 EXPORT_SYMBOL(asm_load_gs_index)
742 _ASM_EXTABLE(.Lgs_change, .Lbad_gs)
743 .section .fixup, "ax"
744 /* running with kernelgs */
745 SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs)
746 swapgs /* switch back to user gs */
747 .macro ZAP_GS
748 /* This can't be a string because the preprocessor needs to see it. */
749 movl $__USER_DS, %eax
750 movl %eax, %gs
751 .endm
752 ALTERNATIVE "", "ZAP_GS", X86_BUG_NULL_SEG
753 xorl %eax, %eax
754 movl %eax, %gs
755 jmp 2b
756 SYM_CODE_END(.Lbad_gs)
757 .previous
759 /*
760 * rdi: New stack pointer points to the top word of the stack
761 * rsi: Function pointer
762 * rdx: Function argument (can be NULL if none)
763 */
764 SYM_FUNC_START(asm_call_on_stack)
765 SYM_INNER_LABEL(asm_call_sysvec_on_stack, SYM_L_GLOBAL)
766 SYM_INNER_LABEL(asm_call_irq_on_stack, SYM_L_GLOBAL)
767 /*
768 * Save the frame pointer unconditionally. This allows the ORC
769 * unwinder to handle the stack switch.
770 */
771 pushq %rbp
772 mov %rsp, %rbp
774 /*
775 * The unwinder relies on the word at the top of the new stack
776 * page linking back to the previous RSP.
777 */
778 mov %rsp, (%rdi)
779 mov %rdi, %rsp
780 /* Move the argument to the right place */
781 mov %rdx, %rdi
783 1:
784 .pushsection .discard.instr_begin
785 .long 1b - .
786 .popsection
788 CALL_NOSPEC rsi
790 2:
791 .pushsection .discard.instr_end
792 .long 2b - .
793 .popsection
795 /* Restore the previous stack pointer from RBP. */
796 leaveq
797 ret
798 SYM_FUNC_END(asm_call_on_stack)
800 #ifdef CONFIG_XEN_PV
801 /*
802 * A note on the "critical region" in our callback handler.
803 * We want to avoid stacking callback handlers due to events occurring
804 * during handling of the last event. To do this, we keep events disabled
805 * until we've done all processing. HOWEVER, we must enable events before
806 * popping the stack frame (can't be done atomically) and so it would still
807 * be possible to get enough handler activations to overflow the stack.
808 * Although unlikely, bugs of that kind are hard to track down, so we'd
809 * like to avoid the possibility.
810 * So, on entry to the handler we detect whether we interrupted an
811 * existing activation in its critical region -- if so, we pop the current
812 * activation and restart the handler using the previous one.
813 *
814 * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs)
815 */
816 SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback)
818 /*
819 * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will
820 * see the correct pointer to the pt_regs
821 */
822 UNWIND_HINT_FUNC
823 movq %rdi, %rsp /* we don't return, adjust the stack frame */
824 UNWIND_HINT_REGS
826 call xen_pv_evtchn_do_upcall
828 jmp error_return
829 SYM_CODE_END(exc_xen_hypervisor_callback)
831 /*
832 * Hypervisor uses this for application faults while it executes.
833 * We get here for two reasons:
834 * 1. Fault while reloading DS, ES, FS or GS
835 * 2. Fault while executing IRET
836 * Category 1 we do not need to fix up as Xen has already reloaded all segment
837 * registers that could be reloaded and zeroed the others.
838 * Category 2 we fix up by killing the current process. We cannot use the
839 * normal Linux return path in this case because if we use the IRET hypercall
840 * to pop the stack frame we end up in an infinite loop of failsafe callbacks.
841 * We distinguish between categories by comparing each saved segment register
842 * with its current contents: any discrepancy means we in category 1.
843 */
844 SYM_CODE_START(xen_failsafe_callback)
845 UNWIND_HINT_EMPTY
846 movl %ds, %ecx
847 cmpw %cx, 0x10(%rsp)
848 jne 1f
849 movl %es, %ecx
850 cmpw %cx, 0x18(%rsp)
851 jne 1f
852 movl %fs, %ecx
853 cmpw %cx, 0x20(%rsp)
854 jne 1f
855 movl %gs, %ecx
856 cmpw %cx, 0x28(%rsp)
857 jne 1f
858 /* All segments match their saved values => Category 2 (Bad IRET). */
859 movq (%rsp), %rcx
860 movq 8(%rsp), %r11
861 addq $0x30, %rsp
862 pushq $0 /* RIP */
863 UNWIND_HINT_IRET_REGS offset=8
864 jmp asm_exc_general_protection
865 1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */
866 movq (%rsp), %rcx
867 movq 8(%rsp), %r11
868 addq $0x30, %rsp
869 UNWIND_HINT_IRET_REGS
870 pushq $-1 /* orig_ax = -1 => not a system call */
871 PUSH_AND_CLEAR_REGS
872 ENCODE_FRAME_POINTER
873 jmp error_return
874 SYM_CODE_END(xen_failsafe_callback)
875 #endif /* CONFIG_XEN_PV */
877 /*
878 * Save all registers in pt_regs. Return GSBASE related information
879 * in EBX depending on the availability of the FSGSBASE instructions:
880 *
881 * FSGSBASE R/EBX
882 * N 0 -> SWAPGS on exit
883 * 1 -> no SWAPGS on exit
884 *
885 * Y GSBASE value at entry, must be restored in paranoid_exit
886 */
887 SYM_CODE_START_LOCAL(paranoid_entry)
888 UNWIND_HINT_FUNC
889 cld
890 PUSH_AND_CLEAR_REGS save_ret=1
891 ENCODE_FRAME_POINTER 8
893 /*
894 * Always stash CR3 in %r14. This value will be restored,
895 * verbatim, at exit. Needed if paranoid_entry interrupted
896 * another entry that already switched to the user CR3 value
897 * but has not yet returned to userspace.
898 *
899 * This is also why CS (stashed in the "iret frame" by the
900 * hardware at entry) can not be used: this may be a return
901 * to kernel code, but with a user CR3 value.
902 *
903 * Switching CR3 does not depend on kernel GSBASE so it can
904 * be done before switching to the kernel GSBASE. This is
905 * required for FSGSBASE because the kernel GSBASE has to
906 * be retrieved from a kernel internal table.
907 */
908 SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg=%rax save_reg=%r14
910 /*
911 * Handling GSBASE depends on the availability of FSGSBASE.
912 *
913 * Without FSGSBASE the kernel enforces that negative GSBASE
914 * values indicate kernel GSBASE. With FSGSBASE no assumptions
915 * can be made about the GSBASE value when entering from user
916 * space.
917 */
918 ALTERNATIVE "jmp .Lparanoid_entry_checkgs", "", X86_FEATURE_FSGSBASE
920 /*
921 * Read the current GSBASE and store it in %rbx unconditionally,
922 * retrieve and set the current CPUs kernel GSBASE. The stored value
923 * has to be restored in paranoid_exit unconditionally.
924 *
925 * The unconditional write to GS base below ensures that no subsequent
926 * loads based on a mispredicted GS base can happen, therefore no LFENCE
927 * is needed here.
928 */
929 SAVE_AND_SET_GSBASE scratch_reg=%rax save_reg=%rbx
930 ret
932 .Lparanoid_entry_checkgs:
933 /* EBX = 1 -> kernel GSBASE active, no restore required */
934 movl $1, %ebx
935 /*
936 * The kernel-enforced convention is a negative GSBASE indicates
937 * a kernel value. No SWAPGS needed on entry and exit.
938 */
939 movl $MSR_GS_BASE, %ecx
940 rdmsr
941 testl %edx, %edx
942 jns .Lparanoid_entry_swapgs
943 ret
945 .Lparanoid_entry_swapgs:
946 SWAPGS
948 /*
949 * The above SAVE_AND_SWITCH_TO_KERNEL_CR3 macro doesn't do an
950 * unconditional CR3 write, even in the PTI case. So do an lfence
951 * to prevent GS speculation, regardless of whether PTI is enabled.
952 */
953 FENCE_SWAPGS_KERNEL_ENTRY
955 /* EBX = 0 -> SWAPGS required on exit */
956 xorl %ebx, %ebx
957 ret
958 SYM_CODE_END(paranoid_entry)
960 /*
961 * "Paranoid" exit path from exception stack. This is invoked
962 * only on return from non-NMI IST interrupts that came
963 * from kernel space.
964 *
965 * We may be returning to very strange contexts (e.g. very early
966 * in syscall entry), so checking for preemption here would
967 * be complicated. Fortunately, there's no good reason to try
968 * to handle preemption here.
969 *
970 * R/EBX contains the GSBASE related information depending on the
971 * availability of the FSGSBASE instructions:
972 *
973 * FSGSBASE R/EBX
974 * N 0 -> SWAPGS on exit
975 * 1 -> no SWAPGS on exit
976 *
977 * Y User space GSBASE, must be restored unconditionally
978 */
979 SYM_CODE_START_LOCAL(paranoid_exit)
980 UNWIND_HINT_REGS
981 /*
982 * The order of operations is important. RESTORE_CR3 requires
983 * kernel GSBASE.
984 *
985 * NB to anyone to try to optimize this code: this code does
986 * not execute at all for exceptions from user mode. Those
987 * exceptions go through error_exit instead.
988 */
989 RESTORE_CR3 scratch_reg=%rax save_reg=%r14
991 /* Handle the three GSBASE cases */
992 ALTERNATIVE "jmp .Lparanoid_exit_checkgs", "", X86_FEATURE_FSGSBASE
994 /* With FSGSBASE enabled, unconditionally restore GSBASE */
995 wrgsbase %rbx
996 jmp restore_regs_and_return_to_kernel
998 .Lparanoid_exit_checkgs:
999 /* On non-FSGSBASE systems, conditionally do SWAPGS */
1000 testl %ebx, %ebx
1001 jnz restore_regs_and_return_to_kernel
1003 /* We are returning to a context with user GSBASE */
1004 SWAPGS_UNSAFE_STACK
1005 jmp restore_regs_and_return_to_kernel
1006 SYM_CODE_END(paranoid_exit)
1008 /*
1009 * Save all registers in pt_regs, and switch GS if needed.
1010 */
1011 SYM_CODE_START_LOCAL(error_entry)
1012 UNWIND_HINT_FUNC
1013 cld
1014 PUSH_AND_CLEAR_REGS save_ret=1
1015 ENCODE_FRAME_POINTER 8
1016 testb $3, CS+8(%rsp)
1017 jz .Lerror_kernelspace
1019 /*
1020 * We entered from user mode or we're pretending to have entered
1021 * from user mode due to an IRET fault.
1022 */
1023 SWAPGS
1024 FENCE_SWAPGS_USER_ENTRY
1025 /* We have user CR3. Change to kernel CR3. */
1026 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1028 .Lerror_entry_from_usermode_after_swapgs:
1029 /* Put us onto the real thread stack. */
1030 popq %r12 /* save return addr in %12 */
1031 movq %rsp, %rdi /* arg0 = pt_regs pointer */
1032 call sync_regs
1033 movq %rax, %rsp /* switch stack */
1034 ENCODE_FRAME_POINTER
1035 pushq %r12
1036 ret
1038 .Lerror_entry_done_lfence:
1039 FENCE_SWAPGS_KERNEL_ENTRY
1040 .Lerror_entry_done:
1041 ret
1043 /*
1044 * There are two places in the kernel that can potentially fault with
1045 * usergs. Handle them here. B stepping K8s sometimes report a
1046 * truncated RIP for IRET exceptions returning to compat mode. Check
1047 * for these here too.
1048 */
1049 .Lerror_kernelspace:
1050 leaq native_irq_return_iret(%rip), %rcx
1051 cmpq %rcx, RIP+8(%rsp)
1052 je .Lerror_bad_iret
1053 movl %ecx, %eax /* zero extend */
1054 cmpq %rax, RIP+8(%rsp)
1055 je .Lbstep_iret
1056 cmpq $.Lgs_change, RIP+8(%rsp)
1057 jne .Lerror_entry_done_lfence
1059 /*
1060 * hack: .Lgs_change can fail with user gsbase. If this happens, fix up
1061 * gsbase and proceed. We'll fix up the exception and land in
1062 * .Lgs_change's error handler with kernel gsbase.
1063 */
1064 SWAPGS
1065 FENCE_SWAPGS_USER_ENTRY
1066 jmp .Lerror_entry_done
1068 .Lbstep_iret:
1069 /* Fix truncated RIP */
1070 movq %rcx, RIP+8(%rsp)
1071 /* fall through */
1073 .Lerror_bad_iret:
1074 /*
1075 * We came from an IRET to user mode, so we have user
1076 * gsbase and CR3. Switch to kernel gsbase and CR3:
1077 */
1078 SWAPGS
1079 FENCE_SWAPGS_USER_ENTRY
1080 SWITCH_TO_KERNEL_CR3 scratch_reg=%rax
1082 /*
1083 * Pretend that the exception came from user mode: set up pt_regs
1084 * as if we faulted immediately after IRET.
1085 */
1086 mov %rsp, %rdi
1087 call fixup_bad_iret
1088 mov %rax, %rsp
1089 jmp .Lerror_entry_from_usermode_after_swapgs
1090 SYM_CODE_END(error_entry)
1092 SYM_CODE_START_LOCAL(error_return)
1093 UNWIND_HINT_REGS
1094 DEBUG_ENTRY_ASSERT_IRQS_OFF
1095 testb $3, CS(%rsp)
1096 jz restore_regs_and_return_to_kernel
1097 jmp swapgs_restore_regs_and_return_to_usermode
1098 SYM_CODE_END(error_return)
1100 /*
1101 * Runs on exception stack. Xen PV does not go through this path at all,
1102 * so we can use real assembly here.
1103 *
1104 * Registers:
1105 * %r14: Used to save/restore the CR3 of the interrupted context
1106 * when PAGE_TABLE_ISOLATION is in use. Do not clobber.
1107 */
1108 SYM_CODE_START(asm_exc_nmi)
1109 UNWIND_HINT_IRET_REGS
1111 /*
1112 * We allow breakpoints in NMIs. If a breakpoint occurs, then
1113 * the iretq it performs will take us out of NMI context.
1114 * This means that we can have nested NMIs where the next
1115 * NMI is using the top of the stack of the previous NMI. We
1116 * can't let it execute because the nested NMI will corrupt the
1117 * stack of the previous NMI. NMI handlers are not re-entrant
1118 * anyway.
1119 *
1120 * To handle this case we do the following:
1121 * Check the a special location on the stack that contains
1122 * a variable that is set when NMIs are executing.
1123 * The interrupted task's stack is also checked to see if it
1124 * is an NMI stack.
1125 * If the variable is not set and the stack is not the NMI
1126 * stack then:
1127 * o Set the special variable on the stack
1128 * o Copy the interrupt frame into an "outermost" location on the
1129 * stack
1130 * o Copy the interrupt frame into an "iret" location on the stack
1131 * o Continue processing the NMI
1132 * If the variable is set or the previous stack is the NMI stack:
1133 * o Modify the "iret" location to jump to the repeat_nmi
1134 * o return back to the first NMI
1135 *
1136 * Now on exit of the first NMI, we first clear the stack variable
1137 * The NMI stack will tell any nested NMIs at that point that it is
1138 * nested. Then we pop the stack normally with iret, and if there was
1139 * a nested NMI that updated the copy interrupt stack frame, a
1140 * jump will be made to the repeat_nmi code that will handle the second
1141 * NMI.
1142 *
1143 * However, espfix prevents us from directly returning to userspace
1144 * with a single IRET instruction. Similarly, IRET to user mode
1145 * can fault. We therefore handle NMIs from user space like
1146 * other IST entries.
1147 */
1149 ASM_CLAC
1151 /* Use %rdx as our temp variable throughout */
1152 pushq %rdx
1154 testb $3, CS-RIP+8(%rsp)
1155 jz .Lnmi_from_kernel
1157 /*
1158 * NMI from user mode. We need to run on the thread stack, but we
1159 * can't go through the normal entry paths: NMIs are masked, and
1160 * we don't want to enable interrupts, because then we'll end
1161 * up in an awkward situation in which IRQs are on but NMIs
1162 * are off.
1163 *
1164 * We also must not push anything to the stack before switching
1165 * stacks lest we corrupt the "NMI executing" variable.
1166 */
1168 swapgs
1169 cld
1170 FENCE_SWAPGS_USER_ENTRY
1171 SWITCH_TO_KERNEL_CR3 scratch_reg=%rdx
1172 movq %rsp, %rdx
1173 movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
1174 UNWIND_HINT_IRET_REGS base=%rdx offset=8
1175 pushq 5*8(%rdx) /* pt_regs->ss */
1176 pushq 4*8(%rdx) /* pt_regs->rsp */
1177 pushq 3*8(%rdx) /* pt_regs->flags */
1178 pushq 2*8(%rdx) /* pt_regs->cs */
1179 pushq 1*8(%rdx) /* pt_regs->rip */
1180 UNWIND_HINT_IRET_REGS
1181 pushq $-1 /* pt_regs->orig_ax */
1182 PUSH_AND_CLEAR_REGS rdx=(%rdx)
1183 ENCODE_FRAME_POINTER
1185 /*
1186 * At this point we no longer need to worry about stack damage
1187 * due to nesting -- we're on the normal thread stack and we're
1188 * done with the NMI stack.
1189 */
1191 movq %rsp, %rdi
1192 movq $-1, %rsi
1193 call exc_nmi
1195 /*
1196 * Return back to user mode. We must *not* do the normal exit
1197 * work, because we don't want to enable interrupts.
1198 */
1199 jmp swapgs_restore_regs_and_return_to_usermode
1201 .Lnmi_from_kernel:
1202 /*
1203 * Here's what our stack frame will look like:
1204 * +---------------------------------------------------------+
1205 * | original SS |
1206 * | original Return RSP |
1207 * | original RFLAGS |
1208 * | original CS |
1209 * | original RIP |
1210 * +---------------------------------------------------------+
1211 * | temp storage for rdx |
1212 * +---------------------------------------------------------+
1213 * | "NMI executing" variable |
1214 * +---------------------------------------------------------+
1215 * | iret SS } Copied from "outermost" frame |
1216 * | iret Return RSP } on each loop iteration; overwritten |
1217 * | iret RFLAGS } by a nested NMI to force another |
1218 * | iret CS } iteration if needed. |
1219 * | iret RIP } |
1220 * +---------------------------------------------------------+
1221 * | outermost SS } initialized in first_nmi; |
1222 * | outermost Return RSP } will not be changed before |
1223 * | outermost RFLAGS } NMI processing is done. |
1224 * | outermost CS } Copied to "iret" frame on each |
1225 * | outermost RIP } iteration. |
1226 * +---------------------------------------------------------+
1227 * | pt_regs |
1228 * +---------------------------------------------------------+
1229 *
1230 * The "original" frame is used by hardware. Before re-enabling
1231 * NMIs, we need to be done with it, and we need to leave enough
1232 * space for the asm code here.
1233 *
1234 * We return by executing IRET while RSP points to the "iret" frame.
1235 * That will either return for real or it will loop back into NMI
1236 * processing.
1237 *
1238 * The "outermost" frame is copied to the "iret" frame on each
1239 * iteration of the loop, so each iteration starts with the "iret"
1240 * frame pointing to the final return target.
1241 */
1243 /*
1244 * Determine whether we're a nested NMI.
1245 *
1246 * If we interrupted kernel code between repeat_nmi and
1247 * end_repeat_nmi, then we are a nested NMI. We must not
1248 * modify the "iret" frame because it's being written by
1249 * the outer NMI. That's okay; the outer NMI handler is
1250 * about to about to call exc_nmi() anyway, so we can just
1251 * resume the outer NMI.
1252 */
1254 movq $repeat_nmi, %rdx
1255 cmpq 8(%rsp), %rdx
1256 ja 1f
1257 movq $end_repeat_nmi, %rdx
1258 cmpq 8(%rsp), %rdx
1259 ja nested_nmi_out
1260 1:
1262 /*
1263 * Now check "NMI executing". If it's set, then we're nested.
1264 * This will not detect if we interrupted an outer NMI just
1265 * before IRET.
1266 */
1267 cmpl $1, -8(%rsp)
1268 je nested_nmi
1270 /*
1271 * Now test if the previous stack was an NMI stack. This covers
1272 * the case where we interrupt an outer NMI after it clears
1273 * "NMI executing" but before IRET. We need to be careful, though:
1274 * there is one case in which RSP could point to the NMI stack
1275 * despite there being no NMI active: naughty userspace controls
1276 * RSP at the very beginning of the SYSCALL targets. We can
1277 * pull a fast one on naughty userspace, though: we program
1278 * SYSCALL to mask DF, so userspace cannot cause DF to be set
1279 * if it controls the kernel's RSP. We set DF before we clear
1280 * "NMI executing".
1281 */
1282 lea 6*8(%rsp), %rdx
1283 /* Compare the NMI stack (rdx) with the stack we came from (4*8(%rsp)) */
1284 cmpq %rdx, 4*8(%rsp)
1285 /* If the stack pointer is above the NMI stack, this is a normal NMI */
1286 ja first_nmi
1288 subq $EXCEPTION_STKSZ, %rdx
1289 cmpq %rdx, 4*8(%rsp)
1290 /* If it is below the NMI stack, it is a normal NMI */
1291 jb first_nmi
1293 /* Ah, it is within the NMI stack. */
1295 testb $(X86_EFLAGS_DF >> 8), (3*8 + 1)(%rsp)
1296 jz first_nmi /* RSP was user controlled. */
1298 /* This is a nested NMI. */
1300 nested_nmi:
1301 /*
1302 * Modify the "iret" frame to point to repeat_nmi, forcing another
1303 * iteration of NMI handling.
1304 */
1305 subq $8, %rsp
1306 leaq -10*8(%rsp), %rdx
1307 pushq $__KERNEL_DS
1308 pushq %rdx
1309 pushfq
1310 pushq $__KERNEL_CS
1311 pushq $repeat_nmi
1313 /* Put stack back */
1314 addq $(6*8), %rsp
1316 nested_nmi_out:
1317 popq %rdx
1319 /* We are returning to kernel mode, so this cannot result in a fault. */
1320 iretq
1322 first_nmi:
1323 /* Restore rdx. */
1324 movq (%rsp), %rdx
1326 /* Make room for "NMI executing". */
1327 pushq $0
1329 /* Leave room for the "iret" frame */
1330 subq $(5*8), %rsp
1332 /* Copy the "original" frame to the "outermost" frame */
1333 .rept 5
1334 pushq 11*8(%rsp)
1335 .endr
1336 UNWIND_HINT_IRET_REGS
1338 /* Everything up to here is safe from nested NMIs */
1340 #ifdef CONFIG_DEBUG_ENTRY
1341 /*
1342 * For ease of testing, unmask NMIs right away. Disabled by
1343 * default because IRET is very expensive.
1344 */
1345 pushq $0 /* SS */
1346 pushq %rsp /* RSP (minus 8 because of the previous push) */
1347 addq $8, (%rsp) /* Fix up RSP */
1348 pushfq /* RFLAGS */
1349 pushq $__KERNEL_CS /* CS */
1350 pushq $1f /* RIP */
1351 iretq /* continues at repeat_nmi below */
1352 UNWIND_HINT_IRET_REGS
1353 1:
1354 #endif
1356 repeat_nmi:
1357 /*
1358 * If there was a nested NMI, the first NMI's iret will return
1359 * here. But NMIs are still enabled and we can take another
1360 * nested NMI. The nested NMI checks the interrupted RIP to see
1361 * if it is between repeat_nmi and end_repeat_nmi, and if so
1362 * it will just return, as we are about to repeat an NMI anyway.
1363 * This makes it safe to copy to the stack frame that a nested
1364 * NMI will update.
1365 *
1366 * RSP is pointing to "outermost RIP". gsbase is unknown, but, if
1367 * we're repeating an NMI, gsbase has the same value that it had on
1368 * the first iteration. paranoid_entry will load the kernel
1369 * gsbase if needed before we call exc_nmi(). "NMI executing"
1370 * is zero.
1371 */
1372 movq $1, 10*8(%rsp) /* Set "NMI executing". */
1374 /*
1375 * Copy the "outermost" frame to the "iret" frame. NMIs that nest
1376 * here must not modify the "iret" frame while we're writing to
1377 * it or it will end up containing garbage.
1378 */
1379 addq $(10*8), %rsp
1380 .rept 5
1381 pushq -6*8(%rsp)
1382 .endr
1383 subq $(5*8), %rsp
1384 end_repeat_nmi:
1386 /*
1387 * Everything below this point can be preempted by a nested NMI.
1388 * If this happens, then the inner NMI will change the "iret"
1389 * frame to point back to repeat_nmi.
1390 */
1391 pushq $-1 /* ORIG_RAX: no syscall to restart */
1393 /*
1394 * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
1395 * as we should not be calling schedule in NMI context.
1396 * Even with normal interrupts enabled. An NMI should not be
1397 * setting NEED_RESCHED or anything that normal interrupts and
1398 * exceptions might do.
1399 */
1400 call paranoid_entry
1401 UNWIND_HINT_REGS
1403 movq %rsp, %rdi
1404 movq $-1, %rsi
1405 call exc_nmi
1407 /* Always restore stashed CR3 value (see paranoid_entry) */
1408 RESTORE_CR3 scratch_reg=%r15 save_reg=%r14
1410 /*
1411 * The above invocation of paranoid_entry stored the GSBASE
1412 * related information in R/EBX depending on the availability
1413 * of FSGSBASE.
1414 *
1415 * If FSGSBASE is enabled, restore the saved GSBASE value
1416 * unconditionally, otherwise take the conditional SWAPGS path.
1417 */
1418 ALTERNATIVE "jmp nmi_no_fsgsbase", "", X86_FEATURE_FSGSBASE
1420 wrgsbase %rbx
1421 jmp nmi_restore
1423 nmi_no_fsgsbase:
1424 /* EBX == 0 -> invoke SWAPGS */
1425 testl %ebx, %ebx
1426 jnz nmi_restore
1428 nmi_swapgs:
1429 SWAPGS_UNSAFE_STACK
1431 nmi_restore:
1432 POP_REGS
1434 /*
1435 * Skip orig_ax and the "outermost" frame to point RSP at the "iret"
1436 * at the "iret" frame.
1437 */
1438 addq $6*8, %rsp
1440 /*
1441 * Clear "NMI executing". Set DF first so that we can easily
1442 * distinguish the remaining code between here and IRET from
1443 * the SYSCALL entry and exit paths.
1444 *
1445 * We arguably should just inspect RIP instead, but I (Andy) wrote
1446 * this code when I had the misapprehension that Xen PV supported
1447 * NMIs, and Xen PV would break that approach.
1448 */
1449 std
1450 movq $0, 5*8(%rsp) /* clear "NMI executing" */
1452 /*
1453 * iretq reads the "iret" frame and exits the NMI stack in a
1454 * single instruction. We are returning to kernel mode, so this
1455 * cannot result in a fault. Similarly, we don't need to worry
1456 * about espfix64 on the way back to kernel mode.
1457 */
1458 iretq
1459 SYM_CODE_END(asm_exc_nmi)
1461 #ifndef CONFIG_IA32_EMULATION
1462 /*
1463 * This handles SYSCALL from 32-bit code. There is no way to program
1464 * MSRs to fully disable 32-bit SYSCALL.
1465 */
1466 SYM_CODE_START(ignore_sysret)
1467 UNWIND_HINT_EMPTY
1468 mov $-ENOSYS, %eax
1469 sysretl
1470 SYM_CODE_END(ignore_sysret)
1471 #endif
1473 .pushsection .text, "ax"
1474 SYM_CODE_START(rewind_stack_do_exit)
1475 UNWIND_HINT_FUNC
1476 /* Prevent any naive code from trying to unwind to our caller. */
1477 xorl %ebp, %ebp
1479 movq PER_CPU_VAR(cpu_current_top_of_stack), %rax
1480 leaq -PTREGS_SIZE(%rax), %rsp
1481 UNWIND_HINT_REGS
1483 call do_exit
1484 SYM_CODE_END(rewind_stack_do_exit)
1485 .popsection