| blob | 86c289832272d71ba48786414bb6e4ecb9b9cb14 |
1 /*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
18 #include <linux/linkage.h>
20 #include <asm/alternative.h>
21 #include <asm/asm-offsets.h>
22 #include <asm/assembler.h>
23 #include <asm/cpufeature.h>
24 #include <asm/debug-monitors.h>
25 #include <asm/esr.h>
26 #include <asm/fpsimdmacros.h>
27 #include <asm/kvm.h>
28 #include <asm/kvm_arm.h>
29 #include <asm/kvm_asm.h>
30 #include <asm/kvm_mmu.h>
31 #include <asm/memory.h>
33 #define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
34 #define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
35 #define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)
36 #define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x)
38 .text
39 .pushsection .hyp.text, "ax"
40 .align PAGE_SHIFT
42 .macro save_common_regs
43 // x2: base address for cpu context
44 // x3: tmp register
46 add x3, x2, #CPU_XREG_OFFSET(19)
47 stp x19, x20, [x3]
48 stp x21, x22, [x3, #16]
49 stp x23, x24, [x3, #32]
50 stp x25, x26, [x3, #48]
51 stp x27, x28, [x3, #64]
52 stp x29, lr, [x3, #80]
54 mrs x19, sp_el0
55 mrs x20, elr_el2 // pc before entering el2
56 mrs x21, spsr_el2 // pstate before entering el2
58 stp x19, x20, [x3, #96]
59 str x21, [x3, #112]
61 mrs x22, sp_el1
62 mrs x23, elr_el1
63 mrs x24, spsr_el1
65 str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
66 str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
67 str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
68 .endm
70 .macro restore_common_regs
71 // x2: base address for cpu context
72 // x3: tmp register
74 ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
75 ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
76 ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
78 msr sp_el1, x22
79 msr elr_el1, x23
80 msr spsr_el1, x24
82 add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0
83 ldp x19, x20, [x3]
84 ldr x21, [x3, #16]
86 msr sp_el0, x19
87 msr elr_el2, x20 // pc on return from el2
88 msr spsr_el2, x21 // pstate on return from el2
90 add x3, x2, #CPU_XREG_OFFSET(19)
91 ldp x19, x20, [x3]
92 ldp x21, x22, [x3, #16]
93 ldp x23, x24, [x3, #32]
94 ldp x25, x26, [x3, #48]
95 ldp x27, x28, [x3, #64]
96 ldp x29, lr, [x3, #80]
97 .endm
99 .macro save_host_regs
100 save_common_regs
101 .endm
103 .macro restore_host_regs
104 restore_common_regs
105 .endm
107 .macro save_fpsimd
108 // x2: cpu context address
109 // x3, x4: tmp regs
110 add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
111 fpsimd_save x3, 4
112 .endm
114 .macro restore_fpsimd
115 // x2: cpu context address
116 // x3, x4: tmp regs
117 add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
118 fpsimd_restore x3, 4
119 .endm
121 .macro save_guest_regs
122 // x0 is the vcpu address
123 // x1 is the return code, do not corrupt!
124 // x2 is the cpu context
125 // x3 is a tmp register
126 // Guest's x0-x3 are on the stack
128 // Compute base to save registers
129 add x3, x2, #CPU_XREG_OFFSET(4)
130 stp x4, x5, [x3]
131 stp x6, x7, [x3, #16]
132 stp x8, x9, [x3, #32]
133 stp x10, x11, [x3, #48]
134 stp x12, x13, [x3, #64]
135 stp x14, x15, [x3, #80]
136 stp x16, x17, [x3, #96]
137 str x18, [x3, #112]
139 pop x6, x7 // x2, x3
140 pop x4, x5 // x0, x1
142 add x3, x2, #CPU_XREG_OFFSET(0)
143 stp x4, x5, [x3]
144 stp x6, x7, [x3, #16]
146 save_common_regs
147 .endm
149 .macro restore_guest_regs
150 // x0 is the vcpu address.
151 // x2 is the cpu context
152 // x3 is a tmp register
154 // Prepare x0-x3 for later restore
155 add x3, x2, #CPU_XREG_OFFSET(0)
156 ldp x4, x5, [x3]
157 ldp x6, x7, [x3, #16]
158 push x4, x5 // Push x0-x3 on the stack
159 push x6, x7
161 // x4-x18
162 ldp x4, x5, [x3, #32]
163 ldp x6, x7, [x3, #48]
164 ldp x8, x9, [x3, #64]
165 ldp x10, x11, [x3, #80]
166 ldp x12, x13, [x3, #96]
167 ldp x14, x15, [x3, #112]
168 ldp x16, x17, [x3, #128]
169 ldr x18, [x3, #144]
171 // x19-x29, lr, sp*, elr*, spsr*
172 restore_common_regs
174 // Last bits of the 64bit state
175 pop x2, x3
176 pop x0, x1
178 // Do not touch any register after this!
179 .endm
181 /*
182 * Macros to perform system register save/restore.
183 *
184 * Ordering here is absolutely critical, and must be kept consistent
185 * in {save,restore}_sysregs, {save,restore}_guest_32bit_state,
186 * and in kvm_asm.h.
187 *
188 * In other words, don't touch any of these unless you know what
189 * you are doing.
190 */
191 .macro save_sysregs
192 // x2: base address for cpu context
193 // x3: tmp register
195 add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
197 mrs x4, vmpidr_el2
198 mrs x5, csselr_el1
199 mrs x6, sctlr_el1
200 mrs x7, actlr_el1
201 mrs x8, cpacr_el1
202 mrs x9, ttbr0_el1
203 mrs x10, ttbr1_el1
204 mrs x11, tcr_el1
205 mrs x12, esr_el1
206 mrs x13, afsr0_el1
207 mrs x14, afsr1_el1
208 mrs x15, far_el1
209 mrs x16, mair_el1
210 mrs x17, vbar_el1
211 mrs x18, contextidr_el1
212 mrs x19, tpidr_el0
213 mrs x20, tpidrro_el0
214 mrs x21, tpidr_el1
215 mrs x22, amair_el1
216 mrs x23, cntkctl_el1
217 mrs x24, par_el1
218 mrs x25, mdscr_el1
220 stp x4, x5, [x3]
221 stp x6, x7, [x3, #16]
222 stp x8, x9, [x3, #32]
223 stp x10, x11, [x3, #48]
224 stp x12, x13, [x3, #64]
225 stp x14, x15, [x3, #80]
226 stp x16, x17, [x3, #96]
227 stp x18, x19, [x3, #112]
228 stp x20, x21, [x3, #128]
229 stp x22, x23, [x3, #144]
230 stp x24, x25, [x3, #160]
231 .endm
233 .macro save_debug type
234 // x4: pointer to register set
235 // x5: number of registers to skip
236 // x6..x22 trashed
238 adr x22, 1f
239 add x22, x22, x5, lsl #2
240 br x22
241 1:
242 mrs x21, \type\()15_el1
243 mrs x20, \type\()14_el1
244 mrs x19, \type\()13_el1
245 mrs x18, \type\()12_el1
246 mrs x17, \type\()11_el1
247 mrs x16, \type\()10_el1
248 mrs x15, \type\()9_el1
249 mrs x14, \type\()8_el1
250 mrs x13, \type\()7_el1
251 mrs x12, \type\()6_el1
252 mrs x11, \type\()5_el1
253 mrs x10, \type\()4_el1
254 mrs x9, \type\()3_el1
255 mrs x8, \type\()2_el1
256 mrs x7, \type\()1_el1
257 mrs x6, \type\()0_el1
259 adr x22, 1f
260 add x22, x22, x5, lsl #2
261 br x22
262 1:
263 str x21, [x4, #(15 * 8)]
264 str x20, [x4, #(14 * 8)]
265 str x19, [x4, #(13 * 8)]
266 str x18, [x4, #(12 * 8)]
267 str x17, [x4, #(11 * 8)]
268 str x16, [x4, #(10 * 8)]
269 str x15, [x4, #(9 * 8)]
270 str x14, [x4, #(8 * 8)]
271 str x13, [x4, #(7 * 8)]
272 str x12, [x4, #(6 * 8)]
273 str x11, [x4, #(5 * 8)]
274 str x10, [x4, #(4 * 8)]
275 str x9, [x4, #(3 * 8)]
276 str x8, [x4, #(2 * 8)]
277 str x7, [x4, #(1 * 8)]
278 str x6, [x4, #(0 * 8)]
279 .endm
281 .macro restore_sysregs
282 // x2: base address for cpu context
283 // x3: tmp register
285 add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
287 ldp x4, x5, [x3]
288 ldp x6, x7, [x3, #16]
289 ldp x8, x9, [x3, #32]
290 ldp x10, x11, [x3, #48]
291 ldp x12, x13, [x3, #64]
292 ldp x14, x15, [x3, #80]
293 ldp x16, x17, [x3, #96]
294 ldp x18, x19, [x3, #112]
295 ldp x20, x21, [x3, #128]
296 ldp x22, x23, [x3, #144]
297 ldp x24, x25, [x3, #160]
299 msr vmpidr_el2, x4
300 msr csselr_el1, x5
301 msr sctlr_el1, x6
302 msr actlr_el1, x7
303 msr cpacr_el1, x8
304 msr ttbr0_el1, x9
305 msr ttbr1_el1, x10
306 msr tcr_el1, x11
307 msr esr_el1, x12
308 msr afsr0_el1, x13
309 msr afsr1_el1, x14
310 msr far_el1, x15
311 msr mair_el1, x16
312 msr vbar_el1, x17
313 msr contextidr_el1, x18
314 msr tpidr_el0, x19
315 msr tpidrro_el0, x20
316 msr tpidr_el1, x21
317 msr amair_el1, x22
318 msr cntkctl_el1, x23
319 msr par_el1, x24
320 msr mdscr_el1, x25
321 .endm
323 .macro restore_debug type
324 // x4: pointer to register set
325 // x5: number of registers to skip
326 // x6..x22 trashed
328 adr x22, 1f
329 add x22, x22, x5, lsl #2
330 br x22
331 1:
332 ldr x21, [x4, #(15 * 8)]
333 ldr x20, [x4, #(14 * 8)]
334 ldr x19, [x4, #(13 * 8)]
335 ldr x18, [x4, #(12 * 8)]
336 ldr x17, [x4, #(11 * 8)]
337 ldr x16, [x4, #(10 * 8)]
338 ldr x15, [x4, #(9 * 8)]
339 ldr x14, [x4, #(8 * 8)]
340 ldr x13, [x4, #(7 * 8)]
341 ldr x12, [x4, #(6 * 8)]
342 ldr x11, [x4, #(5 * 8)]
343 ldr x10, [x4, #(4 * 8)]
344 ldr x9, [x4, #(3 * 8)]
345 ldr x8, [x4, #(2 * 8)]
346 ldr x7, [x4, #(1 * 8)]
347 ldr x6, [x4, #(0 * 8)]
349 adr x22, 1f
350 add x22, x22, x5, lsl #2
351 br x22
352 1:
353 msr \type\()15_el1, x21
354 msr \type\()14_el1, x20
355 msr \type\()13_el1, x19
356 msr \type\()12_el1, x18
357 msr \type\()11_el1, x17
358 msr \type\()10_el1, x16
359 msr \type\()9_el1, x15
360 msr \type\()8_el1, x14
361 msr \type\()7_el1, x13
362 msr \type\()6_el1, x12
363 msr \type\()5_el1, x11
364 msr \type\()4_el1, x10
365 msr \type\()3_el1, x9
366 msr \type\()2_el1, x8
367 msr \type\()1_el1, x7
368 msr \type\()0_el1, x6
369 .endm
371 .macro skip_32bit_state tmp, target
372 // Skip 32bit state if not needed
373 mrs \tmp, hcr_el2
374 tbnz \tmp, #HCR_RW_SHIFT, \target
375 .endm
377 .macro skip_tee_state tmp, target
378 // Skip ThumbEE state if not needed
379 mrs \tmp, id_pfr0_el1
380 tbz \tmp, #12, \target
381 .endm
383 .macro skip_debug_state tmp, target
384 ldr \tmp, [x0, #VCPU_DEBUG_FLAGS]
385 tbz \tmp, #KVM_ARM64_DEBUG_DIRTY_SHIFT, \target
386 .endm
388 /*
389 * Branch to target if CPTR_EL2.TFP bit is set (VFP/SIMD trapping enabled)
390 */
391 .macro skip_fpsimd_state tmp, target
392 mrs \tmp, cptr_el2
393 tbnz \tmp, #CPTR_EL2_TFP_SHIFT, \target
394 .endm
396 .macro compute_debug_state target
397 // Compute debug state: If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY
398 // is set, we do a full save/restore cycle and disable trapping.
399 add x25, x0, #VCPU_CONTEXT
401 // Check the state of MDSCR_EL1
402 ldr x25, [x25, #CPU_SYSREG_OFFSET(MDSCR_EL1)]
403 and x26, x25, #DBG_MDSCR_KDE
404 and x25, x25, #DBG_MDSCR_MDE
405 adds xzr, x25, x26
406 b.eq 9998f // Nothing to see there
408 // If any interesting bits was set, we must set the flag
409 mov x26, #KVM_ARM64_DEBUG_DIRTY
410 str x26, [x0, #VCPU_DEBUG_FLAGS]
411 b 9999f // Don't skip restore
413 9998:
414 // Otherwise load the flags from memory in case we recently
415 // trapped
416 skip_debug_state x25, \target
417 9999:
418 .endm
420 .macro save_guest_32bit_state
421 skip_32bit_state x3, 1f
423 add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
424 mrs x4, spsr_abt
425 mrs x5, spsr_und
426 mrs x6, spsr_irq
427 mrs x7, spsr_fiq
428 stp x4, x5, [x3]
429 stp x6, x7, [x3, #16]
431 add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
432 mrs x4, dacr32_el2
433 mrs x5, ifsr32_el2
434 stp x4, x5, [x3]
436 skip_fpsimd_state x8, 2f
437 mrs x6, fpexc32_el2
438 str x6, [x3, #16]
439 2:
440 skip_debug_state x8, 1f
441 mrs x7, dbgvcr32_el2
442 str x7, [x3, #24]
443 1:
444 .endm
446 .macro restore_guest_32bit_state
447 skip_32bit_state x3, 1f
449 add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
450 ldp x4, x5, [x3]
451 ldp x6, x7, [x3, #16]
452 msr spsr_abt, x4
453 msr spsr_und, x5
454 msr spsr_irq, x6
455 msr spsr_fiq, x7
457 add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
458 ldp x4, x5, [x3]
459 msr dacr32_el2, x4
460 msr ifsr32_el2, x5
462 skip_debug_state x8, 1f
463 ldr x7, [x3, #24]
464 msr dbgvcr32_el2, x7
465 1:
466 .endm
468 .macro activate_traps
469 ldr x2, [x0, #VCPU_HCR_EL2]
471 /*
472 * We are about to set CPTR_EL2.TFP to trap all floating point
473 * register accesses to EL2, however, the ARM ARM clearly states that
474 * traps are only taken to EL2 if the operation would not otherwise
475 * trap to EL1. Therefore, always make sure that for 32-bit guests,
476 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
477 */
478 tbnz x2, #HCR_RW_SHIFT, 99f // open code skip_32bit_state
479 mov x3, #(1 << 30)
480 msr fpexc32_el2, x3
481 isb
482 99:
483 msr hcr_el2, x2
484 mov x2, #CPTR_EL2_TTA
485 orr x2, x2, #CPTR_EL2_TFP
486 msr cptr_el2, x2
488 mov x2, #(1 << 15) // Trap CP15 Cr=15
489 msr hstr_el2, x2
491 // Monitor Debug Config - see kvm_arm_setup_debug()
492 ldr x2, [x0, #VCPU_MDCR_EL2]
493 msr mdcr_el2, x2
494 .endm
496 .macro deactivate_traps
497 mov x2, #HCR_RW
498 msr hcr_el2, x2
499 msr hstr_el2, xzr
501 mrs x2, mdcr_el2
502 and x2, x2, #MDCR_EL2_HPMN_MASK
503 msr mdcr_el2, x2
504 .endm
506 .macro activate_vm
507 ldr x1, [x0, #VCPU_KVM]
508 kern_hyp_va x1
509 ldr x2, [x1, #KVM_VTTBR]
510 msr vttbr_el2, x2
511 .endm
513 .macro deactivate_vm
514 msr vttbr_el2, xzr
515 .endm
517 /*
518 * Call into the vgic backend for state saving
519 */
520 .macro save_vgic_state
521 alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
522 bl __save_vgic_v2_state
523 alternative_else
524 bl __save_vgic_v3_state
525 alternative_endif
526 mrs x24, hcr_el2
527 mov x25, #HCR_INT_OVERRIDE
528 neg x25, x25
529 and x24, x24, x25
530 msr hcr_el2, x24
531 .endm
533 /*
534 * Call into the vgic backend for state restoring
535 */
536 .macro restore_vgic_state
537 mrs x24, hcr_el2
538 ldr x25, [x0, #VCPU_IRQ_LINES]
539 orr x24, x24, #HCR_INT_OVERRIDE
540 orr x24, x24, x25
541 msr hcr_el2, x24
542 alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
543 bl __restore_vgic_v2_state
544 alternative_else
545 bl __restore_vgic_v3_state
546 alternative_endif
547 .endm
549 .macro save_timer_state
550 // x0: vcpu pointer
551 ldr x2, [x0, #VCPU_KVM]
552 kern_hyp_va x2
553 ldr w3, [x2, #KVM_TIMER_ENABLED]
554 cbz w3, 1f
556 mrs x3, cntv_ctl_el0
557 and x3, x3, #3
558 str w3, [x0, #VCPU_TIMER_CNTV_CTL]
560 isb
562 mrs x3, cntv_cval_el0
563 str x3, [x0, #VCPU_TIMER_CNTV_CVAL]
565 1:
566 // Disable the virtual timer
567 msr cntv_ctl_el0, xzr
569 // Allow physical timer/counter access for the host
570 mrs x2, cnthctl_el2
571 orr x2, x2, #3
572 msr cnthctl_el2, x2
574 // Clear cntvoff for the host
575 msr cntvoff_el2, xzr
576 .endm
578 .macro restore_timer_state
579 // x0: vcpu pointer
580 // Disallow physical timer access for the guest
581 // Physical counter access is allowed
582 mrs x2, cnthctl_el2
583 orr x2, x2, #1
584 bic x2, x2, #2
585 msr cnthctl_el2, x2
587 ldr x2, [x0, #VCPU_KVM]
588 kern_hyp_va x2
589 ldr w3, [x2, #KVM_TIMER_ENABLED]
590 cbz w3, 1f
592 ldr x3, [x2, #KVM_TIMER_CNTVOFF]
593 msr cntvoff_el2, x3
594 ldr x2, [x0, #VCPU_TIMER_CNTV_CVAL]
595 msr cntv_cval_el0, x2
596 isb
598 ldr w2, [x0, #VCPU_TIMER_CNTV_CTL]
599 and x2, x2, #3
600 msr cntv_ctl_el0, x2
601 1:
602 .endm
604 __save_sysregs:
605 save_sysregs
606 ret
608 __restore_sysregs:
609 restore_sysregs
610 ret
612 /* Save debug state */
613 __save_debug:
614 // x2: ptr to CPU context
615 // x3: ptr to debug reg struct
616 // x4/x5/x6-22/x24-26: trashed
618 mrs x26, id_aa64dfr0_el1
619 ubfx x24, x26, #12, #4 // Extract BRPs
620 ubfx x25, x26, #20, #4 // Extract WRPs
621 mov w26, #15
622 sub w24, w26, w24 // How many BPs to skip
623 sub w25, w26, w25 // How many WPs to skip
625 mov x5, x24
626 add x4, x3, #DEBUG_BCR
627 save_debug dbgbcr
628 add x4, x3, #DEBUG_BVR
629 save_debug dbgbvr
631 mov x5, x25
632 add x4, x3, #DEBUG_WCR
633 save_debug dbgwcr
634 add x4, x3, #DEBUG_WVR
635 save_debug dbgwvr
637 mrs x21, mdccint_el1
638 str x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
639 ret
641 /* Restore debug state */
642 __restore_debug:
643 // x2: ptr to CPU context
644 // x3: ptr to debug reg struct
645 // x4/x5/x6-22/x24-26: trashed
647 mrs x26, id_aa64dfr0_el1
648 ubfx x24, x26, #12, #4 // Extract BRPs
649 ubfx x25, x26, #20, #4 // Extract WRPs
650 mov w26, #15
651 sub w24, w26, w24 // How many BPs to skip
652 sub w25, w26, w25 // How many WPs to skip
654 mov x5, x24
655 add x4, x3, #DEBUG_BCR
656 restore_debug dbgbcr
657 add x4, x3, #DEBUG_BVR
658 restore_debug dbgbvr
660 mov x5, x25
661 add x4, x3, #DEBUG_WCR
662 restore_debug dbgwcr
663 add x4, x3, #DEBUG_WVR
664 restore_debug dbgwvr
666 ldr x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
667 msr mdccint_el1, x21
669 ret
671 __save_fpsimd:
672 skip_fpsimd_state x3, 1f
673 save_fpsimd
674 1: ret
676 __restore_fpsimd:
677 skip_fpsimd_state x3, 1f
678 restore_fpsimd
679 1: ret
681 switch_to_guest_fpsimd:
682 push x4, lr
684 mrs x2, cptr_el2
685 bic x2, x2, #CPTR_EL2_TFP
686 msr cptr_el2, x2
687 isb
689 mrs x0, tpidr_el2
691 ldr x2, [x0, #VCPU_HOST_CONTEXT]
692 kern_hyp_va x2
693 bl __save_fpsimd
695 add x2, x0, #VCPU_CONTEXT
696 bl __restore_fpsimd
698 skip_32bit_state x3, 1f
699 ldr x4, [x2, #CPU_SYSREG_OFFSET(FPEXC32_EL2)]
700 msr fpexc32_el2, x4
701 1:
702 pop x4, lr
703 pop x2, x3
704 pop x0, x1
706 eret
708 /*
709 * u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);
710 *
711 * This is the world switch. The first half of the function
712 * deals with entering the guest, and anything from __kvm_vcpu_return
713 * to the end of the function deals with reentering the host.
714 * On the enter path, only x0 (vcpu pointer) must be preserved until
715 * the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception
716 * code) must both be preserved until the epilogue.
717 * In both cases, x2 points to the CPU context we're saving/restoring from/to.
718 */
719 ENTRY(__kvm_vcpu_run)
720 kern_hyp_va x0
721 msr tpidr_el2, x0 // Save the vcpu register
723 // Host context
724 ldr x2, [x0, #VCPU_HOST_CONTEXT]
725 kern_hyp_va x2
727 save_host_regs
728 bl __save_sysregs
730 compute_debug_state 1f
731 add x3, x0, #VCPU_HOST_DEBUG_STATE
732 bl __save_debug
733 1:
734 activate_traps
735 activate_vm
737 restore_vgic_state
738 restore_timer_state
740 // Guest context
741 add x2, x0, #VCPU_CONTEXT
743 // We must restore the 32-bit state before the sysregs, thanks
744 // to Cortex-A57 erratum #852523.
745 restore_guest_32bit_state
746 bl __restore_sysregs
748 skip_debug_state x3, 1f
749 ldr x3, [x0, #VCPU_DEBUG_PTR]
750 kern_hyp_va x3
751 bl __restore_debug
752 1:
753 restore_guest_regs
755 // That's it, no more messing around.
756 eret
758 __kvm_vcpu_return:
759 // Assume x0 is the vcpu pointer, x1 the return code
760 // Guest's x0-x3 are on the stack
762 // Guest context
763 add x2, x0, #VCPU_CONTEXT
765 save_guest_regs
766 bl __save_fpsimd
767 bl __save_sysregs
769 skip_debug_state x3, 1f
770 ldr x3, [x0, #VCPU_DEBUG_PTR]
771 kern_hyp_va x3
772 bl __save_debug
773 1:
774 save_guest_32bit_state
776 save_timer_state
777 save_vgic_state
779 deactivate_traps
780 deactivate_vm
782 // Host context
783 ldr x2, [x0, #VCPU_HOST_CONTEXT]
784 kern_hyp_va x2
786 bl __restore_sysregs
787 bl __restore_fpsimd
788 /* Clear FPSIMD and Trace trapping */
789 msr cptr_el2, xzr
791 skip_debug_state x3, 1f
792 // Clear the dirty flag for the next run, as all the state has
793 // already been saved. Note that we nuke the whole 64bit word.
794 // If we ever add more flags, we'll have to be more careful...
795 str xzr, [x0, #VCPU_DEBUG_FLAGS]
796 add x3, x0, #VCPU_HOST_DEBUG_STATE
797 bl __restore_debug
798 1:
799 restore_host_regs
801 mov x0, x1
802 ret
803 END(__kvm_vcpu_run)
805 // void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
806 ENTRY(__kvm_tlb_flush_vmid_ipa)
807 dsb ishst
809 kern_hyp_va x0
810 ldr x2, [x0, #KVM_VTTBR]
811 msr vttbr_el2, x2
812 isb
814 /*
815 * We could do so much better if we had the VA as well.
816 * Instead, we invalidate Stage-2 for this IPA, and the
817 * whole of Stage-1. Weep...
818 */
819 lsr x1, x1, #12
820 tlbi ipas2e1is, x1
821 /*
822 * We have to ensure completion of the invalidation at Stage-2,
823 * since a table walk on another CPU could refill a TLB with a
824 * complete (S1 + S2) walk based on the old Stage-2 mapping if
825 * the Stage-1 invalidation happened first.
826 */
827 dsb ish
828 tlbi vmalle1is
829 dsb ish
830 isb
832 msr vttbr_el2, xzr
833 ret
834 ENDPROC(__kvm_tlb_flush_vmid_ipa)
836 /**
837 * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
838 * @struct kvm *kvm - pointer to kvm structure
839 *
840 * Invalidates all Stage 1 and 2 TLB entries for current VMID.
841 */
842 ENTRY(__kvm_tlb_flush_vmid)
843 dsb ishst
845 kern_hyp_va x0
846 ldr x2, [x0, #KVM_VTTBR]
847 msr vttbr_el2, x2
848 isb
850 tlbi vmalls12e1is
851 dsb ish
852 isb
854 msr vttbr_el2, xzr
855 ret
856 ENDPROC(__kvm_tlb_flush_vmid)
858 ENTRY(__kvm_flush_vm_context)
859 dsb ishst
860 tlbi alle1is
861 ic ialluis
862 dsb ish
863 ret
864 ENDPROC(__kvm_flush_vm_context)
866 __kvm_hyp_panic:
867 // Stash PAR_EL1 before corrupting it in __restore_sysregs
868 mrs x0, par_el1
869 push x0, xzr
871 // Guess the context by looking at VTTBR:
872 // If zero, then we're already a host.
873 // Otherwise restore a minimal host context before panicing.
874 mrs x0, vttbr_el2
875 cbz x0, 1f
877 mrs x0, tpidr_el2
879 deactivate_traps
880 deactivate_vm
882 ldr x2, [x0, #VCPU_HOST_CONTEXT]
883 kern_hyp_va x2
885 bl __restore_sysregs
887 /*
888 * Make sure we have a valid host stack, and don't leave junk in the
889 * frame pointer that will give us a misleading host stack unwinding.
890 */
891 ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
892 msr sp_el1, x22
893 mov x29, xzr
895 1: adr x0, __hyp_panic_str
896 adr x1, 2f
897 ldp x2, x3, [x1]
898 sub x0, x0, x2
899 add x0, x0, x3
900 mrs x1, spsr_el2
901 mrs x2, elr_el2
902 mrs x3, esr_el2
903 mrs x4, far_el2
904 mrs x5, hpfar_el2
905 pop x6, xzr // active context PAR_EL1
906 mrs x7, tpidr_el2
908 mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
909 PSR_MODE_EL1h)
910 msr spsr_el2, lr
911 ldr lr, =panic
912 msr elr_el2, lr
913 eret
915 .align 3
916 2: .quad HYP_PAGE_OFFSET
917 .quad PAGE_OFFSET
918 ENDPROC(__kvm_hyp_panic)
920 __hyp_panic_str:
921 .ascii "HYP panic:\nPS:%08x PC:%016x ESR:%08x\nFAR:%016x HPFAR:%016x PAR:%016x\nVCPU:%p\n\0"
923 .align 2
925 /*
926 * u64 kvm_call_hyp(void *hypfn, ...);
927 *
928 * This is not really a variadic function in the classic C-way and care must
929 * be taken when calling this to ensure parameters are passed in registers
930 * only, since the stack will change between the caller and the callee.
931 *
932 * Call the function with the first argument containing a pointer to the
933 * function you wish to call in Hyp mode, and subsequent arguments will be
934 * passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the
935 * function pointer can be passed). The function being called must be mapped
936 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
937 * passed in r0 and r1.
938 *
939 * A function pointer with a value of 0 has a special meaning, and is
940 * used to implement __hyp_get_vectors in the same way as in
941 * arch/arm64/kernel/hyp_stub.S.
942 */
943 ENTRY(kvm_call_hyp)
944 hvc #0
945 ret
946 ENDPROC(kvm_call_hyp)
948 .macro invalid_vector label, target
949 .align 2
950 \label:
951 b \target
952 ENDPROC(\label)
953 .endm
955 /* None of these should ever happen */
956 invalid_vector el2t_sync_invalid, __kvm_hyp_panic
957 invalid_vector el2t_irq_invalid, __kvm_hyp_panic
958 invalid_vector el2t_fiq_invalid, __kvm_hyp_panic
959 invalid_vector el2t_error_invalid, __kvm_hyp_panic
960 invalid_vector el2h_sync_invalid, __kvm_hyp_panic
961 invalid_vector el2h_irq_invalid, __kvm_hyp_panic
962 invalid_vector el2h_fiq_invalid, __kvm_hyp_panic
963 invalid_vector el2h_error_invalid, __kvm_hyp_panic
964 invalid_vector el1_sync_invalid, __kvm_hyp_panic
965 invalid_vector el1_irq_invalid, __kvm_hyp_panic
966 invalid_vector el1_fiq_invalid, __kvm_hyp_panic
967 invalid_vector el1_error_invalid, __kvm_hyp_panic
969 el1_sync: // Guest trapped into EL2
970 push x0, x1
971 push x2, x3
973 mrs x1, esr_el2
974 lsr x2, x1, #ESR_ELx_EC_SHIFT
976 cmp x2, #ESR_ELx_EC_HVC64
977 b.ne el1_trap
979 mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
980 cbnz x3, el1_trap // called HVC
982 /* Here, we're pretty sure the host called HVC. */
983 pop x2, x3
984 pop x0, x1
986 /* Check for __hyp_get_vectors */
987 cbnz x0, 1f
988 mrs x0, vbar_el2
989 b 2f
991 1: push lr, xzr
993 /*
994 * Compute the function address in EL2, and shuffle the parameters.
995 */
996 kern_hyp_va x0
997 mov lr, x0
998 mov x0, x1
999 mov x1, x2
1000 mov x2, x3
1001 blr lr
1003 pop lr, xzr
1004 2: eret
1006 el1_trap:
1007 /*
1008 * x1: ESR
1009 * x2: ESR_EC
1010 */
1012 /* Guest accessed VFP/SIMD registers, save host, restore Guest */
1013 cmp x2, #ESR_ELx_EC_FP_ASIMD
1014 b.eq switch_to_guest_fpsimd
1016 cmp x2, #ESR_ELx_EC_DABT_LOW
1017 mov x0, #ESR_ELx_EC_IABT_LOW
1018 ccmp x2, x0, #4, ne
1019 b.ne 1f // Not an abort we care about
1021 /* This is an abort. Check for permission fault */
1022 alternative_if_not ARM64_WORKAROUND_834220
1023 and x2, x1, #ESR_ELx_FSC_TYPE
1024 cmp x2, #FSC_PERM
1025 b.ne 1f // Not a permission fault
1026 alternative_else
1027 nop // Use the permission fault path to
1028 nop // check for a valid S1 translation,
1029 nop // regardless of the ESR value.
1030 alternative_endif
1032 /*
1033 * Check for Stage-1 page table walk, which is guaranteed
1034 * to give a valid HPFAR_EL2.
1035 */
1036 tbnz x1, #7, 1f // S1PTW is set
1038 /* Preserve PAR_EL1 */
1039 mrs x3, par_el1
1040 push x3, xzr
1042 /*
1043 * Permission fault, HPFAR_EL2 is invalid.
1044 * Resolve the IPA the hard way using the guest VA.
1045 * Stage-1 translation already validated the memory access rights.
1046 * As such, we can use the EL1 translation regime, and don't have
1047 * to distinguish between EL0 and EL1 access.
1048 */
1049 mrs x2, far_el2
1050 at s1e1r, x2
1051 isb
1053 /* Read result */
1054 mrs x3, par_el1
1055 pop x0, xzr // Restore PAR_EL1 from the stack
1056 msr par_el1, x0
1057 tbnz x3, #0, 3f // Bail out if we failed the translation
1058 ubfx x3, x3, #12, #36 // Extract IPA
1059 lsl x3, x3, #4 // and present it like HPFAR
1060 b 2f
1062 1: mrs x3, hpfar_el2
1063 mrs x2, far_el2
1065 2: mrs x0, tpidr_el2
1066 str w1, [x0, #VCPU_ESR_EL2]
1067 str x2, [x0, #VCPU_FAR_EL2]
1068 str x3, [x0, #VCPU_HPFAR_EL2]
1070 mov x1, #ARM_EXCEPTION_TRAP
1071 b __kvm_vcpu_return
1073 /*
1074 * Translation failed. Just return to the guest and
1075 * let it fault again. Another CPU is probably playing
1076 * behind our back.
1077 */
1078 3: pop x2, x3
1079 pop x0, x1
1081 eret
1083 el1_irq:
1084 push x0, x1
1085 push x2, x3
1086 mrs x0, tpidr_el2
1087 mov x1, #ARM_EXCEPTION_IRQ
1088 b __kvm_vcpu_return
1090 .ltorg
1092 .align 11
1094 ENTRY(__kvm_hyp_vector)
1095 ventry el2t_sync_invalid // Synchronous EL2t
1096 ventry el2t_irq_invalid // IRQ EL2t
1097 ventry el2t_fiq_invalid // FIQ EL2t
1098 ventry el2t_error_invalid // Error EL2t
1100 ventry el2h_sync_invalid // Synchronous EL2h
1101 ventry el2h_irq_invalid // IRQ EL2h
1102 ventry el2h_fiq_invalid // FIQ EL2h
1103 ventry el2h_error_invalid // Error EL2h
1105 ventry el1_sync // Synchronous 64-bit EL1
1106 ventry el1_irq // IRQ 64-bit EL1
1107 ventry el1_fiq_invalid // FIQ 64-bit EL1
1108 ventry el1_error_invalid // Error 64-bit EL1
1110 ventry el1_sync // Synchronous 32-bit EL1
1111 ventry el1_irq // IRQ 32-bit EL1
1112 ventry el1_fiq_invalid // FIQ 32-bit EL1
1113 ventry el1_error_invalid // Error 32-bit EL1
1114 ENDPROC(__kvm_hyp_vector)
1117 ENTRY(__kvm_get_mdcr_el2)
1118 mrs x0, mdcr_el2
1119 ret
1120 ENDPROC(__kvm_get_mdcr_el2)
1122 .popsection