| blob | 37c89ea2c572ed858c0425344b45e0897a89b74f |
1 /*
2 * Copyright (C) 2012,2013 - ARM Ltd
3 * Author: Marc Zyngier <marc.zyngier@arm.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 */
18 #include <linux/linkage.h>
20 #include <asm/alternative.h>
21 #include <asm/asm-offsets.h>
22 #include <asm/assembler.h>
23 #include <asm/cpufeature.h>
24 #include <asm/debug-monitors.h>
25 #include <asm/esr.h>
26 #include <asm/fpsimdmacros.h>
27 #include <asm/kvm.h>
28 #include <asm/kvm_arm.h>
29 #include <asm/kvm_asm.h>
30 #include <asm/kvm_mmu.h>
31 #include <asm/memory.h>
33 #define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
34 #define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
35 #define CPU_SPSR_OFFSET(x) CPU_GP_REG_OFFSET(CPU_SPSR + 8*x)
36 #define CPU_SYSREG_OFFSET(x) (CPU_SYSREGS + 8*x)
38 .text
39 .pushsection .hyp.text, "ax"
40 .align PAGE_SHIFT
42 .macro save_common_regs
43 // x2: base address for cpu context
44 // x3: tmp register
46 add x3, x2, #CPU_XREG_OFFSET(19)
47 stp x19, x20, [x3]
48 stp x21, x22, [x3, #16]
49 stp x23, x24, [x3, #32]
50 stp x25, x26, [x3, #48]
51 stp x27, x28, [x3, #64]
52 stp x29, lr, [x3, #80]
54 mrs x19, sp_el0
55 mrs x20, elr_el2 // pc before entering el2
56 mrs x21, spsr_el2 // pstate before entering el2
58 stp x19, x20, [x3, #96]
59 str x21, [x3, #112]
61 mrs x22, sp_el1
62 mrs x23, elr_el1
63 mrs x24, spsr_el1
65 str x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
66 str x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
67 str x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
68 .endm
70 .macro restore_common_regs
71 // x2: base address for cpu context
72 // x3: tmp register
74 ldr x22, [x2, #CPU_GP_REG_OFFSET(CPU_SP_EL1)]
75 ldr x23, [x2, #CPU_GP_REG_OFFSET(CPU_ELR_EL1)]
76 ldr x24, [x2, #CPU_SPSR_OFFSET(KVM_SPSR_EL1)]
78 msr sp_el1, x22
79 msr elr_el1, x23
80 msr spsr_el1, x24
82 add x3, x2, #CPU_XREG_OFFSET(31) // SP_EL0
83 ldp x19, x20, [x3]
84 ldr x21, [x3, #16]
86 msr sp_el0, x19
87 msr elr_el2, x20 // pc on return from el2
88 msr spsr_el2, x21 // pstate on return from el2
90 add x3, x2, #CPU_XREG_OFFSET(19)
91 ldp x19, x20, [x3]
92 ldp x21, x22, [x3, #16]
93 ldp x23, x24, [x3, #32]
94 ldp x25, x26, [x3, #48]
95 ldp x27, x28, [x3, #64]
96 ldp x29, lr, [x3, #80]
97 .endm
99 .macro save_host_regs
100 save_common_regs
101 .endm
103 .macro restore_host_regs
104 restore_common_regs
105 .endm
107 .macro save_fpsimd
108 // x2: cpu context address
109 // x3, x4: tmp regs
110 add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
111 fpsimd_save x3, 4
112 .endm
114 .macro restore_fpsimd
115 // x2: cpu context address
116 // x3, x4: tmp regs
117 add x3, x2, #CPU_GP_REG_OFFSET(CPU_FP_REGS)
118 fpsimd_restore x3, 4
119 .endm
121 .macro save_guest_regs
122 // x0 is the vcpu address
123 // x1 is the return code, do not corrupt!
124 // x2 is the cpu context
125 // x3 is a tmp register
126 // Guest's x0-x3 are on the stack
128 // Compute base to save registers
129 add x3, x2, #CPU_XREG_OFFSET(4)
130 stp x4, x5, [x3]
131 stp x6, x7, [x3, #16]
132 stp x8, x9, [x3, #32]
133 stp x10, x11, [x3, #48]
134 stp x12, x13, [x3, #64]
135 stp x14, x15, [x3, #80]
136 stp x16, x17, [x3, #96]
137 str x18, [x3, #112]
139 pop x6, x7 // x2, x3
140 pop x4, x5 // x0, x1
142 add x3, x2, #CPU_XREG_OFFSET(0)
143 stp x4, x5, [x3]
144 stp x6, x7, [x3, #16]
146 save_common_regs
147 .endm
149 .macro restore_guest_regs
150 // x0 is the vcpu address.
151 // x2 is the cpu context
152 // x3 is a tmp register
154 // Prepare x0-x3 for later restore
155 add x3, x2, #CPU_XREG_OFFSET(0)
156 ldp x4, x5, [x3]
157 ldp x6, x7, [x3, #16]
158 push x4, x5 // Push x0-x3 on the stack
159 push x6, x7
161 // x4-x18
162 ldp x4, x5, [x3, #32]
163 ldp x6, x7, [x3, #48]
164 ldp x8, x9, [x3, #64]
165 ldp x10, x11, [x3, #80]
166 ldp x12, x13, [x3, #96]
167 ldp x14, x15, [x3, #112]
168 ldp x16, x17, [x3, #128]
169 ldr x18, [x3, #144]
171 // x19-x29, lr, sp*, elr*, spsr*
172 restore_common_regs
174 // Last bits of the 64bit state
175 pop x2, x3
176 pop x0, x1
178 // Do not touch any register after this!
179 .endm
181 /*
182 * Macros to perform system register save/restore.
183 *
184 * Ordering here is absolutely critical, and must be kept consistent
185 * in {save,restore}_sysregs, {save,restore}_guest_32bit_state,
186 * and in kvm_asm.h.
187 *
188 * In other words, don't touch any of these unless you know what
189 * you are doing.
190 */
191 .macro save_sysregs
192 // x2: base address for cpu context
193 // x3: tmp register
195 add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
197 mrs x4, vmpidr_el2
198 mrs x5, csselr_el1
199 mrs x6, sctlr_el1
200 mrs x7, actlr_el1
201 mrs x8, cpacr_el1
202 mrs x9, ttbr0_el1
203 mrs x10, ttbr1_el1
204 mrs x11, tcr_el1
205 mrs x12, esr_el1
206 mrs x13, afsr0_el1
207 mrs x14, afsr1_el1
208 mrs x15, far_el1
209 mrs x16, mair_el1
210 mrs x17, vbar_el1
211 mrs x18, contextidr_el1
212 mrs x19, tpidr_el0
213 mrs x20, tpidrro_el0
214 mrs x21, tpidr_el1
215 mrs x22, amair_el1
216 mrs x23, cntkctl_el1
217 mrs x24, par_el1
218 mrs x25, mdscr_el1
220 stp x4, x5, [x3]
221 stp x6, x7, [x3, #16]
222 stp x8, x9, [x3, #32]
223 stp x10, x11, [x3, #48]
224 stp x12, x13, [x3, #64]
225 stp x14, x15, [x3, #80]
226 stp x16, x17, [x3, #96]
227 stp x18, x19, [x3, #112]
228 stp x20, x21, [x3, #128]
229 stp x22, x23, [x3, #144]
230 stp x24, x25, [x3, #160]
231 .endm
233 .macro save_debug type
234 // x4: pointer to register set
235 // x5: number of registers to skip
236 // x6..x22 trashed
238 adr x22, 1f
239 add x22, x22, x5, lsl #2
240 br x22
241 1:
242 mrs x21, \type\()15_el1
243 mrs x20, \type\()14_el1
244 mrs x19, \type\()13_el1
245 mrs x18, \type\()12_el1
246 mrs x17, \type\()11_el1
247 mrs x16, \type\()10_el1
248 mrs x15, \type\()9_el1
249 mrs x14, \type\()8_el1
250 mrs x13, \type\()7_el1
251 mrs x12, \type\()6_el1
252 mrs x11, \type\()5_el1
253 mrs x10, \type\()4_el1
254 mrs x9, \type\()3_el1
255 mrs x8, \type\()2_el1
256 mrs x7, \type\()1_el1
257 mrs x6, \type\()0_el1
259 adr x22, 1f
260 add x22, x22, x5, lsl #2
261 br x22
262 1:
263 str x21, [x4, #(15 * 8)]
264 str x20, [x4, #(14 * 8)]
265 str x19, [x4, #(13 * 8)]
266 str x18, [x4, #(12 * 8)]
267 str x17, [x4, #(11 * 8)]
268 str x16, [x4, #(10 * 8)]
269 str x15, [x4, #(9 * 8)]
270 str x14, [x4, #(8 * 8)]
271 str x13, [x4, #(7 * 8)]
272 str x12, [x4, #(6 * 8)]
273 str x11, [x4, #(5 * 8)]
274 str x10, [x4, #(4 * 8)]
275 str x9, [x4, #(3 * 8)]
276 str x8, [x4, #(2 * 8)]
277 str x7, [x4, #(1 * 8)]
278 str x6, [x4, #(0 * 8)]
279 .endm
281 .macro restore_sysregs
282 // x2: base address for cpu context
283 // x3: tmp register
285 add x3, x2, #CPU_SYSREG_OFFSET(MPIDR_EL1)
287 ldp x4, x5, [x3]
288 ldp x6, x7, [x3, #16]
289 ldp x8, x9, [x3, #32]
290 ldp x10, x11, [x3, #48]
291 ldp x12, x13, [x3, #64]
292 ldp x14, x15, [x3, #80]
293 ldp x16, x17, [x3, #96]
294 ldp x18, x19, [x3, #112]
295 ldp x20, x21, [x3, #128]
296 ldp x22, x23, [x3, #144]
297 ldp x24, x25, [x3, #160]
299 msr vmpidr_el2, x4
300 msr csselr_el1, x5
301 msr sctlr_el1, x6
302 msr actlr_el1, x7
303 msr cpacr_el1, x8
304 msr ttbr0_el1, x9
305 msr ttbr1_el1, x10
306 msr tcr_el1, x11
307 msr esr_el1, x12
308 msr afsr0_el1, x13
309 msr afsr1_el1, x14
310 msr far_el1, x15
311 msr mair_el1, x16
312 msr vbar_el1, x17
313 msr contextidr_el1, x18
314 msr tpidr_el0, x19
315 msr tpidrro_el0, x20
316 msr tpidr_el1, x21
317 msr amair_el1, x22
318 msr cntkctl_el1, x23
319 msr par_el1, x24
320 msr mdscr_el1, x25
321 .endm
323 .macro restore_debug type
324 // x4: pointer to register set
325 // x5: number of registers to skip
326 // x6..x22 trashed
328 adr x22, 1f
329 add x22, x22, x5, lsl #2
330 br x22
331 1:
332 ldr x21, [x4, #(15 * 8)]
333 ldr x20, [x4, #(14 * 8)]
334 ldr x19, [x4, #(13 * 8)]
335 ldr x18, [x4, #(12 * 8)]
336 ldr x17, [x4, #(11 * 8)]
337 ldr x16, [x4, #(10 * 8)]
338 ldr x15, [x4, #(9 * 8)]
339 ldr x14, [x4, #(8 * 8)]
340 ldr x13, [x4, #(7 * 8)]
341 ldr x12, [x4, #(6 * 8)]
342 ldr x11, [x4, #(5 * 8)]
343 ldr x10, [x4, #(4 * 8)]
344 ldr x9, [x4, #(3 * 8)]
345 ldr x8, [x4, #(2 * 8)]
346 ldr x7, [x4, #(1 * 8)]
347 ldr x6, [x4, #(0 * 8)]
349 adr x22, 1f
350 add x22, x22, x5, lsl #2
351 br x22
352 1:
353 msr \type\()15_el1, x21
354 msr \type\()14_el1, x20
355 msr \type\()13_el1, x19
356 msr \type\()12_el1, x18
357 msr \type\()11_el1, x17
358 msr \type\()10_el1, x16
359 msr \type\()9_el1, x15
360 msr \type\()8_el1, x14
361 msr \type\()7_el1, x13
362 msr \type\()6_el1, x12
363 msr \type\()5_el1, x11
364 msr \type\()4_el1, x10
365 msr \type\()3_el1, x9
366 msr \type\()2_el1, x8
367 msr \type\()1_el1, x7
368 msr \type\()0_el1, x6
369 .endm
371 .macro skip_32bit_state tmp, target
372 // Skip 32bit state if not needed
373 mrs \tmp, hcr_el2
374 tbnz \tmp, #HCR_RW_SHIFT, \target
375 .endm
377 .macro skip_tee_state tmp, target
378 // Skip ThumbEE state if not needed
379 mrs \tmp, id_pfr0_el1
380 tbz \tmp, #12, \target
381 .endm
383 .macro skip_debug_state tmp, target
384 ldr \tmp, [x0, #VCPU_DEBUG_FLAGS]
385 tbz \tmp, #KVM_ARM64_DEBUG_DIRTY_SHIFT, \target
386 .endm
388 /*
389 * Branch to target if CPTR_EL2.TFP bit is set (VFP/SIMD trapping enabled)
390 */
391 .macro skip_fpsimd_state tmp, target
392 mrs \tmp, cptr_el2
393 tbnz \tmp, #CPTR_EL2_TFP_SHIFT, \target
394 .endm
396 .macro compute_debug_state target
397 // Compute debug state: If any of KDE, MDE or KVM_ARM64_DEBUG_DIRTY
398 // is set, we do a full save/restore cycle and disable trapping.
399 add x25, x0, #VCPU_CONTEXT
401 // Check the state of MDSCR_EL1
402 ldr x25, [x25, #CPU_SYSREG_OFFSET(MDSCR_EL1)]
403 and x26, x25, #DBG_MDSCR_KDE
404 and x25, x25, #DBG_MDSCR_MDE
405 adds xzr, x25, x26
406 b.eq 9998f // Nothing to see there
408 // If any interesting bits was set, we must set the flag
409 mov x26, #KVM_ARM64_DEBUG_DIRTY
410 str x26, [x0, #VCPU_DEBUG_FLAGS]
411 b 9999f // Don't skip restore
413 9998:
414 // Otherwise load the flags from memory in case we recently
415 // trapped
416 skip_debug_state x25, \target
417 9999:
418 .endm
420 .macro save_guest_32bit_state
421 skip_32bit_state x3, 1f
423 add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
424 mrs x4, spsr_abt
425 mrs x5, spsr_und
426 mrs x6, spsr_irq
427 mrs x7, spsr_fiq
428 stp x4, x5, [x3]
429 stp x6, x7, [x3, #16]
431 add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
432 mrs x4, dacr32_el2
433 mrs x5, ifsr32_el2
434 stp x4, x5, [x3]
436 skip_fpsimd_state x8, 3f
437 mrs x6, fpexc32_el2
438 str x6, [x3, #16]
439 3:
440 skip_debug_state x8, 2f
441 mrs x7, dbgvcr32_el2
442 str x7, [x3, #24]
443 2:
444 skip_tee_state x8, 1f
446 add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1)
447 mrs x4, teecr32_el1
448 mrs x5, teehbr32_el1
449 stp x4, x5, [x3]
450 1:
451 .endm
453 .macro restore_guest_32bit_state
454 skip_32bit_state x3, 1f
456 add x3, x2, #CPU_SPSR_OFFSET(KVM_SPSR_ABT)
457 ldp x4, x5, [x3]
458 ldp x6, x7, [x3, #16]
459 msr spsr_abt, x4
460 msr spsr_und, x5
461 msr spsr_irq, x6
462 msr spsr_fiq, x7
464 add x3, x2, #CPU_SYSREG_OFFSET(DACR32_EL2)
465 ldp x4, x5, [x3]
466 msr dacr32_el2, x4
467 msr ifsr32_el2, x5
469 skip_debug_state x8, 2f
470 ldr x7, [x3, #24]
471 msr dbgvcr32_el2, x7
472 2:
473 skip_tee_state x8, 1f
475 add x3, x2, #CPU_SYSREG_OFFSET(TEECR32_EL1)
476 ldp x4, x5, [x3]
477 msr teecr32_el1, x4
478 msr teehbr32_el1, x5
479 1:
480 .endm
482 .macro activate_traps
483 ldr x2, [x0, #VCPU_HCR_EL2]
485 /*
486 * We are about to set CPTR_EL2.TFP to trap all floating point
487 * register accesses to EL2, however, the ARM ARM clearly states that
488 * traps are only taken to EL2 if the operation would not otherwise
489 * trap to EL1. Therefore, always make sure that for 32-bit guests,
490 * we set FPEXC.EN to prevent traps to EL1, when setting the TFP bit.
491 */
492 tbnz x2, #HCR_RW_SHIFT, 99f // open code skip_32bit_state
493 mov x3, #(1 << 30)
494 msr fpexc32_el2, x3
495 isb
496 99:
497 msr hcr_el2, x2
498 mov x2, #CPTR_EL2_TTA
499 orr x2, x2, #CPTR_EL2_TFP
500 msr cptr_el2, x2
502 mov x2, #(1 << 15) // Trap CP15 Cr=15
503 msr hstr_el2, x2
505 // Monitor Debug Config - see kvm_arm_setup_debug()
506 ldr x2, [x0, #VCPU_MDCR_EL2]
507 msr mdcr_el2, x2
508 .endm
510 .macro deactivate_traps
511 mov x2, #HCR_RW
512 msr hcr_el2, x2
513 msr hstr_el2, xzr
515 mrs x2, mdcr_el2
516 and x2, x2, #MDCR_EL2_HPMN_MASK
517 msr mdcr_el2, x2
518 .endm
520 .macro activate_vm
521 ldr x1, [x0, #VCPU_KVM]
522 kern_hyp_va x1
523 ldr x2, [x1, #KVM_VTTBR]
524 msr vttbr_el2, x2
525 .endm
527 .macro deactivate_vm
528 msr vttbr_el2, xzr
529 .endm
531 /*
532 * Call into the vgic backend for state saving
533 */
534 .macro save_vgic_state
535 alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
536 bl __save_vgic_v2_state
537 alternative_else
538 bl __save_vgic_v3_state
539 alternative_endif
540 mrs x24, hcr_el2
541 mov x25, #HCR_INT_OVERRIDE
542 neg x25, x25
543 and x24, x24, x25
544 msr hcr_el2, x24
545 .endm
547 /*
548 * Call into the vgic backend for state restoring
549 */
550 .macro restore_vgic_state
551 mrs x24, hcr_el2
552 ldr x25, [x0, #VCPU_IRQ_LINES]
553 orr x24, x24, #HCR_INT_OVERRIDE
554 orr x24, x24, x25
555 msr hcr_el2, x24
556 alternative_if_not ARM64_HAS_SYSREG_GIC_CPUIF
557 bl __restore_vgic_v2_state
558 alternative_else
559 bl __restore_vgic_v3_state
560 alternative_endif
561 .endm
563 .macro save_timer_state
564 // x0: vcpu pointer
565 ldr x2, [x0, #VCPU_KVM]
566 kern_hyp_va x2
567 ldr w3, [x2, #KVM_TIMER_ENABLED]
568 cbz w3, 1f
570 mrs x3, cntv_ctl_el0
571 and x3, x3, #3
572 str w3, [x0, #VCPU_TIMER_CNTV_CTL]
573 bic x3, x3, #1 // Clear Enable
574 msr cntv_ctl_el0, x3
576 isb
578 mrs x3, cntv_cval_el0
579 str x3, [x0, #VCPU_TIMER_CNTV_CVAL]
581 1:
582 // Allow physical timer/counter access for the host
583 mrs x2, cnthctl_el2
584 orr x2, x2, #3
585 msr cnthctl_el2, x2
587 // Clear cntvoff for the host
588 msr cntvoff_el2, xzr
589 .endm
591 .macro restore_timer_state
592 // x0: vcpu pointer
593 // Disallow physical timer access for the guest
594 // Physical counter access is allowed
595 mrs x2, cnthctl_el2
596 orr x2, x2, #1
597 bic x2, x2, #2
598 msr cnthctl_el2, x2
600 ldr x2, [x0, #VCPU_KVM]
601 kern_hyp_va x2
602 ldr w3, [x2, #KVM_TIMER_ENABLED]
603 cbz w3, 1f
605 ldr x3, [x2, #KVM_TIMER_CNTVOFF]
606 msr cntvoff_el2, x3
607 ldr x2, [x0, #VCPU_TIMER_CNTV_CVAL]
608 msr cntv_cval_el0, x2
609 isb
611 ldr w2, [x0, #VCPU_TIMER_CNTV_CTL]
612 and x2, x2, #3
613 msr cntv_ctl_el0, x2
614 1:
615 .endm
617 __save_sysregs:
618 save_sysregs
619 ret
621 __restore_sysregs:
622 restore_sysregs
623 ret
625 /* Save debug state */
626 __save_debug:
627 // x2: ptr to CPU context
628 // x3: ptr to debug reg struct
629 // x4/x5/x6-22/x24-26: trashed
631 mrs x26, id_aa64dfr0_el1
632 ubfx x24, x26, #12, #4 // Extract BRPs
633 ubfx x25, x26, #20, #4 // Extract WRPs
634 mov w26, #15
635 sub w24, w26, w24 // How many BPs to skip
636 sub w25, w26, w25 // How many WPs to skip
638 mov x5, x24
639 add x4, x3, #DEBUG_BCR
640 save_debug dbgbcr
641 add x4, x3, #DEBUG_BVR
642 save_debug dbgbvr
644 mov x5, x25
645 add x4, x3, #DEBUG_WCR
646 save_debug dbgwcr
647 add x4, x3, #DEBUG_WVR
648 save_debug dbgwvr
650 mrs x21, mdccint_el1
651 str x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
652 ret
654 /* Restore debug state */
655 __restore_debug:
656 // x2: ptr to CPU context
657 // x3: ptr to debug reg struct
658 // x4/x5/x6-22/x24-26: trashed
660 mrs x26, id_aa64dfr0_el1
661 ubfx x24, x26, #12, #4 // Extract BRPs
662 ubfx x25, x26, #20, #4 // Extract WRPs
663 mov w26, #15
664 sub w24, w26, w24 // How many BPs to skip
665 sub w25, w26, w25 // How many WPs to skip
667 mov x5, x24
668 add x4, x3, #DEBUG_BCR
669 restore_debug dbgbcr
670 add x4, x3, #DEBUG_BVR
671 restore_debug dbgbvr
673 mov x5, x25
674 add x4, x3, #DEBUG_WCR
675 restore_debug dbgwcr
676 add x4, x3, #DEBUG_WVR
677 restore_debug dbgwvr
679 ldr x21, [x2, #CPU_SYSREG_OFFSET(MDCCINT_EL1)]
680 msr mdccint_el1, x21
682 ret
684 __save_fpsimd:
685 skip_fpsimd_state x3, 1f
686 save_fpsimd
687 1: ret
689 __restore_fpsimd:
690 skip_fpsimd_state x3, 1f
691 restore_fpsimd
692 1: ret
694 switch_to_guest_fpsimd:
695 push x4, lr
697 mrs x2, cptr_el2
698 bic x2, x2, #CPTR_EL2_TFP
699 msr cptr_el2, x2
700 isb
702 mrs x0, tpidr_el2
704 ldr x2, [x0, #VCPU_HOST_CONTEXT]
705 kern_hyp_va x2
706 bl __save_fpsimd
708 add x2, x0, #VCPU_CONTEXT
709 bl __restore_fpsimd
711 skip_32bit_state x3, 1f
712 ldr x4, [x2, #CPU_SYSREG_OFFSET(FPEXC32_EL2)]
713 msr fpexc32_el2, x4
714 1:
715 pop x4, lr
716 pop x2, x3
717 pop x0, x1
719 eret
721 /*
722 * u64 __kvm_vcpu_run(struct kvm_vcpu *vcpu);
723 *
724 * This is the world switch. The first half of the function
725 * deals with entering the guest, and anything from __kvm_vcpu_return
726 * to the end of the function deals with reentering the host.
727 * On the enter path, only x0 (vcpu pointer) must be preserved until
728 * the last moment. On the exit path, x0 (vcpu pointer) and x1 (exception
729 * code) must both be preserved until the epilogue.
730 * In both cases, x2 points to the CPU context we're saving/restoring from/to.
731 */
732 ENTRY(__kvm_vcpu_run)
733 kern_hyp_va x0
734 msr tpidr_el2, x0 // Save the vcpu register
736 // Host context
737 ldr x2, [x0, #VCPU_HOST_CONTEXT]
738 kern_hyp_va x2
740 save_host_regs
741 bl __save_sysregs
743 compute_debug_state 1f
744 add x3, x0, #VCPU_HOST_DEBUG_STATE
745 bl __save_debug
746 1:
747 activate_traps
748 activate_vm
750 restore_vgic_state
751 restore_timer_state
753 // Guest context
754 add x2, x0, #VCPU_CONTEXT
756 bl __restore_sysregs
758 skip_debug_state x3, 1f
759 ldr x3, [x0, #VCPU_DEBUG_PTR]
760 kern_hyp_va x3
761 bl __restore_debug
762 1:
763 restore_guest_32bit_state
764 restore_guest_regs
766 // That's it, no more messing around.
767 eret
769 __kvm_vcpu_return:
770 // Assume x0 is the vcpu pointer, x1 the return code
771 // Guest's x0-x3 are on the stack
773 // Guest context
774 add x2, x0, #VCPU_CONTEXT
776 save_guest_regs
777 bl __save_fpsimd
778 bl __save_sysregs
780 skip_debug_state x3, 1f
781 ldr x3, [x0, #VCPU_DEBUG_PTR]
782 kern_hyp_va x3
783 bl __save_debug
784 1:
785 save_guest_32bit_state
787 save_timer_state
788 save_vgic_state
790 deactivate_traps
791 deactivate_vm
793 // Host context
794 ldr x2, [x0, #VCPU_HOST_CONTEXT]
795 kern_hyp_va x2
797 bl __restore_sysregs
798 bl __restore_fpsimd
799 /* Clear FPSIMD and Trace trapping */
800 msr cptr_el2, xzr
802 skip_debug_state x3, 1f
803 // Clear the dirty flag for the next run, as all the state has
804 // already been saved. Note that we nuke the whole 64bit word.
805 // If we ever add more flags, we'll have to be more careful...
806 str xzr, [x0, #VCPU_DEBUG_FLAGS]
807 add x3, x0, #VCPU_HOST_DEBUG_STATE
808 bl __restore_debug
809 1:
810 restore_host_regs
812 mov x0, x1
813 ret
814 END(__kvm_vcpu_run)
816 // void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
817 ENTRY(__kvm_tlb_flush_vmid_ipa)
818 dsb ishst
820 kern_hyp_va x0
821 ldr x2, [x0, #KVM_VTTBR]
822 msr vttbr_el2, x2
823 isb
825 /*
826 * We could do so much better if we had the VA as well.
827 * Instead, we invalidate Stage-2 for this IPA, and the
828 * whole of Stage-1. Weep...
829 */
830 lsr x1, x1, #12
831 tlbi ipas2e1is, x1
832 /*
833 * We have to ensure completion of the invalidation at Stage-2,
834 * since a table walk on another CPU could refill a TLB with a
835 * complete (S1 + S2) walk based on the old Stage-2 mapping if
836 * the Stage-1 invalidation happened first.
837 */
838 dsb ish
839 tlbi vmalle1is
840 dsb ish
841 isb
843 msr vttbr_el2, xzr
844 ret
845 ENDPROC(__kvm_tlb_flush_vmid_ipa)
847 /**
848 * void __kvm_tlb_flush_vmid(struct kvm *kvm) - Flush per-VMID TLBs
849 * @struct kvm *kvm - pointer to kvm structure
850 *
851 * Invalidates all Stage 1 and 2 TLB entries for current VMID.
852 */
853 ENTRY(__kvm_tlb_flush_vmid)
854 dsb ishst
856 kern_hyp_va x0
857 ldr x2, [x0, #KVM_VTTBR]
858 msr vttbr_el2, x2
859 isb
861 tlbi vmalls12e1is
862 dsb ish
863 isb
865 msr vttbr_el2, xzr
866 ret
867 ENDPROC(__kvm_tlb_flush_vmid)
869 ENTRY(__kvm_flush_vm_context)
870 dsb ishst
871 tlbi alle1is
872 ic ialluis
873 dsb ish
874 ret
875 ENDPROC(__kvm_flush_vm_context)
877 __kvm_hyp_panic:
878 // Guess the context by looking at VTTBR:
879 // If zero, then we're already a host.
880 // Otherwise restore a minimal host context before panicing.
881 mrs x0, vttbr_el2
882 cbz x0, 1f
884 mrs x0, tpidr_el2
886 deactivate_traps
887 deactivate_vm
889 ldr x2, [x0, #VCPU_HOST_CONTEXT]
890 kern_hyp_va x2
892 bl __restore_sysregs
894 1: adr x0, __hyp_panic_str
895 adr x1, 2f
896 ldp x2, x3, [x1]
897 sub x0, x0, x2
898 add x0, x0, x3
899 mrs x1, spsr_el2
900 mrs x2, elr_el2
901 mrs x3, esr_el2
902 mrs x4, far_el2
903 mrs x5, hpfar_el2
904 mrs x6, par_el1
905 mrs x7, tpidr_el2
907 mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\
908 PSR_MODE_EL1h)
909 msr spsr_el2, lr
910 ldr lr, =panic
911 msr elr_el2, lr
912 eret
914 .align 3
915 2: .quad HYP_PAGE_OFFSET
916 .quad PAGE_OFFSET
917 ENDPROC(__kvm_hyp_panic)
919 __hyp_panic_str:
920 .ascii "HYP panic:\nPS:%08x PC:%p ESR:%p\nFAR:%p HPFAR:%p PAR:%p\nVCPU:%p\n\0"
922 .align 2
924 /*
925 * u64 kvm_call_hyp(void *hypfn, ...);
926 *
927 * This is not really a variadic function in the classic C-way and care must
928 * be taken when calling this to ensure parameters are passed in registers
929 * only, since the stack will change between the caller and the callee.
930 *
931 * Call the function with the first argument containing a pointer to the
932 * function you wish to call in Hyp mode, and subsequent arguments will be
933 * passed as x0, x1, and x2 (a maximum of 3 arguments in addition to the
934 * function pointer can be passed). The function being called must be mapped
935 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
936 * passed in r0 and r1.
937 *
938 * A function pointer with a value of 0 has a special meaning, and is
939 * used to implement __hyp_get_vectors in the same way as in
940 * arch/arm64/kernel/hyp_stub.S.
941 */
942 ENTRY(kvm_call_hyp)
943 hvc #0
944 ret
945 ENDPROC(kvm_call_hyp)
947 .macro invalid_vector label, target
948 .align 2
949 \label:
950 b \target
951 ENDPROC(\label)
952 .endm
954 /* None of these should ever happen */
955 invalid_vector el2t_sync_invalid, __kvm_hyp_panic
956 invalid_vector el2t_irq_invalid, __kvm_hyp_panic
957 invalid_vector el2t_fiq_invalid, __kvm_hyp_panic
958 invalid_vector el2t_error_invalid, __kvm_hyp_panic
959 invalid_vector el2h_sync_invalid, __kvm_hyp_panic
960 invalid_vector el2h_irq_invalid, __kvm_hyp_panic
961 invalid_vector el2h_fiq_invalid, __kvm_hyp_panic
962 invalid_vector el2h_error_invalid, __kvm_hyp_panic
963 invalid_vector el1_sync_invalid, __kvm_hyp_panic
964 invalid_vector el1_irq_invalid, __kvm_hyp_panic
965 invalid_vector el1_fiq_invalid, __kvm_hyp_panic
966 invalid_vector el1_error_invalid, __kvm_hyp_panic
968 el1_sync: // Guest trapped into EL2
969 push x0, x1
970 push x2, x3
972 mrs x1, esr_el2
973 lsr x2, x1, #ESR_ELx_EC_SHIFT
975 cmp x2, #ESR_ELx_EC_HVC64
976 b.ne el1_trap
978 mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
979 cbnz x3, el1_trap // called HVC
981 /* Here, we're pretty sure the host called HVC. */
982 pop x2, x3
983 pop x0, x1
985 /* Check for __hyp_get_vectors */
986 cbnz x0, 1f
987 mrs x0, vbar_el2
988 b 2f
990 1: push lr, xzr
992 /*
993 * Compute the function address in EL2, and shuffle the parameters.
994 */
995 kern_hyp_va x0
996 mov lr, x0
997 mov x0, x1
998 mov x1, x2
999 mov x2, x3
1000 blr lr
1002 pop lr, xzr
1003 2: eret
1005 el1_trap:
1006 /*
1007 * x1: ESR
1008 * x2: ESR_EC
1009 */
1011 /* Guest accessed VFP/SIMD registers, save host, restore Guest */
1012 cmp x2, #ESR_ELx_EC_FP_ASIMD
1013 b.eq switch_to_guest_fpsimd
1015 cmp x2, #ESR_ELx_EC_DABT_LOW
1016 mov x0, #ESR_ELx_EC_IABT_LOW
1017 ccmp x2, x0, #4, ne
1018 b.ne 1f // Not an abort we care about
1020 /* This is an abort. Check for permission fault */
1021 and x2, x1, #ESR_ELx_FSC_TYPE
1022 cmp x2, #FSC_PERM
1023 b.ne 1f // Not a permission fault
1025 /*
1026 * Check for Stage-1 page table walk, which is guaranteed
1027 * to give a valid HPFAR_EL2.
1028 */
1029 tbnz x1, #7, 1f // S1PTW is set
1031 /* Preserve PAR_EL1 */
1032 mrs x3, par_el1
1033 push x3, xzr
1035 /*
1036 * Permission fault, HPFAR_EL2 is invalid.
1037 * Resolve the IPA the hard way using the guest VA.
1038 * Stage-1 translation already validated the memory access rights.
1039 * As such, we can use the EL1 translation regime, and don't have
1040 * to distinguish between EL0 and EL1 access.
1041 */
1042 mrs x2, far_el2
1043 at s1e1r, x2
1044 isb
1046 /* Read result */
1047 mrs x3, par_el1
1048 pop x0, xzr // Restore PAR_EL1 from the stack
1049 msr par_el1, x0
1050 tbnz x3, #0, 3f // Bail out if we failed the translation
1051 ubfx x3, x3, #12, #36 // Extract IPA
1052 lsl x3, x3, #4 // and present it like HPFAR
1053 b 2f
1055 1: mrs x3, hpfar_el2
1056 mrs x2, far_el2
1058 2: mrs x0, tpidr_el2
1059 str w1, [x0, #VCPU_ESR_EL2]
1060 str x2, [x0, #VCPU_FAR_EL2]
1061 str x3, [x0, #VCPU_HPFAR_EL2]
1063 mov x1, #ARM_EXCEPTION_TRAP
1064 b __kvm_vcpu_return
1066 /*
1067 * Translation failed. Just return to the guest and
1068 * let it fault again. Another CPU is probably playing
1069 * behind our back.
1070 */
1071 3: pop x2, x3
1072 pop x0, x1
1074 eret
1076 el1_irq:
1077 push x0, x1
1078 push x2, x3
1079 mrs x0, tpidr_el2
1080 mov x1, #ARM_EXCEPTION_IRQ
1081 b __kvm_vcpu_return
1083 .ltorg
1085 .align 11
1087 ENTRY(__kvm_hyp_vector)
1088 ventry el2t_sync_invalid // Synchronous EL2t
1089 ventry el2t_irq_invalid // IRQ EL2t
1090 ventry el2t_fiq_invalid // FIQ EL2t
1091 ventry el2t_error_invalid // Error EL2t
1093 ventry el2h_sync_invalid // Synchronous EL2h
1094 ventry el2h_irq_invalid // IRQ EL2h
1095 ventry el2h_fiq_invalid // FIQ EL2h
1096 ventry el2h_error_invalid // Error EL2h
1098 ventry el1_sync // Synchronous 64-bit EL1
1099 ventry el1_irq // IRQ 64-bit EL1
1100 ventry el1_fiq_invalid // FIQ 64-bit EL1
1101 ventry el1_error_invalid // Error 64-bit EL1
1103 ventry el1_sync // Synchronous 32-bit EL1
1104 ventry el1_irq // IRQ 32-bit EL1
1105 ventry el1_fiq_invalid // FIQ 32-bit EL1
1106 ventry el1_error_invalid // Error 32-bit EL1
1107 ENDPROC(__kvm_hyp_vector)
1110 ENTRY(__kvm_get_mdcr_el2)
1111 mrs x0, mdcr_el2
1112 ret
1113 ENDPROC(__kvm_get_mdcr_el2)
1115 .popsection