| blob | a1467e7689f51fb594e829b914725e5532731851 |
1 /*
2 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
3 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License, version 2, as
7 * published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 */
19 #include <linux/linkage.h>
20 #include <linux/const.h>
21 #include <asm/unified.h>
22 #include <asm/page.h>
23 #include <asm/ptrace.h>
24 #include <asm/asm-offsets.h>
25 #include <asm/kvm_asm.h>
26 #include <asm/kvm_arm.h>
27 #include <asm/vfpmacros.h>
28 #include "interrupts_head.S"
30 .text
32 __kvm_hyp_code_start:
33 .globl __kvm_hyp_code_start
35 /********************************************************************
36 * Flush per-VMID TLBs
37 *
38 * void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
39 *
40 * We rely on the hardware to broadcast the TLB invalidation to all CPUs
41 * inside the inner-shareable domain (which is the case for all v7
42 * implementations). If we come across a non-IS SMP implementation, we'll
43 * have to use an IPI based mechanism. Until then, we stick to the simple
44 * hardware assisted version.
45 *
46 * As v7 does not support flushing per IPA, just nuke the whole TLB
47 * instead, ignoring the ipa value.
48 */
49 ENTRY(__kvm_tlb_flush_vmid_ipa)
50 push {r2, r3}
52 dsb ishst
53 add r0, r0, #KVM_VTTBR
54 ldrd r2, r3, [r0]
55 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
56 isb
57 mcr p15, 0, r0, c8, c3, 0 @ TLBIALLIS (rt ignored)
58 dsb ish
59 isb
60 mov r2, #0
61 mov r3, #0
62 mcrr p15, 6, r2, r3, c2 @ Back to VMID #0
63 isb @ Not necessary if followed by eret
65 pop {r2, r3}
66 bx lr
67 ENDPROC(__kvm_tlb_flush_vmid_ipa)
69 /********************************************************************
70 * Flush TLBs and instruction caches of all CPUs inside the inner-shareable
71 * domain, for all VMIDs
72 *
73 * void __kvm_flush_vm_context(void);
74 */
75 ENTRY(__kvm_flush_vm_context)
76 mov r0, #0 @ rn parameter for c15 flushes is SBZ
78 /* Invalidate NS Non-Hyp TLB Inner Shareable (TLBIALLNSNHIS) */
79 mcr p15, 4, r0, c8, c3, 4
80 /* Invalidate instruction caches Inner Shareable (ICIALLUIS) */
81 mcr p15, 0, r0, c7, c1, 0
82 dsb ish
83 isb @ Not necessary if followed by eret
85 bx lr
86 ENDPROC(__kvm_flush_vm_context)
89 /********************************************************************
90 * Hypervisor world-switch code
91 *
92 *
93 * int __kvm_vcpu_run(struct kvm_vcpu *vcpu)
94 */
95 ENTRY(__kvm_vcpu_run)
96 @ Save the vcpu pointer
97 mcr p15, 4, vcpu, c13, c0, 2 @ HTPIDR
99 save_host_regs
101 restore_vgic_state
102 restore_timer_state
104 @ Store hardware CP15 state and load guest state
105 read_cp15_state store_to_vcpu = 0
106 write_cp15_state read_from_vcpu = 1
108 @ If the host kernel has not been configured with VFPv3 support,
109 @ then it is safer if we deny guests from using it as well.
110 #ifdef CONFIG_VFPv3
111 @ Set FPEXC_EN so the guest doesn't trap floating point instructions
112 VFPFMRX r2, FPEXC @ VMRS
113 push {r2}
114 orr r2, r2, #FPEXC_EN
115 VFPFMXR FPEXC, r2 @ VMSR
116 #endif
118 @ Configure Hyp-role
119 configure_hyp_role vmentry
121 @ Trap coprocessor CRx accesses
122 set_hstr vmentry
123 set_hcptr vmentry, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11))
124 set_hdcr vmentry
126 @ Write configured ID register into MIDR alias
127 ldr r1, [vcpu, #VCPU_MIDR]
128 mcr p15, 4, r1, c0, c0, 0
130 @ Write guest view of MPIDR into VMPIDR
131 ldr r1, [vcpu, #CP15_OFFSET(c0_MPIDR)]
132 mcr p15, 4, r1, c0, c0, 5
134 @ Set up guest memory translation
135 ldr r1, [vcpu, #VCPU_KVM]
136 add r1, r1, #KVM_VTTBR
137 ldrd r2, r3, [r1]
138 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
140 @ We're all done, just restore the GPRs and go to the guest
141 restore_guest_regs
142 clrex @ Clear exclusive monitor
143 eret
145 __kvm_vcpu_return:
146 /*
147 * return convention:
148 * guest r0, r1, r2 saved on the stack
149 * r0: vcpu pointer
150 * r1: exception code
151 */
152 save_guest_regs
154 @ Set VMID == 0
155 mov r2, #0
156 mov r3, #0
157 mcrr p15, 6, r2, r3, c2 @ Write VTTBR
159 @ Don't trap coprocessor accesses for host kernel
160 set_hstr vmexit
161 set_hdcr vmexit
162 set_hcptr vmexit, (HCPTR_TTA | HCPTR_TCP(10) | HCPTR_TCP(11)), after_vfp_restore
164 #ifdef CONFIG_VFPv3
165 @ Switch VFP/NEON hardware state to the host's
166 add r7, vcpu, #VCPU_VFP_GUEST
167 store_vfp_state r7
168 add r7, vcpu, #VCPU_VFP_HOST
169 ldr r7, [r7]
170 restore_vfp_state r7
172 after_vfp_restore:
173 @ Restore FPEXC_EN which we clobbered on entry
174 pop {r2}
175 VFPFMXR FPEXC, r2
176 #else
177 after_vfp_restore:
178 #endif
180 @ Reset Hyp-role
181 configure_hyp_role vmexit
183 @ Let host read hardware MIDR
184 mrc p15, 0, r2, c0, c0, 0
185 mcr p15, 4, r2, c0, c0, 0
187 @ Back to hardware MPIDR
188 mrc p15, 0, r2, c0, c0, 5
189 mcr p15, 4, r2, c0, c0, 5
191 @ Store guest CP15 state and restore host state
192 read_cp15_state store_to_vcpu = 1
193 write_cp15_state read_from_vcpu = 0
195 save_timer_state
196 save_vgic_state
198 restore_host_regs
199 clrex @ Clear exclusive monitor
200 mov r0, r1 @ Return the return code
201 mov r1, #0 @ Clear upper bits in return value
202 bx lr @ return to IOCTL
204 /********************************************************************
205 * Call function in Hyp mode
206 *
207 *
208 * u64 kvm_call_hyp(void *hypfn, ...);
209 *
210 * This is not really a variadic function in the classic C-way and care must
211 * be taken when calling this to ensure parameters are passed in registers
212 * only, since the stack will change between the caller and the callee.
213 *
214 * Call the function with the first argument containing a pointer to the
215 * function you wish to call in Hyp mode, and subsequent arguments will be
216 * passed as r0, r1, and r2 (a maximum of 3 arguments in addition to the
217 * function pointer can be passed). The function being called must be mapped
218 * in Hyp mode (see init_hyp_mode in arch/arm/kvm/arm.c). Return values are
219 * passed in r0 and r1.
220 *
221 * A function pointer with a value of 0xffffffff has a special meaning,
222 * and is used to implement __hyp_get_vectors in the same way as in
223 * arch/arm/kernel/hyp_stub.S.
224 *
225 * The calling convention follows the standard AAPCS:
226 * r0 - r3: caller save
227 * r12: caller save
228 * rest: callee save
229 */
230 ENTRY(kvm_call_hyp)
231 hvc #0
232 bx lr
234 /********************************************************************
235 * Hypervisor exception vector and handlers
236 *
237 *
238 * The KVM/ARM Hypervisor ABI is defined as follows:
239 *
240 * Entry to Hyp mode from the host kernel will happen _only_ when an HVC
241 * instruction is issued since all traps are disabled when running the host
242 * kernel as per the Hyp-mode initialization at boot time.
243 *
244 * HVC instructions cause a trap to the vector page + offset 0x14 (see hyp_hvc
245 * below) when the HVC instruction is called from SVC mode (i.e. a guest or the
246 * host kernel) and they cause a trap to the vector page + offset 0x8 when HVC
247 * instructions are called from within Hyp-mode.
248 *
249 * Hyp-ABI: Calling HYP-mode functions from host (in SVC mode):
250 * Switching to Hyp mode is done through a simple HVC #0 instruction. The
251 * exception vector code will check that the HVC comes from VMID==0 and if
252 * so will push the necessary state (SPSR, lr_usr) on the Hyp stack.
253 * - r0 contains a pointer to a HYP function
254 * - r1, r2, and r3 contain arguments to the above function.
255 * - The HYP function will be called with its arguments in r0, r1 and r2.
256 * On HYP function return, we return directly to SVC.
257 *
258 * Note that the above is used to execute code in Hyp-mode from a host-kernel
259 * point of view, and is a different concept from performing a world-switch and
260 * executing guest code SVC mode (with a VMID != 0).
261 */
263 /* Handle undef, svc, pabt, or dabt by crashing with a user notice */
264 .macro bad_exception exception_code, panic_str
265 push {r0-r2}
266 mrrc p15, 6, r0, r1, c2 @ Read VTTBR
267 lsr r1, r1, #16
268 ands r1, r1, #0xff
269 beq 99f
271 load_vcpu @ Load VCPU pointer
272 .if \exception_code == ARM_EXCEPTION_DATA_ABORT
273 mrc p15, 4, r2, c5, c2, 0 @ HSR
274 mrc p15, 4, r1, c6, c0, 0 @ HDFAR
275 str r2, [vcpu, #VCPU_HSR]
276 str r1, [vcpu, #VCPU_HxFAR]
277 .endif
278 .if \exception_code == ARM_EXCEPTION_PREF_ABORT
279 mrc p15, 4, r2, c5, c2, 0 @ HSR
280 mrc p15, 4, r1, c6, c0, 2 @ HIFAR
281 str r2, [vcpu, #VCPU_HSR]
282 str r1, [vcpu, #VCPU_HxFAR]
283 .endif
284 mov r1, #\exception_code
285 b __kvm_vcpu_return
287 @ We were in the host already. Let's craft a panic-ing return to SVC.
288 99: mrs r2, cpsr
289 bic r2, r2, #MODE_MASK
290 orr r2, r2, #SVC_MODE
291 THUMB( orr r2, r2, #PSR_T_BIT )
292 msr spsr_cxsf, r2
293 mrs r1, ELR_hyp
294 ldr r2, =BSYM(panic)
295 msr ELR_hyp, r2
296 ldr r0, =\panic_str
297 clrex @ Clear exclusive monitor
298 eret
299 .endm
301 .text
303 .align 5
304 __kvm_hyp_vector:
305 .globl __kvm_hyp_vector
307 @ Hyp-mode exception vector
308 W(b) hyp_reset
309 W(b) hyp_undef
310 W(b) hyp_svc
311 W(b) hyp_pabt
312 W(b) hyp_dabt
313 W(b) hyp_hvc
314 W(b) hyp_irq
315 W(b) hyp_fiq
317 .align
318 hyp_reset:
319 b hyp_reset
321 .align
322 hyp_undef:
323 bad_exception ARM_EXCEPTION_UNDEFINED, und_die_str
325 .align
326 hyp_svc:
327 bad_exception ARM_EXCEPTION_HVC, svc_die_str
329 .align
330 hyp_pabt:
331 bad_exception ARM_EXCEPTION_PREF_ABORT, pabt_die_str
333 .align
334 hyp_dabt:
335 bad_exception ARM_EXCEPTION_DATA_ABORT, dabt_die_str
337 .align
338 hyp_hvc:
339 /*
340 * Getting here is either becuase of a trap from a guest or from calling
341 * HVC from the host kernel, which means "switch to Hyp mode".
342 */
343 push {r0, r1, r2}
345 @ Check syndrome register
346 mrc p15, 4, r1, c5, c2, 0 @ HSR
347 lsr r0, r1, #HSR_EC_SHIFT
348 #ifdef CONFIG_VFPv3
349 cmp r0, #HSR_EC_CP_0_13
350 beq switch_to_guest_vfp
351 #endif
352 cmp r0, #HSR_EC_HVC
353 bne guest_trap @ Not HVC instr.
355 /*
356 * Let's check if the HVC came from VMID 0 and allow simple
357 * switch to Hyp mode
358 */
359 mrrc p15, 6, r0, r2, c2
360 lsr r2, r2, #16
361 and r2, r2, #0xff
362 cmp r2, #0
363 bne guest_trap @ Guest called HVC
365 host_switch_to_hyp:
366 pop {r0, r1, r2}
368 /* Check for __hyp_get_vectors */
369 cmp r0, #-1
370 mrceq p15, 4, r0, c12, c0, 0 @ get HVBAR
371 beq 1f
373 push {lr}
374 mrs lr, SPSR
375 push {lr}
377 mov lr, r0
378 mov r0, r1
379 mov r1, r2
380 mov r2, r3
382 THUMB( orr lr, #1)
383 blx lr @ Call the HYP function
385 pop {lr}
386 msr SPSR_csxf, lr
387 pop {lr}
388 1: eret
390 guest_trap:
391 load_vcpu @ Load VCPU pointer to r0
392 str r1, [vcpu, #VCPU_HSR]
394 @ Check if we need the fault information
395 lsr r1, r1, #HSR_EC_SHIFT
396 cmp r1, #HSR_EC_IABT
397 mrceq p15, 4, r2, c6, c0, 2 @ HIFAR
398 beq 2f
399 cmp r1, #HSR_EC_DABT
400 bne 1f
401 mrc p15, 4, r2, c6, c0, 0 @ HDFAR
403 2: str r2, [vcpu, #VCPU_HxFAR]
405 /*
406 * B3.13.5 Reporting exceptions taken to the Non-secure PL2 mode:
407 *
408 * Abort on the stage 2 translation for a memory access from a
409 * Non-secure PL1 or PL0 mode:
410 *
411 * For any Access flag fault or Translation fault, and also for any
412 * Permission fault on the stage 2 translation of a memory access
413 * made as part of a translation table walk for a stage 1 translation,
414 * the HPFAR holds the IPA that caused the fault. Otherwise, the HPFAR
415 * is UNKNOWN.
416 */
418 /* Check for permission fault, and S1PTW */
419 mrc p15, 4, r1, c5, c2, 0 @ HSR
420 and r0, r1, #HSR_FSC_TYPE
421 cmp r0, #FSC_PERM
422 tsteq r1, #(1 << 7) @ S1PTW
423 mrcne p15, 4, r2, c6, c0, 4 @ HPFAR
424 bne 3f
426 /* Preserve PAR */
427 mrrc p15, 0, r0, r1, c7 @ PAR
428 push {r0, r1}
430 /* Resolve IPA using the xFAR */
431 mcr p15, 0, r2, c7, c8, 0 @ ATS1CPR
432 isb
433 mrrc p15, 0, r0, r1, c7 @ PAR
434 tst r0, #1
435 bne 4f @ Failed translation
436 ubfx r2, r0, #12, #20
437 lsl r2, r2, #4
438 orr r2, r2, r1, lsl #24
440 /* Restore PAR */
441 pop {r0, r1}
442 mcrr p15, 0, r0, r1, c7 @ PAR
444 3: load_vcpu @ Load VCPU pointer to r0
445 str r2, [r0, #VCPU_HPFAR]
447 1: mov r1, #ARM_EXCEPTION_HVC
448 b __kvm_vcpu_return
450 4: pop {r0, r1} @ Failed translation, return to guest
451 mcrr p15, 0, r0, r1, c7 @ PAR
452 clrex
453 pop {r0, r1, r2}
454 eret
456 /*
457 * If VFPv3 support is not available, then we will not switch the VFP
458 * registers; however cp10 and cp11 accesses will still trap and fallback
459 * to the regular coprocessor emulation code, which currently will
460 * inject an undefined exception to the guest.
461 */
462 #ifdef CONFIG_VFPv3
463 switch_to_guest_vfp:
464 load_vcpu @ Load VCPU pointer to r0
465 push {r3-r7}
467 @ NEON/VFP used. Turn on VFP access.
468 set_hcptr vmtrap, (HCPTR_TCP(10) | HCPTR_TCP(11))
470 @ Switch VFP/NEON hardware state to the guest's
471 add r7, r0, #VCPU_VFP_HOST
472 ldr r7, [r7]
473 store_vfp_state r7
474 add r7, r0, #VCPU_VFP_GUEST
475 restore_vfp_state r7
477 pop {r3-r7}
478 pop {r0-r2}
479 clrex
480 eret
481 #endif
483 .align
484 hyp_irq:
485 push {r0, r1, r2}
486 mov r1, #ARM_EXCEPTION_IRQ
487 load_vcpu @ Load VCPU pointer to r0
488 b __kvm_vcpu_return
490 .align
491 hyp_fiq:
492 b hyp_fiq
494 .ltorg
496 __kvm_hyp_code_end:
497 .globl __kvm_hyp_code_end
499 .section ".rodata"
501 und_die_str:
502 .ascii "unexpected undefined exception in Hyp mode at: %#08x\n"
503 pabt_die_str:
504 .ascii "unexpected prefetch abort in Hyp mode at: %#08x\n"
505 dabt_die_str:
506 .ascii "unexpected data abort in Hyp mode at: %#08x\n"
507 svc_die_str:
508 .ascii "unexpected HVC/SVC trap in Hyp mode at: %#08x\n"