2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
5 * Alexander Graf <agraf@suse.de>
6 * Kevin Wolf <mail@kevin-wolf.de>
9 * This file is derived from arch/powerpc/kvm/44x.c,
10 * by Hollis Blanchard <hollisb@us.ibm.com>.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License, version 2, as
14 * published by the Free Software Foundation.
17 #include <linux/kvm_host.h>
18 #include <linux/err.h>
19 #include <linux/export.h>
20 #include <linux/slab.h>
23 #include <asm/cputable.h>
24 #include <asm/cacheflush.h>
25 #include <asm/tlbflush.h>
26 #include <asm/uaccess.h>
28 #include <asm/kvm_ppc.h>
29 #include <asm/kvm_book3s.h>
30 #include <asm/mmu_context.h>
32 #include <linux/gfp.h>
33 #include <linux/sched.h>
34 #include <linux/vmalloc.h>
35 #include <linux/highmem.h>
39 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
41 /* #define EXIT_DEBUG */
43 struct kvm_stats_debugfs_item debugfs_entries
[] = {
44 { "exits", VCPU_STAT(sum_exits
) },
45 { "mmio", VCPU_STAT(mmio_exits
) },
46 { "sig", VCPU_STAT(signal_exits
) },
47 { "sysc", VCPU_STAT(syscall_exits
) },
48 { "inst_emu", VCPU_STAT(emulated_inst_exits
) },
49 { "dec", VCPU_STAT(dec_exits
) },
50 { "ext_intr", VCPU_STAT(ext_intr_exits
) },
51 { "queue_intr", VCPU_STAT(queue_intr
) },
52 { "halt_wakeup", VCPU_STAT(halt_wakeup
) },
53 { "pf_storage", VCPU_STAT(pf_storage
) },
54 { "sp_storage", VCPU_STAT(sp_storage
) },
55 { "pf_instruc", VCPU_STAT(pf_instruc
) },
56 { "sp_instruc", VCPU_STAT(sp_instruc
) },
57 { "ld", VCPU_STAT(ld
) },
58 { "ld_slow", VCPU_STAT(ld_slow
) },
59 { "st", VCPU_STAT(st
) },
60 { "st_slow", VCPU_STAT(st_slow
) },
64 void kvmppc_core_load_host_debugstate(struct kvm_vcpu
*vcpu
)
68 void kvmppc_core_load_guest_debugstate(struct kvm_vcpu
*vcpu
)
72 void kvmppc_inject_interrupt(struct kvm_vcpu
*vcpu
, int vec
, u64 flags
)
74 vcpu
->arch
.shared
->srr0
= kvmppc_get_pc(vcpu
);
75 vcpu
->arch
.shared
->srr1
= vcpu
->arch
.shared
->msr
| flags
;
76 kvmppc_set_pc(vcpu
, kvmppc_interrupt_offset(vcpu
) + vec
);
77 vcpu
->arch
.mmu
.reset_msr(vcpu
);
80 static int kvmppc_book3s_vec2irqprio(unsigned int vec
)
85 case 0x100: prio
= BOOK3S_IRQPRIO_SYSTEM_RESET
; break;
86 case 0x200: prio
= BOOK3S_IRQPRIO_MACHINE_CHECK
; break;
87 case 0x300: prio
= BOOK3S_IRQPRIO_DATA_STORAGE
; break;
88 case 0x380: prio
= BOOK3S_IRQPRIO_DATA_SEGMENT
; break;
89 case 0x400: prio
= BOOK3S_IRQPRIO_INST_STORAGE
; break;
90 case 0x480: prio
= BOOK3S_IRQPRIO_INST_SEGMENT
; break;
91 case 0x500: prio
= BOOK3S_IRQPRIO_EXTERNAL
; break;
92 case 0x501: prio
= BOOK3S_IRQPRIO_EXTERNAL_LEVEL
; break;
93 case 0x600: prio
= BOOK3S_IRQPRIO_ALIGNMENT
; break;
94 case 0x700: prio
= BOOK3S_IRQPRIO_PROGRAM
; break;
95 case 0x800: prio
= BOOK3S_IRQPRIO_FP_UNAVAIL
; break;
96 case 0x900: prio
= BOOK3S_IRQPRIO_DECREMENTER
; break;
97 case 0xc00: prio
= BOOK3S_IRQPRIO_SYSCALL
; break;
98 case 0xd00: prio
= BOOK3S_IRQPRIO_DEBUG
; break;
99 case 0xf20: prio
= BOOK3S_IRQPRIO_ALTIVEC
; break;
100 case 0xf40: prio
= BOOK3S_IRQPRIO_VSX
; break;
101 default: prio
= BOOK3S_IRQPRIO_MAX
; break;
107 void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu
*vcpu
,
110 unsigned long old_pending
= vcpu
->arch
.pending_exceptions
;
112 clear_bit(kvmppc_book3s_vec2irqprio(vec
),
113 &vcpu
->arch
.pending_exceptions
);
115 kvmppc_update_int_pending(vcpu
, vcpu
->arch
.pending_exceptions
,
119 void kvmppc_book3s_queue_irqprio(struct kvm_vcpu
*vcpu
, unsigned int vec
)
121 vcpu
->stat
.queue_intr
++;
123 set_bit(kvmppc_book3s_vec2irqprio(vec
),
124 &vcpu
->arch
.pending_exceptions
);
126 printk(KERN_INFO
"Queueing interrupt %x\n", vec
);
131 void kvmppc_core_queue_program(struct kvm_vcpu
*vcpu
, ulong flags
)
133 /* might as well deliver this straight away */
134 kvmppc_inject_interrupt(vcpu
, BOOK3S_INTERRUPT_PROGRAM
, flags
);
137 void kvmppc_core_queue_dec(struct kvm_vcpu
*vcpu
)
139 kvmppc_book3s_queue_irqprio(vcpu
, BOOK3S_INTERRUPT_DECREMENTER
);
142 int kvmppc_core_pending_dec(struct kvm_vcpu
*vcpu
)
144 return test_bit(BOOK3S_IRQPRIO_DECREMENTER
, &vcpu
->arch
.pending_exceptions
);
147 void kvmppc_core_dequeue_dec(struct kvm_vcpu
*vcpu
)
149 kvmppc_book3s_dequeue_irqprio(vcpu
, BOOK3S_INTERRUPT_DECREMENTER
);
152 void kvmppc_core_queue_external(struct kvm_vcpu
*vcpu
,
153 struct kvm_interrupt
*irq
)
155 unsigned int vec
= BOOK3S_INTERRUPT_EXTERNAL
;
157 if (irq
->irq
== KVM_INTERRUPT_SET_LEVEL
)
158 vec
= BOOK3S_INTERRUPT_EXTERNAL_LEVEL
;
160 kvmppc_book3s_queue_irqprio(vcpu
, vec
);
163 void kvmppc_core_dequeue_external(struct kvm_vcpu
*vcpu
)
165 kvmppc_book3s_dequeue_irqprio(vcpu
, BOOK3S_INTERRUPT_EXTERNAL
);
166 kvmppc_book3s_dequeue_irqprio(vcpu
, BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
169 int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu
*vcpu
, unsigned int priority
)
173 bool crit
= kvmppc_critical_section(vcpu
);
176 case BOOK3S_IRQPRIO_DECREMENTER
:
177 deliver
= (vcpu
->arch
.shared
->msr
& MSR_EE
) && !crit
;
178 vec
= BOOK3S_INTERRUPT_DECREMENTER
;
180 case BOOK3S_IRQPRIO_EXTERNAL
:
181 case BOOK3S_IRQPRIO_EXTERNAL_LEVEL
:
182 deliver
= (vcpu
->arch
.shared
->msr
& MSR_EE
) && !crit
;
183 vec
= BOOK3S_INTERRUPT_EXTERNAL
;
185 case BOOK3S_IRQPRIO_SYSTEM_RESET
:
186 vec
= BOOK3S_INTERRUPT_SYSTEM_RESET
;
188 case BOOK3S_IRQPRIO_MACHINE_CHECK
:
189 vec
= BOOK3S_INTERRUPT_MACHINE_CHECK
;
191 case BOOK3S_IRQPRIO_DATA_STORAGE
:
192 vec
= BOOK3S_INTERRUPT_DATA_STORAGE
;
194 case BOOK3S_IRQPRIO_INST_STORAGE
:
195 vec
= BOOK3S_INTERRUPT_INST_STORAGE
;
197 case BOOK3S_IRQPRIO_DATA_SEGMENT
:
198 vec
= BOOK3S_INTERRUPT_DATA_SEGMENT
;
200 case BOOK3S_IRQPRIO_INST_SEGMENT
:
201 vec
= BOOK3S_INTERRUPT_INST_SEGMENT
;
203 case BOOK3S_IRQPRIO_ALIGNMENT
:
204 vec
= BOOK3S_INTERRUPT_ALIGNMENT
;
206 case BOOK3S_IRQPRIO_PROGRAM
:
207 vec
= BOOK3S_INTERRUPT_PROGRAM
;
209 case BOOK3S_IRQPRIO_VSX
:
210 vec
= BOOK3S_INTERRUPT_VSX
;
212 case BOOK3S_IRQPRIO_ALTIVEC
:
213 vec
= BOOK3S_INTERRUPT_ALTIVEC
;
215 case BOOK3S_IRQPRIO_FP_UNAVAIL
:
216 vec
= BOOK3S_INTERRUPT_FP_UNAVAIL
;
218 case BOOK3S_IRQPRIO_SYSCALL
:
219 vec
= BOOK3S_INTERRUPT_SYSCALL
;
221 case BOOK3S_IRQPRIO_DEBUG
:
222 vec
= BOOK3S_INTERRUPT_TRACE
;
224 case BOOK3S_IRQPRIO_PERFORMANCE_MONITOR
:
225 vec
= BOOK3S_INTERRUPT_PERFMON
;
229 printk(KERN_ERR
"KVM: Unknown interrupt: 0x%x\n", priority
);
234 printk(KERN_INFO
"Deliver interrupt 0x%x? %x\n", vec
, deliver
);
238 kvmppc_inject_interrupt(vcpu
, vec
, 0);
244 * This function determines if an irqprio should be cleared once issued.
246 static bool clear_irqprio(struct kvm_vcpu
*vcpu
, unsigned int priority
)
249 case BOOK3S_IRQPRIO_DECREMENTER
:
250 /* DEC interrupts get cleared by mtdec */
252 case BOOK3S_IRQPRIO_EXTERNAL_LEVEL
:
253 /* External interrupts get cleared by userspace */
260 int kvmppc_core_prepare_to_enter(struct kvm_vcpu
*vcpu
)
262 unsigned long *pending
= &vcpu
->arch
.pending_exceptions
;
263 unsigned long old_pending
= vcpu
->arch
.pending_exceptions
;
264 unsigned int priority
;
267 if (vcpu
->arch
.pending_exceptions
)
268 printk(KERN_EMERG
"KVM: Check pending: %lx\n", vcpu
->arch
.pending_exceptions
);
270 priority
= __ffs(*pending
);
271 while (priority
< BOOK3S_IRQPRIO_MAX
) {
272 if (kvmppc_book3s_irqprio_deliver(vcpu
, priority
) &&
273 clear_irqprio(vcpu
, priority
)) {
274 clear_bit(priority
, &vcpu
->arch
.pending_exceptions
);
278 priority
= find_next_bit(pending
,
279 BITS_PER_BYTE
* sizeof(*pending
),
283 /* Tell the guest about our interrupt status */
284 kvmppc_update_int_pending(vcpu
, *pending
, old_pending
);
289 pfn_t
kvmppc_gfn_to_pfn(struct kvm_vcpu
*vcpu
, gfn_t gfn
)
291 ulong mp_pa
= vcpu
->arch
.magic_page_pa
;
293 if (!(vcpu
->arch
.shared
->msr
& MSR_SF
))
294 mp_pa
= (uint32_t)mp_pa
;
296 /* Magic page override */
297 if (unlikely(mp_pa
) &&
298 unlikely(((gfn
<< PAGE_SHIFT
) & KVM_PAM
) ==
299 ((mp_pa
& PAGE_MASK
) & KVM_PAM
))) {
300 ulong shared_page
= ((ulong
)vcpu
->arch
.shared
) & PAGE_MASK
;
303 pfn
= (pfn_t
)virt_to_phys((void*)shared_page
) >> PAGE_SHIFT
;
304 get_page(pfn_to_page(pfn
));
308 return gfn_to_pfn(vcpu
->kvm
, gfn
);
311 static int kvmppc_xlate(struct kvm_vcpu
*vcpu
, ulong eaddr
, bool data
,
312 struct kvmppc_pte
*pte
)
314 int relocated
= (vcpu
->arch
.shared
->msr
& (data
? MSR_DR
: MSR_IR
));
318 r
= vcpu
->arch
.mmu
.xlate(vcpu
, eaddr
, pte
, data
);
321 pte
->raddr
= eaddr
& KVM_PAM
;
322 pte
->vpage
= VSID_REAL
| eaddr
>> 12;
323 pte
->may_read
= true;
324 pte
->may_write
= true;
325 pte
->may_execute
= true;
332 static hva_t
kvmppc_bad_hva(void)
337 static hva_t
kvmppc_pte_to_hva(struct kvm_vcpu
*vcpu
, struct kvmppc_pte
*pte
,
342 if (read
&& !pte
->may_read
)
345 if (!read
&& !pte
->may_write
)
348 hpage
= gfn_to_hva(vcpu
->kvm
, pte
->raddr
>> PAGE_SHIFT
);
349 if (kvm_is_error_hva(hpage
))
352 return hpage
| (pte
->raddr
& ~PAGE_MASK
);
354 return kvmppc_bad_hva();
357 int kvmppc_st(struct kvm_vcpu
*vcpu
, ulong
*eaddr
, int size
, void *ptr
,
360 struct kvmppc_pte pte
;
364 if (kvmppc_xlate(vcpu
, *eaddr
, data
, &pte
))
372 if (kvm_write_guest(vcpu
->kvm
, pte
.raddr
, ptr
, size
))
373 return EMULATE_DO_MMIO
;
378 int kvmppc_ld(struct kvm_vcpu
*vcpu
, ulong
*eaddr
, int size
, void *ptr
,
381 struct kvmppc_pte pte
;
386 if (kvmppc_xlate(vcpu
, *eaddr
, data
, &pte
))
391 hva
= kvmppc_pte_to_hva(vcpu
, &pte
, true);
392 if (kvm_is_error_hva(hva
))
395 if (copy_from_user(ptr
, (void __user
*)hva
, size
)) {
396 printk(KERN_INFO
"kvmppc_ld at 0x%lx failed\n", hva
);
405 return EMULATE_DO_MMIO
;
408 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
413 int kvmppc_subarch_vcpu_init(struct kvm_vcpu
*vcpu
)
418 void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu
*vcpu
)
422 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
426 regs
->pc
= kvmppc_get_pc(vcpu
);
427 regs
->cr
= kvmppc_get_cr(vcpu
);
428 regs
->ctr
= kvmppc_get_ctr(vcpu
);
429 regs
->lr
= kvmppc_get_lr(vcpu
);
430 regs
->xer
= kvmppc_get_xer(vcpu
);
431 regs
->msr
= vcpu
->arch
.shared
->msr
;
432 regs
->srr0
= vcpu
->arch
.shared
->srr0
;
433 regs
->srr1
= vcpu
->arch
.shared
->srr1
;
434 regs
->pid
= vcpu
->arch
.pid
;
435 regs
->sprg0
= vcpu
->arch
.shared
->sprg0
;
436 regs
->sprg1
= vcpu
->arch
.shared
->sprg1
;
437 regs
->sprg2
= vcpu
->arch
.shared
->sprg2
;
438 regs
->sprg3
= vcpu
->arch
.shared
->sprg3
;
439 regs
->sprg4
= vcpu
->arch
.shared
->sprg4
;
440 regs
->sprg5
= vcpu
->arch
.shared
->sprg5
;
441 regs
->sprg6
= vcpu
->arch
.shared
->sprg6
;
442 regs
->sprg7
= vcpu
->arch
.shared
->sprg7
;
444 for (i
= 0; i
< ARRAY_SIZE(regs
->gpr
); i
++)
445 regs
->gpr
[i
] = kvmppc_get_gpr(vcpu
, i
);
450 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
454 kvmppc_set_pc(vcpu
, regs
->pc
);
455 kvmppc_set_cr(vcpu
, regs
->cr
);
456 kvmppc_set_ctr(vcpu
, regs
->ctr
);
457 kvmppc_set_lr(vcpu
, regs
->lr
);
458 kvmppc_set_xer(vcpu
, regs
->xer
);
459 kvmppc_set_msr(vcpu
, regs
->msr
);
460 vcpu
->arch
.shared
->srr0
= regs
->srr0
;
461 vcpu
->arch
.shared
->srr1
= regs
->srr1
;
462 vcpu
->arch
.shared
->sprg0
= regs
->sprg0
;
463 vcpu
->arch
.shared
->sprg1
= regs
->sprg1
;
464 vcpu
->arch
.shared
->sprg2
= regs
->sprg2
;
465 vcpu
->arch
.shared
->sprg3
= regs
->sprg3
;
466 vcpu
->arch
.shared
->sprg4
= regs
->sprg4
;
467 vcpu
->arch
.shared
->sprg5
= regs
->sprg5
;
468 vcpu
->arch
.shared
->sprg6
= regs
->sprg6
;
469 vcpu
->arch
.shared
->sprg7
= regs
->sprg7
;
471 for (i
= 0; i
< ARRAY_SIZE(regs
->gpr
); i
++)
472 kvmppc_set_gpr(vcpu
, i
, regs
->gpr
[i
]);
477 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
482 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
487 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu
*vcpu
, struct kvm_one_reg
*reg
)
490 union kvmppc_one_reg val
;
494 size
= one_reg_size(reg
->id
);
495 if (size
> sizeof(val
))
498 r
= kvmppc_get_one_reg(vcpu
, reg
->id
, &val
);
503 case KVM_REG_PPC_DAR
:
504 val
= get_reg_val(reg
->id
, vcpu
->arch
.shared
->dar
);
506 case KVM_REG_PPC_DSISR
:
507 val
= get_reg_val(reg
->id
, vcpu
->arch
.shared
->dsisr
);
509 case KVM_REG_PPC_FPR0
... KVM_REG_PPC_FPR31
:
510 i
= reg
->id
- KVM_REG_PPC_FPR0
;
511 val
= get_reg_val(reg
->id
, vcpu
->arch
.fpr
[i
]);
513 case KVM_REG_PPC_FPSCR
:
514 val
= get_reg_val(reg
->id
, vcpu
->arch
.fpscr
);
516 #ifdef CONFIG_ALTIVEC
517 case KVM_REG_PPC_VR0
... KVM_REG_PPC_VR31
:
518 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
522 val
.vval
= vcpu
->arch
.vr
[reg
->id
- KVM_REG_PPC_VR0
];
524 case KVM_REG_PPC_VSCR
:
525 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
529 val
= get_reg_val(reg
->id
, vcpu
->arch
.vscr
.u
[3]);
531 #endif /* CONFIG_ALTIVEC */
532 case KVM_REG_PPC_DEBUG_INST
: {
534 r
= copy_to_user((u32 __user
*)(long)reg
->addr
,
535 &opcode
, sizeof(u32
));
538 #ifdef CONFIG_KVM_XICS
539 case KVM_REG_PPC_ICP_STATE
:
540 if (!vcpu
->arch
.icp
) {
544 val
= get_reg_val(reg
->id
, kvmppc_xics_get_icp(vcpu
));
546 #endif /* CONFIG_KVM_XICS */
555 if (copy_to_user((char __user
*)(unsigned long)reg
->addr
, &val
, size
))
561 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu
*vcpu
, struct kvm_one_reg
*reg
)
564 union kvmppc_one_reg val
;
568 size
= one_reg_size(reg
->id
);
569 if (size
> sizeof(val
))
572 if (copy_from_user(&val
, (char __user
*)(unsigned long)reg
->addr
, size
))
575 r
= kvmppc_set_one_reg(vcpu
, reg
->id
, &val
);
580 case KVM_REG_PPC_DAR
:
581 vcpu
->arch
.shared
->dar
= set_reg_val(reg
->id
, val
);
583 case KVM_REG_PPC_DSISR
:
584 vcpu
->arch
.shared
->dsisr
= set_reg_val(reg
->id
, val
);
586 case KVM_REG_PPC_FPR0
... KVM_REG_PPC_FPR31
:
587 i
= reg
->id
- KVM_REG_PPC_FPR0
;
588 vcpu
->arch
.fpr
[i
] = set_reg_val(reg
->id
, val
);
590 case KVM_REG_PPC_FPSCR
:
591 vcpu
->arch
.fpscr
= set_reg_val(reg
->id
, val
);
593 #ifdef CONFIG_ALTIVEC
594 case KVM_REG_PPC_VR0
... KVM_REG_PPC_VR31
:
595 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
599 vcpu
->arch
.vr
[reg
->id
- KVM_REG_PPC_VR0
] = val
.vval
;
601 case KVM_REG_PPC_VSCR
:
602 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
606 vcpu
->arch
.vscr
.u
[3] = set_reg_val(reg
->id
, val
);
608 #endif /* CONFIG_ALTIVEC */
609 #ifdef CONFIG_KVM_XICS
610 case KVM_REG_PPC_ICP_STATE
:
611 if (!vcpu
->arch
.icp
) {
615 r
= kvmppc_xics_set_icp(vcpu
,
616 set_reg_val(reg
->id
, val
));
618 #endif /* CONFIG_KVM_XICS */
628 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
629 struct kvm_translation
*tr
)
634 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu
*vcpu
,
635 struct kvm_guest_debug
*dbg
)
640 void kvmppc_decrementer_func(unsigned long data
)
642 struct kvm_vcpu
*vcpu
= (struct kvm_vcpu
*)data
;
644 kvmppc_core_queue_dec(vcpu
);