2 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
5 * Alexander Graf <agraf@suse.de>
6 * Kevin Wolf <mail@kevin-wolf.de>
9 * This file is derived from arch/powerpc/kvm/44x.c,
10 * by Hollis Blanchard <hollisb@us.ibm.com>.
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License, version 2, as
14 * published by the Free Software Foundation.
17 #include <linux/kvm_host.h>
18 #include <linux/err.h>
19 #include <linux/export.h>
20 #include <linux/slab.h>
23 #include <asm/cputable.h>
24 #include <asm/cacheflush.h>
25 #include <asm/tlbflush.h>
26 #include <asm/uaccess.h>
28 #include <asm/kvm_ppc.h>
29 #include <asm/kvm_book3s.h>
30 #include <asm/mmu_context.h>
32 #include <linux/gfp.h>
33 #include <linux/sched.h>
34 #include <linux/vmalloc.h>
35 #include <linux/highmem.h>
39 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU
41 /* #define EXIT_DEBUG */
43 struct kvm_stats_debugfs_item debugfs_entries
[] = {
44 { "exits", VCPU_STAT(sum_exits
) },
45 { "mmio", VCPU_STAT(mmio_exits
) },
46 { "sig", VCPU_STAT(signal_exits
) },
47 { "sysc", VCPU_STAT(syscall_exits
) },
48 { "inst_emu", VCPU_STAT(emulated_inst_exits
) },
49 { "dec", VCPU_STAT(dec_exits
) },
50 { "ext_intr", VCPU_STAT(ext_intr_exits
) },
51 { "queue_intr", VCPU_STAT(queue_intr
) },
52 { "halt_wakeup", VCPU_STAT(halt_wakeup
) },
53 { "pf_storage", VCPU_STAT(pf_storage
) },
54 { "sp_storage", VCPU_STAT(sp_storage
) },
55 { "pf_instruc", VCPU_STAT(pf_instruc
) },
56 { "sp_instruc", VCPU_STAT(sp_instruc
) },
57 { "ld", VCPU_STAT(ld
) },
58 { "ld_slow", VCPU_STAT(ld_slow
) },
59 { "st", VCPU_STAT(st
) },
60 { "st_slow", VCPU_STAT(st_slow
) },
64 void kvmppc_core_load_host_debugstate(struct kvm_vcpu
*vcpu
)
68 void kvmppc_core_load_guest_debugstate(struct kvm_vcpu
*vcpu
)
72 void kvmppc_inject_interrupt(struct kvm_vcpu
*vcpu
, int vec
, u64 flags
)
74 vcpu
->arch
.shared
->srr0
= kvmppc_get_pc(vcpu
);
75 vcpu
->arch
.shared
->srr1
= vcpu
->arch
.shared
->msr
| flags
;
76 kvmppc_set_pc(vcpu
, kvmppc_interrupt_offset(vcpu
) + vec
);
77 vcpu
->arch
.mmu
.reset_msr(vcpu
);
80 static int kvmppc_book3s_vec2irqprio(unsigned int vec
)
85 case 0x100: prio
= BOOK3S_IRQPRIO_SYSTEM_RESET
; break;
86 case 0x200: prio
= BOOK3S_IRQPRIO_MACHINE_CHECK
; break;
87 case 0x300: prio
= BOOK3S_IRQPRIO_DATA_STORAGE
; break;
88 case 0x380: prio
= BOOK3S_IRQPRIO_DATA_SEGMENT
; break;
89 case 0x400: prio
= BOOK3S_IRQPRIO_INST_STORAGE
; break;
90 case 0x480: prio
= BOOK3S_IRQPRIO_INST_SEGMENT
; break;
91 case 0x500: prio
= BOOK3S_IRQPRIO_EXTERNAL
; break;
92 case 0x501: prio
= BOOK3S_IRQPRIO_EXTERNAL_LEVEL
; break;
93 case 0x600: prio
= BOOK3S_IRQPRIO_ALIGNMENT
; break;
94 case 0x700: prio
= BOOK3S_IRQPRIO_PROGRAM
; break;
95 case 0x800: prio
= BOOK3S_IRQPRIO_FP_UNAVAIL
; break;
96 case 0x900: prio
= BOOK3S_IRQPRIO_DECREMENTER
; break;
97 case 0xc00: prio
= BOOK3S_IRQPRIO_SYSCALL
; break;
98 case 0xd00: prio
= BOOK3S_IRQPRIO_DEBUG
; break;
99 case 0xf20: prio
= BOOK3S_IRQPRIO_ALTIVEC
; break;
100 case 0xf40: prio
= BOOK3S_IRQPRIO_VSX
; break;
101 default: prio
= BOOK3S_IRQPRIO_MAX
; break;
107 static void kvmppc_book3s_dequeue_irqprio(struct kvm_vcpu
*vcpu
,
110 unsigned long old_pending
= vcpu
->arch
.pending_exceptions
;
112 clear_bit(kvmppc_book3s_vec2irqprio(vec
),
113 &vcpu
->arch
.pending_exceptions
);
115 kvmppc_update_int_pending(vcpu
, vcpu
->arch
.pending_exceptions
,
119 void kvmppc_book3s_queue_irqprio(struct kvm_vcpu
*vcpu
, unsigned int vec
)
121 vcpu
->stat
.queue_intr
++;
123 set_bit(kvmppc_book3s_vec2irqprio(vec
),
124 &vcpu
->arch
.pending_exceptions
);
126 printk(KERN_INFO
"Queueing interrupt %x\n", vec
);
131 void kvmppc_core_queue_program(struct kvm_vcpu
*vcpu
, ulong flags
)
133 /* might as well deliver this straight away */
134 kvmppc_inject_interrupt(vcpu
, BOOK3S_INTERRUPT_PROGRAM
, flags
);
137 void kvmppc_core_queue_dec(struct kvm_vcpu
*vcpu
)
139 kvmppc_book3s_queue_irqprio(vcpu
, BOOK3S_INTERRUPT_DECREMENTER
);
142 int kvmppc_core_pending_dec(struct kvm_vcpu
*vcpu
)
144 return test_bit(BOOK3S_IRQPRIO_DECREMENTER
, &vcpu
->arch
.pending_exceptions
);
147 void kvmppc_core_dequeue_dec(struct kvm_vcpu
*vcpu
)
149 kvmppc_book3s_dequeue_irqprio(vcpu
, BOOK3S_INTERRUPT_DECREMENTER
);
152 void kvmppc_core_queue_external(struct kvm_vcpu
*vcpu
,
153 struct kvm_interrupt
*irq
)
155 unsigned int vec
= BOOK3S_INTERRUPT_EXTERNAL
;
157 if (irq
->irq
== KVM_INTERRUPT_SET_LEVEL
)
158 vec
= BOOK3S_INTERRUPT_EXTERNAL_LEVEL
;
160 kvmppc_book3s_queue_irqprio(vcpu
, vec
);
163 void kvmppc_core_dequeue_external(struct kvm_vcpu
*vcpu
,
164 struct kvm_interrupt
*irq
)
166 kvmppc_book3s_dequeue_irqprio(vcpu
, BOOK3S_INTERRUPT_EXTERNAL
);
167 kvmppc_book3s_dequeue_irqprio(vcpu
, BOOK3S_INTERRUPT_EXTERNAL_LEVEL
);
170 int kvmppc_book3s_irqprio_deliver(struct kvm_vcpu
*vcpu
, unsigned int priority
)
174 bool crit
= kvmppc_critical_section(vcpu
);
177 case BOOK3S_IRQPRIO_DECREMENTER
:
178 deliver
= (vcpu
->arch
.shared
->msr
& MSR_EE
) && !crit
;
179 vec
= BOOK3S_INTERRUPT_DECREMENTER
;
181 case BOOK3S_IRQPRIO_EXTERNAL
:
182 case BOOK3S_IRQPRIO_EXTERNAL_LEVEL
:
183 deliver
= (vcpu
->arch
.shared
->msr
& MSR_EE
) && !crit
;
184 vec
= BOOK3S_INTERRUPT_EXTERNAL
;
186 case BOOK3S_IRQPRIO_SYSTEM_RESET
:
187 vec
= BOOK3S_INTERRUPT_SYSTEM_RESET
;
189 case BOOK3S_IRQPRIO_MACHINE_CHECK
:
190 vec
= BOOK3S_INTERRUPT_MACHINE_CHECK
;
192 case BOOK3S_IRQPRIO_DATA_STORAGE
:
193 vec
= BOOK3S_INTERRUPT_DATA_STORAGE
;
195 case BOOK3S_IRQPRIO_INST_STORAGE
:
196 vec
= BOOK3S_INTERRUPT_INST_STORAGE
;
198 case BOOK3S_IRQPRIO_DATA_SEGMENT
:
199 vec
= BOOK3S_INTERRUPT_DATA_SEGMENT
;
201 case BOOK3S_IRQPRIO_INST_SEGMENT
:
202 vec
= BOOK3S_INTERRUPT_INST_SEGMENT
;
204 case BOOK3S_IRQPRIO_ALIGNMENT
:
205 vec
= BOOK3S_INTERRUPT_ALIGNMENT
;
207 case BOOK3S_IRQPRIO_PROGRAM
:
208 vec
= BOOK3S_INTERRUPT_PROGRAM
;
210 case BOOK3S_IRQPRIO_VSX
:
211 vec
= BOOK3S_INTERRUPT_VSX
;
213 case BOOK3S_IRQPRIO_ALTIVEC
:
214 vec
= BOOK3S_INTERRUPT_ALTIVEC
;
216 case BOOK3S_IRQPRIO_FP_UNAVAIL
:
217 vec
= BOOK3S_INTERRUPT_FP_UNAVAIL
;
219 case BOOK3S_IRQPRIO_SYSCALL
:
220 vec
= BOOK3S_INTERRUPT_SYSCALL
;
222 case BOOK3S_IRQPRIO_DEBUG
:
223 vec
= BOOK3S_INTERRUPT_TRACE
;
225 case BOOK3S_IRQPRIO_PERFORMANCE_MONITOR
:
226 vec
= BOOK3S_INTERRUPT_PERFMON
;
230 printk(KERN_ERR
"KVM: Unknown interrupt: 0x%x\n", priority
);
235 printk(KERN_INFO
"Deliver interrupt 0x%x? %x\n", vec
, deliver
);
239 kvmppc_inject_interrupt(vcpu
, vec
, 0);
245 * This function determines if an irqprio should be cleared once issued.
247 static bool clear_irqprio(struct kvm_vcpu
*vcpu
, unsigned int priority
)
250 case BOOK3S_IRQPRIO_DECREMENTER
:
251 /* DEC interrupts get cleared by mtdec */
253 case BOOK3S_IRQPRIO_EXTERNAL_LEVEL
:
254 /* External interrupts get cleared by userspace */
261 int kvmppc_core_prepare_to_enter(struct kvm_vcpu
*vcpu
)
263 unsigned long *pending
= &vcpu
->arch
.pending_exceptions
;
264 unsigned long old_pending
= vcpu
->arch
.pending_exceptions
;
265 unsigned int priority
;
268 if (vcpu
->arch
.pending_exceptions
)
269 printk(KERN_EMERG
"KVM: Check pending: %lx\n", vcpu
->arch
.pending_exceptions
);
271 priority
= __ffs(*pending
);
272 while (priority
< BOOK3S_IRQPRIO_MAX
) {
273 if (kvmppc_book3s_irqprio_deliver(vcpu
, priority
) &&
274 clear_irqprio(vcpu
, priority
)) {
275 clear_bit(priority
, &vcpu
->arch
.pending_exceptions
);
279 priority
= find_next_bit(pending
,
280 BITS_PER_BYTE
* sizeof(*pending
),
284 /* Tell the guest about our interrupt status */
285 kvmppc_update_int_pending(vcpu
, *pending
, old_pending
);
290 pfn_t
kvmppc_gfn_to_pfn(struct kvm_vcpu
*vcpu
, gfn_t gfn
)
292 ulong mp_pa
= vcpu
->arch
.magic_page_pa
;
294 if (!(vcpu
->arch
.shared
->msr
& MSR_SF
))
295 mp_pa
= (uint32_t)mp_pa
;
297 /* Magic page override */
298 if (unlikely(mp_pa
) &&
299 unlikely(((gfn
<< PAGE_SHIFT
) & KVM_PAM
) ==
300 ((mp_pa
& PAGE_MASK
) & KVM_PAM
))) {
301 ulong shared_page
= ((ulong
)vcpu
->arch
.shared
) & PAGE_MASK
;
304 pfn
= (pfn_t
)virt_to_phys((void*)shared_page
) >> PAGE_SHIFT
;
305 get_page(pfn_to_page(pfn
));
309 return gfn_to_pfn(vcpu
->kvm
, gfn
);
312 static int kvmppc_xlate(struct kvm_vcpu
*vcpu
, ulong eaddr
, bool data
,
313 struct kvmppc_pte
*pte
)
315 int relocated
= (vcpu
->arch
.shared
->msr
& (data
? MSR_DR
: MSR_IR
));
319 r
= vcpu
->arch
.mmu
.xlate(vcpu
, eaddr
, pte
, data
);
322 pte
->raddr
= eaddr
& KVM_PAM
;
323 pte
->vpage
= VSID_REAL
| eaddr
>> 12;
324 pte
->may_read
= true;
325 pte
->may_write
= true;
326 pte
->may_execute
= true;
333 static hva_t
kvmppc_bad_hva(void)
338 static hva_t
kvmppc_pte_to_hva(struct kvm_vcpu
*vcpu
, struct kvmppc_pte
*pte
,
343 if (read
&& !pte
->may_read
)
346 if (!read
&& !pte
->may_write
)
349 hpage
= gfn_to_hva(vcpu
->kvm
, pte
->raddr
>> PAGE_SHIFT
);
350 if (kvm_is_error_hva(hpage
))
353 return hpage
| (pte
->raddr
& ~PAGE_MASK
);
355 return kvmppc_bad_hva();
358 int kvmppc_st(struct kvm_vcpu
*vcpu
, ulong
*eaddr
, int size
, void *ptr
,
361 struct kvmppc_pte pte
;
365 if (kvmppc_xlate(vcpu
, *eaddr
, data
, &pte
))
373 if (kvm_write_guest(vcpu
->kvm
, pte
.raddr
, ptr
, size
))
374 return EMULATE_DO_MMIO
;
379 int kvmppc_ld(struct kvm_vcpu
*vcpu
, ulong
*eaddr
, int size
, void *ptr
,
382 struct kvmppc_pte pte
;
387 if (kvmppc_xlate(vcpu
, *eaddr
, data
, &pte
))
392 hva
= kvmppc_pte_to_hva(vcpu
, &pte
, true);
393 if (kvm_is_error_hva(hva
))
396 if (copy_from_user(ptr
, (void __user
*)hva
, size
)) {
397 printk(KERN_INFO
"kvmppc_ld at 0x%lx failed\n", hva
);
406 return EMULATE_DO_MMIO
;
409 int kvm_arch_vcpu_setup(struct kvm_vcpu
*vcpu
)
414 int kvmppc_subarch_vcpu_init(struct kvm_vcpu
*vcpu
)
419 void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu
*vcpu
)
423 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
427 regs
->pc
= kvmppc_get_pc(vcpu
);
428 regs
->cr
= kvmppc_get_cr(vcpu
);
429 regs
->ctr
= kvmppc_get_ctr(vcpu
);
430 regs
->lr
= kvmppc_get_lr(vcpu
);
431 regs
->xer
= kvmppc_get_xer(vcpu
);
432 regs
->msr
= vcpu
->arch
.shared
->msr
;
433 regs
->srr0
= vcpu
->arch
.shared
->srr0
;
434 regs
->srr1
= vcpu
->arch
.shared
->srr1
;
435 regs
->pid
= vcpu
->arch
.pid
;
436 regs
->sprg0
= vcpu
->arch
.shared
->sprg0
;
437 regs
->sprg1
= vcpu
->arch
.shared
->sprg1
;
438 regs
->sprg2
= vcpu
->arch
.shared
->sprg2
;
439 regs
->sprg3
= vcpu
->arch
.shared
->sprg3
;
440 regs
->sprg4
= vcpu
->arch
.shared
->sprg4
;
441 regs
->sprg5
= vcpu
->arch
.shared
->sprg5
;
442 regs
->sprg6
= vcpu
->arch
.shared
->sprg6
;
443 regs
->sprg7
= vcpu
->arch
.shared
->sprg7
;
445 for (i
= 0; i
< ARRAY_SIZE(regs
->gpr
); i
++)
446 regs
->gpr
[i
] = kvmppc_get_gpr(vcpu
, i
);
451 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu
*vcpu
, struct kvm_regs
*regs
)
455 kvmppc_set_pc(vcpu
, regs
->pc
);
456 kvmppc_set_cr(vcpu
, regs
->cr
);
457 kvmppc_set_ctr(vcpu
, regs
->ctr
);
458 kvmppc_set_lr(vcpu
, regs
->lr
);
459 kvmppc_set_xer(vcpu
, regs
->xer
);
460 kvmppc_set_msr(vcpu
, regs
->msr
);
461 vcpu
->arch
.shared
->srr0
= regs
->srr0
;
462 vcpu
->arch
.shared
->srr1
= regs
->srr1
;
463 vcpu
->arch
.shared
->sprg0
= regs
->sprg0
;
464 vcpu
->arch
.shared
->sprg1
= regs
->sprg1
;
465 vcpu
->arch
.shared
->sprg2
= regs
->sprg2
;
466 vcpu
->arch
.shared
->sprg3
= regs
->sprg3
;
467 vcpu
->arch
.shared
->sprg4
= regs
->sprg4
;
468 vcpu
->arch
.shared
->sprg5
= regs
->sprg5
;
469 vcpu
->arch
.shared
->sprg6
= regs
->sprg6
;
470 vcpu
->arch
.shared
->sprg7
= regs
->sprg7
;
472 for (i
= 0; i
< ARRAY_SIZE(regs
->gpr
); i
++)
473 kvmppc_set_gpr(vcpu
, i
, regs
->gpr
[i
]);
478 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
483 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu
*vcpu
, struct kvm_fpu
*fpu
)
488 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu
*vcpu
, struct kvm_one_reg
*reg
)
491 union kvmppc_one_reg val
;
495 size
= one_reg_size(reg
->id
);
496 if (size
> sizeof(val
))
499 r
= kvmppc_get_one_reg(vcpu
, reg
->id
, &val
);
504 case KVM_REG_PPC_DAR
:
505 val
= get_reg_val(reg
->id
, vcpu
->arch
.shared
->dar
);
507 case KVM_REG_PPC_DSISR
:
508 val
= get_reg_val(reg
->id
, vcpu
->arch
.shared
->dsisr
);
510 case KVM_REG_PPC_FPR0
... KVM_REG_PPC_FPR31
:
511 i
= reg
->id
- KVM_REG_PPC_FPR0
;
512 val
= get_reg_val(reg
->id
, vcpu
->arch
.fpr
[i
]);
514 case KVM_REG_PPC_FPSCR
:
515 val
= get_reg_val(reg
->id
, vcpu
->arch
.fpscr
);
517 #ifdef CONFIG_ALTIVEC
518 case KVM_REG_PPC_VR0
... KVM_REG_PPC_VR31
:
519 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
523 val
.vval
= vcpu
->arch
.vr
[reg
->id
- KVM_REG_PPC_VR0
];
525 case KVM_REG_PPC_VSCR
:
526 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
530 val
= get_reg_val(reg
->id
, vcpu
->arch
.vscr
.u
[3]);
532 #endif /* CONFIG_ALTIVEC */
541 if (copy_to_user((char __user
*)(unsigned long)reg
->addr
, &val
, size
))
547 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu
*vcpu
, struct kvm_one_reg
*reg
)
550 union kvmppc_one_reg val
;
554 size
= one_reg_size(reg
->id
);
555 if (size
> sizeof(val
))
558 if (copy_from_user(&val
, (char __user
*)(unsigned long)reg
->addr
, size
))
561 r
= kvmppc_set_one_reg(vcpu
, reg
->id
, &val
);
566 case KVM_REG_PPC_DAR
:
567 vcpu
->arch
.shared
->dar
= set_reg_val(reg
->id
, val
);
569 case KVM_REG_PPC_DSISR
:
570 vcpu
->arch
.shared
->dsisr
= set_reg_val(reg
->id
, val
);
572 case KVM_REG_PPC_FPR0
... KVM_REG_PPC_FPR31
:
573 i
= reg
->id
- KVM_REG_PPC_FPR0
;
574 vcpu
->arch
.fpr
[i
] = set_reg_val(reg
->id
, val
);
576 case KVM_REG_PPC_FPSCR
:
577 vcpu
->arch
.fpscr
= set_reg_val(reg
->id
, val
);
579 #ifdef CONFIG_ALTIVEC
580 case KVM_REG_PPC_VR0
... KVM_REG_PPC_VR31
:
581 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
585 vcpu
->arch
.vr
[reg
->id
- KVM_REG_PPC_VR0
] = val
.vval
;
587 case KVM_REG_PPC_VSCR
:
588 if (!cpu_has_feature(CPU_FTR_ALTIVEC
)) {
592 vcpu
->arch
.vscr
.u
[3] = set_reg_val(reg
->id
, val
);
594 #endif /* CONFIG_ALTIVEC */
604 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu
*vcpu
,
605 struct kvm_translation
*tr
)
610 void kvmppc_decrementer_func(unsigned long data
)
612 struct kvm_vcpu
*vcpu
= (struct kvm_vcpu
*)data
;
614 kvmppc_core_queue_dec(vcpu
);