?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'moduleh/module.h-split'
[linux-2.6/next.git] / arch / powerpc / kvm / book3s_hv.c
blob4591e808f46e1bc424660ffd3b69511e7ec29966
1 /*
2 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
3 * Copyright (C) 2009. SUSE Linux Products GmbH. All rights reserved.
4 *
5 * Authors:
6 * Paul Mackerras <paulus@au1.ibm.com>
7 * Alexander Graf <agraf@suse.de>
8 * Kevin Wolf <mail@kevin-wolf.de>
9 *
10 * Description: KVM functions specific to running on Book 3S
11 * processors in hypervisor mode (specifically POWER7 and later).
12 *
13 * This file is derived from arch/powerpc/kvm/book3s.c,
14 * by Alexander Graf <agraf@suse.de>.
15 *
16 * This program is free software; you can redistribute it and/or modify
17 * it under the terms of the GNU General Public License, version 2, as
18 * published by the Free Software Foundation.
19 */
20
21 #include <linux/kvm_host.h>
22 #include <linux/err.h>
23 #include <linux/slab.h>
24 #include <linux/preempt.h>
25 #include <linux/sched.h>
26 #include <linux/delay.h>
27 #include <linux/export.h>
28 #include <linux/fs.h>
29 #include <linux/anon_inodes.h>
30 #include <linux/cpumask.h>
31 #include <linux/spinlock.h>
32 #include <linux/page-flags.h>
33
34 #include <asm/reg.h>
35 #include <asm/cputable.h>
36 #include <asm/cacheflush.h>
37 #include <asm/tlbflush.h>
38 #include <asm/uaccess.h>
39 #include <asm/io.h>
40 #include <asm/kvm_ppc.h>
41 #include <asm/kvm_book3s.h>
42 #include <asm/mmu_context.h>
43 #include <asm/lppaca.h>
44 #include <asm/processor.h>
45 #include <asm/cputhreads.h>
46 #include <asm/page.h>
47 #include <linux/gfp.h>
48 #include <linux/sched.h>
49 #include <linux/vmalloc.h>
50 #include <linux/highmem.h>
51
52 /*
53 * For now, limit memory to 64GB and require it to be large pages.
54 * This value is chosen because it makes the ram_pginfo array be
55 * 64kB in size, which is about as large as we want to be trying
56 * to allocate with kmalloc.
57 */
58 #define MAX_MEM_ORDER 36
59
60 #define LARGE_PAGE_ORDER 24 /* 16MB pages */
61
62 /* #define EXIT_DEBUG */
63 /* #define EXIT_DEBUG_SIMPLE */
64 /* #define EXIT_DEBUG_INT */
65
66 void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
67 {
68 local_paca->kvm_hstate.kvm_vcpu = vcpu;
69 local_paca->kvm_hstate.kvm_vcore = vcpu->arch.vcore;
70 }
71
72 void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
73 {
74 }
75
76 static void kvmppc_vcpu_blocked(struct kvm_vcpu *vcpu);
77 static void kvmppc_vcpu_unblocked(struct kvm_vcpu *vcpu);
78
79 void kvmppc_vcpu_block(struct kvm_vcpu *vcpu)
80 {
81 u64 now;
82 unsigned long dec_nsec;
83
84 now = get_tb();
85 if (now >= vcpu->arch.dec_expires && !kvmppc_core_pending_dec(vcpu))
86 kvmppc_core_queue_dec(vcpu);
87 if (vcpu->arch.pending_exceptions)
88 return;
89 if (vcpu->arch.dec_expires != ~(u64)0) {
90 dec_nsec = (vcpu->arch.dec_expires - now) * NSEC_PER_SEC /
91 tb_ticks_per_sec;
92 hrtimer_start(&vcpu->arch.dec_timer, ktime_set(0, dec_nsec),
93 HRTIMER_MODE_REL);
94 }
95
96 kvmppc_vcpu_blocked(vcpu);
97
98 kvm_vcpu_block(vcpu);
99 vcpu->stat.halt_wakeup++;
100
101 if (vcpu->arch.dec_expires != ~(u64)0)
102 hrtimer_try_to_cancel(&vcpu->arch.dec_timer);
103
104 kvmppc_vcpu_unblocked(vcpu);
105 }
106
107 void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr)
108 {
109 vcpu->arch.shregs.msr = msr;
110 }
111
112 void kvmppc_set_pvr(struct kvm_vcpu *vcpu, u32 pvr)
113 {
114 vcpu->arch.pvr = pvr;
115 }
116
117 void kvmppc_dump_regs(struct kvm_vcpu *vcpu)
118 {
119 int r;
120
121 pr_err("vcpu %p (%d):\n", vcpu, vcpu->vcpu_id);
122 pr_err("pc = %.16lx msr = %.16llx trap = %x\n",
123 vcpu->arch.pc, vcpu->arch.shregs.msr, vcpu->arch.trap);
124 for (r = 0; r < 16; ++r)
125 pr_err("r%2d = %.16lx r%d = %.16lx\n",
126 r, kvmppc_get_gpr(vcpu, r),
127 r+16, kvmppc_get_gpr(vcpu, r+16));
128 pr_err("ctr = %.16lx lr = %.16lx\n",
129 vcpu->arch.ctr, vcpu->arch.lr);
130 pr_err("srr0 = %.16llx srr1 = %.16llx\n",
131 vcpu->arch.shregs.srr0, vcpu->arch.shregs.srr1);
132 pr_err("sprg0 = %.16llx sprg1 = %.16llx\n",
133 vcpu->arch.shregs.sprg0, vcpu->arch.shregs.sprg1);
134 pr_err("sprg2 = %.16llx sprg3 = %.16llx\n",
135 vcpu->arch.shregs.sprg2, vcpu->arch.shregs.sprg3);
136 pr_err("cr = %.8x xer = %.16lx dsisr = %.8x\n",
137 vcpu->arch.cr, vcpu->arch.xer, vcpu->arch.shregs.dsisr);
138 pr_err("dar = %.16llx\n", vcpu->arch.shregs.dar);
139 pr_err("fault dar = %.16lx dsisr = %.8x\n",
140 vcpu->arch.fault_dar, vcpu->arch.fault_dsisr);
141 pr_err("SLB (%d entries):\n", vcpu->arch.slb_max);
142 for (r = 0; r < vcpu->arch.slb_max; ++r)
143 pr_err(" ESID = %.16llx VSID = %.16llx\n",
144 vcpu->arch.slb[r].orige, vcpu->arch.slb[r].origv);
145 pr_err("lpcr = %.16lx sdr1 = %.16lx last_inst = %.8x\n",
146 vcpu->kvm->arch.lpcr, vcpu->kvm->arch.sdr1,
147 vcpu->arch.last_inst);
148 }
149
150 struct kvm_vcpu *kvmppc_find_vcpu(struct kvm *kvm, int id)
151 {
152 int r;
153 struct kvm_vcpu *v, *ret = NULL;
154
155 mutex_lock(&kvm->lock);
156 kvm_for_each_vcpu(r, v, kvm) {
157 if (v->vcpu_id == id) {
158 ret = v;
159 break;
160 }
161 }
162 mutex_unlock(&kvm->lock);
163 return ret;
164 }
165
166 static void init_vpa(struct kvm_vcpu *vcpu, struct lppaca *vpa)
167 {
168 vpa->shared_proc = 1;
169 vpa->yield_count = 1;
170 }
171
172 static unsigned long do_h_register_vpa(struct kvm_vcpu *vcpu,
173 unsigned long flags,
174 unsigned long vcpuid, unsigned long vpa)
175 {
176 struct kvm *kvm = vcpu->kvm;
177 unsigned long pg_index, ra, len;
178 unsigned long pg_offset;
179 void *va;
180 struct kvm_vcpu *tvcpu;
181
182 tvcpu = kvmppc_find_vcpu(kvm, vcpuid);
183 if (!tvcpu)
184 return H_PARAMETER;
185
186 flags >>= 63 - 18;
187 flags &= 7;
188 if (flags == 0 || flags == 4)
189 return H_PARAMETER;
190 if (flags < 4) {
191 if (vpa & 0x7f)
192 return H_PARAMETER;
193 /* registering new area; convert logical addr to real */
194 pg_index = vpa >> kvm->arch.ram_porder;
195 pg_offset = vpa & (kvm->arch.ram_psize - 1);
196 if (pg_index >= kvm->arch.ram_npages)
197 return H_PARAMETER;
198 if (kvm->arch.ram_pginfo[pg_index].pfn == 0)
199 return H_PARAMETER;
200 ra = kvm->arch.ram_pginfo[pg_index].pfn << PAGE_SHIFT;
201 ra |= pg_offset;
202 va = __va(ra);
203 if (flags <= 1)
204 len = *(unsigned short *)(va + 4);
205 else
206 len = *(unsigned int *)(va + 4);
207 if (pg_offset + len > kvm->arch.ram_psize)
208 return H_PARAMETER;
209 switch (flags) {
210 case 1: /* register VPA */
211 if (len < 640)
212 return H_PARAMETER;
213 tvcpu->arch.vpa = va;
214 init_vpa(vcpu, va);
215 break;
216 case 2: /* register DTL */
217 if (len < 48)
218 return H_PARAMETER;
219 if (!tvcpu->arch.vpa)
220 return H_RESOURCE;
221 len -= len % 48;
222 tvcpu->arch.dtl = va;
223 tvcpu->arch.dtl_end = va + len;
224 break;
225 case 3: /* register SLB shadow buffer */
226 if (len < 8)
227 return H_PARAMETER;
228 if (!tvcpu->arch.vpa)
229 return H_RESOURCE;
230 tvcpu->arch.slb_shadow = va;
231 len = (len - 16) / 16;
232 tvcpu->arch.slb_shadow = va;
233 break;
234 }
235 } else {
236 switch (flags) {
237 case 5: /* unregister VPA */
238 if (tvcpu->arch.slb_shadow || tvcpu->arch.dtl)
239 return H_RESOURCE;
240 tvcpu->arch.vpa = NULL;
241 break;
242 case 6: /* unregister DTL */
243 tvcpu->arch.dtl = NULL;
244 break;
245 case 7: /* unregister SLB shadow buffer */
246 tvcpu->arch.slb_shadow = NULL;
247 break;
248 }
249 }
250 return H_SUCCESS;
251 }
252
253 int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu)
254 {
255 unsigned long req = kvmppc_get_gpr(vcpu, 3);
256 unsigned long target, ret = H_SUCCESS;
257 struct kvm_vcpu *tvcpu;
258
259 switch (req) {
260 case H_CEDE:
261 vcpu->arch.shregs.msr |= MSR_EE;
262 vcpu->arch.ceded = 1;
263 smp_mb();
264 if (!vcpu->arch.prodded)
265 kvmppc_vcpu_block(vcpu);
266 else
267 vcpu->arch.prodded = 0;
268 smp_mb();
269 vcpu->arch.ceded = 0;
270 break;
271 case H_PROD:
272 target = kvmppc_get_gpr(vcpu, 4);
273 tvcpu = kvmppc_find_vcpu(vcpu->kvm, target);
274 if (!tvcpu) {
275 ret = H_PARAMETER;
276 break;
277 }
278 tvcpu->arch.prodded = 1;
279 smp_mb();
280 if (vcpu->arch.ceded) {
281 if (waitqueue_active(&vcpu->wq)) {
282 wake_up_interruptible(&vcpu->wq);
283 vcpu->stat.halt_wakeup++;
284 }
285 }
286 break;
287 case H_CONFER:
288 break;
289 case H_REGISTER_VPA:
290 ret = do_h_register_vpa(vcpu, kvmppc_get_gpr(vcpu, 4),
291 kvmppc_get_gpr(vcpu, 5),
292 kvmppc_get_gpr(vcpu, 6));
293 break;
294 default:
295 return RESUME_HOST;
296 }
297 kvmppc_set_gpr(vcpu, 3, ret);
298 vcpu->arch.hcall_needed = 0;
299 return RESUME_GUEST;
300 }
301
302 static int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu,
303 struct task_struct *tsk)
304 {
305 int r = RESUME_HOST;
306
307 vcpu->stat.sum_exits++;
308
309 run->exit_reason = KVM_EXIT_UNKNOWN;
310 run->ready_for_interrupt_injection = 1;
311 switch (vcpu->arch.trap) {
312 /* We're good on these - the host merely wanted to get our attention */
313 case BOOK3S_INTERRUPT_HV_DECREMENTER:
314 vcpu->stat.dec_exits++;
315 r = RESUME_GUEST;
316 break;
317 case BOOK3S_INTERRUPT_EXTERNAL:
318 vcpu->stat.ext_intr_exits++;
319 r = RESUME_GUEST;
320 break;
321 case BOOK3S_INTERRUPT_PERFMON:
322 r = RESUME_GUEST;
323 break;
324 case BOOK3S_INTERRUPT_PROGRAM:
325 {
326 ulong flags;
327 /*
328 * Normally program interrupts are delivered directly
329 * to the guest by the hardware, but we can get here
330 * as a result of a hypervisor emulation interrupt
331 * (e40) getting turned into a 700 by BML RTAS.
332 */
333 flags = vcpu->arch.shregs.msr & 0x1f0000ull;
334 kvmppc_core_queue_program(vcpu, flags);
335 r = RESUME_GUEST;
336 break;
337 }
338 case BOOK3S_INTERRUPT_SYSCALL:
339 {
340 /* hcall - punt to userspace */
341 int i;
342
343 if (vcpu->arch.shregs.msr & MSR_PR) {
344 /* sc 1 from userspace - reflect to guest syscall */
345 kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_SYSCALL);
346 r = RESUME_GUEST;
347 break;
348 }
349 run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3);
350 for (i = 0; i < 9; ++i)
351 run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i);
352 run->exit_reason = KVM_EXIT_PAPR_HCALL;
353 vcpu->arch.hcall_needed = 1;
354 r = RESUME_HOST;
355 break;
356 }
357 /*
358 * We get these next two if the guest does a bad real-mode access,
359 * as we have enabled VRMA (virtualized real mode area) mode in the
360 * LPCR. We just generate an appropriate DSI/ISI to the guest.
361 */
362 case BOOK3S_INTERRUPT_H_DATA_STORAGE:
363 vcpu->arch.shregs.dsisr = vcpu->arch.fault_dsisr;
364 vcpu->arch.shregs.dar = vcpu->arch.fault_dar;
365 kvmppc_inject_interrupt(vcpu, BOOK3S_INTERRUPT_DATA_STORAGE, 0);
366 r = RESUME_GUEST;
367 break;
368 case BOOK3S_INTERRUPT_H_INST_STORAGE:
369 kvmppc_inject_interrupt(vcpu, BOOK3S_INTERRUPT_INST_STORAGE,
370 0x08000000);
371 r = RESUME_GUEST;
372 break;
373 /*
374 * This occurs if the guest executes an illegal instruction.
375 * We just generate a program interrupt to the guest, since
376 * we don't emulate any guest instructions at this stage.
377 */
378 case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
379 kvmppc_core_queue_program(vcpu, 0x80000);
380 r = RESUME_GUEST;
381 break;
382 default:
383 kvmppc_dump_regs(vcpu);
384 printk(KERN_EMERG "trap=0x%x | pc=0x%lx | msr=0x%llx\n",
385 vcpu->arch.trap, kvmppc_get_pc(vcpu),
386 vcpu->arch.shregs.msr);
387 r = RESUME_HOST;
388 BUG();
389 break;
390 }
391
392
393 if (!(r & RESUME_HOST)) {
394 /* To avoid clobbering exit_reason, only check for signals if
395 * we aren't already exiting to userspace for some other
396 * reason. */
397 if (signal_pending(tsk)) {
398 vcpu->stat.signal_exits++;
399 run->exit_reason = KVM_EXIT_INTR;
400 r = -EINTR;
401 } else {
402 kvmppc_core_deliver_interrupts(vcpu);
403 }
404 }
405
406 return r;
407 }
408
409 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
410 struct kvm_sregs *sregs)
411 {
412 int i;
413
414 sregs->pvr = vcpu->arch.pvr;
415
416 memset(sregs, 0, sizeof(struct kvm_sregs));
417 for (i = 0; i < vcpu->arch.slb_max; i++) {
418 sregs->u.s.ppc64.slb[i].slbe = vcpu->arch.slb[i].orige;
419 sregs->u.s.ppc64.slb[i].slbv = vcpu->arch.slb[i].origv;
420 }
421
422 return 0;
423 }
424
425 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
426 struct kvm_sregs *sregs)
427 {
428 int i, j;
429
430 kvmppc_set_pvr(vcpu, sregs->pvr);
431
432 j = 0;
433 for (i = 0; i < vcpu->arch.slb_nr; i++) {
434 if (sregs->u.s.ppc64.slb[i].slbe & SLB_ESID_V) {
435 vcpu->arch.slb[j].orige = sregs->u.s.ppc64.slb[i].slbe;
436 vcpu->arch.slb[j].origv = sregs->u.s.ppc64.slb[i].slbv;
437 ++j;
438 }
439 }
440 vcpu->arch.slb_max = j;
441
442 return 0;
443 }
444
445 int kvmppc_core_check_processor_compat(void)
446 {
447 if (cpu_has_feature(CPU_FTR_HVMODE))
448 return 0;
449 return -EIO;
450 }
451
452 struct kvm_vcpu *kvmppc_core_vcpu_create(struct kvm *kvm, unsigned int id)
453 {
454 struct kvm_vcpu *vcpu;
455 int err = -EINVAL;
456 int core;
457 struct kvmppc_vcore *vcore;
458
459 core = id / threads_per_core;
460 if (core >= KVM_MAX_VCORES)
461 goto out;
462
463 err = -ENOMEM;
464 vcpu = kzalloc(sizeof(struct kvm_vcpu), GFP_KERNEL);
465 if (!vcpu)
466 goto out;
467
468 err = kvm_vcpu_init(vcpu, kvm, id);
469 if (err)
470 goto free_vcpu;
471
472 vcpu->arch.shared = &vcpu->arch.shregs;
473 vcpu->arch.last_cpu = -1;
474 vcpu->arch.mmcr[0] = MMCR0_FC;
475 vcpu->arch.ctrl = CTRL_RUNLATCH;
476 /* default to host PVR, since we can't spoof it */
477 vcpu->arch.pvr = mfspr(SPRN_PVR);
478 kvmppc_set_pvr(vcpu, vcpu->arch.pvr);
479
480 kvmppc_mmu_book3s_hv_init(vcpu);
481
482 /*
483 * Some vcpus may start out in stopped state. If we initialize
484 * them to busy-in-host state they will stop other vcpus in the
485 * vcore from running. Instead we initialize them to blocked
486 * state, effectively considering them to be stopped until we
487 * see the first run ioctl for them.
488 */
489 vcpu->arch.state = KVMPPC_VCPU_BLOCKED;
490
491 init_waitqueue_head(&vcpu->arch.cpu_run);
492
493 mutex_lock(&kvm->lock);
494 vcore = kvm->arch.vcores[core];
495 if (!vcore) {
496 vcore = kzalloc(sizeof(struct kvmppc_vcore), GFP_KERNEL);
497 if (vcore) {
498 INIT_LIST_HEAD(&vcore->runnable_threads);
499 spin_lock_init(&vcore->lock);
500 }
501 kvm->arch.vcores[core] = vcore;
502 }
503 mutex_unlock(&kvm->lock);
504
505 if (!vcore)
506 goto free_vcpu;
507
508 spin_lock(&vcore->lock);
509 ++vcore->num_threads;
510 ++vcore->n_blocked;
511 spin_unlock(&vcore->lock);
512 vcpu->arch.vcore = vcore;
513
514 return vcpu;
515
516 free_vcpu:
517 kfree(vcpu);
518 out:
519 return ERR_PTR(err);
520 }
521
522 void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
523 {
524 kvm_vcpu_uninit(vcpu);
525 kfree(vcpu);
526 }
527
528 static void kvmppc_vcpu_blocked(struct kvm_vcpu *vcpu)
529 {
530 struct kvmppc_vcore *vc = vcpu->arch.vcore;
531
532 spin_lock(&vc->lock);
533 vcpu->arch.state = KVMPPC_VCPU_BLOCKED;
534 ++vc->n_blocked;
535 if (vc->n_runnable > 0 &&
536 vc->n_runnable + vc->n_blocked == vc->num_threads) {
537 vcpu = list_first_entry(&vc->runnable_threads, struct kvm_vcpu,
538 arch.run_list);
539 wake_up(&vcpu->arch.cpu_run);
540 }
541 spin_unlock(&vc->lock);
542 }
543
544 static void kvmppc_vcpu_unblocked(struct kvm_vcpu *vcpu)
545 {
546 struct kvmppc_vcore *vc = vcpu->arch.vcore;
547
548 spin_lock(&vc->lock);
549 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
550 --vc->n_blocked;
551 spin_unlock(&vc->lock);
552 }
553
554 extern int __kvmppc_vcore_entry(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu);
555 extern void xics_wake_cpu(int cpu);
556
557 static void kvmppc_remove_runnable(struct kvmppc_vcore *vc,
558 struct kvm_vcpu *vcpu)
559 {
560 struct kvm_vcpu *v;
561
562 if (vcpu->arch.state != KVMPPC_VCPU_RUNNABLE)
563 return;
564 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
565 --vc->n_runnable;
566 /* decrement the physical thread id of each following vcpu */
567 v = vcpu;
568 list_for_each_entry_continue(v, &vc->runnable_threads, arch.run_list)
569 --v->arch.ptid;
570 list_del(&vcpu->arch.run_list);
571 }
572
573 static void kvmppc_start_thread(struct kvm_vcpu *vcpu)
574 {
575 int cpu;
576 struct paca_struct *tpaca;
577 struct kvmppc_vcore *vc = vcpu->arch.vcore;
578
579 cpu = vc->pcpu + vcpu->arch.ptid;
580 tpaca = &paca[cpu];
581 tpaca->kvm_hstate.kvm_vcpu = vcpu;
582 tpaca->kvm_hstate.kvm_vcore = vc;
583 smp_wmb();
584 #ifdef CONFIG_PPC_ICP_NATIVE
585 if (vcpu->arch.ptid) {
586 tpaca->cpu_start = 0x80;
587 tpaca->kvm_hstate.in_guest = KVM_GUEST_MODE_GUEST;
588 wmb();
589 xics_wake_cpu(cpu);
590 ++vc->n_woken;
591 }
592 #endif
593 }
594
595 static void kvmppc_wait_for_nap(struct kvmppc_vcore *vc)
596 {
597 int i;
598
599 HMT_low();
600 i = 0;
601 while (vc->nap_count < vc->n_woken) {
602 if (++i >= 1000000) {
603 pr_err("kvmppc_wait_for_nap timeout %d %d\n",
604 vc->nap_count, vc->n_woken);
605 break;
606 }
607 cpu_relax();
608 }
609 HMT_medium();
610 }
611
612 /*
613 * Check that we are on thread 0 and that any other threads in
614 * this core are off-line.
615 */
616 static int on_primary_thread(void)
617 {
618 int cpu = smp_processor_id();
619 int thr = cpu_thread_in_core(cpu);
620
621 if (thr)
622 return 0;
623 while (++thr < threads_per_core)
624 if (cpu_online(cpu + thr))
625 return 0;
626 return 1;
627 }
628
629 /*
630 * Run a set of guest threads on a physical core.
631 * Called with vc->lock held.
632 */
633 static int kvmppc_run_core(struct kvmppc_vcore *vc)
634 {
635 struct kvm_vcpu *vcpu, *vnext;
636 long ret;
637 u64 now;
638
639 /* don't start if any threads have a signal pending */
640 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
641 if (signal_pending(vcpu->arch.run_task))
642 return 0;
643
644 /*
645 * Make sure we are running on thread 0, and that
646 * secondary threads are offline.
647 * XXX we should also block attempts to bring any
648 * secondary threads online.
649 */
650 if (threads_per_core > 1 && !on_primary_thread()) {
651 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
652 vcpu->arch.ret = -EBUSY;
653 goto out;
654 }
655
656 vc->n_woken = 0;
657 vc->nap_count = 0;
658 vc->entry_exit_count = 0;
659 vc->vcore_running = 1;
660 vc->in_guest = 0;
661 vc->pcpu = smp_processor_id();
662 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list)
663 kvmppc_start_thread(vcpu);
664 vcpu = list_first_entry(&vc->runnable_threads, struct kvm_vcpu,
665 arch.run_list);
666
667 spin_unlock(&vc->lock);
668
669 preempt_disable();
670 kvm_guest_enter();
671 __kvmppc_vcore_entry(NULL, vcpu);
672
673 /* wait for secondary threads to finish writing their state to memory */
674 spin_lock(&vc->lock);
675 if (vc->nap_count < vc->n_woken)
676 kvmppc_wait_for_nap(vc);
677 /* prevent other vcpu threads from doing kvmppc_start_thread() now */
678 vc->vcore_running = 2;
679 spin_unlock(&vc->lock);
680
681 /* make sure updates to secondary vcpu structs are visible now */
682 smp_mb();
683 kvm_guest_exit();
684
685 preempt_enable();
686 kvm_resched(vcpu);
687
688 now = get_tb();
689 list_for_each_entry(vcpu, &vc->runnable_threads, arch.run_list) {
690 /* cancel pending dec exception if dec is positive */
691 if (now < vcpu->arch.dec_expires &&
692 kvmppc_core_pending_dec(vcpu))
693 kvmppc_core_dequeue_dec(vcpu);
694 if (!vcpu->arch.trap) {
695 if (signal_pending(vcpu->arch.run_task)) {
696 vcpu->arch.kvm_run->exit_reason = KVM_EXIT_INTR;
697 vcpu->arch.ret = -EINTR;
698 }
699 continue; /* didn't get to run */
700 }
701 ret = kvmppc_handle_exit(vcpu->arch.kvm_run, vcpu,
702 vcpu->arch.run_task);
703 vcpu->arch.ret = ret;
704 vcpu->arch.trap = 0;
705 }
706
707 spin_lock(&vc->lock);
708 out:
709 vc->vcore_running = 0;
710 list_for_each_entry_safe(vcpu, vnext, &vc->runnable_threads,
711 arch.run_list) {
712 if (vcpu->arch.ret != RESUME_GUEST) {
713 kvmppc_remove_runnable(vc, vcpu);
714 wake_up(&vcpu->arch.cpu_run);
715 }
716 }
717
718 return 1;
719 }
720
721 static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu)
722 {
723 int ptid;
724 int wait_state;
725 struct kvmppc_vcore *vc;
726 DEFINE_WAIT(wait);
727
728 /* No need to go into the guest when all we do is going out */
729 if (signal_pending(current)) {
730 kvm_run->exit_reason = KVM_EXIT_INTR;
731 return -EINTR;
732 }
733
734 /* On PPC970, check that we have an RMA region */
735 if (!vcpu->kvm->arch.rma && cpu_has_feature(CPU_FTR_ARCH_201))
736 return -EPERM;
737
738 kvm_run->exit_reason = 0;
739 vcpu->arch.ret = RESUME_GUEST;
740 vcpu->arch.trap = 0;
741
742 flush_fp_to_thread(current);
743 flush_altivec_to_thread(current);
744 flush_vsx_to_thread(current);
745
746 /*
747 * Synchronize with other threads in this virtual core
748 */
749 vc = vcpu->arch.vcore;
750 spin_lock(&vc->lock);
751 /* This happens the first time this is called for a vcpu */
752 if (vcpu->arch.state == KVMPPC_VCPU_BLOCKED)
753 --vc->n_blocked;
754 vcpu->arch.state = KVMPPC_VCPU_RUNNABLE;
755 ptid = vc->n_runnable;
756 vcpu->arch.run_task = current;
757 vcpu->arch.kvm_run = kvm_run;
758 vcpu->arch.ptid = ptid;
759 list_add_tail(&vcpu->arch.run_list, &vc->runnable_threads);
760 ++vc->n_runnable;
761
762 wait_state = TASK_INTERRUPTIBLE;
763 while (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) {
764 if (signal_pending(current)) {
765 if (!vc->vcore_running) {
766 kvm_run->exit_reason = KVM_EXIT_INTR;
767 vcpu->arch.ret = -EINTR;
768 break;
769 }
770 /* have to wait for vcore to stop executing guest */
771 wait_state = TASK_UNINTERRUPTIBLE;
772 smp_send_reschedule(vc->pcpu);
773 }
774
775 if (!vc->vcore_running &&
776 vc->n_runnable + vc->n_blocked == vc->num_threads) {
777 /* we can run now */
778 if (kvmppc_run_core(vc))
779 continue;
780 }
781
782 if (vc->vcore_running == 1 && VCORE_EXIT_COUNT(vc) == 0)
783 kvmppc_start_thread(vcpu);
784
785 /* wait for other threads to come in, or wait for vcore */
786 prepare_to_wait(&vcpu->arch.cpu_run, &wait, wait_state);
787 spin_unlock(&vc->lock);
788 schedule();
789 finish_wait(&vcpu->arch.cpu_run, &wait);
790 spin_lock(&vc->lock);
791 }
792
793 if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE)
794 kvmppc_remove_runnable(vc, vcpu);
795 spin_unlock(&vc->lock);
796
797 return vcpu->arch.ret;
798 }
799
800 int kvmppc_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu)
801 {
802 int r;
803
804 do {
805 r = kvmppc_run_vcpu(run, vcpu);
806
807 if (run->exit_reason == KVM_EXIT_PAPR_HCALL &&
808 !(vcpu->arch.shregs.msr & MSR_PR)) {
809 r = kvmppc_pseries_do_hcall(vcpu);
810 kvmppc_core_deliver_interrupts(vcpu);
811 }
812 } while (r == RESUME_GUEST);
813 return r;
814 }
815
816 static long kvmppc_stt_npages(unsigned long window_size)
817 {
818 return ALIGN((window_size >> SPAPR_TCE_SHIFT)
819 * sizeof(u64), PAGE_SIZE) / PAGE_SIZE;
820 }
821
822 static void release_spapr_tce_table(struct kvmppc_spapr_tce_table *stt)
823 {
824 struct kvm *kvm = stt->kvm;
825 int i;
826
827 mutex_lock(&kvm->lock);
828 list_del(&stt->list);
829 for (i = 0; i < kvmppc_stt_npages(stt->window_size); i++)
830 __free_page(stt->pages[i]);
831 kfree(stt);
832 mutex_unlock(&kvm->lock);
833
834 kvm_put_kvm(kvm);
835 }
836
837 static int kvm_spapr_tce_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
838 {
839 struct kvmppc_spapr_tce_table *stt = vma->vm_file->private_data;
840 struct page *page;
841
842 if (vmf->pgoff >= kvmppc_stt_npages(stt->window_size))
843 return VM_FAULT_SIGBUS;
844
845 page = stt->pages[vmf->pgoff];
846 get_page(page);
847 vmf->page = page;
848 return 0;
849 }
850
851 static const struct vm_operations_struct kvm_spapr_tce_vm_ops = {
852 .fault = kvm_spapr_tce_fault,
853 };
854
855 static int kvm_spapr_tce_mmap(struct file *file, struct vm_area_struct *vma)
856 {
857 vma->vm_ops = &kvm_spapr_tce_vm_ops;
858 return 0;
859 }
860
861 static int kvm_spapr_tce_release(struct inode *inode, struct file *filp)
862 {
863 struct kvmppc_spapr_tce_table *stt = filp->private_data;
864
865 release_spapr_tce_table(stt);
866 return 0;
867 }
868
869 static struct file_operations kvm_spapr_tce_fops = {
870 .mmap = kvm_spapr_tce_mmap,
871 .release = kvm_spapr_tce_release,
872 };
873
874 long kvm_vm_ioctl_create_spapr_tce(struct kvm *kvm,
875 struct kvm_create_spapr_tce *args)
876 {
877 struct kvmppc_spapr_tce_table *stt = NULL;
878 long npages;
879 int ret = -ENOMEM;
880 int i;
881
882 /* Check this LIOBN hasn't been previously allocated */
883 list_for_each_entry(stt, &kvm->arch.spapr_tce_tables, list) {
884 if (stt->liobn == args->liobn)
885 return -EBUSY;
886 }
887
888 npages = kvmppc_stt_npages(args->window_size);
889
890 stt = kzalloc(sizeof(*stt) + npages* sizeof(struct page *),
891 GFP_KERNEL);
892 if (!stt)
893 goto fail;
894
895 stt->liobn = args->liobn;
896 stt->window_size = args->window_size;
897 stt->kvm = kvm;
898
899 for (i = 0; i < npages; i++) {
900 stt->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
901 if (!stt->pages[i])
902 goto fail;
903 }
904
905 kvm_get_kvm(kvm);
906
907 mutex_lock(&kvm->lock);
908 list_add(&stt->list, &kvm->arch.spapr_tce_tables);
909
910 mutex_unlock(&kvm->lock);
911
912 return anon_inode_getfd("kvm-spapr-tce", &kvm_spapr_tce_fops,
913 stt, O_RDWR);
914
915 fail:
916 if (stt) {
917 for (i = 0; i < npages; i++)
918 if (stt->pages[i])
919 __free_page(stt->pages[i]);
920
921 kfree(stt);
922 }
923 return ret;
924 }
925
926 /* Work out RMLS (real mode limit selector) field value for a given RMA size.
927 Assumes POWER7 or PPC970. */
928 static inline int lpcr_rmls(unsigned long rma_size)
929 {
930 switch (rma_size) {
931 case 32ul << 20: /* 32 MB */
932 if (cpu_has_feature(CPU_FTR_ARCH_206))
933 return 8; /* only supported on POWER7 */
934 return -1;
935 case 64ul << 20: /* 64 MB */
936 return 3;
937 case 128ul << 20: /* 128 MB */
938 return 7;
939 case 256ul << 20: /* 256 MB */
940 return 4;
941 case 1ul << 30: /* 1 GB */
942 return 2;
943 case 16ul << 30: /* 16 GB */
944 return 1;
945 case 256ul << 30: /* 256 GB */
946 return 0;
947 default:
948 return -1;
949 }
950 }
951
952 static int kvm_rma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
953 {
954 struct kvmppc_rma_info *ri = vma->vm_file->private_data;
955 struct page *page;
956
957 if (vmf->pgoff >= ri->npages)
958 return VM_FAULT_SIGBUS;
959
960 page = pfn_to_page(ri->base_pfn + vmf->pgoff);
961 get_page(page);
962 vmf->page = page;
963 return 0;
964 }
965
966 static const struct vm_operations_struct kvm_rma_vm_ops = {
967 .fault = kvm_rma_fault,
968 };
969
970 static int kvm_rma_mmap(struct file *file, struct vm_area_struct *vma)
971 {
972 vma->vm_flags |= VM_RESERVED;
973 vma->vm_ops = &kvm_rma_vm_ops;
974 return 0;
975 }
976
977 static int kvm_rma_release(struct inode *inode, struct file *filp)
978 {
979 struct kvmppc_rma_info *ri = filp->private_data;
980
981 kvm_release_rma(ri);
982 return 0;
983 }
984
985 static struct file_operations kvm_rma_fops = {
986 .mmap = kvm_rma_mmap,
987 .release = kvm_rma_release,
988 };
989
990 long kvm_vm_ioctl_allocate_rma(struct kvm *kvm, struct kvm_allocate_rma *ret)
991 {
992 struct kvmppc_rma_info *ri;
993 long fd;
994
995 ri = kvm_alloc_rma();
996 if (!ri)
997 return -ENOMEM;
998
999 fd = anon_inode_getfd("kvm-rma", &kvm_rma_fops, ri, O_RDWR);
1000 if (fd < 0)
1001 kvm_release_rma(ri);
1002
1003 ret->rma_size = ri->npages << PAGE_SHIFT;
1004 return fd;
1005 }
1006
1007 static struct page *hva_to_page(unsigned long addr)
1008 {
1009 struct page *page[1];
1010 int npages;
1011
1012 might_sleep();
1013
1014 npages = get_user_pages_fast(addr, 1, 1, page);
1015
1016 if (unlikely(npages != 1))
1017 return 0;
1018
1019 return page[0];
1020 }
1021
1022 int kvmppc_core_prepare_memory_region(struct kvm *kvm,
1023 struct kvm_userspace_memory_region *mem)
1024 {
1025 unsigned long psize, porder;
1026 unsigned long i, npages, totalpages;
1027 unsigned long pg_ix;
1028 struct kvmppc_pginfo *pginfo;
1029 unsigned long hva;
1030 struct kvmppc_rma_info *ri = NULL;
1031 struct page *page;
1032
1033 /* For now, only allow 16MB pages */
1034 porder = LARGE_PAGE_ORDER;
1035 psize = 1ul << porder;
1036 if ((mem->memory_size & (psize - 1)) ||
1037 (mem->guest_phys_addr & (psize - 1))) {
1038 pr_err("bad memory_size=%llx @ %llx\n",
1039 mem->memory_size, mem->guest_phys_addr);
1040 return -EINVAL;
1041 }
1042
1043 npages = mem->memory_size >> porder;
1044 totalpages = (mem->guest_phys_addr + mem->memory_size) >> porder;
1045
1046 /* More memory than we have space to track? */
1047 if (totalpages > (1ul << (MAX_MEM_ORDER - LARGE_PAGE_ORDER)))
1048 return -EINVAL;
1049
1050 /* Do we already have an RMA registered? */
1051 if (mem->guest_phys_addr == 0 && kvm->arch.rma)
1052 return -EINVAL;
1053
1054 if (totalpages > kvm->arch.ram_npages)
1055 kvm->arch.ram_npages = totalpages;
1056
1057 /* Is this one of our preallocated RMAs? */
1058 if (mem->guest_phys_addr == 0) {
1059 struct vm_area_struct *vma;
1060
1061 down_read(&current->mm->mmap_sem);
1062 vma = find_vma(current->mm, mem->userspace_addr);
1063 if (vma && vma->vm_file &&
1064 vma->vm_file->f_op == &kvm_rma_fops &&
1065 mem->userspace_addr == vma->vm_start)
1066 ri = vma->vm_file->private_data;
1067 up_read(&current->mm->mmap_sem);
1068 if (!ri && cpu_has_feature(CPU_FTR_ARCH_201)) {
1069 pr_err("CPU requires an RMO\n");
1070 return -EINVAL;
1071 }
1072 }
1073
1074 if (ri) {
1075 unsigned long rma_size;
1076 unsigned long lpcr;
1077 long rmls;
1078
1079 rma_size = ri->npages << PAGE_SHIFT;
1080 if (rma_size > mem->memory_size)
1081 rma_size = mem->memory_size;
1082 rmls = lpcr_rmls(rma_size);
1083 if (rmls < 0) {
1084 pr_err("Can't use RMA of 0x%lx bytes\n", rma_size);
1085 return -EINVAL;
1086 }
1087 atomic_inc(&ri->use_count);
1088 kvm->arch.rma = ri;
1089 kvm->arch.n_rma_pages = rma_size >> porder;
1090
1091 /* Update LPCR and RMOR */
1092 lpcr = kvm->arch.lpcr;
1093 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1094 /* PPC970; insert RMLS value (split field) in HID4 */
1095 lpcr &= ~((1ul << HID4_RMLS0_SH) |
1096 (3ul << HID4_RMLS2_SH));
1097 lpcr |= ((rmls >> 2) << HID4_RMLS0_SH) |
1098 ((rmls & 3) << HID4_RMLS2_SH);
1099 /* RMOR is also in HID4 */
1100 lpcr |= ((ri->base_pfn >> (26 - PAGE_SHIFT)) & 0xffff)
1101 << HID4_RMOR_SH;
1102 } else {
1103 /* POWER7 */
1104 lpcr &= ~(LPCR_VPM0 | LPCR_VRMA_L);
1105 lpcr |= rmls << LPCR_RMLS_SH;
1106 kvm->arch.rmor = kvm->arch.rma->base_pfn << PAGE_SHIFT;
1107 }
1108 kvm->arch.lpcr = lpcr;
1109 pr_info("Using RMO at %lx size %lx (LPCR = %lx)\n",
1110 ri->base_pfn << PAGE_SHIFT, rma_size, lpcr);
1111 }
1112
1113 pg_ix = mem->guest_phys_addr >> porder;
1114 pginfo = kvm->arch.ram_pginfo + pg_ix;
1115 for (i = 0; i < npages; ++i, ++pg_ix) {
1116 if (ri && pg_ix < kvm->arch.n_rma_pages) {
1117 pginfo[i].pfn = ri->base_pfn +
1118 (pg_ix << (porder - PAGE_SHIFT));
1119 continue;
1120 }
1121 hva = mem->userspace_addr + (i << porder);
1122 page = hva_to_page(hva);
1123 if (!page) {
1124 pr_err("oops, no pfn for hva %lx\n", hva);
1125 goto err;
1126 }
1127 /* Check it's a 16MB page */
1128 if (!PageHead(page) ||
1129 compound_order(page) != (LARGE_PAGE_ORDER - PAGE_SHIFT)) {
1130 pr_err("page at %lx isn't 16MB (o=%d)\n",
1131 hva, compound_order(page));
1132 goto err;
1133 }
1134 pginfo[i].pfn = page_to_pfn(page);
1135 }
1136
1137 return 0;
1138
1139 err:
1140 return -EINVAL;
1141 }
1142
1143 void kvmppc_core_commit_memory_region(struct kvm *kvm,
1144 struct kvm_userspace_memory_region *mem)
1145 {
1146 if (mem->guest_phys_addr == 0 && mem->memory_size != 0 &&
1147 !kvm->arch.rma)
1148 kvmppc_map_vrma(kvm, mem);
1149 }
1150
1151 int kvmppc_core_init_vm(struct kvm *kvm)
1152 {
1153 long r;
1154 unsigned long npages = 1ul << (MAX_MEM_ORDER - LARGE_PAGE_ORDER);
1155 long err = -ENOMEM;
1156 unsigned long lpcr;
1157
1158 /* Allocate hashed page table */
1159 r = kvmppc_alloc_hpt(kvm);
1160 if (r)
1161 return r;
1162
1163 INIT_LIST_HEAD(&kvm->arch.spapr_tce_tables);
1164
1165 kvm->arch.ram_pginfo = kzalloc(npages * sizeof(struct kvmppc_pginfo),
1166 GFP_KERNEL);
1167 if (!kvm->arch.ram_pginfo) {
1168 pr_err("kvmppc_core_init_vm: couldn't alloc %lu bytes\n",
1169 npages * sizeof(struct kvmppc_pginfo));
1170 goto out_free;
1171 }
1172
1173 kvm->arch.ram_npages = 0;
1174 kvm->arch.ram_psize = 1ul << LARGE_PAGE_ORDER;
1175 kvm->arch.ram_porder = LARGE_PAGE_ORDER;
1176 kvm->arch.rma = NULL;
1177 kvm->arch.n_rma_pages = 0;
1178
1179 kvm->arch.host_sdr1 = mfspr(SPRN_SDR1);
1180
1181 if (cpu_has_feature(CPU_FTR_ARCH_201)) {
1182 /* PPC970; HID4 is effectively the LPCR */
1183 unsigned long lpid = kvm->arch.lpid;
1184 kvm->arch.host_lpid = 0;
1185 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_HID4);
1186 lpcr &= ~((3 << HID4_LPID1_SH) | (0xful << HID4_LPID5_SH));
1187 lpcr |= ((lpid >> 4) << HID4_LPID1_SH) |
1188 ((lpid & 0xf) << HID4_LPID5_SH);
1189 } else {
1190 /* POWER7; init LPCR for virtual RMA mode */
1191 kvm->arch.host_lpid = mfspr(SPRN_LPID);
1192 kvm->arch.host_lpcr = lpcr = mfspr(SPRN_LPCR);
1193 lpcr &= LPCR_PECE | LPCR_LPES;
1194 lpcr |= (4UL << LPCR_DPFD_SH) | LPCR_HDICE |
1195 LPCR_VPM0 | LPCR_VRMA_L;
1196 }
1197 kvm->arch.lpcr = lpcr;
1198
1199 return 0;
1200
1201 out_free:
1202 kvmppc_free_hpt(kvm);
1203 return err;
1204 }
1205
1206 void kvmppc_core_destroy_vm(struct kvm *kvm)
1207 {
1208 struct kvmppc_pginfo *pginfo;
1209 unsigned long i;
1210
1211 if (kvm->arch.ram_pginfo) {
1212 pginfo = kvm->arch.ram_pginfo;
1213 kvm->arch.ram_pginfo = NULL;
1214 for (i = kvm->arch.n_rma_pages; i < kvm->arch.ram_npages; ++i)
1215 if (pginfo[i].pfn)
1216 put_page(pfn_to_page(pginfo[i].pfn));
1217 kfree(pginfo);
1218 }
1219 if (kvm->arch.rma) {
1220 kvm_release_rma(kvm->arch.rma);
1221 kvm->arch.rma = NULL;
1222 }
1223
1224 kvmppc_free_hpt(kvm);
1225 WARN_ON(!list_empty(&kvm->arch.spapr_tce_tables));
1226 }
1227
1228 /* These are stubs for now */
1229 void kvmppc_mmu_pte_pflush(struct kvm_vcpu *vcpu, ulong pa_start, ulong pa_end)
1230 {
1231 }
1232
1233 /* We don't need to emulate any privileged instructions or dcbz */
1234 int kvmppc_core_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu,
1235 unsigned int inst, int *advance)
1236 {
1237 return EMULATE_FAIL;
1238 }
1239
1240 int kvmppc_core_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, int rs)
1241 {
1242 return EMULATE_FAIL;
1243 }
1244
1245 int kvmppc_core_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, int rt)
1246 {
1247 return EMULATE_FAIL;
1248 }
1249
1250 static int kvmppc_book3s_hv_init(void)
1251 {
1252 int r;
1253
1254 r = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
1255
1256 if (r)
1257 return r;
1258
1259 r = kvmppc_mmu_hv_init();
1260
1261 return r;
1262 }
1263
1264 static void kvmppc_book3s_hv_exit(void)
1265 {
1266 kvm_exit();
1267 }
1268
1269 module_init(kvmppc_book3s_hv_init);
1270 module_exit(kvmppc_book3s_hv_exit);
linux-next