?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / arch / loongarch / kvm / vcpu.c
blobcab1818be68d852ac66e4abfb9f941b8236c7e93
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
4 */
5
6 #include <linux/kvm_host.h>
7 #include <linux/entry-kvm.h>
8 #include <asm/fpu.h>
9 #include <asm/lbt.h>
10 #include <asm/loongarch.h>
11 #include <asm/setup.h>
12 #include <asm/time.h>
13
14 #define CREATE_TRACE_POINTS
15 #include "trace.h"
16
17 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
18 KVM_GENERIC_VCPU_STATS(),
19 STATS_DESC_COUNTER(VCPU, int_exits),
20 STATS_DESC_COUNTER(VCPU, idle_exits),
21 STATS_DESC_COUNTER(VCPU, cpucfg_exits),
22 STATS_DESC_COUNTER(VCPU, signal_exits),
23 STATS_DESC_COUNTER(VCPU, hypercall_exits)
24 };
25
26 const struct kvm_stats_header kvm_vcpu_stats_header = {
27 .name_size = KVM_STATS_NAME_SIZE,
28 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
29 .id_offset = sizeof(struct kvm_stats_header),
30 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
31 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
32 sizeof(kvm_vcpu_stats_desc),
33 };
34
35 static inline void kvm_save_host_pmu(struct kvm_vcpu *vcpu)
36 {
37 struct kvm_context *context;
38
39 context = this_cpu_ptr(vcpu->kvm->arch.vmcs);
40 context->perf_cntr[0] = read_csr_perfcntr0();
41 context->perf_cntr[1] = read_csr_perfcntr1();
42 context->perf_cntr[2] = read_csr_perfcntr2();
43 context->perf_cntr[3] = read_csr_perfcntr3();
44 context->perf_ctrl[0] = write_csr_perfctrl0(0);
45 context->perf_ctrl[1] = write_csr_perfctrl1(0);
46 context->perf_ctrl[2] = write_csr_perfctrl2(0);
47 context->perf_ctrl[3] = write_csr_perfctrl3(0);
48 }
49
50 static inline void kvm_restore_host_pmu(struct kvm_vcpu *vcpu)
51 {
52 struct kvm_context *context;
53
54 context = this_cpu_ptr(vcpu->kvm->arch.vmcs);
55 write_csr_perfcntr0(context->perf_cntr[0]);
56 write_csr_perfcntr1(context->perf_cntr[1]);
57 write_csr_perfcntr2(context->perf_cntr[2]);
58 write_csr_perfcntr3(context->perf_cntr[3]);
59 write_csr_perfctrl0(context->perf_ctrl[0]);
60 write_csr_perfctrl1(context->perf_ctrl[1]);
61 write_csr_perfctrl2(context->perf_ctrl[2]);
62 write_csr_perfctrl3(context->perf_ctrl[3]);
63 }
64
65
66 static inline void kvm_save_guest_pmu(struct kvm_vcpu *vcpu)
67 {
68 struct loongarch_csrs *csr = vcpu->arch.csr;
69
70 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0);
71 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1);
72 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2);
73 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3);
74 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0);
75 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1);
76 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2);
77 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
78 }
79
80 static inline void kvm_restore_guest_pmu(struct kvm_vcpu *vcpu)
81 {
82 struct loongarch_csrs *csr = vcpu->arch.csr;
83
84 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0);
85 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1);
86 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2);
87 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3);
88 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0);
89 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1);
90 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2);
91 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
92 }
93
94 static int kvm_own_pmu(struct kvm_vcpu *vcpu)
95 {
96 unsigned long val;
97
98 if (!kvm_guest_has_pmu(&vcpu->arch))
99 return -EINVAL;
100
101 kvm_save_host_pmu(vcpu);
102
103 /* Set PM0-PM(num) to guest */
104 val = read_csr_gcfg() & ~CSR_GCFG_GPERF;
105 val |= (kvm_get_pmu_num(&vcpu->arch) + 1) << CSR_GCFG_GPERF_SHIFT;
106 write_csr_gcfg(val);
107
108 kvm_restore_guest_pmu(vcpu);
109
110 return 0;
111 }
112
113 static void kvm_lose_pmu(struct kvm_vcpu *vcpu)
114 {
115 unsigned long val;
116 struct loongarch_csrs *csr = vcpu->arch.csr;
117
118 if (!(vcpu->arch.aux_inuse & KVM_LARCH_PMU))
119 return;
120
121 kvm_save_guest_pmu(vcpu);
122
123 /* Disable pmu access from guest */
124 write_csr_gcfg(read_csr_gcfg() & ~CSR_GCFG_GPERF);
125
126 /*
127 * Clear KVM_LARCH_PMU if the guest is not using PMU CSRs when
128 * exiting the guest, so that the next time trap into the guest.
129 * We don't need to deal with PMU CSRs contexts.
130 */
131 val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0);
132 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1);
133 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2);
134 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
135 if (!(val & KVM_PMU_EVENT_ENABLED))
136 vcpu->arch.aux_inuse &= ~KVM_LARCH_PMU;
137
138 kvm_restore_host_pmu(vcpu);
139 }
140
141 static void kvm_restore_pmu(struct kvm_vcpu *vcpu)
142 {
143 if ((vcpu->arch.aux_inuse & KVM_LARCH_PMU))
144 kvm_make_request(KVM_REQ_PMU, vcpu);
145 }
146
147 static void kvm_check_pmu(struct kvm_vcpu *vcpu)
148 {
149 if (kvm_check_request(KVM_REQ_PMU, vcpu)) {
150 kvm_own_pmu(vcpu);
151 vcpu->arch.aux_inuse |= KVM_LARCH_PMU;
152 }
153 }
154
155 static void kvm_update_stolen_time(struct kvm_vcpu *vcpu)
156 {
157 u32 version;
158 u64 steal;
159 gpa_t gpa;
160 struct kvm_memslots *slots;
161 struct kvm_steal_time __user *st;
162 struct gfn_to_hva_cache *ghc;
163
164 ghc = &vcpu->arch.st.cache;
165 gpa = vcpu->arch.st.guest_addr;
166 if (!(gpa & KVM_STEAL_PHYS_VALID))
167 return;
168
169 gpa &= KVM_STEAL_PHYS_MASK;
170 slots = kvm_memslots(vcpu->kvm);
171 if (slots->generation != ghc->generation || gpa != ghc->gpa) {
172 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) {
173 ghc->gpa = INVALID_GPA;
174 return;
175 }
176 }
177
178 st = (struct kvm_steal_time __user *)ghc->hva;
179 unsafe_get_user(version, &st->version, out);
180 if (version & 1)
181 version += 1; /* first time write, random junk */
182
183 version += 1;
184 unsafe_put_user(version, &st->version, out);
185 smp_wmb();
186
187 unsafe_get_user(steal, &st->steal, out);
188 steal += current->sched_info.run_delay - vcpu->arch.st.last_steal;
189 vcpu->arch.st.last_steal = current->sched_info.run_delay;
190 unsafe_put_user(steal, &st->steal, out);
191
192 smp_wmb();
193 version += 1;
194 unsafe_put_user(version, &st->version, out);
195 out:
196 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
197 }
198
199 /*
200 * kvm_check_requests - check and handle pending vCPU requests
201 *
202 * Return: RESUME_GUEST if we should enter the guest
203 * RESUME_HOST if we should exit to userspace
204 */
205 static int kvm_check_requests(struct kvm_vcpu *vcpu)
206 {
207 if (!kvm_request_pending(vcpu))
208 return RESUME_GUEST;
209
210 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
211 vcpu->arch.vpid = 0; /* Drop vpid for this vCPU */
212
213 if (kvm_dirty_ring_check_request(vcpu))
214 return RESUME_HOST;
215
216 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu))
217 kvm_update_stolen_time(vcpu);
218
219 return RESUME_GUEST;
220 }
221
222 static void kvm_late_check_requests(struct kvm_vcpu *vcpu)
223 {
224 lockdep_assert_irqs_disabled();
225 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GPA, vcpu))
226 if (vcpu->arch.flush_gpa != INVALID_GPA) {
227 kvm_flush_tlb_gpa(vcpu, vcpu->arch.flush_gpa);
228 vcpu->arch.flush_gpa = INVALID_GPA;
229 }
230 }
231
232 /*
233 * Check and handle pending signal and vCPU requests etc
234 * Run with irq enabled and preempt enabled
235 *
236 * Return: RESUME_GUEST if we should enter the guest
237 * RESUME_HOST if we should exit to userspace
238 * < 0 if we should exit to userspace, where the return value
239 * indicates an error
240 */
241 static int kvm_enter_guest_check(struct kvm_vcpu *vcpu)
242 {
243 int ret;
244
245 /*
246 * Check conditions before entering the guest
247 */
248 ret = xfer_to_guest_mode_handle_work(vcpu);
249 if (ret < 0)
250 return ret;
251
252 ret = kvm_check_requests(vcpu);
253
254 return ret;
255 }
256
257 /*
258 * Called with irq enabled
259 *
260 * Return: RESUME_GUEST if we should enter the guest, and irq disabled
261 * Others if we should exit to userspace
262 */
263 static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu)
264 {
265 int ret;
266
267 do {
268 ret = kvm_enter_guest_check(vcpu);
269 if (ret != RESUME_GUEST)
270 break;
271
272 /*
273 * Handle vcpu timer, interrupts, check requests and
274 * check vmid before vcpu enter guest
275 */
276 local_irq_disable();
277 kvm_deliver_intr(vcpu);
278 kvm_deliver_exception(vcpu);
279 /* Make sure the vcpu mode has been written */
280 smp_store_mb(vcpu->mode, IN_GUEST_MODE);
281 kvm_check_vpid(vcpu);
282 kvm_check_pmu(vcpu);
283
284 /*
285 * Called after function kvm_check_vpid()
286 * Since it updates CSR.GSTAT used by kvm_flush_tlb_gpa(),
287 * and it may also clear KVM_REQ_TLB_FLUSH_GPA pending bit
288 */
289 kvm_late_check_requests(vcpu);
290 vcpu->arch.host_eentry = csr_read64(LOONGARCH_CSR_EENTRY);
291 /* Clear KVM_LARCH_SWCSR_LATEST as CSR will change when enter guest */
292 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST;
293
294 if (kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending()) {
295 /* make sure the vcpu mode has been written */
296 smp_store_mb(vcpu->mode, OUTSIDE_GUEST_MODE);
297 local_irq_enable();
298 ret = -EAGAIN;
299 }
300 } while (ret != RESUME_GUEST);
301
302 return ret;
303 }
304
305 /*
306 * Return 1 for resume guest and "<= 0" for resume host.
307 */
308 static int kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu)
309 {
310 int ret = RESUME_GUEST;
311 unsigned long estat = vcpu->arch.host_estat;
312 u32 intr = estat & 0x1fff; /* Ignore NMI */
313 u32 ecode = (estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT;
314
315 vcpu->mode = OUTSIDE_GUEST_MODE;
316
317 /* Set a default exit reason */
318 run->exit_reason = KVM_EXIT_UNKNOWN;
319
320 kvm_lose_pmu(vcpu);
321
322 guest_timing_exit_irqoff();
323 guest_state_exit_irqoff();
324 local_irq_enable();
325
326 trace_kvm_exit(vcpu, ecode);
327 if (ecode) {
328 ret = kvm_handle_fault(vcpu, ecode);
329 } else {
330 WARN(!intr, "vm exiting with suspicious irq\n");
331 ++vcpu->stat.int_exits;
332 }
333
334 if (ret == RESUME_GUEST)
335 ret = kvm_pre_enter_guest(vcpu);
336
337 if (ret != RESUME_GUEST) {
338 local_irq_disable();
339 return ret;
340 }
341
342 guest_timing_enter_irqoff();
343 guest_state_enter_irqoff();
344 trace_kvm_reenter(vcpu);
345
346 return RESUME_GUEST;
347 }
348
349 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
350 {
351 return !!(vcpu->arch.irq_pending) &&
352 vcpu->arch.mp_state.mp_state == KVM_MP_STATE_RUNNABLE;
353 }
354
355 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
356 {
357 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
358 }
359
360 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
361 {
362 return false;
363 }
364
365 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
366 {
367 return VM_FAULT_SIGBUS;
368 }
369
370 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
371 struct kvm_translation *tr)
372 {
373 return -EINVAL;
374 }
375
376 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
377 {
378 int ret;
379
380 /* Protect from TOD sync and vcpu_load/put() */
381 preempt_disable();
382 ret = kvm_pending_timer(vcpu) ||
383 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT) & (1 << INT_TI);
384 preempt_enable();
385
386 return ret;
387 }
388
389 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu)
390 {
391 int i;
392
393 kvm_debug("vCPU Register Dump:\n");
394 kvm_debug("\tPC = 0x%08lx\n", vcpu->arch.pc);
395 kvm_debug("\tExceptions: %08lx\n", vcpu->arch.irq_pending);
396
397 for (i = 0; i < 32; i += 4) {
398 kvm_debug("\tGPR%02d: %08lx %08lx %08lx %08lx\n", i,
399 vcpu->arch.gprs[i], vcpu->arch.gprs[i + 1],
400 vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]);
401 }
402
403 kvm_debug("\tCRMD: 0x%08lx, ESTAT: 0x%08lx\n",
404 kvm_read_hw_gcsr(LOONGARCH_CSR_CRMD),
405 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT));
406
407 kvm_debug("\tERA: 0x%08lx\n", kvm_read_hw_gcsr(LOONGARCH_CSR_ERA));
408
409 return 0;
410 }
411
412 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
413 struct kvm_mp_state *mp_state)
414 {
415 *mp_state = vcpu->arch.mp_state;
416
417 return 0;
418 }
419
420 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
421 struct kvm_mp_state *mp_state)
422 {
423 int ret = 0;
424
425 switch (mp_state->mp_state) {
426 case KVM_MP_STATE_RUNNABLE:
427 vcpu->arch.mp_state = *mp_state;
428 break;
429 default:
430 ret = -EINVAL;
431 }
432
433 return ret;
434 }
435
436 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
437 struct kvm_guest_debug *dbg)
438 {
439 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK)
440 return -EINVAL;
441
442 if (dbg->control & KVM_GUESTDBG_ENABLE)
443 vcpu->guest_debug = dbg->control;
444 else
445 vcpu->guest_debug = 0;
446
447 return 0;
448 }
449
450 static inline int kvm_set_cpuid(struct kvm_vcpu *vcpu, u64 val)
451 {
452 int cpuid;
453 struct kvm_phyid_map *map;
454 struct loongarch_csrs *csr = vcpu->arch.csr;
455
456 if (val >= KVM_MAX_PHYID)
457 return -EINVAL;
458
459 map = vcpu->kvm->arch.phyid_map;
460 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID);
461
462 spin_lock(&vcpu->kvm->arch.phyid_map_lock);
463 if ((cpuid < KVM_MAX_PHYID) && map->phys_map[cpuid].enabled) {
464 /* Discard duplicated CPUID set operation */
465 if (cpuid == val) {
466 spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
467 return 0;
468 }
469
470 /*
471 * CPUID is already set before
472 * Forbid changing to a different CPUID at runtime
473 */
474 spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
475 return -EINVAL;
476 }
477
478 if (map->phys_map[val].enabled) {
479 /* Discard duplicated CPUID set operation */
480 if (vcpu == map->phys_map[val].vcpu) {
481 spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
482 return 0;
483 }
484
485 /*
486 * New CPUID is already set with other vcpu
487 * Forbid sharing the same CPUID between different vcpus
488 */
489 spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
490 return -EINVAL;
491 }
492
493 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, val);
494 map->phys_map[val].enabled = true;
495 map->phys_map[val].vcpu = vcpu;
496 spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
497
498 return 0;
499 }
500
501 static inline void kvm_drop_cpuid(struct kvm_vcpu *vcpu)
502 {
503 int cpuid;
504 struct kvm_phyid_map *map;
505 struct loongarch_csrs *csr = vcpu->arch.csr;
506
507 map = vcpu->kvm->arch.phyid_map;
508 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID);
509
510 if (cpuid >= KVM_MAX_PHYID)
511 return;
512
513 spin_lock(&vcpu->kvm->arch.phyid_map_lock);
514 if (map->phys_map[cpuid].enabled) {
515 map->phys_map[cpuid].vcpu = NULL;
516 map->phys_map[cpuid].enabled = false;
517 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID);
518 }
519 spin_unlock(&vcpu->kvm->arch.phyid_map_lock);
520 }
521
522 struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid)
523 {
524 struct kvm_phyid_map *map;
525
526 if (cpuid >= KVM_MAX_PHYID)
527 return NULL;
528
529 map = kvm->arch.phyid_map;
530 if (!map->phys_map[cpuid].enabled)
531 return NULL;
532
533 return map->phys_map[cpuid].vcpu;
534 }
535
536 static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val)
537 {
538 unsigned long gintc;
539 struct loongarch_csrs *csr = vcpu->arch.csr;
540
541 if (get_gcsr_flag(id) & INVALID_GCSR)
542 return -EINVAL;
543
544 if (id == LOONGARCH_CSR_ESTAT) {
545 preempt_disable();
546 vcpu_load(vcpu);
547 /*
548 * Sync pending interrupts into ESTAT so that interrupt
549 * remains during VM migration stage
550 */
551 kvm_deliver_intr(vcpu);
552 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST;
553 vcpu_put(vcpu);
554 preempt_enable();
555
556 /* ESTAT IP0~IP7 get from GINTC */
557 gintc = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_GINTC) & 0xff;
558 *val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_ESTAT) | (gintc << 2);
559 return 0;
560 }
561
562 /*
563 * Get software CSR state since software state is consistent
564 * with hardware for synchronous ioctl
565 */
566 *val = kvm_read_sw_gcsr(csr, id);
567
568 return 0;
569 }
570
571 static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val)
572 {
573 int ret = 0, gintc;
574 struct loongarch_csrs *csr = vcpu->arch.csr;
575
576 if (get_gcsr_flag(id) & INVALID_GCSR)
577 return -EINVAL;
578
579 if (id == LOONGARCH_CSR_CPUID)
580 return kvm_set_cpuid(vcpu, val);
581
582 if (id == LOONGARCH_CSR_ESTAT) {
583 /* ESTAT IP0~IP7 inject through GINTC */
584 gintc = (val >> 2) & 0xff;
585 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_GINTC, gintc);
586
587 gintc = val & ~(0xffUL << 2);
588 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_ESTAT, gintc);
589
590 return ret;
591 }
592
593 kvm_write_sw_gcsr(csr, id, val);
594
595 /*
596 * After modifying the PMU CSR register value of the vcpu.
597 * If the PMU CSRs are used, we need to set KVM_REQ_PMU.
598 */
599 if (id >= LOONGARCH_CSR_PERFCTRL0 && id <= LOONGARCH_CSR_PERFCNTR3) {
600 unsigned long val;
601
602 val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0) |
603 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1) |
604 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2) |
605 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3);
606
607 if (val & KVM_PMU_EVENT_ENABLED)
608 kvm_make_request(KVM_REQ_PMU, vcpu);
609 }
610
611 return ret;
612 }
613
614 static int _kvm_get_cpucfg_mask(int id, u64 *v)
615 {
616 if (id < 0 || id >= KVM_MAX_CPUCFG_REGS)
617 return -EINVAL;
618
619 switch (id) {
620 case LOONGARCH_CPUCFG0:
621 *v = GENMASK(31, 0);
622 return 0;
623 case LOONGARCH_CPUCFG1:
624 /* CPUCFG1_MSGINT is not supported by KVM */
625 *v = GENMASK(25, 0);
626 return 0;
627 case LOONGARCH_CPUCFG2:
628 /* CPUCFG2 features unconditionally supported by KVM */
629 *v = CPUCFG2_FP | CPUCFG2_FPSP | CPUCFG2_FPDP |
630 CPUCFG2_FPVERS | CPUCFG2_LLFTP | CPUCFG2_LLFTPREV |
631 CPUCFG2_LSPW | CPUCFG2_LAM;
632 /*
633 * For the ISA extensions listed below, if one is supported
634 * by the host, then it is also supported by KVM.
635 */
636 if (cpu_has_lsx)
637 *v |= CPUCFG2_LSX;
638 if (cpu_has_lasx)
639 *v |= CPUCFG2_LASX;
640 if (cpu_has_lbt_x86)
641 *v |= CPUCFG2_X86BT;
642 if (cpu_has_lbt_arm)
643 *v |= CPUCFG2_ARMBT;
644 if (cpu_has_lbt_mips)
645 *v |= CPUCFG2_MIPSBT;
646
647 return 0;
648 case LOONGARCH_CPUCFG3:
649 *v = GENMASK(16, 0);
650 return 0;
651 case LOONGARCH_CPUCFG4:
652 case LOONGARCH_CPUCFG5:
653 *v = GENMASK(31, 0);
654 return 0;
655 case LOONGARCH_CPUCFG6:
656 if (cpu_has_pmp)
657 *v = GENMASK(14, 0);
658 else
659 *v = 0;
660 return 0;
661 case LOONGARCH_CPUCFG16:
662 *v = GENMASK(16, 0);
663 return 0;
664 case LOONGARCH_CPUCFG17 ... LOONGARCH_CPUCFG20:
665 *v = GENMASK(30, 0);
666 return 0;
667 default:
668 /*
669 * CPUCFG bits should be zero if reserved by HW or not
670 * supported by KVM.
671 */
672 *v = 0;
673 return 0;
674 }
675 }
676
677 static int kvm_check_cpucfg(int id, u64 val)
678 {
679 int ret;
680 u64 mask = 0;
681
682 ret = _kvm_get_cpucfg_mask(id, &mask);
683 if (ret)
684 return ret;
685
686 if (val & ~mask)
687 /* Unsupported features and/or the higher 32 bits should not be set */
688 return -EINVAL;
689
690 switch (id) {
691 case LOONGARCH_CPUCFG2:
692 if (!(val & CPUCFG2_LLFTP))
693 /* Guests must have a constant timer */
694 return -EINVAL;
695 if ((val & CPUCFG2_FP) && (!(val & CPUCFG2_FPSP) || !(val & CPUCFG2_FPDP)))
696 /* Single and double float point must both be set when FP is enabled */
697 return -EINVAL;
698 if ((val & CPUCFG2_LSX) && !(val & CPUCFG2_FP))
699 /* LSX architecturally implies FP but val does not satisfy that */
700 return -EINVAL;
701 if ((val & CPUCFG2_LASX) && !(val & CPUCFG2_LSX))
702 /* LASX architecturally implies LSX and FP but val does not satisfy that */
703 return -EINVAL;
704 return 0;
705 case LOONGARCH_CPUCFG6:
706 if (val & CPUCFG6_PMP) {
707 u32 host = read_cpucfg(LOONGARCH_CPUCFG6);
708 if ((val & CPUCFG6_PMBITS) != (host & CPUCFG6_PMBITS))
709 return -EINVAL;
710 if ((val & CPUCFG6_PMNUM) > (host & CPUCFG6_PMNUM))
711 return -EINVAL;
712 if ((val & CPUCFG6_UPM) && !(host & CPUCFG6_UPM))
713 return -EINVAL;
714 }
715 return 0;
716 default:
717 /*
718 * Values for the other CPUCFG IDs are not being further validated
719 * besides the mask check above.
720 */
721 return 0;
722 }
723 }
724
725 static int kvm_get_one_reg(struct kvm_vcpu *vcpu,
726 const struct kvm_one_reg *reg, u64 *v)
727 {
728 int id, ret = 0;
729 u64 type = reg->id & KVM_REG_LOONGARCH_MASK;
730
731 switch (type) {
732 case KVM_REG_LOONGARCH_CSR:
733 id = KVM_GET_IOC_CSR_IDX(reg->id);
734 ret = _kvm_getcsr(vcpu, id, v);
735 break;
736 case KVM_REG_LOONGARCH_CPUCFG:
737 id = KVM_GET_IOC_CPUCFG_IDX(reg->id);
738 if (id >= 0 && id < KVM_MAX_CPUCFG_REGS)
739 *v = vcpu->arch.cpucfg[id];
740 else
741 ret = -EINVAL;
742 break;
743 case KVM_REG_LOONGARCH_LBT:
744 if (!kvm_guest_has_lbt(&vcpu->arch))
745 return -ENXIO;
746
747 switch (reg->id) {
748 case KVM_REG_LOONGARCH_LBT_SCR0:
749 *v = vcpu->arch.lbt.scr0;
750 break;
751 case KVM_REG_LOONGARCH_LBT_SCR1:
752 *v = vcpu->arch.lbt.scr1;
753 break;
754 case KVM_REG_LOONGARCH_LBT_SCR2:
755 *v = vcpu->arch.lbt.scr2;
756 break;
757 case KVM_REG_LOONGARCH_LBT_SCR3:
758 *v = vcpu->arch.lbt.scr3;
759 break;
760 case KVM_REG_LOONGARCH_LBT_EFLAGS:
761 *v = vcpu->arch.lbt.eflags;
762 break;
763 case KVM_REG_LOONGARCH_LBT_FTOP:
764 *v = vcpu->arch.fpu.ftop;
765 break;
766 default:
767 ret = -EINVAL;
768 break;
769 }
770 break;
771 case KVM_REG_LOONGARCH_KVM:
772 switch (reg->id) {
773 case KVM_REG_LOONGARCH_COUNTER:
774 *v = drdtime() + vcpu->kvm->arch.time_offset;
775 break;
776 case KVM_REG_LOONGARCH_DEBUG_INST:
777 *v = INSN_HVCL | KVM_HCALL_SWDBG;
778 break;
779 default:
780 ret = -EINVAL;
781 break;
782 }
783 break;
784 default:
785 ret = -EINVAL;
786 break;
787 }
788
789 return ret;
790 }
791
792 static int kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
793 {
794 int ret = 0;
795 u64 v, size = reg->id & KVM_REG_SIZE_MASK;
796
797 switch (size) {
798 case KVM_REG_SIZE_U64:
799 ret = kvm_get_one_reg(vcpu, reg, &v);
800 if (ret)
801 return ret;
802 ret = put_user(v, (u64 __user *)(long)reg->addr);
803 break;
804 default:
805 ret = -EINVAL;
806 break;
807 }
808
809 return ret;
810 }
811
812 static int kvm_set_one_reg(struct kvm_vcpu *vcpu,
813 const struct kvm_one_reg *reg, u64 v)
814 {
815 int id, ret = 0;
816 u64 type = reg->id & KVM_REG_LOONGARCH_MASK;
817
818 switch (type) {
819 case KVM_REG_LOONGARCH_CSR:
820 id = KVM_GET_IOC_CSR_IDX(reg->id);
821 ret = _kvm_setcsr(vcpu, id, v);
822 break;
823 case KVM_REG_LOONGARCH_CPUCFG:
824 id = KVM_GET_IOC_CPUCFG_IDX(reg->id);
825 ret = kvm_check_cpucfg(id, v);
826 if (ret)
827 break;
828 vcpu->arch.cpucfg[id] = (u32)v;
829 if (id == LOONGARCH_CPUCFG6)
830 vcpu->arch.max_pmu_csrid =
831 LOONGARCH_CSR_PERFCTRL0 + 2 * kvm_get_pmu_num(&vcpu->arch) + 1;
832 break;
833 case KVM_REG_LOONGARCH_LBT:
834 if (!kvm_guest_has_lbt(&vcpu->arch))
835 return -ENXIO;
836
837 switch (reg->id) {
838 case KVM_REG_LOONGARCH_LBT_SCR0:
839 vcpu->arch.lbt.scr0 = v;
840 break;
841 case KVM_REG_LOONGARCH_LBT_SCR1:
842 vcpu->arch.lbt.scr1 = v;
843 break;
844 case KVM_REG_LOONGARCH_LBT_SCR2:
845 vcpu->arch.lbt.scr2 = v;
846 break;
847 case KVM_REG_LOONGARCH_LBT_SCR3:
848 vcpu->arch.lbt.scr3 = v;
849 break;
850 case KVM_REG_LOONGARCH_LBT_EFLAGS:
851 vcpu->arch.lbt.eflags = v;
852 break;
853 case KVM_REG_LOONGARCH_LBT_FTOP:
854 vcpu->arch.fpu.ftop = v;
855 break;
856 default:
857 ret = -EINVAL;
858 break;
859 }
860 break;
861 case KVM_REG_LOONGARCH_KVM:
862 switch (reg->id) {
863 case KVM_REG_LOONGARCH_COUNTER:
864 /*
865 * gftoffset is relative with board, not vcpu
866 * only set for the first time for smp system
867 */
868 if (vcpu->vcpu_id == 0)
869 vcpu->kvm->arch.time_offset = (signed long)(v - drdtime());
870 break;
871 case KVM_REG_LOONGARCH_VCPU_RESET:
872 vcpu->arch.st.guest_addr = 0;
873 memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending));
874 memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear));
875 break;
876 default:
877 ret = -EINVAL;
878 break;
879 }
880 break;
881 default:
882 ret = -EINVAL;
883 break;
884 }
885
886 return ret;
887 }
888
889 static int kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
890 {
891 int ret = 0;
892 u64 v, size = reg->id & KVM_REG_SIZE_MASK;
893
894 switch (size) {
895 case KVM_REG_SIZE_U64:
896 ret = get_user(v, (u64 __user *)(long)reg->addr);
897 if (ret)
898 return ret;
899 break;
900 default:
901 return -EINVAL;
902 }
903
904 return kvm_set_one_reg(vcpu, reg, v);
905 }
906
907 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
908 {
909 return -ENOIOCTLCMD;
910 }
911
912 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
913 {
914 return -ENOIOCTLCMD;
915 }
916
917 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
918 {
919 int i;
920
921 for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
922 regs->gpr[i] = vcpu->arch.gprs[i];
923
924 regs->pc = vcpu->arch.pc;
925
926 return 0;
927 }
928
929 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
930 {
931 int i;
932
933 for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++)
934 vcpu->arch.gprs[i] = regs->gpr[i];
935
936 vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */
937 vcpu->arch.pc = regs->pc;
938
939 return 0;
940 }
941
942 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
943 struct kvm_enable_cap *cap)
944 {
945 /* FPU is enabled by default, will support LSX/LASX later. */
946 return -EINVAL;
947 }
948
949 static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu,
950 struct kvm_device_attr *attr)
951 {
952 switch (attr->attr) {
953 case LOONGARCH_CPUCFG2:
954 case LOONGARCH_CPUCFG6:
955 return 0;
956 case CPUCFG_KVM_FEATURE:
957 return 0;
958 default:
959 return -ENXIO;
960 }
961
962 return -ENXIO;
963 }
964
965 static int kvm_loongarch_pvtime_has_attr(struct kvm_vcpu *vcpu,
966 struct kvm_device_attr *attr)
967 {
968 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)
969 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA)
970 return -ENXIO;
971
972 return 0;
973 }
974
975 static int kvm_loongarch_vcpu_has_attr(struct kvm_vcpu *vcpu,
976 struct kvm_device_attr *attr)
977 {
978 int ret = -ENXIO;
979
980 switch (attr->group) {
981 case KVM_LOONGARCH_VCPU_CPUCFG:
982 ret = kvm_loongarch_cpucfg_has_attr(vcpu, attr);
983 break;
984 case KVM_LOONGARCH_VCPU_PVTIME_CTRL:
985 ret = kvm_loongarch_pvtime_has_attr(vcpu, attr);
986 break;
987 default:
988 break;
989 }
990
991 return ret;
992 }
993
994 static int kvm_loongarch_cpucfg_get_attr(struct kvm_vcpu *vcpu,
995 struct kvm_device_attr *attr)
996 {
997 int ret = 0;
998 uint64_t val;
999 uint64_t __user *uaddr = (uint64_t __user *)attr->addr;
1000
1001 switch (attr->attr) {
1002 case 0 ... (KVM_MAX_CPUCFG_REGS - 1):
1003 ret = _kvm_get_cpucfg_mask(attr->attr, &val);
1004 if (ret)
1005 return ret;
1006 break;
1007 case CPUCFG_KVM_FEATURE:
1008 val = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK;
1009 break;
1010 default:
1011 return -ENXIO;
1012 }
1013
1014 put_user(val, uaddr);
1015
1016 return ret;
1017 }
1018
1019 static int kvm_loongarch_pvtime_get_attr(struct kvm_vcpu *vcpu,
1020 struct kvm_device_attr *attr)
1021 {
1022 u64 gpa;
1023 u64 __user *user = (u64 __user *)attr->addr;
1024
1025 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)
1026 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA)
1027 return -ENXIO;
1028
1029 gpa = vcpu->arch.st.guest_addr;
1030 if (put_user(gpa, user))
1031 return -EFAULT;
1032
1033 return 0;
1034 }
1035
1036 static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu,
1037 struct kvm_device_attr *attr)
1038 {
1039 int ret = -ENXIO;
1040
1041 switch (attr->group) {
1042 case KVM_LOONGARCH_VCPU_CPUCFG:
1043 ret = kvm_loongarch_cpucfg_get_attr(vcpu, attr);
1044 break;
1045 case KVM_LOONGARCH_VCPU_PVTIME_CTRL:
1046 ret = kvm_loongarch_pvtime_get_attr(vcpu, attr);
1047 break;
1048 default:
1049 break;
1050 }
1051
1052 return ret;
1053 }
1054
1055 static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu,
1056 struct kvm_device_attr *attr)
1057 {
1058 u64 val, valid;
1059 u64 __user *user = (u64 __user *)attr->addr;
1060 struct kvm *kvm = vcpu->kvm;
1061
1062 switch (attr->attr) {
1063 case CPUCFG_KVM_FEATURE:
1064 if (get_user(val, user))
1065 return -EFAULT;
1066
1067 valid = LOONGARCH_PV_FEAT_MASK;
1068 if (val & ~valid)
1069 return -EINVAL;
1070
1071 /* All vCPUs need set the same PV features */
1072 if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED)
1073 && ((kvm->arch.pv_features & valid) != val))
1074 return -EINVAL;
1075 kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED;
1076 return 0;
1077 default:
1078 return -ENXIO;
1079 }
1080 }
1081
1082 static int kvm_loongarch_pvtime_set_attr(struct kvm_vcpu *vcpu,
1083 struct kvm_device_attr *attr)
1084 {
1085 int idx, ret = 0;
1086 u64 gpa, __user *user = (u64 __user *)attr->addr;
1087 struct kvm *kvm = vcpu->kvm;
1088
1089 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)
1090 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA)
1091 return -ENXIO;
1092
1093 if (get_user(gpa, user))
1094 return -EFAULT;
1095
1096 if (gpa & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID))
1097 return -EINVAL;
1098
1099 if (!(gpa & KVM_STEAL_PHYS_VALID)) {
1100 vcpu->arch.st.guest_addr = gpa;
1101 return 0;
1102 }
1103
1104 /* Check the address is in a valid memslot */
1105 idx = srcu_read_lock(&kvm->srcu);
1106 if (kvm_is_error_hva(gfn_to_hva(kvm, gpa >> PAGE_SHIFT)))
1107 ret = -EINVAL;
1108 srcu_read_unlock(&kvm->srcu, idx);
1109
1110 if (!ret) {
1111 vcpu->arch.st.guest_addr = gpa;
1112 vcpu->arch.st.last_steal = current->sched_info.run_delay;
1113 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
1114 }
1115
1116 return ret;
1117 }
1118
1119 static int kvm_loongarch_vcpu_set_attr(struct kvm_vcpu *vcpu,
1120 struct kvm_device_attr *attr)
1121 {
1122 int ret = -ENXIO;
1123
1124 switch (attr->group) {
1125 case KVM_LOONGARCH_VCPU_CPUCFG:
1126 ret = kvm_loongarch_cpucfg_set_attr(vcpu, attr);
1127 break;
1128 case KVM_LOONGARCH_VCPU_PVTIME_CTRL:
1129 ret = kvm_loongarch_pvtime_set_attr(vcpu, attr);
1130 break;
1131 default:
1132 break;
1133 }
1134
1135 return ret;
1136 }
1137
1138 long kvm_arch_vcpu_ioctl(struct file *filp,
1139 unsigned int ioctl, unsigned long arg)
1140 {
1141 long r;
1142 struct kvm_device_attr attr;
1143 void __user *argp = (void __user *)arg;
1144 struct kvm_vcpu *vcpu = filp->private_data;
1145
1146 /*
1147 * Only software CSR should be modified
1148 *
1149 * If any hardware CSR register is modified, vcpu_load/vcpu_put pair
1150 * should be used. Since CSR registers owns by this vcpu, if switch
1151 * to other vcpus, other vcpus need reload CSR registers.
1152 *
1153 * If software CSR is modified, bit KVM_LARCH_HWCSR_USABLE should
1154 * be clear in vcpu->arch.aux_inuse, and vcpu_load will check
1155 * aux_inuse flag and reload CSR registers form software.
1156 */
1157
1158 switch (ioctl) {
1159 case KVM_SET_ONE_REG:
1160 case KVM_GET_ONE_REG: {
1161 struct kvm_one_reg reg;
1162
1163 r = -EFAULT;
1164 if (copy_from_user(&reg, argp, sizeof(reg)))
1165 break;
1166 if (ioctl == KVM_SET_ONE_REG) {
1167 r = kvm_set_reg(vcpu, &reg);
1168 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE;
1169 } else
1170 r = kvm_get_reg(vcpu, &reg);
1171 break;
1172 }
1173 case KVM_ENABLE_CAP: {
1174 struct kvm_enable_cap cap;
1175
1176 r = -EFAULT;
1177 if (copy_from_user(&cap, argp, sizeof(cap)))
1178 break;
1179 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
1180 break;
1181 }
1182 case KVM_HAS_DEVICE_ATTR: {
1183 r = -EFAULT;
1184 if (copy_from_user(&attr, argp, sizeof(attr)))
1185 break;
1186 r = kvm_loongarch_vcpu_has_attr(vcpu, &attr);
1187 break;
1188 }
1189 case KVM_GET_DEVICE_ATTR: {
1190 r = -EFAULT;
1191 if (copy_from_user(&attr, argp, sizeof(attr)))
1192 break;
1193 r = kvm_loongarch_vcpu_get_attr(vcpu, &attr);
1194 break;
1195 }
1196 case KVM_SET_DEVICE_ATTR: {
1197 r = -EFAULT;
1198 if (copy_from_user(&attr, argp, sizeof(attr)))
1199 break;
1200 r = kvm_loongarch_vcpu_set_attr(vcpu, &attr);
1201 break;
1202 }
1203 default:
1204 r = -ENOIOCTLCMD;
1205 break;
1206 }
1207
1208 return r;
1209 }
1210
1211 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1212 {
1213 int i = 0;
1214
1215 fpu->fcc = vcpu->arch.fpu.fcc;
1216 fpu->fcsr = vcpu->arch.fpu.fcsr;
1217 for (i = 0; i < NUM_FPU_REGS; i++)
1218 memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64);
1219
1220 return 0;
1221 }
1222
1223 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
1224 {
1225 int i = 0;
1226
1227 vcpu->arch.fpu.fcc = fpu->fcc;
1228 vcpu->arch.fpu.fcsr = fpu->fcsr;
1229 for (i = 0; i < NUM_FPU_REGS; i++)
1230 memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64);
1231
1232 return 0;
1233 }
1234
1235 #ifdef CONFIG_CPU_HAS_LBT
1236 int kvm_own_lbt(struct kvm_vcpu *vcpu)
1237 {
1238 if (!kvm_guest_has_lbt(&vcpu->arch))
1239 return -EINVAL;
1240
1241 preempt_disable();
1242 set_csr_euen(CSR_EUEN_LBTEN);
1243 _restore_lbt(&vcpu->arch.lbt);
1244 vcpu->arch.aux_inuse |= KVM_LARCH_LBT;
1245 preempt_enable();
1246
1247 return 0;
1248 }
1249
1250 static void kvm_lose_lbt(struct kvm_vcpu *vcpu)
1251 {
1252 preempt_disable();
1253 if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) {
1254 _save_lbt(&vcpu->arch.lbt);
1255 clear_csr_euen(CSR_EUEN_LBTEN);
1256 vcpu->arch.aux_inuse &= ~KVM_LARCH_LBT;
1257 }
1258 preempt_enable();
1259 }
1260
1261 static void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr)
1262 {
1263 /*
1264 * If TM is enabled, top register save/restore will
1265 * cause lbt exception, here enable lbt in advance
1266 */
1267 if (fcsr & FPU_CSR_TM)
1268 kvm_own_lbt(vcpu);
1269 }
1270
1271 static void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu)
1272 {
1273 if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
1274 if (vcpu->arch.aux_inuse & KVM_LARCH_LBT)
1275 return;
1276 kvm_check_fcsr(vcpu, read_fcsr(LOONGARCH_FCSR0));
1277 }
1278 }
1279 #else
1280 static inline void kvm_lose_lbt(struct kvm_vcpu *vcpu) { }
1281 static inline void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) { }
1282 static inline void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) { }
1283 #endif
1284
1285 /* Enable FPU and restore context */
1286 void kvm_own_fpu(struct kvm_vcpu *vcpu)
1287 {
1288 preempt_disable();
1289
1290 /*
1291 * Enable FPU for guest
1292 * Set FR and FRE according to guest context
1293 */
1294 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr);
1295 set_csr_euen(CSR_EUEN_FPEN);
1296
1297 kvm_restore_fpu(&vcpu->arch.fpu);
1298 vcpu->arch.aux_inuse |= KVM_LARCH_FPU;
1299 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU);
1300
1301 preempt_enable();
1302 }
1303
1304 #ifdef CONFIG_CPU_HAS_LSX
1305 /* Enable LSX and restore context */
1306 int kvm_own_lsx(struct kvm_vcpu *vcpu)
1307 {
1308 if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch))
1309 return -EINVAL;
1310
1311 preempt_disable();
1312
1313 /* Enable LSX for guest */
1314 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr);
1315 set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN);
1316 switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
1317 case KVM_LARCH_FPU:
1318 /*
1319 * Guest FPU state already loaded,
1320 * only restore upper LSX state
1321 */
1322 _restore_lsx_upper(&vcpu->arch.fpu);
1323 break;
1324 default:
1325 /* Neither FP or LSX already active,
1326 * restore full LSX state
1327 */
1328 kvm_restore_lsx(&vcpu->arch.fpu);
1329 break;
1330 }
1331
1332 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LSX);
1333 vcpu->arch.aux_inuse |= KVM_LARCH_LSX | KVM_LARCH_FPU;
1334 preempt_enable();
1335
1336 return 0;
1337 }
1338 #endif
1339
1340 #ifdef CONFIG_CPU_HAS_LASX
1341 /* Enable LASX and restore context */
1342 int kvm_own_lasx(struct kvm_vcpu *vcpu)
1343 {
1344 if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch) || !kvm_guest_has_lasx(&vcpu->arch))
1345 return -EINVAL;
1346
1347 preempt_disable();
1348
1349 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr);
1350 set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
1351 switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) {
1352 case KVM_LARCH_LSX:
1353 case KVM_LARCH_LSX | KVM_LARCH_FPU:
1354 /* Guest LSX state already loaded, only restore upper LASX state */
1355 _restore_lasx_upper(&vcpu->arch.fpu);
1356 break;
1357 case KVM_LARCH_FPU:
1358 /* Guest FP state already loaded, only restore upper LSX & LASX state */
1359 _restore_lsx_upper(&vcpu->arch.fpu);
1360 _restore_lasx_upper(&vcpu->arch.fpu);
1361 break;
1362 default:
1363 /* Neither FP or LSX already active, restore full LASX state */
1364 kvm_restore_lasx(&vcpu->arch.fpu);
1365 break;
1366 }
1367
1368 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX);
1369 vcpu->arch.aux_inuse |= KVM_LARCH_LASX | KVM_LARCH_LSX | KVM_LARCH_FPU;
1370 preempt_enable();
1371
1372 return 0;
1373 }
1374 #endif
1375
1376 /* Save context and disable FPU */
1377 void kvm_lose_fpu(struct kvm_vcpu *vcpu)
1378 {
1379 preempt_disable();
1380
1381 kvm_check_fcsr_alive(vcpu);
1382 if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) {
1383 kvm_save_lasx(&vcpu->arch.fpu);
1384 vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU | KVM_LARCH_LASX);
1385 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LASX);
1386
1387 /* Disable LASX & LSX & FPU */
1388 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN);
1389 } else if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) {
1390 kvm_save_lsx(&vcpu->arch.fpu);
1391 vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU);
1392 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX);
1393
1394 /* Disable LSX & FPU */
1395 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN);
1396 } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) {
1397 kvm_save_fpu(&vcpu->arch.fpu);
1398 vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU;
1399 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU);
1400
1401 /* Disable FPU */
1402 clear_csr_euen(CSR_EUEN_FPEN);
1403 }
1404 kvm_lose_lbt(vcpu);
1405
1406 preempt_enable();
1407 }
1408
1409 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
1410 {
1411 int intr = (int)irq->irq;
1412
1413 if (intr > 0)
1414 kvm_queue_irq(vcpu, intr);
1415 else if (intr < 0)
1416 kvm_dequeue_irq(vcpu, -intr);
1417 else {
1418 kvm_err("%s: invalid interrupt ioctl %d\n", __func__, irq->irq);
1419 return -EINVAL;
1420 }
1421
1422 kvm_vcpu_kick(vcpu);
1423
1424 return 0;
1425 }
1426
1427 long kvm_arch_vcpu_async_ioctl(struct file *filp,
1428 unsigned int ioctl, unsigned long arg)
1429 {
1430 void __user *argp = (void __user *)arg;
1431 struct kvm_vcpu *vcpu = filp->private_data;
1432
1433 if (ioctl == KVM_INTERRUPT) {
1434 struct kvm_interrupt irq;
1435
1436 if (copy_from_user(&irq, argp, sizeof(irq)))
1437 return -EFAULT;
1438
1439 kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, irq.irq);
1440
1441 return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
1442 }
1443
1444 return -ENOIOCTLCMD;
1445 }
1446
1447 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
1448 {
1449 return 0;
1450 }
1451
1452 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
1453 {
1454 unsigned long timer_hz;
1455 struct loongarch_csrs *csr;
1456
1457 vcpu->arch.vpid = 0;
1458 vcpu->arch.flush_gpa = INVALID_GPA;
1459
1460 hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED_HARD);
1461 vcpu->arch.swtimer.function = kvm_swtimer_wakeup;
1462
1463 vcpu->arch.handle_exit = kvm_handle_exit;
1464 vcpu->arch.guest_eentry = (unsigned long)kvm_loongarch_ops->exc_entry;
1465 vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL);
1466 if (!vcpu->arch.csr)
1467 return -ENOMEM;
1468
1469 /*
1470 * All kvm exceptions share one exception entry, and host <-> guest
1471 * switch also switch ECFG.VS field, keep host ECFG.VS info here.
1472 */
1473 vcpu->arch.host_ecfg = (read_csr_ecfg() & CSR_ECFG_VS);
1474
1475 /* Init */
1476 vcpu->arch.last_sched_cpu = -1;
1477
1478 /* Init ipi_state lock */
1479 spin_lock_init(&vcpu->arch.ipi_state.lock);
1480
1481 /*
1482 * Initialize guest register state to valid architectural reset state.
1483 */
1484 timer_hz = calc_const_freq();
1485 kvm_init_timer(vcpu, timer_hz);
1486
1487 /* Set Initialize mode for guest */
1488 csr = vcpu->arch.csr;
1489 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CRMD, CSR_CRMD_DA);
1490
1491 /* Set cpuid */
1492 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_TMID, vcpu->vcpu_id);
1493 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID);
1494
1495 /* Start with no pending virtual guest interrupts */
1496 csr->csrs[LOONGARCH_CSR_GINTC] = 0;
1497
1498 return 0;
1499 }
1500
1501 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
1502 {
1503 }
1504
1505 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
1506 {
1507 int cpu;
1508 struct kvm_context *context;
1509
1510 hrtimer_cancel(&vcpu->arch.swtimer);
1511 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache);
1512 kvm_drop_cpuid(vcpu);
1513 kfree(vcpu->arch.csr);
1514
1515 /*
1516 * If the vCPU is freed and reused as another vCPU, we don't want the
1517 * matching pointer wrongly hanging around in last_vcpu.
1518 */
1519 for_each_possible_cpu(cpu) {
1520 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
1521 if (context->last_vcpu == vcpu)
1522 context->last_vcpu = NULL;
1523 }
1524 }
1525
1526 static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1527 {
1528 bool migrated;
1529 struct kvm_context *context;
1530 struct loongarch_csrs *csr = vcpu->arch.csr;
1531
1532 /*
1533 * Have we migrated to a different CPU?
1534 * If so, any old guest TLB state may be stale.
1535 */
1536 migrated = (vcpu->arch.last_sched_cpu != cpu);
1537
1538 /*
1539 * Was this the last vCPU to run on this CPU?
1540 * If not, any old guest state from this vCPU will have been clobbered.
1541 */
1542 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu);
1543 if (migrated || (context->last_vcpu != vcpu))
1544 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE;
1545 context->last_vcpu = vcpu;
1546
1547 /* Restore timer state regardless */
1548 kvm_restore_timer(vcpu);
1549
1550 /* Control guest page CCA attribute */
1551 change_csr_gcfg(CSR_GCFG_MATC_MASK, CSR_GCFG_MATC_ROOT);
1552 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu);
1553
1554 /* Restore hardware PMU CSRs */
1555 kvm_restore_pmu(vcpu);
1556
1557 /* Don't bother restoring registers multiple times unless necessary */
1558 if (vcpu->arch.aux_inuse & KVM_LARCH_HWCSR_USABLE)
1559 return 0;
1560
1561 write_csr_gcntc((ulong)vcpu->kvm->arch.time_offset);
1562
1563 /* Restore guest CSR registers */
1564 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CRMD);
1565 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PRMD);
1566 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EUEN);
1567 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_MISC);
1568 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ECFG);
1569 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ERA);
1570 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADV);
1571 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADI);
1572 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EENTRY);
1573 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX);
1574 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI);
1575 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0);
1576 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1);
1577 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ASID);
1578 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDL);
1579 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDH);
1580 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0);
1581 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1);
1582 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE);
1583 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_RVACFG);
1584 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CPUID);
1585 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS0);
1586 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS1);
1587 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS2);
1588 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS3);
1589 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS4);
1590 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS5);
1591 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS6);
1592 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS7);
1593 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TMID);
1594 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CNTC);
1595 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY);
1596 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV);
1597 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA);
1598 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE);
1599 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0);
1600 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1);
1601 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI);
1602 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD);
1603 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0);
1604 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1);
1605 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2);
1606 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3);
1607 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL);
1608
1609 /* Restore Root.GINTC from unused Guest.GINTC register */
1610 write_csr_gintc(csr->csrs[LOONGARCH_CSR_GINTC]);
1611
1612 /*
1613 * We should clear linked load bit to break interrupted atomics. This
1614 * prevents a SC on the next vCPU from succeeding by matching a LL on
1615 * the previous vCPU.
1616 */
1617 if (vcpu->kvm->created_vcpus > 1)
1618 set_gcsr_llbctl(CSR_LLBCTL_WCLLB);
1619
1620 vcpu->arch.aux_inuse |= KVM_LARCH_HWCSR_USABLE;
1621
1622 return 0;
1623 }
1624
1625 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1626 {
1627 unsigned long flags;
1628
1629 local_irq_save(flags);
1630 /* Restore guest state to registers */
1631 _kvm_vcpu_load(vcpu, cpu);
1632 local_irq_restore(flags);
1633 }
1634
1635 static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu)
1636 {
1637 struct loongarch_csrs *csr = vcpu->arch.csr;
1638
1639 kvm_lose_fpu(vcpu);
1640
1641 /*
1642 * Update CSR state from hardware if software CSR state is stale,
1643 * most CSR registers are kept unchanged during process context
1644 * switch except CSR registers like remaining timer tick value and
1645 * injected interrupt state.
1646 */
1647 if (vcpu->arch.aux_inuse & KVM_LARCH_SWCSR_LATEST)
1648 goto out;
1649
1650 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CRMD);
1651 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRMD);
1652 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EUEN);
1653 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_MISC);
1654 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ECFG);
1655 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ERA);
1656 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADV);
1657 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADI);
1658 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EENTRY);
1659 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX);
1660 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI);
1661 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0);
1662 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1);
1663 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ASID);
1664 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDL);
1665 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDH);
1666 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0);
1667 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1);
1668 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE);
1669 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_RVACFG);
1670 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CPUID);
1671 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG1);
1672 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG2);
1673 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG3);
1674 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS0);
1675 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS1);
1676 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS2);
1677 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS3);
1678 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS4);
1679 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS5);
1680 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS6);
1681 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS7);
1682 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TMID);
1683 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CNTC);
1684 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL);
1685 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY);
1686 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV);
1687 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA);
1688 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE);
1689 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0);
1690 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1);
1691 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI);
1692 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD);
1693 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0);
1694 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1);
1695 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2);
1696 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3);
1697
1698 vcpu->arch.aux_inuse |= KVM_LARCH_SWCSR_LATEST;
1699
1700 out:
1701 kvm_save_timer(vcpu);
1702 /* Save Root.GINTC into unused Guest.GINTC register */
1703 csr->csrs[LOONGARCH_CSR_GINTC] = read_csr_gintc();
1704
1705 return 0;
1706 }
1707
1708 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
1709 {
1710 int cpu;
1711 unsigned long flags;
1712
1713 local_irq_save(flags);
1714 cpu = smp_processor_id();
1715 vcpu->arch.last_sched_cpu = cpu;
1716
1717 /* Save guest state in registers */
1718 _kvm_vcpu_put(vcpu, cpu);
1719 local_irq_restore(flags);
1720 }
1721
1722 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1723 {
1724 int r = -EINTR;
1725 struct kvm_run *run = vcpu->run;
1726
1727 if (vcpu->mmio_needed) {
1728 if (!vcpu->mmio_is_write)
1729 kvm_complete_mmio_read(vcpu, run);
1730 vcpu->mmio_needed = 0;
1731 }
1732
1733 if (run->exit_reason == KVM_EXIT_LOONGARCH_IOCSR) {
1734 if (!run->iocsr_io.is_write)
1735 kvm_complete_iocsr_read(vcpu, run);
1736 }
1737
1738 if (!vcpu->wants_to_run)
1739 return r;
1740
1741 /* Clear exit_reason */
1742 run->exit_reason = KVM_EXIT_UNKNOWN;
1743 lose_fpu(1);
1744 vcpu_load(vcpu);
1745 kvm_sigset_activate(vcpu);
1746 r = kvm_pre_enter_guest(vcpu);
1747 if (r != RESUME_GUEST)
1748 goto out;
1749
1750 guest_timing_enter_irqoff();
1751 guest_state_enter_irqoff();
1752 trace_kvm_enter(vcpu);
1753 r = kvm_loongarch_ops->enter_guest(run, vcpu);
1754
1755 trace_kvm_out(vcpu);
1756 /*
1757 * Guest exit is already recorded at kvm_handle_exit()
1758 * return value must not be RESUME_GUEST
1759 */
1760 local_irq_enable();
1761 out:
1762 kvm_sigset_deactivate(vcpu);
1763 vcpu_put(vcpu);
1764
1765 return r;
1766 }
Kernel DRM miscellaneous fixes and cross-tree changes