?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / arch / riscv / kvm / aia.c
blobdcced4db7fe8c3b6e4d1813a07b346fafad9270a
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 *
6 * Authors:
7 * Anup Patel <apatel@ventanamicro.com>
8 */
9
10 #include <linux/kernel.h>
11 #include <linux/bitops.h>
12 #include <linux/irq.h>
13 #include <linux/irqchip/riscv-imsic.h>
14 #include <linux/irqdomain.h>
15 #include <linux/kvm_host.h>
16 #include <linux/percpu.h>
17 #include <linux/spinlock.h>
18 #include <asm/cpufeature.h>
19 #include <asm/kvm_nacl.h>
20
21 struct aia_hgei_control {
22 raw_spinlock_t lock;
23 unsigned long free_bitmap;
24 struct kvm_vcpu *owners[BITS_PER_LONG];
25 };
26 static DEFINE_PER_CPU(struct aia_hgei_control, aia_hgei);
27 static int hgei_parent_irq;
28
29 unsigned int kvm_riscv_aia_nr_hgei;
30 unsigned int kvm_riscv_aia_max_ids;
31 DEFINE_STATIC_KEY_FALSE(kvm_riscv_aia_available);
32
33 static int aia_find_hgei(struct kvm_vcpu *owner)
34 {
35 int i, hgei;
36 unsigned long flags;
37 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
38
39 raw_spin_lock_irqsave(&hgctrl->lock, flags);
40
41 hgei = -1;
42 for (i = 1; i <= kvm_riscv_aia_nr_hgei; i++) {
43 if (hgctrl->owners[i] == owner) {
44 hgei = i;
45 break;
46 }
47 }
48
49 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
50
51 put_cpu_ptr(&aia_hgei);
52 return hgei;
53 }
54
55 static inline unsigned long aia_hvictl_value(bool ext_irq_pending)
56 {
57 unsigned long hvictl;
58
59 /*
60 * HVICTL.IID == 9 and HVICTL.IPRIO == 0 represents
61 * no interrupt in HVICTL.
62 */
63
64 hvictl = (IRQ_S_EXT << HVICTL_IID_SHIFT) & HVICTL_IID;
65 hvictl |= ext_irq_pending;
66 return hvictl;
67 }
68
69 #ifdef CONFIG_32BIT
70 void kvm_riscv_vcpu_aia_flush_interrupts(struct kvm_vcpu *vcpu)
71 {
72 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
73 unsigned long mask, val;
74
75 if (!kvm_riscv_aia_available())
76 return;
77
78 if (READ_ONCE(vcpu->arch.irqs_pending_mask[1])) {
79 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[1], 0);
80 val = READ_ONCE(vcpu->arch.irqs_pending[1]) & mask;
81
82 csr->hviph &= ~mask;
83 csr->hviph |= val;
84 }
85 }
86
87 void kvm_riscv_vcpu_aia_sync_interrupts(struct kvm_vcpu *vcpu)
88 {
89 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
90
91 if (kvm_riscv_aia_available())
92 csr->vsieh = ncsr_read(CSR_VSIEH);
93 }
94 #endif
95
96 bool kvm_riscv_vcpu_aia_has_interrupts(struct kvm_vcpu *vcpu, u64 mask)
97 {
98 int hgei;
99 unsigned long seip;
100
101 if (!kvm_riscv_aia_available())
102 return false;
103
104 #ifdef CONFIG_32BIT
105 if (READ_ONCE(vcpu->arch.irqs_pending[1]) &
106 (vcpu->arch.aia_context.guest_csr.vsieh & upper_32_bits(mask)))
107 return true;
108 #endif
109
110 seip = vcpu->arch.guest_csr.vsie;
111 seip &= (unsigned long)mask;
112 seip &= BIT(IRQ_S_EXT);
113
114 if (!kvm_riscv_aia_initialized(vcpu->kvm) || !seip)
115 return false;
116
117 hgei = aia_find_hgei(vcpu);
118 if (hgei > 0)
119 return !!(ncsr_read(CSR_HGEIP) & BIT(hgei));
120
121 return false;
122 }
123
124 void kvm_riscv_vcpu_aia_update_hvip(struct kvm_vcpu *vcpu)
125 {
126 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr;
127
128 if (!kvm_riscv_aia_available())
129 return;
130
131 #ifdef CONFIG_32BIT
132 ncsr_write(CSR_HVIPH, vcpu->arch.aia_context.guest_csr.hviph);
133 #endif
134 ncsr_write(CSR_HVICTL, aia_hvictl_value(!!(csr->hvip & BIT(IRQ_VS_EXT))));
135 }
136
137 void kvm_riscv_vcpu_aia_load(struct kvm_vcpu *vcpu, int cpu)
138 {
139 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
140 void *nsh;
141
142 if (!kvm_riscv_aia_available())
143 return;
144
145 if (kvm_riscv_nacl_sync_csr_available()) {
146 nsh = nacl_shmem();
147 nacl_csr_write(nsh, CSR_VSISELECT, csr->vsiselect);
148 nacl_csr_write(nsh, CSR_HVIPRIO1, csr->hviprio1);
149 nacl_csr_write(nsh, CSR_HVIPRIO2, csr->hviprio2);
150 #ifdef CONFIG_32BIT
151 nacl_csr_write(nsh, CSR_VSIEH, csr->vsieh);
152 nacl_csr_write(nsh, CSR_HVIPH, csr->hviph);
153 nacl_csr_write(nsh, CSR_HVIPRIO1H, csr->hviprio1h);
154 nacl_csr_write(nsh, CSR_HVIPRIO2H, csr->hviprio2h);
155 #endif
156 } else {
157 csr_write(CSR_VSISELECT, csr->vsiselect);
158 csr_write(CSR_HVIPRIO1, csr->hviprio1);
159 csr_write(CSR_HVIPRIO2, csr->hviprio2);
160 #ifdef CONFIG_32BIT
161 csr_write(CSR_VSIEH, csr->vsieh);
162 csr_write(CSR_HVIPH, csr->hviph);
163 csr_write(CSR_HVIPRIO1H, csr->hviprio1h);
164 csr_write(CSR_HVIPRIO2H, csr->hviprio2h);
165 #endif
166 }
167 }
168
169 void kvm_riscv_vcpu_aia_put(struct kvm_vcpu *vcpu)
170 {
171 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
172 void *nsh;
173
174 if (!kvm_riscv_aia_available())
175 return;
176
177 if (kvm_riscv_nacl_available()) {
178 nsh = nacl_shmem();
179 csr->vsiselect = nacl_csr_read(nsh, CSR_VSISELECT);
180 csr->hviprio1 = nacl_csr_read(nsh, CSR_HVIPRIO1);
181 csr->hviprio2 = nacl_csr_read(nsh, CSR_HVIPRIO2);
182 #ifdef CONFIG_32BIT
183 csr->vsieh = nacl_csr_read(nsh, CSR_VSIEH);
184 csr->hviph = nacl_csr_read(nsh, CSR_HVIPH);
185 csr->hviprio1h = nacl_csr_read(nsh, CSR_HVIPRIO1H);
186 csr->hviprio2h = nacl_csr_read(nsh, CSR_HVIPRIO2H);
187 #endif
188 } else {
189 csr->vsiselect = csr_read(CSR_VSISELECT);
190 csr->hviprio1 = csr_read(CSR_HVIPRIO1);
191 csr->hviprio2 = csr_read(CSR_HVIPRIO2);
192 #ifdef CONFIG_32BIT
193 csr->vsieh = csr_read(CSR_VSIEH);
194 csr->hviph = csr_read(CSR_HVIPH);
195 csr->hviprio1h = csr_read(CSR_HVIPRIO1H);
196 csr->hviprio2h = csr_read(CSR_HVIPRIO2H);
197 #endif
198 }
199 }
200
201 int kvm_riscv_vcpu_aia_get_csr(struct kvm_vcpu *vcpu,
202 unsigned long reg_num,
203 unsigned long *out_val)
204 {
205 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
206
207 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
208 return -ENOENT;
209
210 *out_val = 0;
211 if (kvm_riscv_aia_available())
212 *out_val = ((unsigned long *)csr)[reg_num];
213
214 return 0;
215 }
216
217 int kvm_riscv_vcpu_aia_set_csr(struct kvm_vcpu *vcpu,
218 unsigned long reg_num,
219 unsigned long val)
220 {
221 struct kvm_vcpu_aia_csr *csr = &vcpu->arch.aia_context.guest_csr;
222
223 if (reg_num >= sizeof(struct kvm_riscv_aia_csr) / sizeof(unsigned long))
224 return -ENOENT;
225
226 if (kvm_riscv_aia_available()) {
227 ((unsigned long *)csr)[reg_num] = val;
228
229 #ifdef CONFIG_32BIT
230 if (reg_num == KVM_REG_RISCV_CSR_AIA_REG(siph))
231 WRITE_ONCE(vcpu->arch.irqs_pending_mask[1], 0);
232 #endif
233 }
234
235 return 0;
236 }
237
238 int kvm_riscv_vcpu_aia_rmw_topei(struct kvm_vcpu *vcpu,
239 unsigned int csr_num,
240 unsigned long *val,
241 unsigned long new_val,
242 unsigned long wr_mask)
243 {
244 /* If AIA not available then redirect trap */
245 if (!kvm_riscv_aia_available())
246 return KVM_INSN_ILLEGAL_TRAP;
247
248 /* If AIA not initialized then forward to user space */
249 if (!kvm_riscv_aia_initialized(vcpu->kvm))
250 return KVM_INSN_EXIT_TO_USER_SPACE;
251
252 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, KVM_RISCV_AIA_IMSIC_TOPEI,
253 val, new_val, wr_mask);
254 }
255
256 /*
257 * External IRQ priority always read-only zero. This means default
258 * priority order is always preferred for external IRQs unless
259 * HVICTL.IID == 9 and HVICTL.IPRIO != 0
260 */
261 static int aia_irq2bitpos[] = {
262 0, 8, -1, -1, 16, 24, -1, -1, /* 0 - 7 */
263 32, -1, -1, -1, -1, 40, 48, 56, /* 8 - 15 */
264 64, 72, 80, 88, 96, 104, 112, 120, /* 16 - 23 */
265 -1, -1, -1, -1, -1, -1, -1, -1, /* 24 - 31 */
266 -1, -1, -1, -1, -1, -1, -1, -1, /* 32 - 39 */
267 -1, -1, -1, -1, -1, -1, -1, -1, /* 40 - 47 */
268 -1, -1, -1, -1, -1, -1, -1, -1, /* 48 - 55 */
269 -1, -1, -1, -1, -1, -1, -1, -1, /* 56 - 63 */
270 };
271
272 static u8 aia_get_iprio8(struct kvm_vcpu *vcpu, unsigned int irq)
273 {
274 unsigned long hviprio;
275 int bitpos = aia_irq2bitpos[irq];
276
277 if (bitpos < 0)
278 return 0;
279
280 switch (bitpos / BITS_PER_LONG) {
281 case 0:
282 hviprio = ncsr_read(CSR_HVIPRIO1);
283 break;
284 case 1:
285 #ifndef CONFIG_32BIT
286 hviprio = ncsr_read(CSR_HVIPRIO2);
287 break;
288 #else
289 hviprio = ncsr_read(CSR_HVIPRIO1H);
290 break;
291 case 2:
292 hviprio = ncsr_read(CSR_HVIPRIO2);
293 break;
294 case 3:
295 hviprio = ncsr_read(CSR_HVIPRIO2H);
296 break;
297 #endif
298 default:
299 return 0;
300 }
301
302 return (hviprio >> (bitpos % BITS_PER_LONG)) & TOPI_IPRIO_MASK;
303 }
304
305 static void aia_set_iprio8(struct kvm_vcpu *vcpu, unsigned int irq, u8 prio)
306 {
307 unsigned long hviprio;
308 int bitpos = aia_irq2bitpos[irq];
309
310 if (bitpos < 0)
311 return;
312
313 switch (bitpos / BITS_PER_LONG) {
314 case 0:
315 hviprio = ncsr_read(CSR_HVIPRIO1);
316 break;
317 case 1:
318 #ifndef CONFIG_32BIT
319 hviprio = ncsr_read(CSR_HVIPRIO2);
320 break;
321 #else
322 hviprio = ncsr_read(CSR_HVIPRIO1H);
323 break;
324 case 2:
325 hviprio = ncsr_read(CSR_HVIPRIO2);
326 break;
327 case 3:
328 hviprio = ncsr_read(CSR_HVIPRIO2H);
329 break;
330 #endif
331 default:
332 return;
333 }
334
335 hviprio &= ~(TOPI_IPRIO_MASK << (bitpos % BITS_PER_LONG));
336 hviprio |= (unsigned long)prio << (bitpos % BITS_PER_LONG);
337
338 switch (bitpos / BITS_PER_LONG) {
339 case 0:
340 ncsr_write(CSR_HVIPRIO1, hviprio);
341 break;
342 case 1:
343 #ifndef CONFIG_32BIT
344 ncsr_write(CSR_HVIPRIO2, hviprio);
345 break;
346 #else
347 ncsr_write(CSR_HVIPRIO1H, hviprio);
348 break;
349 case 2:
350 ncsr_write(CSR_HVIPRIO2, hviprio);
351 break;
352 case 3:
353 ncsr_write(CSR_HVIPRIO2H, hviprio);
354 break;
355 #endif
356 default:
357 return;
358 }
359 }
360
361 static int aia_rmw_iprio(struct kvm_vcpu *vcpu, unsigned int isel,
362 unsigned long *val, unsigned long new_val,
363 unsigned long wr_mask)
364 {
365 int i, first_irq, nirqs;
366 unsigned long old_val;
367 u8 prio;
368
369 #ifndef CONFIG_32BIT
370 if (isel & 0x1)
371 return KVM_INSN_ILLEGAL_TRAP;
372 #endif
373
374 nirqs = 4 * (BITS_PER_LONG / 32);
375 first_irq = (isel - ISELECT_IPRIO0) * 4;
376
377 old_val = 0;
378 for (i = 0; i < nirqs; i++) {
379 prio = aia_get_iprio8(vcpu, first_irq + i);
380 old_val |= (unsigned long)prio << (TOPI_IPRIO_BITS * i);
381 }
382
383 if (val)
384 *val = old_val;
385
386 if (wr_mask) {
387 new_val = (old_val & ~wr_mask) | (new_val & wr_mask);
388 for (i = 0; i < nirqs; i++) {
389 prio = (new_val >> (TOPI_IPRIO_BITS * i)) &
390 TOPI_IPRIO_MASK;
391 aia_set_iprio8(vcpu, first_irq + i, prio);
392 }
393 }
394
395 return KVM_INSN_CONTINUE_NEXT_SEPC;
396 }
397
398 int kvm_riscv_vcpu_aia_rmw_ireg(struct kvm_vcpu *vcpu, unsigned int csr_num,
399 unsigned long *val, unsigned long new_val,
400 unsigned long wr_mask)
401 {
402 unsigned int isel;
403
404 /* If AIA not available then redirect trap */
405 if (!kvm_riscv_aia_available())
406 return KVM_INSN_ILLEGAL_TRAP;
407
408 /* First try to emulate in kernel space */
409 isel = ncsr_read(CSR_VSISELECT) & ISELECT_MASK;
410 if (isel >= ISELECT_IPRIO0 && isel <= ISELECT_IPRIO15)
411 return aia_rmw_iprio(vcpu, isel, val, new_val, wr_mask);
412 else if (isel >= IMSIC_FIRST && isel <= IMSIC_LAST &&
413 kvm_riscv_aia_initialized(vcpu->kvm))
414 return kvm_riscv_vcpu_aia_imsic_rmw(vcpu, isel, val, new_val,
415 wr_mask);
416
417 /* We can't handle it here so redirect to user space */
418 return KVM_INSN_EXIT_TO_USER_SPACE;
419 }
420
421 int kvm_riscv_aia_alloc_hgei(int cpu, struct kvm_vcpu *owner,
422 void __iomem **hgei_va, phys_addr_t *hgei_pa)
423 {
424 int ret = -ENOENT;
425 unsigned long flags;
426 const struct imsic_global_config *gc;
427 const struct imsic_local_config *lc;
428 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
429
430 if (!kvm_riscv_aia_available() || !hgctrl)
431 return -ENODEV;
432
433 raw_spin_lock_irqsave(&hgctrl->lock, flags);
434
435 if (hgctrl->free_bitmap) {
436 ret = __ffs(hgctrl->free_bitmap);
437 hgctrl->free_bitmap &= ~BIT(ret);
438 hgctrl->owners[ret] = owner;
439 }
440
441 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
442
443 gc = imsic_get_global_config();
444 lc = (gc) ? per_cpu_ptr(gc->local, cpu) : NULL;
445 if (lc && ret > 0) {
446 if (hgei_va)
447 *hgei_va = lc->msi_va + (ret * IMSIC_MMIO_PAGE_SZ);
448 if (hgei_pa)
449 *hgei_pa = lc->msi_pa + (ret * IMSIC_MMIO_PAGE_SZ);
450 }
451
452 return ret;
453 }
454
455 void kvm_riscv_aia_free_hgei(int cpu, int hgei)
456 {
457 unsigned long flags;
458 struct aia_hgei_control *hgctrl = per_cpu_ptr(&aia_hgei, cpu);
459
460 if (!kvm_riscv_aia_available() || !hgctrl)
461 return;
462
463 raw_spin_lock_irqsave(&hgctrl->lock, flags);
464
465 if (hgei > 0 && hgei <= kvm_riscv_aia_nr_hgei) {
466 if (!(hgctrl->free_bitmap & BIT(hgei))) {
467 hgctrl->free_bitmap |= BIT(hgei);
468 hgctrl->owners[hgei] = NULL;
469 }
470 }
471
472 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
473 }
474
475 void kvm_riscv_aia_wakeon_hgei(struct kvm_vcpu *owner, bool enable)
476 {
477 int hgei;
478
479 if (!kvm_riscv_aia_available())
480 return;
481
482 hgei = aia_find_hgei(owner);
483 if (hgei > 0) {
484 if (enable)
485 csr_set(CSR_HGEIE, BIT(hgei));
486 else
487 csr_clear(CSR_HGEIE, BIT(hgei));
488 }
489 }
490
491 static irqreturn_t hgei_interrupt(int irq, void *dev_id)
492 {
493 int i;
494 unsigned long hgei_mask, flags;
495 struct aia_hgei_control *hgctrl = get_cpu_ptr(&aia_hgei);
496
497 hgei_mask = csr_read(CSR_HGEIP) & csr_read(CSR_HGEIE);
498 csr_clear(CSR_HGEIE, hgei_mask);
499
500 raw_spin_lock_irqsave(&hgctrl->lock, flags);
501
502 for_each_set_bit(i, &hgei_mask, BITS_PER_LONG) {
503 if (hgctrl->owners[i])
504 kvm_vcpu_kick(hgctrl->owners[i]);
505 }
506
507 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
508
509 put_cpu_ptr(&aia_hgei);
510 return IRQ_HANDLED;
511 }
512
513 static int aia_hgei_init(void)
514 {
515 int cpu, rc;
516 struct irq_domain *domain;
517 struct aia_hgei_control *hgctrl;
518
519 /* Initialize per-CPU guest external interrupt line management */
520 for_each_possible_cpu(cpu) {
521 hgctrl = per_cpu_ptr(&aia_hgei, cpu);
522 raw_spin_lock_init(&hgctrl->lock);
523 if (kvm_riscv_aia_nr_hgei) {
524 hgctrl->free_bitmap =
525 BIT(kvm_riscv_aia_nr_hgei + 1) - 1;
526 hgctrl->free_bitmap &= ~BIT(0);
527 } else
528 hgctrl->free_bitmap = 0;
529 }
530
531 /* Skip SGEI interrupt setup for zero guest external interrupts */
532 if (!kvm_riscv_aia_nr_hgei)
533 goto skip_sgei_interrupt;
534
535 /* Find INTC irq domain */
536 domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(),
537 DOMAIN_BUS_ANY);
538 if (!domain) {
539 kvm_err("unable to find INTC domain\n");
540 return -ENOENT;
541 }
542
543 /* Map per-CPU SGEI interrupt from INTC domain */
544 hgei_parent_irq = irq_create_mapping(domain, IRQ_S_GEXT);
545 if (!hgei_parent_irq) {
546 kvm_err("unable to map SGEI IRQ\n");
547 return -ENOMEM;
548 }
549
550 /* Request per-CPU SGEI interrupt */
551 rc = request_percpu_irq(hgei_parent_irq, hgei_interrupt,
552 "riscv-kvm", &aia_hgei);
553 if (rc) {
554 kvm_err("failed to request SGEI IRQ\n");
555 return rc;
556 }
557
558 skip_sgei_interrupt:
559 return 0;
560 }
561
562 static void aia_hgei_exit(void)
563 {
564 /* Do nothing for zero guest external interrupts */
565 if (!kvm_riscv_aia_nr_hgei)
566 return;
567
568 /* Free per-CPU SGEI interrupt */
569 free_percpu_irq(hgei_parent_irq, &aia_hgei);
570 }
571
572 void kvm_riscv_aia_enable(void)
573 {
574 if (!kvm_riscv_aia_available())
575 return;
576
577 csr_write(CSR_HVICTL, aia_hvictl_value(false));
578 csr_write(CSR_HVIPRIO1, 0x0);
579 csr_write(CSR_HVIPRIO2, 0x0);
580 #ifdef CONFIG_32BIT
581 csr_write(CSR_HVIPH, 0x0);
582 csr_write(CSR_HIDELEGH, 0x0);
583 csr_write(CSR_HVIPRIO1H, 0x0);
584 csr_write(CSR_HVIPRIO2H, 0x0);
585 #endif
586
587 /* Enable per-CPU SGEI interrupt */
588 enable_percpu_irq(hgei_parent_irq,
589 irq_get_trigger_type(hgei_parent_irq));
590 csr_set(CSR_HIE, BIT(IRQ_S_GEXT));
591 /* Enable IRQ filtering for overflow interrupt only if sscofpmf is present */
592 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
593 csr_write(CSR_HVIEN, BIT(IRQ_PMU_OVF));
594 }
595
596 void kvm_riscv_aia_disable(void)
597 {
598 int i;
599 unsigned long flags;
600 struct kvm_vcpu *vcpu;
601 struct aia_hgei_control *hgctrl;
602
603 if (!kvm_riscv_aia_available())
604 return;
605 hgctrl = get_cpu_ptr(&aia_hgei);
606
607 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_SSCOFPMF))
608 csr_clear(CSR_HVIEN, BIT(IRQ_PMU_OVF));
609 /* Disable per-CPU SGEI interrupt */
610 csr_clear(CSR_HIE, BIT(IRQ_S_GEXT));
611 disable_percpu_irq(hgei_parent_irq);
612
613 csr_write(CSR_HVICTL, aia_hvictl_value(false));
614
615 raw_spin_lock_irqsave(&hgctrl->lock, flags);
616
617 for (i = 0; i <= kvm_riscv_aia_nr_hgei; i++) {
618 vcpu = hgctrl->owners[i];
619 if (!vcpu)
620 continue;
621
622 /*
623 * We release hgctrl->lock before notifying IMSIC
624 * so that we don't have lock ordering issues.
625 */
626 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
627
628 /* Notify IMSIC */
629 kvm_riscv_vcpu_aia_imsic_release(vcpu);
630
631 /*
632 * Wakeup VCPU if it was blocked so that it can
633 * run on other HARTs
634 */
635 if (csr_read(CSR_HGEIE) & BIT(i)) {
636 csr_clear(CSR_HGEIE, BIT(i));
637 kvm_vcpu_kick(vcpu);
638 }
639
640 raw_spin_lock_irqsave(&hgctrl->lock, flags);
641 }
642
643 raw_spin_unlock_irqrestore(&hgctrl->lock, flags);
644
645 put_cpu_ptr(&aia_hgei);
646 }
647
648 int kvm_riscv_aia_init(void)
649 {
650 int rc;
651 const struct imsic_global_config *gc;
652
653 if (!riscv_isa_extension_available(NULL, SxAIA))
654 return -ENODEV;
655 gc = imsic_get_global_config();
656
657 /* Figure-out number of bits in HGEIE */
658 csr_write(CSR_HGEIE, -1UL);
659 kvm_riscv_aia_nr_hgei = fls_long(csr_read(CSR_HGEIE));
660 csr_write(CSR_HGEIE, 0);
661 if (kvm_riscv_aia_nr_hgei)
662 kvm_riscv_aia_nr_hgei--;
663
664 /*
665 * Number of usable HGEI lines should be minimum of per-HART
666 * IMSIC guest files and number of bits in HGEIE
667 */
668 if (gc)
669 kvm_riscv_aia_nr_hgei = min((ulong)kvm_riscv_aia_nr_hgei,
670 BIT(gc->guest_index_bits) - 1);
671 else
672 kvm_riscv_aia_nr_hgei = 0;
673
674 /* Find number of guest MSI IDs */
675 kvm_riscv_aia_max_ids = IMSIC_MAX_ID;
676 if (gc && kvm_riscv_aia_nr_hgei)
677 kvm_riscv_aia_max_ids = gc->nr_guest_ids + 1;
678
679 /* Initialize guest external interrupt line management */
680 rc = aia_hgei_init();
681 if (rc)
682 return rc;
683
684 /* Register device operations */
685 rc = kvm_register_device_ops(&kvm_riscv_aia_device_ops,
686 KVM_DEV_TYPE_RISCV_AIA);
687 if (rc) {
688 aia_hgei_exit();
689 return rc;
690 }
691
692 /* Enable KVM AIA support */
693 static_branch_enable(&kvm_riscv_aia_available);
694
695 return 0;
696 }
697
698 void kvm_riscv_aia_exit(void)
699 {
700 if (!kvm_riscv_aia_available())
701 return;
702
703 /* Unregister device operations */
704 kvm_unregister_device_ops(KVM_DEV_TYPE_RISCV_AIA);
705
706 /* Cleanup the HGEI state */
707 aia_hgei_exit();
708 }
Kernel DRM miscellaneous fixes and cross-tree changes