?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'io_uring-5.11-2021-01-16' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / gpu / drm / i915 / gvt / kvmgt.c
blob60f1a386dd06058b0b858becf551c29bfad0328f
1 /*
2 * KVMGT - the implementation of Intel mediated pass-through framework for KVM
3 *
4 * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Kevin Tian <kevin.tian@intel.com>
27 * Jike Song <jike.song@intel.com>
28 * Xiaoguang Chen <xiaoguang.chen@intel.com>
29 */
30
31 #include <linux/init.h>
32 #include <linux/device.h>
33 #include <linux/mm.h>
34 #include <linux/kthread.h>
35 #include <linux/sched/mm.h>
36 #include <linux/types.h>
37 #include <linux/list.h>
38 #include <linux/rbtree.h>
39 #include <linux/spinlock.h>
40 #include <linux/eventfd.h>
41 #include <linux/uuid.h>
42 #include <linux/kvm_host.h>
43 #include <linux/vfio.h>
44 #include <linux/mdev.h>
45 #include <linux/debugfs.h>
46
47 #include <linux/nospec.h>
48
49 #include "i915_drv.h"
50 #include "gvt.h"
51
52 static const struct intel_gvt_ops *intel_gvt_ops;
53
54 /* helper macros copied from vfio-pci */
55 #define VFIO_PCI_OFFSET_SHIFT 40
56 #define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT)
57 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
58 #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
59
60 #define EDID_BLOB_OFFSET (PAGE_SIZE/2)
61
62 #define OPREGION_SIGNATURE "IntelGraphicsMem"
63
64 struct vfio_region;
65 struct intel_vgpu_regops {
66 size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
67 size_t count, loff_t *ppos, bool iswrite);
68 void (*release)(struct intel_vgpu *vgpu,
69 struct vfio_region *region);
70 };
71
72 struct vfio_region {
73 u32 type;
74 u32 subtype;
75 size_t size;
76 u32 flags;
77 const struct intel_vgpu_regops *ops;
78 void *data;
79 };
80
81 struct vfio_edid_region {
82 struct vfio_region_gfx_edid vfio_edid_regs;
83 void *edid_blob;
84 };
85
86 struct kvmgt_pgfn {
87 gfn_t gfn;
88 struct hlist_node hnode;
89 };
90
91 struct kvmgt_guest_info {
92 struct kvm *kvm;
93 struct intel_vgpu *vgpu;
94 struct kvm_page_track_notifier_node track_node;
95 #define NR_BKT (1 << 18)
96 struct hlist_head ptable[NR_BKT];
97 #undef NR_BKT
98 struct dentry *debugfs_cache_entries;
99 };
100
101 struct gvt_dma {
102 struct intel_vgpu *vgpu;
103 struct rb_node gfn_node;
104 struct rb_node dma_addr_node;
105 gfn_t gfn;
106 dma_addr_t dma_addr;
107 unsigned long size;
108 struct kref ref;
109 };
110
111 struct kvmgt_vdev {
112 struct intel_vgpu *vgpu;
113 struct mdev_device *mdev;
114 struct vfio_region *region;
115 int num_regions;
116 struct eventfd_ctx *intx_trigger;
117 struct eventfd_ctx *msi_trigger;
118
119 /*
120 * Two caches are used to avoid mapping duplicated pages (eg.
121 * scratch pages). This help to reduce dma setup overhead.
122 */
123 struct rb_root gfn_cache;
124 struct rb_root dma_addr_cache;
125 unsigned long nr_cache_entries;
126 struct mutex cache_lock;
127
128 struct notifier_block iommu_notifier;
129 struct notifier_block group_notifier;
130 struct kvm *kvm;
131 struct work_struct release_work;
132 atomic_t released;
133 struct vfio_device *vfio_device;
134 struct vfio_group *vfio_group;
135 };
136
137 static inline struct kvmgt_vdev *kvmgt_vdev(struct intel_vgpu *vgpu)
138 {
139 return intel_vgpu_vdev(vgpu);
140 }
141
142 static inline bool handle_valid(unsigned long handle)
143 {
144 return !!(handle & ~0xff);
145 }
146
147 static int kvmgt_guest_init(struct mdev_device *mdev);
148 static void intel_vgpu_release_work(struct work_struct *work);
149 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
150
151 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
152 unsigned long size)
153 {
154 struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
155 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
156 int total_pages;
157 int npage;
158 int ret;
159
160 total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
161
162 for (npage = 0; npage < total_pages; npage++) {
163 unsigned long cur_gfn = gfn + npage;
164
165 ret = vfio_group_unpin_pages(vdev->vfio_group, &cur_gfn, 1);
166 drm_WARN_ON(&i915->drm, ret != 1);
167 }
168 }
169
170 /* Pin a normal or compound guest page for dma. */
171 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
172 unsigned long size, struct page **page)
173 {
174 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
175 unsigned long base_pfn = 0;
176 int total_pages;
177 int npage;
178 int ret;
179
180 total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
181 /*
182 * We pin the pages one-by-one to avoid allocating a big arrary
183 * on stack to hold pfns.
184 */
185 for (npage = 0; npage < total_pages; npage++) {
186 unsigned long cur_gfn = gfn + npage;
187 unsigned long pfn;
188
189 ret = vfio_group_pin_pages(vdev->vfio_group, &cur_gfn, 1,
190 IOMMU_READ | IOMMU_WRITE, &pfn);
191 if (ret != 1) {
192 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n",
193 cur_gfn, ret);
194 goto err;
195 }
196
197 if (!pfn_valid(pfn)) {
198 gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
199 npage++;
200 ret = -EFAULT;
201 goto err;
202 }
203
204 if (npage == 0)
205 base_pfn = pfn;
206 else if (base_pfn + npage != pfn) {
207 gvt_vgpu_err("The pages are not continuous\n");
208 ret = -EINVAL;
209 npage++;
210 goto err;
211 }
212 }
213
214 *page = pfn_to_page(base_pfn);
215 return 0;
216 err:
217 gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
218 return ret;
219 }
220
221 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
222 dma_addr_t *dma_addr, unsigned long size)
223 {
224 struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;
225 struct page *page = NULL;
226 int ret;
227
228 ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
229 if (ret)
230 return ret;
231
232 /* Setup DMA mapping. */
233 *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL);
234 if (dma_mapping_error(dev, *dma_addr)) {
235 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
236 page_to_pfn(page), ret);
237 gvt_unpin_guest_page(vgpu, gfn, size);
238 return -ENOMEM;
239 }
240
241 return 0;
242 }
243
244 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
245 dma_addr_t dma_addr, unsigned long size)
246 {
247 struct device *dev = &vgpu->gvt->gt->i915->drm.pdev->dev;
248
249 dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
250 gvt_unpin_guest_page(vgpu, gfn, size);
251 }
252
253 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
254 dma_addr_t dma_addr)
255 {
256 struct rb_node *node = kvmgt_vdev(vgpu)->dma_addr_cache.rb_node;
257 struct gvt_dma *itr;
258
259 while (node) {
260 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
261
262 if (dma_addr < itr->dma_addr)
263 node = node->rb_left;
264 else if (dma_addr > itr->dma_addr)
265 node = node->rb_right;
266 else
267 return itr;
268 }
269 return NULL;
270 }
271
272 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
273 {
274 struct rb_node *node = kvmgt_vdev(vgpu)->gfn_cache.rb_node;
275 struct gvt_dma *itr;
276
277 while (node) {
278 itr = rb_entry(node, struct gvt_dma, gfn_node);
279
280 if (gfn < itr->gfn)
281 node = node->rb_left;
282 else if (gfn > itr->gfn)
283 node = node->rb_right;
284 else
285 return itr;
286 }
287 return NULL;
288 }
289
290 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
291 dma_addr_t dma_addr, unsigned long size)
292 {
293 struct gvt_dma *new, *itr;
294 struct rb_node **link, *parent = NULL;
295 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
296
297 new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
298 if (!new)
299 return -ENOMEM;
300
301 new->vgpu = vgpu;
302 new->gfn = gfn;
303 new->dma_addr = dma_addr;
304 new->size = size;
305 kref_init(&new->ref);
306
307 /* gfn_cache maps gfn to struct gvt_dma. */
308 link = &vdev->gfn_cache.rb_node;
309 while (*link) {
310 parent = *link;
311 itr = rb_entry(parent, struct gvt_dma, gfn_node);
312
313 if (gfn < itr->gfn)
314 link = &parent->rb_left;
315 else
316 link = &parent->rb_right;
317 }
318 rb_link_node(&new->gfn_node, parent, link);
319 rb_insert_color(&new->gfn_node, &vdev->gfn_cache);
320
321 /* dma_addr_cache maps dma addr to struct gvt_dma. */
322 parent = NULL;
323 link = &vdev->dma_addr_cache.rb_node;
324 while (*link) {
325 parent = *link;
326 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
327
328 if (dma_addr < itr->dma_addr)
329 link = &parent->rb_left;
330 else
331 link = &parent->rb_right;
332 }
333 rb_link_node(&new->dma_addr_node, parent, link);
334 rb_insert_color(&new->dma_addr_node, &vdev->dma_addr_cache);
335
336 vdev->nr_cache_entries++;
337 return 0;
338 }
339
340 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
341 struct gvt_dma *entry)
342 {
343 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
344
345 rb_erase(&entry->gfn_node, &vdev->gfn_cache);
346 rb_erase(&entry->dma_addr_node, &vdev->dma_addr_cache);
347 kfree(entry);
348 vdev->nr_cache_entries--;
349 }
350
351 static void gvt_cache_destroy(struct intel_vgpu *vgpu)
352 {
353 struct gvt_dma *dma;
354 struct rb_node *node = NULL;
355 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
356
357 for (;;) {
358 mutex_lock(&vdev->cache_lock);
359 node = rb_first(&vdev->gfn_cache);
360 if (!node) {
361 mutex_unlock(&vdev->cache_lock);
362 break;
363 }
364 dma = rb_entry(node, struct gvt_dma, gfn_node);
365 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
366 __gvt_cache_remove_entry(vgpu, dma);
367 mutex_unlock(&vdev->cache_lock);
368 }
369 }
370
371 static void gvt_cache_init(struct intel_vgpu *vgpu)
372 {
373 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
374
375 vdev->gfn_cache = RB_ROOT;
376 vdev->dma_addr_cache = RB_ROOT;
377 vdev->nr_cache_entries = 0;
378 mutex_init(&vdev->cache_lock);
379 }
380
381 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
382 {
383 hash_init(info->ptable);
384 }
385
386 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
387 {
388 struct kvmgt_pgfn *p;
389 struct hlist_node *tmp;
390 int i;
391
392 hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
393 hash_del(&p->hnode);
394 kfree(p);
395 }
396 }
397
398 static struct kvmgt_pgfn *
399 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
400 {
401 struct kvmgt_pgfn *p, *res = NULL;
402
403 hash_for_each_possible(info->ptable, p, hnode, gfn) {
404 if (gfn == p->gfn) {
405 res = p;
406 break;
407 }
408 }
409
410 return res;
411 }
412
413 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
414 gfn_t gfn)
415 {
416 struct kvmgt_pgfn *p;
417
418 p = __kvmgt_protect_table_find(info, gfn);
419 return !!p;
420 }
421
422 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
423 {
424 struct kvmgt_pgfn *p;
425
426 if (kvmgt_gfn_is_write_protected(info, gfn))
427 return;
428
429 p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
430 if (WARN(!p, "gfn: 0x%llx\n", gfn))
431 return;
432
433 p->gfn = gfn;
434 hash_add(info->ptable, &p->hnode, gfn);
435 }
436
437 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
438 gfn_t gfn)
439 {
440 struct kvmgt_pgfn *p;
441
442 p = __kvmgt_protect_table_find(info, gfn);
443 if (p) {
444 hash_del(&p->hnode);
445 kfree(p);
446 }
447 }
448
449 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
450 size_t count, loff_t *ppos, bool iswrite)
451 {
452 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
453 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
454 VFIO_PCI_NUM_REGIONS;
455 void *base = vdev->region[i].data;
456 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
457
458
459 if (pos >= vdev->region[i].size || iswrite) {
460 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
461 return -EINVAL;
462 }
463 count = min(count, (size_t)(vdev->region[i].size - pos));
464 memcpy(buf, base + pos, count);
465
466 return count;
467 }
468
469 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
470 struct vfio_region *region)
471 {
472 }
473
474 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
475 .rw = intel_vgpu_reg_rw_opregion,
476 .release = intel_vgpu_reg_release_opregion,
477 };
478
479 static int handle_edid_regs(struct intel_vgpu *vgpu,
480 struct vfio_edid_region *region, char *buf,
481 size_t count, u16 offset, bool is_write)
482 {
483 struct vfio_region_gfx_edid *regs = &region->vfio_edid_regs;
484 unsigned int data;
485
486 if (offset + count > sizeof(*regs))
487 return -EINVAL;
488
489 if (count != 4)
490 return -EINVAL;
491
492 if (is_write) {
493 data = *((unsigned int *)buf);
494 switch (offset) {
495 case offsetof(struct vfio_region_gfx_edid, link_state):
496 if (data == VFIO_DEVICE_GFX_LINK_STATE_UP) {
497 if (!drm_edid_block_valid(
498 (u8 *)region->edid_blob,
499 0,
500 true,
501 NULL)) {
502 gvt_vgpu_err("invalid EDID blob\n");
503 return -EINVAL;
504 }
505 intel_gvt_ops->emulate_hotplug(vgpu, true);
506 } else if (data == VFIO_DEVICE_GFX_LINK_STATE_DOWN)
507 intel_gvt_ops->emulate_hotplug(vgpu, false);
508 else {
509 gvt_vgpu_err("invalid EDID link state %d\n",
510 regs->link_state);
511 return -EINVAL;
512 }
513 regs->link_state = data;
514 break;
515 case offsetof(struct vfio_region_gfx_edid, edid_size):
516 if (data > regs->edid_max_size) {
517 gvt_vgpu_err("EDID size is bigger than %d!\n",
518 regs->edid_max_size);
519 return -EINVAL;
520 }
521 regs->edid_size = data;
522 break;
523 default:
524 /* read-only regs */
525 gvt_vgpu_err("write read-only EDID region at offset %d\n",
526 offset);
527 return -EPERM;
528 }
529 } else {
530 memcpy(buf, (char *)regs + offset, count);
531 }
532
533 return count;
534 }
535
536 static int handle_edid_blob(struct vfio_edid_region *region, char *buf,
537 size_t count, u16 offset, bool is_write)
538 {
539 if (offset + count > region->vfio_edid_regs.edid_size)
540 return -EINVAL;
541
542 if (is_write)
543 memcpy(region->edid_blob + offset, buf, count);
544 else
545 memcpy(buf, region->edid_blob + offset, count);
546
547 return count;
548 }
549
550 static size_t intel_vgpu_reg_rw_edid(struct intel_vgpu *vgpu, char *buf,
551 size_t count, loff_t *ppos, bool iswrite)
552 {
553 int ret;
554 unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
555 VFIO_PCI_NUM_REGIONS;
556 struct vfio_edid_region *region =
557 (struct vfio_edid_region *)kvmgt_vdev(vgpu)->region[i].data;
558 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
559
560 if (pos < region->vfio_edid_regs.edid_offset) {
561 ret = handle_edid_regs(vgpu, region, buf, count, pos, iswrite);
562 } else {
563 pos -= EDID_BLOB_OFFSET;
564 ret = handle_edid_blob(region, buf, count, pos, iswrite);
565 }
566
567 if (ret < 0)
568 gvt_vgpu_err("failed to access EDID region\n");
569
570 return ret;
571 }
572
573 static void intel_vgpu_reg_release_edid(struct intel_vgpu *vgpu,
574 struct vfio_region *region)
575 {
576 kfree(region->data);
577 }
578
579 static const struct intel_vgpu_regops intel_vgpu_regops_edid = {
580 .rw = intel_vgpu_reg_rw_edid,
581 .release = intel_vgpu_reg_release_edid,
582 };
583
584 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
585 unsigned int type, unsigned int subtype,
586 const struct intel_vgpu_regops *ops,
587 size_t size, u32 flags, void *data)
588 {
589 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
590 struct vfio_region *region;
591
592 region = krealloc(vdev->region,
593 (vdev->num_regions + 1) * sizeof(*region),
594 GFP_KERNEL);
595 if (!region)
596 return -ENOMEM;
597
598 vdev->region = region;
599 vdev->region[vdev->num_regions].type = type;
600 vdev->region[vdev->num_regions].subtype = subtype;
601 vdev->region[vdev->num_regions].ops = ops;
602 vdev->region[vdev->num_regions].size = size;
603 vdev->region[vdev->num_regions].flags = flags;
604 vdev->region[vdev->num_regions].data = data;
605 vdev->num_regions++;
606 return 0;
607 }
608
609 static int kvmgt_get_vfio_device(void *p_vgpu)
610 {
611 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
612 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
613
614 vdev->vfio_device = vfio_device_get_from_dev(
615 mdev_dev(vdev->mdev));
616 if (!vdev->vfio_device) {
617 gvt_vgpu_err("failed to get vfio device\n");
618 return -ENODEV;
619 }
620 return 0;
621 }
622
623
624 static int kvmgt_set_opregion(void *p_vgpu)
625 {
626 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
627 void *base;
628 int ret;
629
630 /* Each vgpu has its own opregion, although VFIO would create another
631 * one later. This one is used to expose opregion to VFIO. And the
632 * other one created by VFIO later, is used by guest actually.
633 */
634 base = vgpu_opregion(vgpu)->va;
635 if (!base)
636 return -ENOMEM;
637
638 if (memcmp(base, OPREGION_SIGNATURE, 16)) {
639 memunmap(base);
640 return -EINVAL;
641 }
642
643 ret = intel_vgpu_register_reg(vgpu,
644 PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
645 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
646 &intel_vgpu_regops_opregion, OPREGION_SIZE,
647 VFIO_REGION_INFO_FLAG_READ, base);
648
649 return ret;
650 }
651
652 static int kvmgt_set_edid(void *p_vgpu, int port_num)
653 {
654 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
655 struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
656 struct vfio_edid_region *base;
657 int ret;
658
659 base = kzalloc(sizeof(*base), GFP_KERNEL);
660 if (!base)
661 return -ENOMEM;
662
663 /* TODO: Add multi-port and EDID extension block support */
664 base->vfio_edid_regs.edid_offset = EDID_BLOB_OFFSET;
665 base->vfio_edid_regs.edid_max_size = EDID_SIZE;
666 base->vfio_edid_regs.edid_size = EDID_SIZE;
667 base->vfio_edid_regs.max_xres = vgpu_edid_xres(port->id);
668 base->vfio_edid_regs.max_yres = vgpu_edid_yres(port->id);
669 base->edid_blob = port->edid->edid_block;
670
671 ret = intel_vgpu_register_reg(vgpu,
672 VFIO_REGION_TYPE_GFX,
673 VFIO_REGION_SUBTYPE_GFX_EDID,
674 &intel_vgpu_regops_edid, EDID_SIZE,
675 VFIO_REGION_INFO_FLAG_READ |
676 VFIO_REGION_INFO_FLAG_WRITE |
677 VFIO_REGION_INFO_FLAG_CAPS, base);
678
679 return ret;
680 }
681
682 static void kvmgt_put_vfio_device(void *vgpu)
683 {
684 struct kvmgt_vdev *vdev = kvmgt_vdev((struct intel_vgpu *)vgpu);
685
686 if (WARN_ON(!vdev->vfio_device))
687 return;
688
689 vfio_device_put(vdev->vfio_device);
690 }
691
692 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
693 {
694 struct intel_vgpu *vgpu = NULL;
695 struct intel_vgpu_type *type;
696 struct device *pdev;
697 void *gvt;
698 int ret;
699
700 pdev = mdev_parent_dev(mdev);
701 gvt = kdev_to_i915(pdev)->gvt;
702
703 type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
704 if (!type) {
705 gvt_vgpu_err("failed to find type %s to create\n",
706 kobject_name(kobj));
707 ret = -EINVAL;
708 goto out;
709 }
710
711 vgpu = intel_gvt_ops->vgpu_create(gvt, type);
712 if (IS_ERR_OR_NULL(vgpu)) {
713 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu);
714 gvt_err("failed to create intel vgpu: %d\n", ret);
715 goto out;
716 }
717
718 INIT_WORK(&kvmgt_vdev(vgpu)->release_work, intel_vgpu_release_work);
719
720 kvmgt_vdev(vgpu)->mdev = mdev;
721 mdev_set_drvdata(mdev, vgpu);
722
723 gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
724 dev_name(mdev_dev(mdev)));
725 ret = 0;
726
727 out:
728 return ret;
729 }
730
731 static int intel_vgpu_remove(struct mdev_device *mdev)
732 {
733 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
734
735 if (handle_valid(vgpu->handle))
736 return -EBUSY;
737
738 intel_gvt_ops->vgpu_destroy(vgpu);
739 return 0;
740 }
741
742 static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
743 unsigned long action, void *data)
744 {
745 struct kvmgt_vdev *vdev = container_of(nb,
746 struct kvmgt_vdev,
747 iommu_notifier);
748 struct intel_vgpu *vgpu = vdev->vgpu;
749
750 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
751 struct vfio_iommu_type1_dma_unmap *unmap = data;
752 struct gvt_dma *entry;
753 unsigned long iov_pfn, end_iov_pfn;
754
755 iov_pfn = unmap->iova >> PAGE_SHIFT;
756 end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE;
757
758 mutex_lock(&vdev->cache_lock);
759 for (; iov_pfn < end_iov_pfn; iov_pfn++) {
760 entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
761 if (!entry)
762 continue;
763
764 gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
765 entry->size);
766 __gvt_cache_remove_entry(vgpu, entry);
767 }
768 mutex_unlock(&vdev->cache_lock);
769 }
770
771 return NOTIFY_OK;
772 }
773
774 static int intel_vgpu_group_notifier(struct notifier_block *nb,
775 unsigned long action, void *data)
776 {
777 struct kvmgt_vdev *vdev = container_of(nb,
778 struct kvmgt_vdev,
779 group_notifier);
780
781 /* the only action we care about */
782 if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
783 vdev->kvm = data;
784
785 if (!data)
786 schedule_work(&vdev->release_work);
787 }
788
789 return NOTIFY_OK;
790 }
791
792 static int intel_vgpu_open(struct mdev_device *mdev)
793 {
794 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
795 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
796 unsigned long events;
797 int ret;
798 struct vfio_group *vfio_group;
799
800 vdev->iommu_notifier.notifier_call = intel_vgpu_iommu_notifier;
801 vdev->group_notifier.notifier_call = intel_vgpu_group_notifier;
802
803 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
804 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events,
805 &vdev->iommu_notifier);
806 if (ret != 0) {
807 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n",
808 ret);
809 goto out;
810 }
811
812 events = VFIO_GROUP_NOTIFY_SET_KVM;
813 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events,
814 &vdev->group_notifier);
815 if (ret != 0) {
816 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n",
817 ret);
818 goto undo_iommu;
819 }
820
821 vfio_group = vfio_group_get_external_user_from_dev(mdev_dev(mdev));
822 if (IS_ERR_OR_NULL(vfio_group)) {
823 ret = !vfio_group ? -EFAULT : PTR_ERR(vfio_group);
824 gvt_vgpu_err("vfio_group_get_external_user_from_dev failed\n");
825 goto undo_register;
826 }
827 vdev->vfio_group = vfio_group;
828
829 /* Take a module reference as mdev core doesn't take
830 * a reference for vendor driver.
831 */
832 if (!try_module_get(THIS_MODULE)) {
833 ret = -ENODEV;
834 goto undo_group;
835 }
836
837 ret = kvmgt_guest_init(mdev);
838 if (ret)
839 goto undo_group;
840
841 intel_gvt_ops->vgpu_activate(vgpu);
842
843 atomic_set(&vdev->released, 0);
844 return ret;
845
846 undo_group:
847 vfio_group_put_external_user(vdev->vfio_group);
848 vdev->vfio_group = NULL;
849
850 undo_register:
851 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
852 &vdev->group_notifier);
853
854 undo_iommu:
855 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
856 &vdev->iommu_notifier);
857 out:
858 return ret;
859 }
860
861 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
862 {
863 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
864 struct eventfd_ctx *trigger;
865
866 trigger = vdev->msi_trigger;
867 if (trigger) {
868 eventfd_ctx_put(trigger);
869 vdev->msi_trigger = NULL;
870 }
871 }
872
873 static void __intel_vgpu_release(struct intel_vgpu *vgpu)
874 {
875 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
876 struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
877 struct kvmgt_guest_info *info;
878 int ret;
879
880 if (!handle_valid(vgpu->handle))
881 return;
882
883 if (atomic_cmpxchg(&vdev->released, 0, 1))
884 return;
885
886 intel_gvt_ops->vgpu_release(vgpu);
887
888 ret = vfio_unregister_notifier(mdev_dev(vdev->mdev), VFIO_IOMMU_NOTIFY,
889 &vdev->iommu_notifier);
890 drm_WARN(&i915->drm, ret,
891 "vfio_unregister_notifier for iommu failed: %d\n", ret);
892
893 ret = vfio_unregister_notifier(mdev_dev(vdev->mdev), VFIO_GROUP_NOTIFY,
894 &vdev->group_notifier);
895 drm_WARN(&i915->drm, ret,
896 "vfio_unregister_notifier for group failed: %d\n", ret);
897
898 /* dereference module reference taken at open */
899 module_put(THIS_MODULE);
900
901 info = (struct kvmgt_guest_info *)vgpu->handle;
902 kvmgt_guest_exit(info);
903
904 intel_vgpu_release_msi_eventfd_ctx(vgpu);
905 vfio_group_put_external_user(vdev->vfio_group);
906
907 vdev->kvm = NULL;
908 vgpu->handle = 0;
909 }
910
911 static void intel_vgpu_release(struct mdev_device *mdev)
912 {
913 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
914
915 __intel_vgpu_release(vgpu);
916 }
917
918 static void intel_vgpu_release_work(struct work_struct *work)
919 {
920 struct kvmgt_vdev *vdev = container_of(work, struct kvmgt_vdev,
921 release_work);
922
923 __intel_vgpu_release(vdev->vgpu);
924 }
925
926 static u64 intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
927 {
928 u32 start_lo, start_hi;
929 u32 mem_type;
930
931 start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
932 PCI_BASE_ADDRESS_MEM_MASK;
933 mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
934 PCI_BASE_ADDRESS_MEM_TYPE_MASK;
935
936 switch (mem_type) {
937 case PCI_BASE_ADDRESS_MEM_TYPE_64:
938 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
939 + bar + 4));
940 break;
941 case PCI_BASE_ADDRESS_MEM_TYPE_32:
942 case PCI_BASE_ADDRESS_MEM_TYPE_1M:
943 /* 1M mem BAR treated as 32-bit BAR */
944 default:
945 /* mem unknown type treated as 32-bit BAR */
946 start_hi = 0;
947 break;
948 }
949
950 return ((u64)start_hi << 32) | start_lo;
951 }
952
953 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, u64 off,
954 void *buf, unsigned int count, bool is_write)
955 {
956 u64 bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
957 int ret;
958
959 if (is_write)
960 ret = intel_gvt_ops->emulate_mmio_write(vgpu,
961 bar_start + off, buf, count);
962 else
963 ret = intel_gvt_ops->emulate_mmio_read(vgpu,
964 bar_start + off, buf, count);
965 return ret;
966 }
967
968 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, u64 off)
969 {
970 return off >= vgpu_aperture_offset(vgpu) &&
971 off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
972 }
973
974 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, u64 off,
975 void *buf, unsigned long count, bool is_write)
976 {
977 void __iomem *aperture_va;
978
979 if (!intel_vgpu_in_aperture(vgpu, off) ||
980 !intel_vgpu_in_aperture(vgpu, off + count)) {
981 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
982 return -EINVAL;
983 }
984
985 aperture_va = io_mapping_map_wc(&vgpu->gvt->gt->ggtt->iomap,
986 ALIGN_DOWN(off, PAGE_SIZE),
987 count + offset_in_page(off));
988 if (!aperture_va)
989 return -EIO;
990
991 if (is_write)
992 memcpy_toio(aperture_va + offset_in_page(off), buf, count);
993 else
994 memcpy_fromio(buf, aperture_va + offset_in_page(off), count);
995
996 io_mapping_unmap(aperture_va);
997
998 return 0;
999 }
1000
1001 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
1002 size_t count, loff_t *ppos, bool is_write)
1003 {
1004 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1005 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
1006 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1007 u64 pos = *ppos & VFIO_PCI_OFFSET_MASK;
1008 int ret = -EINVAL;
1009
1010
1011 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) {
1012 gvt_vgpu_err("invalid index: %u\n", index);
1013 return -EINVAL;
1014 }
1015
1016 switch (index) {
1017 case VFIO_PCI_CONFIG_REGION_INDEX:
1018 if (is_write)
1019 ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos,
1020 buf, count);
1021 else
1022 ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos,
1023 buf, count);
1024 break;
1025 case VFIO_PCI_BAR0_REGION_INDEX:
1026 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
1027 buf, count, is_write);
1028 break;
1029 case VFIO_PCI_BAR2_REGION_INDEX:
1030 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
1031 break;
1032 case VFIO_PCI_BAR1_REGION_INDEX:
1033 case VFIO_PCI_BAR3_REGION_INDEX:
1034 case VFIO_PCI_BAR4_REGION_INDEX:
1035 case VFIO_PCI_BAR5_REGION_INDEX:
1036 case VFIO_PCI_VGA_REGION_INDEX:
1037 case VFIO_PCI_ROM_REGION_INDEX:
1038 break;
1039 default:
1040 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1041 return -EINVAL;
1042
1043 index -= VFIO_PCI_NUM_REGIONS;
1044 return vdev->region[index].ops->rw(vgpu, buf, count,
1045 ppos, is_write);
1046 }
1047
1048 return ret == 0 ? count : ret;
1049 }
1050
1051 static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos)
1052 {
1053 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1054 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1055 struct intel_gvt *gvt = vgpu->gvt;
1056 int offset;
1057
1058 /* Only allow MMIO GGTT entry access */
1059 if (index != PCI_BASE_ADDRESS_0)
1060 return false;
1061
1062 offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
1063 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
1064
1065 return (offset >= gvt->device_info.gtt_start_offset &&
1066 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
1067 true : false;
1068 }
1069
1070 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
1071 size_t count, loff_t *ppos)
1072 {
1073 unsigned int done = 0;
1074 int ret;
1075
1076 while (count) {
1077 size_t filled;
1078
1079 /* Only support GGTT entry 8 bytes read */
1080 if (count >= 8 && !(*ppos % 8) &&
1081 gtt_entry(mdev, ppos)) {
1082 u64 val;
1083
1084 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
1085 ppos, false);
1086 if (ret <= 0)
1087 goto read_err;
1088
1089 if (copy_to_user(buf, &val, sizeof(val)))
1090 goto read_err;
1091
1092 filled = 8;
1093 } else if (count >= 4 && !(*ppos % 4)) {
1094 u32 val;
1095
1096 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
1097 ppos, false);
1098 if (ret <= 0)
1099 goto read_err;
1100
1101 if (copy_to_user(buf, &val, sizeof(val)))
1102 goto read_err;
1103
1104 filled = 4;
1105 } else if (count >= 2 && !(*ppos % 2)) {
1106 u16 val;
1107
1108 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
1109 ppos, false);
1110 if (ret <= 0)
1111 goto read_err;
1112
1113 if (copy_to_user(buf, &val, sizeof(val)))
1114 goto read_err;
1115
1116 filled = 2;
1117 } else {
1118 u8 val;
1119
1120 ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos,
1121 false);
1122 if (ret <= 0)
1123 goto read_err;
1124
1125 if (copy_to_user(buf, &val, sizeof(val)))
1126 goto read_err;
1127
1128 filled = 1;
1129 }
1130
1131 count -= filled;
1132 done += filled;
1133 *ppos += filled;
1134 buf += filled;
1135 }
1136
1137 return done;
1138
1139 read_err:
1140 return -EFAULT;
1141 }
1142
1143 static ssize_t intel_vgpu_write(struct mdev_device *mdev,
1144 const char __user *buf,
1145 size_t count, loff_t *ppos)
1146 {
1147 unsigned int done = 0;
1148 int ret;
1149
1150 while (count) {
1151 size_t filled;
1152
1153 /* Only support GGTT entry 8 bytes write */
1154 if (count >= 8 && !(*ppos % 8) &&
1155 gtt_entry(mdev, ppos)) {
1156 u64 val;
1157
1158 if (copy_from_user(&val, buf, sizeof(val)))
1159 goto write_err;
1160
1161 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
1162 ppos, true);
1163 if (ret <= 0)
1164 goto write_err;
1165
1166 filled = 8;
1167 } else if (count >= 4 && !(*ppos % 4)) {
1168 u32 val;
1169
1170 if (copy_from_user(&val, buf, sizeof(val)))
1171 goto write_err;
1172
1173 ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
1174 ppos, true);
1175 if (ret <= 0)
1176 goto write_err;
1177
1178 filled = 4;
1179 } else if (count >= 2 && !(*ppos % 2)) {
1180 u16 val;
1181
1182 if (copy_from_user(&val, buf, sizeof(val)))
1183 goto write_err;
1184
1185 ret = intel_vgpu_rw(mdev, (char *)&val,
1186 sizeof(val), ppos, true);
1187 if (ret <= 0)
1188 goto write_err;
1189
1190 filled = 2;
1191 } else {
1192 u8 val;
1193
1194 if (copy_from_user(&val, buf, sizeof(val)))
1195 goto write_err;
1196
1197 ret = intel_vgpu_rw(mdev, &val, sizeof(val),
1198 ppos, true);
1199 if (ret <= 0)
1200 goto write_err;
1201
1202 filled = 1;
1203 }
1204
1205 count -= filled;
1206 done += filled;
1207 *ppos += filled;
1208 buf += filled;
1209 }
1210
1211 return done;
1212 write_err:
1213 return -EFAULT;
1214 }
1215
1216 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma)
1217 {
1218 unsigned int index;
1219 u64 virtaddr;
1220 unsigned long req_size, pgoff, req_start;
1221 pgprot_t pg_prot;
1222 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1223
1224 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1225 if (index >= VFIO_PCI_ROM_REGION_INDEX)
1226 return -EINVAL;
1227
1228 if (vma->vm_end < vma->vm_start)
1229 return -EINVAL;
1230 if ((vma->vm_flags & VM_SHARED) == 0)
1231 return -EINVAL;
1232 if (index != VFIO_PCI_BAR2_REGION_INDEX)
1233 return -EINVAL;
1234
1235 pg_prot = vma->vm_page_prot;
1236 virtaddr = vma->vm_start;
1237 req_size = vma->vm_end - vma->vm_start;
1238 pgoff = vma->vm_pgoff &
1239 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1240 req_start = pgoff << PAGE_SHIFT;
1241
1242 if (!intel_vgpu_in_aperture(vgpu, req_start))
1243 return -EINVAL;
1244 if (req_start + req_size >
1245 vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu))
1246 return -EINVAL;
1247
1248 pgoff = (gvt_aperture_pa_base(vgpu->gvt) >> PAGE_SHIFT) + pgoff;
1249
1250 return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
1251 }
1252
1253 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1254 {
1255 if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1256 return 1;
1257
1258 return 0;
1259 }
1260
1261 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1262 unsigned int index, unsigned int start,
1263 unsigned int count, u32 flags,
1264 void *data)
1265 {
1266 return 0;
1267 }
1268
1269 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1270 unsigned int index, unsigned int start,
1271 unsigned int count, u32 flags, void *data)
1272 {
1273 return 0;
1274 }
1275
1276 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1277 unsigned int index, unsigned int start, unsigned int count,
1278 u32 flags, void *data)
1279 {
1280 return 0;
1281 }
1282
1283 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1284 unsigned int index, unsigned int start, unsigned int count,
1285 u32 flags, void *data)
1286 {
1287 struct eventfd_ctx *trigger;
1288
1289 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1290 int fd = *(int *)data;
1291
1292 trigger = eventfd_ctx_fdget(fd);
1293 if (IS_ERR(trigger)) {
1294 gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1295 return PTR_ERR(trigger);
1296 }
1297 kvmgt_vdev(vgpu)->msi_trigger = trigger;
1298 } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1299 intel_vgpu_release_msi_eventfd_ctx(vgpu);
1300
1301 return 0;
1302 }
1303
1304 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, u32 flags,
1305 unsigned int index, unsigned int start, unsigned int count,
1306 void *data)
1307 {
1308 int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1309 unsigned int start, unsigned int count, u32 flags,
1310 void *data) = NULL;
1311
1312 switch (index) {
1313 case VFIO_PCI_INTX_IRQ_INDEX:
1314 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1315 case VFIO_IRQ_SET_ACTION_MASK:
1316 func = intel_vgpu_set_intx_mask;
1317 break;
1318 case VFIO_IRQ_SET_ACTION_UNMASK:
1319 func = intel_vgpu_set_intx_unmask;
1320 break;
1321 case VFIO_IRQ_SET_ACTION_TRIGGER:
1322 func = intel_vgpu_set_intx_trigger;
1323 break;
1324 }
1325 break;
1326 case VFIO_PCI_MSI_IRQ_INDEX:
1327 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1328 case VFIO_IRQ_SET_ACTION_MASK:
1329 case VFIO_IRQ_SET_ACTION_UNMASK:
1330 /* XXX Need masking support exported */
1331 break;
1332 case VFIO_IRQ_SET_ACTION_TRIGGER:
1333 func = intel_vgpu_set_msi_trigger;
1334 break;
1335 }
1336 break;
1337 }
1338
1339 if (!func)
1340 return -ENOTTY;
1341
1342 return func(vgpu, index, start, count, flags, data);
1343 }
1344
1345 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
1346 unsigned long arg)
1347 {
1348 struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1349 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
1350 unsigned long minsz;
1351
1352 gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1353
1354 if (cmd == VFIO_DEVICE_GET_INFO) {
1355 struct vfio_device_info info;
1356
1357 minsz = offsetofend(struct vfio_device_info, num_irqs);
1358
1359 if (copy_from_user(&info, (void __user *)arg, minsz))
1360 return -EFAULT;
1361
1362 if (info.argsz < minsz)
1363 return -EINVAL;
1364
1365 info.flags = VFIO_DEVICE_FLAGS_PCI;
1366 info.flags |= VFIO_DEVICE_FLAGS_RESET;
1367 info.num_regions = VFIO_PCI_NUM_REGIONS +
1368 vdev->num_regions;
1369 info.num_irqs = VFIO_PCI_NUM_IRQS;
1370
1371 return copy_to_user((void __user *)arg, &info, minsz) ?
1372 -EFAULT : 0;
1373
1374 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1375 struct vfio_region_info info;
1376 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1377 unsigned int i;
1378 int ret;
1379 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1380 int nr_areas = 1;
1381 int cap_type_id;
1382
1383 minsz = offsetofend(struct vfio_region_info, offset);
1384
1385 if (copy_from_user(&info, (void __user *)arg, minsz))
1386 return -EFAULT;
1387
1388 if (info.argsz < minsz)
1389 return -EINVAL;
1390
1391 switch (info.index) {
1392 case VFIO_PCI_CONFIG_REGION_INDEX:
1393 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1394 info.size = vgpu->gvt->device_info.cfg_space_size;
1395 info.flags = VFIO_REGION_INFO_FLAG_READ |
1396 VFIO_REGION_INFO_FLAG_WRITE;
1397 break;
1398 case VFIO_PCI_BAR0_REGION_INDEX:
1399 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1400 info.size = vgpu->cfg_space.bar[info.index].size;
1401 if (!info.size) {
1402 info.flags = 0;
1403 break;
1404 }
1405
1406 info.flags = VFIO_REGION_INFO_FLAG_READ |
1407 VFIO_REGION_INFO_FLAG_WRITE;
1408 break;
1409 case VFIO_PCI_BAR1_REGION_INDEX:
1410 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1411 info.size = 0;
1412 info.flags = 0;
1413 break;
1414 case VFIO_PCI_BAR2_REGION_INDEX:
1415 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1416 info.flags = VFIO_REGION_INFO_FLAG_CAPS |
1417 VFIO_REGION_INFO_FLAG_MMAP |
1418 VFIO_REGION_INFO_FLAG_READ |
1419 VFIO_REGION_INFO_FLAG_WRITE;
1420 info.size = gvt_aperture_sz(vgpu->gvt);
1421
1422 sparse = kzalloc(struct_size(sparse, areas, nr_areas),
1423 GFP_KERNEL);
1424 if (!sparse)
1425 return -ENOMEM;
1426
1427 sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1428 sparse->header.version = 1;
1429 sparse->nr_areas = nr_areas;
1430 cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1431 sparse->areas[0].offset =
1432 PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1433 sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1434 break;
1435
1436 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1437 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1438 info.size = 0;
1439 info.flags = 0;
1440
1441 gvt_dbg_core("get region info bar:%d\n", info.index);
1442 break;
1443
1444 case VFIO_PCI_ROM_REGION_INDEX:
1445 case VFIO_PCI_VGA_REGION_INDEX:
1446 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1447 info.size = 0;
1448 info.flags = 0;
1449
1450 gvt_dbg_core("get region info index:%d\n", info.index);
1451 break;
1452 default:
1453 {
1454 struct vfio_region_info_cap_type cap_type = {
1455 .header.id = VFIO_REGION_INFO_CAP_TYPE,
1456 .header.version = 1 };
1457
1458 if (info.index >= VFIO_PCI_NUM_REGIONS +
1459 vdev->num_regions)
1460 return -EINVAL;
1461 info.index =
1462 array_index_nospec(info.index,
1463 VFIO_PCI_NUM_REGIONS +
1464 vdev->num_regions);
1465
1466 i = info.index - VFIO_PCI_NUM_REGIONS;
1467
1468 info.offset =
1469 VFIO_PCI_INDEX_TO_OFFSET(info.index);
1470 info.size = vdev->region[i].size;
1471 info.flags = vdev->region[i].flags;
1472
1473 cap_type.type = vdev->region[i].type;
1474 cap_type.subtype = vdev->region[i].subtype;
1475
1476 ret = vfio_info_add_capability(&caps,
1477 &cap_type.header,
1478 sizeof(cap_type));
1479 if (ret)
1480 return ret;
1481 }
1482 }
1483
1484 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1485 switch (cap_type_id) {
1486 case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
1487 ret = vfio_info_add_capability(&caps,
1488 &sparse->header,
1489 struct_size(sparse, areas,
1490 sparse->nr_areas));
1491 if (ret) {
1492 kfree(sparse);
1493 return ret;
1494 }
1495 break;
1496 default:
1497 kfree(sparse);
1498 return -EINVAL;
1499 }
1500 }
1501
1502 if (caps.size) {
1503 info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1504 if (info.argsz < sizeof(info) + caps.size) {
1505 info.argsz = sizeof(info) + caps.size;
1506 info.cap_offset = 0;
1507 } else {
1508 vfio_info_cap_shift(&caps, sizeof(info));
1509 if (copy_to_user((void __user *)arg +
1510 sizeof(info), caps.buf,
1511 caps.size)) {
1512 kfree(caps.buf);
1513 kfree(sparse);
1514 return -EFAULT;
1515 }
1516 info.cap_offset = sizeof(info);
1517 }
1518
1519 kfree(caps.buf);
1520 }
1521
1522 kfree(sparse);
1523 return copy_to_user((void __user *)arg, &info, minsz) ?
1524 -EFAULT : 0;
1525 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1526 struct vfio_irq_info info;
1527
1528 minsz = offsetofend(struct vfio_irq_info, count);
1529
1530 if (copy_from_user(&info, (void __user *)arg, minsz))
1531 return -EFAULT;
1532
1533 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1534 return -EINVAL;
1535
1536 switch (info.index) {
1537 case VFIO_PCI_INTX_IRQ_INDEX:
1538 case VFIO_PCI_MSI_IRQ_INDEX:
1539 break;
1540 default:
1541 return -EINVAL;
1542 }
1543
1544 info.flags = VFIO_IRQ_INFO_EVENTFD;
1545
1546 info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1547
1548 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1549 info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1550 VFIO_IRQ_INFO_AUTOMASKED);
1551 else
1552 info.flags |= VFIO_IRQ_INFO_NORESIZE;
1553
1554 return copy_to_user((void __user *)arg, &info, minsz) ?
1555 -EFAULT : 0;
1556 } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1557 struct vfio_irq_set hdr;
1558 u8 *data = NULL;
1559 int ret = 0;
1560 size_t data_size = 0;
1561
1562 minsz = offsetofend(struct vfio_irq_set, count);
1563
1564 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1565 return -EFAULT;
1566
1567 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1568 int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1569
1570 ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1571 VFIO_PCI_NUM_IRQS, &data_size);
1572 if (ret) {
1573 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1574 return -EINVAL;
1575 }
1576 if (data_size) {
1577 data = memdup_user((void __user *)(arg + minsz),
1578 data_size);
1579 if (IS_ERR(data))
1580 return PTR_ERR(data);
1581 }
1582 }
1583
1584 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1585 hdr.start, hdr.count, data);
1586 kfree(data);
1587
1588 return ret;
1589 } else if (cmd == VFIO_DEVICE_RESET) {
1590 intel_gvt_ops->vgpu_reset(vgpu);
1591 return 0;
1592 } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1593 struct vfio_device_gfx_plane_info dmabuf;
1594 int ret = 0;
1595
1596 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1597 dmabuf_id);
1598 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1599 return -EFAULT;
1600 if (dmabuf.argsz < minsz)
1601 return -EINVAL;
1602
1603 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf);
1604 if (ret != 0)
1605 return ret;
1606
1607 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1608 -EFAULT : 0;
1609 } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1610 __u32 dmabuf_id;
1611 __s32 dmabuf_fd;
1612
1613 if (get_user(dmabuf_id, (__u32 __user *)arg))
1614 return -EFAULT;
1615
1616 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id);
1617 return dmabuf_fd;
1618
1619 }
1620
1621 return -ENOTTY;
1622 }
1623
1624 static ssize_t
1625 vgpu_id_show(struct device *dev, struct device_attribute *attr,
1626 char *buf)
1627 {
1628 struct mdev_device *mdev = mdev_from_dev(dev);
1629
1630 if (mdev) {
1631 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1632 mdev_get_drvdata(mdev);
1633 return sprintf(buf, "%d\n", vgpu->id);
1634 }
1635 return sprintf(buf, "\n");
1636 }
1637
1638 static DEVICE_ATTR_RO(vgpu_id);
1639
1640 static struct attribute *intel_vgpu_attrs[] = {
1641 &dev_attr_vgpu_id.attr,
1642 NULL
1643 };
1644
1645 static const struct attribute_group intel_vgpu_group = {
1646 .name = "intel_vgpu",
1647 .attrs = intel_vgpu_attrs,
1648 };
1649
1650 static const struct attribute_group *intel_vgpu_groups[] = {
1651 &intel_vgpu_group,
1652 NULL,
1653 };
1654
1655 static struct mdev_parent_ops intel_vgpu_ops = {
1656 .mdev_attr_groups = intel_vgpu_groups,
1657 .create = intel_vgpu_create,
1658 .remove = intel_vgpu_remove,
1659
1660 .open = intel_vgpu_open,
1661 .release = intel_vgpu_release,
1662
1663 .read = intel_vgpu_read,
1664 .write = intel_vgpu_write,
1665 .mmap = intel_vgpu_mmap,
1666 .ioctl = intel_vgpu_ioctl,
1667 };
1668
1669 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
1670 {
1671 struct attribute_group **kvm_vgpu_type_groups;
1672
1673 intel_gvt_ops = ops;
1674 if (!intel_gvt_ops->get_gvt_attrs(&kvm_vgpu_type_groups))
1675 return -EFAULT;
1676 intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
1677
1678 return mdev_register_device(dev, &intel_vgpu_ops);
1679 }
1680
1681 static void kvmgt_host_exit(struct device *dev)
1682 {
1683 mdev_unregister_device(dev);
1684 }
1685
1686 static int kvmgt_page_track_add(unsigned long handle, u64 gfn)
1687 {
1688 struct kvmgt_guest_info *info;
1689 struct kvm *kvm;
1690 struct kvm_memory_slot *slot;
1691 int idx;
1692
1693 if (!handle_valid(handle))
1694 return -ESRCH;
1695
1696 info = (struct kvmgt_guest_info *)handle;
1697 kvm = info->kvm;
1698
1699 idx = srcu_read_lock(&kvm->srcu);
1700 slot = gfn_to_memslot(kvm, gfn);
1701 if (!slot) {
1702 srcu_read_unlock(&kvm->srcu, idx);
1703 return -EINVAL;
1704 }
1705
1706 spin_lock(&kvm->mmu_lock);
1707
1708 if (kvmgt_gfn_is_write_protected(info, gfn))
1709 goto out;
1710
1711 kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1712 kvmgt_protect_table_add(info, gfn);
1713
1714 out:
1715 spin_unlock(&kvm->mmu_lock);
1716 srcu_read_unlock(&kvm->srcu, idx);
1717 return 0;
1718 }
1719
1720 static int kvmgt_page_track_remove(unsigned long handle, u64 gfn)
1721 {
1722 struct kvmgt_guest_info *info;
1723 struct kvm *kvm;
1724 struct kvm_memory_slot *slot;
1725 int idx;
1726
1727 if (!handle_valid(handle))
1728 return 0;
1729
1730 info = (struct kvmgt_guest_info *)handle;
1731 kvm = info->kvm;
1732
1733 idx = srcu_read_lock(&kvm->srcu);
1734 slot = gfn_to_memslot(kvm, gfn);
1735 if (!slot) {
1736 srcu_read_unlock(&kvm->srcu, idx);
1737 return -EINVAL;
1738 }
1739
1740 spin_lock(&kvm->mmu_lock);
1741
1742 if (!kvmgt_gfn_is_write_protected(info, gfn))
1743 goto out;
1744
1745 kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1746 kvmgt_protect_table_del(info, gfn);
1747
1748 out:
1749 spin_unlock(&kvm->mmu_lock);
1750 srcu_read_unlock(&kvm->srcu, idx);
1751 return 0;
1752 }
1753
1754 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1755 const u8 *val, int len,
1756 struct kvm_page_track_notifier_node *node)
1757 {
1758 struct kvmgt_guest_info *info = container_of(node,
1759 struct kvmgt_guest_info, track_node);
1760
1761 if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
1762 intel_gvt_ops->write_protect_handler(info->vgpu, gpa,
1763 (void *)val, len);
1764 }
1765
1766 static void kvmgt_page_track_flush_slot(struct kvm *kvm,
1767 struct kvm_memory_slot *slot,
1768 struct kvm_page_track_notifier_node *node)
1769 {
1770 int i;
1771 gfn_t gfn;
1772 struct kvmgt_guest_info *info = container_of(node,
1773 struct kvmgt_guest_info, track_node);
1774
1775 spin_lock(&kvm->mmu_lock);
1776 for (i = 0; i < slot->npages; i++) {
1777 gfn = slot->base_gfn + i;
1778 if (kvmgt_gfn_is_write_protected(info, gfn)) {
1779 kvm_slot_page_track_remove_page(kvm, slot, gfn,
1780 KVM_PAGE_TRACK_WRITE);
1781 kvmgt_protect_table_del(info, gfn);
1782 }
1783 }
1784 spin_unlock(&kvm->mmu_lock);
1785 }
1786
1787 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm)
1788 {
1789 struct intel_vgpu *itr;
1790 struct kvmgt_guest_info *info;
1791 int id;
1792 bool ret = false;
1793
1794 mutex_lock(&vgpu->gvt->lock);
1795 for_each_active_vgpu(vgpu->gvt, itr, id) {
1796 if (!handle_valid(itr->handle))
1797 continue;
1798
1799 info = (struct kvmgt_guest_info *)itr->handle;
1800 if (kvm && kvm == info->kvm) {
1801 ret = true;
1802 goto out;
1803 }
1804 }
1805 out:
1806 mutex_unlock(&vgpu->gvt->lock);
1807 return ret;
1808 }
1809
1810 static int kvmgt_guest_init(struct mdev_device *mdev)
1811 {
1812 struct kvmgt_guest_info *info;
1813 struct intel_vgpu *vgpu;
1814 struct kvmgt_vdev *vdev;
1815 struct kvm *kvm;
1816
1817 vgpu = mdev_get_drvdata(mdev);
1818 if (handle_valid(vgpu->handle))
1819 return -EEXIST;
1820
1821 vdev = kvmgt_vdev(vgpu);
1822 kvm = vdev->kvm;
1823 if (!kvm || kvm->mm != current->mm) {
1824 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
1825 return -ESRCH;
1826 }
1827
1828 if (__kvmgt_vgpu_exist(vgpu, kvm))
1829 return -EEXIST;
1830
1831 info = vzalloc(sizeof(struct kvmgt_guest_info));
1832 if (!info)
1833 return -ENOMEM;
1834
1835 vgpu->handle = (unsigned long)info;
1836 info->vgpu = vgpu;
1837 info->kvm = kvm;
1838 kvm_get_kvm(info->kvm);
1839
1840 kvmgt_protect_table_init(info);
1841 gvt_cache_init(vgpu);
1842
1843 info->track_node.track_write = kvmgt_page_track_write;
1844 info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
1845 kvm_page_track_register_notifier(kvm, &info->track_node);
1846
1847 info->debugfs_cache_entries = debugfs_create_ulong(
1848 "kvmgt_nr_cache_entries",
1849 0444, vgpu->debugfs,
1850 &vdev->nr_cache_entries);
1851 return 0;
1852 }
1853
1854 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info)
1855 {
1856 debugfs_remove(info->debugfs_cache_entries);
1857
1858 kvm_page_track_unregister_notifier(info->kvm, &info->track_node);
1859 kvm_put_kvm(info->kvm);
1860 kvmgt_protect_table_destroy(info);
1861 gvt_cache_destroy(info->vgpu);
1862 vfree(info);
1863
1864 return true;
1865 }
1866
1867 static int kvmgt_attach_vgpu(void *p_vgpu, unsigned long *handle)
1868 {
1869 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
1870
1871 vgpu->vdev = kzalloc(sizeof(struct kvmgt_vdev), GFP_KERNEL);
1872
1873 if (!vgpu->vdev)
1874 return -ENOMEM;
1875
1876 kvmgt_vdev(vgpu)->vgpu = vgpu;
1877
1878 return 0;
1879 }
1880
1881 static void kvmgt_detach_vgpu(void *p_vgpu)
1882 {
1883 int i;
1884 struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
1885 struct kvmgt_vdev *vdev = kvmgt_vdev(vgpu);
1886
1887 if (!vdev->region)
1888 return;
1889
1890 for (i = 0; i < vdev->num_regions; i++)
1891 if (vdev->region[i].ops->release)
1892 vdev->region[i].ops->release(vgpu,
1893 &vdev->region[i]);
1894 vdev->num_regions = 0;
1895 kfree(vdev->region);
1896 vdev->region = NULL;
1897
1898 kfree(vdev);
1899 }
1900
1901 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
1902 {
1903 struct kvmgt_guest_info *info;
1904 struct intel_vgpu *vgpu;
1905 struct kvmgt_vdev *vdev;
1906
1907 if (!handle_valid(handle))
1908 return -ESRCH;
1909
1910 info = (struct kvmgt_guest_info *)handle;
1911 vgpu = info->vgpu;
1912 vdev = kvmgt_vdev(vgpu);
1913
1914 /*
1915 * When guest is poweroff, msi_trigger is set to NULL, but vgpu's
1916 * config and mmio register isn't restored to default during guest
1917 * poweroff. If this vgpu is still used in next vm, this vgpu's pipe
1918 * may be enabled, then once this vgpu is active, it will get inject
1919 * vblank interrupt request. But msi_trigger is null until msi is
1920 * enabled by guest. so if msi_trigger is null, success is still
1921 * returned and don't inject interrupt into guest.
1922 */
1923 if (vdev->msi_trigger == NULL)
1924 return 0;
1925
1926 if (eventfd_signal(vdev->msi_trigger, 1) == 1)
1927 return 0;
1928
1929 return -EFAULT;
1930 }
1931
1932 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
1933 {
1934 struct kvmgt_guest_info *info;
1935 kvm_pfn_t pfn;
1936
1937 if (!handle_valid(handle))
1938 return INTEL_GVT_INVALID_ADDR;
1939
1940 info = (struct kvmgt_guest_info *)handle;
1941
1942 pfn = gfn_to_pfn(info->kvm, gfn);
1943 if (is_error_noslot_pfn(pfn))
1944 return INTEL_GVT_INVALID_ADDR;
1945
1946 return pfn;
1947 }
1948
1949 static int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
1950 unsigned long size, dma_addr_t *dma_addr)
1951 {
1952 struct intel_vgpu *vgpu;
1953 struct kvmgt_vdev *vdev;
1954 struct gvt_dma *entry;
1955 int ret;
1956
1957 if (!handle_valid(handle))
1958 return -EINVAL;
1959
1960 vgpu = ((struct kvmgt_guest_info *)handle)->vgpu;
1961 vdev = kvmgt_vdev(vgpu);
1962
1963 mutex_lock(&vdev->cache_lock);
1964
1965 entry = __gvt_cache_find_gfn(vgpu, gfn);
1966 if (!entry) {
1967 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1968 if (ret)
1969 goto err_unlock;
1970
1971 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);
1972 if (ret)
1973 goto err_unmap;
1974 } else if (entry->size != size) {
1975 /* the same gfn with different size: unmap and re-map */
1976 gvt_dma_unmap_page(vgpu, gfn, entry->dma_addr, entry->size);
1977 __gvt_cache_remove_entry(vgpu, entry);
1978
1979 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1980 if (ret)
1981 goto err_unlock;
1982
1983 ret = __gvt_cache_add(vgpu, gfn, *dma_addr, size);
1984 if (ret)
1985 goto err_unmap;
1986 } else {
1987 kref_get(&entry->ref);
1988 *dma_addr = entry->dma_addr;
1989 }
1990
1991 mutex_unlock(&vdev->cache_lock);
1992 return 0;
1993
1994 err_unmap:
1995 gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
1996 err_unlock:
1997 mutex_unlock(&vdev->cache_lock);
1998 return ret;
1999 }
2000
2001 static int kvmgt_dma_pin_guest_page(unsigned long handle, dma_addr_t dma_addr)
2002 {
2003 struct kvmgt_guest_info *info;
2004 struct kvmgt_vdev *vdev;
2005 struct gvt_dma *entry;
2006 int ret = 0;
2007
2008 if (!handle_valid(handle))
2009 return -ENODEV;
2010
2011 info = (struct kvmgt_guest_info *)handle;
2012 vdev = kvmgt_vdev(info->vgpu);
2013
2014 mutex_lock(&vdev->cache_lock);
2015 entry = __gvt_cache_find_dma_addr(info->vgpu, dma_addr);
2016 if (entry)
2017 kref_get(&entry->ref);
2018 else
2019 ret = -ENOMEM;
2020 mutex_unlock(&vdev->cache_lock);
2021
2022 return ret;
2023 }
2024
2025 static void __gvt_dma_release(struct kref *ref)
2026 {
2027 struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
2028
2029 gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
2030 entry->size);
2031 __gvt_cache_remove_entry(entry->vgpu, entry);
2032 }
2033
2034 static void kvmgt_dma_unmap_guest_page(unsigned long handle, dma_addr_t dma_addr)
2035 {
2036 struct intel_vgpu *vgpu;
2037 struct kvmgt_vdev *vdev;
2038 struct gvt_dma *entry;
2039
2040 if (!handle_valid(handle))
2041 return;
2042
2043 vgpu = ((struct kvmgt_guest_info *)handle)->vgpu;
2044 vdev = kvmgt_vdev(vgpu);
2045
2046 mutex_lock(&vdev->cache_lock);
2047 entry = __gvt_cache_find_dma_addr(vgpu, dma_addr);
2048 if (entry)
2049 kref_put(&entry->ref, __gvt_dma_release);
2050 mutex_unlock(&vdev->cache_lock);
2051 }
2052
2053 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
2054 void *buf, unsigned long len, bool write)
2055 {
2056 struct kvmgt_guest_info *info;
2057
2058 if (!handle_valid(handle))
2059 return -ESRCH;
2060
2061 info = (struct kvmgt_guest_info *)handle;
2062
2063 return vfio_dma_rw(kvmgt_vdev(info->vgpu)->vfio_group,
2064 gpa, buf, len, write);
2065 }
2066
2067 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
2068 void *buf, unsigned long len)
2069 {
2070 return kvmgt_rw_gpa(handle, gpa, buf, len, false);
2071 }
2072
2073 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
2074 void *buf, unsigned long len)
2075 {
2076 return kvmgt_rw_gpa(handle, gpa, buf, len, true);
2077 }
2078
2079 static unsigned long kvmgt_virt_to_pfn(void *addr)
2080 {
2081 return PFN_DOWN(__pa(addr));
2082 }
2083
2084 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn)
2085 {
2086 struct kvmgt_guest_info *info;
2087 struct kvm *kvm;
2088 int idx;
2089 bool ret;
2090
2091 if (!handle_valid(handle))
2092 return false;
2093
2094 info = (struct kvmgt_guest_info *)handle;
2095 kvm = info->kvm;
2096
2097 idx = srcu_read_lock(&kvm->srcu);
2098 ret = kvm_is_visible_gfn(kvm, gfn);
2099 srcu_read_unlock(&kvm->srcu, idx);
2100
2101 return ret;
2102 }
2103
2104 static const struct intel_gvt_mpt kvmgt_mpt = {
2105 .type = INTEL_GVT_HYPERVISOR_KVM,
2106 .host_init = kvmgt_host_init,
2107 .host_exit = kvmgt_host_exit,
2108 .attach_vgpu = kvmgt_attach_vgpu,
2109 .detach_vgpu = kvmgt_detach_vgpu,
2110 .inject_msi = kvmgt_inject_msi,
2111 .from_virt_to_mfn = kvmgt_virt_to_pfn,
2112 .enable_page_track = kvmgt_page_track_add,
2113 .disable_page_track = kvmgt_page_track_remove,
2114 .read_gpa = kvmgt_read_gpa,
2115 .write_gpa = kvmgt_write_gpa,
2116 .gfn_to_mfn = kvmgt_gfn_to_pfn,
2117 .dma_map_guest_page = kvmgt_dma_map_guest_page,
2118 .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page,
2119 .dma_pin_guest_page = kvmgt_dma_pin_guest_page,
2120 .set_opregion = kvmgt_set_opregion,
2121 .set_edid = kvmgt_set_edid,
2122 .get_vfio_device = kvmgt_get_vfio_device,
2123 .put_vfio_device = kvmgt_put_vfio_device,
2124 .is_valid_gfn = kvmgt_is_valid_gfn,
2125 };
2126
2127 static int __init kvmgt_init(void)
2128 {
2129 if (intel_gvt_register_hypervisor(&kvmgt_mpt) < 0)
2130 return -ENODEV;
2131 return 0;
2132 }
2133
2134 static void __exit kvmgt_exit(void)
2135 {
2136 intel_gvt_unregister_hypervisor();
2137 }
2138
2139 module_init(kvmgt_init);
2140 module_exit(kvmgt_exit);
2141
2142 MODULE_LICENSE("GPL and additional rights");
2143 MODULE_AUTHOR("Intel Corporation");
Linux with added FPC-III support