?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / vfio / pci / vfio_pci.c
blobb0f759476900d1520565810a4ead4cc2cc3a1ac2
1 /*
2 * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
3 * Author: Alex Williamson <alex.williamson@redhat.com>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 as
7 * published by the Free Software Foundation.
8 *
9 * Derived from original vfio:
10 * Copyright 2010 Cisco Systems, Inc. All rights reserved.
11 * Author: Tom Lyon, pugs@cisco.com
12 */
13
14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16 #include <linux/device.h>
17 #include <linux/eventfd.h>
18 #include <linux/file.h>
19 #include <linux/interrupt.h>
20 #include <linux/iommu.h>
21 #include <linux/module.h>
22 #include <linux/mutex.h>
23 #include <linux/notifier.h>
24 #include <linux/pci.h>
25 #include <linux/pm_runtime.h>
26 #include <linux/slab.h>
27 #include <linux/types.h>
28 #include <linux/uaccess.h>
29 #include <linux/vfio.h>
30 #include <linux/vgaarb.h>
31
32 #include "vfio_pci_private.h"
33
34 #define DRIVER_VERSION "0.2"
35 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
36 #define DRIVER_DESC "VFIO PCI - User Level meta-driver"
37
38 static char ids[1024] __initdata;
39 module_param_string(ids, ids, sizeof(ids), 0);
40 MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");
41
42 static bool nointxmask;
43 module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);
44 MODULE_PARM_DESC(nointxmask,
45 "Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");
46
47 #ifdef CONFIG_VFIO_PCI_VGA
48 static bool disable_vga;
49 module_param(disable_vga, bool, S_IRUGO);
50 MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");
51 #endif
52
53 static bool disable_idle_d3;
54 module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR);
55 MODULE_PARM_DESC(disable_idle_d3,
56 "Disable using the PCI D3 low power state for idle, unused devices");
57
58 static DEFINE_MUTEX(driver_lock);
59
60 static inline bool vfio_vga_disabled(void)
61 {
62 #ifdef CONFIG_VFIO_PCI_VGA
63 return disable_vga;
64 #else
65 return true;
66 #endif
67 }
68
69 /*
70 * Our VGA arbiter participation is limited since we don't know anything
71 * about the device itself. However, if the device is the only VGA device
72 * downstream of a bridge and VFIO VGA support is disabled, then we can
73 * safely return legacy VGA IO and memory as not decoded since the user
74 * has no way to get to it and routing can be disabled externally at the
75 * bridge.
76 */
77 static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)
78 {
79 struct vfio_pci_device *vdev = opaque;
80 struct pci_dev *tmp = NULL, *pdev = vdev->pdev;
81 unsigned char max_busnr;
82 unsigned int decodes;
83
84 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))
85 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
86 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
87
88 max_busnr = pci_bus_max_busnr(pdev->bus);
89 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
90
91 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
92 if (tmp == pdev ||
93 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||
94 pci_is_root_bus(tmp->bus))
95 continue;
96
97 if (tmp->bus->number >= pdev->bus->number &&
98 tmp->bus->number <= max_busnr) {
99 pci_dev_put(tmp);
100 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
101 break;
102 }
103 }
104
105 return decodes;
106 }
107
108 static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
109 {
110 return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
111 }
112
113 static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
114 {
115 struct resource *res;
116 int bar;
117 struct vfio_pci_dummy_resource *dummy_res;
118
119 INIT_LIST_HEAD(&vdev->dummy_resources_list);
120
121 for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
122 res = vdev->pdev->resource + bar;
123
124 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
125 goto no_mmap;
126
127 if (!(res->flags & IORESOURCE_MEM))
128 goto no_mmap;
129
130 /*
131 * The PCI core shouldn't set up a resource with a
132 * type but zero size. But there may be bugs that
133 * cause us to do that.
134 */
135 if (!resource_size(res))
136 goto no_mmap;
137
138 if (resource_size(res) >= PAGE_SIZE) {
139 vdev->bar_mmap_supported[bar] = true;
140 continue;
141 }
142
143 if (!(res->start & ~PAGE_MASK)) {
144 /*
145 * Add a dummy resource to reserve the remainder
146 * of the exclusive page in case that hot-add
147 * device's bar is assigned into it.
148 */
149 dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);
150 if (dummy_res == NULL)
151 goto no_mmap;
152
153 dummy_res->resource.name = "vfio sub-page reserved";
154 dummy_res->resource.start = res->end + 1;
155 dummy_res->resource.end = res->start + PAGE_SIZE - 1;
156 dummy_res->resource.flags = res->flags;
157 if (request_resource(res->parent,
158 &dummy_res->resource)) {
159 kfree(dummy_res);
160 goto no_mmap;
161 }
162 dummy_res->index = bar;
163 list_add(&dummy_res->res_next,
164 &vdev->dummy_resources_list);
165 vdev->bar_mmap_supported[bar] = true;
166 continue;
167 }
168 /*
169 * Here we don't handle the case when the BAR is not page
170 * aligned because we can't expect the BAR will be
171 * assigned into the same location in a page in guest
172 * when we passthrough the BAR. And it's hard to access
173 * this BAR in userspace because we have no way to get
174 * the BAR's location in a page.
175 */
176 no_mmap:
177 vdev->bar_mmap_supported[bar] = false;
178 }
179 }
180
181 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);
182 static void vfio_pci_disable(struct vfio_pci_device *vdev);
183
184 /*
185 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
186 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
187 * If a device implements the former but not the latter we would typically
188 * expect broken_intx_masking be set and require an exclusive interrupt.
189 * However since we do have control of the device's ability to assert INTx,
190 * we can instead pretend that the device does not implement INTx, virtualizing
191 * the pin register to report zero and maintaining DisINTx set on the host.
192 */
193 static bool vfio_pci_nointx(struct pci_dev *pdev)
194 {
195 switch (pdev->vendor) {
196 case PCI_VENDOR_ID_INTEL:
197 switch (pdev->device) {
198 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
199 case 0x1572:
200 case 0x1574:
201 case 0x1580 ... 0x1581:
202 case 0x1583 ... 0x158b:
203 case 0x37d0 ... 0x37d2:
204 return true;
205 default:
206 return false;
207 }
208 }
209
210 if (!pdev->irq)
211 return true;
212
213 return false;
214 }
215
216 static int vfio_pci_enable(struct vfio_pci_device *vdev)
217 {
218 struct pci_dev *pdev = vdev->pdev;
219 int ret;
220 u16 cmd;
221 u8 msix_pos;
222
223 pci_set_power_state(pdev, PCI_D0);
224
225 /* Don't allow our initial saved state to include busmaster */
226 pci_clear_master(pdev);
227
228 ret = pci_enable_device(pdev);
229 if (ret)
230 return ret;
231
232 /* If reset fails because of the device lock, fail this path entirely */
233 ret = pci_try_reset_function(pdev);
234 if (ret == -EAGAIN) {
235 pci_disable_device(pdev);
236 return ret;
237 }
238
239 vdev->reset_works = !ret;
240 pci_save_state(pdev);
241 vdev->pci_saved_state = pci_store_saved_state(pdev);
242 if (!vdev->pci_saved_state)
243 pr_debug("%s: Couldn't store %s saved state\n",
244 __func__, dev_name(&pdev->dev));
245
246 if (likely(!nointxmask)) {
247 if (vfio_pci_nointx(pdev)) {
248 dev_info(&pdev->dev, "Masking broken INTx support\n");
249 vdev->nointx = true;
250 pci_intx(pdev, 0);
251 } else
252 vdev->pci_2_3 = pci_intx_mask_supported(pdev);
253 }
254
255 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
256 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
257 cmd &= ~PCI_COMMAND_INTX_DISABLE;
258 pci_write_config_word(pdev, PCI_COMMAND, cmd);
259 }
260
261 ret = vfio_config_init(vdev);
262 if (ret) {
263 kfree(vdev->pci_saved_state);
264 vdev->pci_saved_state = NULL;
265 pci_disable_device(pdev);
266 return ret;
267 }
268
269 msix_pos = pdev->msix_cap;
270 if (msix_pos) {
271 u16 flags;
272 u32 table;
273
274 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
275 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
276
277 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
278 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
279 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
280 } else
281 vdev->msix_bar = 0xFF;
282
283 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
284 vdev->has_vga = true;
285
286
287 if (vfio_pci_is_vga(pdev) &&
288 pdev->vendor == PCI_VENDOR_ID_INTEL &&
289 IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {
290 ret = vfio_pci_igd_init(vdev);
291 if (ret) {
292 dev_warn(&vdev->pdev->dev,
293 "Failed to setup Intel IGD regions\n");
294 vfio_pci_disable(vdev);
295 return ret;
296 }
297 }
298
299 vfio_pci_probe_mmaps(vdev);
300
301 return 0;
302 }
303
304 static void vfio_pci_disable(struct vfio_pci_device *vdev)
305 {
306 struct pci_dev *pdev = vdev->pdev;
307 struct vfio_pci_dummy_resource *dummy_res, *tmp;
308 int i, bar;
309
310 /* Stop the device from further DMA */
311 pci_clear_master(pdev);
312
313 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
314 VFIO_IRQ_SET_ACTION_TRIGGER,
315 vdev->irq_type, 0, 0, NULL);
316
317 vdev->virq_disabled = false;
318
319 for (i = 0; i < vdev->num_regions; i++)
320 vdev->region[i].ops->release(vdev, &vdev->region[i]);
321
322 vdev->num_regions = 0;
323 kfree(vdev->region);
324 vdev->region = NULL; /* don't krealloc a freed pointer */
325
326 vfio_config_free(vdev);
327
328 for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
329 if (!vdev->barmap[bar])
330 continue;
331 pci_iounmap(pdev, vdev->barmap[bar]);
332 pci_release_selected_regions(pdev, 1 << bar);
333 vdev->barmap[bar] = NULL;
334 }
335
336 list_for_each_entry_safe(dummy_res, tmp,
337 &vdev->dummy_resources_list, res_next) {
338 list_del(&dummy_res->res_next);
339 release_resource(&dummy_res->resource);
340 kfree(dummy_res);
341 }
342
343 vdev->needs_reset = true;
344
345 /*
346 * If we have saved state, restore it. If we can reset the device,
347 * even better. Resetting with current state seems better than
348 * nothing, but saving and restoring current state without reset
349 * is just busy work.
350 */
351 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
352 pr_info("%s: Couldn't reload %s saved state\n",
353 __func__, dev_name(&pdev->dev));
354
355 if (!vdev->reset_works)
356 goto out;
357
358 pci_save_state(pdev);
359 }
360
361 /*
362 * Disable INTx and MSI, presumably to avoid spurious interrupts
363 * during reset. Stolen from pci_reset_function()
364 */
365 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
366
367 /*
368 * Try to reset the device. The success of this is dependent on
369 * being able to lock the device, which is not always possible.
370 */
371 if (vdev->reset_works && !pci_try_reset_function(pdev))
372 vdev->needs_reset = false;
373
374 pci_restore_state(pdev);
375 out:
376 pci_disable_device(pdev);
377
378 vfio_pci_try_bus_reset(vdev);
379
380 if (!disable_idle_d3)
381 pci_set_power_state(pdev, PCI_D3hot);
382 }
383
384 static void vfio_pci_release(void *device_data)
385 {
386 struct vfio_pci_device *vdev = device_data;
387
388 mutex_lock(&driver_lock);
389
390 if (!(--vdev->refcnt)) {
391 vfio_spapr_pci_eeh_release(vdev->pdev);
392 vfio_pci_disable(vdev);
393 }
394
395 mutex_unlock(&driver_lock);
396
397 module_put(THIS_MODULE);
398 }
399
400 static int vfio_pci_open(void *device_data)
401 {
402 struct vfio_pci_device *vdev = device_data;
403 int ret = 0;
404
405 if (!try_module_get(THIS_MODULE))
406 return -ENODEV;
407
408 mutex_lock(&driver_lock);
409
410 if (!vdev->refcnt) {
411 ret = vfio_pci_enable(vdev);
412 if (ret)
413 goto error;
414
415 vfio_spapr_pci_eeh_open(vdev->pdev);
416 }
417 vdev->refcnt++;
418 error:
419 mutex_unlock(&driver_lock);
420 if (ret)
421 module_put(THIS_MODULE);
422 return ret;
423 }
424
425 static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
426 {
427 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
428 u8 pin;
429 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
430 if (IS_ENABLED(CONFIG_VFIO_PCI_INTX) && !vdev->nointx && pin)
431 return 1;
432
433 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
434 u8 pos;
435 u16 flags;
436
437 pos = vdev->pdev->msi_cap;
438 if (pos) {
439 pci_read_config_word(vdev->pdev,
440 pos + PCI_MSI_FLAGS, &flags);
441 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
442 }
443 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
444 u8 pos;
445 u16 flags;
446
447 pos = vdev->pdev->msix_cap;
448 if (pos) {
449 pci_read_config_word(vdev->pdev,
450 pos + PCI_MSIX_FLAGS, &flags);
451
452 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
453 }
454 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
455 if (pci_is_pcie(vdev->pdev))
456 return 1;
457 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
458 return 1;
459 }
460
461 return 0;
462 }
463
464 static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)
465 {
466 (*(int *)data)++;
467 return 0;
468 }
469
470 struct vfio_pci_fill_info {
471 int max;
472 int cur;
473 struct vfio_pci_dependent_device *devices;
474 };
475
476 static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)
477 {
478 struct vfio_pci_fill_info *fill = data;
479 struct iommu_group *iommu_group;
480
481 if (fill->cur == fill->max)
482 return -EAGAIN; /* Something changed, try again */
483
484 iommu_group = iommu_group_get(&pdev->dev);
485 if (!iommu_group)
486 return -EPERM; /* Cannot reset non-isolated devices */
487
488 fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);
489 fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
490 fill->devices[fill->cur].bus = pdev->bus->number;
491 fill->devices[fill->cur].devfn = pdev->devfn;
492 fill->cur++;
493 iommu_group_put(iommu_group);
494 return 0;
495 }
496
497 struct vfio_pci_group_entry {
498 struct vfio_group *group;
499 int id;
500 };
501
502 struct vfio_pci_group_info {
503 int count;
504 struct vfio_pci_group_entry *groups;
505 };
506
507 static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data)
508 {
509 struct vfio_pci_group_info *info = data;
510 struct iommu_group *group;
511 int id, i;
512
513 group = iommu_group_get(&pdev->dev);
514 if (!group)
515 return -EPERM;
516
517 id = iommu_group_id(group);
518
519 for (i = 0; i < info->count; i++)
520 if (info->groups[i].id == id)
521 break;
522
523 iommu_group_put(group);
524
525 return (i == info->count) ? -EINVAL : 0;
526 }
527
528 static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)
529 {
530 for (; pdev; pdev = pdev->bus->self)
531 if (pdev->bus == slot->bus)
532 return (pdev->slot == slot);
533 return false;
534 }
535
536 struct vfio_pci_walk_info {
537 int (*fn)(struct pci_dev *, void *data);
538 void *data;
539 struct pci_dev *pdev;
540 bool slot;
541 int ret;
542 };
543
544 static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)
545 {
546 struct vfio_pci_walk_info *walk = data;
547
548 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
549 walk->ret = walk->fn(pdev, walk->data);
550
551 return walk->ret;
552 }
553
554 static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
555 int (*fn)(struct pci_dev *,
556 void *data), void *data,
557 bool slot)
558 {
559 struct vfio_pci_walk_info walk = {
560 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
561 };
562
563 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
564
565 return walk.ret;
566 }
567
568 static int msix_mmappable_cap(struct vfio_pci_device *vdev,
569 struct vfio_info_cap *caps)
570 {
571 struct vfio_info_cap_header header = {
572 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,
573 .version = 1
574 };
575
576 return vfio_info_add_capability(caps, &header, sizeof(header));
577 }
578
579 int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
580 unsigned int type, unsigned int subtype,
581 const struct vfio_pci_regops *ops,
582 size_t size, u32 flags, void *data)
583 {
584 struct vfio_pci_region *region;
585
586 region = krealloc(vdev->region,
587 (vdev->num_regions + 1) * sizeof(*region),
588 GFP_KERNEL);
589 if (!region)
590 return -ENOMEM;
591
592 vdev->region = region;
593 vdev->region[vdev->num_regions].type = type;
594 vdev->region[vdev->num_regions].subtype = subtype;
595 vdev->region[vdev->num_regions].ops = ops;
596 vdev->region[vdev->num_regions].size = size;
597 vdev->region[vdev->num_regions].flags = flags;
598 vdev->region[vdev->num_regions].data = data;
599
600 vdev->num_regions++;
601
602 return 0;
603 }
604
605 static long vfio_pci_ioctl(void *device_data,
606 unsigned int cmd, unsigned long arg)
607 {
608 struct vfio_pci_device *vdev = device_data;
609 unsigned long minsz;
610
611 if (cmd == VFIO_DEVICE_GET_INFO) {
612 struct vfio_device_info info;
613
614 minsz = offsetofend(struct vfio_device_info, num_irqs);
615
616 if (copy_from_user(&info, (void __user *)arg, minsz))
617 return -EFAULT;
618
619 if (info.argsz < minsz)
620 return -EINVAL;
621
622 info.flags = VFIO_DEVICE_FLAGS_PCI;
623
624 if (vdev->reset_works)
625 info.flags |= VFIO_DEVICE_FLAGS_RESET;
626
627 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
628 info.num_irqs = VFIO_PCI_NUM_IRQS;
629
630 return copy_to_user((void __user *)arg, &info, minsz) ?
631 -EFAULT : 0;
632
633 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
634 struct pci_dev *pdev = vdev->pdev;
635 struct vfio_region_info info;
636 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
637 int i, ret;
638
639 minsz = offsetofend(struct vfio_region_info, offset);
640
641 if (copy_from_user(&info, (void __user *)arg, minsz))
642 return -EFAULT;
643
644 if (info.argsz < minsz)
645 return -EINVAL;
646
647 switch (info.index) {
648 case VFIO_PCI_CONFIG_REGION_INDEX:
649 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
650 info.size = pdev->cfg_size;
651 info.flags = VFIO_REGION_INFO_FLAG_READ |
652 VFIO_REGION_INFO_FLAG_WRITE;
653 break;
654 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
655 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
656 info.size = pci_resource_len(pdev, info.index);
657 if (!info.size) {
658 info.flags = 0;
659 break;
660 }
661
662 info.flags = VFIO_REGION_INFO_FLAG_READ |
663 VFIO_REGION_INFO_FLAG_WRITE;
664 if (vdev->bar_mmap_supported[info.index]) {
665 info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
666 if (info.index == vdev->msix_bar) {
667 ret = msix_mmappable_cap(vdev, &caps);
668 if (ret)
669 return ret;
670 }
671 }
672
673 break;
674 case VFIO_PCI_ROM_REGION_INDEX:
675 {
676 void __iomem *io;
677 size_t size;
678
679 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
680 info.flags = 0;
681
682 /* Report the BAR size, not the ROM size */
683 info.size = pci_resource_len(pdev, info.index);
684 if (!info.size) {
685 /* Shadow ROMs appear as PCI option ROMs */
686 if (pdev->resource[PCI_ROM_RESOURCE].flags &
687 IORESOURCE_ROM_SHADOW)
688 info.size = 0x20000;
689 else
690 break;
691 }
692
693 /* Is it really there? */
694 io = pci_map_rom(pdev, &size);
695 if (!io || !size) {
696 info.size = 0;
697 break;
698 }
699 pci_unmap_rom(pdev, io);
700
701 info.flags = VFIO_REGION_INFO_FLAG_READ;
702 break;
703 }
704 case VFIO_PCI_VGA_REGION_INDEX:
705 if (!vdev->has_vga)
706 return -EINVAL;
707
708 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
709 info.size = 0xc0000;
710 info.flags = VFIO_REGION_INFO_FLAG_READ |
711 VFIO_REGION_INFO_FLAG_WRITE;
712
713 break;
714 default:
715 {
716 struct vfio_region_info_cap_type cap_type = {
717 .header.id = VFIO_REGION_INFO_CAP_TYPE,
718 .header.version = 1 };
719
720 if (info.index >=
721 VFIO_PCI_NUM_REGIONS + vdev->num_regions)
722 return -EINVAL;
723
724 i = info.index - VFIO_PCI_NUM_REGIONS;
725
726 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
727 info.size = vdev->region[i].size;
728 info.flags = vdev->region[i].flags;
729
730 cap_type.type = vdev->region[i].type;
731 cap_type.subtype = vdev->region[i].subtype;
732
733 ret = vfio_info_add_capability(&caps, &cap_type.header,
734 sizeof(cap_type));
735 if (ret)
736 return ret;
737
738 }
739 }
740
741 if (caps.size) {
742 info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
743 if (info.argsz < sizeof(info) + caps.size) {
744 info.argsz = sizeof(info) + caps.size;
745 info.cap_offset = 0;
746 } else {
747 vfio_info_cap_shift(&caps, sizeof(info));
748 if (copy_to_user((void __user *)arg +
749 sizeof(info), caps.buf,
750 caps.size)) {
751 kfree(caps.buf);
752 return -EFAULT;
753 }
754 info.cap_offset = sizeof(info);
755 }
756
757 kfree(caps.buf);
758 }
759
760 return copy_to_user((void __user *)arg, &info, minsz) ?
761 -EFAULT : 0;
762
763 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
764 struct vfio_irq_info info;
765
766 minsz = offsetofend(struct vfio_irq_info, count);
767
768 if (copy_from_user(&info, (void __user *)arg, minsz))
769 return -EFAULT;
770
771 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
772 return -EINVAL;
773
774 switch (info.index) {
775 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
776 case VFIO_PCI_REQ_IRQ_INDEX:
777 break;
778 case VFIO_PCI_ERR_IRQ_INDEX:
779 if (pci_is_pcie(vdev->pdev))
780 break;
781 /* pass thru to return error */
782 default:
783 return -EINVAL;
784 }
785
786 info.flags = VFIO_IRQ_INFO_EVENTFD;
787
788 info.count = vfio_pci_get_irq_count(vdev, info.index);
789
790 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
791 info.flags |= (VFIO_IRQ_INFO_MASKABLE |
792 VFIO_IRQ_INFO_AUTOMASKED);
793 else
794 info.flags |= VFIO_IRQ_INFO_NORESIZE;
795
796 return copy_to_user((void __user *)arg, &info, minsz) ?
797 -EFAULT : 0;
798
799 } else if (cmd == VFIO_DEVICE_SET_IRQS) {
800 struct vfio_irq_set hdr;
801 u8 *data = NULL;
802 int max, ret = 0;
803 size_t data_size = 0;
804
805 minsz = offsetofend(struct vfio_irq_set, count);
806
807 if (copy_from_user(&hdr, (void __user *)arg, minsz))
808 return -EFAULT;
809
810 max = vfio_pci_get_irq_count(vdev, hdr.index);
811
812 ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
813 VFIO_PCI_NUM_IRQS, &data_size);
814 if (ret)
815 return ret;
816
817 if (data_size) {
818 data = memdup_user((void __user *)(arg + minsz),
819 data_size);
820 if (IS_ERR(data))
821 return PTR_ERR(data);
822 }
823
824 mutex_lock(&vdev->igate);
825
826 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
827 hdr.start, hdr.count, data);
828
829 mutex_unlock(&vdev->igate);
830 kfree(data);
831
832 return ret;
833
834 } else if (cmd == VFIO_DEVICE_RESET) {
835 return vdev->reset_works ?
836 pci_try_reset_function(vdev->pdev) : -EINVAL;
837
838 } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
839 struct vfio_pci_hot_reset_info hdr;
840 struct vfio_pci_fill_info fill = { 0 };
841 struct vfio_pci_dependent_device *devices = NULL;
842 bool slot = false;
843 int ret = 0;
844
845 minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
846
847 if (copy_from_user(&hdr, (void __user *)arg, minsz))
848 return -EFAULT;
849
850 if (hdr.argsz < minsz)
851 return -EINVAL;
852
853 hdr.flags = 0;
854
855 /* Can we do a slot or bus reset or neither? */
856 if (!pci_probe_reset_slot(vdev->pdev->slot))
857 slot = true;
858 else if (pci_probe_reset_bus(vdev->pdev->bus))
859 return -ENODEV;
860
861 /* How many devices are affected? */
862 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
863 vfio_pci_count_devs,
864 &fill.max, slot);
865 if (ret)
866 return ret;
867
868 WARN_ON(!fill.max); /* Should always be at least one */
869
870 /*
871 * If there's enough space, fill it now, otherwise return
872 * -ENOSPC and the number of devices affected.
873 */
874 if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
875 ret = -ENOSPC;
876 hdr.count = fill.max;
877 goto reset_info_exit;
878 }
879
880 devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
881 if (!devices)
882 return -ENOMEM;
883
884 fill.devices = devices;
885
886 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
887 vfio_pci_fill_devs,
888 &fill, slot);
889
890 /*
891 * If a device was removed between counting and filling,
892 * we may come up short of fill.max. If a device was
893 * added, we'll have a return of -EAGAIN above.
894 */
895 if (!ret)
896 hdr.count = fill.cur;
897
898 reset_info_exit:
899 if (copy_to_user((void __user *)arg, &hdr, minsz))
900 ret = -EFAULT;
901
902 if (!ret) {
903 if (copy_to_user((void __user *)(arg + minsz), devices,
904 hdr.count * sizeof(*devices)))
905 ret = -EFAULT;
906 }
907
908 kfree(devices);
909 return ret;
910
911 } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
912 struct vfio_pci_hot_reset hdr;
913 int32_t *group_fds;
914 struct vfio_pci_group_entry *groups;
915 struct vfio_pci_group_info info;
916 bool slot = false;
917 int i, count = 0, ret = 0;
918
919 minsz = offsetofend(struct vfio_pci_hot_reset, count);
920
921 if (copy_from_user(&hdr, (void __user *)arg, minsz))
922 return -EFAULT;
923
924 if (hdr.argsz < minsz || hdr.flags)
925 return -EINVAL;
926
927 /* Can we do a slot or bus reset or neither? */
928 if (!pci_probe_reset_slot(vdev->pdev->slot))
929 slot = true;
930 else if (pci_probe_reset_bus(vdev->pdev->bus))
931 return -ENODEV;
932
933 /*
934 * We can't let userspace give us an arbitrarily large
935 * buffer to copy, so verify how many we think there
936 * could be. Note groups can have multiple devices so
937 * one group per device is the max.
938 */
939 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
940 vfio_pci_count_devs,
941 &count, slot);
942 if (ret)
943 return ret;
944
945 /* Somewhere between 1 and count is OK */
946 if (!hdr.count || hdr.count > count)
947 return -EINVAL;
948
949 group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
950 groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
951 if (!group_fds || !groups) {
952 kfree(group_fds);
953 kfree(groups);
954 return -ENOMEM;
955 }
956
957 if (copy_from_user(group_fds, (void __user *)(arg + minsz),
958 hdr.count * sizeof(*group_fds))) {
959 kfree(group_fds);
960 kfree(groups);
961 return -EFAULT;
962 }
963
964 /*
965 * For each group_fd, get the group through the vfio external
966 * user interface and store the group and iommu ID. This
967 * ensures the group is held across the reset.
968 */
969 for (i = 0; i < hdr.count; i++) {
970 struct vfio_group *group;
971 struct fd f = fdget(group_fds[i]);
972 if (!f.file) {
973 ret = -EBADF;
974 break;
975 }
976
977 group = vfio_group_get_external_user(f.file);
978 fdput(f);
979 if (IS_ERR(group)) {
980 ret = PTR_ERR(group);
981 break;
982 }
983
984 groups[i].group = group;
985 groups[i].id = vfio_external_user_iommu_id(group);
986 }
987
988 kfree(group_fds);
989
990 /* release reference to groups on error */
991 if (ret)
992 goto hot_reset_release;
993
994 info.count = hdr.count;
995 info.groups = groups;
996
997 /*
998 * Test whether all the affected devices are contained
999 * by the set of groups provided by the user.
1000 */
1001 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1002 vfio_pci_validate_devs,
1003 &info, slot);
1004 if (!ret)
1005 /* User has access, do the reset */
1006 ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
1007 pci_try_reset_bus(vdev->pdev->bus);
1008
1009 hot_reset_release:
1010 for (i--; i >= 0; i--)
1011 vfio_group_put_external_user(groups[i].group);
1012
1013 kfree(groups);
1014 return ret;
1015 }
1016
1017 return -ENOTTY;
1018 }
1019
1020 static ssize_t vfio_pci_rw(void *device_data, char __user *buf,
1021 size_t count, loff_t *ppos, bool iswrite)
1022 {
1023 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1024 struct vfio_pci_device *vdev = device_data;
1025
1026 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1027 return -EINVAL;
1028
1029 switch (index) {
1030 case VFIO_PCI_CONFIG_REGION_INDEX:
1031 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1032
1033 case VFIO_PCI_ROM_REGION_INDEX:
1034 if (iswrite)
1035 return -EINVAL;
1036 return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1037
1038 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1039 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1040
1041 case VFIO_PCI_VGA_REGION_INDEX:
1042 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1043 default:
1044 index -= VFIO_PCI_NUM_REGIONS;
1045 return vdev->region[index].ops->rw(vdev, buf,
1046 count, ppos, iswrite);
1047 }
1048
1049 return -EINVAL;
1050 }
1051
1052 static ssize_t vfio_pci_read(void *device_data, char __user *buf,
1053 size_t count, loff_t *ppos)
1054 {
1055 if (!count)
1056 return 0;
1057
1058 return vfio_pci_rw(device_data, buf, count, ppos, false);
1059 }
1060
1061 static ssize_t vfio_pci_write(void *device_data, const char __user *buf,
1062 size_t count, loff_t *ppos)
1063 {
1064 if (!count)
1065 return 0;
1066
1067 return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);
1068 }
1069
1070 static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
1071 {
1072 struct vfio_pci_device *vdev = device_data;
1073 struct pci_dev *pdev = vdev->pdev;
1074 unsigned int index;
1075 u64 phys_len, req_len, pgoff, req_start;
1076 int ret;
1077
1078 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1079
1080 if (vma->vm_end < vma->vm_start)
1081 return -EINVAL;
1082 if ((vma->vm_flags & VM_SHARED) == 0)
1083 return -EINVAL;
1084 if (index >= VFIO_PCI_ROM_REGION_INDEX)
1085 return -EINVAL;
1086 if (!vdev->bar_mmap_supported[index])
1087 return -EINVAL;
1088
1089 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
1090 req_len = vma->vm_end - vma->vm_start;
1091 pgoff = vma->vm_pgoff &
1092 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1093 req_start = pgoff << PAGE_SHIFT;
1094
1095 if (req_start + req_len > phys_len)
1096 return -EINVAL;
1097
1098 /*
1099 * Even though we don't make use of the barmap for the mmap,
1100 * we need to request the region and the barmap tracks that.
1101 */
1102 if (!vdev->barmap[index]) {
1103 ret = pci_request_selected_regions(pdev,
1104 1 << index, "vfio-pci");
1105 if (ret)
1106 return ret;
1107
1108 vdev->barmap[index] = pci_iomap(pdev, index, 0);
1109 if (!vdev->barmap[index]) {
1110 pci_release_selected_regions(pdev, 1 << index);
1111 return -ENOMEM;
1112 }
1113 }
1114
1115 vma->vm_private_data = vdev;
1116 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1117 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
1118
1119 return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1120 req_len, vma->vm_page_prot);
1121 }
1122
1123 static void vfio_pci_request(void *device_data, unsigned int count)
1124 {
1125 struct vfio_pci_device *vdev = device_data;
1126
1127 mutex_lock(&vdev->igate);
1128
1129 if (vdev->req_trigger) {
1130 if (!(count % 10))
1131 dev_notice_ratelimited(&vdev->pdev->dev,
1132 "Relaying device request to user (#%u)\n",
1133 count);
1134 eventfd_signal(vdev->req_trigger, 1);
1135 } else if (count == 0) {
1136 dev_warn(&vdev->pdev->dev,
1137 "No device request channel registered, blocked until released by user\n");
1138 }
1139
1140 mutex_unlock(&vdev->igate);
1141 }
1142
1143 static const struct vfio_device_ops vfio_pci_ops = {
1144 .name = "vfio-pci",
1145 .open = vfio_pci_open,
1146 .release = vfio_pci_release,
1147 .ioctl = vfio_pci_ioctl,
1148 .read = vfio_pci_read,
1149 .write = vfio_pci_write,
1150 .mmap = vfio_pci_mmap,
1151 .request = vfio_pci_request,
1152 };
1153
1154 static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1155 {
1156 struct vfio_pci_device *vdev;
1157 struct iommu_group *group;
1158 int ret;
1159
1160 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
1161 return -EINVAL;
1162
1163 group = vfio_iommu_group_get(&pdev->dev);
1164 if (!group)
1165 return -EINVAL;
1166
1167 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
1168 if (!vdev) {
1169 vfio_iommu_group_put(group, &pdev->dev);
1170 return -ENOMEM;
1171 }
1172
1173 vdev->pdev = pdev;
1174 vdev->irq_type = VFIO_PCI_NUM_IRQS;
1175 mutex_init(&vdev->igate);
1176 spin_lock_init(&vdev->irqlock);
1177
1178 ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
1179 if (ret) {
1180 vfio_iommu_group_put(group, &pdev->dev);
1181 kfree(vdev);
1182 return ret;
1183 }
1184
1185 if (vfio_pci_is_vga(pdev)) {
1186 vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
1187 vga_set_legacy_decoding(pdev,
1188 vfio_pci_set_vga_decode(vdev, false));
1189 }
1190
1191 if (!disable_idle_d3) {
1192 /*
1193 * pci-core sets the device power state to an unknown value at
1194 * bootup and after being removed from a driver. The only
1195 * transition it allows from this unknown state is to D0, which
1196 * typically happens when a driver calls pci_enable_device().
1197 * We're not ready to enable the device yet, but we do want to
1198 * be able to get to D3. Therefore first do a D0 transition
1199 * before going to D3.
1200 */
1201 pci_set_power_state(pdev, PCI_D0);
1202 pci_set_power_state(pdev, PCI_D3hot);
1203 }
1204
1205 return ret;
1206 }
1207
1208 static void vfio_pci_remove(struct pci_dev *pdev)
1209 {
1210 struct vfio_pci_device *vdev;
1211
1212 vdev = vfio_del_group_dev(&pdev->dev);
1213 if (!vdev)
1214 return;
1215
1216 vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
1217 kfree(vdev->region);
1218 kfree(vdev);
1219
1220 if (vfio_pci_is_vga(pdev)) {
1221 vga_client_register(pdev, NULL, NULL, NULL);
1222 vga_set_legacy_decoding(pdev,
1223 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
1224 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM);
1225 }
1226
1227 if (!disable_idle_d3)
1228 pci_set_power_state(pdev, PCI_D0);
1229 }
1230
1231 static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
1232 pci_channel_state_t state)
1233 {
1234 struct vfio_pci_device *vdev;
1235 struct vfio_device *device;
1236
1237 device = vfio_device_get_from_dev(&pdev->dev);
1238 if (device == NULL)
1239 return PCI_ERS_RESULT_DISCONNECT;
1240
1241 vdev = vfio_device_data(device);
1242 if (vdev == NULL) {
1243 vfio_device_put(device);
1244 return PCI_ERS_RESULT_DISCONNECT;
1245 }
1246
1247 mutex_lock(&vdev->igate);
1248
1249 if (vdev->err_trigger)
1250 eventfd_signal(vdev->err_trigger, 1);
1251
1252 mutex_unlock(&vdev->igate);
1253
1254 vfio_device_put(device);
1255
1256 return PCI_ERS_RESULT_CAN_RECOVER;
1257 }
1258
1259 static const struct pci_error_handlers vfio_err_handlers = {
1260 .error_detected = vfio_pci_aer_err_detected,
1261 };
1262
1263 static struct pci_driver vfio_pci_driver = {
1264 .name = "vfio-pci",
1265 .id_table = NULL, /* only dynamic ids */
1266 .probe = vfio_pci_probe,
1267 .remove = vfio_pci_remove,
1268 .err_handler = &vfio_err_handlers,
1269 };
1270
1271 struct vfio_devices {
1272 struct vfio_device **devices;
1273 int cur_index;
1274 int max_index;
1275 };
1276
1277 static int vfio_pci_get_devs(struct pci_dev *pdev, void *data)
1278 {
1279 struct vfio_devices *devs = data;
1280 struct vfio_device *device;
1281
1282 if (devs->cur_index == devs->max_index)
1283 return -ENOSPC;
1284
1285 device = vfio_device_get_from_dev(&pdev->dev);
1286 if (!device)
1287 return -EINVAL;
1288
1289 if (pci_dev_driver(pdev) != &vfio_pci_driver) {
1290 vfio_device_put(device);
1291 return -EBUSY;
1292 }
1293
1294 devs->devices[devs->cur_index++] = device;
1295 return 0;
1296 }
1297
1298 /*
1299 * Attempt to do a bus/slot reset if there are devices affected by a reset for
1300 * this device that are needs_reset and all of the affected devices are unused
1301 * (!refcnt). Callers are required to hold driver_lock when calling this to
1302 * prevent device opens and concurrent bus reset attempts. We prevent device
1303 * unbinds by acquiring and holding a reference to the vfio_device.
1304 *
1305 * NB: vfio-core considers a group to be viable even if some devices are
1306 * bound to drivers like pci-stub or pcieport. Here we require all devices
1307 * to be bound to vfio_pci since that's the only way we can be sure they
1308 * stay put.
1309 */
1310 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
1311 {
1312 struct vfio_devices devs = { .cur_index = 0 };
1313 int i = 0, ret = -EINVAL;
1314 bool needs_reset = false, slot = false;
1315 struct vfio_pci_device *tmp;
1316
1317 if (!pci_probe_reset_slot(vdev->pdev->slot))
1318 slot = true;
1319 else if (pci_probe_reset_bus(vdev->pdev->bus))
1320 return;
1321
1322 if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
1323 &i, slot) || !i)
1324 return;
1325
1326 devs.max_index = i;
1327 devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL);
1328 if (!devs.devices)
1329 return;
1330
1331 if (vfio_pci_for_each_slot_or_bus(vdev->pdev,
1332 vfio_pci_get_devs, &devs, slot))
1333 goto put_devs;
1334
1335 for (i = 0; i < devs.cur_index; i++) {
1336 tmp = vfio_device_data(devs.devices[i]);
1337 if (tmp->needs_reset)
1338 needs_reset = true;
1339 if (tmp->refcnt)
1340 goto put_devs;
1341 }
1342
1343 if (needs_reset)
1344 ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
1345 pci_try_reset_bus(vdev->pdev->bus);
1346
1347 put_devs:
1348 for (i = 0; i < devs.cur_index; i++) {
1349 tmp = vfio_device_data(devs.devices[i]);
1350 if (!ret)
1351 tmp->needs_reset = false;
1352
1353 if (!tmp->refcnt && !disable_idle_d3)
1354 pci_set_power_state(tmp->pdev, PCI_D3hot);
1355
1356 vfio_device_put(devs.devices[i]);
1357 }
1358
1359 kfree(devs.devices);
1360 }
1361
1362 static void __exit vfio_pci_cleanup(void)
1363 {
1364 pci_unregister_driver(&vfio_pci_driver);
1365 vfio_pci_uninit_perm_bits();
1366 }
1367
1368 static void __init vfio_pci_fill_ids(void)
1369 {
1370 char *p, *id;
1371 int rc;
1372
1373 /* no ids passed actually */
1374 if (ids[0] == '\0')
1375 return;
1376
1377 /* add ids specified in the module parameter */
1378 p = ids;
1379 while ((id = strsep(&p, ","))) {
1380 unsigned int vendor, device, subvendor = PCI_ANY_ID,
1381 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
1382 int fields;
1383
1384 if (!strlen(id))
1385 continue;
1386
1387 fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
1388 &vendor, &device, &subvendor, &subdevice,
1389 &class, &class_mask);
1390
1391 if (fields < 2) {
1392 pr_warn("invalid id string \"%s\"\n", id);
1393 continue;
1394 }
1395
1396 rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
1397 subvendor, subdevice, class, class_mask, 0);
1398 if (rc)
1399 pr_warn("failed to add dynamic id [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x (%d)\n",
1400 vendor, device, subvendor, subdevice,
1401 class, class_mask, rc);
1402 else
1403 pr_info("add [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x\n",
1404 vendor, device, subvendor, subdevice,
1405 class, class_mask);
1406 }
1407 }
1408
1409 static int __init vfio_pci_init(void)
1410 {
1411 int ret;
1412
1413 /* Allocate shared config space permision data used by all devices */
1414 ret = vfio_pci_init_perm_bits();
1415 if (ret)
1416 return ret;
1417
1418 /* Register and scan for devices */
1419 ret = pci_register_driver(&vfio_pci_driver);
1420 if (ret)
1421 goto out_driver;
1422
1423 vfio_pci_fill_ids();
1424
1425 return 0;
1426
1427 out_driver:
1428 vfio_pci_uninit_perm_bits();
1429 return ret;
1430 }
1431
1432 module_init(vfio_pci_init);
1433 module_exit(vfio_pci_cleanup);
1434
1435 MODULE_VERSION(DRIVER_VERSION);
1436 MODULE_LICENSE("GPL v2");
1437 MODULE_AUTHOR(DRIVER_AUTHOR);
1438 MODULE_DESCRIPTION(DRIVER_DESC);
CRIS linux kernel tree