4 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
6 * PCI Express I/O Virtualization (IOV) support.
8 * Address Translation Service 1.0
11 #include <linux/pci.h>
12 #include <linux/slab.h>
13 #include <linux/mutex.h>
14 #include <linux/export.h>
15 #include <linux/string.h>
16 #include <linux/delay.h>
17 #include <linux/pci-ats.h>
20 #define VIRTFN_ID_LEN 16
22 static inline u8
virtfn_bus(struct pci_dev
*dev
, int id
)
24 return dev
->bus
->number
+ ((dev
->devfn
+ dev
->sriov
->offset
+
25 dev
->sriov
->stride
* id
) >> 8);
28 static inline u8
virtfn_devfn(struct pci_dev
*dev
, int id
)
30 return (dev
->devfn
+ dev
->sriov
->offset
+
31 dev
->sriov
->stride
* id
) & 0xff;
34 static struct pci_bus
*virtfn_add_bus(struct pci_bus
*bus
, int busnr
)
36 struct pci_bus
*child
;
38 if (bus
->number
== busnr
)
41 child
= pci_find_bus(pci_domain_nr(bus
), busnr
);
45 child
= pci_add_new_bus(bus
, NULL
, busnr
);
49 pci_bus_insert_busn_res(child
, busnr
, busnr
);
54 static void virtfn_remove_bus(struct pci_bus
*physbus
, struct pci_bus
*virtbus
)
56 if (physbus
!= virtbus
&& list_empty(&virtbus
->devices
))
57 pci_remove_bus(virtbus
);
60 static int virtfn_add(struct pci_dev
*dev
, int id
, int reset
)
65 char buf
[VIRTFN_ID_LEN
];
66 struct pci_dev
*virtfn
;
68 struct pci_sriov
*iov
= dev
->sriov
;
71 mutex_lock(&iov
->dev
->sriov
->lock
);
72 bus
= virtfn_add_bus(dev
->bus
, virtfn_bus(dev
, id
));
76 virtfn
= pci_alloc_dev(bus
);
80 virtfn
->devfn
= virtfn_devfn(dev
, id
);
81 virtfn
->vendor
= dev
->vendor
;
82 pci_read_config_word(dev
, iov
->pos
+ PCI_SRIOV_VF_DID
, &virtfn
->device
);
83 pci_setup_device(virtfn
);
84 virtfn
->dev
.parent
= dev
->dev
.parent
;
85 virtfn
->physfn
= pci_dev_get(dev
);
86 virtfn
->is_virtfn
= 1;
87 virtfn
->multifunction
= 0;
89 for (i
= 0; i
< PCI_SRIOV_NUM_BARS
; i
++) {
90 res
= dev
->resource
+ PCI_IOV_RESOURCES
+ i
;
93 virtfn
->resource
[i
].name
= pci_name(virtfn
);
94 virtfn
->resource
[i
].flags
= res
->flags
;
95 size
= resource_size(res
);
96 do_div(size
, iov
->total_VFs
);
97 virtfn
->resource
[i
].start
= res
->start
+ size
* id
;
98 virtfn
->resource
[i
].end
= virtfn
->resource
[i
].start
+ size
- 1;
99 rc
= request_resource(res
, &virtfn
->resource
[i
]);
104 __pci_reset_function(virtfn
);
106 pci_device_add(virtfn
, virtfn
->bus
);
107 mutex_unlock(&iov
->dev
->sriov
->lock
);
109 pci_bus_add_device(virtfn
);
110 sprintf(buf
, "virtfn%u", id
);
111 rc
= sysfs_create_link(&dev
->dev
.kobj
, &virtfn
->dev
.kobj
, buf
);
114 rc
= sysfs_create_link(&virtfn
->dev
.kobj
, &dev
->dev
.kobj
, "physfn");
118 kobject_uevent(&virtfn
->dev
.kobj
, KOBJ_CHANGE
);
123 sysfs_remove_link(&dev
->dev
.kobj
, buf
);
126 mutex_lock(&iov
->dev
->sriov
->lock
);
127 pci_stop_and_remove_bus_device(virtfn
);
129 virtfn_remove_bus(dev
->bus
, bus
);
131 mutex_unlock(&iov
->dev
->sriov
->lock
);
136 static void virtfn_remove(struct pci_dev
*dev
, int id
, int reset
)
138 char buf
[VIRTFN_ID_LEN
];
139 struct pci_dev
*virtfn
;
140 struct pci_sriov
*iov
= dev
->sriov
;
142 virtfn
= pci_get_domain_bus_and_slot(pci_domain_nr(dev
->bus
),
144 virtfn_devfn(dev
, id
));
149 device_release_driver(&virtfn
->dev
);
150 __pci_reset_function(virtfn
);
153 sprintf(buf
, "virtfn%u", id
);
154 sysfs_remove_link(&dev
->dev
.kobj
, buf
);
156 * pci_stop_dev() could have been called for this virtfn already,
157 * so the directory for the virtfn may have been removed before.
158 * Double check to avoid spurious sysfs warnings.
160 if (virtfn
->dev
.kobj
.sd
)
161 sysfs_remove_link(&virtfn
->dev
.kobj
, "physfn");
163 mutex_lock(&iov
->dev
->sriov
->lock
);
164 pci_stop_and_remove_bus_device(virtfn
);
165 virtfn_remove_bus(dev
->bus
, virtfn
->bus
);
166 mutex_unlock(&iov
->dev
->sriov
->lock
);
168 /* balance pci_get_domain_bus_and_slot() */
173 static int sriov_enable(struct pci_dev
*dev
, int nr_virtfn
)
178 u16 offset
, stride
, initial
;
179 struct resource
*res
;
180 struct pci_dev
*pdev
;
181 struct pci_sriov
*iov
= dev
->sriov
;
190 pci_read_config_word(dev
, iov
->pos
+ PCI_SRIOV_INITIAL_VF
, &initial
);
191 if (initial
> iov
->total_VFs
||
192 (!(iov
->cap
& PCI_SRIOV_CAP_VFM
) && (initial
!= iov
->total_VFs
)))
195 if (nr_virtfn
< 0 || nr_virtfn
> iov
->total_VFs
||
196 (!(iov
->cap
& PCI_SRIOV_CAP_VFM
) && (nr_virtfn
> initial
)))
199 pci_read_config_word(dev
, iov
->pos
+ PCI_SRIOV_VF_OFFSET
, &offset
);
200 pci_read_config_word(dev
, iov
->pos
+ PCI_SRIOV_VF_STRIDE
, &stride
);
201 if (!offset
|| (nr_virtfn
> 1 && !stride
))
205 for (i
= 0; i
< PCI_SRIOV_NUM_BARS
; i
++) {
206 bars
|= (1 << (i
+ PCI_IOV_RESOURCES
));
207 res
= dev
->resource
+ PCI_IOV_RESOURCES
+ i
;
211 if (nres
!= iov
->nres
) {
212 dev_err(&dev
->dev
, "not enough MMIO resources for SR-IOV\n");
216 iov
->offset
= offset
;
217 iov
->stride
= stride
;
219 if (virtfn_bus(dev
, nr_virtfn
- 1) > dev
->bus
->busn_res
.end
) {
220 dev_err(&dev
->dev
, "SR-IOV: bus number out of range\n");
224 if (pci_enable_resources(dev
, bars
)) {
225 dev_err(&dev
->dev
, "SR-IOV: IOV BARS not allocated\n");
229 if (iov
->link
!= dev
->devfn
) {
230 pdev
= pci_get_slot(dev
->bus
, iov
->link
);
234 if (!pdev
->is_physfn
) {
239 rc
= sysfs_create_link(&dev
->dev
.kobj
,
240 &pdev
->dev
.kobj
, "dep_link");
246 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_NUM_VF
, nr_virtfn
);
247 iov
->ctrl
|= PCI_SRIOV_CTRL_VFE
| PCI_SRIOV_CTRL_MSE
;
248 pci_cfg_access_lock(dev
);
249 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_CTRL
, iov
->ctrl
);
251 pci_cfg_access_unlock(dev
);
253 iov
->initial_VFs
= initial
;
254 if (nr_virtfn
< initial
)
257 for (i
= 0; i
< initial
; i
++) {
258 rc
= virtfn_add(dev
, i
, 0);
263 kobject_uevent(&dev
->dev
.kobj
, KOBJ_CHANGE
);
264 iov
->num_VFs
= nr_virtfn
;
269 for (j
= 0; j
< i
; j
++)
270 virtfn_remove(dev
, j
, 0);
272 iov
->ctrl
&= ~(PCI_SRIOV_CTRL_VFE
| PCI_SRIOV_CTRL_MSE
);
273 pci_cfg_access_lock(dev
);
274 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_CTRL
, iov
->ctrl
);
275 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_NUM_VF
, 0);
277 pci_cfg_access_unlock(dev
);
279 if (iov
->link
!= dev
->devfn
)
280 sysfs_remove_link(&dev
->dev
.kobj
, "dep_link");
285 static void sriov_disable(struct pci_dev
*dev
)
288 struct pci_sriov
*iov
= dev
->sriov
;
293 for (i
= 0; i
< iov
->num_VFs
; i
++)
294 virtfn_remove(dev
, i
, 0);
296 iov
->ctrl
&= ~(PCI_SRIOV_CTRL_VFE
| PCI_SRIOV_CTRL_MSE
);
297 pci_cfg_access_lock(dev
);
298 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_CTRL
, iov
->ctrl
);
300 pci_cfg_access_unlock(dev
);
302 if (iov
->link
!= dev
->devfn
)
303 sysfs_remove_link(&dev
->dev
.kobj
, "dep_link");
306 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_NUM_VF
, 0);
309 static int sriov_init(struct pci_dev
*dev
, int pos
)
315 u16 ctrl
, total
, offset
, stride
;
316 struct pci_sriov
*iov
;
317 struct resource
*res
;
318 struct pci_dev
*pdev
;
320 if (pci_pcie_type(dev
) != PCI_EXP_TYPE_RC_END
&&
321 pci_pcie_type(dev
) != PCI_EXP_TYPE_ENDPOINT
)
324 pci_read_config_word(dev
, pos
+ PCI_SRIOV_CTRL
, &ctrl
);
325 if (ctrl
& PCI_SRIOV_CTRL_VFE
) {
326 pci_write_config_word(dev
, pos
+ PCI_SRIOV_CTRL
, 0);
330 pci_read_config_word(dev
, pos
+ PCI_SRIOV_TOTAL_VF
, &total
);
335 list_for_each_entry(pdev
, &dev
->bus
->devices
, bus_list
)
340 if (pci_ari_enabled(dev
->bus
))
341 ctrl
|= PCI_SRIOV_CTRL_ARI
;
344 pci_write_config_word(dev
, pos
+ PCI_SRIOV_CTRL
, ctrl
);
345 pci_write_config_word(dev
, pos
+ PCI_SRIOV_NUM_VF
, 0);
346 pci_read_config_word(dev
, pos
+ PCI_SRIOV_VF_OFFSET
, &offset
);
347 pci_read_config_word(dev
, pos
+ PCI_SRIOV_VF_STRIDE
, &stride
);
348 if (!offset
|| (total
> 1 && !stride
))
351 pci_read_config_dword(dev
, pos
+ PCI_SRIOV_SUP_PGSIZE
, &pgsz
);
352 i
= PAGE_SHIFT
> 12 ? PAGE_SHIFT
- 12 : 0;
353 pgsz
&= ~((1 << i
) - 1);
358 pci_write_config_dword(dev
, pos
+ PCI_SRIOV_SYS_PGSIZE
, pgsz
);
361 for (i
= 0; i
< PCI_SRIOV_NUM_BARS
; i
++) {
362 res
= dev
->resource
+ PCI_IOV_RESOURCES
+ i
;
363 i
+= __pci_read_base(dev
, pci_bar_unknown
, res
,
364 pos
+ PCI_SRIOV_BAR
+ i
* 4);
367 if (resource_size(res
) & (PAGE_SIZE
- 1)) {
371 res
->end
= res
->start
+ resource_size(res
) * total
- 1;
375 iov
= kzalloc(sizeof(*iov
), GFP_KERNEL
);
384 iov
->total_VFs
= total
;
385 iov
->offset
= offset
;
386 iov
->stride
= stride
;
389 pci_read_config_dword(dev
, pos
+ PCI_SRIOV_CAP
, &iov
->cap
);
390 pci_read_config_byte(dev
, pos
+ PCI_SRIOV_FUNC_LINK
, &iov
->link
);
391 if (pci_pcie_type(dev
) == PCI_EXP_TYPE_RC_END
)
392 iov
->link
= PCI_DEVFN(PCI_SLOT(dev
->devfn
), iov
->link
);
395 iov
->dev
= pci_dev_get(pdev
);
399 mutex_init(&iov
->lock
);
407 for (i
= 0; i
< PCI_SRIOV_NUM_BARS
; i
++) {
408 res
= dev
->resource
+ PCI_IOV_RESOURCES
+ i
;
415 static void sriov_release(struct pci_dev
*dev
)
417 BUG_ON(dev
->sriov
->num_VFs
);
419 if (dev
!= dev
->sriov
->dev
)
420 pci_dev_put(dev
->sriov
->dev
);
422 mutex_destroy(&dev
->sriov
->lock
);
428 static void sriov_restore_state(struct pci_dev
*dev
)
432 struct pci_sriov
*iov
= dev
->sriov
;
434 pci_read_config_word(dev
, iov
->pos
+ PCI_SRIOV_CTRL
, &ctrl
);
435 if (ctrl
& PCI_SRIOV_CTRL_VFE
)
438 for (i
= PCI_IOV_RESOURCES
; i
<= PCI_IOV_RESOURCE_END
; i
++)
439 pci_update_resource(dev
, i
);
441 pci_write_config_dword(dev
, iov
->pos
+ PCI_SRIOV_SYS_PGSIZE
, iov
->pgsz
);
442 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_NUM_VF
, iov
->num_VFs
);
443 pci_write_config_word(dev
, iov
->pos
+ PCI_SRIOV_CTRL
, iov
->ctrl
);
444 if (iov
->ctrl
& PCI_SRIOV_CTRL_VFE
)
449 * pci_iov_init - initialize the IOV capability
450 * @dev: the PCI device
452 * Returns 0 on success, or negative on failure.
454 int pci_iov_init(struct pci_dev
*dev
)
458 if (!pci_is_pcie(dev
))
461 pos
= pci_find_ext_capability(dev
, PCI_EXT_CAP_ID_SRIOV
);
463 return sriov_init(dev
, pos
);
469 * pci_iov_release - release resources used by the IOV capability
470 * @dev: the PCI device
472 void pci_iov_release(struct pci_dev
*dev
)
479 * pci_iov_resource_bar - get position of the SR-IOV BAR
480 * @dev: the PCI device
481 * @resno: the resource number
482 * @type: the BAR type to be filled in
484 * Returns position of the BAR encapsulated in the SR-IOV capability.
486 int pci_iov_resource_bar(struct pci_dev
*dev
, int resno
,
487 enum pci_bar_type
*type
)
489 if (resno
< PCI_IOV_RESOURCES
|| resno
> PCI_IOV_RESOURCE_END
)
492 BUG_ON(!dev
->is_physfn
);
494 *type
= pci_bar_unknown
;
496 return dev
->sriov
->pos
+ PCI_SRIOV_BAR
+
497 4 * (resno
- PCI_IOV_RESOURCES
);
501 * pci_sriov_resource_alignment - get resource alignment for VF BAR
502 * @dev: the PCI device
503 * @resno: the resource number
505 * Returns the alignment of the VF BAR found in the SR-IOV capability.
506 * This is not the same as the resource size which is defined as
507 * the VF BAR size multiplied by the number of VFs. The alignment
508 * is just the VF BAR size.
510 resource_size_t
pci_sriov_resource_alignment(struct pci_dev
*dev
, int resno
)
513 enum pci_bar_type type
;
514 int reg
= pci_iov_resource_bar(dev
, resno
, &type
);
519 __pci_read_base(dev
, type
, &tmp
, reg
);
520 return resource_alignment(&tmp
);
524 * pci_restore_iov_state - restore the state of the IOV capability
525 * @dev: the PCI device
527 void pci_restore_iov_state(struct pci_dev
*dev
)
530 sriov_restore_state(dev
);
534 * pci_iov_bus_range - find bus range used by Virtual Function
537 * Returns max number of buses (exclude current one) used by Virtual
540 int pci_iov_bus_range(struct pci_bus
*bus
)
546 list_for_each_entry(dev
, &bus
->devices
, bus_list
) {
549 busnr
= virtfn_bus(dev
, dev
->sriov
->total_VFs
- 1);
554 return max
? max
- bus
->number
: 0;
558 * pci_enable_sriov - enable the SR-IOV capability
559 * @dev: the PCI device
560 * @nr_virtfn: number of virtual functions to enable
562 * Returns 0 on success, or negative on failure.
564 int pci_enable_sriov(struct pci_dev
*dev
, int nr_virtfn
)
571 return sriov_enable(dev
, nr_virtfn
);
573 EXPORT_SYMBOL_GPL(pci_enable_sriov
);
576 * pci_disable_sriov - disable the SR-IOV capability
577 * @dev: the PCI device
579 void pci_disable_sriov(struct pci_dev
*dev
)
588 EXPORT_SYMBOL_GPL(pci_disable_sriov
);
591 * pci_num_vf - return number of VFs associated with a PF device_release_driver
592 * @dev: the PCI device
594 * Returns number of VFs, or 0 if SR-IOV is not enabled.
596 int pci_num_vf(struct pci_dev
*dev
)
601 return dev
->sriov
->num_VFs
;
603 EXPORT_SYMBOL_GPL(pci_num_vf
);
606 * pci_vfs_assigned - returns number of VFs are assigned to a guest
607 * @dev: the PCI device
609 * Returns number of VFs belonging to this device that are assigned to a guest.
610 * If device is not a physical function returns 0.
612 int pci_vfs_assigned(struct pci_dev
*dev
)
614 struct pci_dev
*vfdev
;
615 unsigned int vfs_assigned
= 0;
616 unsigned short dev_id
;
618 /* only search if we are a PF */
623 * determine the device ID for the VFs, the vendor ID will be the
624 * same as the PF so there is no need to check for that one
626 pci_read_config_word(dev
, dev
->sriov
->pos
+ PCI_SRIOV_VF_DID
, &dev_id
);
628 /* loop through all the VFs to see if we own any that are assigned */
629 vfdev
= pci_get_device(dev
->vendor
, dev_id
, NULL
);
632 * It is considered assigned if it is a virtual function with
633 * our dev as the physical function and the assigned bit is set
635 if (vfdev
->is_virtfn
&& (vfdev
->physfn
== dev
) &&
636 (vfdev
->dev_flags
& PCI_DEV_FLAGS_ASSIGNED
))
639 vfdev
= pci_get_device(dev
->vendor
, dev_id
, vfdev
);
644 EXPORT_SYMBOL_GPL(pci_vfs_assigned
);
647 * pci_sriov_set_totalvfs -- reduce the TotalVFs available
648 * @dev: the PCI PF device
649 * @numvfs: number that should be used for TotalVFs supported
651 * Should be called from PF driver's probe routine with
652 * device's mutex held.
654 * Returns 0 if PF is an SRIOV-capable device and
655 * value of numvfs valid. If not a PF return -ENOSYS;
656 * if numvfs is invalid return -EINVAL;
657 * if VFs already enabled, return -EBUSY.
659 int pci_sriov_set_totalvfs(struct pci_dev
*dev
, u16 numvfs
)
663 if (numvfs
> dev
->sriov
->total_VFs
)
666 /* Shouldn't change if VFs already enabled */
667 if (dev
->sriov
->ctrl
& PCI_SRIOV_CTRL_VFE
)
670 dev
->sriov
->driver_max_VFs
= numvfs
;
674 EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs
);
677 * pci_sriov_get_totalvfs -- get total VFs supported on this device
678 * @dev: the PCI PF device
680 * For a PCIe device with SRIOV support, return the PCIe
681 * SRIOV capability value of TotalVFs or the value of driver_max_VFs
682 * if the driver reduced it. Otherwise 0.
684 int pci_sriov_get_totalvfs(struct pci_dev
*dev
)
689 if (dev
->sriov
->driver_max_VFs
)
690 return dev
->sriov
->driver_max_VFs
;
692 return dev
->sriov
->total_VFs
;
694 EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs
);