drivers/virtio/virtio_pci_common.c

   1 /*
   2  * Virtio PCI driver - common functionality for all device versions
   3  *
   4  * This module allows virtio devices to be used over a virtual PCI device.
   5  * This can be used with QEMU based VMMs like KVM or Xen.
   6  *
   7  * Copyright IBM Corp. 2007
   8  * Copyright Red Hat, Inc. 2014
   9  *
  10  * Authors:
  11  *  Anthony Liguori  <aliguori@us.ibm.com>
  12  *  Rusty Russell <rusty@rustcorp.com.au>
  13  *  Michael S. Tsirkin <mst@redhat.com>
  14  *
  15  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  16  * See the COPYING file in the top-level directory.
  17  *
  18  */
  19
  20 #include "virtio_pci_common.h"
  21
  22 static bool force_legacy = false;
  23
  24 #if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY)
  25 module_param(force_legacy, bool, 0444);
  26 MODULE_PARM_DESC(force_legacy,
  27                  "Force legacy mode for transitional virtio 1 devices");
  28 #endif
  29
  30 /* wait for pending irq handlers */
  31 void vp_synchronize_vectors(struct virtio_device *vdev)
  32 {
  33         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
  34         int i;
  35
  36         if (vp_dev->intx_enabled)
  37                 synchronize_irq(vp_dev->pci_dev->irq);
  38
  39         for (i = 0; i < vp_dev->msix_vectors; ++i)
  40                 synchronize_irq(vp_dev->msix_entries[i].vector);
  41 }
  42
  43 /* the notify function used when creating a virt queue */
  44 bool vp_notify(struct virtqueue *vq)
  45 {
  46         /* we write the queue's selector into the notification register to
  47          * signal the other end */
  48         iowrite16(vq->index, (void __iomem *)vq->priv);
  49         return true;
  50 }
  51
  52 /* Handle a configuration change: Tell driver if it wants to know. */
  53 static irqreturn_t vp_config_changed(int irq, void *opaque)
  54 {
  55         struct virtio_pci_device *vp_dev = opaque;
  56
  57         virtio_config_changed(&vp_dev->vdev);
  58         return IRQ_HANDLED;
  59 }
  60
  61 /* Notify all virtqueues on an interrupt. */
  62 static irqreturn_t vp_vring_interrupt(int irq, void *opaque)
  63 {
  64         struct virtio_pci_device *vp_dev = opaque;
  65         struct virtio_pci_vq_info *info;
  66         irqreturn_t ret = IRQ_NONE;
  67         unsigned long flags;
  68
  69         spin_lock_irqsave(&vp_dev->lock, flags);
  70         list_for_each_entry(info, &vp_dev->virtqueues, node) {
  71                 if (vring_interrupt(irq, info->vq) == IRQ_HANDLED)
  72                         ret = IRQ_HANDLED;
  73         }
  74         spin_unlock_irqrestore(&vp_dev->lock, flags);
  75
  76         return ret;
  77 }
  78
  79 /* A small wrapper to also acknowledge the interrupt when it's handled.
  80  * I really need an EIO hook for the vring so I can ack the interrupt once we
  81  * know that we'll be handling the IRQ but before we invoke the callback since
  82  * the callback may notify the host which results in the host attempting to
  83  * raise an interrupt that we would then mask once we acknowledged the
  84  * interrupt. */
  85 static irqreturn_t vp_interrupt(int irq, void *opaque)
  86 {
  87         struct virtio_pci_device *vp_dev = opaque;
  88         u8 isr;
  89
  90         /* reading the ISR has the effect of also clearing it so it's very
  91          * important to save off the value. */
  92         isr = ioread8(vp_dev->isr);
  93
  94         /* It's definitely not us if the ISR was not high */
  95         if (!isr)
  96                 return IRQ_NONE;
  97
  98         /* Configuration change?  Tell driver if it wants to know. */
  99         if (isr & VIRTIO_PCI_ISR_CONFIG)
 100                 vp_config_changed(irq, opaque);
 101
 102         return vp_vring_interrupt(irq, opaque);
 103 }
 104
 105 static void vp_free_vectors(struct virtio_device *vdev)
 106 {
 107         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 108         int i;
 109
 110         if (vp_dev->intx_enabled) {
 111                 free_irq(vp_dev->pci_dev->irq, vp_dev);
 112                 vp_dev->intx_enabled = 0;
 113         }
 114
 115         for (i = 0; i < vp_dev->msix_used_vectors; ++i)
 116                 free_irq(vp_dev->msix_entries[i].vector, vp_dev);
 117
 118         for (i = 0; i < vp_dev->msix_vectors; i++)
 119                 if (vp_dev->msix_affinity_masks[i])
 120                         free_cpumask_var(vp_dev->msix_affinity_masks[i]);
 121
 122         if (vp_dev->msix_enabled) {
 123                 /* Disable the vector used for configuration */
 124                 vp_dev->config_vector(vp_dev, VIRTIO_MSI_NO_VECTOR);
 125
 126                 pci_disable_msix(vp_dev->pci_dev);
 127                 vp_dev->msix_enabled = 0;
 128         }
 129
 130         vp_dev->msix_vectors = 0;
 131         vp_dev->msix_used_vectors = 0;
 132         kfree(vp_dev->msix_names);
 133         vp_dev->msix_names = NULL;
 134         kfree(vp_dev->msix_entries);
 135         vp_dev->msix_entries = NULL;
 136         kfree(vp_dev->msix_affinity_masks);
 137         vp_dev->msix_affinity_masks = NULL;
 138 }
 139
 140 static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors,
 141                                    bool per_vq_vectors)
 142 {
 143         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 144         const char *name = dev_name(&vp_dev->vdev.dev);
 145         unsigned i, v;
 146         int err = -ENOMEM;
 147
 148         vp_dev->msix_vectors = nvectors;
 149
 150         vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries,
 151                                        GFP_KERNEL);
 152         if (!vp_dev->msix_entries)
 153                 goto error;
 154         vp_dev->msix_names = kmalloc(nvectors * sizeof *vp_dev->msix_names,
 155                                      GFP_KERNEL);
 156         if (!vp_dev->msix_names)
 157                 goto error;
 158         vp_dev->msix_affinity_masks
 159                 = kzalloc(nvectors * sizeof *vp_dev->msix_affinity_masks,
 160                           GFP_KERNEL);
 161         if (!vp_dev->msix_affinity_masks)
 162                 goto error;
 163         for (i = 0; i < nvectors; ++i)
 164                 if (!alloc_cpumask_var(&vp_dev->msix_affinity_masks[i],
 165                                         GFP_KERNEL))
 166                         goto error;
 167
 168         for (i = 0; i < nvectors; ++i)
 169                 vp_dev->msix_entries[i].entry = i;
 170
 171         err = pci_enable_msix_exact(vp_dev->pci_dev,
 172                                     vp_dev->msix_entries, nvectors);
 173         if (err)
 174                 goto error;
 175         vp_dev->msix_enabled = 1;
 176
 177         /* Set the vector used for configuration */
 178         v = vp_dev->msix_used_vectors;
 179         snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 180                  "%s-config", name);
 181         err = request_irq(vp_dev->msix_entries[v].vector,
 182                           vp_config_changed, 0, vp_dev->msix_names[v],
 183                           vp_dev);
 184         if (err)
 185                 goto error;
 186         ++vp_dev->msix_used_vectors;
 187
 188         v = vp_dev->config_vector(vp_dev, v);
 189         /* Verify we had enough resources to assign the vector */
 190         if (v == VIRTIO_MSI_NO_VECTOR) {
 191                 err = -EBUSY;
 192                 goto error;
 193         }
 194
 195         if (!per_vq_vectors) {
 196                 /* Shared vector for all VQs */
 197                 v = vp_dev->msix_used_vectors;
 198                 snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
 199                          "%s-virtqueues", name);
 200                 err = request_irq(vp_dev->msix_entries[v].vector,
 201                                   vp_vring_interrupt, 0, vp_dev->msix_names[v],
 202                                   vp_dev);
 203                 if (err)
 204                         goto error;
 205                 ++vp_dev->msix_used_vectors;
 206         }
 207         return 0;
 208 error:
 209         vp_free_vectors(vdev);
 210         return err;
 211 }
 212
 213 static int vp_request_intx(struct virtio_device *vdev)
 214 {
 215         int err;
 216         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 217
 218         err = request_irq(vp_dev->pci_dev->irq, vp_interrupt,
 219                           IRQF_SHARED, dev_name(&vdev->dev), vp_dev);
 220         if (!err)
 221                 vp_dev->intx_enabled = 1;
 222         return err;
 223 }
 224
 225 static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned index,
 226                                      void (*callback)(struct virtqueue *vq),
 227                                      const char *name,
 228                                      u16 msix_vec)
 229 {
 230         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 231         struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL);
 232         struct virtqueue *vq;
 233         unsigned long flags;
 234
 235         /* fill out our structure that represents an active queue */
 236         if (!info)
 237                 return ERR_PTR(-ENOMEM);
 238
 239         vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, msix_vec);
 240         if (IS_ERR(vq))
 241                 goto out_info;
 242
 243         info->vq = vq;
 244         if (callback) {
 245                 spin_lock_irqsave(&vp_dev->lock, flags);
 246                 list_add(&info->node, &vp_dev->virtqueues);
 247                 spin_unlock_irqrestore(&vp_dev->lock, flags);
 248         } else {
 249                 INIT_LIST_HEAD(&info->node);
 250         }
 251
 252         vp_dev->vqs[index] = info;
 253         return vq;
 254
 255 out_info:
 256         kfree(info);
 257         return vq;
 258 }
 259
 260 static void vp_del_vq(struct virtqueue *vq)
 261 {
 262         struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev);
 263         struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
 264         unsigned long flags;
 265
 266         spin_lock_irqsave(&vp_dev->lock, flags);
 267         list_del(&info->node);
 268         spin_unlock_irqrestore(&vp_dev->lock, flags);
 269
 270         vp_dev->del_vq(info);
 271         kfree(info);
 272 }
 273
 274 /* the config->del_vqs() implementation */
 275 void vp_del_vqs(struct virtio_device *vdev)
 276 {
 277         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 278         struct virtqueue *vq, *n;
 279         struct virtio_pci_vq_info *info;
 280
 281         list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
 282                 info = vp_dev->vqs[vq->index];
 283                 if (vp_dev->per_vq_vectors &&
 284                         info->msix_vector != VIRTIO_MSI_NO_VECTOR)
 285                         free_irq(vp_dev->msix_entries[info->msix_vector].vector,
 286                                  vq);
 287                 vp_del_vq(vq);
 288         }
 289         vp_dev->per_vq_vectors = false;
 290
 291         vp_free_vectors(vdev);
 292         kfree(vp_dev->vqs);
 293         vp_dev->vqs = NULL;
 294 }
 295
 296 static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 297                               struct virtqueue *vqs[],
 298                               vq_callback_t *callbacks[],
 299                               const char *names[],
 300                               bool use_msix,
 301                               bool per_vq_vectors)
 302 {
 303         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 304         u16 msix_vec;
 305         int i, err, nvectors, allocated_vectors;
 306
 307         vp_dev->vqs = kmalloc(nvqs * sizeof *vp_dev->vqs, GFP_KERNEL);
 308         if (!vp_dev->vqs)
 309                 return -ENOMEM;
 310
 311         if (!use_msix) {
 312                 /* Old style: one normal interrupt for change and all vqs. */
 313                 err = vp_request_intx(vdev);
 314                 if (err)
 315                         goto error_find;
 316         } else {
 317                 if (per_vq_vectors) {
 318                         /* Best option: one for change interrupt, one per vq. */
 319                         nvectors = 1;
 320                         for (i = 0; i < nvqs; ++i)
 321                                 if (callbacks[i])
 322                                         ++nvectors;
 323                 } else {
 324                         /* Second best: one for change, shared for all vqs. */
 325                         nvectors = 2;
 326                 }
 327
 328                 err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors);
 329                 if (err)
 330                         goto error_find;
 331         }
 332
 333         vp_dev->per_vq_vectors = per_vq_vectors;
 334         allocated_vectors = vp_dev->msix_used_vectors;
 335         for (i = 0; i < nvqs; ++i) {
 336                 if (!names[i]) {
 337                         vqs[i] = NULL;
 338                         continue;
 339                 } else if (!callbacks[i] || !vp_dev->msix_enabled)
 340                         msix_vec = VIRTIO_MSI_NO_VECTOR;
 341                 else if (vp_dev->per_vq_vectors)
 342                         msix_vec = allocated_vectors++;
 343                 else
 344                         msix_vec = VP_MSIX_VQ_VECTOR;
 345                 vqs[i] = vp_setup_vq(vdev, i, callbacks[i], names[i], msix_vec);
 346                 if (IS_ERR(vqs[i])) {
 347                         err = PTR_ERR(vqs[i]);
 348                         goto error_find;
 349                 }
 350
 351                 if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR)
 352                         continue;
 353
 354                 /* allocate per-vq irq if available and necessary */
 355                 snprintf(vp_dev->msix_names[msix_vec],
 356                          sizeof *vp_dev->msix_names,
 357                          "%s-%s",
 358                          dev_name(&vp_dev->vdev.dev), names[i]);
 359                 err = request_irq(vp_dev->msix_entries[msix_vec].vector,
 360                                   vring_interrupt, 0,
 361                                   vp_dev->msix_names[msix_vec],
 362                                   vqs[i]);
 363                 if (err) {
 364                         vp_del_vq(vqs[i]);
 365                         goto error_find;
 366                 }
 367         }
 368         return 0;
 369
 370 error_find:
 371         vp_del_vqs(vdev);
 372         return err;
 373 }
 374
 375 /* the config->find_vqs() implementation */
 376 int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs,
 377                 struct virtqueue *vqs[],
 378                 vq_callback_t *callbacks[],
 379                 const char *names[])
 380 {
 381         int err;
 382
 383         /* Try MSI-X with one vector per queue. */
 384         err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names, true, true);
 385         if (!err)
 386                 return 0;
 387         /* Fallback: MSI-X with one vector for config, one shared for queues. */
 388         err = vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
 389                                  true, false);
 390         if (!err)
 391                 return 0;
 392         /* Finally fall back to regular interrupts. */
 393         return vp_try_to_find_vqs(vdev, nvqs, vqs, callbacks, names,
 394                                   false, false);
 395 }
 396
 397 const char *vp_bus_name(struct virtio_device *vdev)
 398 {
 399         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 400
 401         return pci_name(vp_dev->pci_dev);
 402 }
 403
 404 /* Setup the affinity for a virtqueue:
 405  * - force the affinity for per vq vector
 406  * - OR over all affinities for shared MSI
 407  * - ignore the affinity request if we're using INTX
 408  */
 409 int vp_set_vq_affinity(struct virtqueue *vq, int cpu)
 410 {
 411         struct virtio_device *vdev = vq->vdev;
 412         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 413         struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index];
 414         struct cpumask *mask;
 415         unsigned int irq;
 416
 417         if (!vq->callback)
 418                 return -EINVAL;
 419
 420         if (vp_dev->msix_enabled) {
 421                 mask = vp_dev->msix_affinity_masks[info->msix_vector];
 422                 irq = vp_dev->msix_entries[info->msix_vector].vector;
 423                 if (cpu == -1)
 424                         irq_set_affinity_hint(irq, NULL);
 425                 else {
 426                         cpumask_clear(mask);
 427                         cpumask_set_cpu(cpu, mask);
 428                         irq_set_affinity_hint(irq, mask);
 429                 }
 430         }
 431         return 0;
 432 }
 433
 434 #ifdef CONFIG_PM_SLEEP
 435 static int virtio_pci_freeze(struct device *dev)
 436 {
 437         struct pci_dev *pci_dev = to_pci_dev(dev);
 438         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 439         int ret;
 440
 441         ret = virtio_device_freeze(&vp_dev->vdev);
 442
 443         if (!ret)
 444                 pci_disable_device(pci_dev);
 445         return ret;
 446 }
 447
 448 static int virtio_pci_restore(struct device *dev)
 449 {
 450         struct pci_dev *pci_dev = to_pci_dev(dev);
 451         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 452         int ret;
 453
 454         ret = pci_enable_device(pci_dev);
 455         if (ret)
 456                 return ret;
 457
 458         pci_set_master(pci_dev);
 459         return virtio_device_restore(&vp_dev->vdev);
 460 }
 461
 462 static const struct dev_pm_ops virtio_pci_pm_ops = {
 463         SET_SYSTEM_SLEEP_PM_OPS(virtio_pci_freeze, virtio_pci_restore)
 464 };
 465 #endif
 466
 467
 468 /* Qumranet donated their vendor ID for devices 0x1000 thru 0x10FF. */
 469 static const struct pci_device_id virtio_pci_id_table[] = {
 470         { PCI_DEVICE(0x1af4, PCI_ANY_ID) },
 471         { 0 }
 472 };
 473
 474 MODULE_DEVICE_TABLE(pci, virtio_pci_id_table);
 475
 476 static void virtio_pci_release_dev(struct device *_d)
 477 {
 478         struct virtio_device *vdev = dev_to_virtio(_d);
 479         struct virtio_pci_device *vp_dev = to_vp_device(vdev);
 480
 481         /* As struct device is a kobject, it's not safe to
 482          * free the memory (including the reference counter itself)
 483          * until it's release callback. */
 484         kfree(vp_dev);
 485 }
 486
 487 static int virtio_pci_probe(struct pci_dev *pci_dev,
 488                             const struct pci_device_id *id)
 489 {
 490         struct virtio_pci_device *vp_dev;
 491         int rc;
 492
 493         /* allocate our structure and fill it out */
 494         vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL);
 495         if (!vp_dev)
 496                 return -ENOMEM;
 497
 498         pci_set_drvdata(pci_dev, vp_dev);
 499         vp_dev->vdev.dev.parent = &pci_dev->dev;
 500         vp_dev->vdev.dev.release = virtio_pci_release_dev;
 501         vp_dev->pci_dev = pci_dev;
 502         INIT_LIST_HEAD(&vp_dev->virtqueues);
 503         spin_lock_init(&vp_dev->lock);
 504
 505         /* Disable MSI/MSIX to bring device to a known good state. */
 506         pci_msi_off(pci_dev);
 507
 508         /* enable the device */
 509         rc = pci_enable_device(pci_dev);
 510         if (rc)
 511                 goto err_enable_device;
 512
 513         rc = pci_request_regions(pci_dev, "virtio-pci");
 514         if (rc)
 515                 goto err_request_regions;
 516
 517         if (force_legacy) {
 518                 rc = virtio_pci_legacy_probe(vp_dev);
 519                 /* Also try modern mode if we can't map BAR0 (no IO space). */
 520                 if (rc == -ENODEV || rc == -ENOMEM)
 521                         rc = virtio_pci_modern_probe(vp_dev);
 522                 if (rc)
 523                         goto err_probe;
 524         } else {
 525                 rc = virtio_pci_modern_probe(vp_dev);
 526                 if (rc == -ENODEV)
 527                         rc = virtio_pci_legacy_probe(vp_dev);
 528                 if (rc)
 529                         goto err_probe;
 530         }
 531
 532         pci_set_master(pci_dev);
 533
 534         rc = register_virtio_device(&vp_dev->vdev);
 535         if (rc)
 536                 goto err_register;
 537
 538         return 0;
 539
 540 err_register:
 541         if (vp_dev->ioaddr)
 542              virtio_pci_legacy_remove(vp_dev);
 543         else
 544              virtio_pci_modern_remove(vp_dev);
 545 err_probe:
 546         pci_release_regions(pci_dev);
 547 err_request_regions:
 548         pci_disable_device(pci_dev);
 549 err_enable_device:
 550         kfree(vp_dev);
 551         return rc;
 552 }
 553
 554 static void virtio_pci_remove(struct pci_dev *pci_dev)
 555 {
 556         struct virtio_pci_device *vp_dev = pci_get_drvdata(pci_dev);
 557
 558         unregister_virtio_device(&vp_dev->vdev);
 559
 560         if (vp_dev->ioaddr)
 561                 virtio_pci_legacy_remove(vp_dev);
 562         else
 563                 virtio_pci_modern_remove(vp_dev);
 564
 565         pci_release_regions(pci_dev);
 566         pci_disable_device(pci_dev);
 567 }
 568
 569 static struct pci_driver virtio_pci_driver = {
 570         .name           = "virtio-pci",
 571         .id_table       = virtio_pci_id_table,
 572         .probe          = virtio_pci_probe,
 573         .remove         = virtio_pci_remove,
 574 #ifdef CONFIG_PM_SLEEP
 575         .driver.pm      = &virtio_pci_pm_ops,
 576 #endif
 577 };
 578
 579 module_pci_driver(virtio_pci_driver);
 580
 581 MODULE_AUTHOR("Anthony Liguori <aliguori@us.ibm.com>");
 582 MODULE_DESCRIPTION("virtio-pci");
 583 MODULE_LICENSE("GPL");
 584 MODULE_VERSION("1");