drivers/pci/msi/api.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * PCI MSI/MSI-X — Exported APIs for device drivers
   4  *
   5  * Copyright (C) 2003-2004 Intel
   6  * Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
   7  * Copyright (C) 2016 Christoph Hellwig.
   8  * Copyright (C) 2022 Linutronix GmbH
   9  */
  10
  11 #include <linux/export.h>
  12 #include <linux/irq.h>
  13
  14 #include "msi.h"
  15
  16 /**
  17  * pci_enable_msi() - Enable MSI interrupt mode on device
  18  * @dev: the PCI device to operate on
  19  *
  20  * Legacy device driver API to enable MSI interrupts mode on device and
  21  * allocate a single interrupt vector. On success, the allocated vector
  22  * Linux IRQ will be saved at @dev->irq. The driver must invoke
  23  * pci_disable_msi() on cleanup.
  24  *
  25  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
  26  * pair should, in general, be used instead.
  27  *
  28  * Return: 0 on success, errno otherwise
  29  */
  30 int pci_enable_msi(struct pci_dev *dev)
  31 {
  32         int rc = __pci_enable_msi_range(dev, 1, 1, NULL);
  33         if (rc < 0)
  34                 return rc;
  35         return 0;
  36 }
  37 EXPORT_SYMBOL(pci_enable_msi);
  38
  39 /**
  40  * pci_disable_msi() - Disable MSI interrupt mode on device
  41  * @dev: the PCI device to operate on
  42  *
  43  * Legacy device driver API to disable MSI interrupt mode on device,
  44  * free earlier allocated interrupt vectors, and restore INTx emulation.
  45  * The PCI device Linux IRQ (@dev->irq) is restored to its default
  46  * pin-assertion IRQ. This is the cleanup pair of pci_enable_msi().
  47  *
  48  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
  49  * pair should, in general, be used instead.
  50  */
  51 void pci_disable_msi(struct pci_dev *dev)
  52 {
  53         if (!pci_msi_enabled() || !dev || !dev->msi_enabled)
  54                 return;
  55
  56         msi_lock_descs(&dev->dev);
  57         pci_msi_shutdown(dev);
  58         pci_free_msi_irqs(dev);
  59         msi_unlock_descs(&dev->dev);
  60 }
  61 EXPORT_SYMBOL(pci_disable_msi);
  62
  63 /**
  64  * pci_msix_vec_count() - Get number of MSI-X interrupt vectors on device
  65  * @dev: the PCI device to operate on
  66  *
  67  * Return: number of MSI-X interrupt vectors available on this device
  68  * (i.e., the device's MSI-X capability structure "table size"), -EINVAL
  69  * if the device is not MSI-X capable, other errnos otherwise.
  70  */
  71 int pci_msix_vec_count(struct pci_dev *dev)
  72 {
  73         u16 control;
  74
  75         if (!dev->msix_cap)
  76                 return -EINVAL;
  77
  78         pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
  79         return msix_table_size(control);
  80 }
  81 EXPORT_SYMBOL(pci_msix_vec_count);
  82
  83 /**
  84  * pci_enable_msix_range() - Enable MSI-X interrupt mode on device
  85  * @dev:     the PCI device to operate on
  86  * @entries: input/output parameter, array of MSI-X configuration entries
  87  * @minvec:  minimum required number of MSI-X vectors
  88  * @maxvec:  maximum desired number of MSI-X vectors
  89  *
  90  * Legacy device driver API to enable MSI-X interrupt mode on device and
  91  * configure its MSI-X capability structure as appropriate.  The passed
  92  * @entries array must have each of its members "entry" field set to a
  93  * desired (valid) MSI-X vector number, where the range of valid MSI-X
  94  * vector numbers can be queried through pci_msix_vec_count().  If
  95  * successful, the driver must invoke pci_disable_msix() on cleanup.
  96  *
  97  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
  98  * pair should, in general, be used instead.
  99  *
 100  * Return: number of MSI-X vectors allocated (which might be smaller
 101  * than @maxvecs), where Linux IRQ numbers for such allocated vectors
 102  * are saved back in the @entries array elements' "vector" field. Return
 103  * -ENOSPC if less than @minvecs interrupt vectors are available.
 104  * Return -EINVAL if one of the passed @entries members "entry" field
 105  * was invalid or a duplicate, or if plain MSI interrupts mode was
 106  * earlier enabled on device. Return other errnos otherwise.
 107  */
 108 int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
 109                           int minvec, int maxvec)
 110 {
 111         return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0);
 112 }
 113 EXPORT_SYMBOL(pci_enable_msix_range);
 114
 115 /**
 116  * pci_msix_can_alloc_dyn - Query whether dynamic allocation after enabling
 117  *                          MSI-X is supported
 118  *
 119  * @dev:        PCI device to operate on
 120  *
 121  * Return: True if supported, false otherwise
 122  */
 123 bool pci_msix_can_alloc_dyn(struct pci_dev *dev)
 124 {
 125         if (!dev->msix_cap)
 126                 return false;
 127
 128         return pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX_ALLOC_DYN, DENY_LEGACY);
 129 }
 130 EXPORT_SYMBOL_GPL(pci_msix_can_alloc_dyn);
 131
 132 /**
 133  * pci_msix_alloc_irq_at - Allocate an MSI-X interrupt after enabling MSI-X
 134  *                         at a given MSI-X vector index or any free vector index
 135  *
 136  * @dev:        PCI device to operate on
 137  * @index:      Index to allocate. If @index == MSI_ANY_INDEX this allocates
 138  *              the next free index in the MSI-X table
 139  * @affdesc:    Optional pointer to an affinity descriptor structure. NULL otherwise
 140  *
 141  * Return: A struct msi_map
 142  *
 143  *      On success msi_map::index contains the allocated index (>= 0) and
 144  *      msi_map::virq contains the allocated Linux interrupt number (> 0).
 145  *
 146  *      On fail msi_map::index contains the error code and msi_map::virq
 147  *      is set to 0.
 148  */
 149 struct msi_map pci_msix_alloc_irq_at(struct pci_dev *dev, unsigned int index,
 150                                      const struct irq_affinity_desc *affdesc)
 151 {
 152         struct msi_map map = { .index = -ENOTSUPP };
 153
 154         if (!dev->msix_enabled)
 155                 return map;
 156
 157         if (!pci_msix_can_alloc_dyn(dev))
 158                 return map;
 159
 160         return msi_domain_alloc_irq_at(&dev->dev, MSI_DEFAULT_DOMAIN, index, affdesc, NULL);
 161 }
 162 EXPORT_SYMBOL_GPL(pci_msix_alloc_irq_at);
 163
 164 /**
 165  * pci_msix_free_irq - Free an interrupt on a PCI/MSIX interrupt domain
 166  *
 167  * @dev:        The PCI device to operate on
 168  * @map:        A struct msi_map describing the interrupt to free
 169  *
 170  * Undo an interrupt vector allocation. Does not disable MSI-X.
 171  */
 172 void pci_msix_free_irq(struct pci_dev *dev, struct msi_map map)
 173 {
 174         if (WARN_ON_ONCE(map.index < 0 || map.virq <= 0))
 175                 return;
 176         if (WARN_ON_ONCE(!pci_msix_can_alloc_dyn(dev)))
 177                 return;
 178         msi_domain_free_irqs_range(&dev->dev, MSI_DEFAULT_DOMAIN, map.index, map.index);
 179 }
 180 EXPORT_SYMBOL_GPL(pci_msix_free_irq);
 181
 182 /**
 183  * pci_disable_msix() - Disable MSI-X interrupt mode on device
 184  * @dev: the PCI device to operate on
 185  *
 186  * Legacy device driver API to disable MSI-X interrupt mode on device,
 187  * free earlier-allocated interrupt vectors, and restore INTx.
 188  * The PCI device Linux IRQ (@dev->irq) is restored to its default pin
 189  * assertion IRQ. This is the cleanup pair of pci_enable_msix_range().
 190  *
 191  * NOTE: The newer pci_alloc_irq_vectors() / pci_free_irq_vectors() API
 192  * pair should, in general, be used instead.
 193  */
 194 void pci_disable_msix(struct pci_dev *dev)
 195 {
 196         if (!pci_msi_enabled() || !dev || !dev->msix_enabled)
 197                 return;
 198
 199         msi_lock_descs(&dev->dev);
 200         pci_msix_shutdown(dev);
 201         pci_free_msi_irqs(dev);
 202         msi_unlock_descs(&dev->dev);
 203 }
 204 EXPORT_SYMBOL(pci_disable_msix);
 205
 206 /**
 207  * pci_alloc_irq_vectors() - Allocate multiple device interrupt vectors
 208  * @dev:      the PCI device to operate on
 209  * @min_vecs: minimum required number of vectors (must be >= 1)
 210  * @max_vecs: maximum desired number of vectors
 211  * @flags:    One or more of:
 212  *
 213  *            * %PCI_IRQ_MSIX      Allow trying MSI-X vector allocations
 214  *            * %PCI_IRQ_MSI       Allow trying MSI vector allocations
 215  *
 216  *            * %PCI_IRQ_INTX      Allow trying INTx interrupts, if and
 217  *              only if @min_vecs == 1
 218  *
 219  *            * %PCI_IRQ_AFFINITY  Auto-manage IRQs affinity by spreading
 220  *              the vectors around available CPUs
 221  *
 222  * Allocate up to @max_vecs interrupt vectors on device. MSI-X irq
 223  * vector allocation has a higher precedence over plain MSI, which has a
 224  * higher precedence over legacy INTx emulation.
 225  *
 226  * Upon a successful allocation, the caller should use pci_irq_vector()
 227  * to get the Linux IRQ number to be passed to request_threaded_irq().
 228  * The driver must call pci_free_irq_vectors() on cleanup.
 229  *
 230  * Return: number of allocated vectors (which might be smaller than
 231  * @max_vecs), -ENOSPC if less than @min_vecs interrupt vectors are
 232  * available, other errnos otherwise.
 233  */
 234 int pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs,
 235                           unsigned int max_vecs, unsigned int flags)
 236 {
 237         return pci_alloc_irq_vectors_affinity(dev, min_vecs, max_vecs,
 238                                               flags, NULL);
 239 }
 240 EXPORT_SYMBOL(pci_alloc_irq_vectors);
 241
 242 /**
 243  * pci_alloc_irq_vectors_affinity() - Allocate multiple device interrupt
 244  *                                    vectors with affinity requirements
 245  * @dev:      the PCI device to operate on
 246  * @min_vecs: minimum required number of vectors (must be >= 1)
 247  * @max_vecs: maximum desired number of vectors
 248  * @flags:    allocation flags, as in pci_alloc_irq_vectors()
 249  * @affd:     affinity requirements (can be %NULL).
 250  *
 251  * Same as pci_alloc_irq_vectors(), but with the extra @affd parameter.
 252  * Check that function docs, and &struct irq_affinity, for more details.
 253  */
 254 int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
 255                                    unsigned int max_vecs, unsigned int flags,
 256                                    struct irq_affinity *affd)
 257 {
 258         struct irq_affinity msi_default_affd = {0};
 259         int nvecs = -ENOSPC;
 260
 261         if (flags & PCI_IRQ_AFFINITY) {
 262                 if (!affd)
 263                         affd = &msi_default_affd;
 264         } else {
 265                 if (WARN_ON(affd))
 266                         affd = NULL;
 267         }
 268
 269         if (flags & PCI_IRQ_MSIX) {
 270                 nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
 271                                                 affd, flags);
 272                 if (nvecs > 0)
 273                         return nvecs;
 274         }
 275
 276         if (flags & PCI_IRQ_MSI) {
 277                 nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);
 278                 if (nvecs > 0)
 279                         return nvecs;
 280         }
 281
 282         /* use INTx IRQ if allowed */
 283         if (flags & PCI_IRQ_INTX) {
 284                 if (min_vecs == 1 && dev->irq) {
 285                         /*
 286                          * Invoke the affinity spreading logic to ensure that
 287                          * the device driver can adjust queue configuration
 288                          * for the single interrupt case.
 289                          */
 290                         if (affd)
 291                                 irq_create_affinity_masks(1, affd);
 292                         pci_intx(dev, 1);
 293                         return 1;
 294                 }
 295         }
 296
 297         return nvecs;
 298 }
 299 EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
 300
 301 /**
 302  * pci_irq_vector() - Get Linux IRQ number of a device interrupt vector
 303  * @dev: the PCI device to operate on
 304  * @nr:  device-relative interrupt vector index (0-based); has different
 305  *       meanings, depending on interrupt mode:
 306  *
 307  *         * MSI-X     the index in the MSI-X vector table
 308  *         * MSI       the index of the enabled MSI vectors
 309  *         * INTx      must be 0
 310  *
 311  * Return: the Linux IRQ number, or -EINVAL if @nr is out of range
 312  */
 313 int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
 314 {
 315         unsigned int irq;
 316
 317         if (!dev->msi_enabled && !dev->msix_enabled)
 318                 return !nr ? dev->irq : -EINVAL;
 319
 320         irq = msi_get_virq(&dev->dev, nr);
 321         return irq ? irq : -EINVAL;
 322 }
 323 EXPORT_SYMBOL(pci_irq_vector);
 324
 325 /**
 326  * pci_irq_get_affinity() - Get a device interrupt vector affinity
 327  * @dev: the PCI device to operate on
 328  * @nr:  device-relative interrupt vector index (0-based); has different
 329  *       meanings, depending on interrupt mode:
 330  *
 331  *         * MSI-X     the index in the MSI-X vector table
 332  *         * MSI       the index of the enabled MSI vectors
 333  *         * INTx      must be 0
 334  *
 335  * Return: MSI/MSI-X vector affinity, NULL if @nr is out of range or if
 336  * the MSI(-X) vector was allocated without explicit affinity
 337  * requirements (e.g., by pci_enable_msi(), pci_enable_msix_range(), or
 338  * pci_alloc_irq_vectors() without the %PCI_IRQ_AFFINITY flag). Return a
 339  * generic set of CPU IDs representing all possible CPUs available
 340  * during system boot if the device is in legacy INTx mode.
 341  */
 342 const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
 343 {
 344         int idx, irq = pci_irq_vector(dev, nr);
 345         struct msi_desc *desc;
 346
 347         if (WARN_ON_ONCE(irq <= 0))
 348                 return NULL;
 349
 350         desc = irq_get_msi_desc(irq);
 351         /* Non-MSI does not have the information handy */
 352         if (!desc)
 353                 return cpu_possible_mask;
 354
 355         /* MSI[X] interrupts can be allocated without affinity descriptor */
 356         if (!desc->affinity)
 357                 return NULL;
 358
 359         /*
 360          * MSI has a mask array in the descriptor.
 361          * MSI-X has a single mask.
 362          */
 363         idx = dev->msi_enabled ? nr : 0;
 364         return &desc->affinity[idx].mask;
 365 }
 366 EXPORT_SYMBOL(pci_irq_get_affinity);
 367
 368 /**
 369  * pci_free_irq_vectors() - Free previously allocated IRQs for a device
 370  * @dev: the PCI device to operate on
 371  *
 372  * Undo the interrupt vector allocations and possible device MSI/MSI-X
 373  * enablement earlier done through pci_alloc_irq_vectors_affinity() or
 374  * pci_alloc_irq_vectors().
 375  */
 376 void pci_free_irq_vectors(struct pci_dev *dev)
 377 {
 378         pci_disable_msix(dev);
 379         pci_disable_msi(dev);
 380 }
 381 EXPORT_SYMBOL(pci_free_irq_vectors);
 382
 383 /**
 384  * pci_restore_msi_state() - Restore cached MSI(-X) state on device
 385  * @dev: the PCI device to operate on
 386  *
 387  * Write the Linux-cached MSI(-X) state back on device. This is
 388  * typically useful upon system resume, or after an error-recovery PCI
 389  * adapter reset.
 390  */
 391 void pci_restore_msi_state(struct pci_dev *dev)
 392 {
 393         __pci_restore_msi_state(dev);
 394         __pci_restore_msix_state(dev);
 395 }
 396 EXPORT_SYMBOL_GPL(pci_restore_msi_state);
 397
 398 /**
 399  * pci_msi_enabled() - Are MSI(-X) interrupts enabled system-wide?
 400  *
 401  * Return: true if MSI has not been globally disabled through ACPI FADT,
 402  * PCI bridge quirks, or the "pci=nomsi" kernel command-line option.
 403  */
 404 int pci_msi_enabled(void)
 405 {
 406         return pci_msi_enable;
 407 }
 408 EXPORT_SYMBOL(pci_msi_enabled);