drivers/pci/access.c

   1 #include <linux/delay.h>
   2 #include <linux/pci.h>
   3 #include <linux/module.h>
   4 #include <linux/sched/signal.h>
   5 #include <linux/slab.h>
   6 #include <linux/ioport.h>
   7 #include <linux/wait.h>
   8
   9 #include "pci.h"
  10
  11 /*
  12  * This interrupt-safe spinlock protects all accesses to PCI
  13  * configuration space.
  14  */
  15
  16 DEFINE_RAW_SPINLOCK(pci_lock);
  17
  18 /*
  19  *  Wrappers for all PCI configuration access functions.  They just check
  20  *  alignment, do locking and call the low-level functions pointed to
  21  *  by pci_dev->ops.
  22  */
  23
  24 #define PCI_byte_BAD 0
  25 #define PCI_word_BAD (pos & 1)
  26 #define PCI_dword_BAD (pos & 3)
  27
  28 #ifdef CONFIG_PCI_LOCKLESS_CONFIG
  29 # define pci_lock_config(f)     do { (void)(f); } while (0)
  30 # define pci_unlock_config(f)   do { (void)(f); } while (0)
  31 #else
  32 # define pci_lock_config(f)     raw_spin_lock_irqsave(&pci_lock, f)
  33 # define pci_unlock_config(f)   raw_spin_unlock_irqrestore(&pci_lock, f)
  34 #endif
  35
  36 #define PCI_OP_READ(size, type, len) \
  37 int pci_bus_read_config_##size \
  38         (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \
  39 {                                                                       \
  40         int res;                                                        \
  41         unsigned long flags;                                            \
  42         u32 data = 0;                                                   \
  43         if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER;       \
  44         pci_lock_config(flags);                                         \
  45         res = bus->ops->read(bus, devfn, pos, len, &data);              \
  46         *value = (type)data;                                            \
  47         pci_unlock_config(flags);                                       \
  48         return res;                                                     \
  49 }
  50
  51 #define PCI_OP_WRITE(size, type, len) \
  52 int pci_bus_write_config_##size \
  53         (struct pci_bus *bus, unsigned int devfn, int pos, type value)  \
  54 {                                                                       \
  55         int res;                                                        \
  56         unsigned long flags;                                            \
  57         if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER;       \
  58         pci_lock_config(flags);                                         \
  59         res = bus->ops->write(bus, devfn, pos, len, value);             \
  60         pci_unlock_config(flags);                                       \
  61         return res;                                                     \
  62 }
  63
  64 PCI_OP_READ(byte, u8, 1)
  65 PCI_OP_READ(word, u16, 2)
  66 PCI_OP_READ(dword, u32, 4)
  67 PCI_OP_WRITE(byte, u8, 1)
  68 PCI_OP_WRITE(word, u16, 2)
  69 PCI_OP_WRITE(dword, u32, 4)
  70
  71 EXPORT_SYMBOL(pci_bus_read_config_byte);
  72 EXPORT_SYMBOL(pci_bus_read_config_word);
  73 EXPORT_SYMBOL(pci_bus_read_config_dword);
  74 EXPORT_SYMBOL(pci_bus_write_config_byte);
  75 EXPORT_SYMBOL(pci_bus_write_config_word);
  76 EXPORT_SYMBOL(pci_bus_write_config_dword);
  77
  78 int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn,
  79                             int where, int size, u32 *val)
  80 {
  81         void __iomem *addr;
  82
  83         addr = bus->ops->map_bus(bus, devfn, where);
  84         if (!addr) {
  85                 *val = ~0;
  86                 return PCIBIOS_DEVICE_NOT_FOUND;
  87         }
  88
  89         if (size == 1)
  90                 *val = readb(addr);
  91         else if (size == 2)
  92                 *val = readw(addr);
  93         else
  94                 *val = readl(addr);
  95
  96         return PCIBIOS_SUCCESSFUL;
  97 }
  98 EXPORT_SYMBOL_GPL(pci_generic_config_read);
  99
 100 int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn,
 101                              int where, int size, u32 val)
 102 {
 103         void __iomem *addr;
 104
 105         addr = bus->ops->map_bus(bus, devfn, where);
 106         if (!addr)
 107                 return PCIBIOS_DEVICE_NOT_FOUND;
 108
 109         if (size == 1)
 110                 writeb(val, addr);
 111         else if (size == 2)
 112                 writew(val, addr);
 113         else
 114                 writel(val, addr);
 115
 116         return PCIBIOS_SUCCESSFUL;
 117 }
 118 EXPORT_SYMBOL_GPL(pci_generic_config_write);
 119
 120 int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn,
 121                               int where, int size, u32 *val)
 122 {
 123         void __iomem *addr;
 124
 125         addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
 126         if (!addr) {
 127                 *val = ~0;
 128                 return PCIBIOS_DEVICE_NOT_FOUND;
 129         }
 130
 131         *val = readl(addr);
 132
 133         if (size <= 2)
 134                 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
 135
 136         return PCIBIOS_SUCCESSFUL;
 137 }
 138 EXPORT_SYMBOL_GPL(pci_generic_config_read32);
 139
 140 int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
 141                                int where, int size, u32 val)
 142 {
 143         void __iomem *addr;
 144         u32 mask, tmp;
 145
 146         addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
 147         if (!addr)
 148                 return PCIBIOS_DEVICE_NOT_FOUND;
 149
 150         if (size == 4) {
 151                 writel(val, addr);
 152                 return PCIBIOS_SUCCESSFUL;
 153         }
 154
 155         /*
 156          * In general, hardware that supports only 32-bit writes on PCI is
 157          * not spec-compliant.  For example, software may perform a 16-bit
 158          * write.  If the hardware only supports 32-bit accesses, we must
 159          * do a 32-bit read, merge in the 16 bits we intend to write,
 160          * followed by a 32-bit write.  If the 16 bits we *don't* intend to
 161          * write happen to have any RW1C (write-one-to-clear) bits set, we
 162          * just inadvertently cleared something we shouldn't have.
 163          */
 164         dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n",
 165                              size, pci_domain_nr(bus), bus->number,
 166                              PCI_SLOT(devfn), PCI_FUNC(devfn), where);
 167
 168         mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
 169         tmp = readl(addr) & mask;
 170         tmp |= val << ((where & 0x3) * 8);
 171         writel(tmp, addr);
 172
 173         return PCIBIOS_SUCCESSFUL;
 174 }
 175 EXPORT_SYMBOL_GPL(pci_generic_config_write32);
 176
 177 /**
 178  * pci_bus_set_ops - Set raw operations of pci bus
 179  * @bus:        pci bus struct
 180  * @ops:        new raw operations
 181  *
 182  * Return previous raw operations
 183  */
 184 struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops)
 185 {
 186         struct pci_ops *old_ops;
 187         unsigned long flags;
 188
 189         raw_spin_lock_irqsave(&pci_lock, flags);
 190         old_ops = bus->ops;
 191         bus->ops = ops;
 192         raw_spin_unlock_irqrestore(&pci_lock, flags);
 193         return old_ops;
 194 }
 195 EXPORT_SYMBOL(pci_bus_set_ops);
 196
 197 /*
 198  * The following routines are to prevent the user from accessing PCI config
 199  * space when it's unsafe to do so.  Some devices require this during BIST and
 200  * we're required to prevent it during D-state transitions.
 201  *
 202  * We have a bit per device to indicate it's blocked and a global wait queue
 203  * for callers to sleep on until devices are unblocked.
 204  */
 205 static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait);
 206
 207 static noinline void pci_wait_cfg(struct pci_dev *dev)
 208 {
 209         DECLARE_WAITQUEUE(wait, current);
 210
 211         __add_wait_queue(&pci_cfg_wait, &wait);
 212         do {
 213                 set_current_state(TASK_UNINTERRUPTIBLE);
 214                 raw_spin_unlock_irq(&pci_lock);
 215                 schedule();
 216                 raw_spin_lock_irq(&pci_lock);
 217         } while (dev->block_cfg_access);
 218         __remove_wait_queue(&pci_cfg_wait, &wait);
 219 }
 220
 221 /* Returns 0 on success, negative values indicate error. */
 222 #define PCI_USER_READ_CONFIG(size, type)                                        \
 223 int pci_user_read_config_##size                                         \
 224         (struct pci_dev *dev, int pos, type *val)                       \
 225 {                                                                       \
 226         int ret = PCIBIOS_SUCCESSFUL;                                   \
 227         u32 data = -1;                                                  \
 228         if (PCI_##size##_BAD)                                           \
 229                 return -EINVAL;                                         \
 230         raw_spin_lock_irq(&pci_lock);                           \
 231         if (unlikely(dev->block_cfg_access))                            \
 232                 pci_wait_cfg(dev);                                      \
 233         ret = dev->bus->ops->read(dev->bus, dev->devfn,                 \
 234                                         pos, sizeof(type), &data);      \
 235         raw_spin_unlock_irq(&pci_lock);                         \
 236         *val = (type)data;                                              \
 237         return pcibios_err_to_errno(ret);                               \
 238 }                                                                       \
 239 EXPORT_SYMBOL_GPL(pci_user_read_config_##size);
 240
 241 /* Returns 0 on success, negative values indicate error. */
 242 #define PCI_USER_WRITE_CONFIG(size, type)                               \
 243 int pci_user_write_config_##size                                        \
 244         (struct pci_dev *dev, int pos, type val)                        \
 245 {                                                                       \
 246         int ret = PCIBIOS_SUCCESSFUL;                                   \
 247         if (PCI_##size##_BAD)                                           \
 248                 return -EINVAL;                                         \
 249         raw_spin_lock_irq(&pci_lock);                           \
 250         if (unlikely(dev->block_cfg_access))                            \
 251                 pci_wait_cfg(dev);                                      \
 252         ret = dev->bus->ops->write(dev->bus, dev->devfn,                \
 253                                         pos, sizeof(type), val);        \
 254         raw_spin_unlock_irq(&pci_lock);                         \
 255         return pcibios_err_to_errno(ret);                               \
 256 }                                                                       \
 257 EXPORT_SYMBOL_GPL(pci_user_write_config_##size);
 258
 259 PCI_USER_READ_CONFIG(byte, u8)
 260 PCI_USER_READ_CONFIG(word, u16)
 261 PCI_USER_READ_CONFIG(dword, u32)
 262 PCI_USER_WRITE_CONFIG(byte, u8)
 263 PCI_USER_WRITE_CONFIG(word, u16)
 264 PCI_USER_WRITE_CONFIG(dword, u32)
 265
 266 /* VPD access through PCI 2.2+ VPD capability */
 267
 268 /**
 269  * pci_read_vpd - Read one entry from Vital Product Data
 270  * @dev:        pci device struct
 271  * @pos:        offset in vpd space
 272  * @count:      number of bytes to read
 273  * @buf:        pointer to where to store result
 274  */
 275 ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf)
 276 {
 277         if (!dev->vpd || !dev->vpd->ops)
 278                 return -ENODEV;
 279         return dev->vpd->ops->read(dev, pos, count, buf);
 280 }
 281 EXPORT_SYMBOL(pci_read_vpd);
 282
 283 /**
 284  * pci_write_vpd - Write entry to Vital Product Data
 285  * @dev:        pci device struct
 286  * @pos:        offset in vpd space
 287  * @count:      number of bytes to write
 288  * @buf:        buffer containing write data
 289  */
 290 ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf)
 291 {
 292         if (!dev->vpd || !dev->vpd->ops)
 293                 return -ENODEV;
 294         return dev->vpd->ops->write(dev, pos, count, buf);
 295 }
 296 EXPORT_SYMBOL(pci_write_vpd);
 297
 298 /**
 299  * pci_set_vpd_size - Set size of Vital Product Data space
 300  * @dev:        pci device struct
 301  * @len:        size of vpd space
 302  */
 303 int pci_set_vpd_size(struct pci_dev *dev, size_t len)
 304 {
 305         if (!dev->vpd || !dev->vpd->ops)
 306                 return -ENODEV;
 307         return dev->vpd->ops->set_size(dev, len);
 308 }
 309 EXPORT_SYMBOL(pci_set_vpd_size);
 310
 311 #define PCI_VPD_MAX_SIZE (PCI_VPD_ADDR_MASK + 1)
 312
 313 /**
 314  * pci_vpd_size - determine actual size of Vital Product Data
 315  * @dev:        pci device struct
 316  * @old_size:   current assumed size, also maximum allowed size
 317  */
 318 static size_t pci_vpd_size(struct pci_dev *dev, size_t old_size)
 319 {
 320         size_t off = 0;
 321         unsigned char header[1+2];      /* 1 byte tag, 2 bytes length */
 322
 323         while (off < old_size &&
 324                pci_read_vpd(dev, off, 1, header) == 1) {
 325                 unsigned char tag;
 326
 327                 if (header[0] & PCI_VPD_LRDT) {
 328                         /* Large Resource Data Type Tag */
 329                         tag = pci_vpd_lrdt_tag(header);
 330                         /* Only read length from known tag items */
 331                         if ((tag == PCI_VPD_LTIN_ID_STRING) ||
 332                             (tag == PCI_VPD_LTIN_RO_DATA) ||
 333                             (tag == PCI_VPD_LTIN_RW_DATA)) {
 334                                 if (pci_read_vpd(dev, off+1, 2,
 335                                                  &header[1]) != 2) {
 336                                         dev_warn(&dev->dev,
 337                                                  "invalid large VPD tag %02x size at offset %zu",
 338                                                  tag, off + 1);
 339                                         return 0;
 340                                 }
 341                                 off += PCI_VPD_LRDT_TAG_SIZE +
 342                                         pci_vpd_lrdt_size(header);
 343                         }
 344                 } else {
 345                         /* Short Resource Data Type Tag */
 346                         off += PCI_VPD_SRDT_TAG_SIZE +
 347                                 pci_vpd_srdt_size(header);
 348                         tag = pci_vpd_srdt_tag(header);
 349                 }
 350
 351                 if (tag == PCI_VPD_STIN_END)    /* End tag descriptor */
 352                         return off;
 353
 354                 if ((tag != PCI_VPD_LTIN_ID_STRING) &&
 355                     (tag != PCI_VPD_LTIN_RO_DATA) &&
 356                     (tag != PCI_VPD_LTIN_RW_DATA)) {
 357                         dev_warn(&dev->dev,
 358                                  "invalid %s VPD tag %02x at offset %zu",
 359                                  (header[0] & PCI_VPD_LRDT) ? "large" : "short",
 360                                  tag, off);
 361                         return 0;
 362                 }
 363         }
 364         return 0;
 365 }
 366
 367 /*
 368  * Wait for last operation to complete.
 369  * This code has to spin since there is no other notification from the PCI
 370  * hardware. Since the VPD is often implemented by serial attachment to an
 371  * EEPROM, it may take many milliseconds to complete.
 372  *
 373  * Returns 0 on success, negative values indicate error.
 374  */
 375 static int pci_vpd_wait(struct pci_dev *dev)
 376 {
 377         struct pci_vpd *vpd = dev->vpd;
 378         unsigned long timeout = jiffies + msecs_to_jiffies(125);
 379         unsigned long max_sleep = 16;
 380         u16 status;
 381         int ret;
 382
 383         if (!vpd->busy)
 384                 return 0;
 385
 386         while (time_before(jiffies, timeout)) {
 387                 ret = pci_user_read_config_word(dev, vpd->cap + PCI_VPD_ADDR,
 388                                                 &status);
 389                 if (ret < 0)
 390                         return ret;
 391
 392                 if ((status & PCI_VPD_ADDR_F) == vpd->flag) {
 393                         vpd->busy = 0;
 394                         return 0;
 395                 }
 396
 397                 if (fatal_signal_pending(current))
 398                         return -EINTR;
 399
 400                 usleep_range(10, max_sleep);
 401                 if (max_sleep < 1024)
 402                         max_sleep *= 2;
 403         }
 404
 405         dev_warn(&dev->dev, "VPD access failed.  This is likely a firmware bug on this device.  Contact the card vendor for a firmware update\n");
 406         return -ETIMEDOUT;
 407 }
 408
 409 static ssize_t pci_vpd_read(struct pci_dev *dev, loff_t pos, size_t count,
 410                             void *arg)
 411 {
 412         struct pci_vpd *vpd = dev->vpd;
 413         int ret;
 414         loff_t end = pos + count;
 415         u8 *buf = arg;
 416
 417         if (pos < 0)
 418                 return -EINVAL;
 419
 420         if (!vpd->valid) {
 421                 vpd->valid = 1;
 422                 vpd->len = pci_vpd_size(dev, vpd->len);
 423         }
 424
 425         if (vpd->len == 0)
 426                 return -EIO;
 427
 428         if (pos > vpd->len)
 429                 return 0;
 430
 431         if (end > vpd->len) {
 432                 end = vpd->len;
 433                 count = end - pos;
 434         }
 435
 436         if (mutex_lock_killable(&vpd->lock))
 437                 return -EINTR;
 438
 439         ret = pci_vpd_wait(dev);
 440         if (ret < 0)
 441                 goto out;
 442
 443         while (pos < end) {
 444                 u32 val;
 445                 unsigned int i, skip;
 446
 447                 ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR,
 448                                                  pos & ~3);
 449                 if (ret < 0)
 450                         break;
 451                 vpd->busy = 1;
 452                 vpd->flag = PCI_VPD_ADDR_F;
 453                 ret = pci_vpd_wait(dev);
 454                 if (ret < 0)
 455                         break;
 456
 457                 ret = pci_user_read_config_dword(dev, vpd->cap + PCI_VPD_DATA, &val);
 458                 if (ret < 0)
 459                         break;
 460
 461                 skip = pos & 3;
 462                 for (i = 0;  i < sizeof(u32); i++) {
 463                         if (i >= skip) {
 464                                 *buf++ = val;
 465                                 if (++pos == end)
 466                                         break;
 467                         }
 468                         val >>= 8;
 469                 }
 470         }
 471 out:
 472         mutex_unlock(&vpd->lock);
 473         return ret ? ret : count;
 474 }
 475
 476 static ssize_t pci_vpd_write(struct pci_dev *dev, loff_t pos, size_t count,
 477                              const void *arg)
 478 {
 479         struct pci_vpd *vpd = dev->vpd;
 480         const u8 *buf = arg;
 481         loff_t end = pos + count;
 482         int ret = 0;
 483
 484         if (pos < 0 || (pos & 3) || (count & 3))
 485                 return -EINVAL;
 486
 487         if (!vpd->valid) {
 488                 vpd->valid = 1;
 489                 vpd->len = pci_vpd_size(dev, vpd->len);
 490         }
 491
 492         if (vpd->len == 0)
 493                 return -EIO;
 494
 495         if (end > vpd->len)
 496                 return -EINVAL;
 497
 498         if (mutex_lock_killable(&vpd->lock))
 499                 return -EINTR;
 500
 501         ret = pci_vpd_wait(dev);
 502         if (ret < 0)
 503                 goto out;
 504
 505         while (pos < end) {
 506                 u32 val;
 507
 508                 val = *buf++;
 509                 val |= *buf++ << 8;
 510                 val |= *buf++ << 16;
 511                 val |= *buf++ << 24;
 512
 513                 ret = pci_user_write_config_dword(dev, vpd->cap + PCI_VPD_DATA, val);
 514                 if (ret < 0)
 515                         break;
 516                 ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR,
 517                                                  pos | PCI_VPD_ADDR_F);
 518                 if (ret < 0)
 519                         break;
 520
 521                 vpd->busy = 1;
 522                 vpd->flag = 0;
 523                 ret = pci_vpd_wait(dev);
 524                 if (ret < 0)
 525                         break;
 526
 527                 pos += sizeof(u32);
 528         }
 529 out:
 530         mutex_unlock(&vpd->lock);
 531         return ret ? ret : count;
 532 }
 533
 534 static int pci_vpd_set_size(struct pci_dev *dev, size_t len)
 535 {
 536         struct pci_vpd *vpd = dev->vpd;
 537
 538         if (len == 0 || len > PCI_VPD_MAX_SIZE)
 539                 return -EIO;
 540
 541         vpd->valid = 1;
 542         vpd->len = len;
 543
 544         return 0;
 545 }
 546
 547 static const struct pci_vpd_ops pci_vpd_ops = {
 548         .read = pci_vpd_read,
 549         .write = pci_vpd_write,
 550         .set_size = pci_vpd_set_size,
 551 };
 552
 553 static ssize_t pci_vpd_f0_read(struct pci_dev *dev, loff_t pos, size_t count,
 554                                void *arg)
 555 {
 556         struct pci_dev *tdev = pci_get_slot(dev->bus,
 557                                             PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
 558         ssize_t ret;
 559
 560         if (!tdev)
 561                 return -ENODEV;
 562
 563         ret = pci_read_vpd(tdev, pos, count, arg);
 564         pci_dev_put(tdev);
 565         return ret;
 566 }
 567
 568 static ssize_t pci_vpd_f0_write(struct pci_dev *dev, loff_t pos, size_t count,
 569                                 const void *arg)
 570 {
 571         struct pci_dev *tdev = pci_get_slot(dev->bus,
 572                                             PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
 573         ssize_t ret;
 574
 575         if (!tdev)
 576                 return -ENODEV;
 577
 578         ret = pci_write_vpd(tdev, pos, count, arg);
 579         pci_dev_put(tdev);
 580         return ret;
 581 }
 582
 583 static int pci_vpd_f0_set_size(struct pci_dev *dev, size_t len)
 584 {
 585         struct pci_dev *tdev = pci_get_slot(dev->bus,
 586                                             PCI_DEVFN(PCI_SLOT(dev->devfn), 0));
 587         int ret;
 588
 589         if (!tdev)
 590                 return -ENODEV;
 591
 592         ret = pci_set_vpd_size(tdev, len);
 593         pci_dev_put(tdev);
 594         return ret;
 595 }
 596
 597 static const struct pci_vpd_ops pci_vpd_f0_ops = {
 598         .read = pci_vpd_f0_read,
 599         .write = pci_vpd_f0_write,
 600         .set_size = pci_vpd_f0_set_size,
 601 };
 602
 603 int pci_vpd_init(struct pci_dev *dev)
 604 {
 605         struct pci_vpd *vpd;
 606         u8 cap;
 607
 608         cap = pci_find_capability(dev, PCI_CAP_ID_VPD);
 609         if (!cap)
 610                 return -ENODEV;
 611
 612         vpd = kzalloc(sizeof(*vpd), GFP_ATOMIC);
 613         if (!vpd)
 614                 return -ENOMEM;
 615
 616         vpd->len = PCI_VPD_MAX_SIZE;
 617         if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0)
 618                 vpd->ops = &pci_vpd_f0_ops;
 619         else
 620                 vpd->ops = &pci_vpd_ops;
 621         mutex_init(&vpd->lock);
 622         vpd->cap = cap;
 623         vpd->busy = 0;
 624         vpd->valid = 0;
 625         dev->vpd = vpd;
 626         return 0;
 627 }
 628
 629 void pci_vpd_release(struct pci_dev *dev)
 630 {
 631         kfree(dev->vpd);
 632 }
 633
 634 /**
 635  * pci_cfg_access_lock - Lock PCI config reads/writes
 636  * @dev:        pci device struct
 637  *
 638  * When access is locked, any userspace reads or writes to config
 639  * space and concurrent lock requests will sleep until access is
 640  * allowed via pci_cfg_access_unlock() again.
 641  */
 642 void pci_cfg_access_lock(struct pci_dev *dev)
 643 {
 644         might_sleep();
 645
 646         raw_spin_lock_irq(&pci_lock);
 647         if (dev->block_cfg_access)
 648                 pci_wait_cfg(dev);
 649         dev->block_cfg_access = 1;
 650         raw_spin_unlock_irq(&pci_lock);
 651 }
 652 EXPORT_SYMBOL_GPL(pci_cfg_access_lock);
 653
 654 /**
 655  * pci_cfg_access_trylock - try to lock PCI config reads/writes
 656  * @dev:        pci device struct
 657  *
 658  * Same as pci_cfg_access_lock, but will return 0 if access is
 659  * already locked, 1 otherwise. This function can be used from
 660  * atomic contexts.
 661  */
 662 bool pci_cfg_access_trylock(struct pci_dev *dev)
 663 {
 664         unsigned long flags;
 665         bool locked = true;
 666
 667         raw_spin_lock_irqsave(&pci_lock, flags);
 668         if (dev->block_cfg_access)
 669                 locked = false;
 670         else
 671                 dev->block_cfg_access = 1;
 672         raw_spin_unlock_irqrestore(&pci_lock, flags);
 673
 674         return locked;
 675 }
 676 EXPORT_SYMBOL_GPL(pci_cfg_access_trylock);
 677
 678 /**
 679  * pci_cfg_access_unlock - Unlock PCI config reads/writes
 680  * @dev:        pci device struct
 681  *
 682  * This function allows PCI config accesses to resume.
 683  */
 684 void pci_cfg_access_unlock(struct pci_dev *dev)
 685 {
 686         unsigned long flags;
 687
 688         raw_spin_lock_irqsave(&pci_lock, flags);
 689
 690         /* This indicates a problem in the caller, but we don't need
 691          * to kill them, unlike a double-block above. */
 692         WARN_ON(!dev->block_cfg_access);
 693
 694         dev->block_cfg_access = 0;
 695         raw_spin_unlock_irqrestore(&pci_lock, flags);
 696
 697         wake_up_all(&pci_cfg_wait);
 698 }
 699 EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
 700
 701 static inline int pcie_cap_version(const struct pci_dev *dev)
 702 {
 703         return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS;
 704 }
 705
 706 static bool pcie_downstream_port(const struct pci_dev *dev)
 707 {
 708         int type = pci_pcie_type(dev);
 709
 710         return type == PCI_EXP_TYPE_ROOT_PORT ||
 711                type == PCI_EXP_TYPE_DOWNSTREAM ||
 712                type == PCI_EXP_TYPE_PCIE_BRIDGE;
 713 }
 714
 715 bool pcie_cap_has_lnkctl(const struct pci_dev *dev)
 716 {
 717         int type = pci_pcie_type(dev);
 718
 719         return type == PCI_EXP_TYPE_ENDPOINT ||
 720                type == PCI_EXP_TYPE_LEG_END ||
 721                type == PCI_EXP_TYPE_ROOT_PORT ||
 722                type == PCI_EXP_TYPE_UPSTREAM ||
 723                type == PCI_EXP_TYPE_DOWNSTREAM ||
 724                type == PCI_EXP_TYPE_PCI_BRIDGE ||
 725                type == PCI_EXP_TYPE_PCIE_BRIDGE;
 726 }
 727
 728 static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev)
 729 {
 730         return pcie_downstream_port(dev) &&
 731                pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT;
 732 }
 733
 734 static inline bool pcie_cap_has_rtctl(const struct pci_dev *dev)
 735 {
 736         int type = pci_pcie_type(dev);
 737
 738         return type == PCI_EXP_TYPE_ROOT_PORT ||
 739                type == PCI_EXP_TYPE_RC_EC;
 740 }
 741
 742 static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos)
 743 {
 744         if (!pci_is_pcie(dev))
 745                 return false;
 746
 747         switch (pos) {
 748         case PCI_EXP_FLAGS:
 749                 return true;
 750         case PCI_EXP_DEVCAP:
 751         case PCI_EXP_DEVCTL:
 752         case PCI_EXP_DEVSTA:
 753                 return true;
 754         case PCI_EXP_LNKCAP:
 755         case PCI_EXP_LNKCTL:
 756         case PCI_EXP_LNKSTA:
 757                 return pcie_cap_has_lnkctl(dev);
 758         case PCI_EXP_SLTCAP:
 759         case PCI_EXP_SLTCTL:
 760         case PCI_EXP_SLTSTA:
 761                 return pcie_cap_has_sltctl(dev);
 762         case PCI_EXP_RTCTL:
 763         case PCI_EXP_RTCAP:
 764         case PCI_EXP_RTSTA:
 765                 return pcie_cap_has_rtctl(dev);
 766         case PCI_EXP_DEVCAP2:
 767         case PCI_EXP_DEVCTL2:
 768         case PCI_EXP_LNKCAP2:
 769         case PCI_EXP_LNKCTL2:
 770         case PCI_EXP_LNKSTA2:
 771                 return pcie_cap_version(dev) > 1;
 772         default:
 773                 return false;
 774         }
 775 }
 776
 777 /*
 778  * Note that these accessor functions are only for the "PCI Express
 779  * Capability" (see PCIe spec r3.0, sec 7.8).  They do not apply to the
 780  * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.)
 781  */
 782 int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val)
 783 {
 784         int ret;
 785
 786         *val = 0;
 787         if (pos & 1)
 788                 return -EINVAL;
 789
 790         if (pcie_capability_reg_implemented(dev, pos)) {
 791                 ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val);
 792                 /*
 793                  * Reset *val to 0 if pci_read_config_word() fails, it may
 794                  * have been written as 0xFFFF if hardware error happens
 795                  * during pci_read_config_word().
 796                  */
 797                 if (ret)
 798                         *val = 0;
 799                 return ret;
 800         }
 801
 802         /*
 803          * For Functions that do not implement the Slot Capabilities,
 804          * Slot Status, and Slot Control registers, these spaces must
 805          * be hardwired to 0b, with the exception of the Presence Detect
 806          * State bit in the Slot Status register of Downstream Ports,
 807          * which must be hardwired to 1b.  (PCIe Base Spec 3.0, sec 7.8)
 808          */
 809         if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
 810             pos == PCI_EXP_SLTSTA)
 811                 *val = PCI_EXP_SLTSTA_PDS;
 812
 813         return 0;
 814 }
 815 EXPORT_SYMBOL(pcie_capability_read_word);
 816
 817 int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val)
 818 {
 819         int ret;
 820
 821         *val = 0;
 822         if (pos & 3)
 823                 return -EINVAL;
 824
 825         if (pcie_capability_reg_implemented(dev, pos)) {
 826                 ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val);
 827                 /*
 828                  * Reset *val to 0 if pci_read_config_dword() fails, it may
 829                  * have been written as 0xFFFFFFFF if hardware error happens
 830                  * during pci_read_config_dword().
 831                  */
 832                 if (ret)
 833                         *val = 0;
 834                 return ret;
 835         }
 836
 837         if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
 838             pos == PCI_EXP_SLTSTA)
 839                 *val = PCI_EXP_SLTSTA_PDS;
 840
 841         return 0;
 842 }
 843 EXPORT_SYMBOL(pcie_capability_read_dword);
 844
 845 int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val)
 846 {
 847         if (pos & 1)
 848                 return -EINVAL;
 849
 850         if (!pcie_capability_reg_implemented(dev, pos))
 851                 return 0;
 852
 853         return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val);
 854 }
 855 EXPORT_SYMBOL(pcie_capability_write_word);
 856
 857 int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val)
 858 {
 859         if (pos & 3)
 860                 return -EINVAL;
 861
 862         if (!pcie_capability_reg_implemented(dev, pos))
 863                 return 0;
 864
 865         return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val);
 866 }
 867 EXPORT_SYMBOL(pcie_capability_write_dword);
 868
 869 int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos,
 870                                        u16 clear, u16 set)
 871 {
 872         int ret;
 873         u16 val;
 874
 875         ret = pcie_capability_read_word(dev, pos, &val);
 876         if (!ret) {
 877                 val &= ~clear;
 878                 val |= set;
 879                 ret = pcie_capability_write_word(dev, pos, val);
 880         }
 881
 882         return ret;
 883 }
 884 EXPORT_SYMBOL(pcie_capability_clear_and_set_word);
 885
 886 int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos,
 887                                         u32 clear, u32 set)
 888 {
 889         int ret;
 890         u32 val;
 891
 892         ret = pcie_capability_read_dword(dev, pos, &val);
 893         if (!ret) {
 894                 val &= ~clear;
 895                 val |= set;
 896                 ret = pcie_capability_write_dword(dev, pos, val);
 897         }
 898
 899         return ret;
 900 }
 901 EXPORT_SYMBOL(pcie_capability_clear_and_set_dword);
 902
 903 int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val)
 904 {
 905         if (pci_dev_is_disconnected(dev)) {
 906                 *val = ~0;
 907                 return PCIBIOS_DEVICE_NOT_FOUND;
 908         }
 909         return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val);
 910 }
 911 EXPORT_SYMBOL(pci_read_config_byte);
 912
 913 int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val)
 914 {
 915         if (pci_dev_is_disconnected(dev)) {
 916                 *val = ~0;
 917                 return PCIBIOS_DEVICE_NOT_FOUND;
 918         }
 919         return pci_bus_read_config_word(dev->bus, dev->devfn, where, val);
 920 }
 921 EXPORT_SYMBOL(pci_read_config_word);
 922
 923 int pci_read_config_dword(const struct pci_dev *dev, int where,
 924                                         u32 *val)
 925 {
 926         if (pci_dev_is_disconnected(dev)) {
 927                 *val = ~0;
 928                 return PCIBIOS_DEVICE_NOT_FOUND;
 929         }
 930         return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val);
 931 }
 932 EXPORT_SYMBOL(pci_read_config_dword);
 933
 934 int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val)
 935 {
 936         if (pci_dev_is_disconnected(dev))
 937                 return PCIBIOS_DEVICE_NOT_FOUND;
 938         return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val);
 939 }
 940 EXPORT_SYMBOL(pci_write_config_byte);
 941
 942 int pci_write_config_word(const struct pci_dev *dev, int where, u16 val)
 943 {
 944         if (pci_dev_is_disconnected(dev))
 945                 return PCIBIOS_DEVICE_NOT_FOUND;
 946         return pci_bus_write_config_word(dev->bus, dev->devfn, where, val);
 947 }
 948 EXPORT_SYMBOL(pci_write_config_word);
 949
 950 int pci_write_config_dword(const struct pci_dev *dev, int where,
 951                                          u32 val)
 952 {
 953         if (pci_dev_is_disconnected(dev))
 954                 return PCIBIOS_DEVICE_NOT_FOUND;
 955         return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val);
 956 }
 957 EXPORT_SYMBOL(pci_write_config_dword);