drivers/pci/hotplug/pciehp_hpc.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * PCI Express PCI Hot Plug Driver
   4  *
   5  * Copyright (C) 1995,2001 Compaq Computer Corporation
   6  * Copyright (C) 2001 Greg Kroah-Hartman (greg@kroah.com)
   7  * Copyright (C) 2001 IBM Corp.
   8  * Copyright (C) 2003-2004 Intel Corporation
   9  *
  10  * All rights reserved.
  11  *
  12  * Send feedback to <greg@kroah.com>,<kristen.c.accardi@intel.com>
  13  */
  14
  15 #define dev_fmt(fmt) "pciehp: " fmt
  16
  17 #include <linux/kernel.h>
  18 #include <linux/types.h>
  19 #include <linux/jiffies.h>
  20 #include <linux/kthread.h>
  21 #include <linux/pci.h>
  22 #include <linux/pm_runtime.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/slab.h>
  25
  26 #include "../pci.h"
  27 #include "pciehp.h"
  28
  29 static inline struct pci_dev *ctrl_dev(struct controller *ctrl)
  30 {
  31         return ctrl->pcie->port;
  32 }
  33
  34 static irqreturn_t pciehp_isr(int irq, void *dev_id);
  35 static irqreturn_t pciehp_ist(int irq, void *dev_id);
  36 static int pciehp_poll(void *data);
  37
  38 static inline int pciehp_request_irq(struct controller *ctrl)
  39 {
  40         int retval, irq = ctrl->pcie->irq;
  41
  42         if (pciehp_poll_mode) {
  43                 ctrl->poll_thread = kthread_run(&pciehp_poll, ctrl,
  44                                                 "pciehp_poll-%s",
  45                                                 slot_name(ctrl));
  46                 return PTR_ERR_OR_ZERO(ctrl->poll_thread);
  47         }
  48
  49         /* Installs the interrupt handler */
  50         retval = request_threaded_irq(irq, pciehp_isr, pciehp_ist,
  51                                       IRQF_SHARED, "pciehp", ctrl);
  52         if (retval)
  53                 ctrl_err(ctrl, "Cannot get irq %d for the hotplug controller\n",
  54                          irq);
  55         return retval;
  56 }
  57
  58 static inline void pciehp_free_irq(struct controller *ctrl)
  59 {
  60         if (pciehp_poll_mode)
  61                 kthread_stop(ctrl->poll_thread);
  62         else
  63                 free_irq(ctrl->pcie->irq, ctrl);
  64 }
  65
  66 static int pcie_poll_cmd(struct controller *ctrl, int timeout)
  67 {
  68         struct pci_dev *pdev = ctrl_dev(ctrl);
  69         u16 slot_status;
  70
  71         do {
  72                 pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status);
  73                 if (slot_status == (u16) ~0) {
  74                         ctrl_info(ctrl, "%s: no response from device\n",
  75                                   __func__);
  76                         return 0;
  77                 }
  78
  79                 if (slot_status & PCI_EXP_SLTSTA_CC) {
  80                         pcie_capability_write_word(pdev, PCI_EXP_SLTSTA,
  81                                                    PCI_EXP_SLTSTA_CC);
  82                         return 1;
  83                 }
  84                 msleep(10);
  85                 timeout -= 10;
  86         } while (timeout >= 0);
  87         return 0;       /* timeout */
  88 }
  89
  90 static void pcie_wait_cmd(struct controller *ctrl)
  91 {
  92         unsigned int msecs = pciehp_poll_mode ? 2500 : 1000;
  93         unsigned long duration = msecs_to_jiffies(msecs);
  94         unsigned long cmd_timeout = ctrl->cmd_started + duration;
  95         unsigned long now, timeout;
  96         int rc;
  97
  98         /*
  99          * If the controller does not generate notifications for command
 100          * completions, we never need to wait between writes.
 101          */
 102         if (NO_CMD_CMPL(ctrl))
 103                 return;
 104
 105         if (!ctrl->cmd_busy)
 106                 return;
 107
 108         /*
 109          * Even if the command has already timed out, we want to call
 110          * pcie_poll_cmd() so it can clear PCI_EXP_SLTSTA_CC.
 111          */
 112         now = jiffies;
 113         if (time_before_eq(cmd_timeout, now))
 114                 timeout = 1;
 115         else
 116                 timeout = cmd_timeout - now;
 117
 118         if (ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE &&
 119             ctrl->slot_ctrl & PCI_EXP_SLTCTL_CCIE)
 120                 rc = wait_event_timeout(ctrl->queue, !ctrl->cmd_busy, timeout);
 121         else
 122                 rc = pcie_poll_cmd(ctrl, jiffies_to_msecs(timeout));
 123
 124         if (!rc)
 125                 ctrl_info(ctrl, "Timeout on hotplug command %#06x (issued %u msec ago)\n",
 126                           ctrl->slot_ctrl,
 127                           jiffies_to_msecs(jiffies - ctrl->cmd_started));
 128 }
 129
 130 #define CC_ERRATUM_MASK         (PCI_EXP_SLTCTL_PCC |   \
 131                                  PCI_EXP_SLTCTL_PIC |   \
 132                                  PCI_EXP_SLTCTL_AIC |   \
 133                                  PCI_EXP_SLTCTL_EIC)
 134
 135 static void pcie_do_write_cmd(struct controller *ctrl, u16 cmd,
 136                               u16 mask, bool wait)
 137 {
 138         struct pci_dev *pdev = ctrl_dev(ctrl);
 139         u16 slot_ctrl_orig, slot_ctrl;
 140
 141         mutex_lock(&ctrl->ctrl_lock);
 142
 143         /*
 144          * Always wait for any previous command that might still be in progress
 145          */
 146         pcie_wait_cmd(ctrl);
 147
 148         pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl);
 149         if (slot_ctrl == (u16) ~0) {
 150                 ctrl_info(ctrl, "%s: no response from device\n", __func__);
 151                 goto out;
 152         }
 153
 154         slot_ctrl_orig = slot_ctrl;
 155         slot_ctrl &= ~mask;
 156         slot_ctrl |= (cmd & mask);
 157         ctrl->cmd_busy = 1;
 158         smp_mb();
 159         ctrl->slot_ctrl = slot_ctrl;
 160         pcie_capability_write_word(pdev, PCI_EXP_SLTCTL, slot_ctrl);
 161         ctrl->cmd_started = jiffies;
 162
 163         /*
 164          * Controllers with the Intel CF118 and similar errata advertise
 165          * Command Completed support, but they only set Command Completed
 166          * if we change the "Control" bits for power, power indicator,
 167          * attention indicator, or interlock.  If we only change the
 168          * "Enable" bits, they never set the Command Completed bit.
 169          */
 170         if (pdev->broken_cmd_compl &&
 171             (slot_ctrl_orig & CC_ERRATUM_MASK) == (slot_ctrl & CC_ERRATUM_MASK))
 172                 ctrl->cmd_busy = 0;
 173
 174         /*
 175          * Optionally wait for the hardware to be ready for a new command,
 176          * indicating completion of the above issued command.
 177          */
 178         if (wait)
 179                 pcie_wait_cmd(ctrl);
 180
 181 out:
 182         mutex_unlock(&ctrl->ctrl_lock);
 183 }
 184
 185 /**
 186  * pcie_write_cmd - Issue controller command
 187  * @ctrl: controller to which the command is issued
 188  * @cmd:  command value written to slot control register
 189  * @mask: bitmask of slot control register to be modified
 190  */
 191 static void pcie_write_cmd(struct controller *ctrl, u16 cmd, u16 mask)
 192 {
 193         pcie_do_write_cmd(ctrl, cmd, mask, true);
 194 }
 195
 196 /* Same as above without waiting for the hardware to latch */
 197 static void pcie_write_cmd_nowait(struct controller *ctrl, u16 cmd, u16 mask)
 198 {
 199         pcie_do_write_cmd(ctrl, cmd, mask, false);
 200 }
 201
 202 /**
 203  * pciehp_check_link_active() - Is the link active
 204  * @ctrl: PCIe hotplug controller
 205  *
 206  * Check whether the downstream link is currently active. Note it is
 207  * possible that the card is removed immediately after this so the
 208  * caller may need to take it into account.
 209  *
 210  * If the hotplug controller itself is not available anymore returns
 211  * %-ENODEV.
 212  */
 213 int pciehp_check_link_active(struct controller *ctrl)
 214 {
 215         struct pci_dev *pdev = ctrl_dev(ctrl);
 216         u16 lnk_status;
 217         int ret;
 218
 219         ret = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
 220         if (ret == PCIBIOS_DEVICE_NOT_FOUND || lnk_status == (u16)~0)
 221                 return -ENODEV;
 222
 223         ret = !!(lnk_status & PCI_EXP_LNKSTA_DLLLA);
 224         ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status);
 225
 226         return ret;
 227 }
 228
 229 static bool pci_bus_check_dev(struct pci_bus *bus, int devfn)
 230 {
 231         u32 l;
 232         int count = 0;
 233         int delay = 1000, step = 20;
 234         bool found = false;
 235
 236         do {
 237                 found = pci_bus_read_dev_vendor_id(bus, devfn, &l, 0);
 238                 count++;
 239
 240                 if (found)
 241                         break;
 242
 243                 msleep(step);
 244                 delay -= step;
 245         } while (delay > 0);
 246
 247         if (count > 1)
 248                 pr_debug("pci %04x:%02x:%02x.%d id reading try %d times with interval %d ms to get %08x\n",
 249                         pci_domain_nr(bus), bus->number, PCI_SLOT(devfn),
 250                         PCI_FUNC(devfn), count, step, l);
 251
 252         return found;
 253 }
 254
 255 int pciehp_check_link_status(struct controller *ctrl)
 256 {
 257         struct pci_dev *pdev = ctrl_dev(ctrl);
 258         bool found;
 259         u16 lnk_status;
 260
 261         if (!pcie_wait_for_link(pdev, true))
 262                 return -1;
 263
 264         found = pci_bus_check_dev(ctrl->pcie->port->subordinate,
 265                                         PCI_DEVFN(0, 0));
 266
 267         /* ignore link or presence changes up to this point */
 268         if (found)
 269                 atomic_and(~(PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC),
 270                            &ctrl->pending_events);
 271
 272         pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnk_status);
 273         ctrl_dbg(ctrl, "%s: lnk_status = %x\n", __func__, lnk_status);
 274         if ((lnk_status & PCI_EXP_LNKSTA_LT) ||
 275             !(lnk_status & PCI_EXP_LNKSTA_NLW)) {
 276                 ctrl_err(ctrl, "link training error: status %#06x\n",
 277                          lnk_status);
 278                 return -1;
 279         }
 280
 281         pcie_update_link_speed(ctrl->pcie->port->subordinate, lnk_status);
 282
 283         if (!found)
 284                 return -1;
 285
 286         return 0;
 287 }
 288
 289 static int __pciehp_link_set(struct controller *ctrl, bool enable)
 290 {
 291         struct pci_dev *pdev = ctrl_dev(ctrl);
 292         u16 lnk_ctrl;
 293
 294         pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &lnk_ctrl);
 295
 296         if (enable)
 297                 lnk_ctrl &= ~PCI_EXP_LNKCTL_LD;
 298         else
 299                 lnk_ctrl |= PCI_EXP_LNKCTL_LD;
 300
 301         pcie_capability_write_word(pdev, PCI_EXP_LNKCTL, lnk_ctrl);
 302         ctrl_dbg(ctrl, "%s: lnk_ctrl = %x\n", __func__, lnk_ctrl);
 303         return 0;
 304 }
 305
 306 static int pciehp_link_enable(struct controller *ctrl)
 307 {
 308         return __pciehp_link_set(ctrl, true);
 309 }
 310
 311 int pciehp_get_raw_indicator_status(struct hotplug_slot *hotplug_slot,
 312                                     u8 *status)
 313 {
 314         struct controller *ctrl = to_ctrl(hotplug_slot);
 315         struct pci_dev *pdev = ctrl_dev(ctrl);
 316         u16 slot_ctrl;
 317
 318         pci_config_pm_runtime_get(pdev);
 319         pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl);
 320         pci_config_pm_runtime_put(pdev);
 321         *status = (slot_ctrl & (PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC)) >> 6;
 322         return 0;
 323 }
 324
 325 int pciehp_get_attention_status(struct hotplug_slot *hotplug_slot, u8 *status)
 326 {
 327         struct controller *ctrl = to_ctrl(hotplug_slot);
 328         struct pci_dev *pdev = ctrl_dev(ctrl);
 329         u16 slot_ctrl;
 330
 331         pci_config_pm_runtime_get(pdev);
 332         pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl);
 333         pci_config_pm_runtime_put(pdev);
 334         ctrl_dbg(ctrl, "%s: SLOTCTRL %x, value read %x\n", __func__,
 335                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, slot_ctrl);
 336
 337         switch (slot_ctrl & PCI_EXP_SLTCTL_AIC) {
 338         case PCI_EXP_SLTCTL_ATTN_IND_ON:
 339                 *status = 1;    /* On */
 340                 break;
 341         case PCI_EXP_SLTCTL_ATTN_IND_BLINK:
 342                 *status = 2;    /* Blink */
 343                 break;
 344         case PCI_EXP_SLTCTL_ATTN_IND_OFF:
 345                 *status = 0;    /* Off */
 346                 break;
 347         default:
 348                 *status = 0xFF;
 349                 break;
 350         }
 351
 352         return 0;
 353 }
 354
 355 void pciehp_get_power_status(struct controller *ctrl, u8 *status)
 356 {
 357         struct pci_dev *pdev = ctrl_dev(ctrl);
 358         u16 slot_ctrl;
 359
 360         pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &slot_ctrl);
 361         ctrl_dbg(ctrl, "%s: SLOTCTRL %x value read %x\n", __func__,
 362                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, slot_ctrl);
 363
 364         switch (slot_ctrl & PCI_EXP_SLTCTL_PCC) {
 365         case PCI_EXP_SLTCTL_PWR_ON:
 366                 *status = 1;    /* On */
 367                 break;
 368         case PCI_EXP_SLTCTL_PWR_OFF:
 369                 *status = 0;    /* Off */
 370                 break;
 371         default:
 372                 *status = 0xFF;
 373                 break;
 374         }
 375 }
 376
 377 void pciehp_get_latch_status(struct controller *ctrl, u8 *status)
 378 {
 379         struct pci_dev *pdev = ctrl_dev(ctrl);
 380         u16 slot_status;
 381
 382         pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status);
 383         *status = !!(slot_status & PCI_EXP_SLTSTA_MRLSS);
 384 }
 385
 386 /**
 387  * pciehp_card_present() - Is the card present
 388  * @ctrl: PCIe hotplug controller
 389  *
 390  * Function checks whether the card is currently present in the slot and
 391  * in that case returns true. Note it is possible that the card is
 392  * removed immediately after the check so the caller may need to take
 393  * this into account.
 394  *
 395  * It the hotplug controller itself is not available anymore returns
 396  * %-ENODEV.
 397  */
 398 int pciehp_card_present(struct controller *ctrl)
 399 {
 400         struct pci_dev *pdev = ctrl_dev(ctrl);
 401         u16 slot_status;
 402         int ret;
 403
 404         ret = pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status);
 405         if (ret == PCIBIOS_DEVICE_NOT_FOUND || slot_status == (u16)~0)
 406                 return -ENODEV;
 407
 408         return !!(slot_status & PCI_EXP_SLTSTA_PDS);
 409 }
 410
 411 /**
 412  * pciehp_card_present_or_link_active() - whether given slot is occupied
 413  * @ctrl: PCIe hotplug controller
 414  *
 415  * Unlike pciehp_card_present(), which determines presence solely from the
 416  * Presence Detect State bit, this helper also returns true if the Link Active
 417  * bit is set.  This is a concession to broken hotplug ports which hardwire
 418  * Presence Detect State to zero, such as Wilocity's [1ae9:0200].
 419  *
 420  * Returns: %1 if the slot is occupied and %0 if it is not. If the hotplug
 421  *          port is not present anymore returns %-ENODEV.
 422  */
 423 int pciehp_card_present_or_link_active(struct controller *ctrl)
 424 {
 425         int ret;
 426
 427         ret = pciehp_card_present(ctrl);
 428         if (ret)
 429                 return ret;
 430
 431         return pciehp_check_link_active(ctrl);
 432 }
 433
 434 int pciehp_query_power_fault(struct controller *ctrl)
 435 {
 436         struct pci_dev *pdev = ctrl_dev(ctrl);
 437         u16 slot_status;
 438
 439         pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status);
 440         return !!(slot_status & PCI_EXP_SLTSTA_PFD);
 441 }
 442
 443 int pciehp_set_raw_indicator_status(struct hotplug_slot *hotplug_slot,
 444                                     u8 status)
 445 {
 446         struct controller *ctrl = to_ctrl(hotplug_slot);
 447         struct pci_dev *pdev = ctrl_dev(ctrl);
 448
 449         pci_config_pm_runtime_get(pdev);
 450         pcie_write_cmd_nowait(ctrl, status << 6,
 451                               PCI_EXP_SLTCTL_AIC | PCI_EXP_SLTCTL_PIC);
 452         pci_config_pm_runtime_put(pdev);
 453         return 0;
 454 }
 455
 456 /**
 457  * pciehp_set_indicators() - set attention indicator, power indicator, or both
 458  * @ctrl: PCIe hotplug controller
 459  * @pwr: one of:
 460  *      PCI_EXP_SLTCTL_PWR_IND_ON
 461  *      PCI_EXP_SLTCTL_PWR_IND_BLINK
 462  *      PCI_EXP_SLTCTL_PWR_IND_OFF
 463  * @attn: one of:
 464  *      PCI_EXP_SLTCTL_ATTN_IND_ON
 465  *      PCI_EXP_SLTCTL_ATTN_IND_BLINK
 466  *      PCI_EXP_SLTCTL_ATTN_IND_OFF
 467  *
 468  * Either @pwr or @attn can also be INDICATOR_NOOP to leave that indicator
 469  * unchanged.
 470  */
 471 void pciehp_set_indicators(struct controller *ctrl, int pwr, int attn)
 472 {
 473         u16 cmd = 0, mask = 0;
 474
 475         if (PWR_LED(ctrl) && pwr != INDICATOR_NOOP) {
 476                 cmd |= (pwr & PCI_EXP_SLTCTL_PIC);
 477                 mask |= PCI_EXP_SLTCTL_PIC;
 478         }
 479
 480         if (ATTN_LED(ctrl) && attn != INDICATOR_NOOP) {
 481                 cmd |= (attn & PCI_EXP_SLTCTL_AIC);
 482                 mask |= PCI_EXP_SLTCTL_AIC;
 483         }
 484
 485         if (cmd) {
 486                 pcie_write_cmd_nowait(ctrl, cmd, mask);
 487                 ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
 488                          pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, cmd);
 489         }
 490 }
 491
 492 int pciehp_power_on_slot(struct controller *ctrl)
 493 {
 494         struct pci_dev *pdev = ctrl_dev(ctrl);
 495         u16 slot_status;
 496         int retval;
 497
 498         /* Clear power-fault bit from previous power failures */
 499         pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &slot_status);
 500         if (slot_status & PCI_EXP_SLTSTA_PFD)
 501                 pcie_capability_write_word(pdev, PCI_EXP_SLTSTA,
 502                                            PCI_EXP_SLTSTA_PFD);
 503         ctrl->power_fault_detected = 0;
 504
 505         pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_ON, PCI_EXP_SLTCTL_PCC);
 506         ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
 507                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,
 508                  PCI_EXP_SLTCTL_PWR_ON);
 509
 510         retval = pciehp_link_enable(ctrl);
 511         if (retval)
 512                 ctrl_err(ctrl, "%s: Can not enable the link!\n", __func__);
 513
 514         return retval;
 515 }
 516
 517 void pciehp_power_off_slot(struct controller *ctrl)
 518 {
 519         pcie_write_cmd(ctrl, PCI_EXP_SLTCTL_PWR_OFF, PCI_EXP_SLTCTL_PCC);
 520         ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
 521                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL,
 522                  PCI_EXP_SLTCTL_PWR_OFF);
 523 }
 524
 525 static irqreturn_t pciehp_isr(int irq, void *dev_id)
 526 {
 527         struct controller *ctrl = (struct controller *)dev_id;
 528         struct pci_dev *pdev = ctrl_dev(ctrl);
 529         struct device *parent = pdev->dev.parent;
 530         u16 status, events;
 531
 532         /*
 533          * Interrupts only occur in D3hot or shallower and only if enabled
 534          * in the Slot Control register (PCIe r4.0, sec 6.7.3.4).
 535          */
 536         if (pdev->current_state == PCI_D3cold ||
 537             (!(ctrl->slot_ctrl & PCI_EXP_SLTCTL_HPIE) && !pciehp_poll_mode))
 538                 return IRQ_NONE;
 539
 540         /*
 541          * Keep the port accessible by holding a runtime PM ref on its parent.
 542          * Defer resume of the parent to the IRQ thread if it's suspended.
 543          * Mask the interrupt until then.
 544          */
 545         if (parent) {
 546                 pm_runtime_get_noresume(parent);
 547                 if (!pm_runtime_active(parent)) {
 548                         pm_runtime_put(parent);
 549                         disable_irq_nosync(irq);
 550                         atomic_or(RERUN_ISR, &ctrl->pending_events);
 551                         return IRQ_WAKE_THREAD;
 552                 }
 553         }
 554
 555         pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &status);
 556         if (status == (u16) ~0) {
 557                 ctrl_info(ctrl, "%s: no response from device\n", __func__);
 558                 if (parent)
 559                         pm_runtime_put(parent);
 560                 return IRQ_NONE;
 561         }
 562
 563         /*
 564          * Slot Status contains plain status bits as well as event
 565          * notification bits; right now we only want the event bits.
 566          */
 567         events = status & (PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
 568                            PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_CC |
 569                            PCI_EXP_SLTSTA_DLLSC);
 570
 571         /*
 572          * If we've already reported a power fault, don't report it again
 573          * until we've done something to handle it.
 574          */
 575         if (ctrl->power_fault_detected)
 576                 events &= ~PCI_EXP_SLTSTA_PFD;
 577
 578         if (!events) {
 579                 if (parent)
 580                         pm_runtime_put(parent);
 581                 return IRQ_NONE;
 582         }
 583
 584         pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, events);
 585         ctrl_dbg(ctrl, "pending interrupts %#06x from Slot Status\n", events);
 586         if (parent)
 587                 pm_runtime_put(parent);
 588
 589         /*
 590          * Command Completed notifications are not deferred to the
 591          * IRQ thread because it may be waiting for their arrival.
 592          */
 593         if (events & PCI_EXP_SLTSTA_CC) {
 594                 ctrl->cmd_busy = 0;
 595                 smp_mb();
 596                 wake_up(&ctrl->queue);
 597
 598                 if (events == PCI_EXP_SLTSTA_CC)
 599                         return IRQ_HANDLED;
 600
 601                 events &= ~PCI_EXP_SLTSTA_CC;
 602         }
 603
 604         if (pdev->ignore_hotplug) {
 605                 ctrl_dbg(ctrl, "ignoring hotplug event %#06x\n", events);
 606                 return IRQ_HANDLED;
 607         }
 608
 609         /* Save pending events for consumption by IRQ thread. */
 610         atomic_or(events, &ctrl->pending_events);
 611         return IRQ_WAKE_THREAD;
 612 }
 613
 614 static irqreturn_t pciehp_ist(int irq, void *dev_id)
 615 {
 616         struct controller *ctrl = (struct controller *)dev_id;
 617         struct pci_dev *pdev = ctrl_dev(ctrl);
 618         irqreturn_t ret;
 619         u32 events;
 620
 621         ctrl->ist_running = true;
 622         pci_config_pm_runtime_get(pdev);
 623
 624         /* rerun pciehp_isr() if the port was inaccessible on interrupt */
 625         if (atomic_fetch_and(~RERUN_ISR, &ctrl->pending_events) & RERUN_ISR) {
 626                 ret = pciehp_isr(irq, dev_id);
 627                 enable_irq(irq);
 628                 if (ret != IRQ_WAKE_THREAD) {
 629                         pci_config_pm_runtime_put(pdev);
 630                         return ret;
 631                 }
 632         }
 633
 634         synchronize_hardirq(irq);
 635         events = atomic_xchg(&ctrl->pending_events, 0);
 636         if (!events) {
 637                 pci_config_pm_runtime_put(pdev);
 638                 return IRQ_NONE;
 639         }
 640
 641         /* Check Attention Button Pressed */
 642         if (events & PCI_EXP_SLTSTA_ABP) {
 643                 ctrl_info(ctrl, "Slot(%s): Attention button pressed\n",
 644                           slot_name(ctrl));
 645                 pciehp_handle_button_press(ctrl);
 646         }
 647
 648         /* Check Power Fault Detected */
 649         if ((events & PCI_EXP_SLTSTA_PFD) && !ctrl->power_fault_detected) {
 650                 ctrl->power_fault_detected = 1;
 651                 ctrl_err(ctrl, "Slot(%s): Power fault\n", slot_name(ctrl));
 652                 pciehp_set_indicators(ctrl, PCI_EXP_SLTCTL_PWR_IND_OFF,
 653                                       PCI_EXP_SLTCTL_ATTN_IND_ON);
 654         }
 655
 656         /*
 657          * Disable requests have higher priority than Presence Detect Changed
 658          * or Data Link Layer State Changed events.
 659          */
 660         down_read(&ctrl->reset_lock);
 661         if (events & DISABLE_SLOT)
 662                 pciehp_handle_disable_request(ctrl);
 663         else if (events & (PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC))
 664                 pciehp_handle_presence_or_link_change(ctrl, events);
 665         up_read(&ctrl->reset_lock);
 666
 667         pci_config_pm_runtime_put(pdev);
 668         ctrl->ist_running = false;
 669         wake_up(&ctrl->requester);
 670         return IRQ_HANDLED;
 671 }
 672
 673 static int pciehp_poll(void *data)
 674 {
 675         struct controller *ctrl = data;
 676
 677         schedule_timeout_idle(10 * HZ); /* start with 10 sec delay */
 678
 679         while (!kthread_should_stop()) {
 680                 /* poll for interrupt events or user requests */
 681                 while (pciehp_isr(IRQ_NOTCONNECTED, ctrl) == IRQ_WAKE_THREAD ||
 682                        atomic_read(&ctrl->pending_events))
 683                         pciehp_ist(IRQ_NOTCONNECTED, ctrl);
 684
 685                 if (pciehp_poll_time <= 0 || pciehp_poll_time > 60)
 686                         pciehp_poll_time = 2; /* clamp to sane value */
 687
 688                 schedule_timeout_idle(pciehp_poll_time * HZ);
 689         }
 690
 691         return 0;
 692 }
 693
 694 static void pcie_enable_notification(struct controller *ctrl)
 695 {
 696         u16 cmd, mask;
 697
 698         /*
 699          * TBD: Power fault detected software notification support.
 700          *
 701          * Power fault detected software notification is not enabled
 702          * now, because it caused power fault detected interrupt storm
 703          * on some machines. On those machines, power fault detected
 704          * bit in the slot status register was set again immediately
 705          * when it is cleared in the interrupt service routine, and
 706          * next power fault detected interrupt was notified again.
 707          */
 708
 709         /*
 710          * Always enable link events: thus link-up and link-down shall
 711          * always be treated as hotplug and unplug respectively. Enable
 712          * presence detect only if Attention Button is not present.
 713          */
 714         cmd = PCI_EXP_SLTCTL_DLLSCE;
 715         if (ATTN_BUTTN(ctrl))
 716                 cmd |= PCI_EXP_SLTCTL_ABPE;
 717         else
 718                 cmd |= PCI_EXP_SLTCTL_PDCE;
 719         if (!pciehp_poll_mode)
 720                 cmd |= PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE;
 721
 722         mask = (PCI_EXP_SLTCTL_PDCE | PCI_EXP_SLTCTL_ABPE |
 723                 PCI_EXP_SLTCTL_PFDE |
 724                 PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE |
 725                 PCI_EXP_SLTCTL_DLLSCE);
 726
 727         pcie_write_cmd_nowait(ctrl, cmd, mask);
 728         ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
 729                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, cmd);
 730 }
 731
 732 static void pcie_disable_notification(struct controller *ctrl)
 733 {
 734         u16 mask;
 735
 736         mask = (PCI_EXP_SLTCTL_PDCE | PCI_EXP_SLTCTL_ABPE |
 737                 PCI_EXP_SLTCTL_MRLSCE | PCI_EXP_SLTCTL_PFDE |
 738                 PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_CCIE |
 739                 PCI_EXP_SLTCTL_DLLSCE);
 740         pcie_write_cmd(ctrl, 0, mask);
 741         ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
 742                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, 0);
 743 }
 744
 745 void pcie_clear_hotplug_events(struct controller *ctrl)
 746 {
 747         pcie_capability_write_word(ctrl_dev(ctrl), PCI_EXP_SLTSTA,
 748                                    PCI_EXP_SLTSTA_PDC | PCI_EXP_SLTSTA_DLLSC);
 749 }
 750
 751 void pcie_enable_interrupt(struct controller *ctrl)
 752 {
 753         u16 mask;
 754
 755         mask = PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_DLLSCE;
 756         pcie_write_cmd(ctrl, mask, mask);
 757 }
 758
 759 void pcie_disable_interrupt(struct controller *ctrl)
 760 {
 761         u16 mask;
 762
 763         /*
 764          * Mask hot-plug interrupt to prevent it triggering immediately
 765          * when the link goes inactive (we still get PME when any of the
 766          * enabled events is detected). Same goes with Link Layer State
 767          * changed event which generates PME immediately when the link goes
 768          * inactive so mask it as well.
 769          */
 770         mask = PCI_EXP_SLTCTL_HPIE | PCI_EXP_SLTCTL_DLLSCE;
 771         pcie_write_cmd(ctrl, 0, mask);
 772 }
 773
 774 /*
 775  * pciehp has a 1:1 bus:slot relationship so we ultimately want a secondary
 776  * bus reset of the bridge, but at the same time we want to ensure that it is
 777  * not seen as a hot-unplug, followed by the hot-plug of the device. Thus,
 778  * disable link state notification and presence detection change notification
 779  * momentarily, if we see that they could interfere. Also, clear any spurious
 780  * events after.
 781  */
 782 int pciehp_reset_slot(struct hotplug_slot *hotplug_slot, int probe)
 783 {
 784         struct controller *ctrl = to_ctrl(hotplug_slot);
 785         struct pci_dev *pdev = ctrl_dev(ctrl);
 786         u16 stat_mask = 0, ctrl_mask = 0;
 787         int rc;
 788
 789         if (probe)
 790                 return 0;
 791
 792         down_write(&ctrl->reset_lock);
 793
 794         if (!ATTN_BUTTN(ctrl)) {
 795                 ctrl_mask |= PCI_EXP_SLTCTL_PDCE;
 796                 stat_mask |= PCI_EXP_SLTSTA_PDC;
 797         }
 798         ctrl_mask |= PCI_EXP_SLTCTL_DLLSCE;
 799         stat_mask |= PCI_EXP_SLTSTA_DLLSC;
 800
 801         pcie_write_cmd(ctrl, 0, ctrl_mask);
 802         ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
 803                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, 0);
 804
 805         rc = pci_bridge_secondary_bus_reset(ctrl->pcie->port);
 806
 807         pcie_capability_write_word(pdev, PCI_EXP_SLTSTA, stat_mask);
 808         pcie_write_cmd_nowait(ctrl, ctrl_mask, ctrl_mask);
 809         ctrl_dbg(ctrl, "%s: SLOTCTRL %x write cmd %x\n", __func__,
 810                  pci_pcie_cap(ctrl->pcie->port) + PCI_EXP_SLTCTL, ctrl_mask);
 811
 812         up_write(&ctrl->reset_lock);
 813         return rc;
 814 }
 815
 816 int pcie_init_notification(struct controller *ctrl)
 817 {
 818         if (pciehp_request_irq(ctrl))
 819                 return -1;
 820         pcie_enable_notification(ctrl);
 821         ctrl->notification_enabled = 1;
 822         return 0;
 823 }
 824
 825 void pcie_shutdown_notification(struct controller *ctrl)
 826 {
 827         if (ctrl->notification_enabled) {
 828                 pcie_disable_notification(ctrl);
 829                 pciehp_free_irq(ctrl);
 830                 ctrl->notification_enabled = 0;
 831         }
 832 }
 833
 834 static inline void dbg_ctrl(struct controller *ctrl)
 835 {
 836         struct pci_dev *pdev = ctrl->pcie->port;
 837         u16 reg16;
 838
 839         ctrl_dbg(ctrl, "Slot Capabilities      : 0x%08x\n", ctrl->slot_cap);
 840         pcie_capability_read_word(pdev, PCI_EXP_SLTSTA, &reg16);
 841         ctrl_dbg(ctrl, "Slot Status            : 0x%04x\n", reg16);
 842         pcie_capability_read_word(pdev, PCI_EXP_SLTCTL, &reg16);
 843         ctrl_dbg(ctrl, "Slot Control           : 0x%04x\n", reg16);
 844 }
 845
 846 #define FLAG(x, y)      (((x) & (y)) ? '+' : '-')
 847
 848 struct controller *pcie_init(struct pcie_device *dev)
 849 {
 850         struct controller *ctrl;
 851         u32 slot_cap, link_cap;
 852         u8 poweron;
 853         struct pci_dev *pdev = dev->port;
 854         struct pci_bus *subordinate = pdev->subordinate;
 855
 856         ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
 857         if (!ctrl)
 858                 return NULL;
 859
 860         ctrl->pcie = dev;
 861         pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &slot_cap);
 862
 863         if (pdev->hotplug_user_indicators)
 864                 slot_cap &= ~(PCI_EXP_SLTCAP_AIP | PCI_EXP_SLTCAP_PIP);
 865
 866         /*
 867          * We assume no Thunderbolt controllers support Command Complete events,
 868          * but some controllers falsely claim they do.
 869          */
 870         if (pdev->is_thunderbolt)
 871                 slot_cap |= PCI_EXP_SLTCAP_NCCS;
 872
 873         ctrl->slot_cap = slot_cap;
 874         mutex_init(&ctrl->ctrl_lock);
 875         mutex_init(&ctrl->state_lock);
 876         init_rwsem(&ctrl->reset_lock);
 877         init_waitqueue_head(&ctrl->requester);
 878         init_waitqueue_head(&ctrl->queue);
 879         INIT_DELAYED_WORK(&ctrl->button_work, pciehp_queue_pushbutton_work);
 880         dbg_ctrl(ctrl);
 881
 882         down_read(&pci_bus_sem);
 883         ctrl->state = list_empty(&subordinate->devices) ? OFF_STATE : ON_STATE;
 884         up_read(&pci_bus_sem);
 885
 886         /* Check if Data Link Layer Link Active Reporting is implemented */
 887         pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &link_cap);
 888
 889         /* Clear all remaining event bits in Slot Status register. */
 890         pcie_capability_write_word(pdev, PCI_EXP_SLTSTA,
 891                 PCI_EXP_SLTSTA_ABP | PCI_EXP_SLTSTA_PFD |
 892                 PCI_EXP_SLTSTA_MRLSC | PCI_EXP_SLTSTA_CC |
 893                 PCI_EXP_SLTSTA_DLLSC | PCI_EXP_SLTSTA_PDC);
 894
 895         ctrl_info(ctrl, "Slot #%d AttnBtn%c PwrCtrl%c MRL%c AttnInd%c PwrInd%c HotPlug%c Surprise%c Interlock%c NoCompl%c LLActRep%c%s\n",
 896                 (slot_cap & PCI_EXP_SLTCAP_PSN) >> 19,
 897                 FLAG(slot_cap, PCI_EXP_SLTCAP_ABP),
 898                 FLAG(slot_cap, PCI_EXP_SLTCAP_PCP),
 899                 FLAG(slot_cap, PCI_EXP_SLTCAP_MRLSP),
 900                 FLAG(slot_cap, PCI_EXP_SLTCAP_AIP),
 901                 FLAG(slot_cap, PCI_EXP_SLTCAP_PIP),
 902                 FLAG(slot_cap, PCI_EXP_SLTCAP_HPC),
 903                 FLAG(slot_cap, PCI_EXP_SLTCAP_HPS),
 904                 FLAG(slot_cap, PCI_EXP_SLTCAP_EIP),
 905                 FLAG(slot_cap, PCI_EXP_SLTCAP_NCCS),
 906                 FLAG(link_cap, PCI_EXP_LNKCAP_DLLLARC),
 907                 pdev->broken_cmd_compl ? " (with Cmd Compl erratum)" : "");
 908
 909         /*
 910          * If empty slot's power status is on, turn power off.  The IRQ isn't
 911          * requested yet, so avoid triggering a notification with this command.
 912          */
 913         if (POWER_CTRL(ctrl)) {
 914                 pciehp_get_power_status(ctrl, &poweron);
 915                 if (!pciehp_card_present_or_link_active(ctrl) && poweron) {
 916                         pcie_disable_notification(ctrl);
 917                         pciehp_power_off_slot(ctrl);
 918                 }
 919         }
 920
 921         return ctrl;
 922 }
 923
 924 void pciehp_release_ctrl(struct controller *ctrl)
 925 {
 926         cancel_delayed_work_sync(&ctrl->button_work);
 927         kfree(ctrl);
 928 }
 929
 930 static void quirk_cmd_compl(struct pci_dev *pdev)
 931 {
 932         u32 slot_cap;
 933
 934         if (pci_is_pcie(pdev)) {
 935                 pcie_capability_read_dword(pdev, PCI_EXP_SLTCAP, &slot_cap);
 936                 if (slot_cap & PCI_EXP_SLTCAP_HPC &&
 937                     !(slot_cap & PCI_EXP_SLTCAP_NCCS))
 938                         pdev->broken_cmd_compl = 1;
 939         }
 940 }
 941 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_INTEL, PCI_ANY_ID,
 942                               PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
 943 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0400,
 944                               PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
 945 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_QCOM, 0x0401,
 946                               PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);
 947 DECLARE_PCI_FIXUP_CLASS_EARLY(PCI_VENDOR_ID_HXT, 0x0401,
 948                               PCI_CLASS_BRIDGE_PCI, 8, quirk_cmd_compl);