drivers/infiniband/hw/hfi1/aspm.c

   1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
   2 /*
   3  * Copyright(c) 2019 Intel Corporation.
   4  *
   5  */
   6
   7 #include "aspm.h"
   8
   9 /* Time after which the timer interrupt will re-enable ASPM */
  10 #define ASPM_TIMER_MS 1000
  11 /* Time for which interrupts are ignored after a timer has been scheduled */
  12 #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2)
  13 /* Two interrupts within this time trigger ASPM disable */
  14 #define ASPM_TRIGGER_MS 1
  15 #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull)
  16 #define ASPM_L1_SUPPORTED(reg) \
  17         ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2)
  18
  19 uint aspm_mode = ASPM_MODE_DISABLED;
  20 module_param_named(aspm, aspm_mode, uint, 0444);
  21 MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic");
  22
  23 static bool aspm_hw_l1_supported(struct hfi1_devdata *dd)
  24 {
  25         struct pci_dev *parent = dd->pcidev->bus->self;
  26         u32 up, dn;
  27
  28         /*
  29          * If the driver does not have access to the upstream component,
  30          * it cannot support ASPM L1 at all.
  31          */
  32         if (!parent)
  33                 return false;
  34
  35         pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn);
  36         dn = ASPM_L1_SUPPORTED(dn);
  37
  38         pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up);
  39         up = ASPM_L1_SUPPORTED(up);
  40
  41         /* ASPM works on A-step but is reported as not supported */
  42         return (!!dn || is_ax(dd)) && !!up;
  43 }
  44
  45 /* Set L1 entrance latency for slower entry to L1 */
  46 static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd)
  47 {
  48         u32 l1_ent_lat = 0x4u;
  49         u32 reg32;
  50
  51         pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, &reg32);
  52         reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK;
  53         reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT;
  54         pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32);
  55 }
  56
  57 static void aspm_hw_enable_l1(struct hfi1_devdata *dd)
  58 {
  59         struct pci_dev *parent = dd->pcidev->bus->self;
  60
  61         /*
  62          * If the driver does not have access to the upstream component,
  63          * it cannot support ASPM L1 at all.
  64          */
  65         if (!parent)
  66                 return;
  67
  68         /* Enable ASPM L1 first in upstream component and then downstream */
  69         pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
  70                                            PCI_EXP_LNKCTL_ASPMC,
  71                                            PCI_EXP_LNKCTL_ASPM_L1);
  72         pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
  73                                            PCI_EXP_LNKCTL_ASPMC,
  74                                            PCI_EXP_LNKCTL_ASPM_L1);
  75 }
  76
  77 void aspm_hw_disable_l1(struct hfi1_devdata *dd)
  78 {
  79         struct pci_dev *parent = dd->pcidev->bus->self;
  80
  81         /* Disable ASPM L1 first in downstream component and then upstream */
  82         pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL,
  83                                            PCI_EXP_LNKCTL_ASPMC, 0x0);
  84         if (parent)
  85                 pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
  86                                                    PCI_EXP_LNKCTL_ASPMC, 0x0);
  87 }
  88
  89 static  void aspm_enable(struct hfi1_devdata *dd)
  90 {
  91         if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED ||
  92             !dd->aspm_supported)
  93                 return;
  94
  95         aspm_hw_enable_l1(dd);
  96         dd->aspm_enabled = true;
  97 }
  98
  99 static  void aspm_disable(struct hfi1_devdata *dd)
 100 {
 101         if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED)
 102                 return;
 103
 104         aspm_hw_disable_l1(dd);
 105         dd->aspm_enabled = false;
 106 }
 107
 108 static  void aspm_disable_inc(struct hfi1_devdata *dd)
 109 {
 110         unsigned long flags;
 111
 112         spin_lock_irqsave(&dd->aspm_lock, flags);
 113         aspm_disable(dd);
 114         atomic_inc(&dd->aspm_disabled_cnt);
 115         spin_unlock_irqrestore(&dd->aspm_lock, flags);
 116 }
 117
 118 static  void aspm_enable_dec(struct hfi1_devdata *dd)
 119 {
 120         unsigned long flags;
 121
 122         spin_lock_irqsave(&dd->aspm_lock, flags);
 123         if (atomic_dec_and_test(&dd->aspm_disabled_cnt))
 124                 aspm_enable(dd);
 125         spin_unlock_irqrestore(&dd->aspm_lock, flags);
 126 }
 127
 128 /* ASPM processing for each receive context interrupt */
 129 void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd)
 130 {
 131         bool restart_timer;
 132         bool close_interrupts;
 133         unsigned long flags;
 134         ktime_t now, prev;
 135
 136         spin_lock_irqsave(&rcd->aspm_lock, flags);
 137         /* PSM contexts are open */
 138         if (!rcd->aspm_intr_enable)
 139                 goto unlock;
 140
 141         prev = rcd->aspm_ts_last_intr;
 142         now = ktime_get();
 143         rcd->aspm_ts_last_intr = now;
 144
 145         /* An interrupt pair close together in time */
 146         close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS;
 147
 148         /* Don't push out our timer till this much time has elapsed */
 149         restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) >
 150                                     ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC;
 151         restart_timer = restart_timer && close_interrupts;
 152
 153         /* Disable ASPM and schedule timer */
 154         if (rcd->aspm_enabled && close_interrupts) {
 155                 aspm_disable_inc(rcd->dd);
 156                 rcd->aspm_enabled = false;
 157                 restart_timer = true;
 158         }
 159
 160         if (restart_timer) {
 161                 mod_timer(&rcd->aspm_timer,
 162                           jiffies + msecs_to_jiffies(ASPM_TIMER_MS));
 163                 rcd->aspm_ts_timer_sched = now;
 164         }
 165 unlock:
 166         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 167 }
 168
 169 /* Timer function for re-enabling ASPM in the absence of interrupt activity */
 170 static  void aspm_ctx_timer_function(struct timer_list *t)
 171 {
 172         struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer);
 173         unsigned long flags;
 174
 175         spin_lock_irqsave(&rcd->aspm_lock, flags);
 176         aspm_enable_dec(rcd->dd);
 177         rcd->aspm_enabled = true;
 178         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 179 }
 180
 181 /*
 182  * Disable interrupt processing for verbs contexts when PSM or VNIC contexts
 183  * are open.
 184  */
 185 void aspm_disable_all(struct hfi1_devdata *dd)
 186 {
 187         struct hfi1_ctxtdata *rcd;
 188         unsigned long flags;
 189         u16 i;
 190
 191         for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
 192                 rcd = hfi1_rcd_get_by_index(dd, i);
 193                 if (rcd) {
 194                         del_timer_sync(&rcd->aspm_timer);
 195                         spin_lock_irqsave(&rcd->aspm_lock, flags);
 196                         rcd->aspm_intr_enable = false;
 197                         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 198                         hfi1_rcd_put(rcd);
 199                 }
 200         }
 201
 202         aspm_disable(dd);
 203         atomic_set(&dd->aspm_disabled_cnt, 0);
 204 }
 205
 206 /* Re-enable interrupt processing for verbs contexts */
 207 void aspm_enable_all(struct hfi1_devdata *dd)
 208 {
 209         struct hfi1_ctxtdata *rcd;
 210         unsigned long flags;
 211         u16 i;
 212
 213         aspm_enable(dd);
 214
 215         if (aspm_mode != ASPM_MODE_DYNAMIC)
 216                 return;
 217
 218         for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
 219                 rcd = hfi1_rcd_get_by_index(dd, i);
 220                 if (rcd) {
 221                         spin_lock_irqsave(&rcd->aspm_lock, flags);
 222                         rcd->aspm_intr_enable = true;
 223                         rcd->aspm_enabled = true;
 224                         spin_unlock_irqrestore(&rcd->aspm_lock, flags);
 225                         hfi1_rcd_put(rcd);
 226                 }
 227         }
 228 }
 229
 230 static  void aspm_ctx_init(struct hfi1_ctxtdata *rcd)
 231 {
 232         spin_lock_init(&rcd->aspm_lock);
 233         timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0);
 234         rcd->aspm_intr_supported = rcd->dd->aspm_supported &&
 235                 aspm_mode == ASPM_MODE_DYNAMIC &&
 236                 rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt;
 237 }
 238
 239 void aspm_init(struct hfi1_devdata *dd)
 240 {
 241         struct hfi1_ctxtdata *rcd;
 242         u16 i;
 243
 244         spin_lock_init(&dd->aspm_lock);
 245         dd->aspm_supported = aspm_hw_l1_supported(dd);
 246
 247         for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) {
 248                 rcd = hfi1_rcd_get_by_index(dd, i);
 249                 if (rcd)
 250                         aspm_ctx_init(rcd);
 251                 hfi1_rcd_put(rcd);
 252         }
 253
 254         /* Start with ASPM disabled */
 255         aspm_hw_set_l1_ent_latency(dd);
 256         dd->aspm_enabled = false;
 257         aspm_hw_disable_l1(dd);
 258
 259         /* Now turn on ASPM if configured */
 260         aspm_enable_all(dd);
 261 }
 262
 263 void aspm_exit(struct hfi1_devdata *dd)
 264 {
 265         aspm_disable_all(dd);
 266
 267         /* Turn on ASPM on exit to conserve power */
 268         aspm_enable(dd);
 269 }
 270