drivers/pci/ats.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * drivers/pci/ats.c
   4  *
   5  * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
   6  * Copyright (C) 2011 Advanced Micro Devices,
   7  *
   8  * PCI Express I/O Virtualization (IOV) support.
   9  *   Address Translation Service 1.0
  10  *   Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
  11  *   PASID support added by Joerg Roedel <joerg.roedel@amd.com>
  12  */
  13
  14 #include <linux/export.h>
  15 #include <linux/pci-ats.h>
  16 #include <linux/pci.h>
  17 #include <linux/slab.h>
  18
  19 #include "pci.h"
  20
  21 void pci_ats_init(struct pci_dev *dev)
  22 {
  23         int pos;
  24
  25         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
  26         if (!pos)
  27                 return;
  28
  29         dev->ats_cap = pos;
  30 }
  31
  32 /**
  33  * pci_enable_ats - enable the ATS capability
  34  * @dev: the PCI device
  35  * @ps: the IOMMU page shift
  36  *
  37  * Returns 0 on success, or negative on failure.
  38  */
  39 int pci_enable_ats(struct pci_dev *dev, int ps)
  40 {
  41         u16 ctrl;
  42         struct pci_dev *pdev;
  43
  44         if (!dev->ats_cap)
  45                 return -EINVAL;
  46
  47         if (WARN_ON(dev->ats_enabled))
  48                 return -EBUSY;
  49
  50         if (ps < PCI_ATS_MIN_STU)
  51                 return -EINVAL;
  52
  53         /*
  54          * Note that enabling ATS on a VF fails unless it's already enabled
  55          * with the same STU on the PF.
  56          */
  57         ctrl = PCI_ATS_CTRL_ENABLE;
  58         if (dev->is_virtfn) {
  59                 pdev = pci_physfn(dev);
  60                 if (pdev->ats_stu != ps)
  61                         return -EINVAL;
  62
  63                 atomic_inc(&pdev->ats_ref_cnt);  /* count enabled VFs */
  64         } else {
  65                 dev->ats_stu = ps;
  66                 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
  67         }
  68         pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
  69
  70         dev->ats_enabled = 1;
  71         return 0;
  72 }
  73 EXPORT_SYMBOL_GPL(pci_enable_ats);
  74
  75 /**
  76  * pci_disable_ats - disable the ATS capability
  77  * @dev: the PCI device
  78  */
  79 void pci_disable_ats(struct pci_dev *dev)
  80 {
  81         struct pci_dev *pdev;
  82         u16 ctrl;
  83
  84         if (WARN_ON(!dev->ats_enabled))
  85                 return;
  86
  87         if (atomic_read(&dev->ats_ref_cnt))
  88                 return;         /* VFs still enabled */
  89
  90         if (dev->is_virtfn) {
  91                 pdev = pci_physfn(dev);
  92                 atomic_dec(&pdev->ats_ref_cnt);
  93         }
  94
  95         pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
  96         ctrl &= ~PCI_ATS_CTRL_ENABLE;
  97         pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
  98
  99         dev->ats_enabled = 0;
 100 }
 101 EXPORT_SYMBOL_GPL(pci_disable_ats);
 102
 103 void pci_restore_ats_state(struct pci_dev *dev)
 104 {
 105         u16 ctrl;
 106
 107         if (!dev->ats_enabled)
 108                 return;
 109
 110         ctrl = PCI_ATS_CTRL_ENABLE;
 111         if (!dev->is_virtfn)
 112                 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
 113         pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
 114 }
 115 EXPORT_SYMBOL_GPL(pci_restore_ats_state);
 116
 117 /**
 118  * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
 119  * @dev: the PCI device
 120  *
 121  * Returns the queue depth on success, or negative on failure.
 122  *
 123  * The ATS spec uses 0 in the Invalidate Queue Depth field to
 124  * indicate that the function can accept 32 Invalidate Request.
 125  * But here we use the `real' values (i.e. 1~32) for the Queue
 126  * Depth; and 0 indicates the function shares the Queue with
 127  * other functions (doesn't exclusively own a Queue).
 128  */
 129 int pci_ats_queue_depth(struct pci_dev *dev)
 130 {
 131         u16 cap;
 132
 133         if (!dev->ats_cap)
 134                 return -EINVAL;
 135
 136         if (dev->is_virtfn)
 137                 return 0;
 138
 139         pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
 140         return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
 141 }
 142 EXPORT_SYMBOL_GPL(pci_ats_queue_depth);
 143
 144 #ifdef CONFIG_PCI_PRI
 145 /**
 146  * pci_enable_pri - Enable PRI capability
 147  * @ pdev: PCI device structure
 148  *
 149  * Returns 0 on success, negative value on error
 150  */
 151 int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
 152 {
 153         u16 control, status;
 154         u32 max_requests;
 155         int pos;
 156
 157         if (WARN_ON(pdev->pri_enabled))
 158                 return -EBUSY;
 159
 160         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
 161         if (!pos)
 162                 return -EINVAL;
 163
 164         pci_read_config_word(pdev, pos + PCI_PRI_STATUS, &status);
 165         if (!(status & PCI_PRI_STATUS_STOPPED))
 166                 return -EBUSY;
 167
 168         pci_read_config_dword(pdev, pos + PCI_PRI_MAX_REQ, &max_requests);
 169         reqs = min(max_requests, reqs);
 170         pdev->pri_reqs_alloc = reqs;
 171         pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs);
 172
 173         control = PCI_PRI_CTRL_ENABLE;
 174         pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
 175
 176         pdev->pri_enabled = 1;
 177
 178         return 0;
 179 }
 180 EXPORT_SYMBOL_GPL(pci_enable_pri);
 181
 182 /**
 183  * pci_disable_pri - Disable PRI capability
 184  * @pdev: PCI device structure
 185  *
 186  * Only clears the enabled-bit, regardless of its former value
 187  */
 188 void pci_disable_pri(struct pci_dev *pdev)
 189 {
 190         u16 control;
 191         int pos;
 192
 193         if (WARN_ON(!pdev->pri_enabled))
 194                 return;
 195
 196         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
 197         if (!pos)
 198                 return;
 199
 200         pci_read_config_word(pdev, pos + PCI_PRI_CTRL, &control);
 201         control &= ~PCI_PRI_CTRL_ENABLE;
 202         pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
 203
 204         pdev->pri_enabled = 0;
 205 }
 206 EXPORT_SYMBOL_GPL(pci_disable_pri);
 207
 208 /**
 209  * pci_restore_pri_state - Restore PRI
 210  * @pdev: PCI device structure
 211  */
 212 void pci_restore_pri_state(struct pci_dev *pdev)
 213 {
 214         u16 control = PCI_PRI_CTRL_ENABLE;
 215         u32 reqs = pdev->pri_reqs_alloc;
 216         int pos;
 217
 218         if (!pdev->pri_enabled)
 219                 return;
 220
 221         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
 222         if (!pos)
 223                 return;
 224
 225         pci_write_config_dword(pdev, pos + PCI_PRI_ALLOC_REQ, reqs);
 226         pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
 227 }
 228 EXPORT_SYMBOL_GPL(pci_restore_pri_state);
 229
 230 /**
 231  * pci_reset_pri - Resets device's PRI state
 232  * @pdev: PCI device structure
 233  *
 234  * The PRI capability must be disabled before this function is called.
 235  * Returns 0 on success, negative value on error.
 236  */
 237 int pci_reset_pri(struct pci_dev *pdev)
 238 {
 239         u16 control;
 240         int pos;
 241
 242         if (WARN_ON(pdev->pri_enabled))
 243                 return -EBUSY;
 244
 245         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
 246         if (!pos)
 247                 return -EINVAL;
 248
 249         control = PCI_PRI_CTRL_RESET;
 250         pci_write_config_word(pdev, pos + PCI_PRI_CTRL, control);
 251
 252         return 0;
 253 }
 254 EXPORT_SYMBOL_GPL(pci_reset_pri);
 255 #endif /* CONFIG_PCI_PRI */
 256
 257 #ifdef CONFIG_PCI_PASID
 258 /**
 259  * pci_enable_pasid - Enable the PASID capability
 260  * @pdev: PCI device structure
 261  * @features: Features to enable
 262  *
 263  * Returns 0 on success, negative value on error. This function checks
 264  * whether the features are actually supported by the device and returns
 265  * an error if not.
 266  */
 267 int pci_enable_pasid(struct pci_dev *pdev, int features)
 268 {
 269         u16 control, supported;
 270         int pos;
 271
 272         if (WARN_ON(pdev->pasid_enabled))
 273                 return -EBUSY;
 274
 275         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
 276         if (!pos)
 277                 return -EINVAL;
 278
 279         pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
 280         supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
 281
 282         /* User wants to enable anything unsupported? */
 283         if ((supported & features) != features)
 284                 return -EINVAL;
 285
 286         control = PCI_PASID_CTRL_ENABLE | features;
 287         pdev->pasid_features = features;
 288
 289         pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
 290
 291         pdev->pasid_enabled = 1;
 292
 293         return 0;
 294 }
 295 EXPORT_SYMBOL_GPL(pci_enable_pasid);
 296
 297 /**
 298  * pci_disable_pasid - Disable the PASID capability
 299  * @pdev: PCI device structure
 300  */
 301 void pci_disable_pasid(struct pci_dev *pdev)
 302 {
 303         u16 control = 0;
 304         int pos;
 305
 306         if (WARN_ON(!pdev->pasid_enabled))
 307                 return;
 308
 309         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
 310         if (!pos)
 311                 return;
 312
 313         pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
 314
 315         pdev->pasid_enabled = 0;
 316 }
 317 EXPORT_SYMBOL_GPL(pci_disable_pasid);
 318
 319 /**
 320  * pci_restore_pasid_state - Restore PASID capabilities
 321  * @pdev: PCI device structure
 322  */
 323 void pci_restore_pasid_state(struct pci_dev *pdev)
 324 {
 325         u16 control;
 326         int pos;
 327
 328         if (!pdev->pasid_enabled)
 329                 return;
 330
 331         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
 332         if (!pos)
 333                 return;
 334
 335         control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
 336         pci_write_config_word(pdev, pos + PCI_PASID_CTRL, control);
 337 }
 338 EXPORT_SYMBOL_GPL(pci_restore_pasid_state);
 339
 340 /**
 341  * pci_pasid_features - Check which PASID features are supported
 342  * @pdev: PCI device structure
 343  *
 344  * Returns a negative value when no PASI capability is present.
 345  * Otherwise is returns a bitmask with supported features. Current
 346  * features reported are:
 347  * PCI_PASID_CAP_EXEC - Execute permission supported
 348  * PCI_PASID_CAP_PRIV - Privileged mode supported
 349  */
 350 int pci_pasid_features(struct pci_dev *pdev)
 351 {
 352         u16 supported;
 353         int pos;
 354
 355         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
 356         if (!pos)
 357                 return -EINVAL;
 358
 359         pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
 360
 361         supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
 362
 363         return supported;
 364 }
 365 EXPORT_SYMBOL_GPL(pci_pasid_features);
 366
 367 #define PASID_NUMBER_SHIFT      8
 368 #define PASID_NUMBER_MASK       (0x1f << PASID_NUMBER_SHIFT)
 369 /**
 370  * pci_max_pasid - Get maximum number of PASIDs supported by device
 371  * @pdev: PCI device structure
 372  *
 373  * Returns negative value when PASID capability is not present.
 374  * Otherwise it returns the numer of supported PASIDs.
 375  */
 376 int pci_max_pasids(struct pci_dev *pdev)
 377 {
 378         u16 supported;
 379         int pos;
 380
 381         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
 382         if (!pos)
 383                 return -EINVAL;
 384
 385         pci_read_config_word(pdev, pos + PCI_PASID_CAP, &supported);
 386
 387         supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT;
 388
 389         return (1 << supported);
 390 }
 391 EXPORT_SYMBOL_GPL(pci_max_pasids);
 392 #endif /* CONFIG_PCI_PASID */