drivers/pci/ats.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * PCI Express I/O Virtualization (IOV) support
   4  *   Address Translation Service 1.0
   5  *   Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>
   6  *   PASID support added by Joerg Roedel <joerg.roedel@amd.com>
   7  *
   8  * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>
   9  * Copyright (C) 2011 Advanced Micro Devices,
  10  */
  11
  12 #include <linux/export.h>
  13 #include <linux/pci-ats.h>
  14 #include <linux/pci.h>
  15 #include <linux/slab.h>
  16
  17 #include "pci.h"
  18
  19 void pci_ats_init(struct pci_dev *dev)
  20 {
  21         int pos;
  22
  23         if (pci_ats_disabled())
  24                 return;
  25
  26         pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);
  27         if (!pos)
  28                 return;
  29
  30         dev->ats_cap = pos;
  31 }
  32
  33 /**
  34  * pci_ats_supported - check if the device can use ATS
  35  * @dev: the PCI device
  36  *
  37  * Returns true if the device supports ATS and is allowed to use it, false
  38  * otherwise.
  39  */
  40 bool pci_ats_supported(struct pci_dev *dev)
  41 {
  42         if (!dev->ats_cap)
  43                 return false;
  44
  45         return (dev->untrusted == 0);
  46 }
  47 EXPORT_SYMBOL_GPL(pci_ats_supported);
  48
  49 /**
  50  * pci_enable_ats - enable the ATS capability
  51  * @dev: the PCI device
  52  * @ps: the IOMMU page shift
  53  *
  54  * Returns 0 on success, or negative on failure.
  55  */
  56 int pci_enable_ats(struct pci_dev *dev, int ps)
  57 {
  58         u16 ctrl;
  59         struct pci_dev *pdev;
  60
  61         if (!pci_ats_supported(dev))
  62                 return -EINVAL;
  63
  64         if (WARN_ON(dev->ats_enabled))
  65                 return -EBUSY;
  66
  67         if (ps < PCI_ATS_MIN_STU)
  68                 return -EINVAL;
  69
  70         /*
  71          * Note that enabling ATS on a VF fails unless it's already enabled
  72          * with the same STU on the PF.
  73          */
  74         ctrl = PCI_ATS_CTRL_ENABLE;
  75         if (dev->is_virtfn) {
  76                 pdev = pci_physfn(dev);
  77                 if (pdev->ats_stu != ps)
  78                         return -EINVAL;
  79         } else {
  80                 dev->ats_stu = ps;
  81                 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
  82         }
  83         pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
  84
  85         dev->ats_enabled = 1;
  86         return 0;
  87 }
  88 EXPORT_SYMBOL_GPL(pci_enable_ats);
  89
  90 /**
  91  * pci_disable_ats - disable the ATS capability
  92  * @dev: the PCI device
  93  */
  94 void pci_disable_ats(struct pci_dev *dev)
  95 {
  96         u16 ctrl;
  97
  98         if (WARN_ON(!dev->ats_enabled))
  99                 return;
 100
 101         pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);
 102         ctrl &= ~PCI_ATS_CTRL_ENABLE;
 103         pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
 104
 105         dev->ats_enabled = 0;
 106 }
 107 EXPORT_SYMBOL_GPL(pci_disable_ats);
 108
 109 void pci_restore_ats_state(struct pci_dev *dev)
 110 {
 111         u16 ctrl;
 112
 113         if (!dev->ats_enabled)
 114                 return;
 115
 116         ctrl = PCI_ATS_CTRL_ENABLE;
 117         if (!dev->is_virtfn)
 118                 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);
 119         pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);
 120 }
 121
 122 /**
 123  * pci_ats_queue_depth - query the ATS Invalidate Queue Depth
 124  * @dev: the PCI device
 125  *
 126  * Returns the queue depth on success, or negative on failure.
 127  *
 128  * The ATS spec uses 0 in the Invalidate Queue Depth field to
 129  * indicate that the function can accept 32 Invalidate Request.
 130  * But here we use the `real' values (i.e. 1~32) for the Queue
 131  * Depth; and 0 indicates the function shares the Queue with
 132  * other functions (doesn't exclusively own a Queue).
 133  */
 134 int pci_ats_queue_depth(struct pci_dev *dev)
 135 {
 136         u16 cap;
 137
 138         if (!dev->ats_cap)
 139                 return -EINVAL;
 140
 141         if (dev->is_virtfn)
 142                 return 0;
 143
 144         pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);
 145         return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;
 146 }
 147
 148 /**
 149  * pci_ats_page_aligned - Return Page Aligned Request bit status.
 150  * @pdev: the PCI device
 151  *
 152  * Returns 1, if the Untranslated Addresses generated by the device
 153  * are always aligned or 0 otherwise.
 154  *
 155  * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit
 156  * is set, it indicates the Untranslated Addresses generated by the
 157  * device are always aligned to a 4096 byte boundary.
 158  */
 159 int pci_ats_page_aligned(struct pci_dev *pdev)
 160 {
 161         u16 cap;
 162
 163         if (!pdev->ats_cap)
 164                 return 0;
 165
 166         pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);
 167
 168         if (cap & PCI_ATS_CAP_PAGE_ALIGNED)
 169                 return 1;
 170
 171         return 0;
 172 }
 173
 174 #ifdef CONFIG_PCI_PRI
 175 void pci_pri_init(struct pci_dev *pdev)
 176 {
 177         u16 status;
 178
 179         pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);
 180
 181         if (!pdev->pri_cap)
 182                 return;
 183
 184         pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);
 185         if (status & PCI_PRI_STATUS_PASID)
 186                 pdev->pasid_required = 1;
 187 }
 188
 189 /**
 190  * pci_enable_pri - Enable PRI capability
 191  * @ pdev: PCI device structure
 192  *
 193  * Returns 0 on success, negative value on error
 194  */
 195 int pci_enable_pri(struct pci_dev *pdev, u32 reqs)
 196 {
 197         u16 control, status;
 198         u32 max_requests;
 199         int pri = pdev->pri_cap;
 200
 201         /*
 202          * VFs must not implement the PRI Capability.  If their PF
 203          * implements PRI, it is shared by the VFs, so if the PF PRI is
 204          * enabled, it is also enabled for the VF.
 205          */
 206         if (pdev->is_virtfn) {
 207                 if (pci_physfn(pdev)->pri_enabled)
 208                         return 0;
 209                 return -EINVAL;
 210         }
 211
 212         if (WARN_ON(pdev->pri_enabled))
 213                 return -EBUSY;
 214
 215         if (!pri)
 216                 return -EINVAL;
 217
 218         pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);
 219         if (!(status & PCI_PRI_STATUS_STOPPED))
 220                 return -EBUSY;
 221
 222         pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);
 223         reqs = min(max_requests, reqs);
 224         pdev->pri_reqs_alloc = reqs;
 225         pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
 226
 227         control = PCI_PRI_CTRL_ENABLE;
 228         pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
 229
 230         pdev->pri_enabled = 1;
 231
 232         return 0;
 233 }
 234
 235 /**
 236  * pci_disable_pri - Disable PRI capability
 237  * @pdev: PCI device structure
 238  *
 239  * Only clears the enabled-bit, regardless of its former value
 240  */
 241 void pci_disable_pri(struct pci_dev *pdev)
 242 {
 243         u16 control;
 244         int pri = pdev->pri_cap;
 245
 246         /* VFs share the PF PRI */
 247         if (pdev->is_virtfn)
 248                 return;
 249
 250         if (WARN_ON(!pdev->pri_enabled))
 251                 return;
 252
 253         if (!pri)
 254                 return;
 255
 256         pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);
 257         control &= ~PCI_PRI_CTRL_ENABLE;
 258         pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
 259
 260         pdev->pri_enabled = 0;
 261 }
 262 EXPORT_SYMBOL_GPL(pci_disable_pri);
 263
 264 /**
 265  * pci_restore_pri_state - Restore PRI
 266  * @pdev: PCI device structure
 267  */
 268 void pci_restore_pri_state(struct pci_dev *pdev)
 269 {
 270         u16 control = PCI_PRI_CTRL_ENABLE;
 271         u32 reqs = pdev->pri_reqs_alloc;
 272         int pri = pdev->pri_cap;
 273
 274         if (pdev->is_virtfn)
 275                 return;
 276
 277         if (!pdev->pri_enabled)
 278                 return;
 279
 280         if (!pri)
 281                 return;
 282
 283         pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);
 284         pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
 285 }
 286
 287 /**
 288  * pci_reset_pri - Resets device's PRI state
 289  * @pdev: PCI device structure
 290  *
 291  * The PRI capability must be disabled before this function is called.
 292  * Returns 0 on success, negative value on error.
 293  */
 294 int pci_reset_pri(struct pci_dev *pdev)
 295 {
 296         u16 control;
 297         int pri = pdev->pri_cap;
 298
 299         if (pdev->is_virtfn)
 300                 return 0;
 301
 302         if (WARN_ON(pdev->pri_enabled))
 303                 return -EBUSY;
 304
 305         if (!pri)
 306                 return -EINVAL;
 307
 308         control = PCI_PRI_CTRL_RESET;
 309         pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);
 310
 311         return 0;
 312 }
 313
 314 /**
 315  * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit
 316  *                               status.
 317  * @pdev: PCI device structure
 318  *
 319  * Returns 1 if PASID is required in PRG Response Message, 0 otherwise.
 320  */
 321 int pci_prg_resp_pasid_required(struct pci_dev *pdev)
 322 {
 323         if (pdev->is_virtfn)
 324                 pdev = pci_physfn(pdev);
 325
 326         return pdev->pasid_required;
 327 }
 328 #endif /* CONFIG_PCI_PRI */
 329
 330 #ifdef CONFIG_PCI_PASID
 331 void pci_pasid_init(struct pci_dev *pdev)
 332 {
 333         pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);
 334 }
 335
 336 /**
 337  * pci_enable_pasid - Enable the PASID capability
 338  * @pdev: PCI device structure
 339  * @features: Features to enable
 340  *
 341  * Returns 0 on success, negative value on error. This function checks
 342  * whether the features are actually supported by the device and returns
 343  * an error if not.
 344  */
 345 int pci_enable_pasid(struct pci_dev *pdev, int features)
 346 {
 347         u16 control, supported;
 348         int pasid = pdev->pasid_cap;
 349
 350         /*
 351          * VFs must not implement the PASID Capability, but if a PF
 352          * supports PASID, its VFs share the PF PASID configuration.
 353          */
 354         if (pdev->is_virtfn) {
 355                 if (pci_physfn(pdev)->pasid_enabled)
 356                         return 0;
 357                 return -EINVAL;
 358         }
 359
 360         if (WARN_ON(pdev->pasid_enabled))
 361                 return -EBUSY;
 362
 363         if (!pdev->eetlp_prefix_path)
 364                 return -EINVAL;
 365
 366         if (!pasid)
 367                 return -EINVAL;
 368
 369         pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
 370         supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
 371
 372         /* User wants to enable anything unsupported? */
 373         if ((supported & features) != features)
 374                 return -EINVAL;
 375
 376         control = PCI_PASID_CTRL_ENABLE | features;
 377         pdev->pasid_features = features;
 378
 379         pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
 380
 381         pdev->pasid_enabled = 1;
 382
 383         return 0;
 384 }
 385 EXPORT_SYMBOL_GPL(pci_enable_pasid);
 386
 387 /**
 388  * pci_disable_pasid - Disable the PASID capability
 389  * @pdev: PCI device structure
 390  */
 391 void pci_disable_pasid(struct pci_dev *pdev)
 392 {
 393         u16 control = 0;
 394         int pasid = pdev->pasid_cap;
 395
 396         /* VFs share the PF PASID configuration */
 397         if (pdev->is_virtfn)
 398                 return;
 399
 400         if (WARN_ON(!pdev->pasid_enabled))
 401                 return;
 402
 403         if (!pasid)
 404                 return;
 405
 406         pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
 407
 408         pdev->pasid_enabled = 0;
 409 }
 410 EXPORT_SYMBOL_GPL(pci_disable_pasid);
 411
 412 /**
 413  * pci_restore_pasid_state - Restore PASID capabilities
 414  * @pdev: PCI device structure
 415  */
 416 void pci_restore_pasid_state(struct pci_dev *pdev)
 417 {
 418         u16 control;
 419         int pasid = pdev->pasid_cap;
 420
 421         if (pdev->is_virtfn)
 422                 return;
 423
 424         if (!pdev->pasid_enabled)
 425                 return;
 426
 427         if (!pasid)
 428                 return;
 429
 430         control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;
 431         pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);
 432 }
 433
 434 /**
 435  * pci_pasid_features - Check which PASID features are supported
 436  * @pdev: PCI device structure
 437  *
 438  * Returns a negative value when no PASI capability is present.
 439  * Otherwise is returns a bitmask with supported features. Current
 440  * features reported are:
 441  * PCI_PASID_CAP_EXEC - Execute permission supported
 442  * PCI_PASID_CAP_PRIV - Privileged mode supported
 443  */
 444 int pci_pasid_features(struct pci_dev *pdev)
 445 {
 446         u16 supported;
 447         int pasid;
 448
 449         if (pdev->is_virtfn)
 450                 pdev = pci_physfn(pdev);
 451
 452         pasid = pdev->pasid_cap;
 453         if (!pasid)
 454                 return -EINVAL;
 455
 456         pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
 457
 458         supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;
 459
 460         return supported;
 461 }
 462 EXPORT_SYMBOL_GPL(pci_pasid_features);
 463
 464 #define PASID_NUMBER_SHIFT      8
 465 #define PASID_NUMBER_MASK       (0x1f << PASID_NUMBER_SHIFT)
 466 /**
 467  * pci_max_pasid - Get maximum number of PASIDs supported by device
 468  * @pdev: PCI device structure
 469  *
 470  * Returns negative value when PASID capability is not present.
 471  * Otherwise it returns the number of supported PASIDs.
 472  */
 473 int pci_max_pasids(struct pci_dev *pdev)
 474 {
 475         u16 supported;
 476         int pasid;
 477
 478         if (pdev->is_virtfn)
 479                 pdev = pci_physfn(pdev);
 480
 481         pasid = pdev->pasid_cap;
 482         if (!pasid)
 483                 return -EINVAL;
 484
 485         pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);
 486
 487         supported = (supported & PASID_NUMBER_MASK) >> PASID_NUMBER_SHIFT;
 488
 489         return (1 << supported);
 490 }
 491 EXPORT_SYMBOL_GPL(pci_max_pasids);
 492 #endif /* CONFIG_PCI_PASID */