drivers/pci/tph.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * TPH (TLP Processing Hints) support
   4  *
   5  * Copyright (C) 2024 Advanced Micro Devices, Inc.
   6  *     Eric Van Tassell <Eric.VanTassell@amd.com>
   7  *     Wei Huang <wei.huang2@amd.com>
   8  */
   9 #include <linux/pci.h>
  10 #include <linux/pci-acpi.h>
  11 #include <linux/msi.h>
  12 #include <linux/bitfield.h>
  13 #include <linux/pci-tph.h>
  14
  15 #include "pci.h"
  16
  17 /* System-wide TPH disabled */
  18 static bool pci_tph_disabled;
  19
  20 #ifdef CONFIG_ACPI
  21 /*
  22  * The st_info struct defines the Steering Tag (ST) info returned by the
  23  * firmware PCI ACPI _DSM method (rev=0x7, func=0xF, "_DSM to Query Cache
  24  * Locality TPH Features"), as specified in the approved ECN for PCI Firmware
  25  * Spec and available at https://members.pcisig.com/wg/PCI-SIG/document/15470.
  26  *
  27  * @vm_st_valid:  8-bit ST for volatile memory is valid
  28  * @vm_xst_valid: 16-bit extended ST for volatile memory is valid
  29  * @vm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied
  30  * @vm_st:        8-bit ST for volatile mem
  31  * @vm_xst:       16-bit extended ST for volatile mem
  32  * @pm_st_valid:  8-bit ST for persistent memory is valid
  33  * @pm_xst_valid: 16-bit extended ST for persistent memory is valid
  34  * @pm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied
  35  * @pm_st:        8-bit ST for persistent mem
  36  * @pm_xst:       16-bit extended ST for persistent mem
  37  */
  38 union st_info {
  39         struct {
  40                 u64 vm_st_valid : 1;
  41                 u64 vm_xst_valid : 1;
  42                 u64 vm_ph_ignore : 1;
  43                 u64 rsvd1 : 5;
  44                 u64 vm_st : 8;
  45                 u64 vm_xst : 16;
  46                 u64 pm_st_valid : 1;
  47                 u64 pm_xst_valid : 1;
  48                 u64 pm_ph_ignore : 1;
  49                 u64 rsvd2 : 5;
  50                 u64 pm_st : 8;
  51                 u64 pm_xst : 16;
  52         };
  53         u64 value;
  54 };
  55
  56 static u16 tph_extract_tag(enum tph_mem_type mem_type, u8 req_type,
  57                            union st_info *info)
  58 {
  59         switch (req_type) {
  60         case PCI_TPH_REQ_TPH_ONLY: /* 8-bit tag */
  61                 switch (mem_type) {
  62                 case TPH_MEM_TYPE_VM:
  63                         if (info->vm_st_valid)
  64                                 return info->vm_st;
  65                         break;
  66                 case TPH_MEM_TYPE_PM:
  67                         if (info->pm_st_valid)
  68                                 return info->pm_st;
  69                         break;
  70                 }
  71                 break;
  72         case PCI_TPH_REQ_EXT_TPH: /* 16-bit tag */
  73                 switch (mem_type) {
  74                 case TPH_MEM_TYPE_VM:
  75                         if (info->vm_xst_valid)
  76                                 return info->vm_xst;
  77                         break;
  78                 case TPH_MEM_TYPE_PM:
  79                         if (info->pm_xst_valid)
  80                                 return info->pm_xst;
  81                         break;
  82                 }
  83                 break;
  84         default:
  85                 return 0;
  86         }
  87
  88         return 0;
  89 }
  90
  91 #define TPH_ST_DSM_FUNC_INDEX   0xF
  92 static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid,
  93                                   union st_info *st_out)
  94 {
  95         union acpi_object arg3[3], in_obj, *out_obj;
  96
  97         if (!acpi_check_dsm(handle, &pci_acpi_dsm_guid, 7,
  98                             BIT(TPH_ST_DSM_FUNC_INDEX)))
  99                 return AE_ERROR;
 100
 101         /* DWORD: feature ID (0 for processor cache ST query) */
 102         arg3[0].integer.type = ACPI_TYPE_INTEGER;
 103         arg3[0].integer.value = 0;
 104
 105         /* DWORD: target UID */
 106         arg3[1].integer.type = ACPI_TYPE_INTEGER;
 107         arg3[1].integer.value = cpu_uid;
 108
 109         /* QWORD: properties, all 0's */
 110         arg3[2].integer.type = ACPI_TYPE_INTEGER;
 111         arg3[2].integer.value = 0;
 112
 113         in_obj.type = ACPI_TYPE_PACKAGE;
 114         in_obj.package.count = ARRAY_SIZE(arg3);
 115         in_obj.package.elements = arg3;
 116
 117         out_obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 7,
 118                                     TPH_ST_DSM_FUNC_INDEX, &in_obj);
 119         if (!out_obj)
 120                 return AE_ERROR;
 121
 122         if (out_obj->type != ACPI_TYPE_BUFFER) {
 123                 ACPI_FREE(out_obj);
 124                 return AE_ERROR;
 125         }
 126
 127         st_out->value = *((u64 *)(out_obj->buffer.pointer));
 128
 129         ACPI_FREE(out_obj);
 130
 131         return AE_OK;
 132 }
 133 #endif
 134
 135 /* Update the TPH Requester Enable field of TPH Control Register */
 136 static void set_ctrl_reg_req_en(struct pci_dev *pdev, u8 req_type)
 137 {
 138         u32 reg;
 139
 140         pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, &reg);
 141
 142         reg &= ~PCI_TPH_CTRL_REQ_EN_MASK;
 143         reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, req_type);
 144
 145         pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
 146 }
 147
 148 static u8 get_st_modes(struct pci_dev *pdev)
 149 {
 150         u32 reg;
 151
 152         pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
 153         reg &= PCI_TPH_CAP_ST_NS | PCI_TPH_CAP_ST_IV | PCI_TPH_CAP_ST_DS;
 154
 155         return reg;
 156 }
 157
 158 static u32 get_st_table_loc(struct pci_dev *pdev)
 159 {
 160         u32 reg;
 161
 162         pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
 163
 164         return FIELD_GET(PCI_TPH_CAP_LOC_MASK, reg);
 165 }
 166
 167 /*
 168  * Return the size of ST table. If ST table is not in TPH Requester Extended
 169  * Capability space, return 0. Otherwise return the ST Table Size + 1.
 170  */
 171 static u16 get_st_table_size(struct pci_dev *pdev)
 172 {
 173         u32 reg;
 174         u32 loc;
 175
 176         /* Check ST table location first */
 177         loc = get_st_table_loc(pdev);
 178
 179         /* Convert loc to match with PCI_TPH_LOC_* defined in pci_regs.h */
 180         loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc);
 181         if (loc != PCI_TPH_LOC_CAP)
 182                 return 0;
 183
 184         pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
 185
 186         return FIELD_GET(PCI_TPH_CAP_ST_MASK, reg) + 1;
 187 }
 188
 189 /* Return device's Root Port completer capability */
 190 static u8 get_rp_completer_type(struct pci_dev *pdev)
 191 {
 192         struct pci_dev *rp;
 193         u32 reg;
 194         int ret;
 195
 196         rp = pcie_find_root_port(pdev);
 197         if (!rp)
 198                 return 0;
 199
 200         ret = pcie_capability_read_dword(rp, PCI_EXP_DEVCAP2, &reg);
 201         if (ret)
 202                 return 0;
 203
 204         return FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg);
 205 }
 206
 207 /* Write ST to MSI-X vector control reg - Return 0 if OK, otherwise -errno */
 208 static int write_tag_to_msix(struct pci_dev *pdev, int msix_idx, u16 tag)
 209 {
 210 #ifdef CONFIG_PCI_MSI
 211         struct msi_desc *msi_desc = NULL;
 212         void __iomem *vec_ctrl;
 213         u32 val;
 214         int err = 0;
 215
 216         msi_lock_descs(&pdev->dev);
 217
 218         /* Find the msi_desc entry with matching msix_idx */
 219         msi_for_each_desc(msi_desc, &pdev->dev, MSI_DESC_ASSOCIATED) {
 220                 if (msi_desc->msi_index == msix_idx)
 221                         break;
 222         }
 223
 224         if (!msi_desc) {
 225                 err = -ENXIO;
 226                 goto err_out;
 227         }
 228
 229         /* Get the vector control register (offset 0xc) pointed by msix_idx */
 230         vec_ctrl = pdev->msix_base + msix_idx * PCI_MSIX_ENTRY_SIZE;
 231         vec_ctrl += PCI_MSIX_ENTRY_VECTOR_CTRL;
 232
 233         val = readl(vec_ctrl);
 234         val &= ~PCI_MSIX_ENTRY_CTRL_ST;
 235         val |= FIELD_PREP(PCI_MSIX_ENTRY_CTRL_ST, tag);
 236         writel(val, vec_ctrl);
 237
 238         /* Read back to flush the update */
 239         val = readl(vec_ctrl);
 240
 241 err_out:
 242         msi_unlock_descs(&pdev->dev);
 243         return err;
 244 #else
 245         return -ENODEV;
 246 #endif
 247 }
 248
 249 /* Write tag to ST table - Return 0 if OK, otherwise -errno */
 250 static int write_tag_to_st_table(struct pci_dev *pdev, int index, u16 tag)
 251 {
 252         int st_table_size;
 253         int offset;
 254
 255         /* Check if index is out of bound */
 256         st_table_size = get_st_table_size(pdev);
 257         if (index >= st_table_size)
 258                 return -ENXIO;
 259
 260         offset = pdev->tph_cap + PCI_TPH_BASE_SIZEOF + index * sizeof(u16);
 261
 262         return pci_write_config_word(pdev, offset, tag);
 263 }
 264
 265 /**
 266  * pcie_tph_get_cpu_st() - Retrieve Steering Tag for a target memory associated
 267  * with a specific CPU
 268  * @pdev: PCI device
 269  * @mem_type: target memory type (volatile or persistent RAM)
 270  * @cpu_uid: associated CPU id
 271  * @tag: Steering Tag to be returned
 272  *
 273  * Return the Steering Tag for a target memory that is associated with a
 274  * specific CPU as indicated by cpu_uid.
 275  *
 276  * Return: 0 if success, otherwise negative value (-errno)
 277  */
 278 int pcie_tph_get_cpu_st(struct pci_dev *pdev, enum tph_mem_type mem_type,
 279                         unsigned int cpu_uid, u16 *tag)
 280 {
 281 #ifdef CONFIG_ACPI
 282         struct pci_dev *rp;
 283         acpi_handle rp_acpi_handle;
 284         union st_info info;
 285
 286         rp = pcie_find_root_port(pdev);
 287         if (!rp || !rp->bus || !rp->bus->bridge)
 288                 return -ENODEV;
 289
 290         rp_acpi_handle = ACPI_HANDLE(rp->bus->bridge);
 291
 292         if (tph_invoke_dsm(rp_acpi_handle, cpu_uid, &info) != AE_OK) {
 293                 *tag = 0;
 294                 return -EINVAL;
 295         }
 296
 297         *tag = tph_extract_tag(mem_type, pdev->tph_req_type, &info);
 298
 299         pci_dbg(pdev, "get steering tag: mem_type=%s, cpu_uid=%d, tag=%#04x\n",
 300                 (mem_type == TPH_MEM_TYPE_VM) ? "volatile" : "persistent",
 301                 cpu_uid, *tag);
 302
 303         return 0;
 304 #else
 305         return -ENODEV;
 306 #endif
 307 }
 308 EXPORT_SYMBOL(pcie_tph_get_cpu_st);
 309
 310 /**
 311  * pcie_tph_set_st_entry() - Set Steering Tag in the ST table entry
 312  * @pdev: PCI device
 313  * @index: ST table entry index
 314  * @tag: Steering Tag to be written
 315  *
 316  * Figure out the proper location of ST table, either in the MSI-X table or
 317  * in the TPH Extended Capability space, and write the Steering Tag into
 318  * the ST entry pointed by index.
 319  *
 320  * Return: 0 if success, otherwise negative value (-errno)
 321  */
 322 int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, u16 tag)
 323 {
 324         u32 loc;
 325         int err = 0;
 326
 327         if (!pdev->tph_cap)
 328                 return -EINVAL;
 329
 330         if (!pdev->tph_enabled)
 331                 return -EINVAL;
 332
 333         /* No need to write tag if device is in "No ST Mode" */
 334         if (pdev->tph_mode == PCI_TPH_ST_NS_MODE)
 335                 return 0;
 336
 337         /*
 338          * Disable TPH before updating ST to avoid potential instability as
 339          * cautioned in PCIe r6.2, sec 6.17.3, "ST Modes of Operation"
 340          */
 341         set_ctrl_reg_req_en(pdev, PCI_TPH_REQ_DISABLE);
 342
 343         loc = get_st_table_loc(pdev);
 344         /* Convert loc to match with PCI_TPH_LOC_* */
 345         loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc);
 346
 347         switch (loc) {
 348         case PCI_TPH_LOC_MSIX:
 349                 err = write_tag_to_msix(pdev, index, tag);
 350                 break;
 351         case PCI_TPH_LOC_CAP:
 352                 err = write_tag_to_st_table(pdev, index, tag);
 353                 break;
 354         default:
 355                 err = -EINVAL;
 356         }
 357
 358         if (err) {
 359                 pcie_disable_tph(pdev);
 360                 return err;
 361         }
 362
 363         set_ctrl_reg_req_en(pdev, pdev->tph_mode);
 364
 365         pci_dbg(pdev, "set steering tag: %s table, index=%d, tag=%#04x\n",
 366                 (loc == PCI_TPH_LOC_MSIX) ? "MSI-X" : "ST", index, tag);
 367
 368         return 0;
 369 }
 370 EXPORT_SYMBOL(pcie_tph_set_st_entry);
 371
 372 /**
 373  * pcie_disable_tph - Turn off TPH support for device
 374  * @pdev: PCI device
 375  *
 376  * Return: none
 377  */
 378 void pcie_disable_tph(struct pci_dev *pdev)
 379 {
 380         if (!pdev->tph_cap)
 381                 return;
 382
 383         if (!pdev->tph_enabled)
 384                 return;
 385
 386         pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, 0);
 387
 388         pdev->tph_mode = 0;
 389         pdev->tph_req_type = 0;
 390         pdev->tph_enabled = 0;
 391 }
 392 EXPORT_SYMBOL(pcie_disable_tph);
 393
 394 /**
 395  * pcie_enable_tph - Enable TPH support for device using a specific ST mode
 396  * @pdev: PCI device
 397  * @mode: ST mode to enable. Current supported modes include:
 398  *
 399  *   - PCI_TPH_ST_NS_MODE: NO ST Mode
 400  *   - PCI_TPH_ST_IV_MODE: Interrupt Vector Mode
 401  *   - PCI_TPH_ST_DS_MODE: Device Specific Mode
 402  *
 403  * Check whether the mode is actually supported by the device before enabling
 404  * and return an error if not. Additionally determine what types of requests,
 405  * TPH or extended TPH, can be issued by the device based on its TPH requester
 406  * capability and the Root Port's completer capability.
 407  *
 408  * Return: 0 on success, otherwise negative value (-errno)
 409  */
 410 int pcie_enable_tph(struct pci_dev *pdev, int mode)
 411 {
 412         u32 reg;
 413         u8 dev_modes;
 414         u8 rp_req_type;
 415
 416         /* Honor "notph" kernel parameter */
 417         if (pci_tph_disabled)
 418                 return -EINVAL;
 419
 420         if (!pdev->tph_cap)
 421                 return -EINVAL;
 422
 423         if (pdev->tph_enabled)
 424                 return -EBUSY;
 425
 426         /* Sanitize and check ST mode compatibility */
 427         mode &= PCI_TPH_CTRL_MODE_SEL_MASK;
 428         dev_modes = get_st_modes(pdev);
 429         if (!((1 << mode) & dev_modes))
 430                 return -EINVAL;
 431
 432         pdev->tph_mode = mode;
 433
 434         /* Get req_type supported by device and its Root Port */
 435         pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, &reg);
 436         if (FIELD_GET(PCI_TPH_CAP_EXT_TPH, reg))
 437                 pdev->tph_req_type = PCI_TPH_REQ_EXT_TPH;
 438         else
 439                 pdev->tph_req_type = PCI_TPH_REQ_TPH_ONLY;
 440
 441         rp_req_type = get_rp_completer_type(pdev);
 442
 443         /* Final req_type is the smallest value of two */
 444         pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type);
 445
 446         if (pdev->tph_req_type == PCI_TPH_REQ_DISABLE)
 447                 return -EINVAL;
 448
 449         /* Write them into TPH control register */
 450         pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, &reg);
 451
 452         reg &= ~PCI_TPH_CTRL_MODE_SEL_MASK;
 453         reg |= FIELD_PREP(PCI_TPH_CTRL_MODE_SEL_MASK, pdev->tph_mode);
 454
 455         reg &= ~PCI_TPH_CTRL_REQ_EN_MASK;
 456         reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, pdev->tph_req_type);
 457
 458         pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
 459
 460         pdev->tph_enabled = 1;
 461
 462         return 0;
 463 }
 464 EXPORT_SYMBOL(pcie_enable_tph);
 465
 466 void pci_restore_tph_state(struct pci_dev *pdev)
 467 {
 468         struct pci_cap_saved_state *save_state;
 469         int num_entries, i, offset;
 470         u16 *st_entry;
 471         u32 *cap;
 472
 473         if (!pdev->tph_cap)
 474                 return;
 475
 476         if (!pdev->tph_enabled)
 477                 return;
 478
 479         save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH);
 480         if (!save_state)
 481                 return;
 482
 483         /* Restore control register and all ST entries */
 484         cap = &save_state->cap.data[0];
 485         pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, *cap++);
 486         st_entry = (u16 *)cap;
 487         offset = PCI_TPH_BASE_SIZEOF;
 488         num_entries = get_st_table_size(pdev);
 489         for (i = 0; i < num_entries; i++) {
 490                 pci_write_config_word(pdev, pdev->tph_cap + offset,
 491                                       *st_entry++);
 492                 offset += sizeof(u16);
 493         }
 494 }
 495
 496 void pci_save_tph_state(struct pci_dev *pdev)
 497 {
 498         struct pci_cap_saved_state *save_state;
 499         int num_entries, i, offset;
 500         u16 *st_entry;
 501         u32 *cap;
 502
 503         if (!pdev->tph_cap)
 504                 return;
 505
 506         if (!pdev->tph_enabled)
 507                 return;
 508
 509         save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH);
 510         if (!save_state)
 511                 return;
 512
 513         /* Save control register */
 514         cap = &save_state->cap.data[0];
 515         pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, cap++);
 516
 517         /* Save all ST entries in extended capability structure */
 518         st_entry = (u16 *)cap;
 519         offset = PCI_TPH_BASE_SIZEOF;
 520         num_entries = get_st_table_size(pdev);
 521         for (i = 0; i < num_entries; i++) {
 522                 pci_read_config_word(pdev, pdev->tph_cap + offset,
 523                                      st_entry++);
 524                 offset += sizeof(u16);
 525         }
 526 }
 527
 528 void pci_no_tph(void)
 529 {
 530         pci_tph_disabled = true;
 531
 532         pr_info("PCIe TPH is disabled\n");
 533 }
 534
 535 void pci_tph_init(struct pci_dev *pdev)
 536 {
 537         int num_entries;
 538         u32 save_size;
 539
 540         pdev->tph_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_TPH);
 541         if (!pdev->tph_cap)
 542                 return;
 543
 544         num_entries = get_st_table_size(pdev);
 545         save_size = sizeof(u32) + num_entries * sizeof(u16);
 546         pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_TPH, save_size);
 547 }