drivers/dca/dca-core.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Copyright(c) 2007 - 2009 Intel Corporation. All rights reserved.
   4  */
   5
   6 /*
   7  * This driver supports an interface for DCA clients and providers to meet.
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/notifier.h>
  12 #include <linux/device.h>
  13 #include <linux/dca.h>
  14 #include <linux/slab.h>
  15 #include <linux/module.h>
  16
  17 #define DCA_VERSION "1.12.1"
  18
  19 MODULE_VERSION(DCA_VERSION);
  20 MODULE_DESCRIPTION("Intel Direct Cache Access (DCA) service module");
  21 MODULE_LICENSE("GPL");
  22 MODULE_AUTHOR("Intel Corporation");
  23
  24 static DEFINE_RAW_SPINLOCK(dca_lock);
  25
  26 static LIST_HEAD(dca_domains);
  27
  28 static BLOCKING_NOTIFIER_HEAD(dca_provider_chain);
  29
  30 static int dca_providers_blocked;
  31
  32 static struct pci_bus *dca_pci_rc_from_dev(struct device *dev)
  33 {
  34         struct pci_dev *pdev = to_pci_dev(dev);
  35         struct pci_bus *bus = pdev->bus;
  36
  37         while (bus->parent)
  38                 bus = bus->parent;
  39
  40         return bus;
  41 }
  42
  43 static struct dca_domain *dca_allocate_domain(struct pci_bus *rc)
  44 {
  45         struct dca_domain *domain;
  46
  47         domain = kzalloc(sizeof(*domain), GFP_NOWAIT);
  48         if (!domain)
  49                 return NULL;
  50
  51         INIT_LIST_HEAD(&domain->dca_providers);
  52         domain->pci_rc = rc;
  53
  54         return domain;
  55 }
  56
  57 static void dca_free_domain(struct dca_domain *domain)
  58 {
  59         list_del(&domain->node);
  60         kfree(domain);
  61 }
  62
  63 static int dca_provider_ioat_ver_3_0(struct device *dev)
  64 {
  65         struct pci_dev *pdev = to_pci_dev(dev);
  66
  67         return ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
  68                 ((pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG0) ||
  69                 (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG1) ||
  70                 (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG2) ||
  71                 (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG3) ||
  72                 (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG4) ||
  73                 (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG5) ||
  74                 (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG6) ||
  75                 (pdev->device == PCI_DEVICE_ID_INTEL_IOAT_TBG7)));
  76 }
  77
  78 static void unregister_dca_providers(void)
  79 {
  80         struct dca_provider *dca, *_dca;
  81         struct list_head unregistered_providers;
  82         struct dca_domain *domain;
  83         unsigned long flags;
  84
  85         blocking_notifier_call_chain(&dca_provider_chain,
  86                                      DCA_PROVIDER_REMOVE, NULL);
  87
  88         INIT_LIST_HEAD(&unregistered_providers);
  89
  90         raw_spin_lock_irqsave(&dca_lock, flags);
  91
  92         if (list_empty(&dca_domains)) {
  93                 raw_spin_unlock_irqrestore(&dca_lock, flags);
  94                 return;
  95         }
  96
  97         /* at this point only one domain in the list is expected */
  98         domain = list_first_entry(&dca_domains, struct dca_domain, node);
  99
 100         list_for_each_entry_safe(dca, _dca, &domain->dca_providers, node)
 101                 list_move(&dca->node, &unregistered_providers);
 102
 103         dca_free_domain(domain);
 104
 105         raw_spin_unlock_irqrestore(&dca_lock, flags);
 106
 107         list_for_each_entry_safe(dca, _dca, &unregistered_providers, node) {
 108                 dca_sysfs_remove_provider(dca);
 109                 list_del(&dca->node);
 110         }
 111 }
 112
 113 static struct dca_domain *dca_find_domain(struct pci_bus *rc)
 114 {
 115         struct dca_domain *domain;
 116
 117         list_for_each_entry(domain, &dca_domains, node)
 118                 if (domain->pci_rc == rc)
 119                         return domain;
 120
 121         return NULL;
 122 }
 123
 124 static struct dca_domain *dca_get_domain(struct device *dev)
 125 {
 126         struct pci_bus *rc;
 127         struct dca_domain *domain;
 128
 129         rc = dca_pci_rc_from_dev(dev);
 130         domain = dca_find_domain(rc);
 131
 132         if (!domain) {
 133                 if (dca_provider_ioat_ver_3_0(dev) && !list_empty(&dca_domains))
 134                         dca_providers_blocked = 1;
 135         }
 136
 137         return domain;
 138 }
 139
 140 static struct dca_provider *dca_find_provider_by_dev(struct device *dev)
 141 {
 142         struct dca_provider *dca;
 143         struct pci_bus *rc;
 144         struct dca_domain *domain;
 145
 146         if (dev) {
 147                 rc = dca_pci_rc_from_dev(dev);
 148                 domain = dca_find_domain(rc);
 149                 if (!domain)
 150                         return NULL;
 151         } else {
 152                 if (!list_empty(&dca_domains))
 153                         domain = list_first_entry(&dca_domains,
 154                                                   struct dca_domain,
 155                                                   node);
 156                 else
 157                         return NULL;
 158         }
 159
 160         list_for_each_entry(dca, &domain->dca_providers, node)
 161                 if ((!dev) || (dca->ops->dev_managed(dca, dev)))
 162                         return dca;
 163
 164         return NULL;
 165 }
 166
 167 /**
 168  * dca_add_requester - add a dca client to the list
 169  * @dev - the device that wants dca service
 170  */
 171 int dca_add_requester(struct device *dev)
 172 {
 173         struct dca_provider *dca;
 174         int err, slot = -ENODEV;
 175         unsigned long flags;
 176         struct pci_bus *pci_rc;
 177         struct dca_domain *domain;
 178
 179         if (!dev)
 180                 return -EFAULT;
 181
 182         raw_spin_lock_irqsave(&dca_lock, flags);
 183
 184         /* check if the requester has not been added already */
 185         dca = dca_find_provider_by_dev(dev);
 186         if (dca) {
 187                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 188                 return -EEXIST;
 189         }
 190
 191         pci_rc = dca_pci_rc_from_dev(dev);
 192         domain = dca_find_domain(pci_rc);
 193         if (!domain) {
 194                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 195                 return -ENODEV;
 196         }
 197
 198         list_for_each_entry(dca, &domain->dca_providers, node) {
 199                 slot = dca->ops->add_requester(dca, dev);
 200                 if (slot >= 0)
 201                         break;
 202         }
 203
 204         raw_spin_unlock_irqrestore(&dca_lock, flags);
 205
 206         if (slot < 0)
 207                 return slot;
 208
 209         err = dca_sysfs_add_req(dca, dev, slot);
 210         if (err) {
 211                 raw_spin_lock_irqsave(&dca_lock, flags);
 212                 if (dca == dca_find_provider_by_dev(dev))
 213                         dca->ops->remove_requester(dca, dev);
 214                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 215                 return err;
 216         }
 217
 218         return 0;
 219 }
 220 EXPORT_SYMBOL_GPL(dca_add_requester);
 221
 222 /**
 223  * dca_remove_requester - remove a dca client from the list
 224  * @dev - the device that wants dca service
 225  */
 226 int dca_remove_requester(struct device *dev)
 227 {
 228         struct dca_provider *dca;
 229         int slot;
 230         unsigned long flags;
 231
 232         if (!dev)
 233                 return -EFAULT;
 234
 235         raw_spin_lock_irqsave(&dca_lock, flags);
 236         dca = dca_find_provider_by_dev(dev);
 237         if (!dca) {
 238                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 239                 return -ENODEV;
 240         }
 241         slot = dca->ops->remove_requester(dca, dev);
 242         raw_spin_unlock_irqrestore(&dca_lock, flags);
 243
 244         if (slot < 0)
 245                 return slot;
 246
 247         dca_sysfs_remove_req(dca, slot);
 248
 249         return 0;
 250 }
 251 EXPORT_SYMBOL_GPL(dca_remove_requester);
 252
 253 /**
 254  * dca_common_get_tag - return the dca tag (serves both new and old api)
 255  * @dev - the device that wants dca service
 256  * @cpu - the cpuid as returned by get_cpu()
 257  */
 258 static u8 dca_common_get_tag(struct device *dev, int cpu)
 259 {
 260         struct dca_provider *dca;
 261         u8 tag;
 262         unsigned long flags;
 263
 264         raw_spin_lock_irqsave(&dca_lock, flags);
 265
 266         dca = dca_find_provider_by_dev(dev);
 267         if (!dca) {
 268                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 269                 return -ENODEV;
 270         }
 271         tag = dca->ops->get_tag(dca, dev, cpu);
 272
 273         raw_spin_unlock_irqrestore(&dca_lock, flags);
 274         return tag;
 275 }
 276
 277 /**
 278  * dca3_get_tag - return the dca tag to the requester device
 279  *                for the given cpu (new api)
 280  * @dev - the device that wants dca service
 281  * @cpu - the cpuid as returned by get_cpu()
 282  */
 283 u8 dca3_get_tag(struct device *dev, int cpu)
 284 {
 285         if (!dev)
 286                 return -EFAULT;
 287
 288         return dca_common_get_tag(dev, cpu);
 289 }
 290 EXPORT_SYMBOL_GPL(dca3_get_tag);
 291
 292 /**
 293  * dca_get_tag - return the dca tag for the given cpu (old api)
 294  * @cpu - the cpuid as returned by get_cpu()
 295  */
 296 u8 dca_get_tag(int cpu)
 297 {
 298         return dca_common_get_tag(NULL, cpu);
 299 }
 300 EXPORT_SYMBOL_GPL(dca_get_tag);
 301
 302 /**
 303  * alloc_dca_provider - get data struct for describing a dca provider
 304  * @ops - pointer to struct of dca operation function pointers
 305  * @priv_size - size of extra mem to be added for provider's needs
 306  */
 307 struct dca_provider *alloc_dca_provider(const struct dca_ops *ops,
 308                                         int priv_size)
 309 {
 310         struct dca_provider *dca;
 311         int alloc_size;
 312
 313         alloc_size = (sizeof(*dca) + priv_size);
 314         dca = kzalloc(alloc_size, GFP_KERNEL);
 315         if (!dca)
 316                 return NULL;
 317         dca->ops = ops;
 318
 319         return dca;
 320 }
 321 EXPORT_SYMBOL_GPL(alloc_dca_provider);
 322
 323 /**
 324  * free_dca_provider - release the dca provider data struct
 325  * @ops - pointer to struct of dca operation function pointers
 326  * @priv_size - size of extra mem to be added for provider's needs
 327  */
 328 void free_dca_provider(struct dca_provider *dca)
 329 {
 330         kfree(dca);
 331 }
 332 EXPORT_SYMBOL_GPL(free_dca_provider);
 333
 334 /**
 335  * register_dca_provider - register a dca provider
 336  * @dca - struct created by alloc_dca_provider()
 337  * @dev - device providing dca services
 338  */
 339 int register_dca_provider(struct dca_provider *dca, struct device *dev)
 340 {
 341         int err;
 342         unsigned long flags;
 343         struct dca_domain *domain, *newdomain = NULL;
 344
 345         raw_spin_lock_irqsave(&dca_lock, flags);
 346         if (dca_providers_blocked) {
 347                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 348                 return -ENODEV;
 349         }
 350         raw_spin_unlock_irqrestore(&dca_lock, flags);
 351
 352         err = dca_sysfs_add_provider(dca, dev);
 353         if (err)
 354                 return err;
 355
 356         raw_spin_lock_irqsave(&dca_lock, flags);
 357         domain = dca_get_domain(dev);
 358         if (!domain) {
 359                 struct pci_bus *rc;
 360
 361                 if (dca_providers_blocked) {
 362                         raw_spin_unlock_irqrestore(&dca_lock, flags);
 363                         dca_sysfs_remove_provider(dca);
 364                         unregister_dca_providers();
 365                         return -ENODEV;
 366                 }
 367
 368                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 369                 rc = dca_pci_rc_from_dev(dev);
 370                 newdomain = dca_allocate_domain(rc);
 371                 if (!newdomain)
 372                         return -ENODEV;
 373                 raw_spin_lock_irqsave(&dca_lock, flags);
 374                 /* Recheck, we might have raced after dropping the lock */
 375                 domain = dca_get_domain(dev);
 376                 if (!domain) {
 377                         domain = newdomain;
 378                         newdomain = NULL;
 379                         list_add(&domain->node, &dca_domains);
 380                 }
 381         }
 382         list_add(&dca->node, &domain->dca_providers);
 383         raw_spin_unlock_irqrestore(&dca_lock, flags);
 384
 385         blocking_notifier_call_chain(&dca_provider_chain,
 386                                      DCA_PROVIDER_ADD, NULL);
 387         kfree(newdomain);
 388         return 0;
 389 }
 390 EXPORT_SYMBOL_GPL(register_dca_provider);
 391
 392 /**
 393  * unregister_dca_provider - remove a dca provider
 394  * @dca - struct created by alloc_dca_provider()
 395  */
 396 void unregister_dca_provider(struct dca_provider *dca, struct device *dev)
 397 {
 398         unsigned long flags;
 399         struct pci_bus *pci_rc;
 400         struct dca_domain *domain;
 401
 402         blocking_notifier_call_chain(&dca_provider_chain,
 403                                      DCA_PROVIDER_REMOVE, NULL);
 404
 405         raw_spin_lock_irqsave(&dca_lock, flags);
 406
 407         if (list_empty(&dca_domains)) {
 408                 raw_spin_unlock_irqrestore(&dca_lock, flags);
 409                 return;
 410         }
 411
 412         list_del(&dca->node);
 413
 414         pci_rc = dca_pci_rc_from_dev(dev);
 415         domain = dca_find_domain(pci_rc);
 416         if (list_empty(&domain->dca_providers))
 417                 dca_free_domain(domain);
 418
 419         raw_spin_unlock_irqrestore(&dca_lock, flags);
 420
 421         dca_sysfs_remove_provider(dca);
 422 }
 423 EXPORT_SYMBOL_GPL(unregister_dca_provider);
 424
 425 /**
 426  * dca_register_notify - register a client's notifier callback
 427  */
 428 void dca_register_notify(struct notifier_block *nb)
 429 {
 430         blocking_notifier_chain_register(&dca_provider_chain, nb);
 431 }
 432 EXPORT_SYMBOL_GPL(dca_register_notify);
 433
 434 /**
 435  * dca_unregister_notify - remove a client's notifier callback
 436  */
 437 void dca_unregister_notify(struct notifier_block *nb)
 438 {
 439         blocking_notifier_chain_unregister(&dca_provider_chain, nb);
 440 }
 441 EXPORT_SYMBOL_GPL(dca_unregister_notify);
 442
 443 static int __init dca_init(void)
 444 {
 445         pr_info("dca service started, version %s\n", DCA_VERSION);
 446         return dca_sysfs_init();
 447 }
 448
 449 static void __exit dca_exit(void)
 450 {
 451         dca_sysfs_exit();
 452 }
 453
 454 arch_initcall(dca_init);
 455 module_exit(dca_exit);
 456