drivers/misc/ocxl/core.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Copyright 2019 IBM Corp.
   3 #include <linux/idr.h>
   4 #include "ocxl_internal.h"
   5
   6 static struct ocxl_fn *ocxl_fn_get(struct ocxl_fn *fn)
   7 {
   8         return (get_device(&fn->dev) == NULL) ? NULL : fn;
   9 }
  10
  11 static void ocxl_fn_put(struct ocxl_fn *fn)
  12 {
  13         put_device(&fn->dev);
  14 }
  15
  16 static struct ocxl_afu *alloc_afu(struct ocxl_fn *fn)
  17 {
  18         struct ocxl_afu *afu;
  19
  20         afu = kzalloc(sizeof(struct ocxl_afu), GFP_KERNEL);
  21         if (!afu)
  22                 return NULL;
  23
  24         kref_init(&afu->kref);
  25         mutex_init(&afu->contexts_lock);
  26         mutex_init(&afu->afu_control_lock);
  27         idr_init(&afu->contexts_idr);
  28         afu->fn = fn;
  29         ocxl_fn_get(fn);
  30         return afu;
  31 }
  32
  33 static void free_afu(struct kref *kref)
  34 {
  35         struct ocxl_afu *afu = container_of(kref, struct ocxl_afu, kref);
  36
  37         idr_destroy(&afu->contexts_idr);
  38         ocxl_fn_put(afu->fn);
  39         kfree(afu);
  40 }
  41
  42 void ocxl_afu_get(struct ocxl_afu *afu)
  43 {
  44         kref_get(&afu->kref);
  45 }
  46 EXPORT_SYMBOL_GPL(ocxl_afu_get);
  47
  48 void ocxl_afu_put(struct ocxl_afu *afu)
  49 {
  50         kref_put(&afu->kref, free_afu);
  51 }
  52 EXPORT_SYMBOL_GPL(ocxl_afu_put);
  53
  54 static int assign_afu_actag(struct ocxl_afu *afu)
  55 {
  56         struct ocxl_fn *fn = afu->fn;
  57         int actag_count, actag_offset;
  58         struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
  59
  60         /*
  61          * if there were not enough actags for the function, each afu
  62          * reduces its count as well
  63          */
  64         actag_count = afu->config.actag_supported *
  65                 fn->actag_enabled / fn->actag_supported;
  66         actag_offset = ocxl_actag_afu_alloc(fn, actag_count);
  67         if (actag_offset < 0) {
  68                 dev_err(&pci_dev->dev, "Can't allocate %d actags for AFU: %d\n",
  69                         actag_count, actag_offset);
  70                 return actag_offset;
  71         }
  72         afu->actag_base = fn->actag_base + actag_offset;
  73         afu->actag_enabled = actag_count;
  74
  75         ocxl_config_set_afu_actag(pci_dev, afu->config.dvsec_afu_control_pos,
  76                                 afu->actag_base, afu->actag_enabled);
  77         dev_dbg(&pci_dev->dev, "actag base=%d enabled=%d\n",
  78                 afu->actag_base, afu->actag_enabled);
  79         return 0;
  80 }
  81
  82 static void reclaim_afu_actag(struct ocxl_afu *afu)
  83 {
  84         struct ocxl_fn *fn = afu->fn;
  85         int start_offset, size;
  86
  87         start_offset = afu->actag_base - fn->actag_base;
  88         size = afu->actag_enabled;
  89         ocxl_actag_afu_free(afu->fn, start_offset, size);
  90 }
  91
  92 static int assign_afu_pasid(struct ocxl_afu *afu)
  93 {
  94         struct ocxl_fn *fn = afu->fn;
  95         int pasid_count, pasid_offset;
  96         struct pci_dev *pci_dev = to_pci_dev(fn->dev.parent);
  97
  98         /*
  99          * We only support the case where the function configuration
 100          * requested enough PASIDs to cover all AFUs.
 101          */
 102         pasid_count = 1 << afu->config.pasid_supported_log;
 103         pasid_offset = ocxl_pasid_afu_alloc(fn, pasid_count);
 104         if (pasid_offset < 0) {
 105                 dev_err(&pci_dev->dev, "Can't allocate %d PASIDs for AFU: %d\n",
 106                         pasid_count, pasid_offset);
 107                 return pasid_offset;
 108         }
 109         afu->pasid_base = fn->pasid_base + pasid_offset;
 110         afu->pasid_count = 0;
 111         afu->pasid_max = pasid_count;
 112
 113         ocxl_config_set_afu_pasid(pci_dev, afu->config.dvsec_afu_control_pos,
 114                                 afu->pasid_base,
 115                                 afu->config.pasid_supported_log);
 116         dev_dbg(&pci_dev->dev, "PASID base=%d, enabled=%d\n",
 117                 afu->pasid_base, pasid_count);
 118         return 0;
 119 }
 120
 121 static void reclaim_afu_pasid(struct ocxl_afu *afu)
 122 {
 123         struct ocxl_fn *fn = afu->fn;
 124         int start_offset, size;
 125
 126         start_offset = afu->pasid_base - fn->pasid_base;
 127         size = 1 << afu->config.pasid_supported_log;
 128         ocxl_pasid_afu_free(afu->fn, start_offset, size);
 129 }
 130
 131 static int reserve_fn_bar(struct ocxl_fn *fn, int bar)
 132 {
 133         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 134         int rc, idx;
 135
 136         if (bar != 0 && bar != 2 && bar != 4)
 137                 return -EINVAL;
 138
 139         idx = bar >> 1;
 140         if (fn->bar_used[idx]++ == 0) {
 141                 rc = pci_request_region(dev, bar, "ocxl");
 142                 if (rc)
 143                         return rc;
 144         }
 145         return 0;
 146 }
 147
 148 static void release_fn_bar(struct ocxl_fn *fn, int bar)
 149 {
 150         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 151         int idx;
 152
 153         if (bar != 0 && bar != 2 && bar != 4)
 154                 return;
 155
 156         idx = bar >> 1;
 157         if (--fn->bar_used[idx] == 0)
 158                 pci_release_region(dev, bar);
 159         WARN_ON(fn->bar_used[idx] < 0);
 160 }
 161
 162 static int map_mmio_areas(struct ocxl_afu *afu)
 163 {
 164         int rc;
 165         struct pci_dev *pci_dev = to_pci_dev(afu->fn->dev.parent);
 166
 167         rc = reserve_fn_bar(afu->fn, afu->config.global_mmio_bar);
 168         if (rc)
 169                 return rc;
 170
 171         rc = reserve_fn_bar(afu->fn, afu->config.pp_mmio_bar);
 172         if (rc) {
 173                 release_fn_bar(afu->fn, afu->config.global_mmio_bar);
 174                 return rc;
 175         }
 176
 177         afu->global_mmio_start =
 178                 pci_resource_start(pci_dev, afu->config.global_mmio_bar) +
 179                 afu->config.global_mmio_offset;
 180         afu->pp_mmio_start =
 181                 pci_resource_start(pci_dev, afu->config.pp_mmio_bar) +
 182                 afu->config.pp_mmio_offset;
 183
 184         afu->global_mmio_ptr = ioremap(afu->global_mmio_start,
 185                                 afu->config.global_mmio_size);
 186         if (!afu->global_mmio_ptr) {
 187                 release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
 188                 release_fn_bar(afu->fn, afu->config.global_mmio_bar);
 189                 dev_err(&pci_dev->dev, "Error mapping global mmio area\n");
 190                 return -ENOMEM;
 191         }
 192
 193         /*
 194          * Leave an empty page between the per-process mmio area and
 195          * the AFU interrupt mappings
 196          */
 197         afu->irq_base_offset = afu->config.pp_mmio_stride + PAGE_SIZE;
 198         return 0;
 199 }
 200
 201 static void unmap_mmio_areas(struct ocxl_afu *afu)
 202 {
 203         if (afu->global_mmio_ptr) {
 204                 iounmap(afu->global_mmio_ptr);
 205                 afu->global_mmio_ptr = NULL;
 206         }
 207         afu->global_mmio_start = 0;
 208         afu->pp_mmio_start = 0;
 209         release_fn_bar(afu->fn, afu->config.pp_mmio_bar);
 210         release_fn_bar(afu->fn, afu->config.global_mmio_bar);
 211 }
 212
 213 static int configure_afu(struct ocxl_afu *afu, u8 afu_idx, struct pci_dev *dev)
 214 {
 215         int rc;
 216
 217         rc = ocxl_config_read_afu(dev, &afu->fn->config, &afu->config, afu_idx);
 218         if (rc)
 219                 return rc;
 220
 221         rc = assign_afu_actag(afu);
 222         if (rc)
 223                 return rc;
 224
 225         rc = assign_afu_pasid(afu);
 226         if (rc)
 227                 goto err_free_actag;
 228
 229         rc = map_mmio_areas(afu);
 230         if (rc)
 231                 goto err_free_pasid;
 232
 233         return 0;
 234
 235 err_free_pasid:
 236         reclaim_afu_pasid(afu);
 237 err_free_actag:
 238         reclaim_afu_actag(afu);
 239         return rc;
 240 }
 241
 242 static void deconfigure_afu(struct ocxl_afu *afu)
 243 {
 244         unmap_mmio_areas(afu);
 245         reclaim_afu_pasid(afu);
 246         reclaim_afu_actag(afu);
 247 }
 248
 249 static int activate_afu(struct pci_dev *dev, struct ocxl_afu *afu)
 250 {
 251         ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 1);
 252
 253         return 0;
 254 }
 255
 256 static void deactivate_afu(struct ocxl_afu *afu)
 257 {
 258         struct pci_dev *dev = to_pci_dev(afu->fn->dev.parent);
 259
 260         ocxl_config_set_afu_state(dev, afu->config.dvsec_afu_control_pos, 0);
 261 }
 262
 263 static int init_afu(struct pci_dev *dev, struct ocxl_fn *fn, u8 afu_idx)
 264 {
 265         int rc;
 266         struct ocxl_afu *afu;
 267
 268         afu = alloc_afu(fn);
 269         if (!afu)
 270                 return -ENOMEM;
 271
 272         rc = configure_afu(afu, afu_idx, dev);
 273         if (rc) {
 274                 ocxl_afu_put(afu);
 275                 return rc;
 276         }
 277
 278         rc = activate_afu(dev, afu);
 279         if (rc) {
 280                 deconfigure_afu(afu);
 281                 ocxl_afu_put(afu);
 282                 return rc;
 283         }
 284
 285         list_add_tail(&afu->list, &fn->afu_list);
 286
 287         return 0;
 288 }
 289
 290 static void remove_afu(struct ocxl_afu *afu)
 291 {
 292         list_del(&afu->list);
 293         ocxl_context_detach_all(afu);
 294         deactivate_afu(afu);
 295         deconfigure_afu(afu);
 296         ocxl_afu_put(afu); // matches the implicit get in alloc_afu
 297 }
 298
 299 static struct ocxl_fn *alloc_function(void)
 300 {
 301         struct ocxl_fn *fn;
 302
 303         fn = kzalloc(sizeof(struct ocxl_fn), GFP_KERNEL);
 304         if (!fn)
 305                 return NULL;
 306
 307         INIT_LIST_HEAD(&fn->afu_list);
 308         INIT_LIST_HEAD(&fn->pasid_list);
 309         INIT_LIST_HEAD(&fn->actag_list);
 310
 311         return fn;
 312 }
 313
 314 static void free_function(struct ocxl_fn *fn)
 315 {
 316         WARN_ON(!list_empty(&fn->afu_list));
 317         WARN_ON(!list_empty(&fn->pasid_list));
 318         kfree(fn);
 319 }
 320
 321 static void free_function_dev(struct device *dev)
 322 {
 323         struct ocxl_fn *fn = container_of(dev, struct ocxl_fn, dev);
 324
 325         free_function(fn);
 326 }
 327
 328 static int set_function_device(struct ocxl_fn *fn, struct pci_dev *dev)
 329 {
 330         fn->dev.parent = &dev->dev;
 331         fn->dev.release = free_function_dev;
 332         return dev_set_name(&fn->dev, "ocxlfn.%s", dev_name(&dev->dev));
 333 }
 334
 335 static int assign_function_actag(struct ocxl_fn *fn)
 336 {
 337         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 338         u16 base, enabled, supported;
 339         int rc;
 340
 341         rc = ocxl_config_get_actag_info(dev, &base, &enabled, &supported);
 342         if (rc)
 343                 return rc;
 344
 345         fn->actag_base = base;
 346         fn->actag_enabled = enabled;
 347         fn->actag_supported = supported;
 348
 349         ocxl_config_set_actag(dev, fn->config.dvsec_function_pos,
 350                         fn->actag_base, fn->actag_enabled);
 351         dev_dbg(&fn->dev, "actag range starting at %d, enabled %d\n",
 352                 fn->actag_base, fn->actag_enabled);
 353         return 0;
 354 }
 355
 356 static int set_function_pasid(struct ocxl_fn *fn)
 357 {
 358         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 359         int rc, desired_count, max_count;
 360
 361         /* A function may not require any PASID */
 362         if (fn->config.max_pasid_log < 0)
 363                 return 0;
 364
 365         rc = ocxl_config_get_pasid_info(dev, &max_count);
 366         if (rc)
 367                 return rc;
 368
 369         desired_count = 1 << fn->config.max_pasid_log;
 370
 371         if (desired_count > max_count) {
 372                 dev_err(&fn->dev,
 373                         "Function requires more PASIDs than is available (%d vs. %d)\n",
 374                         desired_count, max_count);
 375                 return -ENOSPC;
 376         }
 377
 378         fn->pasid_base = 0;
 379         return 0;
 380 }
 381
 382 static int configure_function(struct ocxl_fn *fn, struct pci_dev *dev)
 383 {
 384         int rc;
 385
 386         rc = pci_enable_device(dev);
 387         if (rc) {
 388                 dev_err(&dev->dev, "pci_enable_device failed: %d\n", rc);
 389                 return rc;
 390         }
 391
 392         /*
 393          * Once it has been confirmed to work on our hardware, we
 394          * should reset the function, to force the adapter to restart
 395          * from scratch.
 396          * A function reset would also reset all its AFUs.
 397          *
 398          * Some hints for implementation:
 399          *
 400          * - there's not status bit to know when the reset is done. We
 401          *   should try reading the config space to know when it's
 402          *   done.
 403          * - probably something like:
 404          *      Reset
 405          *      wait 100ms
 406          *      issue config read
 407          *      allow device up to 1 sec to return success on config
 408          *      read before declaring it broken
 409          *
 410          * Some shared logic on the card (CFG, TLX) won't be reset, so
 411          * there's no guarantee that it will be enough.
 412          */
 413         rc = ocxl_config_read_function(dev, &fn->config);
 414         if (rc)
 415                 return rc;
 416
 417         rc = set_function_device(fn, dev);
 418         if (rc)
 419                 return rc;
 420
 421         rc = assign_function_actag(fn);
 422         if (rc)
 423                 return rc;
 424
 425         rc = set_function_pasid(fn);
 426         if (rc)
 427                 return rc;
 428
 429         rc = ocxl_link_setup(dev, 0, &fn->link);
 430         if (rc)
 431                 return rc;
 432
 433         rc = ocxl_config_set_TL(dev, fn->config.dvsec_tl_pos);
 434         if (rc) {
 435                 ocxl_link_release(dev, fn->link);
 436                 return rc;
 437         }
 438         return 0;
 439 }
 440
 441 static void deconfigure_function(struct ocxl_fn *fn)
 442 {
 443         struct pci_dev *dev = to_pci_dev(fn->dev.parent);
 444
 445         ocxl_link_release(dev, fn->link);
 446         pci_disable_device(dev);
 447 }
 448
 449 static struct ocxl_fn *init_function(struct pci_dev *dev)
 450 {
 451         struct ocxl_fn *fn;
 452         int rc;
 453
 454         fn = alloc_function();
 455         if (!fn)
 456                 return ERR_PTR(-ENOMEM);
 457
 458         rc = configure_function(fn, dev);
 459         if (rc) {
 460                 free_function(fn);
 461                 return ERR_PTR(rc);
 462         }
 463
 464         rc = device_register(&fn->dev);
 465         if (rc) {
 466                 deconfigure_function(fn);
 467                 put_device(&fn->dev);
 468                 return ERR_PTR(rc);
 469         }
 470         return fn;
 471 }
 472
 473 // Device detection & initialisation
 474
 475 struct ocxl_fn *ocxl_function_open(struct pci_dev *dev)
 476 {
 477         int rc, afu_count = 0;
 478         u8 afu;
 479         struct ocxl_fn *fn;
 480
 481         if (!radix_enabled()) {
 482                 dev_err(&dev->dev, "Unsupported memory model (hash)\n");
 483                 return ERR_PTR(-ENODEV);
 484         }
 485
 486         fn = init_function(dev);
 487         if (IS_ERR(fn)) {
 488                 dev_err(&dev->dev, "function init failed: %li\n",
 489                         PTR_ERR(fn));
 490                 return fn;
 491         }
 492
 493         for (afu = 0; afu <= fn->config.max_afu_index; afu++) {
 494                 rc = ocxl_config_check_afu_index(dev, &fn->config, afu);
 495                 if (rc > 0) {
 496                         rc = init_afu(dev, fn, afu);
 497                         if (rc) {
 498                                 dev_err(&dev->dev,
 499                                         "Can't initialize AFU index %d\n", afu);
 500                                 continue;
 501                         }
 502                         afu_count++;
 503                 }
 504         }
 505         dev_info(&dev->dev, "%d AFU(s) configured\n", afu_count);
 506         return fn;
 507 }
 508 EXPORT_SYMBOL_GPL(ocxl_function_open);
 509
 510 struct list_head *ocxl_function_afu_list(struct ocxl_fn *fn)
 511 {
 512         return &fn->afu_list;
 513 }
 514 EXPORT_SYMBOL_GPL(ocxl_function_afu_list);
 515
 516 struct ocxl_afu *ocxl_function_fetch_afu(struct ocxl_fn *fn, u8 afu_idx)
 517 {
 518         struct ocxl_afu *afu;
 519
 520         list_for_each_entry(afu, &fn->afu_list, list) {
 521                 if (afu->config.idx == afu_idx)
 522                         return afu;
 523         }
 524
 525         return NULL;
 526 }
 527 EXPORT_SYMBOL_GPL(ocxl_function_fetch_afu);
 528
 529 const struct ocxl_fn_config *ocxl_function_config(struct ocxl_fn *fn)
 530 {
 531         return &fn->config;
 532 }
 533 EXPORT_SYMBOL_GPL(ocxl_function_config);
 534
 535 void ocxl_function_close(struct ocxl_fn *fn)
 536 {
 537         struct ocxl_afu *afu, *tmp;
 538
 539         list_for_each_entry_safe(afu, tmp, &fn->afu_list, list) {
 540                 remove_afu(afu);
 541         }
 542
 543         deconfigure_function(fn);
 544         device_unregister(&fn->dev);
 545 }
 546 EXPORT_SYMBOL_GPL(ocxl_function_close);
 547
 548 // AFU Metadata
 549
 550 struct ocxl_afu_config *ocxl_afu_config(struct ocxl_afu *afu)
 551 {
 552         return &afu->config;
 553 }
 554 EXPORT_SYMBOL_GPL(ocxl_afu_config);
 555
 556 void ocxl_afu_set_private(struct ocxl_afu *afu, void *private)
 557 {
 558         afu->private = private;
 559 }
 560 EXPORT_SYMBOL_GPL(ocxl_afu_set_private);
 561
 562 void *ocxl_afu_get_private(struct ocxl_afu *afu)
 563 {
 564         if (afu)
 565                 return afu->private;
 566
 567         return NULL;
 568 }
 569 EXPORT_SYMBOL_GPL(ocxl_afu_get_private);