drivers/misc/cxl/native.c

   1 /*
   2  * Copyright 2014 IBM Corp.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public License
   6  * as published by the Free Software Foundation; either version
   7  * 2 of the License, or (at your option) any later version.
   8  */
   9
  10 #include <linux/spinlock.h>
  11 #include <linux/sched.h>
  12 #include <linux/slab.h>
  13 #include <linux/sched.h>
  14 #include <linux/mutex.h>
  15 #include <linux/mm.h>
  16 #include <linux/uaccess.h>
  17 #include <asm/synch.h>
  18 #include <misc/cxl.h>
  19
  20 #include "cxl.h"
  21
  22 static int afu_control(struct cxl_afu *afu, u64 command,
  23                        u64 result, u64 mask, bool enabled)
  24 {
  25         u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  26         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
  27
  28         spin_lock(&afu->afu_cntl_lock);
  29         pr_devel("AFU command starting: %llx\n", command);
  30
  31         cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command);
  32
  33         AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  34         while ((AFU_Cntl & mask) != result) {
  35                 if (time_after_eq(jiffies, timeout)) {
  36                         dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
  37                         spin_unlock(&afu->afu_cntl_lock);
  38                         return -EBUSY;
  39                 }
  40                 pr_devel_ratelimited("AFU control... (0x%.16llx)\n",
  41                                      AFU_Cntl | command);
  42                 cpu_relax();
  43                 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  44         };
  45         pr_devel("AFU command complete: %llx\n", command);
  46         afu->enabled = enabled;
  47         spin_unlock(&afu->afu_cntl_lock);
  48
  49         return 0;
  50 }
  51
  52 static int afu_enable(struct cxl_afu *afu)
  53 {
  54         pr_devel("AFU enable request\n");
  55
  56         return afu_control(afu, CXL_AFU_Cntl_An_E,
  57                            CXL_AFU_Cntl_An_ES_Enabled,
  58                            CXL_AFU_Cntl_An_ES_MASK, true);
  59 }
  60
  61 int cxl_afu_disable(struct cxl_afu *afu)
  62 {
  63         pr_devel("AFU disable request\n");
  64
  65         return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled,
  66                            CXL_AFU_Cntl_An_ES_MASK, false);
  67 }
  68
  69 /* This will disable as well as reset */
  70 int cxl_afu_reset(struct cxl_afu *afu)
  71 {
  72         pr_devel("AFU reset request\n");
  73
  74         return afu_control(afu, CXL_AFU_Cntl_An_RA,
  75                            CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
  76                            CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
  77                            false);
  78 }
  79
  80 static int afu_check_and_enable(struct cxl_afu *afu)
  81 {
  82         if (afu->enabled)
  83                 return 0;
  84         return afu_enable(afu);
  85 }
  86
  87 int cxl_psl_purge(struct cxl_afu *afu)
  88 {
  89         u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
  90         u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
  91         u64 dsisr, dar;
  92         u64 start, end;
  93         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
  94
  95         pr_devel("PSL purge request\n");
  96
  97         if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
  98                 WARN(1, "psl_purge request while AFU not disabled!\n");
  99                 cxl_afu_disable(afu);
 100         }
 101
 102         cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
 103                        PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
 104         start = local_clock();
 105         PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
 106         while ((PSL_CNTL &  CXL_PSL_SCNTL_An_Ps_MASK)
 107                         == CXL_PSL_SCNTL_An_Ps_Pending) {
 108                 if (time_after_eq(jiffies, timeout)) {
 109                         dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
 110                         return -EBUSY;
 111                 }
 112                 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
 113                 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%.16llx  PSL_DSISR: 0x%.16llx\n", PSL_CNTL, dsisr);
 114                 if (dsisr & CXL_PSL_DSISR_TRANS) {
 115                         dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
 116                         dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%.16llx, DAR: 0x%.16llx\n", dsisr, dar);
 117                         cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
 118                 } else if (dsisr) {
 119                         dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%.16llx\n", dsisr);
 120                         cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
 121                 } else {
 122                         cpu_relax();
 123                 }
 124                 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
 125         };
 126         end = local_clock();
 127         pr_devel("PSL purged in %lld ns\n", end - start);
 128
 129         cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
 130                        PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc);
 131         return 0;
 132 }
 133
 134 static int spa_max_procs(int spa_size)
 135 {
 136         /*
 137          * From the CAIA:
 138          *    end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255
 139          * Most of that junk is really just an overly-complicated way of saying
 140          * the last 256 bytes are __aligned(128), so it's really:
 141          *    end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255
 142          * and
 143          *    end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1
 144          * so
 145          *    sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256
 146          * Ignore the alignment (which is safe in this case as long as we are
 147          * careful with our rounding) and solve for n:
 148          */
 149         return ((spa_size / 8) - 96) / 17;
 150 }
 151
 152 static int alloc_spa(struct cxl_afu *afu)
 153 {
 154         u64 spap;
 155
 156         /* Work out how many pages to allocate */
 157         afu->spa_order = 0;
 158         do {
 159                 afu->spa_order++;
 160                 afu->spa_size = (1 << afu->spa_order) * PAGE_SIZE;
 161                 afu->spa_max_procs = spa_max_procs(afu->spa_size);
 162         } while (afu->spa_max_procs < afu->num_procs);
 163
 164         WARN_ON(afu->spa_size > 0x100000); /* Max size supported by the hardware */
 165
 166         if (!(afu->spa = (struct cxl_process_element *)
 167               __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->spa_order))) {
 168                 pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n");
 169                 return -ENOMEM;
 170         }
 171         pr_devel("spa pages: %i afu->spa_max_procs: %i   afu->num_procs: %i\n",
 172                  1<<afu->spa_order, afu->spa_max_procs, afu->num_procs);
 173
 174         afu->sw_command_status = (__be64 *)((char *)afu->spa +
 175                                             ((afu->spa_max_procs + 3) * 128));
 176
 177         spap = virt_to_phys(afu->spa) & CXL_PSL_SPAP_Addr;
 178         spap |= ((afu->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;
 179         spap |= CXL_PSL_SPAP_V;
 180         pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", afu->spa, afu->spa_max_procs, afu->sw_command_status, spap);
 181         cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap);
 182
 183         return 0;
 184 }
 185
 186 static void release_spa(struct cxl_afu *afu)
 187 {
 188         free_pages((unsigned long) afu->spa, afu->spa_order);
 189 }
 190
 191 int cxl_tlb_slb_invalidate(struct cxl *adapter)
 192 {
 193         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
 194
 195         pr_devel("CXL adapter wide TLBIA & SLBIA\n");
 196
 197         cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);
 198
 199         cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
 200         while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
 201                 if (time_after_eq(jiffies, timeout)) {
 202                         dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
 203                         return -EBUSY;
 204                 }
 205                 cpu_relax();
 206         }
 207
 208         cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
 209         while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
 210                 if (time_after_eq(jiffies, timeout)) {
 211                         dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
 212                         return -EBUSY;
 213                 }
 214                 cpu_relax();
 215         }
 216         return 0;
 217 }
 218
 219 int cxl_afu_slbia(struct cxl_afu *afu)
 220 {
 221         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
 222
 223         pr_devel("cxl_afu_slbia issuing SLBIA command\n");
 224         cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL);
 225         while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) {
 226                 if (time_after_eq(jiffies, timeout)) {
 227                         dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n");
 228                         return -EBUSY;
 229                 }
 230                 cpu_relax();
 231         }
 232         return 0;
 233 }
 234
 235 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
 236 {
 237         int rc;
 238
 239         /* 1. Disable SSTP by writing 0 to SSTP1[V] */
 240         cxl_p2n_write(afu, CXL_SSTP1_An, 0);
 241
 242         /* 2. Invalidate all SLB entries */
 243         if ((rc = cxl_afu_slbia(afu)))
 244                 return rc;
 245
 246         /* 3. Set SSTP0_An */
 247         cxl_p2n_write(afu, CXL_SSTP0_An, sstp0);
 248
 249         /* 4. Set SSTP1_An */
 250         cxl_p2n_write(afu, CXL_SSTP1_An, sstp1);
 251
 252         return 0;
 253 }
 254
 255 /* Using per slice version may improve performance here. (ie. SLBIA_An) */
 256 static void slb_invalid(struct cxl_context *ctx)
 257 {
 258         struct cxl *adapter = ctx->afu->adapter;
 259         u64 slbia;
 260
 261         WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex));
 262
 263         cxl_p1_write(adapter, CXL_PSL_LBISEL,
 264                         ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
 265                         be32_to_cpu(ctx->elem->lpid));
 266         cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);
 267
 268         while (1) {
 269                 slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
 270                 if (!(slbia & CXL_TLB_SLB_P))
 271                         break;
 272                 cpu_relax();
 273         }
 274 }
 275
 276 static int do_process_element_cmd(struct cxl_context *ctx,
 277                                   u64 cmd, u64 pe_state)
 278 {
 279         u64 state;
 280         unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
 281
 282         WARN_ON(!ctx->afu->enabled);
 283
 284         ctx->elem->software_state = cpu_to_be32(pe_state);
 285         smp_wmb();
 286         *(ctx->afu->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);
 287         smp_mb();
 288         cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
 289         while (1) {
 290                 if (time_after_eq(jiffies, timeout)) {
 291                         dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n");
 292                         return -EBUSY;
 293                 }
 294                 state = be64_to_cpup(ctx->afu->sw_command_status);
 295                 if (state == ~0ULL) {
 296                         pr_err("cxl: Error adding process element to AFU\n");
 297                         return -1;
 298                 }
 299                 if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK  | CXL_SPA_SW_LINK_MASK)) ==
 300                     (cmd | (cmd >> 16) | ctx->pe))
 301                         break;
 302                 /*
 303                  * The command won't finish in the PSL if there are
 304                  * outstanding DSIs.  Hence we need to yield here in
 305                  * case there are outstanding DSIs that we need to
 306                  * service.  Tuning possiblity: we could wait for a
 307                  * while before sched
 308                  */
 309                 schedule();
 310
 311         }
 312         return 0;
 313 }
 314
 315 static int add_process_element(struct cxl_context *ctx)
 316 {
 317         int rc = 0;
 318
 319         mutex_lock(&ctx->afu->spa_mutex);
 320         pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe);
 321         if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V)))
 322                 ctx->pe_inserted = true;
 323         pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe);
 324         mutex_unlock(&ctx->afu->spa_mutex);
 325         return rc;
 326 }
 327
 328 static int terminate_process_element(struct cxl_context *ctx)
 329 {
 330         int rc = 0;
 331
 332         /* fast path terminate if it's already invalid */
 333         if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
 334                 return rc;
 335
 336         mutex_lock(&ctx->afu->spa_mutex);
 337         pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
 338         rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
 339                                     CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
 340         ctx->elem->software_state = 0;  /* Remove Valid bit */
 341         pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
 342         mutex_unlock(&ctx->afu->spa_mutex);
 343         return rc;
 344 }
 345
 346 static int remove_process_element(struct cxl_context *ctx)
 347 {
 348         int rc = 0;
 349
 350         mutex_lock(&ctx->afu->spa_mutex);
 351         pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
 352         if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0)))
 353                 ctx->pe_inserted = false;
 354         slb_invalid(ctx);
 355         pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
 356         mutex_unlock(&ctx->afu->spa_mutex);
 357
 358         return rc;
 359 }
 360
 361
 362 static void assign_psn_space(struct cxl_context *ctx)
 363 {
 364         if (!ctx->afu->pp_size || ctx->master) {
 365                 ctx->psn_phys = ctx->afu->psn_phys;
 366                 ctx->psn_size = ctx->afu->adapter->ps_size;
 367         } else {
 368                 ctx->psn_phys = ctx->afu->psn_phys +
 369                         (ctx->afu->pp_offset + ctx->afu->pp_size * ctx->pe);
 370                 ctx->psn_size = ctx->afu->pp_size;
 371         }
 372 }
 373
 374 static int activate_afu_directed(struct cxl_afu *afu)
 375 {
 376         int rc;
 377
 378         dev_info(&afu->dev, "Activating AFU directed mode\n");
 379
 380         if (alloc_spa(afu))
 381                 return -ENOMEM;
 382
 383         cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU);
 384         cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
 385         cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);
 386
 387         afu->current_mode = CXL_MODE_DIRECTED;
 388         afu->num_procs = afu->max_procs_virtualised;
 389
 390         if ((rc = cxl_chardev_m_afu_add(afu)))
 391                 return rc;
 392
 393         if ((rc = cxl_sysfs_afu_m_add(afu)))
 394                 goto err;
 395
 396         if ((rc = cxl_chardev_s_afu_add(afu)))
 397                 goto err1;
 398
 399         return 0;
 400 err1:
 401         cxl_sysfs_afu_m_remove(afu);
 402 err:
 403         cxl_chardev_afu_remove(afu);
 404         return rc;
 405 }
 406
 407 #ifdef CONFIG_CPU_LITTLE_ENDIAN
 408 #define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE)
 409 #else
 410 #define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
 411 #endif
 412
 413 static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
 414 {
 415         u64 sr;
 416         int r, result;
 417
 418         assign_psn_space(ctx);
 419
 420         ctx->elem->ctxtime = 0; /* disable */
 421         ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
 422         ctx->elem->haurp = 0; /* disable */
 423         ctx->elem->sdr = cpu_to_be64(mfspr(SPRN_SDR1));
 424
 425         sr = 0;
 426         if (ctx->master)
 427                 sr |= CXL_PSL_SR_An_MP;
 428         if (mfspr(SPRN_LPCR) & LPCR_TC)
 429                 sr |= CXL_PSL_SR_An_TC;
 430         /* HV=0, PR=1, R=1 for userspace
 431          * For kernel contexts: this would need to change
 432          */
 433         sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
 434         set_endian(sr);
 435         sr &= ~(CXL_PSL_SR_An_HV);
 436         if (!test_tsk_thread_flag(current, TIF_32BIT))
 437                 sr |= CXL_PSL_SR_An_SF;
 438         ctx->elem->common.pid = cpu_to_be32(current->pid);
 439         ctx->elem->common.tid = 0;
 440         ctx->elem->sr = cpu_to_be64(sr);
 441
 442         ctx->elem->common.csrp = 0; /* disable */
 443         ctx->elem->common.aurp0 = 0; /* disable */
 444         ctx->elem->common.aurp1 = 0; /* disable */
 445
 446         cxl_prefault(ctx, wed);
 447
 448         ctx->elem->common.sstp0 = cpu_to_be64(ctx->sstp0);
 449         ctx->elem->common.sstp1 = cpu_to_be64(ctx->sstp1);
 450
 451         for (r = 0; r < CXL_IRQ_RANGES; r++) {
 452                 ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
 453                 ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
 454         }
 455
 456         ctx->elem->common.amr = cpu_to_be64(amr);
 457         ctx->elem->common.wed = cpu_to_be64(wed);
 458
 459         /* first guy needs to enable */
 460         if ((result = afu_check_and_enable(ctx->afu)))
 461                 return result;
 462
 463         add_process_element(ctx);
 464
 465         return 0;
 466 }
 467
 468 static int deactivate_afu_directed(struct cxl_afu *afu)
 469 {
 470         dev_info(&afu->dev, "Deactivating AFU directed mode\n");
 471
 472         afu->current_mode = 0;
 473         afu->num_procs = 0;
 474
 475         cxl_sysfs_afu_m_remove(afu);
 476         cxl_chardev_afu_remove(afu);
 477
 478         cxl_afu_reset(afu);
 479         cxl_afu_disable(afu);
 480         cxl_psl_purge(afu);
 481
 482         release_spa(afu);
 483
 484         return 0;
 485 }
 486
 487 static int activate_dedicated_process(struct cxl_afu *afu)
 488 {
 489         dev_info(&afu->dev, "Activating dedicated process mode\n");
 490
 491         cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
 492
 493         cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */
 494         cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0);    /* disable */
 495         cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
 496         cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID));
 497         cxl_p1n_write(afu, CXL_HAURP_An, 0);       /* disable */
 498         cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1));
 499
 500         cxl_p2n_write(afu, CXL_CSRP_An, 0);        /* disable */
 501         cxl_p2n_write(afu, CXL_AURP0_An, 0);       /* disable */
 502         cxl_p2n_write(afu, CXL_AURP1_An, 0);       /* disable */
 503
 504         afu->current_mode = CXL_MODE_DEDICATED;
 505         afu->num_procs = 1;
 506
 507         return cxl_chardev_d_afu_add(afu);
 508 }
 509
 510 static int attach_dedicated(struct cxl_context *ctx, u64 wed, u64 amr)
 511 {
 512         struct cxl_afu *afu = ctx->afu;
 513         u64 sr;
 514         int rc;
 515
 516         sr = 0;
 517         set_endian(sr);
 518         if (ctx->master)
 519                 sr |= CXL_PSL_SR_An_MP;
 520         if (mfspr(SPRN_LPCR) & LPCR_TC)
 521                 sr |= CXL_PSL_SR_An_TC;
 522         sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
 523         if (!test_tsk_thread_flag(current, TIF_32BIT))
 524                 sr |= CXL_PSL_SR_An_SF;
 525         cxl_p2n_write(afu, CXL_PSL_PID_TID_An, (u64)current->pid << 32);
 526         cxl_p1n_write(afu, CXL_PSL_SR_An, sr);
 527
 528         if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1)))
 529                 return rc;
 530
 531         cxl_prefault(ctx, wed);
 532
 533         cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An,
 534                        (((u64)ctx->irqs.offset[0] & 0xffff) << 48) |
 535                        (((u64)ctx->irqs.offset[1] & 0xffff) << 32) |
 536                        (((u64)ctx->irqs.offset[2] & 0xffff) << 16) |
 537                         ((u64)ctx->irqs.offset[3] & 0xffff));
 538         cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64)
 539                        (((u64)ctx->irqs.range[0] & 0xffff) << 48) |
 540                        (((u64)ctx->irqs.range[1] & 0xffff) << 32) |
 541                        (((u64)ctx->irqs.range[2] & 0xffff) << 16) |
 542                         ((u64)ctx->irqs.range[3] & 0xffff));
 543
 544         cxl_p2n_write(afu, CXL_PSL_AMR_An, amr);
 545
 546         /* master only context for dedicated */
 547         assign_psn_space(ctx);
 548
 549         if ((rc = cxl_afu_reset(afu)))
 550                 return rc;
 551
 552         cxl_p2n_write(afu, CXL_PSL_WED_An, wed);
 553
 554         return afu_enable(afu);
 555 }
 556
 557 static int deactivate_dedicated_process(struct cxl_afu *afu)
 558 {
 559         dev_info(&afu->dev, "Deactivating dedicated process mode\n");
 560
 561         afu->current_mode = 0;
 562         afu->num_procs = 0;
 563
 564         cxl_chardev_afu_remove(afu);
 565
 566         return 0;
 567 }
 568
 569 int _cxl_afu_deactivate_mode(struct cxl_afu *afu, int mode)
 570 {
 571         if (mode == CXL_MODE_DIRECTED)
 572                 return deactivate_afu_directed(afu);
 573         if (mode == CXL_MODE_DEDICATED)
 574                 return deactivate_dedicated_process(afu);
 575         return 0;
 576 }
 577
 578 int cxl_afu_deactivate_mode(struct cxl_afu *afu)
 579 {
 580         return _cxl_afu_deactivate_mode(afu, afu->current_mode);
 581 }
 582
 583 int cxl_afu_activate_mode(struct cxl_afu *afu, int mode)
 584 {
 585         if (!mode)
 586                 return 0;
 587         if (!(mode & afu->modes_supported))
 588                 return -EINVAL;
 589
 590         if (mode == CXL_MODE_DIRECTED)
 591                 return activate_afu_directed(afu);
 592         if (mode == CXL_MODE_DEDICATED)
 593                 return activate_dedicated_process(afu);
 594
 595         return -EINVAL;
 596 }
 597
 598 int cxl_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
 599 {
 600         ctx->kernel = kernel;
 601         if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
 602                 return attach_afu_directed(ctx, wed, amr);
 603
 604         if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
 605                 return attach_dedicated(ctx, wed, amr);
 606
 607         return -EINVAL;
 608 }
 609
 610 static inline int detach_process_native_dedicated(struct cxl_context *ctx)
 611 {
 612         cxl_afu_reset(ctx->afu);
 613         cxl_afu_disable(ctx->afu);
 614         cxl_psl_purge(ctx->afu);
 615         return 0;
 616 }
 617
 618 static inline int detach_process_native_afu_directed(struct cxl_context *ctx)
 619 {
 620         if (!ctx->pe_inserted)
 621                 return 0;
 622         if (terminate_process_element(ctx))
 623                 return -1;
 624         if (remove_process_element(ctx))
 625                 return -1;
 626
 627         return 0;
 628 }
 629
 630 int cxl_detach_process(struct cxl_context *ctx)
 631 {
 632         if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
 633                 return detach_process_native_dedicated(ctx);
 634
 635         return detach_process_native_afu_directed(ctx);
 636 }
 637
 638 int cxl_get_irq(struct cxl_afu *afu, struct cxl_irq_info *info)
 639 {
 640         u64 pidtid;
 641
 642         info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
 643         info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
 644         info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An);
 645         pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An);
 646         info->pid = pidtid >> 32;
 647         info->tid = pidtid & 0xffffffff;
 648         info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An);
 649         info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
 650
 651         return 0;
 652 }
 653
 654 static void recover_psl_err(struct cxl_afu *afu, u64 errstat)
 655 {
 656         u64 dsisr;
 657
 658         pr_devel("RECOVERING FROM PSL ERROR... (0x%.16llx)\n", errstat);
 659
 660         /* Clear PSL_DSISR[PE] */
 661         dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
 662         cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE);
 663
 664         /* Write 1s to clear error status bits */
 665         cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat);
 666 }
 667
 668 int cxl_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
 669 {
 670         if (tfc)
 671                 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc);
 672         if (psl_reset_mask)
 673                 recover_psl_err(ctx->afu, psl_reset_mask);
 674
 675         return 0;
 676 }
 677
 678 int cxl_check_error(struct cxl_afu *afu)
 679 {
 680         return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL);
 681 }