drivers/crypto/nx/nx-842-powernv.c

   1 /*
   2  * Driver for IBM PowerNV 842 compression accelerator
   3  *
   4  * Copyright (C) 2015 Dan Streetman, IBM Corp
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  */
  16
  17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  18
  19 #include "nx-842.h"
  20
  21 #include <linux/timer.h>
  22
  23 #include <asm/prom.h>
  24 #include <asm/icswx.h>
  25 #include <asm/vas.h>
  26 #include <asm/reg.h>
  27
  28 MODULE_LICENSE("GPL");
  29 MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>");
  30 MODULE_DESCRIPTION("842 H/W Compression driver for IBM PowerNV processors");
  31 MODULE_ALIAS_CRYPTO("842");
  32 MODULE_ALIAS_CRYPTO("842-nx");
  33
  34 #define WORKMEM_ALIGN   (CRB_ALIGN)
  35 #define CSB_WAIT_MAX    (5000) /* ms */
  36 #define VAS_RETRIES     (10)
  37 /* # of requests allowed per RxFIFO at a time. 0 for unlimited */
  38 #define MAX_CREDITS_PER_RXFIFO  (1024)
  39
  40 struct nx842_workmem {
  41         /* Below fields must be properly aligned */
  42         struct coprocessor_request_block crb; /* CRB_ALIGN align */
  43         struct data_descriptor_entry ddl_in[DDL_LEN_MAX]; /* DDE_ALIGN align */
  44         struct data_descriptor_entry ddl_out[DDL_LEN_MAX]; /* DDE_ALIGN align */
  45         /* Above fields must be properly aligned */
  46
  47         ktime_t start;
  48
  49         char padding[WORKMEM_ALIGN]; /* unused, to allow alignment */
  50 } __packed __aligned(WORKMEM_ALIGN);
  51
  52 struct nx842_coproc {
  53         unsigned int chip_id;
  54         unsigned int ct;
  55         unsigned int ci;        /* Coprocessor instance, used with icswx */
  56         struct {
  57                 struct vas_window *rxwin;
  58                 int id;
  59         } vas;
  60         struct list_head list;
  61 };
  62
  63 /*
  64  * Send the request to NX engine on the chip for the corresponding CPU
  65  * where the process is executing. Use with VAS function.
  66  */
  67 static DEFINE_PER_CPU(struct vas_window *, cpu_txwin);
  68
  69 /* no cpu hotplug on powernv, so this list never changes after init */
  70 static LIST_HEAD(nx842_coprocs);
  71 static unsigned int nx842_ct;   /* used in icswx function */
  72
  73 static int (*nx842_powernv_exec)(const unsigned char *in,
  74                                 unsigned int inlen, unsigned char *out,
  75                                 unsigned int *outlenp, void *workmem, int fc);
  76
  77 /**
  78  * setup_indirect_dde - Setup an indirect DDE
  79  *
  80  * The DDE is setup with the the DDE count, byte count, and address of
  81  * first direct DDE in the list.
  82  */
  83 static void setup_indirect_dde(struct data_descriptor_entry *dde,
  84                                struct data_descriptor_entry *ddl,
  85                                unsigned int dde_count, unsigned int byte_count)
  86 {
  87         dde->flags = 0;
  88         dde->count = dde_count;
  89         dde->index = 0;
  90         dde->length = cpu_to_be32(byte_count);
  91         dde->address = cpu_to_be64(nx842_get_pa(ddl));
  92 }
  93
  94 /**
  95  * setup_direct_dde - Setup single DDE from buffer
  96  *
  97  * The DDE is setup with the buffer and length.  The buffer must be properly
  98  * aligned.  The used length is returned.
  99  * Returns:
 100  *   N    Successfully set up DDE with N bytes
 101  */
 102 static unsigned int setup_direct_dde(struct data_descriptor_entry *dde,
 103                                      unsigned long pa, unsigned int len)
 104 {
 105         unsigned int l = min_t(unsigned int, len, LEN_ON_PAGE(pa));
 106
 107         dde->flags = 0;
 108         dde->count = 0;
 109         dde->index = 0;
 110         dde->length = cpu_to_be32(l);
 111         dde->address = cpu_to_be64(pa);
 112
 113         return l;
 114 }
 115
 116 /**
 117  * setup_ddl - Setup DDL from buffer
 118  *
 119  * Returns:
 120  *   0          Successfully set up DDL
 121  */
 122 static int setup_ddl(struct data_descriptor_entry *dde,
 123                      struct data_descriptor_entry *ddl,
 124                      unsigned char *buf, unsigned int len,
 125                      bool in)
 126 {
 127         unsigned long pa = nx842_get_pa(buf);
 128         int i, ret, total_len = len;
 129
 130         if (!IS_ALIGNED(pa, DDE_BUFFER_ALIGN)) {
 131                 pr_debug("%s buffer pa 0x%lx not 0x%x-byte aligned\n",
 132                          in ? "input" : "output", pa, DDE_BUFFER_ALIGN);
 133                 return -EINVAL;
 134         }
 135
 136         /* only need to check last mult; since buffer must be
 137          * DDE_BUFFER_ALIGN aligned, and that is a multiple of
 138          * DDE_BUFFER_SIZE_MULT, and pre-last page DDE buffers
 139          * are guaranteed a multiple of DDE_BUFFER_SIZE_MULT.
 140          */
 141         if (len % DDE_BUFFER_LAST_MULT) {
 142                 pr_debug("%s buffer len 0x%x not a multiple of 0x%x\n",
 143                          in ? "input" : "output", len, DDE_BUFFER_LAST_MULT);
 144                 if (in)
 145                         return -EINVAL;
 146                 len = round_down(len, DDE_BUFFER_LAST_MULT);
 147         }
 148
 149         /* use a single direct DDE */
 150         if (len <= LEN_ON_PAGE(pa)) {
 151                 ret = setup_direct_dde(dde, pa, len);
 152                 WARN_ON(ret < len);
 153                 return 0;
 154         }
 155
 156         /* use the DDL */
 157         for (i = 0; i < DDL_LEN_MAX && len > 0; i++) {
 158                 ret = setup_direct_dde(&ddl[i], pa, len);
 159                 buf += ret;
 160                 len -= ret;
 161                 pa = nx842_get_pa(buf);
 162         }
 163
 164         if (len > 0) {
 165                 pr_debug("0x%x total %s bytes 0x%x too many for DDL.\n",
 166                          total_len, in ? "input" : "output", len);
 167                 if (in)
 168                         return -EMSGSIZE;
 169                 total_len -= len;
 170         }
 171         setup_indirect_dde(dde, ddl, i, total_len);
 172
 173         return 0;
 174 }
 175
 176 #define CSB_ERR(csb, msg, ...)                                  \
 177         pr_err("ERROR: " msg " : %02x %02x %02x %02x %08x\n",   \
 178                ##__VA_ARGS__, (csb)->flags,                     \
 179                (csb)->cs, (csb)->cc, (csb)->ce,                 \
 180                be32_to_cpu((csb)->count))
 181
 182 #define CSB_ERR_ADDR(csb, msg, ...)                             \
 183         CSB_ERR(csb, msg " at %lx", ##__VA_ARGS__,              \
 184                 (unsigned long)be64_to_cpu((csb)->address))
 185
 186 /**
 187  * wait_for_csb
 188  */
 189 static int wait_for_csb(struct nx842_workmem *wmem,
 190                         struct coprocessor_status_block *csb)
 191 {
 192         ktime_t start = wmem->start, now = ktime_get();
 193         ktime_t timeout = ktime_add_ms(start, CSB_WAIT_MAX);
 194
 195         while (!(READ_ONCE(csb->flags) & CSB_V)) {
 196                 cpu_relax();
 197                 now = ktime_get();
 198                 if (ktime_after(now, timeout))
 199                         break;
 200         }
 201
 202         /* hw has updated csb and output buffer */
 203         barrier();
 204
 205         /* check CSB flags */
 206         if (!(csb->flags & CSB_V)) {
 207                 CSB_ERR(csb, "CSB still not valid after %ld us, giving up",
 208                         (long)ktime_us_delta(now, start));
 209                 return -ETIMEDOUT;
 210         }
 211         if (csb->flags & CSB_F) {
 212                 CSB_ERR(csb, "Invalid CSB format");
 213                 return -EPROTO;
 214         }
 215         if (csb->flags & CSB_CH) {
 216                 CSB_ERR(csb, "Invalid CSB chaining state");
 217                 return -EPROTO;
 218         }
 219
 220         /* verify CSB completion sequence is 0 */
 221         if (csb->cs) {
 222                 CSB_ERR(csb, "Invalid CSB completion sequence");
 223                 return -EPROTO;
 224         }
 225
 226         /* check CSB Completion Code */
 227         switch (csb->cc) {
 228         /* no error */
 229         case CSB_CC_SUCCESS:
 230                 break;
 231         case CSB_CC_TPBC_GT_SPBC:
 232                 /* not an error, but the compressed data is
 233                  * larger than the uncompressed data :(
 234                  */
 235                 break;
 236
 237         /* input data errors */
 238         case CSB_CC_OPERAND_OVERLAP:
 239                 /* input and output buffers overlap */
 240                 CSB_ERR(csb, "Operand Overlap error");
 241                 return -EINVAL;
 242         case CSB_CC_INVALID_OPERAND:
 243                 CSB_ERR(csb, "Invalid operand");
 244                 return -EINVAL;
 245         case CSB_CC_NOSPC:
 246                 /* output buffer too small */
 247                 return -ENOSPC;
 248         case CSB_CC_ABORT:
 249                 CSB_ERR(csb, "Function aborted");
 250                 return -EINTR;
 251         case CSB_CC_CRC_MISMATCH:
 252                 CSB_ERR(csb, "CRC mismatch");
 253                 return -EINVAL;
 254         case CSB_CC_TEMPL_INVALID:
 255                 CSB_ERR(csb, "Compressed data template invalid");
 256                 return -EINVAL;
 257         case CSB_CC_TEMPL_OVERFLOW:
 258                 CSB_ERR(csb, "Compressed data template shows data past end");
 259                 return -EINVAL;
 260         case CSB_CC_EXCEED_BYTE_COUNT:  /* P9 or later */
 261                 /*
 262                  * DDE byte count exceeds the limit specified in Maximum
 263                  * byte count register.
 264                  */
 265                 CSB_ERR(csb, "DDE byte count exceeds the limit");
 266                 return -EINVAL;
 267
 268         /* these should not happen */
 269         case CSB_CC_INVALID_ALIGN:
 270                 /* setup_ddl should have detected this */
 271                 CSB_ERR_ADDR(csb, "Invalid alignment");
 272                 return -EINVAL;
 273         case CSB_CC_DATA_LENGTH:
 274                 /* setup_ddl should have detected this */
 275                 CSB_ERR(csb, "Invalid data length");
 276                 return -EINVAL;
 277         case CSB_CC_WR_TRANSLATION:
 278         case CSB_CC_TRANSLATION:
 279         case CSB_CC_TRANSLATION_DUP1:
 280         case CSB_CC_TRANSLATION_DUP2:
 281         case CSB_CC_TRANSLATION_DUP3:
 282         case CSB_CC_TRANSLATION_DUP4:
 283         case CSB_CC_TRANSLATION_DUP5:
 284         case CSB_CC_TRANSLATION_DUP6:
 285                 /* should not happen, we use physical addrs */
 286                 CSB_ERR_ADDR(csb, "Translation error");
 287                 return -EPROTO;
 288         case CSB_CC_WR_PROTECTION:
 289         case CSB_CC_PROTECTION:
 290         case CSB_CC_PROTECTION_DUP1:
 291         case CSB_CC_PROTECTION_DUP2:
 292         case CSB_CC_PROTECTION_DUP3:
 293         case CSB_CC_PROTECTION_DUP4:
 294         case CSB_CC_PROTECTION_DUP5:
 295         case CSB_CC_PROTECTION_DUP6:
 296                 /* should not happen, we use physical addrs */
 297                 CSB_ERR_ADDR(csb, "Protection error");
 298                 return -EPROTO;
 299         case CSB_CC_PRIVILEGE:
 300                 /* shouldn't happen, we're in HYP mode */
 301                 CSB_ERR(csb, "Insufficient Privilege error");
 302                 return -EPROTO;
 303         case CSB_CC_EXCESSIVE_DDE:
 304                 /* shouldn't happen, setup_ddl doesn't use many dde's */
 305                 CSB_ERR(csb, "Too many DDEs in DDL");
 306                 return -EINVAL;
 307         case CSB_CC_TRANSPORT:
 308         case CSB_CC_INVALID_CRB:        /* P9 or later */
 309                 /* shouldn't happen, we setup CRB correctly */
 310                 CSB_ERR(csb, "Invalid CRB");
 311                 return -EINVAL;
 312         case CSB_CC_INVALID_DDE:        /* P9 or later */
 313                 /*
 314                  * shouldn't happen, setup_direct/indirect_dde creates
 315                  * DDE right
 316                  */
 317                 CSB_ERR(csb, "Invalid DDE");
 318                 return -EINVAL;
 319         case CSB_CC_SEGMENTED_DDL:
 320                 /* shouldn't happen, setup_ddl creates DDL right */
 321                 CSB_ERR(csb, "Segmented DDL error");
 322                 return -EINVAL;
 323         case CSB_CC_DDE_OVERFLOW:
 324                 /* shouldn't happen, setup_ddl creates DDL right */
 325                 CSB_ERR(csb, "DDE overflow error");
 326                 return -EINVAL;
 327         case CSB_CC_SESSION:
 328                 /* should not happen with ICSWX */
 329                 CSB_ERR(csb, "Session violation error");
 330                 return -EPROTO;
 331         case CSB_CC_CHAIN:
 332                 /* should not happen, we don't use chained CRBs */
 333                 CSB_ERR(csb, "Chained CRB error");
 334                 return -EPROTO;
 335         case CSB_CC_SEQUENCE:
 336                 /* should not happen, we don't use chained CRBs */
 337                 CSB_ERR(csb, "CRB seqeunce number error");
 338                 return -EPROTO;
 339         case CSB_CC_UNKNOWN_CODE:
 340                 CSB_ERR(csb, "Unknown subfunction code");
 341                 return -EPROTO;
 342
 343         /* hardware errors */
 344         case CSB_CC_RD_EXTERNAL:
 345         case CSB_CC_RD_EXTERNAL_DUP1:
 346         case CSB_CC_RD_EXTERNAL_DUP2:
 347         case CSB_CC_RD_EXTERNAL_DUP3:
 348                 CSB_ERR_ADDR(csb, "Read error outside coprocessor");
 349                 return -EPROTO;
 350         case CSB_CC_WR_EXTERNAL:
 351                 CSB_ERR_ADDR(csb, "Write error outside coprocessor");
 352                 return -EPROTO;
 353         case CSB_CC_INTERNAL:
 354                 CSB_ERR(csb, "Internal error in coprocessor");
 355                 return -EPROTO;
 356         case CSB_CC_PROVISION:
 357                 CSB_ERR(csb, "Storage provision error");
 358                 return -EPROTO;
 359         case CSB_CC_HW:
 360                 CSB_ERR(csb, "Correctable hardware error");
 361                 return -EPROTO;
 362         case CSB_CC_HW_EXPIRED_TIMER:   /* P9 or later */
 363                 CSB_ERR(csb, "Job did not finish within allowed time");
 364                 return -EPROTO;
 365
 366         default:
 367                 CSB_ERR(csb, "Invalid CC %d", csb->cc);
 368                 return -EPROTO;
 369         }
 370
 371         /* check Completion Extension state */
 372         if (csb->ce & CSB_CE_TERMINATION) {
 373                 CSB_ERR(csb, "CSB request was terminated");
 374                 return -EPROTO;
 375         }
 376         if (csb->ce & CSB_CE_INCOMPLETE) {
 377                 CSB_ERR(csb, "CSB request not complete");
 378                 return -EPROTO;
 379         }
 380         if (!(csb->ce & CSB_CE_TPBC)) {
 381                 CSB_ERR(csb, "TPBC not provided, unknown target length");
 382                 return -EPROTO;
 383         }
 384
 385         /* successful completion */
 386         pr_debug_ratelimited("Processed %u bytes in %lu us\n",
 387                              be32_to_cpu(csb->count),
 388                              (unsigned long)ktime_us_delta(now, start));
 389
 390         return 0;
 391 }
 392
 393 static int nx842_config_crb(const unsigned char *in, unsigned int inlen,
 394                         unsigned char *out, unsigned int outlen,
 395                         struct nx842_workmem *wmem)
 396 {
 397         struct coprocessor_request_block *crb;
 398         struct coprocessor_status_block *csb;
 399         u64 csb_addr;
 400         int ret;
 401
 402         crb = &wmem->crb;
 403         csb = &crb->csb;
 404
 405         /* Clear any previous values */
 406         memset(crb, 0, sizeof(*crb));
 407
 408         /* set up DDLs */
 409         ret = setup_ddl(&crb->source, wmem->ddl_in,
 410                         (unsigned char *)in, inlen, true);
 411         if (ret)
 412                 return ret;
 413
 414         ret = setup_ddl(&crb->target, wmem->ddl_out,
 415                         out, outlen, false);
 416         if (ret)
 417                 return ret;
 418
 419         /* set up CRB's CSB addr */
 420         csb_addr = nx842_get_pa(csb) & CRB_CSB_ADDRESS;
 421         csb_addr |= CRB_CSB_AT; /* Addrs are phys */
 422         crb->csb_addr = cpu_to_be64(csb_addr);
 423
 424         return 0;
 425 }
 426
 427 /**
 428  * nx842_exec_icswx - compress/decompress data using the 842 algorithm
 429  *
 430  * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
 431  * This compresses or decompresses the provided input buffer into the provided
 432  * output buffer.
 433  *
 434  * Upon return from this function @outlen contains the length of the
 435  * output data.  If there is an error then @outlen will be 0 and an
 436  * error will be specified by the return code from this function.
 437  *
 438  * The @workmem buffer should only be used by one function call at a time.
 439  *
 440  * @in: input buffer pointer
 441  * @inlen: input buffer size
 442  * @out: output buffer pointer
 443  * @outlenp: output buffer size pointer
 444  * @workmem: working memory buffer pointer, size determined by
 445  *           nx842_powernv_driver.workmem_size
 446  * @fc: function code, see CCW Function Codes in nx-842.h
 447  *
 448  * Returns:
 449  *   0          Success, output of length @outlenp stored in the buffer at @out
 450  *   -ENODEV    Hardware unavailable
 451  *   -ENOSPC    Output buffer is to small
 452  *   -EMSGSIZE  Input buffer too large
 453  *   -EINVAL    buffer constraints do not fix nx842_constraints
 454  *   -EPROTO    hardware error during operation
 455  *   -ETIMEDOUT hardware did not complete operation in reasonable time
 456  *   -EINTR     operation was aborted
 457  */
 458 static int nx842_exec_icswx(const unsigned char *in, unsigned int inlen,
 459                                   unsigned char *out, unsigned int *outlenp,
 460                                   void *workmem, int fc)
 461 {
 462         struct coprocessor_request_block *crb;
 463         struct coprocessor_status_block *csb;
 464         struct nx842_workmem *wmem;
 465         int ret;
 466         u32 ccw;
 467         unsigned int outlen = *outlenp;
 468
 469         wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
 470
 471         *outlenp = 0;
 472
 473         /* shoudn't happen, we don't load without a coproc */
 474         if (!nx842_ct) {
 475                 pr_err_ratelimited("coprocessor CT is 0");
 476                 return -ENODEV;
 477         }
 478
 479         ret = nx842_config_crb(in, inlen, out, outlen, wmem);
 480         if (ret)
 481                 return ret;
 482
 483         crb = &wmem->crb;
 484         csb = &crb->csb;
 485
 486         /* set up CCW */
 487         ccw = 0;
 488         ccw = SET_FIELD(CCW_CT, ccw, nx842_ct);
 489         ccw = SET_FIELD(CCW_CI_842, ccw, 0); /* use 0 for hw auto-selection */
 490         ccw = SET_FIELD(CCW_FC_842, ccw, fc);
 491
 492         wmem->start = ktime_get();
 493
 494         /* do ICSWX */
 495         ret = icswx(cpu_to_be32(ccw), crb);
 496
 497         pr_debug_ratelimited("icswx CR %x ccw %x crb->ccw %x\n", ret,
 498                              (unsigned int)ccw,
 499                              (unsigned int)be32_to_cpu(crb->ccw));
 500
 501         /*
 502          * NX842 coprocessor sets 3rd bit in CR register with XER[S0].
 503          * XER[S0] is the integer summary overflow bit which is nothing
 504          * to do NX. Since this bit can be set with other return values,
 505          * mask this bit.
 506          */
 507         ret &= ~ICSWX_XERS0;
 508
 509         switch (ret) {
 510         case ICSWX_INITIATED:
 511                 ret = wait_for_csb(wmem, csb);
 512                 break;
 513         case ICSWX_BUSY:
 514                 pr_debug_ratelimited("842 Coprocessor busy\n");
 515                 ret = -EBUSY;
 516                 break;
 517         case ICSWX_REJECTED:
 518                 pr_err_ratelimited("ICSWX rejected\n");
 519                 ret = -EPROTO;
 520                 break;
 521         }
 522
 523         if (!ret)
 524                 *outlenp = be32_to_cpu(csb->count);
 525
 526         return ret;
 527 }
 528
 529 /**
 530  * nx842_exec_vas - compress/decompress data using the 842 algorithm
 531  *
 532  * (De)compression provided by the NX842 coprocessor on IBM PowerNV systems.
 533  * This compresses or decompresses the provided input buffer into the provided
 534  * output buffer.
 535  *
 536  * Upon return from this function @outlen contains the length of the
 537  * output data.  If there is an error then @outlen will be 0 and an
 538  * error will be specified by the return code from this function.
 539  *
 540  * The @workmem buffer should only be used by one function call at a time.
 541  *
 542  * @in: input buffer pointer
 543  * @inlen: input buffer size
 544  * @out: output buffer pointer
 545  * @outlenp: output buffer size pointer
 546  * @workmem: working memory buffer pointer, size determined by
 547  *           nx842_powernv_driver.workmem_size
 548  * @fc: function code, see CCW Function Codes in nx-842.h
 549  *
 550  * Returns:
 551  *   0          Success, output of length @outlenp stored in the buffer
 552  *              at @out
 553  *   -ENODEV    Hardware unavailable
 554  *   -ENOSPC    Output buffer is to small
 555  *   -EMSGSIZE  Input buffer too large
 556  *   -EINVAL    buffer constraints do not fix nx842_constraints
 557  *   -EPROTO    hardware error during operation
 558  *   -ETIMEDOUT hardware did not complete operation in reasonable time
 559  *   -EINTR     operation was aborted
 560  */
 561 static int nx842_exec_vas(const unsigned char *in, unsigned int inlen,
 562                                   unsigned char *out, unsigned int *outlenp,
 563                                   void *workmem, int fc)
 564 {
 565         struct coprocessor_request_block *crb;
 566         struct coprocessor_status_block *csb;
 567         struct nx842_workmem *wmem;
 568         struct vas_window *txwin;
 569         int ret, i = 0;
 570         u32 ccw;
 571         unsigned int outlen = *outlenp;
 572
 573         wmem = PTR_ALIGN(workmem, WORKMEM_ALIGN);
 574
 575         *outlenp = 0;
 576
 577         crb = &wmem->crb;
 578         csb = &crb->csb;
 579
 580         ret = nx842_config_crb(in, inlen, out, outlen, wmem);
 581         if (ret)
 582                 return ret;
 583
 584         ccw = 0;
 585         ccw = SET_FIELD(CCW_FC_842, ccw, fc);
 586         crb->ccw = cpu_to_be32(ccw);
 587
 588         do {
 589                 wmem->start = ktime_get();
 590                 preempt_disable();
 591                 txwin = this_cpu_read(cpu_txwin);
 592
 593                 /*
 594                  * VAS copy CRB into L2 cache. Refer <asm/vas.h>.
 595                  * @crb and @offset.
 596                  */
 597                 vas_copy_crb(crb, 0);
 598
 599                 /*
 600                  * VAS paste previously copied CRB to NX.
 601                  * @txwin, @offset and @last (must be true).
 602                  */
 603                 ret = vas_paste_crb(txwin, 0, 1);
 604                 preempt_enable();
 605                 /*
 606                  * Retry copy/paste function for VAS failures.
 607                  */
 608         } while (ret && (i++ < VAS_RETRIES));
 609
 610         if (ret) {
 611                 pr_err_ratelimited("VAS copy/paste failed\n");
 612                 return ret;
 613         }
 614
 615         ret = wait_for_csb(wmem, csb);
 616         if (!ret)
 617                 *outlenp = be32_to_cpu(csb->count);
 618
 619         return ret;
 620 }
 621
 622 /**
 623  * nx842_powernv_compress - Compress data using the 842 algorithm
 624  *
 625  * Compression provided by the NX842 coprocessor on IBM PowerNV systems.
 626  * The input buffer is compressed and the result is stored in the
 627  * provided output buffer.
 628  *
 629  * Upon return from this function @outlen contains the length of the
 630  * compressed data.  If there is an error then @outlen will be 0 and an
 631  * error will be specified by the return code from this function.
 632  *
 633  * @in: input buffer pointer
 634  * @inlen: input buffer size
 635  * @out: output buffer pointer
 636  * @outlenp: output buffer size pointer
 637  * @workmem: working memory buffer pointer, size determined by
 638  *           nx842_powernv_driver.workmem_size
 639  *
 640  * Returns: see @nx842_powernv_exec()
 641  */
 642 static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen,
 643                                   unsigned char *out, unsigned int *outlenp,
 644                                   void *wmem)
 645 {
 646         return nx842_powernv_exec(in, inlen, out, outlenp,
 647                                       wmem, CCW_FC_842_COMP_CRC);
 648 }
 649
 650 /**
 651  * nx842_powernv_decompress - Decompress data using the 842 algorithm
 652  *
 653  * Decompression provided by the NX842 coprocessor on IBM PowerNV systems.
 654  * The input buffer is decompressed and the result is stored in the
 655  * provided output buffer.
 656  *
 657  * Upon return from this function @outlen contains the length of the
 658  * decompressed data.  If there is an error then @outlen will be 0 and an
 659  * error will be specified by the return code from this function.
 660  *
 661  * @in: input buffer pointer
 662  * @inlen: input buffer size
 663  * @out: output buffer pointer
 664  * @outlenp: output buffer size pointer
 665  * @workmem: working memory buffer pointer, size determined by
 666  *           nx842_powernv_driver.workmem_size
 667  *
 668  * Returns: see @nx842_powernv_exec()
 669  */
 670 static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen,
 671                                     unsigned char *out, unsigned int *outlenp,
 672                                     void *wmem)
 673 {
 674         return nx842_powernv_exec(in, inlen, out, outlenp,
 675                                       wmem, CCW_FC_842_DECOMP_CRC);
 676 }
 677
 678 static inline void nx842_add_coprocs_list(struct nx842_coproc *coproc,
 679                                         int chipid)
 680 {
 681         coproc->chip_id = chipid;
 682         INIT_LIST_HEAD(&coproc->list);
 683         list_add(&coproc->list, &nx842_coprocs);
 684 }
 685
 686 static struct vas_window *nx842_alloc_txwin(struct nx842_coproc *coproc)
 687 {
 688         struct vas_window *txwin = NULL;
 689         struct vas_tx_win_attr txattr;
 690
 691         /*
 692          * Kernel requests will be high priority. So open send
 693          * windows only for high priority RxFIFO entries.
 694          */
 695         vas_init_tx_win_attr(&txattr, coproc->ct);
 696         txattr.lpid = 0;        /* lpid is 0 for kernel requests */
 697         txattr.pid = 0;         /* pid is 0 for kernel requests */
 698
 699         /*
 700          * Open a VAS send window which is used to send request to NX.
 701          */
 702         txwin = vas_tx_win_open(coproc->vas.id, coproc->ct, &txattr);
 703         if (IS_ERR(txwin))
 704                 pr_err("ibm,nx-842: Can not open TX window: %ld\n",
 705                                 PTR_ERR(txwin));
 706
 707         return txwin;
 708 }
 709
 710 /*
 711  * Identify chip ID for each CPU, open send wndow for the corresponding NX
 712  * engine and save txwin in percpu cpu_txwin.
 713  * cpu_txwin is used in copy/paste operation for each compression /
 714  * decompression request.
 715  */
 716 static int nx842_open_percpu_txwins(void)
 717 {
 718         struct nx842_coproc *coproc, *n;
 719         unsigned int i, chip_id;
 720
 721         for_each_possible_cpu(i) {
 722                 struct vas_window *txwin = NULL;
 723
 724                 chip_id = cpu_to_chip_id(i);
 725
 726                 list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
 727                         /*
 728                          * Kernel requests use only high priority FIFOs. So
 729                          * open send windows for these FIFOs.
 730                          */
 731
 732                         if (coproc->ct != VAS_COP_TYPE_842_HIPRI)
 733                                 continue;
 734
 735                         if (coproc->chip_id == chip_id) {
 736                                 txwin = nx842_alloc_txwin(coproc);
 737                                 if (IS_ERR(txwin))
 738                                         return PTR_ERR(txwin);
 739
 740                                 per_cpu(cpu_txwin, i) = txwin;
 741                                 break;
 742                         }
 743                 }
 744
 745                 if (!per_cpu(cpu_txwin, i)) {
 746                         /* shouldn't happen, Each chip will have NX engine */
 747                         pr_err("NX engine is not available for CPU %d\n", i);
 748                         return -EINVAL;
 749                 }
 750         }
 751
 752         return 0;
 753 }
 754
 755 static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id,
 756                                         int vasid)
 757 {
 758         struct vas_window *rxwin = NULL;
 759         struct vas_rx_win_attr rxattr;
 760         struct nx842_coproc *coproc;
 761         u32 lpid, pid, tid, fifo_size;
 762         u64 rx_fifo;
 763         const char *priority;
 764         int ret;
 765
 766         ret = of_property_read_u64(dn, "rx-fifo-address", &rx_fifo);
 767         if (ret) {
 768                 pr_err("Missing rx-fifo-address property\n");
 769                 return ret;
 770         }
 771
 772         ret = of_property_read_u32(dn, "rx-fifo-size", &fifo_size);
 773         if (ret) {
 774                 pr_err("Missing rx-fifo-size property\n");
 775                 return ret;
 776         }
 777
 778         ret = of_property_read_u32(dn, "lpid", &lpid);
 779         if (ret) {
 780                 pr_err("Missing lpid property\n");
 781                 return ret;
 782         }
 783
 784         ret = of_property_read_u32(dn, "pid", &pid);
 785         if (ret) {
 786                 pr_err("Missing pid property\n");
 787                 return ret;
 788         }
 789
 790         ret = of_property_read_u32(dn, "tid", &tid);
 791         if (ret) {
 792                 pr_err("Missing tid property\n");
 793                 return ret;
 794         }
 795
 796         ret = of_property_read_string(dn, "priority", &priority);
 797         if (ret) {
 798                 pr_err("Missing priority property\n");
 799                 return ret;
 800         }
 801
 802         coproc = kzalloc(sizeof(*coproc), GFP_KERNEL);
 803         if (!coproc)
 804                 return -ENOMEM;
 805
 806         if (!strcmp(priority, "High"))
 807                 coproc->ct = VAS_COP_TYPE_842_HIPRI;
 808         else if (!strcmp(priority, "Normal"))
 809                 coproc->ct = VAS_COP_TYPE_842;
 810         else {
 811                 pr_err("Invalid RxFIFO priority value\n");
 812                 ret =  -EINVAL;
 813                 goto err_out;
 814         }
 815
 816         vas_init_rx_win_attr(&rxattr, coproc->ct);
 817         rxattr.rx_fifo = (void *)rx_fifo;
 818         rxattr.rx_fifo_size = fifo_size;
 819         rxattr.lnotify_lpid = lpid;
 820         rxattr.lnotify_pid = pid;
 821         rxattr.lnotify_tid = tid;
 822         rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO;
 823
 824         /*
 825          * Open a VAS receice window which is used to configure RxFIFO
 826          * for NX.
 827          */
 828         rxwin = vas_rx_win_open(vasid, coproc->ct, &rxattr);
 829         if (IS_ERR(rxwin)) {
 830                 ret = PTR_ERR(rxwin);
 831                 pr_err("setting RxFIFO with VAS failed: %d\n",
 832                         ret);
 833                 goto err_out;
 834         }
 835
 836         coproc->vas.rxwin = rxwin;
 837         coproc->vas.id = vasid;
 838         nx842_add_coprocs_list(coproc, chip_id);
 839
 840         return 0;
 841
 842 err_out:
 843         kfree(coproc);
 844         return ret;
 845 }
 846
 847
 848 static int __init nx842_powernv_probe_vas(struct device_node *pn)
 849 {
 850         struct device_node *dn;
 851         int chip_id, vasid, ret = 0;
 852         int nx_fifo_found = 0;
 853
 854         chip_id = of_get_ibm_chip_id(pn);
 855         if (chip_id < 0) {
 856                 pr_err("ibm,chip-id missing\n");
 857                 return -EINVAL;
 858         }
 859
 860         vasid = chip_to_vas_id(chip_id);
 861         if (vasid < 0) {
 862                 pr_err("Unable to map chip_id %d to vasid\n", chip_id);
 863                 return -EINVAL;
 864         }
 865
 866         for_each_child_of_node(pn, dn) {
 867                 if (of_device_is_compatible(dn, "ibm,p9-nx-842")) {
 868                         ret = vas_cfg_coproc_info(dn, chip_id, vasid);
 869                         if (ret) {
 870                                 of_node_put(dn);
 871                                 return ret;
 872                         }
 873                         nx_fifo_found++;
 874                 }
 875         }
 876
 877         if (!nx_fifo_found) {
 878                 pr_err("NX842 FIFO nodes are missing\n");
 879                 ret = -EINVAL;
 880         }
 881
 882         return ret;
 883 }
 884
 885 static int __init nx842_powernv_probe(struct device_node *dn)
 886 {
 887         struct nx842_coproc *coproc;
 888         unsigned int ct, ci;
 889         int chip_id;
 890
 891         chip_id = of_get_ibm_chip_id(dn);
 892         if (chip_id < 0) {
 893                 pr_err("ibm,chip-id missing\n");
 894                 return -EINVAL;
 895         }
 896
 897         if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) {
 898                 pr_err("ibm,842-coprocessor-type missing\n");
 899                 return -EINVAL;
 900         }
 901
 902         if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) {
 903                 pr_err("ibm,842-coprocessor-instance missing\n");
 904                 return -EINVAL;
 905         }
 906
 907         coproc = kmalloc(sizeof(*coproc), GFP_KERNEL);
 908         if (!coproc)
 909                 return -ENOMEM;
 910
 911         coproc->ct = ct;
 912         coproc->ci = ci;
 913         nx842_add_coprocs_list(coproc, chip_id);
 914
 915         pr_info("coprocessor found on chip %d, CT %d CI %d\n", chip_id, ct, ci);
 916
 917         if (!nx842_ct)
 918                 nx842_ct = ct;
 919         else if (nx842_ct != ct)
 920                 pr_err("NX842 chip %d, CT %d != first found CT %d\n",
 921                        chip_id, ct, nx842_ct);
 922
 923         return 0;
 924 }
 925
 926 static void nx842_delete_coprocs(void)
 927 {
 928         struct nx842_coproc *coproc, *n;
 929         struct vas_window *txwin;
 930         int i;
 931
 932         /*
 933          * close percpu txwins that are opened for the corresponding coproc.
 934          */
 935         for_each_possible_cpu(i) {
 936                 txwin = per_cpu(cpu_txwin, i);
 937                 if (txwin)
 938                         vas_win_close(txwin);
 939
 940                 per_cpu(cpu_txwin, i) = 0;
 941         }
 942
 943         list_for_each_entry_safe(coproc, n, &nx842_coprocs, list) {
 944                 if (coproc->vas.rxwin)
 945                         vas_win_close(coproc->vas.rxwin);
 946
 947                 list_del(&coproc->list);
 948                 kfree(coproc);
 949         }
 950 }
 951
 952 static struct nx842_constraints nx842_powernv_constraints = {
 953         .alignment =    DDE_BUFFER_ALIGN,
 954         .multiple =     DDE_BUFFER_LAST_MULT,
 955         .minimum =      DDE_BUFFER_LAST_MULT,
 956         .maximum =      (DDL_LEN_MAX - 1) * PAGE_SIZE,
 957 };
 958
 959 static struct nx842_driver nx842_powernv_driver = {
 960         .name =         KBUILD_MODNAME,
 961         .owner =        THIS_MODULE,
 962         .workmem_size = sizeof(struct nx842_workmem),
 963         .constraints =  &nx842_powernv_constraints,
 964         .compress =     nx842_powernv_compress,
 965         .decompress =   nx842_powernv_decompress,
 966 };
 967
 968 static int nx842_powernv_crypto_init(struct crypto_tfm *tfm)
 969 {
 970         return nx842_crypto_init(tfm, &nx842_powernv_driver);
 971 }
 972
 973 static struct crypto_alg nx842_powernv_alg = {
 974         .cra_name               = "842",
 975         .cra_driver_name        = "842-nx",
 976         .cra_priority           = 300,
 977         .cra_flags              = CRYPTO_ALG_TYPE_COMPRESS,
 978         .cra_ctxsize            = sizeof(struct nx842_crypto_ctx),
 979         .cra_module             = THIS_MODULE,
 980         .cra_init               = nx842_powernv_crypto_init,
 981         .cra_exit               = nx842_crypto_exit,
 982         .cra_u                  = { .compress = {
 983         .coa_compress           = nx842_crypto_compress,
 984         .coa_decompress         = nx842_crypto_decompress } }
 985 };
 986
 987 static __init int nx842_powernv_init(void)
 988 {
 989         struct device_node *dn;
 990         int ret;
 991
 992         /* verify workmem size/align restrictions */
 993         BUILD_BUG_ON(WORKMEM_ALIGN % CRB_ALIGN);
 994         BUILD_BUG_ON(CRB_ALIGN % DDE_ALIGN);
 995         BUILD_BUG_ON(CRB_SIZE % DDE_ALIGN);
 996         /* verify buffer size/align restrictions */
 997         BUILD_BUG_ON(PAGE_SIZE % DDE_BUFFER_ALIGN);
 998         BUILD_BUG_ON(DDE_BUFFER_ALIGN % DDE_BUFFER_SIZE_MULT);
 999         BUILD_BUG_ON(DDE_BUFFER_SIZE_MULT % DDE_BUFFER_LAST_MULT);
1000
1001         for_each_compatible_node(dn, NULL, "ibm,power9-nx") {
1002                 ret = nx842_powernv_probe_vas(dn);
1003                 if (ret) {
1004                         nx842_delete_coprocs();
1005                         return ret;
1006                 }
1007         }
1008
1009         if (list_empty(&nx842_coprocs)) {
1010                 for_each_compatible_node(dn, NULL, "ibm,power-nx")
1011                         nx842_powernv_probe(dn);
1012
1013                 if (!nx842_ct)
1014                         return -ENODEV;
1015
1016                 nx842_powernv_exec = nx842_exec_icswx;
1017         } else {
1018                 ret = nx842_open_percpu_txwins();
1019                 if (ret) {
1020                         nx842_delete_coprocs();
1021                         return ret;
1022                 }
1023
1024                 nx842_powernv_exec = nx842_exec_vas;
1025         }
1026
1027         ret = crypto_register_alg(&nx842_powernv_alg);
1028         if (ret) {
1029                 nx842_delete_coprocs();
1030                 return ret;
1031         }
1032
1033         return 0;
1034 }
1035 module_init(nx842_powernv_init);
1036
1037 static void __exit nx842_powernv_exit(void)
1038 {
1039         crypto_unregister_alg(&nx842_powernv_alg);
1040
1041         nx842_delete_coprocs();
1042 }
1043 module_exit(nx842_powernv_exit);