drivers/crypto/bcm/cipher.h

   1 /*
   2  * Copyright 2016 Broadcom
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License, version 2, as
   6  * published by the Free Software Foundation (the "GPL").
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License version 2 (GPLv2) for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License
  14  * version 2 (GPLv2) along with this source code.
  15  */
  16
  17 #ifndef _CIPHER_H
  18 #define _CIPHER_H
  19
  20 #include <linux/atomic.h>
  21 #include <linux/mailbox/brcm-message.h>
  22 #include <linux/mailbox_client.h>
  23 #include <crypto/aes.h>
  24 #include <crypto/internal/hash.h>
  25 #include <crypto/aead.h>
  26 #include <crypto/sha.h>
  27 #include <crypto/sha3.h>
  28
  29 #include "spu.h"
  30 #include "spum.h"
  31 #include "spu2.h"
  32
  33 /* Driver supports up to MAX_SPUS SPU blocks */
  34 #define MAX_SPUS 16
  35
  36 #define ARC4_MIN_KEY_SIZE   1
  37 #define ARC4_MAX_KEY_SIZE   256
  38 #define ARC4_BLOCK_SIZE     1
  39 #define ARC4_STATE_SIZE     4
  40
  41 #define CCM_AES_IV_SIZE    16
  42 #define GCM_AES_IV_SIZE    12
  43 #define GCM_ESP_IV_SIZE     8
  44 #define CCM_ESP_IV_SIZE     8
  45 #define RFC4543_ICV_SIZE   16
  46
  47 #define MAX_KEY_SIZE    ARC4_MAX_KEY_SIZE
  48 #define MAX_IV_SIZE     AES_BLOCK_SIZE
  49 #define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE
  50 #define MAX_ASSOC_SIZE  512
  51
  52 /* size of salt value for AES-GCM-ESP and AES-CCM-ESP */
  53 #define GCM_ESP_SALT_SIZE   4
  54 #define CCM_ESP_SALT_SIZE   3
  55 #define MAX_SALT_SIZE       GCM_ESP_SALT_SIZE
  56 #define GCM_ESP_SALT_OFFSET 0
  57 #define CCM_ESP_SALT_OFFSET 1
  58
  59 #define GCM_ESP_DIGESTSIZE 16
  60
  61 #define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
  62
  63 /*
  64  * Maximum number of bytes from a non-final hash request that can be deferred
  65  * until more data is available. With new crypto API framework, this
  66  * can be no more than one block of data.
  67  */
  68 #define HASH_CARRY_MAX  MAX_HASH_BLOCK_SIZE
  69
  70 /* Force at least 4-byte alignment of all SPU message fields */
  71 #define SPU_MSG_ALIGN  4
  72
  73 /* Number of times to resend mailbox message if mb queue is full */
  74 #define SPU_MB_RETRY_MAX  1000
  75
  76 /* op_counts[] indexes */
  77 enum op_type {
  78         SPU_OP_CIPHER,
  79         SPU_OP_HASH,
  80         SPU_OP_HMAC,
  81         SPU_OP_AEAD,
  82         SPU_OP_NUM
  83 };
  84
  85 enum spu_spu_type {
  86         SPU_TYPE_SPUM,
  87         SPU_TYPE_SPU2,
  88 };
  89
  90 /*
  91  * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus,
  92  * respectively.
  93  */
  94 enum spu_spu_subtype {
  95         SPU_SUBTYPE_SPUM_NS2,
  96         SPU_SUBTYPE_SPUM_NSP,
  97         SPU_SUBTYPE_SPU2_V1,
  98         SPU_SUBTYPE_SPU2_V2
  99 };
 100
 101 struct spu_type_subtype {
 102         enum spu_spu_type type;
 103         enum spu_spu_subtype subtype;
 104 };
 105
 106 struct cipher_op {
 107         enum spu_cipher_alg alg;
 108         enum spu_cipher_mode mode;
 109 };
 110
 111 struct auth_op {
 112         enum hash_alg alg;
 113         enum hash_mode mode;
 114 };
 115
 116 struct iproc_alg_s {
 117         u32 type;
 118         union {
 119                 struct crypto_alg crypto;
 120                 struct ahash_alg hash;
 121                 struct aead_alg aead;
 122         } alg;
 123         struct cipher_op cipher_info;
 124         struct auth_op auth_info;
 125         bool auth_first;
 126         bool registered;
 127 };
 128
 129 /*
 130  * Buffers for a SPU request/reply message pair. All part of one structure to
 131  * allow a single alloc per request.
 132  */
 133 struct spu_msg_buf {
 134         /* Request message fragments */
 135
 136         /*
 137          * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC,
 138          * and BD header. For SPU2, holds FMD, OMD.
 139          */
 140         u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
 141
 142         /* IV or counter. Size to include salt. Also used for XTS tweek. */
 143         u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
 144
 145         /* Hash digest. request and response. */
 146         u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)];
 147
 148         /* SPU request message padding */
 149         u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
 150
 151         /* SPU-M request message STATUS field */
 152         u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)];
 153
 154         /* Response message fragments */
 155
 156         /* SPU response message header */
 157         u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
 158
 159         /* SPU response message STATUS field padding */
 160         u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)];
 161
 162         /* SPU response message STATUS field */
 163         u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)];
 164
 165         union {
 166                 /* Buffers only used for ablkcipher */
 167                 struct {
 168                         /*
 169                          * Field used for either SUPDT when RC4 is used
 170                          * -OR- tweak value when XTS/AES is used
 171                          */
 172                         u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)];
 173                 } c;
 174
 175                 /* Buffers only used for aead */
 176                 struct {
 177                         /* SPU response pad for GCM data */
 178                         u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
 179
 180                         /* SPU request msg padding for GCM AAD */
 181                         u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
 182
 183                         /* SPU response data to be discarded */
 184                         u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE,
 185                                           SPU_MSG_ALIGN)];
 186                 } a;
 187         };
 188 };
 189
 190 struct iproc_ctx_s {
 191         u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
 192         unsigned int enckeylen;
 193
 194         u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
 195         unsigned int authkeylen;
 196
 197         u8 salt[MAX_SALT_SIZE];
 198         unsigned int salt_len;
 199         unsigned int salt_offset;
 200         u8 iv[MAX_IV_SIZE];
 201
 202         unsigned int digestsize;
 203
 204         struct iproc_alg_s *alg;
 205         bool is_esp;
 206
 207         struct cipher_op cipher;
 208         enum spu_cipher_type cipher_type;
 209
 210         struct auth_op auth;
 211         bool auth_first;
 212
 213         /*
 214          * The maximum length in bytes of the payload in a SPU message for this
 215          * context. For SPU-M, the payload is the combination of AAD and data.
 216          * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF
 217          * indicates that there is no limit to the length of the SPU message
 218          * payload.
 219          */
 220         unsigned int max_payload;
 221
 222         struct crypto_aead *fallback_cipher;
 223
 224         /* auth_type is determined during processing of request */
 225
 226         u8 ipad[MAX_HASH_BLOCK_SIZE];
 227         u8 opad[MAX_HASH_BLOCK_SIZE];
 228
 229         /*
 230          * Buffer to hold SPU message header template. Template is created at
 231          * setkey time for ablkcipher requests, since most of the fields in the
 232          * header are known at that time. At request time, just fill in a few
 233          * missing pieces related to length of data in the request and IVs, etc.
 234          */
 235         u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
 236
 237         /* Length of SPU request header */
 238         u16 spu_req_hdr_len;
 239
 240         /* Expected length of SPU response header */
 241         u16 spu_resp_hdr_len;
 242
 243         /*
 244          * shash descriptor - needed to perform incremental hashing in
 245          * in software, when hw doesn't support it.
 246          */
 247         struct shash_desc *shash;
 248
 249         bool is_rfc4543;        /* RFC 4543 style of GMAC */
 250 };
 251
 252 /* state from iproc_reqctx_s necessary for hash state export/import */
 253 struct spu_hash_export_s {
 254         unsigned int total_todo;
 255         unsigned int total_sent;
 256         u8 hash_carry[HASH_CARRY_MAX];
 257         unsigned int hash_carry_len;
 258         u8 incr_hash[MAX_DIGEST_SIZE];
 259         bool is_sw_hmac;
 260 };
 261
 262 struct iproc_reqctx_s {
 263         /* general context */
 264         struct crypto_async_request *parent;
 265
 266         /* only valid after enqueue() */
 267         struct iproc_ctx_s *ctx;
 268
 269         u8 chan_idx;   /* Mailbox channel to be used to submit this request */
 270
 271         /* total todo, rx'd, and sent for this request */
 272         unsigned int total_todo;
 273         unsigned int total_received;    /* only valid for ablkcipher */
 274         unsigned int total_sent;
 275
 276         /*
 277          * num bytes sent to hw from the src sg in this request. This can differ
 278          * from total_sent for incremental hashing. total_sent includes previous
 279          * init() and update() data. src_sent does not.
 280          */
 281         unsigned int src_sent;
 282
 283         /*
 284          * For AEAD requests, start of associated data. This will typically
 285          * point to the beginning of the src scatterlist from the request,
 286          * since assoc data is at the beginning of the src scatterlist rather
 287          * than in its own sg.
 288          */
 289         struct scatterlist *assoc;
 290
 291         /*
 292          * scatterlist entry and offset to start of data for next chunk. Crypto
 293          * API src scatterlist for AEAD starts with AAD, if present. For first
 294          * chunk, src_sg is sg entry at beginning of input data (after AAD).
 295          * src_skip begins at the offset in that sg entry where data begins.
 296          */
 297         struct scatterlist *src_sg;
 298         int src_nents;          /* Number of src entries with data */
 299         u32 src_skip;           /* bytes of current sg entry already used */
 300
 301         /*
 302          * Same for destination. For AEAD, if there is AAD, output data must
 303          * be written at offset following AAD.
 304          */
 305         struct scatterlist *dst_sg;
 306         int dst_nents;          /* Number of dst entries with data */
 307         u32 dst_skip;           /* bytes of current sg entry already written */
 308
 309         /* Mailbox message used to send this request to PDC driver */
 310         struct brcm_message mb_mssg;
 311
 312         bool bd_suppress;       /* suppress BD field in SPU response? */
 313
 314         /* cipher context */
 315         bool is_encrypt;
 316
 317         /*
 318          * CBC mode: IV.  CTR mode: counter.  Else empty. Used as a DMA
 319          * buffer for AEAD requests. So allocate as DMAable memory. If IV
 320          * concatenated with salt, includes the salt.
 321          */
 322         u8 *iv_ctr;
 323         /* Length of IV or counter, in bytes */
 324         unsigned int iv_ctr_len;
 325
 326         /*
 327          * Hash requests can be of any size, whether initial, update, or final.
 328          * A non-final request must be submitted to the SPU as an integral
 329          * number of blocks. This may leave data at the end of the request
 330          * that is not a full block. Since the request is non-final, it cannot
 331          * be padded. So, we write the remainder to this hash_carry buffer and
 332          * hold it until the next request arrives. The carry data is then
 333          * submitted at the beginning of the data in the next SPU msg.
 334          * hash_carry_len is the number of bytes currently in hash_carry. These
 335          * fields are only used for ahash requests.
 336          */
 337         u8 hash_carry[HASH_CARRY_MAX];
 338         unsigned int hash_carry_len;
 339         unsigned int is_final;  /* is this the final for the hash op? */
 340
 341         /*
 342          * Digest from incremental hash is saved here to include in next hash
 343          * operation. Cannot be stored in req->result for truncated hashes,
 344          * since result may be sized for final digest. Cannot be saved in
 345          * msg_buf because that gets deleted between incremental hash ops
 346          * and is not saved as part of export().
 347          */
 348         u8 incr_hash[MAX_DIGEST_SIZE];
 349
 350         /* hmac context */
 351         bool is_sw_hmac;
 352
 353         /* aead context */
 354         struct crypto_tfm *old_tfm;
 355         crypto_completion_t old_complete;
 356         void *old_data;
 357
 358         gfp_t gfp;
 359
 360         /* Buffers used to build SPU request and response messages */
 361         struct spu_msg_buf msg_buf;
 362 };
 363
 364 /*
 365  * Structure encapsulates a set of function pointers specific to the type of
 366  * SPU hardware running. These functions handling creation and parsing of
 367  * SPU request messages and SPU response messages. Includes hardware-specific
 368  * values read from device tree.
 369  */
 370 struct spu_hw {
 371         void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len);
 372         u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg,
 373                                    enum spu_cipher_mode cipher_mode,
 374                                    unsigned int blocksize);
 375         u32 (*spu_payload_length)(u8 *spu_hdr);
 376         u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len,
 377                                     bool is_hash);
 378         u16 (*spu_hash_pad_len)(enum hash_alg hash_alg,
 379                                 enum hash_mode hash_mode, u32 chunksize,
 380                                 u16 hash_block_size);
 381         u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode,
 382                                    unsigned int data_size);
 383         u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode,
 384                                   unsigned int assoc_len,
 385                                   unsigned int iv_len, bool is_encrypt);
 386         u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode,
 387                              u16 iv_len);
 388         enum hash_type (*spu_hash_type)(u32 src_sent);
 389         u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg,
 390                                enum hash_type);
 391         u32 (*spu_create_request)(u8 *spu_hdr,
 392                                   struct spu_request_opts *req_opts,
 393                                   struct spu_cipher_parms *cipher_parms,
 394                                   struct spu_hash_parms *hash_parms,
 395                                   struct spu_aead_parms *aead_parms,
 396                                   unsigned int data_size);
 397         u16 (*spu_cipher_req_init)(u8 *spu_hdr,
 398                                    struct spu_cipher_parms *cipher_parms);
 399         void (*spu_cipher_req_finish)(u8 *spu_hdr,
 400                                       u16 spu_req_hdr_len,
 401                                       unsigned int is_inbound,
 402                                       struct spu_cipher_parms *cipher_parms,
 403                                       bool update_key,
 404                                       unsigned int data_size);
 405         void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
 406                                 u32 hash_pad_len, enum hash_alg auth_alg,
 407                                 enum hash_mode auth_mode,
 408                                 unsigned int total_sent, u32 status_padding);
 409         u8 (*spu_xts_tweak_in_payload)(void);
 410         u8 (*spu_tx_status_len)(void);
 411         u8 (*spu_rx_status_len)(void);
 412         int (*spu_status_process)(u8 *statp);
 413         void (*spu_ccm_update_iv)(unsigned int digestsize,
 414                                   struct spu_cipher_parms *cipher_parms,
 415                                   unsigned int assoclen, unsigned int chunksize,
 416                                   bool is_encrypt, bool is_esp);
 417         u32 (*spu_wordalign_padlen)(u32 data_size);
 418
 419         /* The base virtual address of the SPU hw registers */
 420         void __iomem *reg_vbase[MAX_SPUS];
 421
 422         /* Version of the SPU hardware */
 423         enum spu_spu_type spu_type;
 424
 425         /* Sub-version of the SPU hardware */
 426         enum spu_spu_subtype spu_subtype;
 427
 428         /* The number of SPUs on this platform */
 429         u32 num_spu;
 430 };
 431
 432 struct device_private {
 433         struct platform_device *pdev[MAX_SPUS];
 434
 435         struct spu_hw spu;
 436
 437         atomic_t session_count; /* number of streams active */
 438         atomic_t stream_count;  /* monotonic counter for streamID's */
 439
 440         /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */
 441         u8 bcm_hdr_len;
 442
 443         /* The index of the channel to use for the next crypto request */
 444         atomic_t next_chan;
 445
 446         struct dentry *debugfs_dir;
 447         struct dentry *debugfs_stats;
 448
 449         /* Number of request bytes processed and result bytes returned */
 450         atomic64_t bytes_in;
 451         atomic64_t bytes_out;
 452
 453         /* Number of operations of each type */
 454         atomic_t op_counts[SPU_OP_NUM];
 455
 456         atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST];
 457         atomic_t hash_cnt[HASH_ALG_LAST];
 458         atomic_t hmac_cnt[HASH_ALG_LAST];
 459         atomic_t aead_cnt[AEAD_TYPE_LAST];
 460
 461         /* Number of calls to setkey() for each operation type */
 462         atomic_t setkey_cnt[SPU_OP_NUM];
 463
 464         /* Number of times request was resubmitted because mb was full */
 465         atomic_t mb_no_spc;
 466
 467         /* Number of mailbox send failures */
 468         atomic_t mb_send_fail;
 469
 470         /* Number of ICV check failures for AEAD messages */
 471         atomic_t bad_icv;
 472
 473         struct mbox_client mcl[MAX_SPUS];
 474         /* Array of mailbox channel pointers, one for each channel */
 475         struct mbox_chan *mbox[MAX_SPUS];
 476
 477         /* Driver initialized */
 478         bool inited;
 479 };
 480
 481 extern struct device_private iproc_priv;
 482
 483 #endif