module/os/freebsd/zfs/crypto_os.c

   1 /*
   2  * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
   3  * Copyright (c) 2018 Sean Eric Fagan <sef@ixsystems.com>
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  25  * SUCH DAMAGE.
  26  *
  27  * Portions of this file are derived from sys/geom/eli/g_eli_hmac.c
  28  */
  29
  30 #include <sys/types.h>
  31 #include <sys/errno.h>
  32
  33 #ifdef _KERNEL
  34 #include <sys/libkern.h>
  35 #include <sys/malloc.h>
  36 #include <sys/sysctl.h>
  37 #include <opencrypto/cryptodev.h>
  38 #include <opencrypto/xform.h>
  39 #endif
  40
  41 #include <sys/zio_crypt.h>
  42 #include <sys/fs/zfs.h>
  43 #include <sys/zio.h>
  44
  45 #include <sys/freebsd_crypto.h>
  46
  47 #define SHA512_HMAC_BLOCK_SIZE  128
  48
  49 static int crypt_sessions = 0;
  50 SYSCTL_DECL(_vfs_zfs);
  51 SYSCTL_INT(_vfs_zfs, OID_AUTO, crypt_sessions, CTLFLAG_RD,
  52         &crypt_sessions, 0, "Number of cryptographic sessions created");
  53
  54 void
  55 crypto_mac_init(struct hmac_ctx *ctx, const crypto_key_t *c_key)
  56 {
  57         uint8_t k_ipad[SHA512_HMAC_BLOCK_SIZE],
  58             k_opad[SHA512_HMAC_BLOCK_SIZE],
  59             key[SHA512_HMAC_BLOCK_SIZE];
  60         SHA512_CTX lctx;
  61         int i;
  62         size_t cl_bytes = CRYPTO_BITS2BYTES(c_key->ck_length);
  63
  64         /*
  65          * This code is based on the similar code in geom/eli/g_eli_hmac.c
  66          */
  67         memset(key, 0, sizeof (key));
  68         if (c_key->ck_length  == 0)
  69                 /* do nothing */;
  70         else if (cl_bytes <= SHA512_HMAC_BLOCK_SIZE)
  71                 memcpy(key, c_key->ck_data, cl_bytes);
  72         else {
  73                 /*
  74                  * If key is longer than 128 bytes reset it to
  75                  * key = SHA512(key).
  76                  */
  77                 SHA512_Init(&lctx);
  78                 SHA512_Update(&lctx, c_key->ck_data, cl_bytes);
  79                 SHA512_Final(key, &lctx);
  80         }
  81
  82         /* XOR key with ipad and opad values. */
  83         for (i = 0; i < sizeof (key); i++) {
  84                 k_ipad[i] = key[i] ^ 0x36;
  85                 k_opad[i] = key[i] ^ 0x5c;
  86         }
  87         memset(key, 0, sizeof (key));
  88
  89         /* Start inner SHA512. */
  90         SHA512_Init(&ctx->innerctx);
  91         SHA512_Update(&ctx->innerctx, k_ipad, sizeof (k_ipad));
  92         memset(k_ipad, 0, sizeof (k_ipad));
  93         /* Start outer SHA512. */
  94         SHA512_Init(&ctx->outerctx);
  95         SHA512_Update(&ctx->outerctx, k_opad, sizeof (k_opad));
  96         memset(k_opad, 0, sizeof (k_opad));
  97 }
  98
  99 void
 100 crypto_mac_update(struct hmac_ctx *ctx, const void *data, size_t datasize)
 101 {
 102         SHA512_Update(&ctx->innerctx, data, datasize);
 103 }
 104
 105 void
 106 crypto_mac_final(struct hmac_ctx *ctx, void *md, size_t mdsize)
 107 {
 108         uint8_t digest[SHA512_DIGEST_LENGTH];
 109
 110         /* Complete inner hash */
 111         SHA512_Final(digest, &ctx->innerctx);
 112
 113         /* Complete outer hash */
 114         SHA512_Update(&ctx->outerctx, digest, sizeof (digest));
 115         SHA512_Final(digest, &ctx->outerctx);
 116
 117         memset(ctx, 0, sizeof (*ctx));
 118         /* mdsize == 0 means "Give me the whole hash!" */
 119         if (mdsize == 0)
 120                 mdsize = SHA512_DIGEST_LENGTH;
 121         memcpy(md, digest, mdsize);
 122         memset(digest, 0, sizeof (digest));
 123 }
 124
 125 void
 126 crypto_mac(const crypto_key_t *key, const void *in_data, size_t in_data_size,
 127     void *out_data, size_t out_data_size)
 128 {
 129         struct hmac_ctx ctx;
 130
 131         crypto_mac_init(&ctx, key);
 132         crypto_mac_update(&ctx, in_data, in_data_size);
 133         crypto_mac_final(&ctx, out_data, out_data_size);
 134 }
 135
 136 static int
 137 freebsd_zfs_crypt_done(struct cryptop *crp)
 138 {
 139         freebsd_crypt_session_t *ses;
 140
 141         ses = crp->crp_opaque;
 142         mtx_lock(&ses->fs_lock);
 143         ses->fs_done = true;
 144         mtx_unlock(&ses->fs_lock);
 145         wakeup(crp);
 146         return (0);
 147 }
 148
 149 static int
 150 freebsd_zfs_crypt_done_sync(struct cryptop *crp)
 151 {
 152
 153         return (0);
 154 }
 155
 156 void
 157 freebsd_crypt_freesession(freebsd_crypt_session_t *sess)
 158 {
 159         mtx_destroy(&sess->fs_lock);
 160         crypto_freesession(sess->fs_sid);
 161         memset(sess, 0, sizeof (*sess));
 162 }
 163
 164 static int
 165 zfs_crypto_dispatch(freebsd_crypt_session_t *session, struct cryptop *crp)
 166 {
 167         int error;
 168
 169         crp->crp_opaque = session;
 170         for (;;) {
 171 #if __FreeBSD_version < 1400004
 172                 boolean_t async = ((crypto_ses2caps(crp->crp_session) &
 173                     CRYPTOCAP_F_SYNC) == 0);
 174 #else
 175                 boolean_t async = !CRYPTO_SESS_SYNC(crp->crp_session);
 176 #endif
 177                 crp->crp_callback = async ? freebsd_zfs_crypt_done :
 178                     freebsd_zfs_crypt_done_sync;
 179                 error = crypto_dispatch(crp);
 180                 if (error == 0) {
 181                         if (async) {
 182                                 mtx_lock(&session->fs_lock);
 183                                 while (session->fs_done == false) {
 184                                         msleep(crp, &session->fs_lock, 0,
 185                                             "zfs_crypto", 0);
 186                                 }
 187                                 mtx_unlock(&session->fs_lock);
 188                         }
 189                         error = crp->crp_etype;
 190                 }
 191
 192                 if (error == ENOMEM) {
 193                         pause("zcrnomem", 1);
 194                 } else if (error != EAGAIN) {
 195                         break;
 196                 }
 197                 crp->crp_etype = 0;
 198                 crp->crp_flags &= ~CRYPTO_F_DONE;
 199                 session->fs_done = false;
 200         }
 201         return (error);
 202 }
 203 static void
 204 freebsd_crypt_uio_debug_log(boolean_t encrypt,
 205     freebsd_crypt_session_t *input_sessionp,
 206     const struct zio_crypt_info *c_info,
 207     zfs_uio_t *data_uio,
 208     crypto_key_t *key,
 209     uint8_t *ivbuf,
 210     size_t datalen,
 211     size_t auth_len)
 212 {
 213 #ifdef FCRYPTO_DEBUG
 214         struct cryptodesc *crd;
 215         uint8_t *p = NULL;
 216         size_t total = 0;
 217
 218         printf("%s(%s, %p, { %s, %d, %d, %s }, %p, { %p, %u }, "
 219             "%p, %u, %u)\n",
 220             __FUNCTION__, encrypt ? "encrypt" : "decrypt", input_sessionp,
 221             c_info->ci_algname, c_info->ci_crypt_type,
 222             (unsigned int)c_info->ci_keylen, c_info->ci_name,
 223             data_uio, key->ck_data,
 224             (unsigned int)key->ck_length,
 225             ivbuf, (unsigned int)datalen, (unsigned int)auth_len);
 226         printf("\tkey = { ");
 227         for (int i = 0; i < key->ck_length / 8; i++) {
 228                 uint8_t *b = (uint8_t *)key->ck_data;
 229                 printf("%02x ", b[i]);
 230         }
 231         printf("}\n");
 232         for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++) {
 233                 printf("\tiovec #%d: <%p, %u>\n", i,
 234                     zfs_uio_iovbase(data_uio, i),
 235                     (unsigned int)zfs_uio_iovlen(data_uio, i));
 236                 total += zfs_uio_iovlen(data_uio, i);
 237         }
 238         zfs_uio_resid(data_uio) = total;
 239 #endif
 240 }
 241 /*
 242  * Create a new cryptographic session.  This should
 243  * happen every time the key changes (including when
 244  * it's first loaded).
 245  */
 246 int
 247 freebsd_crypt_newsession(freebsd_crypt_session_t *sessp,
 248     const struct zio_crypt_info *c_info, crypto_key_t *key)
 249 {
 250         struct crypto_session_params csp = {0};
 251         int error = 0;
 252
 253 #ifdef FCRYPTO_DEBUG
 254         printf("%s(%p, { %s, %d, %d, %s }, { %p, %u })\n",
 255             __FUNCTION__, sessp,
 256             c_info->ci_algname, c_info->ci_crypt_type,
 257             (unsigned int)c_info->ci_keylen, c_info->ci_name,
 258             key->ck_data, (unsigned int)key->ck_length);
 259         printf("\tkey = { ");
 260         for (int i = 0; i < key->ck_length / 8; i++) {
 261                 uint8_t *b = (uint8_t *)key->ck_data;
 262                 printf("%02x ", b[i]);
 263         }
 264         printf("}\n");
 265 #endif
 266         csp.csp_mode = CSP_MODE_AEAD;
 267         csp.csp_cipher_key = key->ck_data;
 268         csp.csp_cipher_klen = key->ck_length / 8;
 269         switch (c_info->ci_crypt_type) {
 270                 case ZC_TYPE_GCM:
 271                 csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
 272                 csp.csp_ivlen = AES_GCM_IV_LEN;
 273                 switch (key->ck_length/8) {
 274                 case AES_128_GMAC_KEY_LEN:
 275                 case AES_192_GMAC_KEY_LEN:
 276                 case AES_256_GMAC_KEY_LEN:
 277                         break;
 278                 default:
 279                         error = EINVAL;
 280                         goto bad;
 281                 }
 282                 break;
 283         case ZC_TYPE_CCM:
 284                 csp.csp_cipher_alg = CRYPTO_AES_CCM_16;
 285                 csp.csp_ivlen = AES_CCM_IV_LEN;
 286                 switch (key->ck_length/8) {
 287                 case AES_128_CBC_MAC_KEY_LEN:
 288                 case AES_192_CBC_MAC_KEY_LEN:
 289                 case AES_256_CBC_MAC_KEY_LEN:
 290                         break;
 291                 default:
 292                         error = EINVAL;
 293                         goto bad;
 294                         break;
 295                 }
 296                 break;
 297         default:
 298                 error = ENOTSUP;
 299                 goto bad;
 300         }
 301
 302         /*
 303          * Disable the use of hardware drivers on FreeBSD 13 and later since
 304          * common crypto offload drivers impose constraints on AES-GCM AAD
 305          * lengths that make them unusable for ZFS, and we currently do not have
 306          * a mechanism to fall back to a software driver for requests not
 307          * handled by a hardware driver.
 308          *
 309          * On 12 we continue to permit the use of hardware drivers since
 310          * CPU-accelerated drivers such as aesni(4) register themselves as
 311          * hardware drivers.
 312          */
 313         error = crypto_newsession(&sessp->fs_sid, &csp, CRYPTOCAP_F_SOFTWARE);
 314         mtx_init(&sessp->fs_lock, "FreeBSD Cryptographic Session Lock",
 315             NULL, MTX_DEF);
 316         crypt_sessions++;
 317 bad:
 318 #ifdef FCRYPTO_DEBUG
 319         if (error)
 320                 printf("%s: returning error %d\n", __FUNCTION__, error);
 321 #endif
 322         return (error);
 323 }
 324
 325 int
 326 freebsd_crypt_uio(boolean_t encrypt,
 327     freebsd_crypt_session_t *input_sessionp,
 328     const struct zio_crypt_info *c_info,
 329     zfs_uio_t *data_uio,
 330     crypto_key_t *key,
 331     uint8_t *ivbuf,
 332     size_t datalen,
 333     size_t auth_len)
 334 {
 335         struct cryptop *crp;
 336         freebsd_crypt_session_t *session = NULL;
 337         int error = 0;
 338         size_t total = 0;
 339
 340         freebsd_crypt_uio_debug_log(encrypt, input_sessionp, c_info, data_uio,
 341             key, ivbuf, datalen, auth_len);
 342         for (int i = 0; i < zfs_uio_iovcnt(data_uio); i++)
 343                 total += zfs_uio_iovlen(data_uio, i);
 344         zfs_uio_resid(data_uio) = total;
 345         if (input_sessionp == NULL) {
 346                 session = kmem_zalloc(sizeof (*session), KM_SLEEP);
 347                 error = freebsd_crypt_newsession(session, c_info, key);
 348                 if (error)
 349                         goto out;
 350         } else
 351                 session = input_sessionp;
 352
 353         crp = crypto_getreq(session->fs_sid, M_WAITOK);
 354         if (encrypt) {
 355                 crp->crp_op = CRYPTO_OP_ENCRYPT |
 356                     CRYPTO_OP_COMPUTE_DIGEST;
 357         } else {
 358                 crp->crp_op = CRYPTO_OP_DECRYPT |
 359                     CRYPTO_OP_VERIFY_DIGEST;
 360         }
 361         crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
 362         crypto_use_uio(crp, GET_UIO_STRUCT(data_uio));
 363
 364         crp->crp_aad_start = 0;
 365         crp->crp_aad_length = auth_len;
 366         crp->crp_payload_start = auth_len;
 367         crp->crp_payload_length = datalen;
 368         crp->crp_digest_start = auth_len + datalen;
 369
 370         memcpy(crp->crp_iv, ivbuf, ZIO_DATA_IV_LEN);
 371         error = zfs_crypto_dispatch(session, crp);
 372         crypto_freereq(crp);
 373 out:
 374 #ifdef FCRYPTO_DEBUG
 375         if (error)
 376                 printf("%s: returning error %d\n", __FUNCTION__, error);
 377 #endif
 378         if (input_sessionp == NULL) {
 379                 freebsd_crypt_freesession(session);
 380                 kmem_free(session, sizeof (*session));
 381         }
 382         return (error);
 383 }