?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge remote-tracking branch 'origin/master'
[unleashed/lotheac.git] / usr / src / uts / common / crypto / io / rsa.c
blob6089ddf770f0192069751ae23046328857e607d5
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * RSA provider for the Kernel Cryptographic Framework (KCF)
28 */
29
30 #include <sys/types.h>
31 #include <sys/systm.h>
32 #include <sys/modctl.h>
33 #include <sys/cmn_err.h>
34 #include <sys/ddi.h>
35 #include <sys/crypto/spi.h>
36 #include <sys/sysmacros.h>
37 #include <sys/strsun.h>
38 #include <sys/md5.h>
39 #include <sys/sha1.h>
40 #define _SHA2_IMPL
41 #include <sys/sha2.h>
42 #include <sys/random.h>
43 #include <sys/crypto/impl.h>
44 #include <sha1/sha1_impl.h>
45 #include <sha2/sha2_impl.h>
46 #include <padding/padding.h>
47 #include <rsa/rsa_impl.h>
48
49 extern struct mod_ops mod_cryptoops;
50
51 /*
52 * Module linkage information for the kernel.
53 */
54 static struct modlcrypto modlcrypto = {
55 &mod_cryptoops,
56 "RSA Kernel SW Provider"
57 };
58
59 static struct modlinkage modlinkage = {
60 MODREV_1,
61 (void *)&modlcrypto,
62 NULL
63 };
64
65 /*
66 * CSPI information (entry points, provider info, etc.)
67 */
68 typedef enum rsa_mech_type {
69 RSA_PKCS_MECH_INFO_TYPE, /* SUN_CKM_RSA_PKCS */
70 RSA_X_509_MECH_INFO_TYPE, /* SUN_CKM_RSA_X_509 */
71 MD5_RSA_PKCS_MECH_INFO_TYPE, /* SUN_MD5_RSA_PKCS */
72 SHA1_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA1_RSA_PKCS */
73 SHA256_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA256_RSA_PKCS */
74 SHA384_RSA_PKCS_MECH_INFO_TYPE, /* SUN_SHA384_RSA_PKCS */
75 SHA512_RSA_PKCS_MECH_INFO_TYPE /* SUN_SHA512_RSA_PKCS */
76 } rsa_mech_type_t;
77
78 /*
79 * Context for RSA_PKCS and RSA_X_509 mechanisms.
80 */
81 typedef struct rsa_ctx {
82 rsa_mech_type_t mech_type;
83 crypto_key_t *key;
84 size_t keychunk_size;
85 } rsa_ctx_t;
86
87 /*
88 * Context for MD5_RSA_PKCS and SHA*_RSA_PKCS mechanisms.
89 */
90 typedef struct digest_rsa_ctx {
91 rsa_mech_type_t mech_type;
92 crypto_key_t *key;
93 size_t keychunk_size;
94 union {
95 MD5_CTX md5ctx;
96 SHA1_CTX sha1ctx;
97 SHA2_CTX sha2ctx;
98 } dctx_u;
99 } digest_rsa_ctx_t;
100
101 #define md5_ctx dctx_u.md5ctx
102 #define sha1_ctx dctx_u.sha1ctx
103 #define sha2_ctx dctx_u.sha2ctx
104
105 /*
106 * Mechanism info structure passed to KCF during registration.
107 */
108 static crypto_mech_info_t rsa_mech_info_tab[] = {
109 /* RSA_PKCS */
110 {SUN_CKM_RSA_PKCS, RSA_PKCS_MECH_INFO_TYPE,
111 CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
112 CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
113 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
114 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
115 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
116 CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
117 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
118
119 /* RSA_X_509 */
120 {SUN_CKM_RSA_X_509, RSA_X_509_MECH_INFO_TYPE,
121 CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
122 CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
123 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
124 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC |
125 CRYPTO_FG_SIGN_RECOVER | CRYPTO_FG_SIGN_RECOVER_ATOMIC |
126 CRYPTO_FG_VERIFY_RECOVER | CRYPTO_FG_VERIFY_RECOVER_ATOMIC,
127 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
128
129 /* MD5_RSA_PKCS */
130 {SUN_CKM_MD5_RSA_PKCS, MD5_RSA_PKCS_MECH_INFO_TYPE,
131 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
132 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
133 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
134
135 /* SHA1_RSA_PKCS */
136 {SUN_CKM_SHA1_RSA_PKCS, SHA1_RSA_PKCS_MECH_INFO_TYPE,
137 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
138 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
139 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
140
141 /* SHA256_RSA_PKCS */
142 {SUN_CKM_SHA256_RSA_PKCS, SHA256_RSA_PKCS_MECH_INFO_TYPE,
143 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
144 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
145 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
146
147 /* SHA384_RSA_PKCS */
148 {SUN_CKM_SHA384_RSA_PKCS, SHA384_RSA_PKCS_MECH_INFO_TYPE,
149 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
150 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
151 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS},
152
153 /* SHA512_RSA_PKCS */
154 {SUN_CKM_SHA512_RSA_PKCS, SHA512_RSA_PKCS_MECH_INFO_TYPE,
155 CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
156 CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
157 RSA_MIN_KEY_LEN, RSA_MAX_KEY_LEN, CRYPTO_KEYSIZE_UNIT_IN_BITS}
158
159 };
160
161 #define RSA_VALID_MECH(mech) \
162 (((mech)->cm_type == RSA_PKCS_MECH_INFO_TYPE || \
163 (mech)->cm_type == RSA_X_509_MECH_INFO_TYPE || \
164 (mech)->cm_type == MD5_RSA_PKCS_MECH_INFO_TYPE || \
165 (mech)->cm_type == SHA1_RSA_PKCS_MECH_INFO_TYPE || \
166 (mech)->cm_type == SHA256_RSA_PKCS_MECH_INFO_TYPE || \
167 (mech)->cm_type == SHA384_RSA_PKCS_MECH_INFO_TYPE || \
168 (mech)->cm_type == SHA512_RSA_PKCS_MECH_INFO_TYPE) ? 1 : 0)
169
170 /* operations are in-place if the output buffer is NULL */
171 #define RSA_ARG_INPLACE(input, output) \
172 if ((output) == NULL) \
173 (output) = (input);
174
175 static void rsa_provider_status(crypto_provider_handle_t, uint_t *);
176
177 static crypto_control_ops_t rsa_control_ops = {
178 rsa_provider_status
179 };
180
181 static int rsa_common_init(crypto_ctx_t *, crypto_mechanism_t *,
182 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
183 static int rsaprov_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
184 crypto_req_handle_t);
185 static int rsa_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
186 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
187 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
188 static int rsaprov_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
189 crypto_req_handle_t);
190 static int rsa_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
191 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
192 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
193
194 /*
195 * The RSA mechanisms do not have multiple-part cipher operations.
196 * So, the update and final routines are set to NULL.
197 */
198 static crypto_cipher_ops_t rsa_cipher_ops = {
199 rsa_common_init,
200 rsaprov_encrypt,
201 NULL,
202 NULL,
203 rsa_encrypt_atomic,
204 rsa_common_init,
205 rsaprov_decrypt,
206 NULL,
207 NULL,
208 rsa_decrypt_atomic
209 };
210
211 static int rsa_sign_verify_common_init(crypto_ctx_t *, crypto_mechanism_t *,
212 crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
213 static int rsaprov_sign(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
214 crypto_req_handle_t);
215 static int rsa_sign_update(crypto_ctx_t *, crypto_data_t *,
216 crypto_req_handle_t);
217 static int rsa_sign_final(crypto_ctx_t *, crypto_data_t *,
218 crypto_req_handle_t);
219 static int rsa_sign_atomic(crypto_provider_handle_t, crypto_session_id_t,
220 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
221 crypto_spi_ctx_template_t, crypto_req_handle_t);
222
223 /*
224 * We use the same routine for sign_init and sign_recover_init fields
225 * as they do the same thing. Same holds for sign and sign_recover fields,
226 * and sign_atomic and sign_recover_atomic fields.
227 */
228 static crypto_sign_ops_t rsa_sign_ops = {
229 rsa_sign_verify_common_init,
230 rsaprov_sign,
231 rsa_sign_update,
232 rsa_sign_final,
233 rsa_sign_atomic,
234 rsa_sign_verify_common_init,
235 rsaprov_sign,
236 rsa_sign_atomic
237 };
238
239 static int rsaprov_verify(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
240 crypto_req_handle_t);
241 static int rsa_verify_update(crypto_ctx_t *, crypto_data_t *,
242 crypto_req_handle_t);
243 static int rsa_verify_final(crypto_ctx_t *, crypto_data_t *,
244 crypto_req_handle_t);
245 static int rsa_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
246 crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
247 crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
248 static int rsa_verify_recover(crypto_ctx_t *, crypto_data_t *,
249 crypto_data_t *, crypto_req_handle_t);
250 static int rsa_verify_recover_atomic(crypto_provider_handle_t,
251 crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *,
252 crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t,
253 crypto_req_handle_t);
254
255 /*
256 * We use the same routine (rsa_sign_verify_common_init) for verify_init
257 * and verify_recover_init fields as they do the same thing.
258 */
259 static crypto_verify_ops_t rsa_verify_ops = {
260 rsa_sign_verify_common_init,
261 rsaprov_verify,
262 rsa_verify_update,
263 rsa_verify_final,
264 rsa_verify_atomic,
265 rsa_sign_verify_common_init,
266 rsa_verify_recover,
267 rsa_verify_recover_atomic
268 };
269
270 static int rsa_free_context(crypto_ctx_t *);
271
272 static crypto_ctx_ops_t rsa_ctx_ops = {
273 NULL,
274 rsa_free_context
275 };
276
277 static crypto_ops_t rsa_crypto_ops = {
278 &rsa_control_ops,
279 NULL,
280 &rsa_cipher_ops,
281 NULL,
282 &rsa_sign_ops,
283 &rsa_verify_ops,
284 NULL,
285 NULL,
286 NULL,
287 NULL,
288 NULL,
289 NULL,
290 NULL,
291 &rsa_ctx_ops,
292 NULL,
293 NULL,
294 NULL,
295 };
296
297 static crypto_provider_info_t rsa_prov_info = {
298 CRYPTO_SPI_VERSION_4,
299 "RSA Software Provider",
300 CRYPTO_SW_PROVIDER,
301 {&modlinkage},
302 NULL,
303 &rsa_crypto_ops,
304 sizeof (rsa_mech_info_tab)/sizeof (crypto_mech_info_t),
305 rsa_mech_info_tab
306 };
307
308 static int rsa_encrypt_common(rsa_mech_type_t, crypto_key_t *,
309 crypto_data_t *, crypto_data_t *);
310 static int rsa_decrypt_common(rsa_mech_type_t, crypto_key_t *,
311 crypto_data_t *, crypto_data_t *);
312 static int rsa_sign_common(rsa_mech_type_t, crypto_key_t *,
313 crypto_data_t *, crypto_data_t *);
314 static int rsa_verify_common(rsa_mech_type_t, crypto_key_t *,
315 crypto_data_t *, crypto_data_t *);
316 static int compare_data(crypto_data_t *, uchar_t *);
317
318 static int core_rsa_encrypt(crypto_key_t *, uchar_t *, int, uchar_t *, int);
319 static int core_rsa_decrypt(crypto_key_t *, uchar_t *, int, uchar_t *);
320
321 static crypto_kcf_provider_handle_t rsa_prov_handle = 0;
322
323 int
324 _init(void)
325 {
326 int ret;
327
328 if ((ret = mod_install(&modlinkage)) != 0)
329 return (ret);
330
331 /* Register with KCF. If the registration fails, remove the module. */
332 if (crypto_register_provider(&rsa_prov_info, &rsa_prov_handle)) {
333 (void) mod_remove(&modlinkage);
334 return (EACCES);
335 }
336
337 return (0);
338 }
339
340 int
341 _fini(void)
342 {
343 /* Unregister from KCF if module is registered */
344 if (rsa_prov_handle != 0) {
345 if (crypto_unregister_provider(rsa_prov_handle))
346 return (EBUSY);
347
348 rsa_prov_handle = 0;
349 }
350
351 return (mod_remove(&modlinkage));
352 }
353
354 int
355 _info(struct modinfo *modinfop)
356 {
357 return (mod_info(&modlinkage, modinfop));
358 }
359
360 /* ARGSUSED */
361 static void
362 rsa_provider_status(crypto_provider_handle_t provider, uint_t *status)
363 {
364 *status = CRYPTO_PROVIDER_READY;
365 }
366
367 static int
368 check_mech_and_key(crypto_mechanism_t *mechanism, crypto_key_t *key)
369 {
370 int rv = CRYPTO_FAILED;
371
372 uchar_t *modulus;
373 ssize_t modulus_len; /* In bytes */
374
375 if (!RSA_VALID_MECH(mechanism))
376 return (CRYPTO_MECHANISM_INVALID);
377
378 /*
379 * We only support RSA keys that are passed as a list of
380 * object attributes.
381 */
382 if (key->ck_format != CRYPTO_KEY_ATTR_LIST) {
383 return (CRYPTO_KEY_TYPE_INCONSISTENT);
384 }
385
386 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
387 &modulus_len)) != CRYPTO_SUCCESS) {
388 return (rv);
389 }
390 if (modulus_len < MIN_RSA_KEYLENGTH_IN_BYTES ||
391 modulus_len > MAX_RSA_KEYLENGTH_IN_BYTES)
392 return (CRYPTO_KEY_SIZE_RANGE);
393
394 return (rv);
395 }
396
397 void
398 kmemset(uint8_t *buf, char pattern, size_t len)
399 {
400 int i = 0;
401
402 while (i < len)
403 buf[i++] = pattern;
404 }
405
406 /*
407 * This function guarantees to return non-zero random numbers.
408 * This is needed as the /dev/urandom kernel interface,
409 * random_get_pseudo_bytes(), may return zeros.
410 */
411 int
412 knzero_random_generator(uint8_t *ran_out, size_t ran_len)
413 {
414 int rv;
415 size_t ebc = 0; /* count of extra bytes in extrarand */
416 size_t i = 0;
417 uint8_t extrarand[32];
418 size_t extrarand_len;
419
420 if ((rv = random_get_pseudo_bytes(ran_out, ran_len)) != 0)
421 return (rv);
422
423 /*
424 * Walk through the returned random numbers pointed by ran_out,
425 * and look for any random number which is zero.
426 * If we find zero, call random_get_pseudo_bytes() to generate
427 * another 32 random numbers pool. Replace any zeros in ran_out[]
428 * from the random number in pool.
429 */
430 while (i < ran_len) {
431 if (ran_out[i] != 0) {
432 i++;
433 continue;
434 }
435
436 /*
437 * Note that it is 'while' so we are guaranteed a
438 * non-zero value on exit.
439 */
440 if (ebc == 0) {
441 /* refresh extrarand */
442 extrarand_len = sizeof (extrarand);
443 if ((rv = random_get_pseudo_bytes(extrarand,
444 extrarand_len)) != 0) {
445 return (rv);
446 }
447
448 ebc = extrarand_len;
449 }
450 /* Replace zero with byte from extrarand. */
451 -- ebc;
452
453 /*
454 * The new random byte zero/non-zero will be checked in
455 * the next pass through the loop.
456 */
457 ran_out[i] = extrarand[ebc];
458 }
459
460 return (CRYPTO_SUCCESS);
461 }
462
463 static int
464 compare_data(crypto_data_t *data, uchar_t *buf)
465 {
466 int len;
467 uchar_t *dptr;
468
469 len = data->cd_length;
470 switch (data->cd_format) {
471 case CRYPTO_DATA_RAW:
472 dptr = (uchar_t *)(data->cd_raw.iov_base +
473 data->cd_offset);
474
475 return (bcmp(dptr, buf, len));
476
477 case CRYPTO_DATA_UIO:
478 return (crypto_uio_data(data, buf, len,
479 COMPARE_TO_DATA, NULL, NULL));
480
481 case CRYPTO_DATA_MBLK:
482 return (crypto_mblk_data(data, buf, len,
483 COMPARE_TO_DATA, NULL, NULL));
484 }
485
486 return (CRYPTO_FAILED);
487 }
488
489 /* ARGSUSED */
490 static int
491 rsa_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
492 crypto_key_t *key, crypto_spi_ctx_template_t template,
493 crypto_req_handle_t req)
494 {
495 int rv;
496 int kmflag;
497 rsa_ctx_t *ctxp;
498
499 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
500 return (rv);
501
502 /*
503 * Allocate a RSA context.
504 */
505 kmflag = crypto_kmflag(req);
506 if ((ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag)) == NULL)
507 return (CRYPTO_HOST_MEMORY);
508
509 if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
510 kmflag)) != CRYPTO_SUCCESS) {
511 kmem_free(ctxp, sizeof (rsa_ctx_t));
512 return (rv);
513 }
514 ctxp->mech_type = mechanism->cm_type;
515
516 ctx->cc_provider_private = ctxp;
517
518 return (CRYPTO_SUCCESS);
519 }
520
521 /* ARGSUSED */
522 static int
523 rsaprov_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
524 crypto_data_t *ciphertext, crypto_req_handle_t req)
525 {
526 int rv;
527 rsa_ctx_t *ctxp;
528
529 ASSERT(ctx->cc_provider_private != NULL);
530 ctxp = ctx->cc_provider_private;
531
532 RSA_ARG_INPLACE(plaintext, ciphertext);
533
534 /*
535 * Note on the KM_SLEEP flag passed to the routine below -
536 * rsaprov_encrypt() is a single-part encryption routine which is
537 * currently usable only by /dev/crypto. Since /dev/crypto calls are
538 * always synchronous, we can safely pass KM_SLEEP here.
539 */
540 rv = rsa_encrypt_common(ctxp->mech_type, ctxp->key, plaintext,
541 ciphertext);
542
543 if (rv != CRYPTO_BUFFER_TOO_SMALL)
544 (void) rsa_free_context(ctx);
545
546 return (rv);
547 }
548
549 /* ARGSUSED */
550 static int
551 rsa_encrypt_atomic(crypto_provider_handle_t provider,
552 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
553 crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
554 crypto_spi_ctx_template_t template, crypto_req_handle_t req)
555 {
556 int rv;
557
558 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
559 return (rv);
560 RSA_ARG_INPLACE(plaintext, ciphertext);
561
562 return (rsa_encrypt_common(mechanism->cm_type, key, plaintext,
563 ciphertext));
564 }
565
566 static int
567 rsa_free_context(crypto_ctx_t *ctx)
568 {
569 rsa_ctx_t *ctxp = ctx->cc_provider_private;
570
571 if (ctxp != NULL) {
572 bzero(ctxp->key, ctxp->keychunk_size);
573 kmem_free(ctxp->key, ctxp->keychunk_size);
574
575 if (ctxp->mech_type == RSA_PKCS_MECH_INFO_TYPE ||
576 ctxp->mech_type == RSA_X_509_MECH_INFO_TYPE)
577 kmem_free(ctxp, sizeof (rsa_ctx_t));
578 else
579 kmem_free(ctxp, sizeof (digest_rsa_ctx_t));
580
581 ctx->cc_provider_private = NULL;
582 }
583
584 return (CRYPTO_SUCCESS);
585 }
586
587 static int
588 rsa_encrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
589 crypto_data_t *plaintext, crypto_data_t *ciphertext)
590 {
591 int rv = CRYPTO_FAILED;
592
593 int plen;
594 uchar_t *ptptr;
595 uchar_t *modulus;
596 ssize_t modulus_len;
597 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
598 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
599 uchar_t cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
600
601 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
602 &modulus_len)) != CRYPTO_SUCCESS) {
603 return (rv);
604 }
605
606 plen = plaintext->cd_length;
607 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
608 if (plen > (modulus_len - MIN_PKCS1_PADLEN))
609 return (CRYPTO_DATA_LEN_RANGE);
610 } else {
611 if (plen > modulus_len)
612 return (CRYPTO_DATA_LEN_RANGE);
613 }
614
615 /*
616 * Output buf len must not be less than RSA modulus size.
617 */
618 if (ciphertext->cd_length < modulus_len) {
619 ciphertext->cd_length = modulus_len;
620 return (CRYPTO_BUFFER_TOO_SMALL);
621 }
622
623 ASSERT(plaintext->cd_length <= sizeof (tmp_data));
624 if ((rv = crypto_get_input_data(plaintext, &ptptr, tmp_data))
625 != CRYPTO_SUCCESS)
626 return (rv);
627
628 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
629 rv = pkcs1_encode(PKCS1_ENCRYPT, ptptr, plen,
630 plain_data, modulus_len);
631
632 if (rv != CRYPTO_SUCCESS)
633 return (rv);
634 } else {
635 bzero(plain_data, modulus_len - plen);
636 bcopy(ptptr, &plain_data[modulus_len - plen], plen);
637 }
638
639 rv = core_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
640 if (rv == CRYPTO_SUCCESS) {
641 /* copy out to ciphertext */
642 if ((rv = crypto_put_output_data(cipher_data,
643 ciphertext, modulus_len)) != CRYPTO_SUCCESS)
644 return (rv);
645
646 ciphertext->cd_length = modulus_len;
647 }
648
649 return (rv);
650 }
651
652 static int
653 core_rsa_encrypt(crypto_key_t *key, uchar_t *in,
654 int in_len, uchar_t *out, int is_public)
655 {
656 int rv;
657 uchar_t *expo, *modulus;
658 ssize_t expo_len;
659 ssize_t modulus_len;
660 RSAbytekey k;
661
662 if (is_public) {
663 if ((rv = crypto_get_key_attr(key, SUN_CKA_PUBLIC_EXPONENT,
664 &expo, &expo_len)) != CRYPTO_SUCCESS)
665 return (rv);
666 } else {
667 /*
668 * SUN_CKA_PRIVATE_EXPONENT is a required attribute for a
669 * RSA secret key. See the comments in core_rsa_decrypt
670 * routine which calls this routine with a private key.
671 */
672 if ((rv = crypto_get_key_attr(key, SUN_CKA_PRIVATE_EXPONENT,
673 &expo, &expo_len)) != CRYPTO_SUCCESS)
674 return (rv);
675 }
676
677 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
678 &modulus_len)) != CRYPTO_SUCCESS) {
679 return (rv);
680 }
681
682 k.modulus = modulus;
683 k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
684 k.pubexpo = expo;
685 k.pubexpo_bytes = expo_len;
686 k.rfunc = NULL;
687
688 rv = rsa_encrypt(&k, in, in_len, out);
689
690 return (rv);
691 }
692
693 /* ARGSUSED */
694 static int
695 rsaprov_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
696 crypto_data_t *plaintext, crypto_req_handle_t req)
697 {
698 int rv;
699 rsa_ctx_t *ctxp;
700
701 ASSERT(ctx->cc_provider_private != NULL);
702 ctxp = ctx->cc_provider_private;
703
704 RSA_ARG_INPLACE(ciphertext, plaintext);
705
706 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
707 rv = rsa_decrypt_common(ctxp->mech_type, ctxp->key,
708 ciphertext, plaintext);
709
710 if (rv != CRYPTO_BUFFER_TOO_SMALL)
711 (void) rsa_free_context(ctx);
712
713 return (rv);
714 }
715
716 /* ARGSUSED */
717 static int
718 rsa_decrypt_atomic(crypto_provider_handle_t provider,
719 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
720 crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
721 crypto_spi_ctx_template_t template, crypto_req_handle_t req)
722 {
723 int rv;
724
725 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
726 return (rv);
727 RSA_ARG_INPLACE(ciphertext, plaintext);
728
729 return (rsa_decrypt_common(mechanism->cm_type, key, ciphertext,
730 plaintext));
731 }
732
733 static int
734 rsa_decrypt_common(rsa_mech_type_t mech_type, crypto_key_t *key,
735 crypto_data_t *ciphertext, crypto_data_t *plaintext)
736 {
737 int rv = CRYPTO_FAILED;
738
739 size_t plain_len;
740 uchar_t *ctptr;
741 uchar_t *modulus;
742 ssize_t modulus_len;
743 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
744 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
745
746 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
747 &modulus_len)) != CRYPTO_SUCCESS) {
748 return (rv);
749 }
750
751 /*
752 * Ciphertext length must be equal to RSA modulus size.
753 */
754 if (ciphertext->cd_length != modulus_len)
755 return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
756
757 ASSERT(ciphertext->cd_length <= sizeof (tmp_data));
758 if ((rv = crypto_get_input_data(ciphertext, &ctptr, tmp_data))
759 != CRYPTO_SUCCESS)
760 return (rv);
761
762 rv = core_rsa_decrypt(key, ctptr, modulus_len, plain_data);
763 if (rv == CRYPTO_SUCCESS) {
764 plain_len = modulus_len;
765
766 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
767 /* Strip off the PKCS block formatting data. */
768 rv = pkcs1_decode(PKCS1_DECRYPT, plain_data,
769 &plain_len);
770 if (rv != CRYPTO_SUCCESS)
771 return (rv);
772 }
773
774 if (plain_len > plaintext->cd_length) {
775 plaintext->cd_length = plain_len;
776 return (CRYPTO_BUFFER_TOO_SMALL);
777 }
778
779 if ((rv = crypto_put_output_data(
780 plain_data + modulus_len - plain_len,
781 plaintext, plain_len)) != CRYPTO_SUCCESS)
782 return (rv);
783
784 plaintext->cd_length = plain_len;
785 }
786
787 return (rv);
788 }
789
790 static int
791 core_rsa_decrypt(crypto_key_t *key, uchar_t *in, int in_len, uchar_t *out)
792 {
793 int rv;
794 uchar_t *modulus, *prime1, *prime2, *expo1, *expo2, *coef;
795 ssize_t modulus_len;
796 ssize_t prime1_len, prime2_len;
797 ssize_t expo1_len, expo2_len, coef_len;
798 RSAbytekey k;
799
800 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
801 &modulus_len)) != CRYPTO_SUCCESS) {
802 return (rv);
803 }
804
805 /*
806 * The following attributes are not required to be
807 * present in a RSA secret key. If any of them is not present
808 * we call the encrypt routine with a flag indicating use of
809 * private exponent (d). Note that SUN_CKA_PRIVATE_EXPONENT is
810 * a required attribute for a RSA secret key.
811 */
812 if ((crypto_get_key_attr(key, SUN_CKA_PRIME_1, &prime1, &prime1_len)
813 != CRYPTO_SUCCESS) ||
814 (crypto_get_key_attr(key, SUN_CKA_PRIME_2, &prime2, &prime2_len)
815 != CRYPTO_SUCCESS) ||
816 (crypto_get_key_attr(key, SUN_CKA_EXPONENT_1, &expo1, &expo1_len)
817 != CRYPTO_SUCCESS) ||
818 (crypto_get_key_attr(key, SUN_CKA_EXPONENT_2, &expo2, &expo2_len)
819 != CRYPTO_SUCCESS) ||
820 (crypto_get_key_attr(key, SUN_CKA_COEFFICIENT, &coef, &coef_len)
821 != CRYPTO_SUCCESS)) {
822 return (core_rsa_encrypt(key, in, in_len, out, 0));
823 }
824
825 k.modulus = modulus;
826 k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
827 k.prime1 = prime1;
828 k.prime1_bytes = prime1_len;
829 k.prime2 = prime2;
830 k.prime2_bytes = prime2_len;
831 k.expo1 = expo1;
832 k.expo1_bytes = expo1_len;
833 k.expo2 = expo2;
834 k.expo2_bytes = expo2_len;
835 k.coeff = coef;
836 k.coeff_bytes = coef_len;
837 k.rfunc = NULL;
838
839 rv = rsa_decrypt(&k, in, in_len, out);
840
841 return (rv);
842 }
843
844 /* ARGSUSED */
845 static int
846 rsa_sign_verify_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
847 crypto_key_t *key, crypto_spi_ctx_template_t ctx_template,
848 crypto_req_handle_t req)
849 {
850 int rv;
851 int kmflag;
852 rsa_ctx_t *ctxp;
853 digest_rsa_ctx_t *dctxp;
854
855 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
856 return (rv);
857
858 /*
859 * Allocate a RSA context.
860 */
861 kmflag = crypto_kmflag(req);
862 switch (mechanism->cm_type) {
863 case MD5_RSA_PKCS_MECH_INFO_TYPE:
864 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
865 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
866 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
867 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
868 dctxp = kmem_zalloc(sizeof (digest_rsa_ctx_t), kmflag);
869 ctxp = (rsa_ctx_t *)dctxp;
870 break;
871 default:
872 ctxp = kmem_zalloc(sizeof (rsa_ctx_t), kmflag);
873 break;
874 }
875
876 if (ctxp == NULL)
877 return (CRYPTO_HOST_MEMORY);
878
879 ctxp->mech_type = mechanism->cm_type;
880 if ((rv = crypto_copy_key_to_ctx(key, &ctxp->key, &ctxp->keychunk_size,
881 kmflag)) != CRYPTO_SUCCESS) {
882 switch (mechanism->cm_type) {
883 case MD5_RSA_PKCS_MECH_INFO_TYPE:
884 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
885 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
886 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
887 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
888 kmem_free(dctxp, sizeof (digest_rsa_ctx_t));
889 break;
890 default:
891 kmem_free(ctxp, sizeof (rsa_ctx_t));
892 break;
893 }
894 return (rv);
895 }
896
897 switch (mechanism->cm_type) {
898 case MD5_RSA_PKCS_MECH_INFO_TYPE:
899 MD5Init(&(dctxp->md5_ctx));
900 break;
901
902 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
903 SHA1Init(&(dctxp->sha1_ctx));
904 break;
905
906 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
907 SHA2Init(SHA256, &(dctxp->sha2_ctx));
908 break;
909
910 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
911 SHA2Init(SHA384, &(dctxp->sha2_ctx));
912 break;
913
914 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
915 SHA2Init(SHA512, &(dctxp->sha2_ctx));
916 break;
917 }
918
919 ctx->cc_provider_private = ctxp;
920
921 return (CRYPTO_SUCCESS);
922 }
923
924 #define SHA1_DIGEST_SIZE 20
925 #define MD5_DIGEST_SIZE 16
926
927 #define INIT_RAW_CRYPTO_DATA(data, base, len, cd_len) \
928 (data).cd_format = CRYPTO_DATA_RAW; \
929 (data).cd_offset = 0; \
930 (data).cd_raw.iov_base = (char *)base; \
931 (data).cd_raw.iov_len = len; \
932 (data).cd_length = cd_len;
933
934 static int
935 rsa_digest_svrfy_common(digest_rsa_ctx_t *ctxp, crypto_data_t *data,
936 crypto_data_t *signature, uchar_t flag)
937 {
938 int rv = CRYPTO_FAILED;
939
940 uchar_t digest[SHA512_DIGEST_LENGTH];
941 /* The der_data size is enough for MD5 also */
942 uchar_t der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
943 ulong_t der_data_len;
944 crypto_data_t der_cd;
945 rsa_mech_type_t mech_type;
946
947 ASSERT(flag & CRYPTO_DO_SIGN || flag & CRYPTO_DO_VERIFY);
948 ASSERT(data != NULL || (flag & CRYPTO_DO_FINAL));
949
950 mech_type = ctxp->mech_type;
951 if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
952 mech_type == RSA_X_509_MECH_INFO_TYPE)
953 return (CRYPTO_MECHANISM_INVALID);
954
955 /*
956 * We need to do the BUFFER_TOO_SMALL check before digesting
957 * the data. No check is needed for verify as signature is not
958 * an output argument for verify.
959 */
960 if (flag & CRYPTO_DO_SIGN) {
961 uchar_t *modulus;
962 ssize_t modulus_len;
963
964 if ((rv = crypto_get_key_attr(ctxp->key, SUN_CKA_MODULUS,
965 &modulus, &modulus_len)) != CRYPTO_SUCCESS) {
966 return (rv);
967 }
968
969 if (signature->cd_length < modulus_len) {
970 signature->cd_length = modulus_len;
971 return (CRYPTO_BUFFER_TOO_SMALL);
972 }
973 }
974
975 if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
976 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
977 digest, MD5Update, MD5Final, flag | CRYPTO_DO_MD5);
978
979 else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
980 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
981 digest, SHA1Update, SHA1Final, flag | CRYPTO_DO_SHA1);
982
983 else
984 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
985 digest, SHA2Update, SHA2Final, flag | CRYPTO_DO_SHA2);
986
987 if (rv != CRYPTO_SUCCESS)
988 return (rv);
989
990
991 /*
992 * Prepare the DER encoding of the DigestInfo value as follows:
993 * MD5: MD5_DER_PREFIX || H
994 * SHA-1: SHA1_DER_PREFIX || H
995 *
996 * See rsa_impl.c for more details.
997 */
998 switch (mech_type) {
999 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1000 bcopy(MD5_DER_PREFIX, der_data, MD5_DER_PREFIX_Len);
1001 bcopy(digest, der_data + MD5_DER_PREFIX_Len, MD5_DIGEST_SIZE);
1002 der_data_len = MD5_DER_PREFIX_Len + MD5_DIGEST_SIZE;
1003 break;
1004
1005 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1006 bcopy(SHA1_DER_PREFIX, der_data, SHA1_DER_PREFIX_Len);
1007 bcopy(digest, der_data + SHA1_DER_PREFIX_Len,
1008 SHA1_DIGEST_SIZE);
1009 der_data_len = SHA1_DER_PREFIX_Len + SHA1_DIGEST_SIZE;
1010 break;
1011
1012 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1013 bcopy(SHA256_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1014 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1015 SHA256_DIGEST_LENGTH);
1016 der_data_len = SHA2_DER_PREFIX_Len + SHA256_DIGEST_LENGTH;
1017 break;
1018
1019 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1020 bcopy(SHA384_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1021 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1022 SHA384_DIGEST_LENGTH);
1023 der_data_len = SHA2_DER_PREFIX_Len + SHA384_DIGEST_LENGTH;
1024 break;
1025
1026 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1027 bcopy(SHA512_DER_PREFIX, der_data, SHA2_DER_PREFIX_Len);
1028 bcopy(digest, der_data + SHA2_DER_PREFIX_Len,
1029 SHA512_DIGEST_LENGTH);
1030 der_data_len = SHA2_DER_PREFIX_Len + SHA512_DIGEST_LENGTH;
1031 break;
1032 }
1033
1034 INIT_RAW_CRYPTO_DATA(der_cd, der_data, der_data_len, der_data_len);
1035 /*
1036 * Now, we are ready to sign or verify the DER_ENCODED data.
1037 */
1038 if (flag & CRYPTO_DO_SIGN)
1039 rv = rsa_sign_common(mech_type, ctxp->key, &der_cd,
1040 signature);
1041 else
1042 rv = rsa_verify_common(mech_type, ctxp->key, &der_cd,
1043 signature);
1044
1045 return (rv);
1046 }
1047
1048 static int
1049 rsa_sign_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1050 crypto_data_t *data, crypto_data_t *signature)
1051 {
1052 int rv = CRYPTO_FAILED;
1053
1054 int dlen;
1055 uchar_t *dataptr, *modulus;
1056 ssize_t modulus_len;
1057 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1058 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1059 uchar_t signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1060
1061 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1062 &modulus_len)) != CRYPTO_SUCCESS) {
1063 return (rv);
1064 }
1065
1066 dlen = data->cd_length;
1067 switch (mech_type) {
1068 case RSA_PKCS_MECH_INFO_TYPE:
1069 if (dlen > (modulus_len - MIN_PKCS1_PADLEN))
1070 return (CRYPTO_DATA_LEN_RANGE);
1071 break;
1072 case RSA_X_509_MECH_INFO_TYPE:
1073 if (dlen > modulus_len)
1074 return (CRYPTO_DATA_LEN_RANGE);
1075 break;
1076 }
1077
1078 if (signature->cd_length < modulus_len) {
1079 signature->cd_length = modulus_len;
1080 return (CRYPTO_BUFFER_TOO_SMALL);
1081 }
1082
1083 ASSERT(data->cd_length <= sizeof (tmp_data));
1084 if ((rv = crypto_get_input_data(data, &dataptr, tmp_data))
1085 != CRYPTO_SUCCESS)
1086 return (rv);
1087
1088 switch (mech_type) {
1089 case RSA_PKCS_MECH_INFO_TYPE:
1090 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1091 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1092 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1093 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1094 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1095 /*
1096 * Add PKCS padding to the input data to format a block
1097 * type "01" encryption block.
1098 */
1099 rv = pkcs1_encode(PKCS1_SIGN, dataptr, dlen, plain_data,
1100 modulus_len);
1101 if (rv != CRYPTO_SUCCESS)
1102 return (rv);
1103
1104 break;
1105
1106 case RSA_X_509_MECH_INFO_TYPE:
1107 bzero(plain_data, modulus_len - dlen);
1108 bcopy(dataptr, &plain_data[modulus_len - dlen], dlen);
1109 break;
1110 }
1111
1112 rv = core_rsa_decrypt(key, plain_data, modulus_len, signed_data);
1113 if (rv == CRYPTO_SUCCESS) {
1114 /* copy out to signature */
1115 if ((rv = crypto_put_output_data(signed_data,
1116 signature, modulus_len)) != CRYPTO_SUCCESS)
1117 return (rv);
1118
1119 signature->cd_length = modulus_len;
1120 }
1121
1122 return (rv);
1123 }
1124
1125 /* ARGSUSED */
1126 static int
1127 rsaprov_sign(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *signature,
1128 crypto_req_handle_t req)
1129 {
1130 int rv;
1131 rsa_ctx_t *ctxp;
1132
1133 ASSERT(ctx->cc_provider_private != NULL);
1134 ctxp = ctx->cc_provider_private;
1135
1136 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1137 switch (ctxp->mech_type) {
1138 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1139 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1140 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1141 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1142 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1143 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1144 signature, CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE |
1145 CRYPTO_DO_FINAL);
1146 break;
1147 default:
1148 rv = rsa_sign_common(ctxp->mech_type, ctxp->key, data,
1149 signature);
1150 break;
1151 }
1152
1153 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1154 (void) rsa_free_context(ctx);
1155
1156 return (rv);
1157 }
1158
1159 /* ARGSUSED */
1160 static int
1161 rsa_sign_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req)
1162 {
1163 int rv;
1164 digest_rsa_ctx_t *ctxp;
1165 rsa_mech_type_t mech_type;
1166
1167 ASSERT(ctx->cc_provider_private != NULL);
1168 ctxp = ctx->cc_provider_private;
1169 mech_type = ctxp->mech_type;
1170
1171 if (mech_type == RSA_PKCS_MECH_INFO_TYPE ||
1172 mech_type == RSA_X_509_MECH_INFO_TYPE)
1173 return (CRYPTO_MECHANISM_INVALID);
1174
1175 if (mech_type == MD5_RSA_PKCS_MECH_INFO_TYPE)
1176 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1177 NULL, MD5Update, MD5Final,
1178 CRYPTO_DO_MD5 | CRYPTO_DO_UPDATE);
1179
1180 else if (mech_type == SHA1_RSA_PKCS_MECH_INFO_TYPE)
1181 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1182 NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1183 CRYPTO_DO_UPDATE);
1184
1185 else
1186 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1187 NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1188 CRYPTO_DO_UPDATE);
1189
1190 return (rv);
1191 }
1192
1193 /* ARGSUSED2 */
1194 static int
1195 rsa_sign_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1196 crypto_req_handle_t req)
1197 {
1198 int rv;
1199 digest_rsa_ctx_t *ctxp;
1200
1201 ASSERT(ctx->cc_provider_private != NULL);
1202 ctxp = ctx->cc_provider_private;
1203
1204 rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1205 CRYPTO_DO_SIGN | CRYPTO_DO_FINAL);
1206 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1207 (void) rsa_free_context(ctx);
1208
1209 return (rv);
1210 }
1211
1212 /* ARGSUSED */
1213 static int
1214 rsa_sign_atomic(crypto_provider_handle_t provider,
1215 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1216 crypto_key_t *key, crypto_data_t *data, crypto_data_t *signature,
1217 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1218 {
1219 int rv;
1220 digest_rsa_ctx_t dctx;
1221
1222 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1223 return (rv);
1224
1225 if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1226 mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1227 rv = rsa_sign_common(mechanism->cm_type, key, data,
1228 signature);
1229
1230 else {
1231 dctx.mech_type = mechanism->cm_type;
1232 dctx.key = key;
1233 switch (mechanism->cm_type) {
1234 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1235 MD5Init(&(dctx.md5_ctx));
1236 break;
1237
1238 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1239 SHA1Init(&(dctx.sha1_ctx));
1240 break;
1241
1242 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1243 SHA2Init(SHA256, &(dctx.sha2_ctx));
1244 break;
1245
1246 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1247 SHA2Init(SHA384, &(dctx.sha2_ctx));
1248 break;
1249
1250 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1251 SHA2Init(SHA512, &(dctx.sha2_ctx));
1252 break;
1253 }
1254
1255 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1256 CRYPTO_DO_SIGN | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1257 }
1258
1259 return (rv);
1260 }
1261
1262 static int
1263 rsa_verify_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1264 crypto_data_t *data, crypto_data_t *signature)
1265 {
1266 int rv = CRYPTO_FAILED;
1267
1268 uchar_t *sigptr, *modulus;
1269 ssize_t modulus_len;
1270 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1271 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1272
1273 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1274 &modulus_len)) != CRYPTO_SUCCESS) {
1275 return (rv);
1276 }
1277
1278 if (signature->cd_length != modulus_len)
1279 return (CRYPTO_SIGNATURE_LEN_RANGE);
1280
1281 ASSERT(signature->cd_length <= sizeof (tmp_data));
1282 if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1283 != CRYPTO_SUCCESS)
1284 return (rv);
1285
1286 rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1287 if (rv != CRYPTO_SUCCESS)
1288 return (rv);
1289
1290 if (mech_type == RSA_X_509_MECH_INFO_TYPE) {
1291 if (compare_data(data, (plain_data + modulus_len
1292 - data->cd_length)) != 0)
1293 rv = CRYPTO_SIGNATURE_INVALID;
1294
1295 } else {
1296 size_t data_len = modulus_len;
1297
1298 /*
1299 * Strip off the encoded padding bytes in front of the
1300 * recovered data, then compare the recovered data with
1301 * the original data.
1302 */
1303 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1304 if (rv != CRYPTO_SUCCESS)
1305 return (rv);
1306
1307 if (data_len != data->cd_length)
1308 return (CRYPTO_SIGNATURE_LEN_RANGE);
1309
1310 if (compare_data(data, (plain_data + modulus_len
1311 - data_len)) != 0)
1312 rv = CRYPTO_SIGNATURE_INVALID;
1313 }
1314
1315 return (rv);
1316 }
1317
1318 /* ARGSUSED */
1319 static int
1320 rsaprov_verify(crypto_ctx_t *ctx, crypto_data_t *data,
1321 crypto_data_t *signature, crypto_req_handle_t req)
1322 {
1323 int rv;
1324 rsa_ctx_t *ctxp;
1325
1326 ASSERT(ctx->cc_provider_private != NULL);
1327 ctxp = ctx->cc_provider_private;
1328
1329 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1330 switch (ctxp->mech_type) {
1331 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1332 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1333 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1334 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1335 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1336 rv = rsa_digest_svrfy_common((digest_rsa_ctx_t *)ctxp, data,
1337 signature, CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE |
1338 CRYPTO_DO_FINAL);
1339 break;
1340 default:
1341 rv = rsa_verify_common(ctxp->mech_type, ctxp->key, data,
1342 signature);
1343 break;
1344 }
1345
1346 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1347 (void) rsa_free_context(ctx);
1348
1349 return (rv);
1350 }
1351
1352 /* ARGSUSED */
1353 static int
1354 rsa_verify_update(crypto_ctx_t *ctx, crypto_data_t *data,
1355 crypto_req_handle_t req)
1356 {
1357 int rv;
1358 digest_rsa_ctx_t *ctxp;
1359
1360 ASSERT(ctx->cc_provider_private != NULL);
1361 ctxp = ctx->cc_provider_private;
1362
1363 switch (ctxp->mech_type) {
1364
1365 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1366 rv = crypto_digest_data(data, &(ctxp->md5_ctx),
1367 NULL, MD5Update, MD5Final, CRYPTO_DO_MD5 |
1368 CRYPTO_DO_UPDATE);
1369 break;
1370
1371 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1372 rv = crypto_digest_data(data, &(ctxp->sha1_ctx),
1373 NULL, SHA1Update, SHA1Final, CRYPTO_DO_SHA1 |
1374 CRYPTO_DO_UPDATE);
1375 break;
1376
1377 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1378 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1379 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1380 rv = crypto_digest_data(data, &(ctxp->sha2_ctx),
1381 NULL, SHA2Update, SHA2Final, CRYPTO_DO_SHA2 |
1382 CRYPTO_DO_UPDATE);
1383 break;
1384
1385 default:
1386 return (CRYPTO_MECHANISM_INVALID);
1387 }
1388
1389 return (rv);
1390 }
1391
1392 /* ARGSUSED2 */
1393 static int
1394 rsa_verify_final(crypto_ctx_t *ctx, crypto_data_t *signature,
1395 crypto_req_handle_t req)
1396 {
1397 int rv;
1398 digest_rsa_ctx_t *ctxp;
1399
1400 ASSERT(ctx->cc_provider_private != NULL);
1401 ctxp = ctx->cc_provider_private;
1402
1403 rv = rsa_digest_svrfy_common(ctxp, NULL, signature,
1404 CRYPTO_DO_VERIFY | CRYPTO_DO_FINAL);
1405 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1406 (void) rsa_free_context(ctx);
1407
1408 return (rv);
1409 }
1410
1411
1412 /* ARGSUSED */
1413 static int
1414 rsa_verify_atomic(crypto_provider_handle_t provider,
1415 crypto_session_id_t session_id,
1416 crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data,
1417 crypto_data_t *signature, crypto_spi_ctx_template_t ctx_template,
1418 crypto_req_handle_t req)
1419 {
1420 int rv;
1421 digest_rsa_ctx_t dctx;
1422
1423 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1424 return (rv);
1425
1426 if (mechanism->cm_type == RSA_PKCS_MECH_INFO_TYPE ||
1427 mechanism->cm_type == RSA_X_509_MECH_INFO_TYPE)
1428 rv = rsa_verify_common(mechanism->cm_type, key, data,
1429 signature);
1430
1431 else {
1432 dctx.mech_type = mechanism->cm_type;
1433 dctx.key = key;
1434
1435 switch (mechanism->cm_type) {
1436 case MD5_RSA_PKCS_MECH_INFO_TYPE:
1437 MD5Init(&(dctx.md5_ctx));
1438 break;
1439
1440 case SHA1_RSA_PKCS_MECH_INFO_TYPE:
1441 SHA1Init(&(dctx.sha1_ctx));
1442 break;
1443
1444 case SHA256_RSA_PKCS_MECH_INFO_TYPE:
1445 SHA2Init(SHA256, &(dctx.sha2_ctx));
1446 break;
1447
1448 case SHA384_RSA_PKCS_MECH_INFO_TYPE:
1449 SHA2Init(SHA384, &(dctx.sha2_ctx));
1450 break;
1451
1452 case SHA512_RSA_PKCS_MECH_INFO_TYPE:
1453 SHA2Init(SHA512, &(dctx.sha2_ctx));
1454 break;
1455 }
1456
1457 rv = rsa_digest_svrfy_common(&dctx, data, signature,
1458 CRYPTO_DO_VERIFY | CRYPTO_DO_UPDATE | CRYPTO_DO_FINAL);
1459 }
1460
1461 return (rv);
1462 }
1463
1464 static int
1465 rsa_verify_recover_common(rsa_mech_type_t mech_type, crypto_key_t *key,
1466 crypto_data_t *signature, crypto_data_t *data)
1467 {
1468 int rv = CRYPTO_FAILED;
1469
1470 size_t data_len;
1471 uchar_t *sigptr, *modulus;
1472 ssize_t modulus_len;
1473 uchar_t plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1474 uchar_t tmp_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1475
1476 if ((rv = crypto_get_key_attr(key, SUN_CKA_MODULUS, &modulus,
1477 &modulus_len)) != CRYPTO_SUCCESS) {
1478 return (rv);
1479 }
1480
1481 if (signature->cd_length != modulus_len)
1482 return (CRYPTO_SIGNATURE_LEN_RANGE);
1483
1484 ASSERT(signature->cd_length <= sizeof (tmp_data));
1485 if ((rv = crypto_get_input_data(signature, &sigptr, tmp_data))
1486 != CRYPTO_SUCCESS)
1487 return (rv);
1488
1489 rv = core_rsa_encrypt(key, sigptr, modulus_len, plain_data, 1);
1490 if (rv != CRYPTO_SUCCESS)
1491 return (rv);
1492
1493 data_len = modulus_len;
1494
1495 if (mech_type == RSA_PKCS_MECH_INFO_TYPE) {
1496 /*
1497 * Strip off the encoded padding bytes in front of the
1498 * recovered data, then compare the recovered data with
1499 * the original data.
1500 */
1501 rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1502 if (rv != CRYPTO_SUCCESS)
1503 return (rv);
1504 }
1505
1506 if (data->cd_length < data_len) {
1507 data->cd_length = data_len;
1508 return (CRYPTO_BUFFER_TOO_SMALL);
1509 }
1510
1511 if ((rv = crypto_put_output_data(plain_data + modulus_len - data_len,
1512 data, data_len)) != CRYPTO_SUCCESS)
1513 return (rv);
1514 data->cd_length = data_len;
1515
1516 return (rv);
1517 }
1518
1519 /* ARGSUSED */
1520 static int
1521 rsa_verify_recover(crypto_ctx_t *ctx, crypto_data_t *signature,
1522 crypto_data_t *data, crypto_req_handle_t req)
1523 {
1524 int rv;
1525 rsa_ctx_t *ctxp;
1526
1527 ASSERT(ctx->cc_provider_private != NULL);
1528 ctxp = ctx->cc_provider_private;
1529
1530 /* See the comments on KM_SLEEP flag in rsaprov_encrypt() */
1531 rv = rsa_verify_recover_common(ctxp->mech_type, ctxp->key,
1532 signature, data);
1533
1534 if (rv != CRYPTO_BUFFER_TOO_SMALL)
1535 (void) rsa_free_context(ctx);
1536
1537 return (rv);
1538 }
1539
1540 /* ARGSUSED */
1541 static int
1542 rsa_verify_recover_atomic(crypto_provider_handle_t provider,
1543 crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1544 crypto_key_t *key, crypto_data_t *signature, crypto_data_t *data,
1545 crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req)
1546 {
1547 int rv;
1548
1549 if ((rv = check_mech_and_key(mechanism, key)) != CRYPTO_SUCCESS)
1550 return (rv);
1551
1552 return (rsa_verify_recover_common(mechanism->cm_type, key,
1553 signature, data));
1554 }