1 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
3 * Copyright (C) 2010 David S. Miller <davem@davemloft.net>
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/kernel.h>
9 #include <linux/module.h>
11 #include <linux/of_device.h>
12 #include <linux/cpumask.h>
13 #include <linux/slab.h>
14 #include <linux/interrupt.h>
15 #include <linux/crypto.h>
16 #include <crypto/md5.h>
17 #include <crypto/sha.h>
18 #include <crypto/aes.h>
19 #include <crypto/des.h>
20 #include <linux/mutex.h>
21 #include <linux/delay.h>
22 #include <linux/sched.h>
24 #include <crypto/internal/hash.h>
25 #include <crypto/scatterwalk.h>
26 #include <crypto/algapi.h>
28 #include <asm/hypervisor.h>
29 #include <asm/mdesc.h>
33 #define DRV_MODULE_NAME "n2_crypto"
34 #define DRV_MODULE_VERSION "0.1"
35 #define DRV_MODULE_RELDATE "April 29, 2010"
37 static char version
[] __devinitdata
=
38 DRV_MODULE_NAME
".c:v" DRV_MODULE_VERSION
" (" DRV_MODULE_RELDATE
")\n";
40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
41 MODULE_DESCRIPTION("Niagara2 Crypto driver");
42 MODULE_LICENSE("GPL");
43 MODULE_VERSION(DRV_MODULE_VERSION
);
45 #define N2_CRA_PRIORITY 300
47 static DEFINE_MUTEX(spu_lock
);
51 unsigned long qhandle
;
58 struct list_head jobs
;
65 struct list_head list
;
68 static struct spu_queue
**cpu_to_cwq
;
69 static struct spu_queue
**cpu_to_mau
;
71 static unsigned long spu_next_offset(struct spu_queue
*q
, unsigned long off
)
73 if (q
->q_type
== HV_NCS_QTYPE_MAU
) {
74 off
+= MAU_ENTRY_SIZE
;
75 if (off
== (MAU_ENTRY_SIZE
* MAU_NUM_ENTRIES
))
78 off
+= CWQ_ENTRY_SIZE
;
79 if (off
== (CWQ_ENTRY_SIZE
* CWQ_NUM_ENTRIES
))
85 struct n2_request_common
{
86 struct list_head entry
;
89 #define OFFSET_NOT_RUNNING (~(unsigned int)0)
91 /* An async job request records the final tail value it used in
92 * n2_request_common->offset, test to see if that offset is in
93 * the range old_head, new_head, inclusive.
95 static inline bool job_finished(struct spu_queue
*q
, unsigned int offset
,
96 unsigned long old_head
, unsigned long new_head
)
98 if (old_head
<= new_head
) {
99 if (offset
> old_head
&& offset
<= new_head
)
102 if (offset
> old_head
|| offset
<= new_head
)
108 /* When the HEAD marker is unequal to the actual HEAD, we get
109 * a virtual device INO interrupt. We should process the
110 * completed CWQ entries and adjust the HEAD marker to clear
113 static irqreturn_t
cwq_intr(int irq
, void *dev_id
)
115 unsigned long off
, new_head
, hv_ret
;
116 struct spu_queue
*q
= dev_id
;
118 pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
119 smp_processor_id(), q
->qhandle
);
123 hv_ret
= sun4v_ncs_gethead(q
->qhandle
, &new_head
);
125 pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
126 smp_processor_id(), new_head
, hv_ret
);
128 for (off
= q
->head
; off
!= new_head
; off
= spu_next_offset(q
, off
)) {
132 hv_ret
= sun4v_ncs_sethead_marker(q
->qhandle
, new_head
);
133 if (hv_ret
== HV_EOK
)
136 spin_unlock(&q
->lock
);
141 static irqreturn_t
mau_intr(int irq
, void *dev_id
)
143 struct spu_queue
*q
= dev_id
;
144 unsigned long head
, hv_ret
;
148 pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
149 smp_processor_id(), q
->qhandle
);
151 hv_ret
= sun4v_ncs_gethead(q
->qhandle
, &head
);
153 pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
154 smp_processor_id(), head
, hv_ret
);
156 sun4v_ncs_sethead_marker(q
->qhandle
, head
);
158 spin_unlock(&q
->lock
);
163 static void *spu_queue_next(struct spu_queue
*q
, void *cur
)
165 return q
->q
+ spu_next_offset(q
, cur
- q
->q
);
168 static int spu_queue_num_free(struct spu_queue
*q
)
170 unsigned long head
= q
->head
;
171 unsigned long tail
= q
->tail
;
172 unsigned long end
= (CWQ_ENTRY_SIZE
* CWQ_NUM_ENTRIES
);
178 diff
= (end
- tail
) + head
;
180 return (diff
/ CWQ_ENTRY_SIZE
) - 1;
183 static void *spu_queue_alloc(struct spu_queue
*q
, int num_entries
)
185 int avail
= spu_queue_num_free(q
);
187 if (avail
>= num_entries
)
188 return q
->q
+ q
->tail
;
193 static unsigned long spu_queue_submit(struct spu_queue
*q
, void *last
)
195 unsigned long hv_ret
, new_tail
;
197 new_tail
= spu_next_offset(q
, last
- q
->q
);
199 hv_ret
= sun4v_ncs_settail(q
->qhandle
, new_tail
);
200 if (hv_ret
== HV_EOK
)
205 static u64
control_word_base(unsigned int len
, unsigned int hmac_key_len
,
206 int enc_type
, int auth_type
,
207 unsigned int hash_len
,
208 bool sfas
, bool sob
, bool eob
, bool encrypt
,
211 u64 word
= (len
- 1) & CONTROL_LEN
;
213 word
|= ((u64
) opcode
<< CONTROL_OPCODE_SHIFT
);
214 word
|= ((u64
) enc_type
<< CONTROL_ENC_TYPE_SHIFT
);
215 word
|= ((u64
) auth_type
<< CONTROL_AUTH_TYPE_SHIFT
);
217 word
|= CONTROL_STORE_FINAL_AUTH_STATE
;
219 word
|= CONTROL_START_OF_BLOCK
;
221 word
|= CONTROL_END_OF_BLOCK
;
223 word
|= CONTROL_ENCRYPT
;
225 word
|= ((u64
) (hmac_key_len
- 1)) << CONTROL_HMAC_KEY_LEN_SHIFT
;
227 word
|= ((u64
) (hash_len
- 1)) << CONTROL_HASH_LEN_SHIFT
;
233 static inline bool n2_should_run_async(struct spu_queue
*qp
, int this_len
)
235 if (this_len
>= 64 ||
236 qp
->head
!= qp
->tail
)
243 struct list_head list
;
246 static void n2_base_ctx_init(struct n2_base_ctx
*ctx
)
248 INIT_LIST_HEAD(&ctx
->list
);
252 struct n2_base_ctx base
;
254 struct crypto_ahash
*fallback_tfm
;
257 struct n2_hash_req_ctx
{
259 struct md5_state md5
;
260 struct sha1_state sha1
;
261 struct sha256_state sha256
;
264 unsigned char hash_key
[64];
265 unsigned char keyed_zero_hash
[32];
267 struct ahash_request fallback_req
;
270 static int n2_hash_async_init(struct ahash_request
*req
)
272 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
273 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
274 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
276 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
277 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
279 return crypto_ahash_init(&rctx
->fallback_req
);
282 static int n2_hash_async_update(struct ahash_request
*req
)
284 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
285 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
286 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
288 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
289 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
290 rctx
->fallback_req
.nbytes
= req
->nbytes
;
291 rctx
->fallback_req
.src
= req
->src
;
293 return crypto_ahash_update(&rctx
->fallback_req
);
296 static int n2_hash_async_final(struct ahash_request
*req
)
298 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
299 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
300 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
302 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
303 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
304 rctx
->fallback_req
.result
= req
->result
;
306 return crypto_ahash_final(&rctx
->fallback_req
);
309 static int n2_hash_async_finup(struct ahash_request
*req
)
311 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
312 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
313 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
315 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
316 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
317 rctx
->fallback_req
.nbytes
= req
->nbytes
;
318 rctx
->fallback_req
.src
= req
->src
;
319 rctx
->fallback_req
.result
= req
->result
;
321 return crypto_ahash_finup(&rctx
->fallback_req
);
324 static int n2_hash_cra_init(struct crypto_tfm
*tfm
)
326 const char *fallback_driver_name
= tfm
->__crt_alg
->cra_name
;
327 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
328 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(ahash
);
329 struct crypto_ahash
*fallback_tfm
;
332 fallback_tfm
= crypto_alloc_ahash(fallback_driver_name
, 0,
333 CRYPTO_ALG_NEED_FALLBACK
);
334 if (IS_ERR(fallback_tfm
)) {
335 pr_warning("Fallback driver '%s' could not be loaded!\n",
336 fallback_driver_name
);
337 err
= PTR_ERR(fallback_tfm
);
341 crypto_ahash_set_reqsize(ahash
, (sizeof(struct n2_hash_req_ctx
) +
342 crypto_ahash_reqsize(fallback_tfm
)));
344 ctx
->fallback_tfm
= fallback_tfm
;
351 static void n2_hash_cra_exit(struct crypto_tfm
*tfm
)
353 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
354 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(ahash
);
356 crypto_free_ahash(ctx
->fallback_tfm
);
359 static unsigned long wait_for_tail(struct spu_queue
*qp
)
361 unsigned long head
, hv_ret
;
364 hv_ret
= sun4v_ncs_gethead(qp
->qhandle
, &head
);
365 if (hv_ret
!= HV_EOK
) {
366 pr_err("Hypervisor error on gethead\n");
369 if (head
== qp
->tail
) {
377 static unsigned long submit_and_wait_for_tail(struct spu_queue
*qp
,
378 struct cwq_initial_entry
*ent
)
380 unsigned long hv_ret
= spu_queue_submit(qp
, ent
);
382 if (hv_ret
== HV_EOK
)
383 hv_ret
= wait_for_tail(qp
);
388 static int n2_hash_async_digest(struct ahash_request
*req
,
389 unsigned int auth_type
, unsigned int digest_size
,
390 unsigned int result_size
, void *hash_loc
)
392 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
393 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
394 struct cwq_initial_entry
*ent
;
395 struct crypto_hash_walk walk
;
396 struct spu_queue
*qp
;
401 /* The total effective length of the operation may not
404 if (unlikely(req
->nbytes
> (1 << 16))) {
405 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
407 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
408 rctx
->fallback_req
.base
.flags
=
409 req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
410 rctx
->fallback_req
.nbytes
= req
->nbytes
;
411 rctx
->fallback_req
.src
= req
->src
;
412 rctx
->fallback_req
.result
= req
->result
;
414 return crypto_ahash_digest(&rctx
->fallback_req
);
417 n2_base_ctx_init(&ctx
->base
);
419 nbytes
= crypto_hash_walk_first(req
, &walk
);
422 qp
= cpu_to_cwq
[cpu
];
426 spin_lock_irqsave(&qp
->lock
, flags
);
428 /* XXX can do better, improve this later by doing a by-hand scatterlist
431 ent
= qp
->q
+ qp
->tail
;
433 ent
->control
= control_word_base(nbytes
, 0, 0,
434 auth_type
, digest_size
,
435 false, true, false, false,
438 ent
->src_addr
= __pa(walk
.data
);
439 ent
->auth_key_addr
= 0UL;
440 ent
->auth_iv_addr
= __pa(hash_loc
);
441 ent
->final_auth_state_addr
= 0UL;
442 ent
->enc_key_addr
= 0UL;
443 ent
->enc_iv_addr
= 0UL;
444 ent
->dest_addr
= __pa(hash_loc
);
446 nbytes
= crypto_hash_walk_done(&walk
, 0);
448 ent
= spu_queue_next(qp
, ent
);
450 ent
->control
= (nbytes
- 1);
451 ent
->src_addr
= __pa(walk
.data
);
452 ent
->auth_key_addr
= 0UL;
453 ent
->auth_iv_addr
= 0UL;
454 ent
->final_auth_state_addr
= 0UL;
455 ent
->enc_key_addr
= 0UL;
456 ent
->enc_iv_addr
= 0UL;
457 ent
->dest_addr
= 0UL;
459 nbytes
= crypto_hash_walk_done(&walk
, 0);
461 ent
->control
|= CONTROL_END_OF_BLOCK
;
463 if (submit_and_wait_for_tail(qp
, ent
) != HV_EOK
)
468 spin_unlock_irqrestore(&qp
->lock
, flags
);
471 memcpy(req
->result
, hash_loc
, result_size
);
478 static int n2_md5_async_digest(struct ahash_request
*req
)
480 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
481 struct md5_state
*m
= &rctx
->u
.md5
;
483 if (unlikely(req
->nbytes
== 0)) {
484 static const char md5_zero
[MD5_DIGEST_SIZE
] = {
485 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
486 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
489 memcpy(req
->result
, md5_zero
, MD5_DIGEST_SIZE
);
492 m
->hash
[0] = cpu_to_le32(0x67452301);
493 m
->hash
[1] = cpu_to_le32(0xefcdab89);
494 m
->hash
[2] = cpu_to_le32(0x98badcfe);
495 m
->hash
[3] = cpu_to_le32(0x10325476);
497 return n2_hash_async_digest(req
, AUTH_TYPE_MD5
,
498 MD5_DIGEST_SIZE
, MD5_DIGEST_SIZE
,
502 static int n2_sha1_async_digest(struct ahash_request
*req
)
504 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
505 struct sha1_state
*s
= &rctx
->u
.sha1
;
507 if (unlikely(req
->nbytes
== 0)) {
508 static const char sha1_zero
[SHA1_DIGEST_SIZE
] = {
509 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
510 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
514 memcpy(req
->result
, sha1_zero
, SHA1_DIGEST_SIZE
);
517 s
->state
[0] = SHA1_H0
;
518 s
->state
[1] = SHA1_H1
;
519 s
->state
[2] = SHA1_H2
;
520 s
->state
[3] = SHA1_H3
;
521 s
->state
[4] = SHA1_H4
;
523 return n2_hash_async_digest(req
, AUTH_TYPE_SHA1
,
524 SHA1_DIGEST_SIZE
, SHA1_DIGEST_SIZE
,
528 static int n2_sha256_async_digest(struct ahash_request
*req
)
530 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
531 struct sha256_state
*s
= &rctx
->u
.sha256
;
533 if (req
->nbytes
== 0) {
534 static const char sha256_zero
[SHA256_DIGEST_SIZE
] = {
535 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
536 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
537 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
538 0x1b, 0x78, 0x52, 0xb8, 0x55
541 memcpy(req
->result
, sha256_zero
, SHA256_DIGEST_SIZE
);
544 s
->state
[0] = SHA256_H0
;
545 s
->state
[1] = SHA256_H1
;
546 s
->state
[2] = SHA256_H2
;
547 s
->state
[3] = SHA256_H3
;
548 s
->state
[4] = SHA256_H4
;
549 s
->state
[5] = SHA256_H5
;
550 s
->state
[6] = SHA256_H6
;
551 s
->state
[7] = SHA256_H7
;
553 return n2_hash_async_digest(req
, AUTH_TYPE_SHA256
,
554 SHA256_DIGEST_SIZE
, SHA256_DIGEST_SIZE
,
558 static int n2_sha224_async_digest(struct ahash_request
*req
)
560 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
561 struct sha256_state
*s
= &rctx
->u
.sha256
;
563 if (req
->nbytes
== 0) {
564 static const char sha224_zero
[SHA224_DIGEST_SIZE
] = {
565 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
566 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
567 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
571 memcpy(req
->result
, sha224_zero
, SHA224_DIGEST_SIZE
);
574 s
->state
[0] = SHA224_H0
;
575 s
->state
[1] = SHA224_H1
;
576 s
->state
[2] = SHA224_H2
;
577 s
->state
[3] = SHA224_H3
;
578 s
->state
[4] = SHA224_H4
;
579 s
->state
[5] = SHA224_H5
;
580 s
->state
[6] = SHA224_H6
;
581 s
->state
[7] = SHA224_H7
;
583 return n2_hash_async_digest(req
, AUTH_TYPE_SHA256
,
584 SHA256_DIGEST_SIZE
, SHA224_DIGEST_SIZE
,
588 struct n2_cipher_context
{
592 u8 aes
[AES_MAX_KEY_SIZE
];
593 u8 des
[DES_KEY_SIZE
];
594 u8 des3
[3 * DES_KEY_SIZE
];
595 u8 arc4
[258]; /* S-box, X, Y */
599 #define N2_CHUNK_ARR_LEN 16
601 struct n2_crypto_chunk
{
602 struct list_head entry
;
603 unsigned long iv_paddr
: 44;
604 unsigned long arr_len
: 20;
605 unsigned long dest_paddr
;
606 unsigned long dest_final
;
608 unsigned long src_paddr
: 44;
609 unsigned long src_len
: 20;
610 } arr
[N2_CHUNK_ARR_LEN
];
613 struct n2_request_context
{
614 struct ablkcipher_walk walk
;
615 struct list_head chunk_list
;
616 struct n2_crypto_chunk chunk
;
620 /* The SPU allows some level of flexibility for partial cipher blocks
621 * being specified in a descriptor.
623 * It merely requires that every descriptor's length field is at least
624 * as large as the cipher block size. This means that a cipher block
625 * can span at most 2 descriptors. However, this does not allow a
626 * partial block to span into the final descriptor as that would
627 * violate the rule (since every descriptor's length must be at lest
628 * the block size). So, for example, assuming an 8 byte block size:
630 * 0xe --> 0xa --> 0x8
632 * is a valid length sequence, whereas:
634 * 0xe --> 0xb --> 0x7
636 * is not a valid sequence.
639 struct n2_cipher_alg
{
640 struct list_head entry
;
642 struct crypto_alg alg
;
645 static inline struct n2_cipher_alg
*n2_cipher_alg(struct crypto_tfm
*tfm
)
647 struct crypto_alg
*alg
= tfm
->__crt_alg
;
649 return container_of(alg
, struct n2_cipher_alg
, alg
);
652 struct n2_cipher_request_context
{
653 struct ablkcipher_walk walk
;
656 static int n2_aes_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
659 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
660 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
661 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
663 ctx
->enc_type
= (n2alg
->enc_type
& ENC_TYPE_CHAINING_MASK
);
666 case AES_KEYSIZE_128
:
667 ctx
->enc_type
|= ENC_TYPE_ALG_AES128
;
669 case AES_KEYSIZE_192
:
670 ctx
->enc_type
|= ENC_TYPE_ALG_AES192
;
672 case AES_KEYSIZE_256
:
673 ctx
->enc_type
|= ENC_TYPE_ALG_AES256
;
676 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
680 ctx
->key_len
= keylen
;
681 memcpy(ctx
->key
.aes
, key
, keylen
);
685 static int n2_des_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
688 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
689 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
690 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
691 u32 tmp
[DES_EXPKEY_WORDS
];
694 ctx
->enc_type
= n2alg
->enc_type
;
696 if (keylen
!= DES_KEY_SIZE
) {
697 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
701 err
= des_ekey(tmp
, key
);
702 if (err
== 0 && (tfm
->crt_flags
& CRYPTO_TFM_REQ_WEAK_KEY
)) {
703 tfm
->crt_flags
|= CRYPTO_TFM_RES_WEAK_KEY
;
707 ctx
->key_len
= keylen
;
708 memcpy(ctx
->key
.des
, key
, keylen
);
712 static int n2_3des_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
715 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
716 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
717 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
719 ctx
->enc_type
= n2alg
->enc_type
;
721 if (keylen
!= (3 * DES_KEY_SIZE
)) {
722 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
725 ctx
->key_len
= keylen
;
726 memcpy(ctx
->key
.des3
, key
, keylen
);
730 static int n2_arc4_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
733 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
734 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
735 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
736 u8
*s
= ctx
->key
.arc4
;
741 ctx
->enc_type
= n2alg
->enc_type
;
746 for (i
= 0; i
< 256; i
++)
748 for (i
= 0; i
< 256; i
++) {
750 j
= (j
+ key
[k
] + a
) & 0xff;
760 static inline int cipher_descriptor_len(int nbytes
, unsigned int block_size
)
762 int this_len
= nbytes
;
764 this_len
-= (nbytes
& (block_size
- 1));
765 return this_len
> (1 << 16) ? (1 << 16) : this_len
;
768 static int __n2_crypt_chunk(struct crypto_tfm
*tfm
, struct n2_crypto_chunk
*cp
,
769 struct spu_queue
*qp
, bool encrypt
)
771 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
772 struct cwq_initial_entry
*ent
;
776 ent
= spu_queue_alloc(qp
, cp
->arr_len
);
778 pr_info("queue_alloc() of %d fails\n",
783 in_place
= (cp
->dest_paddr
== cp
->arr
[0].src_paddr
);
785 ent
->control
= control_word_base(cp
->arr
[0].src_len
,
786 0, ctx
->enc_type
, 0, 0,
787 false, true, false, encrypt
,
789 (in_place
? OPCODE_INPLACE_BIT
: 0));
790 ent
->src_addr
= cp
->arr
[0].src_paddr
;
791 ent
->auth_key_addr
= 0UL;
792 ent
->auth_iv_addr
= 0UL;
793 ent
->final_auth_state_addr
= 0UL;
794 ent
->enc_key_addr
= __pa(&ctx
->key
);
795 ent
->enc_iv_addr
= cp
->iv_paddr
;
796 ent
->dest_addr
= (in_place
? 0UL : cp
->dest_paddr
);
798 for (i
= 1; i
< cp
->arr_len
; i
++) {
799 ent
= spu_queue_next(qp
, ent
);
801 ent
->control
= cp
->arr
[i
].src_len
- 1;
802 ent
->src_addr
= cp
->arr
[i
].src_paddr
;
803 ent
->auth_key_addr
= 0UL;
804 ent
->auth_iv_addr
= 0UL;
805 ent
->final_auth_state_addr
= 0UL;
806 ent
->enc_key_addr
= 0UL;
807 ent
->enc_iv_addr
= 0UL;
808 ent
->dest_addr
= 0UL;
810 ent
->control
|= CONTROL_END_OF_BLOCK
;
812 return (spu_queue_submit(qp
, ent
) != HV_EOK
) ? -EINVAL
: 0;
815 static int n2_compute_chunks(struct ablkcipher_request
*req
)
817 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
818 struct ablkcipher_walk
*walk
= &rctx
->walk
;
819 struct n2_crypto_chunk
*chunk
;
820 unsigned long dest_prev
;
821 unsigned int tot_len
;
825 ablkcipher_walk_init(walk
, req
->dst
, req
->src
, req
->nbytes
);
826 err
= ablkcipher_walk_phys(req
, walk
);
830 INIT_LIST_HEAD(&rctx
->chunk_list
);
832 chunk
= &rctx
->chunk
;
833 INIT_LIST_HEAD(&chunk
->entry
);
835 chunk
->iv_paddr
= 0UL;
837 chunk
->dest_paddr
= 0UL;
839 prev_in_place
= false;
843 while ((nbytes
= walk
->nbytes
) != 0) {
844 unsigned long dest_paddr
, src_paddr
;
848 src_paddr
= (page_to_phys(walk
->src
.page
) +
850 dest_paddr
= (page_to_phys(walk
->dst
.page
) +
852 in_place
= (src_paddr
== dest_paddr
);
853 this_len
= cipher_descriptor_len(nbytes
, walk
->blocksize
);
855 if (chunk
->arr_len
!= 0) {
856 if (in_place
!= prev_in_place
||
858 dest_paddr
!= dest_prev
) ||
859 chunk
->arr_len
== N2_CHUNK_ARR_LEN
||
860 tot_len
+ this_len
> (1 << 16)) {
861 chunk
->dest_final
= dest_prev
;
862 list_add_tail(&chunk
->entry
,
864 chunk
= kzalloc(sizeof(*chunk
), GFP_ATOMIC
);
869 INIT_LIST_HEAD(&chunk
->entry
);
872 if (chunk
->arr_len
== 0) {
873 chunk
->dest_paddr
= dest_paddr
;
876 chunk
->arr
[chunk
->arr_len
].src_paddr
= src_paddr
;
877 chunk
->arr
[chunk
->arr_len
].src_len
= this_len
;
880 dest_prev
= dest_paddr
+ this_len
;
881 prev_in_place
= in_place
;
884 err
= ablkcipher_walk_done(req
, walk
, nbytes
- this_len
);
888 if (!err
&& chunk
->arr_len
!= 0) {
889 chunk
->dest_final
= dest_prev
;
890 list_add_tail(&chunk
->entry
, &rctx
->chunk_list
);
896 static void n2_chunk_complete(struct ablkcipher_request
*req
, void *final_iv
)
898 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
899 struct n2_crypto_chunk
*c
, *tmp
;
902 memcpy(rctx
->walk
.iv
, final_iv
, rctx
->walk
.blocksize
);
904 ablkcipher_walk_complete(&rctx
->walk
);
905 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
, entry
) {
907 if (unlikely(c
!= &rctx
->chunk
))
913 static int n2_do_ecb(struct ablkcipher_request
*req
, bool encrypt
)
915 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
916 struct crypto_tfm
*tfm
= req
->base
.tfm
;
917 int err
= n2_compute_chunks(req
);
918 struct n2_crypto_chunk
*c
, *tmp
;
919 unsigned long flags
, hv_ret
;
920 struct spu_queue
*qp
;
925 qp
= cpu_to_cwq
[get_cpu()];
930 spin_lock_irqsave(&qp
->lock
, flags
);
932 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
, entry
) {
933 err
= __n2_crypt_chunk(tfm
, c
, qp
, encrypt
);
937 if (unlikely(c
!= &rctx
->chunk
))
941 hv_ret
= wait_for_tail(qp
);
942 if (hv_ret
!= HV_EOK
)
946 spin_unlock_irqrestore(&qp
->lock
, flags
);
951 n2_chunk_complete(req
, NULL
);
955 static int n2_encrypt_ecb(struct ablkcipher_request
*req
)
957 return n2_do_ecb(req
, true);
960 static int n2_decrypt_ecb(struct ablkcipher_request
*req
)
962 return n2_do_ecb(req
, false);
965 static int n2_do_chaining(struct ablkcipher_request
*req
, bool encrypt
)
967 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
968 struct crypto_tfm
*tfm
= req
->base
.tfm
;
969 unsigned long flags
, hv_ret
, iv_paddr
;
970 int err
= n2_compute_chunks(req
);
971 struct n2_crypto_chunk
*c
, *tmp
;
972 struct spu_queue
*qp
;
975 final_iv_addr
= NULL
;
980 qp
= cpu_to_cwq
[get_cpu()];
985 spin_lock_irqsave(&qp
->lock
, flags
);
988 iv_paddr
= __pa(rctx
->walk
.iv
);
989 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
,
991 c
->iv_paddr
= iv_paddr
;
992 err
= __n2_crypt_chunk(tfm
, c
, qp
, true);
995 iv_paddr
= c
->dest_final
- rctx
->walk
.blocksize
;
997 if (unlikely(c
!= &rctx
->chunk
))
1000 final_iv_addr
= __va(iv_paddr
);
1002 list_for_each_entry_safe_reverse(c
, tmp
, &rctx
->chunk_list
,
1004 if (c
== &rctx
->chunk
) {
1005 iv_paddr
= __pa(rctx
->walk
.iv
);
1007 iv_paddr
= (tmp
->arr
[tmp
->arr_len
-1].src_paddr
+
1008 tmp
->arr
[tmp
->arr_len
-1].src_len
-
1009 rctx
->walk
.blocksize
);
1011 if (!final_iv_addr
) {
1014 pa
= (c
->arr
[c
->arr_len
-1].src_paddr
+
1015 c
->arr
[c
->arr_len
-1].src_len
-
1016 rctx
->walk
.blocksize
);
1017 final_iv_addr
= rctx
->temp_iv
;
1018 memcpy(rctx
->temp_iv
, __va(pa
),
1019 rctx
->walk
.blocksize
);
1021 c
->iv_paddr
= iv_paddr
;
1022 err
= __n2_crypt_chunk(tfm
, c
, qp
, false);
1025 list_del(&c
->entry
);
1026 if (unlikely(c
!= &rctx
->chunk
))
1031 hv_ret
= wait_for_tail(qp
);
1032 if (hv_ret
!= HV_EOK
)
1036 spin_unlock_irqrestore(&qp
->lock
, flags
);
1041 n2_chunk_complete(req
, err
? NULL
: final_iv_addr
);
1045 static int n2_encrypt_chaining(struct ablkcipher_request
*req
)
1047 return n2_do_chaining(req
, true);
1050 static int n2_decrypt_chaining(struct ablkcipher_request
*req
)
1052 return n2_do_chaining(req
, false);
1055 struct n2_cipher_tmpl
{
1057 const char *drv_name
;
1060 struct ablkcipher_alg ablkcipher
;
1063 static const struct n2_cipher_tmpl cipher_tmpls
[] = {
1064 /* ARC4: only ECB is supported (chaining bits ignored) */
1065 { .name
= "ecb(arc4)",
1066 .drv_name
= "ecb-arc4",
1068 .enc_type
= (ENC_TYPE_ALG_RC4_STREAM
|
1069 ENC_TYPE_CHAINING_ECB
),
1073 .setkey
= n2_arc4_setkey
,
1074 .encrypt
= n2_encrypt_ecb
,
1075 .decrypt
= n2_decrypt_ecb
,
1079 /* DES: ECB CBC and CFB are supported */
1080 { .name
= "ecb(des)",
1081 .drv_name
= "ecb-des",
1082 .block_size
= DES_BLOCK_SIZE
,
1083 .enc_type
= (ENC_TYPE_ALG_DES
|
1084 ENC_TYPE_CHAINING_ECB
),
1086 .min_keysize
= DES_KEY_SIZE
,
1087 .max_keysize
= DES_KEY_SIZE
,
1088 .setkey
= n2_des_setkey
,
1089 .encrypt
= n2_encrypt_ecb
,
1090 .decrypt
= n2_decrypt_ecb
,
1093 { .name
= "cbc(des)",
1094 .drv_name
= "cbc-des",
1095 .block_size
= DES_BLOCK_SIZE
,
1096 .enc_type
= (ENC_TYPE_ALG_DES
|
1097 ENC_TYPE_CHAINING_CBC
),
1099 .ivsize
= DES_BLOCK_SIZE
,
1100 .min_keysize
= DES_KEY_SIZE
,
1101 .max_keysize
= DES_KEY_SIZE
,
1102 .setkey
= n2_des_setkey
,
1103 .encrypt
= n2_encrypt_chaining
,
1104 .decrypt
= n2_decrypt_chaining
,
1107 { .name
= "cfb(des)",
1108 .drv_name
= "cfb-des",
1109 .block_size
= DES_BLOCK_SIZE
,
1110 .enc_type
= (ENC_TYPE_ALG_DES
|
1111 ENC_TYPE_CHAINING_CFB
),
1113 .min_keysize
= DES_KEY_SIZE
,
1114 .max_keysize
= DES_KEY_SIZE
,
1115 .setkey
= n2_des_setkey
,
1116 .encrypt
= n2_encrypt_chaining
,
1117 .decrypt
= n2_decrypt_chaining
,
1121 /* 3DES: ECB CBC and CFB are supported */
1122 { .name
= "ecb(des3_ede)",
1123 .drv_name
= "ecb-3des",
1124 .block_size
= DES_BLOCK_SIZE
,
1125 .enc_type
= (ENC_TYPE_ALG_3DES
|
1126 ENC_TYPE_CHAINING_ECB
),
1128 .min_keysize
= 3 * DES_KEY_SIZE
,
1129 .max_keysize
= 3 * DES_KEY_SIZE
,
1130 .setkey
= n2_3des_setkey
,
1131 .encrypt
= n2_encrypt_ecb
,
1132 .decrypt
= n2_decrypt_ecb
,
1135 { .name
= "cbc(des3_ede)",
1136 .drv_name
= "cbc-3des",
1137 .block_size
= DES_BLOCK_SIZE
,
1138 .enc_type
= (ENC_TYPE_ALG_3DES
|
1139 ENC_TYPE_CHAINING_CBC
),
1141 .ivsize
= DES_BLOCK_SIZE
,
1142 .min_keysize
= 3 * DES_KEY_SIZE
,
1143 .max_keysize
= 3 * DES_KEY_SIZE
,
1144 .setkey
= n2_3des_setkey
,
1145 .encrypt
= n2_encrypt_chaining
,
1146 .decrypt
= n2_decrypt_chaining
,
1149 { .name
= "cfb(des3_ede)",
1150 .drv_name
= "cfb-3des",
1151 .block_size
= DES_BLOCK_SIZE
,
1152 .enc_type
= (ENC_TYPE_ALG_3DES
|
1153 ENC_TYPE_CHAINING_CFB
),
1155 .min_keysize
= 3 * DES_KEY_SIZE
,
1156 .max_keysize
= 3 * DES_KEY_SIZE
,
1157 .setkey
= n2_3des_setkey
,
1158 .encrypt
= n2_encrypt_chaining
,
1159 .decrypt
= n2_decrypt_chaining
,
1162 /* AES: ECB CBC and CTR are supported */
1163 { .name
= "ecb(aes)",
1164 .drv_name
= "ecb-aes",
1165 .block_size
= AES_BLOCK_SIZE
,
1166 .enc_type
= (ENC_TYPE_ALG_AES128
|
1167 ENC_TYPE_CHAINING_ECB
),
1169 .min_keysize
= AES_MIN_KEY_SIZE
,
1170 .max_keysize
= AES_MAX_KEY_SIZE
,
1171 .setkey
= n2_aes_setkey
,
1172 .encrypt
= n2_encrypt_ecb
,
1173 .decrypt
= n2_decrypt_ecb
,
1176 { .name
= "cbc(aes)",
1177 .drv_name
= "cbc-aes",
1178 .block_size
= AES_BLOCK_SIZE
,
1179 .enc_type
= (ENC_TYPE_ALG_AES128
|
1180 ENC_TYPE_CHAINING_CBC
),
1182 .ivsize
= AES_BLOCK_SIZE
,
1183 .min_keysize
= AES_MIN_KEY_SIZE
,
1184 .max_keysize
= AES_MAX_KEY_SIZE
,
1185 .setkey
= n2_aes_setkey
,
1186 .encrypt
= n2_encrypt_chaining
,
1187 .decrypt
= n2_decrypt_chaining
,
1190 { .name
= "ctr(aes)",
1191 .drv_name
= "ctr-aes",
1192 .block_size
= AES_BLOCK_SIZE
,
1193 .enc_type
= (ENC_TYPE_ALG_AES128
|
1194 ENC_TYPE_CHAINING_COUNTER
),
1196 .ivsize
= AES_BLOCK_SIZE
,
1197 .min_keysize
= AES_MIN_KEY_SIZE
,
1198 .max_keysize
= AES_MAX_KEY_SIZE
,
1199 .setkey
= n2_aes_setkey
,
1200 .encrypt
= n2_encrypt_chaining
,
1201 .decrypt
= n2_encrypt_chaining
,
1206 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1208 static LIST_HEAD(cipher_algs
);
1210 struct n2_hash_tmpl
{
1212 int (*digest
)(struct ahash_request
*req
);
1216 static const struct n2_hash_tmpl hash_tmpls
[] = {
1218 .digest
= n2_md5_async_digest
,
1219 .digest_size
= MD5_DIGEST_SIZE
,
1220 .block_size
= MD5_HMAC_BLOCK_SIZE
},
1222 .digest
= n2_sha1_async_digest
,
1223 .digest_size
= SHA1_DIGEST_SIZE
,
1224 .block_size
= SHA1_BLOCK_SIZE
},
1226 .digest
= n2_sha256_async_digest
,
1227 .digest_size
= SHA256_DIGEST_SIZE
,
1228 .block_size
= SHA256_BLOCK_SIZE
},
1230 .digest
= n2_sha224_async_digest
,
1231 .digest_size
= SHA224_DIGEST_SIZE
,
1232 .block_size
= SHA224_BLOCK_SIZE
},
1234 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1236 struct n2_ahash_alg
{
1237 struct list_head entry
;
1238 struct ahash_alg alg
;
1240 static LIST_HEAD(ahash_algs
);
1242 static int algs_registered
;
1244 static void __n2_unregister_algs(void)
1246 struct n2_cipher_alg
*cipher
, *cipher_tmp
;
1247 struct n2_ahash_alg
*alg
, *alg_tmp
;
1249 list_for_each_entry_safe(cipher
, cipher_tmp
, &cipher_algs
, entry
) {
1250 crypto_unregister_alg(&cipher
->alg
);
1251 list_del(&cipher
->entry
);
1254 list_for_each_entry_safe(alg
, alg_tmp
, &ahash_algs
, entry
) {
1255 crypto_unregister_ahash(&alg
->alg
);
1256 list_del(&alg
->entry
);
1261 static int n2_cipher_cra_init(struct crypto_tfm
*tfm
)
1263 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct n2_request_context
);
1267 static int __devinit
__n2_register_one_cipher(const struct n2_cipher_tmpl
*tmpl
)
1269 struct n2_cipher_alg
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1270 struct crypto_alg
*alg
;
1278 snprintf(alg
->cra_name
, CRYPTO_MAX_ALG_NAME
, "%s", tmpl
->name
);
1279 snprintf(alg
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s-n2", tmpl
->drv_name
);
1280 alg
->cra_priority
= N2_CRA_PRIORITY
;
1281 alg
->cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
;
1282 alg
->cra_blocksize
= tmpl
->block_size
;
1283 p
->enc_type
= tmpl
->enc_type
;
1284 alg
->cra_ctxsize
= sizeof(struct n2_cipher_context
);
1285 alg
->cra_type
= &crypto_ablkcipher_type
;
1286 alg
->cra_u
.ablkcipher
= tmpl
->ablkcipher
;
1287 alg
->cra_init
= n2_cipher_cra_init
;
1288 alg
->cra_module
= THIS_MODULE
;
1290 list_add(&p
->entry
, &cipher_algs
);
1291 err
= crypto_register_alg(alg
);
1293 list_del(&p
->entry
);
1299 static int __devinit
__n2_register_one_ahash(const struct n2_hash_tmpl
*tmpl
)
1301 struct n2_ahash_alg
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1302 struct hash_alg_common
*halg
;
1303 struct crypto_alg
*base
;
1304 struct ahash_alg
*ahash
;
1311 ahash
->init
= n2_hash_async_init
;
1312 ahash
->update
= n2_hash_async_update
;
1313 ahash
->final
= n2_hash_async_final
;
1314 ahash
->finup
= n2_hash_async_finup
;
1315 ahash
->digest
= tmpl
->digest
;
1317 halg
= &ahash
->halg
;
1318 halg
->digestsize
= tmpl
->digest_size
;
1321 snprintf(base
->cra_name
, CRYPTO_MAX_ALG_NAME
, "%s", tmpl
->name
);
1322 snprintf(base
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s-n2", tmpl
->name
);
1323 base
->cra_priority
= N2_CRA_PRIORITY
;
1324 base
->cra_flags
= CRYPTO_ALG_TYPE_AHASH
| CRYPTO_ALG_NEED_FALLBACK
;
1325 base
->cra_blocksize
= tmpl
->block_size
;
1326 base
->cra_ctxsize
= sizeof(struct n2_hash_ctx
);
1327 base
->cra_module
= THIS_MODULE
;
1328 base
->cra_init
= n2_hash_cra_init
;
1329 base
->cra_exit
= n2_hash_cra_exit
;
1331 list_add(&p
->entry
, &ahash_algs
);
1332 err
= crypto_register_ahash(ahash
);
1334 list_del(&p
->entry
);
1340 static int __devinit
n2_register_algs(void)
1344 mutex_lock(&spu_lock
);
1345 if (algs_registered
++)
1348 for (i
= 0; i
< NUM_HASH_TMPLS
; i
++) {
1349 err
= __n2_register_one_ahash(&hash_tmpls
[i
]);
1351 __n2_unregister_algs();
1355 for (i
= 0; i
< NUM_CIPHER_TMPLS
; i
++) {
1356 err
= __n2_register_one_cipher(&cipher_tmpls
[i
]);
1358 __n2_unregister_algs();
1364 mutex_unlock(&spu_lock
);
1368 static void __exit
n2_unregister_algs(void)
1370 mutex_lock(&spu_lock
);
1371 if (!--algs_registered
)
1372 __n2_unregister_algs();
1373 mutex_unlock(&spu_lock
);
1376 /* To map CWQ queues to interrupt sources, the hypervisor API provides
1377 * a devino. This isn't very useful to us because all of the
1378 * interrupts listed in the of_device node have been translated to
1379 * Linux virtual IRQ cookie numbers.
1381 * So we have to back-translate, going through the 'intr' and 'ino'
1382 * property tables of the n2cp MDESC node, matching it with the OF
1383 * 'interrupts' property entries, in order to to figure out which
1384 * devino goes to which already-translated IRQ.
1386 static int find_devino_index(struct of_device
*dev
, struct spu_mdesc_info
*ip
,
1387 unsigned long dev_ino
)
1389 const unsigned int *dev_intrs
;
1393 for (i
= 0; i
< ip
->num_intrs
; i
++) {
1394 if (ip
->ino_table
[i
].ino
== dev_ino
)
1397 if (i
== ip
->num_intrs
)
1400 intr
= ip
->ino_table
[i
].intr
;
1402 dev_intrs
= of_get_property(dev
->dev
.of_node
, "interrupts", NULL
);
1406 for (i
= 0; i
< dev
->num_irqs
; i
++) {
1407 if (dev_intrs
[i
] == intr
)
1414 static int spu_map_ino(struct of_device
*dev
, struct spu_mdesc_info
*ip
,
1415 const char *irq_name
, struct spu_queue
*p
,
1416 irq_handler_t handler
)
1421 herr
= sun4v_ncs_qhandle_to_devino(p
->qhandle
, &p
->devino
);
1425 index
= find_devino_index(dev
, ip
, p
->devino
);
1429 p
->irq
= dev
->irqs
[index
];
1431 sprintf(p
->irq_name
, "%s-%d", irq_name
, index
);
1433 return request_irq(p
->irq
, handler
, IRQF_SAMPLE_RANDOM
,
1437 static struct kmem_cache
*queue_cache
[2];
1439 static void *new_queue(unsigned long q_type
)
1441 return kmem_cache_zalloc(queue_cache
[q_type
- 1], GFP_KERNEL
);
1444 static void free_queue(void *p
, unsigned long q_type
)
1446 return kmem_cache_free(queue_cache
[q_type
- 1], p
);
1449 static int queue_cache_init(void)
1451 if (!queue_cache
[HV_NCS_QTYPE_MAU
- 1])
1452 queue_cache
[HV_NCS_QTYPE_MAU
- 1] =
1453 kmem_cache_create("mau_queue",
1456 MAU_ENTRY_SIZE
, 0, NULL
);
1457 if (!queue_cache
[HV_NCS_QTYPE_MAU
- 1])
1460 if (!queue_cache
[HV_NCS_QTYPE_CWQ
- 1])
1461 queue_cache
[HV_NCS_QTYPE_CWQ
- 1] =
1462 kmem_cache_create("cwq_queue",
1465 CWQ_ENTRY_SIZE
, 0, NULL
);
1466 if (!queue_cache
[HV_NCS_QTYPE_CWQ
- 1]) {
1467 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_MAU
- 1]);
1473 static void queue_cache_destroy(void)
1475 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_MAU
- 1]);
1476 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_CWQ
- 1]);
1479 static int spu_queue_register(struct spu_queue
*p
, unsigned long q_type
)
1481 cpumask_var_t old_allowed
;
1482 unsigned long hv_ret
;
1484 if (cpumask_empty(&p
->sharing
))
1487 if (!alloc_cpumask_var(&old_allowed
, GFP_KERNEL
))
1490 cpumask_copy(old_allowed
, ¤t
->cpus_allowed
);
1492 set_cpus_allowed_ptr(current
, &p
->sharing
);
1494 hv_ret
= sun4v_ncs_qconf(q_type
, __pa(p
->q
),
1495 CWQ_NUM_ENTRIES
, &p
->qhandle
);
1497 sun4v_ncs_sethead_marker(p
->qhandle
, 0);
1499 set_cpus_allowed_ptr(current
, old_allowed
);
1501 free_cpumask_var(old_allowed
);
1503 return (hv_ret
? -EINVAL
: 0);
1506 static int spu_queue_setup(struct spu_queue
*p
)
1510 p
->q
= new_queue(p
->q_type
);
1514 err
= spu_queue_register(p
, p
->q_type
);
1516 free_queue(p
->q
, p
->q_type
);
1523 static void spu_queue_destroy(struct spu_queue
*p
)
1525 unsigned long hv_ret
;
1530 hv_ret
= sun4v_ncs_qconf(p
->q_type
, p
->qhandle
, 0, &p
->qhandle
);
1533 free_queue(p
->q
, p
->q_type
);
1536 static void spu_list_destroy(struct list_head
*list
)
1538 struct spu_queue
*p
, *n
;
1540 list_for_each_entry_safe(p
, n
, list
, list
) {
1543 for (i
= 0; i
< NR_CPUS
; i
++) {
1544 if (cpu_to_cwq
[i
] == p
)
1545 cpu_to_cwq
[i
] = NULL
;
1549 free_irq(p
->irq
, p
);
1552 spu_queue_destroy(p
);
1558 /* Walk the backward arcs of a CWQ 'exec-unit' node,
1559 * gathering cpu membership information.
1561 static int spu_mdesc_walk_arcs(struct mdesc_handle
*mdesc
,
1562 struct of_device
*dev
,
1563 u64 node
, struct spu_queue
*p
,
1564 struct spu_queue
**table
)
1568 mdesc_for_each_arc(arc
, mdesc
, node
, MDESC_ARC_TYPE_BACK
) {
1569 u64 tgt
= mdesc_arc_target(mdesc
, arc
);
1570 const char *name
= mdesc_node_name(mdesc
, tgt
);
1573 if (strcmp(name
, "cpu"))
1575 id
= mdesc_get_property(mdesc
, tgt
, "id", NULL
);
1576 if (table
[*id
] != NULL
) {
1577 dev_err(&dev
->dev
, "%s: SPU cpu slot already set.\n",
1578 dev
->dev
.of_node
->full_name
);
1581 cpu_set(*id
, p
->sharing
);
1587 /* Process an 'exec-unit' MDESC node of type 'cwq'. */
1588 static int handle_exec_unit(struct spu_mdesc_info
*ip
, struct list_head
*list
,
1589 struct of_device
*dev
, struct mdesc_handle
*mdesc
,
1590 u64 node
, const char *iname
, unsigned long q_type
,
1591 irq_handler_t handler
, struct spu_queue
**table
)
1593 struct spu_queue
*p
;
1596 p
= kzalloc(sizeof(struct spu_queue
), GFP_KERNEL
);
1598 dev_err(&dev
->dev
, "%s: Could not allocate SPU queue.\n",
1599 dev
->dev
.of_node
->full_name
);
1603 cpus_clear(p
->sharing
);
1604 spin_lock_init(&p
->lock
);
1606 INIT_LIST_HEAD(&p
->jobs
);
1607 list_add(&p
->list
, list
);
1609 err
= spu_mdesc_walk_arcs(mdesc
, dev
, node
, p
, table
);
1613 err
= spu_queue_setup(p
);
1617 return spu_map_ino(dev
, ip
, iname
, p
, handler
);
1620 static int spu_mdesc_scan(struct mdesc_handle
*mdesc
, struct of_device
*dev
,
1621 struct spu_mdesc_info
*ip
, struct list_head
*list
,
1622 const char *exec_name
, unsigned long q_type
,
1623 irq_handler_t handler
, struct spu_queue
**table
)
1628 mdesc_for_each_node_by_name(mdesc
, node
, "exec-unit") {
1631 type
= mdesc_get_property(mdesc
, node
, "type", NULL
);
1632 if (!type
|| strcmp(type
, exec_name
))
1635 err
= handle_exec_unit(ip
, list
, dev
, mdesc
, node
,
1636 exec_name
, q_type
, handler
, table
);
1638 spu_list_destroy(list
);
1646 static int __devinit
get_irq_props(struct mdesc_handle
*mdesc
, u64 node
,
1647 struct spu_mdesc_info
*ip
)
1649 const u64
*intr
, *ino
;
1650 int intr_len
, ino_len
;
1653 intr
= mdesc_get_property(mdesc
, node
, "intr", &intr_len
);
1657 ino
= mdesc_get_property(mdesc
, node
, "ino", &ino_len
);
1661 if (intr_len
!= ino_len
)
1664 ip
->num_intrs
= intr_len
/ sizeof(u64
);
1665 ip
->ino_table
= kzalloc((sizeof(struct ino_blob
) *
1671 for (i
= 0; i
< ip
->num_intrs
; i
++) {
1672 struct ino_blob
*b
= &ip
->ino_table
[i
];
1680 static int __devinit
grab_mdesc_irq_props(struct mdesc_handle
*mdesc
,
1681 struct of_device
*dev
,
1682 struct spu_mdesc_info
*ip
,
1683 const char *node_name
)
1685 const unsigned int *reg
;
1688 reg
= of_get_property(dev
->dev
.of_node
, "reg", NULL
);
1692 mdesc_for_each_node_by_name(mdesc
, node
, "virtual-device") {
1696 name
= mdesc_get_property(mdesc
, node
, "name", NULL
);
1697 if (!name
|| strcmp(name
, node_name
))
1699 chdl
= mdesc_get_property(mdesc
, node
, "cfg-handle", NULL
);
1700 if (!chdl
|| (*chdl
!= *reg
))
1702 ip
->cfg_handle
= *chdl
;
1703 return get_irq_props(mdesc
, node
, ip
);
1709 static unsigned long n2_spu_hvapi_major
;
1710 static unsigned long n2_spu_hvapi_minor
;
1712 static int __devinit
n2_spu_hvapi_register(void)
1716 n2_spu_hvapi_major
= 2;
1717 n2_spu_hvapi_minor
= 0;
1719 err
= sun4v_hvapi_register(HV_GRP_NCS
,
1721 &n2_spu_hvapi_minor
);
1724 pr_info("Registered NCS HVAPI version %lu.%lu\n",
1726 n2_spu_hvapi_minor
);
1731 static void n2_spu_hvapi_unregister(void)
1733 sun4v_hvapi_unregister(HV_GRP_NCS
);
1736 static int global_ref
;
1738 static int __devinit
grab_global_resources(void)
1742 mutex_lock(&spu_lock
);
1747 err
= n2_spu_hvapi_register();
1751 err
= queue_cache_init();
1753 goto out_hvapi_release
;
1756 cpu_to_cwq
= kzalloc(sizeof(struct spu_queue
*) * NR_CPUS
,
1759 goto out_queue_cache_destroy
;
1761 cpu_to_mau
= kzalloc(sizeof(struct spu_queue
*) * NR_CPUS
,
1764 goto out_free_cwq_table
;
1771 mutex_unlock(&spu_lock
);
1778 out_queue_cache_destroy
:
1779 queue_cache_destroy();
1782 n2_spu_hvapi_unregister();
1786 static void release_global_resources(void)
1788 mutex_lock(&spu_lock
);
1789 if (!--global_ref
) {
1796 queue_cache_destroy();
1797 n2_spu_hvapi_unregister();
1799 mutex_unlock(&spu_lock
);
1802 static struct n2_crypto
* __devinit
alloc_n2cp(void)
1804 struct n2_crypto
*np
= kzalloc(sizeof(struct n2_crypto
), GFP_KERNEL
);
1807 INIT_LIST_HEAD(&np
->cwq_list
);
1812 static void free_n2cp(struct n2_crypto
*np
)
1814 if (np
->cwq_info
.ino_table
) {
1815 kfree(np
->cwq_info
.ino_table
);
1816 np
->cwq_info
.ino_table
= NULL
;
1822 static void __devinit
n2_spu_driver_version(void)
1824 static int n2_spu_version_printed
;
1826 if (n2_spu_version_printed
++ == 0)
1827 pr_info("%s", version
);
1830 static int __devinit
n2_crypto_probe(struct of_device
*dev
,
1831 const struct of_device_id
*match
)
1833 struct mdesc_handle
*mdesc
;
1834 const char *full_name
;
1835 struct n2_crypto
*np
;
1838 n2_spu_driver_version();
1840 full_name
= dev
->dev
.of_node
->full_name
;
1841 pr_info("Found N2CP at %s\n", full_name
);
1845 dev_err(&dev
->dev
, "%s: Unable to allocate n2cp.\n",
1850 err
= grab_global_resources();
1852 dev_err(&dev
->dev
, "%s: Unable to grab "
1853 "global resources.\n", full_name
);
1857 mdesc
= mdesc_grab();
1860 dev_err(&dev
->dev
, "%s: Unable to grab MDESC.\n",
1863 goto out_free_global
;
1865 err
= grab_mdesc_irq_props(mdesc
, dev
, &np
->cwq_info
, "n2cp");
1867 dev_err(&dev
->dev
, "%s: Unable to grab IRQ props.\n",
1869 mdesc_release(mdesc
);
1870 goto out_free_global
;
1873 err
= spu_mdesc_scan(mdesc
, dev
, &np
->cwq_info
, &np
->cwq_list
,
1874 "cwq", HV_NCS_QTYPE_CWQ
, cwq_intr
,
1876 mdesc_release(mdesc
);
1879 dev_err(&dev
->dev
, "%s: CWQ MDESC scan failed.\n",
1881 goto out_free_global
;
1884 err
= n2_register_algs();
1886 dev_err(&dev
->dev
, "%s: Unable to register algorithms.\n",
1888 goto out_free_spu_list
;
1891 dev_set_drvdata(&dev
->dev
, np
);
1896 spu_list_destroy(&np
->cwq_list
);
1899 release_global_resources();
1907 static int __devexit
n2_crypto_remove(struct of_device
*dev
)
1909 struct n2_crypto
*np
= dev_get_drvdata(&dev
->dev
);
1911 n2_unregister_algs();
1913 spu_list_destroy(&np
->cwq_list
);
1915 release_global_resources();
1922 static struct n2_mau
* __devinit
alloc_ncp(void)
1924 struct n2_mau
*mp
= kzalloc(sizeof(struct n2_mau
), GFP_KERNEL
);
1927 INIT_LIST_HEAD(&mp
->mau_list
);
1932 static void free_ncp(struct n2_mau
*mp
)
1934 if (mp
->mau_info
.ino_table
) {
1935 kfree(mp
->mau_info
.ino_table
);
1936 mp
->mau_info
.ino_table
= NULL
;
1942 static int __devinit
n2_mau_probe(struct of_device
*dev
,
1943 const struct of_device_id
*match
)
1945 struct mdesc_handle
*mdesc
;
1946 const char *full_name
;
1950 n2_spu_driver_version();
1952 full_name
= dev
->dev
.of_node
->full_name
;
1953 pr_info("Found NCP at %s\n", full_name
);
1957 dev_err(&dev
->dev
, "%s: Unable to allocate ncp.\n",
1962 err
= grab_global_resources();
1964 dev_err(&dev
->dev
, "%s: Unable to grab "
1965 "global resources.\n", full_name
);
1969 mdesc
= mdesc_grab();
1972 dev_err(&dev
->dev
, "%s: Unable to grab MDESC.\n",
1975 goto out_free_global
;
1978 err
= grab_mdesc_irq_props(mdesc
, dev
, &mp
->mau_info
, "ncp");
1980 dev_err(&dev
->dev
, "%s: Unable to grab IRQ props.\n",
1982 mdesc_release(mdesc
);
1983 goto out_free_global
;
1986 err
= spu_mdesc_scan(mdesc
, dev
, &mp
->mau_info
, &mp
->mau_list
,
1987 "mau", HV_NCS_QTYPE_MAU
, mau_intr
,
1989 mdesc_release(mdesc
);
1992 dev_err(&dev
->dev
, "%s: MAU MDESC scan failed.\n",
1994 goto out_free_global
;
1997 dev_set_drvdata(&dev
->dev
, mp
);
2002 release_global_resources();
2010 static int __devexit
n2_mau_remove(struct of_device
*dev
)
2012 struct n2_mau
*mp
= dev_get_drvdata(&dev
->dev
);
2014 spu_list_destroy(&mp
->mau_list
);
2016 release_global_resources();
2023 static struct of_device_id n2_crypto_match
[] = {
2026 .compatible
= "SUNW,n2-cwq",
2030 .compatible
= "SUNW,vf-cwq",
2035 MODULE_DEVICE_TABLE(of
, n2_crypto_match
);
2037 static struct of_platform_driver n2_crypto_driver
= {
2040 .owner
= THIS_MODULE
,
2041 .of_match_table
= n2_crypto_match
,
2043 .probe
= n2_crypto_probe
,
2044 .remove
= __devexit_p(n2_crypto_remove
),
2047 static struct of_device_id n2_mau_match
[] = {
2050 .compatible
= "SUNW,n2-mau",
2054 .compatible
= "SUNW,vf-mau",
2059 MODULE_DEVICE_TABLE(of
, n2_mau_match
);
2061 static struct of_platform_driver n2_mau_driver
= {
2064 .owner
= THIS_MODULE
,
2065 .of_match_table
= n2_mau_match
,
2067 .probe
= n2_mau_probe
,
2068 .remove
= __devexit_p(n2_mau_remove
),
2071 static int __init
n2_init(void)
2073 int err
= of_register_driver(&n2_crypto_driver
, &of_bus_type
);
2076 err
= of_register_driver(&n2_mau_driver
, &of_bus_type
);
2078 of_unregister_driver(&n2_crypto_driver
);
2083 static void __exit
n2_exit(void)
2085 of_unregister_driver(&n2_mau_driver
);
2086 of_unregister_driver(&n2_crypto_driver
);
2089 module_init(n2_init
);
2090 module_exit(n2_exit
);