2 * 2007+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <linux/crypto.h>
17 #include <linux/highmem.h>
18 #include <linux/kthread.h>
19 #include <linux/pagemap.h>
20 #include <linux/scatterlist.h>
21 #include <linux/slab.h>
25 static struct crypto_hash
*pohmelfs_init_hash(struct pohmelfs_sb
*psb
)
28 struct crypto_hash
*hash
;
30 hash
= crypto_alloc_hash(psb
->hash_string
, 0, CRYPTO_ALG_ASYNC
);
33 dprintk("%s: idx: %u: failed to allocate hash '%s', err: %d.\n",
34 __func__
, psb
->idx
, psb
->hash_string
, err
);
38 psb
->crypto_attached_size
= crypto_hash_digestsize(hash
);
40 if (!psb
->hash_keysize
)
43 err
= crypto_hash_setkey(hash
, psb
->hash_key
, psb
->hash_keysize
);
45 dprintk("%s: idx: %u: failed to set key for hash '%s', err: %d.\n",
46 __func__
, psb
->idx
, psb
->hash_string
, err
);
53 crypto_free_hash(hash
);
58 static struct crypto_ablkcipher
*pohmelfs_init_cipher(struct pohmelfs_sb
*psb
)
61 struct crypto_ablkcipher
*cipher
;
63 if (!psb
->cipher_keysize
)
66 cipher
= crypto_alloc_ablkcipher(psb
->cipher_string
, 0, 0);
68 err
= PTR_ERR(cipher
);
69 dprintk("%s: idx: %u: failed to allocate cipher '%s', err: %d.\n",
70 __func__
, psb
->idx
, psb
->cipher_string
, err
);
74 crypto_ablkcipher_clear_flags(cipher
, ~0);
76 err
= crypto_ablkcipher_setkey(cipher
, psb
->cipher_key
, psb
->cipher_keysize
);
78 dprintk("%s: idx: %u: failed to set key for cipher '%s', err: %d.\n",
79 __func__
, psb
->idx
, psb
->cipher_string
, err
);
86 crypto_free_ablkcipher(cipher
);
91 int pohmelfs_crypto_engine_init(struct pohmelfs_crypto_engine
*e
, struct pohmelfs_sb
*psb
)
98 e
->data
= kmalloc(e
->size
, GFP_KERNEL
);
104 if (psb
->hash_string
) {
105 e
->hash
= pohmelfs_init_hash(psb
);
106 if (IS_ERR(e
->hash
)) {
107 err
= PTR_ERR(e
->hash
);
113 if (psb
->cipher_string
) {
114 e
->cipher
= pohmelfs_init_cipher(psb
);
115 if (IS_ERR(e
->cipher
)) {
116 err
= PTR_ERR(e
->cipher
);
118 goto err_out_free_hash
;
125 crypto_free_hash(e
->hash
);
132 void pohmelfs_crypto_engine_exit(struct pohmelfs_crypto_engine
*e
)
134 crypto_free_hash(e
->hash
);
135 crypto_free_ablkcipher(e
->cipher
);
139 static void pohmelfs_crypto_complete(struct crypto_async_request
*req
, int err
)
141 struct pohmelfs_crypto_completion
*c
= req
->data
;
143 if (err
== -EINPROGRESS
)
146 dprintk("%s: req: %p, err: %d.\n", __func__
, req
, err
);
148 complete(&c
->complete
);
151 static int pohmelfs_crypto_process(struct ablkcipher_request
*req
,
152 struct scatterlist
*sg_dst
, struct scatterlist
*sg_src
,
153 void *iv
, int enc
, unsigned long timeout
)
155 struct pohmelfs_crypto_completion complete
;
158 init_completion(&complete
.complete
);
159 complete
.error
= -EINPROGRESS
;
161 ablkcipher_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_BACKLOG
,
162 pohmelfs_crypto_complete
, &complete
);
164 ablkcipher_request_set_crypt(req
, sg_src
, sg_dst
, sg_src
->length
, iv
);
167 err
= crypto_ablkcipher_encrypt(req
);
169 err
= crypto_ablkcipher_decrypt(req
);
174 err
= wait_for_completion_interruptible_timeout(&complete
.complete
,
179 err
= complete
.error
;
188 int pohmelfs_crypto_process_input_data(struct pohmelfs_crypto_engine
*e
, u64 cmd_iv
,
189 void *data
, struct page
*page
, unsigned int size
)
192 struct scatterlist sg
;
194 if (!e
->cipher
&& !e
->hash
)
197 dprintk("%s: eng: %p, iv: %llx, data: %p, page: %p/%lu, size: %u.\n",
198 __func__
, e
, cmd_iv
, data
, page
, (page
) ? page
->index
: 0, size
);
201 sg_init_one(&sg
, data
, size
);
203 sg_init_table(&sg
, 1);
204 sg_set_page(&sg
, page
, size
, 0);
208 struct ablkcipher_request
*req
= e
->data
+ crypto_hash_digestsize(e
->hash
);
211 memset(iv
, 0, sizeof(iv
));
212 memcpy(iv
, &cmd_iv
, sizeof(cmd_iv
));
214 ablkcipher_request_set_tfm(req
, e
->cipher
);
216 err
= pohmelfs_crypto_process(req
, &sg
, &sg
, iv
, 0, e
->timeout
);
222 struct hash_desc desc
;
223 void *dst
= e
->data
+ e
->size
/2;
228 err
= crypto_hash_init(&desc
);
232 err
= crypto_hash_update(&desc
, &sg
, size
);
236 err
= crypto_hash_final(&desc
, dst
);
240 err
= !!memcmp(dst
, e
->data
, crypto_hash_digestsize(e
->hash
));
243 #ifdef CONFIG_POHMELFS_DEBUG
245 unsigned char *recv
= e
->data
, *calc
= dst
;
247 dprintk("%s: eng: %p, hash: %p, cipher: %p: iv : %llx, hash mismatch (recv/calc): ",
248 __func__
, e
, e
->hash
, e
->cipher
, cmd_iv
);
249 for (i
= 0; i
< crypto_hash_digestsize(e
->hash
); ++i
) {
251 dprintka("%02x ", recv
[i
]);
252 if (recv
[i
] != calc
[i
]) {
253 dprintka("| calc byte: %02x.\n", calc
[i
]);
257 dprintka("%02x/%02x ", recv
[i
], calc
[i
]);
264 dprintk("%s: eng: %p, hash: %p, cipher: %p: hashes matched.\n",
265 __func__
, e
, e
->hash
, e
->cipher
);
269 dprintk("%s: eng: %p, size: %u, hash: %p, cipher: %p: completed.\n",
270 __func__
, e
, e
->size
, e
->hash
, e
->cipher
);
275 dprintk("%s: eng: %p, hash: %p, cipher: %p: err: %d.\n",
276 __func__
, e
, e
->hash
, e
->cipher
, err
);
280 static int pohmelfs_trans_iter(struct netfs_trans
*t
, struct pohmelfs_crypto_engine
*e
,
281 int (*iterator
) (struct pohmelfs_crypto_engine
*e
,
282 struct scatterlist
*dst
,
283 struct scatterlist
*src
))
285 void *data
= t
->iovec
.iov_base
+ sizeof(struct netfs_cmd
) + t
->psb
->crypto_attached_size
;
286 unsigned int size
= t
->iovec
.iov_len
- sizeof(struct netfs_cmd
) - t
->psb
->crypto_attached_size
;
287 struct netfs_cmd
*cmd
= data
;
288 unsigned int sz
, pages
= t
->attached_pages
, i
, csize
, cmd_cmd
, dpage_idx
;
289 struct scatterlist sg_src
, sg_dst
;
294 cmd_cmd
= __be16_to_cpu(cmd
->cmd
);
295 csize
= __be32_to_cpu(cmd
->size
);
296 cmd
->iv
= __cpu_to_be64(e
->iv
);
298 if (cmd_cmd
== NETFS_READ_PAGES
|| cmd_cmd
== NETFS_READ_PAGE
)
299 csize
= __be16_to_cpu(cmd
->ext
);
301 sz
= csize
+ __be16_to_cpu(cmd
->cpad
) + sizeof(struct netfs_cmd
);
303 dprintk("%s: size: %u, sz: %u, cmd_size: %u, cmd_cpad: %u.\n",
304 __func__
, size
, sz
, __be32_to_cpu(cmd
->size
), __be16_to_cpu(cmd
->cpad
));
309 sg_init_one(&sg_src
, cmd
->data
, sz
- sizeof(struct netfs_cmd
));
310 sg_init_one(&sg_dst
, cmd
->data
, sz
- sizeof(struct netfs_cmd
));
312 err
= iterator(e
, &sg_dst
, &sg_src
);
321 for (i
= 0; i
< t
->page_num
; ++i
) {
322 struct page
*page
= t
->pages
[i
];
323 struct page
*dpage
= e
->pages
[dpage_idx
];
328 sg_init_table(&sg_src
, 1);
329 sg_init_table(&sg_dst
, 1);
330 sg_set_page(&sg_src
, page
, page_private(page
), 0);
331 sg_set_page(&sg_dst
, dpage
, page_private(page
), 0);
333 err
= iterator(e
, &sg_dst
, &sg_src
);
346 static int pohmelfs_encrypt_iterator(struct pohmelfs_crypto_engine
*e
,
347 struct scatterlist
*sg_dst
, struct scatterlist
*sg_src
)
349 struct ablkcipher_request
*req
= e
->data
;
352 memset(iv
, 0, sizeof(iv
));
354 memcpy(iv
, &e
->iv
, sizeof(e
->iv
));
356 return pohmelfs_crypto_process(req
, sg_dst
, sg_src
, iv
, 1, e
->timeout
);
359 static int pohmelfs_encrypt(struct pohmelfs_crypto_thread
*tc
)
361 struct netfs_trans
*t
= tc
->trans
;
362 struct pohmelfs_crypto_engine
*e
= &tc
->eng
;
363 struct ablkcipher_request
*req
= e
->data
;
365 memset(req
, 0, sizeof(struct ablkcipher_request
));
366 ablkcipher_request_set_tfm(req
, e
->cipher
);
368 e
->iv
= pohmelfs_gen_iv(t
);
370 return pohmelfs_trans_iter(t
, e
, pohmelfs_encrypt_iterator
);
373 static int pohmelfs_hash_iterator(struct pohmelfs_crypto_engine
*e
,
374 struct scatterlist
*sg_dst
, struct scatterlist
*sg_src
)
376 return crypto_hash_update(e
->data
, sg_src
, sg_src
->length
);
379 static int pohmelfs_hash(struct pohmelfs_crypto_thread
*tc
)
381 struct pohmelfs_crypto_engine
*e
= &tc
->eng
;
382 struct hash_desc
*desc
= e
->data
;
383 unsigned char *dst
= tc
->trans
->iovec
.iov_base
+ sizeof(struct netfs_cmd
);
389 err
= crypto_hash_init(desc
);
393 err
= pohmelfs_trans_iter(tc
->trans
, e
, pohmelfs_hash_iterator
);
397 err
= crypto_hash_final(desc
, dst
);
403 dprintk("%s: ", __func__
);
404 for (i
= 0; i
< tc
->psb
->crypto_attached_size
; ++i
)
405 dprintka("%02x ", dst
[i
]);
412 static void pohmelfs_crypto_pages_free(struct pohmelfs_crypto_engine
*e
)
416 for (i
= 0; i
< e
->page_num
; ++i
)
417 __free_page(e
->pages
[i
]);
421 static int pohmelfs_crypto_pages_alloc(struct pohmelfs_crypto_engine
*e
, struct pohmelfs_sb
*psb
)
425 e
->pages
= kmalloc(psb
->trans_max_pages
* sizeof(struct page
*), GFP_KERNEL
);
429 for (i
= 0; i
< psb
->trans_max_pages
; ++i
) {
430 e
->pages
[i
] = alloc_page(GFP_KERNEL
);
446 static void pohmelfs_sys_crypto_exit_one(struct pohmelfs_crypto_thread
*t
)
448 struct pohmelfs_sb
*psb
= t
->psb
;
451 kthread_stop(t
->thread
);
453 mutex_lock(&psb
->crypto_thread_lock
);
454 list_del(&t
->thread_entry
);
455 psb
->crypto_thread_num
--;
456 mutex_unlock(&psb
->crypto_thread_lock
);
458 pohmelfs_crypto_engine_exit(&t
->eng
);
459 pohmelfs_crypto_pages_free(&t
->eng
);
463 static int pohmelfs_crypto_finish(struct netfs_trans
*t
, struct pohmelfs_sb
*psb
, int err
)
465 struct netfs_cmd
*cmd
= t
->iovec
.iov_base
;
466 netfs_convert_cmd(cmd
);
469 err
= netfs_trans_finish_send(t
, psb
);
477 void pohmelfs_crypto_thread_make_ready(struct pohmelfs_crypto_thread
*th
)
479 struct pohmelfs_sb
*psb
= th
->psb
;
484 mutex_lock(&psb
->crypto_thread_lock
);
485 list_move_tail(&th
->thread_entry
, &psb
->crypto_ready_list
);
486 mutex_unlock(&psb
->crypto_thread_lock
);
490 static int pohmelfs_crypto_thread_trans(struct pohmelfs_crypto_thread
*t
)
492 struct netfs_trans
*trans
;
499 err
= pohmelfs_hash(t
);
505 err
= pohmelfs_encrypt(t
);
508 trans
->eng
= &t
->eng
;
516 pohmelfs_crypto_thread_make_ready(t
);
518 pohmelfs_crypto_finish(trans
, t
->psb
, err
);
522 static int pohmelfs_crypto_thread_page(struct pohmelfs_crypto_thread
*t
)
524 struct pohmelfs_crypto_engine
*e
= &t
->eng
;
525 struct page
*page
= t
->page
;
528 WARN_ON(!PageChecked(page
));
530 err
= pohmelfs_crypto_process_input_data(e
, e
->iv
, NULL
, page
, t
->size
);
532 SetPageUptodate(page
);
536 page_cache_release(page
);
538 pohmelfs_crypto_thread_make_ready(t
);
543 static int pohmelfs_crypto_thread_func(void *data
)
545 struct pohmelfs_crypto_thread
*t
= data
;
547 while (!kthread_should_stop()) {
548 wait_event_interruptible(t
->wait
, kthread_should_stop() ||
549 t
->trans
|| t
->page
);
551 if (kthread_should_stop())
554 if (!t
->trans
&& !t
->page
)
557 dprintk("%s: thread: %p, trans: %p, page: %p.\n",
558 __func__
, t
, t
->trans
, t
->page
);
561 pohmelfs_crypto_thread_trans(t
);
563 pohmelfs_crypto_thread_page(t
);
569 static void pohmelfs_crypto_flush(struct pohmelfs_sb
*psb
, struct list_head
*head
)
571 while (!list_empty(head
)) {
572 struct pohmelfs_crypto_thread
*t
= NULL
;
574 mutex_lock(&psb
->crypto_thread_lock
);
575 if (!list_empty(head
)) {
576 t
= list_first_entry(head
, struct pohmelfs_crypto_thread
, thread_entry
);
577 list_del_init(&t
->thread_entry
);
579 mutex_unlock(&psb
->crypto_thread_lock
);
582 pohmelfs_sys_crypto_exit_one(t
);
586 static void pohmelfs_sys_crypto_exit(struct pohmelfs_sb
*psb
)
588 while (!list_empty(&psb
->crypto_active_list
) || !list_empty(&psb
->crypto_ready_list
)) {
589 dprintk("%s: crypto_thread_num: %u.\n", __func__
, psb
->crypto_thread_num
);
590 pohmelfs_crypto_flush(psb
, &psb
->crypto_active_list
);
591 pohmelfs_crypto_flush(psb
, &psb
->crypto_ready_list
);
595 static int pohmelfs_sys_crypto_init(struct pohmelfs_sb
*psb
)
598 struct pohmelfs_crypto_thread
*t
;
599 struct pohmelfs_config
*c
;
600 struct netfs_state
*st
;
603 list_for_each_entry(c
, &psb
->state_list
, config_entry
) {
606 err
= pohmelfs_crypto_engine_init(&st
->eng
, psb
);
610 dprintk("%s: st: %p, eng: %p, hash: %p, cipher: %p.\n",
611 __func__
, st
, &st
->eng
, &st
->eng
.hash
, &st
->eng
.cipher
);
614 for (i
= 0; i
< psb
->crypto_thread_num
; ++i
) {
616 t
= kzalloc(sizeof(struct pohmelfs_crypto_thread
), GFP_KERNEL
);
618 goto err_out_free_state_engines
;
620 init_waitqueue_head(&t
->wait
);
626 err
= pohmelfs_crypto_engine_init(&t
->eng
, psb
);
628 goto err_out_free_state_engines
;
630 err
= pohmelfs_crypto_pages_alloc(&t
->eng
, psb
);
634 t
->thread
= kthread_run(pohmelfs_crypto_thread_func
, t
,
635 "pohmelfs-crypto-%d-%d", psb
->idx
, i
);
636 if (IS_ERR(t
->thread
)) {
637 err
= PTR_ERR(t
->thread
);
643 psb
->crypto_align_size
= crypto_ablkcipher_blocksize(t
->eng
.cipher
);
645 mutex_lock(&psb
->crypto_thread_lock
);
646 list_add_tail(&t
->thread_entry
, &psb
->crypto_ready_list
);
647 mutex_unlock(&psb
->crypto_thread_lock
);
650 psb
->crypto_thread_num
= i
;
654 pohmelfs_sys_crypto_exit_one(t
);
655 err_out_free_state_engines
:
656 list_for_each_entry(c
, &psb
->state_list
, config_entry
) {
658 pohmelfs_crypto_engine_exit(&st
->eng
);
661 pohmelfs_sys_crypto_exit(psb
);
665 void pohmelfs_crypto_exit(struct pohmelfs_sb
*psb
)
667 pohmelfs_sys_crypto_exit(psb
);
669 kfree(psb
->hash_string
);
670 kfree(psb
->cipher_string
);
673 static int pohmelfs_crypt_init_complete(struct page
**pages
, unsigned int page_num
,
674 void *private, int err
)
676 struct pohmelfs_sb
*psb
= private;
679 dprintk("%s: err: %d.\n", __func__
, err
);
686 static int pohmelfs_crypto_init_handshake(struct pohmelfs_sb
*psb
)
688 struct netfs_trans
*t
;
689 struct netfs_crypto_capabilities
*cap
;
690 struct netfs_cmd
*cmd
;
692 int err
= -ENOMEM
, size
;
694 size
= sizeof(struct netfs_crypto_capabilities
) +
695 psb
->cipher_strlen
+ psb
->hash_strlen
+ 2; /* 0 bytes */
697 t
= netfs_trans_alloc(psb
, size
, 0, 0);
701 t
->complete
= pohmelfs_crypt_init_complete
;
704 cmd
= netfs_trans_current(t
);
705 cap
= (struct netfs_crypto_capabilities
*)(cmd
+ 1);
706 str
= (char *)(cap
+ 1);
708 cmd
->cmd
= NETFS_CAPABILITIES
;
709 cmd
->id
= POHMELFS_CRYPTO_CAPABILITIES
;
715 netfs_convert_cmd(cmd
);
716 netfs_trans_update(cmd
, t
, size
);
718 cap
->hash_strlen
= psb
->hash_strlen
;
719 if (cap
->hash_strlen
) {
720 sprintf(str
, "%s", psb
->hash_string
);
721 str
+= cap
->hash_strlen
;
724 cap
->cipher_strlen
= psb
->cipher_strlen
;
725 cap
->cipher_keysize
= psb
->cipher_keysize
;
726 if (cap
->cipher_strlen
)
727 sprintf(str
, "%s", psb
->cipher_string
);
729 netfs_convert_crypto_capabilities(cap
);
732 err
= netfs_trans_finish(t
, psb
);
736 err
= wait_event_interruptible_timeout(psb
->wait
, (psb
->flags
!= ~0),
737 psb
->wait_on_page_timeout
);
744 psb
->perform_crypto
= 1;
748 * At this point NETFS_CAPABILITIES response command
749 * should setup superblock in a way, which is acceptable
750 * for both client and server, so if server refuses connection,
751 * it will send error in transaction response.
763 int pohmelfs_crypto_init(struct pohmelfs_sb
*psb
)
767 if (!psb
->cipher_string
&& !psb
->hash_string
)
770 err
= pohmelfs_crypto_init_handshake(psb
);
774 err
= pohmelfs_sys_crypto_init(psb
);
781 static int pohmelfs_crypto_thread_get(struct pohmelfs_sb
*psb
,
782 int (*action
)(struct pohmelfs_crypto_thread
*t
, void *data
), void *data
)
784 struct pohmelfs_crypto_thread
*t
= NULL
;
788 err
= wait_event_interruptible_timeout(psb
->wait
,
789 !list_empty(&psb
->crypto_ready_list
),
790 psb
->wait_on_page_timeout
);
794 mutex_lock(&psb
->crypto_thread_lock
);
795 if (!list_empty(&psb
->crypto_ready_list
)) {
796 t
= list_entry(psb
->crypto_ready_list
.prev
,
797 struct pohmelfs_crypto_thread
,
800 list_move_tail(&t
->thread_entry
,
801 &psb
->crypto_active_list
);
807 mutex_unlock(&psb
->crypto_thread_lock
);
813 static int pohmelfs_trans_crypt_action(struct pohmelfs_crypto_thread
*t
, void *data
)
815 struct netfs_trans
*trans
= data
;
817 netfs_trans_get(trans
);
820 dprintk("%s: t: %p, gen: %u, thread: %p.\n", __func__
, trans
, trans
->gen
, t
);
824 int pohmelfs_trans_crypt(struct netfs_trans
*trans
, struct pohmelfs_sb
*psb
)
826 if ((!psb
->hash_string
&& !psb
->cipher_string
) || !psb
->perform_crypto
) {
827 netfs_trans_get(trans
);
828 return pohmelfs_crypto_finish(trans
, psb
, 0);
831 return pohmelfs_crypto_thread_get(psb
, pohmelfs_trans_crypt_action
, trans
);
834 struct pohmelfs_crypto_input_action_data
{
836 struct pohmelfs_crypto_engine
*e
;
841 static int pohmelfs_crypt_input_page_action(struct pohmelfs_crypto_thread
*t
, void *data
)
843 struct pohmelfs_crypto_input_action_data
*act
= data
;
845 memcpy(t
->eng
.data
, act
->e
->data
, t
->psb
->crypto_attached_size
);
854 int pohmelfs_crypto_process_input_page(struct pohmelfs_crypto_engine
*e
,
855 struct page
*page
, unsigned int size
, u64 iv
)
857 struct inode
*inode
= page
->mapping
->host
;
858 struct pohmelfs_crypto_input_action_data act
;
866 err
= pohmelfs_crypto_thread_get(POHMELFS_SB(inode
->i_sb
),
867 pohmelfs_crypt_input_page_action
, &act
);
874 SetPageUptodate(page
);
875 page_cache_release(page
);