2 * 2007+ Copyright (c) Evgeniy Polyakov <zbr@ioremap.net>
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
16 #include <linux/crypto.h>
17 #include <linux/highmem.h>
18 #include <linux/kthread.h>
19 #include <linux/pagemap.h>
20 #include <linux/slab.h>
24 static struct crypto_hash
*pohmelfs_init_hash(struct pohmelfs_sb
*psb
)
27 struct crypto_hash
*hash
;
29 hash
= crypto_alloc_hash(psb
->hash_string
, 0, CRYPTO_ALG_ASYNC
);
32 dprintk("%s: idx: %u: failed to allocate hash '%s', err: %d.\n",
33 __func__
, psb
->idx
, psb
->hash_string
, err
);
37 psb
->crypto_attached_size
= crypto_hash_digestsize(hash
);
39 if (!psb
->hash_keysize
)
42 err
= crypto_hash_setkey(hash
, psb
->hash_key
, psb
->hash_keysize
);
44 dprintk("%s: idx: %u: failed to set key for hash '%s', err: %d.\n",
45 __func__
, psb
->idx
, psb
->hash_string
, err
);
52 crypto_free_hash(hash
);
57 static struct crypto_ablkcipher
*pohmelfs_init_cipher(struct pohmelfs_sb
*psb
)
60 struct crypto_ablkcipher
*cipher
;
62 if (!psb
->cipher_keysize
)
65 cipher
= crypto_alloc_ablkcipher(psb
->cipher_string
, 0, 0);
67 err
= PTR_ERR(cipher
);
68 dprintk("%s: idx: %u: failed to allocate cipher '%s', err: %d.\n",
69 __func__
, psb
->idx
, psb
->cipher_string
, err
);
73 crypto_ablkcipher_clear_flags(cipher
, ~0);
75 err
= crypto_ablkcipher_setkey(cipher
, psb
->cipher_key
, psb
->cipher_keysize
);
77 dprintk("%s: idx: %u: failed to set key for cipher '%s', err: %d.\n",
78 __func__
, psb
->idx
, psb
->cipher_string
, err
);
85 crypto_free_ablkcipher(cipher
);
90 int pohmelfs_crypto_engine_init(struct pohmelfs_crypto_engine
*e
, struct pohmelfs_sb
*psb
)
97 e
->data
= kmalloc(e
->size
, GFP_KERNEL
);
103 if (psb
->hash_string
) {
104 e
->hash
= pohmelfs_init_hash(psb
);
105 if (IS_ERR(e
->hash
)) {
106 err
= PTR_ERR(e
->hash
);
112 if (psb
->cipher_string
) {
113 e
->cipher
= pohmelfs_init_cipher(psb
);
114 if (IS_ERR(e
->cipher
)) {
115 err
= PTR_ERR(e
->cipher
);
117 goto err_out_free_hash
;
124 crypto_free_hash(e
->hash
);
131 void pohmelfs_crypto_engine_exit(struct pohmelfs_crypto_engine
*e
)
133 crypto_free_hash(e
->hash
);
134 crypto_free_ablkcipher(e
->cipher
);
138 static void pohmelfs_crypto_complete(struct crypto_async_request
*req
, int err
)
140 struct pohmelfs_crypto_completion
*c
= req
->data
;
142 if (err
== -EINPROGRESS
)
145 dprintk("%s: req: %p, err: %d.\n", __func__
, req
, err
);
147 complete(&c
->complete
);
150 static int pohmelfs_crypto_process(struct ablkcipher_request
*req
,
151 struct scatterlist
*sg_dst
, struct scatterlist
*sg_src
,
152 void *iv
, int enc
, unsigned long timeout
)
154 struct pohmelfs_crypto_completion complete
;
157 init_completion(&complete
.complete
);
158 complete
.error
= -EINPROGRESS
;
160 ablkcipher_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_BACKLOG
,
161 pohmelfs_crypto_complete
, &complete
);
163 ablkcipher_request_set_crypt(req
, sg_src
, sg_dst
, sg_src
->length
, iv
);
166 err
= crypto_ablkcipher_encrypt(req
);
168 err
= crypto_ablkcipher_decrypt(req
);
173 err
= wait_for_completion_interruptible_timeout(&complete
.complete
,
178 err
= complete
.error
;
187 int pohmelfs_crypto_process_input_data(struct pohmelfs_crypto_engine
*e
, u64 cmd_iv
,
188 void *data
, struct page
*page
, unsigned int size
)
191 struct scatterlist sg
;
193 if (!e
->cipher
&& !e
->hash
)
196 dprintk("%s: eng: %p, iv: %llx, data: %p, page: %p/%lu, size: %u.\n",
197 __func__
, e
, cmd_iv
, data
, page
, (page
) ? page
->index
: 0, size
);
200 sg_init_one(&sg
, data
, size
);
202 sg_init_table(&sg
, 1);
203 sg_set_page(&sg
, page
, size
, 0);
207 struct ablkcipher_request
*req
= e
->data
+ crypto_hash_digestsize(e
->hash
);
210 memset(iv
, 0, sizeof(iv
));
211 memcpy(iv
, &cmd_iv
, sizeof(cmd_iv
));
213 ablkcipher_request_set_tfm(req
, e
->cipher
);
215 err
= pohmelfs_crypto_process(req
, &sg
, &sg
, iv
, 0, e
->timeout
);
221 struct hash_desc desc
;
222 void *dst
= e
->data
+ e
->size
/2;
227 err
= crypto_hash_init(&desc
);
231 err
= crypto_hash_update(&desc
, &sg
, size
);
235 err
= crypto_hash_final(&desc
, dst
);
239 err
= !!memcmp(dst
, e
->data
, crypto_hash_digestsize(e
->hash
));
242 #ifdef CONFIG_POHMELFS_DEBUG
244 unsigned char *recv
= e
->data
, *calc
= dst
;
246 dprintk("%s: eng: %p, hash: %p, cipher: %p: iv : %llx, hash mismatch (recv/calc): ",
247 __func__
, e
, e
->hash
, e
->cipher
, cmd_iv
);
248 for (i
= 0; i
< crypto_hash_digestsize(e
->hash
); ++i
) {
250 dprintka("%02x ", recv
[i
]);
251 if (recv
[i
] != calc
[i
]) {
252 dprintka("| calc byte: %02x.\n", calc
[i
]);
256 dprintka("%02x/%02x ", recv
[i
], calc
[i
]);
263 dprintk("%s: eng: %p, hash: %p, cipher: %p: hashes matched.\n",
264 __func__
, e
, e
->hash
, e
->cipher
);
268 dprintk("%s: eng: %p, size: %u, hash: %p, cipher: %p: completed.\n",
269 __func__
, e
, e
->size
, e
->hash
, e
->cipher
);
274 dprintk("%s: eng: %p, hash: %p, cipher: %p: err: %d.\n",
275 __func__
, e
, e
->hash
, e
->cipher
, err
);
279 static int pohmelfs_trans_iter(struct netfs_trans
*t
, struct pohmelfs_crypto_engine
*e
,
280 int (*iterator
) (struct pohmelfs_crypto_engine
*e
,
281 struct scatterlist
*dst
,
282 struct scatterlist
*src
))
284 void *data
= t
->iovec
.iov_base
+ sizeof(struct netfs_cmd
) + t
->psb
->crypto_attached_size
;
285 unsigned int size
= t
->iovec
.iov_len
- sizeof(struct netfs_cmd
) - t
->psb
->crypto_attached_size
;
286 struct netfs_cmd
*cmd
= data
;
287 unsigned int sz
, pages
= t
->attached_pages
, i
, csize
, cmd_cmd
, dpage_idx
;
288 struct scatterlist sg_src
, sg_dst
;
293 cmd_cmd
= __be16_to_cpu(cmd
->cmd
);
294 csize
= __be32_to_cpu(cmd
->size
);
295 cmd
->iv
= __cpu_to_be64(e
->iv
);
297 if (cmd_cmd
== NETFS_READ_PAGES
|| cmd_cmd
== NETFS_READ_PAGE
)
298 csize
= __be16_to_cpu(cmd
->ext
);
300 sz
= csize
+ __be16_to_cpu(cmd
->cpad
) + sizeof(struct netfs_cmd
);
302 dprintk("%s: size: %u, sz: %u, cmd_size: %u, cmd_cpad: %u.\n",
303 __func__
, size
, sz
, __be32_to_cpu(cmd
->size
), __be16_to_cpu(cmd
->cpad
));
308 sg_init_one(&sg_src
, cmd
->data
, sz
- sizeof(struct netfs_cmd
));
309 sg_init_one(&sg_dst
, cmd
->data
, sz
- sizeof(struct netfs_cmd
));
311 err
= iterator(e
, &sg_dst
, &sg_src
);
320 for (i
= 0; i
< t
->page_num
; ++i
) {
321 struct page
*page
= t
->pages
[i
];
322 struct page
*dpage
= e
->pages
[dpage_idx
];
327 sg_init_table(&sg_src
, 1);
328 sg_init_table(&sg_dst
, 1);
329 sg_set_page(&sg_src
, page
, page_private(page
), 0);
330 sg_set_page(&sg_dst
, dpage
, page_private(page
), 0);
332 err
= iterator(e
, &sg_dst
, &sg_src
);
345 static int pohmelfs_encrypt_iterator(struct pohmelfs_crypto_engine
*e
,
346 struct scatterlist
*sg_dst
, struct scatterlist
*sg_src
)
348 struct ablkcipher_request
*req
= e
->data
;
351 memset(iv
, 0, sizeof(iv
));
353 memcpy(iv
, &e
->iv
, sizeof(e
->iv
));
355 return pohmelfs_crypto_process(req
, sg_dst
, sg_src
, iv
, 1, e
->timeout
);
358 static int pohmelfs_encrypt(struct pohmelfs_crypto_thread
*tc
)
360 struct netfs_trans
*t
= tc
->trans
;
361 struct pohmelfs_crypto_engine
*e
= &tc
->eng
;
362 struct ablkcipher_request
*req
= e
->data
;
364 memset(req
, 0, sizeof(struct ablkcipher_request
));
365 ablkcipher_request_set_tfm(req
, e
->cipher
);
367 e
->iv
= pohmelfs_gen_iv(t
);
369 return pohmelfs_trans_iter(t
, e
, pohmelfs_encrypt_iterator
);
372 static int pohmelfs_hash_iterator(struct pohmelfs_crypto_engine
*e
,
373 struct scatterlist
*sg_dst
, struct scatterlist
*sg_src
)
375 return crypto_hash_update(e
->data
, sg_src
, sg_src
->length
);
378 static int pohmelfs_hash(struct pohmelfs_crypto_thread
*tc
)
380 struct pohmelfs_crypto_engine
*e
= &tc
->eng
;
381 struct hash_desc
*desc
= e
->data
;
382 unsigned char *dst
= tc
->trans
->iovec
.iov_base
+ sizeof(struct netfs_cmd
);
388 err
= crypto_hash_init(desc
);
392 err
= pohmelfs_trans_iter(tc
->trans
, e
, pohmelfs_hash_iterator
);
396 err
= crypto_hash_final(desc
, dst
);
402 dprintk("%s: ", __func__
);
403 for (i
= 0; i
< tc
->psb
->crypto_attached_size
; ++i
)
404 dprintka("%02x ", dst
[i
]);
411 static void pohmelfs_crypto_pages_free(struct pohmelfs_crypto_engine
*e
)
415 for (i
= 0; i
< e
->page_num
; ++i
)
416 __free_page(e
->pages
[i
]);
420 static int pohmelfs_crypto_pages_alloc(struct pohmelfs_crypto_engine
*e
, struct pohmelfs_sb
*psb
)
424 e
->pages
= kmalloc(psb
->trans_max_pages
* sizeof(struct page
*), GFP_KERNEL
);
428 for (i
= 0; i
< psb
->trans_max_pages
; ++i
) {
429 e
->pages
[i
] = alloc_page(GFP_KERNEL
);
445 static void pohmelfs_sys_crypto_exit_one(struct pohmelfs_crypto_thread
*t
)
447 struct pohmelfs_sb
*psb
= t
->psb
;
450 kthread_stop(t
->thread
);
452 mutex_lock(&psb
->crypto_thread_lock
);
453 list_del(&t
->thread_entry
);
454 psb
->crypto_thread_num
--;
455 mutex_unlock(&psb
->crypto_thread_lock
);
457 pohmelfs_crypto_engine_exit(&t
->eng
);
458 pohmelfs_crypto_pages_free(&t
->eng
);
462 static int pohmelfs_crypto_finish(struct netfs_trans
*t
, struct pohmelfs_sb
*psb
, int err
)
464 struct netfs_cmd
*cmd
= t
->iovec
.iov_base
;
465 netfs_convert_cmd(cmd
);
468 err
= netfs_trans_finish_send(t
, psb
);
476 void pohmelfs_crypto_thread_make_ready(struct pohmelfs_crypto_thread
*th
)
478 struct pohmelfs_sb
*psb
= th
->psb
;
483 mutex_lock(&psb
->crypto_thread_lock
);
484 list_move_tail(&th
->thread_entry
, &psb
->crypto_ready_list
);
485 mutex_unlock(&psb
->crypto_thread_lock
);
489 static int pohmelfs_crypto_thread_trans(struct pohmelfs_crypto_thread
*t
)
491 struct netfs_trans
*trans
;
498 err
= pohmelfs_hash(t
);
504 err
= pohmelfs_encrypt(t
);
507 trans
->eng
= &t
->eng
;
515 pohmelfs_crypto_thread_make_ready(t
);
517 pohmelfs_crypto_finish(trans
, t
->psb
, err
);
521 static int pohmelfs_crypto_thread_page(struct pohmelfs_crypto_thread
*t
)
523 struct pohmelfs_crypto_engine
*e
= &t
->eng
;
524 struct page
*page
= t
->page
;
527 WARN_ON(!PageChecked(page
));
529 err
= pohmelfs_crypto_process_input_data(e
, e
->iv
, NULL
, page
, t
->size
);
531 SetPageUptodate(page
);
535 page_cache_release(page
);
537 pohmelfs_crypto_thread_make_ready(t
);
542 static int pohmelfs_crypto_thread_func(void *data
)
544 struct pohmelfs_crypto_thread
*t
= data
;
546 while (!kthread_should_stop()) {
547 wait_event_interruptible(t
->wait
, kthread_should_stop() ||
548 t
->trans
|| t
->page
);
550 if (kthread_should_stop())
553 if (!t
->trans
&& !t
->page
)
556 dprintk("%s: thread: %p, trans: %p, page: %p.\n",
557 __func__
, t
, t
->trans
, t
->page
);
560 pohmelfs_crypto_thread_trans(t
);
562 pohmelfs_crypto_thread_page(t
);
568 static void pohmelfs_crypto_flush(struct pohmelfs_sb
*psb
, struct list_head
*head
)
570 while (!list_empty(head
)) {
571 struct pohmelfs_crypto_thread
*t
= NULL
;
573 mutex_lock(&psb
->crypto_thread_lock
);
574 if (!list_empty(head
)) {
575 t
= list_first_entry(head
, struct pohmelfs_crypto_thread
, thread_entry
);
576 list_del_init(&t
->thread_entry
);
578 mutex_unlock(&psb
->crypto_thread_lock
);
581 pohmelfs_sys_crypto_exit_one(t
);
585 static void pohmelfs_sys_crypto_exit(struct pohmelfs_sb
*psb
)
587 while (!list_empty(&psb
->crypto_active_list
) || !list_empty(&psb
->crypto_ready_list
)) {
588 dprintk("%s: crypto_thread_num: %u.\n", __func__
, psb
->crypto_thread_num
);
589 pohmelfs_crypto_flush(psb
, &psb
->crypto_active_list
);
590 pohmelfs_crypto_flush(psb
, &psb
->crypto_ready_list
);
594 static int pohmelfs_sys_crypto_init(struct pohmelfs_sb
*psb
)
597 struct pohmelfs_crypto_thread
*t
;
598 struct pohmelfs_config
*c
;
599 struct netfs_state
*st
;
602 list_for_each_entry(c
, &psb
->state_list
, config_entry
) {
605 err
= pohmelfs_crypto_engine_init(&st
->eng
, psb
);
609 dprintk("%s: st: %p, eng: %p, hash: %p, cipher: %p.\n",
610 __func__
, st
, &st
->eng
, &st
->eng
.hash
, &st
->eng
.cipher
);
613 for (i
= 0; i
< psb
->crypto_thread_num
; ++i
) {
615 t
= kzalloc(sizeof(struct pohmelfs_crypto_thread
), GFP_KERNEL
);
617 goto err_out_free_state_engines
;
619 init_waitqueue_head(&t
->wait
);
625 err
= pohmelfs_crypto_engine_init(&t
->eng
, psb
);
627 goto err_out_free_state_engines
;
629 err
= pohmelfs_crypto_pages_alloc(&t
->eng
, psb
);
633 t
->thread
= kthread_run(pohmelfs_crypto_thread_func
, t
,
634 "pohmelfs-crypto-%d-%d", psb
->idx
, i
);
635 if (IS_ERR(t
->thread
)) {
636 err
= PTR_ERR(t
->thread
);
642 psb
->crypto_align_size
= crypto_ablkcipher_blocksize(t
->eng
.cipher
);
644 mutex_lock(&psb
->crypto_thread_lock
);
645 list_add_tail(&t
->thread_entry
, &psb
->crypto_ready_list
);
646 mutex_unlock(&psb
->crypto_thread_lock
);
649 psb
->crypto_thread_num
= i
;
653 pohmelfs_sys_crypto_exit_one(t
);
654 err_out_free_state_engines
:
655 list_for_each_entry(c
, &psb
->state_list
, config_entry
) {
657 pohmelfs_crypto_engine_exit(&st
->eng
);
660 pohmelfs_sys_crypto_exit(psb
);
664 void pohmelfs_crypto_exit(struct pohmelfs_sb
*psb
)
666 pohmelfs_sys_crypto_exit(psb
);
668 kfree(psb
->hash_string
);
669 kfree(psb
->cipher_string
);
672 static int pohmelfs_crypt_init_complete(struct page
**pages
, unsigned int page_num
,
673 void *private, int err
)
675 struct pohmelfs_sb
*psb
= private;
678 dprintk("%s: err: %d.\n", __func__
, err
);
685 static int pohmelfs_crypto_init_handshake(struct pohmelfs_sb
*psb
)
687 struct netfs_trans
*t
;
688 struct netfs_crypto_capabilities
*cap
;
689 struct netfs_cmd
*cmd
;
691 int err
= -ENOMEM
, size
;
693 size
= sizeof(struct netfs_crypto_capabilities
) +
694 psb
->cipher_strlen
+ psb
->hash_strlen
+ 2; /* 0 bytes */
696 t
= netfs_trans_alloc(psb
, size
, 0, 0);
700 t
->complete
= pohmelfs_crypt_init_complete
;
703 cmd
= netfs_trans_current(t
);
704 cap
= (struct netfs_crypto_capabilities
*)(cmd
+ 1);
705 str
= (char *)(cap
+ 1);
707 cmd
->cmd
= NETFS_CAPABILITIES
;
708 cmd
->id
= POHMELFS_CRYPTO_CAPABILITIES
;
714 netfs_convert_cmd(cmd
);
715 netfs_trans_update(cmd
, t
, size
);
717 cap
->hash_strlen
= psb
->hash_strlen
;
718 if (cap
->hash_strlen
) {
719 sprintf(str
, "%s", psb
->hash_string
);
720 str
+= cap
->hash_strlen
;
723 cap
->cipher_strlen
= psb
->cipher_strlen
;
724 cap
->cipher_keysize
= psb
->cipher_keysize
;
725 if (cap
->cipher_strlen
)
726 sprintf(str
, "%s", psb
->cipher_string
);
728 netfs_convert_crypto_capabilities(cap
);
731 err
= netfs_trans_finish(t
, psb
);
735 err
= wait_event_interruptible_timeout(psb
->wait
, (psb
->flags
!= ~0),
736 psb
->wait_on_page_timeout
);
743 psb
->perform_crypto
= 1;
747 * At this point NETFS_CAPABILITIES response command
748 * should setup superblock in a way, which is acceptable
749 * for both client and server, so if server refuses connection,
750 * it will send error in transaction response.
762 int pohmelfs_crypto_init(struct pohmelfs_sb
*psb
)
766 if (!psb
->cipher_string
&& !psb
->hash_string
)
769 err
= pohmelfs_crypto_init_handshake(psb
);
773 err
= pohmelfs_sys_crypto_init(psb
);
780 static int pohmelfs_crypto_thread_get(struct pohmelfs_sb
*psb
,
781 int (*action
)(struct pohmelfs_crypto_thread
*t
, void *data
), void *data
)
783 struct pohmelfs_crypto_thread
*t
= NULL
;
787 err
= wait_event_interruptible_timeout(psb
->wait
,
788 !list_empty(&psb
->crypto_ready_list
),
789 psb
->wait_on_page_timeout
);
793 mutex_lock(&psb
->crypto_thread_lock
);
794 if (!list_empty(&psb
->crypto_ready_list
)) {
795 t
= list_entry(psb
->crypto_ready_list
.prev
,
796 struct pohmelfs_crypto_thread
,
799 list_move_tail(&t
->thread_entry
,
800 &psb
->crypto_active_list
);
806 mutex_unlock(&psb
->crypto_thread_lock
);
812 static int pohmelfs_trans_crypt_action(struct pohmelfs_crypto_thread
*t
, void *data
)
814 struct netfs_trans
*trans
= data
;
816 netfs_trans_get(trans
);
819 dprintk("%s: t: %p, gen: %u, thread: %p.\n", __func__
, trans
, trans
->gen
, t
);
823 int pohmelfs_trans_crypt(struct netfs_trans
*trans
, struct pohmelfs_sb
*psb
)
825 if ((!psb
->hash_string
&& !psb
->cipher_string
) || !psb
->perform_crypto
) {
826 netfs_trans_get(trans
);
827 return pohmelfs_crypto_finish(trans
, psb
, 0);
830 return pohmelfs_crypto_thread_get(psb
, pohmelfs_trans_crypt_action
, trans
);
833 struct pohmelfs_crypto_input_action_data
{
835 struct pohmelfs_crypto_engine
*e
;
840 static int pohmelfs_crypt_input_page_action(struct pohmelfs_crypto_thread
*t
, void *data
)
842 struct pohmelfs_crypto_input_action_data
*act
= data
;
844 memcpy(t
->eng
.data
, act
->e
->data
, t
->psb
->crypto_attached_size
);
853 int pohmelfs_crypto_process_input_page(struct pohmelfs_crypto_engine
*e
,
854 struct page
*page
, unsigned int size
, u64 iv
)
856 struct inode
*inode
= page
->mapping
->host
;
857 struct pohmelfs_crypto_input_action_data act
;
865 err
= pohmelfs_crypto_thread_get(POHMELFS_SB(inode
->i_sb
),
866 pohmelfs_crypt_input_page_action
, &act
);
873 SetPageUptodate(page
);
874 page_cache_release(page
);