?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.1.5
[linux/fpc-iii.git] / net / tls / tls_device.c
blob14dedb24fa7b6ff1e7fe99f9adbe114eeabf15c1
1 /* Copyright (c) 2018, Mellanox Technologies All rights reserved.
2 *
3 * This software is available to you under a choice of one of two
4 * licenses. You may choose to be licensed under the terms of the GNU
5 * General Public License (GPL) Version 2, available from the file
6 * COPYING in the main directory of this source tree, or the
7 * OpenIB.org BSD license below:
8 *
9 * Redistribution and use in source and binary forms, with or
10 * without modification, are permitted provided that the following
11 * conditions are met:
12 *
13 * - Redistributions of source code must retain the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer.
16 *
17 * - Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials
20 * provided with the distribution.
21 *
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29 * SOFTWARE.
30 */
31
32 #include <crypto/aead.h>
33 #include <linux/highmem.h>
34 #include <linux/module.h>
35 #include <linux/netdevice.h>
36 #include <net/dst.h>
37 #include <net/inet_connection_sock.h>
38 #include <net/tcp.h>
39 #include <net/tls.h>
40
41 /* device_offload_lock is used to synchronize tls_dev_add
42 * against NETDEV_DOWN notifications.
43 */
44 static DECLARE_RWSEM(device_offload_lock);
45
46 static void tls_device_gc_task(struct work_struct *work);
47
48 static DECLARE_WORK(tls_device_gc_work, tls_device_gc_task);
49 static LIST_HEAD(tls_device_gc_list);
50 static LIST_HEAD(tls_device_list);
51 static DEFINE_SPINLOCK(tls_device_lock);
52
53 static void tls_device_free_ctx(struct tls_context *ctx)
54 {
55 if (ctx->tx_conf == TLS_HW) {
56 kfree(tls_offload_ctx_tx(ctx));
57 kfree(ctx->tx.rec_seq);
58 kfree(ctx->tx.iv);
59 }
60
61 if (ctx->rx_conf == TLS_HW)
62 kfree(tls_offload_ctx_rx(ctx));
63
64 kfree(ctx);
65 }
66
67 static void tls_device_gc_task(struct work_struct *work)
68 {
69 struct tls_context *ctx, *tmp;
70 unsigned long flags;
71 LIST_HEAD(gc_list);
72
73 spin_lock_irqsave(&tls_device_lock, flags);
74 list_splice_init(&tls_device_gc_list, &gc_list);
75 spin_unlock_irqrestore(&tls_device_lock, flags);
76
77 list_for_each_entry_safe(ctx, tmp, &gc_list, list) {
78 struct net_device *netdev = ctx->netdev;
79
80 if (netdev && ctx->tx_conf == TLS_HW) {
81 netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
82 TLS_OFFLOAD_CTX_DIR_TX);
83 dev_put(netdev);
84 ctx->netdev = NULL;
85 }
86
87 list_del(&ctx->list);
88 tls_device_free_ctx(ctx);
89 }
90 }
91
92 static void tls_device_attach(struct tls_context *ctx, struct sock *sk,
93 struct net_device *netdev)
94 {
95 if (sk->sk_destruct != tls_device_sk_destruct) {
96 refcount_set(&ctx->refcount, 1);
97 dev_hold(netdev);
98 ctx->netdev = netdev;
99 spin_lock_irq(&tls_device_lock);
100 list_add_tail(&ctx->list, &tls_device_list);
101 spin_unlock_irq(&tls_device_lock);
102
103 ctx->sk_destruct = sk->sk_destruct;
104 sk->sk_destruct = tls_device_sk_destruct;
105 }
106 }
107
108 static void tls_device_queue_ctx_destruction(struct tls_context *ctx)
109 {
110 unsigned long flags;
111
112 spin_lock_irqsave(&tls_device_lock, flags);
113 list_move_tail(&ctx->list, &tls_device_gc_list);
114
115 /* schedule_work inside the spinlock
116 * to make sure tls_device_down waits for that work.
117 */
118 schedule_work(&tls_device_gc_work);
119
120 spin_unlock_irqrestore(&tls_device_lock, flags);
121 }
122
123 /* We assume that the socket is already connected */
124 static struct net_device *get_netdev_for_sock(struct sock *sk)
125 {
126 struct dst_entry *dst = sk_dst_get(sk);
127 struct net_device *netdev = NULL;
128
129 if (likely(dst)) {
130 netdev = dst->dev;
131 dev_hold(netdev);
132 }
133
134 dst_release(dst);
135
136 return netdev;
137 }
138
139 static void destroy_record(struct tls_record_info *record)
140 {
141 int nr_frags = record->num_frags;
142 skb_frag_t *frag;
143
144 while (nr_frags-- > 0) {
145 frag = &record->frags[nr_frags];
146 __skb_frag_unref(frag);
147 }
148 kfree(record);
149 }
150
151 static void delete_all_records(struct tls_offload_context_tx *offload_ctx)
152 {
153 struct tls_record_info *info, *temp;
154
155 list_for_each_entry_safe(info, temp, &offload_ctx->records_list, list) {
156 list_del(&info->list);
157 destroy_record(info);
158 }
159
160 offload_ctx->retransmit_hint = NULL;
161 }
162
163 static void tls_icsk_clean_acked(struct sock *sk, u32 acked_seq)
164 {
165 struct tls_context *tls_ctx = tls_get_ctx(sk);
166 struct tls_record_info *info, *temp;
167 struct tls_offload_context_tx *ctx;
168 u64 deleted_records = 0;
169 unsigned long flags;
170
171 if (!tls_ctx)
172 return;
173
174 ctx = tls_offload_ctx_tx(tls_ctx);
175
176 spin_lock_irqsave(&ctx->lock, flags);
177 info = ctx->retransmit_hint;
178 if (info && !before(acked_seq, info->end_seq)) {
179 ctx->retransmit_hint = NULL;
180 list_del(&info->list);
181 destroy_record(info);
182 deleted_records++;
183 }
184
185 list_for_each_entry_safe(info, temp, &ctx->records_list, list) {
186 if (before(acked_seq, info->end_seq))
187 break;
188 list_del(&info->list);
189
190 destroy_record(info);
191 deleted_records++;
192 }
193
194 ctx->unacked_record_sn += deleted_records;
195 spin_unlock_irqrestore(&ctx->lock, flags);
196 }
197
198 /* At this point, there should be no references on this
199 * socket and no in-flight SKBs associated with this
200 * socket, so it is safe to free all the resources.
201 */
202 void tls_device_sk_destruct(struct sock *sk)
203 {
204 struct tls_context *tls_ctx = tls_get_ctx(sk);
205 struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
206
207 tls_ctx->sk_destruct(sk);
208
209 if (tls_ctx->tx_conf == TLS_HW) {
210 if (ctx->open_record)
211 destroy_record(ctx->open_record);
212 delete_all_records(ctx);
213 crypto_free_aead(ctx->aead_send);
214 clean_acked_data_disable(inet_csk(sk));
215 }
216
217 if (refcount_dec_and_test(&tls_ctx->refcount))
218 tls_device_queue_ctx_destruction(tls_ctx);
219 }
220 EXPORT_SYMBOL(tls_device_sk_destruct);
221
222 void tls_device_free_resources_tx(struct sock *sk)
223 {
224 struct tls_context *tls_ctx = tls_get_ctx(sk);
225
226 tls_free_partial_record(sk, tls_ctx);
227 }
228
229 static void tls_append_frag(struct tls_record_info *record,
230 struct page_frag *pfrag,
231 int size)
232 {
233 skb_frag_t *frag;
234
235 frag = &record->frags[record->num_frags - 1];
236 if (frag->page.p == pfrag->page &&
237 frag->page_offset + frag->size == pfrag->offset) {
238 frag->size += size;
239 } else {
240 ++frag;
241 frag->page.p = pfrag->page;
242 frag->page_offset = pfrag->offset;
243 frag->size = size;
244 ++record->num_frags;
245 get_page(pfrag->page);
246 }
247
248 pfrag->offset += size;
249 record->len += size;
250 }
251
252 static int tls_push_record(struct sock *sk,
253 struct tls_context *ctx,
254 struct tls_offload_context_tx *offload_ctx,
255 struct tls_record_info *record,
256 struct page_frag *pfrag,
257 int flags,
258 unsigned char record_type)
259 {
260 struct tls_prot_info *prot = &ctx->prot_info;
261 struct tcp_sock *tp = tcp_sk(sk);
262 struct page_frag dummy_tag_frag;
263 skb_frag_t *frag;
264 int i;
265
266 /* fill prepend */
267 frag = &record->frags[0];
268 tls_fill_prepend(ctx,
269 skb_frag_address(frag),
270 record->len - prot->prepend_size,
271 record_type,
272 ctx->crypto_send.info.version);
273
274 /* HW doesn't care about the data in the tag, because it fills it. */
275 dummy_tag_frag.page = skb_frag_page(frag);
276 dummy_tag_frag.offset = 0;
277
278 tls_append_frag(record, &dummy_tag_frag, prot->tag_size);
279 record->end_seq = tp->write_seq + record->len;
280 spin_lock_irq(&offload_ctx->lock);
281 list_add_tail(&record->list, &offload_ctx->records_list);
282 spin_unlock_irq(&offload_ctx->lock);
283 offload_ctx->open_record = NULL;
284 tls_advance_record_sn(sk, &ctx->tx, ctx->crypto_send.info.version);
285
286 for (i = 0; i < record->num_frags; i++) {
287 frag = &record->frags[i];
288 sg_unmark_end(&offload_ctx->sg_tx_data[i]);
289 sg_set_page(&offload_ctx->sg_tx_data[i], skb_frag_page(frag),
290 frag->size, frag->page_offset);
291 sk_mem_charge(sk, frag->size);
292 get_page(skb_frag_page(frag));
293 }
294 sg_mark_end(&offload_ctx->sg_tx_data[record->num_frags - 1]);
295
296 /* all ready, send */
297 return tls_push_sg(sk, ctx, offload_ctx->sg_tx_data, 0, flags);
298 }
299
300 static int tls_create_new_record(struct tls_offload_context_tx *offload_ctx,
301 struct page_frag *pfrag,
302 size_t prepend_size)
303 {
304 struct tls_record_info *record;
305 skb_frag_t *frag;
306
307 record = kmalloc(sizeof(*record), GFP_KERNEL);
308 if (!record)
309 return -ENOMEM;
310
311 frag = &record->frags[0];
312 __skb_frag_set_page(frag, pfrag->page);
313 frag->page_offset = pfrag->offset;
314 skb_frag_size_set(frag, prepend_size);
315
316 get_page(pfrag->page);
317 pfrag->offset += prepend_size;
318
319 record->num_frags = 1;
320 record->len = prepend_size;
321 offload_ctx->open_record = record;
322 return 0;
323 }
324
325 static int tls_do_allocation(struct sock *sk,
326 struct tls_offload_context_tx *offload_ctx,
327 struct page_frag *pfrag,
328 size_t prepend_size)
329 {
330 int ret;
331
332 if (!offload_ctx->open_record) {
333 if (unlikely(!skb_page_frag_refill(prepend_size, pfrag,
334 sk->sk_allocation))) {
335 sk->sk_prot->enter_memory_pressure(sk);
336 sk_stream_moderate_sndbuf(sk);
337 return -ENOMEM;
338 }
339
340 ret = tls_create_new_record(offload_ctx, pfrag, prepend_size);
341 if (ret)
342 return ret;
343
344 if (pfrag->size > pfrag->offset)
345 return 0;
346 }
347
348 if (!sk_page_frag_refill(sk, pfrag))
349 return -ENOMEM;
350
351 return 0;
352 }
353
354 static int tls_push_data(struct sock *sk,
355 struct iov_iter *msg_iter,
356 size_t size, int flags,
357 unsigned char record_type)
358 {
359 struct tls_context *tls_ctx = tls_get_ctx(sk);
360 struct tls_prot_info *prot = &tls_ctx->prot_info;
361 struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
362 int tls_push_record_flags = flags | MSG_SENDPAGE_NOTLAST;
363 int more = flags & (MSG_SENDPAGE_NOTLAST | MSG_MORE);
364 struct tls_record_info *record = ctx->open_record;
365 struct page_frag *pfrag;
366 size_t orig_size = size;
367 u32 max_open_record_len;
368 int copy, rc = 0;
369 bool done = false;
370 long timeo;
371
372 if (flags &
373 ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_SENDPAGE_NOTLAST))
374 return -ENOTSUPP;
375
376 if (sk->sk_err)
377 return -sk->sk_err;
378
379 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
380 if (tls_is_partially_sent_record(tls_ctx)) {
381 rc = tls_push_partial_record(sk, tls_ctx, flags);
382 if (rc < 0)
383 return rc;
384 }
385
386 pfrag = sk_page_frag(sk);
387
388 /* TLS_HEADER_SIZE is not counted as part of the TLS record, and
389 * we need to leave room for an authentication tag.
390 */
391 max_open_record_len = TLS_MAX_PAYLOAD_SIZE +
392 prot->prepend_size;
393 do {
394 rc = tls_do_allocation(sk, ctx, pfrag,
395 prot->prepend_size);
396 if (rc) {
397 rc = sk_stream_wait_memory(sk, &timeo);
398 if (!rc)
399 continue;
400
401 record = ctx->open_record;
402 if (!record)
403 break;
404 handle_error:
405 if (record_type != TLS_RECORD_TYPE_DATA) {
406 /* avoid sending partial
407 * record with type !=
408 * application_data
409 */
410 size = orig_size;
411 destroy_record(record);
412 ctx->open_record = NULL;
413 } else if (record->len > prot->prepend_size) {
414 goto last_record;
415 }
416
417 break;
418 }
419
420 record = ctx->open_record;
421 copy = min_t(size_t, size, (pfrag->size - pfrag->offset));
422 copy = min_t(size_t, copy, (max_open_record_len - record->len));
423
424 if (copy_from_iter_nocache(page_address(pfrag->page) +
425 pfrag->offset,
426 copy, msg_iter) != copy) {
427 rc = -EFAULT;
428 goto handle_error;
429 }
430 tls_append_frag(record, pfrag, copy);
431
432 size -= copy;
433 if (!size) {
434 last_record:
435 tls_push_record_flags = flags;
436 if (more) {
437 tls_ctx->pending_open_record_frags =
438 !!record->num_frags;
439 break;
440 }
441
442 done = true;
443 }
444
445 if (done || record->len >= max_open_record_len ||
446 (record->num_frags >= MAX_SKB_FRAGS - 1)) {
447 rc = tls_push_record(sk,
448 tls_ctx,
449 ctx,
450 record,
451 pfrag,
452 tls_push_record_flags,
453 record_type);
454 if (rc < 0)
455 break;
456 }
457 } while (!done);
458
459 if (orig_size - size > 0)
460 rc = orig_size - size;
461
462 return rc;
463 }
464
465 int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
466 {
467 unsigned char record_type = TLS_RECORD_TYPE_DATA;
468 int rc;
469
470 lock_sock(sk);
471
472 if (unlikely(msg->msg_controllen)) {
473 rc = tls_proccess_cmsg(sk, msg, &record_type);
474 if (rc)
475 goto out;
476 }
477
478 rc = tls_push_data(sk, &msg->msg_iter, size,
479 msg->msg_flags, record_type);
480
481 out:
482 release_sock(sk);
483 return rc;
484 }
485
486 int tls_device_sendpage(struct sock *sk, struct page *page,
487 int offset, size_t size, int flags)
488 {
489 struct iov_iter msg_iter;
490 char *kaddr = kmap(page);
491 struct kvec iov;
492 int rc;
493
494 if (flags & MSG_SENDPAGE_NOTLAST)
495 flags |= MSG_MORE;
496
497 lock_sock(sk);
498
499 if (flags & MSG_OOB) {
500 rc = -ENOTSUPP;
501 goto out;
502 }
503
504 iov.iov_base = kaddr + offset;
505 iov.iov_len = size;
506 iov_iter_kvec(&msg_iter, WRITE, &iov, 1, size);
507 rc = tls_push_data(sk, &msg_iter, size,
508 flags, TLS_RECORD_TYPE_DATA);
509 kunmap(page);
510
511 out:
512 release_sock(sk);
513 return rc;
514 }
515
516 struct tls_record_info *tls_get_record(struct tls_offload_context_tx *context,
517 u32 seq, u64 *p_record_sn)
518 {
519 u64 record_sn = context->hint_record_sn;
520 struct tls_record_info *info;
521
522 info = context->retransmit_hint;
523 if (!info ||
524 before(seq, info->end_seq - info->len)) {
525 /* if retransmit_hint is irrelevant start
526 * from the beggining of the list
527 */
528 info = list_first_entry(&context->records_list,
529 struct tls_record_info, list);
530 record_sn = context->unacked_record_sn;
531 }
532
533 list_for_each_entry_from(info, &context->records_list, list) {
534 if (before(seq, info->end_seq)) {
535 if (!context->retransmit_hint ||
536 after(info->end_seq,
537 context->retransmit_hint->end_seq)) {
538 context->hint_record_sn = record_sn;
539 context->retransmit_hint = info;
540 }
541 *p_record_sn = record_sn;
542 return info;
543 }
544 record_sn++;
545 }
546
547 return NULL;
548 }
549 EXPORT_SYMBOL(tls_get_record);
550
551 static int tls_device_push_pending_record(struct sock *sk, int flags)
552 {
553 struct iov_iter msg_iter;
554
555 iov_iter_kvec(&msg_iter, WRITE, NULL, 0, 0);
556 return tls_push_data(sk, &msg_iter, 0, flags, TLS_RECORD_TYPE_DATA);
557 }
558
559 void tls_device_write_space(struct sock *sk, struct tls_context *ctx)
560 {
561 int rc = 0;
562
563 if (!sk->sk_write_pending && tls_is_partially_sent_record(ctx)) {
564 gfp_t sk_allocation = sk->sk_allocation;
565
566 sk->sk_allocation = GFP_ATOMIC;
567 rc = tls_push_partial_record(sk, ctx,
568 MSG_DONTWAIT | MSG_NOSIGNAL);
569 sk->sk_allocation = sk_allocation;
570 }
571 }
572
573 void handle_device_resync(struct sock *sk, u32 seq, u64 rcd_sn)
574 {
575 struct tls_context *tls_ctx = tls_get_ctx(sk);
576 struct net_device *netdev = tls_ctx->netdev;
577 struct tls_offload_context_rx *rx_ctx;
578 u32 is_req_pending;
579 s64 resync_req;
580 u32 req_seq;
581
582 if (tls_ctx->rx_conf != TLS_HW)
583 return;
584
585 rx_ctx = tls_offload_ctx_rx(tls_ctx);
586 resync_req = atomic64_read(&rx_ctx->resync_req);
587 req_seq = ntohl(resync_req >> 32) - ((u32)TLS_HEADER_SIZE - 1);
588 is_req_pending = resync_req;
589
590 if (unlikely(is_req_pending) && req_seq == seq &&
591 atomic64_try_cmpxchg(&rx_ctx->resync_req, &resync_req, 0))
592 netdev->tlsdev_ops->tls_dev_resync_rx(netdev, sk,
593 seq + TLS_HEADER_SIZE - 1,
594 rcd_sn);
595 }
596
597 static int tls_device_reencrypt(struct sock *sk, struct sk_buff *skb)
598 {
599 struct strp_msg *rxm = strp_msg(skb);
600 int err = 0, offset = rxm->offset, copy, nsg, data_len, pos;
601 struct sk_buff *skb_iter, *unused;
602 struct scatterlist sg[1];
603 char *orig_buf, *buf;
604
605 orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE +
606 TLS_CIPHER_AES_GCM_128_IV_SIZE, sk->sk_allocation);
607 if (!orig_buf)
608 return -ENOMEM;
609 buf = orig_buf;
610
611 nsg = skb_cow_data(skb, 0, &unused);
612 if (unlikely(nsg < 0)) {
613 err = nsg;
614 goto free_buf;
615 }
616
617 sg_init_table(sg, 1);
618 sg_set_buf(&sg[0], buf,
619 rxm->full_len + TLS_HEADER_SIZE +
620 TLS_CIPHER_AES_GCM_128_IV_SIZE);
621 skb_copy_bits(skb, offset, buf,
622 TLS_HEADER_SIZE + TLS_CIPHER_AES_GCM_128_IV_SIZE);
623
624 /* We are interested only in the decrypted data not the auth */
625 err = decrypt_skb(sk, skb, sg);
626 if (err != -EBADMSG)
627 goto free_buf;
628 else
629 err = 0;
630
631 data_len = rxm->full_len - TLS_CIPHER_AES_GCM_128_TAG_SIZE;
632
633 if (skb_pagelen(skb) > offset) {
634 copy = min_t(int, skb_pagelen(skb) - offset, data_len);
635
636 if (skb->decrypted)
637 skb_store_bits(skb, offset, buf, copy);
638
639 offset += copy;
640 buf += copy;
641 }
642
643 pos = skb_pagelen(skb);
644 skb_walk_frags(skb, skb_iter) {
645 int frag_pos;
646
647 /* Practically all frags must belong to msg if reencrypt
648 * is needed with current strparser and coalescing logic,
649 * but strparser may "get optimized", so let's be safe.
650 */
651 if (pos + skb_iter->len <= offset)
652 goto done_with_frag;
653 if (pos >= data_len + rxm->offset)
654 break;
655
656 frag_pos = offset - pos;
657 copy = min_t(int, skb_iter->len - frag_pos,
658 data_len + rxm->offset - offset);
659
660 if (skb_iter->decrypted)
661 skb_store_bits(skb_iter, frag_pos, buf, copy);
662
663 offset += copy;
664 buf += copy;
665 done_with_frag:
666 pos += skb_iter->len;
667 }
668
669 free_buf:
670 kfree(orig_buf);
671 return err;
672 }
673
674 int tls_device_decrypted(struct sock *sk, struct sk_buff *skb)
675 {
676 struct tls_context *tls_ctx = tls_get_ctx(sk);
677 struct tls_offload_context_rx *ctx = tls_offload_ctx_rx(tls_ctx);
678 int is_decrypted = skb->decrypted;
679 int is_encrypted = !is_decrypted;
680 struct sk_buff *skb_iter;
681
682 /* Skip if it is already decrypted */
683 if (ctx->sw.decrypted)
684 return 0;
685
686 /* Check if all the data is decrypted already */
687 skb_walk_frags(skb, skb_iter) {
688 is_decrypted &= skb_iter->decrypted;
689 is_encrypted &= !skb_iter->decrypted;
690 }
691
692 ctx->sw.decrypted |= is_decrypted;
693
694 /* Return immedeatly if the record is either entirely plaintext or
695 * entirely ciphertext. Otherwise handle reencrypt partially decrypted
696 * record.
697 */
698 return (is_encrypted || is_decrypted) ? 0 :
699 tls_device_reencrypt(sk, skb);
700 }
701
702 int tls_set_device_offload(struct sock *sk, struct tls_context *ctx)
703 {
704 u16 nonce_size, tag_size, iv_size, rec_seq_size;
705 struct tls_context *tls_ctx = tls_get_ctx(sk);
706 struct tls_prot_info *prot = &tls_ctx->prot_info;
707 struct tls_record_info *start_marker_record;
708 struct tls_offload_context_tx *offload_ctx;
709 struct tls_crypto_info *crypto_info;
710 struct net_device *netdev;
711 char *iv, *rec_seq;
712 struct sk_buff *skb;
713 int rc = -EINVAL;
714 __be64 rcd_sn;
715
716 if (!ctx)
717 goto out;
718
719 if (ctx->priv_ctx_tx) {
720 rc = -EEXIST;
721 goto out;
722 }
723
724 start_marker_record = kmalloc(sizeof(*start_marker_record), GFP_KERNEL);
725 if (!start_marker_record) {
726 rc = -ENOMEM;
727 goto out;
728 }
729
730 offload_ctx = kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_TX, GFP_KERNEL);
731 if (!offload_ctx) {
732 rc = -ENOMEM;
733 goto free_marker_record;
734 }
735
736 crypto_info = &ctx->crypto_send.info;
737 switch (crypto_info->cipher_type) {
738 case TLS_CIPHER_AES_GCM_128:
739 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
740 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
741 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
742 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
743 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
744 rec_seq =
745 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
746 break;
747 default:
748 rc = -EINVAL;
749 goto free_offload_ctx;
750 }
751
752 prot->prepend_size = TLS_HEADER_SIZE + nonce_size;
753 prot->tag_size = tag_size;
754 prot->overhead_size = prot->prepend_size + prot->tag_size;
755 prot->iv_size = iv_size;
756 ctx->tx.iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
757 GFP_KERNEL);
758 if (!ctx->tx.iv) {
759 rc = -ENOMEM;
760 goto free_offload_ctx;
761 }
762
763 memcpy(ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
764
765 prot->rec_seq_size = rec_seq_size;
766 ctx->tx.rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
767 if (!ctx->tx.rec_seq) {
768 rc = -ENOMEM;
769 goto free_iv;
770 }
771
772 rc = tls_sw_fallback_init(sk, offload_ctx, crypto_info);
773 if (rc)
774 goto free_rec_seq;
775
776 /* start at rec_seq - 1 to account for the start marker record */
777 memcpy(&rcd_sn, ctx->tx.rec_seq, sizeof(rcd_sn));
778 offload_ctx->unacked_record_sn = be64_to_cpu(rcd_sn) - 1;
779
780 start_marker_record->end_seq = tcp_sk(sk)->write_seq;
781 start_marker_record->len = 0;
782 start_marker_record->num_frags = 0;
783
784 INIT_LIST_HEAD(&offload_ctx->records_list);
785 list_add_tail(&start_marker_record->list, &offload_ctx->records_list);
786 spin_lock_init(&offload_ctx->lock);
787 sg_init_table(offload_ctx->sg_tx_data,
788 ARRAY_SIZE(offload_ctx->sg_tx_data));
789
790 clean_acked_data_enable(inet_csk(sk), &tls_icsk_clean_acked);
791 ctx->push_pending_record = tls_device_push_pending_record;
792
793 /* TLS offload is greatly simplified if we don't send
794 * SKBs where only part of the payload needs to be encrypted.
795 * So mark the last skb in the write queue as end of record.
796 */
797 skb = tcp_write_queue_tail(sk);
798 if (skb)
799 TCP_SKB_CB(skb)->eor = 1;
800
801 /* We support starting offload on multiple sockets
802 * concurrently, so we only need a read lock here.
803 * This lock must precede get_netdev_for_sock to prevent races between
804 * NETDEV_DOWN and setsockopt.
805 */
806 down_read(&device_offload_lock);
807 netdev = get_netdev_for_sock(sk);
808 if (!netdev) {
809 pr_err_ratelimited("%s: netdev not found\n", __func__);
810 rc = -EINVAL;
811 goto release_lock;
812 }
813
814 if (!(netdev->features & NETIF_F_HW_TLS_TX)) {
815 rc = -ENOTSUPP;
816 goto release_netdev;
817 }
818
819 /* Avoid offloading if the device is down
820 * We don't want to offload new flows after
821 * the NETDEV_DOWN event
822 */
823 if (!(netdev->flags & IFF_UP)) {
824 rc = -EINVAL;
825 goto release_netdev;
826 }
827
828 ctx->priv_ctx_tx = offload_ctx;
829 rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_TX,
830 &ctx->crypto_send.info,
831 tcp_sk(sk)->write_seq);
832 if (rc)
833 goto release_netdev;
834
835 tls_device_attach(ctx, sk, netdev);
836
837 /* following this assignment tls_is_sk_tx_device_offloaded
838 * will return true and the context might be accessed
839 * by the netdev's xmit function.
840 */
841 smp_store_release(&sk->sk_validate_xmit_skb, tls_validate_xmit_skb);
842 dev_put(netdev);
843 up_read(&device_offload_lock);
844 goto out;
845
846 release_netdev:
847 dev_put(netdev);
848 release_lock:
849 up_read(&device_offload_lock);
850 clean_acked_data_disable(inet_csk(sk));
851 crypto_free_aead(offload_ctx->aead_send);
852 free_rec_seq:
853 kfree(ctx->tx.rec_seq);
854 free_iv:
855 kfree(ctx->tx.iv);
856 free_offload_ctx:
857 kfree(offload_ctx);
858 ctx->priv_ctx_tx = NULL;
859 free_marker_record:
860 kfree(start_marker_record);
861 out:
862 return rc;
863 }
864
865 int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
866 {
867 struct tls_offload_context_rx *context;
868 struct net_device *netdev;
869 int rc = 0;
870
871 /* We support starting offload on multiple sockets
872 * concurrently, so we only need a read lock here.
873 * This lock must precede get_netdev_for_sock to prevent races between
874 * NETDEV_DOWN and setsockopt.
875 */
876 down_read(&device_offload_lock);
877 netdev = get_netdev_for_sock(sk);
878 if (!netdev) {
879 pr_err_ratelimited("%s: netdev not found\n", __func__);
880 rc = -EINVAL;
881 goto release_lock;
882 }
883
884 if (!(netdev->features & NETIF_F_HW_TLS_RX)) {
885 pr_err_ratelimited("%s: netdev %s with no TLS offload\n",
886 __func__, netdev->name);
887 rc = -ENOTSUPP;
888 goto release_netdev;
889 }
890
891 /* Avoid offloading if the device is down
892 * We don't want to offload new flows after
893 * the NETDEV_DOWN event
894 */
895 if (!(netdev->flags & IFF_UP)) {
896 rc = -EINVAL;
897 goto release_netdev;
898 }
899
900 context = kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_RX, GFP_KERNEL);
901 if (!context) {
902 rc = -ENOMEM;
903 goto release_netdev;
904 }
905
906 ctx->priv_ctx_rx = context;
907 rc = tls_set_sw_offload(sk, ctx, 0);
908 if (rc)
909 goto release_ctx;
910
911 rc = netdev->tlsdev_ops->tls_dev_add(netdev, sk, TLS_OFFLOAD_CTX_DIR_RX,
912 &ctx->crypto_recv.info,
913 tcp_sk(sk)->copied_seq);
914 if (rc) {
915 pr_err_ratelimited("%s: The netdev has refused to offload this socket\n",
916 __func__);
917 goto free_sw_resources;
918 }
919
920 tls_device_attach(ctx, sk, netdev);
921 goto release_netdev;
922
923 free_sw_resources:
924 up_read(&device_offload_lock);
925 tls_sw_free_resources_rx(sk);
926 down_read(&device_offload_lock);
927 release_ctx:
928 ctx->priv_ctx_rx = NULL;
929 release_netdev:
930 dev_put(netdev);
931 release_lock:
932 up_read(&device_offload_lock);
933 return rc;
934 }
935
936 void tls_device_offload_cleanup_rx(struct sock *sk)
937 {
938 struct tls_context *tls_ctx = tls_get_ctx(sk);
939 struct net_device *netdev;
940
941 down_read(&device_offload_lock);
942 netdev = tls_ctx->netdev;
943 if (!netdev)
944 goto out;
945
946 if (!(netdev->features & NETIF_F_HW_TLS_RX)) {
947 pr_err_ratelimited("%s: device is missing NETIF_F_HW_TLS_RX cap\n",
948 __func__);
949 goto out;
950 }
951
952 netdev->tlsdev_ops->tls_dev_del(netdev, tls_ctx,
953 TLS_OFFLOAD_CTX_DIR_RX);
954
955 if (tls_ctx->tx_conf != TLS_HW) {
956 dev_put(netdev);
957 tls_ctx->netdev = NULL;
958 }
959 out:
960 up_read(&device_offload_lock);
961 tls_sw_release_resources_rx(sk);
962 }
963
964 static int tls_device_down(struct net_device *netdev)
965 {
966 struct tls_context *ctx, *tmp;
967 unsigned long flags;
968 LIST_HEAD(list);
969
970 /* Request a write lock to block new offload attempts */
971 down_write(&device_offload_lock);
972
973 spin_lock_irqsave(&tls_device_lock, flags);
974 list_for_each_entry_safe(ctx, tmp, &tls_device_list, list) {
975 if (ctx->netdev != netdev ||
976 !refcount_inc_not_zero(&ctx->refcount))
977 continue;
978
979 list_move(&ctx->list, &list);
980 }
981 spin_unlock_irqrestore(&tls_device_lock, flags);
982
983 list_for_each_entry_safe(ctx, tmp, &list, list) {
984 if (ctx->tx_conf == TLS_HW)
985 netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
986 TLS_OFFLOAD_CTX_DIR_TX);
987 if (ctx->rx_conf == TLS_HW)
988 netdev->tlsdev_ops->tls_dev_del(netdev, ctx,
989 TLS_OFFLOAD_CTX_DIR_RX);
990 ctx->netdev = NULL;
991 dev_put(netdev);
992 list_del_init(&ctx->list);
993
994 if (refcount_dec_and_test(&ctx->refcount))
995 tls_device_free_ctx(ctx);
996 }
997
998 up_write(&device_offload_lock);
999
1000 flush_work(&tls_device_gc_work);
1001
1002 return NOTIFY_DONE;
1003 }
1004
1005 static int tls_dev_event(struct notifier_block *this, unsigned long event,
1006 void *ptr)
1007 {
1008 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1009
1010 if (!(dev->features & (NETIF_F_HW_TLS_RX | NETIF_F_HW_TLS_TX)))
1011 return NOTIFY_DONE;
1012
1013 switch (event) {
1014 case NETDEV_REGISTER:
1015 case NETDEV_FEAT_CHANGE:
1016 if ((dev->features & NETIF_F_HW_TLS_RX) &&
1017 !dev->tlsdev_ops->tls_dev_resync_rx)
1018 return NOTIFY_BAD;
1019
1020 if (dev->tlsdev_ops &&
1021 dev->tlsdev_ops->tls_dev_add &&
1022 dev->tlsdev_ops->tls_dev_del)
1023 return NOTIFY_DONE;
1024 else
1025 return NOTIFY_BAD;
1026 case NETDEV_DOWN:
1027 return tls_device_down(dev);
1028 }
1029 return NOTIFY_DONE;
1030 }
1031
1032 static struct notifier_block tls_dev_notifier = {
1033 .notifier_call = tls_dev_event,
1034 };
1035
1036 void __init tls_device_init(void)
1037 {
1038 register_netdevice_notifier(&tls_dev_notifier);
1039 }
1040
1041 void __exit tls_device_cleanup(void)
1042 {
1043 unregister_netdevice_notifier(&tls_dev_notifier);
1044 flush_work(&tls_device_gc_work);
1045 }
Linux with added FPC-III support