?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
gpio: rcar: Fix runtime PM imbalance on error
[linux/fpc-iii.git] / drivers / crypto / chelsio / chcr_ktls.c
blobcd1769ecdc1cc99b68fa9b50bd329621d6be320e
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Chelsio Communications. All rights reserved. */
3
4 #ifdef CONFIG_CHELSIO_TLS_DEVICE
5 #include <linux/highmem.h>
6 #include "chcr_ktls.h"
7 #include "clip_tbl.h"
8
9 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
10 /*
11 * chcr_ktls_save_keys: calculate and save crypto keys.
12 * @tx_info - driver specific tls info.
13 * @crypto_info - tls crypto information.
14 * @direction - TX/RX direction.
15 * return - SUCCESS/FAILURE.
16 */
17 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
18 struct tls_crypto_info *crypto_info,
19 enum tls_offload_ctx_dir direction)
20 {
21 int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
22 unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
23 struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
24 struct ktls_key_ctx *kctx = &tx_info->key_ctx;
25 struct crypto_cipher *cipher;
26 unsigned char *key, *salt;
27
28 switch (crypto_info->cipher_type) {
29 case TLS_CIPHER_AES_GCM_128:
30 info_128_gcm =
31 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
32 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
33 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
34 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
35 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
36 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
37 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
38
39 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
40 key = info_128_gcm->key;
41 salt = info_128_gcm->salt;
42 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
43
44 /* The SCMD fields used when encrypting a full TLS
45 * record. Its a one time calculation till the
46 * connection exists.
47 */
48 tx_info->scmd0_seqno_numivs =
49 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
50 SCMD_CIPH_AUTH_SEQ_CTRL_F |
51 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
52 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
53 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
54 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
55 SCMD_NUM_IVS_V(1);
56
57 /* keys will be sent inline. */
58 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
59
60 /* The SCMD fields used when encrypting a partial TLS
61 * record (no trailer and possibly a truncated payload).
62 */
63 tx_info->scmd0_short_seqno_numivs =
64 SCMD_CIPH_AUTH_SEQ_CTRL_F |
65 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
66 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
67 SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
68
69 tx_info->scmd0_short_ivgen_hdrlen =
70 tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
71
72 break;
73
74 default:
75 pr_err("GCM: cipher type 0x%x not supported\n",
76 crypto_info->cipher_type);
77 ret = -EINVAL;
78 goto out;
79 }
80
81 key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
82 roundup(keylen, 16) + ghash_size;
83 /* Calculate the H = CIPH(K, 0 repeated 16 times).
84 * It will go in key context
85 */
86 cipher = crypto_alloc_cipher("aes", 0, 0);
87 if (IS_ERR(cipher)) {
88 ret = -ENOMEM;
89 goto out;
90 }
91
92 ret = crypto_cipher_setkey(cipher, key, keylen);
93 if (ret)
94 goto out1;
95
96 memset(ghash_h, 0, ghash_size);
97 crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
98
99 /* fill the Key context */
100 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
101 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
102 mac_key_size,
103 key_ctx_size >> 4);
104 } else {
105 ret = -EINVAL;
106 goto out1;
107 }
108
109 memcpy(kctx->salt, salt, tx_info->salt_size);
110 memcpy(kctx->key, key, keylen);
111 memcpy(kctx->key + keylen, ghash_h, ghash_size);
112 tx_info->key_ctx_len = key_ctx_size;
113
114 out1:
115 crypto_free_cipher(cipher);
116 out:
117 return ret;
118 }
119
120 static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info,
121 int new_state)
122 {
123 unsigned long flags;
124
125 /* This function can be called from both rx (interrupt context) and tx
126 * queue contexts.
127 */
128 spin_lock_irqsave(&tx_info->lock, flags);
129 switch (tx_info->connection_state) {
130 case KTLS_CONN_CLOSED:
131 tx_info->connection_state = new_state;
132 break;
133
134 case KTLS_CONN_ACT_OPEN_REQ:
135 /* only go forward if state is greater than current state. */
136 if (new_state <= tx_info->connection_state)
137 break;
138 /* update to the next state and also initialize TCB */
139 tx_info->connection_state = new_state;
140 /* FALLTHRU */
141 case KTLS_CONN_ACT_OPEN_RPL:
142 /* if we are stuck in this state, means tcb init might not
143 * received by HW, try sending it again.
144 */
145 if (!chcr_init_tcb_fields(tx_info))
146 tx_info->connection_state = KTLS_CONN_SET_TCB_REQ;
147 break;
148
149 case KTLS_CONN_SET_TCB_REQ:
150 /* only go forward if state is greater than current state. */
151 if (new_state <= tx_info->connection_state)
152 break;
153 /* update to the next state and check if l2t_state is valid */
154 tx_info->connection_state = new_state;
155 /* FALLTHRU */
156 case KTLS_CONN_SET_TCB_RPL:
157 /* Check if l2t state is valid, then move to ready state. */
158 if (cxgb4_check_l2t_valid(tx_info->l2te)) {
159 tx_info->connection_state = KTLS_CONN_TX_READY;
160 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx);
161 }
162 break;
163
164 case KTLS_CONN_TX_READY:
165 /* nothing to be done here */
166 break;
167
168 default:
169 pr_err("unknown KTLS connection state\n");
170 break;
171 }
172 spin_unlock_irqrestore(&tx_info->lock, flags);
173
174 return tx_info->connection_state;
175 }
176 /*
177 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
178 * @sk - tcp socket.
179 * @tx_info - driver specific tls info.
180 * @atid - connection active tid.
181 * return - send success/failure.
182 */
183 static int chcr_ktls_act_open_req(struct sock *sk,
184 struct chcr_ktls_info *tx_info,
185 int atid)
186 {
187 struct inet_sock *inet = inet_sk(sk);
188 struct cpl_t6_act_open_req *cpl6;
189 struct cpl_act_open_req *cpl;
190 struct sk_buff *skb;
191 unsigned int len;
192 int qid_atid;
193 u64 options;
194
195 len = sizeof(*cpl6);
196 skb = alloc_skb(len, GFP_KERNEL);
197 if (unlikely(!skb))
198 return -ENOMEM;
199 /* mark it a control pkt */
200 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
201
202 cpl6 = __skb_put_zero(skb, len);
203 cpl = (struct cpl_act_open_req *)cpl6;
204 INIT_TP_WR(cpl6, 0);
205 qid_atid = TID_QID_V(tx_info->rx_qid) |
206 TID_TID_V(atid);
207 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
208 cpl->local_port = inet->inet_sport;
209 cpl->peer_port = inet->inet_dport;
210 cpl->local_ip = inet->inet_rcv_saddr;
211 cpl->peer_ip = inet->inet_daddr;
212
213 /* fill first 64 bit option field. */
214 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
215 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
216 cpl->opt0 = cpu_to_be64(options);
217
218 /* next 64 bit option field. */
219 options =
220 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
221 cpl->opt2 = htonl(options);
222
223 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
224 }
225
226 /*
227 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
228 * @sk - tcp socket.
229 * @tx_info - driver specific tls info.
230 * @atid - connection active tid.
231 * return - send success/failure.
232 */
233 static int chcr_ktls_act_open_req6(struct sock *sk,
234 struct chcr_ktls_info *tx_info,
235 int atid)
236 {
237 struct inet_sock *inet = inet_sk(sk);
238 struct cpl_t6_act_open_req6 *cpl6;
239 struct cpl_act_open_req6 *cpl;
240 struct sk_buff *skb;
241 unsigned int len;
242 int qid_atid;
243 u64 options;
244
245 len = sizeof(*cpl6);
246 skb = alloc_skb(len, GFP_KERNEL);
247 if (unlikely(!skb))
248 return -ENOMEM;
249 /* mark it a control pkt */
250 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
251
252 cpl6 = __skb_put_zero(skb, len);
253 cpl = (struct cpl_act_open_req6 *)cpl6;
254 INIT_TP_WR(cpl6, 0);
255 qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
256 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
257 cpl->local_port = inet->inet_sport;
258 cpl->peer_port = inet->inet_dport;
259 cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
260 cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
261 cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
262 cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
263
264 /* first 64 bit option field. */
265 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
266 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
267 cpl->opt0 = cpu_to_be64(options);
268 /* next 64 bit option field. */
269 options =
270 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
271 cpl->opt2 = htonl(options);
272
273 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
274 }
275
276 /*
277 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets.
278 * @sk - tcp socket.
279 * @tx_info - driver specific tls info.
280 * return: NET_TX_OK/NET_XMIT_DROP
281 */
282 static int chcr_setup_connection(struct sock *sk,
283 struct chcr_ktls_info *tx_info)
284 {
285 struct tid_info *t = &tx_info->adap->tids;
286 int atid, ret = 0;
287
288 atid = cxgb4_alloc_atid(t, tx_info);
289 if (atid == -1)
290 return -EINVAL;
291
292 tx_info->atid = atid;
293 tx_info->ip_family = sk->sk_family;
294
295 if (sk->sk_family == AF_INET ||
296 (sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
297 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
298 tx_info->ip_family = AF_INET;
299 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
300 } else {
301 tx_info->ip_family = AF_INET6;
302 ret =
303 cxgb4_clip_get(tx_info->netdev,
304 (const u32 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8,
305 1);
306 if (ret)
307 goto out;
308 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
309 }
310
311 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
312 * success, if any other return type clear atid and return that failure.
313 */
314 if (ret) {
315 if (ret == NET_XMIT_CN)
316 ret = 0;
317 else
318 cxgb4_free_atid(t, atid);
319 goto out;
320 }
321
322 /* update the connection state */
323 chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ);
324 out:
325 return ret;
326 }
327
328 /*
329 * chcr_set_tcb_field: update tcb fields.
330 * @tx_info - driver specific tls info.
331 * @word - TCB word.
332 * @mask - TCB word related mask.
333 * @val - TCB word related value.
334 * @no_reply - set 1 if not looking for TP response.
335 */
336 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
337 u64 mask, u64 val, int no_reply)
338 {
339 struct cpl_set_tcb_field *req;
340 struct sk_buff *skb;
341
342 skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
343 if (!skb)
344 return -ENOMEM;
345
346 req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
347 INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
348 req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
349 NO_REPLY_V(no_reply));
350 req->word_cookie = htons(TCB_WORD_V(word));
351 req->mask = cpu_to_be64(mask);
352 req->val = cpu_to_be64(val);
353
354 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
355 return cxgb4_ofld_send(tx_info->netdev, skb);
356 }
357
358 /*
359 * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
360 * @tx_info - driver specific tls info.
361 * return: NET_TX_OK/NET_XMIT_DROP.
362 */
363 static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
364 {
365 return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
366 TCB_T_STATE_V(TCB_T_STATE_M),
367 CHCR_TCB_STATE_CLOSED, 1);
368 }
369
370 /*
371 * chcr_ktls_dev_del: call back for tls_dev_del.
372 * Remove the tid and l2t entry and close the connection.
373 * it per connection basis.
374 * @netdev - net device.
375 * @tls_cts - tls context.
376 * @direction - TX/RX crypto direction
377 */
378 static void chcr_ktls_dev_del(struct net_device *netdev,
379 struct tls_context *tls_ctx,
380 enum tls_offload_ctx_dir direction)
381 {
382 struct chcr_ktls_ofld_ctx_tx *tx_ctx =
383 chcr_get_ktls_tx_context(tls_ctx);
384 struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
385 struct sock *sk;
386
387 if (!tx_info)
388 return;
389 sk = tx_info->sk;
390
391 spin_lock(&tx_info->lock);
392 tx_info->connection_state = KTLS_CONN_CLOSED;
393 spin_unlock(&tx_info->lock);
394
395 /* clear l2t entry */
396 if (tx_info->l2te)
397 cxgb4_l2t_release(tx_info->l2te);
398
399 /* clear clip entry */
400 if (tx_info->ip_family == AF_INET6)
401 cxgb4_clip_release(netdev,
402 (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8,
403 1);
404
405 /* clear tid */
406 if (tx_info->tid != -1) {
407 /* clear tcb state and then release tid */
408 chcr_ktls_mark_tcb_close(tx_info);
409 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
410 tx_info->tid, tx_info->ip_family);
411 }
412
413 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close);
414 kvfree(tx_info);
415 tx_ctx->chcr_info = NULL;
416 }
417
418 /*
419 * chcr_ktls_dev_add: call back for tls_dev_add.
420 * Create a tcb entry for TP. Also add l2t entry for the connection. And
421 * generate keys & save those keys locally.
422 * @netdev - net device.
423 * @tls_cts - tls context.
424 * @direction - TX/RX crypto direction
425 * return: SUCCESS/FAILURE.
426 */
427 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
428 enum tls_offload_ctx_dir direction,
429 struct tls_crypto_info *crypto_info,
430 u32 start_offload_tcp_sn)
431 {
432 struct tls_context *tls_ctx = tls_get_ctx(sk);
433 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
434 struct chcr_ktls_info *tx_info;
435 struct dst_entry *dst;
436 struct adapter *adap;
437 struct port_info *pi;
438 struct neighbour *n;
439 u8 daaddr[16];
440 int ret = -1;
441
442 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
443
444 pi = netdev_priv(netdev);
445 adap = pi->adapter;
446 if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
447 pr_err("not expecting for RX direction\n");
448 ret = -EINVAL;
449 goto out;
450 }
451 if (tx_ctx->chcr_info) {
452 ret = -EINVAL;
453 goto out;
454 }
455
456 tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
457 if (!tx_info) {
458 ret = -ENOMEM;
459 goto out;
460 }
461
462 spin_lock_init(&tx_info->lock);
463
464 /* clear connection state */
465 spin_lock(&tx_info->lock);
466 tx_info->connection_state = KTLS_CONN_CLOSED;
467 spin_unlock(&tx_info->lock);
468
469 tx_info->sk = sk;
470 /* initialize tid and atid to -1, 0 is a also a valid id. */
471 tx_info->tid = -1;
472 tx_info->atid = -1;
473
474 tx_info->adap = adap;
475 tx_info->netdev = netdev;
476 tx_info->first_qset = pi->first_qset;
477 tx_info->tx_chan = pi->tx_chan;
478 tx_info->smt_idx = pi->smt_idx;
479 tx_info->port_id = pi->port_id;
480
481 tx_info->rx_qid = chcr_get_first_rx_qid(adap);
482 if (unlikely(tx_info->rx_qid < 0))
483 goto out2;
484
485 tx_info->prev_seq = start_offload_tcp_sn;
486 tx_info->tcp_start_seq_number = start_offload_tcp_sn;
487
488 /* save crypto keys */
489 ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
490 if (ret < 0)
491 goto out2;
492
493 /* get peer ip */
494 if (sk->sk_family == AF_INET ||
495 (sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
496 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
497 memcpy(daaddr, &sk->sk_daddr, 4);
498 } else {
499 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
500 }
501
502 /* get the l2t index */
503 dst = sk_dst_get(sk);
504 if (!dst) {
505 pr_err("DST entry not found\n");
506 goto out2;
507 }
508 n = dst_neigh_lookup(dst, daaddr);
509 if (!n || !n->dev) {
510 pr_err("neighbour not found\n");
511 dst_release(dst);
512 goto out2;
513 }
514 tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
515
516 neigh_release(n);
517 dst_release(dst);
518
519 if (!tx_info->l2te) {
520 pr_err("l2t entry not found\n");
521 goto out2;
522 }
523
524 tx_ctx->chcr_info = tx_info;
525
526 /* create a filter and call cxgb4_l2t_send to send the packet out, which
527 * will take care of updating l2t entry in hw if not already done.
528 */
529 ret = chcr_setup_connection(sk, tx_info);
530 if (ret)
531 goto out2;
532
533 atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open);
534 return 0;
535 out2:
536 kvfree(tx_info);
537 out:
538 atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail);
539 return ret;
540 }
541
542 static const struct tlsdev_ops chcr_ktls_ops = {
543 .tls_dev_add = chcr_ktls_dev_add,
544 .tls_dev_del = chcr_ktls_dev_del,
545 };
546
547 /*
548 * chcr_enable_ktls: add NETIF_F_HW_TLS_TX flag in all the ports.
549 */
550 void chcr_enable_ktls(struct adapter *adap)
551 {
552 struct net_device *netdev;
553 int i;
554
555 for_each_port(adap, i) {
556 netdev = adap->port[i];
557 netdev->features |= NETIF_F_HW_TLS_TX;
558 netdev->hw_features |= NETIF_F_HW_TLS_TX;
559 netdev->tlsdev_ops = &chcr_ktls_ops;
560 }
561 }
562
563 /*
564 * chcr_disable_ktls: remove NETIF_F_HW_TLS_TX flag from all the ports.
565 */
566 void chcr_disable_ktls(struct adapter *adap)
567 {
568 struct net_device *netdev;
569 int i;
570
571 for_each_port(adap, i) {
572 netdev = adap->port[i];
573 netdev->features &= ~NETIF_F_HW_TLS_TX;
574 netdev->hw_features &= ~NETIF_F_HW_TLS_TX;
575 netdev->tlsdev_ops = NULL;
576 }
577 }
578
579 /*
580 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number
581 * handling.
582 * @tx_info - driver specific tls info.
583 * return: NET_TX_OK/NET_XMIT_DROP
584 */
585 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
586 {
587 int ret = 0;
588
589 /* set tcb in offload and bypass */
590 ret =
591 chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
592 TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
593 TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
594 if (ret)
595 return ret;
596 /* reset snd_una and snd_next fields in tcb */
597 ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
598 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
599 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
600 0, 1);
601 if (ret)
602 return ret;
603
604 /* reset send max */
605 ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
606 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
607 0, 1);
608 if (ret)
609 return ret;
610
611 /* update l2t index and request for tp reply to confirm tcb is
612 * initialised to handle tx traffic.
613 */
614 ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
615 TCB_L2T_IX_V(TCB_L2T_IX_M),
616 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
617 return ret;
618 }
619
620 /*
621 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
622 */
623 int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input)
624 {
625 const struct cpl_act_open_rpl *p = (void *)input;
626 struct chcr_ktls_info *tx_info = NULL;
627 unsigned int atid, tid, status;
628 struct tid_info *t;
629
630 tid = GET_TID(p);
631 status = AOPEN_STATUS_G(ntohl(p->atid_status));
632 atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
633
634 t = &adap->tids;
635 tx_info = lookup_atid(t, atid);
636
637 if (!tx_info || tx_info->atid != atid) {
638 pr_err("tx_info or atid is not correct\n");
639 return -1;
640 }
641
642 if (!status) {
643 tx_info->tid = tid;
644 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
645
646 cxgb4_free_atid(t, atid);
647 tx_info->atid = -1;
648 /* update the connection state */
649 chcr_ktls_update_connection_state(tx_info,
650 KTLS_CONN_ACT_OPEN_RPL);
651 }
652 return 0;
653 }
654
655 /*
656 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
657 */
658 int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
659 {
660 const struct cpl_set_tcb_rpl *p = (void *)input;
661 struct chcr_ktls_info *tx_info = NULL;
662 struct tid_info *t;
663 u32 tid;
664
665 tid = GET_TID(p);
666
667 t = &adap->tids;
668 tx_info = lookup_tid(t, tid);
669 if (!tx_info || tx_info->tid != tid) {
670 pr_err("tx_info or atid is not correct\n");
671 return -1;
672 }
673 /* update the connection state */
674 chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL);
675 return 0;
676 }
677
678 /*
679 * chcr_write_cpl_set_tcb_ulp: update tcb values.
680 * TCB is responsible to create tcp headers, so all the related values
681 * should be correctly updated.
682 * @tx_info - driver specific tls info.
683 * @q - tx queue on which packet is going out.
684 * @tid - TCB identifier.
685 * @pos - current index where should we start writing.
686 * @word - TCB word.
687 * @mask - TCB word related mask.
688 * @val - TCB word related value.
689 * @reply - set 1 if looking for TP response.
690 * return - next position to write.
691 */
692 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
693 struct sge_eth_txq *q, u32 tid,
694 void *pos, u16 word, u64 mask,
695 u64 val, u32 reply)
696 {
697 struct cpl_set_tcb_field_core *cpl;
698 struct ulptx_idata *idata;
699 struct ulp_txpkt *txpkt;
700 void *save_pos = NULL;
701 u8 buf[48] = {0};
702 int left;
703
704 left = (void *)q->q.stat - pos;
705 if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
706 if (!left) {
707 pos = q->q.desc;
708 } else {
709 save_pos = pos;
710 pos = buf;
711 }
712 }
713 /* ULP_TXPKT */
714 txpkt = pos;
715 txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0));
716 txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
717
718 /* ULPTX_IDATA sub-command */
719 idata = (struct ulptx_idata *)(txpkt + 1);
720 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
721 idata->len = htonl(sizeof(*cpl));
722 pos = idata + 1;
723
724 cpl = pos;
725 /* CPL_SET_TCB_FIELD */
726 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
727 cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
728 NO_REPLY_V(!reply));
729 cpl->word_cookie = htons(TCB_WORD_V(word));
730 cpl->mask = cpu_to_be64(mask);
731 cpl->val = cpu_to_be64(val);
732
733 /* ULPTX_NOOP */
734 idata = (struct ulptx_idata *)(cpl + 1);
735 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
736 idata->len = htonl(0);
737
738 if (save_pos) {
739 pos = chcr_copy_to_txd(buf, &q->q, save_pos,
740 CHCR_SET_TCB_FIELD_LEN);
741 } else {
742 /* check again if we are at the end of the queue */
743 if (left == CHCR_SET_TCB_FIELD_LEN)
744 pos = q->q.desc;
745 else
746 pos = idata + 1;
747 }
748
749 return pos;
750 }
751
752 /*
753 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
754 * with updated values like tcp seq, ack, window etc.
755 * @tx_info - driver specific tls info.
756 * @q - TX queue.
757 * @tcp_seq
758 * @tcp_ack
759 * @tcp_win
760 * return: NETDEV_TX_BUSY/NET_TX_OK.
761 */
762 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
763 struct sge_eth_txq *q, u64 tcp_seq,
764 u64 tcp_ack, u64 tcp_win)
765 {
766 bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
767 u32 len, cpl = 0, ndesc, wr_len;
768 struct fw_ulptx_wr *wr;
769 int credits;
770 void *pos;
771
772 wr_len = sizeof(*wr);
773 /* there can be max 4 cpls, check if we have enough credits */
774 len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
775 ndesc = DIV_ROUND_UP(len, 64);
776
777 credits = chcr_txq_avail(&q->q) - ndesc;
778 if (unlikely(credits < 0)) {
779 chcr_eth_txq_stop(q);
780 return NETDEV_TX_BUSY;
781 }
782
783 pos = &q->q.desc[q->q.pidx];
784 /* make space for WR, we'll fill it later when we know all the cpls
785 * being sent out and have complete length.
786 */
787 wr = pos;
788 pos += wr_len;
789 /* update tx_max if its a re-transmit or the first wr */
790 if (first_wr || tcp_seq != tx_info->prev_seq) {
791 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
792 TCB_TX_MAX_W,
793 TCB_TX_MAX_V(TCB_TX_MAX_M),
794 TCB_TX_MAX_V(tcp_seq), 0);
795 cpl++;
796 }
797 /* reset snd una if it's a re-transmit pkt */
798 if (tcp_seq != tx_info->prev_seq) {
799 /* reset snd_una */
800 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
801 TCB_SND_UNA_RAW_W,
802 TCB_SND_UNA_RAW_V
803 (TCB_SND_UNA_RAW_M),
804 TCB_SND_UNA_RAW_V(0), 0);
805 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo);
806 cpl++;
807 }
808 /* update ack */
809 if (first_wr || tx_info->prev_ack != tcp_ack) {
810 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
811 TCB_RCV_NXT_W,
812 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
813 TCB_RCV_NXT_V(tcp_ack), 0);
814 tx_info->prev_ack = tcp_ack;
815 cpl++;
816 }
817 /* update receive window */
818 if (first_wr || tx_info->prev_win != tcp_win) {
819 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
820 TCB_RCV_WND_W,
821 TCB_RCV_WND_V(TCB_RCV_WND_M),
822 TCB_RCV_WND_V(tcp_win), 0);
823 tx_info->prev_win = tcp_win;
824 cpl++;
825 }
826
827 if (cpl) {
828 /* get the actual length */
829 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
830 /* ULPTX wr */
831 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
832 wr->cookie = 0;
833 /* fill len in wr field */
834 wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
835
836 ndesc = DIV_ROUND_UP(len, 64);
837 chcr_txq_advance(&q->q, ndesc);
838 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
839 }
840 return 0;
841 }
842
843 /*
844 * chcr_ktls_skb_copy
845 * @nskb - new skb where the frags to be added.
846 * @skb - old skb from which frags will be copied.
847 */
848 static void chcr_ktls_skb_copy(struct sk_buff *skb, struct sk_buff *nskb)
849 {
850 int i;
851
852 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
853 skb_shinfo(nskb)->frags[i] = skb_shinfo(skb)->frags[i];
854 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
855 }
856
857 skb_shinfo(nskb)->nr_frags = skb_shinfo(skb)->nr_frags;
858 nskb->len += skb->data_len;
859 nskb->data_len = skb->data_len;
860 nskb->truesize += skb->data_len;
861 }
862
863 /*
864 * chcr_ktls_get_tx_flits
865 * returns number of flits to be sent out, it includes key context length, WR
866 * size and skb fragments.
867 */
868 static unsigned int
869 chcr_ktls_get_tx_flits(const struct sk_buff *skb, unsigned int key_ctx_len)
870 {
871 return chcr_sgl_len(skb_shinfo(skb)->nr_frags) +
872 DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
873 }
874
875 /*
876 * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale
877 * other than timestamp.
878 * @skb - skb contains partial record..
879 * return: 1 / 0
880 */
881 static int
882 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
883 {
884 int cnt, opt, optlen;
885 u_char *cp;
886
887 cp = (u_char *)(tcp + 1);
888 cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
889 for (; cnt > 0; cnt -= optlen, cp += optlen) {
890 opt = cp[0];
891 if (opt == TCPOPT_EOL)
892 break;
893 if (opt == TCPOPT_NOP) {
894 optlen = 1;
895 } else {
896 if (cnt < 2)
897 break;
898 optlen = cp[1];
899 if (optlen < 2 || optlen > cnt)
900 break;
901 }
902 switch (opt) {
903 case TCPOPT_NOP:
904 break;
905 default:
906 return 1;
907 }
908 }
909 return 0;
910 }
911
912 /*
913 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
914 * send out separately.
915 * @tx_info - driver specific tls info.
916 * @skb - skb contains partial record..
917 * @q - TX queue.
918 * @tx_chan - channel number.
919 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
920 */
921 static int
922 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
923 struct sge_eth_txq *q, uint32_t tx_chan)
924 {
925 struct fw_eth_tx_pkt_wr *wr;
926 struct cpl_tx_pkt_core *cpl;
927 u32 ctrl, iplen, maclen;
928 struct ipv6hdr *ip6;
929 unsigned int ndesc;
930 struct tcphdr *tcp;
931 int len16, pktlen;
932 struct iphdr *ip;
933 int credits;
934 u8 buf[150];
935 void *pos;
936
937 iplen = skb_network_header_len(skb);
938 maclen = skb_mac_header_len(skb);
939
940 /* packet length = eth hdr len + ip hdr len + tcp hdr len
941 * (including options).
942 */
943 pktlen = skb->len - skb->data_len;
944
945 ctrl = sizeof(*cpl) + pktlen;
946 len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
947 /* check how many descriptors needed */
948 ndesc = DIV_ROUND_UP(len16, 4);
949
950 credits = chcr_txq_avail(&q->q) - ndesc;
951 if (unlikely(credits < 0)) {
952 chcr_eth_txq_stop(q);
953 return NETDEV_TX_BUSY;
954 }
955
956 pos = &q->q.desc[q->q.pidx];
957 wr = pos;
958
959 /* Firmware work request header */
960 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
961 FW_WR_IMMDLEN_V(ctrl));
962
963 wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(len16));
964 wr->r3 = 0;
965
966 cpl = (void *)(wr + 1);
967
968 /* CPL header */
969 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
970 TXPKT_PF_V(tx_info->adap->pf));
971 cpl->pack = 0;
972 cpl->len = htons(pktlen);
973 /* checksum offload */
974 cpl->ctrl1 = 0;
975
976 pos = cpl + 1;
977
978 memcpy(buf, skb->data, pktlen);
979 if (tx_info->ip_family == AF_INET) {
980 /* we need to correct ip header len */
981 ip = (struct iphdr *)(buf + maclen);
982 ip->tot_len = htons(pktlen - maclen);
983 } else {
984 ip6 = (struct ipv6hdr *)(buf + maclen);
985 ip6->payload_len = htons(pktlen - maclen - iplen);
986 }
987 /* now take care of the tcp header, if fin is not set then clear push
988 * bit as well, and if fin is set, it will be sent at the last so we
989 * need to update the tcp sequence number as per the last packet.
990 */
991 tcp = (struct tcphdr *)(buf + maclen + iplen);
992
993 if (!tcp->fin)
994 tcp->psh = 0;
995 else
996 tcp->seq = htonl(tx_info->prev_seq);
997
998 chcr_copy_to_txd(buf, &q->q, pos, pktlen);
999
1000 chcr_txq_advance(&q->q, ndesc);
1001 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1002 return 0;
1003 }
1004
1005 /* chcr_ktls_skb_shift - Shifts request length paged data from skb to another.
1006 * @tgt- buffer into which tail data gets added
1007 * @skb- buffer from which the paged data comes from
1008 * @shiftlen- shift up to this many bytes
1009 */
1010 static int chcr_ktls_skb_shift(struct sk_buff *tgt, struct sk_buff *skb,
1011 int shiftlen)
1012 {
1013 skb_frag_t *fragfrom, *fragto;
1014 int from, to, todo;
1015
1016 WARN_ON(shiftlen > skb->data_len);
1017
1018 todo = shiftlen;
1019 from = 0;
1020 to = 0;
1021 fragfrom = &skb_shinfo(skb)->frags[from];
1022
1023 while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
1024 fragfrom = &skb_shinfo(skb)->frags[from];
1025 fragto = &skb_shinfo(tgt)->frags[to];
1026
1027 if (todo >= skb_frag_size(fragfrom)) {
1028 *fragto = *fragfrom;
1029 todo -= skb_frag_size(fragfrom);
1030 from++;
1031 to++;
1032
1033 } else {
1034 __skb_frag_ref(fragfrom);
1035 skb_frag_page_copy(fragto, fragfrom);
1036 skb_frag_off_copy(fragto, fragfrom);
1037 skb_frag_size_set(fragto, todo);
1038
1039 skb_frag_off_add(fragfrom, todo);
1040 skb_frag_size_sub(fragfrom, todo);
1041 todo = 0;
1042
1043 to++;
1044 break;
1045 }
1046 }
1047
1048 /* Ready to "commit" this state change to tgt */
1049 skb_shinfo(tgt)->nr_frags = to;
1050
1051 /* Reposition in the original skb */
1052 to = 0;
1053 while (from < skb_shinfo(skb)->nr_frags)
1054 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
1055
1056 skb_shinfo(skb)->nr_frags = to;
1057
1058 WARN_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
1059
1060 skb->len -= shiftlen;
1061 skb->data_len -= shiftlen;
1062 skb->truesize -= shiftlen;
1063 tgt->len += shiftlen;
1064 tgt->data_len += shiftlen;
1065 tgt->truesize += shiftlen;
1066
1067 return shiftlen;
1068 }
1069
1070 /*
1071 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1072 * received has partial end part of the record, send out the complete record, so
1073 * that crypto block will be able to generate TAG/HASH.
1074 * @skb - segment which has complete or partial end part.
1075 * @tx_info - driver specific tls info.
1076 * @q - TX queue.
1077 * @tcp_seq
1078 * @tcp_push - tcp push bit.
1079 * @mss - segment size.
1080 * return: NETDEV_TX_BUSY/NET_TX_OK.
1081 */
1082 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1083 struct chcr_ktls_info *tx_info,
1084 struct sge_eth_txq *q, u32 tcp_seq,
1085 bool tcp_push, u32 mss)
1086 {
1087 u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1088 struct adapter *adap = tx_info->adap;
1089 int credits, left, last_desc;
1090 struct tx_sw_desc *sgl_sdesc;
1091 struct cpl_tx_data *tx_data;
1092 struct cpl_tx_sec_pdu *cpl;
1093 struct ulptx_idata *idata;
1094 struct ulp_txpkt *ulptx;
1095 struct fw_ulptx_wr *wr;
1096 void *pos;
1097 u64 *end;
1098
1099 /* get the number of flits required */
1100 flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len);
1101 /* number of descriptors */
1102 ndesc = chcr_flits_to_desc(flits);
1103 /* check if enough credits available */
1104 credits = chcr_txq_avail(&q->q) - ndesc;
1105 if (unlikely(credits < 0)) {
1106 chcr_eth_txq_stop(q);
1107 return NETDEV_TX_BUSY;
1108 }
1109
1110 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1111 /* Credits are below the threshold vaues, stop the queue after
1112 * injecting the Work Request for this packet.
1113 */
1114 chcr_eth_txq_stop(q);
1115 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1116 }
1117
1118 last_desc = q->q.pidx + ndesc - 1;
1119 if (last_desc >= q->q.size)
1120 last_desc -= q->q.size;
1121 sgl_sdesc = &q->q.sdesc[last_desc];
1122
1123 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1124 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1125 q->mapping_err++;
1126 return NETDEV_TX_BUSY;
1127 }
1128
1129 pos = &q->q.desc[q->q.pidx];
1130 end = (u64 *)pos + flits;
1131 /* FW_ULPTX_WR */
1132 wr = pos;
1133 /* WR will need len16 */
1134 len16 = DIV_ROUND_UP(flits, 2);
1135 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1136 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1137 wr->cookie = 0;
1138 pos += sizeof(*wr);
1139 /* ULP_TXPKT */
1140 ulptx = pos;
1141 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1142 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1143 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1144 ULP_TXPKT_RO_F);
1145 ulptx->len = htonl(len16 - 1);
1146 /* ULPTX_IDATA sub-command */
1147 idata = (struct ulptx_idata *)(ulptx + 1);
1148 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1149 /* idata length will include cpl_tx_sec_pdu + key context size +
1150 * cpl_tx_data header.
1151 */
1152 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1153 sizeof(*tx_data));
1154 /* SEC CPL */
1155 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1156 cpl->op_ivinsrtofst =
1157 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1158 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1159 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1160 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1161 cpl->pldlen = htonl(skb->data_len);
1162
1163 /* encryption should start after tls header size + iv size */
1164 cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1165
1166 cpl->aadstart_cipherstop_hi =
1167 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1168 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1169 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1170
1171 /* authentication will also start after tls header + iv size */
1172 cpl->cipherstop_lo_authinsert =
1173 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1174 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1175 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1176
1177 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1178 cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1179 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1180 cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1181
1182 pos = cpl + 1;
1183 /* check if space left to fill the keys */
1184 left = (void *)q->q.stat - pos;
1185 if (!left) {
1186 left = (void *)end - (void *)q->q.stat;
1187 pos = q->q.desc;
1188 end = pos + left;
1189 }
1190
1191 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1192 tx_info->key_ctx_len);
1193 left = (void *)q->q.stat - pos;
1194
1195 if (!left) {
1196 left = (void *)end - (void *)q->q.stat;
1197 pos = q->q.desc;
1198 end = pos + left;
1199 }
1200 /* CPL_TX_DATA */
1201 tx_data = (void *)pos;
1202 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1203 tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(skb->data_len));
1204
1205 tx_data->rsvd = htonl(tcp_seq);
1206
1207 tx_data->flags = htonl(TX_BYPASS_F);
1208 if (tcp_push)
1209 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1210
1211 /* check left again, it might go beyond queue limit */
1212 pos = tx_data + 1;
1213 left = (void *)q->q.stat - pos;
1214
1215 /* check the position again */
1216 if (!left) {
1217 left = (void *)end - (void *)q->q.stat;
1218 pos = q->q.desc;
1219 end = pos + left;
1220 }
1221
1222 /* send the complete packet except the header */
1223 cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1224 sgl_sdesc->addr);
1225 sgl_sdesc->skb = skb;
1226
1227 chcr_txq_advance(&q->q, ndesc);
1228 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1229 atomic64_inc(&adap->chcr_stats.ktls_tx_send_records);
1230
1231 return 0;
1232 }
1233
1234 /*
1235 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1236 * a middle part of a record, fetch the prior data to make it 16 byte aligned
1237 * and then only send it out.
1238 *
1239 * @skb - skb contains partial record..
1240 * @tx_info - driver specific tls info.
1241 * @q - TX queue.
1242 * @tcp_seq
1243 * @tcp_push - tcp push bit.
1244 * @mss - segment size.
1245 * @tls_rec_offset - offset from start of the tls record.
1246 * @perior_data - data before the current segment, required to make this record
1247 * 16 byte aligned.
1248 * @prior_data_len - prior_data length (less than 16)
1249 * return: NETDEV_TX_BUSY/NET_TX_OK.
1250 */
1251 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1252 struct chcr_ktls_info *tx_info,
1253 struct sge_eth_txq *q,
1254 u32 tcp_seq, bool tcp_push, u32 mss,
1255 u32 tls_rec_offset, u8 *prior_data,
1256 u32 prior_data_len)
1257 {
1258 struct adapter *adap = tx_info->adap;
1259 u32 len16, wr_mid = 0, cipher_start;
1260 unsigned int flits = 0, ndesc;
1261 int credits, left, last_desc;
1262 struct tx_sw_desc *sgl_sdesc;
1263 struct cpl_tx_data *tx_data;
1264 struct cpl_tx_sec_pdu *cpl;
1265 struct ulptx_idata *idata;
1266 struct ulp_txpkt *ulptx;
1267 struct fw_ulptx_wr *wr;
1268 __be64 iv_record;
1269 void *pos;
1270 u64 *end;
1271
1272 /* get the number of flits required, it's a partial record so 2 flits
1273 * (AES_BLOCK_SIZE) will be added.
1274 */
1275 flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len) + 2;
1276 /* get the correct 8 byte IV of this record */
1277 iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1278 /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1279 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1280 * data will be added.
1281 */
1282 if (prior_data_len)
1283 flits += 2;
1284 /* number of descriptors */
1285 ndesc = chcr_flits_to_desc(flits);
1286 /* check if enough credits available */
1287 credits = chcr_txq_avail(&q->q) - ndesc;
1288 if (unlikely(credits < 0)) {
1289 chcr_eth_txq_stop(q);
1290 return NETDEV_TX_BUSY;
1291 }
1292
1293 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1294 chcr_eth_txq_stop(q);
1295 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1296 }
1297
1298 last_desc = q->q.pidx + ndesc - 1;
1299 if (last_desc >= q->q.size)
1300 last_desc -= q->q.size;
1301 sgl_sdesc = &q->q.sdesc[last_desc];
1302
1303 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1304 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1305 q->mapping_err++;
1306 return NETDEV_TX_BUSY;
1307 }
1308
1309 pos = &q->q.desc[q->q.pidx];
1310 end = (u64 *)pos + flits;
1311 /* FW_ULPTX_WR */
1312 wr = pos;
1313 /* WR will need len16 */
1314 len16 = DIV_ROUND_UP(flits, 2);
1315 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1316 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1317 wr->cookie = 0;
1318 pos += sizeof(*wr);
1319 /* ULP_TXPKT */
1320 ulptx = pos;
1321 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1322 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1323 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1324 ULP_TXPKT_RO_F);
1325 ulptx->len = htonl(len16 - 1);
1326 /* ULPTX_IDATA sub-command */
1327 idata = (struct ulptx_idata *)(ulptx + 1);
1328 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1329 /* idata length will include cpl_tx_sec_pdu + key context size +
1330 * cpl_tx_data header.
1331 */
1332 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1333 sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1334 /* SEC CPL */
1335 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1336 /* cipher start will have tls header + iv size extra if its a header
1337 * part of tls record. else only 16 byte IV will be added.
1338 */
1339 cipher_start =
1340 AES_BLOCK_LEN + 1 +
1341 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1342
1343 cpl->op_ivinsrtofst =
1344 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1345 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1346 CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1347 cpl->pldlen = htonl(skb->data_len + AES_BLOCK_LEN + prior_data_len);
1348 cpl->aadstart_cipherstop_hi =
1349 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1350 cpl->cipherstop_lo_authinsert = 0;
1351 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1352 cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1353 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1354 cpl->scmd1 = 0;
1355
1356 pos = cpl + 1;
1357 /* check if space left to fill the keys */
1358 left = (void *)q->q.stat - pos;
1359 if (!left) {
1360 left = (void *)end - (void *)q->q.stat;
1361 pos = q->q.desc;
1362 end = pos + left;
1363 }
1364
1365 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1366 tx_info->key_ctx_len);
1367 left = (void *)q->q.stat - pos;
1368
1369 if (!left) {
1370 left = (void *)end - (void *)q->q.stat;
1371 pos = q->q.desc;
1372 end = pos + left;
1373 }
1374 /* CPL_TX_DATA */
1375 tx_data = (void *)pos;
1376 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1377 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1378 TX_LENGTH_V(skb->data_len + prior_data_len));
1379 tx_data->rsvd = htonl(tcp_seq);
1380 tx_data->flags = htonl(TX_BYPASS_F);
1381 if (tcp_push)
1382 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1383
1384 /* check left again, it might go beyond queue limit */
1385 pos = tx_data + 1;
1386 left = (void *)q->q.stat - pos;
1387
1388 /* check the position again */
1389 if (!left) {
1390 left = (void *)end - (void *)q->q.stat;
1391 pos = q->q.desc;
1392 end = pos + left;
1393 }
1394 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1395 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1396 */
1397 memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1398 memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1399 *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1400 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1401 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1402
1403 pos += 16;
1404 /* Prior_data_len will always be less than 16 bytes, fill the
1405 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1406 * to 0.
1407 */
1408 if (prior_data_len)
1409 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1410 /* send the complete packet except the header */
1411 cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1412 sgl_sdesc->addr);
1413 sgl_sdesc->skb = skb;
1414
1415 chcr_txq_advance(&q->q, ndesc);
1416 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1417
1418 return 0;
1419 }
1420
1421 /*
1422 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1423 * only plain text (only tls header and iv)
1424 * @tx_info - driver specific tls info.
1425 * @skb - skb contains partial record..
1426 * @tcp_seq
1427 * @mss - segment size.
1428 * @tcp_push - tcp push bit.
1429 * @q - TX queue.
1430 * @port_id : port number
1431 * @perior_data - data before the current segment, required to make this record
1432 * 16 byte aligned.
1433 * @prior_data_len - prior_data length (less than 16)
1434 * return: NETDEV_TX_BUSY/NET_TX_OK.
1435 */
1436 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1437 struct sk_buff *skb, u32 tcp_seq, u32 mss,
1438 bool tcp_push, struct sge_eth_txq *q,
1439 u32 port_id, u8 *prior_data,
1440 u32 prior_data_len)
1441 {
1442 int credits, left, len16, last_desc;
1443 unsigned int flits = 0, ndesc;
1444 struct tx_sw_desc *sgl_sdesc;
1445 struct cpl_tx_data *tx_data;
1446 struct ulptx_idata *idata;
1447 struct ulp_txpkt *ulptx;
1448 struct fw_ulptx_wr *wr;
1449 u32 wr_mid = 0;
1450 void *pos;
1451 u64 *end;
1452
1453 flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1454 flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags);
1455 if (prior_data_len)
1456 flits += 2;
1457 /* WR will need len16 */
1458 len16 = DIV_ROUND_UP(flits, 2);
1459 /* check how many descriptors needed */
1460 ndesc = DIV_ROUND_UP(flits, 8);
1461
1462 credits = chcr_txq_avail(&q->q) - ndesc;
1463 if (unlikely(credits < 0)) {
1464 chcr_eth_txq_stop(q);
1465 return NETDEV_TX_BUSY;
1466 }
1467
1468 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1469 chcr_eth_txq_stop(q);
1470 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1471 }
1472
1473 last_desc = q->q.pidx + ndesc - 1;
1474 if (last_desc >= q->q.size)
1475 last_desc -= q->q.size;
1476 sgl_sdesc = &q->q.sdesc[last_desc];
1477
1478 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1479 sgl_sdesc->addr) < 0)) {
1480 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1481 q->mapping_err++;
1482 return NETDEV_TX_BUSY;
1483 }
1484
1485 pos = &q->q.desc[q->q.pidx];
1486 end = (u64 *)pos + flits;
1487 /* FW_ULPTX_WR */
1488 wr = pos;
1489 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1490 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1491 wr->cookie = 0;
1492 pos += sizeof(*wr);
1493 /* ULP_TXPKT */
1494 ulptx = (struct ulp_txpkt *)(wr + 1);
1495 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1496 ULP_TXPKT_DATAMODIFY_V(0) |
1497 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1498 ULP_TXPKT_DEST_V(0) |
1499 ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1500 ulptx->len = htonl(len16 - 1);
1501 /* ULPTX_IDATA sub-command */
1502 idata = (struct ulptx_idata *)(ulptx + 1);
1503 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1504 idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1505 /* CPL_TX_DATA */
1506 tx_data = (struct cpl_tx_data *)(idata + 1);
1507 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1508 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1509 TX_LENGTH_V(skb->data_len + prior_data_len));
1510 /* set tcp seq number */
1511 tx_data->rsvd = htonl(tcp_seq);
1512 tx_data->flags = htonl(TX_BYPASS_F);
1513 if (tcp_push)
1514 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1515
1516 pos = tx_data + 1;
1517 /* apart from prior_data_len, we should set remaining part of 16 bytes
1518 * to be zero.
1519 */
1520 if (prior_data_len)
1521 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1522
1523 /* check left again, it might go beyond queue limit */
1524 left = (void *)q->q.stat - pos;
1525
1526 /* check the position again */
1527 if (!left) {
1528 left = (void *)end - (void *)q->q.stat;
1529 pos = q->q.desc;
1530 end = pos + left;
1531 }
1532 /* send the complete packet including the header */
1533 cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1534 sgl_sdesc->addr);
1535 sgl_sdesc->skb = skb;
1536
1537 chcr_txq_advance(&q->q, ndesc);
1538 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1539 return 0;
1540 }
1541
1542 /*
1543 * chcr_ktls_copy_record_in_skb
1544 * @nskb - new skb where the frags to be added.
1545 * @record - specific record which has complete 16k record in frags.
1546 */
1547 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1548 struct tls_record_info *record)
1549 {
1550 int i = 0;
1551
1552 for (i = 0; i < record->num_frags; i++) {
1553 skb_shinfo(nskb)->frags[i] = record->frags[i];
1554 /* increase the frag ref count */
1555 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1556 }
1557
1558 skb_shinfo(nskb)->nr_frags = record->num_frags;
1559 nskb->data_len = record->len;
1560 nskb->len += record->len;
1561 nskb->truesize += record->len;
1562 }
1563
1564 /*
1565 * chcr_ktls_update_snd_una: Reset the SEND_UNA. It will be done to avoid
1566 * sending the same segment again. It will discard the segment which is before
1567 * the current tx max.
1568 * @tx_info - driver specific tls info.
1569 * @q - TX queue.
1570 * return: NET_TX_OK/NET_XMIT_DROP.
1571 */
1572 static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info,
1573 struct sge_eth_txq *q)
1574 {
1575 struct fw_ulptx_wr *wr;
1576 unsigned int ndesc;
1577 int credits;
1578 void *pos;
1579 u32 len;
1580
1581 len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16);
1582 ndesc = DIV_ROUND_UP(len, 64);
1583
1584 credits = chcr_txq_avail(&q->q) - ndesc;
1585 if (unlikely(credits < 0)) {
1586 chcr_eth_txq_stop(q);
1587 return NETDEV_TX_BUSY;
1588 }
1589
1590 pos = &q->q.desc[q->q.pidx];
1591
1592 wr = pos;
1593 /* ULPTX wr */
1594 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1595 wr->cookie = 0;
1596 /* fill len in wr field */
1597 wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
1598
1599 pos += sizeof(*wr);
1600
1601 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
1602 TCB_SND_UNA_RAW_W,
1603 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
1604 TCB_SND_UNA_RAW_V(0), 0);
1605
1606 chcr_txq_advance(&q->q, ndesc);
1607 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1608
1609 return 0;
1610 }
1611
1612 /*
1613 * chcr_end_part_handler: This handler will handle the record which
1614 * is complete or if record's end part is received. T6 adapter has a issue that
1615 * it can't send out TAG with partial record so if its an end part then we have
1616 * to send TAG as well and for which we need to fetch the complete record and
1617 * send it to crypto module.
1618 * @tx_info - driver specific tls info.
1619 * @skb - skb contains partial record.
1620 * @record - complete record of 16K size.
1621 * @tcp_seq
1622 * @mss - segment size in which TP needs to chop a packet.
1623 * @tcp_push_no_fin - tcp push if fin is not set.
1624 * @q - TX queue.
1625 * @tls_end_offset - offset from end of the record.
1626 * @last wr : check if this is the last part of the skb going out.
1627 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1628 */
1629 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1630 struct sk_buff *skb,
1631 struct tls_record_info *record,
1632 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1633 struct sge_eth_txq *q,
1634 u32 tls_end_offset, bool last_wr)
1635 {
1636 struct sk_buff *nskb = NULL;
1637 /* check if it is a complete record */
1638 if (tls_end_offset == record->len) {
1639 nskb = skb;
1640 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_complete_pkts);
1641 } else {
1642 dev_kfree_skb_any(skb);
1643
1644 nskb = alloc_skb(0, GFP_KERNEL);
1645 if (!nskb)
1646 return NETDEV_TX_BUSY;
1647 /* copy complete record in skb */
1648 chcr_ktls_copy_record_in_skb(nskb, record);
1649 /* packet is being sent from the beginning, update the tcp_seq
1650 * accordingly.
1651 */
1652 tcp_seq = tls_record_start_seq(record);
1653 /* reset snd una, so the middle record won't send the already
1654 * sent part.
1655 */
1656 if (chcr_ktls_update_snd_una(tx_info, q))
1657 goto out;
1658 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_end_pkts);
1659 }
1660
1661 if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1662 (last_wr && tcp_push_no_fin),
1663 mss)) {
1664 goto out;
1665 }
1666 return 0;
1667 out:
1668 dev_kfree_skb_any(nskb);
1669 return NETDEV_TX_BUSY;
1670 }
1671
1672 /*
1673 * chcr_short_record_handler: This handler will take care of the records which
1674 * doesn't have end part (1st part or the middle part(/s) of a record). In such
1675 * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1676 * This partial record might be the first part of the record, or the middle
1677 * part. In case of middle record we should fetch the prior data to make it 16
1678 * byte aligned. If it has a partial tls header or iv then get to the start of
1679 * tls header. And if it has partial TAG, then remove the complete TAG and send
1680 * only the payload.
1681 * There is one more possibility that it gets a partial header, send that
1682 * portion as a plaintext.
1683 * @tx_info - driver specific tls info.
1684 * @skb - skb contains partial record..
1685 * @record - complete record of 16K size.
1686 * @tcp_seq
1687 * @mss - segment size in which TP needs to chop a packet.
1688 * @tcp_push_no_fin - tcp push if fin is not set.
1689 * @q - TX queue.
1690 * @tls_end_offset - offset from end of the record.
1691 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1692 */
1693 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1694 struct sk_buff *skb,
1695 struct tls_record_info *record,
1696 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1697 struct sge_eth_txq *q, u32 tls_end_offset)
1698 {
1699 u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1700 u8 prior_data[16] = {0};
1701 u32 prior_data_len = 0;
1702 u32 data_len;
1703
1704 /* check if the skb is ending in middle of tag/HASH, its a big
1705 * trouble, send the packet before the HASH.
1706 */
1707 int remaining_record = tls_end_offset - skb->data_len;
1708
1709 if (remaining_record > 0 &&
1710 remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1711 int trimmed_len = skb->data_len -
1712 (TLS_CIPHER_AES_GCM_128_TAG_SIZE - remaining_record);
1713 struct sk_buff *tmp_skb = NULL;
1714 /* don't process the pkt if it is only a partial tag */
1715 if (skb->data_len < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1716 goto out;
1717
1718 WARN_ON(trimmed_len > skb->data_len);
1719
1720 /* shift to those many bytes */
1721 tmp_skb = alloc_skb(0, GFP_KERNEL);
1722 if (unlikely(!tmp_skb))
1723 goto out;
1724
1725 chcr_ktls_skb_shift(tmp_skb, skb, trimmed_len);
1726 /* free the last trimmed portion */
1727 dev_kfree_skb_any(skb);
1728 skb = tmp_skb;
1729 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_trimmed_pkts);
1730 }
1731 data_len = skb->data_len;
1732 /* check if the middle record's start point is 16 byte aligned. CTR
1733 * needs 16 byte aligned start point to start encryption.
1734 */
1735 if (tls_rec_offset) {
1736 /* there is an offset from start, means its a middle record */
1737 int remaining = 0;
1738
1739 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1740 prior_data_len = tls_rec_offset;
1741 tls_rec_offset = 0;
1742 remaining = 0;
1743 } else {
1744 prior_data_len =
1745 (tls_rec_offset -
1746 (TLS_HEADER_SIZE + tx_info->iv_size))
1747 % AES_BLOCK_LEN;
1748 remaining = tls_rec_offset - prior_data_len;
1749 }
1750
1751 /* if prior_data_len is not zero, means we need to fetch prior
1752 * data to make this record 16 byte aligned, or we need to reach
1753 * to start offset.
1754 */
1755 if (prior_data_len) {
1756 int i = 0;
1757 u8 *data = NULL;
1758 skb_frag_t *f;
1759 u8 *vaddr;
1760 int frag_size = 0, frag_delta = 0;
1761
1762 while (remaining > 0) {
1763 frag_size = skb_frag_size(&record->frags[i]);
1764 if (remaining < frag_size)
1765 break;
1766
1767 remaining -= frag_size;
1768 i++;
1769 }
1770 f = &record->frags[i];
1771 vaddr = kmap_atomic(skb_frag_page(f));
1772
1773 data = vaddr + skb_frag_off(f) + remaining;
1774 frag_delta = skb_frag_size(f) - remaining;
1775
1776 if (frag_delta >= prior_data_len) {
1777 memcpy(prior_data, data, prior_data_len);
1778 kunmap_atomic(vaddr);
1779 } else {
1780 memcpy(prior_data, data, frag_delta);
1781 kunmap_atomic(vaddr);
1782 /* get the next page */
1783 f = &record->frags[i + 1];
1784 vaddr = kmap_atomic(skb_frag_page(f));
1785 data = vaddr + skb_frag_off(f);
1786 memcpy(prior_data + frag_delta,
1787 data, (prior_data_len - frag_delta));
1788 kunmap_atomic(vaddr);
1789 }
1790 /* reset tcp_seq as per the prior_data_required len */
1791 tcp_seq -= prior_data_len;
1792 /* include prio_data_len for further calculation.
1793 */
1794 data_len += prior_data_len;
1795 }
1796 /* reset snd una, so the middle record won't send the already
1797 * sent part.
1798 */
1799 if (chcr_ktls_update_snd_una(tx_info, q))
1800 goto out;
1801 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_middle_pkts);
1802 } else {
1803 /* Else means, its a partial first part of the record. Check if
1804 * its only the header, don't need to send for encryption then.
1805 */
1806 if (data_len <= TLS_HEADER_SIZE + tx_info->iv_size) {
1807 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1808 tcp_push_no_fin, q,
1809 tx_info->port_id,
1810 prior_data,
1811 prior_data_len)) {
1812 goto out;
1813 }
1814 return 0;
1815 }
1816 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_start_pkts);
1817 }
1818
1819 if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1820 mss, tls_rec_offset, prior_data,
1821 prior_data_len)) {
1822 goto out;
1823 }
1824
1825 return 0;
1826 out:
1827 dev_kfree_skb_any(skb);
1828 return NETDEV_TX_BUSY;
1829 }
1830
1831 /* nic tls TX handler */
1832 int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1833 {
1834 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1835 struct tcphdr *th = tcp_hdr(skb);
1836 int data_len, qidx, ret = 0, mss;
1837 struct tls_record_info *record;
1838 struct chcr_stats_debug *stats;
1839 struct chcr_ktls_info *tx_info;
1840 u32 tls_end_offset, tcp_seq;
1841 struct tls_context *tls_ctx;
1842 struct sk_buff *local_skb;
1843 int new_connection_state;
1844 struct sge_eth_txq *q;
1845 struct adapter *adap;
1846 unsigned long flags;
1847
1848 tcp_seq = ntohl(th->seq);
1849
1850 mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : skb->data_len;
1851
1852 /* check if we haven't set it for ktls offload */
1853 if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))
1854 goto out;
1855
1856 tls_ctx = tls_get_ctx(skb->sk);
1857 if (unlikely(tls_ctx->netdev != dev))
1858 goto out;
1859
1860 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1861 tx_info = tx_ctx->chcr_info;
1862
1863 if (unlikely(!tx_info))
1864 goto out;
1865
1866 /* check the connection state, we don't need to pass new connection
1867 * state, state machine will check and update the new state if it is
1868 * stuck due to responses not received from HW.
1869 * Start the tx handling only if state is KTLS_CONN_TX_READY.
1870 */
1871 new_connection_state = chcr_ktls_update_connection_state(tx_info, 0);
1872 if (new_connection_state != KTLS_CONN_TX_READY)
1873 goto out;
1874
1875 /* don't touch the original skb, make a new skb to extract each records
1876 * and send them separately.
1877 */
1878 local_skb = alloc_skb(0, GFP_KERNEL);
1879
1880 if (unlikely(!local_skb))
1881 return NETDEV_TX_BUSY;
1882
1883 adap = tx_info->adap;
1884 stats = &adap->chcr_stats;
1885
1886 qidx = skb->queue_mapping;
1887 q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1888 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1889 /* if tcp options are set but finish is not send the options first */
1890 if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1891 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1892 tx_info->tx_chan);
1893 if (ret)
1894 return NETDEV_TX_BUSY;
1895 }
1896 /* update tcb */
1897 ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, ntohl(th->seq),
1898 ntohl(th->ack_seq),
1899 ntohs(th->window));
1900 if (ret) {
1901 dev_kfree_skb_any(local_skb);
1902 return NETDEV_TX_BUSY;
1903 }
1904
1905 /* copy skb contents into local skb */
1906 chcr_ktls_skb_copy(skb, local_skb);
1907
1908 /* go through the skb and send only one record at a time. */
1909 data_len = skb->data_len;
1910 /* TCP segments can be in received either complete or partial.
1911 * chcr_end_part_handler will handle cases if complete record or end
1912 * part of the record is received. Incase of partial end part of record,
1913 * we will send the complete record again.
1914 */
1915
1916 do {
1917 int i;
1918
1919 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1920 /* lock taken */
1921 spin_lock_irqsave(&tx_ctx->base.lock, flags);
1922 /* fetch the tls record */
1923 record = tls_get_record(&tx_ctx->base, tcp_seq,
1924 &tx_info->record_no);
1925 /* By the time packet reached to us, ACK is received, and record
1926 * won't be found in that case, handle it gracefully.
1927 */
1928 if (unlikely(!record)) {
1929 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1930 atomic64_inc(&stats->ktls_tx_drop_no_sync_data);
1931 goto out;
1932 }
1933
1934 if (unlikely(tls_record_is_start_marker(record))) {
1935 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1936 atomic64_inc(&stats->ktls_tx_skip_no_sync_data);
1937 goto out;
1938 }
1939
1940 /* increase page reference count of the record, so that there
1941 * won't be any chance of page free in middle if in case stack
1942 * receives ACK and try to delete the record.
1943 */
1944 for (i = 0; i < record->num_frags; i++)
1945 __skb_frag_ref(&record->frags[i]);
1946 /* lock cleared */
1947 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1948
1949 tls_end_offset = record->end_seq - tcp_seq;
1950
1951 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
1952 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
1953 /* if a tls record is finishing in this SKB */
1954 if (tls_end_offset <= data_len) {
1955 struct sk_buff *nskb = NULL;
1956
1957 if (tls_end_offset < data_len) {
1958 nskb = alloc_skb(0, GFP_KERNEL);
1959 if (unlikely(!nskb)) {
1960 ret = -ENOMEM;
1961 goto clear_ref;
1962 }
1963
1964 chcr_ktls_skb_shift(nskb, local_skb,
1965 tls_end_offset);
1966 } else {
1967 /* its the only record in this skb, directly
1968 * point it.
1969 */
1970 nskb = local_skb;
1971 }
1972 ret = chcr_end_part_handler(tx_info, nskb, record,
1973 tcp_seq, mss,
1974 (!th->fin && th->psh), q,
1975 tls_end_offset,
1976 (nskb == local_skb));
1977
1978 if (ret && nskb != local_skb)
1979 dev_kfree_skb_any(local_skb);
1980
1981 data_len -= tls_end_offset;
1982 /* tcp_seq increment is required to handle next record.
1983 */
1984 tcp_seq += tls_end_offset;
1985 } else {
1986 ret = chcr_short_record_handler(tx_info, local_skb,
1987 record, tcp_seq, mss,
1988 (!th->fin && th->psh),
1989 q, tls_end_offset);
1990 data_len = 0;
1991 }
1992 clear_ref:
1993 /* clear the frag ref count which increased locally before */
1994 for (i = 0; i < record->num_frags; i++) {
1995 /* clear the frag ref count */
1996 __skb_frag_unref(&record->frags[i]);
1997 }
1998 /* if any failure, come out from the loop. */
1999 if (ret)
2000 goto out;
2001 /* length should never be less than 0 */
2002 WARN_ON(data_len < 0);
2003
2004 } while (data_len > 0);
2005
2006 tx_info->prev_seq = ntohl(th->seq) + skb->data_len;
2007
2008 atomic64_inc(&stats->ktls_tx_encrypted_packets);
2009 atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes);
2010
2011 /* tcp finish is set, send a separate tcp msg including all the options
2012 * as well.
2013 */
2014 if (th->fin)
2015 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2016
2017 out:
2018 dev_kfree_skb_any(skb);
2019 return NETDEV_TX_OK;
2020 }
2021 #endif /* CONFIG_CHELSIO_TLS_DEVICE */
Linux with added FPC-III support