?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
dm writecache: add cond_resched to loop in persistent_memory_claim()
[linux/fpc-iii.git] / drivers / crypto / chelsio / chcr_ipsec.c
blob9fd3b9d1ec2f5d54379542f56927c6152e15d644
1 /*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Atul Gupta (atul.gupta@chelsio.com)
36 */
37
38 #define pr_fmt(fmt) "chcr:" fmt
39
40 #include <linux/kernel.h>
41 #include <linux/module.h>
42 #include <linux/crypto.h>
43 #include <linux/cryptohash.h>
44 #include <linux/skbuff.h>
45 #include <linux/rtnetlink.h>
46 #include <linux/highmem.h>
47 #include <linux/if_vlan.h>
48 #include <linux/ip.h>
49 #include <linux/netdevice.h>
50 #include <net/esp.h>
51 #include <net/xfrm.h>
52 #include <crypto/aes.h>
53 #include <crypto/algapi.h>
54 #include <crypto/hash.h>
55 #include <crypto/sha.h>
56 #include <crypto/authenc.h>
57 #include <crypto/internal/aead.h>
58 #include <crypto/null.h>
59 #include <crypto/internal/skcipher.h>
60 #include <crypto/aead.h>
61 #include <crypto/scatterwalk.h>
62 #include <crypto/internal/hash.h>
63
64 #include "chcr_core.h"
65 #include "chcr_algo.h"
66 #include "chcr_crypto.h"
67
68 /*
69 * Max Tx descriptor space we allow for an Ethernet packet to be inlined
70 * into a WR.
71 */
72 #define MAX_IMM_TX_PKT_LEN 256
73 #define GCM_ESP_IV_SIZE 8
74
75 static int chcr_xfrm_add_state(struct xfrm_state *x);
76 static void chcr_xfrm_del_state(struct xfrm_state *x);
77 static void chcr_xfrm_free_state(struct xfrm_state *x);
78 static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
79 static void chcr_advance_esn_state(struct xfrm_state *x);
80
81 static const struct xfrmdev_ops chcr_xfrmdev_ops = {
82 .xdo_dev_state_add = chcr_xfrm_add_state,
83 .xdo_dev_state_delete = chcr_xfrm_del_state,
84 .xdo_dev_state_free = chcr_xfrm_free_state,
85 .xdo_dev_offload_ok = chcr_ipsec_offload_ok,
86 .xdo_dev_state_advance_esn = chcr_advance_esn_state,
87 };
88
89 /* Add offload xfrms to Chelsio Interface */
90 void chcr_add_xfrmops(const struct cxgb4_lld_info *lld)
91 {
92 struct net_device *netdev = NULL;
93 int i;
94
95 for (i = 0; i < lld->nports; i++) {
96 netdev = lld->ports[i];
97 if (!netdev)
98 continue;
99 netdev->xfrmdev_ops = &chcr_xfrmdev_ops;
100 netdev->hw_enc_features |= NETIF_F_HW_ESP;
101 netdev->features |= NETIF_F_HW_ESP;
102 netdev_change_features(netdev);
103 }
104 }
105
106 static inline int chcr_ipsec_setauthsize(struct xfrm_state *x,
107 struct ipsec_sa_entry *sa_entry)
108 {
109 int hmac_ctrl;
110 int authsize = x->aead->alg_icv_len / 8;
111
112 sa_entry->authsize = authsize;
113
114 switch (authsize) {
115 case ICV_8:
116 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
117 break;
118 case ICV_12:
119 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
120 break;
121 case ICV_16:
122 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
123 break;
124 default:
125 return -EINVAL;
126 }
127 return hmac_ctrl;
128 }
129
130 static inline int chcr_ipsec_setkey(struct xfrm_state *x,
131 struct ipsec_sa_entry *sa_entry)
132 {
133 int keylen = (x->aead->alg_key_len + 7) / 8;
134 unsigned char *key = x->aead->alg_key;
135 int ck_size, key_ctx_size = 0;
136 unsigned char ghash_h[AEAD_H_SIZE];
137 struct crypto_aes_ctx aes;
138 int ret = 0;
139
140 if (keylen > 3) {
141 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
142 memcpy(sa_entry->salt, key + keylen, 4);
143 }
144
145 if (keylen == AES_KEYSIZE_128) {
146 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
147 } else if (keylen == AES_KEYSIZE_192) {
148 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
149 } else if (keylen == AES_KEYSIZE_256) {
150 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
151 } else {
152 pr_err("GCM: Invalid key length %d\n", keylen);
153 ret = -EINVAL;
154 goto out;
155 }
156
157 memcpy(sa_entry->key, key, keylen);
158 sa_entry->enckey_len = keylen;
159 key_ctx_size = sizeof(struct _key_ctx) +
160 ((DIV_ROUND_UP(keylen, 16)) << 4) +
161 AEAD_H_SIZE;
162
163 sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
164 CHCR_KEYCTX_MAC_KEY_SIZE_128,
165 0, 0,
166 key_ctx_size >> 4);
167
168 /* Calculate the H = CIPH(K, 0 repeated 16 times).
169 * It will go in key context
170 */
171 ret = aes_expandkey(&aes, key, keylen);
172 if (ret) {
173 sa_entry->enckey_len = 0;
174 goto out;
175 }
176 memset(ghash_h, 0, AEAD_H_SIZE);
177 aes_encrypt(&aes, ghash_h, ghash_h);
178 memzero_explicit(&aes, sizeof(aes));
179
180 memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *
181 16), ghash_h, AEAD_H_SIZE);
182 sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +
183 AEAD_H_SIZE;
184 out:
185 return ret;
186 }
187
188 /*
189 * chcr_xfrm_add_state
190 * returns 0 on success, negative error if failed to send message to FPGA
191 * positive error if FPGA returned a bad response
192 */
193 static int chcr_xfrm_add_state(struct xfrm_state *x)
194 {
195 struct ipsec_sa_entry *sa_entry;
196 int res = 0;
197
198 if (x->props.aalgo != SADB_AALG_NONE) {
199 pr_debug("CHCR: Cannot offload authenticated xfrm states\n");
200 return -EINVAL;
201 }
202 if (x->props.calgo != SADB_X_CALG_NONE) {
203 pr_debug("CHCR: Cannot offload compressed xfrm states\n");
204 return -EINVAL;
205 }
206 if (x->props.family != AF_INET &&
207 x->props.family != AF_INET6) {
208 pr_debug("CHCR: Only IPv4/6 xfrm state offloaded\n");
209 return -EINVAL;
210 }
211 if (x->props.mode != XFRM_MODE_TRANSPORT &&
212 x->props.mode != XFRM_MODE_TUNNEL) {
213 pr_debug("CHCR: Only transport and tunnel xfrm offload\n");
214 return -EINVAL;
215 }
216 if (x->id.proto != IPPROTO_ESP) {
217 pr_debug("CHCR: Only ESP xfrm state offloaded\n");
218 return -EINVAL;
219 }
220 if (x->encap) {
221 pr_debug("CHCR: Encapsulated xfrm state not offloaded\n");
222 return -EINVAL;
223 }
224 if (!x->aead) {
225 pr_debug("CHCR: Cannot offload xfrm states without aead\n");
226 return -EINVAL;
227 }
228 if (x->aead->alg_icv_len != 128 &&
229 x->aead->alg_icv_len != 96) {
230 pr_debug("CHCR: Cannot offload xfrm states with AEAD ICV length other than 96b & 128b\n");
231 return -EINVAL;
232 }
233 if ((x->aead->alg_key_len != 128 + 32) &&
234 (x->aead->alg_key_len != 256 + 32)) {
235 pr_debug("CHCR: Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
236 return -EINVAL;
237 }
238 if (x->tfcpad) {
239 pr_debug("CHCR: Cannot offload xfrm states with tfc padding\n");
240 return -EINVAL;
241 }
242 if (!x->geniv) {
243 pr_debug("CHCR: Cannot offload xfrm states without geniv\n");
244 return -EINVAL;
245 }
246 if (strcmp(x->geniv, "seqiv")) {
247 pr_debug("CHCR: Cannot offload xfrm states with geniv other than seqiv\n");
248 return -EINVAL;
249 }
250
251 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
252 if (!sa_entry) {
253 res = -ENOMEM;
254 goto out;
255 }
256
257 sa_entry->hmac_ctrl = chcr_ipsec_setauthsize(x, sa_entry);
258 if (x->props.flags & XFRM_STATE_ESN)
259 sa_entry->esn = 1;
260 chcr_ipsec_setkey(x, sa_entry);
261 x->xso.offload_handle = (unsigned long)sa_entry;
262 try_module_get(THIS_MODULE);
263 out:
264 return res;
265 }
266
267 static void chcr_xfrm_del_state(struct xfrm_state *x)
268 {
269 /* do nothing */
270 if (!x->xso.offload_handle)
271 return;
272 }
273
274 static void chcr_xfrm_free_state(struct xfrm_state *x)
275 {
276 struct ipsec_sa_entry *sa_entry;
277
278 if (!x->xso.offload_handle)
279 return;
280
281 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
282 kfree(sa_entry);
283 module_put(THIS_MODULE);
284 }
285
286 static bool chcr_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
287 {
288 if (x->props.family == AF_INET) {
289 /* Offload with IP options is not supported yet */
290 if (ip_hdr(skb)->ihl > 5)
291 return false;
292 } else {
293 /* Offload with IPv6 extension headers is not support yet */
294 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
295 return false;
296 }
297 /* Inline single pdu */
298 if (skb_shinfo(skb)->gso_size)
299 return false;
300 return true;
301 }
302
303 static void chcr_advance_esn_state(struct xfrm_state *x)
304 {
305 /* do nothing */
306 if (!x->xso.offload_handle)
307 return;
308 }
309
310 static inline int is_eth_imm(const struct sk_buff *skb,
311 struct ipsec_sa_entry *sa_entry)
312 {
313 unsigned int kctx_len;
314 int hdrlen;
315
316 kctx_len = sa_entry->kctx_len;
317 hdrlen = sizeof(struct fw_ulptx_wr) +
318 sizeof(struct chcr_ipsec_req) + kctx_len;
319
320 hdrlen += sizeof(struct cpl_tx_pkt);
321 if (sa_entry->esn)
322 hdrlen += (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16)
323 << 4);
324 if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)
325 return hdrlen;
326 return 0;
327 }
328
329 static inline unsigned int calc_tx_sec_flits(const struct sk_buff *skb,
330 struct ipsec_sa_entry *sa_entry,
331 bool *immediate)
332 {
333 unsigned int kctx_len;
334 unsigned int flits;
335 int aadivlen;
336 int hdrlen;
337
338 kctx_len = sa_entry->kctx_len;
339 hdrlen = is_eth_imm(skb, sa_entry);
340 aadivlen = sa_entry->esn ? DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
341 16) : 0;
342 aadivlen <<= 4;
343
344 /* If the skb is small enough, we can pump it out as a work request
345 * with only immediate data. In that case we just have to have the
346 * TX Packet header plus the skb data in the Work Request.
347 */
348
349 if (hdrlen) {
350 *immediate = true;
351 return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));
352 }
353
354 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);
355
356 /* Otherwise, we're going to have to construct a Scatter gather list
357 * of the skb body and fragments. We also include the flits necessary
358 * for the TX Packet Work Request and CPL. We always have a firmware
359 * Write Header (incorporated as part of the cpl_tx_pkt_lso and
360 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL
361 * message or, if we're doing a Large Send Offload, an LSO CPL message
362 * with an embedded TX Packet Write CPL message.
363 */
364 flits += (sizeof(struct fw_ulptx_wr) +
365 sizeof(struct chcr_ipsec_req) +
366 kctx_len +
367 sizeof(struct cpl_tx_pkt_core) +
368 aadivlen) / sizeof(__be64);
369 return flits;
370 }
371
372 inline void *copy_esn_pktxt(struct sk_buff *skb,
373 struct net_device *dev,
374 void *pos,
375 struct ipsec_sa_entry *sa_entry)
376 {
377 struct chcr_ipsec_aadiv *aadiv;
378 struct ulptx_idata *sc_imm;
379 struct ip_esp_hdr *esphdr;
380 struct xfrm_offload *xo;
381 struct sge_eth_txq *q;
382 struct adapter *adap;
383 struct port_info *pi;
384 __be64 seqno;
385 u32 qidx;
386 u32 seqlo;
387 u8 *iv;
388 int eoq;
389 int len;
390
391 pi = netdev_priv(dev);
392 adap = pi->adapter;
393 qidx = skb->queue_mapping;
394 q = &adap->sge.ethtxq[qidx + pi->first_qset];
395
396 /* end of queue, reset pos to start of queue */
397 eoq = (void *)q->q.stat - pos;
398 if (!eoq)
399 pos = q->q.desc;
400
401 len = DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) << 4;
402 memset(pos, 0, len);
403 aadiv = (struct chcr_ipsec_aadiv *)pos;
404 esphdr = (struct ip_esp_hdr *)skb_transport_header(skb);
405 iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
406 xo = xfrm_offload(skb);
407
408 aadiv->spi = (esphdr->spi);
409 seqlo = htonl(esphdr->seq_no);
410 seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32));
411 memcpy(aadiv->seq_no, &seqno, 8);
412 iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);
413 memcpy(aadiv->iv, iv, 8);
414
415 if (is_eth_imm(skb, sa_entry) && !skb_is_nonlinear(skb)) {
416 sc_imm = (struct ulptx_idata *)(pos +
417 (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),
418 sizeof(__be64)) << 3));
419 sc_imm->cmd_more = FILL_CMD_MORE(0);
420 sc_imm->len = cpu_to_be32(skb->len);
421 }
422 pos += len;
423 return pos;
424 }
425
426 inline void *copy_cpltx_pktxt(struct sk_buff *skb,
427 struct net_device *dev,
428 void *pos,
429 struct ipsec_sa_entry *sa_entry)
430 {
431 struct cpl_tx_pkt_core *cpl;
432 struct sge_eth_txq *q;
433 struct adapter *adap;
434 struct port_info *pi;
435 u32 ctrl0, qidx;
436 u64 cntrl = 0;
437 int left;
438
439 pi = netdev_priv(dev);
440 adap = pi->adapter;
441 qidx = skb->queue_mapping;
442 q = &adap->sge.ethtxq[qidx + pi->first_qset];
443
444 left = (void *)q->q.stat - pos;
445 if (!left)
446 pos = q->q.desc;
447
448 cpl = (struct cpl_tx_pkt_core *)pos;
449
450 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;
451 ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |
452 TXPKT_PF_V(adap->pf);
453 if (skb_vlan_tag_present(skb)) {
454 q->vlan_ins++;
455 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));
456 }
457
458 cpl->ctrl0 = htonl(ctrl0);
459 cpl->pack = htons(0);
460 cpl->len = htons(skb->len);
461 cpl->ctrl1 = cpu_to_be64(cntrl);
462
463 pos += sizeof(struct cpl_tx_pkt_core);
464 /* Copy ESN info for HW */
465 if (sa_entry->esn)
466 pos = copy_esn_pktxt(skb, dev, pos, sa_entry);
467 return pos;
468 }
469
470 inline void *copy_key_cpltx_pktxt(struct sk_buff *skb,
471 struct net_device *dev,
472 void *pos,
473 struct ipsec_sa_entry *sa_entry)
474 {
475 struct _key_ctx *key_ctx;
476 int left, eoq, key_len;
477 struct sge_eth_txq *q;
478 struct adapter *adap;
479 struct port_info *pi;
480 unsigned int qidx;
481
482 pi = netdev_priv(dev);
483 adap = pi->adapter;
484 qidx = skb->queue_mapping;
485 q = &adap->sge.ethtxq[qidx + pi->first_qset];
486 key_len = sa_entry->kctx_len;
487
488 /* end of queue, reset pos to start of queue */
489 eoq = (void *)q->q.stat - pos;
490 left = eoq;
491 if (!eoq) {
492 pos = q->q.desc;
493 left = 64 * q->q.size;
494 }
495
496 /* Copy the Key context header */
497 key_ctx = (struct _key_ctx *)pos;
498 key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;
499 memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);
500 pos += sizeof(struct _key_ctx);
501 left -= sizeof(struct _key_ctx);
502
503 if (likely(key_len <= left)) {
504 memcpy(key_ctx->key, sa_entry->key, key_len);
505 pos += key_len;
506 } else {
507 memcpy(pos, sa_entry->key, left);
508 memcpy(q->q.desc, sa_entry->key + left,
509 key_len - left);
510 pos = (u8 *)q->q.desc + (key_len - left);
511 }
512 /* Copy CPL TX PKT XT */
513 pos = copy_cpltx_pktxt(skb, dev, pos, sa_entry);
514
515 return pos;
516 }
517
518 inline void *chcr_crypto_wreq(struct sk_buff *skb,
519 struct net_device *dev,
520 void *pos,
521 int credits,
522 struct ipsec_sa_entry *sa_entry)
523 {
524 struct port_info *pi = netdev_priv(dev);
525 struct adapter *adap = pi->adapter;
526 unsigned int ivsize = GCM_ESP_IV_SIZE;
527 struct chcr_ipsec_wr *wr;
528 bool immediate = false;
529 u16 immdatalen = 0;
530 unsigned int flits;
531 u32 ivinoffset;
532 u32 aadstart;
533 u32 aadstop;
534 u32 ciphstart;
535 u16 sc_more = 0;
536 u32 ivdrop = 0;
537 u32 esnlen = 0;
538 u32 wr_mid;
539 u16 ndesc;
540 int qidx = skb_get_queue_mapping(skb);
541 struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];
542 unsigned int kctx_len = sa_entry->kctx_len;
543 int qid = q->q.cntxt_id;
544
545 atomic_inc(&adap->chcr_stats.ipsec_cnt);
546
547 flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
548 ndesc = DIV_ROUND_UP(flits, 2);
549 if (sa_entry->esn)
550 ivdrop = 1;
551
552 if (immediate)
553 immdatalen = skb->len;
554
555 if (sa_entry->esn) {
556 esnlen = sizeof(struct chcr_ipsec_aadiv);
557 if (!skb_is_nonlinear(skb))
558 sc_more = 1;
559 }
560
561 /* WR Header */
562 wr = (struct chcr_ipsec_wr *)pos;
563 wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
564 wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(ndesc);
565
566 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
567 netif_tx_stop_queue(q->txq);
568 q->q.stops++;
569 if (!q->dbqt)
570 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
571 }
572 wr_mid |= FW_ULPTX_WR_DATA_F;
573 wr->wreq.flowid_len16 = htonl(wr_mid);
574
575 /* ULPTX */
576 wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);
577 wr->req.ulptx.len = htonl(ndesc - 1);
578
579 /* Sub-command */
580 wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen || sc_more);
581 wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
582 sizeof(wr->req.key_ctx) +
583 kctx_len +
584 sizeof(struct cpl_tx_pkt_core) +
585 esnlen +
586 (esnlen ? 0 : immdatalen));
587
588 /* CPL_SEC_PDU */
589 ivinoffset = sa_entry->esn ? (ESN_IV_INSERT_OFFSET + 1) :
590 (skb_transport_offset(skb) +
591 sizeof(struct ip_esp_hdr) + 1);
592 wr->req.sec_cpl.op_ivinsrtofst = htonl(
593 CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
594 CPL_TX_SEC_PDU_CPLLEN_V(2) |
595 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
596 CPL_TX_SEC_PDU_IVINSRTOFST_V(
597 ivinoffset));
598
599 wr->req.sec_cpl.pldlen = htonl(skb->len + esnlen);
600 aadstart = sa_entry->esn ? 1 : (skb_transport_offset(skb) + 1);
601 aadstop = sa_entry->esn ? ESN_IV_INSERT_OFFSET :
602 (skb_transport_offset(skb) +
603 sizeof(struct ip_esp_hdr));
604 ciphstart = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr) +
605 GCM_ESP_IV_SIZE + 1;
606 ciphstart += sa_entry->esn ? esnlen : 0;
607
608 wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
609 aadstart,
610 aadstop,
611 ciphstart, 0);
612
613 wr->req.sec_cpl.cipherstop_lo_authinsert =
614 FILL_SEC_CPL_AUTHINSERT(0, ciphstart,
615 sa_entry->authsize,
616 sa_entry->authsize);
617 wr->req.sec_cpl.seqno_numivs =
618 FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,
619 CHCR_SCMD_CIPHER_MODE_AES_GCM,
620 CHCR_SCMD_AUTH_MODE_GHASH,
621 sa_entry->hmac_ctrl,
622 ivsize >> 1);
623 wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
624 0, ivdrop, 0);
625
626 pos += sizeof(struct fw_ulptx_wr) +
627 sizeof(struct ulp_txpkt) +
628 sizeof(struct ulptx_idata) +
629 sizeof(struct cpl_tx_sec_pdu);
630
631 pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);
632
633 return pos;
634 }
635
636 /**
637 * flits_to_desc - returns the num of Tx descriptors for the given flits
638 * @n: the number of flits
639 *
640 * Returns the number of Tx descriptors needed for the supplied number
641 * of flits.
642 */
643 static inline unsigned int flits_to_desc(unsigned int n)
644 {
645 WARN_ON(n > SGE_MAX_WR_LEN / 8);
646 return DIV_ROUND_UP(n, 8);
647 }
648
649 static inline unsigned int txq_avail(const struct sge_txq *q)
650 {
651 return q->size - 1 - q->in_use;
652 }
653
654 static void eth_txq_stop(struct sge_eth_txq *q)
655 {
656 netif_tx_stop_queue(q->txq);
657 q->q.stops++;
658 }
659
660 static inline void txq_advance(struct sge_txq *q, unsigned int n)
661 {
662 q->in_use += n;
663 q->pidx += n;
664 if (q->pidx >= q->size)
665 q->pidx -= q->size;
666 }
667
668 /*
669 * chcr_ipsec_xmit called from ULD Tx handler
670 */
671 int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)
672 {
673 struct xfrm_state *x = xfrm_input_state(skb);
674 unsigned int last_desc, ndesc, flits = 0;
675 struct ipsec_sa_entry *sa_entry;
676 u64 *pos, *end, *before, *sgl;
677 struct tx_sw_desc *sgl_sdesc;
678 int qidx, left, credits;
679 bool immediate = false;
680 struct sge_eth_txq *q;
681 struct adapter *adap;
682 struct port_info *pi;
683 struct sec_path *sp;
684
685 if (!x->xso.offload_handle)
686 return NETDEV_TX_BUSY;
687
688 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;
689
690 sp = skb_sec_path(skb);
691 if (sp->len != 1) {
692 out_free: dev_kfree_skb_any(skb);
693 return NETDEV_TX_OK;
694 }
695
696 pi = netdev_priv(dev);
697 adap = pi->adapter;
698 qidx = skb->queue_mapping;
699 q = &adap->sge.ethtxq[qidx + pi->first_qset];
700
701 cxgb4_reclaim_completed_tx(adap, &q->q, true);
702
703 flits = calc_tx_sec_flits(skb, sa_entry, &immediate);
704 ndesc = flits_to_desc(flits);
705 credits = txq_avail(&q->q) - ndesc;
706
707 if (unlikely(credits < 0)) {
708 eth_txq_stop(q);
709 dev_err(adap->pdev_dev,
710 "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",
711 dev->name, qidx, credits, ndesc, txq_avail(&q->q),
712 flits);
713 return NETDEV_TX_BUSY;
714 }
715
716 last_desc = q->q.pidx + ndesc - 1;
717 if (last_desc >= q->q.size)
718 last_desc -= q->q.size;
719 sgl_sdesc = &q->q.sdesc[last_desc];
720
721 if (!immediate &&
722 unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
723 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
724 q->mapping_err++;
725 goto out_free;
726 }
727
728 pos = (u64 *)&q->q.desc[q->q.pidx];
729 before = (u64 *)pos;
730 end = (u64 *)pos + flits;
731 /* Setup IPSec CPL */
732 pos = (void *)chcr_crypto_wreq(skb, dev, (void *)pos,
733 credits, sa_entry);
734 if (before > (u64 *)pos) {
735 left = (u8 *)end - (u8 *)q->q.stat;
736 end = (void *)q->q.desc + left;
737 }
738 if (pos == (u64 *)q->q.stat) {
739 left = (u8 *)end - (u8 *)q->q.stat;
740 end = (void *)q->q.desc + left;
741 pos = (void *)q->q.desc;
742 }
743
744 sgl = (void *)pos;
745 if (immediate) {
746 cxgb4_inline_tx_skb(skb, &q->q, sgl);
747 dev_consume_skb_any(skb);
748 } else {
749 cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,
750 0, sgl_sdesc->addr);
751 skb_orphan(skb);
752 sgl_sdesc->skb = skb;
753 }
754 txq_advance(&q->q, ndesc);
755
756 cxgb4_ring_tx_db(adap, &q->q, ndesc);
757 return NETDEV_TX_OK;
758 }
Linux with added FPC-III support