2 * Copyright (c) 2016 Chelsio Communications, Inc.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
9 #include <linux/workqueue.h>
10 #include <linux/kthread.h>
11 #include <linux/sched/signal.h>
13 #include <asm/unaligned.h>
15 #include <target/target_core_base.h>
16 #include <target/target_core_fabric.h>
19 struct sge_opaque_hdr
{
21 dma_addr_t addr
[MAX_SKB_FRAGS
+ 1];
24 static const u8 cxgbit_digest_len
[] = {0, 4, 4, 8};
26 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
27 sizeof(struct fw_ofld_tx_data_wr))
29 static struct sk_buff
*
30 __cxgbit_alloc_skb(struct cxgbit_sock
*csk
, u32 len
, bool iso
)
32 struct sk_buff
*skb
= NULL
;
35 static const u32 hdr_len
= TX_HDR_LEN
+ ISCSI_HDR_LEN
;
38 skb
= alloc_skb_with_frags(hdr_len
, len
,
44 skb_reserve(skb
, TX_HDR_LEN
);
45 skb_reset_transport_header(skb
);
46 __skb_put(skb
, ISCSI_HDR_LEN
);
49 submode
|= (csk
->submode
& CXGBIT_SUBMODE_DCRC
);
52 u32 iso_len
= iso
? sizeof(struct cpl_tx_data_iso
) : 0;
54 skb
= alloc_skb(hdr_len
+ iso_len
, GFP_KERNEL
);
58 skb_reserve(skb
, TX_HDR_LEN
+ iso_len
);
59 skb_reset_transport_header(skb
);
60 __skb_put(skb
, ISCSI_HDR_LEN
);
63 submode
|= (csk
->submode
& CXGBIT_SUBMODE_HCRC
);
64 cxgbit_skcb_submode(skb
) = submode
;
65 cxgbit_skcb_tx_extralen(skb
) = cxgbit_digest_len
[submode
];
66 cxgbit_skcb_flags(skb
) |= SKCBF_TX_NEED_HDR
;
70 static struct sk_buff
*cxgbit_alloc_skb(struct cxgbit_sock
*csk
, u32 len
)
72 return __cxgbit_alloc_skb(csk
, len
, false);
76 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
79 * Returns true if a packet can be sent as an offload WR with immediate
80 * data. We currently use the same limit as for Ethernet packets.
82 static int cxgbit_is_ofld_imm(const struct sk_buff
*skb
)
84 int length
= skb
->len
;
86 if (likely(cxgbit_skcb_flags(skb
) & SKCBF_TX_NEED_HDR
))
87 length
+= sizeof(struct fw_ofld_tx_data_wr
);
89 if (likely(cxgbit_skcb_flags(skb
) & SKCBF_TX_ISO
))
90 length
+= sizeof(struct cpl_tx_data_iso
);
92 #define MAX_IMM_TX_PKT_LEN 256
93 return length
<= MAX_IMM_TX_PKT_LEN
;
97 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
98 * @n: the number of SGL entries
99 * Calculates the number of flits needed for a scatter/gather list that
100 * can hold the given number of entries.
102 static inline unsigned int cxgbit_sgl_len(unsigned int n
)
105 return (3 * n
) / 2 + (n
& 1) + 2;
109 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
112 * Returns the number of flits needed for the given offload packet.
113 * These packets are already fully constructed and no additional headers
116 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff
*skb
)
118 unsigned int flits
, cnt
;
120 if (cxgbit_is_ofld_imm(skb
))
121 return DIV_ROUND_UP(skb
->len
, 8);
122 flits
= skb_transport_offset(skb
) / 8;
123 cnt
= skb_shinfo(skb
)->nr_frags
;
124 if (skb_tail_pointer(skb
) != skb_transport_header(skb
))
126 return flits
+ cxgbit_sgl_len(cnt
);
129 #define CXGBIT_ISO_FSLICE 0x1
130 #define CXGBIT_ISO_LSLICE 0x2
132 cxgbit_cpl_tx_data_iso(struct sk_buff
*skb
, struct cxgbit_iso_info
*iso_info
)
134 struct cpl_tx_data_iso
*cpl
;
135 unsigned int submode
= cxgbit_skcb_submode(skb
);
136 unsigned int fslice
= !!(iso_info
->flags
& CXGBIT_ISO_FSLICE
);
137 unsigned int lslice
= !!(iso_info
->flags
& CXGBIT_ISO_LSLICE
);
139 cpl
= __skb_push(skb
, sizeof(*cpl
));
141 cpl
->op_to_scsi
= htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO
) |
142 CPL_TX_DATA_ISO_FIRST_V(fslice
) |
143 CPL_TX_DATA_ISO_LAST_V(lslice
) |
144 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
145 CPL_TX_DATA_ISO_HDRCRC_V(submode
& 1) |
146 CPL_TX_DATA_ISO_PLDCRC_V(((submode
>> 1) & 1)) |
147 CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
148 CPL_TX_DATA_ISO_SCSI_V(2));
151 cpl
->mpdu
= htons(DIV_ROUND_UP(iso_info
->mpdu
, 4));
152 cpl
->burst_size
= htonl(DIV_ROUND_UP(iso_info
->burst_len
, 4));
153 cpl
->len
= htonl(iso_info
->len
);
154 cpl
->reserved2_seglen_offset
= htonl(0);
155 cpl
->datasn_offset
= htonl(0);
156 cpl
->buffer_offset
= htonl(0);
159 __skb_pull(skb
, sizeof(*cpl
));
163 cxgbit_tx_data_wr(struct cxgbit_sock
*csk
, struct sk_buff
*skb
, u32 dlen
,
164 u32 len
, u32 credits
, u32
compl)
166 struct fw_ofld_tx_data_wr
*req
;
167 const struct cxgb4_lld_info
*lldi
= &csk
->com
.cdev
->lldi
;
168 u32 submode
= cxgbit_skcb_submode(skb
);
170 u32 hdr_size
= sizeof(*req
);
171 u32 opcode
= FW_OFLD_TX_DATA_WR
;
173 u32 force
= is_t5(lldi
->adapter_type
) ? TX_FORCE_V(!submode
) :
176 if (cxgbit_skcb_flags(skb
) & SKCBF_TX_ISO
) {
177 opcode
= FW_ISCSI_TX_DATA_WR
;
178 immlen
+= sizeof(struct cpl_tx_data_iso
);
179 hdr_size
+= sizeof(struct cpl_tx_data_iso
);
183 if (cxgbit_is_ofld_imm(skb
))
186 req
= __skb_push(skb
, hdr_size
);
187 req
->op_to_immdlen
= cpu_to_be32(FW_WR_OP_V(opcode
) |
188 FW_WR_COMPL_V(compl) |
189 FW_WR_IMMDLEN_V(immlen
));
190 req
->flowid_len16
= cpu_to_be32(FW_WR_FLOWID_V(csk
->tid
) |
191 FW_WR_LEN16_V(credits
));
192 req
->plen
= htonl(len
);
193 wr_ulp_mode
= FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI
) |
194 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode
);
196 req
->tunnel_to_proxy
= htonl((wr_ulp_mode
) | force
|
197 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk
->txq
) ? 0 : 1));
200 static void cxgbit_arp_failure_skb_discard(void *handle
, struct sk_buff
*skb
)
205 void cxgbit_push_tx_frames(struct cxgbit_sock
*csk
)
209 while (csk
->wr_cred
&& ((skb
= skb_peek(&csk
->txq
)) != NULL
)) {
217 if (cxgbit_skcb_flags(skb
) & SKCBF_TX_ISO
)
218 iso_cpl_len
= sizeof(struct cpl_tx_data_iso
);
220 if (cxgbit_is_ofld_imm(skb
))
221 credits_needed
= DIV_ROUND_UP(dlen
+ iso_cpl_len
, 16);
223 credits_needed
= DIV_ROUND_UP((8 *
224 cxgbit_calc_tx_flits_ofld(skb
)) +
227 if (likely(cxgbit_skcb_flags(skb
) & SKCBF_TX_NEED_HDR
))
228 credits_needed
+= DIV_ROUND_UP(
229 sizeof(struct fw_ofld_tx_data_wr
), 16);
231 * Assumes the initial credits is large enough to support
232 * fw_flowc_wr plus largest possible first payload
235 if (!test_and_set_bit(CSK_TX_DATA_SENT
, &csk
->com
.flags
)) {
236 flowclen16
= cxgbit_send_tx_flowc_wr(csk
);
237 csk
->wr_cred
-= flowclen16
;
238 csk
->wr_una_cred
+= flowclen16
;
241 if (csk
->wr_cred
< credits_needed
) {
242 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
243 csk
, skb
->len
, skb
->data_len
,
244 credits_needed
, csk
->wr_cred
);
247 __skb_unlink(skb
, &csk
->txq
);
248 set_wr_txq(skb
, CPL_PRIORITY_DATA
, csk
->txq_idx
);
249 skb
->csum
= (__force __wsum
)(credits_needed
+ flowclen16
);
250 csk
->wr_cred
-= credits_needed
;
251 csk
->wr_una_cred
+= credits_needed
;
253 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
254 csk
, skb
->len
, skb
->data_len
, credits_needed
,
255 csk
->wr_cred
, csk
->wr_una_cred
);
257 if (likely(cxgbit_skcb_flags(skb
) & SKCBF_TX_NEED_HDR
)) {
258 len
+= cxgbit_skcb_tx_extralen(skb
);
260 if ((csk
->wr_una_cred
>= (csk
->wr_max_cred
/ 2)) ||
261 (!before(csk
->write_seq
,
262 csk
->snd_una
+ csk
->snd_win
))) {
264 csk
->wr_una_cred
= 0;
267 cxgbit_tx_data_wr(csk
, skb
, dlen
, len
, credits_needed
,
271 } else if ((cxgbit_skcb_flags(skb
) & SKCBF_TX_FLAG_COMPL
) ||
272 (csk
->wr_una_cred
>= (csk
->wr_max_cred
/ 2))) {
273 struct cpl_close_con_req
*req
=
274 (struct cpl_close_con_req
*)skb
->data
;
275 req
->wr
.wr_hi
|= htonl(FW_WR_COMPL_F
);
276 csk
->wr_una_cred
= 0;
279 cxgbit_sock_enqueue_wr(csk
, skb
);
280 t4_set_arp_err_handler(skb
, csk
,
281 cxgbit_arp_failure_skb_discard
);
283 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
284 csk
, csk
->tid
, skb
, len
);
286 cxgbit_l2t_send(csk
->com
.cdev
, skb
, csk
->l2t
);
290 static bool cxgbit_lock_sock(struct cxgbit_sock
*csk
)
292 spin_lock_bh(&csk
->lock
);
294 if (before(csk
->write_seq
, csk
->snd_una
+ csk
->snd_win
))
295 csk
->lock_owner
= true;
297 spin_unlock_bh(&csk
->lock
);
299 return csk
->lock_owner
;
302 static void cxgbit_unlock_sock(struct cxgbit_sock
*csk
)
304 struct sk_buff_head backlogq
;
306 void (*fn
)(struct cxgbit_sock
*, struct sk_buff
*);
308 skb_queue_head_init(&backlogq
);
310 spin_lock_bh(&csk
->lock
);
311 while (skb_queue_len(&csk
->backlogq
)) {
312 skb_queue_splice_init(&csk
->backlogq
, &backlogq
);
313 spin_unlock_bh(&csk
->lock
);
315 while ((skb
= __skb_dequeue(&backlogq
))) {
316 fn
= cxgbit_skcb_rx_backlog_fn(skb
);
320 spin_lock_bh(&csk
->lock
);
323 csk
->lock_owner
= false;
324 spin_unlock_bh(&csk
->lock
);
327 static int cxgbit_queue_skb(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
331 wait_event_interruptible(csk
->ack_waitq
, cxgbit_lock_sock(csk
));
333 if (unlikely((csk
->com
.state
!= CSK_STATE_ESTABLISHED
) ||
334 signal_pending(current
))) {
336 __skb_queue_purge(&csk
->ppodq
);
338 spin_lock_bh(&csk
->lock
);
339 if (csk
->lock_owner
) {
340 spin_unlock_bh(&csk
->lock
);
343 spin_unlock_bh(&csk
->lock
);
347 csk
->write_seq
+= skb
->len
+
348 cxgbit_skcb_tx_extralen(skb
);
350 skb_queue_splice_tail_init(&csk
->ppodq
, &csk
->txq
);
351 __skb_queue_tail(&csk
->txq
, skb
);
352 cxgbit_push_tx_frames(csk
);
355 cxgbit_unlock_sock(csk
);
360 cxgbit_map_skb(struct iscsi_cmd
*cmd
, struct sk_buff
*skb
, u32 data_offset
,
363 u32 i
= 0, nr_frags
= MAX_SKB_FRAGS
;
364 u32 padding
= ((-data_length
) & 3);
365 struct scatterlist
*sg
;
367 unsigned int page_off
;
373 * We know each entry in t_data_sg contains a page.
375 sg
= &cmd
->se_cmd
.t_data_sg
[data_offset
/ PAGE_SIZE
];
376 page_off
= (data_offset
% PAGE_SIZE
);
378 while (data_length
&& (i
< nr_frags
)) {
379 u32 cur_len
= min_t(u32
, data_length
, sg
->length
- page_off
);
384 skb_fill_page_desc(skb
, i
, page
, sg
->offset
+ page_off
,
386 skb
->data_len
+= cur_len
;
388 skb
->truesize
+= cur_len
;
390 data_length
-= cur_len
;
400 page
= alloc_page(GFP_KERNEL
| __GFP_ZERO
);
403 skb_fill_page_desc(skb
, i
, page
, 0, padding
);
404 skb
->data_len
+= padding
;
406 skb
->truesize
+= padding
;
413 cxgbit_tx_datain_iso(struct cxgbit_sock
*csk
, struct iscsi_cmd
*cmd
,
414 struct iscsi_datain_req
*dr
)
416 struct iscsi_conn
*conn
= csk
->conn
;
418 struct iscsi_datain datain
;
419 struct cxgbit_iso_info iso_info
;
420 u32 data_length
= cmd
->se_cmd
.data_length
;
421 u32 mrdsl
= conn
->conn_ops
->MaxRecvDataSegmentLength
;
422 u32 num_pdu
, plen
, tx_data
= 0;
423 bool task_sense
= !!(cmd
->se_cmd
.se_cmd_flags
&
424 SCF_TRANSPORT_TASK_SENSE
);
425 bool set_statsn
= false;
428 while (data_length
) {
429 num_pdu
= (data_length
+ mrdsl
- 1) / mrdsl
;
430 if (num_pdu
> csk
->max_iso_npdu
)
431 num_pdu
= csk
->max_iso_npdu
;
433 plen
= num_pdu
* mrdsl
;
434 if (plen
> data_length
)
437 skb
= __cxgbit_alloc_skb(csk
, 0, true);
441 memset(skb
->data
, 0, ISCSI_HDR_LEN
);
442 cxgbit_skcb_flags(skb
) |= SKCBF_TX_ISO
;
443 cxgbit_skcb_submode(skb
) |= (csk
->submode
&
444 CXGBIT_SUBMODE_DCRC
);
445 cxgbit_skcb_tx_extralen(skb
) = (num_pdu
*
446 cxgbit_digest_len
[cxgbit_skcb_submode(skb
)]) +
447 ((num_pdu
- 1) * ISCSI_HDR_LEN
);
449 memset(&datain
, 0, sizeof(struct iscsi_datain
));
450 memset(&iso_info
, 0, sizeof(iso_info
));
453 iso_info
.flags
|= CXGBIT_ISO_FSLICE
;
455 if (!(data_length
- plen
)) {
456 iso_info
.flags
|= CXGBIT_ISO_LSLICE
;
458 datain
.flags
= ISCSI_FLAG_DATA_STATUS
;
459 iscsit_increment_maxcmdsn(cmd
, conn
->sess
);
460 cmd
->stat_sn
= conn
->stat_sn
++;
465 iso_info
.burst_len
= num_pdu
* mrdsl
;
466 iso_info
.mpdu
= mrdsl
;
467 iso_info
.len
= ISCSI_HDR_LEN
+ plen
;
469 cxgbit_cpl_tx_data_iso(skb
, &iso_info
);
471 datain
.offset
= tx_data
;
472 datain
.data_sn
= cmd
->data_sn
- 1;
474 iscsit_build_datain_pdu(cmd
, conn
, &datain
,
475 (struct iscsi_data_rsp
*)skb
->data
,
478 ret
= cxgbit_map_skb(cmd
, skb
, tx_data
, plen
);
484 ret
= cxgbit_queue_skb(csk
, skb
);
491 cmd
->read_data_done
+= plen
;
492 cmd
->data_sn
+= num_pdu
;
495 dr
->dr_complete
= DATAIN_COMPLETE_NORMAL
;
504 cxgbit_tx_datain(struct cxgbit_sock
*csk
, struct iscsi_cmd
*cmd
,
505 const struct iscsi_datain
*datain
)
510 skb
= cxgbit_alloc_skb(csk
, 0);
514 memcpy(skb
->data
, cmd
->pdu
, ISCSI_HDR_LEN
);
516 if (datain
->length
) {
517 cxgbit_skcb_submode(skb
) |= (csk
->submode
&
518 CXGBIT_SUBMODE_DCRC
);
519 cxgbit_skcb_tx_extralen(skb
) =
520 cxgbit_digest_len
[cxgbit_skcb_submode(skb
)];
523 ret
= cxgbit_map_skb(cmd
, skb
, datain
->offset
, datain
->length
);
529 return cxgbit_queue_skb(csk
, skb
);
533 cxgbit_xmit_datain_pdu(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
534 struct iscsi_datain_req
*dr
,
535 const struct iscsi_datain
*datain
)
537 struct cxgbit_sock
*csk
= conn
->context
;
538 u32 data_length
= cmd
->se_cmd
.data_length
;
539 u32 padding
= ((-data_length
) & 3);
540 u32 mrdsl
= conn
->conn_ops
->MaxRecvDataSegmentLength
;
542 if ((data_length
> mrdsl
) && (!dr
->recovery
) &&
543 (!padding
) && (!datain
->offset
) && csk
->max_iso_npdu
) {
544 atomic_long_add(data_length
- datain
->length
,
545 &conn
->sess
->tx_data_octets
);
546 return cxgbit_tx_datain_iso(csk
, cmd
, dr
);
549 return cxgbit_tx_datain(csk
, cmd
, datain
);
553 cxgbit_xmit_nondatain_pdu(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
554 const void *data_buf
, u32 data_buf_len
)
556 struct cxgbit_sock
*csk
= conn
->context
;
558 u32 padding
= ((-data_buf_len
) & 3);
560 skb
= cxgbit_alloc_skb(csk
, data_buf_len
+ padding
);
564 memcpy(skb
->data
, cmd
->pdu
, ISCSI_HDR_LEN
);
569 skb_store_bits(skb
, ISCSI_HDR_LEN
, data_buf
, data_buf_len
);
572 skb_store_bits(skb
, ISCSI_HDR_LEN
+ data_buf_len
,
573 &pad_bytes
, padding
);
576 cxgbit_skcb_tx_extralen(skb
) = cxgbit_digest_len
[
577 cxgbit_skcb_submode(skb
)];
579 return cxgbit_queue_skb(csk
, skb
);
583 cxgbit_xmit_pdu(struct iscsi_conn
*conn
, struct iscsi_cmd
*cmd
,
584 struct iscsi_datain_req
*dr
, const void *buf
, u32 buf_len
)
587 return cxgbit_xmit_datain_pdu(conn
, cmd
, dr
, buf
);
589 return cxgbit_xmit_nondatain_pdu(conn
, cmd
, buf
, buf_len
);
592 int cxgbit_validate_params(struct iscsi_conn
*conn
)
594 struct cxgbit_sock
*csk
= conn
->context
;
595 struct cxgbit_device
*cdev
= csk
->com
.cdev
;
596 struct iscsi_param
*param
;
599 param
= iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH
,
604 if (kstrtou32(param
->value
, 0, &max_xmitdsl
) < 0)
607 if (max_xmitdsl
> cdev
->mdsl
) {
608 if (iscsi_change_param_sprintf(
609 conn
, "MaxXmitDataSegmentLength=%u", cdev
->mdsl
))
616 static int cxgbit_set_digest(struct cxgbit_sock
*csk
)
618 struct iscsi_conn
*conn
= csk
->conn
;
619 struct iscsi_param
*param
;
621 param
= iscsi_find_param_from_key(HEADERDIGEST
, conn
->param_list
);
623 pr_err("param not found key %s\n", HEADERDIGEST
);
627 if (!strcmp(param
->value
, CRC32C
))
628 csk
->submode
|= CXGBIT_SUBMODE_HCRC
;
630 param
= iscsi_find_param_from_key(DATADIGEST
, conn
->param_list
);
633 pr_err("param not found key %s\n", DATADIGEST
);
637 if (!strcmp(param
->value
, CRC32C
))
638 csk
->submode
|= CXGBIT_SUBMODE_DCRC
;
640 if (cxgbit_setup_conn_digest(csk
)) {
648 static int cxgbit_set_iso_npdu(struct cxgbit_sock
*csk
)
650 struct iscsi_conn
*conn
= csk
->conn
;
651 struct iscsi_conn_ops
*conn_ops
= conn
->conn_ops
;
652 struct iscsi_param
*param
;
654 u32 max_npdu
, max_iso_npdu
;
656 if (conn
->login
->leading_connection
) {
657 param
= iscsi_find_param_from_key(MAXBURSTLENGTH
,
660 pr_err("param not found key %s\n", MAXBURSTLENGTH
);
664 if (kstrtou32(param
->value
, 0, &mbl
) < 0)
667 mbl
= conn
->sess
->sess_ops
->MaxBurstLength
;
670 mrdsl
= conn_ops
->MaxRecvDataSegmentLength
;
671 max_npdu
= mbl
/ mrdsl
;
673 max_iso_npdu
= CXGBIT_MAX_ISO_PAYLOAD
/
674 (ISCSI_HDR_LEN
+ mrdsl
+
675 cxgbit_digest_len
[csk
->submode
]);
677 csk
->max_iso_npdu
= min(max_npdu
, max_iso_npdu
);
679 if (csk
->max_iso_npdu
<= 1)
680 csk
->max_iso_npdu
= 0;
686 * cxgbit_seq_pdu_inorder()
687 * @csk: pointer to cxgbit socket structure
689 * This function checks whether data sequence and data
692 * Return: returns -1 on error, 0 if data sequence and
693 * data pdu are in order, 1 if data sequence or data pdu
696 static int cxgbit_seq_pdu_inorder(struct cxgbit_sock
*csk
)
698 struct iscsi_conn
*conn
= csk
->conn
;
699 struct iscsi_param
*param
;
701 if (conn
->login
->leading_connection
) {
702 param
= iscsi_find_param_from_key(DATASEQUENCEINORDER
,
705 pr_err("param not found key %s\n", DATASEQUENCEINORDER
);
709 if (strcmp(param
->value
, YES
))
712 param
= iscsi_find_param_from_key(DATAPDUINORDER
,
715 pr_err("param not found key %s\n", DATAPDUINORDER
);
719 if (strcmp(param
->value
, YES
))
723 if (!conn
->sess
->sess_ops
->DataSequenceInOrder
)
725 if (!conn
->sess
->sess_ops
->DataPDUInOrder
)
732 static int cxgbit_set_params(struct iscsi_conn
*conn
)
734 struct cxgbit_sock
*csk
= conn
->context
;
735 struct cxgbit_device
*cdev
= csk
->com
.cdev
;
736 struct cxgbi_ppm
*ppm
= *csk
->com
.cdev
->lldi
.iscsi_ppm
;
737 struct iscsi_conn_ops
*conn_ops
= conn
->conn_ops
;
738 struct iscsi_param
*param
;
741 if (conn_ops
->MaxRecvDataSegmentLength
> cdev
->mdsl
)
742 conn_ops
->MaxRecvDataSegmentLength
= cdev
->mdsl
;
744 if (conn
->login
->leading_connection
) {
745 param
= iscsi_find_param_from_key(ERRORRECOVERYLEVEL
,
748 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL
);
751 if (kstrtou8(param
->value
, 0, &erl
) < 0)
754 erl
= conn
->sess
->sess_ops
->ErrorRecoveryLevel
;
760 ret
= cxgbit_seq_pdu_inorder(csk
);
763 } else if (ret
> 0) {
764 if (is_t5(cdev
->lldi
.adapter_type
))
770 if (test_bit(CDEV_ISO_ENABLE
, &cdev
->flags
)) {
771 if (cxgbit_set_iso_npdu(csk
))
776 if (test_bit(CDEV_DDP_ENABLE
, &cdev
->flags
)) {
777 if (cxgbit_setup_conn_pgidx(csk
,
778 ppm
->tformat
.pgsz_idx_dflt
))
780 set_bit(CSK_DDP_ENABLE
, &csk
->com
.flags
);
785 if (cxgbit_set_digest(csk
))
792 cxgbit_put_login_tx(struct iscsi_conn
*conn
, struct iscsi_login
*login
,
795 struct cxgbit_sock
*csk
= conn
->context
;
798 u8 padding
= ((-length
) & 3);
800 skb
= cxgbit_alloc_skb(csk
, length
+ padding
);
803 skb_store_bits(skb
, 0, login
->rsp
, ISCSI_HDR_LEN
);
804 skb_store_bits(skb
, ISCSI_HDR_LEN
, login
->rsp_buf
, length
);
807 skb_store_bits(skb
, ISCSI_HDR_LEN
+ length
,
808 &padding_buf
, padding
);
810 if (login
->login_complete
) {
811 if (cxgbit_set_params(conn
)) {
816 set_bit(CSK_LOGIN_DONE
, &csk
->com
.flags
);
819 if (cxgbit_queue_skb(csk
, skb
))
822 if ((!login
->login_complete
) && (!login
->login_failed
))
823 schedule_delayed_work(&conn
->login_work
, 0);
829 cxgbit_skb_copy_to_sg(struct sk_buff
*skb
, struct scatterlist
*sg
,
830 unsigned int nents
, u32 skip
)
832 struct skb_seq_state st
;
834 unsigned int consumed
= 0, buf_len
;
835 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(skb
);
837 skb_prepare_seq_read(skb
, pdu_cb
->doffset
,
838 pdu_cb
->doffset
+ pdu_cb
->dlen
,
842 buf_len
= skb_seq_read(consumed
, &buf
, &st
);
844 skb_abort_seq_read(&st
);
848 consumed
+= sg_pcopy_from_buffer(sg
, nents
, (void *)buf
,
849 buf_len
, skip
+ consumed
);
853 static struct iscsi_cmd
*cxgbit_allocate_cmd(struct cxgbit_sock
*csk
)
855 struct iscsi_conn
*conn
= csk
->conn
;
856 struct cxgbi_ppm
*ppm
= cdev2ppm(csk
->com
.cdev
);
857 struct cxgbit_cmd
*ccmd
;
858 struct iscsi_cmd
*cmd
;
860 cmd
= iscsit_allocate_cmd(conn
, TASK_INTERRUPTIBLE
);
862 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
866 ccmd
= iscsit_priv_cmd(cmd
);
867 ccmd
->ttinfo
.tag
= ppm
->tformat
.no_ddp_mask
;
868 ccmd
->setup_ddp
= true;
874 cxgbit_handle_immediate_data(struct iscsi_cmd
*cmd
, struct iscsi_scsi_req
*hdr
,
877 struct iscsi_conn
*conn
= cmd
->conn
;
878 struct cxgbit_sock
*csk
= conn
->context
;
879 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
881 if (pdu_cb
->flags
& PDUCBF_RX_DCRC_ERR
) {
882 pr_err("ImmediateData CRC32C DataDigest error\n");
883 if (!conn
->sess
->sess_ops
->ErrorRecoveryLevel
) {
884 pr_err("Unable to recover from"
885 " Immediate Data digest failure while"
887 iscsit_reject_cmd(cmd
, ISCSI_REASON_DATA_DIGEST_ERROR
,
888 (unsigned char *)hdr
);
889 return IMMEDIATE_DATA_CANNOT_RECOVER
;
892 iscsit_reject_cmd(cmd
, ISCSI_REASON_DATA_DIGEST_ERROR
,
893 (unsigned char *)hdr
);
894 return IMMEDIATE_DATA_ERL1_CRC_FAILURE
;
897 if (cmd
->se_cmd
.se_cmd_flags
& SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC
) {
898 struct cxgbit_cmd
*ccmd
= iscsit_priv_cmd(cmd
);
899 struct skb_shared_info
*ssi
= skb_shinfo(csk
->skb
);
900 skb_frag_t
*dfrag
= &ssi
->frags
[pdu_cb
->dfrag_idx
];
902 sg_init_table(&ccmd
->sg
, 1);
903 sg_set_page(&ccmd
->sg
, dfrag
->page
.p
, skb_frag_size(dfrag
),
905 get_page(dfrag
->page
.p
);
907 cmd
->se_cmd
.t_data_sg
= &ccmd
->sg
;
908 cmd
->se_cmd
.t_data_nents
= 1;
910 ccmd
->release
= true;
912 struct scatterlist
*sg
= &cmd
->se_cmd
.t_data_sg
[0];
913 u32 sg_nents
= max(1UL, DIV_ROUND_UP(pdu_cb
->dlen
, PAGE_SIZE
));
915 cxgbit_skb_copy_to_sg(csk
->skb
, sg
, sg_nents
, 0);
918 cmd
->write_data_done
+= pdu_cb
->dlen
;
920 if (cmd
->write_data_done
== cmd
->se_cmd
.data_length
) {
921 spin_lock_bh(&cmd
->istate_lock
);
922 cmd
->cmd_flags
|= ICF_GOT_LAST_DATAOUT
;
923 cmd
->i_state
= ISTATE_RECEIVED_LAST_DATAOUT
;
924 spin_unlock_bh(&cmd
->istate_lock
);
927 return IMMEDIATE_DATA_NORMAL_OPERATION
;
931 cxgbit_get_immediate_data(struct iscsi_cmd
*cmd
, struct iscsi_scsi_req
*hdr
,
934 struct iscsi_conn
*conn
= cmd
->conn
;
935 int cmdsn_ret
= 0, immed_ret
= IMMEDIATE_DATA_NORMAL_OPERATION
;
937 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
940 goto after_immediate_data
;
942 immed_ret
= cxgbit_handle_immediate_data(cmd
, hdr
,
943 cmd
->first_burst_len
);
944 after_immediate_data
:
945 if (immed_ret
== IMMEDIATE_DATA_NORMAL_OPERATION
) {
947 * A PDU/CmdSN carrying Immediate Data passed
948 * DataCRC, check against ExpCmdSN/MaxCmdSN if
949 * Immediate Bit is not set.
951 cmdsn_ret
= iscsit_sequence_cmd(conn
, cmd
,
952 (unsigned char *)hdr
,
954 if (cmdsn_ret
== CMDSN_ERROR_CANNOT_RECOVER
)
957 if (cmd
->sense_reason
|| cmdsn_ret
== CMDSN_LOWER_THAN_EXP
) {
958 target_put_sess_cmd(&cmd
->se_cmd
);
960 } else if (cmd
->unsolicited_data
) {
961 iscsit_set_unsoliticed_dataout(cmd
);
964 } else if (immed_ret
== IMMEDIATE_DATA_ERL1_CRC_FAILURE
) {
966 * Immediate Data failed DataCRC and ERL>=1,
967 * silently drop this PDU and let the initiator
968 * plug the CmdSN gap.
970 * FIXME: Send Unsolicited NOPIN with reserved
971 * TTT here to help the initiator figure out
972 * the missing CmdSN, although they should be
973 * intelligent enough to determine the missing
974 * CmdSN and issue a retry to plug the sequence.
976 cmd
->i_state
= ISTATE_REMOVE
;
977 iscsit_add_cmd_to_immediate_queue(cmd
, conn
, cmd
->i_state
);
978 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
985 cxgbit_handle_scsi_cmd(struct cxgbit_sock
*csk
, struct iscsi_cmd
*cmd
)
987 struct iscsi_conn
*conn
= csk
->conn
;
988 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
989 struct iscsi_scsi_req
*hdr
= (struct iscsi_scsi_req
*)pdu_cb
->hdr
;
991 bool dump_payload
= false;
993 rc
= iscsit_setup_scsi_cmd(conn
, cmd
, (unsigned char *)hdr
);
997 if (pdu_cb
->dlen
&& (pdu_cb
->dlen
== cmd
->se_cmd
.data_length
) &&
998 (pdu_cb
->nr_dfrags
== 1))
999 cmd
->se_cmd
.se_cmd_flags
|= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC
;
1001 rc
= iscsit_process_scsi_cmd(conn
, cmd
, hdr
);
1005 dump_payload
= true;
1010 return cxgbit_get_immediate_data(cmd
, hdr
, dump_payload
);
1013 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock
*csk
)
1015 struct scatterlist
*sg_start
;
1016 struct iscsi_conn
*conn
= csk
->conn
;
1017 struct iscsi_cmd
*cmd
= NULL
;
1018 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
1019 struct iscsi_data
*hdr
= (struct iscsi_data
*)pdu_cb
->hdr
;
1020 u32 data_offset
= be32_to_cpu(hdr
->offset
);
1021 u32 data_len
= pdu_cb
->dlen
;
1022 int rc
, sg_nents
, sg_off
;
1023 bool dcrc_err
= false;
1025 if (pdu_cb
->flags
& PDUCBF_RX_DDP_CMP
) {
1026 u32 offset
= be32_to_cpu(hdr
->offset
);
1028 u32 payload_length
= ntoh24(hdr
->dlength
);
1029 bool success
= false;
1031 cmd
= iscsit_find_cmd_from_itt_or_dump(conn
, hdr
->itt
, 0);
1035 ddp_data_len
= offset
- cmd
->write_data_done
;
1036 atomic_long_add(ddp_data_len
, &conn
->sess
->rx_data_octets
);
1038 cmd
->write_data_done
= offset
;
1039 cmd
->next_burst_len
= ddp_data_len
;
1040 cmd
->data_sn
= be32_to_cpu(hdr
->datasn
);
1042 rc
= __iscsit_check_dataout_hdr(conn
, (unsigned char *)hdr
,
1043 cmd
, payload_length
, &success
);
1049 rc
= iscsit_check_dataout_hdr(conn
, (unsigned char *)hdr
, &cmd
);
1056 if (pdu_cb
->flags
& PDUCBF_RX_DCRC_ERR
) {
1057 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1058 " DataSN: 0x%08x\n",
1059 hdr
->itt
, hdr
->offset
, data_len
,
1066 pr_debug("DataOut data_len: %u, "
1067 "write_data_done: %u, data_length: %u\n",
1068 data_len
, cmd
->write_data_done
,
1069 cmd
->se_cmd
.data_length
);
1071 if (!(pdu_cb
->flags
& PDUCBF_RX_DATA_DDPD
)) {
1072 u32 skip
= data_offset
% PAGE_SIZE
;
1074 sg_off
= data_offset
/ PAGE_SIZE
;
1075 sg_start
= &cmd
->se_cmd
.t_data_sg
[sg_off
];
1076 sg_nents
= max(1UL, DIV_ROUND_UP(skip
+ data_len
, PAGE_SIZE
));
1078 cxgbit_skb_copy_to_sg(csk
->skb
, sg_start
, sg_nents
, skip
);
1083 rc
= iscsit_check_dataout_payload(cmd
, hdr
, dcrc_err
);
1090 static int cxgbit_handle_nop_out(struct cxgbit_sock
*csk
, struct iscsi_cmd
*cmd
)
1092 struct iscsi_conn
*conn
= csk
->conn
;
1093 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
1094 struct iscsi_nopout
*hdr
= (struct iscsi_nopout
*)pdu_cb
->hdr
;
1095 unsigned char *ping_data
= NULL
;
1096 u32 payload_length
= pdu_cb
->dlen
;
1099 ret
= iscsit_setup_nop_out(conn
, cmd
, hdr
);
1103 if (pdu_cb
->flags
& PDUCBF_RX_DCRC_ERR
) {
1104 if (!conn
->sess
->sess_ops
->ErrorRecoveryLevel
) {
1105 pr_err("Unable to recover from"
1106 " NOPOUT Ping DataCRC failure while in"
1112 * drop this PDU and let the
1113 * initiator plug the CmdSN gap.
1115 pr_info("Dropping NOPOUT"
1116 " Command CmdSN: 0x%08x due to"
1117 " DataCRC error.\n", hdr
->cmdsn
);
1124 * Handle NOP-OUT payload for traditional iSCSI sockets
1126 if (payload_length
&& hdr
->ttt
== cpu_to_be32(0xFFFFFFFF)) {
1127 ping_data
= kzalloc(payload_length
+ 1, GFP_KERNEL
);
1129 pr_err("Unable to allocate memory for"
1130 " NOPOUT ping data.\n");
1135 skb_copy_bits(csk
->skb
, pdu_cb
->doffset
,
1136 ping_data
, payload_length
);
1138 ping_data
[payload_length
] = '\0';
1140 * Attach ping data to struct iscsi_cmd->buf_ptr.
1142 cmd
->buf_ptr
= ping_data
;
1143 cmd
->buf_ptr_size
= payload_length
;
1145 pr_debug("Got %u bytes of NOPOUT ping"
1146 " data.\n", payload_length
);
1147 pr_debug("Ping Data: \"%s\"\n", ping_data
);
1150 return iscsit_process_nop_out(conn
, cmd
, hdr
);
1153 iscsit_free_cmd(cmd
, false);
1158 cxgbit_handle_text_cmd(struct cxgbit_sock
*csk
, struct iscsi_cmd
*cmd
)
1160 struct iscsi_conn
*conn
= csk
->conn
;
1161 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
1162 struct iscsi_text
*hdr
= (struct iscsi_text
*)pdu_cb
->hdr
;
1163 u32 payload_length
= pdu_cb
->dlen
;
1165 unsigned char *text_in
= NULL
;
1167 rc
= iscsit_setup_text_cmd(conn
, cmd
, hdr
);
1171 if (pdu_cb
->flags
& PDUCBF_RX_DCRC_ERR
) {
1172 if (!conn
->sess
->sess_ops
->ErrorRecoveryLevel
) {
1173 pr_err("Unable to recover from"
1174 " Text Data digest failure while in"
1179 * drop this PDU and let the
1180 * initiator plug the CmdSN gap.
1182 pr_info("Dropping Text"
1183 " Command CmdSN: 0x%08x due to"
1184 " DataCRC error.\n", hdr
->cmdsn
);
1189 if (payload_length
) {
1190 text_in
= kzalloc(payload_length
, GFP_KERNEL
);
1192 pr_err("Unable to allocate text_in of payload_length: %u\n",
1196 skb_copy_bits(csk
->skb
, pdu_cb
->doffset
,
1197 text_in
, payload_length
);
1199 text_in
[payload_length
- 1] = '\0';
1201 cmd
->text_in_ptr
= text_in
;
1204 return iscsit_process_text_cmd(conn
, cmd
, hdr
);
1207 return iscsit_reject_cmd(cmd
, ISCSI_REASON_PROTOCOL_ERROR
,
1211 static int cxgbit_target_rx_opcode(struct cxgbit_sock
*csk
)
1213 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
1214 struct iscsi_hdr
*hdr
= (struct iscsi_hdr
*)pdu_cb
->hdr
;
1215 struct iscsi_conn
*conn
= csk
->conn
;
1216 struct iscsi_cmd
*cmd
= NULL
;
1217 u8 opcode
= (hdr
->opcode
& ISCSI_OPCODE_MASK
);
1221 case ISCSI_OP_SCSI_CMD
:
1222 cmd
= cxgbit_allocate_cmd(csk
);
1226 ret
= cxgbit_handle_scsi_cmd(csk
, cmd
);
1228 case ISCSI_OP_SCSI_DATA_OUT
:
1229 ret
= cxgbit_handle_iscsi_dataout(csk
);
1231 case ISCSI_OP_NOOP_OUT
:
1232 if (hdr
->ttt
== cpu_to_be32(0xFFFFFFFF)) {
1233 cmd
= cxgbit_allocate_cmd(csk
);
1238 ret
= cxgbit_handle_nop_out(csk
, cmd
);
1240 case ISCSI_OP_SCSI_TMFUNC
:
1241 cmd
= cxgbit_allocate_cmd(csk
);
1245 ret
= iscsit_handle_task_mgt_cmd(conn
, cmd
,
1246 (unsigned char *)hdr
);
1249 if (hdr
->ttt
!= cpu_to_be32(0xFFFFFFFF)) {
1250 cmd
= iscsit_find_cmd_from_itt(conn
, hdr
->itt
);
1254 cmd
= cxgbit_allocate_cmd(csk
);
1259 ret
= cxgbit_handle_text_cmd(csk
, cmd
);
1261 case ISCSI_OP_LOGOUT
:
1262 cmd
= cxgbit_allocate_cmd(csk
);
1266 ret
= iscsit_handle_logout_cmd(conn
, cmd
, (unsigned char *)hdr
);
1268 wait_for_completion_timeout(&conn
->conn_logout_comp
,
1269 SECONDS_FOR_LOGOUT_COMP
1272 case ISCSI_OP_SNACK
:
1273 ret
= iscsit_handle_snack(conn
, (unsigned char *)hdr
);
1276 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode
);
1284 return iscsit_add_reject(conn
, ISCSI_REASON_BOOKMARK_NO_RESOURCES
,
1285 (unsigned char *)hdr
);
1289 static int cxgbit_rx_opcode(struct cxgbit_sock
*csk
)
1291 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
1292 struct iscsi_conn
*conn
= csk
->conn
;
1293 struct iscsi_hdr
*hdr
= pdu_cb
->hdr
;
1296 if (pdu_cb
->flags
& PDUCBF_RX_HCRC_ERR
) {
1297 atomic_long_inc(&conn
->sess
->conn_digest_errors
);
1301 if (conn
->conn_state
== TARG_CONN_STATE_IN_LOGOUT
)
1304 opcode
= hdr
->opcode
& ISCSI_OPCODE_MASK
;
1306 if (conn
->sess
->sess_ops
->SessionType
&&
1307 ((!(opcode
& ISCSI_OP_TEXT
)) ||
1308 (!(opcode
& ISCSI_OP_LOGOUT
)))) {
1309 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1310 " while in Discovery Session, rejecting.\n", opcode
);
1311 iscsit_add_reject(conn
, ISCSI_REASON_PROTOCOL_ERROR
,
1312 (unsigned char *)hdr
);
1316 if (cxgbit_target_rx_opcode(csk
) < 0)
1325 static int cxgbit_rx_login_pdu(struct cxgbit_sock
*csk
)
1327 struct iscsi_conn
*conn
= csk
->conn
;
1328 struct iscsi_login
*login
= conn
->login
;
1329 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_rx_pdu_cb(csk
->skb
);
1330 struct iscsi_login_req
*login_req
;
1332 login_req
= (struct iscsi_login_req
*)login
->req
;
1333 memcpy(login_req
, pdu_cb
->hdr
, sizeof(*login_req
));
1335 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1336 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1337 login_req
->flags
, login_req
->itt
, login_req
->cmdsn
,
1338 login_req
->exp_statsn
, login_req
->cid
, pdu_cb
->dlen
);
1340 * Setup the initial iscsi_login values from the leading
1341 * login request PDU.
1343 if (login
->first_request
) {
1344 login_req
= (struct iscsi_login_req
*)login
->req
;
1345 login
->leading_connection
= (!login_req
->tsih
) ? 1 : 0;
1346 login
->current_stage
= ISCSI_LOGIN_CURRENT_STAGE(
1348 login
->version_min
= login_req
->min_version
;
1349 login
->version_max
= login_req
->max_version
;
1350 memcpy(login
->isid
, login_req
->isid
, 6);
1351 login
->cmd_sn
= be32_to_cpu(login_req
->cmdsn
);
1352 login
->init_task_tag
= login_req
->itt
;
1353 login
->initial_exp_statsn
= be32_to_cpu(login_req
->exp_statsn
);
1354 login
->cid
= be16_to_cpu(login_req
->cid
);
1355 login
->tsih
= be16_to_cpu(login_req
->tsih
);
1358 if (iscsi_target_check_login_request(conn
, login
) < 0)
1361 memset(login
->req_buf
, 0, MAX_KEY_VALUE_PAIRS
);
1362 skb_copy_bits(csk
->skb
, pdu_cb
->doffset
, login
->req_buf
, pdu_cb
->dlen
);
1368 cxgbit_process_iscsi_pdu(struct cxgbit_sock
*csk
, struct sk_buff
*skb
, int idx
)
1370 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_skb_lro_pdu_cb(skb
, idx
);
1373 cxgbit_rx_pdu_cb(skb
) = pdu_cb
;
1377 if (!test_bit(CSK_LOGIN_DONE
, &csk
->com
.flags
)) {
1378 ret
= cxgbit_rx_login_pdu(csk
);
1379 set_bit(CSK_LOGIN_PDU_DONE
, &csk
->com
.flags
);
1381 ret
= cxgbit_rx_opcode(csk
);
1387 static void cxgbit_lro_skb_dump(struct sk_buff
*skb
)
1389 struct skb_shared_info
*ssi
= skb_shinfo(skb
);
1390 struct cxgbit_lro_cb
*lro_cb
= cxgbit_skb_lro_cb(skb
);
1391 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_skb_lro_pdu_cb(skb
, 0);
1394 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1395 skb
, skb
->head
, skb
->data
, skb
->len
, skb
->data_len
,
1397 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1398 skb
, lro_cb
->csk
, lro_cb
->pdu_idx
, lro_cb
->pdu_totallen
);
1400 for (i
= 0; i
< lro_cb
->pdu_idx
; i
++, pdu_cb
++)
1401 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1403 skb
, i
, pdu_cb
->pdulen
, pdu_cb
->flags
, pdu_cb
->seq
,
1404 pdu_cb
->ddigest
, pdu_cb
->frags
);
1405 for (i
= 0; i
< ssi
->nr_frags
; i
++)
1406 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1407 skb
, i
, ssi
->frags
[i
].page_offset
, ssi
->frags
[i
].size
);
1410 static void cxgbit_lro_hskb_reset(struct cxgbit_sock
*csk
)
1412 struct sk_buff
*skb
= csk
->lro_hskb
;
1413 struct skb_shared_info
*ssi
= skb_shinfo(skb
);
1416 memset(skb
->data
, 0, LRO_SKB_MIN_HEADROOM
);
1417 for (i
= 0; i
< ssi
->nr_frags
; i
++)
1418 put_page(skb_frag_page(&ssi
->frags
[i
]));
1421 skb
->truesize
-= skb
->len
;
1426 cxgbit_lro_skb_merge(struct cxgbit_sock
*csk
, struct sk_buff
*skb
, u8 pdu_idx
)
1428 struct sk_buff
*hskb
= csk
->lro_hskb
;
1429 struct cxgbit_lro_pdu_cb
*hpdu_cb
= cxgbit_skb_lro_pdu_cb(hskb
, 0);
1430 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_skb_lro_pdu_cb(skb
, pdu_idx
);
1431 struct skb_shared_info
*hssi
= skb_shinfo(hskb
);
1432 struct skb_shared_info
*ssi
= skb_shinfo(skb
);
1433 unsigned int len
= 0;
1435 if (pdu_cb
->flags
& PDUCBF_RX_HDR
) {
1436 u8 hfrag_idx
= hssi
->nr_frags
;
1438 hpdu_cb
->flags
|= pdu_cb
->flags
;
1439 hpdu_cb
->seq
= pdu_cb
->seq
;
1440 hpdu_cb
->hdr
= pdu_cb
->hdr
;
1441 hpdu_cb
->hlen
= pdu_cb
->hlen
;
1443 memcpy(&hssi
->frags
[hfrag_idx
], &ssi
->frags
[pdu_cb
->hfrag_idx
],
1444 sizeof(skb_frag_t
));
1446 get_page(skb_frag_page(&hssi
->frags
[hfrag_idx
]));
1449 hpdu_cb
->hfrag_idx
= hfrag_idx
;
1451 len
= hssi
->frags
[hfrag_idx
].size
;
1453 hskb
->data_len
+= len
;
1454 hskb
->truesize
+= len
;
1457 if (pdu_cb
->flags
& PDUCBF_RX_DATA
) {
1458 u8 dfrag_idx
= hssi
->nr_frags
, i
;
1460 hpdu_cb
->flags
|= pdu_cb
->flags
;
1461 hpdu_cb
->dfrag_idx
= dfrag_idx
;
1464 for (i
= 0; i
< pdu_cb
->nr_dfrags
; dfrag_idx
++, i
++) {
1465 memcpy(&hssi
->frags
[dfrag_idx
],
1466 &ssi
->frags
[pdu_cb
->dfrag_idx
+ i
],
1467 sizeof(skb_frag_t
));
1469 get_page(skb_frag_page(&hssi
->frags
[dfrag_idx
]));
1471 len
+= hssi
->frags
[dfrag_idx
].size
;
1477 hpdu_cb
->dlen
= pdu_cb
->dlen
;
1478 hpdu_cb
->doffset
= hpdu_cb
->hlen
;
1479 hpdu_cb
->nr_dfrags
= pdu_cb
->nr_dfrags
;
1481 hskb
->data_len
+= len
;
1482 hskb
->truesize
+= len
;
1485 if (pdu_cb
->flags
& PDUCBF_RX_STATUS
) {
1486 hpdu_cb
->flags
|= pdu_cb
->flags
;
1488 if (hpdu_cb
->flags
& PDUCBF_RX_DATA
)
1489 hpdu_cb
->flags
&= ~PDUCBF_RX_DATA_DDPD
;
1491 hpdu_cb
->ddigest
= pdu_cb
->ddigest
;
1492 hpdu_cb
->pdulen
= pdu_cb
->pdulen
;
1496 static int cxgbit_process_lro_skb(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1498 struct cxgbit_lro_cb
*lro_cb
= cxgbit_skb_lro_cb(skb
);
1499 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_skb_lro_pdu_cb(skb
, 0);
1500 u8 pdu_idx
= 0, last_idx
= 0;
1503 if (!pdu_cb
->complete
) {
1504 cxgbit_lro_skb_merge(csk
, skb
, 0);
1506 if (pdu_cb
->flags
& PDUCBF_RX_STATUS
) {
1507 struct sk_buff
*hskb
= csk
->lro_hskb
;
1509 ret
= cxgbit_process_iscsi_pdu(csk
, hskb
, 0);
1511 cxgbit_lro_hskb_reset(csk
);
1520 if (lro_cb
->pdu_idx
)
1521 last_idx
= lro_cb
->pdu_idx
- 1;
1523 for (; pdu_idx
<= last_idx
; pdu_idx
++) {
1524 ret
= cxgbit_process_iscsi_pdu(csk
, skb
, pdu_idx
);
1529 if ((!lro_cb
->complete
) && lro_cb
->pdu_idx
)
1530 cxgbit_lro_skb_merge(csk
, skb
, lro_cb
->pdu_idx
);
1536 static int cxgbit_rx_lro_skb(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1538 struct cxgbit_lro_cb
*lro_cb
= cxgbit_skb_lro_cb(skb
);
1539 struct cxgbit_lro_pdu_cb
*pdu_cb
= cxgbit_skb_lro_pdu_cb(skb
, 0);
1542 if ((pdu_cb
->flags
& PDUCBF_RX_HDR
) &&
1543 (pdu_cb
->seq
!= csk
->rcv_nxt
)) {
1544 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1545 csk
, csk
->tid
, pdu_cb
->seq
, csk
->rcv_nxt
);
1546 cxgbit_lro_skb_dump(skb
);
1550 csk
->rcv_nxt
+= lro_cb
->pdu_totallen
;
1552 ret
= cxgbit_process_lro_skb(csk
, skb
);
1554 csk
->rx_credits
+= lro_cb
->pdu_totallen
;
1556 if (csk
->rx_credits
>= (csk
->rcv_win
/ 4))
1557 cxgbit_rx_data_ack(csk
);
1562 static int cxgbit_rx_skb(struct cxgbit_sock
*csk
, struct sk_buff
*skb
)
1564 struct cxgb4_lld_info
*lldi
= &csk
->com
.cdev
->lldi
;
1567 if (likely(cxgbit_skcb_flags(skb
) & SKCBF_RX_LRO
)) {
1568 if (is_t5(lldi
->adapter_type
))
1569 ret
= cxgbit_rx_lro_skb(csk
, skb
);
1571 ret
= cxgbit_process_lro_skb(csk
, skb
);
1578 static bool cxgbit_rxq_len(struct cxgbit_sock
*csk
, struct sk_buff_head
*rxq
)
1580 spin_lock_bh(&csk
->rxq
.lock
);
1581 if (skb_queue_len(&csk
->rxq
)) {
1582 skb_queue_splice_init(&csk
->rxq
, rxq
);
1583 spin_unlock_bh(&csk
->rxq
.lock
);
1586 spin_unlock_bh(&csk
->rxq
.lock
);
1590 static int cxgbit_wait_rxq(struct cxgbit_sock
*csk
)
1592 struct sk_buff
*skb
;
1593 struct sk_buff_head rxq
;
1595 skb_queue_head_init(&rxq
);
1597 wait_event_interruptible(csk
->waitq
, cxgbit_rxq_len(csk
, &rxq
));
1599 if (signal_pending(current
))
1602 while ((skb
= __skb_dequeue(&rxq
))) {
1603 if (cxgbit_rx_skb(csk
, skb
))
1609 __skb_queue_purge(&rxq
);
1613 int cxgbit_get_login_rx(struct iscsi_conn
*conn
, struct iscsi_login
*login
)
1615 struct cxgbit_sock
*csk
= conn
->context
;
1618 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE
, &csk
->com
.flags
)) {
1619 ret
= cxgbit_wait_rxq(csk
);
1621 clear_bit(CSK_LOGIN_PDU_DONE
, &csk
->com
.flags
);
1629 void cxgbit_get_rx_pdu(struct iscsi_conn
*conn
)
1631 struct cxgbit_sock
*csk
= conn
->context
;
1633 while (!kthread_should_stop()) {
1634 iscsit_thread_check_cpumask(conn
, current
, 0);
1635 if (cxgbit_wait_rxq(csk
))