?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'r6040-next'
[linux/fpc-iii.git] / drivers / target / iscsi / cxgbit / cxgbit_target.c
blobd02bf58aea6d85bfdfa0473b95ca94794135becc
1 /*
2 * Copyright (c) 2016 Chelsio Communications, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9 #include <linux/workqueue.h>
10 #include <linux/kthread.h>
11 #include <asm/unaligned.h>
12 #include <target/target_core_base.h>
13 #include <target/target_core_fabric.h>
14 #include "cxgbit.h"
15
16 struct sge_opaque_hdr {
17 void *dev;
18 dma_addr_t addr[MAX_SKB_FRAGS + 1];
19 };
20
21 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
22
23 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
24 sizeof(struct fw_ofld_tx_data_wr))
25
26 static struct sk_buff *
27 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
28 {
29 struct sk_buff *skb = NULL;
30 u8 submode = 0;
31 int errcode;
32 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
33
34 if (len) {
35 skb = alloc_skb_with_frags(hdr_len, len,
36 0, &errcode,
37 GFP_KERNEL);
38 if (!skb)
39 return NULL;
40
41 skb_reserve(skb, TX_HDR_LEN);
42 skb_reset_transport_header(skb);
43 __skb_put(skb, ISCSI_HDR_LEN);
44 skb->data_len = len;
45 skb->len += len;
46 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
47
48 } else {
49 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
50
51 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
52 if (!skb)
53 return NULL;
54
55 skb_reserve(skb, TX_HDR_LEN + iso_len);
56 skb_reset_transport_header(skb);
57 __skb_put(skb, ISCSI_HDR_LEN);
58 }
59
60 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
61 cxgbit_skcb_submode(skb) = submode;
62 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
63 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
64 return skb;
65 }
66
67 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
68 {
69 return __cxgbit_alloc_skb(csk, len, false);
70 }
71
72 /*
73 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
74 * @skb: the packet
75 *
76 * Returns true if a packet can be sent as an offload WR with immediate
77 * data. We currently use the same limit as for Ethernet packets.
78 */
79 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
80 {
81 int length = skb->len;
82
83 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
84 length += sizeof(struct fw_ofld_tx_data_wr);
85
86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
87 length += sizeof(struct cpl_tx_data_iso);
88
89 #define MAX_IMM_TX_PKT_LEN 256
90 return length <= MAX_IMM_TX_PKT_LEN;
91 }
92
93 /*
94 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
95 * @n: the number of SGL entries
96 * Calculates the number of flits needed for a scatter/gather list that
97 * can hold the given number of entries.
98 */
99 static inline unsigned int cxgbit_sgl_len(unsigned int n)
100 {
101 n--;
102 return (3 * n) / 2 + (n & 1) + 2;
103 }
104
105 /*
106 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
107 * @skb: the packet
108 *
109 * Returns the number of flits needed for the given offload packet.
110 * These packets are already fully constructed and no additional headers
111 * will be added.
112 */
113 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
114 {
115 unsigned int flits, cnt;
116
117 if (cxgbit_is_ofld_imm(skb))
118 return DIV_ROUND_UP(skb->len, 8);
119 flits = skb_transport_offset(skb) / 8;
120 cnt = skb_shinfo(skb)->nr_frags;
121 if (skb_tail_pointer(skb) != skb_transport_header(skb))
122 cnt++;
123 return flits + cxgbit_sgl_len(cnt);
124 }
125
126 #define CXGBIT_ISO_FSLICE 0x1
127 #define CXGBIT_ISO_LSLICE 0x2
128 static void
129 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
130 {
131 struct cpl_tx_data_iso *cpl;
132 unsigned int submode = cxgbit_skcb_submode(skb);
133 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
134 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
135
136 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl));
137
138 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
139 CPL_TX_DATA_ISO_FIRST_V(fslice) |
140 CPL_TX_DATA_ISO_LAST_V(lslice) |
141 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
142 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
143 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
144 CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
145 CPL_TX_DATA_ISO_SCSI_V(2));
146
147 cpl->ahs_len = 0;
148 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
149 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
150 cpl->len = htonl(iso_info->len);
151 cpl->reserved2_seglen_offset = htonl(0);
152 cpl->datasn_offset = htonl(0);
153 cpl->buffer_offset = htonl(0);
154 cpl->reserved3 = 0;
155
156 __skb_pull(skb, sizeof(*cpl));
157 }
158
159 static void
160 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
161 u32 len, u32 credits, u32 compl)
162 {
163 struct fw_ofld_tx_data_wr *req;
164 u32 submode = cxgbit_skcb_submode(skb);
165 u32 wr_ulp_mode = 0;
166 u32 hdr_size = sizeof(*req);
167 u32 opcode = FW_OFLD_TX_DATA_WR;
168 u32 immlen = 0;
169 u32 force = TX_FORCE_V(!submode);
170
171 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
172 opcode = FW_ISCSI_TX_DATA_WR;
173 immlen += sizeof(struct cpl_tx_data_iso);
174 hdr_size += sizeof(struct cpl_tx_data_iso);
175 submode |= 8;
176 }
177
178 if (cxgbit_is_ofld_imm(skb))
179 immlen += dlen;
180
181 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb,
182 hdr_size);
183 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
184 FW_WR_COMPL_V(compl) |
185 FW_WR_IMMDLEN_V(immlen));
186 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
187 FW_WR_LEN16_V(credits));
188 req->plen = htonl(len);
189 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
190 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
191
192 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
193 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
194 }
195
196 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
197 {
198 kfree_skb(skb);
199 }
200
201 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
202 {
203 struct sk_buff *skb;
204
205 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
206 u32 dlen = skb->len;
207 u32 len = skb->len;
208 u32 credits_needed;
209 u32 compl = 0;
210 u32 flowclen16 = 0;
211 u32 iso_cpl_len = 0;
212
213 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
214 iso_cpl_len = sizeof(struct cpl_tx_data_iso);
215
216 if (cxgbit_is_ofld_imm(skb))
217 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
218 else
219 credits_needed = DIV_ROUND_UP((8 *
220 cxgbit_calc_tx_flits_ofld(skb)) +
221 iso_cpl_len, 16);
222
223 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
224 credits_needed += DIV_ROUND_UP(
225 sizeof(struct fw_ofld_tx_data_wr), 16);
226 /*
227 * Assumes the initial credits is large enough to support
228 * fw_flowc_wr plus largest possible first payload
229 */
230
231 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
232 flowclen16 = cxgbit_send_tx_flowc_wr(csk);
233 csk->wr_cred -= flowclen16;
234 csk->wr_una_cred += flowclen16;
235 }
236
237 if (csk->wr_cred < credits_needed) {
238 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
239 csk, skb->len, skb->data_len,
240 credits_needed, csk->wr_cred);
241 break;
242 }
243 __skb_unlink(skb, &csk->txq);
244 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
245 skb->csum = credits_needed + flowclen16;
246 csk->wr_cred -= credits_needed;
247 csk->wr_una_cred += credits_needed;
248
249 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
250 csk, skb->len, skb->data_len, credits_needed,
251 csk->wr_cred, csk->wr_una_cred);
252
253 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
254 len += cxgbit_skcb_tx_extralen(skb);
255
256 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
257 (!before(csk->write_seq,
258 csk->snd_una + csk->snd_win))) {
259 compl = 1;
260 csk->wr_una_cred = 0;
261 }
262
263 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
264 compl);
265 csk->snd_nxt += len;
266
267 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
268 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
269 struct cpl_close_con_req *req =
270 (struct cpl_close_con_req *)skb->data;
271 req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
272 csk->wr_una_cred = 0;
273 }
274
275 cxgbit_sock_enqueue_wr(csk, skb);
276 t4_set_arp_err_handler(skb, csk,
277 cxgbit_arp_failure_skb_discard);
278
279 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
280 csk, csk->tid, skb, len);
281
282 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
283 }
284 }
285
286 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
287 {
288 spin_lock_bh(&csk->lock);
289
290 if (before(csk->write_seq, csk->snd_una + csk->snd_win))
291 csk->lock_owner = true;
292
293 spin_unlock_bh(&csk->lock);
294
295 return csk->lock_owner;
296 }
297
298 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
299 {
300 struct sk_buff_head backlogq;
301 struct sk_buff *skb;
302 void (*fn)(struct cxgbit_sock *, struct sk_buff *);
303
304 skb_queue_head_init(&backlogq);
305
306 spin_lock_bh(&csk->lock);
307 while (skb_queue_len(&csk->backlogq)) {
308 skb_queue_splice_init(&csk->backlogq, &backlogq);
309 spin_unlock_bh(&csk->lock);
310
311 while ((skb = __skb_dequeue(&backlogq))) {
312 fn = cxgbit_skcb_rx_backlog_fn(skb);
313 fn(csk, skb);
314 }
315
316 spin_lock_bh(&csk->lock);
317 }
318
319 csk->lock_owner = false;
320 spin_unlock_bh(&csk->lock);
321 }
322
323 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
324 {
325 int ret = 0;
326
327 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
328
329 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
330 signal_pending(current))) {
331 __kfree_skb(skb);
332 __skb_queue_purge(&csk->ppodq);
333 ret = -1;
334 spin_lock_bh(&csk->lock);
335 if (csk->lock_owner) {
336 spin_unlock_bh(&csk->lock);
337 goto unlock;
338 }
339 spin_unlock_bh(&csk->lock);
340 return ret;
341 }
342
343 csk->write_seq += skb->len +
344 cxgbit_skcb_tx_extralen(skb);
345
346 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
347 __skb_queue_tail(&csk->txq, skb);
348 cxgbit_push_tx_frames(csk);
349
350 unlock:
351 cxgbit_unlock_sock(csk);
352 return ret;
353 }
354
355 static int
356 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
357 u32 data_length)
358 {
359 u32 i = 0, nr_frags = MAX_SKB_FRAGS;
360 u32 padding = ((-data_length) & 3);
361 struct scatterlist *sg;
362 struct page *page;
363 unsigned int page_off;
364
365 if (padding)
366 nr_frags--;
367
368 /*
369 * We know each entry in t_data_sg contains a page.
370 */
371 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
372 page_off = (data_offset % PAGE_SIZE);
373
374 while (data_length && (i < nr_frags)) {
375 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
376
377 page = sg_page(sg);
378
379 get_page(page);
380 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
381 cur_len);
382 skb->data_len += cur_len;
383 skb->len += cur_len;
384 skb->truesize += cur_len;
385
386 data_length -= cur_len;
387 page_off = 0;
388 sg = sg_next(sg);
389 i++;
390 }
391
392 if (data_length)
393 return -1;
394
395 if (padding) {
396 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
397 if (!page)
398 return -1;
399 skb_fill_page_desc(skb, i, page, 0, padding);
400 skb->data_len += padding;
401 skb->len += padding;
402 skb->truesize += padding;
403 }
404
405 return 0;
406 }
407
408 static int
409 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
410 struct iscsi_datain_req *dr)
411 {
412 struct iscsi_conn *conn = csk->conn;
413 struct sk_buff *skb;
414 struct iscsi_datain datain;
415 struct cxgbit_iso_info iso_info;
416 u32 data_length = cmd->se_cmd.data_length;
417 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
418 u32 num_pdu, plen, tx_data = 0;
419 bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
420 SCF_TRANSPORT_TASK_SENSE);
421 bool set_statsn = false;
422 int ret = -1;
423
424 while (data_length) {
425 num_pdu = (data_length + mrdsl - 1) / mrdsl;
426 if (num_pdu > csk->max_iso_npdu)
427 num_pdu = csk->max_iso_npdu;
428
429 plen = num_pdu * mrdsl;
430 if (plen > data_length)
431 plen = data_length;
432
433 skb = __cxgbit_alloc_skb(csk, 0, true);
434 if (unlikely(!skb))
435 return -ENOMEM;
436
437 memset(skb->data, 0, ISCSI_HDR_LEN);
438 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
439 cxgbit_skcb_submode(skb) |= (csk->submode &
440 CXGBIT_SUBMODE_DCRC);
441 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
442 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
443 ((num_pdu - 1) * ISCSI_HDR_LEN);
444
445 memset(&datain, 0, sizeof(struct iscsi_datain));
446 memset(&iso_info, 0, sizeof(iso_info));
447
448 if (!tx_data)
449 iso_info.flags |= CXGBIT_ISO_FSLICE;
450
451 if (!(data_length - plen)) {
452 iso_info.flags |= CXGBIT_ISO_LSLICE;
453 if (!task_sense) {
454 datain.flags = ISCSI_FLAG_DATA_STATUS;
455 iscsit_increment_maxcmdsn(cmd, conn->sess);
456 cmd->stat_sn = conn->stat_sn++;
457 set_statsn = true;
458 }
459 }
460
461 iso_info.burst_len = num_pdu * mrdsl;
462 iso_info.mpdu = mrdsl;
463 iso_info.len = ISCSI_HDR_LEN + plen;
464
465 cxgbit_cpl_tx_data_iso(skb, &iso_info);
466
467 datain.offset = tx_data;
468 datain.data_sn = cmd->data_sn - 1;
469
470 iscsit_build_datain_pdu(cmd, conn, &datain,
471 (struct iscsi_data_rsp *)skb->data,
472 set_statsn);
473
474 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
475 if (unlikely(ret)) {
476 __kfree_skb(skb);
477 goto out;
478 }
479
480 ret = cxgbit_queue_skb(csk, skb);
481 if (unlikely(ret))
482 goto out;
483
484 tx_data += plen;
485 data_length -= plen;
486
487 cmd->read_data_done += plen;
488 cmd->data_sn += num_pdu;
489 }
490
491 dr->dr_complete = DATAIN_COMPLETE_NORMAL;
492
493 return 0;
494
495 out:
496 return ret;
497 }
498
499 static int
500 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
501 const struct iscsi_datain *datain)
502 {
503 struct sk_buff *skb;
504 int ret = 0;
505
506 skb = cxgbit_alloc_skb(csk, 0);
507 if (unlikely(!skb))
508 return -ENOMEM;
509
510 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
511
512 if (datain->length) {
513 cxgbit_skcb_submode(skb) |= (csk->submode &
514 CXGBIT_SUBMODE_DCRC);
515 cxgbit_skcb_tx_extralen(skb) =
516 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
517 }
518
519 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
520 if (ret < 0) {
521 __kfree_skb(skb);
522 return ret;
523 }
524
525 return cxgbit_queue_skb(csk, skb);
526 }
527
528 static int
529 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
530 struct iscsi_datain_req *dr,
531 const struct iscsi_datain *datain)
532 {
533 struct cxgbit_sock *csk = conn->context;
534 u32 data_length = cmd->se_cmd.data_length;
535 u32 padding = ((-data_length) & 3);
536 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
537
538 if ((data_length > mrdsl) && (!dr->recovery) &&
539 (!padding) && (!datain->offset) && csk->max_iso_npdu) {
540 atomic_long_add(data_length - datain->length,
541 &conn->sess->tx_data_octets);
542 return cxgbit_tx_datain_iso(csk, cmd, dr);
543 }
544
545 return cxgbit_tx_datain(csk, cmd, datain);
546 }
547
548 static int
549 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
550 const void *data_buf, u32 data_buf_len)
551 {
552 struct cxgbit_sock *csk = conn->context;
553 struct sk_buff *skb;
554 u32 padding = ((-data_buf_len) & 3);
555
556 skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
557 if (unlikely(!skb))
558 return -ENOMEM;
559
560 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
561
562 if (data_buf_len) {
563 u32 pad_bytes = 0;
564
565 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
566
567 if (padding)
568 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
569 &pad_bytes, padding);
570 }
571
572 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
573 cxgbit_skcb_submode(skb)];
574
575 return cxgbit_queue_skb(csk, skb);
576 }
577
578 int
579 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
580 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
581 {
582 if (dr)
583 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
584 else
585 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
586 }
587
588 int cxgbit_validate_params(struct iscsi_conn *conn)
589 {
590 struct cxgbit_sock *csk = conn->context;
591 struct cxgbit_device *cdev = csk->com.cdev;
592 struct iscsi_param *param;
593 u32 max_xmitdsl;
594
595 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
596 conn->param_list);
597 if (!param)
598 return -1;
599
600 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
601 return -1;
602
603 if (max_xmitdsl > cdev->mdsl) {
604 if (iscsi_change_param_sprintf(
605 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
606 return -1;
607 }
608
609 return 0;
610 }
611
612 static int cxgbit_set_digest(struct cxgbit_sock *csk)
613 {
614 struct iscsi_conn *conn = csk->conn;
615 struct iscsi_param *param;
616
617 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
618 if (!param) {
619 pr_err("param not found key %s\n", HEADERDIGEST);
620 return -1;
621 }
622
623 if (!strcmp(param->value, CRC32C))
624 csk->submode |= CXGBIT_SUBMODE_HCRC;
625
626 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
627 if (!param) {
628 csk->submode = 0;
629 pr_err("param not found key %s\n", DATADIGEST);
630 return -1;
631 }
632
633 if (!strcmp(param->value, CRC32C))
634 csk->submode |= CXGBIT_SUBMODE_DCRC;
635
636 if (cxgbit_setup_conn_digest(csk)) {
637 csk->submode = 0;
638 return -1;
639 }
640
641 return 0;
642 }
643
644 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
645 {
646 struct iscsi_conn *conn = csk->conn;
647 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
648 struct iscsi_param *param;
649 u32 mrdsl, mbl;
650 u32 max_npdu, max_iso_npdu;
651
652 if (conn->login->leading_connection) {
653 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
654 conn->param_list);
655 if (!param) {
656 pr_err("param not found key %s\n", DATASEQUENCEINORDER);
657 return -1;
658 }
659
660 if (strcmp(param->value, YES))
661 return 0;
662
663 param = iscsi_find_param_from_key(DATAPDUINORDER,
664 conn->param_list);
665 if (!param) {
666 pr_err("param not found key %s\n", DATAPDUINORDER);
667 return -1;
668 }
669
670 if (strcmp(param->value, YES))
671 return 0;
672
673 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
674 conn->param_list);
675 if (!param) {
676 pr_err("param not found key %s\n", MAXBURSTLENGTH);
677 return -1;
678 }
679
680 if (kstrtou32(param->value, 0, &mbl) < 0)
681 return -1;
682 } else {
683 if (!conn->sess->sess_ops->DataSequenceInOrder)
684 return 0;
685 if (!conn->sess->sess_ops->DataPDUInOrder)
686 return 0;
687
688 mbl = conn->sess->sess_ops->MaxBurstLength;
689 }
690
691 mrdsl = conn_ops->MaxRecvDataSegmentLength;
692 max_npdu = mbl / mrdsl;
693
694 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD /
695 (ISCSI_HDR_LEN + mrdsl +
696 cxgbit_digest_len[csk->submode]);
697
698 csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
699
700 if (csk->max_iso_npdu <= 1)
701 csk->max_iso_npdu = 0;
702
703 return 0;
704 }
705
706 static int cxgbit_set_params(struct iscsi_conn *conn)
707 {
708 struct cxgbit_sock *csk = conn->context;
709 struct cxgbit_device *cdev = csk->com.cdev;
710 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
711 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
712 struct iscsi_param *param;
713 u8 erl;
714
715 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
716 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
717
718 if (conn->login->leading_connection) {
719 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
720 conn->param_list);
721 if (!param) {
722 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
723 return -1;
724 }
725 if (kstrtou8(param->value, 0, &erl) < 0)
726 return -1;
727 } else {
728 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
729 }
730
731 if (!erl) {
732 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
733 if (cxgbit_set_iso_npdu(csk))
734 return -1;
735 }
736
737 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
738 if (cxgbit_setup_conn_pgidx(csk,
739 ppm->tformat.pgsz_idx_dflt))
740 return -1;
741 set_bit(CSK_DDP_ENABLE, &csk->com.flags);
742 }
743 }
744
745 if (cxgbit_set_digest(csk))
746 return -1;
747
748 return 0;
749 }
750
751 int
752 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
753 u32 length)
754 {
755 struct cxgbit_sock *csk = conn->context;
756 struct sk_buff *skb;
757 u32 padding_buf = 0;
758 u8 padding = ((-length) & 3);
759
760 skb = cxgbit_alloc_skb(csk, length + padding);
761 if (!skb)
762 return -ENOMEM;
763 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
764 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
765
766 if (padding)
767 skb_store_bits(skb, ISCSI_HDR_LEN + length,
768 &padding_buf, padding);
769
770 if (login->login_complete) {
771 if (cxgbit_set_params(conn)) {
772 kfree_skb(skb);
773 return -1;
774 }
775
776 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
777 }
778
779 if (cxgbit_queue_skb(csk, skb))
780 return -1;
781
782 if ((!login->login_complete) && (!login->login_failed))
783 schedule_delayed_work(&conn->login_work, 0);
784
785 return 0;
786 }
787
788 static void
789 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
790 unsigned int nents)
791 {
792 struct skb_seq_state st;
793 const u8 *buf;
794 unsigned int consumed = 0, buf_len;
795 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
796
797 skb_prepare_seq_read(skb, pdu_cb->doffset,
798 pdu_cb->doffset + pdu_cb->dlen,
799 &st);
800
801 while (true) {
802 buf_len = skb_seq_read(consumed, &buf, &st);
803 if (!buf_len) {
804 skb_abort_seq_read(&st);
805 break;
806 }
807
808 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
809 buf_len, consumed);
810 }
811 }
812
813 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
814 {
815 struct iscsi_conn *conn = csk->conn;
816 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
817 struct cxgbit_cmd *ccmd;
818 struct iscsi_cmd *cmd;
819
820 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
821 if (!cmd) {
822 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
823 return NULL;
824 }
825
826 ccmd = iscsit_priv_cmd(cmd);
827 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
828 ccmd->setup_ddp = true;
829
830 return cmd;
831 }
832
833 static int
834 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
835 u32 length)
836 {
837 struct iscsi_conn *conn = cmd->conn;
838 struct cxgbit_sock *csk = conn->context;
839 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
840
841 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
842 pr_err("ImmediateData CRC32C DataDigest error\n");
843 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
844 pr_err("Unable to recover from"
845 " Immediate Data digest failure while"
846 " in ERL=0.\n");
847 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
848 (unsigned char *)hdr);
849 return IMMEDIATE_DATA_CANNOT_RECOVER;
850 }
851
852 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
853 (unsigned char *)hdr);
854 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
855 }
856
857 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
858 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
859 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
860 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
861
862 sg_init_table(&ccmd->sg, 1);
863 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
864 dfrag->page_offset);
865 get_page(dfrag->page.p);
866
867 cmd->se_cmd.t_data_sg = &ccmd->sg;
868 cmd->se_cmd.t_data_nents = 1;
869
870 ccmd->release = true;
871 } else {
872 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
873 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
874
875 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents);
876 }
877
878 cmd->write_data_done += pdu_cb->dlen;
879
880 if (cmd->write_data_done == cmd->se_cmd.data_length) {
881 spin_lock_bh(&cmd->istate_lock);
882 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
883 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
884 spin_unlock_bh(&cmd->istate_lock);
885 }
886
887 return IMMEDIATE_DATA_NORMAL_OPERATION;
888 }
889
890 static int
891 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
892 bool dump_payload)
893 {
894 struct iscsi_conn *conn = cmd->conn;
895 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
896 /*
897 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
898 */
899 if (dump_payload)
900 goto after_immediate_data;
901
902 immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
903 cmd->first_burst_len);
904 after_immediate_data:
905 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
906 /*
907 * A PDU/CmdSN carrying Immediate Data passed
908 * DataCRC, check against ExpCmdSN/MaxCmdSN if
909 * Immediate Bit is not set.
910 */
911 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
912 (unsigned char *)hdr,
913 hdr->cmdsn);
914 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
915 return -1;
916
917 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
918 target_put_sess_cmd(&cmd->se_cmd);
919 return 0;
920 } else if (cmd->unsolicited_data) {
921 iscsit_set_unsoliticed_dataout(cmd);
922 }
923
924 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
925 /*
926 * Immediate Data failed DataCRC and ERL>=1,
927 * silently drop this PDU and let the initiator
928 * plug the CmdSN gap.
929 *
930 * FIXME: Send Unsolicited NOPIN with reserved
931 * TTT here to help the initiator figure out
932 * the missing CmdSN, although they should be
933 * intelligent enough to determine the missing
934 * CmdSN and issue a retry to plug the sequence.
935 */
936 cmd->i_state = ISTATE_REMOVE;
937 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
938 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
939 return -1;
940
941 return 0;
942 }
943
944 static int
945 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
946 {
947 struct iscsi_conn *conn = csk->conn;
948 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
949 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
950 int rc;
951 bool dump_payload = false;
952
953 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
954 if (rc < 0)
955 return rc;
956
957 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
958 (pdu_cb->nr_dfrags == 1))
959 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
960
961 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
962 if (rc < 0)
963 return 0;
964 else if (rc > 0)
965 dump_payload = true;
966
967 if (!pdu_cb->dlen)
968 return 0;
969
970 return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
971 }
972
973 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
974 {
975 struct scatterlist *sg_start;
976 struct iscsi_conn *conn = csk->conn;
977 struct iscsi_cmd *cmd = NULL;
978 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
979 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
980 u32 data_offset = be32_to_cpu(hdr->offset);
981 u32 data_len = pdu_cb->dlen;
982 int rc, sg_nents, sg_off;
983 bool dcrc_err = false;
984
985 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
986 if (rc < 0)
987 return rc;
988 else if (!cmd)
989 return 0;
990
991 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
992 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
993 " DataSN: 0x%08x\n",
994 hdr->itt, hdr->offset, data_len,
995 hdr->datasn);
996
997 dcrc_err = true;
998 goto check_payload;
999 }
1000
1001 pr_debug("DataOut data_len: %u, "
1002 "write_data_done: %u, data_length: %u\n",
1003 data_len, cmd->write_data_done,
1004 cmd->se_cmd.data_length);
1005
1006 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1007 sg_off = data_offset / PAGE_SIZE;
1008 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1009 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE));
1010
1011 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents);
1012 }
1013
1014 check_payload:
1015
1016 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1017 if (rc < 0)
1018 return rc;
1019
1020 return 0;
1021 }
1022
1023 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1024 {
1025 struct iscsi_conn *conn = csk->conn;
1026 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1027 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1028 unsigned char *ping_data = NULL;
1029 u32 payload_length = pdu_cb->dlen;
1030 int ret;
1031
1032 ret = iscsit_setup_nop_out(conn, cmd, hdr);
1033 if (ret < 0)
1034 return 0;
1035
1036 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1037 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1038 pr_err("Unable to recover from"
1039 " NOPOUT Ping DataCRC failure while in"
1040 " ERL=0.\n");
1041 ret = -1;
1042 goto out;
1043 } else {
1044 /*
1045 * drop this PDU and let the
1046 * initiator plug the CmdSN gap.
1047 */
1048 pr_info("Dropping NOPOUT"
1049 " Command CmdSN: 0x%08x due to"
1050 " DataCRC error.\n", hdr->cmdsn);
1051 ret = 0;
1052 goto out;
1053 }
1054 }
1055
1056 /*
1057 * Handle NOP-OUT payload for traditional iSCSI sockets
1058 */
1059 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1060 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1061 if (!ping_data) {
1062 pr_err("Unable to allocate memory for"
1063 " NOPOUT ping data.\n");
1064 ret = -1;
1065 goto out;
1066 }
1067
1068 skb_copy_bits(csk->skb, pdu_cb->doffset,
1069 ping_data, payload_length);
1070
1071 ping_data[payload_length] = '\0';
1072 /*
1073 * Attach ping data to struct iscsi_cmd->buf_ptr.
1074 */
1075 cmd->buf_ptr = ping_data;
1076 cmd->buf_ptr_size = payload_length;
1077
1078 pr_debug("Got %u bytes of NOPOUT ping"
1079 " data.\n", payload_length);
1080 pr_debug("Ping Data: \"%s\"\n", ping_data);
1081 }
1082
1083 return iscsit_process_nop_out(conn, cmd, hdr);
1084 out:
1085 if (cmd)
1086 iscsit_free_cmd(cmd, false);
1087 return ret;
1088 }
1089
1090 static int
1091 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1092 {
1093 struct iscsi_conn *conn = csk->conn;
1094 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1095 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1096 u32 payload_length = pdu_cb->dlen;
1097 int rc;
1098 unsigned char *text_in = NULL;
1099
1100 rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1101 if (rc < 0)
1102 return rc;
1103
1104 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1105 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1106 pr_err("Unable to recover from"
1107 " Text Data digest failure while in"
1108 " ERL=0.\n");
1109 goto reject;
1110 } else {
1111 /*
1112 * drop this PDU and let the
1113 * initiator plug the CmdSN gap.
1114 */
1115 pr_info("Dropping Text"
1116 " Command CmdSN: 0x%08x due to"
1117 " DataCRC error.\n", hdr->cmdsn);
1118 return 0;
1119 }
1120 }
1121
1122 if (payload_length) {
1123 text_in = kzalloc(payload_length, GFP_KERNEL);
1124 if (!text_in) {
1125 pr_err("Unable to allocate text_in of payload_length: %u\n",
1126 payload_length);
1127 return -ENOMEM;
1128 }
1129 skb_copy_bits(csk->skb, pdu_cb->doffset,
1130 text_in, payload_length);
1131
1132 text_in[payload_length - 1] = '\0';
1133
1134 cmd->text_in_ptr = text_in;
1135 }
1136
1137 return iscsit_process_text_cmd(conn, cmd, hdr);
1138
1139 reject:
1140 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1141 pdu_cb->hdr);
1142 }
1143
1144 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1145 {
1146 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1147 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1148 struct iscsi_conn *conn = csk->conn;
1149 struct iscsi_cmd *cmd = NULL;
1150 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1151 int ret = -EINVAL;
1152
1153 switch (opcode) {
1154 case ISCSI_OP_SCSI_CMD:
1155 cmd = cxgbit_allocate_cmd(csk);
1156 if (!cmd)
1157 goto reject;
1158
1159 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1160 break;
1161 case ISCSI_OP_SCSI_DATA_OUT:
1162 ret = cxgbit_handle_iscsi_dataout(csk);
1163 break;
1164 case ISCSI_OP_NOOP_OUT:
1165 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1166 cmd = cxgbit_allocate_cmd(csk);
1167 if (!cmd)
1168 goto reject;
1169 }
1170
1171 ret = cxgbit_handle_nop_out(csk, cmd);
1172 break;
1173 case ISCSI_OP_SCSI_TMFUNC:
1174 cmd = cxgbit_allocate_cmd(csk);
1175 if (!cmd)
1176 goto reject;
1177
1178 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1179 (unsigned char *)hdr);
1180 break;
1181 case ISCSI_OP_TEXT:
1182 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1183 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1184 if (!cmd)
1185 goto reject;
1186 } else {
1187 cmd = cxgbit_allocate_cmd(csk);
1188 if (!cmd)
1189 goto reject;
1190 }
1191
1192 ret = cxgbit_handle_text_cmd(csk, cmd);
1193 break;
1194 case ISCSI_OP_LOGOUT:
1195 cmd = cxgbit_allocate_cmd(csk);
1196 if (!cmd)
1197 goto reject;
1198
1199 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1200 if (ret > 0)
1201 wait_for_completion_timeout(&conn->conn_logout_comp,
1202 SECONDS_FOR_LOGOUT_COMP
1203 * HZ);
1204 break;
1205 case ISCSI_OP_SNACK:
1206 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1207 break;
1208 default:
1209 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1210 dump_stack();
1211 break;
1212 }
1213
1214 return ret;
1215
1216 reject:
1217 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1218 (unsigned char *)hdr);
1219 return ret;
1220 }
1221
1222 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1223 {
1224 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1225 struct iscsi_conn *conn = csk->conn;
1226 struct iscsi_hdr *hdr = pdu_cb->hdr;
1227 u8 opcode;
1228
1229 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1230 atomic_long_inc(&conn->sess->conn_digest_errors);
1231 goto transport_err;
1232 }
1233
1234 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1235 goto transport_err;
1236
1237 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1238
1239 if (conn->sess->sess_ops->SessionType &&
1240 ((!(opcode & ISCSI_OP_TEXT)) ||
1241 (!(opcode & ISCSI_OP_LOGOUT)))) {
1242 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1243 " while in Discovery Session, rejecting.\n", opcode);
1244 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1245 (unsigned char *)hdr);
1246 goto transport_err;
1247 }
1248
1249 if (cxgbit_target_rx_opcode(csk) < 0)
1250 goto transport_err;
1251
1252 return 0;
1253
1254 transport_err:
1255 return -1;
1256 }
1257
1258 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1259 {
1260 struct iscsi_conn *conn = csk->conn;
1261 struct iscsi_login *login = conn->login;
1262 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1263 struct iscsi_login_req *login_req;
1264
1265 login_req = (struct iscsi_login_req *)login->req;
1266 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1267
1268 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1269 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1270 login_req->flags, login_req->itt, login_req->cmdsn,
1271 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1272 /*
1273 * Setup the initial iscsi_login values from the leading
1274 * login request PDU.
1275 */
1276 if (login->first_request) {
1277 login_req = (struct iscsi_login_req *)login->req;
1278 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1279 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
1280 login_req->flags);
1281 login->version_min = login_req->min_version;
1282 login->version_max = login_req->max_version;
1283 memcpy(login->isid, login_req->isid, 6);
1284 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1285 login->init_task_tag = login_req->itt;
1286 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1287 login->cid = be16_to_cpu(login_req->cid);
1288 login->tsih = be16_to_cpu(login_req->tsih);
1289 }
1290
1291 if (iscsi_target_check_login_request(conn, login) < 0)
1292 return -1;
1293
1294 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1295 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1296
1297 return 0;
1298 }
1299
1300 static int
1301 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1302 {
1303 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1304 int ret;
1305
1306 cxgbit_rx_pdu_cb(skb) = pdu_cb;
1307
1308 csk->skb = skb;
1309
1310 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1311 ret = cxgbit_rx_login_pdu(csk);
1312 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1313 } else {
1314 ret = cxgbit_rx_opcode(csk);
1315 }
1316
1317 return ret;
1318 }
1319
1320 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1321 {
1322 struct skb_shared_info *ssi = skb_shinfo(skb);
1323 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1324 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1325 u8 i;
1326
1327 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1328 skb, skb->head, skb->data, skb->len, skb->data_len,
1329 ssi->nr_frags);
1330 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1331 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1332
1333 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1334 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1335 "frags %u.\n",
1336 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1337 pdu_cb->ddigest, pdu_cb->frags);
1338 for (i = 0; i < ssi->nr_frags; i++)
1339 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1340 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1341 }
1342
1343 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1344 {
1345 struct sk_buff *skb = csk->lro_hskb;
1346 struct skb_shared_info *ssi = skb_shinfo(skb);
1347 u8 i;
1348
1349 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1350 for (i = 0; i < ssi->nr_frags; i++)
1351 put_page(skb_frag_page(&ssi->frags[i]));
1352 ssi->nr_frags = 0;
1353 }
1354
1355 static void
1356 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1357 {
1358 struct sk_buff *hskb = csk->lro_hskb;
1359 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1360 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1361 struct skb_shared_info *hssi = skb_shinfo(hskb);
1362 struct skb_shared_info *ssi = skb_shinfo(skb);
1363 unsigned int len = 0;
1364
1365 if (pdu_cb->flags & PDUCBF_RX_HDR) {
1366 hpdu_cb->flags = pdu_cb->flags;
1367 hpdu_cb->seq = pdu_cb->seq;
1368 hpdu_cb->hdr = pdu_cb->hdr;
1369 hpdu_cb->hlen = pdu_cb->hlen;
1370
1371 memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx],
1372 sizeof(skb_frag_t));
1373
1374 get_page(skb_frag_page(&hssi->frags[0]));
1375 hssi->nr_frags = 1;
1376 hpdu_cb->frags = 1;
1377 hpdu_cb->hfrag_idx = 0;
1378
1379 len = hssi->frags[0].size;
1380 hskb->len = len;
1381 hskb->data_len = len;
1382 hskb->truesize = len;
1383 }
1384
1385 if (pdu_cb->flags & PDUCBF_RX_DATA) {
1386 u8 hfrag_idx = 1, i;
1387
1388 hpdu_cb->flags |= pdu_cb->flags;
1389
1390 len = 0;
1391 for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) {
1392 memcpy(&hssi->frags[hfrag_idx],
1393 &ssi->frags[pdu_cb->dfrag_idx + i],
1394 sizeof(skb_frag_t));
1395
1396 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1397
1398 len += hssi->frags[hfrag_idx].size;
1399
1400 hssi->nr_frags++;
1401 hpdu_cb->frags++;
1402 }
1403
1404 hpdu_cb->dlen = pdu_cb->dlen;
1405 hpdu_cb->doffset = hpdu_cb->hlen;
1406 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1407 hpdu_cb->dfrag_idx = 1;
1408 hskb->len += len;
1409 hskb->data_len += len;
1410 hskb->truesize += len;
1411 }
1412
1413 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1414 hpdu_cb->flags |= pdu_cb->flags;
1415
1416 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1417 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1418
1419 hpdu_cb->ddigest = pdu_cb->ddigest;
1420 hpdu_cb->pdulen = pdu_cb->pdulen;
1421 }
1422 }
1423
1424 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1425 {
1426 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1427 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1428 u8 pdu_idx = 0, last_idx = 0;
1429 int ret = 0;
1430
1431 if (!pdu_cb->complete) {
1432 cxgbit_lro_skb_merge(csk, skb, 0);
1433
1434 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1435 struct sk_buff *hskb = csk->lro_hskb;
1436
1437 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1438
1439 cxgbit_lro_hskb_reset(csk);
1440
1441 if (ret < 0)
1442 goto out;
1443 }
1444
1445 pdu_idx = 1;
1446 }
1447
1448 if (lro_cb->pdu_idx)
1449 last_idx = lro_cb->pdu_idx - 1;
1450
1451 for (; pdu_idx <= last_idx; pdu_idx++) {
1452 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1453 if (ret < 0)
1454 goto out;
1455 }
1456
1457 if ((!lro_cb->complete) && lro_cb->pdu_idx)
1458 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1459
1460 out:
1461 return ret;
1462 }
1463
1464 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1465 {
1466 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1467 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1468 int ret = -1;
1469
1470 if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1471 (pdu_cb->seq != csk->rcv_nxt)) {
1472 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1473 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1474 cxgbit_lro_skb_dump(skb);
1475 return ret;
1476 }
1477
1478 csk->rcv_nxt += lro_cb->pdu_totallen;
1479
1480 ret = cxgbit_process_lro_skb(csk, skb);
1481
1482 csk->rx_credits += lro_cb->pdu_totallen;
1483
1484 if (csk->rx_credits >= (csk->rcv_win / 4))
1485 cxgbit_rx_data_ack(csk);
1486
1487 return ret;
1488 }
1489
1490 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1491 {
1492 int ret = -1;
1493
1494 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO))
1495 ret = cxgbit_rx_lro_skb(csk, skb);
1496
1497 __kfree_skb(skb);
1498 return ret;
1499 }
1500
1501 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1502 {
1503 spin_lock_bh(&csk->rxq.lock);
1504 if (skb_queue_len(&csk->rxq)) {
1505 skb_queue_splice_init(&csk->rxq, rxq);
1506 spin_unlock_bh(&csk->rxq.lock);
1507 return true;
1508 }
1509 spin_unlock_bh(&csk->rxq.lock);
1510 return false;
1511 }
1512
1513 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1514 {
1515 struct sk_buff *skb;
1516 struct sk_buff_head rxq;
1517
1518 skb_queue_head_init(&rxq);
1519
1520 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1521
1522 if (signal_pending(current))
1523 goto out;
1524
1525 while ((skb = __skb_dequeue(&rxq))) {
1526 if (cxgbit_rx_skb(csk, skb))
1527 goto out;
1528 }
1529
1530 return 0;
1531 out:
1532 __skb_queue_purge(&rxq);
1533 return -1;
1534 }
1535
1536 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1537 {
1538 struct cxgbit_sock *csk = conn->context;
1539 int ret = -1;
1540
1541 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1542 ret = cxgbit_wait_rxq(csk);
1543 if (ret) {
1544 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1545 break;
1546 }
1547 }
1548
1549 return ret;
1550 }
1551
1552 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1553 {
1554 struct cxgbit_sock *csk = conn->context;
1555
1556 while (!kthread_should_stop()) {
1557 iscsit_thread_check_cpumask(conn, current, 0);
1558 if (cxgbit_wait_rxq(csk))
1559 return;
1560 }
1561 }
Linux with added FPC-III support