?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
sh_eth: fix EESIPR values for SH77{34|63}
[linux/fpc-iii.git] / drivers / target / iscsi / cxgbit / cxgbit_target.c
blob8bcb9b71f764325d585f659fb1430b393b54214c
1 /*
2 * Copyright (c) 2016 Chelsio Communications, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 */
8
9 #include <linux/workqueue.h>
10 #include <linux/kthread.h>
11 #include <asm/unaligned.h>
12 #include <net/tcp.h>
13 #include <target/target_core_base.h>
14 #include <target/target_core_fabric.h>
15 #include "cxgbit.h"
16
17 struct sge_opaque_hdr {
18 void *dev;
19 dma_addr_t addr[MAX_SKB_FRAGS + 1];
20 };
21
22 static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
23
24 #define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
25 sizeof(struct fw_ofld_tx_data_wr))
26
27 static struct sk_buff *
28 __cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
29 {
30 struct sk_buff *skb = NULL;
31 u8 submode = 0;
32 int errcode;
33 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
34
35 if (len) {
36 skb = alloc_skb_with_frags(hdr_len, len,
37 0, &errcode,
38 GFP_KERNEL);
39 if (!skb)
40 return NULL;
41
42 skb_reserve(skb, TX_HDR_LEN);
43 skb_reset_transport_header(skb);
44 __skb_put(skb, ISCSI_HDR_LEN);
45 skb->data_len = len;
46 skb->len += len;
47 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
48
49 } else {
50 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
51
52 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
53 if (!skb)
54 return NULL;
55
56 skb_reserve(skb, TX_HDR_LEN + iso_len);
57 skb_reset_transport_header(skb);
58 __skb_put(skb, ISCSI_HDR_LEN);
59 }
60
61 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
62 cxgbit_skcb_submode(skb) = submode;
63 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
64 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
65 return skb;
66 }
67
68 static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
69 {
70 return __cxgbit_alloc_skb(csk, len, false);
71 }
72
73 /*
74 * cxgbit_is_ofld_imm - check whether a packet can be sent as immediate data
75 * @skb: the packet
76 *
77 * Returns true if a packet can be sent as an offload WR with immediate
78 * data. We currently use the same limit as for Ethernet packets.
79 */
80 static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
81 {
82 int length = skb->len;
83
84 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
85 length += sizeof(struct fw_ofld_tx_data_wr);
86
87 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
88 length += sizeof(struct cpl_tx_data_iso);
89
90 #define MAX_IMM_TX_PKT_LEN 256
91 return length <= MAX_IMM_TX_PKT_LEN;
92 }
93
94 /*
95 * cxgbit_sgl_len - calculates the size of an SGL of the given capacity
96 * @n: the number of SGL entries
97 * Calculates the number of flits needed for a scatter/gather list that
98 * can hold the given number of entries.
99 */
100 static inline unsigned int cxgbit_sgl_len(unsigned int n)
101 {
102 n--;
103 return (3 * n) / 2 + (n & 1) + 2;
104 }
105
106 /*
107 * cxgbit_calc_tx_flits_ofld - calculate # of flits for an offload packet
108 * @skb: the packet
109 *
110 * Returns the number of flits needed for the given offload packet.
111 * These packets are already fully constructed and no additional headers
112 * will be added.
113 */
114 static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
115 {
116 unsigned int flits, cnt;
117
118 if (cxgbit_is_ofld_imm(skb))
119 return DIV_ROUND_UP(skb->len, 8);
120 flits = skb_transport_offset(skb) / 8;
121 cnt = skb_shinfo(skb)->nr_frags;
122 if (skb_tail_pointer(skb) != skb_transport_header(skb))
123 cnt++;
124 return flits + cxgbit_sgl_len(cnt);
125 }
126
127 #define CXGBIT_ISO_FSLICE 0x1
128 #define CXGBIT_ISO_LSLICE 0x2
129 static void
130 cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
131 {
132 struct cpl_tx_data_iso *cpl;
133 unsigned int submode = cxgbit_skcb_submode(skb);
134 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
135 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
136
137 cpl = (struct cpl_tx_data_iso *)__skb_push(skb, sizeof(*cpl));
138
139 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
140 CPL_TX_DATA_ISO_FIRST_V(fslice) |
141 CPL_TX_DATA_ISO_LAST_V(lslice) |
142 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
143 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
144 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
145 CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
146 CPL_TX_DATA_ISO_SCSI_V(2));
147
148 cpl->ahs_len = 0;
149 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
150 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
151 cpl->len = htonl(iso_info->len);
152 cpl->reserved2_seglen_offset = htonl(0);
153 cpl->datasn_offset = htonl(0);
154 cpl->buffer_offset = htonl(0);
155 cpl->reserved3 = 0;
156
157 __skb_pull(skb, sizeof(*cpl));
158 }
159
160 static void
161 cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
162 u32 len, u32 credits, u32 compl)
163 {
164 struct fw_ofld_tx_data_wr *req;
165 u32 submode = cxgbit_skcb_submode(skb);
166 u32 wr_ulp_mode = 0;
167 u32 hdr_size = sizeof(*req);
168 u32 opcode = FW_OFLD_TX_DATA_WR;
169 u32 immlen = 0;
170 u32 force = TX_FORCE_V(!submode);
171
172 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
173 opcode = FW_ISCSI_TX_DATA_WR;
174 immlen += sizeof(struct cpl_tx_data_iso);
175 hdr_size += sizeof(struct cpl_tx_data_iso);
176 submode |= 8;
177 }
178
179 if (cxgbit_is_ofld_imm(skb))
180 immlen += dlen;
181
182 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb,
183 hdr_size);
184 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
185 FW_WR_COMPL_V(compl) |
186 FW_WR_IMMDLEN_V(immlen));
187 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
188 FW_WR_LEN16_V(credits));
189 req->plen = htonl(len);
190 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
191 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
192
193 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
194 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
195 }
196
197 static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
198 {
199 kfree_skb(skb);
200 }
201
202 void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
203 {
204 struct sk_buff *skb;
205
206 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
207 u32 dlen = skb->len;
208 u32 len = skb->len;
209 u32 credits_needed;
210 u32 compl = 0;
211 u32 flowclen16 = 0;
212 u32 iso_cpl_len = 0;
213
214 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
215 iso_cpl_len = sizeof(struct cpl_tx_data_iso);
216
217 if (cxgbit_is_ofld_imm(skb))
218 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
219 else
220 credits_needed = DIV_ROUND_UP((8 *
221 cxgbit_calc_tx_flits_ofld(skb)) +
222 iso_cpl_len, 16);
223
224 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
225 credits_needed += DIV_ROUND_UP(
226 sizeof(struct fw_ofld_tx_data_wr), 16);
227 /*
228 * Assumes the initial credits is large enough to support
229 * fw_flowc_wr plus largest possible first payload
230 */
231
232 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
233 flowclen16 = cxgbit_send_tx_flowc_wr(csk);
234 csk->wr_cred -= flowclen16;
235 csk->wr_una_cred += flowclen16;
236 }
237
238 if (csk->wr_cred < credits_needed) {
239 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
240 csk, skb->len, skb->data_len,
241 credits_needed, csk->wr_cred);
242 break;
243 }
244 __skb_unlink(skb, &csk->txq);
245 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
246 skb->csum = credits_needed + flowclen16;
247 csk->wr_cred -= credits_needed;
248 csk->wr_una_cred += credits_needed;
249
250 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
251 csk, skb->len, skb->data_len, credits_needed,
252 csk->wr_cred, csk->wr_una_cred);
253
254 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
255 len += cxgbit_skcb_tx_extralen(skb);
256
257 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
258 (!before(csk->write_seq,
259 csk->snd_una + csk->snd_win))) {
260 compl = 1;
261 csk->wr_una_cred = 0;
262 }
263
264 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
265 compl);
266 csk->snd_nxt += len;
267
268 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
269 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
270 struct cpl_close_con_req *req =
271 (struct cpl_close_con_req *)skb->data;
272 req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
273 csk->wr_una_cred = 0;
274 }
275
276 cxgbit_sock_enqueue_wr(csk, skb);
277 t4_set_arp_err_handler(skb, csk,
278 cxgbit_arp_failure_skb_discard);
279
280 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
281 csk, csk->tid, skb, len);
282
283 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
284 }
285 }
286
287 static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
288 {
289 spin_lock_bh(&csk->lock);
290
291 if (before(csk->write_seq, csk->snd_una + csk->snd_win))
292 csk->lock_owner = true;
293
294 spin_unlock_bh(&csk->lock);
295
296 return csk->lock_owner;
297 }
298
299 static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
300 {
301 struct sk_buff_head backlogq;
302 struct sk_buff *skb;
303 void (*fn)(struct cxgbit_sock *, struct sk_buff *);
304
305 skb_queue_head_init(&backlogq);
306
307 spin_lock_bh(&csk->lock);
308 while (skb_queue_len(&csk->backlogq)) {
309 skb_queue_splice_init(&csk->backlogq, &backlogq);
310 spin_unlock_bh(&csk->lock);
311
312 while ((skb = __skb_dequeue(&backlogq))) {
313 fn = cxgbit_skcb_rx_backlog_fn(skb);
314 fn(csk, skb);
315 }
316
317 spin_lock_bh(&csk->lock);
318 }
319
320 csk->lock_owner = false;
321 spin_unlock_bh(&csk->lock);
322 }
323
324 static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
325 {
326 int ret = 0;
327
328 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
329
330 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
331 signal_pending(current))) {
332 __kfree_skb(skb);
333 __skb_queue_purge(&csk->ppodq);
334 ret = -1;
335 spin_lock_bh(&csk->lock);
336 if (csk->lock_owner) {
337 spin_unlock_bh(&csk->lock);
338 goto unlock;
339 }
340 spin_unlock_bh(&csk->lock);
341 return ret;
342 }
343
344 csk->write_seq += skb->len +
345 cxgbit_skcb_tx_extralen(skb);
346
347 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
348 __skb_queue_tail(&csk->txq, skb);
349 cxgbit_push_tx_frames(csk);
350
351 unlock:
352 cxgbit_unlock_sock(csk);
353 return ret;
354 }
355
356 static int
357 cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
358 u32 data_length)
359 {
360 u32 i = 0, nr_frags = MAX_SKB_FRAGS;
361 u32 padding = ((-data_length) & 3);
362 struct scatterlist *sg;
363 struct page *page;
364 unsigned int page_off;
365
366 if (padding)
367 nr_frags--;
368
369 /*
370 * We know each entry in t_data_sg contains a page.
371 */
372 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
373 page_off = (data_offset % PAGE_SIZE);
374
375 while (data_length && (i < nr_frags)) {
376 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
377
378 page = sg_page(sg);
379
380 get_page(page);
381 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
382 cur_len);
383 skb->data_len += cur_len;
384 skb->len += cur_len;
385 skb->truesize += cur_len;
386
387 data_length -= cur_len;
388 page_off = 0;
389 sg = sg_next(sg);
390 i++;
391 }
392
393 if (data_length)
394 return -1;
395
396 if (padding) {
397 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
398 if (!page)
399 return -1;
400 skb_fill_page_desc(skb, i, page, 0, padding);
401 skb->data_len += padding;
402 skb->len += padding;
403 skb->truesize += padding;
404 }
405
406 return 0;
407 }
408
409 static int
410 cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
411 struct iscsi_datain_req *dr)
412 {
413 struct iscsi_conn *conn = csk->conn;
414 struct sk_buff *skb;
415 struct iscsi_datain datain;
416 struct cxgbit_iso_info iso_info;
417 u32 data_length = cmd->se_cmd.data_length;
418 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
419 u32 num_pdu, plen, tx_data = 0;
420 bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
421 SCF_TRANSPORT_TASK_SENSE);
422 bool set_statsn = false;
423 int ret = -1;
424
425 while (data_length) {
426 num_pdu = (data_length + mrdsl - 1) / mrdsl;
427 if (num_pdu > csk->max_iso_npdu)
428 num_pdu = csk->max_iso_npdu;
429
430 plen = num_pdu * mrdsl;
431 if (plen > data_length)
432 plen = data_length;
433
434 skb = __cxgbit_alloc_skb(csk, 0, true);
435 if (unlikely(!skb))
436 return -ENOMEM;
437
438 memset(skb->data, 0, ISCSI_HDR_LEN);
439 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
440 cxgbit_skcb_submode(skb) |= (csk->submode &
441 CXGBIT_SUBMODE_DCRC);
442 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
443 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
444 ((num_pdu - 1) * ISCSI_HDR_LEN);
445
446 memset(&datain, 0, sizeof(struct iscsi_datain));
447 memset(&iso_info, 0, sizeof(iso_info));
448
449 if (!tx_data)
450 iso_info.flags |= CXGBIT_ISO_FSLICE;
451
452 if (!(data_length - plen)) {
453 iso_info.flags |= CXGBIT_ISO_LSLICE;
454 if (!task_sense) {
455 datain.flags = ISCSI_FLAG_DATA_STATUS;
456 iscsit_increment_maxcmdsn(cmd, conn->sess);
457 cmd->stat_sn = conn->stat_sn++;
458 set_statsn = true;
459 }
460 }
461
462 iso_info.burst_len = num_pdu * mrdsl;
463 iso_info.mpdu = mrdsl;
464 iso_info.len = ISCSI_HDR_LEN + plen;
465
466 cxgbit_cpl_tx_data_iso(skb, &iso_info);
467
468 datain.offset = tx_data;
469 datain.data_sn = cmd->data_sn - 1;
470
471 iscsit_build_datain_pdu(cmd, conn, &datain,
472 (struct iscsi_data_rsp *)skb->data,
473 set_statsn);
474
475 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
476 if (unlikely(ret)) {
477 __kfree_skb(skb);
478 goto out;
479 }
480
481 ret = cxgbit_queue_skb(csk, skb);
482 if (unlikely(ret))
483 goto out;
484
485 tx_data += plen;
486 data_length -= plen;
487
488 cmd->read_data_done += plen;
489 cmd->data_sn += num_pdu;
490 }
491
492 dr->dr_complete = DATAIN_COMPLETE_NORMAL;
493
494 return 0;
495
496 out:
497 return ret;
498 }
499
500 static int
501 cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
502 const struct iscsi_datain *datain)
503 {
504 struct sk_buff *skb;
505 int ret = 0;
506
507 skb = cxgbit_alloc_skb(csk, 0);
508 if (unlikely(!skb))
509 return -ENOMEM;
510
511 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
512
513 if (datain->length) {
514 cxgbit_skcb_submode(skb) |= (csk->submode &
515 CXGBIT_SUBMODE_DCRC);
516 cxgbit_skcb_tx_extralen(skb) =
517 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
518 }
519
520 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
521 if (ret < 0) {
522 __kfree_skb(skb);
523 return ret;
524 }
525
526 return cxgbit_queue_skb(csk, skb);
527 }
528
529 static int
530 cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
531 struct iscsi_datain_req *dr,
532 const struct iscsi_datain *datain)
533 {
534 struct cxgbit_sock *csk = conn->context;
535 u32 data_length = cmd->se_cmd.data_length;
536 u32 padding = ((-data_length) & 3);
537 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
538
539 if ((data_length > mrdsl) && (!dr->recovery) &&
540 (!padding) && (!datain->offset) && csk->max_iso_npdu) {
541 atomic_long_add(data_length - datain->length,
542 &conn->sess->tx_data_octets);
543 return cxgbit_tx_datain_iso(csk, cmd, dr);
544 }
545
546 return cxgbit_tx_datain(csk, cmd, datain);
547 }
548
549 static int
550 cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
551 const void *data_buf, u32 data_buf_len)
552 {
553 struct cxgbit_sock *csk = conn->context;
554 struct sk_buff *skb;
555 u32 padding = ((-data_buf_len) & 3);
556
557 skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
558 if (unlikely(!skb))
559 return -ENOMEM;
560
561 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
562
563 if (data_buf_len) {
564 u32 pad_bytes = 0;
565
566 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
567
568 if (padding)
569 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
570 &pad_bytes, padding);
571 }
572
573 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
574 cxgbit_skcb_submode(skb)];
575
576 return cxgbit_queue_skb(csk, skb);
577 }
578
579 int
580 cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
581 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
582 {
583 if (dr)
584 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
585 else
586 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
587 }
588
589 int cxgbit_validate_params(struct iscsi_conn *conn)
590 {
591 struct cxgbit_sock *csk = conn->context;
592 struct cxgbit_device *cdev = csk->com.cdev;
593 struct iscsi_param *param;
594 u32 max_xmitdsl;
595
596 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
597 conn->param_list);
598 if (!param)
599 return -1;
600
601 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
602 return -1;
603
604 if (max_xmitdsl > cdev->mdsl) {
605 if (iscsi_change_param_sprintf(
606 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
607 return -1;
608 }
609
610 return 0;
611 }
612
613 static int cxgbit_set_digest(struct cxgbit_sock *csk)
614 {
615 struct iscsi_conn *conn = csk->conn;
616 struct iscsi_param *param;
617
618 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
619 if (!param) {
620 pr_err("param not found key %s\n", HEADERDIGEST);
621 return -1;
622 }
623
624 if (!strcmp(param->value, CRC32C))
625 csk->submode |= CXGBIT_SUBMODE_HCRC;
626
627 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
628 if (!param) {
629 csk->submode = 0;
630 pr_err("param not found key %s\n", DATADIGEST);
631 return -1;
632 }
633
634 if (!strcmp(param->value, CRC32C))
635 csk->submode |= CXGBIT_SUBMODE_DCRC;
636
637 if (cxgbit_setup_conn_digest(csk)) {
638 csk->submode = 0;
639 return -1;
640 }
641
642 return 0;
643 }
644
645 static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
646 {
647 struct iscsi_conn *conn = csk->conn;
648 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
649 struct iscsi_param *param;
650 u32 mrdsl, mbl;
651 u32 max_npdu, max_iso_npdu;
652
653 if (conn->login->leading_connection) {
654 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
655 conn->param_list);
656 if (!param) {
657 pr_err("param not found key %s\n", DATASEQUENCEINORDER);
658 return -1;
659 }
660
661 if (strcmp(param->value, YES))
662 return 0;
663
664 param = iscsi_find_param_from_key(DATAPDUINORDER,
665 conn->param_list);
666 if (!param) {
667 pr_err("param not found key %s\n", DATAPDUINORDER);
668 return -1;
669 }
670
671 if (strcmp(param->value, YES))
672 return 0;
673
674 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
675 conn->param_list);
676 if (!param) {
677 pr_err("param not found key %s\n", MAXBURSTLENGTH);
678 return -1;
679 }
680
681 if (kstrtou32(param->value, 0, &mbl) < 0)
682 return -1;
683 } else {
684 if (!conn->sess->sess_ops->DataSequenceInOrder)
685 return 0;
686 if (!conn->sess->sess_ops->DataPDUInOrder)
687 return 0;
688
689 mbl = conn->sess->sess_ops->MaxBurstLength;
690 }
691
692 mrdsl = conn_ops->MaxRecvDataSegmentLength;
693 max_npdu = mbl / mrdsl;
694
695 max_iso_npdu = CXGBIT_MAX_ISO_PAYLOAD /
696 (ISCSI_HDR_LEN + mrdsl +
697 cxgbit_digest_len[csk->submode]);
698
699 csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
700
701 if (csk->max_iso_npdu <= 1)
702 csk->max_iso_npdu = 0;
703
704 return 0;
705 }
706
707 static int cxgbit_set_params(struct iscsi_conn *conn)
708 {
709 struct cxgbit_sock *csk = conn->context;
710 struct cxgbit_device *cdev = csk->com.cdev;
711 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
712 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
713 struct iscsi_param *param;
714 u8 erl;
715
716 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
717 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
718
719 if (conn->login->leading_connection) {
720 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
721 conn->param_list);
722 if (!param) {
723 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
724 return -1;
725 }
726 if (kstrtou8(param->value, 0, &erl) < 0)
727 return -1;
728 } else {
729 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
730 }
731
732 if (!erl) {
733 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
734 if (cxgbit_set_iso_npdu(csk))
735 return -1;
736 }
737
738 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
739 if (cxgbit_setup_conn_pgidx(csk,
740 ppm->tformat.pgsz_idx_dflt))
741 return -1;
742 set_bit(CSK_DDP_ENABLE, &csk->com.flags);
743 }
744 }
745
746 if (cxgbit_set_digest(csk))
747 return -1;
748
749 return 0;
750 }
751
752 int
753 cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
754 u32 length)
755 {
756 struct cxgbit_sock *csk = conn->context;
757 struct sk_buff *skb;
758 u32 padding_buf = 0;
759 u8 padding = ((-length) & 3);
760
761 skb = cxgbit_alloc_skb(csk, length + padding);
762 if (!skb)
763 return -ENOMEM;
764 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
765 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
766
767 if (padding)
768 skb_store_bits(skb, ISCSI_HDR_LEN + length,
769 &padding_buf, padding);
770
771 if (login->login_complete) {
772 if (cxgbit_set_params(conn)) {
773 kfree_skb(skb);
774 return -1;
775 }
776
777 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
778 }
779
780 if (cxgbit_queue_skb(csk, skb))
781 return -1;
782
783 if ((!login->login_complete) && (!login->login_failed))
784 schedule_delayed_work(&conn->login_work, 0);
785
786 return 0;
787 }
788
789 static void
790 cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
791 unsigned int nents)
792 {
793 struct skb_seq_state st;
794 const u8 *buf;
795 unsigned int consumed = 0, buf_len;
796 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
797
798 skb_prepare_seq_read(skb, pdu_cb->doffset,
799 pdu_cb->doffset + pdu_cb->dlen,
800 &st);
801
802 while (true) {
803 buf_len = skb_seq_read(consumed, &buf, &st);
804 if (!buf_len) {
805 skb_abort_seq_read(&st);
806 break;
807 }
808
809 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
810 buf_len, consumed);
811 }
812 }
813
814 static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
815 {
816 struct iscsi_conn *conn = csk->conn;
817 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
818 struct cxgbit_cmd *ccmd;
819 struct iscsi_cmd *cmd;
820
821 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
822 if (!cmd) {
823 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
824 return NULL;
825 }
826
827 ccmd = iscsit_priv_cmd(cmd);
828 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
829 ccmd->setup_ddp = true;
830
831 return cmd;
832 }
833
834 static int
835 cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
836 u32 length)
837 {
838 struct iscsi_conn *conn = cmd->conn;
839 struct cxgbit_sock *csk = conn->context;
840 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
841
842 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
843 pr_err("ImmediateData CRC32C DataDigest error\n");
844 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
845 pr_err("Unable to recover from"
846 " Immediate Data digest failure while"
847 " in ERL=0.\n");
848 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
849 (unsigned char *)hdr);
850 return IMMEDIATE_DATA_CANNOT_RECOVER;
851 }
852
853 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
854 (unsigned char *)hdr);
855 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
856 }
857
858 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
859 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
860 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
861 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
862
863 sg_init_table(&ccmd->sg, 1);
864 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
865 dfrag->page_offset);
866 get_page(dfrag->page.p);
867
868 cmd->se_cmd.t_data_sg = &ccmd->sg;
869 cmd->se_cmd.t_data_nents = 1;
870
871 ccmd->release = true;
872 } else {
873 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
874 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
875
876 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents);
877 }
878
879 cmd->write_data_done += pdu_cb->dlen;
880
881 if (cmd->write_data_done == cmd->se_cmd.data_length) {
882 spin_lock_bh(&cmd->istate_lock);
883 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
884 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
885 spin_unlock_bh(&cmd->istate_lock);
886 }
887
888 return IMMEDIATE_DATA_NORMAL_OPERATION;
889 }
890
891 static int
892 cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
893 bool dump_payload)
894 {
895 struct iscsi_conn *conn = cmd->conn;
896 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
897 /*
898 * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes.
899 */
900 if (dump_payload)
901 goto after_immediate_data;
902
903 immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
904 cmd->first_burst_len);
905 after_immediate_data:
906 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
907 /*
908 * A PDU/CmdSN carrying Immediate Data passed
909 * DataCRC, check against ExpCmdSN/MaxCmdSN if
910 * Immediate Bit is not set.
911 */
912 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
913 (unsigned char *)hdr,
914 hdr->cmdsn);
915 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
916 return -1;
917
918 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
919 target_put_sess_cmd(&cmd->se_cmd);
920 return 0;
921 } else if (cmd->unsolicited_data) {
922 iscsit_set_unsoliticed_dataout(cmd);
923 }
924
925 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
926 /*
927 * Immediate Data failed DataCRC and ERL>=1,
928 * silently drop this PDU and let the initiator
929 * plug the CmdSN gap.
930 *
931 * FIXME: Send Unsolicited NOPIN with reserved
932 * TTT here to help the initiator figure out
933 * the missing CmdSN, although they should be
934 * intelligent enough to determine the missing
935 * CmdSN and issue a retry to plug the sequence.
936 */
937 cmd->i_state = ISTATE_REMOVE;
938 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
939 } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */
940 return -1;
941
942 return 0;
943 }
944
945 static int
946 cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
947 {
948 struct iscsi_conn *conn = csk->conn;
949 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
950 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
951 int rc;
952 bool dump_payload = false;
953
954 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
955 if (rc < 0)
956 return rc;
957
958 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
959 (pdu_cb->nr_dfrags == 1))
960 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
961
962 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
963 if (rc < 0)
964 return 0;
965 else if (rc > 0)
966 dump_payload = true;
967
968 if (!pdu_cb->dlen)
969 return 0;
970
971 return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
972 }
973
974 static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
975 {
976 struct scatterlist *sg_start;
977 struct iscsi_conn *conn = csk->conn;
978 struct iscsi_cmd *cmd = NULL;
979 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
980 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
981 u32 data_offset = be32_to_cpu(hdr->offset);
982 u32 data_len = pdu_cb->dlen;
983 int rc, sg_nents, sg_off;
984 bool dcrc_err = false;
985
986 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
987 if (rc < 0)
988 return rc;
989 else if (!cmd)
990 return 0;
991
992 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
993 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
994 " DataSN: 0x%08x\n",
995 hdr->itt, hdr->offset, data_len,
996 hdr->datasn);
997
998 dcrc_err = true;
999 goto check_payload;
1000 }
1001
1002 pr_debug("DataOut data_len: %u, "
1003 "write_data_done: %u, data_length: %u\n",
1004 data_len, cmd->write_data_done,
1005 cmd->se_cmd.data_length);
1006
1007 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1008 sg_off = data_offset / PAGE_SIZE;
1009 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1010 sg_nents = max(1UL, DIV_ROUND_UP(data_len, PAGE_SIZE));
1011
1012 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents);
1013 }
1014
1015 check_payload:
1016
1017 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1018 if (rc < 0)
1019 return rc;
1020
1021 return 0;
1022 }
1023
1024 static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1025 {
1026 struct iscsi_conn *conn = csk->conn;
1027 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1028 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1029 unsigned char *ping_data = NULL;
1030 u32 payload_length = pdu_cb->dlen;
1031 int ret;
1032
1033 ret = iscsit_setup_nop_out(conn, cmd, hdr);
1034 if (ret < 0)
1035 return 0;
1036
1037 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1038 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1039 pr_err("Unable to recover from"
1040 " NOPOUT Ping DataCRC failure while in"
1041 " ERL=0.\n");
1042 ret = -1;
1043 goto out;
1044 } else {
1045 /*
1046 * drop this PDU and let the
1047 * initiator plug the CmdSN gap.
1048 */
1049 pr_info("Dropping NOPOUT"
1050 " Command CmdSN: 0x%08x due to"
1051 " DataCRC error.\n", hdr->cmdsn);
1052 ret = 0;
1053 goto out;
1054 }
1055 }
1056
1057 /*
1058 * Handle NOP-OUT payload for traditional iSCSI sockets
1059 */
1060 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1061 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1062 if (!ping_data) {
1063 pr_err("Unable to allocate memory for"
1064 " NOPOUT ping data.\n");
1065 ret = -1;
1066 goto out;
1067 }
1068
1069 skb_copy_bits(csk->skb, pdu_cb->doffset,
1070 ping_data, payload_length);
1071
1072 ping_data[payload_length] = '\0';
1073 /*
1074 * Attach ping data to struct iscsi_cmd->buf_ptr.
1075 */
1076 cmd->buf_ptr = ping_data;
1077 cmd->buf_ptr_size = payload_length;
1078
1079 pr_debug("Got %u bytes of NOPOUT ping"
1080 " data.\n", payload_length);
1081 pr_debug("Ping Data: \"%s\"\n", ping_data);
1082 }
1083
1084 return iscsit_process_nop_out(conn, cmd, hdr);
1085 out:
1086 if (cmd)
1087 iscsit_free_cmd(cmd, false);
1088 return ret;
1089 }
1090
1091 static int
1092 cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1093 {
1094 struct iscsi_conn *conn = csk->conn;
1095 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1096 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1097 u32 payload_length = pdu_cb->dlen;
1098 int rc;
1099 unsigned char *text_in = NULL;
1100
1101 rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1102 if (rc < 0)
1103 return rc;
1104
1105 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1106 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1107 pr_err("Unable to recover from"
1108 " Text Data digest failure while in"
1109 " ERL=0.\n");
1110 goto reject;
1111 } else {
1112 /*
1113 * drop this PDU and let the
1114 * initiator plug the CmdSN gap.
1115 */
1116 pr_info("Dropping Text"
1117 " Command CmdSN: 0x%08x due to"
1118 " DataCRC error.\n", hdr->cmdsn);
1119 return 0;
1120 }
1121 }
1122
1123 if (payload_length) {
1124 text_in = kzalloc(payload_length, GFP_KERNEL);
1125 if (!text_in) {
1126 pr_err("Unable to allocate text_in of payload_length: %u\n",
1127 payload_length);
1128 return -ENOMEM;
1129 }
1130 skb_copy_bits(csk->skb, pdu_cb->doffset,
1131 text_in, payload_length);
1132
1133 text_in[payload_length - 1] = '\0';
1134
1135 cmd->text_in_ptr = text_in;
1136 }
1137
1138 return iscsit_process_text_cmd(conn, cmd, hdr);
1139
1140 reject:
1141 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1142 pdu_cb->hdr);
1143 }
1144
1145 static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1146 {
1147 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1148 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1149 struct iscsi_conn *conn = csk->conn;
1150 struct iscsi_cmd *cmd = NULL;
1151 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1152 int ret = -EINVAL;
1153
1154 switch (opcode) {
1155 case ISCSI_OP_SCSI_CMD:
1156 cmd = cxgbit_allocate_cmd(csk);
1157 if (!cmd)
1158 goto reject;
1159
1160 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1161 break;
1162 case ISCSI_OP_SCSI_DATA_OUT:
1163 ret = cxgbit_handle_iscsi_dataout(csk);
1164 break;
1165 case ISCSI_OP_NOOP_OUT:
1166 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1167 cmd = cxgbit_allocate_cmd(csk);
1168 if (!cmd)
1169 goto reject;
1170 }
1171
1172 ret = cxgbit_handle_nop_out(csk, cmd);
1173 break;
1174 case ISCSI_OP_SCSI_TMFUNC:
1175 cmd = cxgbit_allocate_cmd(csk);
1176 if (!cmd)
1177 goto reject;
1178
1179 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1180 (unsigned char *)hdr);
1181 break;
1182 case ISCSI_OP_TEXT:
1183 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1184 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1185 if (!cmd)
1186 goto reject;
1187 } else {
1188 cmd = cxgbit_allocate_cmd(csk);
1189 if (!cmd)
1190 goto reject;
1191 }
1192
1193 ret = cxgbit_handle_text_cmd(csk, cmd);
1194 break;
1195 case ISCSI_OP_LOGOUT:
1196 cmd = cxgbit_allocate_cmd(csk);
1197 if (!cmd)
1198 goto reject;
1199
1200 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1201 if (ret > 0)
1202 wait_for_completion_timeout(&conn->conn_logout_comp,
1203 SECONDS_FOR_LOGOUT_COMP
1204 * HZ);
1205 break;
1206 case ISCSI_OP_SNACK:
1207 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1208 break;
1209 default:
1210 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1211 dump_stack();
1212 break;
1213 }
1214
1215 return ret;
1216
1217 reject:
1218 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1219 (unsigned char *)hdr);
1220 return ret;
1221 }
1222
1223 static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1224 {
1225 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1226 struct iscsi_conn *conn = csk->conn;
1227 struct iscsi_hdr *hdr = pdu_cb->hdr;
1228 u8 opcode;
1229
1230 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1231 atomic_long_inc(&conn->sess->conn_digest_errors);
1232 goto transport_err;
1233 }
1234
1235 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1236 goto transport_err;
1237
1238 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1239
1240 if (conn->sess->sess_ops->SessionType &&
1241 ((!(opcode & ISCSI_OP_TEXT)) ||
1242 (!(opcode & ISCSI_OP_LOGOUT)))) {
1243 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1244 " while in Discovery Session, rejecting.\n", opcode);
1245 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1246 (unsigned char *)hdr);
1247 goto transport_err;
1248 }
1249
1250 if (cxgbit_target_rx_opcode(csk) < 0)
1251 goto transport_err;
1252
1253 return 0;
1254
1255 transport_err:
1256 return -1;
1257 }
1258
1259 static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1260 {
1261 struct iscsi_conn *conn = csk->conn;
1262 struct iscsi_login *login = conn->login;
1263 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1264 struct iscsi_login_req *login_req;
1265
1266 login_req = (struct iscsi_login_req *)login->req;
1267 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1268
1269 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1270 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1271 login_req->flags, login_req->itt, login_req->cmdsn,
1272 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1273 /*
1274 * Setup the initial iscsi_login values from the leading
1275 * login request PDU.
1276 */
1277 if (login->first_request) {
1278 login_req = (struct iscsi_login_req *)login->req;
1279 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1280 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
1281 login_req->flags);
1282 login->version_min = login_req->min_version;
1283 login->version_max = login_req->max_version;
1284 memcpy(login->isid, login_req->isid, 6);
1285 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1286 login->init_task_tag = login_req->itt;
1287 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1288 login->cid = be16_to_cpu(login_req->cid);
1289 login->tsih = be16_to_cpu(login_req->tsih);
1290 }
1291
1292 if (iscsi_target_check_login_request(conn, login) < 0)
1293 return -1;
1294
1295 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1296 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1297
1298 return 0;
1299 }
1300
1301 static int
1302 cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1303 {
1304 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1305 int ret;
1306
1307 cxgbit_rx_pdu_cb(skb) = pdu_cb;
1308
1309 csk->skb = skb;
1310
1311 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1312 ret = cxgbit_rx_login_pdu(csk);
1313 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1314 } else {
1315 ret = cxgbit_rx_opcode(csk);
1316 }
1317
1318 return ret;
1319 }
1320
1321 static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1322 {
1323 struct skb_shared_info *ssi = skb_shinfo(skb);
1324 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1325 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1326 u8 i;
1327
1328 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1329 skb, skb->head, skb->data, skb->len, skb->data_len,
1330 ssi->nr_frags);
1331 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1332 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1333
1334 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1335 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1336 "frags %u.\n",
1337 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1338 pdu_cb->ddigest, pdu_cb->frags);
1339 for (i = 0; i < ssi->nr_frags; i++)
1340 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1341 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1342 }
1343
1344 static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1345 {
1346 struct sk_buff *skb = csk->lro_hskb;
1347 struct skb_shared_info *ssi = skb_shinfo(skb);
1348 u8 i;
1349
1350 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1351 for (i = 0; i < ssi->nr_frags; i++)
1352 put_page(skb_frag_page(&ssi->frags[i]));
1353 ssi->nr_frags = 0;
1354 }
1355
1356 static void
1357 cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1358 {
1359 struct sk_buff *hskb = csk->lro_hskb;
1360 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1361 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1362 struct skb_shared_info *hssi = skb_shinfo(hskb);
1363 struct skb_shared_info *ssi = skb_shinfo(skb);
1364 unsigned int len = 0;
1365
1366 if (pdu_cb->flags & PDUCBF_RX_HDR) {
1367 hpdu_cb->flags = pdu_cb->flags;
1368 hpdu_cb->seq = pdu_cb->seq;
1369 hpdu_cb->hdr = pdu_cb->hdr;
1370 hpdu_cb->hlen = pdu_cb->hlen;
1371
1372 memcpy(&hssi->frags[0], &ssi->frags[pdu_cb->hfrag_idx],
1373 sizeof(skb_frag_t));
1374
1375 get_page(skb_frag_page(&hssi->frags[0]));
1376 hssi->nr_frags = 1;
1377 hpdu_cb->frags = 1;
1378 hpdu_cb->hfrag_idx = 0;
1379
1380 len = hssi->frags[0].size;
1381 hskb->len = len;
1382 hskb->data_len = len;
1383 hskb->truesize = len;
1384 }
1385
1386 if (pdu_cb->flags & PDUCBF_RX_DATA) {
1387 u8 hfrag_idx = 1, i;
1388
1389 hpdu_cb->flags |= pdu_cb->flags;
1390
1391 len = 0;
1392 for (i = 0; i < pdu_cb->nr_dfrags; hfrag_idx++, i++) {
1393 memcpy(&hssi->frags[hfrag_idx],
1394 &ssi->frags[pdu_cb->dfrag_idx + i],
1395 sizeof(skb_frag_t));
1396
1397 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1398
1399 len += hssi->frags[hfrag_idx].size;
1400
1401 hssi->nr_frags++;
1402 hpdu_cb->frags++;
1403 }
1404
1405 hpdu_cb->dlen = pdu_cb->dlen;
1406 hpdu_cb->doffset = hpdu_cb->hlen;
1407 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1408 hpdu_cb->dfrag_idx = 1;
1409 hskb->len += len;
1410 hskb->data_len += len;
1411 hskb->truesize += len;
1412 }
1413
1414 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1415 hpdu_cb->flags |= pdu_cb->flags;
1416
1417 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1418 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1419
1420 hpdu_cb->ddigest = pdu_cb->ddigest;
1421 hpdu_cb->pdulen = pdu_cb->pdulen;
1422 }
1423 }
1424
1425 static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1426 {
1427 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1428 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1429 u8 pdu_idx = 0, last_idx = 0;
1430 int ret = 0;
1431
1432 if (!pdu_cb->complete) {
1433 cxgbit_lro_skb_merge(csk, skb, 0);
1434
1435 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1436 struct sk_buff *hskb = csk->lro_hskb;
1437
1438 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1439
1440 cxgbit_lro_hskb_reset(csk);
1441
1442 if (ret < 0)
1443 goto out;
1444 }
1445
1446 pdu_idx = 1;
1447 }
1448
1449 if (lro_cb->pdu_idx)
1450 last_idx = lro_cb->pdu_idx - 1;
1451
1452 for (; pdu_idx <= last_idx; pdu_idx++) {
1453 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1454 if (ret < 0)
1455 goto out;
1456 }
1457
1458 if ((!lro_cb->complete) && lro_cb->pdu_idx)
1459 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1460
1461 out:
1462 return ret;
1463 }
1464
1465 static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1466 {
1467 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1468 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1469 int ret = -1;
1470
1471 if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1472 (pdu_cb->seq != csk->rcv_nxt)) {
1473 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1474 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1475 cxgbit_lro_skb_dump(skb);
1476 return ret;
1477 }
1478
1479 csk->rcv_nxt += lro_cb->pdu_totallen;
1480
1481 ret = cxgbit_process_lro_skb(csk, skb);
1482
1483 csk->rx_credits += lro_cb->pdu_totallen;
1484
1485 if (csk->rx_credits >= (csk->rcv_win / 4))
1486 cxgbit_rx_data_ack(csk);
1487
1488 return ret;
1489 }
1490
1491 static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1492 {
1493 int ret = -1;
1494
1495 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO))
1496 ret = cxgbit_rx_lro_skb(csk, skb);
1497
1498 __kfree_skb(skb);
1499 return ret;
1500 }
1501
1502 static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1503 {
1504 spin_lock_bh(&csk->rxq.lock);
1505 if (skb_queue_len(&csk->rxq)) {
1506 skb_queue_splice_init(&csk->rxq, rxq);
1507 spin_unlock_bh(&csk->rxq.lock);
1508 return true;
1509 }
1510 spin_unlock_bh(&csk->rxq.lock);
1511 return false;
1512 }
1513
1514 static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1515 {
1516 struct sk_buff *skb;
1517 struct sk_buff_head rxq;
1518
1519 skb_queue_head_init(&rxq);
1520
1521 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1522
1523 if (signal_pending(current))
1524 goto out;
1525
1526 while ((skb = __skb_dequeue(&rxq))) {
1527 if (cxgbit_rx_skb(csk, skb))
1528 goto out;
1529 }
1530
1531 return 0;
1532 out:
1533 __skb_queue_purge(&rxq);
1534 return -1;
1535 }
1536
1537 int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1538 {
1539 struct cxgbit_sock *csk = conn->context;
1540 int ret = -1;
1541
1542 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1543 ret = cxgbit_wait_rxq(csk);
1544 if (ret) {
1545 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1546 break;
1547 }
1548 }
1549
1550 return ret;
1551 }
1552
1553 void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1554 {
1555 struct cxgbit_sock *csk = conn->context;
1556
1557 while (!kthread_should_stop()) {
1558 iscsit_thread_check_cpumask(conn, current, 0);
1559 if (cxgbit_wait_rxq(csk))
1560 return;
1561 }
1562 }
Linux with added FPC-III support