?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.1.15
[linux/fpc-iii.git] / drivers / target / iscsi / iscsi_target_erl1.c
blobe02e1aaf63c5552a0430aecd21bfdf7e4d7b495b
1 /*******************************************************************************
2 * This file contains error recovery level one used by the iSCSI Target driver.
3 *
4 * (c) Copyright 2007-2013 Datera, Inc.
5 *
6 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 ******************************************************************************/
18
19 #include <linux/list.h>
20 #include <linux/slab.h>
21 #include <scsi/iscsi_proto.h>
22 #include <target/target_core_base.h>
23 #include <target/target_core_fabric.h>
24 #include <target/iscsi/iscsi_transport.h>
25
26 #include <target/iscsi/iscsi_target_core.h>
27 #include "iscsi_target_seq_pdu_list.h"
28 #include "iscsi_target_datain_values.h"
29 #include "iscsi_target_device.h"
30 #include "iscsi_target_tpg.h"
31 #include "iscsi_target_util.h"
32 #include "iscsi_target_erl0.h"
33 #include "iscsi_target_erl1.h"
34 #include "iscsi_target_erl2.h"
35 #include "iscsi_target.h"
36
37 #define OFFLOAD_BUF_SIZE 32768U
38
39 /*
40 * Used to dump excess datain payload for certain error recovery
41 * situations. Receive in OFFLOAD_BUF_SIZE max of datain per rx_data().
42 *
43 * dump_padding_digest denotes if padding and data digests need
44 * to be dumped.
45 */
46 int iscsit_dump_data_payload(
47 struct iscsi_conn *conn,
48 u32 buf_len,
49 int dump_padding_digest)
50 {
51 char *buf;
52 int ret = DATAOUT_WITHIN_COMMAND_RECOVERY, rx_got;
53 u32 length, offset = 0, size;
54 struct kvec iov;
55
56 if (conn->sess->sess_ops->RDMAExtensions)
57 return 0;
58
59 if (dump_padding_digest) {
60 buf_len = ALIGN(buf_len, 4);
61 if (conn->conn_ops->DataDigest)
62 buf_len += ISCSI_CRC_LEN;
63 }
64
65 length = min(buf_len, OFFLOAD_BUF_SIZE);
66
67 buf = kzalloc(length, GFP_ATOMIC);
68 if (!buf) {
69 pr_err("Unable to allocate %u bytes for offload"
70 " buffer.\n", length);
71 return -1;
72 }
73 memset(&iov, 0, sizeof(struct kvec));
74
75 while (offset < buf_len) {
76 size = min(buf_len - offset, length);
77
78 iov.iov_len = size;
79 iov.iov_base = buf;
80
81 rx_got = rx_data(conn, &iov, 1, size);
82 if (rx_got != size) {
83 ret = DATAOUT_CANNOT_RECOVER;
84 break;
85 }
86
87 offset += size;
88 }
89
90 kfree(buf);
91 return ret;
92 }
93
94 /*
95 * Used for retransmitting R2Ts from a R2T SNACK request.
96 */
97 static int iscsit_send_recovery_r2t_for_snack(
98 struct iscsi_cmd *cmd,
99 struct iscsi_r2t *r2t)
100 {
101 /*
102 * If the struct iscsi_r2t has not been sent yet, we can safely
103 * ignore retransmission
104 * of the R2TSN in question.
105 */
106 spin_lock_bh(&cmd->r2t_lock);
107 if (!r2t->sent_r2t) {
108 spin_unlock_bh(&cmd->r2t_lock);
109 return 0;
110 }
111 r2t->sent_r2t = 0;
112 spin_unlock_bh(&cmd->r2t_lock);
113
114 iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, ISTATE_SEND_R2T);
115
116 return 0;
117 }
118
119 static int iscsit_handle_r2t_snack(
120 struct iscsi_cmd *cmd,
121 unsigned char *buf,
122 u32 begrun,
123 u32 runlength)
124 {
125 u32 last_r2tsn;
126 struct iscsi_r2t *r2t;
127
128 /*
129 * Make sure the initiator is not requesting retransmission
130 * of R2TSNs already acknowledged by a TMR TASK_REASSIGN.
131 */
132 if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) &&
133 (begrun <= cmd->acked_data_sn)) {
134 pr_err("ITT: 0x%08x, R2T SNACK requesting"
135 " retransmission of R2TSN: 0x%08x to 0x%08x but already"
136 " acked to R2TSN: 0x%08x by TMR TASK_REASSIGN,"
137 " protocol error.\n", cmd->init_task_tag, begrun,
138 (begrun + runlength), cmd->acked_data_sn);
139
140 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
141 }
142
143 if (runlength) {
144 if ((begrun + runlength) > cmd->r2t_sn) {
145 pr_err("Command ITT: 0x%08x received R2T SNACK"
146 " with BegRun: 0x%08x, RunLength: 0x%08x, exceeds"
147 " current R2TSN: 0x%08x, protocol error.\n",
148 cmd->init_task_tag, begrun, runlength, cmd->r2t_sn);
149 return iscsit_reject_cmd(cmd,
150 ISCSI_REASON_BOOKMARK_INVALID, buf);
151 }
152 last_r2tsn = (begrun + runlength);
153 } else
154 last_r2tsn = cmd->r2t_sn;
155
156 while (begrun < last_r2tsn) {
157 r2t = iscsit_get_holder_for_r2tsn(cmd, begrun);
158 if (!r2t)
159 return -1;
160 if (iscsit_send_recovery_r2t_for_snack(cmd, r2t) < 0)
161 return -1;
162
163 begrun++;
164 }
165
166 return 0;
167 }
168
169 /*
170 * Generates Offsets and NextBurstLength based on Begrun and Runlength
171 * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN.
172 *
173 * For DataSequenceInOrder=Yes and DataPDUInOrder=[Yes,No] only.
174 *
175 * FIXME: How is this handled for a RData SNACK?
176 */
177 int iscsit_create_recovery_datain_values_datasequenceinorder_yes(
178 struct iscsi_cmd *cmd,
179 struct iscsi_datain_req *dr)
180 {
181 u32 data_sn = 0, data_sn_count = 0;
182 u32 pdu_start = 0, seq_no = 0;
183 u32 begrun = dr->begrun;
184 struct iscsi_conn *conn = cmd->conn;
185
186 while (begrun > data_sn++) {
187 data_sn_count++;
188 if ((dr->next_burst_len +
189 conn->conn_ops->MaxRecvDataSegmentLength) <
190 conn->sess->sess_ops->MaxBurstLength) {
191 dr->read_data_done +=
192 conn->conn_ops->MaxRecvDataSegmentLength;
193 dr->next_burst_len +=
194 conn->conn_ops->MaxRecvDataSegmentLength;
195 } else {
196 dr->read_data_done +=
197 (conn->sess->sess_ops->MaxBurstLength -
198 dr->next_burst_len);
199 dr->next_burst_len = 0;
200 pdu_start += data_sn_count;
201 data_sn_count = 0;
202 seq_no++;
203 }
204 }
205
206 if (!conn->sess->sess_ops->DataPDUInOrder) {
207 cmd->seq_no = seq_no;
208 cmd->pdu_start = pdu_start;
209 cmd->pdu_send_order = data_sn_count;
210 }
211
212 return 0;
213 }
214
215 /*
216 * Generates Offsets and NextBurstLength based on Begrun and Runlength
217 * carried in a Data SNACK or ExpDataSN in TMR TASK_REASSIGN.
218 *
219 * For DataSequenceInOrder=No and DataPDUInOrder=[Yes,No] only.
220 *
221 * FIXME: How is this handled for a RData SNACK?
222 */
223 int iscsit_create_recovery_datain_values_datasequenceinorder_no(
224 struct iscsi_cmd *cmd,
225 struct iscsi_datain_req *dr)
226 {
227 int found_seq = 0, i;
228 u32 data_sn, read_data_done = 0, seq_send_order = 0;
229 u32 begrun = dr->begrun;
230 u32 runlength = dr->runlength;
231 struct iscsi_conn *conn = cmd->conn;
232 struct iscsi_seq *first_seq = NULL, *seq = NULL;
233
234 if (!cmd->seq_list) {
235 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
236 return -1;
237 }
238
239 /*
240 * Calculate read_data_done for all sequences containing a
241 * first_datasn and last_datasn less than the BegRun.
242 *
243 * Locate the struct iscsi_seq the BegRun lies within and calculate
244 * NextBurstLenghth up to the DataSN based on MaxRecvDataSegmentLength.
245 *
246 * Also use struct iscsi_seq->seq_send_order to determine where to start.
247 */
248 for (i = 0; i < cmd->seq_count; i++) {
249 seq = &cmd->seq_list[i];
250
251 if (!seq->seq_send_order)
252 first_seq = seq;
253
254 /*
255 * No data has been transferred for this DataIN sequence, so the
256 * seq->first_datasn and seq->last_datasn have not been set.
257 */
258 if (!seq->sent) {
259 pr_err("Ignoring non-sent sequence 0x%08x ->"
260 " 0x%08x\n\n", seq->first_datasn,
261 seq->last_datasn);
262 continue;
263 }
264
265 /*
266 * This DataIN sequence is precedes the received BegRun, add the
267 * total xfer_len of the sequence to read_data_done and reset
268 * seq->pdu_send_order.
269 */
270 if ((seq->first_datasn < begrun) &&
271 (seq->last_datasn < begrun)) {
272 pr_err("Pre BegRun sequence 0x%08x ->"
273 " 0x%08x\n", seq->first_datasn,
274 seq->last_datasn);
275
276 read_data_done += cmd->seq_list[i].xfer_len;
277 seq->next_burst_len = seq->pdu_send_order = 0;
278 continue;
279 }
280
281 /*
282 * The BegRun lies within this DataIN sequence.
283 */
284 if ((seq->first_datasn <= begrun) &&
285 (seq->last_datasn >= begrun)) {
286 pr_err("Found sequence begrun: 0x%08x in"
287 " 0x%08x -> 0x%08x\n", begrun,
288 seq->first_datasn, seq->last_datasn);
289
290 seq_send_order = seq->seq_send_order;
291 data_sn = seq->first_datasn;
292 seq->next_burst_len = seq->pdu_send_order = 0;
293 found_seq = 1;
294
295 /*
296 * For DataPDUInOrder=Yes, while the first DataSN of
297 * the sequence is less than the received BegRun, add
298 * the MaxRecvDataSegmentLength to read_data_done and
299 * to the sequence's next_burst_len;
300 *
301 * For DataPDUInOrder=No, while the first DataSN of the
302 * sequence is less than the received BegRun, find the
303 * struct iscsi_pdu of the DataSN in question and add the
304 * MaxRecvDataSegmentLength to read_data_done and to the
305 * sequence's next_burst_len;
306 */
307 if (conn->sess->sess_ops->DataPDUInOrder) {
308 while (data_sn < begrun) {
309 seq->pdu_send_order++;
310 read_data_done +=
311 conn->conn_ops->MaxRecvDataSegmentLength;
312 seq->next_burst_len +=
313 conn->conn_ops->MaxRecvDataSegmentLength;
314 data_sn++;
315 }
316 } else {
317 int j;
318 struct iscsi_pdu *pdu;
319
320 while (data_sn < begrun) {
321 seq->pdu_send_order++;
322
323 for (j = 0; j < seq->pdu_count; j++) {
324 pdu = &cmd->pdu_list[
325 seq->pdu_start + j];
326 if (pdu->data_sn == data_sn) {
327 read_data_done +=
328 pdu->length;
329 seq->next_burst_len +=
330 pdu->length;
331 }
332 }
333 data_sn++;
334 }
335 }
336 continue;
337 }
338
339 /*
340 * This DataIN sequence is larger than the received BegRun,
341 * reset seq->pdu_send_order and continue.
342 */
343 if ((seq->first_datasn > begrun) ||
344 (seq->last_datasn > begrun)) {
345 pr_err("Post BegRun sequence 0x%08x -> 0x%08x\n",
346 seq->first_datasn, seq->last_datasn);
347
348 seq->next_burst_len = seq->pdu_send_order = 0;
349 continue;
350 }
351 }
352
353 if (!found_seq) {
354 if (!begrun) {
355 if (!first_seq) {
356 pr_err("ITT: 0x%08x, Begrun: 0x%08x"
357 " but first_seq is NULL\n",
358 cmd->init_task_tag, begrun);
359 return -1;
360 }
361 seq_send_order = first_seq->seq_send_order;
362 seq->next_burst_len = seq->pdu_send_order = 0;
363 goto done;
364 }
365
366 pr_err("Unable to locate struct iscsi_seq for ITT: 0x%08x,"
367 " BegRun: 0x%08x, RunLength: 0x%08x while"
368 " DataSequenceInOrder=No and DataPDUInOrder=%s.\n",
369 cmd->init_task_tag, begrun, runlength,
370 (conn->sess->sess_ops->DataPDUInOrder) ? "Yes" : "No");
371 return -1;
372 }
373
374 done:
375 dr->read_data_done = read_data_done;
376 dr->seq_send_order = seq_send_order;
377
378 return 0;
379 }
380
381 static int iscsit_handle_recovery_datain(
382 struct iscsi_cmd *cmd,
383 unsigned char *buf,
384 u32 begrun,
385 u32 runlength)
386 {
387 struct iscsi_conn *conn = cmd->conn;
388 struct iscsi_datain_req *dr;
389 struct se_cmd *se_cmd = &cmd->se_cmd;
390
391 if (!(se_cmd->transport_state & CMD_T_COMPLETE)) {
392 pr_err("Ignoring ITT: 0x%08x Data SNACK\n",
393 cmd->init_task_tag);
394 return 0;
395 }
396
397 /*
398 * Make sure the initiator is not requesting retransmission
399 * of DataSNs already acknowledged by a Data ACK SNACK.
400 */
401 if ((cmd->cmd_flags & ICF_GOT_DATACK_SNACK) &&
402 (begrun <= cmd->acked_data_sn)) {
403 pr_err("ITT: 0x%08x, Data SNACK requesting"
404 " retransmission of DataSN: 0x%08x to 0x%08x but"
405 " already acked to DataSN: 0x%08x by Data ACK SNACK,"
406 " protocol error.\n", cmd->init_task_tag, begrun,
407 (begrun + runlength), cmd->acked_data_sn);
408
409 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR, buf);
410 }
411
412 /*
413 * Make sure BegRun and RunLength in the Data SNACK are sane.
414 * Note: (cmd->data_sn - 1) will carry the maximum DataSN sent.
415 */
416 if ((begrun + runlength) > (cmd->data_sn - 1)) {
417 pr_err("Initiator requesting BegRun: 0x%08x, RunLength"
418 ": 0x%08x greater than maximum DataSN: 0x%08x.\n",
419 begrun, runlength, (cmd->data_sn - 1));
420 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_INVALID,
421 buf);
422 }
423
424 dr = iscsit_allocate_datain_req();
425 if (!dr)
426 return iscsit_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
427 buf);
428
429 dr->data_sn = dr->begrun = begrun;
430 dr->runlength = runlength;
431 dr->generate_recovery_values = 1;
432 dr->recovery = DATAIN_WITHIN_COMMAND_RECOVERY;
433
434 iscsit_attach_datain_req(cmd, dr);
435
436 cmd->i_state = ISTATE_SEND_DATAIN;
437 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
438
439 return 0;
440 }
441
442 int iscsit_handle_recovery_datain_or_r2t(
443 struct iscsi_conn *conn,
444 unsigned char *buf,
445 itt_t init_task_tag,
446 u32 targ_xfer_tag,
447 u32 begrun,
448 u32 runlength)
449 {
450 struct iscsi_cmd *cmd;
451
452 cmd = iscsit_find_cmd_from_itt(conn, init_task_tag);
453 if (!cmd)
454 return 0;
455
456 /*
457 * FIXME: This will not work for bidi commands.
458 */
459 switch (cmd->data_direction) {
460 case DMA_TO_DEVICE:
461 return iscsit_handle_r2t_snack(cmd, buf, begrun, runlength);
462 case DMA_FROM_DEVICE:
463 return iscsit_handle_recovery_datain(cmd, buf, begrun,
464 runlength);
465 default:
466 pr_err("Unknown cmd->data_direction: 0x%02x\n",
467 cmd->data_direction);
468 return -1;
469 }
470
471 return 0;
472 }
473
474 /* #warning FIXME: Status SNACK needs to be dependent on OPCODE!!! */
475 int iscsit_handle_status_snack(
476 struct iscsi_conn *conn,
477 itt_t init_task_tag,
478 u32 targ_xfer_tag,
479 u32 begrun,
480 u32 runlength)
481 {
482 struct iscsi_cmd *cmd = NULL;
483 u32 last_statsn;
484 int found_cmd;
485
486 if (!begrun) {
487 begrun = conn->exp_statsn;
488 } else if (conn->exp_statsn > begrun) {
489 pr_err("Got Status SNACK Begrun: 0x%08x, RunLength:"
490 " 0x%08x but already got ExpStatSN: 0x%08x on CID:"
491 " %hu.\n", begrun, runlength, conn->exp_statsn,
492 conn->cid);
493 return 0;
494 }
495
496 last_statsn = (!runlength) ? conn->stat_sn : (begrun + runlength);
497
498 while (begrun < last_statsn) {
499 found_cmd = 0;
500
501 spin_lock_bh(&conn->cmd_lock);
502 list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) {
503 if (cmd->stat_sn == begrun) {
504 found_cmd = 1;
505 break;
506 }
507 }
508 spin_unlock_bh(&conn->cmd_lock);
509
510 if (!found_cmd) {
511 pr_err("Unable to find StatSN: 0x%08x for"
512 " a Status SNACK, assuming this was a"
513 " protactic SNACK for an untransmitted"
514 " StatSN, ignoring.\n", begrun);
515 begrun++;
516 continue;
517 }
518
519 spin_lock_bh(&cmd->istate_lock);
520 if (cmd->i_state == ISTATE_SEND_DATAIN) {
521 spin_unlock_bh(&cmd->istate_lock);
522 pr_err("Ignoring Status SNACK for BegRun:"
523 " 0x%08x, RunLength: 0x%08x, assuming this was"
524 " a protactic SNACK for an untransmitted"
525 " StatSN\n", begrun, runlength);
526 begrun++;
527 continue;
528 }
529 spin_unlock_bh(&cmd->istate_lock);
530
531 cmd->i_state = ISTATE_SEND_STATUS_RECOVERY;
532 iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state);
533 begrun++;
534 }
535
536 return 0;
537 }
538
539 int iscsit_handle_data_ack(
540 struct iscsi_conn *conn,
541 u32 targ_xfer_tag,
542 u32 begrun,
543 u32 runlength)
544 {
545 struct iscsi_cmd *cmd = NULL;
546
547 cmd = iscsit_find_cmd_from_ttt(conn, targ_xfer_tag);
548 if (!cmd) {
549 pr_err("Data ACK SNACK for TTT: 0x%08x is"
550 " invalid.\n", targ_xfer_tag);
551 return -1;
552 }
553
554 if (begrun <= cmd->acked_data_sn) {
555 pr_err("ITT: 0x%08x Data ACK SNACK BegRUN: 0x%08x is"
556 " less than the already acked DataSN: 0x%08x.\n",
557 cmd->init_task_tag, begrun, cmd->acked_data_sn);
558 return -1;
559 }
560
561 /*
562 * For Data ACK SNACK, BegRun is the next expected DataSN.
563 * (see iSCSI v19: 10.16.6)
564 */
565 cmd->cmd_flags |= ICF_GOT_DATACK_SNACK;
566 cmd->acked_data_sn = (begrun - 1);
567
568 pr_debug("Received Data ACK SNACK for ITT: 0x%08x,"
569 " updated acked DataSN to 0x%08x.\n",
570 cmd->init_task_tag, cmd->acked_data_sn);
571
572 return 0;
573 }
574
575 static int iscsit_send_recovery_r2t(
576 struct iscsi_cmd *cmd,
577 u32 offset,
578 u32 xfer_len)
579 {
580 int ret;
581
582 spin_lock_bh(&cmd->r2t_lock);
583 ret = iscsit_add_r2t_to_list(cmd, offset, xfer_len, 1, 0);
584 spin_unlock_bh(&cmd->r2t_lock);
585
586 return ret;
587 }
588
589 int iscsit_dataout_datapduinorder_no_fbit(
590 struct iscsi_cmd *cmd,
591 struct iscsi_pdu *pdu)
592 {
593 int i, send_recovery_r2t = 0, recovery = 0;
594 u32 length = 0, offset = 0, pdu_count = 0, xfer_len = 0;
595 struct iscsi_conn *conn = cmd->conn;
596 struct iscsi_pdu *first_pdu = NULL;
597
598 /*
599 * Get an struct iscsi_pdu pointer to the first PDU, and total PDU count
600 * of the DataOUT sequence.
601 */
602 if (conn->sess->sess_ops->DataSequenceInOrder) {
603 for (i = 0; i < cmd->pdu_count; i++) {
604 if (cmd->pdu_list[i].seq_no == pdu->seq_no) {
605 if (!first_pdu)
606 first_pdu = &cmd->pdu_list[i];
607 xfer_len += cmd->pdu_list[i].length;
608 pdu_count++;
609 } else if (pdu_count)
610 break;
611 }
612 } else {
613 struct iscsi_seq *seq = cmd->seq_ptr;
614
615 first_pdu = &cmd->pdu_list[seq->pdu_start];
616 pdu_count = seq->pdu_count;
617 }
618
619 if (!first_pdu || !pdu_count)
620 return DATAOUT_CANNOT_RECOVER;
621
622 /*
623 * Loop through the ending DataOUT Sequence checking each struct iscsi_pdu.
624 * The following ugly logic does batching of not received PDUs.
625 */
626 for (i = 0; i < pdu_count; i++) {
627 if (first_pdu[i].status == ISCSI_PDU_RECEIVED_OK) {
628 if (!send_recovery_r2t)
629 continue;
630
631 if (iscsit_send_recovery_r2t(cmd, offset, length) < 0)
632 return DATAOUT_CANNOT_RECOVER;
633
634 send_recovery_r2t = length = offset = 0;
635 continue;
636 }
637 /*
638 * Set recovery = 1 for any missing, CRC failed, or timed
639 * out PDUs to let the DataOUT logic know that this sequence
640 * has not been completed yet.
641 *
642 * Also, only send a Recovery R2T for ISCSI_PDU_NOT_RECEIVED.
643 * We assume if the PDU either failed CRC or timed out
644 * that a Recovery R2T has already been sent.
645 */
646 recovery = 1;
647
648 if (first_pdu[i].status != ISCSI_PDU_NOT_RECEIVED)
649 continue;
650
651 if (!offset)
652 offset = first_pdu[i].offset;
653 length += first_pdu[i].length;
654
655 send_recovery_r2t = 1;
656 }
657
658 if (send_recovery_r2t)
659 if (iscsit_send_recovery_r2t(cmd, offset, length) < 0)
660 return DATAOUT_CANNOT_RECOVER;
661
662 return (!recovery) ? DATAOUT_NORMAL : DATAOUT_WITHIN_COMMAND_RECOVERY;
663 }
664
665 static int iscsit_recalculate_dataout_values(
666 struct iscsi_cmd *cmd,
667 u32 pdu_offset,
668 u32 pdu_length,
669 u32 *r2t_offset,
670 u32 *r2t_length)
671 {
672 int i;
673 struct iscsi_conn *conn = cmd->conn;
674 struct iscsi_pdu *pdu = NULL;
675
676 if (conn->sess->sess_ops->DataSequenceInOrder) {
677 cmd->data_sn = 0;
678
679 if (conn->sess->sess_ops->DataPDUInOrder) {
680 *r2t_offset = cmd->write_data_done;
681 *r2t_length = (cmd->seq_end_offset -
682 cmd->write_data_done);
683 return 0;
684 }
685
686 *r2t_offset = cmd->seq_start_offset;
687 *r2t_length = (cmd->seq_end_offset - cmd->seq_start_offset);
688
689 for (i = 0; i < cmd->pdu_count; i++) {
690 pdu = &cmd->pdu_list[i];
691
692 if (pdu->status != ISCSI_PDU_RECEIVED_OK)
693 continue;
694
695 if ((pdu->offset >= cmd->seq_start_offset) &&
696 ((pdu->offset + pdu->length) <=
697 cmd->seq_end_offset)) {
698 if (!cmd->unsolicited_data)
699 cmd->next_burst_len -= pdu->length;
700 else
701 cmd->first_burst_len -= pdu->length;
702
703 cmd->write_data_done -= pdu->length;
704 pdu->status = ISCSI_PDU_NOT_RECEIVED;
705 }
706 }
707 } else {
708 struct iscsi_seq *seq = NULL;
709
710 seq = iscsit_get_seq_holder(cmd, pdu_offset, pdu_length);
711 if (!seq)
712 return -1;
713
714 *r2t_offset = seq->orig_offset;
715 *r2t_length = seq->xfer_len;
716
717 cmd->write_data_done -= (seq->offset - seq->orig_offset);
718 if (cmd->immediate_data)
719 cmd->first_burst_len = cmd->write_data_done;
720
721 seq->data_sn = 0;
722 seq->offset = seq->orig_offset;
723 seq->next_burst_len = 0;
724 seq->status = DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY;
725
726 if (conn->sess->sess_ops->DataPDUInOrder)
727 return 0;
728
729 for (i = 0; i < seq->pdu_count; i++) {
730 pdu = &cmd->pdu_list[i+seq->pdu_start];
731
732 if (pdu->status != ISCSI_PDU_RECEIVED_OK)
733 continue;
734
735 pdu->status = ISCSI_PDU_NOT_RECEIVED;
736 }
737 }
738
739 return 0;
740 }
741
742 int iscsit_recover_dataout_sequence(
743 struct iscsi_cmd *cmd,
744 u32 pdu_offset,
745 u32 pdu_length)
746 {
747 u32 r2t_length = 0, r2t_offset = 0;
748
749 spin_lock_bh(&cmd->istate_lock);
750 cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY;
751 spin_unlock_bh(&cmd->istate_lock);
752
753 if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length,
754 &r2t_offset, &r2t_length) < 0)
755 return DATAOUT_CANNOT_RECOVER;
756
757 iscsit_send_recovery_r2t(cmd, r2t_offset, r2t_length);
758
759 return DATAOUT_WITHIN_COMMAND_RECOVERY;
760 }
761
762 static struct iscsi_ooo_cmdsn *iscsit_allocate_ooo_cmdsn(void)
763 {
764 struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL;
765
766 ooo_cmdsn = kmem_cache_zalloc(lio_ooo_cache, GFP_ATOMIC);
767 if (!ooo_cmdsn) {
768 pr_err("Unable to allocate memory for"
769 " struct iscsi_ooo_cmdsn.\n");
770 return NULL;
771 }
772 INIT_LIST_HEAD(&ooo_cmdsn->ooo_list);
773
774 return ooo_cmdsn;
775 }
776
777 static int iscsit_attach_ooo_cmdsn(
778 struct iscsi_session *sess,
779 struct iscsi_ooo_cmdsn *ooo_cmdsn)
780 {
781 struct iscsi_ooo_cmdsn *ooo_tail, *ooo_tmp;
782
783 lockdep_assert_held(&sess->cmdsn_mutex);
784
785 /*
786 * We attach the struct iscsi_ooo_cmdsn entry to the out of order
787 * list in increasing CmdSN order.
788 * This allows iscsi_execute_ooo_cmdsns() to detect any
789 * additional CmdSN holes while performing delayed execution.
790 */
791 if (list_empty(&sess->sess_ooo_cmdsn_list))
792 list_add_tail(&ooo_cmdsn->ooo_list,
793 &sess->sess_ooo_cmdsn_list);
794 else {
795 ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev,
796 typeof(*ooo_tail), ooo_list);
797 /*
798 * CmdSN is greater than the tail of the list.
799 */
800 if (iscsi_sna_lt(ooo_tail->cmdsn, ooo_cmdsn->cmdsn))
801 list_add_tail(&ooo_cmdsn->ooo_list,
802 &sess->sess_ooo_cmdsn_list);
803 else {
804 /*
805 * CmdSN is either lower than the head, or somewhere
806 * in the middle.
807 */
808 list_for_each_entry(ooo_tmp, &sess->sess_ooo_cmdsn_list,
809 ooo_list) {
810 if (iscsi_sna_lt(ooo_tmp->cmdsn, ooo_cmdsn->cmdsn))
811 continue;
812
813 /* Insert before this entry */
814 list_add(&ooo_cmdsn->ooo_list,
815 ooo_tmp->ooo_list.prev);
816 break;
817 }
818 }
819 }
820
821 return 0;
822 }
823
824 /*
825 * Removes an struct iscsi_ooo_cmdsn from a session's list,
826 * called with struct iscsi_session->cmdsn_mutex held.
827 */
828 void iscsit_remove_ooo_cmdsn(
829 struct iscsi_session *sess,
830 struct iscsi_ooo_cmdsn *ooo_cmdsn)
831 {
832 list_del(&ooo_cmdsn->ooo_list);
833 kmem_cache_free(lio_ooo_cache, ooo_cmdsn);
834 }
835
836 void iscsit_clear_ooo_cmdsns_for_conn(struct iscsi_conn *conn)
837 {
838 struct iscsi_ooo_cmdsn *ooo_cmdsn;
839 struct iscsi_session *sess = conn->sess;
840
841 mutex_lock(&sess->cmdsn_mutex);
842 list_for_each_entry(ooo_cmdsn, &sess->sess_ooo_cmdsn_list, ooo_list) {
843 if (ooo_cmdsn->cid != conn->cid)
844 continue;
845
846 ooo_cmdsn->cmd = NULL;
847 }
848 mutex_unlock(&sess->cmdsn_mutex);
849 }
850
851 int iscsit_execute_ooo_cmdsns(struct iscsi_session *sess)
852 {
853 int ooo_count = 0;
854 struct iscsi_cmd *cmd = NULL;
855 struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp;
856
857 lockdep_assert_held(&sess->cmdsn_mutex);
858
859 list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp,
860 &sess->sess_ooo_cmdsn_list, ooo_list) {
861 if (ooo_cmdsn->cmdsn != sess->exp_cmd_sn)
862 continue;
863
864 if (!ooo_cmdsn->cmd) {
865 sess->exp_cmd_sn++;
866 iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn);
867 continue;
868 }
869
870 cmd = ooo_cmdsn->cmd;
871 cmd->i_state = cmd->deferred_i_state;
872 ooo_count++;
873 sess->exp_cmd_sn++;
874 pr_debug("Executing out of order CmdSN: 0x%08x,"
875 " incremented ExpCmdSN to 0x%08x.\n",
876 cmd->cmd_sn, sess->exp_cmd_sn);
877
878 iscsit_remove_ooo_cmdsn(sess, ooo_cmdsn);
879
880 if (iscsit_execute_cmd(cmd, 1) < 0)
881 return -1;
882
883 continue;
884 }
885
886 return ooo_count;
887 }
888
889 /*
890 * Called either:
891 *
892 * 1. With sess->cmdsn_mutex held from iscsi_execute_ooo_cmdsns()
893 * or iscsi_check_received_cmdsn().
894 * 2. With no locks held directly from iscsi_handle_XXX_pdu() functions
895 * for immediate commands.
896 */
897 int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo)
898 {
899 struct se_cmd *se_cmd = &cmd->se_cmd;
900 struct iscsi_conn *conn = cmd->conn;
901 int lr = 0;
902
903 spin_lock_bh(&cmd->istate_lock);
904 if (ooo)
905 cmd->cmd_flags &= ~ICF_OOO_CMDSN;
906
907 switch (cmd->iscsi_opcode) {
908 case ISCSI_OP_SCSI_CMD:
909 /*
910 * Go ahead and send the CHECK_CONDITION status for
911 * any SCSI CDB exceptions that may have occurred.
912 */
913 if (cmd->sense_reason) {
914 if (cmd->sense_reason == TCM_RESERVATION_CONFLICT) {
915 cmd->i_state = ISTATE_SEND_STATUS;
916 spin_unlock_bh(&cmd->istate_lock);
917 iscsit_add_cmd_to_response_queue(cmd, cmd->conn,
918 cmd->i_state);
919 return 0;
920 }
921 spin_unlock_bh(&cmd->istate_lock);
922 if (cmd->se_cmd.transport_state & CMD_T_ABORTED)
923 return 0;
924 return transport_send_check_condition_and_sense(se_cmd,
925 cmd->sense_reason, 0);
926 }
927 /*
928 * Special case for delayed CmdSN with Immediate
929 * Data and/or Unsolicited Data Out attached.
930 */
931 if (cmd->immediate_data) {
932 if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) {
933 spin_unlock_bh(&cmd->istate_lock);
934 target_execute_cmd(&cmd->se_cmd);
935 return 0;
936 }
937 spin_unlock_bh(&cmd->istate_lock);
938
939 if (!(cmd->cmd_flags &
940 ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) {
941 if (cmd->se_cmd.transport_state & CMD_T_ABORTED)
942 return 0;
943
944 iscsit_set_dataout_sequence_values(cmd);
945 conn->conn_transport->iscsit_get_dataout(conn, cmd, false);
946 }
947 return 0;
948 }
949 /*
950 * The default handler.
951 */
952 spin_unlock_bh(&cmd->istate_lock);
953
954 if ((cmd->data_direction == DMA_TO_DEVICE) &&
955 !(cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA)) {
956 if (cmd->se_cmd.transport_state & CMD_T_ABORTED)
957 return 0;
958
959 iscsit_set_unsolicited_dataout(cmd);
960 }
961 return transport_handle_cdb_direct(&cmd->se_cmd);
962
963 case ISCSI_OP_NOOP_OUT:
964 case ISCSI_OP_TEXT:
965 spin_unlock_bh(&cmd->istate_lock);
966 iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state);
967 break;
968 case ISCSI_OP_SCSI_TMFUNC:
969 if (cmd->se_cmd.se_tmr_req->response) {
970 spin_unlock_bh(&cmd->istate_lock);
971 iscsit_add_cmd_to_response_queue(cmd, cmd->conn,
972 cmd->i_state);
973 return 0;
974 }
975 spin_unlock_bh(&cmd->istate_lock);
976
977 return transport_generic_handle_tmr(&cmd->se_cmd);
978 case ISCSI_OP_LOGOUT:
979 spin_unlock_bh(&cmd->istate_lock);
980 switch (cmd->logout_reason) {
981 case ISCSI_LOGOUT_REASON_CLOSE_SESSION:
982 lr = iscsit_logout_closesession(cmd, cmd->conn);
983 break;
984 case ISCSI_LOGOUT_REASON_CLOSE_CONNECTION:
985 lr = iscsit_logout_closeconnection(cmd, cmd->conn);
986 break;
987 case ISCSI_LOGOUT_REASON_RECOVERY:
988 lr = iscsit_logout_removeconnforrecovery(cmd, cmd->conn);
989 break;
990 default:
991 pr_err("Unknown iSCSI Logout Request Code:"
992 " 0x%02x\n", cmd->logout_reason);
993 return -1;
994 }
995
996 return lr;
997 default:
998 spin_unlock_bh(&cmd->istate_lock);
999 pr_err("Cannot perform out of order execution for"
1000 " unknown iSCSI Opcode: 0x%02x\n", cmd->iscsi_opcode);
1001 return -1;
1002 }
1003
1004 return 0;
1005 }
1006
1007 void iscsit_free_all_ooo_cmdsns(struct iscsi_session *sess)
1008 {
1009 struct iscsi_ooo_cmdsn *ooo_cmdsn, *ooo_cmdsn_tmp;
1010
1011 mutex_lock(&sess->cmdsn_mutex);
1012 list_for_each_entry_safe(ooo_cmdsn, ooo_cmdsn_tmp,
1013 &sess->sess_ooo_cmdsn_list, ooo_list) {
1014
1015 list_del(&ooo_cmdsn->ooo_list);
1016 kmem_cache_free(lio_ooo_cache, ooo_cmdsn);
1017 }
1018 mutex_unlock(&sess->cmdsn_mutex);
1019 }
1020
1021 int iscsit_handle_ooo_cmdsn(
1022 struct iscsi_session *sess,
1023 struct iscsi_cmd *cmd,
1024 u32 cmdsn)
1025 {
1026 int batch = 0;
1027 struct iscsi_ooo_cmdsn *ooo_cmdsn = NULL, *ooo_tail = NULL;
1028
1029 cmd->deferred_i_state = cmd->i_state;
1030 cmd->i_state = ISTATE_DEFERRED_CMD;
1031 cmd->cmd_flags |= ICF_OOO_CMDSN;
1032
1033 if (list_empty(&sess->sess_ooo_cmdsn_list))
1034 batch = 1;
1035 else {
1036 ooo_tail = list_entry(sess->sess_ooo_cmdsn_list.prev,
1037 typeof(*ooo_tail), ooo_list);
1038 if (ooo_tail->cmdsn != (cmdsn - 1))
1039 batch = 1;
1040 }
1041
1042 ooo_cmdsn = iscsit_allocate_ooo_cmdsn();
1043 if (!ooo_cmdsn)
1044 return -ENOMEM;
1045
1046 ooo_cmdsn->cmd = cmd;
1047 ooo_cmdsn->batch_count = (batch) ?
1048 (cmdsn - sess->exp_cmd_sn) : 1;
1049 ooo_cmdsn->cid = cmd->conn->cid;
1050 ooo_cmdsn->exp_cmdsn = sess->exp_cmd_sn;
1051 ooo_cmdsn->cmdsn = cmdsn;
1052
1053 if (iscsit_attach_ooo_cmdsn(sess, ooo_cmdsn) < 0) {
1054 kmem_cache_free(lio_ooo_cache, ooo_cmdsn);
1055 return -ENOMEM;
1056 }
1057
1058 return 0;
1059 }
1060
1061 static int iscsit_set_dataout_timeout_values(
1062 struct iscsi_cmd *cmd,
1063 u32 *offset,
1064 u32 *length)
1065 {
1066 struct iscsi_conn *conn = cmd->conn;
1067 struct iscsi_r2t *r2t;
1068
1069 if (cmd->unsolicited_data) {
1070 *offset = 0;
1071 *length = (conn->sess->sess_ops->FirstBurstLength >
1072 cmd->se_cmd.data_length) ?
1073 cmd->se_cmd.data_length :
1074 conn->sess->sess_ops->FirstBurstLength;
1075 return 0;
1076 }
1077
1078 spin_lock_bh(&cmd->r2t_lock);
1079 if (list_empty(&cmd->cmd_r2t_list)) {
1080 pr_err("cmd->cmd_r2t_list is empty!\n");
1081 spin_unlock_bh(&cmd->r2t_lock);
1082 return -1;
1083 }
1084
1085 list_for_each_entry(r2t, &cmd->cmd_r2t_list, r2t_list) {
1086 if (r2t->sent_r2t && !r2t->recovery_r2t && !r2t->seq_complete) {
1087 *offset = r2t->offset;
1088 *length = r2t->xfer_len;
1089 spin_unlock_bh(&cmd->r2t_lock);
1090 return 0;
1091 }
1092 }
1093 spin_unlock_bh(&cmd->r2t_lock);
1094
1095 pr_err("Unable to locate any incomplete DataOUT"
1096 " sequences for ITT: 0x%08x.\n", cmd->init_task_tag);
1097
1098 return -1;
1099 }
1100
1101 /*
1102 * NOTE: Called from interrupt (timer) context.
1103 */
1104 void iscsit_handle_dataout_timeout(struct timer_list *t)
1105 {
1106 u32 pdu_length = 0, pdu_offset = 0;
1107 u32 r2t_length = 0, r2t_offset = 0;
1108 struct iscsi_cmd *cmd = from_timer(cmd, t, dataout_timer);
1109 struct iscsi_conn *conn = cmd->conn;
1110 struct iscsi_session *sess = NULL;
1111 struct iscsi_node_attrib *na;
1112
1113 iscsit_inc_conn_usage_count(conn);
1114
1115 spin_lock_bh(&cmd->dataout_timeout_lock);
1116 if (cmd->dataout_timer_flags & ISCSI_TF_STOP) {
1117 spin_unlock_bh(&cmd->dataout_timeout_lock);
1118 iscsit_dec_conn_usage_count(conn);
1119 return;
1120 }
1121 cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING;
1122 sess = conn->sess;
1123 na = iscsit_tpg_get_node_attrib(sess);
1124
1125 if (!sess->sess_ops->ErrorRecoveryLevel) {
1126 pr_err("Unable to recover from DataOut timeout while"
1127 " in ERL=0, closing iSCSI connection for I_T Nexus"
1128 " %s,i,0x%6phN,%s,t,0x%02x\n",
1129 sess->sess_ops->InitiatorName, sess->isid,
1130 sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt);
1131 goto failure;
1132 }
1133
1134 if (++cmd->dataout_timeout_retries == na->dataout_timeout_retries) {
1135 pr_err("Command ITT: 0x%08x exceeded max retries"
1136 " for DataOUT timeout %u, closing iSCSI connection for"
1137 " I_T Nexus %s,i,0x%6phN,%s,t,0x%02x\n",
1138 cmd->init_task_tag, na->dataout_timeout_retries,
1139 sess->sess_ops->InitiatorName, sess->isid,
1140 sess->tpg->tpg_tiqn->tiqn, (u32)sess->tpg->tpgt);
1141 goto failure;
1142 }
1143
1144 cmd->cmd_flags |= ICF_WITHIN_COMMAND_RECOVERY;
1145
1146 if (conn->sess->sess_ops->DataSequenceInOrder) {
1147 if (conn->sess->sess_ops->DataPDUInOrder) {
1148 pdu_offset = cmd->write_data_done;
1149 if ((pdu_offset + (conn->sess->sess_ops->MaxBurstLength -
1150 cmd->next_burst_len)) > cmd->se_cmd.data_length)
1151 pdu_length = (cmd->se_cmd.data_length -
1152 cmd->write_data_done);
1153 else
1154 pdu_length = (conn->sess->sess_ops->MaxBurstLength -
1155 cmd->next_burst_len);
1156 } else {
1157 pdu_offset = cmd->seq_start_offset;
1158 pdu_length = (cmd->seq_end_offset -
1159 cmd->seq_start_offset);
1160 }
1161 } else {
1162 if (iscsit_set_dataout_timeout_values(cmd, &pdu_offset,
1163 &pdu_length) < 0)
1164 goto failure;
1165 }
1166
1167 if (iscsit_recalculate_dataout_values(cmd, pdu_offset, pdu_length,
1168 &r2t_offset, &r2t_length) < 0)
1169 goto failure;
1170
1171 pr_debug("Command ITT: 0x%08x timed out waiting for"
1172 " completion of %sDataOUT Sequence Offset: %u, Length: %u\n",
1173 cmd->init_task_tag, (cmd->unsolicited_data) ? "Unsolicited " :
1174 "", r2t_offset, r2t_length);
1175
1176 if (iscsit_send_recovery_r2t(cmd, r2t_offset, r2t_length) < 0)
1177 goto failure;
1178
1179 iscsit_start_dataout_timer(cmd, conn);
1180 spin_unlock_bh(&cmd->dataout_timeout_lock);
1181 iscsit_dec_conn_usage_count(conn);
1182
1183 return;
1184
1185 failure:
1186 spin_unlock_bh(&cmd->dataout_timeout_lock);
1187 iscsit_fill_cxn_timeout_err_stats(sess);
1188 iscsit_cause_connection_reinstatement(conn, 0);
1189 iscsit_dec_conn_usage_count(conn);
1190 }
1191
1192 void iscsit_mod_dataout_timer(struct iscsi_cmd *cmd)
1193 {
1194 struct iscsi_conn *conn = cmd->conn;
1195 struct iscsi_session *sess = conn->sess;
1196 struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1197
1198 spin_lock_bh(&cmd->dataout_timeout_lock);
1199 if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) {
1200 spin_unlock_bh(&cmd->dataout_timeout_lock);
1201 return;
1202 }
1203
1204 mod_timer(&cmd->dataout_timer,
1205 (get_jiffies_64() + na->dataout_timeout * HZ));
1206 pr_debug("Updated DataOUT timer for ITT: 0x%08x",
1207 cmd->init_task_tag);
1208 spin_unlock_bh(&cmd->dataout_timeout_lock);
1209 }
1210
1211 void iscsit_start_dataout_timer(
1212 struct iscsi_cmd *cmd,
1213 struct iscsi_conn *conn)
1214 {
1215 struct iscsi_session *sess = conn->sess;
1216 struct iscsi_node_attrib *na = iscsit_tpg_get_node_attrib(sess);
1217
1218 lockdep_assert_held(&cmd->dataout_timeout_lock);
1219
1220 if (cmd->dataout_timer_flags & ISCSI_TF_RUNNING)
1221 return;
1222
1223 pr_debug("Starting DataOUT timer for ITT: 0x%08x on"
1224 " CID: %hu.\n", cmd->init_task_tag, conn->cid);
1225
1226 cmd->dataout_timer_flags &= ~ISCSI_TF_STOP;
1227 cmd->dataout_timer_flags |= ISCSI_TF_RUNNING;
1228 mod_timer(&cmd->dataout_timer, jiffies + na->dataout_timeout * HZ);
1229 }
1230
1231 void iscsit_stop_dataout_timer(struct iscsi_cmd *cmd)
1232 {
1233 spin_lock_bh(&cmd->dataout_timeout_lock);
1234 if (!(cmd->dataout_timer_flags & ISCSI_TF_RUNNING)) {
1235 spin_unlock_bh(&cmd->dataout_timeout_lock);
1236 return;
1237 }
1238 cmd->dataout_timer_flags |= ISCSI_TF_STOP;
1239 spin_unlock_bh(&cmd->dataout_timeout_lock);
1240
1241 del_timer_sync(&cmd->dataout_timer);
1242
1243 spin_lock_bh(&cmd->dataout_timeout_lock);
1244 cmd->dataout_timer_flags &= ~ISCSI_TF_RUNNING;
1245 pr_debug("Stopped DataOUT Timer for ITT: 0x%08x\n",
1246 cmd->init_task_tag);
1247 spin_unlock_bh(&cmd->dataout_timeout_lock);
1248 }
1249 EXPORT_SYMBOL(iscsit_stop_dataout_timer);
Linux with added FPC-III support