drivers/target/iscsi/iscsi_target_erl0.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /******************************************************************************
   3  * This file contains error recovery level zero functions used by
   4  * the iSCSI Target driver.
   5  *
   6  * (c) Copyright 2007-2013 Datera, Inc.
   7  *
   8  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   9  *
  10  ******************************************************************************/
  11
  12 #include <linux/sched/signal.h>
  13
  14 #include <scsi/iscsi_proto.h>
  15 #include <target/target_core_base.h>
  16 #include <target/target_core_fabric.h>
  17
  18 #include <target/iscsi/iscsi_target_core.h>
  19 #include "iscsi_target_seq_pdu_list.h"
  20 #include "iscsi_target_erl0.h"
  21 #include "iscsi_target_erl1.h"
  22 #include "iscsi_target_erl2.h"
  23 #include "iscsi_target_util.h"
  24 #include "iscsi_target.h"
  25
  26 /*
  27  *      Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
  28  *      checks against to determine a PDU's Offset+Length is within the current
  29  *      DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
  30  */
  31 void iscsit_set_dataout_sequence_values(
  32         struct iscsi_cmd *cmd)
  33 {
  34         struct iscsi_conn *conn = cmd->conn;
  35         /*
  36          * Still set seq_start_offset and seq_end_offset for Unsolicited
  37          * DataOUT, even if DataSequenceInOrder=No.
  38          */
  39         if (cmd->unsolicited_data) {
  40                 cmd->seq_start_offset = cmd->write_data_done;
  41                 cmd->seq_end_offset = min(cmd->se_cmd.data_length,
  42                                         conn->sess->sess_ops->FirstBurstLength);
  43                 return;
  44         }
  45
  46         if (!conn->sess->sess_ops->DataSequenceInOrder)
  47                 return;
  48
  49         if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
  50                 cmd->seq_start_offset = cmd->write_data_done;
  51                 cmd->seq_end_offset = (cmd->se_cmd.data_length >
  52                         conn->sess->sess_ops->MaxBurstLength) ?
  53                         (cmd->write_data_done +
  54                         conn->sess->sess_ops->MaxBurstLength) : cmd->se_cmd.data_length;
  55         } else {
  56                 cmd->seq_start_offset = cmd->seq_end_offset;
  57                 cmd->seq_end_offset = ((cmd->seq_end_offset +
  58                         conn->sess->sess_ops->MaxBurstLength) >=
  59                         cmd->se_cmd.data_length) ? cmd->se_cmd.data_length :
  60                         (cmd->seq_end_offset +
  61                          conn->sess->sess_ops->MaxBurstLength);
  62         }
  63 }
  64
  65 static int iscsit_dataout_within_command_recovery_check(
  66         struct iscsi_cmd *cmd,
  67         unsigned char *buf)
  68 {
  69         struct iscsi_conn *conn = cmd->conn;
  70         struct iscsi_data *hdr = (struct iscsi_data *) buf;
  71         u32 payload_length = ntoh24(hdr->dlength);
  72
  73         /*
  74          * We do the within-command recovery checks here as it is
  75          * the first function called in iscsi_check_pre_dataout().
  76          * Basically, if we are in within-command recovery and
  77          * the PDU does not contain the offset the sequence needs,
  78          * dump the payload.
  79          *
  80          * This only applies to DataPDUInOrder=Yes, for
  81          * DataPDUInOrder=No we only re-request the failed PDU
  82          * and check that all PDUs in a sequence are received
  83          * upon end of sequence.
  84          */
  85         if (conn->sess->sess_ops->DataSequenceInOrder) {
  86                 if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
  87                     cmd->write_data_done != be32_to_cpu(hdr->offset))
  88                         goto dump;
  89
  90                 cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
  91         } else {
  92                 struct iscsi_seq *seq;
  93
  94                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
  95                                             payload_length);
  96                 if (!seq)
  97                         return DATAOUT_CANNOT_RECOVER;
  98                 /*
  99                  * Set the struct iscsi_seq pointer to reuse later.
 100                  */
 101                 cmd->seq_ptr = seq;
 102
 103                 if (conn->sess->sess_ops->DataPDUInOrder) {
 104                         if (seq->status ==
 105                             DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 106                            (seq->offset != be32_to_cpu(hdr->offset) ||
 107                             seq->data_sn != be32_to_cpu(hdr->datasn)))
 108                                 goto dump;
 109                 } else {
 110                         if (seq->status ==
 111                              DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 112                             seq->data_sn != be32_to_cpu(hdr->datasn))
 113                                 goto dump;
 114                 }
 115
 116                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
 117                         goto dump;
 118
 119                 if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
 120                         seq->status = 0;
 121         }
 122
 123         return DATAOUT_NORMAL;
 124
 125 dump:
 126         pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
 127                 " 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
 128         return iscsit_dump_data_payload(conn, payload_length, 1);
 129 }
 130
 131 static int iscsit_dataout_check_unsolicited_sequence(
 132         struct iscsi_cmd *cmd,
 133         unsigned char *buf)
 134 {
 135         u32 first_burst_len;
 136         struct iscsi_conn *conn = cmd->conn;
 137         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 138         u32 payload_length = ntoh24(hdr->dlength);
 139
 140
 141         if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 142            ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 143                 pr_err("Command ITT: 0x%08x with Offset: %u,"
 144                 " Length: %u outside of Unsolicited Sequence %u:%u while"
 145                 " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 146                 be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 147                         cmd->seq_end_offset);
 148                 return DATAOUT_CANNOT_RECOVER;
 149         }
 150
 151         first_burst_len = (cmd->first_burst_len + payload_length);
 152
 153         if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
 154                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 155                         " for this Unsolicited DataOut Burst.\n",
 156                         first_burst_len, conn->sess->sess_ops->FirstBurstLength);
 157                 transport_send_check_condition_and_sense(&cmd->se_cmd,
 158                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 159                 return DATAOUT_CANNOT_RECOVER;
 160         }
 161
 162         /*
 163          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 164          * checks for the current Unsolicited DataOUT Sequence.
 165          */
 166         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 167                 /*
 168                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 169                  * sequence checks are handled in
 170                  * iscsit_dataout_datapduinorder_no_fbit().
 171                  */
 172                 if (!conn->sess->sess_ops->DataPDUInOrder)
 173                         goto out;
 174
 175                 if ((first_burst_len != cmd->se_cmd.data_length) &&
 176                     (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
 177                         pr_err("Unsolicited non-immediate data"
 178                         " received %u does not equal FirstBurstLength: %u, and"
 179                         " does not equal ExpXferLen %u.\n", first_burst_len,
 180                                 conn->sess->sess_ops->FirstBurstLength,
 181                                 cmd->se_cmd.data_length);
 182                         transport_send_check_condition_and_sense(&cmd->se_cmd,
 183                                         TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 184                         return DATAOUT_CANNOT_RECOVER;
 185                 }
 186         } else {
 187                 if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
 188                         pr_err("Command ITT: 0x%08x reached"
 189                         " FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 190                                 " error.\n", cmd->init_task_tag,
 191                                 conn->sess->sess_ops->FirstBurstLength);
 192                         return DATAOUT_CANNOT_RECOVER;
 193                 }
 194                 if (first_burst_len == cmd->se_cmd.data_length) {
 195                         pr_err("Command ITT: 0x%08x reached"
 196                         " ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 197                         " error.\n", cmd->init_task_tag, cmd->se_cmd.data_length);
 198                         return DATAOUT_CANNOT_RECOVER;
 199                 }
 200         }
 201
 202 out:
 203         return DATAOUT_NORMAL;
 204 }
 205
 206 static int iscsit_dataout_check_sequence(
 207         struct iscsi_cmd *cmd,
 208         unsigned char *buf)
 209 {
 210         u32 next_burst_len;
 211         struct iscsi_conn *conn = cmd->conn;
 212         struct iscsi_seq *seq = NULL;
 213         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 214         u32 payload_length = ntoh24(hdr->dlength);
 215
 216         /*
 217          * For DataSequenceInOrder=Yes: Check that the offset and offset+length
 218          * is within range as defined by iscsi_set_dataout_sequence_values().
 219          *
 220          * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
 221          * offset+length tuple.
 222          */
 223         if (conn->sess->sess_ops->DataSequenceInOrder) {
 224                 /*
 225                  * Due to possibility of recovery DataOUT sent by the initiator
 226                  * fullfilling an Recovery R2T, it's best to just dump the
 227                  * payload here, instead of erroring out.
 228                  */
 229                 if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 230                    ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 231                         pr_err("Command ITT: 0x%08x with Offset: %u,"
 232                         " Length: %u outside of Sequence %u:%u while"
 233                         " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 234                         be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 235                                 cmd->seq_end_offset);
 236
 237                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 238                                 return DATAOUT_CANNOT_RECOVER;
 239                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 240                 }
 241
 242                 next_burst_len = (cmd->next_burst_len + payload_length);
 243         } else {
 244                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 245                                             payload_length);
 246                 if (!seq)
 247                         return DATAOUT_CANNOT_RECOVER;
 248                 /*
 249                  * Set the struct iscsi_seq pointer to reuse later.
 250                  */
 251                 cmd->seq_ptr = seq;
 252
 253                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
 254                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 255                                 return DATAOUT_CANNOT_RECOVER;
 256                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 257                 }
 258
 259                 next_burst_len = (seq->next_burst_len + payload_length);
 260         }
 261
 262         if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
 263                 pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
 264                         " Length: %u exceeds MaxBurstLength: %u. protocol"
 265                         " error.\n", cmd->init_task_tag,
 266                         (next_burst_len - payload_length),
 267                         payload_length, conn->sess->sess_ops->MaxBurstLength);
 268                 return DATAOUT_CANNOT_RECOVER;
 269         }
 270
 271         /*
 272          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 273          * checks for the current DataOUT Sequence.
 274          */
 275         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 276                 /*
 277                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 278                  * sequence checks are handled in
 279                  * iscsit_dataout_datapduinorder_no_fbit().
 280                  */
 281                 if (!conn->sess->sess_ops->DataPDUInOrder)
 282                         goto out;
 283
 284                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 285                         if ((next_burst_len <
 286                              conn->sess->sess_ops->MaxBurstLength) &&
 287                            ((cmd->write_data_done + payload_length) <
 288                              cmd->se_cmd.data_length)) {
 289                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 290                                 " before end of DataOUT sequence, protocol"
 291                                 " error.\n", cmd->init_task_tag);
 292                                 return DATAOUT_CANNOT_RECOVER;
 293                         }
 294                 } else {
 295                         if (next_burst_len < seq->xfer_len) {
 296                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 297                                 " before end of DataOUT sequence, protocol"
 298                                 " error.\n", cmd->init_task_tag);
 299                                 return DATAOUT_CANNOT_RECOVER;
 300                         }
 301                 }
 302         } else {
 303                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 304                         if (next_burst_len ==
 305                                         conn->sess->sess_ops->MaxBurstLength) {
 306                                 pr_err("Command ITT: 0x%08x reached"
 307                                 " MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
 308                                 " not set, protocol error.", cmd->init_task_tag,
 309                                         conn->sess->sess_ops->MaxBurstLength);
 310                                 return DATAOUT_CANNOT_RECOVER;
 311                         }
 312                         if ((cmd->write_data_done + payload_length) ==
 313                                         cmd->se_cmd.data_length) {
 314                                 pr_err("Command ITT: 0x%08x reached"
 315                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 316                                 "CMD_FINAL is not set, protocol error.\n",
 317                                         cmd->init_task_tag);
 318                                 return DATAOUT_CANNOT_RECOVER;
 319                         }
 320                 } else {
 321                         if (next_burst_len == seq->xfer_len) {
 322                                 pr_err("Command ITT: 0x%08x reached"
 323                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 324                                 "CMD_FINAL is not set, protocol error.\n",
 325                                         cmd->init_task_tag);
 326                                 return DATAOUT_CANNOT_RECOVER;
 327                         }
 328                 }
 329         }
 330
 331 out:
 332         return DATAOUT_NORMAL;
 333 }
 334
 335 static int iscsit_dataout_check_datasn(
 336         struct iscsi_cmd *cmd,
 337         unsigned char *buf)
 338 {
 339         u32 data_sn = 0;
 340         struct iscsi_conn *conn = cmd->conn;
 341         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 342         u32 payload_length = ntoh24(hdr->dlength);
 343
 344         /*
 345          * Considering the target has no method of re-requesting DataOUT
 346          * by DataSN, if we receieve a greater DataSN than expected we
 347          * assume the functions for DataPDUInOrder=[Yes,No] below will
 348          * handle it.
 349          *
 350          * If the DataSN is less than expected, dump the payload.
 351          */
 352         if (conn->sess->sess_ops->DataSequenceInOrder)
 353                 data_sn = cmd->data_sn;
 354         else {
 355                 struct iscsi_seq *seq = cmd->seq_ptr;
 356                 data_sn = seq->data_sn;
 357         }
 358
 359         if (be32_to_cpu(hdr->datasn) > data_sn) {
 360                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 361                         " higher than expected 0x%08x.\n", cmd->init_task_tag,
 362                                 be32_to_cpu(hdr->datasn), data_sn);
 363                 goto recover;
 364         } else if (be32_to_cpu(hdr->datasn) < data_sn) {
 365                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 366                         " lower than expected 0x%08x, discarding payload.\n",
 367                         cmd->init_task_tag, be32_to_cpu(hdr->datasn), data_sn);
 368                 goto dump;
 369         }
 370
 371         return DATAOUT_NORMAL;
 372
 373 recover:
 374         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 375                 pr_err("Unable to perform within-command recovery"
 376                                 " while ERL=0.\n");
 377                 return DATAOUT_CANNOT_RECOVER;
 378         }
 379 dump:
 380         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 381                 return DATAOUT_CANNOT_RECOVER;
 382
 383         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 384 }
 385
 386 static int iscsit_dataout_pre_datapduinorder_yes(
 387         struct iscsi_cmd *cmd,
 388         unsigned char *buf)
 389 {
 390         int dump = 0, recovery = 0;
 391         struct iscsi_conn *conn = cmd->conn;
 392         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 393         u32 payload_length = ntoh24(hdr->dlength);
 394
 395         /*
 396          * For DataSequenceInOrder=Yes: If the offset is greater than the global
 397          * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
 398          * occurred and fail the connection.
 399          *
 400          * For DataSequenceInOrder=No: If the offset is greater than the per
 401          * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
 402          * error has occurred and fail the connection.
 403          */
 404         if (conn->sess->sess_ops->DataSequenceInOrder) {
 405                 if (be32_to_cpu(hdr->offset) != cmd->write_data_done) {
 406                         pr_err("Command ITT: 0x%08x, received offset"
 407                         " %u different than expected %u.\n", cmd->init_task_tag,
 408                                 be32_to_cpu(hdr->offset), cmd->write_data_done);
 409                         recovery = 1;
 410                         goto recover;
 411                 }
 412         } else {
 413                 struct iscsi_seq *seq = cmd->seq_ptr;
 414
 415                 if (be32_to_cpu(hdr->offset) > seq->offset) {
 416                         pr_err("Command ITT: 0x%08x, received offset"
 417                         " %u greater than expected %u.\n", cmd->init_task_tag,
 418                                 be32_to_cpu(hdr->offset), seq->offset);
 419                         recovery = 1;
 420                         goto recover;
 421                 } else if (be32_to_cpu(hdr->offset) < seq->offset) {
 422                         pr_err("Command ITT: 0x%08x, received offset"
 423                         " %u less than expected %u, discarding payload.\n",
 424                                 cmd->init_task_tag, be32_to_cpu(hdr->offset),
 425                                 seq->offset);
 426                         dump = 1;
 427                         goto dump;
 428                 }
 429         }
 430
 431         return DATAOUT_NORMAL;
 432
 433 recover:
 434         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 435                 pr_err("Unable to perform within-command recovery"
 436                                 " while ERL=0.\n");
 437                 return DATAOUT_CANNOT_RECOVER;
 438         }
 439 dump:
 440         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 441                 return DATAOUT_CANNOT_RECOVER;
 442
 443         return (recovery) ? iscsit_recover_dataout_sequence(cmd,
 444                 be32_to_cpu(hdr->offset), payload_length) :
 445                (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
 446 }
 447
 448 static int iscsit_dataout_pre_datapduinorder_no(
 449         struct iscsi_cmd *cmd,
 450         unsigned char *buf)
 451 {
 452         struct iscsi_pdu *pdu;
 453         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 454         u32 payload_length = ntoh24(hdr->dlength);
 455
 456         pdu = iscsit_get_pdu_holder(cmd, be32_to_cpu(hdr->offset),
 457                                     payload_length);
 458         if (!pdu)
 459                 return DATAOUT_CANNOT_RECOVER;
 460
 461         cmd->pdu_ptr = pdu;
 462
 463         switch (pdu->status) {
 464         case ISCSI_PDU_NOT_RECEIVED:
 465         case ISCSI_PDU_CRC_FAILED:
 466         case ISCSI_PDU_TIMED_OUT:
 467                 break;
 468         case ISCSI_PDU_RECEIVED_OK:
 469                 pr_err("Command ITT: 0x%08x received already gotten"
 470                         " Offset: %u, Length: %u\n", cmd->init_task_tag,
 471                                 be32_to_cpu(hdr->offset), payload_length);
 472                 return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
 473         default:
 474                 return DATAOUT_CANNOT_RECOVER;
 475         }
 476
 477         return DATAOUT_NORMAL;
 478 }
 479
 480 static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
 481 {
 482         struct iscsi_r2t *r2t;
 483
 484         if (cmd->unsolicited_data)
 485                 return 0;
 486
 487         r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
 488         if (!r2t)
 489                 return -1;
 490
 491         spin_lock_bh(&cmd->r2t_lock);
 492         r2t->seq_complete = 1;
 493         cmd->outstanding_r2ts--;
 494         spin_unlock_bh(&cmd->r2t_lock);
 495
 496         return 0;
 497 }
 498
 499 static int iscsit_dataout_update_datapduinorder_no(
 500         struct iscsi_cmd *cmd,
 501         u32 data_sn,
 502         int f_bit)
 503 {
 504         int ret = 0;
 505         struct iscsi_pdu *pdu = cmd->pdu_ptr;
 506
 507         pdu->data_sn = data_sn;
 508
 509         switch (pdu->status) {
 510         case ISCSI_PDU_NOT_RECEIVED:
 511                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 512                 break;
 513         case ISCSI_PDU_CRC_FAILED:
 514                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 515                 break;
 516         case ISCSI_PDU_TIMED_OUT:
 517                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 518                 break;
 519         default:
 520                 return DATAOUT_CANNOT_RECOVER;
 521         }
 522
 523         if (f_bit) {
 524                 ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
 525                 if (ret == DATAOUT_CANNOT_RECOVER)
 526                         return ret;
 527         }
 528
 529         return DATAOUT_NORMAL;
 530 }
 531
 532 static int iscsit_dataout_post_crc_passed(
 533         struct iscsi_cmd *cmd,
 534         unsigned char *buf)
 535 {
 536         int ret, send_r2t = 0;
 537         struct iscsi_conn *conn = cmd->conn;
 538         struct iscsi_seq *seq = NULL;
 539         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 540         u32 payload_length = ntoh24(hdr->dlength);
 541
 542         if (cmd->unsolicited_data) {
 543                 if ((cmd->first_burst_len + payload_length) ==
 544                      conn->sess->sess_ops->FirstBurstLength) {
 545                         if (iscsit_dataout_update_r2t(cmd, be32_to_cpu(hdr->offset),
 546                                         payload_length) < 0)
 547                                 return DATAOUT_CANNOT_RECOVER;
 548                         send_r2t = 1;
 549                 }
 550
 551                 if (!conn->sess->sess_ops->DataPDUInOrder) {
 552                         ret = iscsit_dataout_update_datapduinorder_no(cmd,
 553                                 be32_to_cpu(hdr->datasn),
 554                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 555                         if (ret == DATAOUT_CANNOT_RECOVER)
 556                                 return ret;
 557                 }
 558
 559                 cmd->first_burst_len += payload_length;
 560
 561                 if (conn->sess->sess_ops->DataSequenceInOrder)
 562                         cmd->data_sn++;
 563                 else {
 564                         seq = cmd->seq_ptr;
 565                         seq->data_sn++;
 566                         seq->offset += payload_length;
 567                 }
 568
 569                 if (send_r2t) {
 570                         if (seq)
 571                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 572                         cmd->first_burst_len = 0;
 573                         cmd->unsolicited_data = 0;
 574                 }
 575         } else {
 576                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 577                         if ((cmd->next_burst_len + payload_length) ==
 578                              conn->sess->sess_ops->MaxBurstLength) {
 579                                 if (iscsit_dataout_update_r2t(cmd,
 580                                                 be32_to_cpu(hdr->offset),
 581                                                 payload_length) < 0)
 582                                         return DATAOUT_CANNOT_RECOVER;
 583                                 send_r2t = 1;
 584                         }
 585
 586                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 587                                 ret = iscsit_dataout_update_datapduinorder_no(
 588                                                 cmd, be32_to_cpu(hdr->datasn),
 589                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 590                                 if (ret == DATAOUT_CANNOT_RECOVER)
 591                                         return ret;
 592                         }
 593
 594                         cmd->next_burst_len += payload_length;
 595                         cmd->data_sn++;
 596
 597                         if (send_r2t)
 598                                 cmd->next_burst_len = 0;
 599                 } else {
 600                         seq = cmd->seq_ptr;
 601
 602                         if ((seq->next_burst_len + payload_length) ==
 603                              seq->xfer_len) {
 604                                 if (iscsit_dataout_update_r2t(cmd,
 605                                                 be32_to_cpu(hdr->offset),
 606                                                 payload_length) < 0)
 607                                         return DATAOUT_CANNOT_RECOVER;
 608                                 send_r2t = 1;
 609                         }
 610
 611                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 612                                 ret = iscsit_dataout_update_datapduinorder_no(
 613                                                 cmd, be32_to_cpu(hdr->datasn),
 614                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 615                                 if (ret == DATAOUT_CANNOT_RECOVER)
 616                                         return ret;
 617                         }
 618
 619                         seq->data_sn++;
 620                         seq->offset += payload_length;
 621                         seq->next_burst_len += payload_length;
 622
 623                         if (send_r2t) {
 624                                 seq->next_burst_len = 0;
 625                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 626                         }
 627                 }
 628         }
 629
 630         if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
 631                 cmd->data_sn = 0;
 632
 633         cmd->write_data_done += payload_length;
 634
 635         if (cmd->write_data_done == cmd->se_cmd.data_length)
 636                 return DATAOUT_SEND_TO_TRANSPORT;
 637         else if (send_r2t)
 638                 return DATAOUT_SEND_R2T;
 639         else
 640                 return DATAOUT_NORMAL;
 641 }
 642
 643 static int iscsit_dataout_post_crc_failed(
 644         struct iscsi_cmd *cmd,
 645         unsigned char *buf)
 646 {
 647         struct iscsi_conn *conn = cmd->conn;
 648         struct iscsi_pdu *pdu;
 649         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 650         u32 payload_length = ntoh24(hdr->dlength);
 651
 652         if (conn->sess->sess_ops->DataPDUInOrder)
 653                 goto recover;
 654         /*
 655          * The rest of this function is only called when DataPDUInOrder=No.
 656          */
 657         pdu = cmd->pdu_ptr;
 658
 659         switch (pdu->status) {
 660         case ISCSI_PDU_NOT_RECEIVED:
 661                 pdu->status = ISCSI_PDU_CRC_FAILED;
 662                 break;
 663         case ISCSI_PDU_CRC_FAILED:
 664                 break;
 665         case ISCSI_PDU_TIMED_OUT:
 666                 pdu->status = ISCSI_PDU_CRC_FAILED;
 667                 break;
 668         default:
 669                 return DATAOUT_CANNOT_RECOVER;
 670         }
 671
 672 recover:
 673         return iscsit_recover_dataout_sequence(cmd, be32_to_cpu(hdr->offset),
 674                                                 payload_length);
 675 }
 676
 677 /*
 678  *      Called from iscsit_handle_data_out() before DataOUT Payload is received
 679  *      and CRC computed.
 680  */
 681 int iscsit_check_pre_dataout(
 682         struct iscsi_cmd *cmd,
 683         unsigned char *buf)
 684 {
 685         int ret;
 686         struct iscsi_conn *conn = cmd->conn;
 687
 688         ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
 689         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 690             (ret == DATAOUT_CANNOT_RECOVER))
 691                 return ret;
 692
 693         ret = iscsit_dataout_check_datasn(cmd, buf);
 694         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 695             (ret == DATAOUT_CANNOT_RECOVER))
 696                 return ret;
 697
 698         if (cmd->unsolicited_data) {
 699                 ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
 700                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 701                     (ret == DATAOUT_CANNOT_RECOVER))
 702                         return ret;
 703         } else {
 704                 ret = iscsit_dataout_check_sequence(cmd, buf);
 705                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 706                     (ret == DATAOUT_CANNOT_RECOVER))
 707                         return ret;
 708         }
 709
 710         return (conn->sess->sess_ops->DataPDUInOrder) ?
 711                 iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
 712                 iscsit_dataout_pre_datapduinorder_no(cmd, buf);
 713 }
 714
 715 /*
 716  *      Called from iscsit_handle_data_out() after DataOUT Payload is received
 717  *      and CRC computed.
 718  */
 719 int iscsit_check_post_dataout(
 720         struct iscsi_cmd *cmd,
 721         unsigned char *buf,
 722         u8 data_crc_failed)
 723 {
 724         struct iscsi_conn *conn = cmd->conn;
 725
 726         cmd->dataout_timeout_retries = 0;
 727
 728         if (!data_crc_failed)
 729                 return iscsit_dataout_post_crc_passed(cmd, buf);
 730         else {
 731                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 732                         pr_err("Unable to recover from DataOUT CRC"
 733                                 " failure while ERL=0, closing session.\n");
 734                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 735                                           buf);
 736                         return DATAOUT_CANNOT_RECOVER;
 737                 }
 738
 739                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, buf);
 740                 return iscsit_dataout_post_crc_failed(cmd, buf);
 741         }
 742 }
 743
 744 void iscsit_handle_time2retain_timeout(struct timer_list *t)
 745 {
 746         struct iscsi_session *sess = from_timer(sess, t, time2retain_timer);
 747         struct iscsi_portal_group *tpg = sess->tpg;
 748         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 749
 750         spin_lock_bh(&se_tpg->session_lock);
 751         if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
 752                 spin_unlock_bh(&se_tpg->session_lock);
 753                 return;
 754         }
 755         if (atomic_read(&sess->session_reinstatement)) {
 756                 pr_err("Exiting Time2Retain handler because"
 757                                 " session_reinstatement=1\n");
 758                 spin_unlock_bh(&se_tpg->session_lock);
 759                 return;
 760         }
 761         sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
 762
 763         pr_err("Time2Retain timer expired for SID: %u, cleaning up"
 764                         " iSCSI session.\n", sess->sid);
 765
 766         iscsit_fill_cxn_timeout_err_stats(sess);
 767         spin_unlock_bh(&se_tpg->session_lock);
 768         iscsit_close_session(sess);
 769 }
 770
 771 void iscsit_start_time2retain_handler(struct iscsi_session *sess)
 772 {
 773         int tpg_active;
 774         /*
 775          * Only start Time2Retain timer when the associated TPG is still in
 776          * an ACTIVE (eg: not disabled or shutdown) state.
 777          */
 778         spin_lock(&sess->tpg->tpg_state_lock);
 779         tpg_active = (sess->tpg->tpg_state == TPG_STATE_ACTIVE);
 780         spin_unlock(&sess->tpg->tpg_state_lock);
 781
 782         if (!tpg_active)
 783                 return;
 784
 785         if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
 786                 return;
 787
 788         pr_debug("Starting Time2Retain timer for %u seconds on"
 789                 " SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
 790
 791         sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
 792         sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
 793         mod_timer(&sess->time2retain_timer,
 794                   jiffies + sess->sess_ops->DefaultTime2Retain * HZ);
 795 }
 796
 797 int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
 798 {
 799         struct iscsi_portal_group *tpg = sess->tpg;
 800         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 801
 802         lockdep_assert_held(&se_tpg->session_lock);
 803
 804         if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
 805                 return -1;
 806
 807         if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
 808                 return 0;
 809
 810         sess->time2retain_timer_flags |= ISCSI_TF_STOP;
 811         spin_unlock(&se_tpg->session_lock);
 812
 813         del_timer_sync(&sess->time2retain_timer);
 814
 815         spin_lock(&se_tpg->session_lock);
 816         sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
 817         pr_debug("Stopped Time2Retain Timer for SID: %u\n",
 818                         sess->sid);
 819         return 0;
 820 }
 821
 822 void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
 823 {
 824         spin_lock_bh(&conn->state_lock);
 825         if (atomic_read(&conn->connection_exit)) {
 826                 spin_unlock_bh(&conn->state_lock);
 827                 goto sleep;
 828         }
 829
 830         if (atomic_read(&conn->transport_failed)) {
 831                 spin_unlock_bh(&conn->state_lock);
 832                 goto sleep;
 833         }
 834         spin_unlock_bh(&conn->state_lock);
 835
 836         if (conn->tx_thread && conn->tx_thread_active)
 837                 send_sig(SIGINT, conn->tx_thread, 1);
 838         if (conn->rx_thread && conn->rx_thread_active)
 839                 send_sig(SIGINT, conn->rx_thread, 1);
 840
 841 sleep:
 842         wait_for_completion(&conn->conn_wait_rcfr_comp);
 843         complete(&conn->conn_post_wait_comp);
 844 }
 845
 846 void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
 847 {
 848         spin_lock_bh(&conn->state_lock);
 849         if (atomic_read(&conn->connection_exit)) {
 850                 spin_unlock_bh(&conn->state_lock);
 851                 return;
 852         }
 853
 854         if (atomic_read(&conn->transport_failed)) {
 855                 spin_unlock_bh(&conn->state_lock);
 856                 return;
 857         }
 858
 859         if (atomic_read(&conn->connection_reinstatement)) {
 860                 spin_unlock_bh(&conn->state_lock);
 861                 return;
 862         }
 863
 864         if (conn->tx_thread && conn->tx_thread_active)
 865                 send_sig(SIGINT, conn->tx_thread, 1);
 866         if (conn->rx_thread && conn->rx_thread_active)
 867                 send_sig(SIGINT, conn->rx_thread, 1);
 868
 869         atomic_set(&conn->connection_reinstatement, 1);
 870         if (!sleep) {
 871                 spin_unlock_bh(&conn->state_lock);
 872                 return;
 873         }
 874
 875         atomic_set(&conn->sleep_on_conn_wait_comp, 1);
 876         spin_unlock_bh(&conn->state_lock);
 877
 878         wait_for_completion(&conn->conn_wait_comp);
 879         complete(&conn->conn_post_wait_comp);
 880 }
 881 EXPORT_SYMBOL(iscsit_cause_connection_reinstatement);
 882
 883 void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
 884 {
 885         pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
 886                         " %u\n", sess->sid);
 887
 888         atomic_set(&sess->session_fall_back_to_erl0, 1);
 889 }
 890
 891 static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
 892 {
 893         struct iscsi_session *sess = conn->sess;
 894
 895         if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
 896             !atomic_read(&sess->session_reinstatement) &&
 897             !atomic_read(&sess->session_fall_back_to_erl0))
 898                 iscsit_connection_recovery_transport_reset(conn);
 899         else {
 900                 pr_debug("Performing cleanup for failed iSCSI"
 901                         " Connection ID: %hu from %s\n", conn->cid,
 902                         sess->sess_ops->InitiatorName);
 903                 iscsit_close_connection(conn);
 904         }
 905 }
 906
 907 void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn, bool *conn_freed)
 908 {
 909         *conn_freed = false;
 910
 911         spin_lock_bh(&conn->state_lock);
 912         if (atomic_read(&conn->connection_exit)) {
 913                 spin_unlock_bh(&conn->state_lock);
 914                 return;
 915         }
 916         atomic_set(&conn->connection_exit, 1);
 917
 918         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
 919                 spin_unlock_bh(&conn->state_lock);
 920                 iscsit_close_connection(conn);
 921                 *conn_freed = true;
 922                 return;
 923         }
 924
 925         if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
 926                 spin_unlock_bh(&conn->state_lock);
 927                 return;
 928         }
 929
 930         pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
 931         conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
 932         spin_unlock_bh(&conn->state_lock);
 933
 934         iscsit_handle_connection_cleanup(conn);
 935         *conn_freed = true;
 936 }