drivers/net/ethernet/qlogic/qed/qed_nvmetcp_fw_funcs.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
   2 /* Copyright 2021 Marvell. All rights reserved. */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/pci.h>
   7 #include <linux/list.h>
   8 #include <linux/mm.h>
   9 #include <linux/types.h>
  10 #include <asm/byteorder.h>
  11 #include <linux/qed/common_hsi.h>
  12 #include <linux/qed/storage_common.h>
  13 #include <linux/qed/nvmetcp_common.h>
  14 #include <linux/qed/qed_nvmetcp_if.h>
  15 #include "qed_nvmetcp_fw_funcs.h"
  16
  17 #define NVMETCP_NUM_SGES_IN_CACHE 0x4
  18
  19 bool nvmetcp_is_slow_sgl(u16 num_sges, bool small_mid_sge)
  20 {
  21         return (num_sges > SCSI_NUM_SGES_SLOW_SGL_THR && small_mid_sge);
  22 }
  23
  24 void init_scsi_sgl_context(struct scsi_sgl_params *ctx_sgl_params,
  25                            struct scsi_cached_sges *ctx_data_desc,
  26                            struct storage_sgl_task_params *sgl_params)
  27 {
  28         u8 num_sges_to_init = (u8)(sgl_params->num_sges > NVMETCP_NUM_SGES_IN_CACHE ?
  29                                    NVMETCP_NUM_SGES_IN_CACHE : sgl_params->num_sges);
  30         u8 sge_index;
  31
  32         /* sgl params */
  33         ctx_sgl_params->sgl_addr.lo = cpu_to_le32(sgl_params->sgl_phys_addr.lo);
  34         ctx_sgl_params->sgl_addr.hi = cpu_to_le32(sgl_params->sgl_phys_addr.hi);
  35         ctx_sgl_params->sgl_total_length = cpu_to_le32(sgl_params->total_buffer_size);
  36         ctx_sgl_params->sgl_num_sges = cpu_to_le16(sgl_params->num_sges);
  37
  38         for (sge_index = 0; sge_index < num_sges_to_init; sge_index++) {
  39                 ctx_data_desc->sge[sge_index].sge_addr.lo =
  40                         cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.lo);
  41                 ctx_data_desc->sge[sge_index].sge_addr.hi =
  42                         cpu_to_le32(sgl_params->sgl[sge_index].sge_addr.hi);
  43                 ctx_data_desc->sge[sge_index].sge_len =
  44                         cpu_to_le32(sgl_params->sgl[sge_index].sge_len);
  45         }
  46 }
  47
  48 static inline u32 calc_rw_task_size(struct nvmetcp_task_params *task_params,
  49                                     enum nvmetcp_task_type task_type)
  50 {
  51         u32 io_size;
  52
  53         if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE)
  54                 io_size = task_params->tx_io_size;
  55         else
  56                 io_size = task_params->rx_io_size;
  57
  58         if (unlikely(!io_size))
  59                 return 0;
  60
  61         return io_size;
  62 }
  63
  64 static inline void init_sqe(struct nvmetcp_task_params *task_params,
  65                             struct storage_sgl_task_params *sgl_task_params,
  66                             enum nvmetcp_task_type task_type)
  67 {
  68         if (!task_params->sqe)
  69                 return;
  70
  71         memset(task_params->sqe, 0, sizeof(*task_params->sqe));
  72         task_params->sqe->task_id = cpu_to_le16(task_params->itid);
  73
  74         switch (task_type) {
  75         case NVMETCP_TASK_TYPE_HOST_WRITE: {
  76                 u32 buf_size = 0;
  77                 u32 num_sges = 0;
  78
  79                 SET_FIELD(task_params->sqe->contlen_cdbsize,
  80                           NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
  81                 SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
  82                           NVMETCP_WQE_TYPE_NORMAL);
  83                 if (task_params->tx_io_size) {
  84                         if (task_params->send_write_incapsule)
  85                                 buf_size = calc_rw_task_size(task_params, task_type);
  86
  87                         if (nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
  88                                                 sgl_task_params->small_mid_sge))
  89                                 num_sges = NVMETCP_WQE_NUM_SGES_SLOWIO;
  90                         else
  91                                 num_sges = min((u16)sgl_task_params->num_sges,
  92                                                (u16)SCSI_NUM_SGES_SLOW_SGL_THR);
  93                 }
  94                 SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES, num_sges);
  95                 SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN, buf_size);
  96         } break;
  97
  98         case NVMETCP_TASK_TYPE_HOST_READ: {
  99                 SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
 100                           NVMETCP_WQE_TYPE_NORMAL);
 101                 SET_FIELD(task_params->sqe->contlen_cdbsize,
 102                           NVMETCP_WQE_CDB_SIZE_OR_NVMETCP_CMD, 1);
 103         } break;
 104
 105         case NVMETCP_TASK_TYPE_INIT_CONN_REQUEST: {
 106                 SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
 107                           NVMETCP_WQE_TYPE_MIDDLE_PATH);
 108
 109                 if (task_params->tx_io_size) {
 110                         SET_FIELD(task_params->sqe->contlen_cdbsize, NVMETCP_WQE_CONT_LEN,
 111                                   task_params->tx_io_size);
 112                         SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_NUM_SGES,
 113                                   min((u16)sgl_task_params->num_sges,
 114                                       (u16)SCSI_NUM_SGES_SLOW_SGL_THR));
 115                 }
 116         } break;
 117
 118         case NVMETCP_TASK_TYPE_CLEANUP:
 119                 SET_FIELD(task_params->sqe->flags, NVMETCP_WQE_WQE_TYPE,
 120                           NVMETCP_WQE_TYPE_TASK_CLEANUP);
 121
 122         default:
 123                 break;
 124         }
 125 }
 126
 127 /* The following function initializes of NVMeTCP task params */
 128 static inline void
 129 init_nvmetcp_task_params(struct e5_nvmetcp_task_context *context,
 130                          struct nvmetcp_task_params *task_params,
 131                          enum nvmetcp_task_type task_type)
 132 {
 133         context->ystorm_st_context.state.cccid = task_params->host_cccid;
 134         SET_FIELD(context->ustorm_st_context.error_flags, USTORM_NVMETCP_TASK_ST_CTX_NVME_TCP, 1);
 135         context->ustorm_st_context.nvme_tcp_opaque_lo = cpu_to_le32(task_params->opq.lo);
 136         context->ustorm_st_context.nvme_tcp_opaque_hi = cpu_to_le32(task_params->opq.hi);
 137 }
 138
 139 /* The following function initializes default values to all tasks */
 140 static inline void
 141 init_default_nvmetcp_task(struct nvmetcp_task_params *task_params,
 142                           void *pdu_header, void *nvme_cmd,
 143                           enum nvmetcp_task_type task_type)
 144 {
 145         struct e5_nvmetcp_task_context *context = task_params->context;
 146         const u8 val_byte = context->mstorm_ag_context.cdu_validation;
 147         u8 dw_index;
 148
 149         memset(context, 0, sizeof(*context));
 150         init_nvmetcp_task_params(context, task_params,
 151                                  (enum nvmetcp_task_type)task_type);
 152
 153         /* Swapping requirements used below, will be removed in future FW versions */
 154         if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE ||
 155             task_type == NVMETCP_TASK_TYPE_HOST_READ) {
 156                 for (dw_index = 0;
 157                      dw_index < QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
 158                      dw_index++)
 159                         context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
 160                                 cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
 161
 162                 for (dw_index = QED_NVMETCP_CMN_HDR_SIZE / sizeof(u32);
 163                      dw_index < QED_NVMETCP_CMD_HDR_SIZE / sizeof(u32);
 164                      dw_index++)
 165                         context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
 166                                 cpu_to_le32(__swab32(((u32 *)nvme_cmd)[dw_index - 2]));
 167         } else {
 168                 for (dw_index = 0;
 169                      dw_index < QED_NVMETCP_NON_IO_HDR_SIZE / sizeof(u32);
 170                      dw_index++)
 171                         context->ystorm_st_context.pdu_hdr.task_hdr.reg[dw_index] =
 172                                 cpu_to_le32(__swab32(((u32 *)pdu_header)[dw_index]));
 173         }
 174
 175         /* M-Storm Context: */
 176         context->mstorm_ag_context.cdu_validation = val_byte;
 177         context->mstorm_st_context.task_type = (u8)(task_type);
 178         context->mstorm_ag_context.task_cid = cpu_to_le16(task_params->conn_icid);
 179
 180         /* Ustorm Context: */
 181         SET_FIELD(context->ustorm_ag_context.flags1, E5_USTORM_NVMETCP_TASK_AG_CTX_R2T2RECV, 1);
 182         context->ustorm_st_context.task_type = (u8)(task_type);
 183         context->ustorm_st_context.cq_rss_number = task_params->cq_rss_number;
 184         context->ustorm_ag_context.icid = cpu_to_le16(task_params->conn_icid);
 185 }
 186
 187 /* The following function initializes the U-Storm Task Contexts */
 188 static inline void
 189 init_ustorm_task_contexts(struct ustorm_nvmetcp_task_st_ctx *ustorm_st_context,
 190                           struct e5_ustorm_nvmetcp_task_ag_ctx *ustorm_ag_context,
 191                           u32 remaining_recv_len,
 192                           u32 expected_data_transfer_len, u8 num_sges,
 193                           bool tx_dif_conn_err_en)
 194 {
 195         /* Remaining data to be received in bytes. Used in validations*/
 196         ustorm_st_context->rem_rcv_len = cpu_to_le32(remaining_recv_len);
 197         ustorm_ag_context->exp_data_acked = cpu_to_le32(expected_data_transfer_len);
 198         ustorm_st_context->exp_data_transfer_len = cpu_to_le32(expected_data_transfer_len);
 199         SET_FIELD(ustorm_st_context->reg1_map, REG1_NUM_SGES, num_sges);
 200         SET_FIELD(ustorm_ag_context->flags2, E5_USTORM_NVMETCP_TASK_AG_CTX_DIF_ERROR_CF_EN,
 201                   tx_dif_conn_err_en ? 1 : 0);
 202 }
 203
 204 /* The following function initializes Local Completion Contexts: */
 205 static inline void
 206 set_local_completion_context(struct e5_nvmetcp_task_context *context)
 207 {
 208         SET_FIELD(context->ystorm_st_context.state.flags,
 209                   YSTORM_NVMETCP_TASK_STATE_LOCAL_COMP, 1);
 210         SET_FIELD(context->ustorm_st_context.flags,
 211                   USTORM_NVMETCP_TASK_ST_CTX_LOCAL_COMP, 1);
 212 }
 213
 214 /* Common Fastpath task init function: */
 215 static inline void
 216 init_rw_nvmetcp_task(struct nvmetcp_task_params *task_params,
 217                      enum nvmetcp_task_type task_type,
 218                      void *pdu_header, void *nvme_cmd,
 219                      struct storage_sgl_task_params *sgl_task_params)
 220 {
 221         struct e5_nvmetcp_task_context *context = task_params->context;
 222         u32 task_size = calc_rw_task_size(task_params, task_type);
 223         bool slow_io = false;
 224         u8 num_sges = 0;
 225
 226         init_default_nvmetcp_task(task_params, pdu_header, nvme_cmd, task_type);
 227
 228         /* Tx/Rx: */
 229         if (task_params->tx_io_size) {
 230                 /* if data to transmit: */
 231                 init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
 232                                       &context->ystorm_st_context.state.data_desc,
 233                                       sgl_task_params);
 234                 slow_io = nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
 235                                               sgl_task_params->small_mid_sge);
 236                 num_sges =
 237                         (u8)(!slow_io ? min((u32)sgl_task_params->num_sges,
 238                                             (u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
 239                                             NVMETCP_WQE_NUM_SGES_SLOWIO);
 240                 if (slow_io) {
 241                         SET_FIELD(context->ystorm_st_context.state.flags,
 242                                   YSTORM_NVMETCP_TASK_STATE_SLOW_IO, 1);
 243                 }
 244         } else if (task_params->rx_io_size) {
 245                 /* if data to receive: */
 246                 init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
 247                                       &context->mstorm_st_context.data_desc,
 248                                       sgl_task_params);
 249                 num_sges =
 250                         (u8)(!nvmetcp_is_slow_sgl(sgl_task_params->num_sges,
 251                                                   sgl_task_params->small_mid_sge) ?
 252                                                   min((u32)sgl_task_params->num_sges,
 253                                                       (u32)SCSI_NUM_SGES_SLOW_SGL_THR) :
 254                                                       NVMETCP_WQE_NUM_SGES_SLOWIO);
 255                 context->mstorm_st_context.rem_task_size = cpu_to_le32(task_size);
 256         }
 257
 258         /* Ustorm context: */
 259         init_ustorm_task_contexts(&context->ustorm_st_context,
 260                                   &context->ustorm_ag_context,
 261                                   /* Remaining Receive length is the Task Size */
 262                                   task_size,
 263                                   /* The size of the transmitted task */
 264                                   task_size,
 265                                   /* num_sges */
 266                                   num_sges,
 267                                   false);
 268
 269         /* Set exp_data_acked */
 270         if (task_type == NVMETCP_TASK_TYPE_HOST_WRITE) {
 271                 if (task_params->send_write_incapsule)
 272                         context->ustorm_ag_context.exp_data_acked = task_size;
 273                 else
 274                         context->ustorm_ag_context.exp_data_acked = 0;
 275         } else if (task_type == NVMETCP_TASK_TYPE_HOST_READ) {
 276                 context->ustorm_ag_context.exp_data_acked = 0;
 277         }
 278
 279         context->ustorm_ag_context.exp_cont_len = 0;
 280         init_sqe(task_params, sgl_task_params, task_type);
 281 }
 282
 283 static void
 284 init_common_initiator_read_task(struct nvmetcp_task_params *task_params,
 285                                 struct nvme_tcp_cmd_pdu *cmd_pdu_header,
 286                                 struct nvme_command *nvme_cmd,
 287                                 struct storage_sgl_task_params *sgl_task_params)
 288 {
 289         init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_READ,
 290                              cmd_pdu_header, nvme_cmd, sgl_task_params);
 291 }
 292
 293 void init_nvmetcp_host_read_task(struct nvmetcp_task_params *task_params,
 294                                  struct nvme_tcp_cmd_pdu *cmd_pdu_header,
 295                                  struct nvme_command *nvme_cmd,
 296                                  struct storage_sgl_task_params *sgl_task_params)
 297 {
 298         init_common_initiator_read_task(task_params, (void *)cmd_pdu_header,
 299                                         (void *)nvme_cmd, sgl_task_params);
 300 }
 301
 302 static void
 303 init_common_initiator_write_task(struct nvmetcp_task_params *task_params,
 304                                  struct nvme_tcp_cmd_pdu *cmd_pdu_header,
 305                                  struct nvme_command *nvme_cmd,
 306                                  struct storage_sgl_task_params *sgl_task_params)
 307 {
 308         init_rw_nvmetcp_task(task_params, NVMETCP_TASK_TYPE_HOST_WRITE,
 309                              cmd_pdu_header, nvme_cmd, sgl_task_params);
 310 }
 311
 312 void init_nvmetcp_host_write_task(struct nvmetcp_task_params *task_params,
 313                                   struct nvme_tcp_cmd_pdu *cmd_pdu_header,
 314                                   struct nvme_command *nvme_cmd,
 315                                   struct storage_sgl_task_params *sgl_task_params)
 316 {
 317         init_common_initiator_write_task(task_params, (void *)cmd_pdu_header,
 318                                          (void *)nvme_cmd, sgl_task_params);
 319 }
 320
 321 static void
 322 init_common_login_request_task(struct nvmetcp_task_params *task_params,
 323                                void *login_req_pdu_header,
 324                                struct storage_sgl_task_params *tx_sgl_task_params,
 325                                struct storage_sgl_task_params *rx_sgl_task_params)
 326 {
 327         struct e5_nvmetcp_task_context *context = task_params->context;
 328
 329         init_default_nvmetcp_task(task_params, (void *)login_req_pdu_header, NULL,
 330                                   NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
 331
 332         /* Ustorm Context: */
 333         init_ustorm_task_contexts(&context->ustorm_st_context,
 334                                   &context->ustorm_ag_context,
 335
 336                                   /* Remaining Receive length is the Task Size */
 337                                   task_params->rx_io_size ?
 338                                   rx_sgl_task_params->total_buffer_size : 0,
 339
 340                                   /* The size of the transmitted task */
 341                                   task_params->tx_io_size ?
 342                                   tx_sgl_task_params->total_buffer_size : 0,
 343                                   0, /* num_sges */
 344                                   0); /* tx_dif_conn_err_en */
 345
 346         /* SGL context: */
 347         if (task_params->tx_io_size)
 348                 init_scsi_sgl_context(&context->ystorm_st_context.state.sgl_params,
 349                                       &context->ystorm_st_context.state.data_desc,
 350                                       tx_sgl_task_params);
 351         if (task_params->rx_io_size)
 352                 init_scsi_sgl_context(&context->mstorm_st_context.sgl_params,
 353                                       &context->mstorm_st_context.data_desc,
 354                                       rx_sgl_task_params);
 355
 356         context->mstorm_st_context.rem_task_size =
 357                 cpu_to_le32(task_params->rx_io_size ?
 358                                  rx_sgl_task_params->total_buffer_size : 0);
 359         init_sqe(task_params, tx_sgl_task_params, NVMETCP_TASK_TYPE_INIT_CONN_REQUEST);
 360 }
 361
 362 /* The following function initializes Login task in Host mode: */
 363 void init_nvmetcp_init_conn_req_task(struct nvmetcp_task_params *task_params,
 364                                      struct nvme_tcp_icreq_pdu *init_conn_req_pdu_hdr,
 365                                      struct storage_sgl_task_params *tx_sgl_task_params,
 366                                      struct storage_sgl_task_params *rx_sgl_task_params)
 367 {
 368         init_common_login_request_task(task_params, init_conn_req_pdu_hdr,
 369                                        tx_sgl_task_params, rx_sgl_task_params);
 370 }
 371
 372 void init_cleanup_task_nvmetcp(struct nvmetcp_task_params *task_params)
 373 {
 374         init_sqe(task_params, NULL, NVMETCP_TASK_TYPE_CLEANUP);
 375 }