1 /*******************************************************************
2 * This file is part of the Emulex Linux Device Driver for *
3 * Fibre Channel Host Bus Adapters. *
4 * Copyright (C) 2017 Broadcom. All Rights Reserved. The term *
5 * “Broadcom” refers to Broadcom Limited and/or its subsidiaries. *
6 * Copyright (C) 2004-2016 Emulex. All rights reserved. *
7 * EMULEX and SLI are trademarks of Emulex. *
9 * Portions Copyright (C) 2004-2005 Christoph Hellwig *
11 * This program is free software; you can redistribute it and/or *
12 * modify it under the terms of version 2 of the GNU General *
13 * Public License as published by the Free Software Foundation. *
14 * This program is distributed in the hope that it will be useful. *
15 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
16 * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
17 * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
18 * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
19 * TO BE LEGALLY INVALID. See the GNU General Public License for *
20 * more details, a copy of which can be found in the file COPYING *
21 * included with this package. *
22 ********************************************************************/
23 #include <linux/pci.h>
24 #include <linux/slab.h>
25 #include <linux/interrupt.h>
26 #include <linux/delay.h>
27 #include <asm/unaligned.h>
28 #include <linux/crc-t10dif.h>
29 #include <net/checksum.h>
31 #include <scsi/scsi.h>
32 #include <scsi/scsi_device.h>
33 #include <scsi/scsi_eh.h>
34 #include <scsi/scsi_host.h>
35 #include <scsi/scsi_tcq.h>
36 #include <scsi/scsi_transport_fc.h>
37 #include <scsi/fc/fc_fs.h>
39 #include <linux/nvme.h>
40 #include <linux/nvme-fc-driver.h>
41 #include <linux/nvme-fc.h>
42 #include "lpfc_version.h"
46 #include "lpfc_sli4.h"
48 #include "lpfc_disc.h"
50 #include "lpfc_nvme.h"
51 #include "lpfc_scsi.h"
52 #include "lpfc_logmsg.h"
53 #include "lpfc_crtn.h"
54 #include "lpfc_vport.h"
55 #include "lpfc_debugfs.h"
57 /* NVME initiator-based functions */
59 static struct lpfc_nvme_buf
*
60 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
,
64 lpfc_release_nvme_buf(struct lpfc_hba
*, struct lpfc_nvme_buf
*);
66 static struct nvme_fc_port_template lpfc_nvme_template
;
69 * lpfc_nvme_create_queue -
70 * @lpfc_pnvme: Pointer to the driver's nvme instance data
71 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
72 * @handle: An opaque driver handle used in follow-up calls.
74 * Driver registers this routine to preallocate and initialize any
75 * internal data structures to bind the @qidx to its internal IO queues.
76 * A hardware queue maps (qidx) to a specific driver MSI-X vector/EQ/CQ/WQ.
80 * -EINVAL - Unsupported input value.
81 * -ENOMEM - Could not alloc necessary memory
84 lpfc_nvme_create_queue(struct nvme_fc_local_port
*pnvme_lport
,
85 unsigned int qidx
, u16 qsize
,
88 struct lpfc_nvme_lport
*lport
;
89 struct lpfc_vport
*vport
;
90 struct lpfc_nvme_qhandle
*qhandle
;
93 if (!pnvme_lport
->private)
96 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
98 qhandle
= kzalloc(sizeof(struct lpfc_nvme_qhandle
), GFP_KERNEL
);
102 qhandle
->cpu_id
= smp_processor_id();
103 qhandle
->qidx
= qidx
;
105 * NVME qidx == 0 is the admin queue, so both admin queue
106 * and first IO queue will use MSI-X vector and associated
107 * EQ/CQ/WQ at index 0. After that they are sequentially assigned.
110 str
= "IO "; /* IO queue */
111 qhandle
->index
= ((qidx
- 1) %
112 vport
->phba
->cfg_nvme_io_channel
);
114 str
= "ADM"; /* Admin queue */
115 qhandle
->index
= qidx
;
118 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
119 "6073 Binding %s HdwQueue %d (cpu %d) to "
120 "io_channel %d qhandle %p\n", str
,
121 qidx
, qhandle
->cpu_id
, qhandle
->index
, qhandle
);
122 *handle
= (void *)qhandle
;
127 * lpfc_nvme_delete_queue -
128 * @lpfc_pnvme: Pointer to the driver's nvme instance data
129 * @qidx: An cpu index used to affinitize IO queues and MSIX vectors.
130 * @handle: An opaque driver handle from lpfc_nvme_create_queue
132 * Driver registers this routine to free
133 * any internal data structures to bind the @qidx to its internal
138 * TODO: What are the failure codes.
141 lpfc_nvme_delete_queue(struct nvme_fc_local_port
*pnvme_lport
,
145 struct lpfc_nvme_lport
*lport
;
146 struct lpfc_vport
*vport
;
148 if (!pnvme_lport
->private)
151 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
152 vport
= lport
->vport
;
154 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
155 "6001 ENTER. lpfc_pnvme %p, qidx x%xi qhandle %p\n",
156 lport
, qidx
, handle
);
161 lpfc_nvme_localport_delete(struct nvme_fc_local_port
*localport
)
163 struct lpfc_nvme_lport
*lport
= localport
->private;
165 lpfc_printf_vlog(lport
->vport
, KERN_INFO
, LOG_NVME
,
166 "6173 localport %p delete complete\n",
169 /* release any threads waiting for the unreg to complete */
170 complete(&lport
->lport_unreg_done
);
173 /* lpfc_nvme_remoteport_delete
175 * @remoteport: Pointer to an nvme transport remoteport instance.
177 * This is a template downcall. NVME transport calls this function
178 * when it has completed the unregistration of a previously
179 * registered remoteport.
185 lpfc_nvme_remoteport_delete(struct nvme_fc_remote_port
*remoteport
)
187 struct lpfc_nvme_rport
*rport
= remoteport
->private;
188 struct lpfc_vport
*vport
;
189 struct lpfc_nodelist
*ndlp
;
199 /* Remove this rport from the lport's list - memory is owned by the
200 * transport. Remove the ndlp reference for the NVME transport before
201 * calling state machine to remove the node.
203 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
204 "6146 remoteport delete of remoteport %p\n",
206 spin_lock_irq(&vport
->phba
->hbalock
);
208 spin_unlock_irq(&vport
->phba
->hbalock
);
210 /* Remove original register reference. The host transport
211 * won't reference this rport/remoteport any further.
220 lpfc_nvme_cmpl_gen_req(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdwqe
,
221 struct lpfc_wcqe_complete
*wcqe
)
223 struct lpfc_vport
*vport
= cmdwqe
->vport
;
224 struct lpfc_nvme_lport
*lport
;
226 struct nvmefc_ls_req
*pnvme_lsreq
;
227 struct lpfc_dmabuf
*buf_ptr
;
228 struct lpfc_nodelist
*ndlp
;
230 atomic_inc(&vport
->phba
->fc4NvmeLsCmpls
);
232 pnvme_lsreq
= (struct nvmefc_ls_req
*)cmdwqe
->context2
;
233 status
= bf_get(lpfc_wcqe_c_status
, wcqe
) & LPFC_IOCB_STATUS_MASK
;
235 lport
= (struct lpfc_nvme_lport
*)vport
->localport
->private;
236 if (bf_get(lpfc_wcqe_c_xb
, wcqe
))
237 atomic_inc(&lport
->cmpl_ls_xb
);
238 atomic_inc(&lport
->cmpl_ls_err
);
241 ndlp
= (struct lpfc_nodelist
*)cmdwqe
->context1
;
242 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
243 "6047 nvme cmpl Enter "
244 "Data %p DID %x Xri: %x status %x cmd:%p lsreg:%p "
246 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
247 cmdwqe
->sli4_xritag
, status
,
248 cmdwqe
, pnvme_lsreq
, cmdwqe
->context3
, ndlp
);
250 lpfc_nvmeio_data(phba
, "NVME LS CMPL: xri x%x stat x%x parm x%x\n",
251 cmdwqe
->sli4_xritag
, status
, wcqe
->parameter
);
253 if (cmdwqe
->context3
) {
254 buf_ptr
= (struct lpfc_dmabuf
*)cmdwqe
->context3
;
255 lpfc_mbuf_free(phba
, buf_ptr
->virt
, buf_ptr
->phys
);
257 cmdwqe
->context3
= NULL
;
259 if (pnvme_lsreq
->done
)
260 pnvme_lsreq
->done(pnvme_lsreq
, status
);
262 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
263 "6046 nvme cmpl without done call back? "
264 "Data %p DID %x Xri: %x status %x\n",
265 pnvme_lsreq
, ndlp
? ndlp
->nlp_DID
: 0,
266 cmdwqe
->sli4_xritag
, status
);
269 cmdwqe
->context1
= NULL
;
271 lpfc_sli_release_iocbq(phba
, cmdwqe
);
275 lpfc_nvme_gen_req(struct lpfc_vport
*vport
, struct lpfc_dmabuf
*bmp
,
276 struct lpfc_dmabuf
*inp
,
277 struct nvmefc_ls_req
*pnvme_lsreq
,
278 void (*cmpl
)(struct lpfc_hba
*, struct lpfc_iocbq
*,
279 struct lpfc_wcqe_complete
*),
280 struct lpfc_nodelist
*ndlp
, uint32_t num_entry
,
281 uint32_t tmo
, uint8_t retry
)
283 struct lpfc_hba
*phba
= vport
->phba
;
285 struct lpfc_iocbq
*genwqe
;
286 struct ulp_bde64
*bpl
;
287 struct ulp_bde64 bde
;
288 int i
, rc
, xmit_len
, first_len
;
290 /* Allocate buffer for command WQE */
291 genwqe
= lpfc_sli_get_iocbq(phba
);
296 memset(wqe
, 0, sizeof(union lpfc_wqe
));
298 genwqe
->context3
= (uint8_t *)bmp
;
299 genwqe
->iocb_flag
|= LPFC_IO_NVME_LS
;
301 /* Save for completion so we can release these resources */
302 genwqe
->context1
= lpfc_nlp_get(ndlp
);
303 genwqe
->context2
= (uint8_t *)pnvme_lsreq
;
304 /* Fill in payload, bp points to frame payload */
307 /* FC spec states we need 3 * ratov for CT requests */
308 tmo
= (3 * phba
->fc_ratov
);
310 /* For this command calculate the xmit length of the request bde. */
313 bpl
= (struct ulp_bde64
*)bmp
->virt
;
314 for (i
= 0; i
< num_entry
; i
++) {
315 bde
.tus
.w
= bpl
[i
].tus
.w
;
316 if (bde
.tus
.f
.bdeFlags
!= BUFF_TYPE_BDE_64
)
318 xmit_len
+= bde
.tus
.f
.bdeSize
;
320 first_len
= xmit_len
;
323 genwqe
->rsvd2
= num_entry
;
324 genwqe
->hba_wqidx
= 0;
327 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64
;
328 wqe
->generic
.bde
.tus
.f
.bdeSize
= first_len
;
329 wqe
->generic
.bde
.addrLow
= bpl
[0].addrLow
;
330 wqe
->generic
.bde
.addrHigh
= bpl
[0].addrHigh
;
333 wqe
->gen_req
.request_payload_len
= first_len
;
338 bf_set(wqe_dfctl
, &wqe
->gen_req
.wge_ctl
, 0);
339 bf_set(wqe_si
, &wqe
->gen_req
.wge_ctl
, 1);
340 bf_set(wqe_la
, &wqe
->gen_req
.wge_ctl
, 1);
341 bf_set(wqe_rctl
, &wqe
->gen_req
.wge_ctl
, FC_RCTL_ELS4_REQ
);
342 bf_set(wqe_type
, &wqe
->gen_req
.wge_ctl
, FC_TYPE_NVME
);
345 bf_set(wqe_ctxt_tag
, &wqe
->gen_req
.wqe_com
,
346 phba
->sli4_hba
.rpi_ids
[ndlp
->nlp_rpi
]);
347 bf_set(wqe_xri_tag
, &wqe
->gen_req
.wqe_com
, genwqe
->sli4_xritag
);
350 bf_set(wqe_tmo
, &wqe
->gen_req
.wqe_com
, (vport
->phba
->fc_ratov
-1));
351 bf_set(wqe_class
, &wqe
->gen_req
.wqe_com
, CLASS3
);
352 bf_set(wqe_cmnd
, &wqe
->gen_req
.wqe_com
, CMD_GEN_REQUEST64_WQE
);
353 bf_set(wqe_ct
, &wqe
->gen_req
.wqe_com
, SLI4_CT_RPI
);
356 wqe
->gen_req
.wqe_com
.abort_tag
= genwqe
->iotag
;
359 bf_set(wqe_reqtag
, &wqe
->gen_req
.wqe_com
, genwqe
->iotag
);
362 bf_set(wqe_dbde
, &wqe
->gen_req
.wqe_com
, 1);
363 bf_set(wqe_iod
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_IOD_READ
);
364 bf_set(wqe_qosd
, &wqe
->gen_req
.wqe_com
, 1);
365 bf_set(wqe_lenloc
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
366 bf_set(wqe_ebde_cnt
, &wqe
->gen_req
.wqe_com
, 0);
369 bf_set(wqe_cqid
, &wqe
->gen_req
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
370 bf_set(wqe_cmd_type
, &wqe
->gen_req
.wqe_com
, OTHER_COMMAND
);
373 /* Issue GEN REQ WQE for NPORT <did> */
374 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_ELS
,
375 "6050 Issue GEN REQ WQE to NPORT x%x "
376 "Data: x%x x%x wq:%p lsreq:%p bmp:%p xmit:%d 1st:%d\n",
377 ndlp
->nlp_DID
, genwqe
->iotag
,
379 genwqe
, pnvme_lsreq
, bmp
, xmit_len
, first_len
);
380 genwqe
->wqe_cmpl
= cmpl
;
381 genwqe
->iocb_cmpl
= NULL
;
382 genwqe
->drvrTimeout
= tmo
+ LPFC_DRVR_TIMEOUT
;
383 genwqe
->vport
= vport
;
384 genwqe
->retry
= retry
;
386 lpfc_nvmeio_data(phba
, "NVME LS XMIT: xri x%x iotag x%x to x%06x\n",
387 genwqe
->sli4_xritag
, genwqe
->iotag
, ndlp
->nlp_DID
);
389 rc
= lpfc_sli4_issue_wqe(phba
, LPFC_ELS_RING
, genwqe
);
391 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_ELS
,
392 "6045 Issue GEN REQ WQE to NPORT x%x "
394 ndlp
->nlp_DID
, genwqe
->iotag
,
396 lpfc_sli_release_iocbq(phba
, genwqe
);
403 * lpfc_nvme_ls_req - Issue an Link Service request
404 * @lpfc_pnvme: Pointer to the driver's nvme instance data
405 * @lpfc_nvme_lport: Pointer to the driver's local port data
406 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
408 * Driver registers this routine to handle any link service request
409 * from the nvme_fc transport to a remote nvme-aware port.
413 * TODO: What are the failure codes.
416 lpfc_nvme_ls_req(struct nvme_fc_local_port
*pnvme_lport
,
417 struct nvme_fc_remote_port
*pnvme_rport
,
418 struct nvmefc_ls_req
*pnvme_lsreq
)
421 struct lpfc_nvme_lport
*lport
;
422 struct lpfc_vport
*vport
;
423 struct lpfc_nodelist
*ndlp
;
424 struct ulp_bde64
*bpl
;
425 struct lpfc_dmabuf
*bmp
;
426 uint16_t ntype
, nstate
;
428 /* there are two dma buf in the request, actually there is one and
429 * the second one is just the start address + cmd size.
430 * Before calling lpfc_nvme_gen_req these buffers need to be wrapped
431 * in a lpfc_dmabuf struct. When freeing we just free the wrapper
432 * because the nvem layer owns the data bufs.
433 * We do not have to break these packets open, we don't care what is in
434 * them. And we do not have to look at the resonse data, we only care
435 * that we got a response. All of the caring is going to happen in the
439 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
440 vport
= lport
->vport
;
442 if (vport
->load_flag
& FC_UNLOADING
)
445 if (vport
->load_flag
& FC_UNLOADING
)
448 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
449 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
450 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
451 "6051 DID x%06x not an active rport.\n",
452 pnvme_rport
->port_id
);
456 /* The remote node has to be a mapped nvme target or an
457 * unmapped nvme initiator or it's an error.
459 ntype
= ndlp
->nlp_type
;
460 nstate
= ndlp
->nlp_state
;
461 if ((ntype
& NLP_NVME_TARGET
&& nstate
!= NLP_STE_MAPPED_NODE
) ||
462 (ntype
& NLP_NVME_INITIATOR
&& nstate
!= NLP_STE_UNMAPPED_NODE
)) {
463 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
464 "6088 DID x%06x not ready for "
465 "IO. State x%x, Type x%x\n",
466 pnvme_rport
->port_id
,
467 ndlp
->nlp_state
, ndlp
->nlp_type
);
470 bmp
= kmalloc(sizeof(struct lpfc_dmabuf
), GFP_KERNEL
);
473 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
474 "6044 Could not find node for DID %x\n",
475 pnvme_rport
->port_id
);
478 INIT_LIST_HEAD(&bmp
->list
);
479 bmp
->virt
= lpfc_mbuf_alloc(vport
->phba
, MEM_PRI
, &(bmp
->phys
));
481 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
482 "6042 Could not find node for DID %x\n",
483 pnvme_rport
->port_id
);
487 bpl
= (struct ulp_bde64
*)bmp
->virt
;
488 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rqstdma
));
489 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rqstdma
));
490 bpl
->tus
.f
.bdeFlags
= 0;
491 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rqstlen
;
492 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
495 bpl
->addrHigh
= le32_to_cpu(putPaddrHigh(pnvme_lsreq
->rspdma
));
496 bpl
->addrLow
= le32_to_cpu(putPaddrLow(pnvme_lsreq
->rspdma
));
497 bpl
->tus
.f
.bdeFlags
= BUFF_TYPE_BDE_64I
;
498 bpl
->tus
.f
.bdeSize
= pnvme_lsreq
->rsplen
;
499 bpl
->tus
.w
= le32_to_cpu(bpl
->tus
.w
);
501 /* Expand print to include key fields. */
502 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
503 "6149 ENTER. lport %p, rport %p lsreq%p rqstlen:%d "
504 "rsplen:%d %pad %pad\n",
505 pnvme_lport
, pnvme_rport
,
506 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
507 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
508 &pnvme_lsreq
->rspdma
);
510 atomic_inc(&vport
->phba
->fc4NvmeLsRequests
);
512 /* Hardcode the wait to 30 seconds. Connections are failing otherwise.
513 * This code allows it all to work.
515 ret
= lpfc_nvme_gen_req(vport
, bmp
, pnvme_lsreq
->rqstaddr
,
516 pnvme_lsreq
, lpfc_nvme_cmpl_gen_req
,
518 if (ret
!= WQE_SUCCESS
) {
519 atomic_inc(&lport
->xmt_ls_err
);
520 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
521 "6052 EXIT. issue ls wqe failed lport %p, "
522 "rport %p lsreq%p Status %x DID %x\n",
523 pnvme_lport
, pnvme_rport
, pnvme_lsreq
,
525 lpfc_mbuf_free(vport
->phba
, bmp
->virt
, bmp
->phys
);
530 /* Stub in routine and return 0 for now. */
535 * lpfc_nvme_ls_abort - Issue an Link Service request
536 * @lpfc_pnvme: Pointer to the driver's nvme instance data
537 * @lpfc_nvme_lport: Pointer to the driver's local port data
538 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
540 * Driver registers this routine to handle any link service request
541 * from the nvme_fc transport to a remote nvme-aware port.
545 * TODO: What are the failure codes.
548 lpfc_nvme_ls_abort(struct nvme_fc_local_port
*pnvme_lport
,
549 struct nvme_fc_remote_port
*pnvme_rport
,
550 struct nvmefc_ls_req
*pnvme_lsreq
)
552 struct lpfc_nvme_lport
*lport
;
553 struct lpfc_vport
*vport
;
554 struct lpfc_hba
*phba
;
555 struct lpfc_nodelist
*ndlp
;
556 LIST_HEAD(abort_list
);
557 struct lpfc_sli_ring
*pring
;
558 struct lpfc_iocbq
*wqe
, *next_wqe
;
560 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
561 vport
= lport
->vport
;
564 if (vport
->load_flag
& FC_UNLOADING
)
567 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
569 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
570 "6049 Could not find node for DID %x\n",
571 pnvme_rport
->port_id
);
575 /* Expand print to include key fields. */
576 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
577 "6040 ENTER. lport %p, rport %p lsreq %p rqstlen:%d "
578 "rsplen:%d %pad %pad\n",
579 pnvme_lport
, pnvme_rport
,
580 pnvme_lsreq
, pnvme_lsreq
->rqstlen
,
581 pnvme_lsreq
->rsplen
, &pnvme_lsreq
->rqstdma
,
582 &pnvme_lsreq
->rspdma
);
585 * Lock the ELS ring txcmplq and build a local list of all ELS IOs
586 * that need an ABTS. The IOs need to stay on the txcmplq so that
587 * the abort operation completes them successfully.
589 pring
= phba
->sli4_hba
.nvmels_wq
->pring
;
590 spin_lock_irq(&phba
->hbalock
);
591 spin_lock(&pring
->ring_lock
);
592 list_for_each_entry_safe(wqe
, next_wqe
, &pring
->txcmplq
, list
) {
593 /* Add to abort_list on on NDLP match. */
594 if (lpfc_check_sli_ndlp(phba
, pring
, wqe
, ndlp
)) {
595 wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
596 list_add_tail(&wqe
->dlist
, &abort_list
);
599 spin_unlock(&pring
->ring_lock
);
600 spin_unlock_irq(&phba
->hbalock
);
602 /* Abort the targeted IOs and remove them from the abort list. */
603 list_for_each_entry_safe(wqe
, next_wqe
, &abort_list
, dlist
) {
604 atomic_inc(&lport
->xmt_ls_abort
);
605 spin_lock_irq(&phba
->hbalock
);
606 list_del_init(&wqe
->dlist
);
607 lpfc_sli_issue_abort_iotag(phba
, pring
, wqe
);
608 spin_unlock_irq(&phba
->hbalock
);
612 /* Fix up the existing sgls for NVME IO. */
614 lpfc_nvme_adj_fcp_sgls(struct lpfc_vport
*vport
,
615 struct lpfc_nvme_buf
*lpfc_ncmd
,
616 struct nvmefc_fcp_req
*nCmd
)
618 struct sli4_sge
*sgl
;
619 union lpfc_wqe128
*wqe
;
620 uint32_t *wptr
, *dptr
;
623 * Adjust the FCP_CMD and FCP_RSP DMA data and sge_len to
624 * match NVME. NVME sends 96 bytes. Also, use the
625 * nvme commands command and response dma addresses
626 * rather than the virtual memory to ease the restore
629 sgl
= lpfc_ncmd
->nvme_sgl
;
630 sgl
->sge_len
= cpu_to_le32(nCmd
->cmdlen
);
634 /* Setup the physical region for the FCP RSP */
635 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(nCmd
->rspdma
));
636 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(nCmd
->rspdma
));
637 sgl
->word2
= le32_to_cpu(sgl
->word2
);
639 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
641 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
642 sgl
->word2
= cpu_to_le32(sgl
->word2
);
643 sgl
->sge_len
= cpu_to_le32(nCmd
->rsplen
);
646 * Get a local pointer to the built-in wqe and correct
647 * the cmd size to match NVME's 96 bytes and fix
651 /* 128 byte wqe support here */
652 wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
654 /* Word 0-2 - NVME CMND IU (embedded payload) */
655 wqe
->generic
.bde
.tus
.f
.bdeFlags
= BUFF_TYPE_BDE_IMMED
;
656 wqe
->generic
.bde
.tus
.f
.bdeSize
= 60;
657 wqe
->generic
.bde
.addrHigh
= 0;
658 wqe
->generic
.bde
.addrLow
= 64; /* Word 16 */
661 bf_set(payload_offset_len
, &wqe
->fcp_icmd
,
662 (nCmd
->rsplen
+ nCmd
->cmdlen
));
665 bf_set(wqe_nvme
, &wqe
->fcp_icmd
.wqe_com
, 1);
666 bf_set(wqe_wqes
, &wqe
->fcp_icmd
.wqe_com
, 1);
669 * Embed the payload in the last half of the WQE
670 * WQE words 16-30 get the NVME CMD IU payload
672 * WQE words 16-19 get payload Words 1-4
673 * WQE words 20-21 get payload Words 6-7
674 * WQE words 22-29 get payload Words 16-23
676 wptr
= &wqe
->words
[16]; /* WQE ptr */
677 dptr
= (uint32_t *)nCmd
->cmdaddr
; /* payload ptr */
678 dptr
++; /* Skip Word 0 in payload */
680 *wptr
++ = *dptr
++; /* Word 1 */
681 *wptr
++ = *dptr
++; /* Word 2 */
682 *wptr
++ = *dptr
++; /* Word 3 */
683 *wptr
++ = *dptr
++; /* Word 4 */
684 dptr
++; /* Skip Word 5 in payload */
685 *wptr
++ = *dptr
++; /* Word 6 */
686 *wptr
++ = *dptr
++; /* Word 7 */
687 dptr
+= 8; /* Skip Words 8-15 in payload */
688 *wptr
++ = *dptr
++; /* Word 16 */
689 *wptr
++ = *dptr
++; /* Word 17 */
690 *wptr
++ = *dptr
++; /* Word 18 */
691 *wptr
++ = *dptr
++; /* Word 19 */
692 *wptr
++ = *dptr
++; /* Word 20 */
693 *wptr
++ = *dptr
++; /* Word 21 */
694 *wptr
++ = *dptr
++; /* Word 22 */
695 *wptr
= *dptr
; /* Word 23 */
698 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
700 lpfc_nvme_ktime(struct lpfc_hba
*phba
,
701 struct lpfc_nvme_buf
*lpfc_ncmd
)
703 uint64_t seg1
, seg2
, seg3
, seg4
;
706 if (!lpfc_ncmd
->ts_last_cmd
||
707 !lpfc_ncmd
->ts_cmd_start
||
708 !lpfc_ncmd
->ts_cmd_wqput
||
709 !lpfc_ncmd
->ts_isr_cmpl
||
710 !lpfc_ncmd
->ts_data_nvme
)
713 if (lpfc_ncmd
->ts_data_nvme
< lpfc_ncmd
->ts_cmd_start
)
715 if (lpfc_ncmd
->ts_cmd_start
< lpfc_ncmd
->ts_last_cmd
)
717 if (lpfc_ncmd
->ts_cmd_wqput
< lpfc_ncmd
->ts_cmd_start
)
719 if (lpfc_ncmd
->ts_isr_cmpl
< lpfc_ncmd
->ts_cmd_wqput
)
721 if (lpfc_ncmd
->ts_data_nvme
< lpfc_ncmd
->ts_isr_cmpl
)
724 * Segment 1 - Time from Last FCP command cmpl is handed
725 * off to NVME Layer to start of next command.
726 * Segment 2 - Time from Driver receives a IO cmd start
727 * from NVME Layer to WQ put is done on IO cmd.
728 * Segment 3 - Time from Driver WQ put is done on IO cmd
729 * to MSI-X ISR for IO cmpl.
730 * Segment 4 - Time from MSI-X ISR for IO cmpl to when
731 * cmpl is handled off to the NVME Layer.
733 seg1
= lpfc_ncmd
->ts_cmd_start
- lpfc_ncmd
->ts_last_cmd
;
734 if (seg1
> 5000000) /* 5 ms - for sequential IOs only */
737 /* Calculate times relative to start of IO */
738 seg2
= (lpfc_ncmd
->ts_cmd_wqput
- lpfc_ncmd
->ts_cmd_start
);
740 seg3
= lpfc_ncmd
->ts_isr_cmpl
- lpfc_ncmd
->ts_cmd_start
;
746 seg4
= lpfc_ncmd
->ts_data_nvme
- lpfc_ncmd
->ts_cmd_start
;
751 phba
->ktime_data_samples
++;
752 phba
->ktime_seg1_total
+= seg1
;
753 if (seg1
< phba
->ktime_seg1_min
)
754 phba
->ktime_seg1_min
= seg1
;
755 else if (seg1
> phba
->ktime_seg1_max
)
756 phba
->ktime_seg1_max
= seg1
;
757 phba
->ktime_seg2_total
+= seg2
;
758 if (seg2
< phba
->ktime_seg2_min
)
759 phba
->ktime_seg2_min
= seg2
;
760 else if (seg2
> phba
->ktime_seg2_max
)
761 phba
->ktime_seg2_max
= seg2
;
762 phba
->ktime_seg3_total
+= seg3
;
763 if (seg3
< phba
->ktime_seg3_min
)
764 phba
->ktime_seg3_min
= seg3
;
765 else if (seg3
> phba
->ktime_seg3_max
)
766 phba
->ktime_seg3_max
= seg3
;
767 phba
->ktime_seg4_total
+= seg4
;
768 if (seg4
< phba
->ktime_seg4_min
)
769 phba
->ktime_seg4_min
= seg4
;
770 else if (seg4
> phba
->ktime_seg4_max
)
771 phba
->ktime_seg4_max
= seg4
;
773 lpfc_ncmd
->ts_last_cmd
= 0;
774 lpfc_ncmd
->ts_cmd_start
= 0;
775 lpfc_ncmd
->ts_cmd_wqput
= 0;
776 lpfc_ncmd
->ts_isr_cmpl
= 0;
777 lpfc_ncmd
->ts_data_nvme
= 0;
782 * lpfc_nvme_io_cmd_wqe_cmpl - Complete an NVME-over-FCP IO
783 * @lpfc_pnvme: Pointer to the driver's nvme instance data
784 * @lpfc_nvme_lport: Pointer to the driver's local port data
785 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
787 * Driver registers this routine as it io request handler. This
788 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
789 * data structure to the rport indicated in @lpfc_nvme_rport.
793 * TODO: What are the failure codes.
796 lpfc_nvme_io_cmd_wqe_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*pwqeIn
,
797 struct lpfc_wcqe_complete
*wcqe
)
799 struct lpfc_nvme_buf
*lpfc_ncmd
=
800 (struct lpfc_nvme_buf
*)pwqeIn
->context1
;
801 struct lpfc_vport
*vport
= pwqeIn
->vport
;
802 struct nvmefc_fcp_req
*nCmd
;
803 struct nvme_fc_ersp_iu
*ep
;
804 struct nvme_fc_cmd_iu
*cp
;
805 struct lpfc_nvme_rport
*rport
;
806 struct lpfc_nodelist
*ndlp
;
807 struct lpfc_nvme_fcpreq_priv
*freqpriv
;
808 struct lpfc_nvme_lport
*lport
;
810 uint32_t code
, status
;
811 uint16_t cid
, sqhd
, data
;
814 /* Sanity check on return of outstanding command */
815 if (!lpfc_ncmd
|| !lpfc_ncmd
->nvmeCmd
|| !lpfc_ncmd
->nrport
) {
816 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
817 "6071 Completion pointers bad on wqe %p.\n",
821 atomic_inc(&phba
->fc4NvmeIoCmpls
);
823 nCmd
= lpfc_ncmd
->nvmeCmd
;
824 rport
= lpfc_ncmd
->nrport
;
825 status
= bf_get(lpfc_wcqe_c_status
, wcqe
);
827 lport
= (struct lpfc_nvme_lport
*)vport
->localport
->private;
828 if (bf_get(lpfc_wcqe_c_xb
, wcqe
))
829 atomic_inc(&lport
->cmpl_fcp_xb
);
830 atomic_inc(&lport
->cmpl_fcp_err
);
833 lpfc_nvmeio_data(phba
, "NVME FCP CMPL: xri x%x stat x%x parm x%x\n",
834 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
835 status
, wcqe
->parameter
);
837 * Catch race where our node has transitioned, but the
838 * transport is still transitioning.
841 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
842 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
843 "6061 rport %p, DID x%06x node not ready.\n",
844 rport
, rport
->remoteport
->port_id
);
846 ndlp
= lpfc_findnode_did(vport
, rport
->remoteport
->port_id
);
848 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
849 "6062 Ignoring NVME cmpl. No ndlp\n");
854 code
= bf_get(lpfc_wcqe_c_code
, wcqe
);
855 if (code
== CQE_CODE_NVME_ERSP
) {
856 /* For this type of CQE, we need to rebuild the rsp */
857 ep
= (struct nvme_fc_ersp_iu
*)nCmd
->rspaddr
;
860 * Get Command Id from cmd to plug into response. This
861 * code is not needed in the next NVME Transport drop.
863 cp
= (struct nvme_fc_cmd_iu
*)nCmd
->cmdaddr
;
864 cid
= cp
->sqe
.common
.command_id
;
867 * RSN is in CQE word 2
868 * SQHD is in CQE Word 3 bits 15:0
869 * Cmd Specific info is in CQE Word 1
870 * and in CQE Word 0 bits 15:0
872 sqhd
= bf_get(lpfc_wcqe_c_sqhead
, wcqe
);
874 /* Now lets build the NVME ERSP IU */
875 ep
->iu_len
= cpu_to_be16(8);
876 ep
->rsn
= wcqe
->parameter
;
877 ep
->xfrd_len
= cpu_to_be32(nCmd
->payload_length
);
879 ptr
= (uint32_t *)&ep
->cqe
.result
.u64
;
880 *ptr
++ = wcqe
->total_data_placed
;
881 data
= bf_get(lpfc_wcqe_c_ersp0
, wcqe
);
882 *ptr
= (uint32_t)data
;
883 ep
->cqe
.sq_head
= sqhd
;
884 ep
->cqe
.sq_id
= nCmd
->sqid
;
885 ep
->cqe
.command_id
= cid
;
888 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
889 lpfc_ncmd
->result
= 0;
890 nCmd
->rcv_rsplen
= LPFC_NVME_ERSP_LEN
;
891 nCmd
->transferred_length
= nCmd
->payload_length
;
893 lpfc_ncmd
->status
= (status
& LPFC_IOCB_STATUS_MASK
);
894 lpfc_ncmd
->result
= (wcqe
->parameter
& IOERR_PARAM_MASK
);
896 /* For NVME, the only failure path that results in an
897 * IO error is when the adapter rejects it. All other
898 * conditions are a success case and resolved by the
900 * IOSTAT_FCP_RSP_ERROR means:
901 * 1. Length of data received doesn't match total
902 * transfer length in WQE
903 * 2. If the RSP payload does NOT match these cases:
904 * a. RSP length 12/24 bytes and all zeros
907 switch (lpfc_ncmd
->status
) {
909 nCmd
->transferred_length
= wcqe
->total_data_placed
;
910 nCmd
->rcv_rsplen
= 0;
913 case IOSTAT_FCP_RSP_ERROR
:
914 nCmd
->transferred_length
= wcqe
->total_data_placed
;
915 nCmd
->rcv_rsplen
= wcqe
->parameter
;
918 if (nCmd
->rcv_rsplen
== LPFC_NVME_ERSP_LEN
)
920 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
921 "6081 NVME Completion Protocol Error: "
922 "xri %x status x%x result x%x "
924 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
925 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
926 wcqe
->total_data_placed
);
928 case IOSTAT_LOCAL_REJECT
:
929 /* Let fall through to set command final state. */
930 if (lpfc_ncmd
->result
== IOERR_ABORT_REQUESTED
)
931 lpfc_printf_vlog(vport
, KERN_INFO
,
933 "6032 Delay Aborted cmd %p "
934 "nvme cmd %p, xri x%x, "
937 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
938 bf_get(lpfc_wcqe_c_xb
, wcqe
));
941 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
942 "6072 NVME Completion Error: xri %x "
943 "status x%x result x%x placed x%x\n",
944 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
945 lpfc_ncmd
->status
, lpfc_ncmd
->result
,
946 wcqe
->total_data_placed
);
947 nCmd
->transferred_length
= 0;
948 nCmd
->rcv_rsplen
= 0;
949 nCmd
->status
= NVME_SC_INTERNAL
;
953 /* pick up SLI4 exhange busy condition */
954 if (bf_get(lpfc_wcqe_c_xb
, wcqe
))
955 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
957 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
959 if (ndlp
&& NLP_CHK_NODE_ACT(ndlp
))
960 atomic_dec(&ndlp
->cmd_pending
);
962 /* Update stats and complete the IO. There is
963 * no need for dma unprep because the nvme_transport
964 * owns the dma address.
966 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
967 if (lpfc_ncmd
->ts_cmd_start
) {
968 lpfc_ncmd
->ts_isr_cmpl
= pwqeIn
->isr_timestamp
;
969 lpfc_ncmd
->ts_data_nvme
= ktime_get_ns();
970 phba
->ktime_last_cmd
= lpfc_ncmd
->ts_data_nvme
;
971 lpfc_nvme_ktime(phba
, lpfc_ncmd
);
973 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
974 if (lpfc_ncmd
->cpu
!= smp_processor_id())
975 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
976 "6701 CPU Check cmpl: "
977 "cpu %d expect %d\n",
978 smp_processor_id(), lpfc_ncmd
->cpu
);
979 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
980 phba
->cpucheck_cmpl_io
[lpfc_ncmd
->cpu
]++;
983 freqpriv
= nCmd
->private;
984 freqpriv
->nvme_buf
= NULL
;
986 /* NVME targets need completion held off until the abort exchange
987 * completes unless the NVME Rport is getting unregistered.
990 if (!(lpfc_ncmd
->flags
& LPFC_SBUF_XBUSY
)) {
992 lpfc_ncmd
->nvmeCmd
= NULL
;
995 spin_lock_irqsave(&phba
->hbalock
, flags
);
996 lpfc_ncmd
->nrport
= NULL
;
997 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
999 /* Call release with XB=1 to queue the IO into the abort list. */
1000 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1005 * lpfc_nvme_prep_io_cmd - Issue an NVME-over-FCP IO
1006 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1007 * @lpfc_nvme_lport: Pointer to the driver's local port data
1008 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1009 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1010 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1012 * Driver registers this routine as it io request handler. This
1013 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1014 * data structure to the rport indicated in @lpfc_nvme_rport.
1018 * TODO: What are the failure codes.
1021 lpfc_nvme_prep_io_cmd(struct lpfc_vport
*vport
,
1022 struct lpfc_nvme_buf
*lpfc_ncmd
,
1023 struct lpfc_nodelist
*pnode
)
1025 struct lpfc_hba
*phba
= vport
->phba
;
1026 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
1027 struct lpfc_iocbq
*pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
1028 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
1031 if (!pnode
|| !NLP_CHK_NODE_ACT(pnode
))
1035 * There are three possibilities here - use scatter-gather segment, use
1036 * the single mapping, or neither.
1038 wqe
->fcp_iwrite
.initial_xfer_len
= 0;
1040 if (nCmd
->io_dir
== NVMEFC_FCP_WRITE
) {
1042 if ((phba
->cfg_nvme_enable_fb
) &&
1043 (pnode
->nlp_flag
& NLP_FIRSTBURST
)) {
1044 req_len
= lpfc_ncmd
->nvmeCmd
->payload_length
;
1045 if (req_len
< pnode
->nvme_fb_size
)
1046 wqe
->fcp_iwrite
.initial_xfer_len
=
1049 wqe
->fcp_iwrite
.initial_xfer_len
=
1050 pnode
->nvme_fb_size
;
1054 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
1055 CMD_FCP_IWRITE64_WQE
);
1056 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
1060 bf_set(wqe_qosd
, &wqe
->fcp_iwrite
.wqe_com
, 0);
1061 bf_set(wqe_iod
, &wqe
->fcp_iwrite
.wqe_com
,
1062 LPFC_WQE_IOD_WRITE
);
1063 bf_set(wqe_lenloc
, &wqe
->fcp_iwrite
.wqe_com
,
1064 LPFC_WQE_LENLOC_WORD4
);
1065 if (phba
->cfg_nvme_oas
)
1066 bf_set(wqe_oas
, &wqe
->fcp_iwrite
.wqe_com
, 1);
1069 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
1072 atomic_inc(&phba
->fc4NvmeOutputRequests
);
1075 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
,
1076 CMD_FCP_IREAD64_WQE
);
1077 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
,
1081 bf_set(wqe_qosd
, &wqe
->fcp_iread
.wqe_com
, 0);
1082 bf_set(wqe_iod
, &wqe
->fcp_iread
.wqe_com
,
1084 bf_set(wqe_lenloc
, &wqe
->fcp_iread
.wqe_com
,
1085 LPFC_WQE_LENLOC_WORD4
);
1086 if (phba
->cfg_nvme_oas
)
1087 bf_set(wqe_oas
, &wqe
->fcp_iread
.wqe_com
, 1);
1090 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
,
1093 atomic_inc(&phba
->fc4NvmeInputRequests
);
1097 wqe
->fcp_icmd
.rsrvd4
= 0;
1100 bf_set(wqe_cmnd
, &wqe
->generic
.wqe_com
, CMD_FCP_ICMND64_WQE
);
1101 bf_set(wqe_pu
, &wqe
->generic
.wqe_com
, 0);
1104 bf_set(wqe_qosd
, &wqe
->fcp_icmd
.wqe_com
, 1);
1105 bf_set(wqe_iod
, &wqe
->fcp_icmd
.wqe_com
, LPFC_WQE_IOD_WRITE
);
1106 bf_set(wqe_lenloc
, &wqe
->fcp_icmd
.wqe_com
,
1107 LPFC_WQE_LENLOC_NONE
);
1108 if (phba
->cfg_nvme_oas
)
1109 bf_set(wqe_oas
, &wqe
->fcp_icmd
.wqe_com
, 1);
1112 bf_set(wqe_cmd_type
, &wqe
->generic
.wqe_com
, NVME_READ_CMD
);
1114 atomic_inc(&phba
->fc4NvmeControlRequests
);
1117 * Finish initializing those WQE fields that are independent
1118 * of the nvme_cmnd request_buffer
1122 bf_set(wqe_ctxt_tag
, &wqe
->generic
.wqe_com
,
1123 phba
->sli4_hba
.rpi_ids
[pnode
->nlp_rpi
]);
1124 bf_set(wqe_xri_tag
, &wqe
->generic
.wqe_com
, pwqeq
->sli4_xritag
);
1127 /* Preserve Class data in the ndlp. */
1128 bf_set(wqe_class
, &wqe
->generic
.wqe_com
,
1129 (pnode
->nlp_fcp_info
& 0x0f));
1132 wqe
->generic
.wqe_com
.abort_tag
= pwqeq
->iotag
;
1135 bf_set(wqe_reqtag
, &wqe
->generic
.wqe_com
, pwqeq
->iotag
);
1138 bf_set(wqe_cqid
, &wqe
->generic
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1140 pwqeq
->vport
= vport
;
1146 * lpfc_nvme_prep_io_dma - Issue an NVME-over-FCP IO
1147 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1148 * @lpfc_nvme_lport: Pointer to the driver's local port data
1149 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1150 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1151 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1153 * Driver registers this routine as it io request handler. This
1154 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1155 * data structure to the rport indicated in @lpfc_nvme_rport.
1159 * TODO: What are the failure codes.
1162 lpfc_nvme_prep_io_dma(struct lpfc_vport
*vport
,
1163 struct lpfc_nvme_buf
*lpfc_ncmd
)
1165 struct lpfc_hba
*phba
= vport
->phba
;
1166 struct nvmefc_fcp_req
*nCmd
= lpfc_ncmd
->nvmeCmd
;
1167 union lpfc_wqe128
*wqe
= (union lpfc_wqe128
*)&lpfc_ncmd
->cur_iocbq
.wqe
;
1168 struct sli4_sge
*sgl
= lpfc_ncmd
->nvme_sgl
;
1169 struct scatterlist
*data_sg
;
1170 struct sli4_sge
*first_data_sgl
;
1171 dma_addr_t physaddr
;
1172 uint32_t num_bde
= 0;
1174 uint32_t dma_offset
= 0;
1177 /* Fix up the command and response DMA stuff. */
1178 lpfc_nvme_adj_fcp_sgls(vport
, lpfc_ncmd
, nCmd
);
1181 * There are three possibilities here - use scatter-gather segment, use
1182 * the single mapping, or neither.
1186 * Jump over the cmd and rsp SGEs. The fix routine
1187 * has already adjusted for this.
1191 first_data_sgl
= sgl
;
1192 lpfc_ncmd
->seg_cnt
= nCmd
->sg_cnt
;
1193 if (lpfc_ncmd
->seg_cnt
> lpfc_nvme_template
.max_sgl_segments
) {
1194 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1195 "6058 Too many sg segments from "
1196 "NVME Transport. Max %d, "
1197 "nvmeIO sg_cnt %d\n",
1198 phba
->cfg_nvme_seg_cnt
+ 1,
1199 lpfc_ncmd
->seg_cnt
);
1200 lpfc_ncmd
->seg_cnt
= 0;
1205 * The driver established a maximum scatter-gather segment count
1206 * during probe that limits the number of sg elements in any
1207 * single nvme command. Just run through the seg_cnt and format
1210 nseg
= nCmd
->sg_cnt
;
1211 data_sg
= nCmd
->first_sgl
;
1212 for (i
= 0; i
< nseg
; i
++) {
1213 if (data_sg
== NULL
) {
1214 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1215 "6059 dptr err %d, nseg %d\n",
1217 lpfc_ncmd
->seg_cnt
= 0;
1220 physaddr
= data_sg
->dma_address
;
1221 dma_len
= data_sg
->length
;
1222 sgl
->addr_lo
= cpu_to_le32(putPaddrLow(physaddr
));
1223 sgl
->addr_hi
= cpu_to_le32(putPaddrHigh(physaddr
));
1224 sgl
->word2
= le32_to_cpu(sgl
->word2
);
1225 if ((num_bde
+ 1) == nseg
)
1226 bf_set(lpfc_sli4_sge_last
, sgl
, 1);
1228 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
1229 bf_set(lpfc_sli4_sge_offset
, sgl
, dma_offset
);
1230 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_DATA
);
1231 sgl
->word2
= cpu_to_le32(sgl
->word2
);
1232 sgl
->sge_len
= cpu_to_le32(dma_len
);
1234 dma_offset
+= dma_len
;
1235 data_sg
= sg_next(data_sg
);
1239 /* For this clause to be valid, the payload_length
1240 * and sg_cnt must zero.
1242 if (nCmd
->payload_length
!= 0) {
1243 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
1244 "6063 NVME DMA Prep Err: sg_cnt %d "
1245 "payload_length x%x\n",
1246 nCmd
->sg_cnt
, nCmd
->payload_length
);
1252 * Due to difference in data length between DIF/non-DIF paths,
1253 * we need to set word 4 of WQE here
1255 wqe
->fcp_iread
.total_xfer_len
= nCmd
->payload_length
;
1260 * lpfc_nvme_fcp_io_submit - Issue an NVME-over-FCP IO
1261 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1262 * @lpfc_nvme_lport: Pointer to the driver's local port data
1263 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1264 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1265 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1267 * Driver registers this routine as it io request handler. This
1268 * routine issues an fcp WQE with data from the @lpfc_nvme_fcpreq
1269 * data structure to the rport
1270 indicated in @lpfc_nvme_rport.
1274 * TODO: What are the failure codes.
1277 lpfc_nvme_fcp_io_submit(struct nvme_fc_local_port
*pnvme_lport
,
1278 struct nvme_fc_remote_port
*pnvme_rport
,
1279 void *hw_queue_handle
,
1280 struct nvmefc_fcp_req
*pnvme_fcreq
)
1284 struct lpfc_nvme_lport
*lport
;
1285 struct lpfc_vport
*vport
;
1286 struct lpfc_hba
*phba
;
1287 struct lpfc_nodelist
*ndlp
;
1288 struct lpfc_nvme_buf
*lpfc_ncmd
;
1289 struct lpfc_nvme_rport
*rport
;
1290 struct lpfc_nvme_qhandle
*lpfc_queue_info
;
1291 struct lpfc_nvme_fcpreq_priv
*freqpriv
;
1292 struct nvme_common_command
*sqe
;
1293 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1297 /* Validate pointers. LLDD fault handling with transport does
1298 * have timing races.
1300 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1301 if (unlikely(!lport
)) {
1306 vport
= lport
->vport
;
1308 if (unlikely(!hw_queue_handle
)) {
1309 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1310 "6129 Fail Abort, NULL hw_queue_handle\n");
1317 if (vport
->load_flag
& FC_UNLOADING
) {
1322 if (vport
->load_flag
& FC_UNLOADING
) {
1327 freqpriv
= pnvme_fcreq
->private;
1328 if (unlikely(!freqpriv
)) {
1333 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1335 start
= ktime_get_ns();
1337 rport
= (struct lpfc_nvme_rport
*)pnvme_rport
->private;
1338 lpfc_queue_info
= (struct lpfc_nvme_qhandle
*)hw_queue_handle
;
1341 * Catch race where our node has transitioned, but the
1342 * transport is still transitioning.
1345 if (!ndlp
|| !NLP_CHK_NODE_ACT(ndlp
)) {
1346 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1347 "6053 rport %p, ndlp %p, DID x%06x "
1348 "ndlp not ready.\n",
1349 rport
, ndlp
, pnvme_rport
->port_id
);
1351 ndlp
= lpfc_findnode_did(vport
, pnvme_rport
->port_id
);
1353 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
1354 "6066 Missing node for DID %x\n",
1355 pnvme_rport
->port_id
);
1356 atomic_inc(&lport
->xmt_fcp_bad_ndlp
);
1362 /* The remote node has to be a mapped target or it's an error. */
1363 if ((ndlp
->nlp_type
& NLP_NVME_TARGET
) &&
1364 (ndlp
->nlp_state
!= NLP_STE_MAPPED_NODE
)) {
1365 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NODE
| LOG_NVME_IOERR
,
1366 "6036 rport %p, DID x%06x not ready for "
1367 "IO. State x%x, Type x%x\n",
1368 rport
, pnvme_rport
->port_id
,
1369 ndlp
->nlp_state
, ndlp
->nlp_type
);
1370 atomic_inc(&lport
->xmt_fcp_bad_ndlp
);
1376 /* Currently only NVME Keep alive commands should be expedited
1377 * if the driver runs out of a resource. These should only be
1378 * issued on the admin queue, qidx 0
1380 if (!lpfc_queue_info
->qidx
&& !pnvme_fcreq
->sg_cnt
) {
1381 sqe
= &((struct nvme_fc_cmd_iu
*)
1382 pnvme_fcreq
->cmdaddr
)->sqe
.common
;
1383 if (sqe
->opcode
== nvme_admin_keep_alive
)
1387 /* The node is shared with FCP IO, make sure the IO pending count does
1388 * not exceed the programmed depth.
1390 if ((atomic_read(&ndlp
->cmd_pending
) >= ndlp
->cmd_qdepth
) &&
1392 atomic_inc(&lport
->xmt_fcp_qdepth
);
1397 lpfc_ncmd
= lpfc_get_nvme_buf(phba
, ndlp
, expedite
);
1398 if (lpfc_ncmd
== NULL
) {
1399 atomic_inc(&lport
->xmt_fcp_noxri
);
1400 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
1401 "6065 driver's buffer pool is empty, "
1406 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1408 lpfc_ncmd
->ts_cmd_start
= start
;
1409 lpfc_ncmd
->ts_last_cmd
= phba
->ktime_last_cmd
;
1411 lpfc_ncmd
->ts_cmd_start
= 0;
1416 * Store the data needed by the driver to issue, abort, and complete
1418 * Do not let the IO hang out forever. There is no midlayer issuing
1419 * an abort so inform the FW of the maximum IO pending time.
1421 freqpriv
->nvme_buf
= lpfc_ncmd
;
1422 lpfc_ncmd
->nvmeCmd
= pnvme_fcreq
;
1423 lpfc_ncmd
->nrport
= rport
;
1424 lpfc_ncmd
->ndlp
= ndlp
;
1425 lpfc_ncmd
->start_time
= jiffies
;
1427 lpfc_nvme_prep_io_cmd(vport
, lpfc_ncmd
, ndlp
);
1428 ret
= lpfc_nvme_prep_io_dma(vport
, lpfc_ncmd
);
1431 goto out_free_nvme_buf
;
1434 atomic_inc(&ndlp
->cmd_pending
);
1437 * Issue the IO on the WQ indicated by index in the hw_queue_handle.
1438 * This identfier was create in our hardware queue create callback
1439 * routine. The driver now is dependent on the IO queue steering from
1440 * the transport. We are trusting the upper NVME layers know which
1441 * index to use and that they have affinitized a CPU to this hardware
1442 * queue. A hardware queue maps to a driver MSI-X vector/EQ/CQ/WQ.
1444 lpfc_ncmd
->cur_iocbq
.hba_wqidx
= lpfc_queue_info
->index
;
1446 lpfc_nvmeio_data(phba
, "NVME FCP XMIT: xri x%x idx %d to %06x\n",
1447 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
1448 lpfc_queue_info
->index
, ndlp
->nlp_DID
);
1450 ret
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, &lpfc_ncmd
->cur_iocbq
);
1452 atomic_inc(&lport
->xmt_fcp_wqerr
);
1453 atomic_dec(&ndlp
->cmd_pending
);
1454 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
1455 "6113 FCP could not issue WQE err %x "
1456 "sid: x%x did: x%x oxid: x%x\n",
1457 ret
, vport
->fc_myDID
, ndlp
->nlp_DID
,
1458 lpfc_ncmd
->cur_iocbq
.sli4_xritag
);
1459 goto out_free_nvme_buf
;
1462 #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
1463 if (lpfc_ncmd
->ts_cmd_start
)
1464 lpfc_ncmd
->ts_cmd_wqput
= ktime_get_ns();
1466 if (phba
->cpucheck_on
& LPFC_CHECK_NVME_IO
) {
1467 lpfc_ncmd
->cpu
= smp_processor_id();
1468 if (lpfc_ncmd
->cpu
!= lpfc_queue_info
->index
) {
1469 /* Check for admin queue */
1470 if (lpfc_queue_info
->qidx
) {
1471 lpfc_printf_vlog(vport
,
1472 KERN_ERR
, LOG_NVME_IOERR
,
1473 "6702 CPU Check cmd: "
1476 lpfc_queue_info
->index
);
1478 lpfc_ncmd
->cpu
= lpfc_queue_info
->index
;
1480 if (lpfc_ncmd
->cpu
< LPFC_CHECK_CPU_CNT
)
1481 phba
->cpucheck_xmt_io
[lpfc_ncmd
->cpu
]++;
1487 if (lpfc_ncmd
->nvmeCmd
->sg_cnt
) {
1488 if (lpfc_ncmd
->nvmeCmd
->io_dir
== NVMEFC_FCP_WRITE
)
1489 atomic_dec(&phba
->fc4NvmeOutputRequests
);
1491 atomic_dec(&phba
->fc4NvmeInputRequests
);
1493 atomic_dec(&phba
->fc4NvmeControlRequests
);
1494 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
1500 * lpfc_nvme_abort_fcreq_cmpl - Complete an NVME FCP abort request.
1501 * @phba: Pointer to HBA context object
1502 * @cmdiocb: Pointer to command iocb object.
1503 * @rspiocb: Pointer to response iocb object.
1505 * This is the callback function for any NVME FCP IO that was aborted.
1511 lpfc_nvme_abort_fcreq_cmpl(struct lpfc_hba
*phba
, struct lpfc_iocbq
*cmdiocb
,
1512 struct lpfc_wcqe_complete
*abts_cmpl
)
1514 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME
,
1515 "6145 ABORT_XRI_CN completing on rpi x%x "
1516 "original iotag x%x, abort cmd iotag x%x "
1517 "req_tag x%x, status x%x, hwstatus x%x\n",
1518 cmdiocb
->iocb
.un
.acxri
.abortContextTag
,
1519 cmdiocb
->iocb
.un
.acxri
.abortIoTag
,
1521 bf_get(lpfc_wcqe_c_request_tag
, abts_cmpl
),
1522 bf_get(lpfc_wcqe_c_status
, abts_cmpl
),
1523 bf_get(lpfc_wcqe_c_hw_status
, abts_cmpl
));
1524 lpfc_sli_release_iocbq(phba
, cmdiocb
);
1528 * lpfc_nvme_fcp_abort - Issue an NVME-over-FCP ABTS
1529 * @lpfc_pnvme: Pointer to the driver's nvme instance data
1530 * @lpfc_nvme_lport: Pointer to the driver's local port data
1531 * @lpfc_nvme_rport: Pointer to the rport getting the @lpfc_nvme_ereq
1532 * @lpfc_nvme_fcreq: IO request from nvme fc to driver.
1533 * @hw_queue_handle: Driver-returned handle in lpfc_nvme_create_queue
1535 * Driver registers this routine as its nvme request io abort handler. This
1536 * routine issues an fcp Abort WQE with data from the @lpfc_nvme_fcpreq
1537 * data structure to the rport indicated in @lpfc_nvme_rport. This routine
1538 * is executed asynchronously - one the target is validated as "MAPPED" and
1539 * ready for IO, the driver issues the abort request and returns.
1545 lpfc_nvme_fcp_abort(struct nvme_fc_local_port
*pnvme_lport
,
1546 struct nvme_fc_remote_port
*pnvme_rport
,
1547 void *hw_queue_handle
,
1548 struct nvmefc_fcp_req
*pnvme_fcreq
)
1550 struct lpfc_nvme_lport
*lport
;
1551 struct lpfc_vport
*vport
;
1552 struct lpfc_hba
*phba
;
1553 struct lpfc_nvme_buf
*lpfc_nbuf
;
1554 struct lpfc_iocbq
*abts_buf
;
1555 struct lpfc_iocbq
*nvmereq_wqe
;
1556 struct lpfc_nvme_fcpreq_priv
*freqpriv
;
1557 union lpfc_wqe
*abts_wqe
;
1558 unsigned long flags
;
1561 /* Validate pointers. LLDD fault handling with transport does
1562 * have timing races.
1564 lport
= (struct lpfc_nvme_lport
*)pnvme_lport
->private;
1565 if (unlikely(!lport
))
1568 vport
= lport
->vport
;
1570 if (unlikely(!hw_queue_handle
)) {
1571 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1572 "6129 Fail Abort, HW Queue Handle NULL.\n");
1577 freqpriv
= pnvme_fcreq
->private;
1579 if (unlikely(!freqpriv
))
1581 if (vport
->load_flag
& FC_UNLOADING
)
1584 /* Announce entry to new IO submit field. */
1585 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1586 "6002 Abort Request to rport DID x%06x "
1587 "for nvme_fc_req %p\n",
1588 pnvme_rport
->port_id
,
1591 /* If the hba is getting reset, this flag is set. It is
1592 * cleared when the reset is complete and rings reestablished.
1594 spin_lock_irqsave(&phba
->hbalock
, flags
);
1595 /* driver queued commands are in process of being flushed */
1596 if (phba
->hba_flag
& HBA_NVME_IOQ_FLUSH
) {
1597 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1598 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1599 "6139 Driver in reset cleanup - flushing "
1600 "NVME Req now. hba_flag x%x\n",
1605 lpfc_nbuf
= freqpriv
->nvme_buf
;
1607 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1608 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1609 "6140 NVME IO req has no matching lpfc nvme "
1610 "io buffer. Skipping abort req.\n");
1612 } else if (!lpfc_nbuf
->nvmeCmd
) {
1613 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1614 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1615 "6141 lpfc NVME IO req has no nvme_fcreq "
1616 "io buffer. Skipping abort req.\n");
1619 nvmereq_wqe
= &lpfc_nbuf
->cur_iocbq
;
1622 * The lpfc_nbuf and the mapped nvme_fcreq in the driver's
1623 * state must match the nvme_fcreq passed by the nvme
1624 * transport. If they don't match, it is likely the driver
1625 * has already completed the NVME IO and the nvme transport
1626 * has not seen it yet.
1628 if (lpfc_nbuf
->nvmeCmd
!= pnvme_fcreq
) {
1629 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1630 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1631 "6143 NVME req mismatch: "
1632 "lpfc_nbuf %p nvmeCmd %p, "
1633 "pnvme_fcreq %p. Skipping Abort xri x%x\n",
1634 lpfc_nbuf
, lpfc_nbuf
->nvmeCmd
,
1635 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1639 /* Don't abort IOs no longer on the pending queue. */
1640 if (!(nvmereq_wqe
->iocb_flag
& LPFC_IO_ON_TXCMPLQ
)) {
1641 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1642 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1643 "6142 NVME IO req %p not queued - skipping "
1644 "abort req xri x%x\n",
1645 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1649 atomic_inc(&lport
->xmt_fcp_abort
);
1650 lpfc_nvmeio_data(phba
, "NVME FCP ABORT: xri x%x idx %d to %06x\n",
1651 nvmereq_wqe
->sli4_xritag
,
1652 nvmereq_wqe
->hba_wqidx
, pnvme_rport
->port_id
);
1654 /* Outstanding abort is in progress */
1655 if (nvmereq_wqe
->iocb_flag
& LPFC_DRIVER_ABORTED
) {
1656 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1657 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1658 "6144 Outstanding NVME I/O Abort Request "
1659 "still pending on nvme_fcreq %p, "
1660 "lpfc_ncmd %p xri x%x\n",
1661 pnvme_fcreq
, lpfc_nbuf
,
1662 nvmereq_wqe
->sli4_xritag
);
1666 abts_buf
= __lpfc_sli_get_iocbq(phba
);
1668 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1669 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1670 "6136 No available abort wqes. Skipping "
1671 "Abts req for nvme_fcreq %p xri x%x\n",
1672 pnvme_fcreq
, nvmereq_wqe
->sli4_xritag
);
1676 /* Ready - mark outstanding as aborted by driver. */
1677 nvmereq_wqe
->iocb_flag
|= LPFC_DRIVER_ABORTED
;
1679 /* Complete prepping the abort wqe and issue to the FW. */
1680 abts_wqe
= &abts_buf
->wqe
;
1682 /* WQEs are reused. Clear stale data and set key fields to
1683 * zero like ia, iaab, iaar, xri_tag, and ctxt_tag.
1685 memset(abts_wqe
, 0, sizeof(union lpfc_wqe
));
1686 bf_set(abort_cmd_criteria
, &abts_wqe
->abort_cmd
, T_XRI_TAG
);
1689 bf_set(wqe_ct
, &abts_wqe
->abort_cmd
.wqe_com
, 0);
1690 bf_set(wqe_cmnd
, &abts_wqe
->abort_cmd
.wqe_com
, CMD_ABORT_XRI_CX
);
1691 bf_set(wqe_class
, &abts_wqe
->abort_cmd
.wqe_com
,
1692 nvmereq_wqe
->iocb
.ulpClass
);
1694 /* word 8 - tell the FW to abort the IO associated with this
1695 * outstanding exchange ID.
1697 abts_wqe
->abort_cmd
.wqe_com
.abort_tag
= nvmereq_wqe
->sli4_xritag
;
1699 /* word 9 - this is the iotag for the abts_wqe completion. */
1700 bf_set(wqe_reqtag
, &abts_wqe
->abort_cmd
.wqe_com
,
1704 bf_set(wqe_wqid
, &abts_wqe
->abort_cmd
.wqe_com
, nvmereq_wqe
->hba_wqidx
);
1705 bf_set(wqe_qosd
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1706 bf_set(wqe_lenloc
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_LENLOC_NONE
);
1709 bf_set(wqe_cmd_type
, &abts_wqe
->abort_cmd
.wqe_com
, OTHER_COMMAND
);
1710 bf_set(wqe_wqec
, &abts_wqe
->abort_cmd
.wqe_com
, 1);
1711 bf_set(wqe_cqid
, &abts_wqe
->abort_cmd
.wqe_com
, LPFC_WQE_CQ_ID_DEFAULT
);
1713 /* ABTS WQE must go to the same WQ as the WQE to be aborted */
1714 abts_buf
->iocb_flag
|= LPFC_IO_NVME
;
1715 abts_buf
->hba_wqidx
= nvmereq_wqe
->hba_wqidx
;
1716 abts_buf
->vport
= vport
;
1717 abts_buf
->wqe_cmpl
= lpfc_nvme_abort_fcreq_cmpl
;
1718 ret_val
= lpfc_sli4_issue_wqe(phba
, LPFC_FCP_RING
, abts_buf
);
1719 spin_unlock_irqrestore(&phba
->hbalock
, flags
);
1721 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_ABTS
,
1722 "6137 Failed abts issue_wqe with status x%x "
1723 "for nvme_fcreq %p.\n",
1724 ret_val
, pnvme_fcreq
);
1725 lpfc_sli_release_iocbq(phba
, abts_buf
);
1729 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_ABTS
,
1730 "6138 Transport Abort NVME Request Issued for "
1731 "ox_id x%x on reqtag x%x\n",
1732 nvmereq_wqe
->sli4_xritag
,
1736 /* Declare and initialization an instance of the FC NVME template. */
1737 static struct nvme_fc_port_template lpfc_nvme_template
= {
1738 /* initiator-based functions */
1739 .localport_delete
= lpfc_nvme_localport_delete
,
1740 .remoteport_delete
= lpfc_nvme_remoteport_delete
,
1741 .create_queue
= lpfc_nvme_create_queue
,
1742 .delete_queue
= lpfc_nvme_delete_queue
,
1743 .ls_req
= lpfc_nvme_ls_req
,
1744 .fcp_io
= lpfc_nvme_fcp_io_submit
,
1745 .ls_abort
= lpfc_nvme_ls_abort
,
1746 .fcp_abort
= lpfc_nvme_fcp_abort
,
1749 .max_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1750 .max_dif_sgl_segments
= LPFC_NVME_DEFAULT_SEGS
,
1751 .dma_boundary
= 0xFFFFFFFF,
1753 /* Sizes of additional private data for data structures.
1754 * No use for the last two sizes at this time.
1756 .local_priv_sz
= sizeof(struct lpfc_nvme_lport
),
1757 .remote_priv_sz
= sizeof(struct lpfc_nvme_rport
),
1758 .lsrqst_priv_sz
= 0,
1759 .fcprqst_priv_sz
= sizeof(struct lpfc_nvme_fcpreq_priv
),
1763 * lpfc_sli4_post_nvme_sgl_block - post a block of nvme sgl list to firmware
1764 * @phba: pointer to lpfc hba data structure.
1765 * @nblist: pointer to nvme buffer list.
1766 * @count: number of scsi buffers on the list.
1768 * This routine is invoked to post a block of @count scsi sgl pages from a
1769 * SCSI buffer list @nblist to the HBA using non-embedded mailbox command.
1774 lpfc_sli4_post_nvme_sgl_block(struct lpfc_hba
*phba
,
1775 struct list_head
*nblist
,
1778 struct lpfc_nvme_buf
*lpfc_ncmd
;
1779 struct lpfc_mbx_post_uembed_sgl_page1
*sgl
;
1780 struct sgl_page_pairs
*sgl_pg_pairs
;
1783 uint32_t reqlen
, alloclen
, pg_pairs
;
1785 uint16_t xritag_start
= 0;
1787 uint32_t shdr_status
, shdr_add_status
;
1788 dma_addr_t pdma_phys_bpl1
;
1789 union lpfc_sli4_cfg_shdr
*shdr
;
1791 /* Calculate the requested length of the dma memory */
1792 reqlen
= count
* sizeof(struct sgl_page_pairs
) +
1793 sizeof(union lpfc_sli4_cfg_shdr
) + sizeof(uint32_t);
1794 if (reqlen
> SLI4_PAGE_SIZE
) {
1795 lpfc_printf_log(phba
, KERN_WARNING
, LOG_INIT
,
1796 "6118 Block sgl registration required DMA "
1797 "size (%d) great than a page\n", reqlen
);
1800 mbox
= mempool_alloc(phba
->mbox_mem_pool
, GFP_KERNEL
);
1802 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1803 "6119 Failed to allocate mbox cmd memory\n");
1807 /* Allocate DMA memory and set up the non-embedded mailbox command */
1808 alloclen
= lpfc_sli4_config(phba
, mbox
, LPFC_MBOX_SUBSYSTEM_FCOE
,
1809 LPFC_MBOX_OPCODE_FCOE_POST_SGL_PAGES
, reqlen
,
1810 LPFC_SLI4_MBX_NEMBED
);
1812 if (alloclen
< reqlen
) {
1813 lpfc_printf_log(phba
, KERN_ERR
, LOG_INIT
,
1814 "6120 Allocated DMA memory size (%d) is "
1815 "less than the requested DMA memory "
1816 "size (%d)\n", alloclen
, reqlen
);
1817 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1821 /* Get the first SGE entry from the non-embedded DMA memory */
1822 viraddr
= mbox
->sge_array
->addr
[0];
1824 /* Set up the SGL pages in the non-embedded DMA pages */
1825 sgl
= (struct lpfc_mbx_post_uembed_sgl_page1
*)viraddr
;
1826 sgl_pg_pairs
= &sgl
->sgl_pg_pairs
;
1829 list_for_each_entry(lpfc_ncmd
, nblist
, list
) {
1830 /* Set up the sge entry */
1831 sgl_pg_pairs
->sgl_pg0_addr_lo
=
1832 cpu_to_le32(putPaddrLow(lpfc_ncmd
->dma_phys_sgl
));
1833 sgl_pg_pairs
->sgl_pg0_addr_hi
=
1834 cpu_to_le32(putPaddrHigh(lpfc_ncmd
->dma_phys_sgl
));
1835 if (phba
->cfg_sg_dma_buf_size
> SGL_PAGE_SIZE
)
1836 pdma_phys_bpl1
= lpfc_ncmd
->dma_phys_sgl
+
1840 sgl_pg_pairs
->sgl_pg1_addr_lo
=
1841 cpu_to_le32(putPaddrLow(pdma_phys_bpl1
));
1842 sgl_pg_pairs
->sgl_pg1_addr_hi
=
1843 cpu_to_le32(putPaddrHigh(pdma_phys_bpl1
));
1844 /* Keep the first xritag on the list */
1846 xritag_start
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1850 bf_set(lpfc_post_sgl_pages_xri
, sgl
, xritag_start
);
1851 bf_set(lpfc_post_sgl_pages_xricnt
, sgl
, pg_pairs
);
1852 /* Perform endian conversion if necessary */
1853 sgl
->word0
= cpu_to_le32(sgl
->word0
);
1855 if (!phba
->sli4_hba
.intr_enable
)
1856 rc
= lpfc_sli_issue_mbox(phba
, mbox
, MBX_POLL
);
1858 mbox_tmo
= lpfc_mbox_tmo_val(phba
, mbox
);
1859 rc
= lpfc_sli_issue_mbox_wait(phba
, mbox
, mbox_tmo
);
1861 shdr
= (union lpfc_sli4_cfg_shdr
*)&sgl
->cfg_shdr
;
1862 shdr_status
= bf_get(lpfc_mbox_hdr_status
, &shdr
->response
);
1863 shdr_add_status
= bf_get(lpfc_mbox_hdr_add_status
, &shdr
->response
);
1864 if (rc
!= MBX_TIMEOUT
)
1865 lpfc_sli4_mbox_cmd_free(phba
, mbox
);
1866 if (shdr_status
|| shdr_add_status
|| rc
) {
1867 lpfc_printf_log(phba
, KERN_ERR
, LOG_SLI
,
1868 "6125 POST_SGL_BLOCK mailbox command failed "
1869 "status x%x add_status x%x mbx status x%x\n",
1870 shdr_status
, shdr_add_status
, rc
);
1877 * lpfc_post_nvme_sgl_list - Post blocks of nvme buffer sgls from a list
1878 * @phba: pointer to lpfc hba data structure.
1879 * @post_nblist: pointer to the nvme buffer list.
1881 * This routine walks a list of nvme buffers that was passed in. It attempts
1882 * to construct blocks of nvme buffer sgls which contains contiguous xris and
1883 * uses the non-embedded SGL block post mailbox commands to post to the port.
1884 * For single NVME buffer sgl with non-contiguous xri, if any, it shall use
1885 * embedded SGL post mailbox command for posting. The @post_nblist passed in
1886 * must be local list, thus no lock is needed when manipulate the list.
1888 * Returns: 0 = failure, non-zero number of successfully posted buffers.
1891 lpfc_post_nvme_sgl_list(struct lpfc_hba
*phba
,
1892 struct list_head
*post_nblist
, int sb_count
)
1894 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
1895 int status
, sgl_size
;
1896 int post_cnt
= 0, block_cnt
= 0, num_posting
= 0, num_posted
= 0;
1897 dma_addr_t pdma_phys_sgl1
;
1898 int last_xritag
= NO_XRI
;
1900 LIST_HEAD(prep_nblist
);
1901 LIST_HEAD(blck_nblist
);
1902 LIST_HEAD(nvme_nblist
);
1908 sgl_size
= phba
->cfg_sg_dma_buf_size
;
1910 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
, post_nblist
, list
) {
1911 list_del_init(&lpfc_ncmd
->list
);
1913 if ((last_xritag
!= NO_XRI
) &&
1914 (lpfc_ncmd
->cur_iocbq
.sli4_xritag
!= last_xritag
+ 1)) {
1915 /* a hole in xri block, form a sgl posting block */
1916 list_splice_init(&prep_nblist
, &blck_nblist
);
1917 post_cnt
= block_cnt
- 1;
1918 /* prepare list for next posting block */
1919 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1922 /* prepare list for next posting block */
1923 list_add_tail(&lpfc_ncmd
->list
, &prep_nblist
);
1924 /* enough sgls for non-embed sgl mbox command */
1925 if (block_cnt
== LPFC_NEMBED_MBOX_SGL_CNT
) {
1926 list_splice_init(&prep_nblist
, &blck_nblist
);
1927 post_cnt
= block_cnt
;
1932 last_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1934 /* end of repost sgl list condition for NVME buffers */
1935 if (num_posting
== sb_count
) {
1936 if (post_cnt
== 0) {
1937 /* last sgl posting block */
1938 list_splice_init(&prep_nblist
, &blck_nblist
);
1939 post_cnt
= block_cnt
;
1940 } else if (block_cnt
== 1) {
1941 /* last single sgl with non-contiguous xri */
1942 if (sgl_size
> SGL_PAGE_SIZE
)
1944 lpfc_ncmd
->dma_phys_sgl
+
1948 cur_xritag
= lpfc_ncmd
->cur_iocbq
.sli4_xritag
;
1949 status
= lpfc_sli4_post_sgl(phba
,
1950 lpfc_ncmd
->dma_phys_sgl
,
1951 pdma_phys_sgl1
, cur_xritag
);
1953 /* failure, put on abort nvme list */
1954 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1956 /* success, put on NVME buffer list */
1957 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1958 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1961 /* success, put on NVME buffer sgl list */
1962 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1966 /* continue until a nembed page worth of sgls */
1970 /* post block of NVME buffer list sgls */
1971 status
= lpfc_sli4_post_nvme_sgl_block(phba
, &blck_nblist
,
1974 /* don't reset xirtag due to hole in xri block */
1976 last_xritag
= NO_XRI
;
1978 /* reset NVME buffer post count for next round of posting */
1981 /* put posted NVME buffer-sgl posted on NVME buffer sgl list */
1982 while (!list_empty(&blck_nblist
)) {
1983 list_remove_head(&blck_nblist
, lpfc_ncmd
,
1984 struct lpfc_nvme_buf
, list
);
1986 /* failure, put on abort nvme list */
1987 lpfc_ncmd
->flags
|= LPFC_SBUF_XBUSY
;
1989 /* success, put on NVME buffer list */
1990 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
1991 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
1994 list_add_tail(&lpfc_ncmd
->list
, &nvme_nblist
);
1997 /* Push NVME buffers with sgl posted to the available list */
1998 while (!list_empty(&nvme_nblist
)) {
1999 list_remove_head(&nvme_nblist
, lpfc_ncmd
,
2000 struct lpfc_nvme_buf
, list
);
2001 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
2007 * lpfc_repost_nvme_sgl_list - Repost all the allocated nvme buffer sgls
2008 * @phba: pointer to lpfc hba data structure.
2010 * This routine walks the list of nvme buffers that have been allocated and
2011 * repost them to the port by using SGL block post. This is needed after a
2012 * pci_function_reset/warm_start or start. The lpfc_hba_down_post_s4 routine
2013 * is responsible for moving all nvme buffers on the lpfc_abts_nvme_sgl_list
2014 * to the lpfc_nvme_buf_list. If the repost fails, reject all nvme buffers.
2016 * Returns: 0 = success, non-zero failure.
2019 lpfc_repost_nvme_sgl_list(struct lpfc_hba
*phba
)
2021 LIST_HEAD(post_nblist
);
2022 int num_posted
, rc
= 0;
2024 /* get all NVME buffers need to repost to a local list */
2025 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
2026 spin_lock(&phba
->nvme_buf_list_put_lock
);
2027 list_splice_init(&phba
->lpfc_nvme_buf_list_get
, &post_nblist
);
2028 list_splice(&phba
->lpfc_nvme_buf_list_put
, &post_nblist
);
2029 phba
->get_nvme_bufs
= 0;
2030 phba
->put_nvme_bufs
= 0;
2031 spin_unlock(&phba
->nvme_buf_list_put_lock
);
2032 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
2034 /* post the list of nvme buffer sgls to port if available */
2035 if (!list_empty(&post_nblist
)) {
2036 num_posted
= lpfc_post_nvme_sgl_list(phba
, &post_nblist
,
2037 phba
->sli4_hba
.nvme_xri_cnt
);
2038 /* failed to post any nvme buffer, return error */
2039 if (num_posted
== 0)
2046 * lpfc_new_nvme_buf - Scsi buffer allocator for HBA with SLI4 IF spec
2047 * @vport: The virtual port for which this call being executed.
2048 * @num_to_allocate: The requested number of buffers to allocate.
2050 * This routine allocates nvme buffers for device with SLI-4 interface spec,
2051 * the nvme buffer contains all the necessary information needed to initiate
2052 * a NVME I/O. After allocating up to @num_to_allocate NVME buffers and put
2053 * them on a list, it post them to the port by using SGL block post.
2056 * int - number of nvme buffers that were allocated and posted.
2057 * 0 = failure, less than num_to_alloc is a partial failure.
2060 lpfc_new_nvme_buf(struct lpfc_vport
*vport
, int num_to_alloc
)
2062 struct lpfc_hba
*phba
= vport
->phba
;
2063 struct lpfc_nvme_buf
*lpfc_ncmd
;
2064 struct lpfc_iocbq
*pwqeq
;
2065 union lpfc_wqe128
*wqe
;
2066 struct sli4_sge
*sgl
;
2067 dma_addr_t pdma_phys_sgl
;
2068 uint16_t iotag
, lxri
= 0;
2069 int bcnt
, num_posted
, sgl_size
;
2070 LIST_HEAD(prep_nblist
);
2071 LIST_HEAD(post_nblist
);
2072 LIST_HEAD(nvme_nblist
);
2074 sgl_size
= phba
->cfg_sg_dma_buf_size
;
2076 for (bcnt
= 0; bcnt
< num_to_alloc
; bcnt
++) {
2077 lpfc_ncmd
= kzalloc(sizeof(struct lpfc_nvme_buf
), GFP_KERNEL
);
2081 * Get memory from the pci pool to map the virt space to
2082 * pci bus space for an I/O. The DMA buffer includes the
2083 * number of SGE's necessary to support the sg_tablesize.
2085 lpfc_ncmd
->data
= dma_pool_zalloc(phba
->lpfc_sg_dma_buf_pool
,
2087 &lpfc_ncmd
->dma_handle
);
2088 if (!lpfc_ncmd
->data
) {
2093 lxri
= lpfc_sli4_next_xritag(phba
);
2094 if (lxri
== NO_XRI
) {
2095 dma_pool_free(phba
->lpfc_sg_dma_buf_pool
,
2096 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
2100 pwqeq
= &(lpfc_ncmd
->cur_iocbq
);
2101 wqe
= (union lpfc_wqe128
*)&pwqeq
->wqe
;
2103 /* Allocate iotag for lpfc_ncmd->cur_iocbq. */
2104 iotag
= lpfc_sli_next_iotag(phba
, pwqeq
);
2106 dma_pool_free(phba
->lpfc_sg_dma_buf_pool
,
2107 lpfc_ncmd
->data
, lpfc_ncmd
->dma_handle
);
2109 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
2110 "6121 Failed to allocated IOTAG for"
2111 " XRI:0x%x\n", lxri
);
2112 lpfc_sli4_free_xri(phba
, lxri
);
2115 pwqeq
->sli4_lxritag
= lxri
;
2116 pwqeq
->sli4_xritag
= phba
->sli4_hba
.xri_ids
[lxri
];
2117 pwqeq
->iocb_flag
|= LPFC_IO_NVME
;
2118 pwqeq
->context1
= lpfc_ncmd
;
2119 pwqeq
->wqe_cmpl
= lpfc_nvme_io_cmd_wqe_cmpl
;
2121 /* Initialize local short-hand pointers. */
2122 lpfc_ncmd
->nvme_sgl
= lpfc_ncmd
->data
;
2123 sgl
= lpfc_ncmd
->nvme_sgl
;
2124 pdma_phys_sgl
= lpfc_ncmd
->dma_handle
;
2125 lpfc_ncmd
->dma_phys_sgl
= pdma_phys_sgl
;
2127 /* Rsp SGE will be filled in when we rcv an IO
2128 * from the NVME Layer to be sent.
2129 * The cmd is going to be embedded so we need a SKIP SGE.
2131 bf_set(lpfc_sli4_sge_type
, sgl
, LPFC_SGE_TYPE_SKIP
);
2132 bf_set(lpfc_sli4_sge_last
, sgl
, 0);
2133 sgl
->word2
= cpu_to_le32(sgl
->word2
);
2134 /* Fill in word 3 / sgl_len during cmd submission */
2136 lpfc_ncmd
->cur_iocbq
.context1
= lpfc_ncmd
;
2139 bf_set(wqe_erp
, &wqe
->generic
.wqe_com
, 0);
2140 /* NVME upper layers will time things out, if needed */
2141 bf_set(wqe_tmo
, &wqe
->generic
.wqe_com
, 0);
2144 bf_set(wqe_ebde_cnt
, &wqe
->generic
.wqe_com
, 0);
2145 bf_set(wqe_dbde
, &wqe
->generic
.wqe_com
, 1);
2147 /* add the nvme buffer to a post list */
2148 list_add_tail(&lpfc_ncmd
->list
, &post_nblist
);
2149 spin_lock_irq(&phba
->nvme_buf_list_get_lock
);
2150 phba
->sli4_hba
.nvme_xri_cnt
++;
2151 spin_unlock_irq(&phba
->nvme_buf_list_get_lock
);
2153 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME
,
2154 "6114 Allocate %d out of %d requested new NVME "
2155 "buffers\n", bcnt
, num_to_alloc
);
2157 /* post the list of nvme buffer sgls to port if available */
2158 if (!list_empty(&post_nblist
))
2159 num_posted
= lpfc_post_nvme_sgl_list(phba
,
2160 &post_nblist
, bcnt
);
2167 static inline struct lpfc_nvme_buf
*
2168 lpfc_nvme_buf(struct lpfc_hba
*phba
)
2170 struct lpfc_nvme_buf
*lpfc_ncmd
, *lpfc_ncmd_next
;
2172 list_for_each_entry_safe(lpfc_ncmd
, lpfc_ncmd_next
,
2173 &phba
->lpfc_nvme_buf_list_get
, list
) {
2174 list_del_init(&lpfc_ncmd
->list
);
2175 phba
->get_nvme_bufs
--;
2182 * lpfc_get_nvme_buf - Get a nvme buffer from lpfc_nvme_buf_list of the HBA
2183 * @phba: The HBA for which this call is being executed.
2185 * This routine removes a nvme buffer from head of @phba lpfc_nvme_buf_list list
2186 * and returns to caller.
2190 * Pointer to lpfc_nvme_buf - Success
2192 static struct lpfc_nvme_buf
*
2193 lpfc_get_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nodelist
*ndlp
,
2196 struct lpfc_nvme_buf
*lpfc_ncmd
= NULL
;
2197 unsigned long iflag
= 0;
2199 spin_lock_irqsave(&phba
->nvme_buf_list_get_lock
, iflag
);
2200 if (phba
->get_nvme_bufs
> LPFC_NVME_EXPEDITE_XRICNT
|| expedite
)
2201 lpfc_ncmd
= lpfc_nvme_buf(phba
);
2203 spin_lock(&phba
->nvme_buf_list_put_lock
);
2204 list_splice(&phba
->lpfc_nvme_buf_list_put
,
2205 &phba
->lpfc_nvme_buf_list_get
);
2206 phba
->get_nvme_bufs
+= phba
->put_nvme_bufs
;
2207 INIT_LIST_HEAD(&phba
->lpfc_nvme_buf_list_put
);
2208 phba
->put_nvme_bufs
= 0;
2209 spin_unlock(&phba
->nvme_buf_list_put_lock
);
2210 if (phba
->get_nvme_bufs
> LPFC_NVME_EXPEDITE_XRICNT
|| expedite
)
2211 lpfc_ncmd
= lpfc_nvme_buf(phba
);
2213 spin_unlock_irqrestore(&phba
->nvme_buf_list_get_lock
, iflag
);
2218 * lpfc_release_nvme_buf: Return a nvme buffer back to hba nvme buf list.
2219 * @phba: The Hba for which this call is being executed.
2220 * @lpfc_ncmd: The nvme buffer which is being released.
2222 * This routine releases @lpfc_ncmd nvme buffer by adding it to tail of @phba
2223 * lpfc_nvme_buf_list list. For SLI4 XRI's are tied to the nvme buffer
2224 * and cannot be reused for at least RA_TOV amount of time if it was
2228 lpfc_release_nvme_buf(struct lpfc_hba
*phba
, struct lpfc_nvme_buf
*lpfc_ncmd
)
2230 unsigned long iflag
= 0;
2232 lpfc_ncmd
->nonsg_phys
= 0;
2233 if (lpfc_ncmd
->flags
& LPFC_SBUF_XBUSY
) {
2234 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2235 "6310 XB release deferred for "
2236 "ox_id x%x on reqtag x%x\n",
2237 lpfc_ncmd
->cur_iocbq
.sli4_xritag
,
2238 lpfc_ncmd
->cur_iocbq
.iotag
);
2240 spin_lock_irqsave(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2242 list_add_tail(&lpfc_ncmd
->list
,
2243 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
);
2244 spin_unlock_irqrestore(&phba
->sli4_hba
.abts_nvme_buf_list_lock
,
2247 lpfc_ncmd
->nvmeCmd
= NULL
;
2248 lpfc_ncmd
->cur_iocbq
.iocb_flag
= LPFC_IO_NVME
;
2249 spin_lock_irqsave(&phba
->nvme_buf_list_put_lock
, iflag
);
2250 list_add_tail(&lpfc_ncmd
->list
, &phba
->lpfc_nvme_buf_list_put
);
2251 phba
->put_nvme_bufs
++;
2252 spin_unlock_irqrestore(&phba
->nvme_buf_list_put_lock
, iflag
);
2257 * lpfc_nvme_create_localport - Create/Bind an nvme localport instance.
2258 * @pvport - the lpfc_vport instance requesting a localport.
2260 * This routine is invoked to create an nvme localport instance to bind
2261 * to the nvme_fc_transport. It is called once during driver load
2262 * like lpfc_create_shost after all other services are initialized.
2263 * It requires a vport, vpi, and wwns at call time. Other localport
2264 * parameters are modified as the driver's FCID and the Fabric WWN
2269 * -ENOMEM - no heap memory available
2270 * other values - from nvme registration upcall
2273 lpfc_nvme_create_localport(struct lpfc_vport
*vport
)
2276 struct lpfc_hba
*phba
= vport
->phba
;
2277 struct nvme_fc_port_info nfcp_info
;
2278 struct nvme_fc_local_port
*localport
;
2279 struct lpfc_nvme_lport
*lport
;
2282 /* Initialize this localport instance. The vport wwn usage ensures
2283 * that NPIV is accounted for.
2285 memset(&nfcp_info
, 0, sizeof(struct nvme_fc_port_info
));
2286 nfcp_info
.port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2287 nfcp_info
.node_name
= wwn_to_u64(vport
->fc_nodename
.u
.wwn
);
2288 nfcp_info
.port_name
= wwn_to_u64(vport
->fc_portname
.u
.wwn
);
2290 /* Limit to LPFC_MAX_NVME_SEG_CNT.
2291 * For now need + 1 to get around NVME transport logic.
2293 if (phba
->cfg_sg_seg_cnt
> LPFC_MAX_NVME_SEG_CNT
) {
2294 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
| LOG_INIT
,
2295 "6300 Reducing sg segment cnt to %d\n",
2296 LPFC_MAX_NVME_SEG_CNT
);
2297 phba
->cfg_nvme_seg_cnt
= LPFC_MAX_NVME_SEG_CNT
;
2299 phba
->cfg_nvme_seg_cnt
= phba
->cfg_sg_seg_cnt
;
2301 lpfc_nvme_template
.max_sgl_segments
= phba
->cfg_nvme_seg_cnt
+ 1;
2302 lpfc_nvme_template
.max_hw_queues
= phba
->cfg_nvme_io_channel
;
2304 /* localport is allocated from the stack, but the registration
2305 * call allocates heap memory as well as the private area.
2307 #if (IS_ENABLED(CONFIG_NVME_FC))
2308 ret
= nvme_fc_register_localport(&nfcp_info
, &lpfc_nvme_template
,
2309 &vport
->phba
->pcidev
->dev
, &localport
);
2314 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
| LOG_NVME_DISC
,
2315 "6005 Successfully registered local "
2316 "NVME port num %d, localP %p, private %p, "
2318 localport
->port_num
, localport
,
2320 lpfc_nvme_template
.max_sgl_segments
);
2322 /* Private is our lport size declared in the template. */
2323 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2324 vport
->localport
= localport
;
2325 lport
->vport
= vport
;
2326 vport
->nvmei_support
= 1;
2328 atomic_set(&lport
->xmt_fcp_noxri
, 0);
2329 atomic_set(&lport
->xmt_fcp_bad_ndlp
, 0);
2330 atomic_set(&lport
->xmt_fcp_qdepth
, 0);
2331 atomic_set(&lport
->xmt_fcp_wqerr
, 0);
2332 atomic_set(&lport
->xmt_fcp_abort
, 0);
2333 atomic_set(&lport
->xmt_ls_abort
, 0);
2334 atomic_set(&lport
->xmt_ls_err
, 0);
2335 atomic_set(&lport
->cmpl_fcp_xb
, 0);
2336 atomic_set(&lport
->cmpl_fcp_err
, 0);
2337 atomic_set(&lport
->cmpl_ls_xb
, 0);
2338 atomic_set(&lport
->cmpl_ls_err
, 0);
2340 /* Don't post more new bufs if repost already recovered
2343 if (phba
->sli4_hba
.nvme_xri_cnt
== 0) {
2344 len
= lpfc_new_nvme_buf(vport
,
2345 phba
->sli4_hba
.nvme_xri_max
);
2346 vport
->phba
->total_nvme_bufs
+= len
;
2353 /* lpfc_nvme_lport_unreg_wait - Wait for the host to complete an lport unreg.
2355 * The driver has to wait for the host nvme transport to callback
2356 * indicating the localport has successfully unregistered all
2357 * resources. Since this is an uninterruptible wait, loop every ten
2358 * seconds and print a message indicating no progress.
2360 * An uninterruptible wait is used because of the risk of transport-to-
2361 * driver state mismatch.
2364 lpfc_nvme_lport_unreg_wait(struct lpfc_vport
*vport
,
2365 struct lpfc_nvme_lport
*lport
)
2367 #if (IS_ENABLED(CONFIG_NVME_FC))
2371 /* Host transport has to clean up and confirm requiring an indefinite
2372 * wait. Print a message if a 10 second wait expires and renew the
2373 * wait. This is unexpected.
2375 wait_tmo
= msecs_to_jiffies(LPFC_NVME_WAIT_TMO
* 1000);
2377 ret
= wait_for_completion_timeout(&lport
->lport_unreg_done
,
2379 if (unlikely(!ret
)) {
2380 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_IOERR
,
2381 "6176 Lport %p Localport %p wait "
2382 "timed out. Renewing.\n",
2383 lport
, vport
->localport
);
2388 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_IOERR
,
2389 "6177 Lport %p Localport %p Complete Success\n",
2390 lport
, vport
->localport
);
2395 * lpfc_nvme_destroy_localport - Destroy lpfc_nvme bound to nvme transport.
2396 * @pnvme: pointer to lpfc nvme data structure.
2398 * This routine is invoked to destroy all lports bound to the phba.
2399 * The lport memory was allocated by the nvme fc transport and is
2400 * released there. This routine ensures all rports bound to the
2401 * lport have been disconnected.
2405 lpfc_nvme_destroy_localport(struct lpfc_vport
*vport
)
2407 #if (IS_ENABLED(CONFIG_NVME_FC))
2408 struct nvme_fc_local_port
*localport
;
2409 struct lpfc_nvme_lport
*lport
;
2412 if (vport
->nvmei_support
== 0)
2415 localport
= vport
->localport
;
2416 vport
->localport
= NULL
;
2417 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2419 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2420 "6011 Destroying NVME localport %p\n",
2423 /* lport's rport list is clear. Unregister
2424 * lport and release resources.
2426 init_completion(&lport
->lport_unreg_done
);
2427 ret
= nvme_fc_unregister_localport(localport
);
2429 /* Wait for completion. This either blocks
2430 * indefinitely or succeeds
2432 lpfc_nvme_lport_unreg_wait(vport
, lport
);
2434 /* Regardless of the unregister upcall response, clear
2435 * nvmei_support. All rports are unregistered and the
2436 * driver will clean up.
2438 vport
->nvmei_support
= 0;
2440 lpfc_printf_vlog(vport
,
2441 KERN_INFO
, LOG_NVME_DISC
,
2442 "6009 Unregistered lport Success\n");
2444 lpfc_printf_vlog(vport
,
2445 KERN_INFO
, LOG_NVME_DISC
,
2446 "6010 Unregistered lport "
2447 "Failed, status x%x\n",
2454 lpfc_nvme_update_localport(struct lpfc_vport
*vport
)
2456 #if (IS_ENABLED(CONFIG_NVME_FC))
2457 struct nvme_fc_local_port
*localport
;
2458 struct lpfc_nvme_lport
*lport
;
2460 localport
= vport
->localport
;
2462 lpfc_printf_vlog(vport
, KERN_WARNING
, LOG_NVME
,
2463 "6710 Update NVME fail. No localport\n");
2466 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2468 lpfc_printf_vlog(vport
, KERN_WARNING
, LOG_NVME
,
2469 "6171 Update NVME fail. localP %p, No lport\n",
2473 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME
,
2474 "6012 Update NVME lport %p did x%x\n",
2475 localport
, vport
->fc_myDID
);
2477 localport
->port_id
= vport
->fc_myDID
;
2478 if (localport
->port_id
== 0)
2479 localport
->port_role
= FC_PORT_ROLE_NVME_DISCOVERY
;
2481 localport
->port_role
= FC_PORT_ROLE_NVME_INITIATOR
;
2483 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2484 "6030 bound lport %p to DID x%06x\n",
2485 lport
, localport
->port_id
);
2490 lpfc_nvme_register_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2492 #if (IS_ENABLED(CONFIG_NVME_FC))
2494 struct nvme_fc_local_port
*localport
;
2495 struct lpfc_nvme_lport
*lport
;
2496 struct lpfc_nvme_rport
*rport
;
2497 struct nvme_fc_remote_port
*remote_port
;
2498 struct nvme_fc_port_info rpinfo
;
2499 struct lpfc_nodelist
*prev_ndlp
;
2501 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
, LOG_NVME_DISC
,
2502 "6006 Register NVME PORT. DID x%06x nlptype x%x\n",
2503 ndlp
->nlp_DID
, ndlp
->nlp_type
);
2505 localport
= vport
->localport
;
2509 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2511 /* NVME rports are not preserved across devloss.
2512 * Just register this instance. Note, rpinfo->dev_loss_tmo
2513 * is left 0 to indicate accept transport defaults. The
2514 * driver communicates port role capabilities consistent
2515 * with the PRLI response data.
2517 memset(&rpinfo
, 0, sizeof(struct nvme_fc_port_info
));
2518 rpinfo
.port_id
= ndlp
->nlp_DID
;
2519 if (ndlp
->nlp_type
& NLP_NVME_TARGET
)
2520 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_TARGET
;
2521 if (ndlp
->nlp_type
& NLP_NVME_INITIATOR
)
2522 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_INITIATOR
;
2524 if (ndlp
->nlp_type
& NLP_NVME_DISCOVERY
)
2525 rpinfo
.port_role
|= FC_PORT_ROLE_NVME_DISCOVERY
;
2527 rpinfo
.port_name
= wwn_to_u64(ndlp
->nlp_portname
.u
.wwn
);
2528 rpinfo
.node_name
= wwn_to_u64(ndlp
->nlp_nodename
.u
.wwn
);
2532 ret
= nvme_fc_register_remoteport(localport
, &rpinfo
, &remote_port
);
2534 /* If the ndlp already has an nrport, this is just
2535 * a resume of the existing rport. Else this is a
2538 rport
= remote_port
->private;
2540 if (ndlp
->nrport
== remote_port
->private) {
2541 /* Same remoteport. Just reuse. */
2542 lpfc_printf_vlog(ndlp
->vport
, KERN_INFO
,
2544 "6014 Rebinding lport to "
2545 "remoteport %p wwpn 0x%llx, "
2546 "Data: x%x x%x %p x%x x%06x\n",
2548 remote_port
->port_name
,
2549 remote_port
->port_id
,
2550 remote_port
->port_role
,
2556 prev_ndlp
= rport
->ndlp
;
2558 /* Sever the ndlp<->rport association
2559 * before dropping the ndlp ref from
2562 spin_lock_irq(&vport
->phba
->hbalock
);
2563 ndlp
->nrport
= NULL
;
2564 spin_unlock_irq(&vport
->phba
->hbalock
);
2566 rport
->remoteport
= NULL
;
2571 /* Clean bind the rport to the ndlp. */
2572 rport
->remoteport
= remote_port
;
2573 rport
->lport
= lport
;
2575 spin_lock_irq(&vport
->phba
->hbalock
);
2576 ndlp
->nrport
= rport
;
2577 spin_unlock_irq(&vport
->phba
->hbalock
);
2578 lpfc_printf_vlog(vport
, KERN_INFO
,
2579 LOG_NVME_DISC
| LOG_NODE
,
2580 "6022 Binding new rport to "
2581 "lport %p Remoteport %p WWNN 0x%llx, "
2582 "Rport WWPN 0x%llx DID "
2583 "x%06x Role x%x, ndlp %p\n",
2585 rpinfo
.node_name
, rpinfo
.port_name
,
2586 rpinfo
.port_id
, rpinfo
.port_role
,
2589 lpfc_printf_vlog(vport
, KERN_ERR
,
2590 LOG_NVME_DISC
| LOG_NODE
,
2591 "6031 RemotePort Registration failed "
2592 "err: %d, DID x%06x\n",
2593 ret
, ndlp
->nlp_DID
);
2602 /* lpfc_nvme_unregister_port - unbind the DID and port_role from this rport.
2604 * There is no notion of Devloss or rport recovery from the current
2605 * nvme_transport perspective. Loss of an rport just means IO cannot
2606 * be sent and recovery is completely up to the initator.
2607 * For now, the driver just unbinds the DID and port_role so that
2608 * no further IO can be issued. Changes are planned for later.
2610 * Notes - the ndlp reference count is not decremented here since
2611 * since there is no nvme_transport api for devloss. Node ref count
2612 * is only adjusted in driver unload.
2615 lpfc_nvme_unregister_port(struct lpfc_vport
*vport
, struct lpfc_nodelist
*ndlp
)
2617 #if (IS_ENABLED(CONFIG_NVME_FC))
2619 struct nvme_fc_local_port
*localport
;
2620 struct lpfc_nvme_lport
*lport
;
2621 struct lpfc_nvme_rport
*rport
;
2622 struct nvme_fc_remote_port
*remoteport
;
2624 localport
= vport
->localport
;
2626 /* This is fundamental error. The localport is always
2627 * available until driver unload. Just exit.
2632 lport
= (struct lpfc_nvme_lport
*)localport
->private;
2636 rport
= ndlp
->nrport
;
2640 remoteport
= rport
->remoteport
;
2641 lpfc_printf_vlog(vport
, KERN_INFO
, LOG_NVME_DISC
,
2642 "6033 Unreg nvme remoteport %p, portname x%llx, "
2643 "port_id x%06x, portstate x%x port type x%x\n",
2644 remoteport
, remoteport
->port_name
,
2645 remoteport
->port_id
, remoteport
->port_state
,
2648 /* Sanity check ndlp type. Only call for NVME ports. Don't
2649 * clear any rport state until the transport calls back.
2652 if (ndlp
->nlp_type
& NLP_NVME_TARGET
) {
2653 /* No concern about the role change on the nvme remoteport.
2654 * The transport will update it.
2656 ndlp
->upcall_flags
|= NLP_WAIT_FOR_UNREG
;
2657 ret
= nvme_fc_unregister_remoteport(remoteport
);
2660 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2661 "6167 NVME unregister failed %d "
2663 ret
, remoteport
->port_state
);
2670 lpfc_printf_vlog(vport
, KERN_ERR
, LOG_NVME_DISC
,
2671 "6168 State error: lport %p, rport%p FCID x%06x\n",
2672 vport
->localport
, ndlp
->rport
, ndlp
->nlp_DID
);
2676 * lpfc_sli4_nvme_xri_aborted - Fast-path process of NVME xri abort
2677 * @phba: pointer to lpfc hba data structure.
2678 * @axri: pointer to the fcp xri abort wcqe structure.
2680 * This routine is invoked by the worker thread to process a SLI4 fast-path
2681 * NVME aborted xri. Aborted NVME IO commands are completed to the transport
2685 lpfc_sli4_nvme_xri_aborted(struct lpfc_hba
*phba
,
2686 struct sli4_wcqe_xri_aborted
*axri
)
2688 uint16_t xri
= bf_get(lpfc_wcqe_xa_xri
, axri
);
2689 struct lpfc_nvme_buf
*lpfc_ncmd
, *next_lpfc_ncmd
;
2690 struct nvmefc_fcp_req
*nvme_cmd
= NULL
;
2691 struct lpfc_nodelist
*ndlp
;
2692 unsigned long iflag
= 0;
2694 if (!(phba
->cfg_enable_fc4_type
& LPFC_ENABLE_NVME
))
2696 spin_lock_irqsave(&phba
->hbalock
, iflag
);
2697 spin_lock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2698 list_for_each_entry_safe(lpfc_ncmd
, next_lpfc_ncmd
,
2699 &phba
->sli4_hba
.lpfc_abts_nvme_buf_list
,
2701 if (lpfc_ncmd
->cur_iocbq
.sli4_xritag
== xri
) {
2702 list_del_init(&lpfc_ncmd
->list
);
2703 lpfc_ncmd
->flags
&= ~LPFC_SBUF_XBUSY
;
2704 lpfc_ncmd
->status
= IOSTAT_SUCCESS
;
2706 &phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2708 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);
2709 ndlp
= lpfc_ncmd
->ndlp
;
2711 lpfc_sli4_abts_err_handler(phba
, ndlp
, axri
);
2713 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2714 "6311 nvme_cmd %p xri x%x tag x%x "
2715 "abort complete and xri released\n",
2716 lpfc_ncmd
->nvmeCmd
, xri
,
2717 lpfc_ncmd
->cur_iocbq
.iotag
);
2719 /* Aborted NVME commands are required to not complete
2720 * before the abort exchange command fully completes.
2721 * Once completed, it is available via the put list.
2723 if (lpfc_ncmd
->nvmeCmd
) {
2724 nvme_cmd
= lpfc_ncmd
->nvmeCmd
;
2725 nvme_cmd
->done(nvme_cmd
);
2726 lpfc_ncmd
->nvmeCmd
= NULL
;
2728 lpfc_release_nvme_buf(phba
, lpfc_ncmd
);
2732 spin_unlock(&phba
->sli4_hba
.abts_nvme_buf_list_lock
);
2733 spin_unlock_irqrestore(&phba
->hbalock
, iflag
);
2735 lpfc_printf_log(phba
, KERN_INFO
, LOG_NVME_ABTS
,
2736 "6312 XRI Aborted xri x%x not found\n", xri
);
2741 * lpfc_nvme_wait_for_io_drain - Wait for all NVME wqes to complete
2742 * @phba: Pointer to HBA context object.
2744 * This function flushes all wqes in the nvme rings and frees all resources
2745 * in the txcmplq. This function does not issue abort wqes for the IO
2746 * commands in txcmplq, they will just be returned with
2747 * IOERR_SLI_DOWN. This function is invoked with EEH when device's PCI
2748 * slot has been permanently disabled.
2751 lpfc_nvme_wait_for_io_drain(struct lpfc_hba
*phba
)
2753 struct lpfc_sli_ring
*pring
;
2754 u32 i
, wait_cnt
= 0;
2756 if (phba
->sli_rev
< LPFC_SLI_REV4
)
2759 /* Cycle through all NVME rings and make sure all outstanding
2760 * WQEs have been removed from the txcmplqs.
2762 for (i
= 0; i
< phba
->cfg_nvme_io_channel
; i
++) {
2763 pring
= phba
->sli4_hba
.nvme_wq
[i
]->pring
;
2765 /* Retrieve everything on the txcmplq */
2766 while (!list_empty(&pring
->txcmplq
)) {
2767 msleep(LPFC_XRI_EXCH_BUSY_WAIT_T1
);
2770 /* The sleep is 10mS. Every ten seconds,
2771 * dump a message. Something is wrong.
2773 if ((wait_cnt
% 1000) == 0) {
2774 lpfc_printf_log(phba
, KERN_ERR
, LOG_NVME_IOERR
,
2775 "6178 NVME IO not empty, "
2776 "cnt %d\n", wait_cnt
);