?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
mic: vop: Fix use-after-free on remove
[linux/fpc-iii.git] / drivers / nvme / target / fc.c
blobf98f5c5bea2647f759c67fbeb0cf6d567615b138
1 /*
2 * Copyright (c) 2016 Avago Technologies. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful.
9 * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
10 * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
11 * PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE DISCLAIMED, EXCEPT TO
12 * THE EXTENT THAT SUCH DISCLAIMERS ARE HELD TO BE LEGALLY INVALID.
13 * See the GNU General Public License for more details, a copy of which
14 * can be found in the file COPYING included with this package
15 *
16 */
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18 #include <linux/module.h>
19 #include <linux/slab.h>
20 #include <linux/blk-mq.h>
21 #include <linux/parser.h>
22 #include <linux/random.h>
23 #include <uapi/scsi/fc/fc_fs.h>
24 #include <uapi/scsi/fc/fc_els.h>
25
26 #include "nvmet.h"
27 #include <linux/nvme-fc-driver.h>
28 #include <linux/nvme-fc.h>
29
30
31 /* *************************** Data Structures/Defines ****************** */
32
33
34 #define NVMET_LS_CTX_COUNT 256
35
36 /* for this implementation, assume small single frame rqst/rsp */
37 #define NVME_FC_MAX_LS_BUFFER_SIZE 2048
38
39 struct nvmet_fc_tgtport;
40 struct nvmet_fc_tgt_assoc;
41
42 struct nvmet_fc_ls_iod {
43 struct nvmefc_tgt_ls_req *lsreq;
44 struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */
45
46 struct list_head ls_list; /* tgtport->ls_list */
47
48 struct nvmet_fc_tgtport *tgtport;
49 struct nvmet_fc_tgt_assoc *assoc;
50
51 u8 *rqstbuf;
52 u8 *rspbuf;
53 u16 rqstdatalen;
54 dma_addr_t rspdma;
55
56 struct scatterlist sg[2];
57
58 struct work_struct work;
59 } __aligned(sizeof(unsigned long long));
60
61 /* desired maximum for a single sequence - if sg list allows it */
62 #define NVMET_FC_MAX_SEQ_LENGTH (256 * 1024)
63
64 enum nvmet_fcp_datadir {
65 NVMET_FCP_NODATA,
66 NVMET_FCP_WRITE,
67 NVMET_FCP_READ,
68 NVMET_FCP_ABORTED,
69 };
70
71 struct nvmet_fc_fcp_iod {
72 struct nvmefc_tgt_fcp_req *fcpreq;
73
74 struct nvme_fc_cmd_iu cmdiubuf;
75 struct nvme_fc_ersp_iu rspiubuf;
76 dma_addr_t rspdma;
77 struct scatterlist *next_sg;
78 struct scatterlist *data_sg;
79 int data_sg_cnt;
80 u32 offset;
81 enum nvmet_fcp_datadir io_dir;
82 bool active;
83 bool abort;
84 bool aborted;
85 bool writedataactive;
86 spinlock_t flock;
87
88 struct nvmet_req req;
89 struct work_struct defer_work;
90
91 struct nvmet_fc_tgtport *tgtport;
92 struct nvmet_fc_tgt_queue *queue;
93
94 struct list_head fcp_list; /* tgtport->fcp_list */
95 };
96
97 struct nvmet_fc_tgtport {
98
99 struct nvmet_fc_target_port fc_target_port;
100
101 struct list_head tgt_list; /* nvmet_fc_target_list */
102 struct device *dev; /* dev for dma mapping */
103 struct nvmet_fc_target_template *ops;
104
105 struct nvmet_fc_ls_iod *iod;
106 spinlock_t lock;
107 struct list_head ls_list;
108 struct list_head ls_busylist;
109 struct list_head assoc_list;
110 struct ida assoc_cnt;
111 struct nvmet_fc_port_entry *pe;
112 struct kref ref;
113 u32 max_sg_cnt;
114 };
115
116 struct nvmet_fc_port_entry {
117 struct nvmet_fc_tgtport *tgtport;
118 struct nvmet_port *port;
119 u64 node_name;
120 u64 port_name;
121 struct list_head pe_list;
122 };
123
124 struct nvmet_fc_defer_fcp_req {
125 struct list_head req_list;
126 struct nvmefc_tgt_fcp_req *fcp_req;
127 };
128
129 struct nvmet_fc_tgt_queue {
130 bool ninetypercent;
131 u16 qid;
132 u16 sqsize;
133 u16 ersp_ratio;
134 __le16 sqhd;
135 atomic_t connected;
136 atomic_t sqtail;
137 atomic_t zrspcnt;
138 atomic_t rsn;
139 spinlock_t qlock;
140 struct nvmet_cq nvme_cq;
141 struct nvmet_sq nvme_sq;
142 struct nvmet_fc_tgt_assoc *assoc;
143 struct nvmet_fc_fcp_iod *fod; /* array of fcp_iods */
144 struct list_head fod_list;
145 struct list_head pending_cmd_list;
146 struct list_head avail_defer_list;
147 struct workqueue_struct *work_q;
148 struct kref ref;
149 } __aligned(sizeof(unsigned long long));
150
151 struct nvmet_fc_tgt_assoc {
152 u64 association_id;
153 u32 a_id;
154 struct nvmet_fc_tgtport *tgtport;
155 struct list_head a_list;
156 struct nvmet_fc_tgt_queue *queues[NVMET_NR_QUEUES + 1];
157 struct kref ref;
158 struct work_struct del_work;
159 };
160
161
162 static inline int
163 nvmet_fc_iodnum(struct nvmet_fc_ls_iod *iodptr)
164 {
165 return (iodptr - iodptr->tgtport->iod);
166 }
167
168 static inline int
169 nvmet_fc_fodnum(struct nvmet_fc_fcp_iod *fodptr)
170 {
171 return (fodptr - fodptr->queue->fod);
172 }
173
174
175 /*
176 * Association and Connection IDs:
177 *
178 * Association ID will have random number in upper 6 bytes and zero
179 * in lower 2 bytes
180 *
181 * Connection IDs will be Association ID with QID or'd in lower 2 bytes
182 *
183 * note: Association ID = Connection ID for queue 0
184 */
185 #define BYTES_FOR_QID sizeof(u16)
186 #define BYTES_FOR_QID_SHIFT (BYTES_FOR_QID * 8)
187 #define NVMET_FC_QUEUEID_MASK ((u64)((1 << BYTES_FOR_QID_SHIFT) - 1))
188
189 static inline u64
190 nvmet_fc_makeconnid(struct nvmet_fc_tgt_assoc *assoc, u16 qid)
191 {
192 return (assoc->association_id | qid);
193 }
194
195 static inline u64
196 nvmet_fc_getassociationid(u64 connectionid)
197 {
198 return connectionid & ~NVMET_FC_QUEUEID_MASK;
199 }
200
201 static inline u16
202 nvmet_fc_getqueueid(u64 connectionid)
203 {
204 return (u16)(connectionid & NVMET_FC_QUEUEID_MASK);
205 }
206
207 static inline struct nvmet_fc_tgtport *
208 targetport_to_tgtport(struct nvmet_fc_target_port *targetport)
209 {
210 return container_of(targetport, struct nvmet_fc_tgtport,
211 fc_target_port);
212 }
213
214 static inline struct nvmet_fc_fcp_iod *
215 nvmet_req_to_fod(struct nvmet_req *nvme_req)
216 {
217 return container_of(nvme_req, struct nvmet_fc_fcp_iod, req);
218 }
219
220
221 /* *************************** Globals **************************** */
222
223
224 static DEFINE_SPINLOCK(nvmet_fc_tgtlock);
225
226 static LIST_HEAD(nvmet_fc_target_list);
227 static DEFINE_IDA(nvmet_fc_tgtport_cnt);
228 static LIST_HEAD(nvmet_fc_portentry_list);
229
230
231 static void nvmet_fc_handle_ls_rqst_work(struct work_struct *work);
232 static void nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work);
233 static void nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc);
234 static int nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc);
235 static void nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue);
236 static int nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue);
237 static void nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport);
238 static int nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport);
239 static void nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
240 struct nvmet_fc_fcp_iod *fod);
241 static void nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc);
242
243
244 /* *********************** FC-NVME DMA Handling **************************** */
245
246 /*
247 * The fcloop device passes in a NULL device pointer. Real LLD's will
248 * pass in a valid device pointer. If NULL is passed to the dma mapping
249 * routines, depending on the platform, it may or may not succeed, and
250 * may crash.
251 *
252 * As such:
253 * Wrapper all the dma routines and check the dev pointer.
254 *
255 * If simple mappings (return just a dma address, we'll noop them,
256 * returning a dma address of 0.
257 *
258 * On more complex mappings (dma_map_sg), a pseudo routine fills
259 * in the scatter list, setting all dma addresses to 0.
260 */
261
262 static inline dma_addr_t
263 fc_dma_map_single(struct device *dev, void *ptr, size_t size,
264 enum dma_data_direction dir)
265 {
266 return dev ? dma_map_single(dev, ptr, size, dir) : (dma_addr_t)0L;
267 }
268
269 static inline int
270 fc_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
271 {
272 return dev ? dma_mapping_error(dev, dma_addr) : 0;
273 }
274
275 static inline void
276 fc_dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
277 enum dma_data_direction dir)
278 {
279 if (dev)
280 dma_unmap_single(dev, addr, size, dir);
281 }
282
283 static inline void
284 fc_dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
285 enum dma_data_direction dir)
286 {
287 if (dev)
288 dma_sync_single_for_cpu(dev, addr, size, dir);
289 }
290
291 static inline void
292 fc_dma_sync_single_for_device(struct device *dev, dma_addr_t addr, size_t size,
293 enum dma_data_direction dir)
294 {
295 if (dev)
296 dma_sync_single_for_device(dev, addr, size, dir);
297 }
298
299 /* pseudo dma_map_sg call */
300 static int
301 fc_map_sg(struct scatterlist *sg, int nents)
302 {
303 struct scatterlist *s;
304 int i;
305
306 WARN_ON(nents == 0 || sg[0].length == 0);
307
308 for_each_sg(sg, s, nents, i) {
309 s->dma_address = 0L;
310 #ifdef CONFIG_NEED_SG_DMA_LENGTH
311 s->dma_length = s->length;
312 #endif
313 }
314 return nents;
315 }
316
317 static inline int
318 fc_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
319 enum dma_data_direction dir)
320 {
321 return dev ? dma_map_sg(dev, sg, nents, dir) : fc_map_sg(sg, nents);
322 }
323
324 static inline void
325 fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
326 enum dma_data_direction dir)
327 {
328 if (dev)
329 dma_unmap_sg(dev, sg, nents, dir);
330 }
331
332
333 /* *********************** FC-NVME Port Management ************************ */
334
335
336 static int
337 nvmet_fc_alloc_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
338 {
339 struct nvmet_fc_ls_iod *iod;
340 int i;
341
342 iod = kcalloc(NVMET_LS_CTX_COUNT, sizeof(struct nvmet_fc_ls_iod),
343 GFP_KERNEL);
344 if (!iod)
345 return -ENOMEM;
346
347 tgtport->iod = iod;
348
349 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
350 INIT_WORK(&iod->work, nvmet_fc_handle_ls_rqst_work);
351 iod->tgtport = tgtport;
352 list_add_tail(&iod->ls_list, &tgtport->ls_list);
353
354 iod->rqstbuf = kcalloc(2, NVME_FC_MAX_LS_BUFFER_SIZE,
355 GFP_KERNEL);
356 if (!iod->rqstbuf)
357 goto out_fail;
358
359 iod->rspbuf = iod->rqstbuf + NVME_FC_MAX_LS_BUFFER_SIZE;
360
361 iod->rspdma = fc_dma_map_single(tgtport->dev, iod->rspbuf,
362 NVME_FC_MAX_LS_BUFFER_SIZE,
363 DMA_TO_DEVICE);
364 if (fc_dma_mapping_error(tgtport->dev, iod->rspdma))
365 goto out_fail;
366 }
367
368 return 0;
369
370 out_fail:
371 kfree(iod->rqstbuf);
372 list_del(&iod->ls_list);
373 for (iod--, i--; i >= 0; iod--, i--) {
374 fc_dma_unmap_single(tgtport->dev, iod->rspdma,
375 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
376 kfree(iod->rqstbuf);
377 list_del(&iod->ls_list);
378 }
379
380 kfree(iod);
381
382 return -EFAULT;
383 }
384
385 static void
386 nvmet_fc_free_ls_iodlist(struct nvmet_fc_tgtport *tgtport)
387 {
388 struct nvmet_fc_ls_iod *iod = tgtport->iod;
389 int i;
390
391 for (i = 0; i < NVMET_LS_CTX_COUNT; iod++, i++) {
392 fc_dma_unmap_single(tgtport->dev,
393 iod->rspdma, NVME_FC_MAX_LS_BUFFER_SIZE,
394 DMA_TO_DEVICE);
395 kfree(iod->rqstbuf);
396 list_del(&iod->ls_list);
397 }
398 kfree(tgtport->iod);
399 }
400
401 static struct nvmet_fc_ls_iod *
402 nvmet_fc_alloc_ls_iod(struct nvmet_fc_tgtport *tgtport)
403 {
404 struct nvmet_fc_ls_iod *iod;
405 unsigned long flags;
406
407 spin_lock_irqsave(&tgtport->lock, flags);
408 iod = list_first_entry_or_null(&tgtport->ls_list,
409 struct nvmet_fc_ls_iod, ls_list);
410 if (iod)
411 list_move_tail(&iod->ls_list, &tgtport->ls_busylist);
412 spin_unlock_irqrestore(&tgtport->lock, flags);
413 return iod;
414 }
415
416
417 static void
418 nvmet_fc_free_ls_iod(struct nvmet_fc_tgtport *tgtport,
419 struct nvmet_fc_ls_iod *iod)
420 {
421 unsigned long flags;
422
423 spin_lock_irqsave(&tgtport->lock, flags);
424 list_move(&iod->ls_list, &tgtport->ls_list);
425 spin_unlock_irqrestore(&tgtport->lock, flags);
426 }
427
428 static void
429 nvmet_fc_prep_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
430 struct nvmet_fc_tgt_queue *queue)
431 {
432 struct nvmet_fc_fcp_iod *fod = queue->fod;
433 int i;
434
435 for (i = 0; i < queue->sqsize; fod++, i++) {
436 INIT_WORK(&fod->defer_work, nvmet_fc_fcp_rqst_op_defer_work);
437 fod->tgtport = tgtport;
438 fod->queue = queue;
439 fod->active = false;
440 fod->abort = false;
441 fod->aborted = false;
442 fod->fcpreq = NULL;
443 list_add_tail(&fod->fcp_list, &queue->fod_list);
444 spin_lock_init(&fod->flock);
445
446 fod->rspdma = fc_dma_map_single(tgtport->dev, &fod->rspiubuf,
447 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
448 if (fc_dma_mapping_error(tgtport->dev, fod->rspdma)) {
449 list_del(&fod->fcp_list);
450 for (fod--, i--; i >= 0; fod--, i--) {
451 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
452 sizeof(fod->rspiubuf),
453 DMA_TO_DEVICE);
454 fod->rspdma = 0L;
455 list_del(&fod->fcp_list);
456 }
457
458 return;
459 }
460 }
461 }
462
463 static void
464 nvmet_fc_destroy_fcp_iodlist(struct nvmet_fc_tgtport *tgtport,
465 struct nvmet_fc_tgt_queue *queue)
466 {
467 struct nvmet_fc_fcp_iod *fod = queue->fod;
468 int i;
469
470 for (i = 0; i < queue->sqsize; fod++, i++) {
471 if (fod->rspdma)
472 fc_dma_unmap_single(tgtport->dev, fod->rspdma,
473 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
474 }
475 }
476
477 static struct nvmet_fc_fcp_iod *
478 nvmet_fc_alloc_fcp_iod(struct nvmet_fc_tgt_queue *queue)
479 {
480 struct nvmet_fc_fcp_iod *fod;
481
482 lockdep_assert_held(&queue->qlock);
483
484 fod = list_first_entry_or_null(&queue->fod_list,
485 struct nvmet_fc_fcp_iod, fcp_list);
486 if (fod) {
487 list_del(&fod->fcp_list);
488 fod->active = true;
489 /*
490 * no queue reference is taken, as it was taken by the
491 * queue lookup just prior to the allocation. The iod
492 * will "inherit" that reference.
493 */
494 }
495 return fod;
496 }
497
498
499 static void
500 nvmet_fc_queue_fcp_req(struct nvmet_fc_tgtport *tgtport,
501 struct nvmet_fc_tgt_queue *queue,
502 struct nvmefc_tgt_fcp_req *fcpreq)
503 {
504 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
505
506 /*
507 * put all admin cmds on hw queue id 0. All io commands go to
508 * the respective hw queue based on a modulo basis
509 */
510 fcpreq->hwqid = queue->qid ?
511 ((queue->qid - 1) % tgtport->ops->max_hw_queues) : 0;
512
513 nvmet_fc_handle_fcp_rqst(tgtport, fod);
514 }
515
516 static void
517 nvmet_fc_fcp_rqst_op_defer_work(struct work_struct *work)
518 {
519 struct nvmet_fc_fcp_iod *fod =
520 container_of(work, struct nvmet_fc_fcp_iod, defer_work);
521
522 /* Submit deferred IO for processing */
523 nvmet_fc_queue_fcp_req(fod->tgtport, fod->queue, fod->fcpreq);
524
525 }
526
527 static void
528 nvmet_fc_free_fcp_iod(struct nvmet_fc_tgt_queue *queue,
529 struct nvmet_fc_fcp_iod *fod)
530 {
531 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
532 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
533 struct nvmet_fc_defer_fcp_req *deferfcp;
534 unsigned long flags;
535
536 fc_dma_sync_single_for_cpu(tgtport->dev, fod->rspdma,
537 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
538
539 fcpreq->nvmet_fc_private = NULL;
540
541 fod->active = false;
542 fod->abort = false;
543 fod->aborted = false;
544 fod->writedataactive = false;
545 fod->fcpreq = NULL;
546
547 tgtport->ops->fcp_req_release(&tgtport->fc_target_port, fcpreq);
548
549 /* release the queue lookup reference on the completed IO */
550 nvmet_fc_tgt_q_put(queue);
551
552 spin_lock_irqsave(&queue->qlock, flags);
553 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
554 struct nvmet_fc_defer_fcp_req, req_list);
555 if (!deferfcp) {
556 list_add_tail(&fod->fcp_list, &fod->queue->fod_list);
557 spin_unlock_irqrestore(&queue->qlock, flags);
558 return;
559 }
560
561 /* Re-use the fod for the next pending cmd that was deferred */
562 list_del(&deferfcp->req_list);
563
564 fcpreq = deferfcp->fcp_req;
565
566 /* deferfcp can be reused for another IO at a later date */
567 list_add_tail(&deferfcp->req_list, &queue->avail_defer_list);
568
569 spin_unlock_irqrestore(&queue->qlock, flags);
570
571 /* Save NVME CMD IO in fod */
572 memcpy(&fod->cmdiubuf, fcpreq->rspaddr, fcpreq->rsplen);
573
574 /* Setup new fcpreq to be processed */
575 fcpreq->rspaddr = NULL;
576 fcpreq->rsplen = 0;
577 fcpreq->nvmet_fc_private = fod;
578 fod->fcpreq = fcpreq;
579 fod->active = true;
580
581 /* inform LLDD IO is now being processed */
582 tgtport->ops->defer_rcv(&tgtport->fc_target_port, fcpreq);
583
584 /*
585 * Leave the queue lookup get reference taken when
586 * fod was originally allocated.
587 */
588
589 queue_work(queue->work_q, &fod->defer_work);
590 }
591
592 static struct nvmet_fc_tgt_queue *
593 nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
594 u16 qid, u16 sqsize)
595 {
596 struct nvmet_fc_tgt_queue *queue;
597 unsigned long flags;
598 int ret;
599
600 if (qid > NVMET_NR_QUEUES)
601 return NULL;
602
603 queue = kzalloc((sizeof(*queue) +
604 (sizeof(struct nvmet_fc_fcp_iod) * sqsize)),
605 GFP_KERNEL);
606 if (!queue)
607 return NULL;
608
609 if (!nvmet_fc_tgt_a_get(assoc))
610 goto out_free_queue;
611
612 queue->work_q = alloc_workqueue("ntfc%d.%d.%d", 0, 0,
613 assoc->tgtport->fc_target_port.port_num,
614 assoc->a_id, qid);
615 if (!queue->work_q)
616 goto out_a_put;
617
618 queue->fod = (struct nvmet_fc_fcp_iod *)&queue[1];
619 queue->qid = qid;
620 queue->sqsize = sqsize;
621 queue->assoc = assoc;
622 INIT_LIST_HEAD(&queue->fod_list);
623 INIT_LIST_HEAD(&queue->avail_defer_list);
624 INIT_LIST_HEAD(&queue->pending_cmd_list);
625 atomic_set(&queue->connected, 0);
626 atomic_set(&queue->sqtail, 0);
627 atomic_set(&queue->rsn, 1);
628 atomic_set(&queue->zrspcnt, 0);
629 spin_lock_init(&queue->qlock);
630 kref_init(&queue->ref);
631
632 nvmet_fc_prep_fcp_iodlist(assoc->tgtport, queue);
633
634 ret = nvmet_sq_init(&queue->nvme_sq);
635 if (ret)
636 goto out_fail_iodlist;
637
638 WARN_ON(assoc->queues[qid]);
639 spin_lock_irqsave(&assoc->tgtport->lock, flags);
640 assoc->queues[qid] = queue;
641 spin_unlock_irqrestore(&assoc->tgtport->lock, flags);
642
643 return queue;
644
645 out_fail_iodlist:
646 nvmet_fc_destroy_fcp_iodlist(assoc->tgtport, queue);
647 destroy_workqueue(queue->work_q);
648 out_a_put:
649 nvmet_fc_tgt_a_put(assoc);
650 out_free_queue:
651 kfree(queue);
652 return NULL;
653 }
654
655
656 static void
657 nvmet_fc_tgt_queue_free(struct kref *ref)
658 {
659 struct nvmet_fc_tgt_queue *queue =
660 container_of(ref, struct nvmet_fc_tgt_queue, ref);
661 unsigned long flags;
662
663 spin_lock_irqsave(&queue->assoc->tgtport->lock, flags);
664 queue->assoc->queues[queue->qid] = NULL;
665 spin_unlock_irqrestore(&queue->assoc->tgtport->lock, flags);
666
667 nvmet_fc_destroy_fcp_iodlist(queue->assoc->tgtport, queue);
668
669 nvmet_fc_tgt_a_put(queue->assoc);
670
671 destroy_workqueue(queue->work_q);
672
673 kfree(queue);
674 }
675
676 static void
677 nvmet_fc_tgt_q_put(struct nvmet_fc_tgt_queue *queue)
678 {
679 kref_put(&queue->ref, nvmet_fc_tgt_queue_free);
680 }
681
682 static int
683 nvmet_fc_tgt_q_get(struct nvmet_fc_tgt_queue *queue)
684 {
685 return kref_get_unless_zero(&queue->ref);
686 }
687
688
689 static void
690 nvmet_fc_delete_target_queue(struct nvmet_fc_tgt_queue *queue)
691 {
692 struct nvmet_fc_tgtport *tgtport = queue->assoc->tgtport;
693 struct nvmet_fc_fcp_iod *fod = queue->fod;
694 struct nvmet_fc_defer_fcp_req *deferfcp, *tempptr;
695 unsigned long flags;
696 int i, writedataactive;
697 bool disconnect;
698
699 disconnect = atomic_xchg(&queue->connected, 0);
700
701 spin_lock_irqsave(&queue->qlock, flags);
702 /* about outstanding io's */
703 for (i = 0; i < queue->sqsize; fod++, i++) {
704 if (fod->active) {
705 spin_lock(&fod->flock);
706 fod->abort = true;
707 writedataactive = fod->writedataactive;
708 spin_unlock(&fod->flock);
709 /*
710 * only call lldd abort routine if waiting for
711 * writedata. other outstanding ops should finish
712 * on their own.
713 */
714 if (writedataactive) {
715 spin_lock(&fod->flock);
716 fod->aborted = true;
717 spin_unlock(&fod->flock);
718 tgtport->ops->fcp_abort(
719 &tgtport->fc_target_port, fod->fcpreq);
720 }
721 }
722 }
723
724 /* Cleanup defer'ed IOs in queue */
725 list_for_each_entry_safe(deferfcp, tempptr, &queue->avail_defer_list,
726 req_list) {
727 list_del(&deferfcp->req_list);
728 kfree(deferfcp);
729 }
730
731 for (;;) {
732 deferfcp = list_first_entry_or_null(&queue->pending_cmd_list,
733 struct nvmet_fc_defer_fcp_req, req_list);
734 if (!deferfcp)
735 break;
736
737 list_del(&deferfcp->req_list);
738 spin_unlock_irqrestore(&queue->qlock, flags);
739
740 tgtport->ops->defer_rcv(&tgtport->fc_target_port,
741 deferfcp->fcp_req);
742
743 tgtport->ops->fcp_abort(&tgtport->fc_target_port,
744 deferfcp->fcp_req);
745
746 tgtport->ops->fcp_req_release(&tgtport->fc_target_port,
747 deferfcp->fcp_req);
748
749 /* release the queue lookup reference */
750 nvmet_fc_tgt_q_put(queue);
751
752 kfree(deferfcp);
753
754 spin_lock_irqsave(&queue->qlock, flags);
755 }
756 spin_unlock_irqrestore(&queue->qlock, flags);
757
758 flush_workqueue(queue->work_q);
759
760 if (disconnect)
761 nvmet_sq_destroy(&queue->nvme_sq);
762
763 nvmet_fc_tgt_q_put(queue);
764 }
765
766 static struct nvmet_fc_tgt_queue *
767 nvmet_fc_find_target_queue(struct nvmet_fc_tgtport *tgtport,
768 u64 connection_id)
769 {
770 struct nvmet_fc_tgt_assoc *assoc;
771 struct nvmet_fc_tgt_queue *queue;
772 u64 association_id = nvmet_fc_getassociationid(connection_id);
773 u16 qid = nvmet_fc_getqueueid(connection_id);
774 unsigned long flags;
775
776 if (qid > NVMET_NR_QUEUES)
777 return NULL;
778
779 spin_lock_irqsave(&tgtport->lock, flags);
780 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
781 if (association_id == assoc->association_id) {
782 queue = assoc->queues[qid];
783 if (queue &&
784 (!atomic_read(&queue->connected) ||
785 !nvmet_fc_tgt_q_get(queue)))
786 queue = NULL;
787 spin_unlock_irqrestore(&tgtport->lock, flags);
788 return queue;
789 }
790 }
791 spin_unlock_irqrestore(&tgtport->lock, flags);
792 return NULL;
793 }
794
795 static void
796 nvmet_fc_delete_assoc(struct work_struct *work)
797 {
798 struct nvmet_fc_tgt_assoc *assoc =
799 container_of(work, struct nvmet_fc_tgt_assoc, del_work);
800
801 nvmet_fc_delete_target_assoc(assoc);
802 nvmet_fc_tgt_a_put(assoc);
803 }
804
805 static struct nvmet_fc_tgt_assoc *
806 nvmet_fc_alloc_target_assoc(struct nvmet_fc_tgtport *tgtport)
807 {
808 struct nvmet_fc_tgt_assoc *assoc, *tmpassoc;
809 unsigned long flags;
810 u64 ran;
811 int idx;
812 bool needrandom = true;
813
814 assoc = kzalloc(sizeof(*assoc), GFP_KERNEL);
815 if (!assoc)
816 return NULL;
817
818 idx = ida_simple_get(&tgtport->assoc_cnt, 0, 0, GFP_KERNEL);
819 if (idx < 0)
820 goto out_free_assoc;
821
822 if (!nvmet_fc_tgtport_get(tgtport))
823 goto out_ida_put;
824
825 assoc->tgtport = tgtport;
826 assoc->a_id = idx;
827 INIT_LIST_HEAD(&assoc->a_list);
828 kref_init(&assoc->ref);
829 INIT_WORK(&assoc->del_work, nvmet_fc_delete_assoc);
830
831 while (needrandom) {
832 get_random_bytes(&ran, sizeof(ran) - BYTES_FOR_QID);
833 ran = ran << BYTES_FOR_QID_SHIFT;
834
835 spin_lock_irqsave(&tgtport->lock, flags);
836 needrandom = false;
837 list_for_each_entry(tmpassoc, &tgtport->assoc_list, a_list)
838 if (ran == tmpassoc->association_id) {
839 needrandom = true;
840 break;
841 }
842 if (!needrandom) {
843 assoc->association_id = ran;
844 list_add_tail(&assoc->a_list, &tgtport->assoc_list);
845 }
846 spin_unlock_irqrestore(&tgtport->lock, flags);
847 }
848
849 return assoc;
850
851 out_ida_put:
852 ida_simple_remove(&tgtport->assoc_cnt, idx);
853 out_free_assoc:
854 kfree(assoc);
855 return NULL;
856 }
857
858 static void
859 nvmet_fc_target_assoc_free(struct kref *ref)
860 {
861 struct nvmet_fc_tgt_assoc *assoc =
862 container_of(ref, struct nvmet_fc_tgt_assoc, ref);
863 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
864 unsigned long flags;
865
866 spin_lock_irqsave(&tgtport->lock, flags);
867 list_del(&assoc->a_list);
868 spin_unlock_irqrestore(&tgtport->lock, flags);
869 ida_simple_remove(&tgtport->assoc_cnt, assoc->a_id);
870 kfree(assoc);
871 nvmet_fc_tgtport_put(tgtport);
872 }
873
874 static void
875 nvmet_fc_tgt_a_put(struct nvmet_fc_tgt_assoc *assoc)
876 {
877 kref_put(&assoc->ref, nvmet_fc_target_assoc_free);
878 }
879
880 static int
881 nvmet_fc_tgt_a_get(struct nvmet_fc_tgt_assoc *assoc)
882 {
883 return kref_get_unless_zero(&assoc->ref);
884 }
885
886 static void
887 nvmet_fc_delete_target_assoc(struct nvmet_fc_tgt_assoc *assoc)
888 {
889 struct nvmet_fc_tgtport *tgtport = assoc->tgtport;
890 struct nvmet_fc_tgt_queue *queue;
891 unsigned long flags;
892 int i;
893
894 spin_lock_irqsave(&tgtport->lock, flags);
895 for (i = NVMET_NR_QUEUES; i >= 0; i--) {
896 queue = assoc->queues[i];
897 if (queue) {
898 if (!nvmet_fc_tgt_q_get(queue))
899 continue;
900 spin_unlock_irqrestore(&tgtport->lock, flags);
901 nvmet_fc_delete_target_queue(queue);
902 nvmet_fc_tgt_q_put(queue);
903 spin_lock_irqsave(&tgtport->lock, flags);
904 }
905 }
906 spin_unlock_irqrestore(&tgtport->lock, flags);
907
908 nvmet_fc_tgt_a_put(assoc);
909 }
910
911 static struct nvmet_fc_tgt_assoc *
912 nvmet_fc_find_target_assoc(struct nvmet_fc_tgtport *tgtport,
913 u64 association_id)
914 {
915 struct nvmet_fc_tgt_assoc *assoc;
916 struct nvmet_fc_tgt_assoc *ret = NULL;
917 unsigned long flags;
918
919 spin_lock_irqsave(&tgtport->lock, flags);
920 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
921 if (association_id == assoc->association_id) {
922 ret = assoc;
923 nvmet_fc_tgt_a_get(assoc);
924 break;
925 }
926 }
927 spin_unlock_irqrestore(&tgtport->lock, flags);
928
929 return ret;
930 }
931
932 static void
933 nvmet_fc_portentry_bind(struct nvmet_fc_tgtport *tgtport,
934 struct nvmet_fc_port_entry *pe,
935 struct nvmet_port *port)
936 {
937 lockdep_assert_held(&nvmet_fc_tgtlock);
938
939 pe->tgtport = tgtport;
940 tgtport->pe = pe;
941
942 pe->port = port;
943 port->priv = pe;
944
945 pe->node_name = tgtport->fc_target_port.node_name;
946 pe->port_name = tgtport->fc_target_port.port_name;
947 INIT_LIST_HEAD(&pe->pe_list);
948
949 list_add_tail(&pe->pe_list, &nvmet_fc_portentry_list);
950 }
951
952 static void
953 nvmet_fc_portentry_unbind(struct nvmet_fc_port_entry *pe)
954 {
955 unsigned long flags;
956
957 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
958 if (pe->tgtport)
959 pe->tgtport->pe = NULL;
960 list_del(&pe->pe_list);
961 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
962 }
963
964 /*
965 * called when a targetport deregisters. Breaks the relationship
966 * with the nvmet port, but leaves the port_entry in place so that
967 * re-registration can resume operation.
968 */
969 static void
970 nvmet_fc_portentry_unbind_tgt(struct nvmet_fc_tgtport *tgtport)
971 {
972 struct nvmet_fc_port_entry *pe;
973 unsigned long flags;
974
975 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
976 pe = tgtport->pe;
977 if (pe)
978 pe->tgtport = NULL;
979 tgtport->pe = NULL;
980 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
981 }
982
983 /*
984 * called when a new targetport is registered. Looks in the
985 * existing nvmet port_entries to see if the nvmet layer is
986 * configured for the targetport's wwn's. (the targetport existed,
987 * nvmet configured, the lldd unregistered the tgtport, and is now
988 * reregistering the same targetport). If so, set the nvmet port
989 * port entry on the targetport.
990 */
991 static void
992 nvmet_fc_portentry_rebind_tgt(struct nvmet_fc_tgtport *tgtport)
993 {
994 struct nvmet_fc_port_entry *pe;
995 unsigned long flags;
996
997 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
998 list_for_each_entry(pe, &nvmet_fc_portentry_list, pe_list) {
999 if (tgtport->fc_target_port.node_name == pe->node_name &&
1000 tgtport->fc_target_port.port_name == pe->port_name) {
1001 WARN_ON(pe->tgtport);
1002 tgtport->pe = pe;
1003 pe->tgtport = tgtport;
1004 break;
1005 }
1006 }
1007 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1008 }
1009
1010 /**
1011 * nvme_fc_register_targetport - transport entry point called by an
1012 * LLDD to register the existence of a local
1013 * NVME subystem FC port.
1014 * @pinfo: pointer to information about the port to be registered
1015 * @template: LLDD entrypoints and operational parameters for the port
1016 * @dev: physical hardware device node port corresponds to. Will be
1017 * used for DMA mappings
1018 * @portptr: pointer to a local port pointer. Upon success, the routine
1019 * will allocate a nvme_fc_local_port structure and place its
1020 * address in the local port pointer. Upon failure, local port
1021 * pointer will be set to NULL.
1022 *
1023 * Returns:
1024 * a completion status. Must be 0 upon success; a negative errno
1025 * (ex: -ENXIO) upon failure.
1026 */
1027 int
1028 nvmet_fc_register_targetport(struct nvmet_fc_port_info *pinfo,
1029 struct nvmet_fc_target_template *template,
1030 struct device *dev,
1031 struct nvmet_fc_target_port **portptr)
1032 {
1033 struct nvmet_fc_tgtport *newrec;
1034 unsigned long flags;
1035 int ret, idx;
1036
1037 if (!template->xmt_ls_rsp || !template->fcp_op ||
1038 !template->fcp_abort ||
1039 !template->fcp_req_release || !template->targetport_delete ||
1040 !template->max_hw_queues || !template->max_sgl_segments ||
1041 !template->max_dif_sgl_segments || !template->dma_boundary) {
1042 ret = -EINVAL;
1043 goto out_regtgt_failed;
1044 }
1045
1046 newrec = kzalloc((sizeof(*newrec) + template->target_priv_sz),
1047 GFP_KERNEL);
1048 if (!newrec) {
1049 ret = -ENOMEM;
1050 goto out_regtgt_failed;
1051 }
1052
1053 idx = ida_simple_get(&nvmet_fc_tgtport_cnt, 0, 0, GFP_KERNEL);
1054 if (idx < 0) {
1055 ret = -ENOSPC;
1056 goto out_fail_kfree;
1057 }
1058
1059 if (!get_device(dev) && dev) {
1060 ret = -ENODEV;
1061 goto out_ida_put;
1062 }
1063
1064 newrec->fc_target_port.node_name = pinfo->node_name;
1065 newrec->fc_target_port.port_name = pinfo->port_name;
1066 newrec->fc_target_port.private = &newrec[1];
1067 newrec->fc_target_port.port_id = pinfo->port_id;
1068 newrec->fc_target_port.port_num = idx;
1069 INIT_LIST_HEAD(&newrec->tgt_list);
1070 newrec->dev = dev;
1071 newrec->ops = template;
1072 spin_lock_init(&newrec->lock);
1073 INIT_LIST_HEAD(&newrec->ls_list);
1074 INIT_LIST_HEAD(&newrec->ls_busylist);
1075 INIT_LIST_HEAD(&newrec->assoc_list);
1076 kref_init(&newrec->ref);
1077 ida_init(&newrec->assoc_cnt);
1078 newrec->max_sg_cnt = template->max_sgl_segments;
1079
1080 ret = nvmet_fc_alloc_ls_iodlist(newrec);
1081 if (ret) {
1082 ret = -ENOMEM;
1083 goto out_free_newrec;
1084 }
1085
1086 nvmet_fc_portentry_rebind_tgt(newrec);
1087
1088 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1089 list_add_tail(&newrec->tgt_list, &nvmet_fc_target_list);
1090 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1091
1092 *portptr = &newrec->fc_target_port;
1093 return 0;
1094
1095 out_free_newrec:
1096 put_device(dev);
1097 out_ida_put:
1098 ida_simple_remove(&nvmet_fc_tgtport_cnt, idx);
1099 out_fail_kfree:
1100 kfree(newrec);
1101 out_regtgt_failed:
1102 *portptr = NULL;
1103 return ret;
1104 }
1105 EXPORT_SYMBOL_GPL(nvmet_fc_register_targetport);
1106
1107
1108 static void
1109 nvmet_fc_free_tgtport(struct kref *ref)
1110 {
1111 struct nvmet_fc_tgtport *tgtport =
1112 container_of(ref, struct nvmet_fc_tgtport, ref);
1113 struct device *dev = tgtport->dev;
1114 unsigned long flags;
1115
1116 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1117 list_del(&tgtport->tgt_list);
1118 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1119
1120 nvmet_fc_free_ls_iodlist(tgtport);
1121
1122 /* let the LLDD know we've finished tearing it down */
1123 tgtport->ops->targetport_delete(&tgtport->fc_target_port);
1124
1125 ida_simple_remove(&nvmet_fc_tgtport_cnt,
1126 tgtport->fc_target_port.port_num);
1127
1128 ida_destroy(&tgtport->assoc_cnt);
1129
1130 kfree(tgtport);
1131
1132 put_device(dev);
1133 }
1134
1135 static void
1136 nvmet_fc_tgtport_put(struct nvmet_fc_tgtport *tgtport)
1137 {
1138 kref_put(&tgtport->ref, nvmet_fc_free_tgtport);
1139 }
1140
1141 static int
1142 nvmet_fc_tgtport_get(struct nvmet_fc_tgtport *tgtport)
1143 {
1144 return kref_get_unless_zero(&tgtport->ref);
1145 }
1146
1147 static void
1148 __nvmet_fc_free_assocs(struct nvmet_fc_tgtport *tgtport)
1149 {
1150 struct nvmet_fc_tgt_assoc *assoc, *next;
1151 unsigned long flags;
1152
1153 spin_lock_irqsave(&tgtport->lock, flags);
1154 list_for_each_entry_safe(assoc, next,
1155 &tgtport->assoc_list, a_list) {
1156 if (!nvmet_fc_tgt_a_get(assoc))
1157 continue;
1158 spin_unlock_irqrestore(&tgtport->lock, flags);
1159 nvmet_fc_delete_target_assoc(assoc);
1160 nvmet_fc_tgt_a_put(assoc);
1161 spin_lock_irqsave(&tgtport->lock, flags);
1162 }
1163 spin_unlock_irqrestore(&tgtport->lock, flags);
1164 }
1165
1166 /*
1167 * nvmet layer has called to terminate an association
1168 */
1169 static void
1170 nvmet_fc_delete_ctrl(struct nvmet_ctrl *ctrl)
1171 {
1172 struct nvmet_fc_tgtport *tgtport, *next;
1173 struct nvmet_fc_tgt_assoc *assoc;
1174 struct nvmet_fc_tgt_queue *queue;
1175 unsigned long flags;
1176 bool found_ctrl = false;
1177
1178 /* this is a bit ugly, but don't want to make locks layered */
1179 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1180 list_for_each_entry_safe(tgtport, next, &nvmet_fc_target_list,
1181 tgt_list) {
1182 if (!nvmet_fc_tgtport_get(tgtport))
1183 continue;
1184 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1185
1186 spin_lock_irqsave(&tgtport->lock, flags);
1187 list_for_each_entry(assoc, &tgtport->assoc_list, a_list) {
1188 queue = assoc->queues[0];
1189 if (queue && queue->nvme_sq.ctrl == ctrl) {
1190 if (nvmet_fc_tgt_a_get(assoc))
1191 found_ctrl = true;
1192 break;
1193 }
1194 }
1195 spin_unlock_irqrestore(&tgtport->lock, flags);
1196
1197 nvmet_fc_tgtport_put(tgtport);
1198
1199 if (found_ctrl) {
1200 schedule_work(&assoc->del_work);
1201 return;
1202 }
1203
1204 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
1205 }
1206 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
1207 }
1208
1209 /**
1210 * nvme_fc_unregister_targetport - transport entry point called by an
1211 * LLDD to deregister/remove a previously
1212 * registered a local NVME subsystem FC port.
1213 * @target_port: pointer to the (registered) target port that is to be
1214 * deregistered.
1215 *
1216 * Returns:
1217 * a completion status. Must be 0 upon success; a negative errno
1218 * (ex: -ENXIO) upon failure.
1219 */
1220 int
1221 nvmet_fc_unregister_targetport(struct nvmet_fc_target_port *target_port)
1222 {
1223 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1224
1225 nvmet_fc_portentry_unbind_tgt(tgtport);
1226
1227 /* terminate any outstanding associations */
1228 __nvmet_fc_free_assocs(tgtport);
1229
1230 nvmet_fc_tgtport_put(tgtport);
1231
1232 return 0;
1233 }
1234 EXPORT_SYMBOL_GPL(nvmet_fc_unregister_targetport);
1235
1236
1237 /* *********************** FC-NVME LS Handling **************************** */
1238
1239
1240 static void
1241 nvmet_fc_format_rsp_hdr(void *buf, u8 ls_cmd, __be32 desc_len, u8 rqst_ls_cmd)
1242 {
1243 struct fcnvme_ls_acc_hdr *acc = buf;
1244
1245 acc->w0.ls_cmd = ls_cmd;
1246 acc->desc_list_len = desc_len;
1247 acc->rqst.desc_tag = cpu_to_be32(FCNVME_LSDESC_RQST);
1248 acc->rqst.desc_len =
1249 fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rqst));
1250 acc->rqst.w0.ls_cmd = rqst_ls_cmd;
1251 }
1252
1253 static int
1254 nvmet_fc_format_rjt(void *buf, u16 buflen, u8 ls_cmd,
1255 u8 reason, u8 explanation, u8 vendor)
1256 {
1257 struct fcnvme_ls_rjt *rjt = buf;
1258
1259 nvmet_fc_format_rsp_hdr(buf, FCNVME_LSDESC_RQST,
1260 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_rjt)),
1261 ls_cmd);
1262 rjt->rjt.desc_tag = cpu_to_be32(FCNVME_LSDESC_RJT);
1263 rjt->rjt.desc_len = fcnvme_lsdesc_len(sizeof(struct fcnvme_lsdesc_rjt));
1264 rjt->rjt.reason_code = reason;
1265 rjt->rjt.reason_explanation = explanation;
1266 rjt->rjt.vendor = vendor;
1267
1268 return sizeof(struct fcnvme_ls_rjt);
1269 }
1270
1271 /* Validation Error indexes into the string table below */
1272 enum {
1273 VERR_NO_ERROR = 0,
1274 VERR_CR_ASSOC_LEN = 1,
1275 VERR_CR_ASSOC_RQST_LEN = 2,
1276 VERR_CR_ASSOC_CMD = 3,
1277 VERR_CR_ASSOC_CMD_LEN = 4,
1278 VERR_ERSP_RATIO = 5,
1279 VERR_ASSOC_ALLOC_FAIL = 6,
1280 VERR_QUEUE_ALLOC_FAIL = 7,
1281 VERR_CR_CONN_LEN = 8,
1282 VERR_CR_CONN_RQST_LEN = 9,
1283 VERR_ASSOC_ID = 10,
1284 VERR_ASSOC_ID_LEN = 11,
1285 VERR_NO_ASSOC = 12,
1286 VERR_CONN_ID = 13,
1287 VERR_CONN_ID_LEN = 14,
1288 VERR_NO_CONN = 15,
1289 VERR_CR_CONN_CMD = 16,
1290 VERR_CR_CONN_CMD_LEN = 17,
1291 VERR_DISCONN_LEN = 18,
1292 VERR_DISCONN_RQST_LEN = 19,
1293 VERR_DISCONN_CMD = 20,
1294 VERR_DISCONN_CMD_LEN = 21,
1295 VERR_DISCONN_SCOPE = 22,
1296 VERR_RS_LEN = 23,
1297 VERR_RS_RQST_LEN = 24,
1298 VERR_RS_CMD = 25,
1299 VERR_RS_CMD_LEN = 26,
1300 VERR_RS_RCTL = 27,
1301 VERR_RS_RO = 28,
1302 };
1303
1304 static char *validation_errors[] = {
1305 "OK",
1306 "Bad CR_ASSOC Length",
1307 "Bad CR_ASSOC Rqst Length",
1308 "Not CR_ASSOC Cmd",
1309 "Bad CR_ASSOC Cmd Length",
1310 "Bad Ersp Ratio",
1311 "Association Allocation Failed",
1312 "Queue Allocation Failed",
1313 "Bad CR_CONN Length",
1314 "Bad CR_CONN Rqst Length",
1315 "Not Association ID",
1316 "Bad Association ID Length",
1317 "No Association",
1318 "Not Connection ID",
1319 "Bad Connection ID Length",
1320 "No Connection",
1321 "Not CR_CONN Cmd",
1322 "Bad CR_CONN Cmd Length",
1323 "Bad DISCONN Length",
1324 "Bad DISCONN Rqst Length",
1325 "Not DISCONN Cmd",
1326 "Bad DISCONN Cmd Length",
1327 "Bad Disconnect Scope",
1328 "Bad RS Length",
1329 "Bad RS Rqst Length",
1330 "Not RS Cmd",
1331 "Bad RS Cmd Length",
1332 "Bad RS R_CTL",
1333 "Bad RS Relative Offset",
1334 };
1335
1336 static void
1337 nvmet_fc_ls_create_association(struct nvmet_fc_tgtport *tgtport,
1338 struct nvmet_fc_ls_iod *iod)
1339 {
1340 struct fcnvme_ls_cr_assoc_rqst *rqst =
1341 (struct fcnvme_ls_cr_assoc_rqst *)iod->rqstbuf;
1342 struct fcnvme_ls_cr_assoc_acc *acc =
1343 (struct fcnvme_ls_cr_assoc_acc *)iod->rspbuf;
1344 struct nvmet_fc_tgt_queue *queue;
1345 int ret = 0;
1346
1347 memset(acc, 0, sizeof(*acc));
1348
1349 /*
1350 * FC-NVME spec changes. There are initiators sending different
1351 * lengths as padding sizes for Create Association Cmd descriptor
1352 * was incorrect.
1353 * Accept anything of "minimum" length. Assume format per 1.15
1354 * spec (with HOSTID reduced to 16 bytes), ignore how long the
1355 * trailing pad length is.
1356 */
1357 if (iod->rqstdatalen < FCNVME_LSDESC_CRA_RQST_MINLEN)
1358 ret = VERR_CR_ASSOC_LEN;
1359 else if (be32_to_cpu(rqst->desc_list_len) <
1360 FCNVME_LSDESC_CRA_RQST_MIN_LISTLEN)
1361 ret = VERR_CR_ASSOC_RQST_LEN;
1362 else if (rqst->assoc_cmd.desc_tag !=
1363 cpu_to_be32(FCNVME_LSDESC_CREATE_ASSOC_CMD))
1364 ret = VERR_CR_ASSOC_CMD;
1365 else if (be32_to_cpu(rqst->assoc_cmd.desc_len) <
1366 FCNVME_LSDESC_CRA_CMD_DESC_MIN_DESCLEN)
1367 ret = VERR_CR_ASSOC_CMD_LEN;
1368 else if (!rqst->assoc_cmd.ersp_ratio ||
1369 (be16_to_cpu(rqst->assoc_cmd.ersp_ratio) >=
1370 be16_to_cpu(rqst->assoc_cmd.sqsize)))
1371 ret = VERR_ERSP_RATIO;
1372
1373 else {
1374 /* new association w/ admin queue */
1375 iod->assoc = nvmet_fc_alloc_target_assoc(tgtport);
1376 if (!iod->assoc)
1377 ret = VERR_ASSOC_ALLOC_FAIL;
1378 else {
1379 queue = nvmet_fc_alloc_target_queue(iod->assoc, 0,
1380 be16_to_cpu(rqst->assoc_cmd.sqsize));
1381 if (!queue)
1382 ret = VERR_QUEUE_ALLOC_FAIL;
1383 }
1384 }
1385
1386 if (ret) {
1387 dev_err(tgtport->dev,
1388 "Create Association LS failed: %s\n",
1389 validation_errors[ret]);
1390 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1391 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1392 FCNVME_RJT_RC_LOGIC,
1393 FCNVME_RJT_EXP_NONE, 0);
1394 return;
1395 }
1396
1397 queue->ersp_ratio = be16_to_cpu(rqst->assoc_cmd.ersp_ratio);
1398 atomic_set(&queue->connected, 1);
1399 queue->sqhd = 0; /* best place to init value */
1400
1401 /* format a response */
1402
1403 iod->lsreq->rsplen = sizeof(*acc);
1404
1405 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1406 fcnvme_lsdesc_len(
1407 sizeof(struct fcnvme_ls_cr_assoc_acc)),
1408 FCNVME_LS_CREATE_ASSOCIATION);
1409 acc->associd.desc_tag = cpu_to_be32(FCNVME_LSDESC_ASSOC_ID);
1410 acc->associd.desc_len =
1411 fcnvme_lsdesc_len(
1412 sizeof(struct fcnvme_lsdesc_assoc_id));
1413 acc->associd.association_id =
1414 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc, 0));
1415 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1416 acc->connectid.desc_len =
1417 fcnvme_lsdesc_len(
1418 sizeof(struct fcnvme_lsdesc_conn_id));
1419 acc->connectid.connection_id = acc->associd.association_id;
1420 }
1421
1422 static void
1423 nvmet_fc_ls_create_connection(struct nvmet_fc_tgtport *tgtport,
1424 struct nvmet_fc_ls_iod *iod)
1425 {
1426 struct fcnvme_ls_cr_conn_rqst *rqst =
1427 (struct fcnvme_ls_cr_conn_rqst *)iod->rqstbuf;
1428 struct fcnvme_ls_cr_conn_acc *acc =
1429 (struct fcnvme_ls_cr_conn_acc *)iod->rspbuf;
1430 struct nvmet_fc_tgt_queue *queue;
1431 int ret = 0;
1432
1433 memset(acc, 0, sizeof(*acc));
1434
1435 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_cr_conn_rqst))
1436 ret = VERR_CR_CONN_LEN;
1437 else if (rqst->desc_list_len !=
1438 fcnvme_lsdesc_len(
1439 sizeof(struct fcnvme_ls_cr_conn_rqst)))
1440 ret = VERR_CR_CONN_RQST_LEN;
1441 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1442 ret = VERR_ASSOC_ID;
1443 else if (rqst->associd.desc_len !=
1444 fcnvme_lsdesc_len(
1445 sizeof(struct fcnvme_lsdesc_assoc_id)))
1446 ret = VERR_ASSOC_ID_LEN;
1447 else if (rqst->connect_cmd.desc_tag !=
1448 cpu_to_be32(FCNVME_LSDESC_CREATE_CONN_CMD))
1449 ret = VERR_CR_CONN_CMD;
1450 else if (rqst->connect_cmd.desc_len !=
1451 fcnvme_lsdesc_len(
1452 sizeof(struct fcnvme_lsdesc_cr_conn_cmd)))
1453 ret = VERR_CR_CONN_CMD_LEN;
1454 else if (!rqst->connect_cmd.ersp_ratio ||
1455 (be16_to_cpu(rqst->connect_cmd.ersp_ratio) >=
1456 be16_to_cpu(rqst->connect_cmd.sqsize)))
1457 ret = VERR_ERSP_RATIO;
1458
1459 else {
1460 /* new io queue */
1461 iod->assoc = nvmet_fc_find_target_assoc(tgtport,
1462 be64_to_cpu(rqst->associd.association_id));
1463 if (!iod->assoc)
1464 ret = VERR_NO_ASSOC;
1465 else {
1466 queue = nvmet_fc_alloc_target_queue(iod->assoc,
1467 be16_to_cpu(rqst->connect_cmd.qid),
1468 be16_to_cpu(rqst->connect_cmd.sqsize));
1469 if (!queue)
1470 ret = VERR_QUEUE_ALLOC_FAIL;
1471
1472 /* release get taken in nvmet_fc_find_target_assoc */
1473 nvmet_fc_tgt_a_put(iod->assoc);
1474 }
1475 }
1476
1477 if (ret) {
1478 dev_err(tgtport->dev,
1479 "Create Connection LS failed: %s\n",
1480 validation_errors[ret]);
1481 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1482 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1483 (ret == VERR_NO_ASSOC) ?
1484 FCNVME_RJT_RC_INV_ASSOC :
1485 FCNVME_RJT_RC_LOGIC,
1486 FCNVME_RJT_EXP_NONE, 0);
1487 return;
1488 }
1489
1490 queue->ersp_ratio = be16_to_cpu(rqst->connect_cmd.ersp_ratio);
1491 atomic_set(&queue->connected, 1);
1492 queue->sqhd = 0; /* best place to init value */
1493
1494 /* format a response */
1495
1496 iod->lsreq->rsplen = sizeof(*acc);
1497
1498 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1499 fcnvme_lsdesc_len(sizeof(struct fcnvme_ls_cr_conn_acc)),
1500 FCNVME_LS_CREATE_CONNECTION);
1501 acc->connectid.desc_tag = cpu_to_be32(FCNVME_LSDESC_CONN_ID);
1502 acc->connectid.desc_len =
1503 fcnvme_lsdesc_len(
1504 sizeof(struct fcnvme_lsdesc_conn_id));
1505 acc->connectid.connection_id =
1506 cpu_to_be64(nvmet_fc_makeconnid(iod->assoc,
1507 be16_to_cpu(rqst->connect_cmd.qid)));
1508 }
1509
1510 static void
1511 nvmet_fc_ls_disconnect(struct nvmet_fc_tgtport *tgtport,
1512 struct nvmet_fc_ls_iod *iod)
1513 {
1514 struct fcnvme_ls_disconnect_rqst *rqst =
1515 (struct fcnvme_ls_disconnect_rqst *)iod->rqstbuf;
1516 struct fcnvme_ls_disconnect_acc *acc =
1517 (struct fcnvme_ls_disconnect_acc *)iod->rspbuf;
1518 struct nvmet_fc_tgt_queue *queue = NULL;
1519 struct nvmet_fc_tgt_assoc *assoc;
1520 int ret = 0;
1521 bool del_assoc = false;
1522
1523 memset(acc, 0, sizeof(*acc));
1524
1525 if (iod->rqstdatalen < sizeof(struct fcnvme_ls_disconnect_rqst))
1526 ret = VERR_DISCONN_LEN;
1527 else if (rqst->desc_list_len !=
1528 fcnvme_lsdesc_len(
1529 sizeof(struct fcnvme_ls_disconnect_rqst)))
1530 ret = VERR_DISCONN_RQST_LEN;
1531 else if (rqst->associd.desc_tag != cpu_to_be32(FCNVME_LSDESC_ASSOC_ID))
1532 ret = VERR_ASSOC_ID;
1533 else if (rqst->associd.desc_len !=
1534 fcnvme_lsdesc_len(
1535 sizeof(struct fcnvme_lsdesc_assoc_id)))
1536 ret = VERR_ASSOC_ID_LEN;
1537 else if (rqst->discon_cmd.desc_tag !=
1538 cpu_to_be32(FCNVME_LSDESC_DISCONN_CMD))
1539 ret = VERR_DISCONN_CMD;
1540 else if (rqst->discon_cmd.desc_len !=
1541 fcnvme_lsdesc_len(
1542 sizeof(struct fcnvme_lsdesc_disconn_cmd)))
1543 ret = VERR_DISCONN_CMD_LEN;
1544 else if ((rqst->discon_cmd.scope != FCNVME_DISCONN_ASSOCIATION) &&
1545 (rqst->discon_cmd.scope != FCNVME_DISCONN_CONNECTION))
1546 ret = VERR_DISCONN_SCOPE;
1547 else {
1548 /* match an active association */
1549 assoc = nvmet_fc_find_target_assoc(tgtport,
1550 be64_to_cpu(rqst->associd.association_id));
1551 iod->assoc = assoc;
1552 if (assoc) {
1553 if (rqst->discon_cmd.scope ==
1554 FCNVME_DISCONN_CONNECTION) {
1555 queue = nvmet_fc_find_target_queue(tgtport,
1556 be64_to_cpu(
1557 rqst->discon_cmd.id));
1558 if (!queue) {
1559 nvmet_fc_tgt_a_put(assoc);
1560 ret = VERR_NO_CONN;
1561 }
1562 }
1563 } else
1564 ret = VERR_NO_ASSOC;
1565 }
1566
1567 if (ret) {
1568 dev_err(tgtport->dev,
1569 "Disconnect LS failed: %s\n",
1570 validation_errors[ret]);
1571 iod->lsreq->rsplen = nvmet_fc_format_rjt(acc,
1572 NVME_FC_MAX_LS_BUFFER_SIZE, rqst->w0.ls_cmd,
1573 (ret == VERR_NO_ASSOC) ?
1574 FCNVME_RJT_RC_INV_ASSOC :
1575 (ret == VERR_NO_CONN) ?
1576 FCNVME_RJT_RC_INV_CONN :
1577 FCNVME_RJT_RC_LOGIC,
1578 FCNVME_RJT_EXP_NONE, 0);
1579 return;
1580 }
1581
1582 /* format a response */
1583
1584 iod->lsreq->rsplen = sizeof(*acc);
1585
1586 nvmet_fc_format_rsp_hdr(acc, FCNVME_LS_ACC,
1587 fcnvme_lsdesc_len(
1588 sizeof(struct fcnvme_ls_disconnect_acc)),
1589 FCNVME_LS_DISCONNECT);
1590
1591
1592 /* are we to delete a Connection ID (queue) */
1593 if (queue) {
1594 int qid = queue->qid;
1595
1596 nvmet_fc_delete_target_queue(queue);
1597
1598 /* release the get taken by find_target_queue */
1599 nvmet_fc_tgt_q_put(queue);
1600
1601 /* tear association down if io queue terminated */
1602 if (!qid)
1603 del_assoc = true;
1604 }
1605
1606 /* release get taken in nvmet_fc_find_target_assoc */
1607 nvmet_fc_tgt_a_put(iod->assoc);
1608
1609 if (del_assoc)
1610 nvmet_fc_delete_target_assoc(iod->assoc);
1611 }
1612
1613
1614 /* *********************** NVME Ctrl Routines **************************** */
1615
1616
1617 static void nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req);
1618
1619 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops;
1620
1621 static void
1622 nvmet_fc_xmt_ls_rsp_done(struct nvmefc_tgt_ls_req *lsreq)
1623 {
1624 struct nvmet_fc_ls_iod *iod = lsreq->nvmet_fc_private;
1625 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1626
1627 fc_dma_sync_single_for_cpu(tgtport->dev, iod->rspdma,
1628 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1629 nvmet_fc_free_ls_iod(tgtport, iod);
1630 nvmet_fc_tgtport_put(tgtport);
1631 }
1632
1633 static void
1634 nvmet_fc_xmt_ls_rsp(struct nvmet_fc_tgtport *tgtport,
1635 struct nvmet_fc_ls_iod *iod)
1636 {
1637 int ret;
1638
1639 fc_dma_sync_single_for_device(tgtport->dev, iod->rspdma,
1640 NVME_FC_MAX_LS_BUFFER_SIZE, DMA_TO_DEVICE);
1641
1642 ret = tgtport->ops->xmt_ls_rsp(&tgtport->fc_target_port, iod->lsreq);
1643 if (ret)
1644 nvmet_fc_xmt_ls_rsp_done(iod->lsreq);
1645 }
1646
1647 /*
1648 * Actual processing routine for received FC-NVME LS Requests from the LLD
1649 */
1650 static void
1651 nvmet_fc_handle_ls_rqst(struct nvmet_fc_tgtport *tgtport,
1652 struct nvmet_fc_ls_iod *iod)
1653 {
1654 struct fcnvme_ls_rqst_w0 *w0 =
1655 (struct fcnvme_ls_rqst_w0 *)iod->rqstbuf;
1656
1657 iod->lsreq->nvmet_fc_private = iod;
1658 iod->lsreq->rspbuf = iod->rspbuf;
1659 iod->lsreq->rspdma = iod->rspdma;
1660 iod->lsreq->done = nvmet_fc_xmt_ls_rsp_done;
1661 /* Be preventative. handlers will later set to valid length */
1662 iod->lsreq->rsplen = 0;
1663
1664 iod->assoc = NULL;
1665
1666 /*
1667 * handlers:
1668 * parse request input, execute the request, and format the
1669 * LS response
1670 */
1671 switch (w0->ls_cmd) {
1672 case FCNVME_LS_CREATE_ASSOCIATION:
1673 /* Creates Association and initial Admin Queue/Connection */
1674 nvmet_fc_ls_create_association(tgtport, iod);
1675 break;
1676 case FCNVME_LS_CREATE_CONNECTION:
1677 /* Creates an IO Queue/Connection */
1678 nvmet_fc_ls_create_connection(tgtport, iod);
1679 break;
1680 case FCNVME_LS_DISCONNECT:
1681 /* Terminate a Queue/Connection or the Association */
1682 nvmet_fc_ls_disconnect(tgtport, iod);
1683 break;
1684 default:
1685 iod->lsreq->rsplen = nvmet_fc_format_rjt(iod->rspbuf,
1686 NVME_FC_MAX_LS_BUFFER_SIZE, w0->ls_cmd,
1687 FCNVME_RJT_RC_INVAL, FCNVME_RJT_EXP_NONE, 0);
1688 }
1689
1690 nvmet_fc_xmt_ls_rsp(tgtport, iod);
1691 }
1692
1693 /*
1694 * Actual processing routine for received FC-NVME LS Requests from the LLD
1695 */
1696 static void
1697 nvmet_fc_handle_ls_rqst_work(struct work_struct *work)
1698 {
1699 struct nvmet_fc_ls_iod *iod =
1700 container_of(work, struct nvmet_fc_ls_iod, work);
1701 struct nvmet_fc_tgtport *tgtport = iod->tgtport;
1702
1703 nvmet_fc_handle_ls_rqst(tgtport, iod);
1704 }
1705
1706
1707 /**
1708 * nvmet_fc_rcv_ls_req - transport entry point called by an LLDD
1709 * upon the reception of a NVME LS request.
1710 *
1711 * The nvmet-fc layer will copy payload to an internal structure for
1712 * processing. As such, upon completion of the routine, the LLDD may
1713 * immediately free/reuse the LS request buffer passed in the call.
1714 *
1715 * If this routine returns error, the LLDD should abort the exchange.
1716 *
1717 * @target_port: pointer to the (registered) target port the LS was
1718 * received on.
1719 * @lsreq: pointer to a lsreq request structure to be used to reference
1720 * the exchange corresponding to the LS.
1721 * @lsreqbuf: pointer to the buffer containing the LS Request
1722 * @lsreqbuf_len: length, in bytes, of the received LS request
1723 */
1724 int
1725 nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
1726 struct nvmefc_tgt_ls_req *lsreq,
1727 void *lsreqbuf, u32 lsreqbuf_len)
1728 {
1729 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
1730 struct nvmet_fc_ls_iod *iod;
1731
1732 if (lsreqbuf_len > NVME_FC_MAX_LS_BUFFER_SIZE)
1733 return -E2BIG;
1734
1735 if (!nvmet_fc_tgtport_get(tgtport))
1736 return -ESHUTDOWN;
1737
1738 iod = nvmet_fc_alloc_ls_iod(tgtport);
1739 if (!iod) {
1740 nvmet_fc_tgtport_put(tgtport);
1741 return -ENOENT;
1742 }
1743
1744 iod->lsreq = lsreq;
1745 iod->fcpreq = NULL;
1746 memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
1747 iod->rqstdatalen = lsreqbuf_len;
1748
1749 schedule_work(&iod->work);
1750
1751 return 0;
1752 }
1753 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_ls_req);
1754
1755
1756 /*
1757 * **********************
1758 * Start of FCP handling
1759 * **********************
1760 */
1761
1762 static int
1763 nvmet_fc_alloc_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1764 {
1765 struct scatterlist *sg;
1766 unsigned int nent;
1767
1768 sg = sgl_alloc(fod->req.transfer_len, GFP_KERNEL, &nent);
1769 if (!sg)
1770 goto out;
1771
1772 fod->data_sg = sg;
1773 fod->data_sg_cnt = nent;
1774 fod->data_sg_cnt = fc_dma_map_sg(fod->tgtport->dev, sg, nent,
1775 ((fod->io_dir == NVMET_FCP_WRITE) ?
1776 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1777 /* note: write from initiator perspective */
1778 fod->next_sg = fod->data_sg;
1779
1780 return 0;
1781
1782 out:
1783 return NVME_SC_INTERNAL;
1784 }
1785
1786 static void
1787 nvmet_fc_free_tgt_pgs(struct nvmet_fc_fcp_iod *fod)
1788 {
1789 if (!fod->data_sg || !fod->data_sg_cnt)
1790 return;
1791
1792 fc_dma_unmap_sg(fod->tgtport->dev, fod->data_sg, fod->data_sg_cnt,
1793 ((fod->io_dir == NVMET_FCP_WRITE) ?
1794 DMA_FROM_DEVICE : DMA_TO_DEVICE));
1795 sgl_free(fod->data_sg);
1796 fod->data_sg = NULL;
1797 fod->data_sg_cnt = 0;
1798 }
1799
1800
1801 static bool
1802 queue_90percent_full(struct nvmet_fc_tgt_queue *q, u32 sqhd)
1803 {
1804 u32 sqtail, used;
1805
1806 /* egad, this is ugly. And sqtail is just a best guess */
1807 sqtail = atomic_read(&q->sqtail) % q->sqsize;
1808
1809 used = (sqtail < sqhd) ? (sqtail + q->sqsize - sqhd) : (sqtail - sqhd);
1810 return ((used * 10) >= (((u32)(q->sqsize - 1) * 9)));
1811 }
1812
1813 /*
1814 * Prep RSP payload.
1815 * May be a NVMET_FCOP_RSP or NVMET_FCOP_READDATA_RSP op
1816 */
1817 static void
1818 nvmet_fc_prep_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1819 struct nvmet_fc_fcp_iod *fod)
1820 {
1821 struct nvme_fc_ersp_iu *ersp = &fod->rspiubuf;
1822 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
1823 struct nvme_completion *cqe = &ersp->cqe;
1824 u32 *cqewd = (u32 *)cqe;
1825 bool send_ersp = false;
1826 u32 rsn, rspcnt, xfr_length;
1827
1828 if (fod->fcpreq->op == NVMET_FCOP_READDATA_RSP)
1829 xfr_length = fod->req.transfer_len;
1830 else
1831 xfr_length = fod->offset;
1832
1833 /*
1834 * check to see if we can send a 0's rsp.
1835 * Note: to send a 0's response, the NVME-FC host transport will
1836 * recreate the CQE. The host transport knows: sq id, SQHD (last
1837 * seen in an ersp), and command_id. Thus it will create a
1838 * zero-filled CQE with those known fields filled in. Transport
1839 * must send an ersp for any condition where the cqe won't match
1840 * this.
1841 *
1842 * Here are the FC-NVME mandated cases where we must send an ersp:
1843 * every N responses, where N=ersp_ratio
1844 * force fabric commands to send ersp's (not in FC-NVME but good
1845 * practice)
1846 * normal cmds: any time status is non-zero, or status is zero
1847 * but words 0 or 1 are non-zero.
1848 * the SQ is 90% or more full
1849 * the cmd is a fused command
1850 * transferred data length not equal to cmd iu length
1851 */
1852 rspcnt = atomic_inc_return(&fod->queue->zrspcnt);
1853 if (!(rspcnt % fod->queue->ersp_ratio) ||
1854 sqe->opcode == nvme_fabrics_command ||
1855 xfr_length != fod->req.transfer_len ||
1856 (le16_to_cpu(cqe->status) & 0xFFFE) || cqewd[0] || cqewd[1] ||
1857 (sqe->flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND)) ||
1858 queue_90percent_full(fod->queue, le16_to_cpu(cqe->sq_head)))
1859 send_ersp = true;
1860
1861 /* re-set the fields */
1862 fod->fcpreq->rspaddr = ersp;
1863 fod->fcpreq->rspdma = fod->rspdma;
1864
1865 if (!send_ersp) {
1866 memset(ersp, 0, NVME_FC_SIZEOF_ZEROS_RSP);
1867 fod->fcpreq->rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
1868 } else {
1869 ersp->iu_len = cpu_to_be16(sizeof(*ersp)/sizeof(u32));
1870 rsn = atomic_inc_return(&fod->queue->rsn);
1871 ersp->rsn = cpu_to_be32(rsn);
1872 ersp->xfrd_len = cpu_to_be32(xfr_length);
1873 fod->fcpreq->rsplen = sizeof(*ersp);
1874 }
1875
1876 fc_dma_sync_single_for_device(tgtport->dev, fod->rspdma,
1877 sizeof(fod->rspiubuf), DMA_TO_DEVICE);
1878 }
1879
1880 static void nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq);
1881
1882 static void
1883 nvmet_fc_abort_op(struct nvmet_fc_tgtport *tgtport,
1884 struct nvmet_fc_fcp_iod *fod)
1885 {
1886 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1887
1888 /* data no longer needed */
1889 nvmet_fc_free_tgt_pgs(fod);
1890
1891 /*
1892 * if an ABTS was received or we issued the fcp_abort early
1893 * don't call abort routine again.
1894 */
1895 /* no need to take lock - lock was taken earlier to get here */
1896 if (!fod->aborted)
1897 tgtport->ops->fcp_abort(&tgtport->fc_target_port, fcpreq);
1898
1899 nvmet_fc_free_fcp_iod(fod->queue, fod);
1900 }
1901
1902 static void
1903 nvmet_fc_xmt_fcp_rsp(struct nvmet_fc_tgtport *tgtport,
1904 struct nvmet_fc_fcp_iod *fod)
1905 {
1906 int ret;
1907
1908 fod->fcpreq->op = NVMET_FCOP_RSP;
1909 fod->fcpreq->timeout = 0;
1910
1911 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1912
1913 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1914 if (ret)
1915 nvmet_fc_abort_op(tgtport, fod);
1916 }
1917
1918 static void
1919 nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport,
1920 struct nvmet_fc_fcp_iod *fod, u8 op)
1921 {
1922 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
1923 struct scatterlist *sg = fod->next_sg;
1924 unsigned long flags;
1925 u32 remaininglen = fod->req.transfer_len - fod->offset;
1926 u32 tlen = 0;
1927 int ret;
1928
1929 fcpreq->op = op;
1930 fcpreq->offset = fod->offset;
1931 fcpreq->timeout = NVME_FC_TGTOP_TIMEOUT_SEC;
1932
1933 /*
1934 * for next sequence:
1935 * break at a sg element boundary
1936 * attempt to keep sequence length capped at
1937 * NVMET_FC_MAX_SEQ_LENGTH but allow sequence to
1938 * be longer if a single sg element is larger
1939 * than that amount. This is done to avoid creating
1940 * a new sg list to use for the tgtport api.
1941 */
1942 fcpreq->sg = sg;
1943 fcpreq->sg_cnt = 0;
1944 while (tlen < remaininglen &&
1945 fcpreq->sg_cnt < tgtport->max_sg_cnt &&
1946 tlen + sg_dma_len(sg) < NVMET_FC_MAX_SEQ_LENGTH) {
1947 fcpreq->sg_cnt++;
1948 tlen += sg_dma_len(sg);
1949 sg = sg_next(sg);
1950 }
1951 if (tlen < remaininglen && fcpreq->sg_cnt == 0) {
1952 fcpreq->sg_cnt++;
1953 tlen += min_t(u32, sg_dma_len(sg), remaininglen);
1954 sg = sg_next(sg);
1955 }
1956 if (tlen < remaininglen)
1957 fod->next_sg = sg;
1958 else
1959 fod->next_sg = NULL;
1960
1961 fcpreq->transfer_length = tlen;
1962 fcpreq->transferred_length = 0;
1963 fcpreq->fcp_error = 0;
1964 fcpreq->rsplen = 0;
1965
1966 /*
1967 * If the last READDATA request: check if LLDD supports
1968 * combined xfr with response.
1969 */
1970 if ((op == NVMET_FCOP_READDATA) &&
1971 ((fod->offset + fcpreq->transfer_length) == fod->req.transfer_len) &&
1972 (tgtport->ops->target_features & NVMET_FCTGTFEAT_READDATA_RSP)) {
1973 fcpreq->op = NVMET_FCOP_READDATA_RSP;
1974 nvmet_fc_prep_fcp_rsp(tgtport, fod);
1975 }
1976
1977 ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq);
1978 if (ret) {
1979 /*
1980 * should be ok to set w/o lock as its in the thread of
1981 * execution (not an async timer routine) and doesn't
1982 * contend with any clearing action
1983 */
1984 fod->abort = true;
1985
1986 if (op == NVMET_FCOP_WRITEDATA) {
1987 spin_lock_irqsave(&fod->flock, flags);
1988 fod->writedataactive = false;
1989 spin_unlock_irqrestore(&fod->flock, flags);
1990 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
1991 } else /* NVMET_FCOP_READDATA or NVMET_FCOP_READDATA_RSP */ {
1992 fcpreq->fcp_error = ret;
1993 fcpreq->transferred_length = 0;
1994 nvmet_fc_xmt_fcp_op_done(fod->fcpreq);
1995 }
1996 }
1997 }
1998
1999 static inline bool
2000 __nvmet_fc_fod_op_abort(struct nvmet_fc_fcp_iod *fod, bool abort)
2001 {
2002 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2003 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2004
2005 /* if in the middle of an io and we need to tear down */
2006 if (abort) {
2007 if (fcpreq->op == NVMET_FCOP_WRITEDATA) {
2008 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2009 return true;
2010 }
2011
2012 nvmet_fc_abort_op(tgtport, fod);
2013 return true;
2014 }
2015
2016 return false;
2017 }
2018
2019 /*
2020 * actual done handler for FCP operations when completed by the lldd
2021 */
2022 static void
2023 nvmet_fc_fod_op_done(struct nvmet_fc_fcp_iod *fod)
2024 {
2025 struct nvmefc_tgt_fcp_req *fcpreq = fod->fcpreq;
2026 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2027 unsigned long flags;
2028 bool abort;
2029
2030 spin_lock_irqsave(&fod->flock, flags);
2031 abort = fod->abort;
2032 fod->writedataactive = false;
2033 spin_unlock_irqrestore(&fod->flock, flags);
2034
2035 switch (fcpreq->op) {
2036
2037 case NVMET_FCOP_WRITEDATA:
2038 if (__nvmet_fc_fod_op_abort(fod, abort))
2039 return;
2040 if (fcpreq->fcp_error ||
2041 fcpreq->transferred_length != fcpreq->transfer_length) {
2042 spin_lock(&fod->flock);
2043 fod->abort = true;
2044 spin_unlock(&fod->flock);
2045
2046 nvmet_req_complete(&fod->req, NVME_SC_INTERNAL);
2047 return;
2048 }
2049
2050 fod->offset += fcpreq->transferred_length;
2051 if (fod->offset != fod->req.transfer_len) {
2052 spin_lock_irqsave(&fod->flock, flags);
2053 fod->writedataactive = true;
2054 spin_unlock_irqrestore(&fod->flock, flags);
2055
2056 /* transfer the next chunk */
2057 nvmet_fc_transfer_fcp_data(tgtport, fod,
2058 NVMET_FCOP_WRITEDATA);
2059 return;
2060 }
2061
2062 /* data transfer complete, resume with nvmet layer */
2063 nvmet_req_execute(&fod->req);
2064 break;
2065
2066 case NVMET_FCOP_READDATA:
2067 case NVMET_FCOP_READDATA_RSP:
2068 if (__nvmet_fc_fod_op_abort(fod, abort))
2069 return;
2070 if (fcpreq->fcp_error ||
2071 fcpreq->transferred_length != fcpreq->transfer_length) {
2072 nvmet_fc_abort_op(tgtport, fod);
2073 return;
2074 }
2075
2076 /* success */
2077
2078 if (fcpreq->op == NVMET_FCOP_READDATA_RSP) {
2079 /* data no longer needed */
2080 nvmet_fc_free_tgt_pgs(fod);
2081 nvmet_fc_free_fcp_iod(fod->queue, fod);
2082 return;
2083 }
2084
2085 fod->offset += fcpreq->transferred_length;
2086 if (fod->offset != fod->req.transfer_len) {
2087 /* transfer the next chunk */
2088 nvmet_fc_transfer_fcp_data(tgtport, fod,
2089 NVMET_FCOP_READDATA);
2090 return;
2091 }
2092
2093 /* data transfer complete, send response */
2094
2095 /* data no longer needed */
2096 nvmet_fc_free_tgt_pgs(fod);
2097
2098 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2099
2100 break;
2101
2102 case NVMET_FCOP_RSP:
2103 if (__nvmet_fc_fod_op_abort(fod, abort))
2104 return;
2105 nvmet_fc_free_fcp_iod(fod->queue, fod);
2106 break;
2107
2108 default:
2109 break;
2110 }
2111 }
2112
2113 static void
2114 nvmet_fc_xmt_fcp_op_done(struct nvmefc_tgt_fcp_req *fcpreq)
2115 {
2116 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2117
2118 nvmet_fc_fod_op_done(fod);
2119 }
2120
2121 /*
2122 * actual completion handler after execution by the nvmet layer
2123 */
2124 static void
2125 __nvmet_fc_fcp_nvme_cmd_done(struct nvmet_fc_tgtport *tgtport,
2126 struct nvmet_fc_fcp_iod *fod, int status)
2127 {
2128 struct nvme_common_command *sqe = &fod->cmdiubuf.sqe.common;
2129 struct nvme_completion *cqe = &fod->rspiubuf.cqe;
2130 unsigned long flags;
2131 bool abort;
2132
2133 spin_lock_irqsave(&fod->flock, flags);
2134 abort = fod->abort;
2135 spin_unlock_irqrestore(&fod->flock, flags);
2136
2137 /* if we have a CQE, snoop the last sq_head value */
2138 if (!status)
2139 fod->queue->sqhd = cqe->sq_head;
2140
2141 if (abort) {
2142 nvmet_fc_abort_op(tgtport, fod);
2143 return;
2144 }
2145
2146 /* if an error handling the cmd post initial parsing */
2147 if (status) {
2148 /* fudge up a failed CQE status for our transport error */
2149 memset(cqe, 0, sizeof(*cqe));
2150 cqe->sq_head = fod->queue->sqhd; /* echo last cqe sqhd */
2151 cqe->sq_id = cpu_to_le16(fod->queue->qid);
2152 cqe->command_id = sqe->command_id;
2153 cqe->status = cpu_to_le16(status);
2154 } else {
2155
2156 /*
2157 * try to push the data even if the SQE status is non-zero.
2158 * There may be a status where data still was intended to
2159 * be moved
2160 */
2161 if ((fod->io_dir == NVMET_FCP_READ) && (fod->data_sg_cnt)) {
2162 /* push the data over before sending rsp */
2163 nvmet_fc_transfer_fcp_data(tgtport, fod,
2164 NVMET_FCOP_READDATA);
2165 return;
2166 }
2167
2168 /* writes & no data - fall thru */
2169 }
2170
2171 /* data no longer needed */
2172 nvmet_fc_free_tgt_pgs(fod);
2173
2174 nvmet_fc_xmt_fcp_rsp(tgtport, fod);
2175 }
2176
2177
2178 static void
2179 nvmet_fc_fcp_nvme_cmd_done(struct nvmet_req *nvme_req)
2180 {
2181 struct nvmet_fc_fcp_iod *fod = nvmet_req_to_fod(nvme_req);
2182 struct nvmet_fc_tgtport *tgtport = fod->tgtport;
2183
2184 __nvmet_fc_fcp_nvme_cmd_done(tgtport, fod, 0);
2185 }
2186
2187
2188 /*
2189 * Actual processing routine for received FC-NVME I/O Requests from the LLD
2190 */
2191 static void
2192 nvmet_fc_handle_fcp_rqst(struct nvmet_fc_tgtport *tgtport,
2193 struct nvmet_fc_fcp_iod *fod)
2194 {
2195 struct nvme_fc_cmd_iu *cmdiu = &fod->cmdiubuf;
2196 u32 xfrlen = be32_to_cpu(cmdiu->data_len);
2197 int ret;
2198
2199 /*
2200 * if there is no nvmet mapping to the targetport there
2201 * shouldn't be requests. just terminate them.
2202 */
2203 if (!tgtport->pe)
2204 goto transport_error;
2205
2206 /*
2207 * Fused commands are currently not supported in the linux
2208 * implementation.
2209 *
2210 * As such, the implementation of the FC transport does not
2211 * look at the fused commands and order delivery to the upper
2212 * layer until we have both based on csn.
2213 */
2214
2215 fod->fcpreq->done = nvmet_fc_xmt_fcp_op_done;
2216
2217 if (cmdiu->flags & FCNVME_CMD_FLAGS_WRITE) {
2218 fod->io_dir = NVMET_FCP_WRITE;
2219 if (!nvme_is_write(&cmdiu->sqe))
2220 goto transport_error;
2221 } else if (cmdiu->flags & FCNVME_CMD_FLAGS_READ) {
2222 fod->io_dir = NVMET_FCP_READ;
2223 if (nvme_is_write(&cmdiu->sqe))
2224 goto transport_error;
2225 } else {
2226 fod->io_dir = NVMET_FCP_NODATA;
2227 if (xfrlen)
2228 goto transport_error;
2229 }
2230
2231 fod->req.cmd = &fod->cmdiubuf.sqe;
2232 fod->req.rsp = &fod->rspiubuf.cqe;
2233 fod->req.port = tgtport->pe->port;
2234
2235 /* clear any response payload */
2236 memset(&fod->rspiubuf, 0, sizeof(fod->rspiubuf));
2237
2238 fod->data_sg = NULL;
2239 fod->data_sg_cnt = 0;
2240
2241 ret = nvmet_req_init(&fod->req,
2242 &fod->queue->nvme_cq,
2243 &fod->queue->nvme_sq,
2244 &nvmet_fc_tgt_fcp_ops);
2245 if (!ret) {
2246 /* bad SQE content or invalid ctrl state */
2247 /* nvmet layer has already called op done to send rsp. */
2248 return;
2249 }
2250
2251 fod->req.transfer_len = xfrlen;
2252
2253 /* keep a running counter of tail position */
2254 atomic_inc(&fod->queue->sqtail);
2255
2256 if (fod->req.transfer_len) {
2257 ret = nvmet_fc_alloc_tgt_pgs(fod);
2258 if (ret) {
2259 nvmet_req_complete(&fod->req, ret);
2260 return;
2261 }
2262 }
2263 fod->req.sg = fod->data_sg;
2264 fod->req.sg_cnt = fod->data_sg_cnt;
2265 fod->offset = 0;
2266
2267 if (fod->io_dir == NVMET_FCP_WRITE) {
2268 /* pull the data over before invoking nvmet layer */
2269 nvmet_fc_transfer_fcp_data(tgtport, fod, NVMET_FCOP_WRITEDATA);
2270 return;
2271 }
2272
2273 /*
2274 * Reads or no data:
2275 *
2276 * can invoke the nvmet_layer now. If read data, cmd completion will
2277 * push the data
2278 */
2279 nvmet_req_execute(&fod->req);
2280 return;
2281
2282 transport_error:
2283 nvmet_fc_abort_op(tgtport, fod);
2284 }
2285
2286 /**
2287 * nvmet_fc_rcv_fcp_req - transport entry point called by an LLDD
2288 * upon the reception of a NVME FCP CMD IU.
2289 *
2290 * Pass a FC-NVME FCP CMD IU received from the FC link to the nvmet-fc
2291 * layer for processing.
2292 *
2293 * The nvmet_fc layer allocates a local job structure (struct
2294 * nvmet_fc_fcp_iod) from the queue for the io and copies the
2295 * CMD IU buffer to the job structure. As such, on a successful
2296 * completion (returns 0), the LLDD may immediately free/reuse
2297 * the CMD IU buffer passed in the call.
2298 *
2299 * However, in some circumstances, due to the packetized nature of FC
2300 * and the api of the FC LLDD which may issue a hw command to send the
2301 * response, but the LLDD may not get the hw completion for that command
2302 * and upcall the nvmet_fc layer before a new command may be
2303 * asynchronously received - its possible for a command to be received
2304 * before the LLDD and nvmet_fc have recycled the job structure. It gives
2305 * the appearance of more commands received than fits in the sq.
2306 * To alleviate this scenario, a temporary queue is maintained in the
2307 * transport for pending LLDD requests waiting for a queue job structure.
2308 * In these "overrun" cases, a temporary queue element is allocated
2309 * the LLDD request and CMD iu buffer information remembered, and the
2310 * routine returns a -EOVERFLOW status. Subsequently, when a queue job
2311 * structure is freed, it is immediately reallocated for anything on the
2312 * pending request list. The LLDDs defer_rcv() callback is called,
2313 * informing the LLDD that it may reuse the CMD IU buffer, and the io
2314 * is then started normally with the transport.
2315 *
2316 * The LLDD, when receiving an -EOVERFLOW completion status, is to treat
2317 * the completion as successful but must not reuse the CMD IU buffer
2318 * until the LLDD's defer_rcv() callback has been called for the
2319 * corresponding struct nvmefc_tgt_fcp_req pointer.
2320 *
2321 * If there is any other condition in which an error occurs, the
2322 * transport will return a non-zero status indicating the error.
2323 * In all cases other than -EOVERFLOW, the transport has not accepted the
2324 * request and the LLDD should abort the exchange.
2325 *
2326 * @target_port: pointer to the (registered) target port the FCP CMD IU
2327 * was received on.
2328 * @fcpreq: pointer to a fcpreq request structure to be used to reference
2329 * the exchange corresponding to the FCP Exchange.
2330 * @cmdiubuf: pointer to the buffer containing the FCP CMD IU
2331 * @cmdiubuf_len: length, in bytes, of the received FCP CMD IU
2332 */
2333 int
2334 nvmet_fc_rcv_fcp_req(struct nvmet_fc_target_port *target_port,
2335 struct nvmefc_tgt_fcp_req *fcpreq,
2336 void *cmdiubuf, u32 cmdiubuf_len)
2337 {
2338 struct nvmet_fc_tgtport *tgtport = targetport_to_tgtport(target_port);
2339 struct nvme_fc_cmd_iu *cmdiu = cmdiubuf;
2340 struct nvmet_fc_tgt_queue *queue;
2341 struct nvmet_fc_fcp_iod *fod;
2342 struct nvmet_fc_defer_fcp_req *deferfcp;
2343 unsigned long flags;
2344
2345 /* validate iu, so the connection id can be used to find the queue */
2346 if ((cmdiubuf_len != sizeof(*cmdiu)) ||
2347 (cmdiu->scsi_id != NVME_CMD_SCSI_ID) ||
2348 (cmdiu->fc_id != NVME_CMD_FC_ID) ||
2349 (be16_to_cpu(cmdiu->iu_len) != (sizeof(*cmdiu)/4)))
2350 return -EIO;
2351
2352 queue = nvmet_fc_find_target_queue(tgtport,
2353 be64_to_cpu(cmdiu->connection_id));
2354 if (!queue)
2355 return -ENOTCONN;
2356
2357 /*
2358 * note: reference taken by find_target_queue
2359 * After successful fod allocation, the fod will inherit the
2360 * ownership of that reference and will remove the reference
2361 * when the fod is freed.
2362 */
2363
2364 spin_lock_irqsave(&queue->qlock, flags);
2365
2366 fod = nvmet_fc_alloc_fcp_iod(queue);
2367 if (fod) {
2368 spin_unlock_irqrestore(&queue->qlock, flags);
2369
2370 fcpreq->nvmet_fc_private = fod;
2371 fod->fcpreq = fcpreq;
2372
2373 memcpy(&fod->cmdiubuf, cmdiubuf, cmdiubuf_len);
2374
2375 nvmet_fc_queue_fcp_req(tgtport, queue, fcpreq);
2376
2377 return 0;
2378 }
2379
2380 if (!tgtport->ops->defer_rcv) {
2381 spin_unlock_irqrestore(&queue->qlock, flags);
2382 /* release the queue lookup reference */
2383 nvmet_fc_tgt_q_put(queue);
2384 return -ENOENT;
2385 }
2386
2387 deferfcp = list_first_entry_or_null(&queue->avail_defer_list,
2388 struct nvmet_fc_defer_fcp_req, req_list);
2389 if (deferfcp) {
2390 /* Just re-use one that was previously allocated */
2391 list_del(&deferfcp->req_list);
2392 } else {
2393 spin_unlock_irqrestore(&queue->qlock, flags);
2394
2395 /* Now we need to dynamically allocate one */
2396 deferfcp = kmalloc(sizeof(*deferfcp), GFP_KERNEL);
2397 if (!deferfcp) {
2398 /* release the queue lookup reference */
2399 nvmet_fc_tgt_q_put(queue);
2400 return -ENOMEM;
2401 }
2402 spin_lock_irqsave(&queue->qlock, flags);
2403 }
2404
2405 /* For now, use rspaddr / rsplen to save payload information */
2406 fcpreq->rspaddr = cmdiubuf;
2407 fcpreq->rsplen = cmdiubuf_len;
2408 deferfcp->fcp_req = fcpreq;
2409
2410 /* defer processing till a fod becomes available */
2411 list_add_tail(&deferfcp->req_list, &queue->pending_cmd_list);
2412
2413 /* NOTE: the queue lookup reference is still valid */
2414
2415 spin_unlock_irqrestore(&queue->qlock, flags);
2416
2417 return -EOVERFLOW;
2418 }
2419 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_req);
2420
2421 /**
2422 * nvmet_fc_rcv_fcp_abort - transport entry point called by an LLDD
2423 * upon the reception of an ABTS for a FCP command
2424 *
2425 * Notify the transport that an ABTS has been received for a FCP command
2426 * that had been given to the transport via nvmet_fc_rcv_fcp_req(). The
2427 * LLDD believes the command is still being worked on
2428 * (template_ops->fcp_req_release() has not been called).
2429 *
2430 * The transport will wait for any outstanding work (an op to the LLDD,
2431 * which the lldd should complete with error due to the ABTS; or the
2432 * completion from the nvmet layer of the nvme command), then will
2433 * stop processing and call the nvmet_fc_rcv_fcp_req() callback to
2434 * return the i/o context to the LLDD. The LLDD may send the BA_ACC
2435 * to the ABTS either after return from this function (assuming any
2436 * outstanding op work has been terminated) or upon the callback being
2437 * called.
2438 *
2439 * @target_port: pointer to the (registered) target port the FCP CMD IU
2440 * was received on.
2441 * @fcpreq: pointer to the fcpreq request structure that corresponds
2442 * to the exchange that received the ABTS.
2443 */
2444 void
2445 nvmet_fc_rcv_fcp_abort(struct nvmet_fc_target_port *target_port,
2446 struct nvmefc_tgt_fcp_req *fcpreq)
2447 {
2448 struct nvmet_fc_fcp_iod *fod = fcpreq->nvmet_fc_private;
2449 struct nvmet_fc_tgt_queue *queue;
2450 unsigned long flags;
2451
2452 if (!fod || fod->fcpreq != fcpreq)
2453 /* job appears to have already completed, ignore abort */
2454 return;
2455
2456 queue = fod->queue;
2457
2458 spin_lock_irqsave(&queue->qlock, flags);
2459 if (fod->active) {
2460 /*
2461 * mark as abort. The abort handler, invoked upon completion
2462 * of any work, will detect the aborted status and do the
2463 * callback.
2464 */
2465 spin_lock(&fod->flock);
2466 fod->abort = true;
2467 fod->aborted = true;
2468 spin_unlock(&fod->flock);
2469 }
2470 spin_unlock_irqrestore(&queue->qlock, flags);
2471 }
2472 EXPORT_SYMBOL_GPL(nvmet_fc_rcv_fcp_abort);
2473
2474
2475 struct nvmet_fc_traddr {
2476 u64 nn;
2477 u64 pn;
2478 };
2479
2480 static int
2481 __nvme_fc_parse_u64(substring_t *sstr, u64 *val)
2482 {
2483 u64 token64;
2484
2485 if (match_u64(sstr, &token64))
2486 return -EINVAL;
2487 *val = token64;
2488
2489 return 0;
2490 }
2491
2492 /*
2493 * This routine validates and extracts the WWN's from the TRADDR string.
2494 * As kernel parsers need the 0x to determine number base, universally
2495 * build string to parse with 0x prefix before parsing name strings.
2496 */
2497 static int
2498 nvme_fc_parse_traddr(struct nvmet_fc_traddr *traddr, char *buf, size_t blen)
2499 {
2500 char name[2 + NVME_FC_TRADDR_HEXNAMELEN + 1];
2501 substring_t wwn = { name, &name[sizeof(name)-1] };
2502 int nnoffset, pnoffset;
2503
2504 /* validate if string is one of the 2 allowed formats */
2505 if (strnlen(buf, blen) == NVME_FC_TRADDR_MAXLENGTH &&
2506 !strncmp(buf, "nn-0x", NVME_FC_TRADDR_OXNNLEN) &&
2507 !strncmp(&buf[NVME_FC_TRADDR_MAX_PN_OFFSET],
2508 "pn-0x", NVME_FC_TRADDR_OXNNLEN)) {
2509 nnoffset = NVME_FC_TRADDR_OXNNLEN;
2510 pnoffset = NVME_FC_TRADDR_MAX_PN_OFFSET +
2511 NVME_FC_TRADDR_OXNNLEN;
2512 } else if ((strnlen(buf, blen) == NVME_FC_TRADDR_MINLENGTH &&
2513 !strncmp(buf, "nn-", NVME_FC_TRADDR_NNLEN) &&
2514 !strncmp(&buf[NVME_FC_TRADDR_MIN_PN_OFFSET],
2515 "pn-", NVME_FC_TRADDR_NNLEN))) {
2516 nnoffset = NVME_FC_TRADDR_NNLEN;
2517 pnoffset = NVME_FC_TRADDR_MIN_PN_OFFSET + NVME_FC_TRADDR_NNLEN;
2518 } else
2519 goto out_einval;
2520
2521 name[0] = '0';
2522 name[1] = 'x';
2523 name[2 + NVME_FC_TRADDR_HEXNAMELEN] = 0;
2524
2525 memcpy(&name[2], &buf[nnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2526 if (__nvme_fc_parse_u64(&wwn, &traddr->nn))
2527 goto out_einval;
2528
2529 memcpy(&name[2], &buf[pnoffset], NVME_FC_TRADDR_HEXNAMELEN);
2530 if (__nvme_fc_parse_u64(&wwn, &traddr->pn))
2531 goto out_einval;
2532
2533 return 0;
2534
2535 out_einval:
2536 pr_warn("%s: bad traddr string\n", __func__);
2537 return -EINVAL;
2538 }
2539
2540 static int
2541 nvmet_fc_add_port(struct nvmet_port *port)
2542 {
2543 struct nvmet_fc_tgtport *tgtport;
2544 struct nvmet_fc_port_entry *pe;
2545 struct nvmet_fc_traddr traddr = { 0L, 0L };
2546 unsigned long flags;
2547 int ret;
2548
2549 /* validate the address info */
2550 if ((port->disc_addr.trtype != NVMF_TRTYPE_FC) ||
2551 (port->disc_addr.adrfam != NVMF_ADDR_FAMILY_FC))
2552 return -EINVAL;
2553
2554 /* map the traddr address info to a target port */
2555
2556 ret = nvme_fc_parse_traddr(&traddr, port->disc_addr.traddr,
2557 sizeof(port->disc_addr.traddr));
2558 if (ret)
2559 return ret;
2560
2561 pe = kzalloc(sizeof(*pe), GFP_KERNEL);
2562 if (!pe)
2563 return -ENOMEM;
2564
2565 ret = -ENXIO;
2566 spin_lock_irqsave(&nvmet_fc_tgtlock, flags);
2567 list_for_each_entry(tgtport, &nvmet_fc_target_list, tgt_list) {
2568 if ((tgtport->fc_target_port.node_name == traddr.nn) &&
2569 (tgtport->fc_target_port.port_name == traddr.pn)) {
2570 /* a FC port can only be 1 nvmet port id */
2571 if (!tgtport->pe) {
2572 nvmet_fc_portentry_bind(tgtport, pe, port);
2573 ret = 0;
2574 } else
2575 ret = -EALREADY;
2576 break;
2577 }
2578 }
2579 spin_unlock_irqrestore(&nvmet_fc_tgtlock, flags);
2580
2581 if (ret)
2582 kfree(pe);
2583
2584 return ret;
2585 }
2586
2587 static void
2588 nvmet_fc_remove_port(struct nvmet_port *port)
2589 {
2590 struct nvmet_fc_port_entry *pe = port->priv;
2591
2592 nvmet_fc_portentry_unbind(pe);
2593
2594 kfree(pe);
2595 }
2596
2597 static const struct nvmet_fabrics_ops nvmet_fc_tgt_fcp_ops = {
2598 .owner = THIS_MODULE,
2599 .type = NVMF_TRTYPE_FC,
2600 .msdbd = 1,
2601 .add_port = nvmet_fc_add_port,
2602 .remove_port = nvmet_fc_remove_port,
2603 .queue_response = nvmet_fc_fcp_nvme_cmd_done,
2604 .delete_ctrl = nvmet_fc_delete_ctrl,
2605 };
2606
2607 static int __init nvmet_fc_init_module(void)
2608 {
2609 return nvmet_register_transport(&nvmet_fc_tgt_fcp_ops);
2610 }
2611
2612 static void __exit nvmet_fc_exit_module(void)
2613 {
2614 /* sanity check - all lports should be removed */
2615 if (!list_empty(&nvmet_fc_target_list))
2616 pr_warn("%s: targetport list not empty\n", __func__);
2617
2618 nvmet_unregister_transport(&nvmet_fc_tgt_fcp_ops);
2619
2620 ida_destroy(&nvmet_fc_tgtport_cnt);
2621 }
2622
2623 module_init(nvmet_fc_init_module);
2624 module_exit(nvmet_fc_exit_module);
2625
2626 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support