?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-v5.11-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git/rostedt...
[linux/fpc-iii.git] / drivers / nvme / host / pci.c
blob50d9a20568a28df74abd85584259b246dbe67eac
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NVM Express device driver
4 * Copyright (c) 2011-2014, Intel Corporation.
5 */
6
7 #include <linux/acpi.h>
8 #include <linux/aer.h>
9 #include <linux/async.h>
10 #include <linux/blkdev.h>
11 #include <linux/blk-mq.h>
12 #include <linux/blk-mq-pci.h>
13 #include <linux/dmi.h>
14 #include <linux/init.h>
15 #include <linux/interrupt.h>
16 #include <linux/io.h>
17 #include <linux/mm.h>
18 #include <linux/module.h>
19 #include <linux/mutex.h>
20 #include <linux/once.h>
21 #include <linux/pci.h>
22 #include <linux/suspend.h>
23 #include <linux/t10-pi.h>
24 #include <linux/types.h>
25 #include <linux/io-64-nonatomic-lo-hi.h>
26 #include <linux/sed-opal.h>
27 #include <linux/pci-p2pdma.h>
28
29 #include "trace.h"
30 #include "nvme.h"
31
32 #define SQ_SIZE(q) ((q)->q_depth << (q)->sqes)
33 #define CQ_SIZE(q) ((q)->q_depth * sizeof(struct nvme_completion))
34
35 #define SGES_PER_PAGE (PAGE_SIZE / sizeof(struct nvme_sgl_desc))
36
37 /*
38 * These can be higher, but we need to ensure that any command doesn't
39 * require an sg allocation that needs more than a page of data.
40 */
41 #define NVME_MAX_KB_SZ 4096
42 #define NVME_MAX_SEGS 127
43
44 static int use_threaded_interrupts;
45 module_param(use_threaded_interrupts, int, 0);
46
47 static bool use_cmb_sqes = true;
48 module_param(use_cmb_sqes, bool, 0444);
49 MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");
50
51 static unsigned int max_host_mem_size_mb = 128;
52 module_param(max_host_mem_size_mb, uint, 0444);
53 MODULE_PARM_DESC(max_host_mem_size_mb,
54 "Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
55
56 static unsigned int sgl_threshold = SZ_32K;
57 module_param(sgl_threshold, uint, 0644);
58 MODULE_PARM_DESC(sgl_threshold,
59 "Use SGLs when average request segment size is larger or equal to "
60 "this size. Use 0 to disable SGLs.");
61
62 static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
63 static const struct kernel_param_ops io_queue_depth_ops = {
64 .set = io_queue_depth_set,
65 .get = param_get_uint,
66 };
67
68 static unsigned int io_queue_depth = 1024;
69 module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
70 MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2");
71
72 static int io_queue_count_set(const char *val, const struct kernel_param *kp)
73 {
74 unsigned int n;
75 int ret;
76
77 ret = kstrtouint(val, 10, &n);
78 if (ret != 0 || n > num_possible_cpus())
79 return -EINVAL;
80 return param_set_uint(val, kp);
81 }
82
83 static const struct kernel_param_ops io_queue_count_ops = {
84 .set = io_queue_count_set,
85 .get = param_get_uint,
86 };
87
88 static unsigned int write_queues;
89 module_param_cb(write_queues, &io_queue_count_ops, &write_queues, 0644);
90 MODULE_PARM_DESC(write_queues,
91 "Number of queues to use for writes. If not set, reads and writes "
92 "will share a queue set.");
93
94 static unsigned int poll_queues;
95 module_param_cb(poll_queues, &io_queue_count_ops, &poll_queues, 0644);
96 MODULE_PARM_DESC(poll_queues, "Number of queues to use for polled IO.");
97
98 static bool noacpi;
99 module_param(noacpi, bool, 0444);
100 MODULE_PARM_DESC(noacpi, "disable acpi bios quirks");
101
102 struct nvme_dev;
103 struct nvme_queue;
104
105 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
106 static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode);
107
108 /*
109 * Represents an NVM Express device. Each nvme_dev is a PCI function.
110 */
111 struct nvme_dev {
112 struct nvme_queue *queues;
113 struct blk_mq_tag_set tagset;
114 struct blk_mq_tag_set admin_tagset;
115 u32 __iomem *dbs;
116 struct device *dev;
117 struct dma_pool *prp_page_pool;
118 struct dma_pool *prp_small_pool;
119 unsigned online_queues;
120 unsigned max_qid;
121 unsigned io_queues[HCTX_MAX_TYPES];
122 unsigned int num_vecs;
123 u32 q_depth;
124 int io_sqes;
125 u32 db_stride;
126 void __iomem *bar;
127 unsigned long bar_mapped_size;
128 struct work_struct remove_work;
129 struct mutex shutdown_lock;
130 bool subsystem;
131 u64 cmb_size;
132 bool cmb_use_sqes;
133 u32 cmbsz;
134 u32 cmbloc;
135 struct nvme_ctrl ctrl;
136 u32 last_ps;
137
138 mempool_t *iod_mempool;
139
140 /* shadow doorbell buffer support: */
141 u32 *dbbuf_dbs;
142 dma_addr_t dbbuf_dbs_dma_addr;
143 u32 *dbbuf_eis;
144 dma_addr_t dbbuf_eis_dma_addr;
145
146 /* host memory buffer support: */
147 u64 host_mem_size;
148 u32 nr_host_mem_descs;
149 dma_addr_t host_mem_descs_dma;
150 struct nvme_host_mem_buf_desc *host_mem_descs;
151 void **host_mem_desc_bufs;
152 unsigned int nr_allocated_queues;
153 unsigned int nr_write_queues;
154 unsigned int nr_poll_queues;
155 };
156
157 static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
158 {
159 int ret;
160 u32 n;
161
162 ret = kstrtou32(val, 10, &n);
163 if (ret != 0 || n < 2)
164 return -EINVAL;
165
166 return param_set_uint(val, kp);
167 }
168
169 static inline unsigned int sq_idx(unsigned int qid, u32 stride)
170 {
171 return qid * 2 * stride;
172 }
173
174 static inline unsigned int cq_idx(unsigned int qid, u32 stride)
175 {
176 return (qid * 2 + 1) * stride;
177 }
178
179 static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
180 {
181 return container_of(ctrl, struct nvme_dev, ctrl);
182 }
183
184 /*
185 * An NVM Express queue. Each device has at least two (one for admin
186 * commands and one for I/O commands).
187 */
188 struct nvme_queue {
189 struct nvme_dev *dev;
190 spinlock_t sq_lock;
191 void *sq_cmds;
192 /* only used for poll queues: */
193 spinlock_t cq_poll_lock ____cacheline_aligned_in_smp;
194 struct nvme_completion *cqes;
195 dma_addr_t sq_dma_addr;
196 dma_addr_t cq_dma_addr;
197 u32 __iomem *q_db;
198 u32 q_depth;
199 u16 cq_vector;
200 u16 sq_tail;
201 u16 last_sq_tail;
202 u16 cq_head;
203 u16 qid;
204 u8 cq_phase;
205 u8 sqes;
206 unsigned long flags;
207 #define NVMEQ_ENABLED 0
208 #define NVMEQ_SQ_CMB 1
209 #define NVMEQ_DELETE_ERROR 2
210 #define NVMEQ_POLLED 3
211 u32 *dbbuf_sq_db;
212 u32 *dbbuf_cq_db;
213 u32 *dbbuf_sq_ei;
214 u32 *dbbuf_cq_ei;
215 struct completion delete_done;
216 };
217
218 /*
219 * The nvme_iod describes the data in an I/O.
220 *
221 * The sg pointer contains the list of PRP/SGL chunk allocations in addition
222 * to the actual struct scatterlist.
223 */
224 struct nvme_iod {
225 struct nvme_request req;
226 struct nvme_queue *nvmeq;
227 bool use_sgl;
228 int aborted;
229 int npages; /* In the PRP list. 0 means small pool in use */
230 int nents; /* Used in scatterlist */
231 dma_addr_t first_dma;
232 unsigned int dma_len; /* length of single DMA segment mapping */
233 dma_addr_t meta_dma;
234 struct scatterlist *sg;
235 };
236
237 static inline unsigned int nvme_dbbuf_size(struct nvme_dev *dev)
238 {
239 return dev->nr_allocated_queues * 8 * dev->db_stride;
240 }
241
242 static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
243 {
244 unsigned int mem_size = nvme_dbbuf_size(dev);
245
246 if (dev->dbbuf_dbs)
247 return 0;
248
249 dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
250 &dev->dbbuf_dbs_dma_addr,
251 GFP_KERNEL);
252 if (!dev->dbbuf_dbs)
253 return -ENOMEM;
254 dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
255 &dev->dbbuf_eis_dma_addr,
256 GFP_KERNEL);
257 if (!dev->dbbuf_eis) {
258 dma_free_coherent(dev->dev, mem_size,
259 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
260 dev->dbbuf_dbs = NULL;
261 return -ENOMEM;
262 }
263
264 return 0;
265 }
266
267 static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
268 {
269 unsigned int mem_size = nvme_dbbuf_size(dev);
270
271 if (dev->dbbuf_dbs) {
272 dma_free_coherent(dev->dev, mem_size,
273 dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
274 dev->dbbuf_dbs = NULL;
275 }
276 if (dev->dbbuf_eis) {
277 dma_free_coherent(dev->dev, mem_size,
278 dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
279 dev->dbbuf_eis = NULL;
280 }
281 }
282
283 static void nvme_dbbuf_init(struct nvme_dev *dev,
284 struct nvme_queue *nvmeq, int qid)
285 {
286 if (!dev->dbbuf_dbs || !qid)
287 return;
288
289 nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
290 nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
291 nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
292 nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
293 }
294
295 static void nvme_dbbuf_free(struct nvme_queue *nvmeq)
296 {
297 if (!nvmeq->qid)
298 return;
299
300 nvmeq->dbbuf_sq_db = NULL;
301 nvmeq->dbbuf_cq_db = NULL;
302 nvmeq->dbbuf_sq_ei = NULL;
303 nvmeq->dbbuf_cq_ei = NULL;
304 }
305
306 static void nvme_dbbuf_set(struct nvme_dev *dev)
307 {
308 struct nvme_command c;
309 unsigned int i;
310
311 if (!dev->dbbuf_dbs)
312 return;
313
314 memset(&c, 0, sizeof(c));
315 c.dbbuf.opcode = nvme_admin_dbbuf;
316 c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
317 c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);
318
319 if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
320 dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
321 /* Free memory and continue on */
322 nvme_dbbuf_dma_free(dev);
323
324 for (i = 1; i <= dev->online_queues; i++)
325 nvme_dbbuf_free(&dev->queues[i]);
326 }
327 }
328
329 static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
330 {
331 return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
332 }
333
334 /* Update dbbuf and return true if an MMIO is required */
335 static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
336 volatile u32 *dbbuf_ei)
337 {
338 if (dbbuf_db) {
339 u16 old_value;
340
341 /*
342 * Ensure that the queue is written before updating
343 * the doorbell in memory
344 */
345 wmb();
346
347 old_value = *dbbuf_db;
348 *dbbuf_db = value;
349
350 /*
351 * Ensure that the doorbell is updated before reading the event
352 * index from memory. The controller needs to provide similar
353 * ordering to ensure the envent index is updated before reading
354 * the doorbell.
355 */
356 mb();
357
358 if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
359 return false;
360 }
361
362 return true;
363 }
364
365 /*
366 * Will slightly overestimate the number of pages needed. This is OK
367 * as it only leads to a small amount of wasted memory for the lifetime of
368 * the I/O.
369 */
370 static int nvme_pci_npages_prp(void)
371 {
372 unsigned nprps = DIV_ROUND_UP(NVME_MAX_KB_SZ + NVME_CTRL_PAGE_SIZE,
373 NVME_CTRL_PAGE_SIZE);
374 return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
375 }
376
377 /*
378 * Calculates the number of pages needed for the SGL segments. For example a 4k
379 * page can accommodate 256 SGL descriptors.
380 */
381 static int nvme_pci_npages_sgl(void)
382 {
383 return DIV_ROUND_UP(NVME_MAX_SEGS * sizeof(struct nvme_sgl_desc),
384 PAGE_SIZE);
385 }
386
387 static size_t nvme_pci_iod_alloc_size(void)
388 {
389 size_t npages = max(nvme_pci_npages_prp(), nvme_pci_npages_sgl());
390
391 return sizeof(__le64 *) * npages +
392 sizeof(struct scatterlist) * NVME_MAX_SEGS;
393 }
394
395 static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
396 unsigned int hctx_idx)
397 {
398 struct nvme_dev *dev = data;
399 struct nvme_queue *nvmeq = &dev->queues[0];
400
401 WARN_ON(hctx_idx != 0);
402 WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
403
404 hctx->driver_data = nvmeq;
405 return 0;
406 }
407
408 static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
409 unsigned int hctx_idx)
410 {
411 struct nvme_dev *dev = data;
412 struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
413
414 WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
415 hctx->driver_data = nvmeq;
416 return 0;
417 }
418
419 static int nvme_init_request(struct blk_mq_tag_set *set, struct request *req,
420 unsigned int hctx_idx, unsigned int numa_node)
421 {
422 struct nvme_dev *dev = set->driver_data;
423 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
424 int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
425 struct nvme_queue *nvmeq = &dev->queues[queue_idx];
426
427 BUG_ON(!nvmeq);
428 iod->nvmeq = nvmeq;
429
430 nvme_req(req)->ctrl = &dev->ctrl;
431 return 0;
432 }
433
434 static int queue_irq_offset(struct nvme_dev *dev)
435 {
436 /* if we have more than 1 vec, admin queue offsets us by 1 */
437 if (dev->num_vecs > 1)
438 return 1;
439
440 return 0;
441 }
442
443 static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
444 {
445 struct nvme_dev *dev = set->driver_data;
446 int i, qoff, offset;
447
448 offset = queue_irq_offset(dev);
449 for (i = 0, qoff = 0; i < set->nr_maps; i++) {
450 struct blk_mq_queue_map *map = &set->map[i];
451
452 map->nr_queues = dev->io_queues[i];
453 if (!map->nr_queues) {
454 BUG_ON(i == HCTX_TYPE_DEFAULT);
455 continue;
456 }
457
458 /*
459 * The poll queue(s) doesn't have an IRQ (and hence IRQ
460 * affinity), so use the regular blk-mq cpu mapping
461 */
462 map->queue_offset = qoff;
463 if (i != HCTX_TYPE_POLL && offset)
464 blk_mq_pci_map_queues(map, to_pci_dev(dev->dev), offset);
465 else
466 blk_mq_map_queues(map);
467 qoff += map->nr_queues;
468 offset += map->nr_queues;
469 }
470
471 return 0;
472 }
473
474 /*
475 * Write sq tail if we are asked to, or if the next command would wrap.
476 */
477 static inline void nvme_write_sq_db(struct nvme_queue *nvmeq, bool write_sq)
478 {
479 if (!write_sq) {
480 u16 next_tail = nvmeq->sq_tail + 1;
481
482 if (next_tail == nvmeq->q_depth)
483 next_tail = 0;
484 if (next_tail != nvmeq->last_sq_tail)
485 return;
486 }
487
488 if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail,
489 nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))
490 writel(nvmeq->sq_tail, nvmeq->q_db);
491 nvmeq->last_sq_tail = nvmeq->sq_tail;
492 }
493
494 /**
495 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
496 * @nvmeq: The queue to use
497 * @cmd: The command to send
498 * @write_sq: whether to write to the SQ doorbell
499 */
500 static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd,
501 bool write_sq)
502 {
503 spin_lock(&nvmeq->sq_lock);
504 memcpy(nvmeq->sq_cmds + (nvmeq->sq_tail << nvmeq->sqes),
505 cmd, sizeof(*cmd));
506 if (++nvmeq->sq_tail == nvmeq->q_depth)
507 nvmeq->sq_tail = 0;
508 nvme_write_sq_db(nvmeq, write_sq);
509 spin_unlock(&nvmeq->sq_lock);
510 }
511
512 static void nvme_commit_rqs(struct blk_mq_hw_ctx *hctx)
513 {
514 struct nvme_queue *nvmeq = hctx->driver_data;
515
516 spin_lock(&nvmeq->sq_lock);
517 if (nvmeq->sq_tail != nvmeq->last_sq_tail)
518 nvme_write_sq_db(nvmeq, true);
519 spin_unlock(&nvmeq->sq_lock);
520 }
521
522 static void **nvme_pci_iod_list(struct request *req)
523 {
524 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
525 return (void **)(iod->sg + blk_rq_nr_phys_segments(req));
526 }
527
528 static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
529 {
530 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
531 int nseg = blk_rq_nr_phys_segments(req);
532 unsigned int avg_seg_size;
533
534 avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
535
536 if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1))))
537 return false;
538 if (!iod->nvmeq->qid)
539 return false;
540 if (!sgl_threshold || avg_seg_size < sgl_threshold)
541 return false;
542 return true;
543 }
544
545 static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
546 {
547 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
548 const int last_prp = NVME_CTRL_PAGE_SIZE / sizeof(__le64) - 1;
549 dma_addr_t dma_addr = iod->first_dma, next_dma_addr;
550 int i;
551
552 if (iod->dma_len) {
553 dma_unmap_page(dev->dev, dma_addr, iod->dma_len,
554 rq_dma_dir(req));
555 return;
556 }
557
558 WARN_ON_ONCE(!iod->nents);
559
560 if (is_pci_p2pdma_page(sg_page(iod->sg)))
561 pci_p2pdma_unmap_sg(dev->dev, iod->sg, iod->nents,
562 rq_dma_dir(req));
563 else
564 dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
565
566
567 if (iod->npages == 0)
568 dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
569 dma_addr);
570
571 for (i = 0; i < iod->npages; i++) {
572 void *addr = nvme_pci_iod_list(req)[i];
573
574 if (iod->use_sgl) {
575 struct nvme_sgl_desc *sg_list = addr;
576
577 next_dma_addr =
578 le64_to_cpu((sg_list[SGES_PER_PAGE - 1]).addr);
579 } else {
580 __le64 *prp_list = addr;
581
582 next_dma_addr = le64_to_cpu(prp_list[last_prp]);
583 }
584
585 dma_pool_free(dev->prp_page_pool, addr, dma_addr);
586 dma_addr = next_dma_addr;
587 }
588
589 mempool_free(iod->sg, dev->iod_mempool);
590 }
591
592 static void nvme_print_sgl(struct scatterlist *sgl, int nents)
593 {
594 int i;
595 struct scatterlist *sg;
596
597 for_each_sg(sgl, sg, nents, i) {
598 dma_addr_t phys = sg_phys(sg);
599 pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
600 "dma_address:%pad dma_length:%d\n",
601 i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
602 sg_dma_len(sg));
603 }
604 }
605
606 static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
607 struct request *req, struct nvme_rw_command *cmnd)
608 {
609 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
610 struct dma_pool *pool;
611 int length = blk_rq_payload_bytes(req);
612 struct scatterlist *sg = iod->sg;
613 int dma_len = sg_dma_len(sg);
614 u64 dma_addr = sg_dma_address(sg);
615 int offset = dma_addr & (NVME_CTRL_PAGE_SIZE - 1);
616 __le64 *prp_list;
617 void **list = nvme_pci_iod_list(req);
618 dma_addr_t prp_dma;
619 int nprps, i;
620
621 length -= (NVME_CTRL_PAGE_SIZE - offset);
622 if (length <= 0) {
623 iod->first_dma = 0;
624 goto done;
625 }
626
627 dma_len -= (NVME_CTRL_PAGE_SIZE - offset);
628 if (dma_len) {
629 dma_addr += (NVME_CTRL_PAGE_SIZE - offset);
630 } else {
631 sg = sg_next(sg);
632 dma_addr = sg_dma_address(sg);
633 dma_len = sg_dma_len(sg);
634 }
635
636 if (length <= NVME_CTRL_PAGE_SIZE) {
637 iod->first_dma = dma_addr;
638 goto done;
639 }
640
641 nprps = DIV_ROUND_UP(length, NVME_CTRL_PAGE_SIZE);
642 if (nprps <= (256 / 8)) {
643 pool = dev->prp_small_pool;
644 iod->npages = 0;
645 } else {
646 pool = dev->prp_page_pool;
647 iod->npages = 1;
648 }
649
650 prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
651 if (!prp_list) {
652 iod->first_dma = dma_addr;
653 iod->npages = -1;
654 return BLK_STS_RESOURCE;
655 }
656 list[0] = prp_list;
657 iod->first_dma = prp_dma;
658 i = 0;
659 for (;;) {
660 if (i == NVME_CTRL_PAGE_SIZE >> 3) {
661 __le64 *old_prp_list = prp_list;
662 prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
663 if (!prp_list)
664 return BLK_STS_RESOURCE;
665 list[iod->npages++] = prp_list;
666 prp_list[0] = old_prp_list[i - 1];
667 old_prp_list[i - 1] = cpu_to_le64(prp_dma);
668 i = 1;
669 }
670 prp_list[i++] = cpu_to_le64(dma_addr);
671 dma_len -= NVME_CTRL_PAGE_SIZE;
672 dma_addr += NVME_CTRL_PAGE_SIZE;
673 length -= NVME_CTRL_PAGE_SIZE;
674 if (length <= 0)
675 break;
676 if (dma_len > 0)
677 continue;
678 if (unlikely(dma_len < 0))
679 goto bad_sgl;
680 sg = sg_next(sg);
681 dma_addr = sg_dma_address(sg);
682 dma_len = sg_dma_len(sg);
683 }
684
685 done:
686 cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
687 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);
688
689 return BLK_STS_OK;
690
691 bad_sgl:
692 WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),
693 "Invalid SGL for payload:%d nents:%d\n",
694 blk_rq_payload_bytes(req), iod->nents);
695 return BLK_STS_IOERR;
696 }
697
698 static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
699 struct scatterlist *sg)
700 {
701 sge->addr = cpu_to_le64(sg_dma_address(sg));
702 sge->length = cpu_to_le32(sg_dma_len(sg));
703 sge->type = NVME_SGL_FMT_DATA_DESC << 4;
704 }
705
706 static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
707 dma_addr_t dma_addr, int entries)
708 {
709 sge->addr = cpu_to_le64(dma_addr);
710 if (entries < SGES_PER_PAGE) {
711 sge->length = cpu_to_le32(entries * sizeof(*sge));
712 sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
713 } else {
714 sge->length = cpu_to_le32(PAGE_SIZE);
715 sge->type = NVME_SGL_FMT_SEG_DESC << 4;
716 }
717 }
718
719 static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
720 struct request *req, struct nvme_rw_command *cmd, int entries)
721 {
722 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
723 struct dma_pool *pool;
724 struct nvme_sgl_desc *sg_list;
725 struct scatterlist *sg = iod->sg;
726 dma_addr_t sgl_dma;
727 int i = 0;
728
729 /* setting the transfer type as SGL */
730 cmd->flags = NVME_CMD_SGL_METABUF;
731
732 if (entries == 1) {
733 nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg);
734 return BLK_STS_OK;
735 }
736
737 if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
738 pool = dev->prp_small_pool;
739 iod->npages = 0;
740 } else {
741 pool = dev->prp_page_pool;
742 iod->npages = 1;
743 }
744
745 sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
746 if (!sg_list) {
747 iod->npages = -1;
748 return BLK_STS_RESOURCE;
749 }
750
751 nvme_pci_iod_list(req)[0] = sg_list;
752 iod->first_dma = sgl_dma;
753
754 nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);
755
756 do {
757 if (i == SGES_PER_PAGE) {
758 struct nvme_sgl_desc *old_sg_desc = sg_list;
759 struct nvme_sgl_desc *link = &old_sg_desc[i - 1];
760
761 sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
762 if (!sg_list)
763 return BLK_STS_RESOURCE;
764
765 i = 0;
766 nvme_pci_iod_list(req)[iod->npages++] = sg_list;
767 sg_list[i++] = *link;
768 nvme_pci_sgl_set_seg(link, sgl_dma, entries);
769 }
770
771 nvme_pci_sgl_set_data(&sg_list[i++], sg);
772 sg = sg_next(sg);
773 } while (--entries > 0);
774
775 return BLK_STS_OK;
776 }
777
778 static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
779 struct request *req, struct nvme_rw_command *cmnd,
780 struct bio_vec *bv)
781 {
782 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
783 unsigned int offset = bv->bv_offset & (NVME_CTRL_PAGE_SIZE - 1);
784 unsigned int first_prp_len = NVME_CTRL_PAGE_SIZE - offset;
785
786 iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
787 if (dma_mapping_error(dev->dev, iod->first_dma))
788 return BLK_STS_RESOURCE;
789 iod->dma_len = bv->bv_len;
790
791 cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma);
792 if (bv->bv_len > first_prp_len)
793 cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len);
794 return BLK_STS_OK;
795 }
796
797 static blk_status_t nvme_setup_sgl_simple(struct nvme_dev *dev,
798 struct request *req, struct nvme_rw_command *cmnd,
799 struct bio_vec *bv)
800 {
801 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
802
803 iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
804 if (dma_mapping_error(dev->dev, iod->first_dma))
805 return BLK_STS_RESOURCE;
806 iod->dma_len = bv->bv_len;
807
808 cmnd->flags = NVME_CMD_SGL_METABUF;
809 cmnd->dptr.sgl.addr = cpu_to_le64(iod->first_dma);
810 cmnd->dptr.sgl.length = cpu_to_le32(iod->dma_len);
811 cmnd->dptr.sgl.type = NVME_SGL_FMT_DATA_DESC << 4;
812 return BLK_STS_OK;
813 }
814
815 static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
816 struct nvme_command *cmnd)
817 {
818 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
819 blk_status_t ret = BLK_STS_RESOURCE;
820 int nr_mapped;
821
822 if (blk_rq_nr_phys_segments(req) == 1) {
823 struct bio_vec bv = req_bvec(req);
824
825 if (!is_pci_p2pdma_page(bv.bv_page)) {
826 if (bv.bv_offset + bv.bv_len <= NVME_CTRL_PAGE_SIZE * 2)
827 return nvme_setup_prp_simple(dev, req,
828 &cmnd->rw, &bv);
829
830 if (iod->nvmeq->qid &&
831 dev->ctrl.sgls & ((1 << 0) | (1 << 1)))
832 return nvme_setup_sgl_simple(dev, req,
833 &cmnd->rw, &bv);
834 }
835 }
836
837 iod->dma_len = 0;
838 iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
839 if (!iod->sg)
840 return BLK_STS_RESOURCE;
841 sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
842 iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
843 if (!iod->nents)
844 goto out;
845
846 if (is_pci_p2pdma_page(sg_page(iod->sg)))
847 nr_mapped = pci_p2pdma_map_sg_attrs(dev->dev, iod->sg,
848 iod->nents, rq_dma_dir(req), DMA_ATTR_NO_WARN);
849 else
850 nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents,
851 rq_dma_dir(req), DMA_ATTR_NO_WARN);
852 if (!nr_mapped)
853 goto out;
854
855 iod->use_sgl = nvme_pci_use_sgls(dev, req);
856 if (iod->use_sgl)
857 ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped);
858 else
859 ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);
860 out:
861 if (ret != BLK_STS_OK)
862 nvme_unmap_data(dev, req);
863 return ret;
864 }
865
866 static blk_status_t nvme_map_metadata(struct nvme_dev *dev, struct request *req,
867 struct nvme_command *cmnd)
868 {
869 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
870
871 iod->meta_dma = dma_map_bvec(dev->dev, rq_integrity_vec(req),
872 rq_dma_dir(req), 0);
873 if (dma_mapping_error(dev->dev, iod->meta_dma))
874 return BLK_STS_IOERR;
875 cmnd->rw.metadata = cpu_to_le64(iod->meta_dma);
876 return BLK_STS_OK;
877 }
878
879 /*
880 * NOTE: ns is NULL when called on the admin queue.
881 */
882 static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
883 const struct blk_mq_queue_data *bd)
884 {
885 struct nvme_ns *ns = hctx->queue->queuedata;
886 struct nvme_queue *nvmeq = hctx->driver_data;
887 struct nvme_dev *dev = nvmeq->dev;
888 struct request *req = bd->rq;
889 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
890 struct nvme_command cmnd;
891 blk_status_t ret;
892
893 iod->aborted = 0;
894 iod->npages = -1;
895 iod->nents = 0;
896
897 /*
898 * We should not need to do this, but we're still using this to
899 * ensure we can drain requests on a dying queue.
900 */
901 if (unlikely(!test_bit(NVMEQ_ENABLED, &nvmeq->flags)))
902 return BLK_STS_IOERR;
903
904 ret = nvme_setup_cmd(ns, req, &cmnd);
905 if (ret)
906 return ret;
907
908 if (blk_rq_nr_phys_segments(req)) {
909 ret = nvme_map_data(dev, req, &cmnd);
910 if (ret)
911 goto out_free_cmd;
912 }
913
914 if (blk_integrity_rq(req)) {
915 ret = nvme_map_metadata(dev, req, &cmnd);
916 if (ret)
917 goto out_unmap_data;
918 }
919
920 blk_mq_start_request(req);
921 nvme_submit_cmd(nvmeq, &cmnd, bd->last);
922 return BLK_STS_OK;
923 out_unmap_data:
924 nvme_unmap_data(dev, req);
925 out_free_cmd:
926 nvme_cleanup_cmd(req);
927 return ret;
928 }
929
930 static void nvme_pci_complete_rq(struct request *req)
931 {
932 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
933 struct nvme_dev *dev = iod->nvmeq->dev;
934
935 if (blk_integrity_rq(req))
936 dma_unmap_page(dev->dev, iod->meta_dma,
937 rq_integrity_vec(req)->bv_len, rq_data_dir(req));
938 if (blk_rq_nr_phys_segments(req))
939 nvme_unmap_data(dev, req);
940 nvme_complete_rq(req);
941 }
942
943 /* We read the CQE phase first to check if the rest of the entry is valid */
944 static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
945 {
946 struct nvme_completion *hcqe = &nvmeq->cqes[nvmeq->cq_head];
947
948 return (le16_to_cpu(READ_ONCE(hcqe->status)) & 1) == nvmeq->cq_phase;
949 }
950
951 static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
952 {
953 u16 head = nvmeq->cq_head;
954
955 if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
956 nvmeq->dbbuf_cq_ei))
957 writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
958 }
959
960 static inline struct blk_mq_tags *nvme_queue_tagset(struct nvme_queue *nvmeq)
961 {
962 if (!nvmeq->qid)
963 return nvmeq->dev->admin_tagset.tags[0];
964 return nvmeq->dev->tagset.tags[nvmeq->qid - 1];
965 }
966
967 static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
968 {
969 struct nvme_completion *cqe = &nvmeq->cqes[idx];
970 __u16 command_id = READ_ONCE(cqe->command_id);
971 struct request *req;
972
973 /*
974 * AEN requests are special as they don't time out and can
975 * survive any kind of queue freeze and often don't respond to
976 * aborts. We don't even bother to allocate a struct request
977 * for them but rather special case them here.
978 */
979 if (unlikely(nvme_is_aen_req(nvmeq->qid, command_id))) {
980 nvme_complete_async_event(&nvmeq->dev->ctrl,
981 cqe->status, &cqe->result);
982 return;
983 }
984
985 req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), command_id);
986 if (unlikely(!req)) {
987 dev_warn(nvmeq->dev->ctrl.device,
988 "invalid id %d completed on queue %d\n",
989 command_id, le16_to_cpu(cqe->sq_id));
990 return;
991 }
992
993 trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail);
994 if (!nvme_try_complete_req(req, cqe->status, cqe->result))
995 nvme_pci_complete_rq(req);
996 }
997
998 static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
999 {
1000 u16 tmp = nvmeq->cq_head + 1;
1001
1002 if (tmp == nvmeq->q_depth) {
1003 nvmeq->cq_head = 0;
1004 nvmeq->cq_phase ^= 1;
1005 } else {
1006 nvmeq->cq_head = tmp;
1007 }
1008 }
1009
1010 static inline int nvme_process_cq(struct nvme_queue *nvmeq)
1011 {
1012 int found = 0;
1013
1014 while (nvme_cqe_pending(nvmeq)) {
1015 found++;
1016 /*
1017 * load-load control dependency between phase and the rest of
1018 * the cqe requires a full read memory barrier
1019 */
1020 dma_rmb();
1021 nvme_handle_cqe(nvmeq, nvmeq->cq_head);
1022 nvme_update_cq_head(nvmeq);
1023 }
1024
1025 if (found)
1026 nvme_ring_cq_doorbell(nvmeq);
1027 return found;
1028 }
1029
1030 static irqreturn_t nvme_irq(int irq, void *data)
1031 {
1032 struct nvme_queue *nvmeq = data;
1033 irqreturn_t ret = IRQ_NONE;
1034
1035 /*
1036 * The rmb/wmb pair ensures we see all updates from a previous run of
1037 * the irq handler, even if that was on another CPU.
1038 */
1039 rmb();
1040 if (nvme_process_cq(nvmeq))
1041 ret = IRQ_HANDLED;
1042 wmb();
1043
1044 return ret;
1045 }
1046
1047 static irqreturn_t nvme_irq_check(int irq, void *data)
1048 {
1049 struct nvme_queue *nvmeq = data;
1050
1051 if (nvme_cqe_pending(nvmeq))
1052 return IRQ_WAKE_THREAD;
1053 return IRQ_NONE;
1054 }
1055
1056 /*
1057 * Poll for completions for any interrupt driven queue
1058 * Can be called from any context.
1059 */
1060 static void nvme_poll_irqdisable(struct nvme_queue *nvmeq)
1061 {
1062 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1063
1064 WARN_ON_ONCE(test_bit(NVMEQ_POLLED, &nvmeq->flags));
1065
1066 disable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
1067 nvme_process_cq(nvmeq);
1068 enable_irq(pci_irq_vector(pdev, nvmeq->cq_vector));
1069 }
1070
1071 static int nvme_poll(struct blk_mq_hw_ctx *hctx)
1072 {
1073 struct nvme_queue *nvmeq = hctx->driver_data;
1074 bool found;
1075
1076 if (!nvme_cqe_pending(nvmeq))
1077 return 0;
1078
1079 spin_lock(&nvmeq->cq_poll_lock);
1080 found = nvme_process_cq(nvmeq);
1081 spin_unlock(&nvmeq->cq_poll_lock);
1082
1083 return found;
1084 }
1085
1086 static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
1087 {
1088 struct nvme_dev *dev = to_nvme_dev(ctrl);
1089 struct nvme_queue *nvmeq = &dev->queues[0];
1090 struct nvme_command c;
1091
1092 memset(&c, 0, sizeof(c));
1093 c.common.opcode = nvme_admin_async_event;
1094 c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1095 nvme_submit_cmd(nvmeq, &c, true);
1096 }
1097
1098 static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1099 {
1100 struct nvme_command c;
1101
1102 memset(&c, 0, sizeof(c));
1103 c.delete_queue.opcode = opcode;
1104 c.delete_queue.qid = cpu_to_le16(id);
1105
1106 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1107 }
1108
1109 static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1110 struct nvme_queue *nvmeq, s16 vector)
1111 {
1112 struct nvme_command c;
1113 int flags = NVME_QUEUE_PHYS_CONTIG;
1114
1115 if (!test_bit(NVMEQ_POLLED, &nvmeq->flags))
1116 flags |= NVME_CQ_IRQ_ENABLED;
1117
1118 /*
1119 * Note: we (ab)use the fact that the prp fields survive if no data
1120 * is attached to the request.
1121 */
1122 memset(&c, 0, sizeof(c));
1123 c.create_cq.opcode = nvme_admin_create_cq;
1124 c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
1125 c.create_cq.cqid = cpu_to_le16(qid);
1126 c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1127 c.create_cq.cq_flags = cpu_to_le16(flags);
1128 c.create_cq.irq_vector = cpu_to_le16(vector);
1129
1130 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1131 }
1132
1133 static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
1134 struct nvme_queue *nvmeq)
1135 {
1136 struct nvme_ctrl *ctrl = &dev->ctrl;
1137 struct nvme_command c;
1138 int flags = NVME_QUEUE_PHYS_CONTIG;
1139
1140 /*
1141 * Some drives have a bug that auto-enables WRRU if MEDIUM isn't
1142 * set. Since URGENT priority is zeroes, it makes all queues
1143 * URGENT.
1144 */
1145 if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ)
1146 flags |= NVME_SQ_PRIO_MEDIUM;
1147
1148 /*
1149 * Note: we (ab)use the fact that the prp fields survive if no data
1150 * is attached to the request.
1151 */
1152 memset(&c, 0, sizeof(c));
1153 c.create_sq.opcode = nvme_admin_create_sq;
1154 c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
1155 c.create_sq.sqid = cpu_to_le16(qid);
1156 c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
1157 c.create_sq.sq_flags = cpu_to_le16(flags);
1158 c.create_sq.cqid = cpu_to_le16(qid);
1159
1160 return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1161 }
1162
1163 static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
1164 {
1165 return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
1166 }
1167
1168 static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
1169 {
1170 return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
1171 }
1172
1173 static void abort_endio(struct request *req, blk_status_t error)
1174 {
1175 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1176 struct nvme_queue *nvmeq = iod->nvmeq;
1177
1178 dev_warn(nvmeq->dev->ctrl.device,
1179 "Abort status: 0x%x", nvme_req(req)->status);
1180 atomic_inc(&nvmeq->dev->ctrl.abort_limit);
1181 blk_mq_free_request(req);
1182 }
1183
1184 static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
1185 {
1186 /* If true, indicates loss of adapter communication, possibly by a
1187 * NVMe Subsystem reset.
1188 */
1189 bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);
1190
1191 /* If there is a reset/reinit ongoing, we shouldn't reset again. */
1192 switch (dev->ctrl.state) {
1193 case NVME_CTRL_RESETTING:
1194 case NVME_CTRL_CONNECTING:
1195 return false;
1196 default:
1197 break;
1198 }
1199
1200 /* We shouldn't reset unless the controller is on fatal error state
1201 * _or_ if we lost the communication with it.
1202 */
1203 if (!(csts & NVME_CSTS_CFS) && !nssro)
1204 return false;
1205
1206 return true;
1207 }
1208
1209 static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
1210 {
1211 /* Read a config register to help see what died. */
1212 u16 pci_status;
1213 int result;
1214
1215 result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
1216 &pci_status);
1217 if (result == PCIBIOS_SUCCESSFUL)
1218 dev_warn(dev->ctrl.device,
1219 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
1220 csts, pci_status);
1221 else
1222 dev_warn(dev->ctrl.device,
1223 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
1224 csts, result);
1225 }
1226
1227 static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
1228 {
1229 struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
1230 struct nvme_queue *nvmeq = iod->nvmeq;
1231 struct nvme_dev *dev = nvmeq->dev;
1232 struct request *abort_req;
1233 struct nvme_command cmd;
1234 u32 csts = readl(dev->bar + NVME_REG_CSTS);
1235
1236 /* If PCI error recovery process is happening, we cannot reset or
1237 * the recovery mechanism will surely fail.
1238 */
1239 mb();
1240 if (pci_channel_offline(to_pci_dev(dev->dev)))
1241 return BLK_EH_RESET_TIMER;
1242
1243 /*
1244 * Reset immediately if the controller is failed
1245 */
1246 if (nvme_should_reset(dev, csts)) {
1247 nvme_warn_reset(dev, csts);
1248 nvme_dev_disable(dev, false);
1249 nvme_reset_ctrl(&dev->ctrl);
1250 return BLK_EH_DONE;
1251 }
1252
1253 /*
1254 * Did we miss an interrupt?
1255 */
1256 if (test_bit(NVMEQ_POLLED, &nvmeq->flags))
1257 nvme_poll(req->mq_hctx);
1258 else
1259 nvme_poll_irqdisable(nvmeq);
1260
1261 if (blk_mq_request_completed(req)) {
1262 dev_warn(dev->ctrl.device,
1263 "I/O %d QID %d timeout, completion polled\n",
1264 req->tag, nvmeq->qid);
1265 return BLK_EH_DONE;
1266 }
1267
1268 /*
1269 * Shutdown immediately if controller times out while starting. The
1270 * reset work will see the pci device disabled when it gets the forced
1271 * cancellation error. All outstanding requests are completed on
1272 * shutdown, so we return BLK_EH_DONE.
1273 */
1274 switch (dev->ctrl.state) {
1275 case NVME_CTRL_CONNECTING:
1276 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
1277 fallthrough;
1278 case NVME_CTRL_DELETING:
1279 dev_warn_ratelimited(dev->ctrl.device,
1280 "I/O %d QID %d timeout, disable controller\n",
1281 req->tag, nvmeq->qid);
1282 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1283 nvme_dev_disable(dev, true);
1284 return BLK_EH_DONE;
1285 case NVME_CTRL_RESETTING:
1286 return BLK_EH_RESET_TIMER;
1287 default:
1288 break;
1289 }
1290
1291 /*
1292 * Shutdown the controller immediately and schedule a reset if the
1293 * command was already aborted once before and still hasn't been
1294 * returned to the driver, or if this is the admin queue.
1295 */
1296 if (!nvmeq->qid || iod->aborted) {
1297 dev_warn(dev->ctrl.device,
1298 "I/O %d QID %d timeout, reset controller\n",
1299 req->tag, nvmeq->qid);
1300 nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1301 nvme_dev_disable(dev, false);
1302 nvme_reset_ctrl(&dev->ctrl);
1303
1304 return BLK_EH_DONE;
1305 }
1306
1307 if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1308 atomic_inc(&dev->ctrl.abort_limit);
1309 return BLK_EH_RESET_TIMER;
1310 }
1311 iod->aborted = 1;
1312
1313 memset(&cmd, 0, sizeof(cmd));
1314 cmd.abort.opcode = nvme_admin_abort_cmd;
1315 cmd.abort.cid = req->tag;
1316 cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1317
1318 dev_warn(nvmeq->dev->ctrl.device,
1319 "I/O %d QID %d timeout, aborting\n",
1320 req->tag, nvmeq->qid);
1321
1322 abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
1323 BLK_MQ_REQ_NOWAIT);
1324 if (IS_ERR(abort_req)) {
1325 atomic_inc(&dev->ctrl.abort_limit);
1326 return BLK_EH_RESET_TIMER;
1327 }
1328
1329 abort_req->end_io_data = NULL;
1330 blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
1331
1332 /*
1333 * The aborted req will be completed on receiving the abort req.
1334 * We enable the timer again. If hit twice, it'll cause a device reset,
1335 * as the device then is in a faulty state.
1336 */
1337 return BLK_EH_RESET_TIMER;
1338 }
1339
1340 static void nvme_free_queue(struct nvme_queue *nvmeq)
1341 {
1342 dma_free_coherent(nvmeq->dev->dev, CQ_SIZE(nvmeq),
1343 (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1344 if (!nvmeq->sq_cmds)
1345 return;
1346
1347 if (test_and_clear_bit(NVMEQ_SQ_CMB, &nvmeq->flags)) {
1348 pci_free_p2pmem(to_pci_dev(nvmeq->dev->dev),
1349 nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1350 } else {
1351 dma_free_coherent(nvmeq->dev->dev, SQ_SIZE(nvmeq),
1352 nvmeq->sq_cmds, nvmeq->sq_dma_addr);
1353 }
1354 }
1355
1356 static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1357 {
1358 int i;
1359
1360 for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
1361 dev->ctrl.queue_count--;
1362 nvme_free_queue(&dev->queues[i]);
1363 }
1364 }
1365
1366 /**
1367 * nvme_suspend_queue - put queue into suspended state
1368 * @nvmeq: queue to suspend
1369 */
1370 static int nvme_suspend_queue(struct nvme_queue *nvmeq)
1371 {
1372 if (!test_and_clear_bit(NVMEQ_ENABLED, &nvmeq->flags))
1373 return 1;
1374
1375 /* ensure that nvme_queue_rq() sees NVMEQ_ENABLED cleared */
1376 mb();
1377
1378 nvmeq->dev->online_queues--;
1379 if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1380 blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);
1381 if (!test_and_clear_bit(NVMEQ_POLLED, &nvmeq->flags))
1382 pci_free_irq(to_pci_dev(nvmeq->dev->dev), nvmeq->cq_vector, nvmeq);
1383 return 0;
1384 }
1385
1386 static void nvme_suspend_io_queues(struct nvme_dev *dev)
1387 {
1388 int i;
1389
1390 for (i = dev->ctrl.queue_count - 1; i > 0; i--)
1391 nvme_suspend_queue(&dev->queues[i]);
1392 }
1393
1394 static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
1395 {
1396 struct nvme_queue *nvmeq = &dev->queues[0];
1397
1398 if (shutdown)
1399 nvme_shutdown_ctrl(&dev->ctrl);
1400 else
1401 nvme_disable_ctrl(&dev->ctrl);
1402
1403 nvme_poll_irqdisable(nvmeq);
1404 }
1405
1406 /*
1407 * Called only on a device that has been disabled and after all other threads
1408 * that can check this device's completion queues have synced, except
1409 * nvme_poll(). This is the last chance for the driver to see a natural
1410 * completion before nvme_cancel_request() terminates all incomplete requests.
1411 */
1412 static void nvme_reap_pending_cqes(struct nvme_dev *dev)
1413 {
1414 int i;
1415
1416 for (i = dev->ctrl.queue_count - 1; i > 0; i--) {
1417 spin_lock(&dev->queues[i].cq_poll_lock);
1418 nvme_process_cq(&dev->queues[i]);
1419 spin_unlock(&dev->queues[i].cq_poll_lock);
1420 }
1421 }
1422
1423 static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
1424 int entry_size)
1425 {
1426 int q_depth = dev->q_depth;
1427 unsigned q_size_aligned = roundup(q_depth * entry_size,
1428 NVME_CTRL_PAGE_SIZE);
1429
1430 if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1431 u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1432
1433 mem_per_q = round_down(mem_per_q, NVME_CTRL_PAGE_SIZE);
1434 q_depth = div_u64(mem_per_q, entry_size);
1435
1436 /*
1437 * Ensure the reduced q_depth is above some threshold where it
1438 * would be better to map queues in system memory with the
1439 * original depth
1440 */
1441 if (q_depth < 64)
1442 return -ENOMEM;
1443 }
1444
1445 return q_depth;
1446 }
1447
1448 static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
1449 int qid)
1450 {
1451 struct pci_dev *pdev = to_pci_dev(dev->dev);
1452
1453 if (qid && dev->cmb_use_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
1454 nvmeq->sq_cmds = pci_alloc_p2pmem(pdev, SQ_SIZE(nvmeq));
1455 if (nvmeq->sq_cmds) {
1456 nvmeq->sq_dma_addr = pci_p2pmem_virt_to_bus(pdev,
1457 nvmeq->sq_cmds);
1458 if (nvmeq->sq_dma_addr) {
1459 set_bit(NVMEQ_SQ_CMB, &nvmeq->flags);
1460 return 0;
1461 }
1462
1463 pci_free_p2pmem(pdev, nvmeq->sq_cmds, SQ_SIZE(nvmeq));
1464 }
1465 }
1466
1467 nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(nvmeq),
1468 &nvmeq->sq_dma_addr, GFP_KERNEL);
1469 if (!nvmeq->sq_cmds)
1470 return -ENOMEM;
1471 return 0;
1472 }
1473
1474 static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
1475 {
1476 struct nvme_queue *nvmeq = &dev->queues[qid];
1477
1478 if (dev->ctrl.queue_count > qid)
1479 return 0;
1480
1481 nvmeq->sqes = qid ? dev->io_sqes : NVME_ADM_SQES;
1482 nvmeq->q_depth = depth;
1483 nvmeq->cqes = dma_alloc_coherent(dev->dev, CQ_SIZE(nvmeq),
1484 &nvmeq->cq_dma_addr, GFP_KERNEL);
1485 if (!nvmeq->cqes)
1486 goto free_nvmeq;
1487
1488 if (nvme_alloc_sq_cmds(dev, nvmeq, qid))
1489 goto free_cqdma;
1490
1491 nvmeq->dev = dev;
1492 spin_lock_init(&nvmeq->sq_lock);
1493 spin_lock_init(&nvmeq->cq_poll_lock);
1494 nvmeq->cq_head = 0;
1495 nvmeq->cq_phase = 1;
1496 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1497 nvmeq->qid = qid;
1498 dev->ctrl.queue_count++;
1499
1500 return 0;
1501
1502 free_cqdma:
1503 dma_free_coherent(dev->dev, CQ_SIZE(nvmeq), (void *)nvmeq->cqes,
1504 nvmeq->cq_dma_addr);
1505 free_nvmeq:
1506 return -ENOMEM;
1507 }
1508
1509 static int queue_request_irq(struct nvme_queue *nvmeq)
1510 {
1511 struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
1512 int nr = nvmeq->dev->ctrl.instance;
1513
1514 if (use_threaded_interrupts) {
1515 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
1516 nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
1517 } else {
1518 return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
1519 NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
1520 }
1521 }
1522
1523 static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
1524 {
1525 struct nvme_dev *dev = nvmeq->dev;
1526
1527 nvmeq->sq_tail = 0;
1528 nvmeq->last_sq_tail = 0;
1529 nvmeq->cq_head = 0;
1530 nvmeq->cq_phase = 1;
1531 nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1532 memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq));
1533 nvme_dbbuf_init(dev, nvmeq, qid);
1534 dev->online_queues++;
1535 wmb(); /* ensure the first interrupt sees the initialization */
1536 }
1537
1538 static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)
1539 {
1540 struct nvme_dev *dev = nvmeq->dev;
1541 int result;
1542 u16 vector = 0;
1543
1544 clear_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
1545
1546 /*
1547 * A queue's vector matches the queue identifier unless the controller
1548 * has only one vector available.
1549 */
1550 if (!polled)
1551 vector = dev->num_vecs == 1 ? 0 : qid;
1552 else
1553 set_bit(NVMEQ_POLLED, &nvmeq->flags);
1554
1555 result = adapter_alloc_cq(dev, qid, nvmeq, vector);
1556 if (result)
1557 return result;
1558
1559 result = adapter_alloc_sq(dev, qid, nvmeq);
1560 if (result < 0)
1561 return result;
1562 if (result)
1563 goto release_cq;
1564
1565 nvmeq->cq_vector = vector;
1566 nvme_init_queue(nvmeq, qid);
1567
1568 if (!polled) {
1569 result = queue_request_irq(nvmeq);
1570 if (result < 0)
1571 goto release_sq;
1572 }
1573
1574 set_bit(NVMEQ_ENABLED, &nvmeq->flags);
1575 return result;
1576
1577 release_sq:
1578 dev->online_queues--;
1579 adapter_delete_sq(dev, qid);
1580 release_cq:
1581 adapter_delete_cq(dev, qid);
1582 return result;
1583 }
1584
1585 static const struct blk_mq_ops nvme_mq_admin_ops = {
1586 .queue_rq = nvme_queue_rq,
1587 .complete = nvme_pci_complete_rq,
1588 .init_hctx = nvme_admin_init_hctx,
1589 .init_request = nvme_init_request,
1590 .timeout = nvme_timeout,
1591 };
1592
1593 static const struct blk_mq_ops nvme_mq_ops = {
1594 .queue_rq = nvme_queue_rq,
1595 .complete = nvme_pci_complete_rq,
1596 .commit_rqs = nvme_commit_rqs,
1597 .init_hctx = nvme_init_hctx,
1598 .init_request = nvme_init_request,
1599 .map_queues = nvme_pci_map_queues,
1600 .timeout = nvme_timeout,
1601 .poll = nvme_poll,
1602 };
1603
1604 static void nvme_dev_remove_admin(struct nvme_dev *dev)
1605 {
1606 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1607 /*
1608 * If the controller was reset during removal, it's possible
1609 * user requests may be waiting on a stopped queue. Start the
1610 * queue to flush these to completion.
1611 */
1612 blk_mq_unquiesce_queue(dev->ctrl.admin_q);
1613 blk_cleanup_queue(dev->ctrl.admin_q);
1614 blk_mq_free_tag_set(&dev->admin_tagset);
1615 }
1616 }
1617
1618 static int nvme_alloc_admin_tags(struct nvme_dev *dev)
1619 {
1620 if (!dev->ctrl.admin_q) {
1621 dev->admin_tagset.ops = &nvme_mq_admin_ops;
1622 dev->admin_tagset.nr_hw_queues = 1;
1623
1624 dev->admin_tagset.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
1625 dev->admin_tagset.timeout = NVME_ADMIN_TIMEOUT;
1626 dev->admin_tagset.numa_node = dev->ctrl.numa_node;
1627 dev->admin_tagset.cmd_size = sizeof(struct nvme_iod);
1628 dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
1629 dev->admin_tagset.driver_data = dev;
1630
1631 if (blk_mq_alloc_tag_set(&dev->admin_tagset))
1632 return -ENOMEM;
1633 dev->ctrl.admin_tagset = &dev->admin_tagset;
1634
1635 dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
1636 if (IS_ERR(dev->ctrl.admin_q)) {
1637 blk_mq_free_tag_set(&dev->admin_tagset);
1638 return -ENOMEM;
1639 }
1640 if (!blk_get_queue(dev->ctrl.admin_q)) {
1641 nvme_dev_remove_admin(dev);
1642 dev->ctrl.admin_q = NULL;
1643 return -ENODEV;
1644 }
1645 } else
1646 blk_mq_unquiesce_queue(dev->ctrl.admin_q);
1647
1648 return 0;
1649 }
1650
1651 static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
1652 {
1653 return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
1654 }
1655
1656 static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
1657 {
1658 struct pci_dev *pdev = to_pci_dev(dev->dev);
1659
1660 if (size <= dev->bar_mapped_size)
1661 return 0;
1662 if (size > pci_resource_len(pdev, 0))
1663 return -ENOMEM;
1664 if (dev->bar)
1665 iounmap(dev->bar);
1666 dev->bar = ioremap(pci_resource_start(pdev, 0), size);
1667 if (!dev->bar) {
1668 dev->bar_mapped_size = 0;
1669 return -ENOMEM;
1670 }
1671 dev->bar_mapped_size = size;
1672 dev->dbs = dev->bar + NVME_REG_DBS;
1673
1674 return 0;
1675 }
1676
1677 static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
1678 {
1679 int result;
1680 u32 aqa;
1681 struct nvme_queue *nvmeq;
1682
1683 result = nvme_remap_bar(dev, db_bar_size(dev, 0));
1684 if (result < 0)
1685 return result;
1686
1687 dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
1688 NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
1689
1690 if (dev->subsystem &&
1691 (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
1692 writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1693
1694 result = nvme_disable_ctrl(&dev->ctrl);
1695 if (result < 0)
1696 return result;
1697
1698 result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1699 if (result)
1700 return result;
1701
1702 dev->ctrl.numa_node = dev_to_node(dev->dev);
1703
1704 nvmeq = &dev->queues[0];
1705 aqa = nvmeq->q_depth - 1;
1706 aqa |= aqa << 16;
1707
1708 writel(aqa, dev->bar + NVME_REG_AQA);
1709 lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
1710 lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
1711
1712 result = nvme_enable_ctrl(&dev->ctrl);
1713 if (result)
1714 return result;
1715
1716 nvmeq->cq_vector = 0;
1717 nvme_init_queue(nvmeq, 0);
1718 result = queue_request_irq(nvmeq);
1719 if (result) {
1720 dev->online_queues--;
1721 return result;
1722 }
1723
1724 set_bit(NVMEQ_ENABLED, &nvmeq->flags);
1725 return result;
1726 }
1727
1728 static int nvme_create_io_queues(struct nvme_dev *dev)
1729 {
1730 unsigned i, max, rw_queues;
1731 int ret = 0;
1732
1733 for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
1734 if (nvme_alloc_queue(dev, i, dev->q_depth)) {
1735 ret = -ENOMEM;
1736 break;
1737 }
1738 }
1739
1740 max = min(dev->max_qid, dev->ctrl.queue_count - 1);
1741 if (max != 1 && dev->io_queues[HCTX_TYPE_POLL]) {
1742 rw_queues = dev->io_queues[HCTX_TYPE_DEFAULT] +
1743 dev->io_queues[HCTX_TYPE_READ];
1744 } else {
1745 rw_queues = max;
1746 }
1747
1748 for (i = dev->online_queues; i <= max; i++) {
1749 bool polled = i > rw_queues;
1750
1751 ret = nvme_create_queue(&dev->queues[i], i, polled);
1752 if (ret)
1753 break;
1754 }
1755
1756 /*
1757 * Ignore failing Create SQ/CQ commands, we can continue with less
1758 * than the desired amount of queues, and even a controller without
1759 * I/O queues can still be used to issue admin commands. This might
1760 * be useful to upgrade a buggy firmware for example.
1761 */
1762 return ret >= 0 ? 0 : ret;
1763 }
1764
1765 static ssize_t nvme_cmb_show(struct device *dev,
1766 struct device_attribute *attr,
1767 char *buf)
1768 {
1769 struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));
1770
1771 return scnprintf(buf, PAGE_SIZE, "cmbloc : x%08x\ncmbsz : x%08x\n",
1772 ndev->cmbloc, ndev->cmbsz);
1773 }
1774 static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL);
1775
1776 static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
1777 {
1778 u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;
1779
1780 return 1ULL << (12 + 4 * szu);
1781 }
1782
1783 static u32 nvme_cmb_size(struct nvme_dev *dev)
1784 {
1785 return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
1786 }
1787
1788 static void nvme_map_cmb(struct nvme_dev *dev)
1789 {
1790 u64 size, offset;
1791 resource_size_t bar_size;
1792 struct pci_dev *pdev = to_pci_dev(dev->dev);
1793 int bar;
1794
1795 if (dev->cmb_size)
1796 return;
1797
1798 dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1799 if (!dev->cmbsz)
1800 return;
1801 dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1802
1803 size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
1804 offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
1805 bar = NVME_CMB_BIR(dev->cmbloc);
1806 bar_size = pci_resource_len(pdev, bar);
1807
1808 if (offset > bar_size)
1809 return;
1810
1811 /*
1812 * Controllers may support a CMB size larger than their BAR,
1813 * for example, due to being behind a bridge. Reduce the CMB to
1814 * the reported size of the BAR
1815 */
1816 if (size > bar_size - offset)
1817 size = bar_size - offset;
1818
1819 if (pci_p2pdma_add_resource(pdev, bar, size, offset)) {
1820 dev_warn(dev->ctrl.device,
1821 "failed to register the CMB\n");
1822 return;
1823 }
1824
1825 dev->cmb_size = size;
1826 dev->cmb_use_sqes = use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS);
1827
1828 if ((dev->cmbsz & (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS)) ==
1829 (NVME_CMBSZ_WDS | NVME_CMBSZ_RDS))
1830 pci_p2pmem_publish(pdev, true);
1831
1832 if (sysfs_add_file_to_group(&dev->ctrl.device->kobj,
1833 &dev_attr_cmb.attr, NULL))
1834 dev_warn(dev->ctrl.device,
1835 "failed to add sysfs attribute for CMB\n");
1836 }
1837
1838 static inline void nvme_release_cmb(struct nvme_dev *dev)
1839 {
1840 if (dev->cmb_size) {
1841 sysfs_remove_file_from_group(&dev->ctrl.device->kobj,
1842 &dev_attr_cmb.attr, NULL);
1843 dev->cmb_size = 0;
1844 }
1845 }
1846
1847 static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
1848 {
1849 u32 host_mem_size = dev->host_mem_size >> NVME_CTRL_PAGE_SHIFT;
1850 u64 dma_addr = dev->host_mem_descs_dma;
1851 struct nvme_command c;
1852 int ret;
1853
1854 memset(&c, 0, sizeof(c));
1855 c.features.opcode = nvme_admin_set_features;
1856 c.features.fid = cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
1857 c.features.dword11 = cpu_to_le32(bits);
1858 c.features.dword12 = cpu_to_le32(host_mem_size);
1859 c.features.dword13 = cpu_to_le32(lower_32_bits(dma_addr));
1860 c.features.dword14 = cpu_to_le32(upper_32_bits(dma_addr));
1861 c.features.dword15 = cpu_to_le32(dev->nr_host_mem_descs);
1862
1863 ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
1864 if (ret) {
1865 dev_warn(dev->ctrl.device,
1866 "failed to set host mem (err %d, flags %#x).\n",
1867 ret, bits);
1868 }
1869 return ret;
1870 }
1871
1872 static void nvme_free_host_mem(struct nvme_dev *dev)
1873 {
1874 int i;
1875
1876 for (i = 0; i < dev->nr_host_mem_descs; i++) {
1877 struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
1878 size_t size = le32_to_cpu(desc->size) * NVME_CTRL_PAGE_SIZE;
1879
1880 dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i],
1881 le64_to_cpu(desc->addr),
1882 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
1883 }
1884
1885 kfree(dev->host_mem_desc_bufs);
1886 dev->host_mem_desc_bufs = NULL;
1887 dma_free_coherent(dev->dev,
1888 dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
1889 dev->host_mem_descs, dev->host_mem_descs_dma);
1890 dev->host_mem_descs = NULL;
1891 dev->nr_host_mem_descs = 0;
1892 }
1893
1894 static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
1895 u32 chunk_size)
1896 {
1897 struct nvme_host_mem_buf_desc *descs;
1898 u32 max_entries, len;
1899 dma_addr_t descs_dma;
1900 int i = 0;
1901 void **bufs;
1902 u64 size, tmp;
1903
1904 tmp = (preferred + chunk_size - 1);
1905 do_div(tmp, chunk_size);
1906 max_entries = tmp;
1907
1908 if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
1909 max_entries = dev->ctrl.hmmaxd;
1910
1911 descs = dma_alloc_coherent(dev->dev, max_entries * sizeof(*descs),
1912 &descs_dma, GFP_KERNEL);
1913 if (!descs)
1914 goto out;
1915
1916 bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
1917 if (!bufs)
1918 goto out_free_descs;
1919
1920 for (size = 0; size < preferred && i < max_entries; size += len) {
1921 dma_addr_t dma_addr;
1922
1923 len = min_t(u64, chunk_size, preferred - size);
1924 bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
1925 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
1926 if (!bufs[i])
1927 break;
1928
1929 descs[i].addr = cpu_to_le64(dma_addr);
1930 descs[i].size = cpu_to_le32(len / NVME_CTRL_PAGE_SIZE);
1931 i++;
1932 }
1933
1934 if (!size)
1935 goto out_free_bufs;
1936
1937 dev->nr_host_mem_descs = i;
1938 dev->host_mem_size = size;
1939 dev->host_mem_descs = descs;
1940 dev->host_mem_descs_dma = descs_dma;
1941 dev->host_mem_desc_bufs = bufs;
1942 return 0;
1943
1944 out_free_bufs:
1945 while (--i >= 0) {
1946 size_t size = le32_to_cpu(descs[i].size) * NVME_CTRL_PAGE_SIZE;
1947
1948 dma_free_attrs(dev->dev, size, bufs[i],
1949 le64_to_cpu(descs[i].addr),
1950 DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
1951 }
1952
1953 kfree(bufs);
1954 out_free_descs:
1955 dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
1956 descs_dma);
1957 out:
1958 dev->host_mem_descs = NULL;
1959 return -ENOMEM;
1960 }
1961
1962 static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
1963 {
1964 u64 min_chunk = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
1965 u64 hmminds = max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
1966 u64 chunk_size;
1967
1968 /* start big and work our way down */
1969 for (chunk_size = min_chunk; chunk_size >= hmminds; chunk_size /= 2) {
1970 if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
1971 if (!min || dev->host_mem_size >= min)
1972 return 0;
1973 nvme_free_host_mem(dev);
1974 }
1975 }
1976
1977 return -ENOMEM;
1978 }
1979
1980 static int nvme_setup_host_mem(struct nvme_dev *dev)
1981 {
1982 u64 max = (u64)max_host_mem_size_mb * SZ_1M;
1983 u64 preferred = (u64)dev->ctrl.hmpre * 4096;
1984 u64 min = (u64)dev->ctrl.hmmin * 4096;
1985 u32 enable_bits = NVME_HOST_MEM_ENABLE;
1986 int ret;
1987
1988 preferred = min(preferred, max);
1989 if (min > max) {
1990 dev_warn(dev->ctrl.device,
1991 "min host memory (%lld MiB) above limit (%d MiB).\n",
1992 min >> ilog2(SZ_1M), max_host_mem_size_mb);
1993 nvme_free_host_mem(dev);
1994 return 0;
1995 }
1996
1997 /*
1998 * If we already have a buffer allocated check if we can reuse it.
1999 */
2000 if (dev->host_mem_descs) {
2001 if (dev->host_mem_size >= min)
2002 enable_bits |= NVME_HOST_MEM_RETURN;
2003 else
2004 nvme_free_host_mem(dev);
2005 }
2006
2007 if (!dev->host_mem_descs) {
2008 if (nvme_alloc_host_mem(dev, min, preferred)) {
2009 dev_warn(dev->ctrl.device,
2010 "failed to allocate host memory buffer.\n");
2011 return 0; /* controller must work without HMB */
2012 }
2013
2014 dev_info(dev->ctrl.device,
2015 "allocated %lld MiB host memory buffer.\n",
2016 dev->host_mem_size >> ilog2(SZ_1M));
2017 }
2018
2019 ret = nvme_set_host_mem(dev, enable_bits);
2020 if (ret)
2021 nvme_free_host_mem(dev);
2022 return ret;
2023 }
2024
2025 /*
2026 * nirqs is the number of interrupts available for write and read
2027 * queues. The core already reserved an interrupt for the admin queue.
2028 */
2029 static void nvme_calc_irq_sets(struct irq_affinity *affd, unsigned int nrirqs)
2030 {
2031 struct nvme_dev *dev = affd->priv;
2032 unsigned int nr_read_queues, nr_write_queues = dev->nr_write_queues;
2033
2034 /*
2035 * If there is no interrupt available for queues, ensure that
2036 * the default queue is set to 1. The affinity set size is
2037 * also set to one, but the irq core ignores it for this case.
2038 *
2039 * If only one interrupt is available or 'write_queue' == 0, combine
2040 * write and read queues.
2041 *
2042 * If 'write_queues' > 0, ensure it leaves room for at least one read
2043 * queue.
2044 */
2045 if (!nrirqs) {
2046 nrirqs = 1;
2047 nr_read_queues = 0;
2048 } else if (nrirqs == 1 || !nr_write_queues) {
2049 nr_read_queues = 0;
2050 } else if (nr_write_queues >= nrirqs) {
2051 nr_read_queues = 1;
2052 } else {
2053 nr_read_queues = nrirqs - nr_write_queues;
2054 }
2055
2056 dev->io_queues[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2057 affd->set_size[HCTX_TYPE_DEFAULT] = nrirqs - nr_read_queues;
2058 dev->io_queues[HCTX_TYPE_READ] = nr_read_queues;
2059 affd->set_size[HCTX_TYPE_READ] = nr_read_queues;
2060 affd->nr_sets = nr_read_queues ? 2 : 1;
2061 }
2062
2063 static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
2064 {
2065 struct pci_dev *pdev = to_pci_dev(dev->dev);
2066 struct irq_affinity affd = {
2067 .pre_vectors = 1,
2068 .calc_sets = nvme_calc_irq_sets,
2069 .priv = dev,
2070 };
2071 unsigned int irq_queues, poll_queues;
2072
2073 /*
2074 * Poll queues don't need interrupts, but we need at least one I/O queue
2075 * left over for non-polled I/O.
2076 */
2077 poll_queues = min(dev->nr_poll_queues, nr_io_queues - 1);
2078 dev->io_queues[HCTX_TYPE_POLL] = poll_queues;
2079
2080 /*
2081 * Initialize for the single interrupt case, will be updated in
2082 * nvme_calc_irq_sets().
2083 */
2084 dev->io_queues[HCTX_TYPE_DEFAULT] = 1;
2085 dev->io_queues[HCTX_TYPE_READ] = 0;
2086
2087 /*
2088 * We need interrupts for the admin queue and each non-polled I/O queue,
2089 * but some Apple controllers require all queues to use the first
2090 * vector.
2091 */
2092 irq_queues = 1;
2093 if (!(dev->ctrl.quirks & NVME_QUIRK_SINGLE_VECTOR))
2094 irq_queues += (nr_io_queues - poll_queues);
2095 return pci_alloc_irq_vectors_affinity(pdev, 1, irq_queues,
2096 PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY, &affd);
2097 }
2098
2099 static void nvme_disable_io_queues(struct nvme_dev *dev)
2100 {
2101 if (__nvme_disable_io_queues(dev, nvme_admin_delete_sq))
2102 __nvme_disable_io_queues(dev, nvme_admin_delete_cq);
2103 }
2104
2105 static unsigned int nvme_max_io_queues(struct nvme_dev *dev)
2106 {
2107 /*
2108 * If tags are shared with admin queue (Apple bug), then
2109 * make sure we only use one IO queue.
2110 */
2111 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2112 return 1;
2113 return num_possible_cpus() + dev->nr_write_queues + dev->nr_poll_queues;
2114 }
2115
2116 static int nvme_setup_io_queues(struct nvme_dev *dev)
2117 {
2118 struct nvme_queue *adminq = &dev->queues[0];
2119 struct pci_dev *pdev = to_pci_dev(dev->dev);
2120 unsigned int nr_io_queues;
2121 unsigned long size;
2122 int result;
2123
2124 /*
2125 * Sample the module parameters once at reset time so that we have
2126 * stable values to work with.
2127 */
2128 dev->nr_write_queues = write_queues;
2129 dev->nr_poll_queues = poll_queues;
2130
2131 nr_io_queues = dev->nr_allocated_queues - 1;
2132 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
2133 if (result < 0)
2134 return result;
2135
2136 if (nr_io_queues == 0)
2137 return 0;
2138
2139 clear_bit(NVMEQ_ENABLED, &adminq->flags);
2140
2141 if (dev->cmb_use_sqes) {
2142 result = nvme_cmb_qdepth(dev, nr_io_queues,
2143 sizeof(struct nvme_command));
2144 if (result > 0)
2145 dev->q_depth = result;
2146 else
2147 dev->cmb_use_sqes = false;
2148 }
2149
2150 do {
2151 size = db_bar_size(dev, nr_io_queues);
2152 result = nvme_remap_bar(dev, size);
2153 if (!result)
2154 break;
2155 if (!--nr_io_queues)
2156 return -ENOMEM;
2157 } while (1);
2158 adminq->q_db = dev->dbs;
2159
2160 retry:
2161 /* Deregister the admin queue's interrupt */
2162 pci_free_irq(pdev, 0, adminq);
2163
2164 /*
2165 * If we enable msix early due to not intx, disable it again before
2166 * setting up the full range we need.
2167 */
2168 pci_free_irq_vectors(pdev);
2169
2170 result = nvme_setup_irqs(dev, nr_io_queues);
2171 if (result <= 0)
2172 return -EIO;
2173
2174 dev->num_vecs = result;
2175 result = max(result - 1, 1);
2176 dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
2177
2178 /*
2179 * Should investigate if there's a performance win from allocating
2180 * more queues than interrupt vectors; it might allow the submission
2181 * path to scale better, even if the receive path is limited by the
2182 * number of interrupts.
2183 */
2184 result = queue_request_irq(adminq);
2185 if (result)
2186 return result;
2187 set_bit(NVMEQ_ENABLED, &adminq->flags);
2188
2189 result = nvme_create_io_queues(dev);
2190 if (result || dev->online_queues < 2)
2191 return result;
2192
2193 if (dev->online_queues - 1 < dev->max_qid) {
2194 nr_io_queues = dev->online_queues - 1;
2195 nvme_disable_io_queues(dev);
2196 nvme_suspend_io_queues(dev);
2197 goto retry;
2198 }
2199 dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
2200 dev->io_queues[HCTX_TYPE_DEFAULT],
2201 dev->io_queues[HCTX_TYPE_READ],
2202 dev->io_queues[HCTX_TYPE_POLL]);
2203 return 0;
2204 }
2205
2206 static void nvme_del_queue_end(struct request *req, blk_status_t error)
2207 {
2208 struct nvme_queue *nvmeq = req->end_io_data;
2209
2210 blk_mq_free_request(req);
2211 complete(&nvmeq->delete_done);
2212 }
2213
2214 static void nvme_del_cq_end(struct request *req, blk_status_t error)
2215 {
2216 struct nvme_queue *nvmeq = req->end_io_data;
2217
2218 if (error)
2219 set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
2220
2221 nvme_del_queue_end(req, error);
2222 }
2223
2224 static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
2225 {
2226 struct request_queue *q = nvmeq->dev->ctrl.admin_q;
2227 struct request *req;
2228 struct nvme_command cmd;
2229
2230 memset(&cmd, 0, sizeof(cmd));
2231 cmd.delete_queue.opcode = opcode;
2232 cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2233
2234 req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT);
2235 if (IS_ERR(req))
2236 return PTR_ERR(req);
2237
2238 req->end_io_data = nvmeq;
2239
2240 init_completion(&nvmeq->delete_done);
2241 blk_execute_rq_nowait(q, NULL, req, false,
2242 opcode == nvme_admin_delete_cq ?
2243 nvme_del_cq_end : nvme_del_queue_end);
2244 return 0;
2245 }
2246
2247 static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode)
2248 {
2249 int nr_queues = dev->online_queues - 1, sent = 0;
2250 unsigned long timeout;
2251
2252 retry:
2253 timeout = NVME_ADMIN_TIMEOUT;
2254 while (nr_queues > 0) {
2255 if (nvme_delete_queue(&dev->queues[nr_queues], opcode))
2256 break;
2257 nr_queues--;
2258 sent++;
2259 }
2260 while (sent) {
2261 struct nvme_queue *nvmeq = &dev->queues[nr_queues + sent];
2262
2263 timeout = wait_for_completion_io_timeout(&nvmeq->delete_done,
2264 timeout);
2265 if (timeout == 0)
2266 return false;
2267
2268 sent--;
2269 if (nr_queues)
2270 goto retry;
2271 }
2272 return true;
2273 }
2274
2275 static void nvme_dev_add(struct nvme_dev *dev)
2276 {
2277 int ret;
2278
2279 if (!dev->ctrl.tagset) {
2280 dev->tagset.ops = &nvme_mq_ops;
2281 dev->tagset.nr_hw_queues = dev->online_queues - 1;
2282 dev->tagset.nr_maps = 2; /* default + read */
2283 if (dev->io_queues[HCTX_TYPE_POLL])
2284 dev->tagset.nr_maps++;
2285 dev->tagset.timeout = NVME_IO_TIMEOUT;
2286 dev->tagset.numa_node = dev->ctrl.numa_node;
2287 dev->tagset.queue_depth = min_t(unsigned int, dev->q_depth,
2288 BLK_MQ_MAX_DEPTH) - 1;
2289 dev->tagset.cmd_size = sizeof(struct nvme_iod);
2290 dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
2291 dev->tagset.driver_data = dev;
2292
2293 /*
2294 * Some Apple controllers requires tags to be unique
2295 * across admin and IO queue, so reserve the first 32
2296 * tags of the IO queue.
2297 */
2298 if (dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS)
2299 dev->tagset.reserved_tags = NVME_AQ_DEPTH;
2300
2301 ret = blk_mq_alloc_tag_set(&dev->tagset);
2302 if (ret) {
2303 dev_warn(dev->ctrl.device,
2304 "IO queues tagset allocation failed %d\n", ret);
2305 return;
2306 }
2307 dev->ctrl.tagset = &dev->tagset;
2308 } else {
2309 blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);
2310
2311 /* Free previously allocated queues that are no longer usable */
2312 nvme_free_queues(dev, dev->online_queues);
2313 }
2314
2315 nvme_dbbuf_set(dev);
2316 }
2317
2318 static int nvme_pci_enable(struct nvme_dev *dev)
2319 {
2320 int result = -ENOMEM;
2321 struct pci_dev *pdev = to_pci_dev(dev->dev);
2322
2323 if (pci_enable_device_mem(pdev))
2324 return result;
2325
2326 pci_set_master(pdev);
2327
2328 if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)))
2329 goto disable;
2330
2331 if (readl(dev->bar + NVME_REG_CSTS) == -1) {
2332 result = -ENODEV;
2333 goto disable;
2334 }
2335
2336 /*
2337 * Some devices and/or platforms don't advertise or work with INTx
2338 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
2339 * adjust this later.
2340 */
2341 result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
2342 if (result < 0)
2343 return result;
2344
2345 dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
2346
2347 dev->q_depth = min_t(u32, NVME_CAP_MQES(dev->ctrl.cap) + 1,
2348 io_queue_depth);
2349 dev->ctrl.sqsize = dev->q_depth - 1; /* 0's based queue depth */
2350 dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
2351 dev->dbs = dev->bar + 4096;
2352
2353 /*
2354 * Some Apple controllers require a non-standard SQE size.
2355 * Interestingly they also seem to ignore the CC:IOSQES register
2356 * so we don't bother updating it here.
2357 */
2358 if (dev->ctrl.quirks & NVME_QUIRK_128_BYTES_SQES)
2359 dev->io_sqes = 7;
2360 else
2361 dev->io_sqes = NVME_NVM_IOSQES;
2362
2363 /*
2364 * Temporary fix for the Apple controller found in the MacBook8,1 and
2365 * some MacBook7,1 to avoid controller resets and data loss.
2366 */
2367 if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
2368 dev->q_depth = 2;
2369 dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
2370 "set queue depth=%u to work around controller resets\n",
2371 dev->q_depth);
2372 } else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
2373 (pdev->device == 0xa821 || pdev->device == 0xa822) &&
2374 NVME_CAP_MQES(dev->ctrl.cap) == 0) {
2375 dev->q_depth = 64;
2376 dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
2377 "set queue depth=%u\n", dev->q_depth);
2378 }
2379
2380 /*
2381 * Controllers with the shared tags quirk need the IO queue to be
2382 * big enough so that we get 32 tags for the admin queue
2383 */
2384 if ((dev->ctrl.quirks & NVME_QUIRK_SHARED_TAGS) &&
2385 (dev->q_depth < (NVME_AQ_DEPTH + 2))) {
2386 dev->q_depth = NVME_AQ_DEPTH + 2;
2387 dev_warn(dev->ctrl.device, "IO queue depth clamped to %d\n",
2388 dev->q_depth);
2389 }
2390
2391
2392 nvme_map_cmb(dev);
2393
2394 pci_enable_pcie_error_reporting(pdev);
2395 pci_save_state(pdev);
2396 return 0;
2397
2398 disable:
2399 pci_disable_device(pdev);
2400 return result;
2401 }
2402
2403 static void nvme_dev_unmap(struct nvme_dev *dev)
2404 {
2405 if (dev->bar)
2406 iounmap(dev->bar);
2407 pci_release_mem_regions(to_pci_dev(dev->dev));
2408 }
2409
2410 static void nvme_pci_disable(struct nvme_dev *dev)
2411 {
2412 struct pci_dev *pdev = to_pci_dev(dev->dev);
2413
2414 pci_free_irq_vectors(pdev);
2415
2416 if (pci_is_enabled(pdev)) {
2417 pci_disable_pcie_error_reporting(pdev);
2418 pci_disable_device(pdev);
2419 }
2420 }
2421
2422 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
2423 {
2424 bool dead = true, freeze = false;
2425 struct pci_dev *pdev = to_pci_dev(dev->dev);
2426
2427 mutex_lock(&dev->shutdown_lock);
2428 if (pci_is_enabled(pdev)) {
2429 u32 csts = readl(dev->bar + NVME_REG_CSTS);
2430
2431 if (dev->ctrl.state == NVME_CTRL_LIVE ||
2432 dev->ctrl.state == NVME_CTRL_RESETTING) {
2433 freeze = true;
2434 nvme_start_freeze(&dev->ctrl);
2435 }
2436 dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
2437 pdev->error_state != pci_channel_io_normal);
2438 }
2439
2440 /*
2441 * Give the controller a chance to complete all entered requests if
2442 * doing a safe shutdown.
2443 */
2444 if (!dead && shutdown && freeze)
2445 nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
2446
2447 nvme_stop_queues(&dev->ctrl);
2448
2449 if (!dead && dev->ctrl.queue_count > 0) {
2450 nvme_disable_io_queues(dev);
2451 nvme_disable_admin_queue(dev, shutdown);
2452 }
2453 nvme_suspend_io_queues(dev);
2454 nvme_suspend_queue(&dev->queues[0]);
2455 nvme_pci_disable(dev);
2456 nvme_reap_pending_cqes(dev);
2457
2458 blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
2459 blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl);
2460 blk_mq_tagset_wait_completed_request(&dev->tagset);
2461 blk_mq_tagset_wait_completed_request(&dev->admin_tagset);
2462
2463 /*
2464 * The driver will not be starting up queues again if shutting down so
2465 * must flush all entered requests to their failed completion to avoid
2466 * deadlocking blk-mq hot-cpu notifier.
2467 */
2468 if (shutdown) {
2469 nvme_start_queues(&dev->ctrl);
2470 if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q))
2471 blk_mq_unquiesce_queue(dev->ctrl.admin_q);
2472 }
2473 mutex_unlock(&dev->shutdown_lock);
2474 }
2475
2476 static int nvme_disable_prepare_reset(struct nvme_dev *dev, bool shutdown)
2477 {
2478 if (!nvme_wait_reset(&dev->ctrl))
2479 return -EBUSY;
2480 nvme_dev_disable(dev, shutdown);
2481 return 0;
2482 }
2483
2484 static int nvme_setup_prp_pools(struct nvme_dev *dev)
2485 {
2486 dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
2487 NVME_CTRL_PAGE_SIZE,
2488 NVME_CTRL_PAGE_SIZE, 0);
2489 if (!dev->prp_page_pool)
2490 return -ENOMEM;
2491
2492 /* Optimisation for I/Os between 4k and 128k */
2493 dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2494 256, 256, 0);
2495 if (!dev->prp_small_pool) {
2496 dma_pool_destroy(dev->prp_page_pool);
2497 return -ENOMEM;
2498 }
2499 return 0;
2500 }
2501
2502 static void nvme_release_prp_pools(struct nvme_dev *dev)
2503 {
2504 dma_pool_destroy(dev->prp_page_pool);
2505 dma_pool_destroy(dev->prp_small_pool);
2506 }
2507
2508 static void nvme_free_tagset(struct nvme_dev *dev)
2509 {
2510 if (dev->tagset.tags)
2511 blk_mq_free_tag_set(&dev->tagset);
2512 dev->ctrl.tagset = NULL;
2513 }
2514
2515 static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2516 {
2517 struct nvme_dev *dev = to_nvme_dev(ctrl);
2518
2519 nvme_dbbuf_dma_free(dev);
2520 nvme_free_tagset(dev);
2521 if (dev->ctrl.admin_q)
2522 blk_put_queue(dev->ctrl.admin_q);
2523 free_opal_dev(dev->ctrl.opal_dev);
2524 mempool_destroy(dev->iod_mempool);
2525 put_device(dev->dev);
2526 kfree(dev->queues);
2527 kfree(dev);
2528 }
2529
2530 static void nvme_remove_dead_ctrl(struct nvme_dev *dev)
2531 {
2532 /*
2533 * Set state to deleting now to avoid blocking nvme_wait_reset(), which
2534 * may be holding this pci_dev's device lock.
2535 */
2536 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
2537 nvme_get_ctrl(&dev->ctrl);
2538 nvme_dev_disable(dev, false);
2539 nvme_kill_queues(&dev->ctrl);
2540 if (!queue_work(nvme_wq, &dev->remove_work))
2541 nvme_put_ctrl(&dev->ctrl);
2542 }
2543
2544 static void nvme_reset_work(struct work_struct *work)
2545 {
2546 struct nvme_dev *dev =
2547 container_of(work, struct nvme_dev, ctrl.reset_work);
2548 bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
2549 int result;
2550
2551 if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING)) {
2552 result = -ENODEV;
2553 goto out;
2554 }
2555
2556 /*
2557 * If we're called to reset a live controller first shut it down before
2558 * moving on.
2559 */
2560 if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
2561 nvme_dev_disable(dev, false);
2562 nvme_sync_queues(&dev->ctrl);
2563
2564 mutex_lock(&dev->shutdown_lock);
2565 result = nvme_pci_enable(dev);
2566 if (result)
2567 goto out_unlock;
2568
2569 result = nvme_pci_configure_admin_queue(dev);
2570 if (result)
2571 goto out_unlock;
2572
2573 result = nvme_alloc_admin_tags(dev);
2574 if (result)
2575 goto out_unlock;
2576
2577 /*
2578 * Limit the max command size to prevent iod->sg allocations going
2579 * over a single page.
2580 */
2581 dev->ctrl.max_hw_sectors = min_t(u32,
2582 NVME_MAX_KB_SZ << 1, dma_max_mapping_size(dev->dev) >> 9);
2583 dev->ctrl.max_segments = NVME_MAX_SEGS;
2584
2585 /*
2586 * Don't limit the IOMMU merged segment size.
2587 */
2588 dma_set_max_seg_size(dev->dev, 0xffffffff);
2589
2590 mutex_unlock(&dev->shutdown_lock);
2591
2592 /*
2593 * Introduce CONNECTING state from nvme-fc/rdma transports to mark the
2594 * initializing procedure here.
2595 */
2596 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
2597 dev_warn(dev->ctrl.device,
2598 "failed to mark controller CONNECTING\n");
2599 result = -EBUSY;
2600 goto out;
2601 }
2602
2603 /*
2604 * We do not support an SGL for metadata (yet), so we are limited to a
2605 * single integrity segment for the separate metadata pointer.
2606 */
2607 dev->ctrl.max_integrity_segments = 1;
2608
2609 result = nvme_init_identify(&dev->ctrl);
2610 if (result)
2611 goto out;
2612
2613 if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) {
2614 if (!dev->ctrl.opal_dev)
2615 dev->ctrl.opal_dev =
2616 init_opal_dev(&dev->ctrl, &nvme_sec_submit);
2617 else if (was_suspend)
2618 opal_unlock_from_suspend(dev->ctrl.opal_dev);
2619 } else {
2620 free_opal_dev(dev->ctrl.opal_dev);
2621 dev->ctrl.opal_dev = NULL;
2622 }
2623
2624 if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
2625 result = nvme_dbbuf_dma_alloc(dev);
2626 if (result)
2627 dev_warn(dev->dev,
2628 "unable to allocate dma for dbbuf\n");
2629 }
2630
2631 if (dev->ctrl.hmpre) {
2632 result = nvme_setup_host_mem(dev);
2633 if (result < 0)
2634 goto out;
2635 }
2636
2637 result = nvme_setup_io_queues(dev);
2638 if (result)
2639 goto out;
2640
2641 /*
2642 * Keep the controller around but remove all namespaces if we don't have
2643 * any working I/O queue.
2644 */
2645 if (dev->online_queues < 2) {
2646 dev_warn(dev->ctrl.device, "IO queues not created\n");
2647 nvme_kill_queues(&dev->ctrl);
2648 nvme_remove_namespaces(&dev->ctrl);
2649 nvme_free_tagset(dev);
2650 } else {
2651 nvme_start_queues(&dev->ctrl);
2652 nvme_wait_freeze(&dev->ctrl);
2653 nvme_dev_add(dev);
2654 nvme_unfreeze(&dev->ctrl);
2655 }
2656
2657 /*
2658 * If only admin queue live, keep it to do further investigation or
2659 * recovery.
2660 */
2661 if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_LIVE)) {
2662 dev_warn(dev->ctrl.device,
2663 "failed to mark controller live state\n");
2664 result = -ENODEV;
2665 goto out;
2666 }
2667
2668 nvme_start_ctrl(&dev->ctrl);
2669 return;
2670
2671 out_unlock:
2672 mutex_unlock(&dev->shutdown_lock);
2673 out:
2674 if (result)
2675 dev_warn(dev->ctrl.device,
2676 "Removing after probe failure status: %d\n", result);
2677 nvme_remove_dead_ctrl(dev);
2678 }
2679
2680 static void nvme_remove_dead_ctrl_work(struct work_struct *work)
2681 {
2682 struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2683 struct pci_dev *pdev = to_pci_dev(dev->dev);
2684
2685 if (pci_get_drvdata(pdev))
2686 device_release_driver(&pdev->dev);
2687 nvme_put_ctrl(&dev->ctrl);
2688 }
2689
2690 static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
2691 {
2692 *val = readl(to_nvme_dev(ctrl)->bar + off);
2693 return 0;
2694 }
2695
2696 static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2697 {
2698 writel(val, to_nvme_dev(ctrl)->bar + off);
2699 return 0;
2700 }
2701
2702 static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
2703 {
2704 *val = lo_hi_readq(to_nvme_dev(ctrl)->bar + off);
2705 return 0;
2706 }
2707
2708 static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
2709 {
2710 struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);
2711
2712 return snprintf(buf, size, "%s\n", dev_name(&pdev->dev));
2713 }
2714
2715 static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
2716 .name = "pcie",
2717 .module = THIS_MODULE,
2718 .flags = NVME_F_METADATA_SUPPORTED |
2719 NVME_F_PCI_P2PDMA,
2720 .reg_read32 = nvme_pci_reg_read32,
2721 .reg_write32 = nvme_pci_reg_write32,
2722 .reg_read64 = nvme_pci_reg_read64,
2723 .free_ctrl = nvme_pci_free_ctrl,
2724 .submit_async_event = nvme_pci_submit_async_event,
2725 .get_address = nvme_pci_get_address,
2726 };
2727
2728 static int nvme_dev_map(struct nvme_dev *dev)
2729 {
2730 struct pci_dev *pdev = to_pci_dev(dev->dev);
2731
2732 if (pci_request_mem_regions(pdev, "nvme"))
2733 return -ENODEV;
2734
2735 if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
2736 goto release;
2737
2738 return 0;
2739 release:
2740 pci_release_mem_regions(pdev);
2741 return -ENODEV;
2742 }
2743
2744 static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
2745 {
2746 if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
2747 /*
2748 * Several Samsung devices seem to drop off the PCIe bus
2749 * randomly when APST is on and uses the deepest sleep state.
2750 * This has been observed on a Samsung "SM951 NVMe SAMSUNG
2751 * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
2752 * 950 PRO 256GB", but it seems to be restricted to two Dell
2753 * laptops.
2754 */
2755 if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
2756 (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
2757 dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
2758 return NVME_QUIRK_NO_DEEPEST_PS;
2759 } else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
2760 /*
2761 * Samsung SSD 960 EVO drops off the PCIe bus after system
2762 * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
2763 * within few minutes after bootup on a Coffee Lake board -
2764 * ASUS PRIME Z370-A
2765 */
2766 if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") &&
2767 (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") ||
2768 dmi_match(DMI_BOARD_NAME, "PRIME Z370-A")))
2769 return NVME_QUIRK_NO_APST;
2770 } else if ((pdev->vendor == 0x144d && (pdev->device == 0xa801 ||
2771 pdev->device == 0xa808 || pdev->device == 0xa809)) ||
2772 (pdev->vendor == 0x1e0f && pdev->device == 0x0001)) {
2773 /*
2774 * Forcing to use host managed nvme power settings for
2775 * lowest idle power with quick resume latency on
2776 * Samsung and Toshiba SSDs based on suspend behavior
2777 * on Coffee Lake board for LENOVO C640
2778 */
2779 if ((dmi_match(DMI_BOARD_VENDOR, "LENOVO")) &&
2780 dmi_match(DMI_BOARD_NAME, "LNVNB161216"))
2781 return NVME_QUIRK_SIMPLE_SUSPEND;
2782 }
2783
2784 return 0;
2785 }
2786
2787 #ifdef CONFIG_ACPI
2788 static bool nvme_acpi_storage_d3(struct pci_dev *dev)
2789 {
2790 struct acpi_device *adev;
2791 struct pci_dev *root;
2792 acpi_handle handle;
2793 acpi_status status;
2794 u8 val;
2795
2796 /*
2797 * Look for _DSD property specifying that the storage device on the port
2798 * must use D3 to support deep platform power savings during
2799 * suspend-to-idle.
2800 */
2801 root = pcie_find_root_port(dev);
2802 if (!root)
2803 return false;
2804
2805 adev = ACPI_COMPANION(&root->dev);
2806 if (!adev)
2807 return false;
2808
2809 /*
2810 * The property is defined in the PXSX device for South complex ports
2811 * and in the PEGP device for North complex ports.
2812 */
2813 status = acpi_get_handle(adev->handle, "PXSX", &handle);
2814 if (ACPI_FAILURE(status)) {
2815 status = acpi_get_handle(adev->handle, "PEGP", &handle);
2816 if (ACPI_FAILURE(status))
2817 return false;
2818 }
2819
2820 if (acpi_bus_get_device(handle, &adev))
2821 return false;
2822
2823 if (fwnode_property_read_u8(acpi_fwnode_handle(adev), "StorageD3Enable",
2824 &val))
2825 return false;
2826 return val == 1;
2827 }
2828 #else
2829 static inline bool nvme_acpi_storage_d3(struct pci_dev *dev)
2830 {
2831 return false;
2832 }
2833 #endif /* CONFIG_ACPI */
2834
2835 static void nvme_async_probe(void *data, async_cookie_t cookie)
2836 {
2837 struct nvme_dev *dev = data;
2838
2839 flush_work(&dev->ctrl.reset_work);
2840 flush_work(&dev->ctrl.scan_work);
2841 nvme_put_ctrl(&dev->ctrl);
2842 }
2843
2844 static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2845 {
2846 int node, result = -ENOMEM;
2847 struct nvme_dev *dev;
2848 unsigned long quirks = id->driver_data;
2849 size_t alloc_size;
2850
2851 node = dev_to_node(&pdev->dev);
2852 if (node == NUMA_NO_NODE)
2853 set_dev_node(&pdev->dev, first_memory_node);
2854
2855 dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
2856 if (!dev)
2857 return -ENOMEM;
2858
2859 dev->nr_write_queues = write_queues;
2860 dev->nr_poll_queues = poll_queues;
2861 dev->nr_allocated_queues = nvme_max_io_queues(dev) + 1;
2862 dev->queues = kcalloc_node(dev->nr_allocated_queues,
2863 sizeof(struct nvme_queue), GFP_KERNEL, node);
2864 if (!dev->queues)
2865 goto free;
2866
2867 dev->dev = get_device(&pdev->dev);
2868 pci_set_drvdata(pdev, dev);
2869
2870 result = nvme_dev_map(dev);
2871 if (result)
2872 goto put_pci;
2873
2874 INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
2875 INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2876 mutex_init(&dev->shutdown_lock);
2877
2878 result = nvme_setup_prp_pools(dev);
2879 if (result)
2880 goto unmap;
2881
2882 quirks |= check_vendor_combination_bug(pdev);
2883
2884 if (!noacpi && nvme_acpi_storage_d3(pdev)) {
2885 /*
2886 * Some systems use a bios work around to ask for D3 on
2887 * platforms that support kernel managed suspend.
2888 */
2889 dev_info(&pdev->dev,
2890 "platform quirk: setting simple suspend\n");
2891 quirks |= NVME_QUIRK_SIMPLE_SUSPEND;
2892 }
2893
2894 /*
2895 * Double check that our mempool alloc size will cover the biggest
2896 * command we support.
2897 */
2898 alloc_size = nvme_pci_iod_alloc_size();
2899 WARN_ON_ONCE(alloc_size > PAGE_SIZE);
2900
2901 dev->iod_mempool = mempool_create_node(1, mempool_kmalloc,
2902 mempool_kfree,
2903 (void *) alloc_size,
2904 GFP_KERNEL, node);
2905 if (!dev->iod_mempool) {
2906 result = -ENOMEM;
2907 goto release_pools;
2908 }
2909
2910 result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
2911 quirks);
2912 if (result)
2913 goto release_mempool;
2914
2915 dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));
2916
2917 nvme_reset_ctrl(&dev->ctrl);
2918 async_schedule(nvme_async_probe, dev);
2919
2920 return 0;
2921
2922 release_mempool:
2923 mempool_destroy(dev->iod_mempool);
2924 release_pools:
2925 nvme_release_prp_pools(dev);
2926 unmap:
2927 nvme_dev_unmap(dev);
2928 put_pci:
2929 put_device(dev->dev);
2930 free:
2931 kfree(dev->queues);
2932 kfree(dev);
2933 return result;
2934 }
2935
2936 static void nvme_reset_prepare(struct pci_dev *pdev)
2937 {
2938 struct nvme_dev *dev = pci_get_drvdata(pdev);
2939
2940 /*
2941 * We don't need to check the return value from waiting for the reset
2942 * state as pci_dev device lock is held, making it impossible to race
2943 * with ->remove().
2944 */
2945 nvme_disable_prepare_reset(dev, false);
2946 nvme_sync_queues(&dev->ctrl);
2947 }
2948
2949 static void nvme_reset_done(struct pci_dev *pdev)
2950 {
2951 struct nvme_dev *dev = pci_get_drvdata(pdev);
2952
2953 if (!nvme_try_sched_reset(&dev->ctrl))
2954 flush_work(&dev->ctrl.reset_work);
2955 }
2956
2957 static void nvme_shutdown(struct pci_dev *pdev)
2958 {
2959 struct nvme_dev *dev = pci_get_drvdata(pdev);
2960
2961 nvme_disable_prepare_reset(dev, true);
2962 }
2963
2964 /*
2965 * The driver's remove may be called on a device in a partially initialized
2966 * state. This function must not have any dependencies on the device state in
2967 * order to proceed.
2968 */
2969 static void nvme_remove(struct pci_dev *pdev)
2970 {
2971 struct nvme_dev *dev = pci_get_drvdata(pdev);
2972
2973 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);
2974 pci_set_drvdata(pdev, NULL);
2975
2976 if (!pci_device_is_present(pdev)) {
2977 nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
2978 nvme_dev_disable(dev, true);
2979 nvme_dev_remove_admin(dev);
2980 }
2981
2982 flush_work(&dev->ctrl.reset_work);
2983 nvme_stop_ctrl(&dev->ctrl);
2984 nvme_remove_namespaces(&dev->ctrl);
2985 nvme_dev_disable(dev, true);
2986 nvme_release_cmb(dev);
2987 nvme_free_host_mem(dev);
2988 nvme_dev_remove_admin(dev);
2989 nvme_free_queues(dev, 0);
2990 nvme_release_prp_pools(dev);
2991 nvme_dev_unmap(dev);
2992 nvme_uninit_ctrl(&dev->ctrl);
2993 }
2994
2995 #ifdef CONFIG_PM_SLEEP
2996 static int nvme_get_power_state(struct nvme_ctrl *ctrl, u32 *ps)
2997 {
2998 return nvme_get_features(ctrl, NVME_FEAT_POWER_MGMT, 0, NULL, 0, ps);
2999 }
3000
3001 static int nvme_set_power_state(struct nvme_ctrl *ctrl, u32 ps)
3002 {
3003 return nvme_set_features(ctrl, NVME_FEAT_POWER_MGMT, ps, NULL, 0, NULL);
3004 }
3005
3006 static int nvme_resume(struct device *dev)
3007 {
3008 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3009 struct nvme_ctrl *ctrl = &ndev->ctrl;
3010
3011 if (ndev->last_ps == U32_MAX ||
3012 nvme_set_power_state(ctrl, ndev->last_ps) != 0)
3013 return nvme_try_sched_reset(&ndev->ctrl);
3014 return 0;
3015 }
3016
3017 static int nvme_suspend(struct device *dev)
3018 {
3019 struct pci_dev *pdev = to_pci_dev(dev);
3020 struct nvme_dev *ndev = pci_get_drvdata(pdev);
3021 struct nvme_ctrl *ctrl = &ndev->ctrl;
3022 int ret = -EBUSY;
3023
3024 ndev->last_ps = U32_MAX;
3025
3026 /*
3027 * The platform does not remove power for a kernel managed suspend so
3028 * use host managed nvme power settings for lowest idle power if
3029 * possible. This should have quicker resume latency than a full device
3030 * shutdown. But if the firmware is involved after the suspend or the
3031 * device does not support any non-default power states, shut down the
3032 * device fully.
3033 *
3034 * If ASPM is not enabled for the device, shut down the device and allow
3035 * the PCI bus layer to put it into D3 in order to take the PCIe link
3036 * down, so as to allow the platform to achieve its minimum low-power
3037 * state (which may not be possible if the link is up).
3038 *
3039 * If a host memory buffer is enabled, shut down the device as the NVMe
3040 * specification allows the device to access the host memory buffer in
3041 * host DRAM from all power states, but hosts will fail access to DRAM
3042 * during S3.
3043 */
3044 if (pm_suspend_via_firmware() || !ctrl->npss ||
3045 !pcie_aspm_enabled(pdev) ||
3046 ndev->nr_host_mem_descs ||
3047 (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND))
3048 return nvme_disable_prepare_reset(ndev, true);
3049
3050 nvme_start_freeze(ctrl);
3051 nvme_wait_freeze(ctrl);
3052 nvme_sync_queues(ctrl);
3053
3054 if (ctrl->state != NVME_CTRL_LIVE)
3055 goto unfreeze;
3056
3057 ret = nvme_get_power_state(ctrl, &ndev->last_ps);
3058 if (ret < 0)
3059 goto unfreeze;
3060
3061 /*
3062 * A saved state prevents pci pm from generically controlling the
3063 * device's power. If we're using protocol specific settings, we don't
3064 * want pci interfering.
3065 */
3066 pci_save_state(pdev);
3067
3068 ret = nvme_set_power_state(ctrl, ctrl->npss);
3069 if (ret < 0)
3070 goto unfreeze;
3071
3072 if (ret) {
3073 /* discard the saved state */
3074 pci_load_saved_state(pdev, NULL);
3075
3076 /*
3077 * Clearing npss forces a controller reset on resume. The
3078 * correct value will be rediscovered then.
3079 */
3080 ret = nvme_disable_prepare_reset(ndev, true);
3081 ctrl->npss = 0;
3082 }
3083 unfreeze:
3084 nvme_unfreeze(ctrl);
3085 return ret;
3086 }
3087
3088 static int nvme_simple_suspend(struct device *dev)
3089 {
3090 struct nvme_dev *ndev = pci_get_drvdata(to_pci_dev(dev));
3091
3092 return nvme_disable_prepare_reset(ndev, true);
3093 }
3094
3095 static int nvme_simple_resume(struct device *dev)
3096 {
3097 struct pci_dev *pdev = to_pci_dev(dev);
3098 struct nvme_dev *ndev = pci_get_drvdata(pdev);
3099
3100 return nvme_try_sched_reset(&ndev->ctrl);
3101 }
3102
3103 static const struct dev_pm_ops nvme_dev_pm_ops = {
3104 .suspend = nvme_suspend,
3105 .resume = nvme_resume,
3106 .freeze = nvme_simple_suspend,
3107 .thaw = nvme_simple_resume,
3108 .poweroff = nvme_simple_suspend,
3109 .restore = nvme_simple_resume,
3110 };
3111 #endif /* CONFIG_PM_SLEEP */
3112
3113 static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
3114 pci_channel_state_t state)
3115 {
3116 struct nvme_dev *dev = pci_get_drvdata(pdev);
3117
3118 /*
3119 * A frozen channel requires a reset. When detected, this method will
3120 * shutdown the controller to quiesce. The controller will be restarted
3121 * after the slot reset through driver's slot_reset callback.
3122 */
3123 switch (state) {
3124 case pci_channel_io_normal:
3125 return PCI_ERS_RESULT_CAN_RECOVER;
3126 case pci_channel_io_frozen:
3127 dev_warn(dev->ctrl.device,
3128 "frozen state error detected, reset controller\n");
3129 nvme_dev_disable(dev, false);
3130 return PCI_ERS_RESULT_NEED_RESET;
3131 case pci_channel_io_perm_failure:
3132 dev_warn(dev->ctrl.device,
3133 "failure state error detected, request disconnect\n");
3134 return PCI_ERS_RESULT_DISCONNECT;
3135 }
3136 return PCI_ERS_RESULT_NEED_RESET;
3137 }
3138
3139 static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
3140 {
3141 struct nvme_dev *dev = pci_get_drvdata(pdev);
3142
3143 dev_info(dev->ctrl.device, "restart after slot reset\n");
3144 pci_restore_state(pdev);
3145 nvme_reset_ctrl(&dev->ctrl);
3146 return PCI_ERS_RESULT_RECOVERED;
3147 }
3148
3149 static void nvme_error_resume(struct pci_dev *pdev)
3150 {
3151 struct nvme_dev *dev = pci_get_drvdata(pdev);
3152
3153 flush_work(&dev->ctrl.reset_work);
3154 }
3155
3156 static const struct pci_error_handlers nvme_err_handler = {
3157 .error_detected = nvme_error_detected,
3158 .slot_reset = nvme_slot_reset,
3159 .resume = nvme_error_resume,
3160 .reset_prepare = nvme_reset_prepare,
3161 .reset_done = nvme_reset_done,
3162 };
3163
3164 static const struct pci_device_id nvme_id_table[] = {
3165 { PCI_VDEVICE(INTEL, 0x0953), /* Intel 750/P3500/P3600/P3700 */
3166 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3167 NVME_QUIRK_DEALLOCATE_ZEROES, },
3168 { PCI_VDEVICE(INTEL, 0x0a53), /* Intel P3520 */
3169 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3170 NVME_QUIRK_DEALLOCATE_ZEROES, },
3171 { PCI_VDEVICE(INTEL, 0x0a54), /* Intel P4500/P4600 */
3172 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3173 NVME_QUIRK_DEALLOCATE_ZEROES, },
3174 { PCI_VDEVICE(INTEL, 0x0a55), /* Dell Express Flash P4600 */
3175 .driver_data = NVME_QUIRK_STRIPE_SIZE |
3176 NVME_QUIRK_DEALLOCATE_ZEROES, },
3177 { PCI_VDEVICE(INTEL, 0xf1a5), /* Intel 600P/P3100 */
3178 .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3179 NVME_QUIRK_MEDIUM_PRIO_SQ |
3180 NVME_QUIRK_NO_TEMP_THRESH_CHANGE |
3181 NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3182 { PCI_VDEVICE(INTEL, 0xf1a6), /* Intel 760p/Pro 7600p */
3183 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3184 { PCI_VDEVICE(INTEL, 0x5845), /* Qemu emulated controller */
3185 .driver_data = NVME_QUIRK_IDENTIFY_CNS |
3186 NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3187 { PCI_DEVICE(0x126f, 0x2263), /* Silicon Motion unidentified */
3188 .driver_data = NVME_QUIRK_NO_NS_DESC_LIST, },
3189 { PCI_DEVICE(0x1bb1, 0x0100), /* Seagate Nytro Flash Storage */
3190 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3191 { PCI_DEVICE(0x1c58, 0x0003), /* HGST adapter */
3192 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3193 { PCI_DEVICE(0x1c58, 0x0023), /* WDC SN200 adapter */
3194 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3195 { PCI_DEVICE(0x1c5f, 0x0540), /* Memblaze Pblaze4 adapter */
3196 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3197 { PCI_DEVICE(0x144d, 0xa821), /* Samsung PM1725 */
3198 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
3199 { PCI_DEVICE(0x144d, 0xa822), /* Samsung PM1725a */
3200 .driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
3201 NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3202 { PCI_DEVICE(0x1d1d, 0x1f1f), /* LighNVM qemu device */
3203 .driver_data = NVME_QUIRK_LIGHTNVM, },
3204 { PCI_DEVICE(0x1d1d, 0x2807), /* CNEX WL */
3205 .driver_data = NVME_QUIRK_LIGHTNVM, },
3206 { PCI_DEVICE(0x1d1d, 0x2601), /* CNEX Granby */
3207 .driver_data = NVME_QUIRK_LIGHTNVM, },
3208 { PCI_DEVICE(0x10ec, 0x5762), /* ADATA SX6000LNP */
3209 .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3210 { PCI_DEVICE(0x1cc1, 0x8201), /* ADATA SX8200PNP 512GB */
3211 .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
3212 NVME_QUIRK_IGNORE_DEV_SUBNQN, },
3213 { PCI_DEVICE(0x1c5c, 0x1504), /* SK Hynix PC400 */
3214 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3215 { PCI_DEVICE(0x15b7, 0x2001), /* Sandisk Skyhawk */
3216 .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
3217 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001),
3218 .driver_data = NVME_QUIRK_SINGLE_VECTOR },
3219 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
3220 { PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2005),
3221 .driver_data = NVME_QUIRK_SINGLE_VECTOR |
3222 NVME_QUIRK_128_BYTES_SQES |
3223 NVME_QUIRK_SHARED_TAGS },
3224
3225 { PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
3226 { 0, }
3227 };
3228 MODULE_DEVICE_TABLE(pci, nvme_id_table);
3229
3230 static struct pci_driver nvme_driver = {
3231 .name = "nvme",
3232 .id_table = nvme_id_table,
3233 .probe = nvme_probe,
3234 .remove = nvme_remove,
3235 .shutdown = nvme_shutdown,
3236 #ifdef CONFIG_PM_SLEEP
3237 .driver = {
3238 .pm = &nvme_dev_pm_ops,
3239 },
3240 #endif
3241 .sriov_configure = pci_sriov_configure_simple,
3242 .err_handler = &nvme_err_handler,
3243 };
3244
3245 static int __init nvme_init(void)
3246 {
3247 BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
3248 BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
3249 BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
3250 BUILD_BUG_ON(IRQ_AFFINITY_MAX_SETS < 2);
3251
3252 return pci_register_driver(&nvme_driver);
3253 }
3254
3255 static void __exit nvme_exit(void)
3256 {
3257 pci_unregister_driver(&nvme_driver);
3258 flush_workqueue(nvme_wq);
3259 }
3260
3261 MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
3262 MODULE_LICENSE("GPL");
3263 MODULE_VERSION("1.0");
3264 module_init(nvme_init);
3265 module_exit(nvme_exit);
Linux with added FPC-III support