?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / block / null_blk.c
blob287a09611c0f8addd756c41d4733b6773f25badc
1 /*
2 * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
3 * Shaohua Li <shli@fb.com>
4 */
5 #include <linux/module.h>
6
7 #include <linux/moduleparam.h>
8 #include <linux/sched.h>
9 #include <linux/fs.h>
10 #include <linux/blkdev.h>
11 #include <linux/init.h>
12 #include <linux/slab.h>
13 #include <linux/blk-mq.h>
14 #include <linux/hrtimer.h>
15 #include <linux/configfs.h>
16 #include <linux/badblocks.h>
17 #include <linux/fault-inject.h>
18
19 #define SECTOR_SHIFT 9
20 #define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
21 #define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
22 #define SECTOR_SIZE (1 << SECTOR_SHIFT)
23 #define SECTOR_MASK (PAGE_SECTORS - 1)
24
25 #define FREE_BATCH 16
26
27 #define TICKS_PER_SEC 50ULL
28 #define TIMER_INTERVAL (NSEC_PER_SEC / TICKS_PER_SEC)
29
30 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
31 static DECLARE_FAULT_ATTR(null_timeout_attr);
32 #endif
33
34 static inline u64 mb_per_tick(int mbps)
35 {
36 return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
37 }
38
39 struct nullb_cmd {
40 struct list_head list;
41 struct llist_node ll_list;
42 struct __call_single_data csd;
43 struct request *rq;
44 struct bio *bio;
45 unsigned int tag;
46 blk_status_t error;
47 struct nullb_queue *nq;
48 struct hrtimer timer;
49 };
50
51 struct nullb_queue {
52 unsigned long *tag_map;
53 wait_queue_head_t wait;
54 unsigned int queue_depth;
55 struct nullb_device *dev;
56
57 struct nullb_cmd *cmds;
58 };
59
60 /*
61 * Status flags for nullb_device.
62 *
63 * CONFIGURED: Device has been configured and turned on. Cannot reconfigure.
64 * UP: Device is currently on and visible in userspace.
65 * THROTTLED: Device is being throttled.
66 * CACHE: Device is using a write-back cache.
67 */
68 enum nullb_device_flags {
69 NULLB_DEV_FL_CONFIGURED = 0,
70 NULLB_DEV_FL_UP = 1,
71 NULLB_DEV_FL_THROTTLED = 2,
72 NULLB_DEV_FL_CACHE = 3,
73 };
74
75 /*
76 * nullb_page is a page in memory for nullb devices.
77 *
78 * @page: The page holding the data.
79 * @bitmap: The bitmap represents which sector in the page has data.
80 * Each bit represents one block size. For example, sector 8
81 * will use the 7th bit
82 * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
83 * page is being flushing to storage. FREE means the cache page is freed and
84 * should be skipped from flushing to storage. Please see
85 * null_make_cache_space
86 */
87 struct nullb_page {
88 struct page *page;
89 unsigned long bitmap;
90 };
91 #define NULLB_PAGE_LOCK (sizeof(unsigned long) * 8 - 1)
92 #define NULLB_PAGE_FREE (sizeof(unsigned long) * 8 - 2)
93
94 struct nullb_device {
95 struct nullb *nullb;
96 struct config_item item;
97 struct radix_tree_root data; /* data stored in the disk */
98 struct radix_tree_root cache; /* disk cache data */
99 unsigned long flags; /* device flags */
100 unsigned int curr_cache;
101 struct badblocks badblocks;
102
103 unsigned long size; /* device size in MB */
104 unsigned long completion_nsec; /* time in ns to complete a request */
105 unsigned long cache_size; /* disk cache size in MB */
106 unsigned int submit_queues; /* number of submission queues */
107 unsigned int home_node; /* home node for the device */
108 unsigned int queue_mode; /* block interface */
109 unsigned int blocksize; /* block size */
110 unsigned int irqmode; /* IRQ completion handler */
111 unsigned int hw_queue_depth; /* queue depth */
112 unsigned int index; /* index of the disk, only valid with a disk */
113 unsigned int mbps; /* Bandwidth throttle cap (in MB/s) */
114 bool blocking; /* blocking blk-mq device */
115 bool use_per_node_hctx; /* use per-node allocation for hardware context */
116 bool power; /* power on/off the device */
117 bool memory_backed; /* if data is stored in memory */
118 bool discard; /* if support discard */
119 };
120
121 struct nullb {
122 struct nullb_device *dev;
123 struct list_head list;
124 unsigned int index;
125 struct request_queue *q;
126 struct gendisk *disk;
127 struct blk_mq_tag_set *tag_set;
128 struct blk_mq_tag_set __tag_set;
129 unsigned int queue_depth;
130 atomic_long_t cur_bytes;
131 struct hrtimer bw_timer;
132 unsigned long cache_flush_pos;
133 spinlock_t lock;
134
135 struct nullb_queue *queues;
136 unsigned int nr_queues;
137 char disk_name[DISK_NAME_LEN];
138 };
139
140 static LIST_HEAD(nullb_list);
141 static struct mutex lock;
142 static int null_major;
143 static DEFINE_IDA(nullb_indexes);
144 static struct blk_mq_tag_set tag_set;
145
146 enum {
147 NULL_IRQ_NONE = 0,
148 NULL_IRQ_SOFTIRQ = 1,
149 NULL_IRQ_TIMER = 2,
150 };
151
152 enum {
153 NULL_Q_BIO = 0,
154 NULL_Q_RQ = 1,
155 NULL_Q_MQ = 2,
156 };
157
158 static int g_no_sched;
159 module_param_named(no_sched, g_no_sched, int, S_IRUGO);
160 MODULE_PARM_DESC(no_sched, "No io scheduler");
161
162 static int g_submit_queues = 1;
163 module_param_named(submit_queues, g_submit_queues, int, S_IRUGO);
164 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
165
166 static int g_home_node = NUMA_NO_NODE;
167 module_param_named(home_node, g_home_node, int, S_IRUGO);
168 MODULE_PARM_DESC(home_node, "Home node for the device");
169
170 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
171 static char g_timeout_str[80];
172 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), S_IRUGO);
173 #endif
174
175 static int g_queue_mode = NULL_Q_MQ;
176
177 static int null_param_store_val(const char *str, int *val, int min, int max)
178 {
179 int ret, new_val;
180
181 ret = kstrtoint(str, 10, &new_val);
182 if (ret)
183 return -EINVAL;
184
185 if (new_val < min || new_val > max)
186 return -EINVAL;
187
188 *val = new_val;
189 return 0;
190 }
191
192 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
193 {
194 return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
195 }
196
197 static const struct kernel_param_ops null_queue_mode_param_ops = {
198 .set = null_set_queue_mode,
199 .get = param_get_int,
200 };
201
202 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, S_IRUGO);
203 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
204
205 static int g_gb = 250;
206 module_param_named(gb, g_gb, int, S_IRUGO);
207 MODULE_PARM_DESC(gb, "Size in GB");
208
209 static int g_bs = 512;
210 module_param_named(bs, g_bs, int, S_IRUGO);
211 MODULE_PARM_DESC(bs, "Block size (in bytes)");
212
213 static int nr_devices = 1;
214 module_param(nr_devices, int, S_IRUGO);
215 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
216
217 static bool g_blocking;
218 module_param_named(blocking, g_blocking, bool, S_IRUGO);
219 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
220
221 static bool shared_tags;
222 module_param(shared_tags, bool, S_IRUGO);
223 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
224
225 static int g_irqmode = NULL_IRQ_SOFTIRQ;
226
227 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
228 {
229 return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
230 NULL_IRQ_TIMER);
231 }
232
233 static const struct kernel_param_ops null_irqmode_param_ops = {
234 .set = null_set_irqmode,
235 .get = param_get_int,
236 };
237
238 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, S_IRUGO);
239 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
240
241 static unsigned long g_completion_nsec = 10000;
242 module_param_named(completion_nsec, g_completion_nsec, ulong, S_IRUGO);
243 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
244
245 static int g_hw_queue_depth = 64;
246 module_param_named(hw_queue_depth, g_hw_queue_depth, int, S_IRUGO);
247 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
248
249 static bool g_use_per_node_hctx;
250 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, S_IRUGO);
251 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
252
253 static struct nullb_device *null_alloc_dev(void);
254 static void null_free_dev(struct nullb_device *dev);
255 static void null_del_dev(struct nullb *nullb);
256 static int null_add_dev(struct nullb_device *dev);
257 static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
258
259 static inline struct nullb_device *to_nullb_device(struct config_item *item)
260 {
261 return item ? container_of(item, struct nullb_device, item) : NULL;
262 }
263
264 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
265 {
266 return snprintf(page, PAGE_SIZE, "%u\n", val);
267 }
268
269 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
270 char *page)
271 {
272 return snprintf(page, PAGE_SIZE, "%lu\n", val);
273 }
274
275 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
276 {
277 return snprintf(page, PAGE_SIZE, "%u\n", val);
278 }
279
280 static ssize_t nullb_device_uint_attr_store(unsigned int *val,
281 const char *page, size_t count)
282 {
283 unsigned int tmp;
284 int result;
285
286 result = kstrtouint(page, 0, &tmp);
287 if (result)
288 return result;
289
290 *val = tmp;
291 return count;
292 }
293
294 static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
295 const char *page, size_t count)
296 {
297 int result;
298 unsigned long tmp;
299
300 result = kstrtoul(page, 0, &tmp);
301 if (result)
302 return result;
303
304 *val = tmp;
305 return count;
306 }
307
308 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
309 size_t count)
310 {
311 bool tmp;
312 int result;
313
314 result = kstrtobool(page, &tmp);
315 if (result)
316 return result;
317
318 *val = tmp;
319 return count;
320 }
321
322 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
323 #define NULLB_DEVICE_ATTR(NAME, TYPE) \
324 static ssize_t \
325 nullb_device_##NAME##_show(struct config_item *item, char *page) \
326 { \
327 return nullb_device_##TYPE##_attr_show( \
328 to_nullb_device(item)->NAME, page); \
329 } \
330 static ssize_t \
331 nullb_device_##NAME##_store(struct config_item *item, const char *page, \
332 size_t count) \
333 { \
334 if (test_bit(NULLB_DEV_FL_CONFIGURED, &to_nullb_device(item)->flags)) \
335 return -EBUSY; \
336 return nullb_device_##TYPE##_attr_store( \
337 &to_nullb_device(item)->NAME, page, count); \
338 } \
339 CONFIGFS_ATTR(nullb_device_, NAME);
340
341 NULLB_DEVICE_ATTR(size, ulong);
342 NULLB_DEVICE_ATTR(completion_nsec, ulong);
343 NULLB_DEVICE_ATTR(submit_queues, uint);
344 NULLB_DEVICE_ATTR(home_node, uint);
345 NULLB_DEVICE_ATTR(queue_mode, uint);
346 NULLB_DEVICE_ATTR(blocksize, uint);
347 NULLB_DEVICE_ATTR(irqmode, uint);
348 NULLB_DEVICE_ATTR(hw_queue_depth, uint);
349 NULLB_DEVICE_ATTR(index, uint);
350 NULLB_DEVICE_ATTR(blocking, bool);
351 NULLB_DEVICE_ATTR(use_per_node_hctx, bool);
352 NULLB_DEVICE_ATTR(memory_backed, bool);
353 NULLB_DEVICE_ATTR(discard, bool);
354 NULLB_DEVICE_ATTR(mbps, uint);
355 NULLB_DEVICE_ATTR(cache_size, ulong);
356
357 static ssize_t nullb_device_power_show(struct config_item *item, char *page)
358 {
359 return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
360 }
361
362 static ssize_t nullb_device_power_store(struct config_item *item,
363 const char *page, size_t count)
364 {
365 struct nullb_device *dev = to_nullb_device(item);
366 bool newp = false;
367 ssize_t ret;
368
369 ret = nullb_device_bool_attr_store(&newp, page, count);
370 if (ret < 0)
371 return ret;
372
373 if (!dev->power && newp) {
374 if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
375 return count;
376 if (null_add_dev(dev)) {
377 clear_bit(NULLB_DEV_FL_UP, &dev->flags);
378 return -ENOMEM;
379 }
380
381 set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
382 dev->power = newp;
383 } else if (dev->power && !newp) {
384 mutex_lock(&lock);
385 dev->power = newp;
386 null_del_dev(dev->nullb);
387 mutex_unlock(&lock);
388 clear_bit(NULLB_DEV_FL_UP, &dev->flags);
389 }
390
391 return count;
392 }
393
394 CONFIGFS_ATTR(nullb_device_, power);
395
396 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
397 {
398 struct nullb_device *t_dev = to_nullb_device(item);
399
400 return badblocks_show(&t_dev->badblocks, page, 0);
401 }
402
403 static ssize_t nullb_device_badblocks_store(struct config_item *item,
404 const char *page, size_t count)
405 {
406 struct nullb_device *t_dev = to_nullb_device(item);
407 char *orig, *buf, *tmp;
408 u64 start, end;
409 int ret;
410
411 orig = kstrndup(page, count, GFP_KERNEL);
412 if (!orig)
413 return -ENOMEM;
414
415 buf = strstrip(orig);
416
417 ret = -EINVAL;
418 if (buf[0] != '+' && buf[0] != '-')
419 goto out;
420 tmp = strchr(&buf[1], '-');
421 if (!tmp)
422 goto out;
423 *tmp = '\0';
424 ret = kstrtoull(buf + 1, 0, &start);
425 if (ret)
426 goto out;
427 ret = kstrtoull(tmp + 1, 0, &end);
428 if (ret)
429 goto out;
430 ret = -EINVAL;
431 if (start > end)
432 goto out;
433 /* enable badblocks */
434 cmpxchg(&t_dev->badblocks.shift, -1, 0);
435 if (buf[0] == '+')
436 ret = badblocks_set(&t_dev->badblocks, start,
437 end - start + 1, 1);
438 else
439 ret = badblocks_clear(&t_dev->badblocks, start,
440 end - start + 1);
441 if (ret == 0)
442 ret = count;
443 out:
444 kfree(orig);
445 return ret;
446 }
447 CONFIGFS_ATTR(nullb_device_, badblocks);
448
449 static struct configfs_attribute *nullb_device_attrs[] = {
450 &nullb_device_attr_size,
451 &nullb_device_attr_completion_nsec,
452 &nullb_device_attr_submit_queues,
453 &nullb_device_attr_home_node,
454 &nullb_device_attr_queue_mode,
455 &nullb_device_attr_blocksize,
456 &nullb_device_attr_irqmode,
457 &nullb_device_attr_hw_queue_depth,
458 &nullb_device_attr_index,
459 &nullb_device_attr_blocking,
460 &nullb_device_attr_use_per_node_hctx,
461 &nullb_device_attr_power,
462 &nullb_device_attr_memory_backed,
463 &nullb_device_attr_discard,
464 &nullb_device_attr_mbps,
465 &nullb_device_attr_cache_size,
466 &nullb_device_attr_badblocks,
467 NULL,
468 };
469
470 static void nullb_device_release(struct config_item *item)
471 {
472 struct nullb_device *dev = to_nullb_device(item);
473
474 null_free_device_storage(dev, false);
475 null_free_dev(dev);
476 }
477
478 static struct configfs_item_operations nullb_device_ops = {
479 .release = nullb_device_release,
480 };
481
482 static const struct config_item_type nullb_device_type = {
483 .ct_item_ops = &nullb_device_ops,
484 .ct_attrs = nullb_device_attrs,
485 .ct_owner = THIS_MODULE,
486 };
487
488 static struct
489 config_item *nullb_group_make_item(struct config_group *group, const char *name)
490 {
491 struct nullb_device *dev;
492
493 dev = null_alloc_dev();
494 if (!dev)
495 return ERR_PTR(-ENOMEM);
496
497 config_item_init_type_name(&dev->item, name, &nullb_device_type);
498
499 return &dev->item;
500 }
501
502 static void
503 nullb_group_drop_item(struct config_group *group, struct config_item *item)
504 {
505 struct nullb_device *dev = to_nullb_device(item);
506
507 if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
508 mutex_lock(&lock);
509 dev->power = false;
510 null_del_dev(dev->nullb);
511 mutex_unlock(&lock);
512 }
513
514 config_item_put(item);
515 }
516
517 static ssize_t memb_group_features_show(struct config_item *item, char *page)
518 {
519 return snprintf(page, PAGE_SIZE, "memory_backed,discard,bandwidth,cache,badblocks\n");
520 }
521
522 CONFIGFS_ATTR_RO(memb_group_, features);
523
524 static struct configfs_attribute *nullb_group_attrs[] = {
525 &memb_group_attr_features,
526 NULL,
527 };
528
529 static struct configfs_group_operations nullb_group_ops = {
530 .make_item = nullb_group_make_item,
531 .drop_item = nullb_group_drop_item,
532 };
533
534 static const struct config_item_type nullb_group_type = {
535 .ct_group_ops = &nullb_group_ops,
536 .ct_attrs = nullb_group_attrs,
537 .ct_owner = THIS_MODULE,
538 };
539
540 static struct configfs_subsystem nullb_subsys = {
541 .su_group = {
542 .cg_item = {
543 .ci_namebuf = "nullb",
544 .ci_type = &nullb_group_type,
545 },
546 },
547 };
548
549 static inline int null_cache_active(struct nullb *nullb)
550 {
551 return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
552 }
553
554 static struct nullb_device *null_alloc_dev(void)
555 {
556 struct nullb_device *dev;
557
558 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
559 if (!dev)
560 return NULL;
561 INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
562 INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
563 if (badblocks_init(&dev->badblocks, 0)) {
564 kfree(dev);
565 return NULL;
566 }
567
568 dev->size = g_gb * 1024;
569 dev->completion_nsec = g_completion_nsec;
570 dev->submit_queues = g_submit_queues;
571 dev->home_node = g_home_node;
572 dev->queue_mode = g_queue_mode;
573 dev->blocksize = g_bs;
574 dev->irqmode = g_irqmode;
575 dev->hw_queue_depth = g_hw_queue_depth;
576 dev->blocking = g_blocking;
577 dev->use_per_node_hctx = g_use_per_node_hctx;
578 return dev;
579 }
580
581 static void null_free_dev(struct nullb_device *dev)
582 {
583 if (!dev)
584 return;
585
586 badblocks_exit(&dev->badblocks);
587 kfree(dev);
588 }
589
590 static void put_tag(struct nullb_queue *nq, unsigned int tag)
591 {
592 clear_bit_unlock(tag, nq->tag_map);
593
594 if (waitqueue_active(&nq->wait))
595 wake_up(&nq->wait);
596 }
597
598 static unsigned int get_tag(struct nullb_queue *nq)
599 {
600 unsigned int tag;
601
602 do {
603 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
604 if (tag >= nq->queue_depth)
605 return -1U;
606 } while (test_and_set_bit_lock(tag, nq->tag_map));
607
608 return tag;
609 }
610
611 static void free_cmd(struct nullb_cmd *cmd)
612 {
613 put_tag(cmd->nq, cmd->tag);
614 }
615
616 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
617
618 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
619 {
620 struct nullb_cmd *cmd;
621 unsigned int tag;
622
623 tag = get_tag(nq);
624 if (tag != -1U) {
625 cmd = &nq->cmds[tag];
626 cmd->tag = tag;
627 cmd->nq = nq;
628 if (nq->dev->irqmode == NULL_IRQ_TIMER) {
629 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
630 HRTIMER_MODE_REL);
631 cmd->timer.function = null_cmd_timer_expired;
632 }
633 return cmd;
634 }
635
636 return NULL;
637 }
638
639 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
640 {
641 struct nullb_cmd *cmd;
642 DEFINE_WAIT(wait);
643
644 cmd = __alloc_cmd(nq);
645 if (cmd || !can_wait)
646 return cmd;
647
648 do {
649 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
650 cmd = __alloc_cmd(nq);
651 if (cmd)
652 break;
653
654 io_schedule();
655 } while (1);
656
657 finish_wait(&nq->wait, &wait);
658 return cmd;
659 }
660
661 static void end_cmd(struct nullb_cmd *cmd)
662 {
663 struct request_queue *q = NULL;
664 int queue_mode = cmd->nq->dev->queue_mode;
665
666 if (cmd->rq)
667 q = cmd->rq->q;
668
669 switch (queue_mode) {
670 case NULL_Q_MQ:
671 blk_mq_end_request(cmd->rq, cmd->error);
672 return;
673 case NULL_Q_RQ:
674 INIT_LIST_HEAD(&cmd->rq->queuelist);
675 blk_end_request_all(cmd->rq, cmd->error);
676 break;
677 case NULL_Q_BIO:
678 cmd->bio->bi_status = cmd->error;
679 bio_endio(cmd->bio);
680 break;
681 }
682
683 free_cmd(cmd);
684
685 /* Restart queue if needed, as we are freeing a tag */
686 if (queue_mode == NULL_Q_RQ && blk_queue_stopped(q)) {
687 unsigned long flags;
688
689 spin_lock_irqsave(q->queue_lock, flags);
690 blk_start_queue_async(q);
691 spin_unlock_irqrestore(q->queue_lock, flags);
692 }
693 }
694
695 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
696 {
697 end_cmd(container_of(timer, struct nullb_cmd, timer));
698
699 return HRTIMER_NORESTART;
700 }
701
702 static void null_cmd_end_timer(struct nullb_cmd *cmd)
703 {
704 ktime_t kt = cmd->nq->dev->completion_nsec;
705
706 hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
707 }
708
709 static void null_softirq_done_fn(struct request *rq)
710 {
711 struct nullb *nullb = rq->q->queuedata;
712
713 if (nullb->dev->queue_mode == NULL_Q_MQ)
714 end_cmd(blk_mq_rq_to_pdu(rq));
715 else
716 end_cmd(rq->special);
717 }
718
719 static struct nullb_page *null_alloc_page(gfp_t gfp_flags)
720 {
721 struct nullb_page *t_page;
722
723 t_page = kmalloc(sizeof(struct nullb_page), gfp_flags);
724 if (!t_page)
725 goto out;
726
727 t_page->page = alloc_pages(gfp_flags, 0);
728 if (!t_page->page)
729 goto out_freepage;
730
731 t_page->bitmap = 0;
732 return t_page;
733 out_freepage:
734 kfree(t_page);
735 out:
736 return NULL;
737 }
738
739 static void null_free_page(struct nullb_page *t_page)
740 {
741 __set_bit(NULLB_PAGE_FREE, &t_page->bitmap);
742 if (test_bit(NULLB_PAGE_LOCK, &t_page->bitmap))
743 return;
744 __free_page(t_page->page);
745 kfree(t_page);
746 }
747
748 static void null_free_sector(struct nullb *nullb, sector_t sector,
749 bool is_cache)
750 {
751 unsigned int sector_bit;
752 u64 idx;
753 struct nullb_page *t_page, *ret;
754 struct radix_tree_root *root;
755
756 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
757 idx = sector >> PAGE_SECTORS_SHIFT;
758 sector_bit = (sector & SECTOR_MASK);
759
760 t_page = radix_tree_lookup(root, idx);
761 if (t_page) {
762 __clear_bit(sector_bit, &t_page->bitmap);
763
764 if (!t_page->bitmap) {
765 ret = radix_tree_delete_item(root, idx, t_page);
766 WARN_ON(ret != t_page);
767 null_free_page(ret);
768 if (is_cache)
769 nullb->dev->curr_cache -= PAGE_SIZE;
770 }
771 }
772 }
773
774 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
775 struct nullb_page *t_page, bool is_cache)
776 {
777 struct radix_tree_root *root;
778
779 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
780
781 if (radix_tree_insert(root, idx, t_page)) {
782 null_free_page(t_page);
783 t_page = radix_tree_lookup(root, idx);
784 WARN_ON(!t_page || t_page->page->index != idx);
785 } else if (is_cache)
786 nullb->dev->curr_cache += PAGE_SIZE;
787
788 return t_page;
789 }
790
791 static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
792 {
793 unsigned long pos = 0;
794 int nr_pages;
795 struct nullb_page *ret, *t_pages[FREE_BATCH];
796 struct radix_tree_root *root;
797
798 root = is_cache ? &dev->cache : &dev->data;
799
800 do {
801 int i;
802
803 nr_pages = radix_tree_gang_lookup(root,
804 (void **)t_pages, pos, FREE_BATCH);
805
806 for (i = 0; i < nr_pages; i++) {
807 pos = t_pages[i]->page->index;
808 ret = radix_tree_delete_item(root, pos, t_pages[i]);
809 WARN_ON(ret != t_pages[i]);
810 null_free_page(ret);
811 }
812
813 pos++;
814 } while (nr_pages == FREE_BATCH);
815
816 if (is_cache)
817 dev->curr_cache = 0;
818 }
819
820 static struct nullb_page *__null_lookup_page(struct nullb *nullb,
821 sector_t sector, bool for_write, bool is_cache)
822 {
823 unsigned int sector_bit;
824 u64 idx;
825 struct nullb_page *t_page;
826 struct radix_tree_root *root;
827
828 idx = sector >> PAGE_SECTORS_SHIFT;
829 sector_bit = (sector & SECTOR_MASK);
830
831 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
832 t_page = radix_tree_lookup(root, idx);
833 WARN_ON(t_page && t_page->page->index != idx);
834
835 if (t_page && (for_write || test_bit(sector_bit, &t_page->bitmap)))
836 return t_page;
837
838 return NULL;
839 }
840
841 static struct nullb_page *null_lookup_page(struct nullb *nullb,
842 sector_t sector, bool for_write, bool ignore_cache)
843 {
844 struct nullb_page *page = NULL;
845
846 if (!ignore_cache)
847 page = __null_lookup_page(nullb, sector, for_write, true);
848 if (page)
849 return page;
850 return __null_lookup_page(nullb, sector, for_write, false);
851 }
852
853 static struct nullb_page *null_insert_page(struct nullb *nullb,
854 sector_t sector, bool ignore_cache)
855 {
856 u64 idx;
857 struct nullb_page *t_page;
858
859 t_page = null_lookup_page(nullb, sector, true, ignore_cache);
860 if (t_page)
861 return t_page;
862
863 spin_unlock_irq(&nullb->lock);
864
865 t_page = null_alloc_page(GFP_NOIO);
866 if (!t_page)
867 goto out_lock;
868
869 if (radix_tree_preload(GFP_NOIO))
870 goto out_freepage;
871
872 spin_lock_irq(&nullb->lock);
873 idx = sector >> PAGE_SECTORS_SHIFT;
874 t_page->page->index = idx;
875 t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
876 radix_tree_preload_end();
877
878 return t_page;
879 out_freepage:
880 null_free_page(t_page);
881 out_lock:
882 spin_lock_irq(&nullb->lock);
883 return null_lookup_page(nullb, sector, true, ignore_cache);
884 }
885
886 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
887 {
888 int i;
889 unsigned int offset;
890 u64 idx;
891 struct nullb_page *t_page, *ret;
892 void *dst, *src;
893
894 idx = c_page->page->index;
895
896 t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
897
898 __clear_bit(NULLB_PAGE_LOCK, &c_page->bitmap);
899 if (test_bit(NULLB_PAGE_FREE, &c_page->bitmap)) {
900 null_free_page(c_page);
901 if (t_page && t_page->bitmap == 0) {
902 ret = radix_tree_delete_item(&nullb->dev->data,
903 idx, t_page);
904 null_free_page(t_page);
905 }
906 return 0;
907 }
908
909 if (!t_page)
910 return -ENOMEM;
911
912 src = kmap_atomic(c_page->page);
913 dst = kmap_atomic(t_page->page);
914
915 for (i = 0; i < PAGE_SECTORS;
916 i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
917 if (test_bit(i, &c_page->bitmap)) {
918 offset = (i << SECTOR_SHIFT);
919 memcpy(dst + offset, src + offset,
920 nullb->dev->blocksize);
921 __set_bit(i, &t_page->bitmap);
922 }
923 }
924
925 kunmap_atomic(dst);
926 kunmap_atomic(src);
927
928 ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
929 null_free_page(ret);
930 nullb->dev->curr_cache -= PAGE_SIZE;
931
932 return 0;
933 }
934
935 static int null_make_cache_space(struct nullb *nullb, unsigned long n)
936 {
937 int i, err, nr_pages;
938 struct nullb_page *c_pages[FREE_BATCH];
939 unsigned long flushed = 0, one_round;
940
941 again:
942 if ((nullb->dev->cache_size * 1024 * 1024) >
943 nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
944 return 0;
945
946 nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
947 (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
948 /*
949 * nullb_flush_cache_page could unlock before using the c_pages. To
950 * avoid race, we don't allow page free
951 */
952 for (i = 0; i < nr_pages; i++) {
953 nullb->cache_flush_pos = c_pages[i]->page->index;
954 /*
955 * We found the page which is being flushed to disk by other
956 * threads
957 */
958 if (test_bit(NULLB_PAGE_LOCK, &c_pages[i]->bitmap))
959 c_pages[i] = NULL;
960 else
961 __set_bit(NULLB_PAGE_LOCK, &c_pages[i]->bitmap);
962 }
963
964 one_round = 0;
965 for (i = 0; i < nr_pages; i++) {
966 if (c_pages[i] == NULL)
967 continue;
968 err = null_flush_cache_page(nullb, c_pages[i]);
969 if (err)
970 return err;
971 one_round++;
972 }
973 flushed += one_round << PAGE_SHIFT;
974
975 if (n > flushed) {
976 if (nr_pages == 0)
977 nullb->cache_flush_pos = 0;
978 if (one_round == 0) {
979 /* give other threads a chance */
980 spin_unlock_irq(&nullb->lock);
981 spin_lock_irq(&nullb->lock);
982 }
983 goto again;
984 }
985 return 0;
986 }
987
988 static int copy_to_nullb(struct nullb *nullb, struct page *source,
989 unsigned int off, sector_t sector, size_t n, bool is_fua)
990 {
991 size_t temp, count = 0;
992 unsigned int offset;
993 struct nullb_page *t_page;
994 void *dst, *src;
995
996 while (count < n) {
997 temp = min_t(size_t, nullb->dev->blocksize, n - count);
998
999 if (null_cache_active(nullb) && !is_fua)
1000 null_make_cache_space(nullb, PAGE_SIZE);
1001
1002 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1003 t_page = null_insert_page(nullb, sector,
1004 !null_cache_active(nullb) || is_fua);
1005 if (!t_page)
1006 return -ENOSPC;
1007
1008 src = kmap_atomic(source);
1009 dst = kmap_atomic(t_page->page);
1010 memcpy(dst + offset, src + off + count, temp);
1011 kunmap_atomic(dst);
1012 kunmap_atomic(src);
1013
1014 __set_bit(sector & SECTOR_MASK, &t_page->bitmap);
1015
1016 if (is_fua)
1017 null_free_sector(nullb, sector, true);
1018
1019 count += temp;
1020 sector += temp >> SECTOR_SHIFT;
1021 }
1022 return 0;
1023 }
1024
1025 static int copy_from_nullb(struct nullb *nullb, struct page *dest,
1026 unsigned int off, sector_t sector, size_t n)
1027 {
1028 size_t temp, count = 0;
1029 unsigned int offset;
1030 struct nullb_page *t_page;
1031 void *dst, *src;
1032
1033 while (count < n) {
1034 temp = min_t(size_t, nullb->dev->blocksize, n - count);
1035
1036 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1037 t_page = null_lookup_page(nullb, sector, false,
1038 !null_cache_active(nullb));
1039
1040 dst = kmap_atomic(dest);
1041 if (!t_page) {
1042 memset(dst + off + count, 0, temp);
1043 goto next;
1044 }
1045 src = kmap_atomic(t_page->page);
1046 memcpy(dst + off + count, src + offset, temp);
1047 kunmap_atomic(src);
1048 next:
1049 kunmap_atomic(dst);
1050
1051 count += temp;
1052 sector += temp >> SECTOR_SHIFT;
1053 }
1054 return 0;
1055 }
1056
1057 static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n)
1058 {
1059 size_t temp;
1060
1061 spin_lock_irq(&nullb->lock);
1062 while (n > 0) {
1063 temp = min_t(size_t, n, nullb->dev->blocksize);
1064 null_free_sector(nullb, sector, false);
1065 if (null_cache_active(nullb))
1066 null_free_sector(nullb, sector, true);
1067 sector += temp >> SECTOR_SHIFT;
1068 n -= temp;
1069 }
1070 spin_unlock_irq(&nullb->lock);
1071 }
1072
1073 static int null_handle_flush(struct nullb *nullb)
1074 {
1075 int err;
1076
1077 if (!null_cache_active(nullb))
1078 return 0;
1079
1080 spin_lock_irq(&nullb->lock);
1081 while (true) {
1082 err = null_make_cache_space(nullb,
1083 nullb->dev->cache_size * 1024 * 1024);
1084 if (err || nullb->dev->curr_cache == 0)
1085 break;
1086 }
1087
1088 WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1089 spin_unlock_irq(&nullb->lock);
1090 return err;
1091 }
1092
1093 static int null_transfer(struct nullb *nullb, struct page *page,
1094 unsigned int len, unsigned int off, bool is_write, sector_t sector,
1095 bool is_fua)
1096 {
1097 int err = 0;
1098
1099 if (!is_write) {
1100 err = copy_from_nullb(nullb, page, off, sector, len);
1101 flush_dcache_page(page);
1102 } else {
1103 flush_dcache_page(page);
1104 err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
1105 }
1106
1107 return err;
1108 }
1109
1110 static int null_handle_rq(struct nullb_cmd *cmd)
1111 {
1112 struct request *rq = cmd->rq;
1113 struct nullb *nullb = cmd->nq->dev->nullb;
1114 int err;
1115 unsigned int len;
1116 sector_t sector;
1117 struct req_iterator iter;
1118 struct bio_vec bvec;
1119
1120 sector = blk_rq_pos(rq);
1121
1122 if (req_op(rq) == REQ_OP_DISCARD) {
1123 null_handle_discard(nullb, sector, blk_rq_bytes(rq));
1124 return 0;
1125 }
1126
1127 spin_lock_irq(&nullb->lock);
1128 rq_for_each_segment(bvec, rq, iter) {
1129 len = bvec.bv_len;
1130 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1131 op_is_write(req_op(rq)), sector,
1132 req_op(rq) & REQ_FUA);
1133 if (err) {
1134 spin_unlock_irq(&nullb->lock);
1135 return err;
1136 }
1137 sector += len >> SECTOR_SHIFT;
1138 }
1139 spin_unlock_irq(&nullb->lock);
1140
1141 return 0;
1142 }
1143
1144 static int null_handle_bio(struct nullb_cmd *cmd)
1145 {
1146 struct bio *bio = cmd->bio;
1147 struct nullb *nullb = cmd->nq->dev->nullb;
1148 int err;
1149 unsigned int len;
1150 sector_t sector;
1151 struct bio_vec bvec;
1152 struct bvec_iter iter;
1153
1154 sector = bio->bi_iter.bi_sector;
1155
1156 if (bio_op(bio) == REQ_OP_DISCARD) {
1157 null_handle_discard(nullb, sector,
1158 bio_sectors(bio) << SECTOR_SHIFT);
1159 return 0;
1160 }
1161
1162 spin_lock_irq(&nullb->lock);
1163 bio_for_each_segment(bvec, bio, iter) {
1164 len = bvec.bv_len;
1165 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1166 op_is_write(bio_op(bio)), sector,
1167 bio_op(bio) & REQ_FUA);
1168 if (err) {
1169 spin_unlock_irq(&nullb->lock);
1170 return err;
1171 }
1172 sector += len >> SECTOR_SHIFT;
1173 }
1174 spin_unlock_irq(&nullb->lock);
1175 return 0;
1176 }
1177
1178 static void null_stop_queue(struct nullb *nullb)
1179 {
1180 struct request_queue *q = nullb->q;
1181
1182 if (nullb->dev->queue_mode == NULL_Q_MQ)
1183 blk_mq_stop_hw_queues(q);
1184 else {
1185 spin_lock_irq(q->queue_lock);
1186 blk_stop_queue(q);
1187 spin_unlock_irq(q->queue_lock);
1188 }
1189 }
1190
1191 static void null_restart_queue_async(struct nullb *nullb)
1192 {
1193 struct request_queue *q = nullb->q;
1194 unsigned long flags;
1195
1196 if (nullb->dev->queue_mode == NULL_Q_MQ)
1197 blk_mq_start_stopped_hw_queues(q, true);
1198 else {
1199 spin_lock_irqsave(q->queue_lock, flags);
1200 blk_start_queue_async(q);
1201 spin_unlock_irqrestore(q->queue_lock, flags);
1202 }
1203 }
1204
1205 static blk_status_t null_handle_cmd(struct nullb_cmd *cmd)
1206 {
1207 struct nullb_device *dev = cmd->nq->dev;
1208 struct nullb *nullb = dev->nullb;
1209 int err = 0;
1210
1211 if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
1212 struct request *rq = cmd->rq;
1213
1214 if (!hrtimer_active(&nullb->bw_timer))
1215 hrtimer_restart(&nullb->bw_timer);
1216
1217 if (atomic_long_sub_return(blk_rq_bytes(rq),
1218 &nullb->cur_bytes) < 0) {
1219 null_stop_queue(nullb);
1220 /* race with timer */
1221 if (atomic_long_read(&nullb->cur_bytes) > 0)
1222 null_restart_queue_async(nullb);
1223 if (dev->queue_mode == NULL_Q_RQ) {
1224 struct request_queue *q = nullb->q;
1225
1226 spin_lock_irq(q->queue_lock);
1227 rq->rq_flags |= RQF_DONTPREP;
1228 blk_requeue_request(q, rq);
1229 spin_unlock_irq(q->queue_lock);
1230 return BLK_STS_OK;
1231 } else
1232 /* requeue request */
1233 return BLK_STS_DEV_RESOURCE;
1234 }
1235 }
1236
1237 if (nullb->dev->badblocks.shift != -1) {
1238 int bad_sectors;
1239 sector_t sector, size, first_bad;
1240 bool is_flush = true;
1241
1242 if (dev->queue_mode == NULL_Q_BIO &&
1243 bio_op(cmd->bio) != REQ_OP_FLUSH) {
1244 is_flush = false;
1245 sector = cmd->bio->bi_iter.bi_sector;
1246 size = bio_sectors(cmd->bio);
1247 }
1248 if (dev->queue_mode != NULL_Q_BIO &&
1249 req_op(cmd->rq) != REQ_OP_FLUSH) {
1250 is_flush = false;
1251 sector = blk_rq_pos(cmd->rq);
1252 size = blk_rq_sectors(cmd->rq);
1253 }
1254 if (!is_flush && badblocks_check(&nullb->dev->badblocks, sector,
1255 size, &first_bad, &bad_sectors)) {
1256 cmd->error = BLK_STS_IOERR;
1257 goto out;
1258 }
1259 }
1260
1261 if (dev->memory_backed) {
1262 if (dev->queue_mode == NULL_Q_BIO) {
1263 if (bio_op(cmd->bio) == REQ_OP_FLUSH)
1264 err = null_handle_flush(nullb);
1265 else
1266 err = null_handle_bio(cmd);
1267 } else {
1268 if (req_op(cmd->rq) == REQ_OP_FLUSH)
1269 err = null_handle_flush(nullb);
1270 else
1271 err = null_handle_rq(cmd);
1272 }
1273 }
1274 cmd->error = errno_to_blk_status(err);
1275 out:
1276 /* Complete IO by inline, softirq or timer */
1277 switch (dev->irqmode) {
1278 case NULL_IRQ_SOFTIRQ:
1279 switch (dev->queue_mode) {
1280 case NULL_Q_MQ:
1281 blk_mq_complete_request(cmd->rq);
1282 break;
1283 case NULL_Q_RQ:
1284 blk_complete_request(cmd->rq);
1285 break;
1286 case NULL_Q_BIO:
1287 /*
1288 * XXX: no proper submitting cpu information available.
1289 */
1290 end_cmd(cmd);
1291 break;
1292 }
1293 break;
1294 case NULL_IRQ_NONE:
1295 end_cmd(cmd);
1296 break;
1297 case NULL_IRQ_TIMER:
1298 null_cmd_end_timer(cmd);
1299 break;
1300 }
1301 return BLK_STS_OK;
1302 }
1303
1304 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1305 {
1306 struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1307 ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1308 unsigned int mbps = nullb->dev->mbps;
1309
1310 if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1311 return HRTIMER_NORESTART;
1312
1313 atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1314 null_restart_queue_async(nullb);
1315
1316 hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1317
1318 return HRTIMER_RESTART;
1319 }
1320
1321 static void nullb_setup_bwtimer(struct nullb *nullb)
1322 {
1323 ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1324
1325 hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1326 nullb->bw_timer.function = nullb_bwtimer_fn;
1327 atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1328 hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1329 }
1330
1331 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
1332 {
1333 int index = 0;
1334
1335 if (nullb->nr_queues != 1)
1336 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
1337
1338 return &nullb->queues[index];
1339 }
1340
1341 static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
1342 {
1343 struct nullb *nullb = q->queuedata;
1344 struct nullb_queue *nq = nullb_to_queue(nullb);
1345 struct nullb_cmd *cmd;
1346
1347 cmd = alloc_cmd(nq, 1);
1348 cmd->bio = bio;
1349
1350 null_handle_cmd(cmd);
1351 return BLK_QC_T_NONE;
1352 }
1353
1354 static enum blk_eh_timer_return null_rq_timed_out_fn(struct request *rq)
1355 {
1356 pr_info("null: rq %p timed out\n", rq);
1357 return BLK_EH_HANDLED;
1358 }
1359
1360 static int null_rq_prep_fn(struct request_queue *q, struct request *req)
1361 {
1362 struct nullb *nullb = q->queuedata;
1363 struct nullb_queue *nq = nullb_to_queue(nullb);
1364 struct nullb_cmd *cmd;
1365
1366 cmd = alloc_cmd(nq, 0);
1367 if (cmd) {
1368 cmd->rq = req;
1369 req->special = cmd;
1370 return BLKPREP_OK;
1371 }
1372 blk_stop_queue(q);
1373
1374 return BLKPREP_DEFER;
1375 }
1376
1377 static bool should_timeout_request(struct request *rq)
1378 {
1379 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1380 if (g_timeout_str[0])
1381 return should_fail(&null_timeout_attr, 1);
1382 #endif
1383
1384 return false;
1385 }
1386
1387 static void null_request_fn(struct request_queue *q)
1388 {
1389 struct request *rq;
1390
1391 while ((rq = blk_fetch_request(q)) != NULL) {
1392 struct nullb_cmd *cmd = rq->special;
1393
1394 if (!should_timeout_request(rq)) {
1395 spin_unlock_irq(q->queue_lock);
1396 null_handle_cmd(cmd);
1397 spin_lock_irq(q->queue_lock);
1398 }
1399 }
1400 }
1401
1402 static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
1403 {
1404 pr_info("null: rq %p timed out\n", rq);
1405 return BLK_EH_HANDLED;
1406 }
1407
1408 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1409 const struct blk_mq_queue_data *bd)
1410 {
1411 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
1412 struct nullb_queue *nq = hctx->driver_data;
1413
1414 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1415
1416 if (nq->dev->irqmode == NULL_IRQ_TIMER) {
1417 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1418 cmd->timer.function = null_cmd_timer_expired;
1419 }
1420 cmd->rq = bd->rq;
1421 cmd->nq = nq;
1422
1423 blk_mq_start_request(bd->rq);
1424
1425 if (!should_timeout_request(bd->rq))
1426 return null_handle_cmd(cmd);
1427
1428 return BLK_STS_OK;
1429 }
1430
1431 static const struct blk_mq_ops null_mq_ops = {
1432 .queue_rq = null_queue_rq,
1433 .complete = null_softirq_done_fn,
1434 .timeout = null_timeout_rq,
1435 };
1436
1437 static void cleanup_queue(struct nullb_queue *nq)
1438 {
1439 kfree(nq->tag_map);
1440 kfree(nq->cmds);
1441 }
1442
1443 static void cleanup_queues(struct nullb *nullb)
1444 {
1445 int i;
1446
1447 for (i = 0; i < nullb->nr_queues; i++)
1448 cleanup_queue(&nullb->queues[i]);
1449
1450 kfree(nullb->queues);
1451 }
1452
1453 static void null_del_dev(struct nullb *nullb)
1454 {
1455 struct nullb_device *dev = nullb->dev;
1456
1457 ida_simple_remove(&nullb_indexes, nullb->index);
1458
1459 list_del_init(&nullb->list);
1460
1461 del_gendisk(nullb->disk);
1462
1463 if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1464 hrtimer_cancel(&nullb->bw_timer);
1465 atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1466 null_restart_queue_async(nullb);
1467 }
1468
1469 blk_cleanup_queue(nullb->q);
1470 if (dev->queue_mode == NULL_Q_MQ &&
1471 nullb->tag_set == &nullb->__tag_set)
1472 blk_mq_free_tag_set(nullb->tag_set);
1473 put_disk(nullb->disk);
1474 cleanup_queues(nullb);
1475 if (null_cache_active(nullb))
1476 null_free_device_storage(nullb->dev, true);
1477 kfree(nullb);
1478 dev->nullb = NULL;
1479 }
1480
1481 static void null_config_discard(struct nullb *nullb)
1482 {
1483 if (nullb->dev->discard == false)
1484 return;
1485 nullb->q->limits.discard_granularity = nullb->dev->blocksize;
1486 nullb->q->limits.discard_alignment = nullb->dev->blocksize;
1487 blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
1488 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, nullb->q);
1489 }
1490
1491 static int null_open(struct block_device *bdev, fmode_t mode)
1492 {
1493 return 0;
1494 }
1495
1496 static void null_release(struct gendisk *disk, fmode_t mode)
1497 {
1498 }
1499
1500 static const struct block_device_operations null_fops = {
1501 .owner = THIS_MODULE,
1502 .open = null_open,
1503 .release = null_release,
1504 };
1505
1506 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1507 {
1508 BUG_ON(!nullb);
1509 BUG_ON(!nq);
1510
1511 init_waitqueue_head(&nq->wait);
1512 nq->queue_depth = nullb->queue_depth;
1513 nq->dev = nullb->dev;
1514 }
1515
1516 static void null_init_queues(struct nullb *nullb)
1517 {
1518 struct request_queue *q = nullb->q;
1519 struct blk_mq_hw_ctx *hctx;
1520 struct nullb_queue *nq;
1521 int i;
1522
1523 queue_for_each_hw_ctx(q, hctx, i) {
1524 if (!hctx->nr_ctx || !hctx->tags)
1525 continue;
1526 nq = &nullb->queues[i];
1527 hctx->driver_data = nq;
1528 null_init_queue(nullb, nq);
1529 nullb->nr_queues++;
1530 }
1531 }
1532
1533 static int setup_commands(struct nullb_queue *nq)
1534 {
1535 struct nullb_cmd *cmd;
1536 int i, tag_size;
1537
1538 nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL);
1539 if (!nq->cmds)
1540 return -ENOMEM;
1541
1542 tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
1543 nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL);
1544 if (!nq->tag_map) {
1545 kfree(nq->cmds);
1546 return -ENOMEM;
1547 }
1548
1549 for (i = 0; i < nq->queue_depth; i++) {
1550 cmd = &nq->cmds[i];
1551 INIT_LIST_HEAD(&cmd->list);
1552 cmd->ll_list.next = NULL;
1553 cmd->tag = -1U;
1554 }
1555
1556 return 0;
1557 }
1558
1559 static int setup_queues(struct nullb *nullb)
1560 {
1561 nullb->queues = kzalloc(nullb->dev->submit_queues *
1562 sizeof(struct nullb_queue), GFP_KERNEL);
1563 if (!nullb->queues)
1564 return -ENOMEM;
1565
1566 nullb->nr_queues = 0;
1567 nullb->queue_depth = nullb->dev->hw_queue_depth;
1568
1569 return 0;
1570 }
1571
1572 static int init_driver_queues(struct nullb *nullb)
1573 {
1574 struct nullb_queue *nq;
1575 int i, ret = 0;
1576
1577 for (i = 0; i < nullb->dev->submit_queues; i++) {
1578 nq = &nullb->queues[i];
1579
1580 null_init_queue(nullb, nq);
1581
1582 ret = setup_commands(nq);
1583 if (ret)
1584 return ret;
1585 nullb->nr_queues++;
1586 }
1587 return 0;
1588 }
1589
1590 static int null_gendisk_register(struct nullb *nullb)
1591 {
1592 struct gendisk *disk;
1593 sector_t size;
1594
1595 disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
1596 if (!disk)
1597 return -ENOMEM;
1598 size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
1599 set_capacity(disk, size >> 9);
1600
1601 disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
1602 disk->major = null_major;
1603 disk->first_minor = nullb->index;
1604 disk->fops = &null_fops;
1605 disk->private_data = nullb;
1606 disk->queue = nullb->q;
1607 strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1608
1609 add_disk(disk);
1610 return 0;
1611 }
1612
1613 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
1614 {
1615 set->ops = &null_mq_ops;
1616 set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
1617 g_submit_queues;
1618 set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
1619 g_hw_queue_depth;
1620 set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
1621 set->cmd_size = sizeof(struct nullb_cmd);
1622 set->flags = BLK_MQ_F_SHOULD_MERGE;
1623 if (g_no_sched)
1624 set->flags |= BLK_MQ_F_NO_SCHED;
1625 set->driver_data = NULL;
1626
1627 if ((nullb && nullb->dev->blocking) || g_blocking)
1628 set->flags |= BLK_MQ_F_BLOCKING;
1629
1630 return blk_mq_alloc_tag_set(set);
1631 }
1632
1633 static void null_validate_conf(struct nullb_device *dev)
1634 {
1635 dev->blocksize = round_down(dev->blocksize, 512);
1636 dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
1637
1638 if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
1639 if (dev->submit_queues != nr_online_nodes)
1640 dev->submit_queues = nr_online_nodes;
1641 } else if (dev->submit_queues > nr_cpu_ids)
1642 dev->submit_queues = nr_cpu_ids;
1643 else if (dev->submit_queues == 0)
1644 dev->submit_queues = 1;
1645
1646 dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
1647 dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1648
1649 /* Do memory allocation, so set blocking */
1650 if (dev->memory_backed)
1651 dev->blocking = true;
1652 else /* cache is meaningless */
1653 dev->cache_size = 0;
1654 dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1655 dev->cache_size);
1656 dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1657 /* can not stop a queue */
1658 if (dev->queue_mode == NULL_Q_BIO)
1659 dev->mbps = 0;
1660 }
1661
1662 static bool null_setup_fault(void)
1663 {
1664 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1665 if (!g_timeout_str[0])
1666 return true;
1667
1668 if (!setup_fault_attr(&null_timeout_attr, g_timeout_str))
1669 return false;
1670
1671 null_timeout_attr.verbose = 0;
1672 #endif
1673 return true;
1674 }
1675
1676 static int null_add_dev(struct nullb_device *dev)
1677 {
1678 struct nullb *nullb;
1679 int rv;
1680
1681 null_validate_conf(dev);
1682
1683 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1684 if (!nullb) {
1685 rv = -ENOMEM;
1686 goto out;
1687 }
1688 nullb->dev = dev;
1689 dev->nullb = nullb;
1690
1691 spin_lock_init(&nullb->lock);
1692
1693 rv = setup_queues(nullb);
1694 if (rv)
1695 goto out_free_nullb;
1696
1697 if (dev->queue_mode == NULL_Q_MQ) {
1698 if (shared_tags) {
1699 nullb->tag_set = &tag_set;
1700 rv = 0;
1701 } else {
1702 nullb->tag_set = &nullb->__tag_set;
1703 rv = null_init_tag_set(nullb, nullb->tag_set);
1704 }
1705
1706 if (rv)
1707 goto out_cleanup_queues;
1708
1709 if (!null_setup_fault())
1710 goto out_cleanup_queues;
1711
1712 nullb->tag_set->timeout = 5 * HZ;
1713 nullb->q = blk_mq_init_queue(nullb->tag_set);
1714 if (IS_ERR(nullb->q)) {
1715 rv = -ENOMEM;
1716 goto out_cleanup_tags;
1717 }
1718 null_init_queues(nullb);
1719 } else if (dev->queue_mode == NULL_Q_BIO) {
1720 nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node);
1721 if (!nullb->q) {
1722 rv = -ENOMEM;
1723 goto out_cleanup_queues;
1724 }
1725 blk_queue_make_request(nullb->q, null_queue_bio);
1726 rv = init_driver_queues(nullb);
1727 if (rv)
1728 goto out_cleanup_blk_queue;
1729 } else {
1730 nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock,
1731 dev->home_node);
1732 if (!nullb->q) {
1733 rv = -ENOMEM;
1734 goto out_cleanup_queues;
1735 }
1736
1737 if (!null_setup_fault())
1738 goto out_cleanup_blk_queue;
1739
1740 blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
1741 blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
1742 blk_queue_rq_timed_out(nullb->q, null_rq_timed_out_fn);
1743 nullb->q->rq_timeout = 5 * HZ;
1744 rv = init_driver_queues(nullb);
1745 if (rv)
1746 goto out_cleanup_blk_queue;
1747 }
1748
1749 if (dev->mbps) {
1750 set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
1751 nullb_setup_bwtimer(nullb);
1752 }
1753
1754 if (dev->cache_size > 0) {
1755 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
1756 blk_queue_write_cache(nullb->q, true, true);
1757 blk_queue_flush_queueable(nullb->q, true);
1758 }
1759
1760 nullb->q->queuedata = nullb;
1761 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
1762 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q);
1763
1764 mutex_lock(&lock);
1765 nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
1766 dev->index = nullb->index;
1767 mutex_unlock(&lock);
1768
1769 blk_queue_logical_block_size(nullb->q, dev->blocksize);
1770 blk_queue_physical_block_size(nullb->q, dev->blocksize);
1771
1772 null_config_discard(nullb);
1773
1774 sprintf(nullb->disk_name, "nullb%d", nullb->index);
1775
1776 rv = null_gendisk_register(nullb);
1777 if (rv)
1778 goto out_cleanup_blk_queue;
1779
1780 mutex_lock(&lock);
1781 list_add_tail(&nullb->list, &nullb_list);
1782 mutex_unlock(&lock);
1783
1784 return 0;
1785 out_cleanup_blk_queue:
1786 blk_cleanup_queue(nullb->q);
1787 out_cleanup_tags:
1788 if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
1789 blk_mq_free_tag_set(nullb->tag_set);
1790 out_cleanup_queues:
1791 cleanup_queues(nullb);
1792 out_free_nullb:
1793 kfree(nullb);
1794 out:
1795 return rv;
1796 }
1797
1798 static int __init null_init(void)
1799 {
1800 int ret = 0;
1801 unsigned int i;
1802 struct nullb *nullb;
1803 struct nullb_device *dev;
1804
1805 /* check for nullb_page.bitmap */
1806 if (sizeof(unsigned long) * 8 - 2 < (PAGE_SIZE >> SECTOR_SHIFT))
1807 return -EINVAL;
1808
1809 if (g_bs > PAGE_SIZE) {
1810 pr_warn("null_blk: invalid block size\n");
1811 pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
1812 g_bs = PAGE_SIZE;
1813 }
1814
1815 if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
1816 if (g_submit_queues != nr_online_nodes) {
1817 pr_warn("null_blk: submit_queues param is set to %u.\n",
1818 nr_online_nodes);
1819 g_submit_queues = nr_online_nodes;
1820 }
1821 } else if (g_submit_queues > nr_cpu_ids)
1822 g_submit_queues = nr_cpu_ids;
1823 else if (g_submit_queues <= 0)
1824 g_submit_queues = 1;
1825
1826 if (g_queue_mode == NULL_Q_MQ && shared_tags) {
1827 ret = null_init_tag_set(NULL, &tag_set);
1828 if (ret)
1829 return ret;
1830 }
1831
1832 config_group_init(&nullb_subsys.su_group);
1833 mutex_init(&nullb_subsys.su_mutex);
1834
1835 ret = configfs_register_subsystem(&nullb_subsys);
1836 if (ret)
1837 goto err_tagset;
1838
1839 mutex_init(&lock);
1840
1841 null_major = register_blkdev(0, "nullb");
1842 if (null_major < 0) {
1843 ret = null_major;
1844 goto err_conf;
1845 }
1846
1847 for (i = 0; i < nr_devices; i++) {
1848 dev = null_alloc_dev();
1849 if (!dev) {
1850 ret = -ENOMEM;
1851 goto err_dev;
1852 }
1853 ret = null_add_dev(dev);
1854 if (ret) {
1855 null_free_dev(dev);
1856 goto err_dev;
1857 }
1858 }
1859
1860 pr_info("null: module loaded\n");
1861 return 0;
1862
1863 err_dev:
1864 while (!list_empty(&nullb_list)) {
1865 nullb = list_entry(nullb_list.next, struct nullb, list);
1866 dev = nullb->dev;
1867 null_del_dev(nullb);
1868 null_free_dev(dev);
1869 }
1870 unregister_blkdev(null_major, "nullb");
1871 err_conf:
1872 configfs_unregister_subsystem(&nullb_subsys);
1873 err_tagset:
1874 if (g_queue_mode == NULL_Q_MQ && shared_tags)
1875 blk_mq_free_tag_set(&tag_set);
1876 return ret;
1877 }
1878
1879 static void __exit null_exit(void)
1880 {
1881 struct nullb *nullb;
1882
1883 configfs_unregister_subsystem(&nullb_subsys);
1884
1885 unregister_blkdev(null_major, "nullb");
1886
1887 mutex_lock(&lock);
1888 while (!list_empty(&nullb_list)) {
1889 struct nullb_device *dev;
1890
1891 nullb = list_entry(nullb_list.next, struct nullb, list);
1892 dev = nullb->dev;
1893 null_del_dev(nullb);
1894 null_free_dev(dev);
1895 }
1896 mutex_unlock(&lock);
1897
1898 if (g_queue_mode == NULL_Q_MQ && shared_tags)
1899 blk_mq_free_tag_set(&tag_set);
1900 }
1901
1902 module_init(null_init);
1903 module_exit(null_exit);
1904
1905 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
1906 MODULE_LICENSE("GPL");
CRIS linux kernel tree