?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
PCI: hotplug: Embed hotplug_slot
[linux/fpc-iii.git] / drivers / block / null_blk_main.c
blob093b614d652445a337db00ea8beaded767073415
1 /*
2 * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
3 * Shaohua Li <shli@fb.com>
4 */
5 #include <linux/module.h>
6
7 #include <linux/moduleparam.h>
8 #include <linux/sched.h>
9 #include <linux/fs.h>
10 #include <linux/init.h>
11 #include "null_blk.h"
12
13 #define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
14 #define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
15 #define SECTOR_MASK (PAGE_SECTORS - 1)
16
17 #define FREE_BATCH 16
18
19 #define TICKS_PER_SEC 50ULL
20 #define TIMER_INTERVAL (NSEC_PER_SEC / TICKS_PER_SEC)
21
22 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
23 static DECLARE_FAULT_ATTR(null_timeout_attr);
24 static DECLARE_FAULT_ATTR(null_requeue_attr);
25 #endif
26
27 static inline u64 mb_per_tick(int mbps)
28 {
29 return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
30 }
31
32 /*
33 * Status flags for nullb_device.
34 *
35 * CONFIGURED: Device has been configured and turned on. Cannot reconfigure.
36 * UP: Device is currently on and visible in userspace.
37 * THROTTLED: Device is being throttled.
38 * CACHE: Device is using a write-back cache.
39 */
40 enum nullb_device_flags {
41 NULLB_DEV_FL_CONFIGURED = 0,
42 NULLB_DEV_FL_UP = 1,
43 NULLB_DEV_FL_THROTTLED = 2,
44 NULLB_DEV_FL_CACHE = 3,
45 };
46
47 #define MAP_SZ ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
48 /*
49 * nullb_page is a page in memory for nullb devices.
50 *
51 * @page: The page holding the data.
52 * @bitmap: The bitmap represents which sector in the page has data.
53 * Each bit represents one block size. For example, sector 8
54 * will use the 7th bit
55 * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
56 * page is being flushing to storage. FREE means the cache page is freed and
57 * should be skipped from flushing to storage. Please see
58 * null_make_cache_space
59 */
60 struct nullb_page {
61 struct page *page;
62 DECLARE_BITMAP(bitmap, MAP_SZ);
63 };
64 #define NULLB_PAGE_LOCK (MAP_SZ - 1)
65 #define NULLB_PAGE_FREE (MAP_SZ - 2)
66
67 static LIST_HEAD(nullb_list);
68 static struct mutex lock;
69 static int null_major;
70 static DEFINE_IDA(nullb_indexes);
71 static struct blk_mq_tag_set tag_set;
72
73 enum {
74 NULL_IRQ_NONE = 0,
75 NULL_IRQ_SOFTIRQ = 1,
76 NULL_IRQ_TIMER = 2,
77 };
78
79 enum {
80 NULL_Q_BIO = 0,
81 NULL_Q_RQ = 1,
82 NULL_Q_MQ = 2,
83 };
84
85 static int g_no_sched;
86 module_param_named(no_sched, g_no_sched, int, 0444);
87 MODULE_PARM_DESC(no_sched, "No io scheduler");
88
89 static int g_submit_queues = 1;
90 module_param_named(submit_queues, g_submit_queues, int, 0444);
91 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
92
93 static int g_home_node = NUMA_NO_NODE;
94 module_param_named(home_node, g_home_node, int, 0444);
95 MODULE_PARM_DESC(home_node, "Home node for the device");
96
97 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
98 static char g_timeout_str[80];
99 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
100
101 static char g_requeue_str[80];
102 module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
103 #endif
104
105 static int g_queue_mode = NULL_Q_MQ;
106
107 static int null_param_store_val(const char *str, int *val, int min, int max)
108 {
109 int ret, new_val;
110
111 ret = kstrtoint(str, 10, &new_val);
112 if (ret)
113 return -EINVAL;
114
115 if (new_val < min || new_val > max)
116 return -EINVAL;
117
118 *val = new_val;
119 return 0;
120 }
121
122 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
123 {
124 return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
125 }
126
127 static const struct kernel_param_ops null_queue_mode_param_ops = {
128 .set = null_set_queue_mode,
129 .get = param_get_int,
130 };
131
132 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
133 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
134
135 static int g_gb = 250;
136 module_param_named(gb, g_gb, int, 0444);
137 MODULE_PARM_DESC(gb, "Size in GB");
138
139 static int g_bs = 512;
140 module_param_named(bs, g_bs, int, 0444);
141 MODULE_PARM_DESC(bs, "Block size (in bytes)");
142
143 static int nr_devices = 1;
144 module_param(nr_devices, int, 0444);
145 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
146
147 static bool g_blocking;
148 module_param_named(blocking, g_blocking, bool, 0444);
149 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
150
151 static bool shared_tags;
152 module_param(shared_tags, bool, 0444);
153 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
154
155 static int g_irqmode = NULL_IRQ_SOFTIRQ;
156
157 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
158 {
159 return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
160 NULL_IRQ_TIMER);
161 }
162
163 static const struct kernel_param_ops null_irqmode_param_ops = {
164 .set = null_set_irqmode,
165 .get = param_get_int,
166 };
167
168 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
169 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
170
171 static unsigned long g_completion_nsec = 10000;
172 module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
173 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
174
175 static int g_hw_queue_depth = 64;
176 module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
177 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
178
179 static bool g_use_per_node_hctx;
180 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
181 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
182
183 static bool g_zoned;
184 module_param_named(zoned, g_zoned, bool, S_IRUGO);
185 MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
186
187 static unsigned long g_zone_size = 256;
188 module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
189 MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
190
191 static struct nullb_device *null_alloc_dev(void);
192 static void null_free_dev(struct nullb_device *dev);
193 static void null_del_dev(struct nullb *nullb);
194 static int null_add_dev(struct nullb_device *dev);
195 static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
196
197 static inline struct nullb_device *to_nullb_device(struct config_item *item)
198 {
199 return item ? container_of(item, struct nullb_device, item) : NULL;
200 }
201
202 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
203 {
204 return snprintf(page, PAGE_SIZE, "%u\n", val);
205 }
206
207 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
208 char *page)
209 {
210 return snprintf(page, PAGE_SIZE, "%lu\n", val);
211 }
212
213 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
214 {
215 return snprintf(page, PAGE_SIZE, "%u\n", val);
216 }
217
218 static ssize_t nullb_device_uint_attr_store(unsigned int *val,
219 const char *page, size_t count)
220 {
221 unsigned int tmp;
222 int result;
223
224 result = kstrtouint(page, 0, &tmp);
225 if (result)
226 return result;
227
228 *val = tmp;
229 return count;
230 }
231
232 static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
233 const char *page, size_t count)
234 {
235 int result;
236 unsigned long tmp;
237
238 result = kstrtoul(page, 0, &tmp);
239 if (result)
240 return result;
241
242 *val = tmp;
243 return count;
244 }
245
246 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
247 size_t count)
248 {
249 bool tmp;
250 int result;
251
252 result = kstrtobool(page, &tmp);
253 if (result)
254 return result;
255
256 *val = tmp;
257 return count;
258 }
259
260 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
261 #define NULLB_DEVICE_ATTR(NAME, TYPE) \
262 static ssize_t \
263 nullb_device_##NAME##_show(struct config_item *item, char *page) \
264 { \
265 return nullb_device_##TYPE##_attr_show( \
266 to_nullb_device(item)->NAME, page); \
267 } \
268 static ssize_t \
269 nullb_device_##NAME##_store(struct config_item *item, const char *page, \
270 size_t count) \
271 { \
272 if (test_bit(NULLB_DEV_FL_CONFIGURED, &to_nullb_device(item)->flags)) \
273 return -EBUSY; \
274 return nullb_device_##TYPE##_attr_store( \
275 &to_nullb_device(item)->NAME, page, count); \
276 } \
277 CONFIGFS_ATTR(nullb_device_, NAME);
278
279 NULLB_DEVICE_ATTR(size, ulong);
280 NULLB_DEVICE_ATTR(completion_nsec, ulong);
281 NULLB_DEVICE_ATTR(submit_queues, uint);
282 NULLB_DEVICE_ATTR(home_node, uint);
283 NULLB_DEVICE_ATTR(queue_mode, uint);
284 NULLB_DEVICE_ATTR(blocksize, uint);
285 NULLB_DEVICE_ATTR(irqmode, uint);
286 NULLB_DEVICE_ATTR(hw_queue_depth, uint);
287 NULLB_DEVICE_ATTR(index, uint);
288 NULLB_DEVICE_ATTR(blocking, bool);
289 NULLB_DEVICE_ATTR(use_per_node_hctx, bool);
290 NULLB_DEVICE_ATTR(memory_backed, bool);
291 NULLB_DEVICE_ATTR(discard, bool);
292 NULLB_DEVICE_ATTR(mbps, uint);
293 NULLB_DEVICE_ATTR(cache_size, ulong);
294 NULLB_DEVICE_ATTR(zoned, bool);
295 NULLB_DEVICE_ATTR(zone_size, ulong);
296
297 static ssize_t nullb_device_power_show(struct config_item *item, char *page)
298 {
299 return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
300 }
301
302 static ssize_t nullb_device_power_store(struct config_item *item,
303 const char *page, size_t count)
304 {
305 struct nullb_device *dev = to_nullb_device(item);
306 bool newp = false;
307 ssize_t ret;
308
309 ret = nullb_device_bool_attr_store(&newp, page, count);
310 if (ret < 0)
311 return ret;
312
313 if (!dev->power && newp) {
314 if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
315 return count;
316 if (null_add_dev(dev)) {
317 clear_bit(NULLB_DEV_FL_UP, &dev->flags);
318 return -ENOMEM;
319 }
320
321 set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
322 dev->power = newp;
323 } else if (dev->power && !newp) {
324 mutex_lock(&lock);
325 dev->power = newp;
326 null_del_dev(dev->nullb);
327 mutex_unlock(&lock);
328 clear_bit(NULLB_DEV_FL_UP, &dev->flags);
329 clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
330 }
331
332 return count;
333 }
334
335 CONFIGFS_ATTR(nullb_device_, power);
336
337 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
338 {
339 struct nullb_device *t_dev = to_nullb_device(item);
340
341 return badblocks_show(&t_dev->badblocks, page, 0);
342 }
343
344 static ssize_t nullb_device_badblocks_store(struct config_item *item,
345 const char *page, size_t count)
346 {
347 struct nullb_device *t_dev = to_nullb_device(item);
348 char *orig, *buf, *tmp;
349 u64 start, end;
350 int ret;
351
352 orig = kstrndup(page, count, GFP_KERNEL);
353 if (!orig)
354 return -ENOMEM;
355
356 buf = strstrip(orig);
357
358 ret = -EINVAL;
359 if (buf[0] != '+' && buf[0] != '-')
360 goto out;
361 tmp = strchr(&buf[1], '-');
362 if (!tmp)
363 goto out;
364 *tmp = '\0';
365 ret = kstrtoull(buf + 1, 0, &start);
366 if (ret)
367 goto out;
368 ret = kstrtoull(tmp + 1, 0, &end);
369 if (ret)
370 goto out;
371 ret = -EINVAL;
372 if (start > end)
373 goto out;
374 /* enable badblocks */
375 cmpxchg(&t_dev->badblocks.shift, -1, 0);
376 if (buf[0] == '+')
377 ret = badblocks_set(&t_dev->badblocks, start,
378 end - start + 1, 1);
379 else
380 ret = badblocks_clear(&t_dev->badblocks, start,
381 end - start + 1);
382 if (ret == 0)
383 ret = count;
384 out:
385 kfree(orig);
386 return ret;
387 }
388 CONFIGFS_ATTR(nullb_device_, badblocks);
389
390 static struct configfs_attribute *nullb_device_attrs[] = {
391 &nullb_device_attr_size,
392 &nullb_device_attr_completion_nsec,
393 &nullb_device_attr_submit_queues,
394 &nullb_device_attr_home_node,
395 &nullb_device_attr_queue_mode,
396 &nullb_device_attr_blocksize,
397 &nullb_device_attr_irqmode,
398 &nullb_device_attr_hw_queue_depth,
399 &nullb_device_attr_index,
400 &nullb_device_attr_blocking,
401 &nullb_device_attr_use_per_node_hctx,
402 &nullb_device_attr_power,
403 &nullb_device_attr_memory_backed,
404 &nullb_device_attr_discard,
405 &nullb_device_attr_mbps,
406 &nullb_device_attr_cache_size,
407 &nullb_device_attr_badblocks,
408 &nullb_device_attr_zoned,
409 &nullb_device_attr_zone_size,
410 NULL,
411 };
412
413 static void nullb_device_release(struct config_item *item)
414 {
415 struct nullb_device *dev = to_nullb_device(item);
416
417 null_free_device_storage(dev, false);
418 null_free_dev(dev);
419 }
420
421 static struct configfs_item_operations nullb_device_ops = {
422 .release = nullb_device_release,
423 };
424
425 static const struct config_item_type nullb_device_type = {
426 .ct_item_ops = &nullb_device_ops,
427 .ct_attrs = nullb_device_attrs,
428 .ct_owner = THIS_MODULE,
429 };
430
431 static struct
432 config_item *nullb_group_make_item(struct config_group *group, const char *name)
433 {
434 struct nullb_device *dev;
435
436 dev = null_alloc_dev();
437 if (!dev)
438 return ERR_PTR(-ENOMEM);
439
440 config_item_init_type_name(&dev->item, name, &nullb_device_type);
441
442 return &dev->item;
443 }
444
445 static void
446 nullb_group_drop_item(struct config_group *group, struct config_item *item)
447 {
448 struct nullb_device *dev = to_nullb_device(item);
449
450 if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
451 mutex_lock(&lock);
452 dev->power = false;
453 null_del_dev(dev->nullb);
454 mutex_unlock(&lock);
455 }
456
457 config_item_put(item);
458 }
459
460 static ssize_t memb_group_features_show(struct config_item *item, char *page)
461 {
462 return snprintf(page, PAGE_SIZE, "memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size\n");
463 }
464
465 CONFIGFS_ATTR_RO(memb_group_, features);
466
467 static struct configfs_attribute *nullb_group_attrs[] = {
468 &memb_group_attr_features,
469 NULL,
470 };
471
472 static struct configfs_group_operations nullb_group_ops = {
473 .make_item = nullb_group_make_item,
474 .drop_item = nullb_group_drop_item,
475 };
476
477 static const struct config_item_type nullb_group_type = {
478 .ct_group_ops = &nullb_group_ops,
479 .ct_attrs = nullb_group_attrs,
480 .ct_owner = THIS_MODULE,
481 };
482
483 static struct configfs_subsystem nullb_subsys = {
484 .su_group = {
485 .cg_item = {
486 .ci_namebuf = "nullb",
487 .ci_type = &nullb_group_type,
488 },
489 },
490 };
491
492 static inline int null_cache_active(struct nullb *nullb)
493 {
494 return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
495 }
496
497 static struct nullb_device *null_alloc_dev(void)
498 {
499 struct nullb_device *dev;
500
501 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
502 if (!dev)
503 return NULL;
504 INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
505 INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
506 if (badblocks_init(&dev->badblocks, 0)) {
507 kfree(dev);
508 return NULL;
509 }
510
511 dev->size = g_gb * 1024;
512 dev->completion_nsec = g_completion_nsec;
513 dev->submit_queues = g_submit_queues;
514 dev->home_node = g_home_node;
515 dev->queue_mode = g_queue_mode;
516 dev->blocksize = g_bs;
517 dev->irqmode = g_irqmode;
518 dev->hw_queue_depth = g_hw_queue_depth;
519 dev->blocking = g_blocking;
520 dev->use_per_node_hctx = g_use_per_node_hctx;
521 dev->zoned = g_zoned;
522 dev->zone_size = g_zone_size;
523 return dev;
524 }
525
526 static void null_free_dev(struct nullb_device *dev)
527 {
528 if (!dev)
529 return;
530
531 null_zone_exit(dev);
532 badblocks_exit(&dev->badblocks);
533 kfree(dev);
534 }
535
536 static void put_tag(struct nullb_queue *nq, unsigned int tag)
537 {
538 clear_bit_unlock(tag, nq->tag_map);
539
540 if (waitqueue_active(&nq->wait))
541 wake_up(&nq->wait);
542 }
543
544 static unsigned int get_tag(struct nullb_queue *nq)
545 {
546 unsigned int tag;
547
548 do {
549 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
550 if (tag >= nq->queue_depth)
551 return -1U;
552 } while (test_and_set_bit_lock(tag, nq->tag_map));
553
554 return tag;
555 }
556
557 static void free_cmd(struct nullb_cmd *cmd)
558 {
559 put_tag(cmd->nq, cmd->tag);
560 }
561
562 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
563
564 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
565 {
566 struct nullb_cmd *cmd;
567 unsigned int tag;
568
569 tag = get_tag(nq);
570 if (tag != -1U) {
571 cmd = &nq->cmds[tag];
572 cmd->tag = tag;
573 cmd->nq = nq;
574 if (nq->dev->irqmode == NULL_IRQ_TIMER) {
575 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
576 HRTIMER_MODE_REL);
577 cmd->timer.function = null_cmd_timer_expired;
578 }
579 return cmd;
580 }
581
582 return NULL;
583 }
584
585 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
586 {
587 struct nullb_cmd *cmd;
588 DEFINE_WAIT(wait);
589
590 cmd = __alloc_cmd(nq);
591 if (cmd || !can_wait)
592 return cmd;
593
594 do {
595 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
596 cmd = __alloc_cmd(nq);
597 if (cmd)
598 break;
599
600 io_schedule();
601 } while (1);
602
603 finish_wait(&nq->wait, &wait);
604 return cmd;
605 }
606
607 static void end_cmd(struct nullb_cmd *cmd)
608 {
609 struct request_queue *q = NULL;
610 int queue_mode = cmd->nq->dev->queue_mode;
611
612 if (cmd->rq)
613 q = cmd->rq->q;
614
615 switch (queue_mode) {
616 case NULL_Q_MQ:
617 blk_mq_end_request(cmd->rq, cmd->error);
618 return;
619 case NULL_Q_RQ:
620 INIT_LIST_HEAD(&cmd->rq->queuelist);
621 blk_end_request_all(cmd->rq, cmd->error);
622 break;
623 case NULL_Q_BIO:
624 cmd->bio->bi_status = cmd->error;
625 bio_endio(cmd->bio);
626 break;
627 }
628
629 free_cmd(cmd);
630
631 /* Restart queue if needed, as we are freeing a tag */
632 if (queue_mode == NULL_Q_RQ && blk_queue_stopped(q)) {
633 unsigned long flags;
634
635 spin_lock_irqsave(q->queue_lock, flags);
636 blk_start_queue_async(q);
637 spin_unlock_irqrestore(q->queue_lock, flags);
638 }
639 }
640
641 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
642 {
643 end_cmd(container_of(timer, struct nullb_cmd, timer));
644
645 return HRTIMER_NORESTART;
646 }
647
648 static void null_cmd_end_timer(struct nullb_cmd *cmd)
649 {
650 ktime_t kt = cmd->nq->dev->completion_nsec;
651
652 hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
653 }
654
655 static void null_softirq_done_fn(struct request *rq)
656 {
657 struct nullb *nullb = rq->q->queuedata;
658
659 if (nullb->dev->queue_mode == NULL_Q_MQ)
660 end_cmd(blk_mq_rq_to_pdu(rq));
661 else
662 end_cmd(rq->special);
663 }
664
665 static struct nullb_page *null_alloc_page(gfp_t gfp_flags)
666 {
667 struct nullb_page *t_page;
668
669 t_page = kmalloc(sizeof(struct nullb_page), gfp_flags);
670 if (!t_page)
671 goto out;
672
673 t_page->page = alloc_pages(gfp_flags, 0);
674 if (!t_page->page)
675 goto out_freepage;
676
677 memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
678 return t_page;
679 out_freepage:
680 kfree(t_page);
681 out:
682 return NULL;
683 }
684
685 static void null_free_page(struct nullb_page *t_page)
686 {
687 __set_bit(NULLB_PAGE_FREE, t_page->bitmap);
688 if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
689 return;
690 __free_page(t_page->page);
691 kfree(t_page);
692 }
693
694 static bool null_page_empty(struct nullb_page *page)
695 {
696 int size = MAP_SZ - 2;
697
698 return find_first_bit(page->bitmap, size) == size;
699 }
700
701 static void null_free_sector(struct nullb *nullb, sector_t sector,
702 bool is_cache)
703 {
704 unsigned int sector_bit;
705 u64 idx;
706 struct nullb_page *t_page, *ret;
707 struct radix_tree_root *root;
708
709 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
710 idx = sector >> PAGE_SECTORS_SHIFT;
711 sector_bit = (sector & SECTOR_MASK);
712
713 t_page = radix_tree_lookup(root, idx);
714 if (t_page) {
715 __clear_bit(sector_bit, t_page->bitmap);
716
717 if (null_page_empty(t_page)) {
718 ret = radix_tree_delete_item(root, idx, t_page);
719 WARN_ON(ret != t_page);
720 null_free_page(ret);
721 if (is_cache)
722 nullb->dev->curr_cache -= PAGE_SIZE;
723 }
724 }
725 }
726
727 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
728 struct nullb_page *t_page, bool is_cache)
729 {
730 struct radix_tree_root *root;
731
732 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
733
734 if (radix_tree_insert(root, idx, t_page)) {
735 null_free_page(t_page);
736 t_page = radix_tree_lookup(root, idx);
737 WARN_ON(!t_page || t_page->page->index != idx);
738 } else if (is_cache)
739 nullb->dev->curr_cache += PAGE_SIZE;
740
741 return t_page;
742 }
743
744 static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
745 {
746 unsigned long pos = 0;
747 int nr_pages;
748 struct nullb_page *ret, *t_pages[FREE_BATCH];
749 struct radix_tree_root *root;
750
751 root = is_cache ? &dev->cache : &dev->data;
752
753 do {
754 int i;
755
756 nr_pages = radix_tree_gang_lookup(root,
757 (void **)t_pages, pos, FREE_BATCH);
758
759 for (i = 0; i < nr_pages; i++) {
760 pos = t_pages[i]->page->index;
761 ret = radix_tree_delete_item(root, pos, t_pages[i]);
762 WARN_ON(ret != t_pages[i]);
763 null_free_page(ret);
764 }
765
766 pos++;
767 } while (nr_pages == FREE_BATCH);
768
769 if (is_cache)
770 dev->curr_cache = 0;
771 }
772
773 static struct nullb_page *__null_lookup_page(struct nullb *nullb,
774 sector_t sector, bool for_write, bool is_cache)
775 {
776 unsigned int sector_bit;
777 u64 idx;
778 struct nullb_page *t_page;
779 struct radix_tree_root *root;
780
781 idx = sector >> PAGE_SECTORS_SHIFT;
782 sector_bit = (sector & SECTOR_MASK);
783
784 root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
785 t_page = radix_tree_lookup(root, idx);
786 WARN_ON(t_page && t_page->page->index != idx);
787
788 if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
789 return t_page;
790
791 return NULL;
792 }
793
794 static struct nullb_page *null_lookup_page(struct nullb *nullb,
795 sector_t sector, bool for_write, bool ignore_cache)
796 {
797 struct nullb_page *page = NULL;
798
799 if (!ignore_cache)
800 page = __null_lookup_page(nullb, sector, for_write, true);
801 if (page)
802 return page;
803 return __null_lookup_page(nullb, sector, for_write, false);
804 }
805
806 static struct nullb_page *null_insert_page(struct nullb *nullb,
807 sector_t sector, bool ignore_cache)
808 __releases(&nullb->lock)
809 __acquires(&nullb->lock)
810 {
811 u64 idx;
812 struct nullb_page *t_page;
813
814 t_page = null_lookup_page(nullb, sector, true, ignore_cache);
815 if (t_page)
816 return t_page;
817
818 spin_unlock_irq(&nullb->lock);
819
820 t_page = null_alloc_page(GFP_NOIO);
821 if (!t_page)
822 goto out_lock;
823
824 if (radix_tree_preload(GFP_NOIO))
825 goto out_freepage;
826
827 spin_lock_irq(&nullb->lock);
828 idx = sector >> PAGE_SECTORS_SHIFT;
829 t_page->page->index = idx;
830 t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
831 radix_tree_preload_end();
832
833 return t_page;
834 out_freepage:
835 null_free_page(t_page);
836 out_lock:
837 spin_lock_irq(&nullb->lock);
838 return null_lookup_page(nullb, sector, true, ignore_cache);
839 }
840
841 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
842 {
843 int i;
844 unsigned int offset;
845 u64 idx;
846 struct nullb_page *t_page, *ret;
847 void *dst, *src;
848
849 idx = c_page->page->index;
850
851 t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
852
853 __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
854 if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
855 null_free_page(c_page);
856 if (t_page && null_page_empty(t_page)) {
857 ret = radix_tree_delete_item(&nullb->dev->data,
858 idx, t_page);
859 null_free_page(t_page);
860 }
861 return 0;
862 }
863
864 if (!t_page)
865 return -ENOMEM;
866
867 src = kmap_atomic(c_page->page);
868 dst = kmap_atomic(t_page->page);
869
870 for (i = 0; i < PAGE_SECTORS;
871 i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
872 if (test_bit(i, c_page->bitmap)) {
873 offset = (i << SECTOR_SHIFT);
874 memcpy(dst + offset, src + offset,
875 nullb->dev->blocksize);
876 __set_bit(i, t_page->bitmap);
877 }
878 }
879
880 kunmap_atomic(dst);
881 kunmap_atomic(src);
882
883 ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
884 null_free_page(ret);
885 nullb->dev->curr_cache -= PAGE_SIZE;
886
887 return 0;
888 }
889
890 static int null_make_cache_space(struct nullb *nullb, unsigned long n)
891 {
892 int i, err, nr_pages;
893 struct nullb_page *c_pages[FREE_BATCH];
894 unsigned long flushed = 0, one_round;
895
896 again:
897 if ((nullb->dev->cache_size * 1024 * 1024) >
898 nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
899 return 0;
900
901 nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
902 (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
903 /*
904 * nullb_flush_cache_page could unlock before using the c_pages. To
905 * avoid race, we don't allow page free
906 */
907 for (i = 0; i < nr_pages; i++) {
908 nullb->cache_flush_pos = c_pages[i]->page->index;
909 /*
910 * We found the page which is being flushed to disk by other
911 * threads
912 */
913 if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
914 c_pages[i] = NULL;
915 else
916 __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
917 }
918
919 one_round = 0;
920 for (i = 0; i < nr_pages; i++) {
921 if (c_pages[i] == NULL)
922 continue;
923 err = null_flush_cache_page(nullb, c_pages[i]);
924 if (err)
925 return err;
926 one_round++;
927 }
928 flushed += one_round << PAGE_SHIFT;
929
930 if (n > flushed) {
931 if (nr_pages == 0)
932 nullb->cache_flush_pos = 0;
933 if (one_round == 0) {
934 /* give other threads a chance */
935 spin_unlock_irq(&nullb->lock);
936 spin_lock_irq(&nullb->lock);
937 }
938 goto again;
939 }
940 return 0;
941 }
942
943 static int copy_to_nullb(struct nullb *nullb, struct page *source,
944 unsigned int off, sector_t sector, size_t n, bool is_fua)
945 {
946 size_t temp, count = 0;
947 unsigned int offset;
948 struct nullb_page *t_page;
949 void *dst, *src;
950
951 while (count < n) {
952 temp = min_t(size_t, nullb->dev->blocksize, n - count);
953
954 if (null_cache_active(nullb) && !is_fua)
955 null_make_cache_space(nullb, PAGE_SIZE);
956
957 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
958 t_page = null_insert_page(nullb, sector,
959 !null_cache_active(nullb) || is_fua);
960 if (!t_page)
961 return -ENOSPC;
962
963 src = kmap_atomic(source);
964 dst = kmap_atomic(t_page->page);
965 memcpy(dst + offset, src + off + count, temp);
966 kunmap_atomic(dst);
967 kunmap_atomic(src);
968
969 __set_bit(sector & SECTOR_MASK, t_page->bitmap);
970
971 if (is_fua)
972 null_free_sector(nullb, sector, true);
973
974 count += temp;
975 sector += temp >> SECTOR_SHIFT;
976 }
977 return 0;
978 }
979
980 static int copy_from_nullb(struct nullb *nullb, struct page *dest,
981 unsigned int off, sector_t sector, size_t n)
982 {
983 size_t temp, count = 0;
984 unsigned int offset;
985 struct nullb_page *t_page;
986 void *dst, *src;
987
988 while (count < n) {
989 temp = min_t(size_t, nullb->dev->blocksize, n - count);
990
991 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
992 t_page = null_lookup_page(nullb, sector, false,
993 !null_cache_active(nullb));
994
995 dst = kmap_atomic(dest);
996 if (!t_page) {
997 memset(dst + off + count, 0, temp);
998 goto next;
999 }
1000 src = kmap_atomic(t_page->page);
1001 memcpy(dst + off + count, src + offset, temp);
1002 kunmap_atomic(src);
1003 next:
1004 kunmap_atomic(dst);
1005
1006 count += temp;
1007 sector += temp >> SECTOR_SHIFT;
1008 }
1009 return 0;
1010 }
1011
1012 static void null_handle_discard(struct nullb *nullb, sector_t sector, size_t n)
1013 {
1014 size_t temp;
1015
1016 spin_lock_irq(&nullb->lock);
1017 while (n > 0) {
1018 temp = min_t(size_t, n, nullb->dev->blocksize);
1019 null_free_sector(nullb, sector, false);
1020 if (null_cache_active(nullb))
1021 null_free_sector(nullb, sector, true);
1022 sector += temp >> SECTOR_SHIFT;
1023 n -= temp;
1024 }
1025 spin_unlock_irq(&nullb->lock);
1026 }
1027
1028 static int null_handle_flush(struct nullb *nullb)
1029 {
1030 int err;
1031
1032 if (!null_cache_active(nullb))
1033 return 0;
1034
1035 spin_lock_irq(&nullb->lock);
1036 while (true) {
1037 err = null_make_cache_space(nullb,
1038 nullb->dev->cache_size * 1024 * 1024);
1039 if (err || nullb->dev->curr_cache == 0)
1040 break;
1041 }
1042
1043 WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1044 spin_unlock_irq(&nullb->lock);
1045 return err;
1046 }
1047
1048 static int null_transfer(struct nullb *nullb, struct page *page,
1049 unsigned int len, unsigned int off, bool is_write, sector_t sector,
1050 bool is_fua)
1051 {
1052 int err = 0;
1053
1054 if (!is_write) {
1055 err = copy_from_nullb(nullb, page, off, sector, len);
1056 flush_dcache_page(page);
1057 } else {
1058 flush_dcache_page(page);
1059 err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
1060 }
1061
1062 return err;
1063 }
1064
1065 static int null_handle_rq(struct nullb_cmd *cmd)
1066 {
1067 struct request *rq = cmd->rq;
1068 struct nullb *nullb = cmd->nq->dev->nullb;
1069 int err;
1070 unsigned int len;
1071 sector_t sector;
1072 struct req_iterator iter;
1073 struct bio_vec bvec;
1074
1075 sector = blk_rq_pos(rq);
1076
1077 if (req_op(rq) == REQ_OP_DISCARD) {
1078 null_handle_discard(nullb, sector, blk_rq_bytes(rq));
1079 return 0;
1080 }
1081
1082 spin_lock_irq(&nullb->lock);
1083 rq_for_each_segment(bvec, rq, iter) {
1084 len = bvec.bv_len;
1085 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1086 op_is_write(req_op(rq)), sector,
1087 req_op(rq) & REQ_FUA);
1088 if (err) {
1089 spin_unlock_irq(&nullb->lock);
1090 return err;
1091 }
1092 sector += len >> SECTOR_SHIFT;
1093 }
1094 spin_unlock_irq(&nullb->lock);
1095
1096 return 0;
1097 }
1098
1099 static int null_handle_bio(struct nullb_cmd *cmd)
1100 {
1101 struct bio *bio = cmd->bio;
1102 struct nullb *nullb = cmd->nq->dev->nullb;
1103 int err;
1104 unsigned int len;
1105 sector_t sector;
1106 struct bio_vec bvec;
1107 struct bvec_iter iter;
1108
1109 sector = bio->bi_iter.bi_sector;
1110
1111 if (bio_op(bio) == REQ_OP_DISCARD) {
1112 null_handle_discard(nullb, sector,
1113 bio_sectors(bio) << SECTOR_SHIFT);
1114 return 0;
1115 }
1116
1117 spin_lock_irq(&nullb->lock);
1118 bio_for_each_segment(bvec, bio, iter) {
1119 len = bvec.bv_len;
1120 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1121 op_is_write(bio_op(bio)), sector,
1122 bio_op(bio) & REQ_FUA);
1123 if (err) {
1124 spin_unlock_irq(&nullb->lock);
1125 return err;
1126 }
1127 sector += len >> SECTOR_SHIFT;
1128 }
1129 spin_unlock_irq(&nullb->lock);
1130 return 0;
1131 }
1132
1133 static void null_stop_queue(struct nullb *nullb)
1134 {
1135 struct request_queue *q = nullb->q;
1136
1137 if (nullb->dev->queue_mode == NULL_Q_MQ)
1138 blk_mq_stop_hw_queues(q);
1139 else {
1140 spin_lock_irq(q->queue_lock);
1141 blk_stop_queue(q);
1142 spin_unlock_irq(q->queue_lock);
1143 }
1144 }
1145
1146 static void null_restart_queue_async(struct nullb *nullb)
1147 {
1148 struct request_queue *q = nullb->q;
1149 unsigned long flags;
1150
1151 if (nullb->dev->queue_mode == NULL_Q_MQ)
1152 blk_mq_start_stopped_hw_queues(q, true);
1153 else {
1154 spin_lock_irqsave(q->queue_lock, flags);
1155 blk_start_queue_async(q);
1156 spin_unlock_irqrestore(q->queue_lock, flags);
1157 }
1158 }
1159
1160 static bool cmd_report_zone(struct nullb *nullb, struct nullb_cmd *cmd)
1161 {
1162 struct nullb_device *dev = cmd->nq->dev;
1163
1164 if (dev->queue_mode == NULL_Q_BIO) {
1165 if (bio_op(cmd->bio) == REQ_OP_ZONE_REPORT) {
1166 cmd->error = null_zone_report(nullb, cmd->bio);
1167 return true;
1168 }
1169 } else {
1170 if (req_op(cmd->rq) == REQ_OP_ZONE_REPORT) {
1171 cmd->error = null_zone_report(nullb, cmd->rq->bio);
1172 return true;
1173 }
1174 }
1175
1176 return false;
1177 }
1178
1179 static blk_status_t null_handle_cmd(struct nullb_cmd *cmd)
1180 {
1181 struct nullb_device *dev = cmd->nq->dev;
1182 struct nullb *nullb = dev->nullb;
1183 int err = 0;
1184
1185 if (cmd_report_zone(nullb, cmd))
1186 goto out;
1187
1188 if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
1189 struct request *rq = cmd->rq;
1190
1191 if (!hrtimer_active(&nullb->bw_timer))
1192 hrtimer_restart(&nullb->bw_timer);
1193
1194 if (atomic_long_sub_return(blk_rq_bytes(rq),
1195 &nullb->cur_bytes) < 0) {
1196 null_stop_queue(nullb);
1197 /* race with timer */
1198 if (atomic_long_read(&nullb->cur_bytes) > 0)
1199 null_restart_queue_async(nullb);
1200 if (dev->queue_mode == NULL_Q_RQ) {
1201 struct request_queue *q = nullb->q;
1202
1203 spin_lock_irq(q->queue_lock);
1204 rq->rq_flags |= RQF_DONTPREP;
1205 blk_requeue_request(q, rq);
1206 spin_unlock_irq(q->queue_lock);
1207 return BLK_STS_OK;
1208 } else
1209 /* requeue request */
1210 return BLK_STS_DEV_RESOURCE;
1211 }
1212 }
1213
1214 if (nullb->dev->badblocks.shift != -1) {
1215 int bad_sectors;
1216 sector_t sector, size, first_bad;
1217 bool is_flush = true;
1218
1219 if (dev->queue_mode == NULL_Q_BIO &&
1220 bio_op(cmd->bio) != REQ_OP_FLUSH) {
1221 is_flush = false;
1222 sector = cmd->bio->bi_iter.bi_sector;
1223 size = bio_sectors(cmd->bio);
1224 }
1225 if (dev->queue_mode != NULL_Q_BIO &&
1226 req_op(cmd->rq) != REQ_OP_FLUSH) {
1227 is_flush = false;
1228 sector = blk_rq_pos(cmd->rq);
1229 size = blk_rq_sectors(cmd->rq);
1230 }
1231 if (!is_flush && badblocks_check(&nullb->dev->badblocks, sector,
1232 size, &first_bad, &bad_sectors)) {
1233 cmd->error = BLK_STS_IOERR;
1234 goto out;
1235 }
1236 }
1237
1238 if (dev->memory_backed) {
1239 if (dev->queue_mode == NULL_Q_BIO) {
1240 if (bio_op(cmd->bio) == REQ_OP_FLUSH)
1241 err = null_handle_flush(nullb);
1242 else
1243 err = null_handle_bio(cmd);
1244 } else {
1245 if (req_op(cmd->rq) == REQ_OP_FLUSH)
1246 err = null_handle_flush(nullb);
1247 else
1248 err = null_handle_rq(cmd);
1249 }
1250 }
1251 cmd->error = errno_to_blk_status(err);
1252
1253 if (!cmd->error && dev->zoned) {
1254 sector_t sector;
1255 unsigned int nr_sectors;
1256 int op;
1257
1258 if (dev->queue_mode == NULL_Q_BIO) {
1259 op = bio_op(cmd->bio);
1260 sector = cmd->bio->bi_iter.bi_sector;
1261 nr_sectors = cmd->bio->bi_iter.bi_size >> 9;
1262 } else {
1263 op = req_op(cmd->rq);
1264 sector = blk_rq_pos(cmd->rq);
1265 nr_sectors = blk_rq_sectors(cmd->rq);
1266 }
1267
1268 if (op == REQ_OP_WRITE)
1269 null_zone_write(cmd, sector, nr_sectors);
1270 else if (op == REQ_OP_ZONE_RESET)
1271 null_zone_reset(cmd, sector);
1272 }
1273 out:
1274 /* Complete IO by inline, softirq or timer */
1275 switch (dev->irqmode) {
1276 case NULL_IRQ_SOFTIRQ:
1277 switch (dev->queue_mode) {
1278 case NULL_Q_MQ:
1279 blk_mq_complete_request(cmd->rq);
1280 break;
1281 case NULL_Q_RQ:
1282 blk_complete_request(cmd->rq);
1283 break;
1284 case NULL_Q_BIO:
1285 /*
1286 * XXX: no proper submitting cpu information available.
1287 */
1288 end_cmd(cmd);
1289 break;
1290 }
1291 break;
1292 case NULL_IRQ_NONE:
1293 end_cmd(cmd);
1294 break;
1295 case NULL_IRQ_TIMER:
1296 null_cmd_end_timer(cmd);
1297 break;
1298 }
1299 return BLK_STS_OK;
1300 }
1301
1302 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1303 {
1304 struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1305 ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1306 unsigned int mbps = nullb->dev->mbps;
1307
1308 if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1309 return HRTIMER_NORESTART;
1310
1311 atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1312 null_restart_queue_async(nullb);
1313
1314 hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1315
1316 return HRTIMER_RESTART;
1317 }
1318
1319 static void nullb_setup_bwtimer(struct nullb *nullb)
1320 {
1321 ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1322
1323 hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1324 nullb->bw_timer.function = nullb_bwtimer_fn;
1325 atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1326 hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1327 }
1328
1329 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
1330 {
1331 int index = 0;
1332
1333 if (nullb->nr_queues != 1)
1334 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
1335
1336 return &nullb->queues[index];
1337 }
1338
1339 static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
1340 {
1341 struct nullb *nullb = q->queuedata;
1342 struct nullb_queue *nq = nullb_to_queue(nullb);
1343 struct nullb_cmd *cmd;
1344
1345 cmd = alloc_cmd(nq, 1);
1346 cmd->bio = bio;
1347
1348 null_handle_cmd(cmd);
1349 return BLK_QC_T_NONE;
1350 }
1351
1352 static enum blk_eh_timer_return null_rq_timed_out_fn(struct request *rq)
1353 {
1354 pr_info("null: rq %p timed out\n", rq);
1355 __blk_complete_request(rq);
1356 return BLK_EH_DONE;
1357 }
1358
1359 static int null_rq_prep_fn(struct request_queue *q, struct request *req)
1360 {
1361 struct nullb *nullb = q->queuedata;
1362 struct nullb_queue *nq = nullb_to_queue(nullb);
1363 struct nullb_cmd *cmd;
1364
1365 cmd = alloc_cmd(nq, 0);
1366 if (cmd) {
1367 cmd->rq = req;
1368 req->special = cmd;
1369 return BLKPREP_OK;
1370 }
1371 blk_stop_queue(q);
1372
1373 return BLKPREP_DEFER;
1374 }
1375
1376 static bool should_timeout_request(struct request *rq)
1377 {
1378 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1379 if (g_timeout_str[0])
1380 return should_fail(&null_timeout_attr, 1);
1381 #endif
1382 return false;
1383 }
1384
1385 static bool should_requeue_request(struct request *rq)
1386 {
1387 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1388 if (g_requeue_str[0])
1389 return should_fail(&null_requeue_attr, 1);
1390 #endif
1391 return false;
1392 }
1393
1394 static void null_request_fn(struct request_queue *q)
1395 {
1396 struct request *rq;
1397
1398 while ((rq = blk_fetch_request(q)) != NULL) {
1399 struct nullb_cmd *cmd = rq->special;
1400
1401 /* just ignore the request */
1402 if (should_timeout_request(rq))
1403 continue;
1404 if (should_requeue_request(rq)) {
1405 blk_requeue_request(q, rq);
1406 continue;
1407 }
1408
1409 spin_unlock_irq(q->queue_lock);
1410 null_handle_cmd(cmd);
1411 spin_lock_irq(q->queue_lock);
1412 }
1413 }
1414
1415 static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
1416 {
1417 pr_info("null: rq %p timed out\n", rq);
1418 blk_mq_complete_request(rq);
1419 return BLK_EH_DONE;
1420 }
1421
1422 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1423 const struct blk_mq_queue_data *bd)
1424 {
1425 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
1426 struct nullb_queue *nq = hctx->driver_data;
1427
1428 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1429
1430 if (nq->dev->irqmode == NULL_IRQ_TIMER) {
1431 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1432 cmd->timer.function = null_cmd_timer_expired;
1433 }
1434 cmd->rq = bd->rq;
1435 cmd->nq = nq;
1436
1437 blk_mq_start_request(bd->rq);
1438
1439 if (should_requeue_request(bd->rq)) {
1440 /*
1441 * Alternate between hitting the core BUSY path, and the
1442 * driver driven requeue path
1443 */
1444 nq->requeue_selection++;
1445 if (nq->requeue_selection & 1)
1446 return BLK_STS_RESOURCE;
1447 else {
1448 blk_mq_requeue_request(bd->rq, true);
1449 return BLK_STS_OK;
1450 }
1451 }
1452 if (should_timeout_request(bd->rq))
1453 return BLK_STS_OK;
1454
1455 return null_handle_cmd(cmd);
1456 }
1457
1458 static const struct blk_mq_ops null_mq_ops = {
1459 .queue_rq = null_queue_rq,
1460 .complete = null_softirq_done_fn,
1461 .timeout = null_timeout_rq,
1462 };
1463
1464 static void cleanup_queue(struct nullb_queue *nq)
1465 {
1466 kfree(nq->tag_map);
1467 kfree(nq->cmds);
1468 }
1469
1470 static void cleanup_queues(struct nullb *nullb)
1471 {
1472 int i;
1473
1474 for (i = 0; i < nullb->nr_queues; i++)
1475 cleanup_queue(&nullb->queues[i]);
1476
1477 kfree(nullb->queues);
1478 }
1479
1480 static void null_del_dev(struct nullb *nullb)
1481 {
1482 struct nullb_device *dev = nullb->dev;
1483
1484 ida_simple_remove(&nullb_indexes, nullb->index);
1485
1486 list_del_init(&nullb->list);
1487
1488 del_gendisk(nullb->disk);
1489
1490 if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1491 hrtimer_cancel(&nullb->bw_timer);
1492 atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1493 null_restart_queue_async(nullb);
1494 }
1495
1496 blk_cleanup_queue(nullb->q);
1497 if (dev->queue_mode == NULL_Q_MQ &&
1498 nullb->tag_set == &nullb->__tag_set)
1499 blk_mq_free_tag_set(nullb->tag_set);
1500 put_disk(nullb->disk);
1501 cleanup_queues(nullb);
1502 if (null_cache_active(nullb))
1503 null_free_device_storage(nullb->dev, true);
1504 kfree(nullb);
1505 dev->nullb = NULL;
1506 }
1507
1508 static void null_config_discard(struct nullb *nullb)
1509 {
1510 if (nullb->dev->discard == false)
1511 return;
1512 nullb->q->limits.discard_granularity = nullb->dev->blocksize;
1513 nullb->q->limits.discard_alignment = nullb->dev->blocksize;
1514 blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
1515 blk_queue_flag_set(QUEUE_FLAG_DISCARD, nullb->q);
1516 }
1517
1518 static int null_open(struct block_device *bdev, fmode_t mode)
1519 {
1520 return 0;
1521 }
1522
1523 static void null_release(struct gendisk *disk, fmode_t mode)
1524 {
1525 }
1526
1527 static const struct block_device_operations null_fops = {
1528 .owner = THIS_MODULE,
1529 .open = null_open,
1530 .release = null_release,
1531 };
1532
1533 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1534 {
1535 BUG_ON(!nullb);
1536 BUG_ON(!nq);
1537
1538 init_waitqueue_head(&nq->wait);
1539 nq->queue_depth = nullb->queue_depth;
1540 nq->dev = nullb->dev;
1541 }
1542
1543 static void null_init_queues(struct nullb *nullb)
1544 {
1545 struct request_queue *q = nullb->q;
1546 struct blk_mq_hw_ctx *hctx;
1547 struct nullb_queue *nq;
1548 int i;
1549
1550 queue_for_each_hw_ctx(q, hctx, i) {
1551 if (!hctx->nr_ctx || !hctx->tags)
1552 continue;
1553 nq = &nullb->queues[i];
1554 hctx->driver_data = nq;
1555 null_init_queue(nullb, nq);
1556 nullb->nr_queues++;
1557 }
1558 }
1559
1560 static int setup_commands(struct nullb_queue *nq)
1561 {
1562 struct nullb_cmd *cmd;
1563 int i, tag_size;
1564
1565 nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);
1566 if (!nq->cmds)
1567 return -ENOMEM;
1568
1569 tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
1570 nq->tag_map = kcalloc(tag_size, sizeof(unsigned long), GFP_KERNEL);
1571 if (!nq->tag_map) {
1572 kfree(nq->cmds);
1573 return -ENOMEM;
1574 }
1575
1576 for (i = 0; i < nq->queue_depth; i++) {
1577 cmd = &nq->cmds[i];
1578 INIT_LIST_HEAD(&cmd->list);
1579 cmd->ll_list.next = NULL;
1580 cmd->tag = -1U;
1581 }
1582
1583 return 0;
1584 }
1585
1586 static int setup_queues(struct nullb *nullb)
1587 {
1588 nullb->queues = kcalloc(nullb->dev->submit_queues,
1589 sizeof(struct nullb_queue),
1590 GFP_KERNEL);
1591 if (!nullb->queues)
1592 return -ENOMEM;
1593
1594 nullb->nr_queues = 0;
1595 nullb->queue_depth = nullb->dev->hw_queue_depth;
1596
1597 return 0;
1598 }
1599
1600 static int init_driver_queues(struct nullb *nullb)
1601 {
1602 struct nullb_queue *nq;
1603 int i, ret = 0;
1604
1605 for (i = 0; i < nullb->dev->submit_queues; i++) {
1606 nq = &nullb->queues[i];
1607
1608 null_init_queue(nullb, nq);
1609
1610 ret = setup_commands(nq);
1611 if (ret)
1612 return ret;
1613 nullb->nr_queues++;
1614 }
1615 return 0;
1616 }
1617
1618 static int null_gendisk_register(struct nullb *nullb)
1619 {
1620 struct gendisk *disk;
1621 sector_t size;
1622
1623 disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
1624 if (!disk)
1625 return -ENOMEM;
1626 size = (sector_t)nullb->dev->size * 1024 * 1024ULL;
1627 set_capacity(disk, size >> 9);
1628
1629 disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
1630 disk->major = null_major;
1631 disk->first_minor = nullb->index;
1632 disk->fops = &null_fops;
1633 disk->private_data = nullb;
1634 disk->queue = nullb->q;
1635 strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1636
1637 add_disk(disk);
1638 return 0;
1639 }
1640
1641 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
1642 {
1643 set->ops = &null_mq_ops;
1644 set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
1645 g_submit_queues;
1646 set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
1647 g_hw_queue_depth;
1648 set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
1649 set->cmd_size = sizeof(struct nullb_cmd);
1650 set->flags = BLK_MQ_F_SHOULD_MERGE;
1651 if (g_no_sched)
1652 set->flags |= BLK_MQ_F_NO_SCHED;
1653 set->driver_data = NULL;
1654
1655 if ((nullb && nullb->dev->blocking) || g_blocking)
1656 set->flags |= BLK_MQ_F_BLOCKING;
1657
1658 return blk_mq_alloc_tag_set(set);
1659 }
1660
1661 static void null_validate_conf(struct nullb_device *dev)
1662 {
1663 dev->blocksize = round_down(dev->blocksize, 512);
1664 dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
1665
1666 if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
1667 if (dev->submit_queues != nr_online_nodes)
1668 dev->submit_queues = nr_online_nodes;
1669 } else if (dev->submit_queues > nr_cpu_ids)
1670 dev->submit_queues = nr_cpu_ids;
1671 else if (dev->submit_queues == 0)
1672 dev->submit_queues = 1;
1673
1674 dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
1675 dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1676
1677 /* Do memory allocation, so set blocking */
1678 if (dev->memory_backed)
1679 dev->blocking = true;
1680 else /* cache is meaningless */
1681 dev->cache_size = 0;
1682 dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1683 dev->cache_size);
1684 dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1685 /* can not stop a queue */
1686 if (dev->queue_mode == NULL_Q_BIO)
1687 dev->mbps = 0;
1688 }
1689
1690 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1691 static bool __null_setup_fault(struct fault_attr *attr, char *str)
1692 {
1693 if (!str[0])
1694 return true;
1695
1696 if (!setup_fault_attr(attr, str))
1697 return false;
1698
1699 attr->verbose = 0;
1700 return true;
1701 }
1702 #endif
1703
1704 static bool null_setup_fault(void)
1705 {
1706 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1707 if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
1708 return false;
1709 if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
1710 return false;
1711 #endif
1712 return true;
1713 }
1714
1715 static int null_add_dev(struct nullb_device *dev)
1716 {
1717 struct nullb *nullb;
1718 int rv;
1719
1720 null_validate_conf(dev);
1721
1722 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1723 if (!nullb) {
1724 rv = -ENOMEM;
1725 goto out;
1726 }
1727 nullb->dev = dev;
1728 dev->nullb = nullb;
1729
1730 spin_lock_init(&nullb->lock);
1731
1732 rv = setup_queues(nullb);
1733 if (rv)
1734 goto out_free_nullb;
1735
1736 if (dev->queue_mode == NULL_Q_MQ) {
1737 if (shared_tags) {
1738 nullb->tag_set = &tag_set;
1739 rv = 0;
1740 } else {
1741 nullb->tag_set = &nullb->__tag_set;
1742 rv = null_init_tag_set(nullb, nullb->tag_set);
1743 }
1744
1745 if (rv)
1746 goto out_cleanup_queues;
1747
1748 if (!null_setup_fault())
1749 goto out_cleanup_queues;
1750
1751 nullb->tag_set->timeout = 5 * HZ;
1752 nullb->q = blk_mq_init_queue(nullb->tag_set);
1753 if (IS_ERR(nullb->q)) {
1754 rv = -ENOMEM;
1755 goto out_cleanup_tags;
1756 }
1757 null_init_queues(nullb);
1758 } else if (dev->queue_mode == NULL_Q_BIO) {
1759 nullb->q = blk_alloc_queue_node(GFP_KERNEL, dev->home_node,
1760 NULL);
1761 if (!nullb->q) {
1762 rv = -ENOMEM;
1763 goto out_cleanup_queues;
1764 }
1765 blk_queue_make_request(nullb->q, null_queue_bio);
1766 rv = init_driver_queues(nullb);
1767 if (rv)
1768 goto out_cleanup_blk_queue;
1769 } else {
1770 nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock,
1771 dev->home_node);
1772 if (!nullb->q) {
1773 rv = -ENOMEM;
1774 goto out_cleanup_queues;
1775 }
1776
1777 if (!null_setup_fault())
1778 goto out_cleanup_blk_queue;
1779
1780 blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
1781 blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
1782 blk_queue_rq_timed_out(nullb->q, null_rq_timed_out_fn);
1783 nullb->q->rq_timeout = 5 * HZ;
1784 rv = init_driver_queues(nullb);
1785 if (rv)
1786 goto out_cleanup_blk_queue;
1787 }
1788
1789 if (dev->mbps) {
1790 set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
1791 nullb_setup_bwtimer(nullb);
1792 }
1793
1794 if (dev->cache_size > 0) {
1795 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
1796 blk_queue_write_cache(nullb->q, true, true);
1797 blk_queue_flush_queueable(nullb->q, true);
1798 }
1799
1800 if (dev->zoned) {
1801 rv = null_zone_init(dev);
1802 if (rv)
1803 goto out_cleanup_blk_queue;
1804
1805 blk_queue_chunk_sectors(nullb->q, dev->zone_size_sects);
1806 nullb->q->limits.zoned = BLK_ZONED_HM;
1807 }
1808
1809 nullb->q->queuedata = nullb;
1810 blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
1811 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q);
1812
1813 mutex_lock(&lock);
1814 nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
1815 dev->index = nullb->index;
1816 mutex_unlock(&lock);
1817
1818 blk_queue_logical_block_size(nullb->q, dev->blocksize);
1819 blk_queue_physical_block_size(nullb->q, dev->blocksize);
1820
1821 null_config_discard(nullb);
1822
1823 sprintf(nullb->disk_name, "nullb%d", nullb->index);
1824
1825 rv = null_gendisk_register(nullb);
1826 if (rv)
1827 goto out_cleanup_zone;
1828
1829 mutex_lock(&lock);
1830 list_add_tail(&nullb->list, &nullb_list);
1831 mutex_unlock(&lock);
1832
1833 return 0;
1834 out_cleanup_zone:
1835 if (dev->zoned)
1836 null_zone_exit(dev);
1837 out_cleanup_blk_queue:
1838 blk_cleanup_queue(nullb->q);
1839 out_cleanup_tags:
1840 if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
1841 blk_mq_free_tag_set(nullb->tag_set);
1842 out_cleanup_queues:
1843 cleanup_queues(nullb);
1844 out_free_nullb:
1845 kfree(nullb);
1846 out:
1847 return rv;
1848 }
1849
1850 static int __init null_init(void)
1851 {
1852 int ret = 0;
1853 unsigned int i;
1854 struct nullb *nullb;
1855 struct nullb_device *dev;
1856
1857 if (g_bs > PAGE_SIZE) {
1858 pr_warn("null_blk: invalid block size\n");
1859 pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
1860 g_bs = PAGE_SIZE;
1861 }
1862
1863 if (!is_power_of_2(g_zone_size)) {
1864 pr_err("null_blk: zone_size must be power-of-two\n");
1865 return -EINVAL;
1866 }
1867
1868 if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
1869 if (g_submit_queues != nr_online_nodes) {
1870 pr_warn("null_blk: submit_queues param is set to %u.\n",
1871 nr_online_nodes);
1872 g_submit_queues = nr_online_nodes;
1873 }
1874 } else if (g_submit_queues > nr_cpu_ids)
1875 g_submit_queues = nr_cpu_ids;
1876 else if (g_submit_queues <= 0)
1877 g_submit_queues = 1;
1878
1879 if (g_queue_mode == NULL_Q_MQ && shared_tags) {
1880 ret = null_init_tag_set(NULL, &tag_set);
1881 if (ret)
1882 return ret;
1883 }
1884
1885 config_group_init(&nullb_subsys.su_group);
1886 mutex_init(&nullb_subsys.su_mutex);
1887
1888 ret = configfs_register_subsystem(&nullb_subsys);
1889 if (ret)
1890 goto err_tagset;
1891
1892 mutex_init(&lock);
1893
1894 null_major = register_blkdev(0, "nullb");
1895 if (null_major < 0) {
1896 ret = null_major;
1897 goto err_conf;
1898 }
1899
1900 for (i = 0; i < nr_devices; i++) {
1901 dev = null_alloc_dev();
1902 if (!dev) {
1903 ret = -ENOMEM;
1904 goto err_dev;
1905 }
1906 ret = null_add_dev(dev);
1907 if (ret) {
1908 null_free_dev(dev);
1909 goto err_dev;
1910 }
1911 }
1912
1913 pr_info("null: module loaded\n");
1914 return 0;
1915
1916 err_dev:
1917 while (!list_empty(&nullb_list)) {
1918 nullb = list_entry(nullb_list.next, struct nullb, list);
1919 dev = nullb->dev;
1920 null_del_dev(nullb);
1921 null_free_dev(dev);
1922 }
1923 unregister_blkdev(null_major, "nullb");
1924 err_conf:
1925 configfs_unregister_subsystem(&nullb_subsys);
1926 err_tagset:
1927 if (g_queue_mode == NULL_Q_MQ && shared_tags)
1928 blk_mq_free_tag_set(&tag_set);
1929 return ret;
1930 }
1931
1932 static void __exit null_exit(void)
1933 {
1934 struct nullb *nullb;
1935
1936 configfs_unregister_subsystem(&nullb_subsys);
1937
1938 unregister_blkdev(null_major, "nullb");
1939
1940 mutex_lock(&lock);
1941 while (!list_empty(&nullb_list)) {
1942 struct nullb_device *dev;
1943
1944 nullb = list_entry(nullb_list.next, struct nullb, list);
1945 dev = nullb->dev;
1946 null_del_dev(nullb);
1947 null_free_dev(dev);
1948 }
1949 mutex_unlock(&lock);
1950
1951 if (g_queue_mode == NULL_Q_MQ && shared_tags)
1952 blk_mq_free_tag_set(&tag_set);
1953 }
1954
1955 module_init(null_init);
1956 module_exit(null_exit);
1957
1958 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
1959 MODULE_LICENSE("GPL");
Linux with added FPC-III support