block/blk-mq-tag.c

   1 #include <linux/kernel.h>
   2 #include <linux/module.h>
   3 #include <linux/percpu_ida.h>
   4
   5 #include <linux/blk-mq.h>
   6 #include "blk.h"
   7 #include "blk-mq.h"
   8 #include "blk-mq-tag.h"
   9
  10 /*
  11  * Per tagged queue (tag address space) map
  12  */
  13 struct blk_mq_tags {
  14         unsigned int nr_tags;
  15         unsigned int nr_reserved_tags;
  16         unsigned int nr_batch_move;
  17         unsigned int nr_max_cache;
  18
  19         struct percpu_ida free_tags;
  20         struct percpu_ida reserved_tags;
  21 };
  22
  23 void blk_mq_wait_for_tags(struct blk_mq_tags *tags)
  24 {
  25         int tag = blk_mq_get_tag(tags, __GFP_WAIT, false);
  26         blk_mq_put_tag(tags, tag);
  27 }
  28
  29 bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
  30 {
  31         return !tags ||
  32                 percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids) != 0;
  33 }
  34
  35 static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp)
  36 {
  37         int tag;
  38
  39         tag = percpu_ida_alloc(&tags->free_tags, (gfp & __GFP_WAIT) ?
  40                                TASK_UNINTERRUPTIBLE : TASK_RUNNING);
  41         if (tag < 0)
  42                 return BLK_MQ_TAG_FAIL;
  43         return tag + tags->nr_reserved_tags;
  44 }
  45
  46 static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags,
  47                                               gfp_t gfp)
  48 {
  49         int tag;
  50
  51         if (unlikely(!tags->nr_reserved_tags)) {
  52                 WARN_ON_ONCE(1);
  53                 return BLK_MQ_TAG_FAIL;
  54         }
  55
  56         tag = percpu_ida_alloc(&tags->reserved_tags, (gfp & __GFP_WAIT) ?
  57                                TASK_UNINTERRUPTIBLE : TASK_RUNNING);
  58         if (tag < 0)
  59                 return BLK_MQ_TAG_FAIL;
  60         return tag;
  61 }
  62
  63 unsigned int blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp, bool reserved)
  64 {
  65         if (!reserved)
  66                 return __blk_mq_get_tag(tags, gfp);
  67
  68         return __blk_mq_get_reserved_tag(tags, gfp);
  69 }
  70
  71 static void __blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag)
  72 {
  73         BUG_ON(tag >= tags->nr_tags);
  74
  75         percpu_ida_free(&tags->free_tags, tag - tags->nr_reserved_tags);
  76 }
  77
  78 static void __blk_mq_put_reserved_tag(struct blk_mq_tags *tags,
  79                                       unsigned int tag)
  80 {
  81         BUG_ON(tag >= tags->nr_reserved_tags);
  82
  83         percpu_ida_free(&tags->reserved_tags, tag);
  84 }
  85
  86 void blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag)
  87 {
  88         if (tag >= tags->nr_reserved_tags)
  89                 __blk_mq_put_tag(tags, tag);
  90         else
  91                 __blk_mq_put_reserved_tag(tags, tag);
  92 }
  93
  94 static int __blk_mq_tag_iter(unsigned id, void *data)
  95 {
  96         unsigned long *tag_map = data;
  97         __set_bit(id, tag_map);
  98         return 0;
  99 }
 100
 101 void blk_mq_tag_busy_iter(struct blk_mq_tags *tags,
 102                           void (*fn)(void *, unsigned long *), void *data)
 103 {
 104         unsigned long *tag_map;
 105         size_t map_size;
 106
 107         map_size = ALIGN(tags->nr_tags, BITS_PER_LONG) / BITS_PER_LONG;
 108         tag_map = kzalloc(map_size * sizeof(unsigned long), GFP_ATOMIC);
 109         if (!tag_map)
 110                 return;
 111
 112         percpu_ida_for_each_free(&tags->free_tags, __blk_mq_tag_iter, tag_map);
 113         if (tags->nr_reserved_tags)
 114                 percpu_ida_for_each_free(&tags->reserved_tags, __blk_mq_tag_iter,
 115                         tag_map);
 116
 117         fn(data, tag_map);
 118         kfree(tag_map);
 119 }
 120
 121 struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
 122                                      unsigned int reserved_tags, int node)
 123 {
 124         unsigned int nr_tags, nr_cache;
 125         struct blk_mq_tags *tags;
 126         int ret;
 127
 128         if (total_tags > BLK_MQ_TAG_MAX) {
 129                 pr_err("blk-mq: tag depth too large\n");
 130                 return NULL;
 131         }
 132
 133         tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
 134         if (!tags)
 135                 return NULL;
 136
 137         nr_tags = total_tags - reserved_tags;
 138         nr_cache = nr_tags / num_possible_cpus();
 139
 140         if (nr_cache < BLK_MQ_TAG_CACHE_MIN)
 141                 nr_cache = BLK_MQ_TAG_CACHE_MIN;
 142         else if (nr_cache > BLK_MQ_TAG_CACHE_MAX)
 143                 nr_cache = BLK_MQ_TAG_CACHE_MAX;
 144
 145         tags->nr_tags = total_tags;
 146         tags->nr_reserved_tags = reserved_tags;
 147         tags->nr_max_cache = nr_cache;
 148         tags->nr_batch_move = max(1u, nr_cache / 2);
 149
 150         ret = __percpu_ida_init(&tags->free_tags, tags->nr_tags -
 151                                 tags->nr_reserved_tags,
 152                                 tags->nr_max_cache,
 153                                 tags->nr_batch_move);
 154         if (ret)
 155                 goto err_free_tags;
 156
 157         if (reserved_tags) {
 158                 /*
 159                  * With max_cahe and batch set to 1, the allocator fallbacks to
 160                  * no cached. It's fine reserved tags allocation is slow.
 161                  */
 162                 ret = __percpu_ida_init(&tags->reserved_tags, reserved_tags,
 163                                 1, 1);
 164                 if (ret)
 165                         goto err_reserved_tags;
 166         }
 167
 168         return tags;
 169
 170 err_reserved_tags:
 171         percpu_ida_destroy(&tags->free_tags);
 172 err_free_tags:
 173         kfree(tags);
 174         return NULL;
 175 }
 176
 177 void blk_mq_free_tags(struct blk_mq_tags *tags)
 178 {
 179         percpu_ida_destroy(&tags->free_tags);
 180         percpu_ida_destroy(&tags->reserved_tags);
 181         kfree(tags);
 182 }
 183
 184 ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page)
 185 {
 186         char *orig_page = page;
 187         int cpu;
 188
 189         if (!tags)
 190                 return 0;
 191
 192         page += sprintf(page, "nr_tags=%u, reserved_tags=%u, batch_move=%u,"
 193                         " max_cache=%u\n", tags->nr_tags, tags->nr_reserved_tags,
 194                         tags->nr_batch_move, tags->nr_max_cache);
 195
 196         page += sprintf(page, "nr_free=%u, nr_reserved=%u\n",
 197                         percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids),
 198                         percpu_ida_free_tags(&tags->reserved_tags, nr_cpu_ids));
 199
 200         for_each_possible_cpu(cpu) {
 201                 page += sprintf(page, "  cpu%02u: nr_free=%u\n", cpu,
 202                                 percpu_ida_free_tags(&tags->free_tags, cpu));
 203         }
 204
 205         return page - orig_page;
 206 }