block/blk-ioc.c

   1 /*
   2  * Functions related to io context handling
   3  */
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/init.h>
   7 #include <linux/bio.h>
   8 #include <linux/blkdev.h>
   9 #include <linux/bootmem.h>      /* for max_pfn/max_low_pfn */
  10
  11 #include "blk.h"
  12
  13 /*
  14  * For io context allocations
  15  */
  16 static struct kmem_cache *iocontext_cachep;
  17
  18 static void cfq_dtor(struct io_context *ioc)
  19 {
  20         struct cfq_io_context *cic[1];
  21         int r;
  22
  23         /*
  24          * We don't have a specific key to lookup with, so use the gang
  25          * lookup to just retrieve the first item stored. The cfq exit
  26          * function will iterate the full tree, so any member will do.
  27          */
  28         r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
  29         if (r > 0)
  30                 cic[0]->dtor(ioc);
  31 }
  32
  33 /*
  34  * IO Context helper functions. put_io_context() returns 1 if there are no
  35  * more users of this io context, 0 otherwise.
  36  */
  37 int put_io_context(struct io_context *ioc)
  38 {
  39         if (ioc == NULL)
  40                 return 1;
  41
  42         BUG_ON(atomic_read(&ioc->refcount) == 0);
  43
  44         if (atomic_dec_and_test(&ioc->refcount)) {
  45                 rcu_read_lock();
  46                 if (ioc->aic && ioc->aic->dtor)
  47                         ioc->aic->dtor(ioc->aic);
  48                 rcu_read_unlock();
  49                 cfq_dtor(ioc);
  50
  51                 kmem_cache_free(iocontext_cachep, ioc);
  52                 return 1;
  53         }
  54         return 0;
  55 }
  56 EXPORT_SYMBOL(put_io_context);
  57
  58 static void cfq_exit(struct io_context *ioc)
  59 {
  60         struct cfq_io_context *cic[1];
  61         int r;
  62
  63         rcu_read_lock();
  64         /*
  65          * See comment for cfq_dtor()
  66          */
  67         r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1);
  68         rcu_read_unlock();
  69
  70         if (r > 0)
  71                 cic[0]->exit(ioc);
  72 }
  73
  74 /* Called by the exitting task */
  75 void exit_io_context(void)
  76 {
  77         struct io_context *ioc;
  78
  79         task_lock(current);
  80         ioc = current->io_context;
  81         current->io_context = NULL;
  82         task_unlock(current);
  83
  84         if (atomic_dec_and_test(&ioc->nr_tasks)) {
  85                 if (ioc->aic && ioc->aic->exit)
  86                         ioc->aic->exit(ioc->aic);
  87                 cfq_exit(ioc);
  88
  89                 put_io_context(ioc);
  90         }
  91 }
  92
  93 struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
  94 {
  95         struct io_context *ret;
  96
  97         ret = kmem_cache_alloc_node(iocontext_cachep, gfp_flags, node);
  98         if (ret) {
  99                 atomic_set(&ret->refcount, 1);
 100                 atomic_set(&ret->nr_tasks, 1);
 101                 spin_lock_init(&ret->lock);
 102                 ret->ioprio_changed = 0;
 103                 ret->ioprio = 0;
 104                 ret->last_waited = jiffies; /* doesn't matter... */
 105                 ret->nr_batch_requests = 0; /* because this is 0 */
 106                 ret->aic = NULL;
 107                 INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH);
 108                 ret->ioc_data = NULL;
 109         }
 110
 111         return ret;
 112 }
 113
 114 /*
 115  * If the current task has no IO context then create one and initialise it.
 116  * Otherwise, return its existing IO context.
 117  *
 118  * This returned IO context doesn't have a specifically elevated refcount,
 119  * but since the current task itself holds a reference, the context can be
 120  * used in general code, so long as it stays within `current` context.
 121  */
 122 struct io_context *current_io_context(gfp_t gfp_flags, int node)
 123 {
 124         struct task_struct *tsk = current;
 125         struct io_context *ret;
 126
 127         ret = tsk->io_context;
 128         if (likely(ret))
 129                 return ret;
 130
 131         ret = alloc_io_context(gfp_flags, node);
 132         if (ret) {
 133                 /* make sure set_task_ioprio() sees the settings above */
 134                 smp_wmb();
 135                 tsk->io_context = ret;
 136         }
 137
 138         return ret;
 139 }
 140
 141 /*
 142  * If the current task has no IO context then create one and initialise it.
 143  * If it does have a context, take a ref on it.
 144  *
 145  * This is always called in the context of the task which submitted the I/O.
 146  */
 147 struct io_context *get_io_context(gfp_t gfp_flags, int node)
 148 {
 149         struct io_context *ret = NULL;
 150
 151         /*
 152          * Check for unlikely race with exiting task. ioc ref count is
 153          * zero when ioc is being detached.
 154          */
 155         do {
 156                 ret = current_io_context(gfp_flags, node);
 157                 if (unlikely(!ret))
 158                         break;
 159         } while (!atomic_inc_not_zero(&ret->refcount));
 160
 161         return ret;
 162 }
 163 EXPORT_SYMBOL(get_io_context);
 164
 165 void copy_io_context(struct io_context **pdst, struct io_context **psrc)
 166 {
 167         struct io_context *src = *psrc;
 168         struct io_context *dst = *pdst;
 169
 170         if (src) {
 171                 BUG_ON(atomic_read(&src->refcount) == 0);
 172                 atomic_inc(&src->refcount);
 173                 put_io_context(dst);
 174                 *pdst = src;
 175         }
 176 }
 177 EXPORT_SYMBOL(copy_io_context);
 178
 179 void swap_io_context(struct io_context **ioc1, struct io_context **ioc2)
 180 {
 181         struct io_context *temp;
 182         temp = *ioc1;
 183         *ioc1 = *ioc2;
 184         *ioc2 = temp;
 185 }
 186 EXPORT_SYMBOL(swap_io_context);
 187
 188 int __init blk_ioc_init(void)
 189 {
 190         iocontext_cachep = kmem_cache_create("blkdev_ioc",
 191                         sizeof(struct io_context), 0, SLAB_PANIC, NULL);
 192         return 0;
 193 }
 194 subsys_initcall(blk_ioc_init);