| blob | bce8f5593a441a2c5669315d22cff02a42780832 |
1 From c3fae37fd2ed5b983d31d5d6e1ffcee4b2f7d01a Mon Sep 17 00:00:00 2001
2 From: Arianna Avanzini <avanzini.arianna@gmail.com>
3 Date: Tue, 18 Oct 2011 21:22:45 +0200
4 Subject: [PATCH 1/3] block: prepare I/O context code for BFQ-v3r1 for 3.1
6 BFQ uses struct cfq_io_context to store its per-process per-device data,
7 reusing the same code for cic handling of CFQ. The code is not shared
8 ATM to minimize the impact of these patches.
10 This patch introduces a new hlist to each io_context to store all the
11 cic's allocated by BFQ to allow calling the right destructor on module
12 unload; the radix tree used for cic lookup needs to be duplicated
13 because it can contain dead keys inserted by a scheduler and later
14 retrieved by the other one.
16 Update the io_context exit and free paths to take care also of
17 the BFQ cic's.
19 Change the type of cfqq inside struct cfq_io_context to void *
20 to use it also for BFQ per-queue data.
22 A new bfq-specific ioprio_changed field is necessary, too, to avoid
23 clobbering cfq's one, so switch ioprio_changed to a bitmap, with one
24 element per scheduler.
26 Signed-off-by: Fabio Checconi <fabio@gandalf.sssup.it>
27 Signed-off-by: Paolo Valente <paolo.valente@unimore.it>
28 Signed-off-by: Arianna Avanzini <avanzini.arianna@gmail.com>
29 ---
30 block/Kconfig.iosched | 26 ++++++++++++++++++++++++++
31 block/blk-ioc.c | 30 +++++++++++++++++-------------
32 block/cfq-iosched.c | 10 +++++++---
33 fs/ioprio.c | 9 +++++++--
34 include/linux/iocontext.h | 18 +++++++++++++++---
35 5 files changed, 72 insertions(+), 21 deletions(-)
37 diff --git a/block/Kconfig.iosched b/block/Kconfig.iosched
38 index 3199b76..5905452 100644
39 --- a/block/Kconfig.iosched
40 +++ b/block/Kconfig.iosched
41 @@ -43,6 +43,28 @@ config CFQ_GROUP_IOSCHED
42 ---help---
43 Enable group IO scheduling in CFQ.
45 +config IOSCHED_BFQ
46 + tristate "BFQ I/O scheduler"
47 + depends on EXPERIMENTAL
48 + default n
49 + ---help---
50 + The BFQ I/O scheduler tries to distribute bandwidth among
51 + all processes according to their weights.
52 + It aims at distributing the bandwidth as desired, independently of
53 + the disk parameters and with any workload. It also tries to
54 + guarantee low latency to interactive and soft real-time
55 + applications. If compiled built-in (saying Y here), BFQ can
56 + be configured to support hierarchical scheduling.
57 +
58 +config CGROUP_BFQIO
59 + bool "BFQ hierarchical scheduling support"
60 + depends on CGROUPS && IOSCHED_BFQ=y
61 + default n
62 + ---help---
63 + Enable hierarchical scheduling in BFQ, using the cgroups
64 + filesystem interface. The name of the subsystem will be
65 + bfqio.
66 +
67 choice
68 prompt "Default I/O scheduler"
69 default DEFAULT_CFQ
70 @@ -56,6 +78,9 @@ choice
71 config DEFAULT_CFQ
72 bool "CFQ" if IOSCHED_CFQ=y
74 + config DEFAULT_BFQ
75 + bool "BFQ" if IOSCHED_BFQ=y
76 +
77 config DEFAULT_NOOP
78 bool "No-op"
80 @@ -65,6 +90,7 @@ config DEFAULT_IOSCHED
81 string
82 default "deadline" if DEFAULT_DEADLINE
83 default "cfq" if DEFAULT_CFQ
84 + default "bfq" if DEFAULT_BFQ
85 default "noop" if DEFAULT_NOOP
87 endmenu
88 diff --git a/block/blk-ioc.c b/block/blk-ioc.c
89 index 6f9bbd9..d0d16d4 100644
90 --- a/block/blk-ioc.c
91 +++ b/block/blk-ioc.c
92 @@ -5,6 +5,7 @@
93 #include <linux/module.h>
94 #include <linux/init.h>
95 #include <linux/bio.h>
96 +#include <linux/bitmap.h>
97 #include <linux/blkdev.h>
98 #include <linux/bootmem.h> /* for max_pfn/max_low_pfn */
99 #include <linux/slab.h>
100 @@ -16,13 +17,12 @@
101 */
102 static struct kmem_cache *iocontext_cachep;
104 -static void cfq_dtor(struct io_context *ioc)
105 +static void hlist_sched_dtor(struct io_context *ioc, struct hlist_head *list)
106 {
107 - if (!hlist_empty(&ioc->cic_list)) {
108 + if (!hlist_empty(list)) {
109 struct cfq_io_context *cic;
111 - cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context,
112 - cic_list);
113 + cic = hlist_entry(list->first, struct cfq_io_context, cic_list);
114 cic->dtor(ioc);
115 }
116 }
117 @@ -40,7 +40,9 @@ int put_io_context(struct io_context *ioc)
119 if (atomic_long_dec_and_test(&ioc->refcount)) {
120 rcu_read_lock();
121 - cfq_dtor(ioc);
122 +
123 + hlist_sched_dtor(ioc, &ioc->cic_list);
124 + hlist_sched_dtor(ioc, &ioc->bfq_cic_list);
125 rcu_read_unlock();
127 kmem_cache_free(iocontext_cachep, ioc);
128 @@ -50,15 +52,14 @@ int put_io_context(struct io_context *ioc)
129 }
130 EXPORT_SYMBOL(put_io_context);
132 -static void cfq_exit(struct io_context *ioc)
133 +static void hlist_sched_exit(struct io_context *ioc, struct hlist_head *list)
134 {
135 rcu_read_lock();
137 - if (!hlist_empty(&ioc->cic_list)) {
138 + if (!hlist_empty(list)) {
139 struct cfq_io_context *cic;
141 - cic = hlist_entry(ioc->cic_list.first, struct cfq_io_context,
142 - cic_list);
143 + cic = hlist_entry(list->first, struct cfq_io_context, cic_list);
144 cic->exit(ioc);
145 }
146 rcu_read_unlock();
147 @@ -74,9 +75,10 @@ void exit_io_context(struct task_struct *task)
148 task->io_context = NULL;
149 task_unlock(task);
151 - if (atomic_dec_and_test(&ioc->nr_tasks))
152 - cfq_exit(ioc);
153 -
154 + if (atomic_dec_and_test(&ioc->nr_tasks)) {
155 + hlist_sched_exit(ioc, &ioc->cic_list);
156 + hlist_sched_exit(ioc, &ioc->bfq_cic_list);
157 + }
158 put_io_context(ioc);
159 }
161 @@ -89,12 +91,14 @@ struct io_context *alloc_io_context(gfp_t gfp_flags, int node)
162 atomic_long_set(&ioc->refcount, 1);
163 atomic_set(&ioc->nr_tasks, 1);
164 spin_lock_init(&ioc->lock);
165 - ioc->ioprio_changed = 0;
166 + bitmap_zero(ioc->ioprio_changed, IOC_IOPRIO_CHANGED_BITS);
167 ioc->ioprio = 0;
168 ioc->last_waited = 0; /* doesn't matter... */
169 ioc->nr_batch_requests = 0; /* because this is 0 */
170 INIT_RADIX_TREE(&ioc->radix_root, GFP_ATOMIC | __GFP_HIGH);
171 INIT_HLIST_HEAD(&ioc->cic_list);
172 + INIT_RADIX_TREE(&ioc->bfq_radix_root, GFP_ATOMIC | __GFP_HIGH);
173 + INIT_HLIST_HEAD(&ioc->bfq_cic_list);
174 ioc->ioc_data = NULL;
175 #if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE)
176 ioc->cgroup_changed = 0;
177 diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
178 index 16ace89..ec4bcf1 100644
179 --- a/block/cfq-iosched.c
180 +++ b/block/cfq-iosched.c
181 @@ -2934,7 +2934,6 @@ static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic)
182 static void cfq_ioc_set_ioprio(struct io_context *ioc)
183 {
184 call_for_each_cic(ioc, changed_ioprio);
185 - ioc->ioprio_changed = 0;
186 }
188 static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
189 @@ -3219,8 +3218,13 @@ cfq_get_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
190 goto err_free;
192 out:
193 - smp_read_barrier_depends();
194 - if (unlikely(ioc->ioprio_changed))
195 + /*
196 + * test_and_clear_bit() implies a memory barrier, paired with
197 + * the wmb() in fs/ioprio.c, so the value seen for ioprio is the
198 + * new one.
199 + */
200 + if (unlikely(test_and_clear_bit(IOC_CFQ_IOPRIO_CHANGED,
201 + ioc->ioprio_changed)))
202 cfq_ioc_set_ioprio(ioc);
204 #ifdef CONFIG_CFQ_GROUP_IOSCHED
205 diff --git a/fs/ioprio.c b/fs/ioprio.c
206 index 7da2a06..95a6c2b 100644
207 --- a/fs/ioprio.c
208 +++ b/fs/ioprio.c
209 @@ -30,7 +30,7 @@
211 int set_task_ioprio(struct task_struct *task, int ioprio)
212 {
213 - int err;
214 + int err, i;
215 struct io_context *ioc;
216 const struct cred *cred = current_cred(), *tcred;
218 @@ -60,12 +60,17 @@ int set_task_ioprio(struct task_struct *task, int ioprio)
219 err = -ENOMEM;
220 break;
221 }
222 + /* let other ioc users see the new values */
223 + smp_wmb();
224 task->io_context = ioc;
225 } while (1);
227 if (!err) {
228 ioc->ioprio = ioprio;
229 - ioc->ioprio_changed = 1;
230 + /* make sure schedulers see the new ioprio value */
231 + wmb();
232 + for (i = 0; i < IOC_IOPRIO_CHANGED_BITS; i++)
233 + set_bit(i, ioc->ioprio_changed);
234 }
236 task_unlock(task);
237 diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h
238 index 5037a0a..69fdd58 100644
239 --- a/include/linux/iocontext.h
240 +++ b/include/linux/iocontext.h
241 @@ -1,10 +1,10 @@
242 #ifndef IOCONTEXT_H
243 #define IOCONTEXT_H
245 +#include <linux/bitmap.h>
246 #include <linux/radix-tree.h>
247 #include <linux/rcupdate.h>
249 -struct cfq_queue;
250 struct cfq_ttime {
251 unsigned long last_end_request;
253 @@ -16,7 +16,7 @@ struct cfq_ttime {
254 struct cfq_io_context {
255 void *key;
257 - struct cfq_queue *cfqq[2];
258 + void *cfqq[2];
260 struct io_context *ioc;
262 @@ -32,6 +32,16 @@ struct cfq_io_context {
263 };
265 /*
266 + * Indexes into the ioprio_changed bitmap. A bit set indicates that
267 + * the corresponding I/O scheduler needs to see a ioprio update.
268 + */
269 +enum {
270 + IOC_CFQ_IOPRIO_CHANGED,
271 + IOC_BFQ_IOPRIO_CHANGED,
272 + IOC_IOPRIO_CHANGED_BITS
273 +};
274 +
275 +/*
276 * I/O subsystem state of the associated processes. It is refcounted
277 * and kmalloc'ed. These could be shared between processes.
278 */
279 @@ -43,7 +53,7 @@ struct io_context {
280 spinlock_t lock;
282 unsigned short ioprio;
283 - unsigned short ioprio_changed;
284 + DECLARE_BITMAP(ioprio_changed, IOC_IOPRIO_CHANGED_BITS);
286 #if defined(CONFIG_BLK_CGROUP) || defined(CONFIG_BLK_CGROUP_MODULE)
287 unsigned short cgroup_changed;
288 @@ -57,6 +67,8 @@ struct io_context {
290 struct radix_tree_root radix_root;
291 struct hlist_head cic_list;
292 + struct radix_tree_root bfq_radix_root;
293 + struct hlist_head bfq_cic_list;
294 void __rcu *ioc_data;
295 };
297 --
298 1.7.2.5