2 * Some low level IO code, and hacks for various block layer limitations
4 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
5 * Copyright 2012 Google, Inc.
12 #include <linux/blkdev.h>
14 static unsigned bch_bio_max_sectors(struct bio
*bio
)
16 struct request_queue
*q
= bdev_get_queue(bio
->bi_bdev
);
18 struct bvec_iter iter
;
19 unsigned ret
= 0, seg
= 0;
21 if (bio
->bi_rw
& REQ_DISCARD
)
22 return min(bio_sectors(bio
), q
->limits
.max_discard_sectors
);
24 bio_for_each_segment(bv
, bio
, iter
) {
25 struct bvec_merge_data bvm
= {
26 .bi_bdev
= bio
->bi_bdev
,
27 .bi_sector
= bio
->bi_iter
.bi_sector
,
32 if (seg
== min_t(unsigned, BIO_MAX_PAGES
,
33 queue_max_segments(q
)))
36 if (q
->merge_bvec_fn
&&
37 q
->merge_bvec_fn(q
, &bvm
, &bv
) < (int) bv
.bv_len
)
41 ret
+= bv
.bv_len
>> 9;
44 ret
= min(ret
, queue_max_sectors(q
));
47 ret
= max_t(int, ret
, bio_iovec(bio
).bv_len
>> 9);
52 static void bch_bio_submit_split_done(struct closure
*cl
)
54 struct bio_split_hook
*s
= container_of(cl
, struct bio_split_hook
, cl
);
56 s
->bio
->bi_end_io
= s
->bi_end_io
;
57 s
->bio
->bi_private
= s
->bi_private
;
60 closure_debug_destroy(&s
->cl
);
61 mempool_free(s
, s
->p
->bio_split_hook
);
64 static void bch_bio_submit_split_endio(struct bio
*bio
, int error
)
66 struct closure
*cl
= bio
->bi_private
;
67 struct bio_split_hook
*s
= container_of(cl
, struct bio_split_hook
, cl
);
70 clear_bit(BIO_UPTODATE
, &s
->bio
->bi_flags
);
76 void bch_generic_make_request(struct bio
*bio
, struct bio_split_pool
*p
)
78 struct bio_split_hook
*s
;
81 if (!bio_has_data(bio
) && !(bio
->bi_rw
& REQ_DISCARD
))
84 if (bio_sectors(bio
) <= bch_bio_max_sectors(bio
))
87 s
= mempool_alloc(p
->bio_split_hook
, GFP_NOIO
);
88 closure_init(&s
->cl
, NULL
);
92 s
->bi_end_io
= bio
->bi_end_io
;
93 s
->bi_private
= bio
->bi_private
;
97 n
= bio_next_split(bio
, bch_bio_max_sectors(bio
),
98 GFP_NOIO
, s
->p
->bio_split
);
100 n
->bi_end_io
= bch_bio_submit_split_endio
;
101 n
->bi_private
= &s
->cl
;
104 generic_make_request(n
);
107 continue_at(&s
->cl
, bch_bio_submit_split_done
, NULL
);
110 generic_make_request(bio
);
113 /* Bios with headers */
115 void bch_bbio_free(struct bio
*bio
, struct cache_set
*c
)
117 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
118 mempool_free(b
, c
->bio_meta
);
121 struct bio
*bch_bbio_alloc(struct cache_set
*c
)
123 struct bbio
*b
= mempool_alloc(c
->bio_meta
, GFP_NOIO
);
124 struct bio
*bio
= &b
->bio
;
127 bio
->bi_flags
|= BIO_POOL_NONE
<< BIO_POOL_OFFSET
;
128 bio
->bi_max_vecs
= bucket_pages(c
);
129 bio
->bi_io_vec
= bio
->bi_inline_vecs
;
134 void __bch_submit_bbio(struct bio
*bio
, struct cache_set
*c
)
136 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
138 bio
->bi_iter
.bi_sector
= PTR_OFFSET(&b
->key
, 0);
139 bio
->bi_bdev
= PTR_CACHE(c
, &b
->key
, 0)->bdev
;
141 b
->submit_time_us
= local_clock_us();
142 closure_bio_submit(bio
, bio
->bi_private
, PTR_CACHE(c
, &b
->key
, 0));
145 void bch_submit_bbio(struct bio
*bio
, struct cache_set
*c
,
146 struct bkey
*k
, unsigned ptr
)
148 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
149 bch_bkey_copy_single_ptr(&b
->key
, k
, ptr
);
150 __bch_submit_bbio(bio
, c
);
155 void bch_count_io_errors(struct cache
*ca
, int error
, const char *m
)
158 * The halflife of an error is:
159 * log2(1/2)/log2(127/128) * refresh ~= 88 * refresh
162 if (ca
->set
->error_decay
) {
163 unsigned count
= atomic_inc_return(&ca
->io_count
);
165 while (count
> ca
->set
->error_decay
) {
167 unsigned old
= count
;
168 unsigned new = count
- ca
->set
->error_decay
;
171 * First we subtract refresh from count; each time we
172 * succesfully do so, we rescale the errors once:
175 count
= atomic_cmpxchg(&ca
->io_count
, old
, new);
180 errors
= atomic_read(&ca
->io_errors
);
183 new = ((uint64_t) errors
* 127) / 128;
184 errors
= atomic_cmpxchg(&ca
->io_errors
,
186 } while (old
!= errors
);
192 char buf
[BDEVNAME_SIZE
];
193 unsigned errors
= atomic_add_return(1 << IO_ERROR_SHIFT
,
195 errors
>>= IO_ERROR_SHIFT
;
197 if (errors
< ca
->set
->error_limit
)
198 pr_err("%s: IO error on %s, recovering",
199 bdevname(ca
->bdev
, buf
), m
);
201 bch_cache_set_error(ca
->set
,
202 "%s: too many IO errors %s",
203 bdevname(ca
->bdev
, buf
), m
);
207 void bch_bbio_count_io_errors(struct cache_set
*c
, struct bio
*bio
,
208 int error
, const char *m
)
210 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
211 struct cache
*ca
= PTR_CACHE(c
, &b
->key
, 0);
213 unsigned threshold
= bio
->bi_rw
& REQ_WRITE
214 ? c
->congested_write_threshold_us
215 : c
->congested_read_threshold_us
;
218 unsigned t
= local_clock_us();
220 int us
= t
- b
->submit_time_us
;
221 int congested
= atomic_read(&c
->congested
);
223 if (us
> (int) threshold
) {
225 c
->congested_last_us
= t
;
227 ms
= min(ms
, CONGESTED_MAX
+ congested
);
228 atomic_sub(ms
, &c
->congested
);
229 } else if (congested
< 0)
230 atomic_inc(&c
->congested
);
233 bch_count_io_errors(ca
, error
, m
);
236 void bch_bbio_endio(struct cache_set
*c
, struct bio
*bio
,
237 int error
, const char *m
)
239 struct closure
*cl
= bio
->bi_private
;
241 bch_bbio_count_io_errors(c
, bio
, error
, m
);