?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.18.10
[linux/fpc-iii.git] / fs / btrfs / extent-tree.c
blob4ab0bccfa281a10f6b8db6fbced51a6a051aab6a
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
5
6 #include <linux/sched.h>
7 #include <linux/sched/signal.h>
8 #include <linux/pagemap.h>
9 #include <linux/writeback.h>
10 #include <linux/blkdev.h>
11 #include <linux/sort.h>
12 #include <linux/rcupdate.h>
13 #include <linux/kthread.h>
14 #include <linux/slab.h>
15 #include <linux/ratelimit.h>
16 #include <linux/percpu_counter.h>
17 #include <linux/lockdep.h>
18 #include <linux/crc32c.h>
19 #include "tree-log.h"
20 #include "disk-io.h"
21 #include "print-tree.h"
22 #include "volumes.h"
23 #include "raid56.h"
24 #include "locking.h"
25 #include "free-space-cache.h"
26 #include "free-space-tree.h"
27 #include "math.h"
28 #include "sysfs.h"
29 #include "qgroup.h"
30 #include "ref-verify.h"
31
32 #undef SCRAMBLE_DELAYED_REFS
33
34 /*
35 * control flags for do_chunk_alloc's force field
36 * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
37 * if we really need one.
38 *
39 * CHUNK_ALLOC_LIMITED means to only try and allocate one
40 * if we have very few chunks already allocated. This is
41 * used as part of the clustering code to help make sure
42 * we have a good pool of storage to cluster in, without
43 * filling the FS with empty chunks
44 *
45 * CHUNK_ALLOC_FORCE means it must try to allocate one
46 *
47 */
48 enum {
49 CHUNK_ALLOC_NO_FORCE = 0,
50 CHUNK_ALLOC_LIMITED = 1,
51 CHUNK_ALLOC_FORCE = 2,
52 };
53
54 static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
55 struct btrfs_fs_info *fs_info,
56 struct btrfs_delayed_ref_node *node, u64 parent,
57 u64 root_objectid, u64 owner_objectid,
58 u64 owner_offset, int refs_to_drop,
59 struct btrfs_delayed_extent_op *extra_op);
60 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
61 struct extent_buffer *leaf,
62 struct btrfs_extent_item *ei);
63 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
64 struct btrfs_fs_info *fs_info,
65 u64 parent, u64 root_objectid,
66 u64 flags, u64 owner, u64 offset,
67 struct btrfs_key *ins, int ref_mod);
68 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
69 struct btrfs_delayed_ref_node *node,
70 struct btrfs_delayed_extent_op *extent_op);
71 static int do_chunk_alloc(struct btrfs_trans_handle *trans,
72 struct btrfs_fs_info *fs_info, u64 flags,
73 int force);
74 static int find_next_key(struct btrfs_path *path, int level,
75 struct btrfs_key *key);
76 static void dump_space_info(struct btrfs_fs_info *fs_info,
77 struct btrfs_space_info *info, u64 bytes,
78 int dump_block_groups);
79 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
80 u64 num_bytes);
81 static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info,
82 struct btrfs_space_info *space_info,
83 u64 num_bytes);
84 static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info,
85 struct btrfs_space_info *space_info,
86 u64 num_bytes);
87
88 static noinline int
89 block_group_cache_done(struct btrfs_block_group_cache *cache)
90 {
91 smp_mb();
92 return cache->cached == BTRFS_CACHE_FINISHED ||
93 cache->cached == BTRFS_CACHE_ERROR;
94 }
95
96 static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
97 {
98 return (cache->flags & bits) == bits;
99 }
100
101 void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
102 {
103 atomic_inc(&cache->count);
104 }
105
106 void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
107 {
108 if (atomic_dec_and_test(&cache->count)) {
109 WARN_ON(cache->pinned > 0);
110 WARN_ON(cache->reserved > 0);
111
112 /*
113 * If not empty, someone is still holding mutex of
114 * full_stripe_lock, which can only be released by caller.
115 * And it will definitely cause use-after-free when caller
116 * tries to release full stripe lock.
117 *
118 * No better way to resolve, but only to warn.
119 */
120 WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root));
121 kfree(cache->free_space_ctl);
122 kfree(cache);
123 }
124 }
125
126 /*
127 * this adds the block group to the fs_info rb tree for the block group
128 * cache
129 */
130 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
131 struct btrfs_block_group_cache *block_group)
132 {
133 struct rb_node **p;
134 struct rb_node *parent = NULL;
135 struct btrfs_block_group_cache *cache;
136
137 spin_lock(&info->block_group_cache_lock);
138 p = &info->block_group_cache_tree.rb_node;
139
140 while (*p) {
141 parent = *p;
142 cache = rb_entry(parent, struct btrfs_block_group_cache,
143 cache_node);
144 if (block_group->key.objectid < cache->key.objectid) {
145 p = &(*p)->rb_left;
146 } else if (block_group->key.objectid > cache->key.objectid) {
147 p = &(*p)->rb_right;
148 } else {
149 spin_unlock(&info->block_group_cache_lock);
150 return -EEXIST;
151 }
152 }
153
154 rb_link_node(&block_group->cache_node, parent, p);
155 rb_insert_color(&block_group->cache_node,
156 &info->block_group_cache_tree);
157
158 if (info->first_logical_byte > block_group->key.objectid)
159 info->first_logical_byte = block_group->key.objectid;
160
161 spin_unlock(&info->block_group_cache_lock);
162
163 return 0;
164 }
165
166 /*
167 * This will return the block group at or after bytenr if contains is 0, else
168 * it will return the block group that contains the bytenr
169 */
170 static struct btrfs_block_group_cache *
171 block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
172 int contains)
173 {
174 struct btrfs_block_group_cache *cache, *ret = NULL;
175 struct rb_node *n;
176 u64 end, start;
177
178 spin_lock(&info->block_group_cache_lock);
179 n = info->block_group_cache_tree.rb_node;
180
181 while (n) {
182 cache = rb_entry(n, struct btrfs_block_group_cache,
183 cache_node);
184 end = cache->key.objectid + cache->key.offset - 1;
185 start = cache->key.objectid;
186
187 if (bytenr < start) {
188 if (!contains && (!ret || start < ret->key.objectid))
189 ret = cache;
190 n = n->rb_left;
191 } else if (bytenr > start) {
192 if (contains && bytenr <= end) {
193 ret = cache;
194 break;
195 }
196 n = n->rb_right;
197 } else {
198 ret = cache;
199 break;
200 }
201 }
202 if (ret) {
203 btrfs_get_block_group(ret);
204 if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
205 info->first_logical_byte = ret->key.objectid;
206 }
207 spin_unlock(&info->block_group_cache_lock);
208
209 return ret;
210 }
211
212 static int add_excluded_extent(struct btrfs_fs_info *fs_info,
213 u64 start, u64 num_bytes)
214 {
215 u64 end = start + num_bytes - 1;
216 set_extent_bits(&fs_info->freed_extents[0],
217 start, end, EXTENT_UPTODATE);
218 set_extent_bits(&fs_info->freed_extents[1],
219 start, end, EXTENT_UPTODATE);
220 return 0;
221 }
222
223 static void free_excluded_extents(struct btrfs_fs_info *fs_info,
224 struct btrfs_block_group_cache *cache)
225 {
226 u64 start, end;
227
228 start = cache->key.objectid;
229 end = start + cache->key.offset - 1;
230
231 clear_extent_bits(&fs_info->freed_extents[0],
232 start, end, EXTENT_UPTODATE);
233 clear_extent_bits(&fs_info->freed_extents[1],
234 start, end, EXTENT_UPTODATE);
235 }
236
237 static int exclude_super_stripes(struct btrfs_fs_info *fs_info,
238 struct btrfs_block_group_cache *cache)
239 {
240 u64 bytenr;
241 u64 *logical;
242 int stripe_len;
243 int i, nr, ret;
244
245 if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
246 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
247 cache->bytes_super += stripe_len;
248 ret = add_excluded_extent(fs_info, cache->key.objectid,
249 stripe_len);
250 if (ret)
251 return ret;
252 }
253
254 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
255 bytenr = btrfs_sb_offset(i);
256 ret = btrfs_rmap_block(fs_info, cache->key.objectid,
257 bytenr, &logical, &nr, &stripe_len);
258 if (ret)
259 return ret;
260
261 while (nr--) {
262 u64 start, len;
263
264 if (logical[nr] > cache->key.objectid +
265 cache->key.offset)
266 continue;
267
268 if (logical[nr] + stripe_len <= cache->key.objectid)
269 continue;
270
271 start = logical[nr];
272 if (start < cache->key.objectid) {
273 start = cache->key.objectid;
274 len = (logical[nr] + stripe_len) - start;
275 } else {
276 len = min_t(u64, stripe_len,
277 cache->key.objectid +
278 cache->key.offset - start);
279 }
280
281 cache->bytes_super += len;
282 ret = add_excluded_extent(fs_info, start, len);
283 if (ret) {
284 kfree(logical);
285 return ret;
286 }
287 }
288
289 kfree(logical);
290 }
291 return 0;
292 }
293
294 static struct btrfs_caching_control *
295 get_caching_control(struct btrfs_block_group_cache *cache)
296 {
297 struct btrfs_caching_control *ctl;
298
299 spin_lock(&cache->lock);
300 if (!cache->caching_ctl) {
301 spin_unlock(&cache->lock);
302 return NULL;
303 }
304
305 ctl = cache->caching_ctl;
306 refcount_inc(&ctl->count);
307 spin_unlock(&cache->lock);
308 return ctl;
309 }
310
311 static void put_caching_control(struct btrfs_caching_control *ctl)
312 {
313 if (refcount_dec_and_test(&ctl->count))
314 kfree(ctl);
315 }
316
317 #ifdef CONFIG_BTRFS_DEBUG
318 static void fragment_free_space(struct btrfs_block_group_cache *block_group)
319 {
320 struct btrfs_fs_info *fs_info = block_group->fs_info;
321 u64 start = block_group->key.objectid;
322 u64 len = block_group->key.offset;
323 u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ?
324 fs_info->nodesize : fs_info->sectorsize;
325 u64 step = chunk << 1;
326
327 while (len > chunk) {
328 btrfs_remove_free_space(block_group, start, chunk);
329 start += step;
330 if (len < step)
331 len = 0;
332 else
333 len -= step;
334 }
335 }
336 #endif
337
338 /*
339 * this is only called by cache_block_group, since we could have freed extents
340 * we need to check the pinned_extents for any extents that can't be used yet
341 * since their free space will be released as soon as the transaction commits.
342 */
343 u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
344 u64 start, u64 end)
345 {
346 struct btrfs_fs_info *info = block_group->fs_info;
347 u64 extent_start, extent_end, size, total_added = 0;
348 int ret;
349
350 while (start < end) {
351 ret = find_first_extent_bit(info->pinned_extents, start,
352 &extent_start, &extent_end,
353 EXTENT_DIRTY | EXTENT_UPTODATE,
354 NULL);
355 if (ret)
356 break;
357
358 if (extent_start <= start) {
359 start = extent_end + 1;
360 } else if (extent_start > start && extent_start < end) {
361 size = extent_start - start;
362 total_added += size;
363 ret = btrfs_add_free_space(block_group, start,
364 size);
365 BUG_ON(ret); /* -ENOMEM or logic error */
366 start = extent_end + 1;
367 } else {
368 break;
369 }
370 }
371
372 if (start < end) {
373 size = end - start;
374 total_added += size;
375 ret = btrfs_add_free_space(block_group, start, size);
376 BUG_ON(ret); /* -ENOMEM or logic error */
377 }
378
379 return total_added;
380 }
381
382 static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
383 {
384 struct btrfs_block_group_cache *block_group = caching_ctl->block_group;
385 struct btrfs_fs_info *fs_info = block_group->fs_info;
386 struct btrfs_root *extent_root = fs_info->extent_root;
387 struct btrfs_path *path;
388 struct extent_buffer *leaf;
389 struct btrfs_key key;
390 u64 total_found = 0;
391 u64 last = 0;
392 u32 nritems;
393 int ret;
394 bool wakeup = true;
395
396 path = btrfs_alloc_path();
397 if (!path)
398 return -ENOMEM;
399
400 last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
401
402 #ifdef CONFIG_BTRFS_DEBUG
403 /*
404 * If we're fragmenting we don't want to make anybody think we can
405 * allocate from this block group until we've had a chance to fragment
406 * the free space.
407 */
408 if (btrfs_should_fragment_free_space(block_group))
409 wakeup = false;
410 #endif
411 /*
412 * We don't want to deadlock with somebody trying to allocate a new
413 * extent for the extent root while also trying to search the extent
414 * root to add free space. So we skip locking and search the commit
415 * root, since its read-only
416 */
417 path->skip_locking = 1;
418 path->search_commit_root = 1;
419 path->reada = READA_FORWARD;
420
421 key.objectid = last;
422 key.offset = 0;
423 key.type = BTRFS_EXTENT_ITEM_KEY;
424
425 next:
426 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
427 if (ret < 0)
428 goto out;
429
430 leaf = path->nodes[0];
431 nritems = btrfs_header_nritems(leaf);
432
433 while (1) {
434 if (btrfs_fs_closing(fs_info) > 1) {
435 last = (u64)-1;
436 break;
437 }
438
439 if (path->slots[0] < nritems) {
440 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
441 } else {
442 ret = find_next_key(path, 0, &key);
443 if (ret)
444 break;
445
446 if (need_resched() ||
447 rwsem_is_contended(&fs_info->commit_root_sem)) {
448 if (wakeup)
449 caching_ctl->progress = last;
450 btrfs_release_path(path);
451 up_read(&fs_info->commit_root_sem);
452 mutex_unlock(&caching_ctl->mutex);
453 cond_resched();
454 mutex_lock(&caching_ctl->mutex);
455 down_read(&fs_info->commit_root_sem);
456 goto next;
457 }
458
459 ret = btrfs_next_leaf(extent_root, path);
460 if (ret < 0)
461 goto out;
462 if (ret)
463 break;
464 leaf = path->nodes[0];
465 nritems = btrfs_header_nritems(leaf);
466 continue;
467 }
468
469 if (key.objectid < last) {
470 key.objectid = last;
471 key.offset = 0;
472 key.type = BTRFS_EXTENT_ITEM_KEY;
473
474 if (wakeup)
475 caching_ctl->progress = last;
476 btrfs_release_path(path);
477 goto next;
478 }
479
480 if (key.objectid < block_group->key.objectid) {
481 path->slots[0]++;
482 continue;
483 }
484
485 if (key.objectid >= block_group->key.objectid +
486 block_group->key.offset)
487 break;
488
489 if (key.type == BTRFS_EXTENT_ITEM_KEY ||
490 key.type == BTRFS_METADATA_ITEM_KEY) {
491 total_found += add_new_free_space(block_group, last,
492 key.objectid);
493 if (key.type == BTRFS_METADATA_ITEM_KEY)
494 last = key.objectid +
495 fs_info->nodesize;
496 else
497 last = key.objectid + key.offset;
498
499 if (total_found > CACHING_CTL_WAKE_UP) {
500 total_found = 0;
501 if (wakeup)
502 wake_up(&caching_ctl->wait);
503 }
504 }
505 path->slots[0]++;
506 }
507 ret = 0;
508
509 total_found += add_new_free_space(block_group, last,
510 block_group->key.objectid +
511 block_group->key.offset);
512 caching_ctl->progress = (u64)-1;
513
514 out:
515 btrfs_free_path(path);
516 return ret;
517 }
518
519 static noinline void caching_thread(struct btrfs_work *work)
520 {
521 struct btrfs_block_group_cache *block_group;
522 struct btrfs_fs_info *fs_info;
523 struct btrfs_caching_control *caching_ctl;
524 int ret;
525
526 caching_ctl = container_of(work, struct btrfs_caching_control, work);
527 block_group = caching_ctl->block_group;
528 fs_info = block_group->fs_info;
529
530 mutex_lock(&caching_ctl->mutex);
531 down_read(&fs_info->commit_root_sem);
532
533 if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
534 ret = load_free_space_tree(caching_ctl);
535 else
536 ret = load_extent_tree_free(caching_ctl);
537
538 spin_lock(&block_group->lock);
539 block_group->caching_ctl = NULL;
540 block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
541 spin_unlock(&block_group->lock);
542
543 #ifdef CONFIG_BTRFS_DEBUG
544 if (btrfs_should_fragment_free_space(block_group)) {
545 u64 bytes_used;
546
547 spin_lock(&block_group->space_info->lock);
548 spin_lock(&block_group->lock);
549 bytes_used = block_group->key.offset -
550 btrfs_block_group_used(&block_group->item);
551 block_group->space_info->bytes_used += bytes_used >> 1;
552 spin_unlock(&block_group->lock);
553 spin_unlock(&block_group->space_info->lock);
554 fragment_free_space(block_group);
555 }
556 #endif
557
558 caching_ctl->progress = (u64)-1;
559
560 up_read(&fs_info->commit_root_sem);
561 free_excluded_extents(fs_info, block_group);
562 mutex_unlock(&caching_ctl->mutex);
563
564 wake_up(&caching_ctl->wait);
565
566 put_caching_control(caching_ctl);
567 btrfs_put_block_group(block_group);
568 }
569
570 static int cache_block_group(struct btrfs_block_group_cache *cache,
571 int load_cache_only)
572 {
573 DEFINE_WAIT(wait);
574 struct btrfs_fs_info *fs_info = cache->fs_info;
575 struct btrfs_caching_control *caching_ctl;
576 int ret = 0;
577
578 caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
579 if (!caching_ctl)
580 return -ENOMEM;
581
582 INIT_LIST_HEAD(&caching_ctl->list);
583 mutex_init(&caching_ctl->mutex);
584 init_waitqueue_head(&caching_ctl->wait);
585 caching_ctl->block_group = cache;
586 caching_ctl->progress = cache->key.objectid;
587 refcount_set(&caching_ctl->count, 1);
588 btrfs_init_work(&caching_ctl->work, btrfs_cache_helper,
589 caching_thread, NULL, NULL);
590
591 spin_lock(&cache->lock);
592 /*
593 * This should be a rare occasion, but this could happen I think in the
594 * case where one thread starts to load the space cache info, and then
595 * some other thread starts a transaction commit which tries to do an
596 * allocation while the other thread is still loading the space cache
597 * info. The previous loop should have kept us from choosing this block
598 * group, but if we've moved to the state where we will wait on caching
599 * block groups we need to first check if we're doing a fast load here,
600 * so we can wait for it to finish, otherwise we could end up allocating
601 * from a block group who's cache gets evicted for one reason or
602 * another.
603 */
604 while (cache->cached == BTRFS_CACHE_FAST) {
605 struct btrfs_caching_control *ctl;
606
607 ctl = cache->caching_ctl;
608 refcount_inc(&ctl->count);
609 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
610 spin_unlock(&cache->lock);
611
612 schedule();
613
614 finish_wait(&ctl->wait, &wait);
615 put_caching_control(ctl);
616 spin_lock(&cache->lock);
617 }
618
619 if (cache->cached != BTRFS_CACHE_NO) {
620 spin_unlock(&cache->lock);
621 kfree(caching_ctl);
622 return 0;
623 }
624 WARN_ON(cache->caching_ctl);
625 cache->caching_ctl = caching_ctl;
626 cache->cached = BTRFS_CACHE_FAST;
627 spin_unlock(&cache->lock);
628
629 if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
630 mutex_lock(&caching_ctl->mutex);
631 ret = load_free_space_cache(fs_info, cache);
632
633 spin_lock(&cache->lock);
634 if (ret == 1) {
635 cache->caching_ctl = NULL;
636 cache->cached = BTRFS_CACHE_FINISHED;
637 cache->last_byte_to_unpin = (u64)-1;
638 caching_ctl->progress = (u64)-1;
639 } else {
640 if (load_cache_only) {
641 cache->caching_ctl = NULL;
642 cache->cached = BTRFS_CACHE_NO;
643 } else {
644 cache->cached = BTRFS_CACHE_STARTED;
645 cache->has_caching_ctl = 1;
646 }
647 }
648 spin_unlock(&cache->lock);
649 #ifdef CONFIG_BTRFS_DEBUG
650 if (ret == 1 &&
651 btrfs_should_fragment_free_space(cache)) {
652 u64 bytes_used;
653
654 spin_lock(&cache->space_info->lock);
655 spin_lock(&cache->lock);
656 bytes_used = cache->key.offset -
657 btrfs_block_group_used(&cache->item);
658 cache->space_info->bytes_used += bytes_used >> 1;
659 spin_unlock(&cache->lock);
660 spin_unlock(&cache->space_info->lock);
661 fragment_free_space(cache);
662 }
663 #endif
664 mutex_unlock(&caching_ctl->mutex);
665
666 wake_up(&caching_ctl->wait);
667 if (ret == 1) {
668 put_caching_control(caching_ctl);
669 free_excluded_extents(fs_info, cache);
670 return 0;
671 }
672 } else {
673 /*
674 * We're either using the free space tree or no caching at all.
675 * Set cached to the appropriate value and wakeup any waiters.
676 */
677 spin_lock(&cache->lock);
678 if (load_cache_only) {
679 cache->caching_ctl = NULL;
680 cache->cached = BTRFS_CACHE_NO;
681 } else {
682 cache->cached = BTRFS_CACHE_STARTED;
683 cache->has_caching_ctl = 1;
684 }
685 spin_unlock(&cache->lock);
686 wake_up(&caching_ctl->wait);
687 }
688
689 if (load_cache_only) {
690 put_caching_control(caching_ctl);
691 return 0;
692 }
693
694 down_write(&fs_info->commit_root_sem);
695 refcount_inc(&caching_ctl->count);
696 list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
697 up_write(&fs_info->commit_root_sem);
698
699 btrfs_get_block_group(cache);
700
701 btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work);
702
703 return ret;
704 }
705
706 /*
707 * return the block group that starts at or after bytenr
708 */
709 static struct btrfs_block_group_cache *
710 btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
711 {
712 return block_group_cache_tree_search(info, bytenr, 0);
713 }
714
715 /*
716 * return the block group that contains the given bytenr
717 */
718 struct btrfs_block_group_cache *btrfs_lookup_block_group(
719 struct btrfs_fs_info *info,
720 u64 bytenr)
721 {
722 return block_group_cache_tree_search(info, bytenr, 1);
723 }
724
725 static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
726 u64 flags)
727 {
728 struct list_head *head = &info->space_info;
729 struct btrfs_space_info *found;
730
731 flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
732
733 rcu_read_lock();
734 list_for_each_entry_rcu(found, head, list) {
735 if (found->flags & flags) {
736 rcu_read_unlock();
737 return found;
738 }
739 }
740 rcu_read_unlock();
741 return NULL;
742 }
743
744 static void add_pinned_bytes(struct btrfs_fs_info *fs_info, s64 num_bytes,
745 bool metadata, u64 root_objectid)
746 {
747 struct btrfs_space_info *space_info;
748 u64 flags;
749
750 if (metadata) {
751 if (root_objectid == BTRFS_CHUNK_TREE_OBJECTID)
752 flags = BTRFS_BLOCK_GROUP_SYSTEM;
753 else
754 flags = BTRFS_BLOCK_GROUP_METADATA;
755 } else {
756 flags = BTRFS_BLOCK_GROUP_DATA;
757 }
758
759 space_info = __find_space_info(fs_info, flags);
760 ASSERT(space_info);
761 percpu_counter_add(&space_info->total_bytes_pinned, num_bytes);
762 }
763
764 /*
765 * after adding space to the filesystem, we need to clear the full flags
766 * on all the space infos.
767 */
768 void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
769 {
770 struct list_head *head = &info->space_info;
771 struct btrfs_space_info *found;
772
773 rcu_read_lock();
774 list_for_each_entry_rcu(found, head, list)
775 found->full = 0;
776 rcu_read_unlock();
777 }
778
779 /* simple helper to search for an existing data extent at a given offset */
780 int btrfs_lookup_data_extent(struct btrfs_fs_info *fs_info, u64 start, u64 len)
781 {
782 int ret;
783 struct btrfs_key key;
784 struct btrfs_path *path;
785
786 path = btrfs_alloc_path();
787 if (!path)
788 return -ENOMEM;
789
790 key.objectid = start;
791 key.offset = len;
792 key.type = BTRFS_EXTENT_ITEM_KEY;
793 ret = btrfs_search_slot(NULL, fs_info->extent_root, &key, path, 0, 0);
794 btrfs_free_path(path);
795 return ret;
796 }
797
798 /*
799 * helper function to lookup reference count and flags of a tree block.
800 *
801 * the head node for delayed ref is used to store the sum of all the
802 * reference count modifications queued up in the rbtree. the head
803 * node may also store the extent flags to set. This way you can check
804 * to see what the reference count and extent flags would be if all of
805 * the delayed refs are not processed.
806 */
807 int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
808 struct btrfs_fs_info *fs_info, u64 bytenr,
809 u64 offset, int metadata, u64 *refs, u64 *flags)
810 {
811 struct btrfs_delayed_ref_head *head;
812 struct btrfs_delayed_ref_root *delayed_refs;
813 struct btrfs_path *path;
814 struct btrfs_extent_item *ei;
815 struct extent_buffer *leaf;
816 struct btrfs_key key;
817 u32 item_size;
818 u64 num_refs;
819 u64 extent_flags;
820 int ret;
821
822 /*
823 * If we don't have skinny metadata, don't bother doing anything
824 * different
825 */
826 if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA)) {
827 offset = fs_info->nodesize;
828 metadata = 0;
829 }
830
831 path = btrfs_alloc_path();
832 if (!path)
833 return -ENOMEM;
834
835 if (!trans) {
836 path->skip_locking = 1;
837 path->search_commit_root = 1;
838 }
839
840 search_again:
841 key.objectid = bytenr;
842 key.offset = offset;
843 if (metadata)
844 key.type = BTRFS_METADATA_ITEM_KEY;
845 else
846 key.type = BTRFS_EXTENT_ITEM_KEY;
847
848 ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 0);
849 if (ret < 0)
850 goto out_free;
851
852 if (ret > 0 && metadata && key.type == BTRFS_METADATA_ITEM_KEY) {
853 if (path->slots[0]) {
854 path->slots[0]--;
855 btrfs_item_key_to_cpu(path->nodes[0], &key,
856 path->slots[0]);
857 if (key.objectid == bytenr &&
858 key.type == BTRFS_EXTENT_ITEM_KEY &&
859 key.offset == fs_info->nodesize)
860 ret = 0;
861 }
862 }
863
864 if (ret == 0) {
865 leaf = path->nodes[0];
866 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
867 if (item_size >= sizeof(*ei)) {
868 ei = btrfs_item_ptr(leaf, path->slots[0],
869 struct btrfs_extent_item);
870 num_refs = btrfs_extent_refs(leaf, ei);
871 extent_flags = btrfs_extent_flags(leaf, ei);
872 } else {
873 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
874 struct btrfs_extent_item_v0 *ei0;
875 BUG_ON(item_size != sizeof(*ei0));
876 ei0 = btrfs_item_ptr(leaf, path->slots[0],
877 struct btrfs_extent_item_v0);
878 num_refs = btrfs_extent_refs_v0(leaf, ei0);
879 /* FIXME: this isn't correct for data */
880 extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
881 #else
882 BUG();
883 #endif
884 }
885 BUG_ON(num_refs == 0);
886 } else {
887 num_refs = 0;
888 extent_flags = 0;
889 ret = 0;
890 }
891
892 if (!trans)
893 goto out;
894
895 delayed_refs = &trans->transaction->delayed_refs;
896 spin_lock(&delayed_refs->lock);
897 head = btrfs_find_delayed_ref_head(delayed_refs, bytenr);
898 if (head) {
899 if (!mutex_trylock(&head->mutex)) {
900 refcount_inc(&head->refs);
901 spin_unlock(&delayed_refs->lock);
902
903 btrfs_release_path(path);
904
905 /*
906 * Mutex was contended, block until it's released and try
907 * again
908 */
909 mutex_lock(&head->mutex);
910 mutex_unlock(&head->mutex);
911 btrfs_put_delayed_ref_head(head);
912 goto search_again;
913 }
914 spin_lock(&head->lock);
915 if (head->extent_op && head->extent_op->update_flags)
916 extent_flags |= head->extent_op->flags_to_set;
917 else
918 BUG_ON(num_refs == 0);
919
920 num_refs += head->ref_mod;
921 spin_unlock(&head->lock);
922 mutex_unlock(&head->mutex);
923 }
924 spin_unlock(&delayed_refs->lock);
925 out:
926 WARN_ON(num_refs == 0);
927 if (refs)
928 *refs = num_refs;
929 if (flags)
930 *flags = extent_flags;
931 out_free:
932 btrfs_free_path(path);
933 return ret;
934 }
935
936 /*
937 * Back reference rules. Back refs have three main goals:
938 *
939 * 1) differentiate between all holders of references to an extent so that
940 * when a reference is dropped we can make sure it was a valid reference
941 * before freeing the extent.
942 *
943 * 2) Provide enough information to quickly find the holders of an extent
944 * if we notice a given block is corrupted or bad.
945 *
946 * 3) Make it easy to migrate blocks for FS shrinking or storage pool
947 * maintenance. This is actually the same as #2, but with a slightly
948 * different use case.
949 *
950 * There are two kinds of back refs. The implicit back refs is optimized
951 * for pointers in non-shared tree blocks. For a given pointer in a block,
952 * back refs of this kind provide information about the block's owner tree
953 * and the pointer's key. These information allow us to find the block by
954 * b-tree searching. The full back refs is for pointers in tree blocks not
955 * referenced by their owner trees. The location of tree block is recorded
956 * in the back refs. Actually the full back refs is generic, and can be
957 * used in all cases the implicit back refs is used. The major shortcoming
958 * of the full back refs is its overhead. Every time a tree block gets
959 * COWed, we have to update back refs entry for all pointers in it.
960 *
961 * For a newly allocated tree block, we use implicit back refs for
962 * pointers in it. This means most tree related operations only involve
963 * implicit back refs. For a tree block created in old transaction, the
964 * only way to drop a reference to it is COW it. So we can detect the
965 * event that tree block loses its owner tree's reference and do the
966 * back refs conversion.
967 *
968 * When a tree block is COWed through a tree, there are four cases:
969 *
970 * The reference count of the block is one and the tree is the block's
971 * owner tree. Nothing to do in this case.
972 *
973 * The reference count of the block is one and the tree is not the
974 * block's owner tree. In this case, full back refs is used for pointers
975 * in the block. Remove these full back refs, add implicit back refs for
976 * every pointers in the new block.
977 *
978 * The reference count of the block is greater than one and the tree is
979 * the block's owner tree. In this case, implicit back refs is used for
980 * pointers in the block. Add full back refs for every pointers in the
981 * block, increase lower level extents' reference counts. The original
982 * implicit back refs are entailed to the new block.
983 *
984 * The reference count of the block is greater than one and the tree is
985 * not the block's owner tree. Add implicit back refs for every pointer in
986 * the new block, increase lower level extents' reference count.
987 *
988 * Back Reference Key composing:
989 *
990 * The key objectid corresponds to the first byte in the extent,
991 * The key type is used to differentiate between types of back refs.
992 * There are different meanings of the key offset for different types
993 * of back refs.
994 *
995 * File extents can be referenced by:
996 *
997 * - multiple snapshots, subvolumes, or different generations in one subvol
998 * - different files inside a single subvolume
999 * - different offsets inside a file (bookend extents in file.c)
1000 *
1001 * The extent ref structure for the implicit back refs has fields for:
1002 *
1003 * - Objectid of the subvolume root
1004 * - objectid of the file holding the reference
1005 * - original offset in the file
1006 * - how many bookend extents
1007 *
1008 * The key offset for the implicit back refs is hash of the first
1009 * three fields.
1010 *
1011 * The extent ref structure for the full back refs has field for:
1012 *
1013 * - number of pointers in the tree leaf
1014 *
1015 * The key offset for the implicit back refs is the first byte of
1016 * the tree leaf
1017 *
1018 * When a file extent is allocated, The implicit back refs is used.
1019 * the fields are filled in:
1020 *
1021 * (root_key.objectid, inode objectid, offset in file, 1)
1022 *
1023 * When a file extent is removed file truncation, we find the
1024 * corresponding implicit back refs and check the following fields:
1025 *
1026 * (btrfs_header_owner(leaf), inode objectid, offset in file)
1027 *
1028 * Btree extents can be referenced by:
1029 *
1030 * - Different subvolumes
1031 *
1032 * Both the implicit back refs and the full back refs for tree blocks
1033 * only consist of key. The key offset for the implicit back refs is
1034 * objectid of block's owner tree. The key offset for the full back refs
1035 * is the first byte of parent block.
1036 *
1037 * When implicit back refs is used, information about the lowest key and
1038 * level of the tree block are required. These information are stored in
1039 * tree block info structure.
1040 */
1041
1042 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1043 static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
1044 struct btrfs_fs_info *fs_info,
1045 struct btrfs_path *path,
1046 u64 owner, u32 extra_size)
1047 {
1048 struct btrfs_root *root = fs_info->extent_root;
1049 struct btrfs_extent_item *item;
1050 struct btrfs_extent_item_v0 *ei0;
1051 struct btrfs_extent_ref_v0 *ref0;
1052 struct btrfs_tree_block_info *bi;
1053 struct extent_buffer *leaf;
1054 struct btrfs_key key;
1055 struct btrfs_key found_key;
1056 u32 new_size = sizeof(*item);
1057 u64 refs;
1058 int ret;
1059
1060 leaf = path->nodes[0];
1061 BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));
1062
1063 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1064 ei0 = btrfs_item_ptr(leaf, path->slots[0],
1065 struct btrfs_extent_item_v0);
1066 refs = btrfs_extent_refs_v0(leaf, ei0);
1067
1068 if (owner == (u64)-1) {
1069 while (1) {
1070 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1071 ret = btrfs_next_leaf(root, path);
1072 if (ret < 0)
1073 return ret;
1074 BUG_ON(ret > 0); /* Corruption */
1075 leaf = path->nodes[0];
1076 }
1077 btrfs_item_key_to_cpu(leaf, &found_key,
1078 path->slots[0]);
1079 BUG_ON(key.objectid != found_key.objectid);
1080 if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
1081 path->slots[0]++;
1082 continue;
1083 }
1084 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1085 struct btrfs_extent_ref_v0);
1086 owner = btrfs_ref_objectid_v0(leaf, ref0);
1087 break;
1088 }
1089 }
1090 btrfs_release_path(path);
1091
1092 if (owner < BTRFS_FIRST_FREE_OBJECTID)
1093 new_size += sizeof(*bi);
1094
1095 new_size -= sizeof(*ei0);
1096 ret = btrfs_search_slot(trans, root, &key, path,
1097 new_size + extra_size, 1);
1098 if (ret < 0)
1099 return ret;
1100 BUG_ON(ret); /* Corruption */
1101
1102 btrfs_extend_item(fs_info, path, new_size);
1103
1104 leaf = path->nodes[0];
1105 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1106 btrfs_set_extent_refs(leaf, item, refs);
1107 /* FIXME: get real generation */
1108 btrfs_set_extent_generation(leaf, item, 0);
1109 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1110 btrfs_set_extent_flags(leaf, item,
1111 BTRFS_EXTENT_FLAG_TREE_BLOCK |
1112 BTRFS_BLOCK_FLAG_FULL_BACKREF);
1113 bi = (struct btrfs_tree_block_info *)(item + 1);
1114 /* FIXME: get first key of the block */
1115 memzero_extent_buffer(leaf, (unsigned long)bi, sizeof(*bi));
1116 btrfs_set_tree_block_level(leaf, bi, (int)owner);
1117 } else {
1118 btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
1119 }
1120 btrfs_mark_buffer_dirty(leaf);
1121 return 0;
1122 }
1123 #endif
1124
1125 /*
1126 * is_data == BTRFS_REF_TYPE_BLOCK, tree block type is required,
1127 * is_data == BTRFS_REF_TYPE_DATA, data type is requried,
1128 * is_data == BTRFS_REF_TYPE_ANY, either type is OK.
1129 */
1130 int btrfs_get_extent_inline_ref_type(const struct extent_buffer *eb,
1131 struct btrfs_extent_inline_ref *iref,
1132 enum btrfs_inline_ref_type is_data)
1133 {
1134 int type = btrfs_extent_inline_ref_type(eb, iref);
1135 u64 offset = btrfs_extent_inline_ref_offset(eb, iref);
1136
1137 if (type == BTRFS_TREE_BLOCK_REF_KEY ||
1138 type == BTRFS_SHARED_BLOCK_REF_KEY ||
1139 type == BTRFS_SHARED_DATA_REF_KEY ||
1140 type == BTRFS_EXTENT_DATA_REF_KEY) {
1141 if (is_data == BTRFS_REF_TYPE_BLOCK) {
1142 if (type == BTRFS_TREE_BLOCK_REF_KEY)
1143 return type;
1144 if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
1145 ASSERT(eb->fs_info);
1146 /*
1147 * Every shared one has parent tree
1148 * block, which must be aligned to
1149 * nodesize.
1150 */
1151 if (offset &&
1152 IS_ALIGNED(offset, eb->fs_info->nodesize))
1153 return type;
1154 }
1155 } else if (is_data == BTRFS_REF_TYPE_DATA) {
1156 if (type == BTRFS_EXTENT_DATA_REF_KEY)
1157 return type;
1158 if (type == BTRFS_SHARED_DATA_REF_KEY) {
1159 ASSERT(eb->fs_info);
1160 /*
1161 * Every shared one has parent tree
1162 * block, which must be aligned to
1163 * nodesize.
1164 */
1165 if (offset &&
1166 IS_ALIGNED(offset, eb->fs_info->nodesize))
1167 return type;
1168 }
1169 } else {
1170 ASSERT(is_data == BTRFS_REF_TYPE_ANY);
1171 return type;
1172 }
1173 }
1174
1175 btrfs_print_leaf((struct extent_buffer *)eb);
1176 btrfs_err(eb->fs_info, "eb %llu invalid extent inline ref type %d",
1177 eb->start, type);
1178 WARN_ON(1);
1179
1180 return BTRFS_REF_TYPE_INVALID;
1181 }
1182
1183 static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
1184 {
1185 u32 high_crc = ~(u32)0;
1186 u32 low_crc = ~(u32)0;
1187 __le64 lenum;
1188
1189 lenum = cpu_to_le64(root_objectid);
1190 high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1191 lenum = cpu_to_le64(owner);
1192 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1193 lenum = cpu_to_le64(offset);
1194 low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1195
1196 return ((u64)high_crc << 31) ^ (u64)low_crc;
1197 }
1198
1199 static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
1200 struct btrfs_extent_data_ref *ref)
1201 {
1202 return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
1203 btrfs_extent_data_ref_objectid(leaf, ref),
1204 btrfs_extent_data_ref_offset(leaf, ref));
1205 }
1206
1207 static int match_extent_data_ref(struct extent_buffer *leaf,
1208 struct btrfs_extent_data_ref *ref,
1209 u64 root_objectid, u64 owner, u64 offset)
1210 {
1211 if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
1212 btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
1213 btrfs_extent_data_ref_offset(leaf, ref) != offset)
1214 return 0;
1215 return 1;
1216 }
1217
1218 static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
1219 struct btrfs_fs_info *fs_info,
1220 struct btrfs_path *path,
1221 u64 bytenr, u64 parent,
1222 u64 root_objectid,
1223 u64 owner, u64 offset)
1224 {
1225 struct btrfs_root *root = fs_info->extent_root;
1226 struct btrfs_key key;
1227 struct btrfs_extent_data_ref *ref;
1228 struct extent_buffer *leaf;
1229 u32 nritems;
1230 int ret;
1231 int recow;
1232 int err = -ENOENT;
1233
1234 key.objectid = bytenr;
1235 if (parent) {
1236 key.type = BTRFS_SHARED_DATA_REF_KEY;
1237 key.offset = parent;
1238 } else {
1239 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1240 key.offset = hash_extent_data_ref(root_objectid,
1241 owner, offset);
1242 }
1243 again:
1244 recow = 0;
1245 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1246 if (ret < 0) {
1247 err = ret;
1248 goto fail;
1249 }
1250
1251 if (parent) {
1252 if (!ret)
1253 return 0;
1254 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1255 key.type = BTRFS_EXTENT_REF_V0_KEY;
1256 btrfs_release_path(path);
1257 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1258 if (ret < 0) {
1259 err = ret;
1260 goto fail;
1261 }
1262 if (!ret)
1263 return 0;
1264 #endif
1265 goto fail;
1266 }
1267
1268 leaf = path->nodes[0];
1269 nritems = btrfs_header_nritems(leaf);
1270 while (1) {
1271 if (path->slots[0] >= nritems) {
1272 ret = btrfs_next_leaf(root, path);
1273 if (ret < 0)
1274 err = ret;
1275 if (ret)
1276 goto fail;
1277
1278 leaf = path->nodes[0];
1279 nritems = btrfs_header_nritems(leaf);
1280 recow = 1;
1281 }
1282
1283 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1284 if (key.objectid != bytenr ||
1285 key.type != BTRFS_EXTENT_DATA_REF_KEY)
1286 goto fail;
1287
1288 ref = btrfs_item_ptr(leaf, path->slots[0],
1289 struct btrfs_extent_data_ref);
1290
1291 if (match_extent_data_ref(leaf, ref, root_objectid,
1292 owner, offset)) {
1293 if (recow) {
1294 btrfs_release_path(path);
1295 goto again;
1296 }
1297 err = 0;
1298 break;
1299 }
1300 path->slots[0]++;
1301 }
1302 fail:
1303 return err;
1304 }
1305
1306 static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
1307 struct btrfs_fs_info *fs_info,
1308 struct btrfs_path *path,
1309 u64 bytenr, u64 parent,
1310 u64 root_objectid, u64 owner,
1311 u64 offset, int refs_to_add)
1312 {
1313 struct btrfs_root *root = fs_info->extent_root;
1314 struct btrfs_key key;
1315 struct extent_buffer *leaf;
1316 u32 size;
1317 u32 num_refs;
1318 int ret;
1319
1320 key.objectid = bytenr;
1321 if (parent) {
1322 key.type = BTRFS_SHARED_DATA_REF_KEY;
1323 key.offset = parent;
1324 size = sizeof(struct btrfs_shared_data_ref);
1325 } else {
1326 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1327 key.offset = hash_extent_data_ref(root_objectid,
1328 owner, offset);
1329 size = sizeof(struct btrfs_extent_data_ref);
1330 }
1331
1332 ret = btrfs_insert_empty_item(trans, root, path, &key, size);
1333 if (ret && ret != -EEXIST)
1334 goto fail;
1335
1336 leaf = path->nodes[0];
1337 if (parent) {
1338 struct btrfs_shared_data_ref *ref;
1339 ref = btrfs_item_ptr(leaf, path->slots[0],
1340 struct btrfs_shared_data_ref);
1341 if (ret == 0) {
1342 btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
1343 } else {
1344 num_refs = btrfs_shared_data_ref_count(leaf, ref);
1345 num_refs += refs_to_add;
1346 btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
1347 }
1348 } else {
1349 struct btrfs_extent_data_ref *ref;
1350 while (ret == -EEXIST) {
1351 ref = btrfs_item_ptr(leaf, path->slots[0],
1352 struct btrfs_extent_data_ref);
1353 if (match_extent_data_ref(leaf, ref, root_objectid,
1354 owner, offset))
1355 break;
1356 btrfs_release_path(path);
1357 key.offset++;
1358 ret = btrfs_insert_empty_item(trans, root, path, &key,
1359 size);
1360 if (ret && ret != -EEXIST)
1361 goto fail;
1362
1363 leaf = path->nodes[0];
1364 }
1365 ref = btrfs_item_ptr(leaf, path->slots[0],
1366 struct btrfs_extent_data_ref);
1367 if (ret == 0) {
1368 btrfs_set_extent_data_ref_root(leaf, ref,
1369 root_objectid);
1370 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
1371 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
1372 btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
1373 } else {
1374 num_refs = btrfs_extent_data_ref_count(leaf, ref);
1375 num_refs += refs_to_add;
1376 btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
1377 }
1378 }
1379 btrfs_mark_buffer_dirty(leaf);
1380 ret = 0;
1381 fail:
1382 btrfs_release_path(path);
1383 return ret;
1384 }
1385
1386 static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
1387 struct btrfs_fs_info *fs_info,
1388 struct btrfs_path *path,
1389 int refs_to_drop, int *last_ref)
1390 {
1391 struct btrfs_key key;
1392 struct btrfs_extent_data_ref *ref1 = NULL;
1393 struct btrfs_shared_data_ref *ref2 = NULL;
1394 struct extent_buffer *leaf;
1395 u32 num_refs = 0;
1396 int ret = 0;
1397
1398 leaf = path->nodes[0];
1399 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1400
1401 if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1402 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1403 struct btrfs_extent_data_ref);
1404 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1405 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1406 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1407 struct btrfs_shared_data_ref);
1408 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1409 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1410 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1411 struct btrfs_extent_ref_v0 *ref0;
1412 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1413 struct btrfs_extent_ref_v0);
1414 num_refs = btrfs_ref_count_v0(leaf, ref0);
1415 #endif
1416 } else {
1417 BUG();
1418 }
1419
1420 BUG_ON(num_refs < refs_to_drop);
1421 num_refs -= refs_to_drop;
1422
1423 if (num_refs == 0) {
1424 ret = btrfs_del_item(trans, fs_info->extent_root, path);
1425 *last_ref = 1;
1426 } else {
1427 if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
1428 btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
1429 else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
1430 btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
1431 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1432 else {
1433 struct btrfs_extent_ref_v0 *ref0;
1434 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1435 struct btrfs_extent_ref_v0);
1436 btrfs_set_ref_count_v0(leaf, ref0, num_refs);
1437 }
1438 #endif
1439 btrfs_mark_buffer_dirty(leaf);
1440 }
1441 return ret;
1442 }
1443
1444 static noinline u32 extent_data_ref_count(struct btrfs_path *path,
1445 struct btrfs_extent_inline_ref *iref)
1446 {
1447 struct btrfs_key key;
1448 struct extent_buffer *leaf;
1449 struct btrfs_extent_data_ref *ref1;
1450 struct btrfs_shared_data_ref *ref2;
1451 u32 num_refs = 0;
1452 int type;
1453
1454 leaf = path->nodes[0];
1455 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1456 if (iref) {
1457 /*
1458 * If type is invalid, we should have bailed out earlier than
1459 * this call.
1460 */
1461 type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA);
1462 ASSERT(type != BTRFS_REF_TYPE_INVALID);
1463 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1464 ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
1465 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1466 } else {
1467 ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
1468 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1469 }
1470 } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1471 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1472 struct btrfs_extent_data_ref);
1473 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1474 } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1475 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1476 struct btrfs_shared_data_ref);
1477 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1478 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1479 } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1480 struct btrfs_extent_ref_v0 *ref0;
1481 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1482 struct btrfs_extent_ref_v0);
1483 num_refs = btrfs_ref_count_v0(leaf, ref0);
1484 #endif
1485 } else {
1486 WARN_ON(1);
1487 }
1488 return num_refs;
1489 }
1490
1491 static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
1492 struct btrfs_fs_info *fs_info,
1493 struct btrfs_path *path,
1494 u64 bytenr, u64 parent,
1495 u64 root_objectid)
1496 {
1497 struct btrfs_root *root = fs_info->extent_root;
1498 struct btrfs_key key;
1499 int ret;
1500
1501 key.objectid = bytenr;
1502 if (parent) {
1503 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1504 key.offset = parent;
1505 } else {
1506 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1507 key.offset = root_objectid;
1508 }
1509
1510 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1511 if (ret > 0)
1512 ret = -ENOENT;
1513 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1514 if (ret == -ENOENT && parent) {
1515 btrfs_release_path(path);
1516 key.type = BTRFS_EXTENT_REF_V0_KEY;
1517 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1518 if (ret > 0)
1519 ret = -ENOENT;
1520 }
1521 #endif
1522 return ret;
1523 }
1524
1525 static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
1526 struct btrfs_fs_info *fs_info,
1527 struct btrfs_path *path,
1528 u64 bytenr, u64 parent,
1529 u64 root_objectid)
1530 {
1531 struct btrfs_key key;
1532 int ret;
1533
1534 key.objectid = bytenr;
1535 if (parent) {
1536 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1537 key.offset = parent;
1538 } else {
1539 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1540 key.offset = root_objectid;
1541 }
1542
1543 ret = btrfs_insert_empty_item(trans, fs_info->extent_root,
1544 path, &key, 0);
1545 btrfs_release_path(path);
1546 return ret;
1547 }
1548
1549 static inline int extent_ref_type(u64 parent, u64 owner)
1550 {
1551 int type;
1552 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1553 if (parent > 0)
1554 type = BTRFS_SHARED_BLOCK_REF_KEY;
1555 else
1556 type = BTRFS_TREE_BLOCK_REF_KEY;
1557 } else {
1558 if (parent > 0)
1559 type = BTRFS_SHARED_DATA_REF_KEY;
1560 else
1561 type = BTRFS_EXTENT_DATA_REF_KEY;
1562 }
1563 return type;
1564 }
1565
1566 static int find_next_key(struct btrfs_path *path, int level,
1567 struct btrfs_key *key)
1568
1569 {
1570 for (; level < BTRFS_MAX_LEVEL; level++) {
1571 if (!path->nodes[level])
1572 break;
1573 if (path->slots[level] + 1 >=
1574 btrfs_header_nritems(path->nodes[level]))
1575 continue;
1576 if (level == 0)
1577 btrfs_item_key_to_cpu(path->nodes[level], key,
1578 path->slots[level] + 1);
1579 else
1580 btrfs_node_key_to_cpu(path->nodes[level], key,
1581 path->slots[level] + 1);
1582 return 0;
1583 }
1584 return 1;
1585 }
1586
1587 /*
1588 * look for inline back ref. if back ref is found, *ref_ret is set
1589 * to the address of inline back ref, and 0 is returned.
1590 *
1591 * if back ref isn't found, *ref_ret is set to the address where it
1592 * should be inserted, and -ENOENT is returned.
1593 *
1594 * if insert is true and there are too many inline back refs, the path
1595 * points to the extent item, and -EAGAIN is returned.
1596 *
1597 * NOTE: inline back refs are ordered in the same way that back ref
1598 * items in the tree are ordered.
1599 */
1600 static noinline_for_stack
1601 int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
1602 struct btrfs_fs_info *fs_info,
1603 struct btrfs_path *path,
1604 struct btrfs_extent_inline_ref **ref_ret,
1605 u64 bytenr, u64 num_bytes,
1606 u64 parent, u64 root_objectid,
1607 u64 owner, u64 offset, int insert)
1608 {
1609 struct btrfs_root *root = fs_info->extent_root;
1610 struct btrfs_key key;
1611 struct extent_buffer *leaf;
1612 struct btrfs_extent_item *ei;
1613 struct btrfs_extent_inline_ref *iref;
1614 u64 flags;
1615 u64 item_size;
1616 unsigned long ptr;
1617 unsigned long end;
1618 int extra_size;
1619 int type;
1620 int want;
1621 int ret;
1622 int err = 0;
1623 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA);
1624 int needed;
1625
1626 key.objectid = bytenr;
1627 key.type = BTRFS_EXTENT_ITEM_KEY;
1628 key.offset = num_bytes;
1629
1630 want = extent_ref_type(parent, owner);
1631 if (insert) {
1632 extra_size = btrfs_extent_inline_ref_size(want);
1633 path->keep_locks = 1;
1634 } else
1635 extra_size = -1;
1636
1637 /*
1638 * Owner is our parent level, so we can just add one to get the level
1639 * for the block we are interested in.
1640 */
1641 if (skinny_metadata && owner < BTRFS_FIRST_FREE_OBJECTID) {
1642 key.type = BTRFS_METADATA_ITEM_KEY;
1643 key.offset = owner;
1644 }
1645
1646 again:
1647 ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1648 if (ret < 0) {
1649 err = ret;
1650 goto out;
1651 }
1652
1653 /*
1654 * We may be a newly converted file system which still has the old fat
1655 * extent entries for metadata, so try and see if we have one of those.
1656 */
1657 if (ret > 0 && skinny_metadata) {
1658 skinny_metadata = false;
1659 if (path->slots[0]) {
1660 path->slots[0]--;
1661 btrfs_item_key_to_cpu(path->nodes[0], &key,
1662 path->slots[0]);
1663 if (key.objectid == bytenr &&
1664 key.type == BTRFS_EXTENT_ITEM_KEY &&
1665 key.offset == num_bytes)
1666 ret = 0;
1667 }
1668 if (ret) {
1669 key.objectid = bytenr;
1670 key.type = BTRFS_EXTENT_ITEM_KEY;
1671 key.offset = num_bytes;
1672 btrfs_release_path(path);
1673 goto again;
1674 }
1675 }
1676
1677 if (ret && !insert) {
1678 err = -ENOENT;
1679 goto out;
1680 } else if (WARN_ON(ret)) {
1681 err = -EIO;
1682 goto out;
1683 }
1684
1685 leaf = path->nodes[0];
1686 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1687 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1688 if (item_size < sizeof(*ei)) {
1689 if (!insert) {
1690 err = -ENOENT;
1691 goto out;
1692 }
1693 ret = convert_extent_item_v0(trans, fs_info, path, owner,
1694 extra_size);
1695 if (ret < 0) {
1696 err = ret;
1697 goto out;
1698 }
1699 leaf = path->nodes[0];
1700 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1701 }
1702 #endif
1703 BUG_ON(item_size < sizeof(*ei));
1704
1705 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1706 flags = btrfs_extent_flags(leaf, ei);
1707
1708 ptr = (unsigned long)(ei + 1);
1709 end = (unsigned long)ei + item_size;
1710
1711 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK && !skinny_metadata) {
1712 ptr += sizeof(struct btrfs_tree_block_info);
1713 BUG_ON(ptr > end);
1714 }
1715
1716 if (owner >= BTRFS_FIRST_FREE_OBJECTID)
1717 needed = BTRFS_REF_TYPE_DATA;
1718 else
1719 needed = BTRFS_REF_TYPE_BLOCK;
1720
1721 err = -ENOENT;
1722 while (1) {
1723 if (ptr >= end) {
1724 WARN_ON(ptr > end);
1725 break;
1726 }
1727 iref = (struct btrfs_extent_inline_ref *)ptr;
1728 type = btrfs_get_extent_inline_ref_type(leaf, iref, needed);
1729 if (type == BTRFS_REF_TYPE_INVALID) {
1730 err = -EINVAL;
1731 goto out;
1732 }
1733
1734 if (want < type)
1735 break;
1736 if (want > type) {
1737 ptr += btrfs_extent_inline_ref_size(type);
1738 continue;
1739 }
1740
1741 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1742 struct btrfs_extent_data_ref *dref;
1743 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1744 if (match_extent_data_ref(leaf, dref, root_objectid,
1745 owner, offset)) {
1746 err = 0;
1747 break;
1748 }
1749 if (hash_extent_data_ref_item(leaf, dref) <
1750 hash_extent_data_ref(root_objectid, owner, offset))
1751 break;
1752 } else {
1753 u64 ref_offset;
1754 ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
1755 if (parent > 0) {
1756 if (parent == ref_offset) {
1757 err = 0;
1758 break;
1759 }
1760 if (ref_offset < parent)
1761 break;
1762 } else {
1763 if (root_objectid == ref_offset) {
1764 err = 0;
1765 break;
1766 }
1767 if (ref_offset < root_objectid)
1768 break;
1769 }
1770 }
1771 ptr += btrfs_extent_inline_ref_size(type);
1772 }
1773 if (err == -ENOENT && insert) {
1774 if (item_size + extra_size >=
1775 BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
1776 err = -EAGAIN;
1777 goto out;
1778 }
1779 /*
1780 * To add new inline back ref, we have to make sure
1781 * there is no corresponding back ref item.
1782 * For simplicity, we just do not add new inline back
1783 * ref if there is any kind of item for this block
1784 */
1785 if (find_next_key(path, 0, &key) == 0 &&
1786 key.objectid == bytenr &&
1787 key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1788 err = -EAGAIN;
1789 goto out;
1790 }
1791 }
1792 *ref_ret = (struct btrfs_extent_inline_ref *)ptr;
1793 out:
1794 if (insert) {
1795 path->keep_locks = 0;
1796 btrfs_unlock_up_safe(path, 1);
1797 }
1798 return err;
1799 }
1800
1801 /*
1802 * helper to add new inline back ref
1803 */
1804 static noinline_for_stack
1805 void setup_inline_extent_backref(struct btrfs_fs_info *fs_info,
1806 struct btrfs_path *path,
1807 struct btrfs_extent_inline_ref *iref,
1808 u64 parent, u64 root_objectid,
1809 u64 owner, u64 offset, int refs_to_add,
1810 struct btrfs_delayed_extent_op *extent_op)
1811 {
1812 struct extent_buffer *leaf;
1813 struct btrfs_extent_item *ei;
1814 unsigned long ptr;
1815 unsigned long end;
1816 unsigned long item_offset;
1817 u64 refs;
1818 int size;
1819 int type;
1820
1821 leaf = path->nodes[0];
1822 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1823 item_offset = (unsigned long)iref - (unsigned long)ei;
1824
1825 type = extent_ref_type(parent, owner);
1826 size = btrfs_extent_inline_ref_size(type);
1827
1828 btrfs_extend_item(fs_info, path, size);
1829
1830 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1831 refs = btrfs_extent_refs(leaf, ei);
1832 refs += refs_to_add;
1833 btrfs_set_extent_refs(leaf, ei, refs);
1834 if (extent_op)
1835 __run_delayed_extent_op(extent_op, leaf, ei);
1836
1837 ptr = (unsigned long)ei + item_offset;
1838 end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
1839 if (ptr < end - size)
1840 memmove_extent_buffer(leaf, ptr + size, ptr,
1841 end - size - ptr);
1842
1843 iref = (struct btrfs_extent_inline_ref *)ptr;
1844 btrfs_set_extent_inline_ref_type(leaf, iref, type);
1845 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1846 struct btrfs_extent_data_ref *dref;
1847 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1848 btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
1849 btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
1850 btrfs_set_extent_data_ref_offset(leaf, dref, offset);
1851 btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
1852 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1853 struct btrfs_shared_data_ref *sref;
1854 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1855 btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
1856 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1857 } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
1858 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1859 } else {
1860 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
1861 }
1862 btrfs_mark_buffer_dirty(leaf);
1863 }
1864
1865 static int lookup_extent_backref(struct btrfs_trans_handle *trans,
1866 struct btrfs_fs_info *fs_info,
1867 struct btrfs_path *path,
1868 struct btrfs_extent_inline_ref **ref_ret,
1869 u64 bytenr, u64 num_bytes, u64 parent,
1870 u64 root_objectid, u64 owner, u64 offset)
1871 {
1872 int ret;
1873
1874 ret = lookup_inline_extent_backref(trans, fs_info, path, ref_ret,
1875 bytenr, num_bytes, parent,
1876 root_objectid, owner, offset, 0);
1877 if (ret != -ENOENT)
1878 return ret;
1879
1880 btrfs_release_path(path);
1881 *ref_ret = NULL;
1882
1883 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1884 ret = lookup_tree_block_ref(trans, fs_info, path, bytenr,
1885 parent, root_objectid);
1886 } else {
1887 ret = lookup_extent_data_ref(trans, fs_info, path, bytenr,
1888 parent, root_objectid, owner,
1889 offset);
1890 }
1891 return ret;
1892 }
1893
1894 /*
1895 * helper to update/remove inline back ref
1896 */
1897 static noinline_for_stack
1898 void update_inline_extent_backref(struct btrfs_fs_info *fs_info,
1899 struct btrfs_path *path,
1900 struct btrfs_extent_inline_ref *iref,
1901 int refs_to_mod,
1902 struct btrfs_delayed_extent_op *extent_op,
1903 int *last_ref)
1904 {
1905 struct extent_buffer *leaf;
1906 struct btrfs_extent_item *ei;
1907 struct btrfs_extent_data_ref *dref = NULL;
1908 struct btrfs_shared_data_ref *sref = NULL;
1909 unsigned long ptr;
1910 unsigned long end;
1911 u32 item_size;
1912 int size;
1913 int type;
1914 u64 refs;
1915
1916 leaf = path->nodes[0];
1917 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1918 refs = btrfs_extent_refs(leaf, ei);
1919 WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
1920 refs += refs_to_mod;
1921 btrfs_set_extent_refs(leaf, ei, refs);
1922 if (extent_op)
1923 __run_delayed_extent_op(extent_op, leaf, ei);
1924
1925 /*
1926 * If type is invalid, we should have bailed out after
1927 * lookup_inline_extent_backref().
1928 */
1929 type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_ANY);
1930 ASSERT(type != BTRFS_REF_TYPE_INVALID);
1931
1932 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1933 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1934 refs = btrfs_extent_data_ref_count(leaf, dref);
1935 } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1936 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1937 refs = btrfs_shared_data_ref_count(leaf, sref);
1938 } else {
1939 refs = 1;
1940 BUG_ON(refs_to_mod != -1);
1941 }
1942
1943 BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
1944 refs += refs_to_mod;
1945
1946 if (refs > 0) {
1947 if (type == BTRFS_EXTENT_DATA_REF_KEY)
1948 btrfs_set_extent_data_ref_count(leaf, dref, refs);
1949 else
1950 btrfs_set_shared_data_ref_count(leaf, sref, refs);
1951 } else {
1952 *last_ref = 1;
1953 size = btrfs_extent_inline_ref_size(type);
1954 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1955 ptr = (unsigned long)iref;
1956 end = (unsigned long)ei + item_size;
1957 if (ptr + size < end)
1958 memmove_extent_buffer(leaf, ptr, ptr + size,
1959 end - ptr - size);
1960 item_size -= size;
1961 btrfs_truncate_item(fs_info, path, item_size, 1);
1962 }
1963 btrfs_mark_buffer_dirty(leaf);
1964 }
1965
1966 static noinline_for_stack
1967 int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
1968 struct btrfs_fs_info *fs_info,
1969 struct btrfs_path *path,
1970 u64 bytenr, u64 num_bytes, u64 parent,
1971 u64 root_objectid, u64 owner,
1972 u64 offset, int refs_to_add,
1973 struct btrfs_delayed_extent_op *extent_op)
1974 {
1975 struct btrfs_extent_inline_ref *iref;
1976 int ret;
1977
1978 ret = lookup_inline_extent_backref(trans, fs_info, path, &iref,
1979 bytenr, num_bytes, parent,
1980 root_objectid, owner, offset, 1);
1981 if (ret == 0) {
1982 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1983 update_inline_extent_backref(fs_info, path, iref,
1984 refs_to_add, extent_op, NULL);
1985 } else if (ret == -ENOENT) {
1986 setup_inline_extent_backref(fs_info, path, iref, parent,
1987 root_objectid, owner, offset,
1988 refs_to_add, extent_op);
1989 ret = 0;
1990 }
1991 return ret;
1992 }
1993
1994 static int insert_extent_backref(struct btrfs_trans_handle *trans,
1995 struct btrfs_fs_info *fs_info,
1996 struct btrfs_path *path,
1997 u64 bytenr, u64 parent, u64 root_objectid,
1998 u64 owner, u64 offset, int refs_to_add)
1999 {
2000 int ret;
2001 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
2002 BUG_ON(refs_to_add != 1);
2003 ret = insert_tree_block_ref(trans, fs_info, path, bytenr,
2004 parent, root_objectid);
2005 } else {
2006 ret = insert_extent_data_ref(trans, fs_info, path, bytenr,
2007 parent, root_objectid,
2008 owner, offset, refs_to_add);
2009 }
2010 return ret;
2011 }
2012
2013 static int remove_extent_backref(struct btrfs_trans_handle *trans,
2014 struct btrfs_fs_info *fs_info,
2015 struct btrfs_path *path,
2016 struct btrfs_extent_inline_ref *iref,
2017 int refs_to_drop, int is_data, int *last_ref)
2018 {
2019 int ret = 0;
2020
2021 BUG_ON(!is_data && refs_to_drop != 1);
2022 if (iref) {
2023 update_inline_extent_backref(fs_info, path, iref,
2024 -refs_to_drop, NULL, last_ref);
2025 } else if (is_data) {
2026 ret = remove_extent_data_ref(trans, fs_info, path, refs_to_drop,
2027 last_ref);
2028 } else {
2029 *last_ref = 1;
2030 ret = btrfs_del_item(trans, fs_info->extent_root, path);
2031 }
2032 return ret;
2033 }
2034
2035 #define in_range(b, first, len) ((b) >= (first) && (b) < (first) + (len))
2036 static int btrfs_issue_discard(struct block_device *bdev, u64 start, u64 len,
2037 u64 *discarded_bytes)
2038 {
2039 int j, ret = 0;
2040 u64 bytes_left, end;
2041 u64 aligned_start = ALIGN(start, 1 << 9);
2042
2043 if (WARN_ON(start != aligned_start)) {
2044 len -= aligned_start - start;
2045 len = round_down(len, 1 << 9);
2046 start = aligned_start;
2047 }
2048
2049 *discarded_bytes = 0;
2050
2051 if (!len)
2052 return 0;
2053
2054 end = start + len;
2055 bytes_left = len;
2056
2057 /* Skip any superblocks on this device. */
2058 for (j = 0; j < BTRFS_SUPER_MIRROR_MAX; j++) {
2059 u64 sb_start = btrfs_sb_offset(j);
2060 u64 sb_end = sb_start + BTRFS_SUPER_INFO_SIZE;
2061 u64 size = sb_start - start;
2062
2063 if (!in_range(sb_start, start, bytes_left) &&
2064 !in_range(sb_end, start, bytes_left) &&
2065 !in_range(start, sb_start, BTRFS_SUPER_INFO_SIZE))
2066 continue;
2067
2068 /*
2069 * Superblock spans beginning of range. Adjust start and
2070 * try again.
2071 */
2072 if (sb_start <= start) {
2073 start += sb_end - start;
2074 if (start > end) {
2075 bytes_left = 0;
2076 break;
2077 }
2078 bytes_left = end - start;
2079 continue;
2080 }
2081
2082 if (size) {
2083 ret = blkdev_issue_discard(bdev, start >> 9, size >> 9,
2084 GFP_NOFS, 0);
2085 if (!ret)
2086 *discarded_bytes += size;
2087 else if (ret != -EOPNOTSUPP)
2088 return ret;
2089 }
2090
2091 start = sb_end;
2092 if (start > end) {
2093 bytes_left = 0;
2094 break;
2095 }
2096 bytes_left = end - start;
2097 }
2098
2099 if (bytes_left) {
2100 ret = blkdev_issue_discard(bdev, start >> 9, bytes_left >> 9,
2101 GFP_NOFS, 0);
2102 if (!ret)
2103 *discarded_bytes += bytes_left;
2104 }
2105 return ret;
2106 }
2107
2108 int btrfs_discard_extent(struct btrfs_fs_info *fs_info, u64 bytenr,
2109 u64 num_bytes, u64 *actual_bytes)
2110 {
2111 int ret;
2112 u64 discarded_bytes = 0;
2113 struct btrfs_bio *bbio = NULL;
2114
2115
2116 /*
2117 * Avoid races with device replace and make sure our bbio has devices
2118 * associated to its stripes that don't go away while we are discarding.
2119 */
2120 btrfs_bio_counter_inc_blocked(fs_info);
2121 /* Tell the block device(s) that the sectors can be discarded */
2122 ret = btrfs_map_block(fs_info, BTRFS_MAP_DISCARD, bytenr, &num_bytes,
2123 &bbio, 0);
2124 /* Error condition is -ENOMEM */
2125 if (!ret) {
2126 struct btrfs_bio_stripe *stripe = bbio->stripes;
2127 int i;
2128
2129
2130 for (i = 0; i < bbio->num_stripes; i++, stripe++) {
2131 u64 bytes;
2132 struct request_queue *req_q;
2133
2134 if (!stripe->dev->bdev) {
2135 ASSERT(btrfs_test_opt(fs_info, DEGRADED));
2136 continue;
2137 }
2138 req_q = bdev_get_queue(stripe->dev->bdev);
2139 if (!blk_queue_discard(req_q))
2140 continue;
2141
2142 ret = btrfs_issue_discard(stripe->dev->bdev,
2143 stripe->physical,
2144 stripe->length,
2145 &bytes);
2146 if (!ret)
2147 discarded_bytes += bytes;
2148 else if (ret != -EOPNOTSUPP)
2149 break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
2150
2151 /*
2152 * Just in case we get back EOPNOTSUPP for some reason,
2153 * just ignore the return value so we don't screw up
2154 * people calling discard_extent.
2155 */
2156 ret = 0;
2157 }
2158 btrfs_put_bbio(bbio);
2159 }
2160 btrfs_bio_counter_dec(fs_info);
2161
2162 if (actual_bytes)
2163 *actual_bytes = discarded_bytes;
2164
2165
2166 if (ret == -EOPNOTSUPP)
2167 ret = 0;
2168 return ret;
2169 }
2170
2171 /* Can return -ENOMEM */
2172 int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
2173 struct btrfs_root *root,
2174 u64 bytenr, u64 num_bytes, u64 parent,
2175 u64 root_objectid, u64 owner, u64 offset)
2176 {
2177 struct btrfs_fs_info *fs_info = root->fs_info;
2178 int old_ref_mod, new_ref_mod;
2179 int ret;
2180
2181 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
2182 root_objectid == BTRFS_TREE_LOG_OBJECTID);
2183
2184 btrfs_ref_tree_mod(root, bytenr, num_bytes, parent, root_objectid,
2185 owner, offset, BTRFS_ADD_DELAYED_REF);
2186
2187 if (owner < BTRFS_FIRST_FREE_OBJECTID) {
2188 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
2189 num_bytes, parent,
2190 root_objectid, (int)owner,
2191 BTRFS_ADD_DELAYED_REF, NULL,
2192 &old_ref_mod, &new_ref_mod);
2193 } else {
2194 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
2195 num_bytes, parent,
2196 root_objectid, owner, offset,
2197 0, BTRFS_ADD_DELAYED_REF,
2198 &old_ref_mod, &new_ref_mod);
2199 }
2200
2201 if (ret == 0 && old_ref_mod < 0 && new_ref_mod >= 0) {
2202 bool metadata = owner < BTRFS_FIRST_FREE_OBJECTID;
2203
2204 add_pinned_bytes(fs_info, -num_bytes, metadata, root_objectid);
2205 }
2206
2207 return ret;
2208 }
2209
2210 static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
2211 struct btrfs_fs_info *fs_info,
2212 struct btrfs_delayed_ref_node *node,
2213 u64 parent, u64 root_objectid,
2214 u64 owner, u64 offset, int refs_to_add,
2215 struct btrfs_delayed_extent_op *extent_op)
2216 {
2217 struct btrfs_path *path;
2218 struct extent_buffer *leaf;
2219 struct btrfs_extent_item *item;
2220 struct btrfs_key key;
2221 u64 bytenr = node->bytenr;
2222 u64 num_bytes = node->num_bytes;
2223 u64 refs;
2224 int ret;
2225
2226 path = btrfs_alloc_path();
2227 if (!path)
2228 return -ENOMEM;
2229
2230 path->reada = READA_FORWARD;
2231 path->leave_spinning = 1;
2232 /* this will setup the path even if it fails to insert the back ref */
2233 ret = insert_inline_extent_backref(trans, fs_info, path, bytenr,
2234 num_bytes, parent, root_objectid,
2235 owner, offset,
2236 refs_to_add, extent_op);
2237 if ((ret < 0 && ret != -EAGAIN) || !ret)
2238 goto out;
2239
2240 /*
2241 * Ok we had -EAGAIN which means we didn't have space to insert and
2242 * inline extent ref, so just update the reference count and add a
2243 * normal backref.
2244 */
2245 leaf = path->nodes[0];
2246 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2247 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2248 refs = btrfs_extent_refs(leaf, item);
2249 btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
2250 if (extent_op)
2251 __run_delayed_extent_op(extent_op, leaf, item);
2252
2253 btrfs_mark_buffer_dirty(leaf);
2254 btrfs_release_path(path);
2255
2256 path->reada = READA_FORWARD;
2257 path->leave_spinning = 1;
2258 /* now insert the actual backref */
2259 ret = insert_extent_backref(trans, fs_info, path, bytenr, parent,
2260 root_objectid, owner, offset, refs_to_add);
2261 if (ret)
2262 btrfs_abort_transaction(trans, ret);
2263 out:
2264 btrfs_free_path(path);
2265 return ret;
2266 }
2267
2268 static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
2269 struct btrfs_fs_info *fs_info,
2270 struct btrfs_delayed_ref_node *node,
2271 struct btrfs_delayed_extent_op *extent_op,
2272 int insert_reserved)
2273 {
2274 int ret = 0;
2275 struct btrfs_delayed_data_ref *ref;
2276 struct btrfs_key ins;
2277 u64 parent = 0;
2278 u64 ref_root = 0;
2279 u64 flags = 0;
2280
2281 ins.objectid = node->bytenr;
2282 ins.offset = node->num_bytes;
2283 ins.type = BTRFS_EXTENT_ITEM_KEY;
2284
2285 ref = btrfs_delayed_node_to_data_ref(node);
2286 trace_run_delayed_data_ref(fs_info, node, ref, node->action);
2287
2288 if (node->type == BTRFS_SHARED_DATA_REF_KEY)
2289 parent = ref->parent;
2290 ref_root = ref->root;
2291
2292 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2293 if (extent_op)
2294 flags |= extent_op->flags_to_set;
2295 ret = alloc_reserved_file_extent(trans, fs_info,
2296 parent, ref_root, flags,
2297 ref->objectid, ref->offset,
2298 &ins, node->ref_mod);
2299 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
2300 ret = __btrfs_inc_extent_ref(trans, fs_info, node, parent,
2301 ref_root, ref->objectid,
2302 ref->offset, node->ref_mod,
2303 extent_op);
2304 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
2305 ret = __btrfs_free_extent(trans, fs_info, node, parent,
2306 ref_root, ref->objectid,
2307 ref->offset, node->ref_mod,
2308 extent_op);
2309 } else {
2310 BUG();
2311 }
2312 return ret;
2313 }
2314
2315 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
2316 struct extent_buffer *leaf,
2317 struct btrfs_extent_item *ei)
2318 {
2319 u64 flags = btrfs_extent_flags(leaf, ei);
2320 if (extent_op->update_flags) {
2321 flags |= extent_op->flags_to_set;
2322 btrfs_set_extent_flags(leaf, ei, flags);
2323 }
2324
2325 if (extent_op->update_key) {
2326 struct btrfs_tree_block_info *bi;
2327 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
2328 bi = (struct btrfs_tree_block_info *)(ei + 1);
2329 btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
2330 }
2331 }
2332
2333 static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
2334 struct btrfs_fs_info *fs_info,
2335 struct btrfs_delayed_ref_head *head,
2336 struct btrfs_delayed_extent_op *extent_op)
2337 {
2338 struct btrfs_key key;
2339 struct btrfs_path *path;
2340 struct btrfs_extent_item *ei;
2341 struct extent_buffer *leaf;
2342 u32 item_size;
2343 int ret;
2344 int err = 0;
2345 int metadata = !extent_op->is_data;
2346
2347 if (trans->aborted)
2348 return 0;
2349
2350 if (metadata && !btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2351 metadata = 0;
2352
2353 path = btrfs_alloc_path();
2354 if (!path)
2355 return -ENOMEM;
2356
2357 key.objectid = head->bytenr;
2358
2359 if (metadata) {
2360 key.type = BTRFS_METADATA_ITEM_KEY;
2361 key.offset = extent_op->level;
2362 } else {
2363 key.type = BTRFS_EXTENT_ITEM_KEY;
2364 key.offset = head->num_bytes;
2365 }
2366
2367 again:
2368 path->reada = READA_FORWARD;
2369 path->leave_spinning = 1;
2370 ret = btrfs_search_slot(trans, fs_info->extent_root, &key, path, 0, 1);
2371 if (ret < 0) {
2372 err = ret;
2373 goto out;
2374 }
2375 if (ret > 0) {
2376 if (metadata) {
2377 if (path->slots[0] > 0) {
2378 path->slots[0]--;
2379 btrfs_item_key_to_cpu(path->nodes[0], &key,
2380 path->slots[0]);
2381 if (key.objectid == head->bytenr &&
2382 key.type == BTRFS_EXTENT_ITEM_KEY &&
2383 key.offset == head->num_bytes)
2384 ret = 0;
2385 }
2386 if (ret > 0) {
2387 btrfs_release_path(path);
2388 metadata = 0;
2389
2390 key.objectid = head->bytenr;
2391 key.offset = head->num_bytes;
2392 key.type = BTRFS_EXTENT_ITEM_KEY;
2393 goto again;
2394 }
2395 } else {
2396 err = -EIO;
2397 goto out;
2398 }
2399 }
2400
2401 leaf = path->nodes[0];
2402 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2403 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
2404 if (item_size < sizeof(*ei)) {
2405 ret = convert_extent_item_v0(trans, fs_info, path, (u64)-1, 0);
2406 if (ret < 0) {
2407 err = ret;
2408 goto out;
2409 }
2410 leaf = path->nodes[0];
2411 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2412 }
2413 #endif
2414 BUG_ON(item_size < sizeof(*ei));
2415 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2416 __run_delayed_extent_op(extent_op, leaf, ei);
2417
2418 btrfs_mark_buffer_dirty(leaf);
2419 out:
2420 btrfs_free_path(path);
2421 return err;
2422 }
2423
2424 static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
2425 struct btrfs_fs_info *fs_info,
2426 struct btrfs_delayed_ref_node *node,
2427 struct btrfs_delayed_extent_op *extent_op,
2428 int insert_reserved)
2429 {
2430 int ret = 0;
2431 struct btrfs_delayed_tree_ref *ref;
2432 u64 parent = 0;
2433 u64 ref_root = 0;
2434
2435 ref = btrfs_delayed_node_to_tree_ref(node);
2436 trace_run_delayed_tree_ref(fs_info, node, ref, node->action);
2437
2438 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2439 parent = ref->parent;
2440 ref_root = ref->root;
2441
2442 if (node->ref_mod != 1) {
2443 btrfs_err(fs_info,
2444 "btree block(%llu) has %d references rather than 1: action %d ref_root %llu parent %llu",
2445 node->bytenr, node->ref_mod, node->action, ref_root,
2446 parent);
2447 return -EIO;
2448 }
2449 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2450 BUG_ON(!extent_op || !extent_op->update_flags);
2451 ret = alloc_reserved_tree_block(trans, node, extent_op);
2452 } else if (node->action == BTRFS_ADD_DELAYED_REF) {
2453 ret = __btrfs_inc_extent_ref(trans, fs_info, node,
2454 parent, ref_root,
2455 ref->level, 0, 1,
2456 extent_op);
2457 } else if (node->action == BTRFS_DROP_DELAYED_REF) {
2458 ret = __btrfs_free_extent(trans, fs_info, node,
2459 parent, ref_root,
2460 ref->level, 0, 1, extent_op);
2461 } else {
2462 BUG();
2463 }
2464 return ret;
2465 }
2466
2467 /* helper function to actually process a single delayed ref entry */
2468 static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
2469 struct btrfs_fs_info *fs_info,
2470 struct btrfs_delayed_ref_node *node,
2471 struct btrfs_delayed_extent_op *extent_op,
2472 int insert_reserved)
2473 {
2474 int ret = 0;
2475
2476 if (trans->aborted) {
2477 if (insert_reserved)
2478 btrfs_pin_extent(fs_info, node->bytenr,
2479 node->num_bytes, 1);
2480 return 0;
2481 }
2482
2483 if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
2484 node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2485 ret = run_delayed_tree_ref(trans, fs_info, node, extent_op,
2486 insert_reserved);
2487 else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
2488 node->type == BTRFS_SHARED_DATA_REF_KEY)
2489 ret = run_delayed_data_ref(trans, fs_info, node, extent_op,
2490 insert_reserved);
2491 else
2492 BUG();
2493 return ret;
2494 }
2495
2496 static inline struct btrfs_delayed_ref_node *
2497 select_delayed_ref(struct btrfs_delayed_ref_head *head)
2498 {
2499 struct btrfs_delayed_ref_node *ref;
2500
2501 if (RB_EMPTY_ROOT(&head->ref_tree))
2502 return NULL;
2503
2504 /*
2505 * Select a delayed ref of type BTRFS_ADD_DELAYED_REF first.
2506 * This is to prevent a ref count from going down to zero, which deletes
2507 * the extent item from the extent tree, when there still are references
2508 * to add, which would fail because they would not find the extent item.
2509 */
2510 if (!list_empty(&head->ref_add_list))
2511 return list_first_entry(&head->ref_add_list,
2512 struct btrfs_delayed_ref_node, add_list);
2513
2514 ref = rb_entry(rb_first(&head->ref_tree),
2515 struct btrfs_delayed_ref_node, ref_node);
2516 ASSERT(list_empty(&ref->add_list));
2517 return ref;
2518 }
2519
2520 static void unselect_delayed_ref_head(struct btrfs_delayed_ref_root *delayed_refs,
2521 struct btrfs_delayed_ref_head *head)
2522 {
2523 spin_lock(&delayed_refs->lock);
2524 head->processing = 0;
2525 delayed_refs->num_heads_ready++;
2526 spin_unlock(&delayed_refs->lock);
2527 btrfs_delayed_ref_unlock(head);
2528 }
2529
2530 static int cleanup_extent_op(struct btrfs_trans_handle *trans,
2531 struct btrfs_fs_info *fs_info,
2532 struct btrfs_delayed_ref_head *head)
2533 {
2534 struct btrfs_delayed_extent_op *extent_op = head->extent_op;
2535 int ret;
2536
2537 if (!extent_op)
2538 return 0;
2539 head->extent_op = NULL;
2540 if (head->must_insert_reserved) {
2541 btrfs_free_delayed_extent_op(extent_op);
2542 return 0;
2543 }
2544 spin_unlock(&head->lock);
2545 ret = run_delayed_extent_op(trans, fs_info, head, extent_op);
2546 btrfs_free_delayed_extent_op(extent_op);
2547 return ret ? ret : 1;
2548 }
2549
2550 static int cleanup_ref_head(struct btrfs_trans_handle *trans,
2551 struct btrfs_fs_info *fs_info,
2552 struct btrfs_delayed_ref_head *head)
2553 {
2554 struct btrfs_delayed_ref_root *delayed_refs;
2555 int ret;
2556
2557 delayed_refs = &trans->transaction->delayed_refs;
2558
2559 ret = cleanup_extent_op(trans, fs_info, head);
2560 if (ret < 0) {
2561 unselect_delayed_ref_head(delayed_refs, head);
2562 btrfs_debug(fs_info, "run_delayed_extent_op returned %d", ret);
2563 return ret;
2564 } else if (ret) {
2565 return ret;
2566 }
2567
2568 /*
2569 * Need to drop our head ref lock and re-acquire the delayed ref lock
2570 * and then re-check to make sure nobody got added.
2571 */
2572 spin_unlock(&head->lock);
2573 spin_lock(&delayed_refs->lock);
2574 spin_lock(&head->lock);
2575 if (!RB_EMPTY_ROOT(&head->ref_tree) || head->extent_op) {
2576 spin_unlock(&head->lock);
2577 spin_unlock(&delayed_refs->lock);
2578 return 1;
2579 }
2580 delayed_refs->num_heads--;
2581 rb_erase(&head->href_node, &delayed_refs->href_root);
2582 RB_CLEAR_NODE(&head->href_node);
2583 spin_unlock(&head->lock);
2584 spin_unlock(&delayed_refs->lock);
2585 atomic_dec(&delayed_refs->num_entries);
2586
2587 trace_run_delayed_ref_head(fs_info, head, 0);
2588
2589 if (head->total_ref_mod < 0) {
2590 struct btrfs_space_info *space_info;
2591 u64 flags;
2592
2593 if (head->is_data)
2594 flags = BTRFS_BLOCK_GROUP_DATA;
2595 else if (head->is_system)
2596 flags = BTRFS_BLOCK_GROUP_SYSTEM;
2597 else
2598 flags = BTRFS_BLOCK_GROUP_METADATA;
2599 space_info = __find_space_info(fs_info, flags);
2600 ASSERT(space_info);
2601 percpu_counter_add(&space_info->total_bytes_pinned,
2602 -head->num_bytes);
2603
2604 if (head->is_data) {
2605 spin_lock(&delayed_refs->lock);
2606 delayed_refs->pending_csums -= head->num_bytes;
2607 spin_unlock(&delayed_refs->lock);
2608 }
2609 }
2610
2611 if (head->must_insert_reserved) {
2612 btrfs_pin_extent(fs_info, head->bytenr,
2613 head->num_bytes, 1);
2614 if (head->is_data) {
2615 ret = btrfs_del_csums(trans, fs_info, head->bytenr,
2616 head->num_bytes);
2617 }
2618 }
2619
2620 /* Also free its reserved qgroup space */
2621 btrfs_qgroup_free_delayed_ref(fs_info, head->qgroup_ref_root,
2622 head->qgroup_reserved);
2623 btrfs_delayed_ref_unlock(head);
2624 btrfs_put_delayed_ref_head(head);
2625 return 0;
2626 }
2627
2628 /*
2629 * Returns 0 on success or if called with an already aborted transaction.
2630 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
2631 */
2632 static noinline int __btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
2633 unsigned long nr)
2634 {
2635 struct btrfs_fs_info *fs_info = trans->fs_info;
2636 struct btrfs_delayed_ref_root *delayed_refs;
2637 struct btrfs_delayed_ref_node *ref;
2638 struct btrfs_delayed_ref_head *locked_ref = NULL;
2639 struct btrfs_delayed_extent_op *extent_op;
2640 ktime_t start = ktime_get();
2641 int ret;
2642 unsigned long count = 0;
2643 unsigned long actual_count = 0;
2644 int must_insert_reserved = 0;
2645
2646 delayed_refs = &trans->transaction->delayed_refs;
2647 while (1) {
2648 if (!locked_ref) {
2649 if (count >= nr)
2650 break;
2651
2652 spin_lock(&delayed_refs->lock);
2653 locked_ref = btrfs_select_ref_head(trans);
2654 if (!locked_ref) {
2655 spin_unlock(&delayed_refs->lock);
2656 break;
2657 }
2658
2659 /* grab the lock that says we are going to process
2660 * all the refs for this head */
2661 ret = btrfs_delayed_ref_lock(trans, locked_ref);
2662 spin_unlock(&delayed_refs->lock);
2663 /*
2664 * we may have dropped the spin lock to get the head
2665 * mutex lock, and that might have given someone else
2666 * time to free the head. If that's true, it has been
2667 * removed from our list and we can move on.
2668 */
2669 if (ret == -EAGAIN) {
2670 locked_ref = NULL;
2671 count++;
2672 continue;
2673 }
2674 }
2675
2676 /*
2677 * We need to try and merge add/drops of the same ref since we
2678 * can run into issues with relocate dropping the implicit ref
2679 * and then it being added back again before the drop can
2680 * finish. If we merged anything we need to re-loop so we can
2681 * get a good ref.
2682 * Or we can get node references of the same type that weren't
2683 * merged when created due to bumps in the tree mod seq, and
2684 * we need to merge them to prevent adding an inline extent
2685 * backref before dropping it (triggering a BUG_ON at
2686 * insert_inline_extent_backref()).
2687 */
2688 spin_lock(&locked_ref->lock);
2689 btrfs_merge_delayed_refs(trans, delayed_refs, locked_ref);
2690
2691 ref = select_delayed_ref(locked_ref);
2692
2693 if (ref && ref->seq &&
2694 btrfs_check_delayed_seq(fs_info, ref->seq)) {
2695 spin_unlock(&locked_ref->lock);
2696 unselect_delayed_ref_head(delayed_refs, locked_ref);
2697 locked_ref = NULL;
2698 cond_resched();
2699 count++;
2700 continue;
2701 }
2702
2703 /*
2704 * We're done processing refs in this ref_head, clean everything
2705 * up and move on to the next ref_head.
2706 */
2707 if (!ref) {
2708 ret = cleanup_ref_head(trans, fs_info, locked_ref);
2709 if (ret > 0 ) {
2710 /* We dropped our lock, we need to loop. */
2711 ret = 0;
2712 continue;
2713 } else if (ret) {
2714 return ret;
2715 }
2716 locked_ref = NULL;
2717 count++;
2718 continue;
2719 }
2720
2721 actual_count++;
2722 ref->in_tree = 0;
2723 rb_erase(&ref->ref_node, &locked_ref->ref_tree);
2724 RB_CLEAR_NODE(&ref->ref_node);
2725 if (!list_empty(&ref->add_list))
2726 list_del(&ref->add_list);
2727 /*
2728 * When we play the delayed ref, also correct the ref_mod on
2729 * head
2730 */
2731 switch (ref->action) {
2732 case BTRFS_ADD_DELAYED_REF:
2733 case BTRFS_ADD_DELAYED_EXTENT:
2734 locked_ref->ref_mod -= ref->ref_mod;
2735 break;
2736 case BTRFS_DROP_DELAYED_REF:
2737 locked_ref->ref_mod += ref->ref_mod;
2738 break;
2739 default:
2740 WARN_ON(1);
2741 }
2742 atomic_dec(&delayed_refs->num_entries);
2743
2744 /*
2745 * Record the must-insert_reserved flag before we drop the spin
2746 * lock.
2747 */
2748 must_insert_reserved = locked_ref->must_insert_reserved;
2749 locked_ref->must_insert_reserved = 0;
2750
2751 extent_op = locked_ref->extent_op;
2752 locked_ref->extent_op = NULL;
2753 spin_unlock(&locked_ref->lock);
2754
2755 ret = run_one_delayed_ref(trans, fs_info, ref, extent_op,
2756 must_insert_reserved);
2757
2758 btrfs_free_delayed_extent_op(extent_op);
2759 if (ret) {
2760 unselect_delayed_ref_head(delayed_refs, locked_ref);
2761 btrfs_put_delayed_ref(ref);
2762 btrfs_debug(fs_info, "run_one_delayed_ref returned %d",
2763 ret);
2764 return ret;
2765 }
2766
2767 btrfs_put_delayed_ref(ref);
2768 count++;
2769 cond_resched();
2770 }
2771
2772 /*
2773 * We don't want to include ref heads since we can have empty ref heads
2774 * and those will drastically skew our runtime down since we just do
2775 * accounting, no actual extent tree updates.
2776 */
2777 if (actual_count > 0) {
2778 u64 runtime = ktime_to_ns(ktime_sub(ktime_get(), start));
2779 u64 avg;
2780
2781 /*
2782 * We weigh the current average higher than our current runtime
2783 * to avoid large swings in the average.
2784 */
2785 spin_lock(&delayed_refs->lock);
2786 avg = fs_info->avg_delayed_ref_runtime * 3 + runtime;
2787 fs_info->avg_delayed_ref_runtime = avg >> 2; /* div by 4 */
2788 spin_unlock(&delayed_refs->lock);
2789 }
2790 return 0;
2791 }
2792
2793 #ifdef SCRAMBLE_DELAYED_REFS
2794 /*
2795 * Normally delayed refs get processed in ascending bytenr order. This
2796 * correlates in most cases to the order added. To expose dependencies on this
2797 * order, we start to process the tree in the middle instead of the beginning
2798 */
2799 static u64 find_middle(struct rb_root *root)
2800 {
2801 struct rb_node *n = root->rb_node;
2802 struct btrfs_delayed_ref_node *entry;
2803 int alt = 1;
2804 u64 middle;
2805 u64 first = 0, last = 0;
2806
2807 n = rb_first(root);
2808 if (n) {
2809 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2810 first = entry->bytenr;
2811 }
2812 n = rb_last(root);
2813 if (n) {
2814 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2815 last = entry->bytenr;
2816 }
2817 n = root->rb_node;
2818
2819 while (n) {
2820 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2821 WARN_ON(!entry->in_tree);
2822
2823 middle = entry->bytenr;
2824
2825 if (alt)
2826 n = n->rb_left;
2827 else
2828 n = n->rb_right;
2829
2830 alt = 1 - alt;
2831 }
2832 return middle;
2833 }
2834 #endif
2835
2836 static inline u64 heads_to_leaves(struct btrfs_fs_info *fs_info, u64 heads)
2837 {
2838 u64 num_bytes;
2839
2840 num_bytes = heads * (sizeof(struct btrfs_extent_item) +
2841 sizeof(struct btrfs_extent_inline_ref));
2842 if (!btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2843 num_bytes += heads * sizeof(struct btrfs_tree_block_info);
2844
2845 /*
2846 * We don't ever fill up leaves all the way so multiply by 2 just to be
2847 * closer to what we're really going to want to use.
2848 */
2849 return div_u64(num_bytes, BTRFS_LEAF_DATA_SIZE(fs_info));
2850 }
2851
2852 /*
2853 * Takes the number of bytes to be csumm'ed and figures out how many leaves it
2854 * would require to store the csums for that many bytes.
2855 */
2856 u64 btrfs_csum_bytes_to_leaves(struct btrfs_fs_info *fs_info, u64 csum_bytes)
2857 {
2858 u64 csum_size;
2859 u64 num_csums_per_leaf;
2860 u64 num_csums;
2861
2862 csum_size = BTRFS_MAX_ITEM_SIZE(fs_info);
2863 num_csums_per_leaf = div64_u64(csum_size,
2864 (u64)btrfs_super_csum_size(fs_info->super_copy));
2865 num_csums = div64_u64(csum_bytes, fs_info->sectorsize);
2866 num_csums += num_csums_per_leaf - 1;
2867 num_csums = div64_u64(num_csums, num_csums_per_leaf);
2868 return num_csums;
2869 }
2870
2871 int btrfs_check_space_for_delayed_refs(struct btrfs_trans_handle *trans,
2872 struct btrfs_fs_info *fs_info)
2873 {
2874 struct btrfs_block_rsv *global_rsv;
2875 u64 num_heads = trans->transaction->delayed_refs.num_heads_ready;
2876 u64 csum_bytes = trans->transaction->delayed_refs.pending_csums;
2877 unsigned int num_dirty_bgs = trans->transaction->num_dirty_bgs;
2878 u64 num_bytes, num_dirty_bgs_bytes;
2879 int ret = 0;
2880
2881 num_bytes = btrfs_calc_trans_metadata_size(fs_info, 1);
2882 num_heads = heads_to_leaves(fs_info, num_heads);
2883 if (num_heads > 1)
2884 num_bytes += (num_heads - 1) * fs_info->nodesize;
2885 num_bytes <<= 1;
2886 num_bytes += btrfs_csum_bytes_to_leaves(fs_info, csum_bytes) *
2887 fs_info->nodesize;
2888 num_dirty_bgs_bytes = btrfs_calc_trans_metadata_size(fs_info,
2889 num_dirty_bgs);
2890 global_rsv = &fs_info->global_block_rsv;
2891
2892 /*
2893 * If we can't allocate any more chunks lets make sure we have _lots_ of
2894 * wiggle room since running delayed refs can create more delayed refs.
2895 */
2896 if (global_rsv->space_info->full) {
2897 num_dirty_bgs_bytes <<= 1;
2898 num_bytes <<= 1;
2899 }
2900
2901 spin_lock(&global_rsv->lock);
2902 if (global_rsv->reserved <= num_bytes + num_dirty_bgs_bytes)
2903 ret = 1;
2904 spin_unlock(&global_rsv->lock);
2905 return ret;
2906 }
2907
2908 int btrfs_should_throttle_delayed_refs(struct btrfs_trans_handle *trans,
2909 struct btrfs_fs_info *fs_info)
2910 {
2911 u64 num_entries =
2912 atomic_read(&trans->transaction->delayed_refs.num_entries);
2913 u64 avg_runtime;
2914 u64 val;
2915
2916 smp_mb();
2917 avg_runtime = fs_info->avg_delayed_ref_runtime;
2918 val = num_entries * avg_runtime;
2919 if (val >= NSEC_PER_SEC)
2920 return 1;
2921 if (val >= NSEC_PER_SEC / 2)
2922 return 2;
2923
2924 return btrfs_check_space_for_delayed_refs(trans, fs_info);
2925 }
2926
2927 struct async_delayed_refs {
2928 struct btrfs_root *root;
2929 u64 transid;
2930 int count;
2931 int error;
2932 int sync;
2933 struct completion wait;
2934 struct btrfs_work work;
2935 };
2936
2937 static inline struct async_delayed_refs *
2938 to_async_delayed_refs(struct btrfs_work *work)
2939 {
2940 return container_of(work, struct async_delayed_refs, work);
2941 }
2942
2943 static void delayed_ref_async_start(struct btrfs_work *work)
2944 {
2945 struct async_delayed_refs *async = to_async_delayed_refs(work);
2946 struct btrfs_trans_handle *trans;
2947 struct btrfs_fs_info *fs_info = async->root->fs_info;
2948 int ret;
2949
2950 /* if the commit is already started, we don't need to wait here */
2951 if (btrfs_transaction_blocked(fs_info))
2952 goto done;
2953
2954 trans = btrfs_join_transaction(async->root);
2955 if (IS_ERR(trans)) {
2956 async->error = PTR_ERR(trans);
2957 goto done;
2958 }
2959
2960 /*
2961 * trans->sync means that when we call end_transaction, we won't
2962 * wait on delayed refs
2963 */
2964 trans->sync = true;
2965
2966 /* Don't bother flushing if we got into a different transaction */
2967 if (trans->transid > async->transid)
2968 goto end;
2969
2970 ret = btrfs_run_delayed_refs(trans, async->count);
2971 if (ret)
2972 async->error = ret;
2973 end:
2974 ret = btrfs_end_transaction(trans);
2975 if (ret && !async->error)
2976 async->error = ret;
2977 done:
2978 if (async->sync)
2979 complete(&async->wait);
2980 else
2981 kfree(async);
2982 }
2983
2984 int btrfs_async_run_delayed_refs(struct btrfs_fs_info *fs_info,
2985 unsigned long count, u64 transid, int wait)
2986 {
2987 struct async_delayed_refs *async;
2988 int ret;
2989
2990 async = kmalloc(sizeof(*async), GFP_NOFS);
2991 if (!async)
2992 return -ENOMEM;
2993
2994 async->root = fs_info->tree_root;
2995 async->count = count;
2996 async->error = 0;
2997 async->transid = transid;
2998 if (wait)
2999 async->sync = 1;
3000 else
3001 async->sync = 0;
3002 init_completion(&async->wait);
3003
3004 btrfs_init_work(&async->work, btrfs_extent_refs_helper,
3005 delayed_ref_async_start, NULL, NULL);
3006
3007 btrfs_queue_work(fs_info->extent_workers, &async->work);
3008
3009 if (wait) {
3010 wait_for_completion(&async->wait);
3011 ret = async->error;
3012 kfree(async);
3013 return ret;
3014 }
3015 return 0;
3016 }
3017
3018 /*
3019 * this starts processing the delayed reference count updates and
3020 * extent insertions we have queued up so far. count can be
3021 * 0, which means to process everything in the tree at the start
3022 * of the run (but not newly added entries), or it can be some target
3023 * number you'd like to process.
3024 *
3025 * Returns 0 on success or if called with an aborted transaction
3026 * Returns <0 on error and aborts the transaction
3027 */
3028 int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
3029 unsigned long count)
3030 {
3031 struct btrfs_fs_info *fs_info = trans->fs_info;
3032 struct rb_node *node;
3033 struct btrfs_delayed_ref_root *delayed_refs;
3034 struct btrfs_delayed_ref_head *head;
3035 int ret;
3036 int run_all = count == (unsigned long)-1;
3037 bool can_flush_pending_bgs = trans->can_flush_pending_bgs;
3038
3039 /* We'll clean this up in btrfs_cleanup_transaction */
3040 if (trans->aborted)
3041 return 0;
3042
3043 if (test_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags))
3044 return 0;
3045
3046 delayed_refs = &trans->transaction->delayed_refs;
3047 if (count == 0)
3048 count = atomic_read(&delayed_refs->num_entries) * 2;
3049
3050 again:
3051 #ifdef SCRAMBLE_DELAYED_REFS
3052 delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
3053 #endif
3054 trans->can_flush_pending_bgs = false;
3055 ret = __btrfs_run_delayed_refs(trans, count);
3056 if (ret < 0) {
3057 btrfs_abort_transaction(trans, ret);
3058 return ret;
3059 }
3060
3061 if (run_all) {
3062 if (!list_empty(&trans->new_bgs))
3063 btrfs_create_pending_block_groups(trans);
3064
3065 spin_lock(&delayed_refs->lock);
3066 node = rb_first(&delayed_refs->href_root);
3067 if (!node) {
3068 spin_unlock(&delayed_refs->lock);
3069 goto out;
3070 }
3071 head = rb_entry(node, struct btrfs_delayed_ref_head,
3072 href_node);
3073 refcount_inc(&head->refs);
3074 spin_unlock(&delayed_refs->lock);
3075
3076 /* Mutex was contended, block until it's released and retry. */
3077 mutex_lock(&head->mutex);
3078 mutex_unlock(&head->mutex);
3079
3080 btrfs_put_delayed_ref_head(head);
3081 cond_resched();
3082 goto again;
3083 }
3084 out:
3085 trans->can_flush_pending_bgs = can_flush_pending_bgs;
3086 return 0;
3087 }
3088
3089 int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
3090 struct btrfs_fs_info *fs_info,
3091 u64 bytenr, u64 num_bytes, u64 flags,
3092 int level, int is_data)
3093 {
3094 struct btrfs_delayed_extent_op *extent_op;
3095 int ret;
3096
3097 extent_op = btrfs_alloc_delayed_extent_op();
3098 if (!extent_op)
3099 return -ENOMEM;
3100
3101 extent_op->flags_to_set = flags;
3102 extent_op->update_flags = true;
3103 extent_op->update_key = false;
3104 extent_op->is_data = is_data ? true : false;
3105 extent_op->level = level;
3106
3107 ret = btrfs_add_delayed_extent_op(fs_info, trans, bytenr,
3108 num_bytes, extent_op);
3109 if (ret)
3110 btrfs_free_delayed_extent_op(extent_op);
3111 return ret;
3112 }
3113
3114 static noinline int check_delayed_ref(struct btrfs_root *root,
3115 struct btrfs_path *path,
3116 u64 objectid, u64 offset, u64 bytenr)
3117 {
3118 struct btrfs_delayed_ref_head *head;
3119 struct btrfs_delayed_ref_node *ref;
3120 struct btrfs_delayed_data_ref *data_ref;
3121 struct btrfs_delayed_ref_root *delayed_refs;
3122 struct btrfs_transaction *cur_trans;
3123 struct rb_node *node;
3124 int ret = 0;
3125
3126 spin_lock(&root->fs_info->trans_lock);
3127 cur_trans = root->fs_info->running_transaction;
3128 if (cur_trans)
3129 refcount_inc(&cur_trans->use_count);
3130 spin_unlock(&root->fs_info->trans_lock);
3131 if (!cur_trans)
3132 return 0;
3133
3134 delayed_refs = &cur_trans->delayed_refs;
3135 spin_lock(&delayed_refs->lock);
3136 head = btrfs_find_delayed_ref_head(delayed_refs, bytenr);
3137 if (!head) {
3138 spin_unlock(&delayed_refs->lock);
3139 btrfs_put_transaction(cur_trans);
3140 return 0;
3141 }
3142
3143 if (!mutex_trylock(&head->mutex)) {
3144 refcount_inc(&head->refs);
3145 spin_unlock(&delayed_refs->lock);
3146
3147 btrfs_release_path(path);
3148
3149 /*
3150 * Mutex was contended, block until it's released and let
3151 * caller try again
3152 */
3153 mutex_lock(&head->mutex);
3154 mutex_unlock(&head->mutex);
3155 btrfs_put_delayed_ref_head(head);
3156 btrfs_put_transaction(cur_trans);
3157 return -EAGAIN;
3158 }
3159 spin_unlock(&delayed_refs->lock);
3160
3161 spin_lock(&head->lock);
3162 /*
3163 * XXX: We should replace this with a proper search function in the
3164 * future.
3165 */
3166 for (node = rb_first(&head->ref_tree); node; node = rb_next(node)) {
3167 ref = rb_entry(node, struct btrfs_delayed_ref_node, ref_node);
3168 /* If it's a shared ref we know a cross reference exists */
3169 if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) {
3170 ret = 1;
3171 break;
3172 }
3173
3174 data_ref = btrfs_delayed_node_to_data_ref(ref);
3175
3176 /*
3177 * If our ref doesn't match the one we're currently looking at
3178 * then we have a cross reference.
3179 */
3180 if (data_ref->root != root->root_key.objectid ||
3181 data_ref->objectid != objectid ||
3182 data_ref->offset != offset) {
3183 ret = 1;
3184 break;
3185 }
3186 }
3187 spin_unlock(&head->lock);
3188 mutex_unlock(&head->mutex);
3189 btrfs_put_transaction(cur_trans);
3190 return ret;
3191 }
3192
3193 static noinline int check_committed_ref(struct btrfs_root *root,
3194 struct btrfs_path *path,
3195 u64 objectid, u64 offset, u64 bytenr)
3196 {
3197 struct btrfs_fs_info *fs_info = root->fs_info;
3198 struct btrfs_root *extent_root = fs_info->extent_root;
3199 struct extent_buffer *leaf;
3200 struct btrfs_extent_data_ref *ref;
3201 struct btrfs_extent_inline_ref *iref;
3202 struct btrfs_extent_item *ei;
3203 struct btrfs_key key;
3204 u32 item_size;
3205 int type;
3206 int ret;
3207
3208 key.objectid = bytenr;
3209 key.offset = (u64)-1;
3210 key.type = BTRFS_EXTENT_ITEM_KEY;
3211
3212 ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
3213 if (ret < 0)
3214 goto out;
3215 BUG_ON(ret == 0); /* Corruption */
3216
3217 ret = -ENOENT;
3218 if (path->slots[0] == 0)
3219 goto out;
3220
3221 path->slots[0]--;
3222 leaf = path->nodes[0];
3223 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
3224
3225 if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
3226 goto out;
3227
3228 ret = 1;
3229 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
3230 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
3231 if (item_size < sizeof(*ei)) {
3232 WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
3233 goto out;
3234 }
3235 #endif
3236 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
3237
3238 if (item_size != sizeof(*ei) +
3239 btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
3240 goto out;
3241
3242 if (btrfs_extent_generation(leaf, ei) <=
3243 btrfs_root_last_snapshot(&root->root_item))
3244 goto out;
3245
3246 iref = (struct btrfs_extent_inline_ref *)(ei + 1);
3247
3248 type = btrfs_get_extent_inline_ref_type(leaf, iref, BTRFS_REF_TYPE_DATA);
3249 if (type != BTRFS_EXTENT_DATA_REF_KEY)
3250 goto out;
3251
3252 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
3253 if (btrfs_extent_refs(leaf, ei) !=
3254 btrfs_extent_data_ref_count(leaf, ref) ||
3255 btrfs_extent_data_ref_root(leaf, ref) !=
3256 root->root_key.objectid ||
3257 btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
3258 btrfs_extent_data_ref_offset(leaf, ref) != offset)
3259 goto out;
3260
3261 ret = 0;
3262 out:
3263 return ret;
3264 }
3265
3266 int btrfs_cross_ref_exist(struct btrfs_root *root, u64 objectid, u64 offset,
3267 u64 bytenr)
3268 {
3269 struct btrfs_path *path;
3270 int ret;
3271 int ret2;
3272
3273 path = btrfs_alloc_path();
3274 if (!path)
3275 return -ENOMEM;
3276
3277 do {
3278 ret = check_committed_ref(root, path, objectid,
3279 offset, bytenr);
3280 if (ret && ret != -ENOENT)
3281 goto out;
3282
3283 ret2 = check_delayed_ref(root, path, objectid,
3284 offset, bytenr);
3285 } while (ret2 == -EAGAIN);
3286
3287 if (ret2 && ret2 != -ENOENT) {
3288 ret = ret2;
3289 goto out;
3290 }
3291
3292 if (ret != -ENOENT || ret2 != -ENOENT)
3293 ret = 0;
3294 out:
3295 btrfs_free_path(path);
3296 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
3297 WARN_ON(ret > 0);
3298 return ret;
3299 }
3300
3301 static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
3302 struct btrfs_root *root,
3303 struct extent_buffer *buf,
3304 int full_backref, int inc)
3305 {
3306 struct btrfs_fs_info *fs_info = root->fs_info;
3307 u64 bytenr;
3308 u64 num_bytes;
3309 u64 parent;
3310 u64 ref_root;
3311 u32 nritems;
3312 struct btrfs_key key;
3313 struct btrfs_file_extent_item *fi;
3314 int i;
3315 int level;
3316 int ret = 0;
3317 int (*process_func)(struct btrfs_trans_handle *,
3318 struct btrfs_root *,
3319 u64, u64, u64, u64, u64, u64);
3320
3321
3322 if (btrfs_is_testing(fs_info))
3323 return 0;
3324
3325 ref_root = btrfs_header_owner(buf);
3326 nritems = btrfs_header_nritems(buf);
3327 level = btrfs_header_level(buf);
3328
3329 if (!test_bit(BTRFS_ROOT_REF_COWS, &root->state) && level == 0)
3330 return 0;
3331
3332 if (inc)
3333 process_func = btrfs_inc_extent_ref;
3334 else
3335 process_func = btrfs_free_extent;
3336
3337 if (full_backref)
3338 parent = buf->start;
3339 else
3340 parent = 0;
3341
3342 for (i = 0; i < nritems; i++) {
3343 if (level == 0) {
3344 btrfs_item_key_to_cpu(buf, &key, i);
3345 if (key.type != BTRFS_EXTENT_DATA_KEY)
3346 continue;
3347 fi = btrfs_item_ptr(buf, i,
3348 struct btrfs_file_extent_item);
3349 if (btrfs_file_extent_type(buf, fi) ==
3350 BTRFS_FILE_EXTENT_INLINE)
3351 continue;
3352 bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
3353 if (bytenr == 0)
3354 continue;
3355
3356 num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
3357 key.offset -= btrfs_file_extent_offset(buf, fi);
3358 ret = process_func(trans, root, bytenr, num_bytes,
3359 parent, ref_root, key.objectid,
3360 key.offset);
3361 if (ret)
3362 goto fail;
3363 } else {
3364 bytenr = btrfs_node_blockptr(buf, i);
3365 num_bytes = fs_info->nodesize;
3366 ret = process_func(trans, root, bytenr, num_bytes,
3367 parent, ref_root, level - 1, 0);
3368 if (ret)
3369 goto fail;
3370 }
3371 }
3372 return 0;
3373 fail:
3374 return ret;
3375 }
3376
3377 int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3378 struct extent_buffer *buf, int full_backref)
3379 {
3380 return __btrfs_mod_ref(trans, root, buf, full_backref, 1);
3381 }
3382
3383 int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3384 struct extent_buffer *buf, int full_backref)
3385 {
3386 return __btrfs_mod_ref(trans, root, buf, full_backref, 0);
3387 }
3388
3389 static int write_one_cache_group(struct btrfs_trans_handle *trans,
3390 struct btrfs_fs_info *fs_info,
3391 struct btrfs_path *path,
3392 struct btrfs_block_group_cache *cache)
3393 {
3394 int ret;
3395 struct btrfs_root *extent_root = fs_info->extent_root;
3396 unsigned long bi;
3397 struct extent_buffer *leaf;
3398
3399 ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
3400 if (ret) {
3401 if (ret > 0)
3402 ret = -ENOENT;
3403 goto fail;
3404 }
3405
3406 leaf = path->nodes[0];
3407 bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
3408 write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
3409 btrfs_mark_buffer_dirty(leaf);
3410 fail:
3411 btrfs_release_path(path);
3412 return ret;
3413
3414 }
3415
3416 static struct btrfs_block_group_cache *
3417 next_block_group(struct btrfs_fs_info *fs_info,
3418 struct btrfs_block_group_cache *cache)
3419 {
3420 struct rb_node *node;
3421
3422 spin_lock(&fs_info->block_group_cache_lock);
3423
3424 /* If our block group was removed, we need a full search. */
3425 if (RB_EMPTY_NODE(&cache->cache_node)) {
3426 const u64 next_bytenr = cache->key.objectid + cache->key.offset;
3427
3428 spin_unlock(&fs_info->block_group_cache_lock);
3429 btrfs_put_block_group(cache);
3430 cache = btrfs_lookup_first_block_group(fs_info, next_bytenr); return cache;
3431 }
3432 node = rb_next(&cache->cache_node);
3433 btrfs_put_block_group(cache);
3434 if (node) {
3435 cache = rb_entry(node, struct btrfs_block_group_cache,
3436 cache_node);
3437 btrfs_get_block_group(cache);
3438 } else
3439 cache = NULL;
3440 spin_unlock(&fs_info->block_group_cache_lock);
3441 return cache;
3442 }
3443
3444 static int cache_save_setup(struct btrfs_block_group_cache *block_group,
3445 struct btrfs_trans_handle *trans,
3446 struct btrfs_path *path)
3447 {
3448 struct btrfs_fs_info *fs_info = block_group->fs_info;
3449 struct btrfs_root *root = fs_info->tree_root;
3450 struct inode *inode = NULL;
3451 struct extent_changeset *data_reserved = NULL;
3452 u64 alloc_hint = 0;
3453 int dcs = BTRFS_DC_ERROR;
3454 u64 num_pages = 0;
3455 int retries = 0;
3456 int ret = 0;
3457
3458 /*
3459 * If this block group is smaller than 100 megs don't bother caching the
3460 * block group.
3461 */
3462 if (block_group->key.offset < (100 * SZ_1M)) {
3463 spin_lock(&block_group->lock);
3464 block_group->disk_cache_state = BTRFS_DC_WRITTEN;
3465 spin_unlock(&block_group->lock);
3466 return 0;
3467 }
3468
3469 if (trans->aborted)
3470 return 0;
3471 again:
3472 inode = lookup_free_space_inode(fs_info, block_group, path);
3473 if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
3474 ret = PTR_ERR(inode);
3475 btrfs_release_path(path);
3476 goto out;
3477 }
3478
3479 if (IS_ERR(inode)) {
3480 BUG_ON(retries);
3481 retries++;
3482
3483 if (block_group->ro)
3484 goto out_free;
3485
3486 ret = create_free_space_inode(fs_info, trans, block_group,
3487 path);
3488 if (ret)
3489 goto out_free;
3490 goto again;
3491 }
3492
3493 /*
3494 * We want to set the generation to 0, that way if anything goes wrong
3495 * from here on out we know not to trust this cache when we load up next
3496 * time.
3497 */
3498 BTRFS_I(inode)->generation = 0;
3499 ret = btrfs_update_inode(trans, root, inode);
3500 if (ret) {
3501 /*
3502 * So theoretically we could recover from this, simply set the
3503 * super cache generation to 0 so we know to invalidate the
3504 * cache, but then we'd have to keep track of the block groups
3505 * that fail this way so we know we _have_ to reset this cache
3506 * before the next commit or risk reading stale cache. So to
3507 * limit our exposure to horrible edge cases lets just abort the
3508 * transaction, this only happens in really bad situations
3509 * anyway.
3510 */
3511 btrfs_abort_transaction(trans, ret);
3512 goto out_put;
3513 }
3514 WARN_ON(ret);
3515
3516 /* We've already setup this transaction, go ahead and exit */
3517 if (block_group->cache_generation == trans->transid &&
3518 i_size_read(inode)) {
3519 dcs = BTRFS_DC_SETUP;
3520 goto out_put;
3521 }
3522
3523 if (i_size_read(inode) > 0) {
3524 ret = btrfs_check_trunc_cache_free_space(fs_info,
3525 &fs_info->global_block_rsv);
3526 if (ret)
3527 goto out_put;
3528
3529 ret = btrfs_truncate_free_space_cache(trans, NULL, inode);
3530 if (ret)
3531 goto out_put;
3532 }
3533
3534 spin_lock(&block_group->lock);
3535 if (block_group->cached != BTRFS_CACHE_FINISHED ||
3536 !btrfs_test_opt(fs_info, SPACE_CACHE)) {
3537 /*
3538 * don't bother trying to write stuff out _if_
3539 * a) we're not cached,
3540 * b) we're with nospace_cache mount option,
3541 * c) we're with v2 space_cache (FREE_SPACE_TREE).
3542 */
3543 dcs = BTRFS_DC_WRITTEN;
3544 spin_unlock(&block_group->lock);
3545 goto out_put;
3546 }
3547 spin_unlock(&block_group->lock);
3548
3549 /*
3550 * We hit an ENOSPC when setting up the cache in this transaction, just
3551 * skip doing the setup, we've already cleared the cache so we're safe.
3552 */
3553 if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) {
3554 ret = -ENOSPC;
3555 goto out_put;
3556 }
3557
3558 /*
3559 * Try to preallocate enough space based on how big the block group is.
3560 * Keep in mind this has to include any pinned space which could end up
3561 * taking up quite a bit since it's not folded into the other space
3562 * cache.
3563 */
3564 num_pages = div_u64(block_group->key.offset, SZ_256M);
3565 if (!num_pages)
3566 num_pages = 1;
3567
3568 num_pages *= 16;
3569 num_pages *= PAGE_SIZE;
3570
3571 ret = btrfs_check_data_free_space(inode, &data_reserved, 0, num_pages);
3572 if (ret)
3573 goto out_put;
3574
3575 ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
3576 num_pages, num_pages,
3577 &alloc_hint);
3578 /*
3579 * Our cache requires contiguous chunks so that we don't modify a bunch
3580 * of metadata or split extents when writing the cache out, which means
3581 * we can enospc if we are heavily fragmented in addition to just normal
3582 * out of space conditions. So if we hit this just skip setting up any
3583 * other block groups for this transaction, maybe we'll unpin enough
3584 * space the next time around.
3585 */
3586 if (!ret)
3587 dcs = BTRFS_DC_SETUP;
3588 else if (ret == -ENOSPC)
3589 set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags);
3590
3591 out_put:
3592 iput(inode);
3593 out_free:
3594 btrfs_release_path(path);
3595 out:
3596 spin_lock(&block_group->lock);
3597 if (!ret && dcs == BTRFS_DC_SETUP)
3598 block_group->cache_generation = trans->transid;
3599 block_group->disk_cache_state = dcs;
3600 spin_unlock(&block_group->lock);
3601
3602 extent_changeset_free(data_reserved);
3603 return ret;
3604 }
3605
3606 int btrfs_setup_space_cache(struct btrfs_trans_handle *trans,
3607 struct btrfs_fs_info *fs_info)
3608 {
3609 struct btrfs_block_group_cache *cache, *tmp;
3610 struct btrfs_transaction *cur_trans = trans->transaction;
3611 struct btrfs_path *path;
3612
3613 if (list_empty(&cur_trans->dirty_bgs) ||
3614 !btrfs_test_opt(fs_info, SPACE_CACHE))
3615 return 0;
3616
3617 path = btrfs_alloc_path();
3618 if (!path)
3619 return -ENOMEM;
3620
3621 /* Could add new block groups, use _safe just in case */
3622 list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs,
3623 dirty_list) {
3624 if (cache->disk_cache_state == BTRFS_DC_CLEAR)
3625 cache_save_setup(cache, trans, path);
3626 }
3627
3628 btrfs_free_path(path);
3629 return 0;
3630 }
3631
3632 /*
3633 * transaction commit does final block group cache writeback during a
3634 * critical section where nothing is allowed to change the FS. This is
3635 * required in order for the cache to actually match the block group,
3636 * but can introduce a lot of latency into the commit.
3637 *
3638 * So, btrfs_start_dirty_block_groups is here to kick off block group
3639 * cache IO. There's a chance we'll have to redo some of it if the
3640 * block group changes again during the commit, but it greatly reduces
3641 * the commit latency by getting rid of the easy block groups while
3642 * we're still allowing others to join the commit.
3643 */
3644 int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
3645 {
3646 struct btrfs_fs_info *fs_info = trans->fs_info;
3647 struct btrfs_block_group_cache *cache;
3648 struct btrfs_transaction *cur_trans = trans->transaction;
3649 int ret = 0;
3650 int should_put;
3651 struct btrfs_path *path = NULL;
3652 LIST_HEAD(dirty);
3653 struct list_head *io = &cur_trans->io_bgs;
3654 int num_started = 0;
3655 int loops = 0;
3656
3657 spin_lock(&cur_trans->dirty_bgs_lock);
3658 if (list_empty(&cur_trans->dirty_bgs)) {
3659 spin_unlock(&cur_trans->dirty_bgs_lock);
3660 return 0;
3661 }
3662 list_splice_init(&cur_trans->dirty_bgs, &dirty);
3663 spin_unlock(&cur_trans->dirty_bgs_lock);
3664
3665 again:
3666 /*
3667 * make sure all the block groups on our dirty list actually
3668 * exist
3669 */
3670 btrfs_create_pending_block_groups(trans);
3671
3672 if (!path) {
3673 path = btrfs_alloc_path();
3674 if (!path)
3675 return -ENOMEM;
3676 }
3677
3678 /*
3679 * cache_write_mutex is here only to save us from balance or automatic
3680 * removal of empty block groups deleting this block group while we are
3681 * writing out the cache
3682 */
3683 mutex_lock(&trans->transaction->cache_write_mutex);
3684 while (!list_empty(&dirty)) {
3685 cache = list_first_entry(&dirty,
3686 struct btrfs_block_group_cache,
3687 dirty_list);
3688 /*
3689 * this can happen if something re-dirties a block
3690 * group that is already under IO. Just wait for it to
3691 * finish and then do it all again
3692 */
3693 if (!list_empty(&cache->io_list)) {
3694 list_del_init(&cache->io_list);
3695 btrfs_wait_cache_io(trans, cache, path);
3696 btrfs_put_block_group(cache);
3697 }
3698
3699
3700 /*
3701 * btrfs_wait_cache_io uses the cache->dirty_list to decide
3702 * if it should update the cache_state. Don't delete
3703 * until after we wait.
3704 *
3705 * Since we're not running in the commit critical section
3706 * we need the dirty_bgs_lock to protect from update_block_group
3707 */
3708 spin_lock(&cur_trans->dirty_bgs_lock);
3709 list_del_init(&cache->dirty_list);
3710 spin_unlock(&cur_trans->dirty_bgs_lock);
3711
3712 should_put = 1;
3713
3714 cache_save_setup(cache, trans, path);
3715
3716 if (cache->disk_cache_state == BTRFS_DC_SETUP) {
3717 cache->io_ctl.inode = NULL;
3718 ret = btrfs_write_out_cache(fs_info, trans,
3719 cache, path);
3720 if (ret == 0 && cache->io_ctl.inode) {
3721 num_started++;
3722 should_put = 0;
3723
3724 /*
3725 * The cache_write_mutex is protecting the
3726 * io_list, also refer to the definition of
3727 * btrfs_transaction::io_bgs for more details
3728 */
3729 list_add_tail(&cache->io_list, io);
3730 } else {
3731 /*
3732 * if we failed to write the cache, the
3733 * generation will be bad and life goes on
3734 */
3735 ret = 0;
3736 }
3737 }
3738 if (!ret) {
3739 ret = write_one_cache_group(trans, fs_info,
3740 path, cache);
3741 /*
3742 * Our block group might still be attached to the list
3743 * of new block groups in the transaction handle of some
3744 * other task (struct btrfs_trans_handle->new_bgs). This
3745 * means its block group item isn't yet in the extent
3746 * tree. If this happens ignore the error, as we will
3747 * try again later in the critical section of the
3748 * transaction commit.
3749 */
3750 if (ret == -ENOENT) {
3751 ret = 0;
3752 spin_lock(&cur_trans->dirty_bgs_lock);
3753 if (list_empty(&cache->dirty_list)) {
3754 list_add_tail(&cache->dirty_list,
3755 &cur_trans->dirty_bgs);
3756 btrfs_get_block_group(cache);
3757 }
3758 spin_unlock(&cur_trans->dirty_bgs_lock);
3759 } else if (ret) {
3760 btrfs_abort_transaction(trans, ret);
3761 }
3762 }
3763
3764 /* if its not on the io list, we need to put the block group */
3765 if (should_put)
3766 btrfs_put_block_group(cache);
3767
3768 if (ret)
3769 break;
3770
3771 /*
3772 * Avoid blocking other tasks for too long. It might even save
3773 * us from writing caches for block groups that are going to be
3774 * removed.
3775 */
3776 mutex_unlock(&trans->transaction->cache_write_mutex);
3777 mutex_lock(&trans->transaction->cache_write_mutex);
3778 }
3779 mutex_unlock(&trans->transaction->cache_write_mutex);
3780
3781 /*
3782 * go through delayed refs for all the stuff we've just kicked off
3783 * and then loop back (just once)
3784 */
3785 ret = btrfs_run_delayed_refs(trans, 0);
3786 if (!ret && loops == 0) {
3787 loops++;
3788 spin_lock(&cur_trans->dirty_bgs_lock);
3789 list_splice_init(&cur_trans->dirty_bgs, &dirty);
3790 /*
3791 * dirty_bgs_lock protects us from concurrent block group
3792 * deletes too (not just cache_write_mutex).
3793 */
3794 if (!list_empty(&dirty)) {
3795 spin_unlock(&cur_trans->dirty_bgs_lock);
3796 goto again;
3797 }
3798 spin_unlock(&cur_trans->dirty_bgs_lock);
3799 } else if (ret < 0) {
3800 btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
3801 }
3802
3803 btrfs_free_path(path);
3804 return ret;
3805 }
3806
3807 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
3808 struct btrfs_fs_info *fs_info)
3809 {
3810 struct btrfs_block_group_cache *cache;
3811 struct btrfs_transaction *cur_trans = trans->transaction;
3812 int ret = 0;
3813 int should_put;
3814 struct btrfs_path *path;
3815 struct list_head *io = &cur_trans->io_bgs;
3816 int num_started = 0;
3817
3818 path = btrfs_alloc_path();
3819 if (!path)
3820 return -ENOMEM;
3821
3822 /*
3823 * Even though we are in the critical section of the transaction commit,
3824 * we can still have concurrent tasks adding elements to this
3825 * transaction's list of dirty block groups. These tasks correspond to
3826 * endio free space workers started when writeback finishes for a
3827 * space cache, which run inode.c:btrfs_finish_ordered_io(), and can
3828 * allocate new block groups as a result of COWing nodes of the root
3829 * tree when updating the free space inode. The writeback for the space
3830 * caches is triggered by an earlier call to
3831 * btrfs_start_dirty_block_groups() and iterations of the following
3832 * loop.
3833 * Also we want to do the cache_save_setup first and then run the
3834 * delayed refs to make sure we have the best chance at doing this all
3835 * in one shot.
3836 */
3837 spin_lock(&cur_trans->dirty_bgs_lock);
3838 while (!list_empty(&cur_trans->dirty_bgs)) {
3839 cache = list_first_entry(&cur_trans->dirty_bgs,
3840 struct btrfs_block_group_cache,
3841 dirty_list);
3842
3843 /*
3844 * this can happen if cache_save_setup re-dirties a block
3845 * group that is already under IO. Just wait for it to
3846 * finish and then do it all again
3847 */
3848 if (!list_empty(&cache->io_list)) {
3849 spin_unlock(&cur_trans->dirty_bgs_lock);
3850 list_del_init(&cache->io_list);
3851 btrfs_wait_cache_io(trans, cache, path);
3852 btrfs_put_block_group(cache);
3853 spin_lock(&cur_trans->dirty_bgs_lock);
3854 }
3855
3856 /*
3857 * don't remove from the dirty list until after we've waited
3858 * on any pending IO
3859 */
3860 list_del_init(&cache->dirty_list);
3861 spin_unlock(&cur_trans->dirty_bgs_lock);
3862 should_put = 1;
3863
3864 cache_save_setup(cache, trans, path);
3865
3866 if (!ret)
3867 ret = btrfs_run_delayed_refs(trans,
3868 (unsigned long) -1);
3869
3870 if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) {
3871 cache->io_ctl.inode = NULL;
3872 ret = btrfs_write_out_cache(fs_info, trans,
3873 cache, path);
3874 if (ret == 0 && cache->io_ctl.inode) {
3875 num_started++;
3876 should_put = 0;
3877 list_add_tail(&cache->io_list, io);
3878 } else {
3879 /*
3880 * if we failed to write the cache, the
3881 * generation will be bad and life goes on
3882 */
3883 ret = 0;
3884 }
3885 }
3886 if (!ret) {
3887 ret = write_one_cache_group(trans, fs_info,
3888 path, cache);
3889 /*
3890 * One of the free space endio workers might have
3891 * created a new block group while updating a free space
3892 * cache's inode (at inode.c:btrfs_finish_ordered_io())
3893 * and hasn't released its transaction handle yet, in
3894 * which case the new block group is still attached to
3895 * its transaction handle and its creation has not
3896 * finished yet (no block group item in the extent tree
3897 * yet, etc). If this is the case, wait for all free
3898 * space endio workers to finish and retry. This is a
3899 * a very rare case so no need for a more efficient and
3900 * complex approach.
3901 */
3902 if (ret == -ENOENT) {
3903 wait_event(cur_trans->writer_wait,
3904 atomic_read(&cur_trans->num_writers) == 1);
3905 ret = write_one_cache_group(trans, fs_info,
3906 path, cache);
3907 }
3908 if (ret)
3909 btrfs_abort_transaction(trans, ret);
3910 }
3911
3912 /* if its not on the io list, we need to put the block group */
3913 if (should_put)
3914 btrfs_put_block_group(cache);
3915 spin_lock(&cur_trans->dirty_bgs_lock);
3916 }
3917 spin_unlock(&cur_trans->dirty_bgs_lock);
3918
3919 /*
3920 * Refer to the definition of io_bgs member for details why it's safe
3921 * to use it without any locking
3922 */
3923 while (!list_empty(io)) {
3924 cache = list_first_entry(io, struct btrfs_block_group_cache,
3925 io_list);
3926 list_del_init(&cache->io_list);
3927 btrfs_wait_cache_io(trans, cache, path);
3928 btrfs_put_block_group(cache);
3929 }
3930
3931 btrfs_free_path(path);
3932 return ret;
3933 }
3934
3935 int btrfs_extent_readonly(struct btrfs_fs_info *fs_info, u64 bytenr)
3936 {
3937 struct btrfs_block_group_cache *block_group;
3938 int readonly = 0;
3939
3940 block_group = btrfs_lookup_block_group(fs_info, bytenr);
3941 if (!block_group || block_group->ro)
3942 readonly = 1;
3943 if (block_group)
3944 btrfs_put_block_group(block_group);
3945 return readonly;
3946 }
3947
3948 bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
3949 {
3950 struct btrfs_block_group_cache *bg;
3951 bool ret = true;
3952
3953 bg = btrfs_lookup_block_group(fs_info, bytenr);
3954 if (!bg)
3955 return false;
3956
3957 spin_lock(&bg->lock);
3958 if (bg->ro)
3959 ret = false;
3960 else
3961 atomic_inc(&bg->nocow_writers);
3962 spin_unlock(&bg->lock);
3963
3964 /* no put on block group, done by btrfs_dec_nocow_writers */
3965 if (!ret)
3966 btrfs_put_block_group(bg);
3967
3968 return ret;
3969
3970 }
3971
3972 void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
3973 {
3974 struct btrfs_block_group_cache *bg;
3975
3976 bg = btrfs_lookup_block_group(fs_info, bytenr);
3977 ASSERT(bg);
3978 if (atomic_dec_and_test(&bg->nocow_writers))
3979 wake_up_var(&bg->nocow_writers);
3980 /*
3981 * Once for our lookup and once for the lookup done by a previous call
3982 * to btrfs_inc_nocow_writers()
3983 */
3984 btrfs_put_block_group(bg);
3985 btrfs_put_block_group(bg);
3986 }
3987
3988 void btrfs_wait_nocow_writers(struct btrfs_block_group_cache *bg)
3989 {
3990 wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers));
3991 }
3992
3993 static const char *alloc_name(u64 flags)
3994 {
3995 switch (flags) {
3996 case BTRFS_BLOCK_GROUP_METADATA|BTRFS_BLOCK_GROUP_DATA:
3997 return "mixed";
3998 case BTRFS_BLOCK_GROUP_METADATA:
3999 return "metadata";
4000 case BTRFS_BLOCK_GROUP_DATA:
4001 return "data";
4002 case BTRFS_BLOCK_GROUP_SYSTEM:
4003 return "system";
4004 default:
4005 WARN_ON(1);
4006 return "invalid-combination";
4007 };
4008 }
4009
4010 static int create_space_info(struct btrfs_fs_info *info, u64 flags)
4011 {
4012
4013 struct btrfs_space_info *space_info;
4014 int i;
4015 int ret;
4016
4017 space_info = kzalloc(sizeof(*space_info), GFP_NOFS);
4018 if (!space_info)
4019 return -ENOMEM;
4020
4021 ret = percpu_counter_init(&space_info->total_bytes_pinned, 0,
4022 GFP_KERNEL);
4023 if (ret) {
4024 kfree(space_info);
4025 return ret;
4026 }
4027
4028 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
4029 INIT_LIST_HEAD(&space_info->block_groups[i]);
4030 init_rwsem(&space_info->groups_sem);
4031 spin_lock_init(&space_info->lock);
4032 space_info->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
4033 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
4034 init_waitqueue_head(&space_info->wait);
4035 INIT_LIST_HEAD(&space_info->ro_bgs);
4036 INIT_LIST_HEAD(&space_info->tickets);
4037 INIT_LIST_HEAD(&space_info->priority_tickets);
4038
4039 ret = kobject_init_and_add(&space_info->kobj, &space_info_ktype,
4040 info->space_info_kobj, "%s",
4041 alloc_name(space_info->flags));
4042 if (ret) {
4043 percpu_counter_destroy(&space_info->total_bytes_pinned);
4044 kfree(space_info);
4045 return ret;
4046 }
4047
4048 list_add_rcu(&space_info->list, &info->space_info);
4049 if (flags & BTRFS_BLOCK_GROUP_DATA)
4050 info->data_sinfo = space_info;
4051
4052 return ret;
4053 }
4054
4055 static void update_space_info(struct btrfs_fs_info *info, u64 flags,
4056 u64 total_bytes, u64 bytes_used,
4057 u64 bytes_readonly,
4058 struct btrfs_space_info **space_info)
4059 {
4060 struct btrfs_space_info *found;
4061 int factor;
4062
4063 if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
4064 BTRFS_BLOCK_GROUP_RAID10))
4065 factor = 2;
4066 else
4067 factor = 1;
4068
4069 found = __find_space_info(info, flags);
4070 ASSERT(found);
4071 spin_lock(&found->lock);
4072 found->total_bytes += total_bytes;
4073 found->disk_total += total_bytes * factor;
4074 found->bytes_used += bytes_used;
4075 found->disk_used += bytes_used * factor;
4076 found->bytes_readonly += bytes_readonly;
4077 if (total_bytes > 0)
4078 found->full = 0;
4079 space_info_add_new_bytes(info, found, total_bytes -
4080 bytes_used - bytes_readonly);
4081 spin_unlock(&found->lock);
4082 *space_info = found;
4083 }
4084
4085 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
4086 {
4087 u64 extra_flags = chunk_to_extended(flags) &
4088 BTRFS_EXTENDED_PROFILE_MASK;
4089
4090 write_seqlock(&fs_info->profiles_lock);
4091 if (flags & BTRFS_BLOCK_GROUP_DATA)
4092 fs_info->avail_data_alloc_bits |= extra_flags;
4093 if (flags & BTRFS_BLOCK_GROUP_METADATA)
4094 fs_info->avail_metadata_alloc_bits |= extra_flags;
4095 if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
4096 fs_info->avail_system_alloc_bits |= extra_flags;
4097 write_sequnlock(&fs_info->profiles_lock);
4098 }
4099
4100 /*
4101 * returns target flags in extended format or 0 if restripe for this
4102 * chunk_type is not in progress
4103 *
4104 * should be called with balance_lock held
4105 */
4106 static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
4107 {
4108 struct btrfs_balance_control *bctl = fs_info->balance_ctl;
4109 u64 target = 0;
4110
4111 if (!bctl)
4112 return 0;
4113
4114 if (flags & BTRFS_BLOCK_GROUP_DATA &&
4115 bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
4116 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
4117 } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
4118 bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
4119 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
4120 } else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
4121 bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
4122 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
4123 }
4124
4125 return target;
4126 }
4127
4128 /*
4129 * @flags: available profiles in extended format (see ctree.h)
4130 *
4131 * Returns reduced profile in chunk format. If profile changing is in
4132 * progress (either running or paused) picks the target profile (if it's
4133 * already available), otherwise falls back to plain reducing.
4134 */
4135 static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags)
4136 {
4137 u64 num_devices = fs_info->fs_devices->rw_devices;
4138 u64 target;
4139 u64 raid_type;
4140 u64 allowed = 0;
4141
4142 /*
4143 * see if restripe for this chunk_type is in progress, if so
4144 * try to reduce to the target profile
4145 */
4146 spin_lock(&fs_info->balance_lock);
4147 target = get_restripe_target(fs_info, flags);
4148 if (target) {
4149 /* pick target profile only if it's already available */
4150 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
4151 spin_unlock(&fs_info->balance_lock);
4152 return extended_to_chunk(target);
4153 }
4154 }
4155 spin_unlock(&fs_info->balance_lock);
4156
4157 /* First, mask out the RAID levels which aren't possible */
4158 for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
4159 if (num_devices >= btrfs_raid_array[raid_type].devs_min)
4160 allowed |= btrfs_raid_array[raid_type].bg_flag;
4161 }
4162 allowed &= flags;
4163
4164 if (allowed & BTRFS_BLOCK_GROUP_RAID6)
4165 allowed = BTRFS_BLOCK_GROUP_RAID6;
4166 else if (allowed & BTRFS_BLOCK_GROUP_RAID5)
4167 allowed = BTRFS_BLOCK_GROUP_RAID5;
4168 else if (allowed & BTRFS_BLOCK_GROUP_RAID10)
4169 allowed = BTRFS_BLOCK_GROUP_RAID10;
4170 else if (allowed & BTRFS_BLOCK_GROUP_RAID1)
4171 allowed = BTRFS_BLOCK_GROUP_RAID1;
4172 else if (allowed & BTRFS_BLOCK_GROUP_RAID0)
4173 allowed = BTRFS_BLOCK_GROUP_RAID0;
4174
4175 flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK;
4176
4177 return extended_to_chunk(flags | allowed);
4178 }
4179
4180 static u64 get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags)
4181 {
4182 unsigned seq;
4183 u64 flags;
4184
4185 do {
4186 flags = orig_flags;
4187 seq = read_seqbegin(&fs_info->profiles_lock);
4188
4189 if (flags & BTRFS_BLOCK_GROUP_DATA)
4190 flags |= fs_info->avail_data_alloc_bits;
4191 else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
4192 flags |= fs_info->avail_system_alloc_bits;
4193 else if (flags & BTRFS_BLOCK_GROUP_METADATA)
4194 flags |= fs_info->avail_metadata_alloc_bits;
4195 } while (read_seqretry(&fs_info->profiles_lock, seq));
4196
4197 return btrfs_reduce_alloc_profile(fs_info, flags);
4198 }
4199
4200 static u64 get_alloc_profile_by_root(struct btrfs_root *root, int data)
4201 {
4202 struct btrfs_fs_info *fs_info = root->fs_info;
4203 u64 flags;
4204 u64 ret;
4205
4206 if (data)
4207 flags = BTRFS_BLOCK_GROUP_DATA;
4208 else if (root == fs_info->chunk_root)
4209 flags = BTRFS_BLOCK_GROUP_SYSTEM;
4210 else
4211 flags = BTRFS_BLOCK_GROUP_METADATA;
4212
4213 ret = get_alloc_profile(fs_info, flags);
4214 return ret;
4215 }
4216
4217 u64 btrfs_data_alloc_profile(struct btrfs_fs_info *fs_info)
4218 {
4219 return get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_DATA);
4220 }
4221
4222 u64 btrfs_metadata_alloc_profile(struct btrfs_fs_info *fs_info)
4223 {
4224 return get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4225 }
4226
4227 u64 btrfs_system_alloc_profile(struct btrfs_fs_info *fs_info)
4228 {
4229 return get_alloc_profile(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
4230 }
4231
4232 static u64 btrfs_space_info_used(struct btrfs_space_info *s_info,
4233 bool may_use_included)
4234 {
4235 ASSERT(s_info);
4236 return s_info->bytes_used + s_info->bytes_reserved +
4237 s_info->bytes_pinned + s_info->bytes_readonly +
4238 (may_use_included ? s_info->bytes_may_use : 0);
4239 }
4240
4241 int btrfs_alloc_data_chunk_ondemand(struct btrfs_inode *inode, u64 bytes)
4242 {
4243 struct btrfs_root *root = inode->root;
4244 struct btrfs_fs_info *fs_info = root->fs_info;
4245 struct btrfs_space_info *data_sinfo = fs_info->data_sinfo;
4246 u64 used;
4247 int ret = 0;
4248 int need_commit = 2;
4249 int have_pinned_space;
4250
4251 /* make sure bytes are sectorsize aligned */
4252 bytes = ALIGN(bytes, fs_info->sectorsize);
4253
4254 if (btrfs_is_free_space_inode(inode)) {
4255 need_commit = 0;
4256 ASSERT(current->journal_info);
4257 }
4258
4259 again:
4260 /* make sure we have enough space to handle the data first */
4261 spin_lock(&data_sinfo->lock);
4262 used = btrfs_space_info_used(data_sinfo, true);
4263
4264 if (used + bytes > data_sinfo->total_bytes) {
4265 struct btrfs_trans_handle *trans;
4266
4267 /*
4268 * if we don't have enough free bytes in this space then we need
4269 * to alloc a new chunk.
4270 */
4271 if (!data_sinfo->full) {
4272 u64 alloc_target;
4273
4274 data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
4275 spin_unlock(&data_sinfo->lock);
4276
4277 alloc_target = btrfs_data_alloc_profile(fs_info);
4278 /*
4279 * It is ugly that we don't call nolock join
4280 * transaction for the free space inode case here.
4281 * But it is safe because we only do the data space
4282 * reservation for the free space cache in the
4283 * transaction context, the common join transaction
4284 * just increase the counter of the current transaction
4285 * handler, doesn't try to acquire the trans_lock of
4286 * the fs.
4287 */
4288 trans = btrfs_join_transaction(root);
4289 if (IS_ERR(trans))
4290 return PTR_ERR(trans);
4291
4292 ret = do_chunk_alloc(trans, fs_info, alloc_target,
4293 CHUNK_ALLOC_NO_FORCE);
4294 btrfs_end_transaction(trans);
4295 if (ret < 0) {
4296 if (ret != -ENOSPC)
4297 return ret;
4298 else {
4299 have_pinned_space = 1;
4300 goto commit_trans;
4301 }
4302 }
4303
4304 goto again;
4305 }
4306
4307 /*
4308 * If we don't have enough pinned space to deal with this
4309 * allocation, and no removed chunk in current transaction,
4310 * don't bother committing the transaction.
4311 */
4312 have_pinned_space = percpu_counter_compare(
4313 &data_sinfo->total_bytes_pinned,
4314 used + bytes - data_sinfo->total_bytes);
4315 spin_unlock(&data_sinfo->lock);
4316
4317 /* commit the current transaction and try again */
4318 commit_trans:
4319 if (need_commit) {
4320 need_commit--;
4321
4322 if (need_commit > 0) {
4323 btrfs_start_delalloc_roots(fs_info, -1);
4324 btrfs_wait_ordered_roots(fs_info, U64_MAX, 0,
4325 (u64)-1);
4326 }
4327
4328 trans = btrfs_join_transaction(root);
4329 if (IS_ERR(trans))
4330 return PTR_ERR(trans);
4331 if (have_pinned_space >= 0 ||
4332 test_bit(BTRFS_TRANS_HAVE_FREE_BGS,
4333 &trans->transaction->flags) ||
4334 need_commit > 0) {
4335 ret = btrfs_commit_transaction(trans);
4336 if (ret)
4337 return ret;
4338 /*
4339 * The cleaner kthread might still be doing iput
4340 * operations. Wait for it to finish so that
4341 * more space is released.
4342 */
4343 mutex_lock(&fs_info->cleaner_delayed_iput_mutex);
4344 mutex_unlock(&fs_info->cleaner_delayed_iput_mutex);
4345 goto again;
4346 } else {
4347 btrfs_end_transaction(trans);
4348 }
4349 }
4350
4351 trace_btrfs_space_reservation(fs_info,
4352 "space_info:enospc",
4353 data_sinfo->flags, bytes, 1);
4354 return -ENOSPC;
4355 }
4356 data_sinfo->bytes_may_use += bytes;
4357 trace_btrfs_space_reservation(fs_info, "space_info",
4358 data_sinfo->flags, bytes, 1);
4359 spin_unlock(&data_sinfo->lock);
4360
4361 return 0;
4362 }
4363
4364 int btrfs_check_data_free_space(struct inode *inode,
4365 struct extent_changeset **reserved, u64 start, u64 len)
4366 {
4367 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4368 int ret;
4369
4370 /* align the range */
4371 len = round_up(start + len, fs_info->sectorsize) -
4372 round_down(start, fs_info->sectorsize);
4373 start = round_down(start, fs_info->sectorsize);
4374
4375 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode), len);
4376 if (ret < 0)
4377 return ret;
4378
4379 /* Use new btrfs_qgroup_reserve_data to reserve precious data space. */
4380 ret = btrfs_qgroup_reserve_data(inode, reserved, start, len);
4381 if (ret < 0)
4382 btrfs_free_reserved_data_space_noquota(inode, start, len);
4383 else
4384 ret = 0;
4385 return ret;
4386 }
4387
4388 /*
4389 * Called if we need to clear a data reservation for this inode
4390 * Normally in a error case.
4391 *
4392 * This one will *NOT* use accurate qgroup reserved space API, just for case
4393 * which we can't sleep and is sure it won't affect qgroup reserved space.
4394 * Like clear_bit_hook().
4395 */
4396 void btrfs_free_reserved_data_space_noquota(struct inode *inode, u64 start,
4397 u64 len)
4398 {
4399 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4400 struct btrfs_space_info *data_sinfo;
4401
4402 /* Make sure the range is aligned to sectorsize */
4403 len = round_up(start + len, fs_info->sectorsize) -
4404 round_down(start, fs_info->sectorsize);
4405 start = round_down(start, fs_info->sectorsize);
4406
4407 data_sinfo = fs_info->data_sinfo;
4408 spin_lock(&data_sinfo->lock);
4409 if (WARN_ON(data_sinfo->bytes_may_use < len))
4410 data_sinfo->bytes_may_use = 0;
4411 else
4412 data_sinfo->bytes_may_use -= len;
4413 trace_btrfs_space_reservation(fs_info, "space_info",
4414 data_sinfo->flags, len, 0);
4415 spin_unlock(&data_sinfo->lock);
4416 }
4417
4418 /*
4419 * Called if we need to clear a data reservation for this inode
4420 * Normally in a error case.
4421 *
4422 * This one will handle the per-inode data rsv map for accurate reserved
4423 * space framework.
4424 */
4425 void btrfs_free_reserved_data_space(struct inode *inode,
4426 struct extent_changeset *reserved, u64 start, u64 len)
4427 {
4428 struct btrfs_root *root = BTRFS_I(inode)->root;
4429
4430 /* Make sure the range is aligned to sectorsize */
4431 len = round_up(start + len, root->fs_info->sectorsize) -
4432 round_down(start, root->fs_info->sectorsize);
4433 start = round_down(start, root->fs_info->sectorsize);
4434
4435 btrfs_free_reserved_data_space_noquota(inode, start, len);
4436 btrfs_qgroup_free_data(inode, reserved, start, len);
4437 }
4438
4439 static void force_metadata_allocation(struct btrfs_fs_info *info)
4440 {
4441 struct list_head *head = &info->space_info;
4442 struct btrfs_space_info *found;
4443
4444 rcu_read_lock();
4445 list_for_each_entry_rcu(found, head, list) {
4446 if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
4447 found->force_alloc = CHUNK_ALLOC_FORCE;
4448 }
4449 rcu_read_unlock();
4450 }
4451
4452 static inline u64 calc_global_rsv_need_space(struct btrfs_block_rsv *global)
4453 {
4454 return (global->size << 1);
4455 }
4456
4457 static int should_alloc_chunk(struct btrfs_fs_info *fs_info,
4458 struct btrfs_space_info *sinfo, int force)
4459 {
4460 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
4461 u64 bytes_used = btrfs_space_info_used(sinfo, false);
4462 u64 thresh;
4463
4464 if (force == CHUNK_ALLOC_FORCE)
4465 return 1;
4466
4467 /*
4468 * We need to take into account the global rsv because for all intents
4469 * and purposes it's used space. Don't worry about locking the
4470 * global_rsv, it doesn't change except when the transaction commits.
4471 */
4472 if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
4473 bytes_used += calc_global_rsv_need_space(global_rsv);
4474
4475 /*
4476 * in limited mode, we want to have some free space up to
4477 * about 1% of the FS size.
4478 */
4479 if (force == CHUNK_ALLOC_LIMITED) {
4480 thresh = btrfs_super_total_bytes(fs_info->super_copy);
4481 thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1));
4482
4483 if (sinfo->total_bytes - bytes_used < thresh)
4484 return 1;
4485 }
4486
4487 if (bytes_used + SZ_2M < div_factor(sinfo->total_bytes, 8))
4488 return 0;
4489 return 1;
4490 }
4491
4492 static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type)
4493 {
4494 u64 num_dev;
4495
4496 if (type & (BTRFS_BLOCK_GROUP_RAID10 |
4497 BTRFS_BLOCK_GROUP_RAID0 |
4498 BTRFS_BLOCK_GROUP_RAID5 |
4499 BTRFS_BLOCK_GROUP_RAID6))
4500 num_dev = fs_info->fs_devices->rw_devices;
4501 else if (type & BTRFS_BLOCK_GROUP_RAID1)
4502 num_dev = 2;
4503 else
4504 num_dev = 1; /* DUP or single */
4505
4506 return num_dev;
4507 }
4508
4509 /*
4510 * If @is_allocation is true, reserve space in the system space info necessary
4511 * for allocating a chunk, otherwise if it's false, reserve space necessary for
4512 * removing a chunk.
4513 */
4514 void check_system_chunk(struct btrfs_trans_handle *trans,
4515 struct btrfs_fs_info *fs_info, u64 type)
4516 {
4517 struct btrfs_space_info *info;
4518 u64 left;
4519 u64 thresh;
4520 int ret = 0;
4521 u64 num_devs;
4522
4523 /*
4524 * Needed because we can end up allocating a system chunk and for an
4525 * atomic and race free space reservation in the chunk block reserve.
4526 */
4527 lockdep_assert_held(&fs_info->chunk_mutex);
4528
4529 info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
4530 spin_lock(&info->lock);
4531 left = info->total_bytes - btrfs_space_info_used(info, true);
4532 spin_unlock(&info->lock);
4533
4534 num_devs = get_profile_num_devs(fs_info, type);
4535
4536 /* num_devs device items to update and 1 chunk item to add or remove */
4537 thresh = btrfs_calc_trunc_metadata_size(fs_info, num_devs) +
4538 btrfs_calc_trans_metadata_size(fs_info, 1);
4539
4540 if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
4541 btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu",
4542 left, thresh, type);
4543 dump_space_info(fs_info, info, 0, 0);
4544 }
4545
4546 if (left < thresh) {
4547 u64 flags = btrfs_system_alloc_profile(fs_info);
4548
4549 /*
4550 * Ignore failure to create system chunk. We might end up not
4551 * needing it, as we might not need to COW all nodes/leafs from
4552 * the paths we visit in the chunk tree (they were already COWed
4553 * or created in the current transaction for example).
4554 */
4555 ret = btrfs_alloc_chunk(trans, fs_info, flags);
4556 }
4557
4558 if (!ret) {
4559 ret = btrfs_block_rsv_add(fs_info->chunk_root,
4560 &fs_info->chunk_block_rsv,
4561 thresh, BTRFS_RESERVE_NO_FLUSH);
4562 if (!ret)
4563 trans->chunk_bytes_reserved += thresh;
4564 }
4565 }
4566
4567 /*
4568 * If force is CHUNK_ALLOC_FORCE:
4569 * - return 1 if it successfully allocates a chunk,
4570 * - return errors including -ENOSPC otherwise.
4571 * If force is NOT CHUNK_ALLOC_FORCE:
4572 * - return 0 if it doesn't need to allocate a new chunk,
4573 * - return 1 if it successfully allocates a chunk,
4574 * - return errors including -ENOSPC otherwise.
4575 */
4576 static int do_chunk_alloc(struct btrfs_trans_handle *trans,
4577 struct btrfs_fs_info *fs_info, u64 flags, int force)
4578 {
4579 struct btrfs_space_info *space_info;
4580 int wait_for_alloc = 0;
4581 int ret = 0;
4582
4583 /* Don't re-enter if we're already allocating a chunk */
4584 if (trans->allocating_chunk)
4585 return -ENOSPC;
4586
4587 space_info = __find_space_info(fs_info, flags);
4588 ASSERT(space_info);
4589
4590 again:
4591 spin_lock(&space_info->lock);
4592 if (force < space_info->force_alloc)
4593 force = space_info->force_alloc;
4594 if (space_info->full) {
4595 if (should_alloc_chunk(fs_info, space_info, force))
4596 ret = -ENOSPC;
4597 else
4598 ret = 0;
4599 spin_unlock(&space_info->lock);
4600 return ret;
4601 }
4602
4603 if (!should_alloc_chunk(fs_info, space_info, force)) {
4604 spin_unlock(&space_info->lock);
4605 return 0;
4606 } else if (space_info->chunk_alloc) {
4607 wait_for_alloc = 1;
4608 } else {
4609 space_info->chunk_alloc = 1;
4610 }
4611
4612 spin_unlock(&space_info->lock);
4613
4614 mutex_lock(&fs_info->chunk_mutex);
4615
4616 /*
4617 * The chunk_mutex is held throughout the entirety of a chunk
4618 * allocation, so once we've acquired the chunk_mutex we know that the
4619 * other guy is done and we need to recheck and see if we should
4620 * allocate.
4621 */
4622 if (wait_for_alloc) {
4623 mutex_unlock(&fs_info->chunk_mutex);
4624 wait_for_alloc = 0;
4625 cond_resched();
4626 goto again;
4627 }
4628
4629 trans->allocating_chunk = true;
4630
4631 /*
4632 * If we have mixed data/metadata chunks we want to make sure we keep
4633 * allocating mixed chunks instead of individual chunks.
4634 */
4635 if (btrfs_mixed_space_info(space_info))
4636 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
4637
4638 /*
4639 * if we're doing a data chunk, go ahead and make sure that
4640 * we keep a reasonable number of metadata chunks allocated in the
4641 * FS as well.
4642 */
4643 if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
4644 fs_info->data_chunk_allocations++;
4645 if (!(fs_info->data_chunk_allocations %
4646 fs_info->metadata_ratio))
4647 force_metadata_allocation(fs_info);
4648 }
4649
4650 /*
4651 * Check if we have enough space in SYSTEM chunk because we may need
4652 * to update devices.
4653 */
4654 check_system_chunk(trans, fs_info, flags);
4655
4656 ret = btrfs_alloc_chunk(trans, fs_info, flags);
4657 trans->allocating_chunk = false;
4658
4659 spin_lock(&space_info->lock);
4660 if (ret < 0) {
4661 if (ret == -ENOSPC)
4662 space_info->full = 1;
4663 else
4664 goto out;
4665 } else {
4666 ret = 1;
4667 }
4668
4669 space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
4670 out:
4671 space_info->chunk_alloc = 0;
4672 spin_unlock(&space_info->lock);
4673 mutex_unlock(&fs_info->chunk_mutex);
4674 /*
4675 * When we allocate a new chunk we reserve space in the chunk block
4676 * reserve to make sure we can COW nodes/leafs in the chunk tree or
4677 * add new nodes/leafs to it if we end up needing to do it when
4678 * inserting the chunk item and updating device items as part of the
4679 * second phase of chunk allocation, performed by
4680 * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a
4681 * large number of new block groups to create in our transaction
4682 * handle's new_bgs list to avoid exhausting the chunk block reserve
4683 * in extreme cases - like having a single transaction create many new
4684 * block groups when starting to write out the free space caches of all
4685 * the block groups that were made dirty during the lifetime of the
4686 * transaction.
4687 */
4688 if (trans->can_flush_pending_bgs &&
4689 trans->chunk_bytes_reserved >= (u64)SZ_2M) {
4690 btrfs_create_pending_block_groups(trans);
4691 btrfs_trans_release_chunk_metadata(trans);
4692 }
4693 return ret;
4694 }
4695
4696 static int can_overcommit(struct btrfs_fs_info *fs_info,
4697 struct btrfs_space_info *space_info, u64 bytes,
4698 enum btrfs_reserve_flush_enum flush,
4699 bool system_chunk)
4700 {
4701 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
4702 u64 profile;
4703 u64 space_size;
4704 u64 avail;
4705 u64 used;
4706
4707 /* Don't overcommit when in mixed mode. */
4708 if (space_info->flags & BTRFS_BLOCK_GROUP_DATA)
4709 return 0;
4710
4711 if (system_chunk)
4712 profile = btrfs_system_alloc_profile(fs_info);
4713 else
4714 profile = btrfs_metadata_alloc_profile(fs_info);
4715
4716 used = btrfs_space_info_used(space_info, false);
4717
4718 /*
4719 * We only want to allow over committing if we have lots of actual space
4720 * free, but if we don't have enough space to handle the global reserve
4721 * space then we could end up having a real enospc problem when trying
4722 * to allocate a chunk or some other such important allocation.
4723 */
4724 spin_lock(&global_rsv->lock);
4725 space_size = calc_global_rsv_need_space(global_rsv);
4726 spin_unlock(&global_rsv->lock);
4727 if (used + space_size >= space_info->total_bytes)
4728 return 0;
4729
4730 used += space_info->bytes_may_use;
4731
4732 avail = atomic64_read(&fs_info->free_chunk_space);
4733
4734 /*
4735 * If we have dup, raid1 or raid10 then only half of the free
4736 * space is actually useable. For raid56, the space info used
4737 * doesn't include the parity drive, so we don't have to
4738 * change the math
4739 */
4740 if (profile & (BTRFS_BLOCK_GROUP_DUP |
4741 BTRFS_BLOCK_GROUP_RAID1 |
4742 BTRFS_BLOCK_GROUP_RAID10))
4743 avail >>= 1;
4744
4745 /*
4746 * If we aren't flushing all things, let us overcommit up to
4747 * 1/2th of the space. If we can flush, don't let us overcommit
4748 * too much, let it overcommit up to 1/8 of the space.
4749 */
4750 if (flush == BTRFS_RESERVE_FLUSH_ALL)
4751 avail >>= 3;
4752 else
4753 avail >>= 1;
4754
4755 if (used + bytes < space_info->total_bytes + avail)
4756 return 1;
4757 return 0;
4758 }
4759
4760 static void btrfs_writeback_inodes_sb_nr(struct btrfs_fs_info *fs_info,
4761 unsigned long nr_pages, int nr_items)
4762 {
4763 struct super_block *sb = fs_info->sb;
4764
4765 if (down_read_trylock(&sb->s_umount)) {
4766 writeback_inodes_sb_nr(sb, nr_pages, WB_REASON_FS_FREE_SPACE);
4767 up_read(&sb->s_umount);
4768 } else {
4769 /*
4770 * We needn't worry the filesystem going from r/w to r/o though
4771 * we don't acquire ->s_umount mutex, because the filesystem
4772 * should guarantee the delalloc inodes list be empty after
4773 * the filesystem is readonly(all dirty pages are written to
4774 * the disk).
4775 */
4776 btrfs_start_delalloc_roots(fs_info, nr_items);
4777 if (!current->journal_info)
4778 btrfs_wait_ordered_roots(fs_info, nr_items, 0, (u64)-1);
4779 }
4780 }
4781
4782 static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info,
4783 u64 to_reclaim)
4784 {
4785 u64 bytes;
4786 u64 nr;
4787
4788 bytes = btrfs_calc_trans_metadata_size(fs_info, 1);
4789 nr = div64_u64(to_reclaim, bytes);
4790 if (!nr)
4791 nr = 1;
4792 return nr;
4793 }
4794
4795 #define EXTENT_SIZE_PER_ITEM SZ_256K
4796
4797 /*
4798 * shrink metadata reservation for delalloc
4799 */
4800 static void shrink_delalloc(struct btrfs_fs_info *fs_info, u64 to_reclaim,
4801 u64 orig, bool wait_ordered)
4802 {
4803 struct btrfs_space_info *space_info;
4804 struct btrfs_trans_handle *trans;
4805 u64 delalloc_bytes;
4806 u64 max_reclaim;
4807 u64 items;
4808 long time_left;
4809 unsigned long nr_pages;
4810 int loops;
4811
4812 /* Calc the number of the pages we need flush for space reservation */
4813 items = calc_reclaim_items_nr(fs_info, to_reclaim);
4814 to_reclaim = items * EXTENT_SIZE_PER_ITEM;
4815
4816 trans = (struct btrfs_trans_handle *)current->journal_info;
4817 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4818
4819 delalloc_bytes = percpu_counter_sum_positive(
4820 &fs_info->delalloc_bytes);
4821 if (delalloc_bytes == 0) {
4822 if (trans)
4823 return;
4824 if (wait_ordered)
4825 btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1);
4826 return;
4827 }
4828
4829 loops = 0;
4830 while (delalloc_bytes && loops < 3) {
4831 max_reclaim = min(delalloc_bytes, to_reclaim);
4832 nr_pages = max_reclaim >> PAGE_SHIFT;
4833 btrfs_writeback_inodes_sb_nr(fs_info, nr_pages, items);
4834 /*
4835 * We need to wait for the async pages to actually start before
4836 * we do anything.
4837 */
4838 max_reclaim = atomic_read(&fs_info->async_delalloc_pages);
4839 if (!max_reclaim)
4840 goto skip_async;
4841
4842 if (max_reclaim <= nr_pages)
4843 max_reclaim = 0;
4844 else
4845 max_reclaim -= nr_pages;
4846
4847 wait_event(fs_info->async_submit_wait,
4848 atomic_read(&fs_info->async_delalloc_pages) <=
4849 (int)max_reclaim);
4850 skip_async:
4851 spin_lock(&space_info->lock);
4852 if (list_empty(&space_info->tickets) &&
4853 list_empty(&space_info->priority_tickets)) {
4854 spin_unlock(&space_info->lock);
4855 break;
4856 }
4857 spin_unlock(&space_info->lock);
4858
4859 loops++;
4860 if (wait_ordered && !trans) {
4861 btrfs_wait_ordered_roots(fs_info, items, 0, (u64)-1);
4862 } else {
4863 time_left = schedule_timeout_killable(1);
4864 if (time_left)
4865 break;
4866 }
4867 delalloc_bytes = percpu_counter_sum_positive(
4868 &fs_info->delalloc_bytes);
4869 }
4870 }
4871
4872 struct reserve_ticket {
4873 u64 bytes;
4874 int error;
4875 struct list_head list;
4876 wait_queue_head_t wait;
4877 };
4878
4879 /**
4880 * maybe_commit_transaction - possibly commit the transaction if its ok to
4881 * @root - the root we're allocating for
4882 * @bytes - the number of bytes we want to reserve
4883 * @force - force the commit
4884 *
4885 * This will check to make sure that committing the transaction will actually
4886 * get us somewhere and then commit the transaction if it does. Otherwise it
4887 * will return -ENOSPC.
4888 */
4889 static int may_commit_transaction(struct btrfs_fs_info *fs_info,
4890 struct btrfs_space_info *space_info)
4891 {
4892 struct reserve_ticket *ticket = NULL;
4893 struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv;
4894 struct btrfs_trans_handle *trans;
4895 u64 bytes;
4896
4897 trans = (struct btrfs_trans_handle *)current->journal_info;
4898 if (trans)
4899 return -EAGAIN;
4900
4901 spin_lock(&space_info->lock);
4902 if (!list_empty(&space_info->priority_tickets))
4903 ticket = list_first_entry(&space_info->priority_tickets,
4904 struct reserve_ticket, list);
4905 else if (!list_empty(&space_info->tickets))
4906 ticket = list_first_entry(&space_info->tickets,
4907 struct reserve_ticket, list);
4908 bytes = (ticket) ? ticket->bytes : 0;
4909 spin_unlock(&space_info->lock);
4910
4911 if (!bytes)
4912 return 0;
4913
4914 /* See if there is enough pinned space to make this reservation */
4915 if (percpu_counter_compare(&space_info->total_bytes_pinned,
4916 bytes) >= 0)
4917 goto commit;
4918
4919 /*
4920 * See if there is some space in the delayed insertion reservation for
4921 * this reservation.
4922 */
4923 if (space_info != delayed_rsv->space_info)
4924 return -ENOSPC;
4925
4926 spin_lock(&delayed_rsv->lock);
4927 if (delayed_rsv->size > bytes)
4928 bytes = 0;
4929 else
4930 bytes -= delayed_rsv->size;
4931 spin_unlock(&delayed_rsv->lock);
4932
4933 if (percpu_counter_compare(&space_info->total_bytes_pinned,
4934 bytes) < 0) {
4935 return -ENOSPC;
4936 }
4937
4938 commit:
4939 trans = btrfs_join_transaction(fs_info->extent_root);
4940 if (IS_ERR(trans))
4941 return -ENOSPC;
4942
4943 return btrfs_commit_transaction(trans);
4944 }
4945
4946 /*
4947 * Try to flush some data based on policy set by @state. This is only advisory
4948 * and may fail for various reasons. The caller is supposed to examine the
4949 * state of @space_info to detect the outcome.
4950 */
4951 static void flush_space(struct btrfs_fs_info *fs_info,
4952 struct btrfs_space_info *space_info, u64 num_bytes,
4953 int state)
4954 {
4955 struct btrfs_root *root = fs_info->extent_root;
4956 struct btrfs_trans_handle *trans;
4957 int nr;
4958 int ret = 0;
4959
4960 switch (state) {
4961 case FLUSH_DELAYED_ITEMS_NR:
4962 case FLUSH_DELAYED_ITEMS:
4963 if (state == FLUSH_DELAYED_ITEMS_NR)
4964 nr = calc_reclaim_items_nr(fs_info, num_bytes) * 2;
4965 else
4966 nr = -1;
4967
4968 trans = btrfs_join_transaction(root);
4969 if (IS_ERR(trans)) {
4970 ret = PTR_ERR(trans);
4971 break;
4972 }
4973 ret = btrfs_run_delayed_items_nr(trans, nr);
4974 btrfs_end_transaction(trans);
4975 break;
4976 case FLUSH_DELALLOC:
4977 case FLUSH_DELALLOC_WAIT:
4978 shrink_delalloc(fs_info, num_bytes * 2, num_bytes,
4979 state == FLUSH_DELALLOC_WAIT);
4980 break;
4981 case ALLOC_CHUNK:
4982 trans = btrfs_join_transaction(root);
4983 if (IS_ERR(trans)) {
4984 ret = PTR_ERR(trans);
4985 break;
4986 }
4987 ret = do_chunk_alloc(trans, fs_info,
4988 btrfs_metadata_alloc_profile(fs_info),
4989 CHUNK_ALLOC_NO_FORCE);
4990 btrfs_end_transaction(trans);
4991 if (ret > 0 || ret == -ENOSPC)
4992 ret = 0;
4993 break;
4994 case COMMIT_TRANS:
4995 ret = may_commit_transaction(fs_info, space_info);
4996 break;
4997 default:
4998 ret = -ENOSPC;
4999 break;
5000 }
5001
5002 trace_btrfs_flush_space(fs_info, space_info->flags, num_bytes, state,
5003 ret);
5004 return;
5005 }
5006
5007 static inline u64
5008 btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info,
5009 struct btrfs_space_info *space_info,
5010 bool system_chunk)
5011 {
5012 struct reserve_ticket *ticket;
5013 u64 used;
5014 u64 expected;
5015 u64 to_reclaim = 0;
5016
5017 list_for_each_entry(ticket, &space_info->tickets, list)
5018 to_reclaim += ticket->bytes;
5019 list_for_each_entry(ticket, &space_info->priority_tickets, list)
5020 to_reclaim += ticket->bytes;
5021 if (to_reclaim)
5022 return to_reclaim;
5023
5024 to_reclaim = min_t(u64, num_online_cpus() * SZ_1M, SZ_16M);
5025 if (can_overcommit(fs_info, space_info, to_reclaim,
5026 BTRFS_RESERVE_FLUSH_ALL, system_chunk))
5027 return 0;
5028
5029 used = btrfs_space_info_used(space_info, true);
5030
5031 if (can_overcommit(fs_info, space_info, SZ_1M,
5032 BTRFS_RESERVE_FLUSH_ALL, system_chunk))
5033 expected = div_factor_fine(space_info->total_bytes, 95);
5034 else
5035 expected = div_factor_fine(space_info->total_bytes, 90);
5036
5037 if (used > expected)
5038 to_reclaim = used - expected;
5039 else
5040 to_reclaim = 0;
5041 to_reclaim = min(to_reclaim, space_info->bytes_may_use +
5042 space_info->bytes_reserved);
5043 return to_reclaim;
5044 }
5045
5046 static inline int need_do_async_reclaim(struct btrfs_fs_info *fs_info,
5047 struct btrfs_space_info *space_info,
5048 u64 used, bool system_chunk)
5049 {
5050 u64 thresh = div_factor_fine(space_info->total_bytes, 98);
5051
5052 /* If we're just plain full then async reclaim just slows us down. */
5053 if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh)
5054 return 0;
5055
5056 if (!btrfs_calc_reclaim_metadata_size(fs_info, space_info,
5057 system_chunk))
5058 return 0;
5059
5060 return (used >= thresh && !btrfs_fs_closing(fs_info) &&
5061 !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state));
5062 }
5063
5064 static void wake_all_tickets(struct list_head *head)
5065 {
5066 struct reserve_ticket *ticket;
5067
5068 while (!list_empty(head)) {
5069 ticket = list_first_entry(head, struct reserve_ticket, list);
5070 list_del_init(&ticket->list);
5071 ticket->error = -ENOSPC;
5072 wake_up(&ticket->wait);
5073 }
5074 }
5075
5076 /*
5077 * This is for normal flushers, we can wait all goddamned day if we want to. We
5078 * will loop and continuously try to flush as long as we are making progress.
5079 * We count progress as clearing off tickets each time we have to loop.
5080 */
5081 static void btrfs_async_reclaim_metadata_space(struct work_struct *work)
5082 {
5083 struct btrfs_fs_info *fs_info;
5084 struct btrfs_space_info *space_info;
5085 u64 to_reclaim;
5086 int flush_state;
5087 int commit_cycles = 0;
5088 u64 last_tickets_id;
5089
5090 fs_info = container_of(work, struct btrfs_fs_info, async_reclaim_work);
5091 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
5092
5093 spin_lock(&space_info->lock);
5094 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info,
5095 false);
5096 if (!to_reclaim) {
5097 space_info->flush = 0;
5098 spin_unlock(&space_info->lock);
5099 return;
5100 }
5101 last_tickets_id = space_info->tickets_id;
5102 spin_unlock(&space_info->lock);
5103
5104 flush_state = FLUSH_DELAYED_ITEMS_NR;
5105 do {
5106 flush_space(fs_info, space_info, to_reclaim, flush_state);
5107 spin_lock(&space_info->lock);
5108 if (list_empty(&space_info->tickets)) {
5109 space_info->flush = 0;
5110 spin_unlock(&space_info->lock);
5111 return;
5112 }
5113 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info,
5114 space_info,
5115 false);
5116 if (last_tickets_id == space_info->tickets_id) {
5117 flush_state++;
5118 } else {
5119 last_tickets_id = space_info->tickets_id;
5120 flush_state = FLUSH_DELAYED_ITEMS_NR;
5121 if (commit_cycles)
5122 commit_cycles--;
5123 }
5124
5125 if (flush_state > COMMIT_TRANS) {
5126 commit_cycles++;
5127 if (commit_cycles > 2) {
5128 wake_all_tickets(&space_info->tickets);
5129 space_info->flush = 0;
5130 } else {
5131 flush_state = FLUSH_DELAYED_ITEMS_NR;
5132 }
5133 }
5134 spin_unlock(&space_info->lock);
5135 } while (flush_state <= COMMIT_TRANS);
5136 }
5137
5138 void btrfs_init_async_reclaim_work(struct work_struct *work)
5139 {
5140 INIT_WORK(work, btrfs_async_reclaim_metadata_space);
5141 }
5142
5143 static void priority_reclaim_metadata_space(struct btrfs_fs_info *fs_info,
5144 struct btrfs_space_info *space_info,
5145 struct reserve_ticket *ticket)
5146 {
5147 u64 to_reclaim;
5148 int flush_state = FLUSH_DELAYED_ITEMS_NR;
5149
5150 spin_lock(&space_info->lock);
5151 to_reclaim = btrfs_calc_reclaim_metadata_size(fs_info, space_info,
5152 false);
5153 if (!to_reclaim) {
5154 spin_unlock(&space_info->lock);
5155 return;
5156 }
5157 spin_unlock(&space_info->lock);
5158
5159 do {
5160 flush_space(fs_info, space_info, to_reclaim, flush_state);
5161 flush_state++;
5162 spin_lock(&space_info->lock);
5163 if (ticket->bytes == 0) {
5164 spin_unlock(&space_info->lock);
5165 return;
5166 }
5167 spin_unlock(&space_info->lock);
5168
5169 /*
5170 * Priority flushers can't wait on delalloc without
5171 * deadlocking.
5172 */
5173 if (flush_state == FLUSH_DELALLOC ||
5174 flush_state == FLUSH_DELALLOC_WAIT)
5175 flush_state = ALLOC_CHUNK;
5176 } while (flush_state < COMMIT_TRANS);
5177 }
5178
5179 static int wait_reserve_ticket(struct btrfs_fs_info *fs_info,
5180 struct btrfs_space_info *space_info,
5181 struct reserve_ticket *ticket, u64 orig_bytes)
5182
5183 {
5184 DEFINE_WAIT(wait);
5185 int ret = 0;
5186
5187 spin_lock(&space_info->lock);
5188 while (ticket->bytes > 0 && ticket->error == 0) {
5189 ret = prepare_to_wait_event(&ticket->wait, &wait, TASK_KILLABLE);
5190 if (ret) {
5191 ret = -EINTR;
5192 break;
5193 }
5194 spin_unlock(&space_info->lock);
5195
5196 schedule();
5197
5198 finish_wait(&ticket->wait, &wait);
5199 spin_lock(&space_info->lock);
5200 }
5201 if (!ret)
5202 ret = ticket->error;
5203 if (!list_empty(&ticket->list))
5204 list_del_init(&ticket->list);
5205 if (ticket->bytes && ticket->bytes < orig_bytes) {
5206 u64 num_bytes = orig_bytes - ticket->bytes;
5207 space_info->bytes_may_use -= num_bytes;
5208 trace_btrfs_space_reservation(fs_info, "space_info",
5209 space_info->flags, num_bytes, 0);
5210 }
5211 spin_unlock(&space_info->lock);
5212
5213 return ret;
5214 }
5215
5216 /**
5217 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
5218 * @root - the root we're allocating for
5219 * @space_info - the space info we want to allocate from
5220 * @orig_bytes - the number of bytes we want
5221 * @flush - whether or not we can flush to make our reservation
5222 *
5223 * This will reserve orig_bytes number of bytes from the space info associated
5224 * with the block_rsv. If there is not enough space it will make an attempt to
5225 * flush out space to make room. It will do this by flushing delalloc if
5226 * possible or committing the transaction. If flush is 0 then no attempts to
5227 * regain reservations will be made and this will fail if there is not enough
5228 * space already.
5229 */
5230 static int __reserve_metadata_bytes(struct btrfs_fs_info *fs_info,
5231 struct btrfs_space_info *space_info,
5232 u64 orig_bytes,
5233 enum btrfs_reserve_flush_enum flush,
5234 bool system_chunk)
5235 {
5236 struct reserve_ticket ticket;
5237 u64 used;
5238 int ret = 0;
5239
5240 ASSERT(orig_bytes);
5241 ASSERT(!current->journal_info || flush != BTRFS_RESERVE_FLUSH_ALL);
5242
5243 spin_lock(&space_info->lock);
5244 ret = -ENOSPC;
5245 used = btrfs_space_info_used(space_info, true);
5246
5247 /*
5248 * If we have enough space then hooray, make our reservation and carry
5249 * on. If not see if we can overcommit, and if we can, hooray carry on.
5250 * If not things get more complicated.
5251 */
5252 if (used + orig_bytes <= space_info->total_bytes) {
5253 space_info->bytes_may_use += orig_bytes;
5254 trace_btrfs_space_reservation(fs_info, "space_info",
5255 space_info->flags, orig_bytes, 1);
5256 ret = 0;
5257 } else if (can_overcommit(fs_info, space_info, orig_bytes, flush,
5258 system_chunk)) {
5259 space_info->bytes_may_use += orig_bytes;
5260 trace_btrfs_space_reservation(fs_info, "space_info",
5261 space_info->flags, orig_bytes, 1);
5262 ret = 0;
5263 }
5264
5265 /*
5266 * If we couldn't make a reservation then setup our reservation ticket
5267 * and kick the async worker if it's not already running.
5268 *
5269 * If we are a priority flusher then we just need to add our ticket to
5270 * the list and we will do our own flushing further down.
5271 */
5272 if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
5273 ticket.bytes = orig_bytes;
5274 ticket.error = 0;
5275 init_waitqueue_head(&ticket.wait);
5276 if (flush == BTRFS_RESERVE_FLUSH_ALL) {
5277 list_add_tail(&ticket.list, &space_info->tickets);
5278 if (!space_info->flush) {
5279 space_info->flush = 1;
5280 trace_btrfs_trigger_flush(fs_info,
5281 space_info->flags,
5282 orig_bytes, flush,
5283 "enospc");
5284 queue_work(system_unbound_wq,
5285 &fs_info->async_reclaim_work);
5286 }
5287 } else {
5288 list_add_tail(&ticket.list,
5289 &space_info->priority_tickets);
5290 }
5291 } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {
5292 used += orig_bytes;
5293 /*
5294 * We will do the space reservation dance during log replay,
5295 * which means we won't have fs_info->fs_root set, so don't do
5296 * the async reclaim as we will panic.
5297 */
5298 if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) &&
5299 need_do_async_reclaim(fs_info, space_info,
5300 used, system_chunk) &&
5301 !work_busy(&fs_info->async_reclaim_work)) {
5302 trace_btrfs_trigger_flush(fs_info, space_info->flags,
5303 orig_bytes, flush, "preempt");
5304 queue_work(system_unbound_wq,
5305 &fs_info->async_reclaim_work);
5306 }
5307 }
5308 spin_unlock(&space_info->lock);
5309 if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
5310 return ret;
5311
5312 if (flush == BTRFS_RESERVE_FLUSH_ALL)
5313 return wait_reserve_ticket(fs_info, space_info, &ticket,
5314 orig_bytes);
5315
5316 ret = 0;
5317 priority_reclaim_metadata_space(fs_info, space_info, &ticket);
5318 spin_lock(&space_info->lock);
5319 if (ticket.bytes) {
5320 if (ticket.bytes < orig_bytes) {
5321 u64 num_bytes = orig_bytes - ticket.bytes;
5322 space_info->bytes_may_use -= num_bytes;
5323 trace_btrfs_space_reservation(fs_info, "space_info",
5324 space_info->flags,
5325 num_bytes, 0);
5326
5327 }
5328 list_del_init(&ticket.list);
5329 ret = -ENOSPC;
5330 }
5331 spin_unlock(&space_info->lock);
5332 ASSERT(list_empty(&ticket.list));
5333 return ret;
5334 }
5335
5336 /**
5337 * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
5338 * @root - the root we're allocating for
5339 * @block_rsv - the block_rsv we're allocating for
5340 * @orig_bytes - the number of bytes we want
5341 * @flush - whether or not we can flush to make our reservation
5342 *
5343 * This will reserve orgi_bytes number of bytes from the space info associated
5344 * with the block_rsv. If there is not enough space it will make an attempt to
5345 * flush out space to make room. It will do this by flushing delalloc if
5346 * possible or committing the transaction. If flush is 0 then no attempts to
5347 * regain reservations will be made and this will fail if there is not enough
5348 * space already.
5349 */
5350 static int reserve_metadata_bytes(struct btrfs_root *root,
5351 struct btrfs_block_rsv *block_rsv,
5352 u64 orig_bytes,
5353 enum btrfs_reserve_flush_enum flush)
5354 {
5355 struct btrfs_fs_info *fs_info = root->fs_info;
5356 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5357 int ret;
5358 bool system_chunk = (root == fs_info->chunk_root);
5359
5360 ret = __reserve_metadata_bytes(fs_info, block_rsv->space_info,
5361 orig_bytes, flush, system_chunk);
5362 if (ret == -ENOSPC &&
5363 unlikely(root->orphan_cleanup_state == ORPHAN_CLEANUP_STARTED)) {
5364 if (block_rsv != global_rsv &&
5365 !block_rsv_use_bytes(global_rsv, orig_bytes))
5366 ret = 0;
5367 }
5368 if (ret == -ENOSPC) {
5369 trace_btrfs_space_reservation(fs_info, "space_info:enospc",
5370 block_rsv->space_info->flags,
5371 orig_bytes, 1);
5372
5373 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG))
5374 dump_space_info(fs_info, block_rsv->space_info,
5375 orig_bytes, 0);
5376 }
5377 return ret;
5378 }
5379
5380 static struct btrfs_block_rsv *get_block_rsv(
5381 const struct btrfs_trans_handle *trans,
5382 const struct btrfs_root *root)
5383 {
5384 struct btrfs_fs_info *fs_info = root->fs_info;
5385 struct btrfs_block_rsv *block_rsv = NULL;
5386
5387 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
5388 (root == fs_info->csum_root && trans->adding_csums) ||
5389 (root == fs_info->uuid_root))
5390 block_rsv = trans->block_rsv;
5391
5392 if (!block_rsv)
5393 block_rsv = root->block_rsv;
5394
5395 if (!block_rsv)
5396 block_rsv = &fs_info->empty_block_rsv;
5397
5398 return block_rsv;
5399 }
5400
5401 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
5402 u64 num_bytes)
5403 {
5404 int ret = -ENOSPC;
5405 spin_lock(&block_rsv->lock);
5406 if (block_rsv->reserved >= num_bytes) {
5407 block_rsv->reserved -= num_bytes;
5408 if (block_rsv->reserved < block_rsv->size)
5409 block_rsv->full = 0;
5410 ret = 0;
5411 }
5412 spin_unlock(&block_rsv->lock);
5413 return ret;
5414 }
5415
5416 static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
5417 u64 num_bytes, int update_size)
5418 {
5419 spin_lock(&block_rsv->lock);
5420 block_rsv->reserved += num_bytes;
5421 if (update_size)
5422 block_rsv->size += num_bytes;
5423 else if (block_rsv->reserved >= block_rsv->size)
5424 block_rsv->full = 1;
5425 spin_unlock(&block_rsv->lock);
5426 }
5427
5428 int btrfs_cond_migrate_bytes(struct btrfs_fs_info *fs_info,
5429 struct btrfs_block_rsv *dest, u64 num_bytes,
5430 int min_factor)
5431 {
5432 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5433 u64 min_bytes;
5434
5435 if (global_rsv->space_info != dest->space_info)
5436 return -ENOSPC;
5437
5438 spin_lock(&global_rsv->lock);
5439 min_bytes = div_factor(global_rsv->size, min_factor);
5440 if (global_rsv->reserved < min_bytes + num_bytes) {
5441 spin_unlock(&global_rsv->lock);
5442 return -ENOSPC;
5443 }
5444 global_rsv->reserved -= num_bytes;
5445 if (global_rsv->reserved < global_rsv->size)
5446 global_rsv->full = 0;
5447 spin_unlock(&global_rsv->lock);
5448
5449 block_rsv_add_bytes(dest, num_bytes, 1);
5450 return 0;
5451 }
5452
5453 /*
5454 * This is for space we already have accounted in space_info->bytes_may_use, so
5455 * basically when we're returning space from block_rsv's.
5456 */
5457 static void space_info_add_old_bytes(struct btrfs_fs_info *fs_info,
5458 struct btrfs_space_info *space_info,
5459 u64 num_bytes)
5460 {
5461 struct reserve_ticket *ticket;
5462 struct list_head *head;
5463 u64 used;
5464 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_NO_FLUSH;
5465 bool check_overcommit = false;
5466
5467 spin_lock(&space_info->lock);
5468 head = &space_info->priority_tickets;
5469
5470 /*
5471 * If we are over our limit then we need to check and see if we can
5472 * overcommit, and if we can't then we just need to free up our space
5473 * and not satisfy any requests.
5474 */
5475 used = btrfs_space_info_used(space_info, true);
5476 if (used - num_bytes >= space_info->total_bytes)
5477 check_overcommit = true;
5478 again:
5479 while (!list_empty(head) && num_bytes) {
5480 ticket = list_first_entry(head, struct reserve_ticket,
5481 list);
5482 /*
5483 * We use 0 bytes because this space is already reserved, so
5484 * adding the ticket space would be a double count.
5485 */
5486 if (check_overcommit &&
5487 !can_overcommit(fs_info, space_info, 0, flush, false))
5488 break;
5489 if (num_bytes >= ticket->bytes) {
5490 list_del_init(&ticket->list);
5491 num_bytes -= ticket->bytes;
5492 ticket->bytes = 0;
5493 space_info->tickets_id++;
5494 wake_up(&ticket->wait);
5495 } else {
5496 ticket->bytes -= num_bytes;
5497 num_bytes = 0;
5498 }
5499 }
5500
5501 if (num_bytes && head == &space_info->priority_tickets) {
5502 head = &space_info->tickets;
5503 flush = BTRFS_RESERVE_FLUSH_ALL;
5504 goto again;
5505 }
5506 space_info->bytes_may_use -= num_bytes;
5507 trace_btrfs_space_reservation(fs_info, "space_info",
5508 space_info->flags, num_bytes, 0);
5509 spin_unlock(&space_info->lock);
5510 }
5511
5512 /*
5513 * This is for newly allocated space that isn't accounted in
5514 * space_info->bytes_may_use yet. So if we allocate a chunk or unpin an extent
5515 * we use this helper.
5516 */
5517 static void space_info_add_new_bytes(struct btrfs_fs_info *fs_info,
5518 struct btrfs_space_info *space_info,
5519 u64 num_bytes)
5520 {
5521 struct reserve_ticket *ticket;
5522 struct list_head *head = &space_info->priority_tickets;
5523
5524 again:
5525 while (!list_empty(head) && num_bytes) {
5526 ticket = list_first_entry(head, struct reserve_ticket,
5527 list);
5528 if (num_bytes >= ticket->bytes) {
5529 trace_btrfs_space_reservation(fs_info, "space_info",
5530 space_info->flags,
5531 ticket->bytes, 1);
5532 list_del_init(&ticket->list);
5533 num_bytes -= ticket->bytes;
5534 space_info->bytes_may_use += ticket->bytes;
5535 ticket->bytes = 0;
5536 space_info->tickets_id++;
5537 wake_up(&ticket->wait);
5538 } else {
5539 trace_btrfs_space_reservation(fs_info, "space_info",
5540 space_info->flags,
5541 num_bytes, 1);
5542 space_info->bytes_may_use += num_bytes;
5543 ticket->bytes -= num_bytes;
5544 num_bytes = 0;
5545 }
5546 }
5547
5548 if (num_bytes && head == &space_info->priority_tickets) {
5549 head = &space_info->tickets;
5550 goto again;
5551 }
5552 }
5553
5554 static u64 block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
5555 struct btrfs_block_rsv *block_rsv,
5556 struct btrfs_block_rsv *dest, u64 num_bytes,
5557 u64 *qgroup_to_release_ret)
5558 {
5559 struct btrfs_space_info *space_info = block_rsv->space_info;
5560 u64 qgroup_to_release = 0;
5561 u64 ret;
5562
5563 spin_lock(&block_rsv->lock);
5564 if (num_bytes == (u64)-1) {
5565 num_bytes = block_rsv->size;
5566 qgroup_to_release = block_rsv->qgroup_rsv_size;
5567 }
5568 block_rsv->size -= num_bytes;
5569 if (block_rsv->reserved >= block_rsv->size) {
5570 num_bytes = block_rsv->reserved - block_rsv->size;
5571 block_rsv->reserved = block_rsv->size;
5572 block_rsv->full = 1;
5573 } else {
5574 num_bytes = 0;
5575 }
5576 if (block_rsv->qgroup_rsv_reserved >= block_rsv->qgroup_rsv_size) {
5577 qgroup_to_release = block_rsv->qgroup_rsv_reserved -
5578 block_rsv->qgroup_rsv_size;
5579 block_rsv->qgroup_rsv_reserved = block_rsv->qgroup_rsv_size;
5580 } else {
5581 qgroup_to_release = 0;
5582 }
5583 spin_unlock(&block_rsv->lock);
5584
5585 ret = num_bytes;
5586 if (num_bytes > 0) {
5587 if (dest) {
5588 spin_lock(&dest->lock);
5589 if (!dest->full) {
5590 u64 bytes_to_add;
5591
5592 bytes_to_add = dest->size - dest->reserved;
5593 bytes_to_add = min(num_bytes, bytes_to_add);
5594 dest->reserved += bytes_to_add;
5595 if (dest->reserved >= dest->size)
5596 dest->full = 1;
5597 num_bytes -= bytes_to_add;
5598 }
5599 spin_unlock(&dest->lock);
5600 }
5601 if (num_bytes)
5602 space_info_add_old_bytes(fs_info, space_info,
5603 num_bytes);
5604 }
5605 if (qgroup_to_release_ret)
5606 *qgroup_to_release_ret = qgroup_to_release;
5607 return ret;
5608 }
5609
5610 int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src,
5611 struct btrfs_block_rsv *dst, u64 num_bytes,
5612 int update_size)
5613 {
5614 int ret;
5615
5616 ret = block_rsv_use_bytes(src, num_bytes);
5617 if (ret)
5618 return ret;
5619
5620 block_rsv_add_bytes(dst, num_bytes, update_size);
5621 return 0;
5622 }
5623
5624 void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
5625 {
5626 memset(rsv, 0, sizeof(*rsv));
5627 spin_lock_init(&rsv->lock);
5628 rsv->type = type;
5629 }
5630
5631 void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info,
5632 struct btrfs_block_rsv *rsv,
5633 unsigned short type)
5634 {
5635 btrfs_init_block_rsv(rsv, type);
5636 rsv->space_info = __find_space_info(fs_info,
5637 BTRFS_BLOCK_GROUP_METADATA);
5638 }
5639
5640 struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info,
5641 unsigned short type)
5642 {
5643 struct btrfs_block_rsv *block_rsv;
5644
5645 block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
5646 if (!block_rsv)
5647 return NULL;
5648
5649 btrfs_init_metadata_block_rsv(fs_info, block_rsv, type);
5650 return block_rsv;
5651 }
5652
5653 void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info,
5654 struct btrfs_block_rsv *rsv)
5655 {
5656 if (!rsv)
5657 return;
5658 btrfs_block_rsv_release(fs_info, rsv, (u64)-1);
5659 kfree(rsv);
5660 }
5661
5662 void __btrfs_free_block_rsv(struct btrfs_block_rsv *rsv)
5663 {
5664 kfree(rsv);
5665 }
5666
5667 int btrfs_block_rsv_add(struct btrfs_root *root,
5668 struct btrfs_block_rsv *block_rsv, u64 num_bytes,
5669 enum btrfs_reserve_flush_enum flush)
5670 {
5671 int ret;
5672
5673 if (num_bytes == 0)
5674 return 0;
5675
5676 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
5677 if (!ret) {
5678 block_rsv_add_bytes(block_rsv, num_bytes, 1);
5679 return 0;
5680 }
5681
5682 return ret;
5683 }
5684
5685 int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_factor)
5686 {
5687 u64 num_bytes = 0;
5688 int ret = -ENOSPC;
5689
5690 if (!block_rsv)
5691 return 0;
5692
5693 spin_lock(&block_rsv->lock);
5694 num_bytes = div_factor(block_rsv->size, min_factor);
5695 if (block_rsv->reserved >= num_bytes)
5696 ret = 0;
5697 spin_unlock(&block_rsv->lock);
5698
5699 return ret;
5700 }
5701
5702 int btrfs_block_rsv_refill(struct btrfs_root *root,
5703 struct btrfs_block_rsv *block_rsv, u64 min_reserved,
5704 enum btrfs_reserve_flush_enum flush)
5705 {
5706 u64 num_bytes = 0;
5707 int ret = -ENOSPC;
5708
5709 if (!block_rsv)
5710 return 0;
5711
5712 spin_lock(&block_rsv->lock);
5713 num_bytes = min_reserved;
5714 if (block_rsv->reserved >= num_bytes)
5715 ret = 0;
5716 else
5717 num_bytes -= block_rsv->reserved;
5718 spin_unlock(&block_rsv->lock);
5719
5720 if (!ret)
5721 return 0;
5722
5723 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
5724 if (!ret) {
5725 block_rsv_add_bytes(block_rsv, num_bytes, 0);
5726 return 0;
5727 }
5728
5729 return ret;
5730 }
5731
5732 /**
5733 * btrfs_inode_rsv_refill - refill the inode block rsv.
5734 * @inode - the inode we are refilling.
5735 * @flush - the flusing restriction.
5736 *
5737 * Essentially the same as btrfs_block_rsv_refill, except it uses the
5738 * block_rsv->size as the minimum size. We'll either refill the missing amount
5739 * or return if we already have enough space. This will also handle the resreve
5740 * tracepoint for the reserved amount.
5741 */
5742 static int btrfs_inode_rsv_refill(struct btrfs_inode *inode,
5743 enum btrfs_reserve_flush_enum flush)
5744 {
5745 struct btrfs_root *root = inode->root;
5746 struct btrfs_block_rsv *block_rsv = &inode->block_rsv;
5747 u64 num_bytes = 0;
5748 u64 qgroup_num_bytes = 0;
5749 int ret = -ENOSPC;
5750
5751 spin_lock(&block_rsv->lock);
5752 if (block_rsv->reserved < block_rsv->size)
5753 num_bytes = block_rsv->size - block_rsv->reserved;
5754 if (block_rsv->qgroup_rsv_reserved < block_rsv->qgroup_rsv_size)
5755 qgroup_num_bytes = block_rsv->qgroup_rsv_size -
5756 block_rsv->qgroup_rsv_reserved;
5757 spin_unlock(&block_rsv->lock);
5758
5759 if (num_bytes == 0)
5760 return 0;
5761
5762 ret = btrfs_qgroup_reserve_meta_prealloc(root, qgroup_num_bytes, true);
5763 if (ret)
5764 return ret;
5765 ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
5766 if (!ret) {
5767 block_rsv_add_bytes(block_rsv, num_bytes, 0);
5768 trace_btrfs_space_reservation(root->fs_info, "delalloc",
5769 btrfs_ino(inode), num_bytes, 1);
5770
5771 /* Don't forget to increase qgroup_rsv_reserved */
5772 spin_lock(&block_rsv->lock);
5773 block_rsv->qgroup_rsv_reserved += qgroup_num_bytes;
5774 spin_unlock(&block_rsv->lock);
5775 } else
5776 btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes);
5777 return ret;
5778 }
5779
5780 /**
5781 * btrfs_inode_rsv_release - release any excessive reservation.
5782 * @inode - the inode we need to release from.
5783 * @qgroup_free - free or convert qgroup meta.
5784 * Unlike normal operation, qgroup meta reservation needs to know if we are
5785 * freeing qgroup reservation or just converting it into per-trans. Normally
5786 * @qgroup_free is true for error handling, and false for normal release.
5787 *
5788 * This is the same as btrfs_block_rsv_release, except that it handles the
5789 * tracepoint for the reservation.
5790 */
5791 static void btrfs_inode_rsv_release(struct btrfs_inode *inode, bool qgroup_free)
5792 {
5793 struct btrfs_fs_info *fs_info = inode->root->fs_info;
5794 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5795 struct btrfs_block_rsv *block_rsv = &inode->block_rsv;
5796 u64 released = 0;
5797 u64 qgroup_to_release = 0;
5798
5799 /*
5800 * Since we statically set the block_rsv->size we just want to say we
5801 * are releasing 0 bytes, and then we'll just get the reservation over
5802 * the size free'd.
5803 */
5804 released = block_rsv_release_bytes(fs_info, block_rsv, global_rsv, 0,
5805 &qgroup_to_release);
5806 if (released > 0)
5807 trace_btrfs_space_reservation(fs_info, "delalloc",
5808 btrfs_ino(inode), released, 0);
5809 if (qgroup_free)
5810 btrfs_qgroup_free_meta_prealloc(inode->root, qgroup_to_release);
5811 else
5812 btrfs_qgroup_convert_reserved_meta(inode->root,
5813 qgroup_to_release);
5814 }
5815
5816 void btrfs_block_rsv_release(struct btrfs_fs_info *fs_info,
5817 struct btrfs_block_rsv *block_rsv,
5818 u64 num_bytes)
5819 {
5820 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5821
5822 if (global_rsv == block_rsv ||
5823 block_rsv->space_info != global_rsv->space_info)
5824 global_rsv = NULL;
5825 block_rsv_release_bytes(fs_info, block_rsv, global_rsv, num_bytes, NULL);
5826 }
5827
5828 static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
5829 {
5830 struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
5831 struct btrfs_space_info *sinfo = block_rsv->space_info;
5832 u64 num_bytes;
5833
5834 /*
5835 * The global block rsv is based on the size of the extent tree, the
5836 * checksum tree and the root tree. If the fs is empty we want to set
5837 * it to a minimal amount for safety.
5838 */
5839 num_bytes = btrfs_root_used(&fs_info->extent_root->root_item) +
5840 btrfs_root_used(&fs_info->csum_root->root_item) +
5841 btrfs_root_used(&fs_info->tree_root->root_item);
5842 num_bytes = max_t(u64, num_bytes, SZ_16M);
5843
5844 spin_lock(&sinfo->lock);
5845 spin_lock(&block_rsv->lock);
5846
5847 block_rsv->size = min_t(u64, num_bytes, SZ_512M);
5848
5849 if (block_rsv->reserved < block_rsv->size) {
5850 num_bytes = btrfs_space_info_used(sinfo, true);
5851 if (sinfo->total_bytes > num_bytes) {
5852 num_bytes = sinfo->total_bytes - num_bytes;
5853 num_bytes = min(num_bytes,
5854 block_rsv->size - block_rsv->reserved);
5855 block_rsv->reserved += num_bytes;
5856 sinfo->bytes_may_use += num_bytes;
5857 trace_btrfs_space_reservation(fs_info, "space_info",
5858 sinfo->flags, num_bytes,
5859 1);
5860 }
5861 } else if (block_rsv->reserved > block_rsv->size) {
5862 num_bytes = block_rsv->reserved - block_rsv->size;
5863 sinfo->bytes_may_use -= num_bytes;
5864 trace_btrfs_space_reservation(fs_info, "space_info",
5865 sinfo->flags, num_bytes, 0);
5866 block_rsv->reserved = block_rsv->size;
5867 }
5868
5869 if (block_rsv->reserved == block_rsv->size)
5870 block_rsv->full = 1;
5871 else
5872 block_rsv->full = 0;
5873
5874 spin_unlock(&block_rsv->lock);
5875 spin_unlock(&sinfo->lock);
5876 }
5877
5878 static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
5879 {
5880 struct btrfs_space_info *space_info;
5881
5882 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
5883 fs_info->chunk_block_rsv.space_info = space_info;
5884
5885 space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
5886 fs_info->global_block_rsv.space_info = space_info;
5887 fs_info->trans_block_rsv.space_info = space_info;
5888 fs_info->empty_block_rsv.space_info = space_info;
5889 fs_info->delayed_block_rsv.space_info = space_info;
5890
5891 fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
5892 fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
5893 fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
5894 fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
5895 if (fs_info->quota_root)
5896 fs_info->quota_root->block_rsv = &fs_info->global_block_rsv;
5897 fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
5898
5899 update_global_block_rsv(fs_info);
5900 }
5901
5902 static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
5903 {
5904 block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
5905 (u64)-1, NULL);
5906 WARN_ON(fs_info->trans_block_rsv.size > 0);
5907 WARN_ON(fs_info->trans_block_rsv.reserved > 0);
5908 WARN_ON(fs_info->chunk_block_rsv.size > 0);
5909 WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
5910 WARN_ON(fs_info->delayed_block_rsv.size > 0);
5911 WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
5912 }
5913
5914
5915 /*
5916 * To be called after all the new block groups attached to the transaction
5917 * handle have been created (btrfs_create_pending_block_groups()).
5918 */
5919 void btrfs_trans_release_chunk_metadata(struct btrfs_trans_handle *trans)
5920 {
5921 struct btrfs_fs_info *fs_info = trans->fs_info;
5922
5923 if (!trans->chunk_bytes_reserved)
5924 return;
5925
5926 WARN_ON_ONCE(!list_empty(&trans->new_bgs));
5927
5928 block_rsv_release_bytes(fs_info, &fs_info->chunk_block_rsv, NULL,
5929 trans->chunk_bytes_reserved, NULL);
5930 trans->chunk_bytes_reserved = 0;
5931 }
5932
5933 /*
5934 * btrfs_subvolume_reserve_metadata() - reserve space for subvolume operation
5935 * root: the root of the parent directory
5936 * rsv: block reservation
5937 * items: the number of items that we need do reservation
5938 * use_global_rsv: allow fallback to the global block reservation
5939 *
5940 * This function is used to reserve the space for snapshot/subvolume
5941 * creation and deletion. Those operations are different with the
5942 * common file/directory operations, they change two fs/file trees
5943 * and root tree, the number of items that the qgroup reserves is
5944 * different with the free space reservation. So we can not use
5945 * the space reservation mechanism in start_transaction().
5946 */
5947 int btrfs_subvolume_reserve_metadata(struct btrfs_root *root,
5948 struct btrfs_block_rsv *rsv, int items,
5949 bool use_global_rsv)
5950 {
5951 u64 qgroup_num_bytes = 0;
5952 u64 num_bytes;
5953 int ret;
5954 struct btrfs_fs_info *fs_info = root->fs_info;
5955 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5956
5957 if (test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) {
5958 /* One for parent inode, two for dir entries */
5959 qgroup_num_bytes = 3 * fs_info->nodesize;
5960 ret = btrfs_qgroup_reserve_meta_prealloc(root,
5961 qgroup_num_bytes, true);
5962 if (ret)
5963 return ret;
5964 }
5965
5966 num_bytes = btrfs_calc_trans_metadata_size(fs_info, items);
5967 rsv->space_info = __find_space_info(fs_info,
5968 BTRFS_BLOCK_GROUP_METADATA);
5969 ret = btrfs_block_rsv_add(root, rsv, num_bytes,
5970 BTRFS_RESERVE_FLUSH_ALL);
5971
5972 if (ret == -ENOSPC && use_global_rsv)
5973 ret = btrfs_block_rsv_migrate(global_rsv, rsv, num_bytes, 1);
5974
5975 if (ret && qgroup_num_bytes)
5976 btrfs_qgroup_free_meta_prealloc(root, qgroup_num_bytes);
5977
5978 return ret;
5979 }
5980
5981 void btrfs_subvolume_release_metadata(struct btrfs_fs_info *fs_info,
5982 struct btrfs_block_rsv *rsv)
5983 {
5984 btrfs_block_rsv_release(fs_info, rsv, (u64)-1);
5985 }
5986
5987 static void btrfs_calculate_inode_block_rsv_size(struct btrfs_fs_info *fs_info,
5988 struct btrfs_inode *inode)
5989 {
5990 struct btrfs_block_rsv *block_rsv = &inode->block_rsv;
5991 u64 reserve_size = 0;
5992 u64 qgroup_rsv_size = 0;
5993 u64 csum_leaves;
5994 unsigned outstanding_extents;
5995
5996 lockdep_assert_held(&inode->lock);
5997 outstanding_extents = inode->outstanding_extents;
5998 if (outstanding_extents)
5999 reserve_size = btrfs_calc_trans_metadata_size(fs_info,
6000 outstanding_extents + 1);
6001 csum_leaves = btrfs_csum_bytes_to_leaves(fs_info,
6002 inode->csum_bytes);
6003 reserve_size += btrfs_calc_trans_metadata_size(fs_info,
6004 csum_leaves);
6005 /*
6006 * For qgroup rsv, the calculation is very simple:
6007 * account one nodesize for each outstanding extent
6008 *
6009 * This is overestimating in most cases.
6010 */
6011 qgroup_rsv_size = outstanding_extents * fs_info->nodesize;
6012
6013 spin_lock(&block_rsv->lock);
6014 block_rsv->size = reserve_size;
6015 block_rsv->qgroup_rsv_size = qgroup_rsv_size;
6016 spin_unlock(&block_rsv->lock);
6017 }
6018
6019 int btrfs_delalloc_reserve_metadata(struct btrfs_inode *inode, u64 num_bytes)
6020 {
6021 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6022 unsigned nr_extents;
6023 enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
6024 int ret = 0;
6025 bool delalloc_lock = true;
6026
6027 /* If we are a free space inode we need to not flush since we will be in
6028 * the middle of a transaction commit. We also don't need the delalloc
6029 * mutex since we won't race with anybody. We need this mostly to make
6030 * lockdep shut its filthy mouth.
6031 *
6032 * If we have a transaction open (can happen if we call truncate_block
6033 * from truncate), then we need FLUSH_LIMIT so we don't deadlock.
6034 */
6035 if (btrfs_is_free_space_inode(inode)) {
6036 flush = BTRFS_RESERVE_NO_FLUSH;
6037 delalloc_lock = false;
6038 } else {
6039 if (current->journal_info)
6040 flush = BTRFS_RESERVE_FLUSH_LIMIT;
6041
6042 if (btrfs_transaction_in_commit(fs_info))
6043 schedule_timeout(1);
6044 }
6045
6046 if (delalloc_lock)
6047 mutex_lock(&inode->delalloc_mutex);
6048
6049 num_bytes = ALIGN(num_bytes, fs_info->sectorsize);
6050
6051 /* Add our new extents and calculate the new rsv size. */
6052 spin_lock(&inode->lock);
6053 nr_extents = count_max_extents(num_bytes);
6054 btrfs_mod_outstanding_extents(inode, nr_extents);
6055 inode->csum_bytes += num_bytes;
6056 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
6057 spin_unlock(&inode->lock);
6058
6059 ret = btrfs_inode_rsv_refill(inode, flush);
6060 if (unlikely(ret))
6061 goto out_fail;
6062
6063 if (delalloc_lock)
6064 mutex_unlock(&inode->delalloc_mutex);
6065 return 0;
6066
6067 out_fail:
6068 spin_lock(&inode->lock);
6069 nr_extents = count_max_extents(num_bytes);
6070 btrfs_mod_outstanding_extents(inode, -nr_extents);
6071 inode->csum_bytes -= num_bytes;
6072 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
6073 spin_unlock(&inode->lock);
6074
6075 btrfs_inode_rsv_release(inode, true);
6076 if (delalloc_lock)
6077 mutex_unlock(&inode->delalloc_mutex);
6078 return ret;
6079 }
6080
6081 /**
6082 * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
6083 * @inode: the inode to release the reservation for.
6084 * @num_bytes: the number of bytes we are releasing.
6085 * @qgroup_free: free qgroup reservation or convert it to per-trans reservation
6086 *
6087 * This will release the metadata reservation for an inode. This can be called
6088 * once we complete IO for a given set of bytes to release their metadata
6089 * reservations, or on error for the same reason.
6090 */
6091 void btrfs_delalloc_release_metadata(struct btrfs_inode *inode, u64 num_bytes,
6092 bool qgroup_free)
6093 {
6094 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6095
6096 num_bytes = ALIGN(num_bytes, fs_info->sectorsize);
6097 spin_lock(&inode->lock);
6098 inode->csum_bytes -= num_bytes;
6099 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
6100 spin_unlock(&inode->lock);
6101
6102 if (btrfs_is_testing(fs_info))
6103 return;
6104
6105 btrfs_inode_rsv_release(inode, qgroup_free);
6106 }
6107
6108 /**
6109 * btrfs_delalloc_release_extents - release our outstanding_extents
6110 * @inode: the inode to balance the reservation for.
6111 * @num_bytes: the number of bytes we originally reserved with
6112 * @qgroup_free: do we need to free qgroup meta reservation or convert them.
6113 *
6114 * When we reserve space we increase outstanding_extents for the extents we may
6115 * add. Once we've set the range as delalloc or created our ordered extents we
6116 * have outstanding_extents to track the real usage, so we use this to free our
6117 * temporarily tracked outstanding_extents. This _must_ be used in conjunction
6118 * with btrfs_delalloc_reserve_metadata.
6119 */
6120 void btrfs_delalloc_release_extents(struct btrfs_inode *inode, u64 num_bytes,
6121 bool qgroup_free)
6122 {
6123 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6124 unsigned num_extents;
6125
6126 spin_lock(&inode->lock);
6127 num_extents = count_max_extents(num_bytes);
6128 btrfs_mod_outstanding_extents(inode, -num_extents);
6129 btrfs_calculate_inode_block_rsv_size(fs_info, inode);
6130 spin_unlock(&inode->lock);
6131
6132 if (btrfs_is_testing(fs_info))
6133 return;
6134
6135 btrfs_inode_rsv_release(inode, qgroup_free);
6136 }
6137
6138 /**
6139 * btrfs_delalloc_reserve_space - reserve data and metadata space for
6140 * delalloc
6141 * @inode: inode we're writing to
6142 * @start: start range we are writing to
6143 * @len: how long the range we are writing to
6144 * @reserved: mandatory parameter, record actually reserved qgroup ranges of
6145 * current reservation.
6146 *
6147 * This will do the following things
6148 *
6149 * o reserve space in data space info for num bytes
6150 * and reserve precious corresponding qgroup space
6151 * (Done in check_data_free_space)
6152 *
6153 * o reserve space for metadata space, based on the number of outstanding
6154 * extents and how much csums will be needed
6155 * also reserve metadata space in a per root over-reserve method.
6156 * o add to the inodes->delalloc_bytes
6157 * o add it to the fs_info's delalloc inodes list.
6158 * (Above 3 all done in delalloc_reserve_metadata)
6159 *
6160 * Return 0 for success
6161 * Return <0 for error(-ENOSPC or -EQUOT)
6162 */
6163 int btrfs_delalloc_reserve_space(struct inode *inode,
6164 struct extent_changeset **reserved, u64 start, u64 len)
6165 {
6166 int ret;
6167
6168 ret = btrfs_check_data_free_space(inode, reserved, start, len);
6169 if (ret < 0)
6170 return ret;
6171 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), len);
6172 if (ret < 0)
6173 btrfs_free_reserved_data_space(inode, *reserved, start, len);
6174 return ret;
6175 }
6176
6177 /**
6178 * btrfs_delalloc_release_space - release data and metadata space for delalloc
6179 * @inode: inode we're releasing space for
6180 * @start: start position of the space already reserved
6181 * @len: the len of the space already reserved
6182 * @release_bytes: the len of the space we consumed or didn't use
6183 *
6184 * This function will release the metadata space that was not used and will
6185 * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
6186 * list if there are no delalloc bytes left.
6187 * Also it will handle the qgroup reserved space.
6188 */
6189 void btrfs_delalloc_release_space(struct inode *inode,
6190 struct extent_changeset *reserved,
6191 u64 start, u64 len, bool qgroup_free)
6192 {
6193 btrfs_delalloc_release_metadata(BTRFS_I(inode), len, qgroup_free);
6194 btrfs_free_reserved_data_space(inode, reserved, start, len);
6195 }
6196
6197 static int update_block_group(struct btrfs_trans_handle *trans,
6198 struct btrfs_fs_info *info, u64 bytenr,
6199 u64 num_bytes, int alloc)
6200 {
6201 struct btrfs_block_group_cache *cache = NULL;
6202 u64 total = num_bytes;
6203 u64 old_val;
6204 u64 byte_in_group;
6205 int factor;
6206
6207 /* block accounting for super block */
6208 spin_lock(&info->delalloc_root_lock);
6209 old_val = btrfs_super_bytes_used(info->super_copy);
6210 if (alloc)
6211 old_val += num_bytes;
6212 else
6213 old_val -= num_bytes;
6214 btrfs_set_super_bytes_used(info->super_copy, old_val);
6215 spin_unlock(&info->delalloc_root_lock);
6216
6217 while (total) {
6218 cache = btrfs_lookup_block_group(info, bytenr);
6219 if (!cache)
6220 return -ENOENT;
6221 if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
6222 BTRFS_BLOCK_GROUP_RAID1 |
6223 BTRFS_BLOCK_GROUP_RAID10))
6224 factor = 2;
6225 else
6226 factor = 1;
6227 /*
6228 * If this block group has free space cache written out, we
6229 * need to make sure to load it if we are removing space. This
6230 * is because we need the unpinning stage to actually add the
6231 * space back to the block group, otherwise we will leak space.
6232 */
6233 if (!alloc && cache->cached == BTRFS_CACHE_NO)
6234 cache_block_group(cache, 1);
6235
6236 byte_in_group = bytenr - cache->key.objectid;
6237 WARN_ON(byte_in_group > cache->key.offset);
6238
6239 spin_lock(&cache->space_info->lock);
6240 spin_lock(&cache->lock);
6241
6242 if (btrfs_test_opt(info, SPACE_CACHE) &&
6243 cache->disk_cache_state < BTRFS_DC_CLEAR)
6244 cache->disk_cache_state = BTRFS_DC_CLEAR;
6245
6246 old_val = btrfs_block_group_used(&cache->item);
6247 num_bytes = min(total, cache->key.offset - byte_in_group);
6248 if (alloc) {
6249 old_val += num_bytes;
6250 btrfs_set_block_group_used(&cache->item, old_val);
6251 cache->reserved -= num_bytes;
6252 cache->space_info->bytes_reserved -= num_bytes;
6253 cache->space_info->bytes_used += num_bytes;
6254 cache->space_info->disk_used += num_bytes * factor;
6255 spin_unlock(&cache->lock);
6256 spin_unlock(&cache->space_info->lock);
6257 } else {
6258 old_val -= num_bytes;
6259 btrfs_set_block_group_used(&cache->item, old_val);
6260 cache->pinned += num_bytes;
6261 cache->space_info->bytes_pinned += num_bytes;
6262 cache->space_info->bytes_used -= num_bytes;
6263 cache->space_info->disk_used -= num_bytes * factor;
6264 spin_unlock(&cache->lock);
6265 spin_unlock(&cache->space_info->lock);
6266
6267 trace_btrfs_space_reservation(info, "pinned",
6268 cache->space_info->flags,
6269 num_bytes, 1);
6270 percpu_counter_add(&cache->space_info->total_bytes_pinned,
6271 num_bytes);
6272 set_extent_dirty(info->pinned_extents,
6273 bytenr, bytenr + num_bytes - 1,
6274 GFP_NOFS | __GFP_NOFAIL);
6275 }
6276
6277 spin_lock(&trans->transaction->dirty_bgs_lock);
6278 if (list_empty(&cache->dirty_list)) {
6279 list_add_tail(&cache->dirty_list,
6280 &trans->transaction->dirty_bgs);
6281 trans->transaction->num_dirty_bgs++;
6282 btrfs_get_block_group(cache);
6283 }
6284 spin_unlock(&trans->transaction->dirty_bgs_lock);
6285
6286 /*
6287 * No longer have used bytes in this block group, queue it for
6288 * deletion. We do this after adding the block group to the
6289 * dirty list to avoid races between cleaner kthread and space
6290 * cache writeout.
6291 */
6292 if (!alloc && old_val == 0) {
6293 spin_lock(&info->unused_bgs_lock);
6294 if (list_empty(&cache->bg_list)) {
6295 btrfs_get_block_group(cache);
6296 trace_btrfs_add_unused_block_group(cache);
6297 list_add_tail(&cache->bg_list,
6298 &info->unused_bgs);
6299 }
6300 spin_unlock(&info->unused_bgs_lock);
6301 }
6302
6303 btrfs_put_block_group(cache);
6304 total -= num_bytes;
6305 bytenr += num_bytes;
6306 }
6307 return 0;
6308 }
6309
6310 static u64 first_logical_byte(struct btrfs_fs_info *fs_info, u64 search_start)
6311 {
6312 struct btrfs_block_group_cache *cache;
6313 u64 bytenr;
6314
6315 spin_lock(&fs_info->block_group_cache_lock);
6316 bytenr = fs_info->first_logical_byte;
6317 spin_unlock(&fs_info->block_group_cache_lock);
6318
6319 if (bytenr < (u64)-1)
6320 return bytenr;
6321
6322 cache = btrfs_lookup_first_block_group(fs_info, search_start);
6323 if (!cache)
6324 return 0;
6325
6326 bytenr = cache->key.objectid;
6327 btrfs_put_block_group(cache);
6328
6329 return bytenr;
6330 }
6331
6332 static int pin_down_extent(struct btrfs_fs_info *fs_info,
6333 struct btrfs_block_group_cache *cache,
6334 u64 bytenr, u64 num_bytes, int reserved)
6335 {
6336 spin_lock(&cache->space_info->lock);
6337 spin_lock(&cache->lock);
6338 cache->pinned += num_bytes;
6339 cache->space_info->bytes_pinned += num_bytes;
6340 if (reserved) {
6341 cache->reserved -= num_bytes;
6342 cache->space_info->bytes_reserved -= num_bytes;
6343 }
6344 spin_unlock(&cache->lock);
6345 spin_unlock(&cache->space_info->lock);
6346
6347 trace_btrfs_space_reservation(fs_info, "pinned",
6348 cache->space_info->flags, num_bytes, 1);
6349 percpu_counter_add(&cache->space_info->total_bytes_pinned, num_bytes);
6350 set_extent_dirty(fs_info->pinned_extents, bytenr,
6351 bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
6352 return 0;
6353 }
6354
6355 /*
6356 * this function must be called within transaction
6357 */
6358 int btrfs_pin_extent(struct btrfs_fs_info *fs_info,
6359 u64 bytenr, u64 num_bytes, int reserved)
6360 {
6361 struct btrfs_block_group_cache *cache;
6362
6363 cache = btrfs_lookup_block_group(fs_info, bytenr);
6364 BUG_ON(!cache); /* Logic error */
6365
6366 pin_down_extent(fs_info, cache, bytenr, num_bytes, reserved);
6367
6368 btrfs_put_block_group(cache);
6369 return 0;
6370 }
6371
6372 /*
6373 * this function must be called within transaction
6374 */
6375 int btrfs_pin_extent_for_log_replay(struct btrfs_fs_info *fs_info,
6376 u64 bytenr, u64 num_bytes)
6377 {
6378 struct btrfs_block_group_cache *cache;
6379 int ret;
6380
6381 cache = btrfs_lookup_block_group(fs_info, bytenr);
6382 if (!cache)
6383 return -EINVAL;
6384
6385 /*
6386 * pull in the free space cache (if any) so that our pin
6387 * removes the free space from the cache. We have load_only set
6388 * to one because the slow code to read in the free extents does check
6389 * the pinned extents.
6390 */
6391 cache_block_group(cache, 1);
6392
6393 pin_down_extent(fs_info, cache, bytenr, num_bytes, 0);
6394
6395 /* remove us from the free space cache (if we're there at all) */
6396 ret = btrfs_remove_free_space(cache, bytenr, num_bytes);
6397 btrfs_put_block_group(cache);
6398 return ret;
6399 }
6400
6401 static int __exclude_logged_extent(struct btrfs_fs_info *fs_info,
6402 u64 start, u64 num_bytes)
6403 {
6404 int ret;
6405 struct btrfs_block_group_cache *block_group;
6406 struct btrfs_caching_control *caching_ctl;
6407
6408 block_group = btrfs_lookup_block_group(fs_info, start);
6409 if (!block_group)
6410 return -EINVAL;
6411
6412 cache_block_group(block_group, 0);
6413 caching_ctl = get_caching_control(block_group);
6414
6415 if (!caching_ctl) {
6416 /* Logic error */
6417 BUG_ON(!block_group_cache_done(block_group));
6418 ret = btrfs_remove_free_space(block_group, start, num_bytes);
6419 } else {
6420 mutex_lock(&caching_ctl->mutex);
6421
6422 if (start >= caching_ctl->progress) {
6423 ret = add_excluded_extent(fs_info, start, num_bytes);
6424 } else if (start + num_bytes <= caching_ctl->progress) {
6425 ret = btrfs_remove_free_space(block_group,
6426 start, num_bytes);
6427 } else {
6428 num_bytes = caching_ctl->progress - start;
6429 ret = btrfs_remove_free_space(block_group,
6430 start, num_bytes);
6431 if (ret)
6432 goto out_lock;
6433
6434 num_bytes = (start + num_bytes) -
6435 caching_ctl->progress;
6436 start = caching_ctl->progress;
6437 ret = add_excluded_extent(fs_info, start, num_bytes);
6438 }
6439 out_lock:
6440 mutex_unlock(&caching_ctl->mutex);
6441 put_caching_control(caching_ctl);
6442 }
6443 btrfs_put_block_group(block_group);
6444 return ret;
6445 }
6446
6447 int btrfs_exclude_logged_extents(struct btrfs_fs_info *fs_info,
6448 struct extent_buffer *eb)
6449 {
6450 struct btrfs_file_extent_item *item;
6451 struct btrfs_key key;
6452 int found_type;
6453 int i;
6454 int ret = 0;
6455
6456 if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS))
6457 return 0;
6458
6459 for (i = 0; i < btrfs_header_nritems(eb); i++) {
6460 btrfs_item_key_to_cpu(eb, &key, i);
6461 if (key.type != BTRFS_EXTENT_DATA_KEY)
6462 continue;
6463 item = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
6464 found_type = btrfs_file_extent_type(eb, item);
6465 if (found_type == BTRFS_FILE_EXTENT_INLINE)
6466 continue;
6467 if (btrfs_file_extent_disk_bytenr(eb, item) == 0)
6468 continue;
6469 key.objectid = btrfs_file_extent_disk_bytenr(eb, item);
6470 key.offset = btrfs_file_extent_disk_num_bytes(eb, item);
6471 ret = __exclude_logged_extent(fs_info, key.objectid, key.offset);
6472 if (ret)
6473 break;
6474 }
6475
6476 return ret;
6477 }
6478
6479 static void
6480 btrfs_inc_block_group_reservations(struct btrfs_block_group_cache *bg)
6481 {
6482 atomic_inc(&bg->reservations);
6483 }
6484
6485 void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info,
6486 const u64 start)
6487 {
6488 struct btrfs_block_group_cache *bg;
6489
6490 bg = btrfs_lookup_block_group(fs_info, start);
6491 ASSERT(bg);
6492 if (atomic_dec_and_test(&bg->reservations))
6493 wake_up_var(&bg->reservations);
6494 btrfs_put_block_group(bg);
6495 }
6496
6497 void btrfs_wait_block_group_reservations(struct btrfs_block_group_cache *bg)
6498 {
6499 struct btrfs_space_info *space_info = bg->space_info;
6500
6501 ASSERT(bg->ro);
6502
6503 if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA))
6504 return;
6505
6506 /*
6507 * Our block group is read only but before we set it to read only,
6508 * some task might have had allocated an extent from it already, but it
6509 * has not yet created a respective ordered extent (and added it to a
6510 * root's list of ordered extents).
6511 * Therefore wait for any task currently allocating extents, since the
6512 * block group's reservations counter is incremented while a read lock
6513 * on the groups' semaphore is held and decremented after releasing
6514 * the read access on that semaphore and creating the ordered extent.
6515 */
6516 down_write(&space_info->groups_sem);
6517 up_write(&space_info->groups_sem);
6518
6519 wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
6520 }
6521
6522 /**
6523 * btrfs_add_reserved_bytes - update the block_group and space info counters
6524 * @cache: The cache we are manipulating
6525 * @ram_bytes: The number of bytes of file content, and will be same to
6526 * @num_bytes except for the compress path.
6527 * @num_bytes: The number of bytes in question
6528 * @delalloc: The blocks are allocated for the delalloc write
6529 *
6530 * This is called by the allocator when it reserves space. If this is a
6531 * reservation and the block group has become read only we cannot make the
6532 * reservation and return -EAGAIN, otherwise this function always succeeds.
6533 */
6534 static int btrfs_add_reserved_bytes(struct btrfs_block_group_cache *cache,
6535 u64 ram_bytes, u64 num_bytes, int delalloc)
6536 {
6537 struct btrfs_space_info *space_info = cache->space_info;
6538 int ret = 0;
6539
6540 spin_lock(&space_info->lock);
6541 spin_lock(&cache->lock);
6542 if (cache->ro) {
6543 ret = -EAGAIN;
6544 } else {
6545 cache->reserved += num_bytes;
6546 space_info->bytes_reserved += num_bytes;
6547
6548 trace_btrfs_space_reservation(cache->fs_info,
6549 "space_info", space_info->flags,
6550 ram_bytes, 0);
6551 space_info->bytes_may_use -= ram_bytes;
6552 if (delalloc)
6553 cache->delalloc_bytes += num_bytes;
6554 }
6555 spin_unlock(&cache->lock);
6556 spin_unlock(&space_info->lock);
6557 return ret;
6558 }
6559
6560 /**
6561 * btrfs_free_reserved_bytes - update the block_group and space info counters
6562 * @cache: The cache we are manipulating
6563 * @num_bytes: The number of bytes in question
6564 * @delalloc: The blocks are allocated for the delalloc write
6565 *
6566 * This is called by somebody who is freeing space that was never actually used
6567 * on disk. For example if you reserve some space for a new leaf in transaction
6568 * A and before transaction A commits you free that leaf, you call this with
6569 * reserve set to 0 in order to clear the reservation.
6570 */
6571
6572 static int btrfs_free_reserved_bytes(struct btrfs_block_group_cache *cache,
6573 u64 num_bytes, int delalloc)
6574 {
6575 struct btrfs_space_info *space_info = cache->space_info;
6576 int ret = 0;
6577
6578 spin_lock(&space_info->lock);
6579 spin_lock(&cache->lock);
6580 if (cache->ro)
6581 space_info->bytes_readonly += num_bytes;
6582 cache->reserved -= num_bytes;
6583 space_info->bytes_reserved -= num_bytes;
6584
6585 if (delalloc)
6586 cache->delalloc_bytes -= num_bytes;
6587 spin_unlock(&cache->lock);
6588 spin_unlock(&space_info->lock);
6589 return ret;
6590 }
6591 void btrfs_prepare_extent_commit(struct btrfs_fs_info *fs_info)
6592 {
6593 struct btrfs_caching_control *next;
6594 struct btrfs_caching_control *caching_ctl;
6595 struct btrfs_block_group_cache *cache;
6596
6597 down_write(&fs_info->commit_root_sem);
6598
6599 list_for_each_entry_safe(caching_ctl, next,
6600 &fs_info->caching_block_groups, list) {
6601 cache = caching_ctl->block_group;
6602 if (block_group_cache_done(cache)) {
6603 cache->last_byte_to_unpin = (u64)-1;
6604 list_del_init(&caching_ctl->list);
6605 put_caching_control(caching_ctl);
6606 } else {
6607 cache->last_byte_to_unpin = caching_ctl->progress;
6608 }
6609 }
6610
6611 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
6612 fs_info->pinned_extents = &fs_info->freed_extents[1];
6613 else
6614 fs_info->pinned_extents = &fs_info->freed_extents[0];
6615
6616 up_write(&fs_info->commit_root_sem);
6617
6618 update_global_block_rsv(fs_info);
6619 }
6620
6621 /*
6622 * Returns the free cluster for the given space info and sets empty_cluster to
6623 * what it should be based on the mount options.
6624 */
6625 static struct btrfs_free_cluster *
6626 fetch_cluster_info(struct btrfs_fs_info *fs_info,
6627 struct btrfs_space_info *space_info, u64 *empty_cluster)
6628 {
6629 struct btrfs_free_cluster *ret = NULL;
6630
6631 *empty_cluster = 0;
6632 if (btrfs_mixed_space_info(space_info))
6633 return ret;
6634
6635 if (space_info->flags & BTRFS_BLOCK_GROUP_METADATA) {
6636 ret = &fs_info->meta_alloc_cluster;
6637 if (btrfs_test_opt(fs_info, SSD))
6638 *empty_cluster = SZ_2M;
6639 else
6640 *empty_cluster = SZ_64K;
6641 } else if ((space_info->flags & BTRFS_BLOCK_GROUP_DATA) &&
6642 btrfs_test_opt(fs_info, SSD_SPREAD)) {
6643 *empty_cluster = SZ_2M;
6644 ret = &fs_info->data_alloc_cluster;
6645 }
6646
6647 return ret;
6648 }
6649
6650 static int unpin_extent_range(struct btrfs_fs_info *fs_info,
6651 u64 start, u64 end,
6652 const bool return_free_space)
6653 {
6654 struct btrfs_block_group_cache *cache = NULL;
6655 struct btrfs_space_info *space_info;
6656 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
6657 struct btrfs_free_cluster *cluster = NULL;
6658 u64 len;
6659 u64 total_unpinned = 0;
6660 u64 empty_cluster = 0;
6661 bool readonly;
6662
6663 while (start <= end) {
6664 readonly = false;
6665 if (!cache ||
6666 start >= cache->key.objectid + cache->key.offset) {
6667 if (cache)
6668 btrfs_put_block_group(cache);
6669 total_unpinned = 0;
6670 cache = btrfs_lookup_block_group(fs_info, start);
6671 BUG_ON(!cache); /* Logic error */
6672
6673 cluster = fetch_cluster_info(fs_info,
6674 cache->space_info,
6675 &empty_cluster);
6676 empty_cluster <<= 1;
6677 }
6678
6679 len = cache->key.objectid + cache->key.offset - start;
6680 len = min(len, end + 1 - start);
6681
6682 if (start < cache->last_byte_to_unpin) {
6683 len = min(len, cache->last_byte_to_unpin - start);
6684 if (return_free_space)
6685 btrfs_add_free_space(cache, start, len);
6686 }
6687
6688 start += len;
6689 total_unpinned += len;
6690 space_info = cache->space_info;
6691
6692 /*
6693 * If this space cluster has been marked as fragmented and we've
6694 * unpinned enough in this block group to potentially allow a
6695 * cluster to be created inside of it go ahead and clear the
6696 * fragmented check.
6697 */
6698 if (cluster && cluster->fragmented &&
6699 total_unpinned > empty_cluster) {
6700 spin_lock(&cluster->lock);
6701 cluster->fragmented = 0;
6702 spin_unlock(&cluster->lock);
6703 }
6704
6705 spin_lock(&space_info->lock);
6706 spin_lock(&cache->lock);
6707 cache->pinned -= len;
6708 space_info->bytes_pinned -= len;
6709
6710 trace_btrfs_space_reservation(fs_info, "pinned",
6711 space_info->flags, len, 0);
6712 space_info->max_extent_size = 0;
6713 percpu_counter_add(&space_info->total_bytes_pinned, -len);
6714 if (cache->ro) {
6715 space_info->bytes_readonly += len;
6716 readonly = true;
6717 }
6718 spin_unlock(&cache->lock);
6719 if (!readonly && return_free_space &&
6720 global_rsv->space_info == space_info) {
6721 u64 to_add = len;
6722
6723 spin_lock(&global_rsv->lock);
6724 if (!global_rsv->full) {
6725 to_add = min(len, global_rsv->size -
6726 global_rsv->reserved);
6727 global_rsv->reserved += to_add;
6728 space_info->bytes_may_use += to_add;
6729 if (global_rsv->reserved >= global_rsv->size)
6730 global_rsv->full = 1;
6731 trace_btrfs_space_reservation(fs_info,
6732 "space_info",
6733 space_info->flags,
6734 to_add, 1);
6735 len -= to_add;
6736 }
6737 spin_unlock(&global_rsv->lock);
6738 /* Add to any tickets we may have */
6739 if (len)
6740 space_info_add_new_bytes(fs_info, space_info,
6741 len);
6742 }
6743 spin_unlock(&space_info->lock);
6744 }
6745
6746 if (cache)
6747 btrfs_put_block_group(cache);
6748 return 0;
6749 }
6750
6751 int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans)
6752 {
6753 struct btrfs_fs_info *fs_info = trans->fs_info;
6754 struct btrfs_block_group_cache *block_group, *tmp;
6755 struct list_head *deleted_bgs;
6756 struct extent_io_tree *unpin;
6757 u64 start;
6758 u64 end;
6759 int ret;
6760
6761 if (fs_info->pinned_extents == &fs_info->freed_extents[0])
6762 unpin = &fs_info->freed_extents[1];
6763 else
6764 unpin = &fs_info->freed_extents[0];
6765
6766 while (!trans->aborted) {
6767 mutex_lock(&fs_info->unused_bg_unpin_mutex);
6768 ret = find_first_extent_bit(unpin, 0, &start, &end,
6769 EXTENT_DIRTY, NULL);
6770 if (ret) {
6771 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
6772 break;
6773 }
6774
6775 if (btrfs_test_opt(fs_info, DISCARD))
6776 ret = btrfs_discard_extent(fs_info, start,
6777 end + 1 - start, NULL);
6778
6779 clear_extent_dirty(unpin, start, end);
6780 unpin_extent_range(fs_info, start, end, true);
6781 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
6782 cond_resched();
6783 }
6784
6785 /*
6786 * Transaction is finished. We don't need the lock anymore. We
6787 * do need to clean up the block groups in case of a transaction
6788 * abort.
6789 */
6790 deleted_bgs = &trans->transaction->deleted_bgs;
6791 list_for_each_entry_safe(block_group, tmp, deleted_bgs, bg_list) {
6792 u64 trimmed = 0;
6793
6794 ret = -EROFS;
6795 if (!trans->aborted)
6796 ret = btrfs_discard_extent(fs_info,
6797 block_group->key.objectid,
6798 block_group->key.offset,
6799 &trimmed);
6800
6801 list_del_init(&block_group->bg_list);
6802 btrfs_put_block_group_trimming(block_group);
6803 btrfs_put_block_group(block_group);
6804
6805 if (ret) {
6806 const char *errstr = btrfs_decode_error(ret);
6807 btrfs_warn(fs_info,
6808 "discard failed while removing blockgroup: errno=%d %s",
6809 ret, errstr);
6810 }
6811 }
6812
6813 return 0;
6814 }
6815
6816 static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
6817 struct btrfs_fs_info *info,
6818 struct btrfs_delayed_ref_node *node, u64 parent,
6819 u64 root_objectid, u64 owner_objectid,
6820 u64 owner_offset, int refs_to_drop,
6821 struct btrfs_delayed_extent_op *extent_op)
6822 {
6823 struct btrfs_key key;
6824 struct btrfs_path *path;
6825 struct btrfs_root *extent_root = info->extent_root;
6826 struct extent_buffer *leaf;
6827 struct btrfs_extent_item *ei;
6828 struct btrfs_extent_inline_ref *iref;
6829 int ret;
6830 int is_data;
6831 int extent_slot = 0;
6832 int found_extent = 0;
6833 int num_to_del = 1;
6834 u32 item_size;
6835 u64 refs;
6836 u64 bytenr = node->bytenr;
6837 u64 num_bytes = node->num_bytes;
6838 int last_ref = 0;
6839 bool skinny_metadata = btrfs_fs_incompat(info, SKINNY_METADATA);
6840
6841 path = btrfs_alloc_path();
6842 if (!path)
6843 return -ENOMEM;
6844
6845 path->reada = READA_FORWARD;
6846 path->leave_spinning = 1;
6847
6848 is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
6849 BUG_ON(!is_data && refs_to_drop != 1);
6850
6851 if (is_data)
6852 skinny_metadata = false;
6853
6854 ret = lookup_extent_backref(trans, info, path, &iref,
6855 bytenr, num_bytes, parent,
6856 root_objectid, owner_objectid,
6857 owner_offset);
6858 if (ret == 0) {
6859 extent_slot = path->slots[0];
6860 while (extent_slot >= 0) {
6861 btrfs_item_key_to_cpu(path->nodes[0], &key,
6862 extent_slot);
6863 if (key.objectid != bytenr)
6864 break;
6865 if (key.type == BTRFS_EXTENT_ITEM_KEY &&
6866 key.offset == num_bytes) {
6867 found_extent = 1;
6868 break;
6869 }
6870 if (key.type == BTRFS_METADATA_ITEM_KEY &&
6871 key.offset == owner_objectid) {
6872 found_extent = 1;
6873 break;
6874 }
6875 if (path->slots[0] - extent_slot > 5)
6876 break;
6877 extent_slot--;
6878 }
6879 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
6880 item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
6881 if (found_extent && item_size < sizeof(*ei))
6882 found_extent = 0;
6883 #endif
6884 if (!found_extent) {
6885 BUG_ON(iref);
6886 ret = remove_extent_backref(trans, info, path, NULL,
6887 refs_to_drop,
6888 is_data, &last_ref);
6889 if (ret) {
6890 btrfs_abort_transaction(trans, ret);
6891 goto out;
6892 }
6893 btrfs_release_path(path);
6894 path->leave_spinning = 1;
6895
6896 key.objectid = bytenr;
6897 key.type = BTRFS_EXTENT_ITEM_KEY;
6898 key.offset = num_bytes;
6899
6900 if (!is_data && skinny_metadata) {
6901 key.type = BTRFS_METADATA_ITEM_KEY;
6902 key.offset = owner_objectid;
6903 }
6904
6905 ret = btrfs_search_slot(trans, extent_root,
6906 &key, path, -1, 1);
6907 if (ret > 0 && skinny_metadata && path->slots[0]) {
6908 /*
6909 * Couldn't find our skinny metadata item,
6910 * see if we have ye olde extent item.
6911 */
6912 path->slots[0]--;
6913 btrfs_item_key_to_cpu(path->nodes[0], &key,
6914 path->slots[0]);
6915 if (key.objectid == bytenr &&
6916 key.type == BTRFS_EXTENT_ITEM_KEY &&
6917 key.offset == num_bytes)
6918 ret = 0;
6919 }
6920
6921 if (ret > 0 && skinny_metadata) {
6922 skinny_metadata = false;
6923 key.objectid = bytenr;
6924 key.type = BTRFS_EXTENT_ITEM_KEY;
6925 key.offset = num_bytes;
6926 btrfs_release_path(path);
6927 ret = btrfs_search_slot(trans, extent_root,
6928 &key, path, -1, 1);
6929 }
6930
6931 if (ret) {
6932 btrfs_err(info,
6933 "umm, got %d back from search, was looking for %llu",
6934 ret, bytenr);
6935 if (ret > 0)
6936 btrfs_print_leaf(path->nodes[0]);
6937 }
6938 if (ret < 0) {
6939 btrfs_abort_transaction(trans, ret);
6940 goto out;
6941 }
6942 extent_slot = path->slots[0];
6943 }
6944 } else if (WARN_ON(ret == -ENOENT)) {
6945 btrfs_print_leaf(path->nodes[0]);
6946 btrfs_err(info,
6947 "unable to find ref byte nr %llu parent %llu root %llu owner %llu offset %llu",
6948 bytenr, parent, root_objectid, owner_objectid,
6949 owner_offset);
6950 btrfs_abort_transaction(trans, ret);
6951 goto out;
6952 } else {
6953 btrfs_abort_transaction(trans, ret);
6954 goto out;
6955 }
6956
6957 leaf = path->nodes[0];
6958 item_size = btrfs_item_size_nr(leaf, extent_slot);
6959 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
6960 if (item_size < sizeof(*ei)) {
6961 BUG_ON(found_extent || extent_slot != path->slots[0]);
6962 ret = convert_extent_item_v0(trans, info, path, owner_objectid,
6963 0);
6964 if (ret < 0) {
6965 btrfs_abort_transaction(trans, ret);
6966 goto out;
6967 }
6968
6969 btrfs_release_path(path);
6970 path->leave_spinning = 1;
6971
6972 key.objectid = bytenr;
6973 key.type = BTRFS_EXTENT_ITEM_KEY;
6974 key.offset = num_bytes;
6975
6976 ret = btrfs_search_slot(trans, extent_root, &key, path,
6977 -1, 1);
6978 if (ret) {
6979 btrfs_err(info,
6980 "umm, got %d back from search, was looking for %llu",
6981 ret, bytenr);
6982 btrfs_print_leaf(path->nodes[0]);
6983 }
6984 if (ret < 0) {
6985 btrfs_abort_transaction(trans, ret);
6986 goto out;
6987 }
6988
6989 extent_slot = path->slots[0];
6990 leaf = path->nodes[0];
6991 item_size = btrfs_item_size_nr(leaf, extent_slot);
6992 }
6993 #endif
6994 BUG_ON(item_size < sizeof(*ei));
6995 ei = btrfs_item_ptr(leaf, extent_slot,
6996 struct btrfs_extent_item);
6997 if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID &&
6998 key.type == BTRFS_EXTENT_ITEM_KEY) {
6999 struct btrfs_tree_block_info *bi;
7000 BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
7001 bi = (struct btrfs_tree_block_info *)(ei + 1);
7002 WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
7003 }
7004
7005 refs = btrfs_extent_refs(leaf, ei);
7006 if (refs < refs_to_drop) {
7007 btrfs_err(info,
7008 "trying to drop %d refs but we only have %Lu for bytenr %Lu",
7009 refs_to_drop, refs, bytenr);
7010 ret = -EINVAL;
7011 btrfs_abort_transaction(trans, ret);
7012 goto out;
7013 }
7014 refs -= refs_to_drop;
7015
7016 if (refs > 0) {
7017 if (extent_op)
7018 __run_delayed_extent_op(extent_op, leaf, ei);
7019 /*
7020 * In the case of inline back ref, reference count will
7021 * be updated by remove_extent_backref
7022 */
7023 if (iref) {
7024 BUG_ON(!found_extent);
7025 } else {
7026 btrfs_set_extent_refs(leaf, ei, refs);
7027 btrfs_mark_buffer_dirty(leaf);
7028 }
7029 if (found_extent) {
7030 ret = remove_extent_backref(trans, info, path,
7031 iref, refs_to_drop,
7032 is_data, &last_ref);
7033 if (ret) {
7034 btrfs_abort_transaction(trans, ret);
7035 goto out;
7036 }
7037 }
7038 } else {
7039 if (found_extent) {
7040 BUG_ON(is_data && refs_to_drop !=
7041 extent_data_ref_count(path, iref));
7042 if (iref) {
7043 BUG_ON(path->slots[0] != extent_slot);
7044 } else {
7045 BUG_ON(path->slots[0] != extent_slot + 1);
7046 path->slots[0] = extent_slot;
7047 num_to_del = 2;
7048 }
7049 }
7050
7051 last_ref = 1;
7052 ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
7053 num_to_del);
7054 if (ret) {
7055 btrfs_abort_transaction(trans, ret);
7056 goto out;
7057 }
7058 btrfs_release_path(path);
7059
7060 if (is_data) {
7061 ret = btrfs_del_csums(trans, info, bytenr, num_bytes);
7062 if (ret) {
7063 btrfs_abort_transaction(trans, ret);
7064 goto out;
7065 }
7066 }
7067
7068 ret = add_to_free_space_tree(trans, bytenr, num_bytes);
7069 if (ret) {
7070 btrfs_abort_transaction(trans, ret);
7071 goto out;
7072 }
7073
7074 ret = update_block_group(trans, info, bytenr, num_bytes, 0);
7075 if (ret) {
7076 btrfs_abort_transaction(trans, ret);
7077 goto out;
7078 }
7079 }
7080 btrfs_release_path(path);
7081
7082 out:
7083 btrfs_free_path(path);
7084 return ret;
7085 }
7086
7087 /*
7088 * when we free an block, it is possible (and likely) that we free the last
7089 * delayed ref for that extent as well. This searches the delayed ref tree for
7090 * a given extent, and if there are no other delayed refs to be processed, it
7091 * removes it from the tree.
7092 */
7093 static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
7094 u64 bytenr)
7095 {
7096 struct btrfs_delayed_ref_head *head;
7097 struct btrfs_delayed_ref_root *delayed_refs;
7098 int ret = 0;
7099
7100 delayed_refs = &trans->transaction->delayed_refs;
7101 spin_lock(&delayed_refs->lock);
7102 head = btrfs_find_delayed_ref_head(delayed_refs, bytenr);
7103 if (!head)
7104 goto out_delayed_unlock;
7105
7106 spin_lock(&head->lock);
7107 if (!RB_EMPTY_ROOT(&head->ref_tree))
7108 goto out;
7109
7110 if (head->extent_op) {
7111 if (!head->must_insert_reserved)
7112 goto out;
7113 btrfs_free_delayed_extent_op(head->extent_op);
7114 head->extent_op = NULL;
7115 }
7116
7117 /*
7118 * waiting for the lock here would deadlock. If someone else has it
7119 * locked they are already in the process of dropping it anyway
7120 */
7121 if (!mutex_trylock(&head->mutex))
7122 goto out;
7123
7124 /*
7125 * at this point we have a head with no other entries. Go
7126 * ahead and process it.
7127 */
7128 rb_erase(&head->href_node, &delayed_refs->href_root);
7129 RB_CLEAR_NODE(&head->href_node);
7130 atomic_dec(&delayed_refs->num_entries);
7131
7132 /*
7133 * we don't take a ref on the node because we're removing it from the
7134 * tree, so we just steal the ref the tree was holding.
7135 */
7136 delayed_refs->num_heads--;
7137 if (head->processing == 0)
7138 delayed_refs->num_heads_ready--;
7139 head->processing = 0;
7140 spin_unlock(&head->lock);
7141 spin_unlock(&delayed_refs->lock);
7142
7143 BUG_ON(head->extent_op);
7144 if (head->must_insert_reserved)
7145 ret = 1;
7146
7147 mutex_unlock(&head->mutex);
7148 btrfs_put_delayed_ref_head(head);
7149 return ret;
7150 out:
7151 spin_unlock(&head->lock);
7152
7153 out_delayed_unlock:
7154 spin_unlock(&delayed_refs->lock);
7155 return 0;
7156 }
7157
7158 void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
7159 struct btrfs_root *root,
7160 struct extent_buffer *buf,
7161 u64 parent, int last_ref)
7162 {
7163 struct btrfs_fs_info *fs_info = root->fs_info;
7164 int pin = 1;
7165 int ret;
7166
7167 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
7168 int old_ref_mod, new_ref_mod;
7169
7170 btrfs_ref_tree_mod(root, buf->start, buf->len, parent,
7171 root->root_key.objectid,
7172 btrfs_header_level(buf), 0,
7173 BTRFS_DROP_DELAYED_REF);
7174 ret = btrfs_add_delayed_tree_ref(fs_info, trans, buf->start,
7175 buf->len, parent,
7176 root->root_key.objectid,
7177 btrfs_header_level(buf),
7178 BTRFS_DROP_DELAYED_REF, NULL,
7179 &old_ref_mod, &new_ref_mod);
7180 BUG_ON(ret); /* -ENOMEM */
7181 pin = old_ref_mod >= 0 && new_ref_mod < 0;
7182 }
7183
7184 if (last_ref && btrfs_header_generation(buf) == trans->transid) {
7185 struct btrfs_block_group_cache *cache;
7186
7187 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
7188 ret = check_ref_cleanup(trans, buf->start);
7189 if (!ret)
7190 goto out;
7191 }
7192
7193 pin = 0;
7194 cache = btrfs_lookup_block_group(fs_info, buf->start);
7195
7196 if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
7197 pin_down_extent(fs_info, cache, buf->start,
7198 buf->len, 1);
7199 btrfs_put_block_group(cache);
7200 goto out;
7201 }
7202
7203 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));
7204
7205 btrfs_add_free_space(cache, buf->start, buf->len);
7206 btrfs_free_reserved_bytes(cache, buf->len, 0);
7207 btrfs_put_block_group(cache);
7208 trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len);
7209 }
7210 out:
7211 if (pin)
7212 add_pinned_bytes(fs_info, buf->len, true,
7213 root->root_key.objectid);
7214
7215 if (last_ref) {
7216 /*
7217 * Deleting the buffer, clear the corrupt flag since it doesn't
7218 * matter anymore.
7219 */
7220 clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
7221 }
7222 }
7223
7224 /* Can return -ENOMEM */
7225 int btrfs_free_extent(struct btrfs_trans_handle *trans,
7226 struct btrfs_root *root,
7227 u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
7228 u64 owner, u64 offset)
7229 {
7230 struct btrfs_fs_info *fs_info = root->fs_info;
7231 int old_ref_mod, new_ref_mod;
7232 int ret;
7233
7234 if (btrfs_is_testing(fs_info))
7235 return 0;
7236
7237 if (root_objectid != BTRFS_TREE_LOG_OBJECTID)
7238 btrfs_ref_tree_mod(root, bytenr, num_bytes, parent,
7239 root_objectid, owner, offset,
7240 BTRFS_DROP_DELAYED_REF);
7241
7242 /*
7243 * tree log blocks never actually go into the extent allocation
7244 * tree, just update pinning info and exit early.
7245 */
7246 if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
7247 WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
7248 /* unlocks the pinned mutex */
7249 btrfs_pin_extent(fs_info, bytenr, num_bytes, 1);
7250 old_ref_mod = new_ref_mod = 0;
7251 ret = 0;
7252 } else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
7253 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
7254 num_bytes, parent,
7255 root_objectid, (int)owner,
7256 BTRFS_DROP_DELAYED_REF, NULL,
7257 &old_ref_mod, &new_ref_mod);
7258 } else {
7259 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
7260 num_bytes, parent,
7261 root_objectid, owner, offset,
7262 0, BTRFS_DROP_DELAYED_REF,
7263 &old_ref_mod, &new_ref_mod);
7264 }
7265
7266 if (ret == 0 && old_ref_mod >= 0 && new_ref_mod < 0) {
7267 bool metadata = owner < BTRFS_FIRST_FREE_OBJECTID;
7268
7269 add_pinned_bytes(fs_info, num_bytes, metadata, root_objectid);
7270 }
7271
7272 return ret;
7273 }
7274
7275 /*
7276 * when we wait for progress in the block group caching, its because
7277 * our allocation attempt failed at least once. So, we must sleep
7278 * and let some progress happen before we try again.
7279 *
7280 * This function will sleep at least once waiting for new free space to
7281 * show up, and then it will check the block group free space numbers
7282 * for our min num_bytes. Another option is to have it go ahead
7283 * and look in the rbtree for a free extent of a given size, but this
7284 * is a good start.
7285 *
7286 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using
7287 * any of the information in this block group.
7288 */
7289 static noinline void
7290 wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
7291 u64 num_bytes)
7292 {
7293 struct btrfs_caching_control *caching_ctl;
7294
7295 caching_ctl = get_caching_control(cache);
7296 if (!caching_ctl)
7297 return;
7298
7299 wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
7300 (cache->free_space_ctl->free_space >= num_bytes));
7301
7302 put_caching_control(caching_ctl);
7303 }
7304
7305 static noinline int
7306 wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
7307 {
7308 struct btrfs_caching_control *caching_ctl;
7309 int ret = 0;
7310
7311 caching_ctl = get_caching_control(cache);
7312 if (!caching_ctl)
7313 return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
7314
7315 wait_event(caching_ctl->wait, block_group_cache_done(cache));
7316 if (cache->cached == BTRFS_CACHE_ERROR)
7317 ret = -EIO;
7318 put_caching_control(caching_ctl);
7319 return ret;
7320 }
7321
7322 enum btrfs_loop_type {
7323 LOOP_CACHING_NOWAIT = 0,
7324 LOOP_CACHING_WAIT = 1,
7325 LOOP_ALLOC_CHUNK = 2,
7326 LOOP_NO_EMPTY_SIZE = 3,
7327 };
7328
7329 static inline void
7330 btrfs_lock_block_group(struct btrfs_block_group_cache *cache,
7331 int delalloc)
7332 {
7333 if (delalloc)
7334 down_read(&cache->data_rwsem);
7335 }
7336
7337 static inline void
7338 btrfs_grab_block_group(struct btrfs_block_group_cache *cache,
7339 int delalloc)
7340 {
7341 btrfs_get_block_group(cache);
7342 if (delalloc)
7343 down_read(&cache->data_rwsem);
7344 }
7345
7346 static struct btrfs_block_group_cache *
7347 btrfs_lock_cluster(struct btrfs_block_group_cache *block_group,
7348 struct btrfs_free_cluster *cluster,
7349 int delalloc)
7350 {
7351 struct btrfs_block_group_cache *used_bg = NULL;
7352
7353 spin_lock(&cluster->refill_lock);
7354 while (1) {
7355 used_bg = cluster->block_group;
7356 if (!used_bg)
7357 return NULL;
7358
7359 if (used_bg == block_group)
7360 return used_bg;
7361
7362 btrfs_get_block_group(used_bg);
7363
7364 if (!delalloc)
7365 return used_bg;
7366
7367 if (down_read_trylock(&used_bg->data_rwsem))
7368 return used_bg;
7369
7370 spin_unlock(&cluster->refill_lock);
7371
7372 /* We should only have one-level nested. */
7373 down_read_nested(&used_bg->data_rwsem, SINGLE_DEPTH_NESTING);
7374
7375 spin_lock(&cluster->refill_lock);
7376 if (used_bg == cluster->block_group)
7377 return used_bg;
7378
7379 up_read(&used_bg->data_rwsem);
7380 btrfs_put_block_group(used_bg);
7381 }
7382 }
7383
7384 static inline void
7385 btrfs_release_block_group(struct btrfs_block_group_cache *cache,
7386 int delalloc)
7387 {
7388 if (delalloc)
7389 up_read(&cache->data_rwsem);
7390 btrfs_put_block_group(cache);
7391 }
7392
7393 /*
7394 * walks the btree of allocated extents and find a hole of a given size.
7395 * The key ins is changed to record the hole:
7396 * ins->objectid == start position
7397 * ins->flags = BTRFS_EXTENT_ITEM_KEY
7398 * ins->offset == the size of the hole.
7399 * Any available blocks before search_start are skipped.
7400 *
7401 * If there is no suitable free space, we will record the max size of
7402 * the free space extent currently.
7403 */
7404 static noinline int find_free_extent(struct btrfs_fs_info *fs_info,
7405 u64 ram_bytes, u64 num_bytes, u64 empty_size,
7406 u64 hint_byte, struct btrfs_key *ins,
7407 u64 flags, int delalloc)
7408 {
7409 int ret = 0;
7410 struct btrfs_root *root = fs_info->extent_root;
7411 struct btrfs_free_cluster *last_ptr = NULL;
7412 struct btrfs_block_group_cache *block_group = NULL;
7413 u64 search_start = 0;
7414 u64 max_extent_size = 0;
7415 u64 empty_cluster = 0;
7416 struct btrfs_space_info *space_info;
7417 int loop = 0;
7418 int index = btrfs_bg_flags_to_raid_index(flags);
7419 bool failed_cluster_refill = false;
7420 bool failed_alloc = false;
7421 bool use_cluster = true;
7422 bool have_caching_bg = false;
7423 bool orig_have_caching_bg = false;
7424 bool full_search = false;
7425
7426 WARN_ON(num_bytes < fs_info->sectorsize);
7427 ins->type = BTRFS_EXTENT_ITEM_KEY;
7428 ins->objectid = 0;
7429 ins->offset = 0;
7430
7431 trace_find_free_extent(fs_info, num_bytes, empty_size, flags);
7432
7433 space_info = __find_space_info(fs_info, flags);
7434 if (!space_info) {
7435 btrfs_err(fs_info, "No space info for %llu", flags);
7436 return -ENOSPC;
7437 }
7438
7439 /*
7440 * If our free space is heavily fragmented we may not be able to make
7441 * big contiguous allocations, so instead of doing the expensive search
7442 * for free space, simply return ENOSPC with our max_extent_size so we
7443 * can go ahead and search for a more manageable chunk.
7444 *
7445 * If our max_extent_size is large enough for our allocation simply
7446 * disable clustering since we will likely not be able to find enough
7447 * space to create a cluster and induce latency trying.
7448 */
7449 if (unlikely(space_info->max_extent_size)) {
7450 spin_lock(&space_info->lock);
7451 if (space_info->max_extent_size &&
7452 num_bytes > space_info->max_extent_size) {
7453 ins->offset = space_info->max_extent_size;
7454 spin_unlock(&space_info->lock);
7455 return -ENOSPC;
7456 } else if (space_info->max_extent_size) {
7457 use_cluster = false;
7458 }
7459 spin_unlock(&space_info->lock);
7460 }
7461
7462 last_ptr = fetch_cluster_info(fs_info, space_info, &empty_cluster);
7463 if (last_ptr) {
7464 spin_lock(&last_ptr->lock);
7465 if (last_ptr->block_group)
7466 hint_byte = last_ptr->window_start;
7467 if (last_ptr->fragmented) {
7468 /*
7469 * We still set window_start so we can keep track of the
7470 * last place we found an allocation to try and save
7471 * some time.
7472 */
7473 hint_byte = last_ptr->window_start;
7474 use_cluster = false;
7475 }
7476 spin_unlock(&last_ptr->lock);
7477 }
7478
7479 search_start = max(search_start, first_logical_byte(fs_info, 0));
7480 search_start = max(search_start, hint_byte);
7481 if (search_start == hint_byte) {
7482 block_group = btrfs_lookup_block_group(fs_info, search_start);
7483 /*
7484 * we don't want to use the block group if it doesn't match our
7485 * allocation bits, or if its not cached.
7486 *
7487 * However if we are re-searching with an ideal block group
7488 * picked out then we don't care that the block group is cached.
7489 */
7490 if (block_group && block_group_bits(block_group, flags) &&
7491 block_group->cached != BTRFS_CACHE_NO) {
7492 down_read(&space_info->groups_sem);
7493 if (list_empty(&block_group->list) ||
7494 block_group->ro) {
7495 /*
7496 * someone is removing this block group,
7497 * we can't jump into the have_block_group
7498 * target because our list pointers are not
7499 * valid
7500 */
7501 btrfs_put_block_group(block_group);
7502 up_read(&space_info->groups_sem);
7503 } else {
7504 index = btrfs_bg_flags_to_raid_index(
7505 block_group->flags);
7506 btrfs_lock_block_group(block_group, delalloc);
7507 goto have_block_group;
7508 }
7509 } else if (block_group) {
7510 btrfs_put_block_group(block_group);
7511 }
7512 }
7513 search:
7514 have_caching_bg = false;
7515 if (index == 0 || index == btrfs_bg_flags_to_raid_index(flags))
7516 full_search = true;
7517 down_read(&space_info->groups_sem);
7518 list_for_each_entry(block_group, &space_info->block_groups[index],
7519 list) {
7520 u64 offset;
7521 int cached;
7522
7523 /* If the block group is read-only, we can skip it entirely. */
7524 if (unlikely(block_group->ro))
7525 continue;
7526
7527 btrfs_grab_block_group(block_group, delalloc);
7528 search_start = block_group->key.objectid;
7529
7530 /*
7531 * this can happen if we end up cycling through all the
7532 * raid types, but we want to make sure we only allocate
7533 * for the proper type.
7534 */
7535 if (!block_group_bits(block_group, flags)) {
7536 u64 extra = BTRFS_BLOCK_GROUP_DUP |
7537 BTRFS_BLOCK_GROUP_RAID1 |
7538 BTRFS_BLOCK_GROUP_RAID5 |
7539 BTRFS_BLOCK_GROUP_RAID6 |
7540 BTRFS_BLOCK_GROUP_RAID10;
7541
7542 /*
7543 * if they asked for extra copies and this block group
7544 * doesn't provide them, bail. This does allow us to
7545 * fill raid0 from raid1.
7546 */
7547 if ((flags & extra) && !(block_group->flags & extra))
7548 goto loop;
7549 }
7550
7551 have_block_group:
7552 cached = block_group_cache_done(block_group);
7553 if (unlikely(!cached)) {
7554 have_caching_bg = true;
7555 ret = cache_block_group(block_group, 0);
7556 BUG_ON(ret < 0);
7557 ret = 0;
7558 }
7559
7560 if (unlikely(block_group->cached == BTRFS_CACHE_ERROR))
7561 goto loop;
7562
7563 /*
7564 * Ok we want to try and use the cluster allocator, so
7565 * lets look there
7566 */
7567 if (last_ptr && use_cluster) {
7568 struct btrfs_block_group_cache *used_block_group;
7569 unsigned long aligned_cluster;
7570 /*
7571 * the refill lock keeps out other
7572 * people trying to start a new cluster
7573 */
7574 used_block_group = btrfs_lock_cluster(block_group,
7575 last_ptr,
7576 delalloc);
7577 if (!used_block_group)
7578 goto refill_cluster;
7579
7580 if (used_block_group != block_group &&
7581 (used_block_group->ro ||
7582 !block_group_bits(used_block_group, flags)))
7583 goto release_cluster;
7584
7585 offset = btrfs_alloc_from_cluster(used_block_group,
7586 last_ptr,
7587 num_bytes,
7588 used_block_group->key.objectid,
7589 &max_extent_size);
7590 if (offset) {
7591 /* we have a block, we're done */
7592 spin_unlock(&last_ptr->refill_lock);
7593 trace_btrfs_reserve_extent_cluster(
7594 used_block_group,
7595 search_start, num_bytes);
7596 if (used_block_group != block_group) {
7597 btrfs_release_block_group(block_group,
7598 delalloc);
7599 block_group = used_block_group;
7600 }
7601 goto checks;
7602 }
7603
7604 WARN_ON(last_ptr->block_group != used_block_group);
7605 release_cluster:
7606 /* If we are on LOOP_NO_EMPTY_SIZE, we can't
7607 * set up a new clusters, so lets just skip it
7608 * and let the allocator find whatever block
7609 * it can find. If we reach this point, we
7610 * will have tried the cluster allocator
7611 * plenty of times and not have found
7612 * anything, so we are likely way too
7613 * fragmented for the clustering stuff to find
7614 * anything.
7615 *
7616 * However, if the cluster is taken from the
7617 * current block group, release the cluster
7618 * first, so that we stand a better chance of
7619 * succeeding in the unclustered
7620 * allocation. */
7621 if (loop >= LOOP_NO_EMPTY_SIZE &&
7622 used_block_group != block_group) {
7623 spin_unlock(&last_ptr->refill_lock);
7624 btrfs_release_block_group(used_block_group,
7625 delalloc);
7626 goto unclustered_alloc;
7627 }
7628
7629 /*
7630 * this cluster didn't work out, free it and
7631 * start over
7632 */
7633 btrfs_return_cluster_to_free_space(NULL, last_ptr);
7634
7635 if (used_block_group != block_group)
7636 btrfs_release_block_group(used_block_group,
7637 delalloc);
7638 refill_cluster:
7639 if (loop >= LOOP_NO_EMPTY_SIZE) {
7640 spin_unlock(&last_ptr->refill_lock);
7641 goto unclustered_alloc;
7642 }
7643
7644 aligned_cluster = max_t(unsigned long,
7645 empty_cluster + empty_size,
7646 block_group->full_stripe_len);
7647
7648 /* allocate a cluster in this block group */
7649 ret = btrfs_find_space_cluster(fs_info, block_group,
7650 last_ptr, search_start,
7651 num_bytes,
7652 aligned_cluster);
7653 if (ret == 0) {
7654 /*
7655 * now pull our allocation out of this
7656 * cluster
7657 */
7658 offset = btrfs_alloc_from_cluster(block_group,
7659 last_ptr,
7660 num_bytes,
7661 search_start,
7662 &max_extent_size);
7663 if (offset) {
7664 /* we found one, proceed */
7665 spin_unlock(&last_ptr->refill_lock);
7666 trace_btrfs_reserve_extent_cluster(
7667 block_group, search_start,
7668 num_bytes);
7669 goto checks;
7670 }
7671 } else if (!cached && loop > LOOP_CACHING_NOWAIT
7672 && !failed_cluster_refill) {
7673 spin_unlock(&last_ptr->refill_lock);
7674
7675 failed_cluster_refill = true;
7676 wait_block_group_cache_progress(block_group,
7677 num_bytes + empty_cluster + empty_size);
7678 goto have_block_group;
7679 }
7680
7681 /*
7682 * at this point we either didn't find a cluster
7683 * or we weren't able to allocate a block from our
7684 * cluster. Free the cluster we've been trying
7685 * to use, and go to the next block group
7686 */
7687 btrfs_return_cluster_to_free_space(NULL, last_ptr);
7688 spin_unlock(&last_ptr->refill_lock);
7689 goto loop;
7690 }
7691
7692 unclustered_alloc:
7693 /*
7694 * We are doing an unclustered alloc, set the fragmented flag so
7695 * we don't bother trying to setup a cluster again until we get
7696 * more space.
7697 */
7698 if (unlikely(last_ptr)) {
7699 spin_lock(&last_ptr->lock);
7700 last_ptr->fragmented = 1;
7701 spin_unlock(&last_ptr->lock);
7702 }
7703 if (cached) {
7704 struct btrfs_free_space_ctl *ctl =
7705 block_group->free_space_ctl;
7706
7707 spin_lock(&ctl->tree_lock);
7708 if (ctl->free_space <
7709 num_bytes + empty_cluster + empty_size) {
7710 if (ctl->free_space > max_extent_size)
7711 max_extent_size = ctl->free_space;
7712 spin_unlock(&ctl->tree_lock);
7713 goto loop;
7714 }
7715 spin_unlock(&ctl->tree_lock);
7716 }
7717
7718 offset = btrfs_find_space_for_alloc(block_group, search_start,
7719 num_bytes, empty_size,
7720 &max_extent_size);
7721 /*
7722 * If we didn't find a chunk, and we haven't failed on this
7723 * block group before, and this block group is in the middle of
7724 * caching and we are ok with waiting, then go ahead and wait
7725 * for progress to be made, and set failed_alloc to true.
7726 *
7727 * If failed_alloc is true then we've already waited on this
7728 * block group once and should move on to the next block group.
7729 */
7730 if (!offset && !failed_alloc && !cached &&
7731 loop > LOOP_CACHING_NOWAIT) {
7732 wait_block_group_cache_progress(block_group,
7733 num_bytes + empty_size);
7734 failed_alloc = true;
7735 goto have_block_group;
7736 } else if (!offset) {
7737 goto loop;
7738 }
7739 checks:
7740 search_start = ALIGN(offset, fs_info->stripesize);
7741
7742 /* move on to the next group */
7743 if (search_start + num_bytes >
7744 block_group->key.objectid + block_group->key.offset) {
7745 btrfs_add_free_space(block_group, offset, num_bytes);
7746 goto loop;
7747 }
7748
7749 if (offset < search_start)
7750 btrfs_add_free_space(block_group, offset,
7751 search_start - offset);
7752 BUG_ON(offset > search_start);
7753
7754 ret = btrfs_add_reserved_bytes(block_group, ram_bytes,
7755 num_bytes, delalloc);
7756 if (ret == -EAGAIN) {
7757 btrfs_add_free_space(block_group, offset, num_bytes);
7758 goto loop;
7759 }
7760 btrfs_inc_block_group_reservations(block_group);
7761
7762 /* we are all good, lets return */
7763 ins->objectid = search_start;
7764 ins->offset = num_bytes;
7765
7766 trace_btrfs_reserve_extent(block_group, search_start, num_bytes);
7767 btrfs_release_block_group(block_group, delalloc);
7768 break;
7769 loop:
7770 failed_cluster_refill = false;
7771 failed_alloc = false;
7772 BUG_ON(btrfs_bg_flags_to_raid_index(block_group->flags) !=
7773 index);
7774 btrfs_release_block_group(block_group, delalloc);
7775 cond_resched();
7776 }
7777 up_read(&space_info->groups_sem);
7778
7779 if ((loop == LOOP_CACHING_NOWAIT) && have_caching_bg
7780 && !orig_have_caching_bg)
7781 orig_have_caching_bg = true;
7782
7783 if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
7784 goto search;
7785
7786 if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
7787 goto search;
7788
7789 /*
7790 * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
7791 * caching kthreads as we move along
7792 * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
7793 * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
7794 * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
7795 * again
7796 */
7797 if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
7798 index = 0;
7799 if (loop == LOOP_CACHING_NOWAIT) {
7800 /*
7801 * We want to skip the LOOP_CACHING_WAIT step if we
7802 * don't have any uncached bgs and we've already done a
7803 * full search through.
7804 */
7805 if (orig_have_caching_bg || !full_search)
7806 loop = LOOP_CACHING_WAIT;
7807 else
7808 loop = LOOP_ALLOC_CHUNK;
7809 } else {
7810 loop++;
7811 }
7812
7813 if (loop == LOOP_ALLOC_CHUNK) {
7814 struct btrfs_trans_handle *trans;
7815 int exist = 0;
7816
7817 trans = current->journal_info;
7818 if (trans)
7819 exist = 1;
7820 else
7821 trans = btrfs_join_transaction(root);
7822
7823 if (IS_ERR(trans)) {
7824 ret = PTR_ERR(trans);
7825 goto out;
7826 }
7827
7828 ret = do_chunk_alloc(trans, fs_info, flags,
7829 CHUNK_ALLOC_FORCE);
7830
7831 /*
7832 * If we can't allocate a new chunk we've already looped
7833 * through at least once, move on to the NO_EMPTY_SIZE
7834 * case.
7835 */
7836 if (ret == -ENOSPC)
7837 loop = LOOP_NO_EMPTY_SIZE;
7838
7839 /*
7840 * Do not bail out on ENOSPC since we
7841 * can do more things.
7842 */
7843 if (ret < 0 && ret != -ENOSPC)
7844 btrfs_abort_transaction(trans, ret);
7845 else
7846 ret = 0;
7847 if (!exist)
7848 btrfs_end_transaction(trans);
7849 if (ret)
7850 goto out;
7851 }
7852
7853 if (loop == LOOP_NO_EMPTY_SIZE) {
7854 /*
7855 * Don't loop again if we already have no empty_size and
7856 * no empty_cluster.
7857 */
7858 if (empty_size == 0 &&
7859 empty_cluster == 0) {
7860 ret = -ENOSPC;
7861 goto out;
7862 }
7863 empty_size = 0;
7864 empty_cluster = 0;
7865 }
7866
7867 goto search;
7868 } else if (!ins->objectid) {
7869 ret = -ENOSPC;
7870 } else if (ins->objectid) {
7871 if (!use_cluster && last_ptr) {
7872 spin_lock(&last_ptr->lock);
7873 last_ptr->window_start = ins->objectid;
7874 spin_unlock(&last_ptr->lock);
7875 }
7876 ret = 0;
7877 }
7878 out:
7879 if (ret == -ENOSPC) {
7880 spin_lock(&space_info->lock);
7881 space_info->max_extent_size = max_extent_size;
7882 spin_unlock(&space_info->lock);
7883 ins->offset = max_extent_size;
7884 }
7885 return ret;
7886 }
7887
7888 static void dump_space_info(struct btrfs_fs_info *fs_info,
7889 struct btrfs_space_info *info, u64 bytes,
7890 int dump_block_groups)
7891 {
7892 struct btrfs_block_group_cache *cache;
7893 int index = 0;
7894
7895 spin_lock(&info->lock);
7896 btrfs_info(fs_info, "space_info %llu has %llu free, is %sfull",
7897 info->flags,
7898 info->total_bytes - btrfs_space_info_used(info, true),
7899 info->full ? "" : "not ");
7900 btrfs_info(fs_info,
7901 "space_info total=%llu, used=%llu, pinned=%llu, reserved=%llu, may_use=%llu, readonly=%llu",
7902 info->total_bytes, info->bytes_used, info->bytes_pinned,
7903 info->bytes_reserved, info->bytes_may_use,
7904 info->bytes_readonly);
7905 spin_unlock(&info->lock);
7906
7907 if (!dump_block_groups)
7908 return;
7909
7910 down_read(&info->groups_sem);
7911 again:
7912 list_for_each_entry(cache, &info->block_groups[index], list) {
7913 spin_lock(&cache->lock);
7914 btrfs_info(fs_info,
7915 "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s",
7916 cache->key.objectid, cache->key.offset,
7917 btrfs_block_group_used(&cache->item), cache->pinned,
7918 cache->reserved, cache->ro ? "[readonly]" : "");
7919 btrfs_dump_free_space(cache, bytes);
7920 spin_unlock(&cache->lock);
7921 }
7922 if (++index < BTRFS_NR_RAID_TYPES)
7923 goto again;
7924 up_read(&info->groups_sem);
7925 }
7926
7927 /*
7928 * btrfs_reserve_extent - entry point to the extent allocator. Tries to find a
7929 * hole that is at least as big as @num_bytes.
7930 *
7931 * @root - The root that will contain this extent
7932 *
7933 * @ram_bytes - The amount of space in ram that @num_bytes take. This
7934 * is used for accounting purposes. This value differs
7935 * from @num_bytes only in the case of compressed extents.
7936 *
7937 * @num_bytes - Number of bytes to allocate on-disk.
7938 *
7939 * @min_alloc_size - Indicates the minimum amount of space that the
7940 * allocator should try to satisfy. In some cases
7941 * @num_bytes may be larger than what is required and if
7942 * the filesystem is fragmented then allocation fails.
7943 * However, the presence of @min_alloc_size gives a
7944 * chance to try and satisfy the smaller allocation.
7945 *
7946 * @empty_size - A hint that you plan on doing more COW. This is the
7947 * size in bytes the allocator should try to find free
7948 * next to the block it returns. This is just a hint and
7949 * may be ignored by the allocator.
7950 *
7951 * @hint_byte - Hint to the allocator to start searching above the byte
7952 * address passed. It might be ignored.
7953 *
7954 * @ins - This key is modified to record the found hole. It will
7955 * have the following values:
7956 * ins->objectid == start position
7957 * ins->flags = BTRFS_EXTENT_ITEM_KEY
7958 * ins->offset == the size of the hole.
7959 *
7960 * @is_data - Boolean flag indicating whether an extent is
7961 * allocated for data (true) or metadata (false)
7962 *
7963 * @delalloc - Boolean flag indicating whether this allocation is for
7964 * delalloc or not. If 'true' data_rwsem of block groups
7965 * is going to be acquired.
7966 *
7967 *
7968 * Returns 0 when an allocation succeeded or < 0 when an error occurred. In
7969 * case -ENOSPC is returned then @ins->offset will contain the size of the
7970 * largest available hole the allocator managed to find.
7971 */
7972 int btrfs_reserve_extent(struct btrfs_root *root, u64 ram_bytes,
7973 u64 num_bytes, u64 min_alloc_size,
7974 u64 empty_size, u64 hint_byte,
7975 struct btrfs_key *ins, int is_data, int delalloc)
7976 {
7977 struct btrfs_fs_info *fs_info = root->fs_info;
7978 bool final_tried = num_bytes == min_alloc_size;
7979 u64 flags;
7980 int ret;
7981
7982 flags = get_alloc_profile_by_root(root, is_data);
7983 again:
7984 WARN_ON(num_bytes < fs_info->sectorsize);
7985 ret = find_free_extent(fs_info, ram_bytes, num_bytes, empty_size,
7986 hint_byte, ins, flags, delalloc);
7987 if (!ret && !is_data) {
7988 btrfs_dec_block_group_reservations(fs_info, ins->objectid);
7989 } else if (ret == -ENOSPC) {
7990 if (!final_tried && ins->offset) {
7991 num_bytes = min(num_bytes >> 1, ins->offset);
7992 num_bytes = round_down(num_bytes,
7993 fs_info->sectorsize);
7994 num_bytes = max(num_bytes, min_alloc_size);
7995 ram_bytes = num_bytes;
7996 if (num_bytes == min_alloc_size)
7997 final_tried = true;
7998 goto again;
7999 } else if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
8000 struct btrfs_space_info *sinfo;
8001
8002 sinfo = __find_space_info(fs_info, flags);
8003 btrfs_err(fs_info,
8004 "allocation failed flags %llu, wanted %llu",
8005 flags, num_bytes);
8006 if (sinfo)
8007 dump_space_info(fs_info, sinfo, num_bytes, 1);
8008 }
8009 }
8010
8011 return ret;
8012 }
8013
8014 static int __btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info,
8015 u64 start, u64 len,
8016 int pin, int delalloc)
8017 {
8018 struct btrfs_block_group_cache *cache;
8019 int ret = 0;
8020
8021 cache = btrfs_lookup_block_group(fs_info, start);
8022 if (!cache) {
8023 btrfs_err(fs_info, "Unable to find block group for %llu",
8024 start);
8025 return -ENOSPC;
8026 }
8027
8028 if (pin)
8029 pin_down_extent(fs_info, cache, start, len, 1);
8030 else {
8031 if (btrfs_test_opt(fs_info, DISCARD))
8032 ret = btrfs_discard_extent(fs_info, start, len, NULL);
8033 btrfs_add_free_space(cache, start, len);
8034 btrfs_free_reserved_bytes(cache, len, delalloc);
8035 trace_btrfs_reserved_extent_free(fs_info, start, len);
8036 }
8037
8038 btrfs_put_block_group(cache);
8039 return ret;
8040 }
8041
8042 int btrfs_free_reserved_extent(struct btrfs_fs_info *fs_info,
8043 u64 start, u64 len, int delalloc)
8044 {
8045 return __btrfs_free_reserved_extent(fs_info, start, len, 0, delalloc);
8046 }
8047
8048 int btrfs_free_and_pin_reserved_extent(struct btrfs_fs_info *fs_info,
8049 u64 start, u64 len)
8050 {
8051 return __btrfs_free_reserved_extent(fs_info, start, len, 1, 0);
8052 }
8053
8054 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
8055 struct btrfs_fs_info *fs_info,
8056 u64 parent, u64 root_objectid,
8057 u64 flags, u64 owner, u64 offset,
8058 struct btrfs_key *ins, int ref_mod)
8059 {
8060 int ret;
8061 struct btrfs_extent_item *extent_item;
8062 struct btrfs_extent_inline_ref *iref;
8063 struct btrfs_path *path;
8064 struct extent_buffer *leaf;
8065 int type;
8066 u32 size;
8067
8068 if (parent > 0)
8069 type = BTRFS_SHARED_DATA_REF_KEY;
8070 else
8071 type = BTRFS_EXTENT_DATA_REF_KEY;
8072
8073 size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
8074
8075 path = btrfs_alloc_path();
8076 if (!path)
8077 return -ENOMEM;
8078
8079 path->leave_spinning = 1;
8080 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
8081 ins, size);
8082 if (ret) {
8083 btrfs_free_path(path);
8084 return ret;
8085 }
8086
8087 leaf = path->nodes[0];
8088 extent_item = btrfs_item_ptr(leaf, path->slots[0],
8089 struct btrfs_extent_item);
8090 btrfs_set_extent_refs(leaf, extent_item, ref_mod);
8091 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
8092 btrfs_set_extent_flags(leaf, extent_item,
8093 flags | BTRFS_EXTENT_FLAG_DATA);
8094
8095 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
8096 btrfs_set_extent_inline_ref_type(leaf, iref, type);
8097 if (parent > 0) {
8098 struct btrfs_shared_data_ref *ref;
8099 ref = (struct btrfs_shared_data_ref *)(iref + 1);
8100 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
8101 btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
8102 } else {
8103 struct btrfs_extent_data_ref *ref;
8104 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
8105 btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
8106 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
8107 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
8108 btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
8109 }
8110
8111 btrfs_mark_buffer_dirty(path->nodes[0]);
8112 btrfs_free_path(path);
8113
8114 ret = remove_from_free_space_tree(trans, ins->objectid, ins->offset);
8115 if (ret)
8116 return ret;
8117
8118 ret = update_block_group(trans, fs_info, ins->objectid, ins->offset, 1);
8119 if (ret) { /* -ENOENT, logic error */
8120 btrfs_err(fs_info, "update block group failed for %llu %llu",
8121 ins->objectid, ins->offset);
8122 BUG();
8123 }
8124 trace_btrfs_reserved_extent_alloc(fs_info, ins->objectid, ins->offset);
8125 return ret;
8126 }
8127
8128 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
8129 struct btrfs_delayed_ref_node *node,
8130 struct btrfs_delayed_extent_op *extent_op)
8131 {
8132 struct btrfs_fs_info *fs_info = trans->fs_info;
8133 int ret;
8134 struct btrfs_extent_item *extent_item;
8135 struct btrfs_key extent_key;
8136 struct btrfs_tree_block_info *block_info;
8137 struct btrfs_extent_inline_ref *iref;
8138 struct btrfs_path *path;
8139 struct extent_buffer *leaf;
8140 struct btrfs_delayed_tree_ref *ref;
8141 u32 size = sizeof(*extent_item) + sizeof(*iref);
8142 u64 num_bytes;
8143 u64 flags = extent_op->flags_to_set;
8144 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA);
8145
8146 ref = btrfs_delayed_node_to_tree_ref(node);
8147
8148 extent_key.objectid = node->bytenr;
8149 if (skinny_metadata) {
8150 extent_key.offset = ref->level;
8151 extent_key.type = BTRFS_METADATA_ITEM_KEY;
8152 num_bytes = fs_info->nodesize;
8153 } else {
8154 extent_key.offset = node->num_bytes;
8155 extent_key.type = BTRFS_EXTENT_ITEM_KEY;
8156 size += sizeof(*block_info);
8157 num_bytes = node->num_bytes;
8158 }
8159
8160 path = btrfs_alloc_path();
8161 if (!path) {
8162 btrfs_free_and_pin_reserved_extent(fs_info,
8163 extent_key.objectid,
8164 fs_info->nodesize);
8165 return -ENOMEM;
8166 }
8167
8168 path->leave_spinning = 1;
8169 ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
8170 &extent_key, size);
8171 if (ret) {
8172 btrfs_free_path(path);
8173 btrfs_free_and_pin_reserved_extent(fs_info,
8174 extent_key.objectid,
8175 fs_info->nodesize);
8176 return ret;
8177 }
8178
8179 leaf = path->nodes[0];
8180 extent_item = btrfs_item_ptr(leaf, path->slots[0],
8181 struct btrfs_extent_item);
8182 btrfs_set_extent_refs(leaf, extent_item, 1);
8183 btrfs_set_extent_generation(leaf, extent_item, trans->transid);
8184 btrfs_set_extent_flags(leaf, extent_item,
8185 flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
8186
8187 if (skinny_metadata) {
8188 iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
8189 } else {
8190 block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
8191 btrfs_set_tree_block_key(leaf, block_info, &extent_op->key);
8192 btrfs_set_tree_block_level(leaf, block_info, ref->level);
8193 iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
8194 }
8195
8196 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) {
8197 BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
8198 btrfs_set_extent_inline_ref_type(leaf, iref,
8199 BTRFS_SHARED_BLOCK_REF_KEY);
8200 btrfs_set_extent_inline_ref_offset(leaf, iref, ref->parent);
8201 } else {
8202 btrfs_set_extent_inline_ref_type(leaf, iref,
8203 BTRFS_TREE_BLOCK_REF_KEY);
8204 btrfs_set_extent_inline_ref_offset(leaf, iref, ref->root);
8205 }
8206
8207 btrfs_mark_buffer_dirty(leaf);
8208 btrfs_free_path(path);
8209
8210 ret = remove_from_free_space_tree(trans, extent_key.objectid,
8211 num_bytes);
8212 if (ret)
8213 return ret;
8214
8215 ret = update_block_group(trans, fs_info, extent_key.objectid,
8216 fs_info->nodesize, 1);
8217 if (ret) { /* -ENOENT, logic error */
8218 btrfs_err(fs_info, "update block group failed for %llu %llu",
8219 extent_key.objectid, extent_key.offset);
8220 BUG();
8221 }
8222
8223 trace_btrfs_reserved_extent_alloc(fs_info, extent_key.objectid,
8224 fs_info->nodesize);
8225 return ret;
8226 }
8227
8228 int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
8229 struct btrfs_root *root, u64 owner,
8230 u64 offset, u64 ram_bytes,
8231 struct btrfs_key *ins)
8232 {
8233 struct btrfs_fs_info *fs_info = root->fs_info;
8234 int ret;
8235
8236 BUG_ON(root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID);
8237
8238 btrfs_ref_tree_mod(root, ins->objectid, ins->offset, 0,
8239 root->root_key.objectid, owner, offset,
8240 BTRFS_ADD_DELAYED_EXTENT);
8241
8242 ret = btrfs_add_delayed_data_ref(fs_info, trans, ins->objectid,
8243 ins->offset, 0,
8244 root->root_key.objectid, owner,
8245 offset, ram_bytes,
8246 BTRFS_ADD_DELAYED_EXTENT, NULL, NULL);
8247 return ret;
8248 }
8249
8250 /*
8251 * this is used by the tree logging recovery code. It records that
8252 * an extent has been allocated and makes sure to clear the free
8253 * space cache bits as well
8254 */
8255 int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
8256 struct btrfs_fs_info *fs_info,
8257 u64 root_objectid, u64 owner, u64 offset,
8258 struct btrfs_key *ins)
8259 {
8260 int ret;
8261 struct btrfs_block_group_cache *block_group;
8262 struct btrfs_space_info *space_info;
8263
8264 /*
8265 * Mixed block groups will exclude before processing the log so we only
8266 * need to do the exclude dance if this fs isn't mixed.
8267 */
8268 if (!btrfs_fs_incompat(fs_info, MIXED_GROUPS)) {
8269 ret = __exclude_logged_extent(fs_info, ins->objectid,
8270 ins->offset);
8271 if (ret)
8272 return ret;
8273 }
8274
8275 block_group = btrfs_lookup_block_group(fs_info, ins->objectid);
8276 if (!block_group)
8277 return -EINVAL;
8278
8279 space_info = block_group->space_info;
8280 spin_lock(&space_info->lock);
8281 spin_lock(&block_group->lock);
8282 space_info->bytes_reserved += ins->offset;
8283 block_group->reserved += ins->offset;
8284 spin_unlock(&block_group->lock);
8285 spin_unlock(&space_info->lock);
8286
8287 ret = alloc_reserved_file_extent(trans, fs_info, 0, root_objectid,
8288 0, owner, offset, ins, 1);
8289 btrfs_put_block_group(block_group);
8290 return ret;
8291 }
8292
8293 static struct extent_buffer *
8294 btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root,
8295 u64 bytenr, int level)
8296 {
8297 struct btrfs_fs_info *fs_info = root->fs_info;
8298 struct extent_buffer *buf;
8299
8300 buf = btrfs_find_create_tree_block(fs_info, bytenr);
8301 if (IS_ERR(buf))
8302 return buf;
8303
8304 btrfs_set_header_generation(buf, trans->transid);
8305 btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
8306 btrfs_tree_lock(buf);
8307 clean_tree_block(fs_info, buf);
8308 clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
8309
8310 btrfs_set_lock_blocking(buf);
8311 set_extent_buffer_uptodate(buf);
8312
8313 if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
8314 buf->log_index = root->log_transid % 2;
8315 /*
8316 * we allow two log transactions at a time, use different
8317 * EXENT bit to differentiate dirty pages.
8318 */
8319 if (buf->log_index == 0)
8320 set_extent_dirty(&root->dirty_log_pages, buf->start,
8321 buf->start + buf->len - 1, GFP_NOFS);
8322 else
8323 set_extent_new(&root->dirty_log_pages, buf->start,
8324 buf->start + buf->len - 1);
8325 } else {
8326 buf->log_index = -1;
8327 set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
8328 buf->start + buf->len - 1, GFP_NOFS);
8329 }
8330 trans->dirty = true;
8331 /* this returns a buffer locked for blocking */
8332 return buf;
8333 }
8334
8335 static struct btrfs_block_rsv *
8336 use_block_rsv(struct btrfs_trans_handle *trans,
8337 struct btrfs_root *root, u32 blocksize)
8338 {
8339 struct btrfs_fs_info *fs_info = root->fs_info;
8340 struct btrfs_block_rsv *block_rsv;
8341 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
8342 int ret;
8343 bool global_updated = false;
8344
8345 block_rsv = get_block_rsv(trans, root);
8346
8347 if (unlikely(block_rsv->size == 0))
8348 goto try_reserve;
8349 again:
8350 ret = block_rsv_use_bytes(block_rsv, blocksize);
8351 if (!ret)
8352 return block_rsv;
8353
8354 if (block_rsv->failfast)
8355 return ERR_PTR(ret);
8356
8357 if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) {
8358 global_updated = true;
8359 update_global_block_rsv(fs_info);
8360 goto again;
8361 }
8362
8363 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
8364 static DEFINE_RATELIMIT_STATE(_rs,
8365 DEFAULT_RATELIMIT_INTERVAL * 10,
8366 /*DEFAULT_RATELIMIT_BURST*/ 1);
8367 if (__ratelimit(&_rs))
8368 WARN(1, KERN_DEBUG
8369 "BTRFS: block rsv returned %d\n", ret);
8370 }
8371 try_reserve:
8372 ret = reserve_metadata_bytes(root, block_rsv, blocksize,
8373 BTRFS_RESERVE_NO_FLUSH);
8374 if (!ret)
8375 return block_rsv;
8376 /*
8377 * If we couldn't reserve metadata bytes try and use some from
8378 * the global reserve if its space type is the same as the global
8379 * reservation.
8380 */
8381 if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL &&
8382 block_rsv->space_info == global_rsv->space_info) {
8383 ret = block_rsv_use_bytes(global_rsv, blocksize);
8384 if (!ret)
8385 return global_rsv;
8386 }
8387 return ERR_PTR(ret);
8388 }
8389
8390 static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
8391 struct btrfs_block_rsv *block_rsv, u32 blocksize)
8392 {
8393 block_rsv_add_bytes(block_rsv, blocksize, 0);
8394 block_rsv_release_bytes(fs_info, block_rsv, NULL, 0, NULL);
8395 }
8396
8397 /*
8398 * finds a free extent and does all the dirty work required for allocation
8399 * returns the tree buffer or an ERR_PTR on error.
8400 */
8401 struct extent_buffer *btrfs_alloc_tree_block(struct btrfs_trans_handle *trans,
8402 struct btrfs_root *root,
8403 u64 parent, u64 root_objectid,
8404 const struct btrfs_disk_key *key,
8405 int level, u64 hint,
8406 u64 empty_size)
8407 {
8408 struct btrfs_fs_info *fs_info = root->fs_info;
8409 struct btrfs_key ins;
8410 struct btrfs_block_rsv *block_rsv;
8411 struct extent_buffer *buf;
8412 struct btrfs_delayed_extent_op *extent_op;
8413 u64 flags = 0;
8414 int ret;
8415 u32 blocksize = fs_info->nodesize;
8416 bool skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA);
8417
8418 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
8419 if (btrfs_is_testing(fs_info)) {
8420 buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr,
8421 level);
8422 if (!IS_ERR(buf))
8423 root->alloc_bytenr += blocksize;
8424 return buf;
8425 }
8426 #endif
8427
8428 block_rsv = use_block_rsv(trans, root, blocksize);
8429 if (IS_ERR(block_rsv))
8430 return ERR_CAST(block_rsv);
8431
8432 ret = btrfs_reserve_extent(root, blocksize, blocksize, blocksize,
8433 empty_size, hint, &ins, 0, 0);
8434 if (ret)
8435 goto out_unuse;
8436
8437 buf = btrfs_init_new_buffer(trans, root, ins.objectid, level);
8438 if (IS_ERR(buf)) {
8439 ret = PTR_ERR(buf);
8440 goto out_free_reserved;
8441 }
8442
8443 if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
8444 if (parent == 0)
8445 parent = ins.objectid;
8446 flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
8447 } else
8448 BUG_ON(parent > 0);
8449
8450 if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
8451 extent_op = btrfs_alloc_delayed_extent_op();
8452 if (!extent_op) {
8453 ret = -ENOMEM;
8454 goto out_free_buf;
8455 }
8456 if (key)
8457 memcpy(&extent_op->key, key, sizeof(extent_op->key));
8458 else
8459 memset(&extent_op->key, 0, sizeof(extent_op->key));
8460 extent_op->flags_to_set = flags;
8461 extent_op->update_key = skinny_metadata ? false : true;
8462 extent_op->update_flags = true;
8463 extent_op->is_data = false;
8464 extent_op->level = level;
8465
8466 btrfs_ref_tree_mod(root, ins.objectid, ins.offset, parent,
8467 root_objectid, level, 0,
8468 BTRFS_ADD_DELAYED_EXTENT);
8469 ret = btrfs_add_delayed_tree_ref(fs_info, trans, ins.objectid,
8470 ins.offset, parent,
8471 root_objectid, level,
8472 BTRFS_ADD_DELAYED_EXTENT,
8473 extent_op, NULL, NULL);
8474 if (ret)
8475 goto out_free_delayed;
8476 }
8477 return buf;
8478
8479 out_free_delayed:
8480 btrfs_free_delayed_extent_op(extent_op);
8481 out_free_buf:
8482 free_extent_buffer(buf);
8483 out_free_reserved:
8484 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 0);
8485 out_unuse:
8486 unuse_block_rsv(fs_info, block_rsv, blocksize);
8487 return ERR_PTR(ret);
8488 }
8489
8490 struct walk_control {
8491 u64 refs[BTRFS_MAX_LEVEL];
8492 u64 flags[BTRFS_MAX_LEVEL];
8493 struct btrfs_key update_progress;
8494 int stage;
8495 int level;
8496 int shared_level;
8497 int update_ref;
8498 int keep_locks;
8499 int reada_slot;
8500 int reada_count;
8501 int for_reloc;
8502 };
8503
8504 #define DROP_REFERENCE 1
8505 #define UPDATE_BACKREF 2
8506
8507 static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
8508 struct btrfs_root *root,
8509 struct walk_control *wc,
8510 struct btrfs_path *path)
8511 {
8512 struct btrfs_fs_info *fs_info = root->fs_info;
8513 u64 bytenr;
8514 u64 generation;
8515 u64 refs;
8516 u64 flags;
8517 u32 nritems;
8518 struct btrfs_key key;
8519 struct extent_buffer *eb;
8520 int ret;
8521 int slot;
8522 int nread = 0;
8523
8524 if (path->slots[wc->level] < wc->reada_slot) {
8525 wc->reada_count = wc->reada_count * 2 / 3;
8526 wc->reada_count = max(wc->reada_count, 2);
8527 } else {
8528 wc->reada_count = wc->reada_count * 3 / 2;
8529 wc->reada_count = min_t(int, wc->reada_count,
8530 BTRFS_NODEPTRS_PER_BLOCK(fs_info));
8531 }
8532
8533 eb = path->nodes[wc->level];
8534 nritems = btrfs_header_nritems(eb);
8535
8536 for (slot = path->slots[wc->level]; slot < nritems; slot++) {
8537 if (nread >= wc->reada_count)
8538 break;
8539
8540 cond_resched();
8541 bytenr = btrfs_node_blockptr(eb, slot);
8542 generation = btrfs_node_ptr_generation(eb, slot);
8543
8544 if (slot == path->slots[wc->level])
8545 goto reada;
8546
8547 if (wc->stage == UPDATE_BACKREF &&
8548 generation <= root->root_key.offset)
8549 continue;
8550
8551 /* We don't lock the tree block, it's OK to be racy here */
8552 ret = btrfs_lookup_extent_info(trans, fs_info, bytenr,
8553 wc->level - 1, 1, &refs,
8554 &flags);
8555 /* We don't care about errors in readahead. */
8556 if (ret < 0)
8557 continue;
8558 BUG_ON(refs == 0);
8559
8560 if (wc->stage == DROP_REFERENCE) {
8561 if (refs == 1)
8562 goto reada;
8563
8564 if (wc->level == 1 &&
8565 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8566 continue;
8567 if (!wc->update_ref ||
8568 generation <= root->root_key.offset)
8569 continue;
8570 btrfs_node_key_to_cpu(eb, &key, slot);
8571 ret = btrfs_comp_cpu_keys(&key,
8572 &wc->update_progress);
8573 if (ret < 0)
8574 continue;
8575 } else {
8576 if (wc->level == 1 &&
8577 (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8578 continue;
8579 }
8580 reada:
8581 readahead_tree_block(fs_info, bytenr);
8582 nread++;
8583 }
8584 wc->reada_slot = slot;
8585 }
8586
8587 /*
8588 * helper to process tree block while walking down the tree.
8589 *
8590 * when wc->stage == UPDATE_BACKREF, this function updates
8591 * back refs for pointers in the block.
8592 *
8593 * NOTE: return value 1 means we should stop walking down.
8594 */
8595 static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
8596 struct btrfs_root *root,
8597 struct btrfs_path *path,
8598 struct walk_control *wc, int lookup_info)
8599 {
8600 struct btrfs_fs_info *fs_info = root->fs_info;
8601 int level = wc->level;
8602 struct extent_buffer *eb = path->nodes[level];
8603 u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
8604 int ret;
8605
8606 if (wc->stage == UPDATE_BACKREF &&
8607 btrfs_header_owner(eb) != root->root_key.objectid)
8608 return 1;
8609
8610 /*
8611 * when reference count of tree block is 1, it won't increase
8612 * again. once full backref flag is set, we never clear it.
8613 */
8614 if (lookup_info &&
8615 ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
8616 (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
8617 BUG_ON(!path->locks[level]);
8618 ret = btrfs_lookup_extent_info(trans, fs_info,
8619 eb->start, level, 1,
8620 &wc->refs[level],
8621 &wc->flags[level]);
8622 BUG_ON(ret == -ENOMEM);
8623 if (ret)
8624 return ret;
8625 BUG_ON(wc->refs[level] == 0);
8626 }
8627
8628 if (wc->stage == DROP_REFERENCE) {
8629 if (wc->refs[level] > 1)
8630 return 1;
8631
8632 if (path->locks[level] && !wc->keep_locks) {
8633 btrfs_tree_unlock_rw(eb, path->locks[level]);
8634 path->locks[level] = 0;
8635 }
8636 return 0;
8637 }
8638
8639 /* wc->stage == UPDATE_BACKREF */
8640 if (!(wc->flags[level] & flag)) {
8641 BUG_ON(!path->locks[level]);
8642 ret = btrfs_inc_ref(trans, root, eb, 1);
8643 BUG_ON(ret); /* -ENOMEM */
8644 ret = btrfs_dec_ref(trans, root, eb, 0);
8645 BUG_ON(ret); /* -ENOMEM */
8646 ret = btrfs_set_disk_extent_flags(trans, fs_info, eb->start,
8647 eb->len, flag,
8648 btrfs_header_level(eb), 0);
8649 BUG_ON(ret); /* -ENOMEM */
8650 wc->flags[level] |= flag;
8651 }
8652
8653 /*
8654 * the block is shared by multiple trees, so it's not good to
8655 * keep the tree lock
8656 */
8657 if (path->locks[level] && level > 0) {
8658 btrfs_tree_unlock_rw(eb, path->locks[level]);
8659 path->locks[level] = 0;
8660 }
8661 return 0;
8662 }
8663
8664 /*
8665 * helper to process tree block pointer.
8666 *
8667 * when wc->stage == DROP_REFERENCE, this function checks
8668 * reference count of the block pointed to. if the block
8669 * is shared and we need update back refs for the subtree
8670 * rooted at the block, this function changes wc->stage to
8671 * UPDATE_BACKREF. if the block is shared and there is no
8672 * need to update back, this function drops the reference
8673 * to the block.
8674 *
8675 * NOTE: return value 1 means we should stop walking down.
8676 */
8677 static noinline int do_walk_down(struct btrfs_trans_handle *trans,
8678 struct btrfs_root *root,
8679 struct btrfs_path *path,
8680 struct walk_control *wc, int *lookup_info)
8681 {
8682 struct btrfs_fs_info *fs_info = root->fs_info;
8683 u64 bytenr;
8684 u64 generation;
8685 u64 parent;
8686 u32 blocksize;
8687 struct btrfs_key key;
8688 struct btrfs_key first_key;
8689 struct extent_buffer *next;
8690 int level = wc->level;
8691 int reada = 0;
8692 int ret = 0;
8693 bool need_account = false;
8694
8695 generation = btrfs_node_ptr_generation(path->nodes[level],
8696 path->slots[level]);
8697 /*
8698 * if the lower level block was created before the snapshot
8699 * was created, we know there is no need to update back refs
8700 * for the subtree
8701 */
8702 if (wc->stage == UPDATE_BACKREF &&
8703 generation <= root->root_key.offset) {
8704 *lookup_info = 1;
8705 return 1;
8706 }
8707
8708 bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
8709 btrfs_node_key_to_cpu(path->nodes[level], &first_key,
8710 path->slots[level]);
8711 blocksize = fs_info->nodesize;
8712
8713 next = find_extent_buffer(fs_info, bytenr);
8714 if (!next) {
8715 next = btrfs_find_create_tree_block(fs_info, bytenr);
8716 if (IS_ERR(next))
8717 return PTR_ERR(next);
8718
8719 btrfs_set_buffer_lockdep_class(root->root_key.objectid, next,
8720 level - 1);
8721 reada = 1;
8722 }
8723 btrfs_tree_lock(next);
8724 btrfs_set_lock_blocking(next);
8725
8726 ret = btrfs_lookup_extent_info(trans, fs_info, bytenr, level - 1, 1,
8727 &wc->refs[level - 1],
8728 &wc->flags[level - 1]);
8729 if (ret < 0)
8730 goto out_unlock;
8731
8732 if (unlikely(wc->refs[level - 1] == 0)) {
8733 btrfs_err(fs_info, "Missing references.");
8734 ret = -EIO;
8735 goto out_unlock;
8736 }
8737 *lookup_info = 0;
8738
8739 if (wc->stage == DROP_REFERENCE) {
8740 if (wc->refs[level - 1] > 1) {
8741 need_account = true;
8742 if (level == 1 &&
8743 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8744 goto skip;
8745
8746 if (!wc->update_ref ||
8747 generation <= root->root_key.offset)
8748 goto skip;
8749
8750 btrfs_node_key_to_cpu(path->nodes[level], &key,
8751 path->slots[level]);
8752 ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
8753 if (ret < 0)
8754 goto skip;
8755
8756 wc->stage = UPDATE_BACKREF;
8757 wc->shared_level = level - 1;
8758 }
8759 } else {
8760 if (level == 1 &&
8761 (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
8762 goto skip;
8763 }
8764
8765 if (!btrfs_buffer_uptodate(next, generation, 0)) {
8766 btrfs_tree_unlock(next);
8767 free_extent_buffer(next);
8768 next = NULL;
8769 *lookup_info = 1;
8770 }
8771
8772 if (!next) {
8773 if (reada && level == 1)
8774 reada_walk_down(trans, root, wc, path);
8775 next = read_tree_block(fs_info, bytenr, generation, level - 1,
8776 &first_key);
8777 if (IS_ERR(next)) {
8778 return PTR_ERR(next);
8779 } else if (!extent_buffer_uptodate(next)) {
8780 free_extent_buffer(next);
8781 return -EIO;
8782 }
8783 btrfs_tree_lock(next);
8784 btrfs_set_lock_blocking(next);
8785 }
8786
8787 level--;
8788 ASSERT(level == btrfs_header_level(next));
8789 if (level != btrfs_header_level(next)) {
8790 btrfs_err(root->fs_info, "mismatched level");
8791 ret = -EIO;
8792 goto out_unlock;
8793 }
8794 path->nodes[level] = next;
8795 path->slots[level] = 0;
8796 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
8797 wc->level = level;
8798 if (wc->level == 1)
8799 wc->reada_slot = 0;
8800 return 0;
8801 skip:
8802 wc->refs[level - 1] = 0;
8803 wc->flags[level - 1] = 0;
8804 if (wc->stage == DROP_REFERENCE) {
8805 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
8806 parent = path->nodes[level]->start;
8807 } else {
8808 ASSERT(root->root_key.objectid ==
8809 btrfs_header_owner(path->nodes[level]));
8810 if (root->root_key.objectid !=
8811 btrfs_header_owner(path->nodes[level])) {
8812 btrfs_err(root->fs_info,
8813 "mismatched block owner");
8814 ret = -EIO;
8815 goto out_unlock;
8816 }
8817 parent = 0;
8818 }
8819
8820 if (need_account) {
8821 ret = btrfs_qgroup_trace_subtree(trans, root, next,
8822 generation, level - 1);
8823 if (ret) {
8824 btrfs_err_rl(fs_info,
8825 "Error %d accounting shared subtree. Quota is out of sync, rescan required.",
8826 ret);
8827 }
8828 }
8829 ret = btrfs_free_extent(trans, root, bytenr, blocksize,
8830 parent, root->root_key.objectid,
8831 level - 1, 0);
8832 if (ret)
8833 goto out_unlock;
8834 }
8835
8836 *lookup_info = 1;
8837 ret = 1;
8838
8839 out_unlock:
8840 btrfs_tree_unlock(next);
8841 free_extent_buffer(next);
8842
8843 return ret;
8844 }
8845
8846 /*
8847 * helper to process tree block while walking up the tree.
8848 *
8849 * when wc->stage == DROP_REFERENCE, this function drops
8850 * reference count on the block.
8851 *
8852 * when wc->stage == UPDATE_BACKREF, this function changes
8853 * wc->stage back to DROP_REFERENCE if we changed wc->stage
8854 * to UPDATE_BACKREF previously while processing the block.
8855 *
8856 * NOTE: return value 1 means we should stop walking up.
8857 */
8858 static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
8859 struct btrfs_root *root,
8860 struct btrfs_path *path,
8861 struct walk_control *wc)
8862 {
8863 struct btrfs_fs_info *fs_info = root->fs_info;
8864 int ret;
8865 int level = wc->level;
8866 struct extent_buffer *eb = path->nodes[level];
8867 u64 parent = 0;
8868
8869 if (wc->stage == UPDATE_BACKREF) {
8870 BUG_ON(wc->shared_level < level);
8871 if (level < wc->shared_level)
8872 goto out;
8873
8874 ret = find_next_key(path, level + 1, &wc->update_progress);
8875 if (ret > 0)
8876 wc->update_ref = 0;
8877
8878 wc->stage = DROP_REFERENCE;
8879 wc->shared_level = -1;
8880 path->slots[level] = 0;
8881
8882 /*
8883 * check reference count again if the block isn't locked.
8884 * we should start walking down the tree again if reference
8885 * count is one.
8886 */
8887 if (!path->locks[level]) {
8888 BUG_ON(level == 0);
8889 btrfs_tree_lock(eb);
8890 btrfs_set_lock_blocking(eb);
8891 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
8892
8893 ret = btrfs_lookup_extent_info(trans, fs_info,
8894 eb->start, level, 1,
8895 &wc->refs[level],
8896 &wc->flags[level]);
8897 if (ret < 0) {
8898 btrfs_tree_unlock_rw(eb, path->locks[level]);
8899 path->locks[level] = 0;
8900 return ret;
8901 }
8902 BUG_ON(wc->refs[level] == 0);
8903 if (wc->refs[level] == 1) {
8904 btrfs_tree_unlock_rw(eb, path->locks[level]);
8905 path->locks[level] = 0;
8906 return 1;
8907 }
8908 }
8909 }
8910
8911 /* wc->stage == DROP_REFERENCE */
8912 BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
8913
8914 if (wc->refs[level] == 1) {
8915 if (level == 0) {
8916 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
8917 ret = btrfs_dec_ref(trans, root, eb, 1);
8918 else
8919 ret = btrfs_dec_ref(trans, root, eb, 0);
8920 BUG_ON(ret); /* -ENOMEM */
8921 ret = btrfs_qgroup_trace_leaf_items(trans, fs_info, eb);
8922 if (ret) {
8923 btrfs_err_rl(fs_info,
8924 "error %d accounting leaf items. Quota is out of sync, rescan required.",
8925 ret);
8926 }
8927 }
8928 /* make block locked assertion in clean_tree_block happy */
8929 if (!path->locks[level] &&
8930 btrfs_header_generation(eb) == trans->transid) {
8931 btrfs_tree_lock(eb);
8932 btrfs_set_lock_blocking(eb);
8933 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
8934 }
8935 clean_tree_block(fs_info, eb);
8936 }
8937
8938 if (eb == root->node) {
8939 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
8940 parent = eb->start;
8941 else
8942 BUG_ON(root->root_key.objectid !=
8943 btrfs_header_owner(eb));
8944 } else {
8945 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
8946 parent = path->nodes[level + 1]->start;
8947 else
8948 BUG_ON(root->root_key.objectid !=
8949 btrfs_header_owner(path->nodes[level + 1]));
8950 }
8951
8952 btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
8953 out:
8954 wc->refs[level] = 0;
8955 wc->flags[level] = 0;
8956 return 0;
8957 }
8958
8959 static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
8960 struct btrfs_root *root,
8961 struct btrfs_path *path,
8962 struct walk_control *wc)
8963 {
8964 int level = wc->level;
8965 int lookup_info = 1;
8966 int ret;
8967
8968 while (level >= 0) {
8969 ret = walk_down_proc(trans, root, path, wc, lookup_info);
8970 if (ret > 0)
8971 break;
8972
8973 if (level == 0)
8974 break;
8975
8976 if (path->slots[level] >=
8977 btrfs_header_nritems(path->nodes[level]))
8978 break;
8979
8980 ret = do_walk_down(trans, root, path, wc, &lookup_info);
8981 if (ret > 0) {
8982 path->slots[level]++;
8983 continue;
8984 } else if (ret < 0)
8985 return ret;
8986 level = wc->level;
8987 }
8988 return 0;
8989 }
8990
8991 static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
8992 struct btrfs_root *root,
8993 struct btrfs_path *path,
8994 struct walk_control *wc, int max_level)
8995 {
8996 int level = wc->level;
8997 int ret;
8998
8999 path->slots[level] = btrfs_header_nritems(path->nodes[level]);
9000 while (level < max_level && path->nodes[level]) {
9001 wc->level = level;
9002 if (path->slots[level] + 1 <
9003 btrfs_header_nritems(path->nodes[level])) {
9004 path->slots[level]++;
9005 return 0;
9006 } else {
9007 ret = walk_up_proc(trans, root, path, wc);
9008 if (ret > 0)
9009 return 0;
9010
9011 if (path->locks[level]) {
9012 btrfs_tree_unlock_rw(path->nodes[level],
9013 path->locks[level]);
9014 path->locks[level] = 0;
9015 }
9016 free_extent_buffer(path->nodes[level]);
9017 path->nodes[level] = NULL;
9018 level++;
9019 }
9020 }
9021 return 1;
9022 }
9023
9024 /*
9025 * drop a subvolume tree.
9026 *
9027 * this function traverses the tree freeing any blocks that only
9028 * referenced by the tree.
9029 *
9030 * when a shared tree block is found. this function decreases its
9031 * reference count by one. if update_ref is true, this function
9032 * also make sure backrefs for the shared block and all lower level
9033 * blocks are properly updated.
9034 *
9035 * If called with for_reloc == 0, may exit early with -EAGAIN
9036 */
9037 int btrfs_drop_snapshot(struct btrfs_root *root,
9038 struct btrfs_block_rsv *block_rsv, int update_ref,
9039 int for_reloc)
9040 {
9041 struct btrfs_fs_info *fs_info = root->fs_info;
9042 struct btrfs_path *path;
9043 struct btrfs_trans_handle *trans;
9044 struct btrfs_root *tree_root = fs_info->tree_root;
9045 struct btrfs_root_item *root_item = &root->root_item;
9046 struct walk_control *wc;
9047 struct btrfs_key key;
9048 int err = 0;
9049 int ret;
9050 int level;
9051 bool root_dropped = false;
9052
9053 btrfs_debug(fs_info, "Drop subvolume %llu", root->objectid);
9054
9055 path = btrfs_alloc_path();
9056 if (!path) {
9057 err = -ENOMEM;
9058 goto out;
9059 }
9060
9061 wc = kzalloc(sizeof(*wc), GFP_NOFS);
9062 if (!wc) {
9063 btrfs_free_path(path);
9064 err = -ENOMEM;
9065 goto out;
9066 }
9067
9068 trans = btrfs_start_transaction(tree_root, 0);
9069 if (IS_ERR(trans)) {
9070 err = PTR_ERR(trans);
9071 goto out_free;
9072 }
9073
9074 if (block_rsv)
9075 trans->block_rsv = block_rsv;
9076
9077 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
9078 level = btrfs_header_level(root->node);
9079 path->nodes[level] = btrfs_lock_root_node(root);
9080 btrfs_set_lock_blocking(path->nodes[level]);
9081 path->slots[level] = 0;
9082 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9083 memset(&wc->update_progress, 0,
9084 sizeof(wc->update_progress));
9085 } else {
9086 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
9087 memcpy(&wc->update_progress, &key,
9088 sizeof(wc->update_progress));
9089
9090 level = root_item->drop_level;
9091 BUG_ON(level == 0);
9092 path->lowest_level = level;
9093 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
9094 path->lowest_level = 0;
9095 if (ret < 0) {
9096 err = ret;
9097 goto out_end_trans;
9098 }
9099 WARN_ON(ret > 0);
9100
9101 /*
9102 * unlock our path, this is safe because only this
9103 * function is allowed to delete this snapshot
9104 */
9105 btrfs_unlock_up_safe(path, 0);
9106
9107 level = btrfs_header_level(root->node);
9108 while (1) {
9109 btrfs_tree_lock(path->nodes[level]);
9110 btrfs_set_lock_blocking(path->nodes[level]);
9111 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9112
9113 ret = btrfs_lookup_extent_info(trans, fs_info,
9114 path->nodes[level]->start,
9115 level, 1, &wc->refs[level],
9116 &wc->flags[level]);
9117 if (ret < 0) {
9118 err = ret;
9119 goto out_end_trans;
9120 }
9121 BUG_ON(wc->refs[level] == 0);
9122
9123 if (level == root_item->drop_level)
9124 break;
9125
9126 btrfs_tree_unlock(path->nodes[level]);
9127 path->locks[level] = 0;
9128 WARN_ON(wc->refs[level] != 1);
9129 level--;
9130 }
9131 }
9132
9133 wc->level = level;
9134 wc->shared_level = -1;
9135 wc->stage = DROP_REFERENCE;
9136 wc->update_ref = update_ref;
9137 wc->keep_locks = 0;
9138 wc->for_reloc = for_reloc;
9139 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info);
9140
9141 while (1) {
9142
9143 ret = walk_down_tree(trans, root, path, wc);
9144 if (ret < 0) {
9145 err = ret;
9146 break;
9147 }
9148
9149 ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
9150 if (ret < 0) {
9151 err = ret;
9152 break;
9153 }
9154
9155 if (ret > 0) {
9156 BUG_ON(wc->stage != DROP_REFERENCE);
9157 break;
9158 }
9159
9160 if (wc->stage == DROP_REFERENCE) {
9161 level = wc->level;
9162 btrfs_node_key(path->nodes[level],
9163 &root_item->drop_progress,
9164 path->slots[level]);
9165 root_item->drop_level = level;
9166 }
9167
9168 BUG_ON(wc->level == 0);
9169 if (btrfs_should_end_transaction(trans) ||
9170 (!for_reloc && btrfs_need_cleaner_sleep(fs_info))) {
9171 ret = btrfs_update_root(trans, tree_root,
9172 &root->root_key,
9173 root_item);
9174 if (ret) {
9175 btrfs_abort_transaction(trans, ret);
9176 err = ret;
9177 goto out_end_trans;
9178 }
9179
9180 btrfs_end_transaction_throttle(trans);
9181 if (!for_reloc && btrfs_need_cleaner_sleep(fs_info)) {
9182 btrfs_debug(fs_info,
9183 "drop snapshot early exit");
9184 err = -EAGAIN;
9185 goto out_free;
9186 }
9187
9188 trans = btrfs_start_transaction(tree_root, 0);
9189 if (IS_ERR(trans)) {
9190 err = PTR_ERR(trans);
9191 goto out_free;
9192 }
9193 if (block_rsv)
9194 trans->block_rsv = block_rsv;
9195 }
9196 }
9197 btrfs_release_path(path);
9198 if (err)
9199 goto out_end_trans;
9200
9201 ret = btrfs_del_root(trans, fs_info, &root->root_key);
9202 if (ret) {
9203 btrfs_abort_transaction(trans, ret);
9204 err = ret;
9205 goto out_end_trans;
9206 }
9207
9208 if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
9209 ret = btrfs_find_root(tree_root, &root->root_key, path,
9210 NULL, NULL);
9211 if (ret < 0) {
9212 btrfs_abort_transaction(trans, ret);
9213 err = ret;
9214 goto out_end_trans;
9215 } else if (ret > 0) {
9216 /* if we fail to delete the orphan item this time
9217 * around, it'll get picked up the next time.
9218 *
9219 * The most common failure here is just -ENOENT.
9220 */
9221 btrfs_del_orphan_item(trans, tree_root,
9222 root->root_key.objectid);
9223 }
9224 }
9225
9226 if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) {
9227 btrfs_add_dropped_root(trans, root);
9228 } else {
9229 free_extent_buffer(root->node);
9230 free_extent_buffer(root->commit_root);
9231 btrfs_put_fs_root(root);
9232 }
9233 root_dropped = true;
9234 out_end_trans:
9235 btrfs_end_transaction_throttle(trans);
9236 out_free:
9237 kfree(wc);
9238 btrfs_free_path(path);
9239 out:
9240 /*
9241 * So if we need to stop dropping the snapshot for whatever reason we
9242 * need to make sure to add it back to the dead root list so that we
9243 * keep trying to do the work later. This also cleans up roots if we
9244 * don't have it in the radix (like when we recover after a power fail
9245 * or unmount) so we don't leak memory.
9246 */
9247 if (!for_reloc && !root_dropped)
9248 btrfs_add_dead_root(root);
9249 if (err && err != -EAGAIN)
9250 btrfs_handle_fs_error(fs_info, err, NULL);
9251 return err;
9252 }
9253
9254 /*
9255 * drop subtree rooted at tree block 'node'.
9256 *
9257 * NOTE: this function will unlock and release tree block 'node'
9258 * only used by relocation code
9259 */
9260 int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
9261 struct btrfs_root *root,
9262 struct extent_buffer *node,
9263 struct extent_buffer *parent)
9264 {
9265 struct btrfs_fs_info *fs_info = root->fs_info;
9266 struct btrfs_path *path;
9267 struct walk_control *wc;
9268 int level;
9269 int parent_level;
9270 int ret = 0;
9271 int wret;
9272
9273 BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
9274
9275 path = btrfs_alloc_path();
9276 if (!path)
9277 return -ENOMEM;
9278
9279 wc = kzalloc(sizeof(*wc), GFP_NOFS);
9280 if (!wc) {
9281 btrfs_free_path(path);
9282 return -ENOMEM;
9283 }
9284
9285 btrfs_assert_tree_locked(parent);
9286 parent_level = btrfs_header_level(parent);
9287 extent_buffer_get(parent);
9288 path->nodes[parent_level] = parent;
9289 path->slots[parent_level] = btrfs_header_nritems(parent);
9290
9291 btrfs_assert_tree_locked(node);
9292 level = btrfs_header_level(node);
9293 path->nodes[level] = node;
9294 path->slots[level] = 0;
9295 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
9296
9297 wc->refs[parent_level] = 1;
9298 wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
9299 wc->level = level;
9300 wc->shared_level = -1;
9301 wc->stage = DROP_REFERENCE;
9302 wc->update_ref = 0;
9303 wc->keep_locks = 1;
9304 wc->for_reloc = 1;
9305 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info);
9306
9307 while (1) {
9308 wret = walk_down_tree(trans, root, path, wc);
9309 if (wret < 0) {
9310 ret = wret;
9311 break;
9312 }
9313
9314 wret = walk_up_tree(trans, root, path, wc, parent_level);
9315 if (wret < 0)
9316 ret = wret;
9317 if (wret != 0)
9318 break;
9319 }
9320
9321 kfree(wc);
9322 btrfs_free_path(path);
9323 return ret;
9324 }
9325
9326 static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags)
9327 {
9328 u64 num_devices;
9329 u64 stripped;
9330
9331 /*
9332 * if restripe for this chunk_type is on pick target profile and
9333 * return, otherwise do the usual balance
9334 */
9335 stripped = get_restripe_target(fs_info, flags);
9336 if (stripped)
9337 return extended_to_chunk(stripped);
9338
9339 num_devices = fs_info->fs_devices->rw_devices;
9340
9341 stripped = BTRFS_BLOCK_GROUP_RAID0 |
9342 BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 |
9343 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;
9344
9345 if (num_devices == 1) {
9346 stripped |= BTRFS_BLOCK_GROUP_DUP;
9347 stripped = flags & ~stripped;
9348
9349 /* turn raid0 into single device chunks */
9350 if (flags & BTRFS_BLOCK_GROUP_RAID0)
9351 return stripped;
9352
9353 /* turn mirroring into duplication */
9354 if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
9355 BTRFS_BLOCK_GROUP_RAID10))
9356 return stripped | BTRFS_BLOCK_GROUP_DUP;
9357 } else {
9358 /* they already had raid on here, just return */
9359 if (flags & stripped)
9360 return flags;
9361
9362 stripped |= BTRFS_BLOCK_GROUP_DUP;
9363 stripped = flags & ~stripped;
9364
9365 /* switch duplicated blocks with raid1 */
9366 if (flags & BTRFS_BLOCK_GROUP_DUP)
9367 return stripped | BTRFS_BLOCK_GROUP_RAID1;
9368
9369 /* this is drive concat, leave it alone */
9370 }
9371
9372 return flags;
9373 }
9374
9375 static int inc_block_group_ro(struct btrfs_block_group_cache *cache, int force)
9376 {
9377 struct btrfs_space_info *sinfo = cache->space_info;
9378 u64 num_bytes;
9379 u64 min_allocable_bytes;
9380 int ret = -ENOSPC;
9381
9382 /*
9383 * We need some metadata space and system metadata space for
9384 * allocating chunks in some corner cases until we force to set
9385 * it to be readonly.
9386 */
9387 if ((sinfo->flags &
9388 (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
9389 !force)
9390 min_allocable_bytes = SZ_1M;
9391 else
9392 min_allocable_bytes = 0;
9393
9394 spin_lock(&sinfo->lock);
9395 spin_lock(&cache->lock);
9396
9397 if (cache->ro) {
9398 cache->ro++;
9399 ret = 0;
9400 goto out;
9401 }
9402
9403 num_bytes = cache->key.offset - cache->reserved - cache->pinned -
9404 cache->bytes_super - btrfs_block_group_used(&cache->item);
9405
9406 if (btrfs_space_info_used(sinfo, true) + num_bytes +
9407 min_allocable_bytes <= sinfo->total_bytes) {
9408 sinfo->bytes_readonly += num_bytes;
9409 cache->ro++;
9410 list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
9411 ret = 0;
9412 }
9413 out:
9414 spin_unlock(&cache->lock);
9415 spin_unlock(&sinfo->lock);
9416 return ret;
9417 }
9418
9419 int btrfs_inc_block_group_ro(struct btrfs_fs_info *fs_info,
9420 struct btrfs_block_group_cache *cache)
9421
9422 {
9423 struct btrfs_trans_handle *trans;
9424 u64 alloc_flags;
9425 int ret;
9426
9427 again:
9428 trans = btrfs_join_transaction(fs_info->extent_root);
9429 if (IS_ERR(trans))
9430 return PTR_ERR(trans);
9431
9432 /*
9433 * we're not allowed to set block groups readonly after the dirty
9434 * block groups cache has started writing. If it already started,
9435 * back off and let this transaction commit
9436 */
9437 mutex_lock(&fs_info->ro_block_group_mutex);
9438 if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) {
9439 u64 transid = trans->transid;
9440
9441 mutex_unlock(&fs_info->ro_block_group_mutex);
9442 btrfs_end_transaction(trans);
9443
9444 ret = btrfs_wait_for_commit(fs_info, transid);
9445 if (ret)
9446 return ret;
9447 goto again;
9448 }
9449
9450 /*
9451 * if we are changing raid levels, try to allocate a corresponding
9452 * block group with the new raid level.
9453 */
9454 alloc_flags = update_block_group_flags(fs_info, cache->flags);
9455 if (alloc_flags != cache->flags) {
9456 ret = do_chunk_alloc(trans, fs_info, alloc_flags,
9457 CHUNK_ALLOC_FORCE);
9458 /*
9459 * ENOSPC is allowed here, we may have enough space
9460 * already allocated at the new raid level to
9461 * carry on
9462 */
9463 if (ret == -ENOSPC)
9464 ret = 0;
9465 if (ret < 0)
9466 goto out;
9467 }
9468
9469 ret = inc_block_group_ro(cache, 0);
9470 if (!ret)
9471 goto out;
9472 alloc_flags = get_alloc_profile(fs_info, cache->space_info->flags);
9473 ret = do_chunk_alloc(trans, fs_info, alloc_flags,
9474 CHUNK_ALLOC_FORCE);
9475 if (ret < 0)
9476 goto out;
9477 ret = inc_block_group_ro(cache, 0);
9478 out:
9479 if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) {
9480 alloc_flags = update_block_group_flags(fs_info, cache->flags);
9481 mutex_lock(&fs_info->chunk_mutex);
9482 check_system_chunk(trans, fs_info, alloc_flags);
9483 mutex_unlock(&fs_info->chunk_mutex);
9484 }
9485 mutex_unlock(&fs_info->ro_block_group_mutex);
9486
9487 btrfs_end_transaction(trans);
9488 return ret;
9489 }
9490
9491 int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
9492 struct btrfs_fs_info *fs_info, u64 type)
9493 {
9494 u64 alloc_flags = get_alloc_profile(fs_info, type);
9495
9496 return do_chunk_alloc(trans, fs_info, alloc_flags, CHUNK_ALLOC_FORCE);
9497 }
9498
9499 /*
9500 * helper to account the unused space of all the readonly block group in the
9501 * space_info. takes mirrors into account.
9502 */
9503 u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
9504 {
9505 struct btrfs_block_group_cache *block_group;
9506 u64 free_bytes = 0;
9507 int factor;
9508
9509 /* It's df, we don't care if it's racy */
9510 if (list_empty(&sinfo->ro_bgs))
9511 return 0;
9512
9513 spin_lock(&sinfo->lock);
9514 list_for_each_entry(block_group, &sinfo->ro_bgs, ro_list) {
9515 spin_lock(&block_group->lock);
9516
9517 if (!block_group->ro) {
9518 spin_unlock(&block_group->lock);
9519 continue;
9520 }
9521
9522 if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
9523 BTRFS_BLOCK_GROUP_RAID10 |
9524 BTRFS_BLOCK_GROUP_DUP))
9525 factor = 2;
9526 else
9527 factor = 1;
9528
9529 free_bytes += (block_group->key.offset -
9530 btrfs_block_group_used(&block_group->item)) *
9531 factor;
9532
9533 spin_unlock(&block_group->lock);
9534 }
9535 spin_unlock(&sinfo->lock);
9536
9537 return free_bytes;
9538 }
9539
9540 void btrfs_dec_block_group_ro(struct btrfs_block_group_cache *cache)
9541 {
9542 struct btrfs_space_info *sinfo = cache->space_info;
9543 u64 num_bytes;
9544
9545 BUG_ON(!cache->ro);
9546
9547 spin_lock(&sinfo->lock);
9548 spin_lock(&cache->lock);
9549 if (!--cache->ro) {
9550 num_bytes = cache->key.offset - cache->reserved -
9551 cache->pinned - cache->bytes_super -
9552 btrfs_block_group_used(&cache->item);
9553 sinfo->bytes_readonly -= num_bytes;
9554 list_del_init(&cache->ro_list);
9555 }
9556 spin_unlock(&cache->lock);
9557 spin_unlock(&sinfo->lock);
9558 }
9559
9560 /*
9561 * checks to see if its even possible to relocate this block group.
9562 *
9563 * @return - -1 if it's not a good idea to relocate this block group, 0 if its
9564 * ok to go ahead and try.
9565 */
9566 int btrfs_can_relocate(struct btrfs_fs_info *fs_info, u64 bytenr)
9567 {
9568 struct btrfs_root *root = fs_info->extent_root;
9569 struct btrfs_block_group_cache *block_group;
9570 struct btrfs_space_info *space_info;
9571 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
9572 struct btrfs_device *device;
9573 struct btrfs_trans_handle *trans;
9574 u64 min_free;
9575 u64 dev_min = 1;
9576 u64 dev_nr = 0;
9577 u64 target;
9578 int debug;
9579 int index;
9580 int full = 0;
9581 int ret = 0;
9582
9583 debug = btrfs_test_opt(fs_info, ENOSPC_DEBUG);
9584
9585 block_group = btrfs_lookup_block_group(fs_info, bytenr);
9586
9587 /* odd, couldn't find the block group, leave it alone */
9588 if (!block_group) {
9589 if (debug)
9590 btrfs_warn(fs_info,
9591 "can't find block group for bytenr %llu",
9592 bytenr);
9593 return -1;
9594 }
9595
9596 min_free = btrfs_block_group_used(&block_group->item);
9597
9598 /* no bytes used, we're good */
9599 if (!min_free)
9600 goto out;
9601
9602 space_info = block_group->space_info;
9603 spin_lock(&space_info->lock);
9604
9605 full = space_info->full;
9606
9607 /*
9608 * if this is the last block group we have in this space, we can't
9609 * relocate it unless we're able to allocate a new chunk below.
9610 *
9611 * Otherwise, we need to make sure we have room in the space to handle
9612 * all of the extents from this block group. If we can, we're good
9613 */
9614 if ((space_info->total_bytes != block_group->key.offset) &&
9615 (btrfs_space_info_used(space_info, false) + min_free <
9616 space_info->total_bytes)) {
9617 spin_unlock(&space_info->lock);
9618 goto out;
9619 }
9620 spin_unlock(&space_info->lock);
9621
9622 /*
9623 * ok we don't have enough space, but maybe we have free space on our
9624 * devices to allocate new chunks for relocation, so loop through our
9625 * alloc devices and guess if we have enough space. if this block
9626 * group is going to be restriped, run checks against the target
9627 * profile instead of the current one.
9628 */
9629 ret = -1;
9630
9631 /*
9632 * index:
9633 * 0: raid10
9634 * 1: raid1
9635 * 2: dup
9636 * 3: raid0
9637 * 4: single
9638 */
9639 target = get_restripe_target(fs_info, block_group->flags);
9640 if (target) {
9641 index = btrfs_bg_flags_to_raid_index(extended_to_chunk(target));
9642 } else {
9643 /*
9644 * this is just a balance, so if we were marked as full
9645 * we know there is no space for a new chunk
9646 */
9647 if (full) {
9648 if (debug)
9649 btrfs_warn(fs_info,
9650 "no space to alloc new chunk for block group %llu",
9651 block_group->key.objectid);
9652 goto out;
9653 }
9654
9655 index = btrfs_bg_flags_to_raid_index(block_group->flags);
9656 }
9657
9658 if (index == BTRFS_RAID_RAID10) {
9659 dev_min = 4;
9660 /* Divide by 2 */
9661 min_free >>= 1;
9662 } else if (index == BTRFS_RAID_RAID1) {
9663 dev_min = 2;
9664 } else if (index == BTRFS_RAID_DUP) {
9665 /* Multiply by 2 */
9666 min_free <<= 1;
9667 } else if (index == BTRFS_RAID_RAID0) {
9668 dev_min = fs_devices->rw_devices;
9669 min_free = div64_u64(min_free, dev_min);
9670 }
9671
9672 /* We need to do this so that we can look at pending chunks */
9673 trans = btrfs_join_transaction(root);
9674 if (IS_ERR(trans)) {
9675 ret = PTR_ERR(trans);
9676 goto out;
9677 }
9678
9679 mutex_lock(&fs_info->chunk_mutex);
9680 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
9681 u64 dev_offset;
9682
9683 /*
9684 * check to make sure we can actually find a chunk with enough
9685 * space to fit our block group in.
9686 */
9687 if (device->total_bytes > device->bytes_used + min_free &&
9688 !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
9689 ret = find_free_dev_extent(trans, device, min_free,
9690 &dev_offset, NULL);
9691 if (!ret)
9692 dev_nr++;
9693
9694 if (dev_nr >= dev_min)
9695 break;
9696
9697 ret = -1;
9698 }
9699 }
9700 if (debug && ret == -1)
9701 btrfs_warn(fs_info,
9702 "no space to allocate a new chunk for block group %llu",
9703 block_group->key.objectid);
9704 mutex_unlock(&fs_info->chunk_mutex);
9705 btrfs_end_transaction(trans);
9706 out:
9707 btrfs_put_block_group(block_group);
9708 return ret;
9709 }
9710
9711 static int find_first_block_group(struct btrfs_fs_info *fs_info,
9712 struct btrfs_path *path,
9713 struct btrfs_key *key)
9714 {
9715 struct btrfs_root *root = fs_info->extent_root;
9716 int ret = 0;
9717 struct btrfs_key found_key;
9718 struct extent_buffer *leaf;
9719 int slot;
9720
9721 ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
9722 if (ret < 0)
9723 goto out;
9724
9725 while (1) {
9726 slot = path->slots[0];
9727 leaf = path->nodes[0];
9728 if (slot >= btrfs_header_nritems(leaf)) {
9729 ret = btrfs_next_leaf(root, path);
9730 if (ret == 0)
9731 continue;
9732 if (ret < 0)
9733 goto out;
9734 break;
9735 }
9736 btrfs_item_key_to_cpu(leaf, &found_key, slot);
9737
9738 if (found_key.objectid >= key->objectid &&
9739 found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
9740 struct extent_map_tree *em_tree;
9741 struct extent_map *em;
9742
9743 em_tree = &root->fs_info->mapping_tree.map_tree;
9744 read_lock(&em_tree->lock);
9745 em = lookup_extent_mapping(em_tree, found_key.objectid,
9746 found_key.offset);
9747 read_unlock(&em_tree->lock);
9748 if (!em) {
9749 btrfs_err(fs_info,
9750 "logical %llu len %llu found bg but no related chunk",
9751 found_key.objectid, found_key.offset);
9752 ret = -ENOENT;
9753 } else {
9754 ret = 0;
9755 }
9756 free_extent_map(em);
9757 goto out;
9758 }
9759 path->slots[0]++;
9760 }
9761 out:
9762 return ret;
9763 }
9764
9765 void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
9766 {
9767 struct btrfs_block_group_cache *block_group;
9768 u64 last = 0;
9769
9770 while (1) {
9771 struct inode *inode;
9772
9773 block_group = btrfs_lookup_first_block_group(info, last);
9774 while (block_group) {
9775 spin_lock(&block_group->lock);
9776 if (block_group->iref)
9777 break;
9778 spin_unlock(&block_group->lock);
9779 block_group = next_block_group(info, block_group);
9780 }
9781 if (!block_group) {
9782 if (last == 0)
9783 break;
9784 last = 0;
9785 continue;
9786 }
9787
9788 inode = block_group->inode;
9789 block_group->iref = 0;
9790 block_group->inode = NULL;
9791 spin_unlock(&block_group->lock);
9792 ASSERT(block_group->io_ctl.inode == NULL);
9793 iput(inode);
9794 last = block_group->key.objectid + block_group->key.offset;
9795 btrfs_put_block_group(block_group);
9796 }
9797 }
9798
9799 /*
9800 * Must be called only after stopping all workers, since we could have block
9801 * group caching kthreads running, and therefore they could race with us if we
9802 * freed the block groups before stopping them.
9803 */
9804 int btrfs_free_block_groups(struct btrfs_fs_info *info)
9805 {
9806 struct btrfs_block_group_cache *block_group;
9807 struct btrfs_space_info *space_info;
9808 struct btrfs_caching_control *caching_ctl;
9809 struct rb_node *n;
9810
9811 down_write(&info->commit_root_sem);
9812 while (!list_empty(&info->caching_block_groups)) {
9813 caching_ctl = list_entry(info->caching_block_groups.next,
9814 struct btrfs_caching_control, list);
9815 list_del(&caching_ctl->list);
9816 put_caching_control(caching_ctl);
9817 }
9818 up_write(&info->commit_root_sem);
9819
9820 spin_lock(&info->unused_bgs_lock);
9821 while (!list_empty(&info->unused_bgs)) {
9822 block_group = list_first_entry(&info->unused_bgs,
9823 struct btrfs_block_group_cache,
9824 bg_list);
9825 list_del_init(&block_group->bg_list);
9826 btrfs_put_block_group(block_group);
9827 }
9828 spin_unlock(&info->unused_bgs_lock);
9829
9830 spin_lock(&info->block_group_cache_lock);
9831 while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
9832 block_group = rb_entry(n, struct btrfs_block_group_cache,
9833 cache_node);
9834 rb_erase(&block_group->cache_node,
9835 &info->block_group_cache_tree);
9836 RB_CLEAR_NODE(&block_group->cache_node);
9837 spin_unlock(&info->block_group_cache_lock);
9838
9839 down_write(&block_group->space_info->groups_sem);
9840 list_del(&block_group->list);
9841 up_write(&block_group->space_info->groups_sem);
9842
9843 /*
9844 * We haven't cached this block group, which means we could
9845 * possibly have excluded extents on this block group.
9846 */
9847 if (block_group->cached == BTRFS_CACHE_NO ||
9848 block_group->cached == BTRFS_CACHE_ERROR)
9849 free_excluded_extents(info, block_group);
9850
9851 btrfs_remove_free_space_cache(block_group);
9852 ASSERT(block_group->cached != BTRFS_CACHE_STARTED);
9853 ASSERT(list_empty(&block_group->dirty_list));
9854 ASSERT(list_empty(&block_group->io_list));
9855 ASSERT(list_empty(&block_group->bg_list));
9856 ASSERT(atomic_read(&block_group->count) == 1);
9857 btrfs_put_block_group(block_group);
9858
9859 spin_lock(&info->block_group_cache_lock);
9860 }
9861 spin_unlock(&info->block_group_cache_lock);
9862
9863 /* now that all the block groups are freed, go through and
9864 * free all the space_info structs. This is only called during
9865 * the final stages of unmount, and so we know nobody is
9866 * using them. We call synchronize_rcu() once before we start,
9867 * just to be on the safe side.
9868 */
9869 synchronize_rcu();
9870
9871 release_global_block_rsv(info);
9872
9873 while (!list_empty(&info->space_info)) {
9874 int i;
9875
9876 space_info = list_entry(info->space_info.next,
9877 struct btrfs_space_info,
9878 list);
9879
9880 /*
9881 * Do not hide this behind enospc_debug, this is actually
9882 * important and indicates a real bug if this happens.
9883 */
9884 if (WARN_ON(space_info->bytes_pinned > 0 ||
9885 space_info->bytes_reserved > 0 ||
9886 space_info->bytes_may_use > 0))
9887 dump_space_info(info, space_info, 0, 0);
9888 list_del(&space_info->list);
9889 for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
9890 struct kobject *kobj;
9891 kobj = space_info->block_group_kobjs[i];
9892 space_info->block_group_kobjs[i] = NULL;
9893 if (kobj) {
9894 kobject_del(kobj);
9895 kobject_put(kobj);
9896 }
9897 }
9898 kobject_del(&space_info->kobj);
9899 kobject_put(&space_info->kobj);
9900 }
9901 return 0;
9902 }
9903
9904 /* link_block_group will queue up kobjects to add when we're reclaim-safe */
9905 void btrfs_add_raid_kobjects(struct btrfs_fs_info *fs_info)
9906 {
9907 struct btrfs_space_info *space_info;
9908 struct raid_kobject *rkobj;
9909 LIST_HEAD(list);
9910 int index;
9911 int ret = 0;
9912
9913 spin_lock(&fs_info->pending_raid_kobjs_lock);
9914 list_splice_init(&fs_info->pending_raid_kobjs, &list);
9915 spin_unlock(&fs_info->pending_raid_kobjs_lock);
9916
9917 list_for_each_entry(rkobj, &list, list) {
9918 space_info = __find_space_info(fs_info, rkobj->flags);
9919 index = btrfs_bg_flags_to_raid_index(rkobj->flags);
9920
9921 ret = kobject_add(&rkobj->kobj, &space_info->kobj,
9922 "%s", get_raid_name(index));
9923 if (ret) {
9924 kobject_put(&rkobj->kobj);
9925 break;
9926 }
9927 }
9928 if (ret)
9929 btrfs_warn(fs_info,
9930 "failed to add kobject for block cache, ignoring");
9931 }
9932
9933 static void link_block_group(struct btrfs_block_group_cache *cache)
9934 {
9935 struct btrfs_space_info *space_info = cache->space_info;
9936 struct btrfs_fs_info *fs_info = cache->fs_info;
9937 int index = btrfs_bg_flags_to_raid_index(cache->flags);
9938 bool first = false;
9939
9940 down_write(&space_info->groups_sem);
9941 if (list_empty(&space_info->block_groups[index]))
9942 first = true;
9943 list_add_tail(&cache->list, &space_info->block_groups[index]);
9944 up_write(&space_info->groups_sem);
9945
9946 if (first) {
9947 struct raid_kobject *rkobj = kzalloc(sizeof(*rkobj), GFP_NOFS);
9948 if (!rkobj) {
9949 btrfs_warn(cache->fs_info,
9950 "couldn't alloc memory for raid level kobject");
9951 return;
9952 }
9953 rkobj->flags = cache->flags;
9954 kobject_init(&rkobj->kobj, &btrfs_raid_ktype);
9955
9956 spin_lock(&fs_info->pending_raid_kobjs_lock);
9957 list_add_tail(&rkobj->list, &fs_info->pending_raid_kobjs);
9958 spin_unlock(&fs_info->pending_raid_kobjs_lock);
9959 space_info->block_group_kobjs[index] = &rkobj->kobj;
9960 }
9961 }
9962
9963 static struct btrfs_block_group_cache *
9964 btrfs_create_block_group_cache(struct btrfs_fs_info *fs_info,
9965 u64 start, u64 size)
9966 {
9967 struct btrfs_block_group_cache *cache;
9968
9969 cache = kzalloc(sizeof(*cache), GFP_NOFS);
9970 if (!cache)
9971 return NULL;
9972
9973 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
9974 GFP_NOFS);
9975 if (!cache->free_space_ctl) {
9976 kfree(cache);
9977 return NULL;
9978 }
9979
9980 cache->key.objectid = start;
9981 cache->key.offset = size;
9982 cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
9983
9984 cache->fs_info = fs_info;
9985 cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
9986 set_free_space_tree_thresholds(cache);
9987
9988 atomic_set(&cache->count, 1);
9989 spin_lock_init(&cache->lock);
9990 init_rwsem(&cache->data_rwsem);
9991 INIT_LIST_HEAD(&cache->list);
9992 INIT_LIST_HEAD(&cache->cluster_list);
9993 INIT_LIST_HEAD(&cache->bg_list);
9994 INIT_LIST_HEAD(&cache->ro_list);
9995 INIT_LIST_HEAD(&cache->dirty_list);
9996 INIT_LIST_HEAD(&cache->io_list);
9997 btrfs_init_free_space_ctl(cache);
9998 atomic_set(&cache->trimming, 0);
9999 mutex_init(&cache->free_space_lock);
10000 btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root);
10001
10002 return cache;
10003 }
10004
10005 int btrfs_read_block_groups(struct btrfs_fs_info *info)
10006 {
10007 struct btrfs_path *path;
10008 int ret;
10009 struct btrfs_block_group_cache *cache;
10010 struct btrfs_space_info *space_info;
10011 struct btrfs_key key;
10012 struct btrfs_key found_key;
10013 struct extent_buffer *leaf;
10014 int need_clear = 0;
10015 u64 cache_gen;
10016 u64 feature;
10017 int mixed;
10018
10019 feature = btrfs_super_incompat_flags(info->super_copy);
10020 mixed = !!(feature & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS);
10021
10022 key.objectid = 0;
10023 key.offset = 0;
10024 key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
10025 path = btrfs_alloc_path();
10026 if (!path)
10027 return -ENOMEM;
10028 path->reada = READA_FORWARD;
10029
10030 cache_gen = btrfs_super_cache_generation(info->super_copy);
10031 if (btrfs_test_opt(info, SPACE_CACHE) &&
10032 btrfs_super_generation(info->super_copy) != cache_gen)
10033 need_clear = 1;
10034 if (btrfs_test_opt(info, CLEAR_CACHE))
10035 need_clear = 1;
10036
10037 while (1) {
10038 ret = find_first_block_group(info, path, &key);
10039 if (ret > 0)
10040 break;
10041 if (ret != 0)
10042 goto error;
10043
10044 leaf = path->nodes[0];
10045 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
10046
10047 cache = btrfs_create_block_group_cache(info, found_key.objectid,
10048 found_key.offset);
10049 if (!cache) {
10050 ret = -ENOMEM;
10051 goto error;
10052 }
10053
10054 if (need_clear) {
10055 /*
10056 * When we mount with old space cache, we need to
10057 * set BTRFS_DC_CLEAR and set dirty flag.
10058 *
10059 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
10060 * truncate the old free space cache inode and
10061 * setup a new one.
10062 * b) Setting 'dirty flag' makes sure that we flush
10063 * the new space cache info onto disk.
10064 */
10065 if (btrfs_test_opt(info, SPACE_CACHE))
10066 cache->disk_cache_state = BTRFS_DC_CLEAR;
10067 }
10068
10069 read_extent_buffer(leaf, &cache->item,
10070 btrfs_item_ptr_offset(leaf, path->slots[0]),
10071 sizeof(cache->item));
10072 cache->flags = btrfs_block_group_flags(&cache->item);
10073 if (!mixed &&
10074 ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) &&
10075 (cache->flags & BTRFS_BLOCK_GROUP_DATA))) {
10076 btrfs_err(info,
10077 "bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups",
10078 cache->key.objectid);
10079 ret = -EINVAL;
10080 goto error;
10081 }
10082
10083 key.objectid = found_key.objectid + found_key.offset;
10084 btrfs_release_path(path);
10085
10086 /*
10087 * We need to exclude the super stripes now so that the space
10088 * info has super bytes accounted for, otherwise we'll think
10089 * we have more space than we actually do.
10090 */
10091 ret = exclude_super_stripes(info, cache);
10092 if (ret) {
10093 /*
10094 * We may have excluded something, so call this just in
10095 * case.
10096 */
10097 free_excluded_extents(info, cache);
10098 btrfs_put_block_group(cache);
10099 goto error;
10100 }
10101
10102 /*
10103 * check for two cases, either we are full, and therefore
10104 * don't need to bother with the caching work since we won't
10105 * find any space, or we are empty, and we can just add all
10106 * the space in and be done with it. This saves us _alot_ of
10107 * time, particularly in the full case.
10108 */
10109 if (found_key.offset == btrfs_block_group_used(&cache->item)) {
10110 cache->last_byte_to_unpin = (u64)-1;
10111 cache->cached = BTRFS_CACHE_FINISHED;
10112 free_excluded_extents(info, cache);
10113 } else if (btrfs_block_group_used(&cache->item) == 0) {
10114 cache->last_byte_to_unpin = (u64)-1;
10115 cache->cached = BTRFS_CACHE_FINISHED;
10116 add_new_free_space(cache, found_key.objectid,
10117 found_key.objectid +
10118 found_key.offset);
10119 free_excluded_extents(info, cache);
10120 }
10121
10122 ret = btrfs_add_block_group_cache(info, cache);
10123 if (ret) {
10124 btrfs_remove_free_space_cache(cache);
10125 btrfs_put_block_group(cache);
10126 goto error;
10127 }
10128
10129 trace_btrfs_add_block_group(info, cache, 0);
10130 update_space_info(info, cache->flags, found_key.offset,
10131 btrfs_block_group_used(&cache->item),
10132 cache->bytes_super, &space_info);
10133
10134 cache->space_info = space_info;
10135
10136 link_block_group(cache);
10137
10138 set_avail_alloc_bits(info, cache->flags);
10139 if (btrfs_chunk_readonly(info, cache->key.objectid)) {
10140 inc_block_group_ro(cache, 1);
10141 } else if (btrfs_block_group_used(&cache->item) == 0) {
10142 spin_lock(&info->unused_bgs_lock);
10143 /* Should always be true but just in case. */
10144 if (list_empty(&cache->bg_list)) {
10145 btrfs_get_block_group(cache);
10146 trace_btrfs_add_unused_block_group(cache);
10147 list_add_tail(&cache->bg_list,
10148 &info->unused_bgs);
10149 }
10150 spin_unlock(&info->unused_bgs_lock);
10151 }
10152 }
10153
10154 list_for_each_entry_rcu(space_info, &info->space_info, list) {
10155 if (!(get_alloc_profile(info, space_info->flags) &
10156 (BTRFS_BLOCK_GROUP_RAID10 |
10157 BTRFS_BLOCK_GROUP_RAID1 |
10158 BTRFS_BLOCK_GROUP_RAID5 |
10159 BTRFS_BLOCK_GROUP_RAID6 |
10160 BTRFS_BLOCK_GROUP_DUP)))
10161 continue;
10162 /*
10163 * avoid allocating from un-mirrored block group if there are
10164 * mirrored block groups.
10165 */
10166 list_for_each_entry(cache,
10167 &space_info->block_groups[BTRFS_RAID_RAID0],
10168 list)
10169 inc_block_group_ro(cache, 1);
10170 list_for_each_entry(cache,
10171 &space_info->block_groups[BTRFS_RAID_SINGLE],
10172 list)
10173 inc_block_group_ro(cache, 1);
10174 }
10175
10176 btrfs_add_raid_kobjects(info);
10177 init_global_block_rsv(info);
10178 ret = 0;
10179 error:
10180 btrfs_free_path(path);
10181 return ret;
10182 }
10183
10184 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans)
10185 {
10186 struct btrfs_fs_info *fs_info = trans->fs_info;
10187 struct btrfs_block_group_cache *block_group, *tmp;
10188 struct btrfs_root *extent_root = fs_info->extent_root;
10189 struct btrfs_block_group_item item;
10190 struct btrfs_key key;
10191 int ret = 0;
10192 bool can_flush_pending_bgs = trans->can_flush_pending_bgs;
10193
10194 trans->can_flush_pending_bgs = false;
10195 list_for_each_entry_safe(block_group, tmp, &trans->new_bgs, bg_list) {
10196 if (ret)
10197 goto next;
10198
10199 spin_lock(&block_group->lock);
10200 memcpy(&item, &block_group->item, sizeof(item));
10201 memcpy(&key, &block_group->key, sizeof(key));
10202 spin_unlock(&block_group->lock);
10203
10204 ret = btrfs_insert_item(trans, extent_root, &key, &item,
10205 sizeof(item));
10206 if (ret)
10207 btrfs_abort_transaction(trans, ret);
10208 ret = btrfs_finish_chunk_alloc(trans, fs_info, key.objectid,
10209 key.offset);
10210 if (ret)
10211 btrfs_abort_transaction(trans, ret);
10212 add_block_group_free_space(trans, block_group);
10213 /* already aborted the transaction if it failed. */
10214 next:
10215 list_del_init(&block_group->bg_list);
10216 }
10217 trans->can_flush_pending_bgs = can_flush_pending_bgs;
10218 }
10219
10220 int btrfs_make_block_group(struct btrfs_trans_handle *trans,
10221 struct btrfs_fs_info *fs_info, u64 bytes_used,
10222 u64 type, u64 chunk_offset, u64 size)
10223 {
10224 struct btrfs_block_group_cache *cache;
10225 int ret;
10226
10227 btrfs_set_log_full_commit(fs_info, trans);
10228
10229 cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size);
10230 if (!cache)
10231 return -ENOMEM;
10232
10233 btrfs_set_block_group_used(&cache->item, bytes_used);
10234 btrfs_set_block_group_chunk_objectid(&cache->item,
10235 BTRFS_FIRST_CHUNK_TREE_OBJECTID);
10236 btrfs_set_block_group_flags(&cache->item, type);
10237
10238 cache->flags = type;
10239 cache->last_byte_to_unpin = (u64)-1;
10240 cache->cached = BTRFS_CACHE_FINISHED;
10241 cache->needs_free_space = 1;
10242 ret = exclude_super_stripes(fs_info, cache);
10243 if (ret) {
10244 /*
10245 * We may have excluded something, so call this just in
10246 * case.
10247 */
10248 free_excluded_extents(fs_info, cache);
10249 btrfs_put_block_group(cache);
10250 return ret;
10251 }
10252
10253 add_new_free_space(cache, chunk_offset, chunk_offset + size);
10254
10255 free_excluded_extents(fs_info, cache);
10256
10257 #ifdef CONFIG_BTRFS_DEBUG
10258 if (btrfs_should_fragment_free_space(cache)) {
10259 u64 new_bytes_used = size - bytes_used;
10260
10261 bytes_used += new_bytes_used >> 1;
10262 fragment_free_space(cache);
10263 }
10264 #endif
10265 /*
10266 * Ensure the corresponding space_info object is created and
10267 * assigned to our block group. We want our bg to be added to the rbtree
10268 * with its ->space_info set.
10269 */
10270 cache->space_info = __find_space_info(fs_info, cache->flags);
10271 ASSERT(cache->space_info);
10272
10273 ret = btrfs_add_block_group_cache(fs_info, cache);
10274 if (ret) {
10275 btrfs_remove_free_space_cache(cache);
10276 btrfs_put_block_group(cache);
10277 return ret;
10278 }
10279
10280 /*
10281 * Now that our block group has its ->space_info set and is inserted in
10282 * the rbtree, update the space info's counters.
10283 */
10284 trace_btrfs_add_block_group(fs_info, cache, 1);
10285 update_space_info(fs_info, cache->flags, size, bytes_used,
10286 cache->bytes_super, &cache->space_info);
10287 update_global_block_rsv(fs_info);
10288
10289 link_block_group(cache);
10290
10291 list_add_tail(&cache->bg_list, &trans->new_bgs);
10292
10293 set_avail_alloc_bits(fs_info, type);
10294 return 0;
10295 }
10296
10297 static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
10298 {
10299 u64 extra_flags = chunk_to_extended(flags) &
10300 BTRFS_EXTENDED_PROFILE_MASK;
10301
10302 write_seqlock(&fs_info->profiles_lock);
10303 if (flags & BTRFS_BLOCK_GROUP_DATA)
10304 fs_info->avail_data_alloc_bits &= ~extra_flags;
10305 if (flags & BTRFS_BLOCK_GROUP_METADATA)
10306 fs_info->avail_metadata_alloc_bits &= ~extra_flags;
10307 if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
10308 fs_info->avail_system_alloc_bits &= ~extra_flags;
10309 write_sequnlock(&fs_info->profiles_lock);
10310 }
10311
10312 int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
10313 struct btrfs_fs_info *fs_info, u64 group_start,
10314 struct extent_map *em)
10315 {
10316 struct btrfs_root *root = fs_info->extent_root;
10317 struct btrfs_path *path;
10318 struct btrfs_block_group_cache *block_group;
10319 struct btrfs_free_cluster *cluster;
10320 struct btrfs_root *tree_root = fs_info->tree_root;
10321 struct btrfs_key key;
10322 struct inode *inode;
10323 struct kobject *kobj = NULL;
10324 int ret;
10325 int index;
10326 int factor;
10327 struct btrfs_caching_control *caching_ctl = NULL;
10328 bool remove_em;
10329
10330 block_group = btrfs_lookup_block_group(fs_info, group_start);
10331 BUG_ON(!block_group);
10332 BUG_ON(!block_group->ro);
10333
10334 trace_btrfs_remove_block_group(block_group);
10335 /*
10336 * Free the reserved super bytes from this block group before
10337 * remove it.
10338 */
10339 free_excluded_extents(fs_info, block_group);
10340 btrfs_free_ref_tree_range(fs_info, block_group->key.objectid,
10341 block_group->key.offset);
10342
10343 memcpy(&key, &block_group->key, sizeof(key));
10344 index = btrfs_bg_flags_to_raid_index(block_group->flags);
10345 if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
10346 BTRFS_BLOCK_GROUP_RAID1 |
10347 BTRFS_BLOCK_GROUP_RAID10))
10348 factor = 2;
10349 else
10350 factor = 1;
10351
10352 /* make sure this block group isn't part of an allocation cluster */
10353 cluster = &fs_info->data_alloc_cluster;
10354 spin_lock(&cluster->refill_lock);
10355 btrfs_return_cluster_to_free_space(block_group, cluster);
10356 spin_unlock(&cluster->refill_lock);
10357
10358 /*
10359 * make sure this block group isn't part of a metadata
10360 * allocation cluster
10361 */
10362 cluster = &fs_info->meta_alloc_cluster;
10363 spin_lock(&cluster->refill_lock);
10364 btrfs_return_cluster_to_free_space(block_group, cluster);
10365 spin_unlock(&cluster->refill_lock);
10366
10367 path = btrfs_alloc_path();
10368 if (!path) {
10369 ret = -ENOMEM;
10370 goto out;
10371 }
10372
10373 /*
10374 * get the inode first so any iput calls done for the io_list
10375 * aren't the final iput (no unlinks allowed now)
10376 */
10377 inode = lookup_free_space_inode(fs_info, block_group, path);
10378
10379 mutex_lock(&trans->transaction->cache_write_mutex);
10380 /*
10381 * make sure our free spache cache IO is done before remove the
10382 * free space inode
10383 */
10384 spin_lock(&trans->transaction->dirty_bgs_lock);
10385 if (!list_empty(&block_group->io_list)) {
10386 list_del_init(&block_group->io_list);
10387
10388 WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode);
10389
10390 spin_unlock(&trans->transaction->dirty_bgs_lock);
10391 btrfs_wait_cache_io(trans, block_group, path);
10392 btrfs_put_block_group(block_group);
10393 spin_lock(&trans->transaction->dirty_bgs_lock);
10394 }
10395
10396 if (!list_empty(&block_group->dirty_list)) {
10397 list_del_init(&block_group->dirty_list);
10398 btrfs_put_block_group(block_group);
10399 }
10400 spin_unlock(&trans->transaction->dirty_bgs_lock);
10401 mutex_unlock(&trans->transaction->cache_write_mutex);
10402
10403 if (!IS_ERR(inode)) {
10404 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
10405 if (ret) {
10406 btrfs_add_delayed_iput(inode);
10407 goto out;
10408 }
10409 clear_nlink(inode);
10410 /* One for the block groups ref */
10411 spin_lock(&block_group->lock);
10412 if (block_group->iref) {
10413 block_group->iref = 0;
10414 block_group->inode = NULL;
10415 spin_unlock(&block_group->lock);
10416 iput(inode);
10417 } else {
10418 spin_unlock(&block_group->lock);
10419 }
10420 /* One for our lookup ref */
10421 btrfs_add_delayed_iput(inode);
10422 }
10423
10424 key.objectid = BTRFS_FREE_SPACE_OBJECTID;
10425 key.offset = block_group->key.objectid;
10426 key.type = 0;
10427
10428 ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
10429 if (ret < 0)
10430 goto out;
10431 if (ret > 0)
10432 btrfs_release_path(path);
10433 if (ret == 0) {
10434 ret = btrfs_del_item(trans, tree_root, path);
10435 if (ret)
10436 goto out;
10437 btrfs_release_path(path);
10438 }
10439
10440 spin_lock(&fs_info->block_group_cache_lock);
10441 rb_erase(&block_group->cache_node,
10442 &fs_info->block_group_cache_tree);
10443 RB_CLEAR_NODE(&block_group->cache_node);
10444
10445 if (fs_info->first_logical_byte == block_group->key.objectid)
10446 fs_info->first_logical_byte = (u64)-1;
10447 spin_unlock(&fs_info->block_group_cache_lock);
10448
10449 down_write(&block_group->space_info->groups_sem);
10450 /*
10451 * we must use list_del_init so people can check to see if they
10452 * are still on the list after taking the semaphore
10453 */
10454 list_del_init(&block_group->list);
10455 if (list_empty(&block_group->space_info->block_groups[index])) {
10456 kobj = block_group->space_info->block_group_kobjs[index];
10457 block_group->space_info->block_group_kobjs[index] = NULL;
10458 clear_avail_alloc_bits(fs_info, block_group->flags);
10459 }
10460 up_write(&block_group->space_info->groups_sem);
10461 if (kobj) {
10462 kobject_del(kobj);
10463 kobject_put(kobj);
10464 }
10465
10466 if (block_group->has_caching_ctl)
10467 caching_ctl = get_caching_control(block_group);
10468 if (block_group->cached == BTRFS_CACHE_STARTED)
10469 wait_block_group_cache_done(block_group);
10470 if (block_group->has_caching_ctl) {
10471 down_write(&fs_info->commit_root_sem);
10472 if (!caching_ctl) {
10473 struct btrfs_caching_control *ctl;
10474
10475 list_for_each_entry(ctl,
10476 &fs_info->caching_block_groups, list)
10477 if (ctl->block_group == block_group) {
10478 caching_ctl = ctl;
10479 refcount_inc(&caching_ctl->count);
10480 break;
10481 }
10482 }
10483 if (caching_ctl)
10484 list_del_init(&caching_ctl->list);
10485 up_write(&fs_info->commit_root_sem);
10486 if (caching_ctl) {
10487 /* Once for the caching bgs list and once for us. */
10488 put_caching_control(caching_ctl);
10489 put_caching_control(caching_ctl);
10490 }
10491 }
10492
10493 spin_lock(&trans->transaction->dirty_bgs_lock);
10494 if (!list_empty(&block_group->dirty_list)) {
10495 WARN_ON(1);
10496 }
10497 if (!list_empty(&block_group->io_list)) {
10498 WARN_ON(1);
10499 }
10500 spin_unlock(&trans->transaction->dirty_bgs_lock);
10501 btrfs_remove_free_space_cache(block_group);
10502
10503 spin_lock(&block_group->space_info->lock);
10504 list_del_init(&block_group->ro_list);
10505
10506 if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
10507 WARN_ON(block_group->space_info->total_bytes
10508 < block_group->key.offset);
10509 WARN_ON(block_group->space_info->bytes_readonly
10510 < block_group->key.offset);
10511 WARN_ON(block_group->space_info->disk_total
10512 < block_group->key.offset * factor);
10513 }
10514 block_group->space_info->total_bytes -= block_group->key.offset;
10515 block_group->space_info->bytes_readonly -= block_group->key.offset;
10516 block_group->space_info->disk_total -= block_group->key.offset * factor;
10517
10518 spin_unlock(&block_group->space_info->lock);
10519
10520 memcpy(&key, &block_group->key, sizeof(key));
10521
10522 mutex_lock(&fs_info->chunk_mutex);
10523 if (!list_empty(&em->list)) {
10524 /* We're in the transaction->pending_chunks list. */
10525 free_extent_map(em);
10526 }
10527 spin_lock(&block_group->lock);
10528 block_group->removed = 1;
10529 /*
10530 * At this point trimming can't start on this block group, because we
10531 * removed the block group from the tree fs_info->block_group_cache_tree
10532 * so no one can't find it anymore and even if someone already got this
10533 * block group before we removed it from the rbtree, they have already
10534 * incremented block_group->trimming - if they didn't, they won't find
10535 * any free space entries because we already removed them all when we
10536 * called btrfs_remove_free_space_cache().
10537 *
10538 * And we must not remove the extent map from the fs_info->mapping_tree
10539 * to prevent the same logical address range and physical device space
10540 * ranges from being reused for a new block group. This is because our
10541 * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
10542 * completely transactionless, so while it is trimming a range the
10543 * currently running transaction might finish and a new one start,
10544 * allowing for new block groups to be created that can reuse the same
10545 * physical device locations unless we take this special care.
10546 *
10547 * There may also be an implicit trim operation if the file system
10548 * is mounted with -odiscard. The same protections must remain
10549 * in place until the extents have been discarded completely when
10550 * the transaction commit has completed.
10551 */
10552 remove_em = (atomic_read(&block_group->trimming) == 0);
10553 /*
10554 * Make sure a trimmer task always sees the em in the pinned_chunks list
10555 * if it sees block_group->removed == 1 (needs to lock block_group->lock
10556 * before checking block_group->removed).
10557 */
10558 if (!remove_em) {
10559 /*
10560 * Our em might be in trans->transaction->pending_chunks which
10561 * is protected by fs_info->chunk_mutex ([lock|unlock]_chunks),
10562 * and so is the fs_info->pinned_chunks list.
10563 *
10564 * So at this point we must be holding the chunk_mutex to avoid
10565 * any races with chunk allocation (more specifically at
10566 * volumes.c:contains_pending_extent()), to ensure it always
10567 * sees the em, either in the pending_chunks list or in the
10568 * pinned_chunks list.
10569 */
10570 list_move_tail(&em->list, &fs_info->pinned_chunks);
10571 }
10572 spin_unlock(&block_group->lock);
10573
10574 if (remove_em) {
10575 struct extent_map_tree *em_tree;
10576
10577 em_tree = &fs_info->mapping_tree.map_tree;
10578 write_lock(&em_tree->lock);
10579 /*
10580 * The em might be in the pending_chunks list, so make sure the
10581 * chunk mutex is locked, since remove_extent_mapping() will
10582 * delete us from that list.
10583 */
10584 remove_extent_mapping(em_tree, em);
10585 write_unlock(&em_tree->lock);
10586 /* once for the tree */
10587 free_extent_map(em);
10588 }
10589
10590 mutex_unlock(&fs_info->chunk_mutex);
10591
10592 ret = remove_block_group_free_space(trans, block_group);
10593 if (ret)
10594 goto out;
10595
10596 btrfs_put_block_group(block_group);
10597 btrfs_put_block_group(block_group);
10598
10599 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
10600 if (ret > 0)
10601 ret = -EIO;
10602 if (ret < 0)
10603 goto out;
10604
10605 ret = btrfs_del_item(trans, root, path);
10606 out:
10607 btrfs_free_path(path);
10608 return ret;
10609 }
10610
10611 struct btrfs_trans_handle *
10612 btrfs_start_trans_remove_block_group(struct btrfs_fs_info *fs_info,
10613 const u64 chunk_offset)
10614 {
10615 struct extent_map_tree *em_tree = &fs_info->mapping_tree.map_tree;
10616 struct extent_map *em;
10617 struct map_lookup *map;
10618 unsigned int num_items;
10619
10620 read_lock(&em_tree->lock);
10621 em = lookup_extent_mapping(em_tree, chunk_offset, 1);
10622 read_unlock(&em_tree->lock);
10623 ASSERT(em && em->start == chunk_offset);
10624
10625 /*
10626 * We need to reserve 3 + N units from the metadata space info in order
10627 * to remove a block group (done at btrfs_remove_chunk() and at
10628 * btrfs_remove_block_group()), which are used for:
10629 *
10630 * 1 unit for adding the free space inode's orphan (located in the tree
10631 * of tree roots).
10632 * 1 unit for deleting the block group item (located in the extent
10633 * tree).
10634 * 1 unit for deleting the free space item (located in tree of tree
10635 * roots).
10636 * N units for deleting N device extent items corresponding to each
10637 * stripe (located in the device tree).
10638 *
10639 * In order to remove a block group we also need to reserve units in the
10640 * system space info in order to update the chunk tree (update one or
10641 * more device items and remove one chunk item), but this is done at
10642 * btrfs_remove_chunk() through a call to check_system_chunk().
10643 */
10644 map = em->map_lookup;
10645 num_items = 3 + map->num_stripes;
10646 free_extent_map(em);
10647
10648 return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root,
10649 num_items, 1);
10650 }
10651
10652 /*
10653 * Process the unused_bgs list and remove any that don't have any allocated
10654 * space inside of them.
10655 */
10656 void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
10657 {
10658 struct btrfs_block_group_cache *block_group;
10659 struct btrfs_space_info *space_info;
10660 struct btrfs_trans_handle *trans;
10661 int ret = 0;
10662
10663 if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
10664 return;
10665
10666 spin_lock(&fs_info->unused_bgs_lock);
10667 while (!list_empty(&fs_info->unused_bgs)) {
10668 u64 start, end;
10669 int trimming;
10670
10671 block_group = list_first_entry(&fs_info->unused_bgs,
10672 struct btrfs_block_group_cache,
10673 bg_list);
10674 list_del_init(&block_group->bg_list);
10675
10676 space_info = block_group->space_info;
10677
10678 if (ret || btrfs_mixed_space_info(space_info)) {
10679 btrfs_put_block_group(block_group);
10680 continue;
10681 }
10682 spin_unlock(&fs_info->unused_bgs_lock);
10683
10684 mutex_lock(&fs_info->delete_unused_bgs_mutex);
10685
10686 /* Don't want to race with allocators so take the groups_sem */
10687 down_write(&space_info->groups_sem);
10688 spin_lock(&block_group->lock);
10689 if (block_group->reserved || block_group->pinned ||
10690 btrfs_block_group_used(&block_group->item) ||
10691 block_group->ro ||
10692 list_is_singular(&block_group->list)) {
10693 /*
10694 * We want to bail if we made new allocations or have
10695 * outstanding allocations in this block group. We do
10696 * the ro check in case balance is currently acting on
10697 * this block group.
10698 */
10699 trace_btrfs_skip_unused_block_group(block_group);
10700 spin_unlock(&block_group->lock);
10701 up_write(&space_info->groups_sem);
10702 goto next;
10703 }
10704 spin_unlock(&block_group->lock);
10705
10706 /* We don't want to force the issue, only flip if it's ok. */
10707 ret = inc_block_group_ro(block_group, 0);
10708 up_write(&space_info->groups_sem);
10709 if (ret < 0) {
10710 ret = 0;
10711 goto next;
10712 }
10713
10714 /*
10715 * Want to do this before we do anything else so we can recover
10716 * properly if we fail to join the transaction.
10717 */
10718 trans = btrfs_start_trans_remove_block_group(fs_info,
10719 block_group->key.objectid);
10720 if (IS_ERR(trans)) {
10721 btrfs_dec_block_group_ro(block_group);
10722 ret = PTR_ERR(trans);
10723 goto next;
10724 }
10725
10726 /*
10727 * We could have pending pinned extents for this block group,
10728 * just delete them, we don't care about them anymore.
10729 */
10730 start = block_group->key.objectid;
10731 end = start + block_group->key.offset - 1;
10732 /*
10733 * Hold the unused_bg_unpin_mutex lock to avoid racing with
10734 * btrfs_finish_extent_commit(). If we are at transaction N,
10735 * another task might be running finish_extent_commit() for the
10736 * previous transaction N - 1, and have seen a range belonging
10737 * to the block group in freed_extents[] before we were able to
10738 * clear the whole block group range from freed_extents[]. This
10739 * means that task can lookup for the block group after we
10740 * unpinned it from freed_extents[] and removed it, leading to
10741 * a BUG_ON() at btrfs_unpin_extent_range().
10742 */
10743 mutex_lock(&fs_info->unused_bg_unpin_mutex);
10744 ret = clear_extent_bits(&fs_info->freed_extents[0], start, end,
10745 EXTENT_DIRTY);
10746 if (ret) {
10747 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
10748 btrfs_dec_block_group_ro(block_group);
10749 goto end_trans;
10750 }
10751 ret = clear_extent_bits(&fs_info->freed_extents[1], start, end,
10752 EXTENT_DIRTY);
10753 if (ret) {
10754 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
10755 btrfs_dec_block_group_ro(block_group);
10756 goto end_trans;
10757 }
10758 mutex_unlock(&fs_info->unused_bg_unpin_mutex);
10759
10760 /* Reset pinned so btrfs_put_block_group doesn't complain */
10761 spin_lock(&space_info->lock);
10762 spin_lock(&block_group->lock);
10763
10764 space_info->bytes_pinned -= block_group->pinned;
10765 space_info->bytes_readonly += block_group->pinned;
10766 percpu_counter_add(&space_info->total_bytes_pinned,
10767 -block_group->pinned);
10768 block_group->pinned = 0;
10769
10770 spin_unlock(&block_group->lock);
10771 spin_unlock(&space_info->lock);
10772
10773 /* DISCARD can flip during remount */
10774 trimming = btrfs_test_opt(fs_info, DISCARD);
10775
10776 /* Implicit trim during transaction commit. */
10777 if (trimming)
10778 btrfs_get_block_group_trimming(block_group);
10779
10780 /*
10781 * Btrfs_remove_chunk will abort the transaction if things go
10782 * horribly wrong.
10783 */
10784 ret = btrfs_remove_chunk(trans, fs_info,
10785 block_group->key.objectid);
10786
10787 if (ret) {
10788 if (trimming)
10789 btrfs_put_block_group_trimming(block_group);
10790 goto end_trans;
10791 }
10792
10793 /*
10794 * If we're not mounted with -odiscard, we can just forget
10795 * about this block group. Otherwise we'll need to wait
10796 * until transaction commit to do the actual discard.
10797 */
10798 if (trimming) {
10799 spin_lock(&fs_info->unused_bgs_lock);
10800 /*
10801 * A concurrent scrub might have added us to the list
10802 * fs_info->unused_bgs, so use a list_move operation
10803 * to add the block group to the deleted_bgs list.
10804 */
10805 list_move(&block_group->bg_list,
10806 &trans->transaction->deleted_bgs);
10807 spin_unlock(&fs_info->unused_bgs_lock);
10808 btrfs_get_block_group(block_group);
10809 }
10810 end_trans:
10811 btrfs_end_transaction(trans);
10812 next:
10813 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
10814 btrfs_put_block_group(block_group);
10815 spin_lock(&fs_info->unused_bgs_lock);
10816 }
10817 spin_unlock(&fs_info->unused_bgs_lock);
10818 }
10819
10820 int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
10821 {
10822 struct btrfs_super_block *disk_super;
10823 u64 features;
10824 u64 flags;
10825 int mixed = 0;
10826 int ret;
10827
10828 disk_super = fs_info->super_copy;
10829 if (!btrfs_super_root(disk_super))
10830 return -EINVAL;
10831
10832 features = btrfs_super_incompat_flags(disk_super);
10833 if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
10834 mixed = 1;
10835
10836 flags = BTRFS_BLOCK_GROUP_SYSTEM;
10837 ret = create_space_info(fs_info, flags);
10838 if (ret)
10839 goto out;
10840
10841 if (mixed) {
10842 flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
10843 ret = create_space_info(fs_info, flags);
10844 } else {
10845 flags = BTRFS_BLOCK_GROUP_METADATA;
10846 ret = create_space_info(fs_info, flags);
10847 if (ret)
10848 goto out;
10849
10850 flags = BTRFS_BLOCK_GROUP_DATA;
10851 ret = create_space_info(fs_info, flags);
10852 }
10853 out:
10854 return ret;
10855 }
10856
10857 int btrfs_error_unpin_extent_range(struct btrfs_fs_info *fs_info,
10858 u64 start, u64 end)
10859 {
10860 return unpin_extent_range(fs_info, start, end, false);
10861 }
10862
10863 /*
10864 * It used to be that old block groups would be left around forever.
10865 * Iterating over them would be enough to trim unused space. Since we
10866 * now automatically remove them, we also need to iterate over unallocated
10867 * space.
10868 *
10869 * We don't want a transaction for this since the discard may take a
10870 * substantial amount of time. We don't require that a transaction be
10871 * running, but we do need to take a running transaction into account
10872 * to ensure that we're not discarding chunks that were released in
10873 * the current transaction.
10874 *
10875 * Holding the chunks lock will prevent other threads from allocating
10876 * or releasing chunks, but it won't prevent a running transaction
10877 * from committing and releasing the memory that the pending chunks
10878 * list head uses. For that, we need to take a reference to the
10879 * transaction.
10880 */
10881 static int btrfs_trim_free_extents(struct btrfs_device *device,
10882 u64 minlen, u64 *trimmed)
10883 {
10884 u64 start = 0, len = 0;
10885 int ret;
10886
10887 *trimmed = 0;
10888
10889 /* Not writeable = nothing to do. */
10890 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
10891 return 0;
10892
10893 /* No free space = nothing to do. */
10894 if (device->total_bytes <= device->bytes_used)
10895 return 0;
10896
10897 ret = 0;
10898
10899 while (1) {
10900 struct btrfs_fs_info *fs_info = device->fs_info;
10901 struct btrfs_transaction *trans;
10902 u64 bytes;
10903
10904 ret = mutex_lock_interruptible(&fs_info->chunk_mutex);
10905 if (ret)
10906 return ret;
10907
10908 down_read(&fs_info->commit_root_sem);
10909
10910 spin_lock(&fs_info->trans_lock);
10911 trans = fs_info->running_transaction;
10912 if (trans)
10913 refcount_inc(&trans->use_count);
10914 spin_unlock(&fs_info->trans_lock);
10915
10916 ret = find_free_dev_extent_start(trans, device, minlen, start,
10917 &start, &len);
10918 if (trans)
10919 btrfs_put_transaction(trans);
10920
10921 if (ret) {
10922 up_read(&fs_info->commit_root_sem);
10923 mutex_unlock(&fs_info->chunk_mutex);
10924 if (ret == -ENOSPC)
10925 ret = 0;
10926 break;
10927 }
10928
10929 ret = btrfs_issue_discard(device->bdev, start, len, &bytes);
10930 up_read(&fs_info->commit_root_sem);
10931 mutex_unlock(&fs_info->chunk_mutex);
10932
10933 if (ret)
10934 break;
10935
10936 start += len;
10937 *trimmed += bytes;
10938
10939 if (fatal_signal_pending(current)) {
10940 ret = -ERESTARTSYS;
10941 break;
10942 }
10943
10944 cond_resched();
10945 }
10946
10947 return ret;
10948 }
10949
10950 int btrfs_trim_fs(struct btrfs_fs_info *fs_info, struct fstrim_range *range)
10951 {
10952 struct btrfs_block_group_cache *cache = NULL;
10953 struct btrfs_device *device;
10954 struct list_head *devices;
10955 u64 group_trimmed;
10956 u64 start;
10957 u64 end;
10958 u64 trimmed = 0;
10959 u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
10960 int ret = 0;
10961
10962 /*
10963 * try to trim all FS space, our block group may start from non-zero.
10964 */
10965 if (range->len == total_bytes)
10966 cache = btrfs_lookup_first_block_group(fs_info, range->start);
10967 else
10968 cache = btrfs_lookup_block_group(fs_info, range->start);
10969
10970 while (cache) {
10971 if (cache->key.objectid >= (range->start + range->len)) {
10972 btrfs_put_block_group(cache);
10973 break;
10974 }
10975
10976 start = max(range->start, cache->key.objectid);
10977 end = min(range->start + range->len,
10978 cache->key.objectid + cache->key.offset);
10979
10980 if (end - start >= range->minlen) {
10981 if (!block_group_cache_done(cache)) {
10982 ret = cache_block_group(cache, 0);
10983 if (ret) {
10984 btrfs_put_block_group(cache);
10985 break;
10986 }
10987 ret = wait_block_group_cache_done(cache);
10988 if (ret) {
10989 btrfs_put_block_group(cache);
10990 break;
10991 }
10992 }
10993 ret = btrfs_trim_block_group(cache,
10994 &group_trimmed,
10995 start,
10996 end,
10997 range->minlen);
10998
10999 trimmed += group_trimmed;
11000 if (ret) {
11001 btrfs_put_block_group(cache);
11002 break;
11003 }
11004 }
11005
11006 cache = next_block_group(fs_info, cache);
11007 }
11008
11009 mutex_lock(&fs_info->fs_devices->device_list_mutex);
11010 devices = &fs_info->fs_devices->alloc_list;
11011 list_for_each_entry(device, devices, dev_alloc_list) {
11012 ret = btrfs_trim_free_extents(device, range->minlen,
11013 &group_trimmed);
11014 if (ret)
11015 break;
11016
11017 trimmed += group_trimmed;
11018 }
11019 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
11020
11021 range->len = trimmed;
11022 return ret;
11023 }
11024
11025 /*
11026 * btrfs_{start,end}_write_no_snapshotting() are similar to
11027 * mnt_{want,drop}_write(), they are used to prevent some tasks from writing
11028 * data into the page cache through nocow before the subvolume is snapshoted,
11029 * but flush the data into disk after the snapshot creation, or to prevent
11030 * operations while snapshotting is ongoing and that cause the snapshot to be
11031 * inconsistent (writes followed by expanding truncates for example).
11032 */
11033 void btrfs_end_write_no_snapshotting(struct btrfs_root *root)
11034 {
11035 percpu_counter_dec(&root->subv_writers->counter);
11036 cond_wake_up(&root->subv_writers->wait);
11037 }
11038
11039 int btrfs_start_write_no_snapshotting(struct btrfs_root *root)
11040 {
11041 if (atomic_read(&root->will_be_snapshotted))
11042 return 0;
11043
11044 percpu_counter_inc(&root->subv_writers->counter);
11045 /*
11046 * Make sure counter is updated before we check for snapshot creation.
11047 */
11048 smp_mb();
11049 if (atomic_read(&root->will_be_snapshotted)) {
11050 btrfs_end_write_no_snapshotting(root);
11051 return 0;
11052 }
11053 return 1;
11054 }
11055
11056 void btrfs_wait_for_snapshot_creation(struct btrfs_root *root)
11057 {
11058 while (true) {
11059 int ret;
11060
11061 ret = btrfs_start_write_no_snapshotting(root);
11062 if (ret)
11063 break;
11064 wait_var_event(&root->will_be_snapshotted,
11065 !atomic_read(&root->will_be_snapshotted));
11066 }
11067 }
Linux with added FPC-III support