?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'riscv-for-linus-4.15-rc2_cleanups' of git://git.kernel.org/pub/scm/linux...
[linux/fpc-iii.git] / fs / btrfs / inode.c
blob993061f83067a9a65c5a58908908f5948423f11a
1 /*
2 * Copyright (C) 2007 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
19 #include <linux/kernel.h>
20 #include <linux/bio.h>
21 #include <linux/buffer_head.h>
22 #include <linux/file.h>
23 #include <linux/fs.h>
24 #include <linux/pagemap.h>
25 #include <linux/highmem.h>
26 #include <linux/time.h>
27 #include <linux/init.h>
28 #include <linux/string.h>
29 #include <linux/backing-dev.h>
30 #include <linux/mpage.h>
31 #include <linux/swap.h>
32 #include <linux/writeback.h>
33 #include <linux/compat.h>
34 #include <linux/bit_spinlock.h>
35 #include <linux/xattr.h>
36 #include <linux/posix_acl.h>
37 #include <linux/falloc.h>
38 #include <linux/slab.h>
39 #include <linux/ratelimit.h>
40 #include <linux/mount.h>
41 #include <linux/btrfs.h>
42 #include <linux/blkdev.h>
43 #include <linux/posix_acl_xattr.h>
44 #include <linux/uio.h>
45 #include <linux/magic.h>
46 #include "ctree.h"
47 #include "disk-io.h"
48 #include "transaction.h"
49 #include "btrfs_inode.h"
50 #include "print-tree.h"
51 #include "ordered-data.h"
52 #include "xattr.h"
53 #include "tree-log.h"
54 #include "volumes.h"
55 #include "compression.h"
56 #include "locking.h"
57 #include "free-space-cache.h"
58 #include "inode-map.h"
59 #include "backref.h"
60 #include "hash.h"
61 #include "props.h"
62 #include "qgroup.h"
63 #include "dedupe.h"
64
65 struct btrfs_iget_args {
66 struct btrfs_key *location;
67 struct btrfs_root *root;
68 };
69
70 struct btrfs_dio_data {
71 u64 reserve;
72 u64 unsubmitted_oe_range_start;
73 u64 unsubmitted_oe_range_end;
74 int overwrite;
75 };
76
77 static const struct inode_operations btrfs_dir_inode_operations;
78 static const struct inode_operations btrfs_symlink_inode_operations;
79 static const struct inode_operations btrfs_dir_ro_inode_operations;
80 static const struct inode_operations btrfs_special_inode_operations;
81 static const struct inode_operations btrfs_file_inode_operations;
82 static const struct address_space_operations btrfs_aops;
83 static const struct address_space_operations btrfs_symlink_aops;
84 static const struct file_operations btrfs_dir_file_operations;
85 static const struct extent_io_ops btrfs_extent_io_ops;
86
87 static struct kmem_cache *btrfs_inode_cachep;
88 struct kmem_cache *btrfs_trans_handle_cachep;
89 struct kmem_cache *btrfs_path_cachep;
90 struct kmem_cache *btrfs_free_space_cachep;
91
92 #define S_SHIFT 12
93 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
94 [S_IFREG >> S_SHIFT] = BTRFS_FT_REG_FILE,
95 [S_IFDIR >> S_SHIFT] = BTRFS_FT_DIR,
96 [S_IFCHR >> S_SHIFT] = BTRFS_FT_CHRDEV,
97 [S_IFBLK >> S_SHIFT] = BTRFS_FT_BLKDEV,
98 [S_IFIFO >> S_SHIFT] = BTRFS_FT_FIFO,
99 [S_IFSOCK >> S_SHIFT] = BTRFS_FT_SOCK,
100 [S_IFLNK >> S_SHIFT] = BTRFS_FT_SYMLINK,
101 };
102
103 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
104 static int btrfs_truncate(struct inode *inode);
105 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
106 static noinline int cow_file_range(struct inode *inode,
107 struct page *locked_page,
108 u64 start, u64 end, u64 delalloc_end,
109 int *page_started, unsigned long *nr_written,
110 int unlock, struct btrfs_dedupe_hash *hash);
111 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
112 u64 orig_start, u64 block_start,
113 u64 block_len, u64 orig_block_len,
114 u64 ram_bytes, int compress_type,
115 int type);
116
117 static void __endio_write_update_ordered(struct inode *inode,
118 const u64 offset, const u64 bytes,
119 const bool uptodate);
120
121 /*
122 * Cleanup all submitted ordered extents in specified range to handle errors
123 * from the fill_dellaloc() callback.
124 *
125 * NOTE: caller must ensure that when an error happens, it can not call
126 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
127 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
128 * to be released, which we want to happen only when finishing the ordered
129 * extent (btrfs_finish_ordered_io()). Also note that the caller of the
130 * fill_delalloc() callback already does proper cleanup for the first page of
131 * the range, that is, it invokes the callback writepage_end_io_hook() for the
132 * range of the first page.
133 */
134 static inline void btrfs_cleanup_ordered_extents(struct inode *inode,
135 const u64 offset,
136 const u64 bytes)
137 {
138 unsigned long index = offset >> PAGE_SHIFT;
139 unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;
140 struct page *page;
141
142 while (index <= end_index) {
143 page = find_get_page(inode->i_mapping, index);
144 index++;
145 if (!page)
146 continue;
147 ClearPagePrivate2(page);
148 put_page(page);
149 }
150 return __endio_write_update_ordered(inode, offset + PAGE_SIZE,
151 bytes - PAGE_SIZE, false);
152 }
153
154 static int btrfs_dirty_inode(struct inode *inode);
155
156 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
157 void btrfs_test_inode_set_ops(struct inode *inode)
158 {
159 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
160 }
161 #endif
162
163 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
164 struct inode *inode, struct inode *dir,
165 const struct qstr *qstr)
166 {
167 int err;
168
169 err = btrfs_init_acl(trans, inode, dir);
170 if (!err)
171 err = btrfs_xattr_security_init(trans, inode, dir, qstr);
172 return err;
173 }
174
175 /*
176 * this does all the hard work for inserting an inline extent into
177 * the btree. The caller should have done a btrfs_drop_extents so that
178 * no overlapping inline items exist in the btree
179 */
180 static int insert_inline_extent(struct btrfs_trans_handle *trans,
181 struct btrfs_path *path, int extent_inserted,
182 struct btrfs_root *root, struct inode *inode,
183 u64 start, size_t size, size_t compressed_size,
184 int compress_type,
185 struct page **compressed_pages)
186 {
187 struct extent_buffer *leaf;
188 struct page *page = NULL;
189 char *kaddr;
190 unsigned long ptr;
191 struct btrfs_file_extent_item *ei;
192 int ret;
193 size_t cur_size = size;
194 unsigned long offset;
195
196 if (compressed_size && compressed_pages)
197 cur_size = compressed_size;
198
199 inode_add_bytes(inode, size);
200
201 if (!extent_inserted) {
202 struct btrfs_key key;
203 size_t datasize;
204
205 key.objectid = btrfs_ino(BTRFS_I(inode));
206 key.offset = start;
207 key.type = BTRFS_EXTENT_DATA_KEY;
208
209 datasize = btrfs_file_extent_calc_inline_size(cur_size);
210 path->leave_spinning = 1;
211 ret = btrfs_insert_empty_item(trans, root, path, &key,
212 datasize);
213 if (ret)
214 goto fail;
215 }
216 leaf = path->nodes[0];
217 ei = btrfs_item_ptr(leaf, path->slots[0],
218 struct btrfs_file_extent_item);
219 btrfs_set_file_extent_generation(leaf, ei, trans->transid);
220 btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
221 btrfs_set_file_extent_encryption(leaf, ei, 0);
222 btrfs_set_file_extent_other_encoding(leaf, ei, 0);
223 btrfs_set_file_extent_ram_bytes(leaf, ei, size);
224 ptr = btrfs_file_extent_inline_start(ei);
225
226 if (compress_type != BTRFS_COMPRESS_NONE) {
227 struct page *cpage;
228 int i = 0;
229 while (compressed_size > 0) {
230 cpage = compressed_pages[i];
231 cur_size = min_t(unsigned long, compressed_size,
232 PAGE_SIZE);
233
234 kaddr = kmap_atomic(cpage);
235 write_extent_buffer(leaf, kaddr, ptr, cur_size);
236 kunmap_atomic(kaddr);
237
238 i++;
239 ptr += cur_size;
240 compressed_size -= cur_size;
241 }
242 btrfs_set_file_extent_compression(leaf, ei,
243 compress_type);
244 } else {
245 page = find_get_page(inode->i_mapping,
246 start >> PAGE_SHIFT);
247 btrfs_set_file_extent_compression(leaf, ei, 0);
248 kaddr = kmap_atomic(page);
249 offset = start & (PAGE_SIZE - 1);
250 write_extent_buffer(leaf, kaddr + offset, ptr, size);
251 kunmap_atomic(kaddr);
252 put_page(page);
253 }
254 btrfs_mark_buffer_dirty(leaf);
255 btrfs_release_path(path);
256
257 /*
258 * we're an inline extent, so nobody can
259 * extend the file past i_size without locking
260 * a page we already have locked.
261 *
262 * We must do any isize and inode updates
263 * before we unlock the pages. Otherwise we
264 * could end up racing with unlink.
265 */
266 BTRFS_I(inode)->disk_i_size = inode->i_size;
267 ret = btrfs_update_inode(trans, root, inode);
268
269 fail:
270 return ret;
271 }
272
273
274 /*
275 * conditionally insert an inline extent into the file. This
276 * does the checks required to make sure the data is small enough
277 * to fit as an inline extent.
278 */
279 static noinline int cow_file_range_inline(struct btrfs_root *root,
280 struct inode *inode, u64 start,
281 u64 end, size_t compressed_size,
282 int compress_type,
283 struct page **compressed_pages)
284 {
285 struct btrfs_fs_info *fs_info = root->fs_info;
286 struct btrfs_trans_handle *trans;
287 u64 isize = i_size_read(inode);
288 u64 actual_end = min(end + 1, isize);
289 u64 inline_len = actual_end - start;
290 u64 aligned_end = ALIGN(end, fs_info->sectorsize);
291 u64 data_len = inline_len;
292 int ret;
293 struct btrfs_path *path;
294 int extent_inserted = 0;
295 u32 extent_item_size;
296
297 if (compressed_size)
298 data_len = compressed_size;
299
300 if (start > 0 ||
301 actual_end > fs_info->sectorsize ||
302 data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
303 (!compressed_size &&
304 (actual_end & (fs_info->sectorsize - 1)) == 0) ||
305 end + 1 < isize ||
306 data_len > fs_info->max_inline) {
307 return 1;
308 }
309
310 path = btrfs_alloc_path();
311 if (!path)
312 return -ENOMEM;
313
314 trans = btrfs_join_transaction(root);
315 if (IS_ERR(trans)) {
316 btrfs_free_path(path);
317 return PTR_ERR(trans);
318 }
319 trans->block_rsv = &BTRFS_I(inode)->block_rsv;
320
321 if (compressed_size && compressed_pages)
322 extent_item_size = btrfs_file_extent_calc_inline_size(
323 compressed_size);
324 else
325 extent_item_size = btrfs_file_extent_calc_inline_size(
326 inline_len);
327
328 ret = __btrfs_drop_extents(trans, root, inode, path,
329 start, aligned_end, NULL,
330 1, 1, extent_item_size, &extent_inserted);
331 if (ret) {
332 btrfs_abort_transaction(trans, ret);
333 goto out;
334 }
335
336 if (isize > actual_end)
337 inline_len = min_t(u64, isize, actual_end);
338 ret = insert_inline_extent(trans, path, extent_inserted,
339 root, inode, start,
340 inline_len, compressed_size,
341 compress_type, compressed_pages);
342 if (ret && ret != -ENOSPC) {
343 btrfs_abort_transaction(trans, ret);
344 goto out;
345 } else if (ret == -ENOSPC) {
346 ret = 1;
347 goto out;
348 }
349
350 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
351 btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0);
352 out:
353 /*
354 * Don't forget to free the reserved space, as for inlined extent
355 * it won't count as data extent, free them directly here.
356 * And at reserve time, it's always aligned to page size, so
357 * just free one page here.
358 */
359 btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE);
360 btrfs_free_path(path);
361 btrfs_end_transaction(trans);
362 return ret;
363 }
364
365 struct async_extent {
366 u64 start;
367 u64 ram_size;
368 u64 compressed_size;
369 struct page **pages;
370 unsigned long nr_pages;
371 int compress_type;
372 struct list_head list;
373 };
374
375 struct async_cow {
376 struct inode *inode;
377 struct btrfs_root *root;
378 struct page *locked_page;
379 u64 start;
380 u64 end;
381 unsigned int write_flags;
382 struct list_head extents;
383 struct btrfs_work work;
384 };
385
386 static noinline int add_async_extent(struct async_cow *cow,
387 u64 start, u64 ram_size,
388 u64 compressed_size,
389 struct page **pages,
390 unsigned long nr_pages,
391 int compress_type)
392 {
393 struct async_extent *async_extent;
394
395 async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
396 BUG_ON(!async_extent); /* -ENOMEM */
397 async_extent->start = start;
398 async_extent->ram_size = ram_size;
399 async_extent->compressed_size = compressed_size;
400 async_extent->pages = pages;
401 async_extent->nr_pages = nr_pages;
402 async_extent->compress_type = compress_type;
403 list_add_tail(&async_extent->list, &cow->extents);
404 return 0;
405 }
406
407 static inline int inode_need_compress(struct inode *inode, u64 start, u64 end)
408 {
409 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
410
411 /* force compress */
412 if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
413 return 1;
414 /* defrag ioctl */
415 if (BTRFS_I(inode)->defrag_compress)
416 return 1;
417 /* bad compression ratios */
418 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
419 return 0;
420 if (btrfs_test_opt(fs_info, COMPRESS) ||
421 BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
422 BTRFS_I(inode)->prop_compress)
423 return btrfs_compress_heuristic(inode, start, end);
424 return 0;
425 }
426
427 static inline void inode_should_defrag(struct btrfs_inode *inode,
428 u64 start, u64 end, u64 num_bytes, u64 small_write)
429 {
430 /* If this is a small write inside eof, kick off a defrag */
431 if (num_bytes < small_write &&
432 (start > 0 || end + 1 < inode->disk_i_size))
433 btrfs_add_inode_defrag(NULL, inode);
434 }
435
436 /*
437 * we create compressed extents in two phases. The first
438 * phase compresses a range of pages that have already been
439 * locked (both pages and state bits are locked).
440 *
441 * This is done inside an ordered work queue, and the compression
442 * is spread across many cpus. The actual IO submission is step
443 * two, and the ordered work queue takes care of making sure that
444 * happens in the same order things were put onto the queue by
445 * writepages and friends.
446 *
447 * If this code finds it can't get good compression, it puts an
448 * entry onto the work queue to write the uncompressed bytes. This
449 * makes sure that both compressed inodes and uncompressed inodes
450 * are written in the same order that the flusher thread sent them
451 * down.
452 */
453 static noinline void compress_file_range(struct inode *inode,
454 struct page *locked_page,
455 u64 start, u64 end,
456 struct async_cow *async_cow,
457 int *num_added)
458 {
459 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
460 struct btrfs_root *root = BTRFS_I(inode)->root;
461 u64 blocksize = fs_info->sectorsize;
462 u64 actual_end;
463 u64 isize = i_size_read(inode);
464 int ret = 0;
465 struct page **pages = NULL;
466 unsigned long nr_pages;
467 unsigned long total_compressed = 0;
468 unsigned long total_in = 0;
469 int i;
470 int will_compress;
471 int compress_type = fs_info->compress_type;
472 int redirty = 0;
473
474 inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
475 SZ_16K);
476
477 actual_end = min_t(u64, isize, end + 1);
478 again:
479 will_compress = 0;
480 nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
481 BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
482 nr_pages = min_t(unsigned long, nr_pages,
483 BTRFS_MAX_COMPRESSED / PAGE_SIZE);
484
485 /*
486 * we don't want to send crud past the end of i_size through
487 * compression, that's just a waste of CPU time. So, if the
488 * end of the file is before the start of our current
489 * requested range of bytes, we bail out to the uncompressed
490 * cleanup code that can deal with all of this.
491 *
492 * It isn't really the fastest way to fix things, but this is a
493 * very uncommon corner.
494 */
495 if (actual_end <= start)
496 goto cleanup_and_bail_uncompressed;
497
498 total_compressed = actual_end - start;
499
500 /*
501 * skip compression for a small file range(<=blocksize) that
502 * isn't an inline extent, since it doesn't save disk space at all.
503 */
504 if (total_compressed <= blocksize &&
505 (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
506 goto cleanup_and_bail_uncompressed;
507
508 total_compressed = min_t(unsigned long, total_compressed,
509 BTRFS_MAX_UNCOMPRESSED);
510 total_in = 0;
511 ret = 0;
512
513 /*
514 * we do compression for mount -o compress and when the
515 * inode has not been flagged as nocompress. This flag can
516 * change at any time if we discover bad compression ratios.
517 */
518 if (inode_need_compress(inode, start, end)) {
519 WARN_ON(pages);
520 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
521 if (!pages) {
522 /* just bail out to the uncompressed code */
523 goto cont;
524 }
525
526 if (BTRFS_I(inode)->defrag_compress)
527 compress_type = BTRFS_I(inode)->defrag_compress;
528 else if (BTRFS_I(inode)->prop_compress)
529 compress_type = BTRFS_I(inode)->prop_compress;
530
531 /*
532 * we need to call clear_page_dirty_for_io on each
533 * page in the range. Otherwise applications with the file
534 * mmap'd can wander in and change the page contents while
535 * we are compressing them.
536 *
537 * If the compression fails for any reason, we set the pages
538 * dirty again later on.
539 */
540 extent_range_clear_dirty_for_io(inode, start, end);
541 redirty = 1;
542
543 /* Compression level is applied here and only here */
544 ret = btrfs_compress_pages(
545 compress_type | (fs_info->compress_level << 4),
546 inode->i_mapping, start,
547 pages,
548 &nr_pages,
549 &total_in,
550 &total_compressed);
551
552 if (!ret) {
553 unsigned long offset = total_compressed &
554 (PAGE_SIZE - 1);
555 struct page *page = pages[nr_pages - 1];
556 char *kaddr;
557
558 /* zero the tail end of the last page, we might be
559 * sending it down to disk
560 */
561 if (offset) {
562 kaddr = kmap_atomic(page);
563 memset(kaddr + offset, 0,
564 PAGE_SIZE - offset);
565 kunmap_atomic(kaddr);
566 }
567 will_compress = 1;
568 }
569 }
570 cont:
571 if (start == 0) {
572 /* lets try to make an inline extent */
573 if (ret || total_in < actual_end) {
574 /* we didn't compress the entire range, try
575 * to make an uncompressed inline extent.
576 */
577 ret = cow_file_range_inline(root, inode, start, end,
578 0, BTRFS_COMPRESS_NONE, NULL);
579 } else {
580 /* try making a compressed inline extent */
581 ret = cow_file_range_inline(root, inode, start, end,
582 total_compressed,
583 compress_type, pages);
584 }
585 if (ret <= 0) {
586 unsigned long clear_flags = EXTENT_DELALLOC |
587 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
588 EXTENT_DO_ACCOUNTING;
589 unsigned long page_error_op;
590
591 page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
592
593 /*
594 * inline extent creation worked or returned error,
595 * we don't need to create any more async work items.
596 * Unlock and free up our temp pages.
597 *
598 * We use DO_ACCOUNTING here because we need the
599 * delalloc_release_metadata to be done _after_ we drop
600 * our outstanding extent for clearing delalloc for this
601 * range.
602 */
603 extent_clear_unlock_delalloc(inode, start, end, end,
604 NULL, clear_flags,
605 PAGE_UNLOCK |
606 PAGE_CLEAR_DIRTY |
607 PAGE_SET_WRITEBACK |
608 page_error_op |
609 PAGE_END_WRITEBACK);
610 goto free_pages_out;
611 }
612 }
613
614 if (will_compress) {
615 /*
616 * we aren't doing an inline extent round the compressed size
617 * up to a block size boundary so the allocator does sane
618 * things
619 */
620 total_compressed = ALIGN(total_compressed, blocksize);
621
622 /*
623 * one last check to make sure the compression is really a
624 * win, compare the page count read with the blocks on disk,
625 * compression must free at least one sector size
626 */
627 total_in = ALIGN(total_in, PAGE_SIZE);
628 if (total_compressed + blocksize <= total_in) {
629 *num_added += 1;
630
631 /*
632 * The async work queues will take care of doing actual
633 * allocation on disk for these compressed pages, and
634 * will submit them to the elevator.
635 */
636 add_async_extent(async_cow, start, total_in,
637 total_compressed, pages, nr_pages,
638 compress_type);
639
640 if (start + total_in < end) {
641 start += total_in;
642 pages = NULL;
643 cond_resched();
644 goto again;
645 }
646 return;
647 }
648 }
649 if (pages) {
650 /*
651 * the compression code ran but failed to make things smaller,
652 * free any pages it allocated and our page pointer array
653 */
654 for (i = 0; i < nr_pages; i++) {
655 WARN_ON(pages[i]->mapping);
656 put_page(pages[i]);
657 }
658 kfree(pages);
659 pages = NULL;
660 total_compressed = 0;
661 nr_pages = 0;
662
663 /* flag the file so we don't compress in the future */
664 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
665 !(BTRFS_I(inode)->prop_compress)) {
666 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
667 }
668 }
669 cleanup_and_bail_uncompressed:
670 /*
671 * No compression, but we still need to write the pages in the file
672 * we've been given so far. redirty the locked page if it corresponds
673 * to our extent and set things up for the async work queue to run
674 * cow_file_range to do the normal delalloc dance.
675 */
676 if (page_offset(locked_page) >= start &&
677 page_offset(locked_page) <= end)
678 __set_page_dirty_nobuffers(locked_page);
679 /* unlocked later on in the async handlers */
680
681 if (redirty)
682 extent_range_redirty_for_io(inode, start, end);
683 add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0,
684 BTRFS_COMPRESS_NONE);
685 *num_added += 1;
686
687 return;
688
689 free_pages_out:
690 for (i = 0; i < nr_pages; i++) {
691 WARN_ON(pages[i]->mapping);
692 put_page(pages[i]);
693 }
694 kfree(pages);
695 }
696
697 static void free_async_extent_pages(struct async_extent *async_extent)
698 {
699 int i;
700
701 if (!async_extent->pages)
702 return;
703
704 for (i = 0; i < async_extent->nr_pages; i++) {
705 WARN_ON(async_extent->pages[i]->mapping);
706 put_page(async_extent->pages[i]);
707 }
708 kfree(async_extent->pages);
709 async_extent->nr_pages = 0;
710 async_extent->pages = NULL;
711 }
712
713 /*
714 * phase two of compressed writeback. This is the ordered portion
715 * of the code, which only gets called in the order the work was
716 * queued. We walk all the async extents created by compress_file_range
717 * and send them down to the disk.
718 */
719 static noinline void submit_compressed_extents(struct inode *inode,
720 struct async_cow *async_cow)
721 {
722 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
723 struct async_extent *async_extent;
724 u64 alloc_hint = 0;
725 struct btrfs_key ins;
726 struct extent_map *em;
727 struct btrfs_root *root = BTRFS_I(inode)->root;
728 struct extent_io_tree *io_tree;
729 int ret = 0;
730
731 again:
732 while (!list_empty(&async_cow->extents)) {
733 async_extent = list_entry(async_cow->extents.next,
734 struct async_extent, list);
735 list_del(&async_extent->list);
736
737 io_tree = &BTRFS_I(inode)->io_tree;
738
739 retry:
740 /* did the compression code fall back to uncompressed IO? */
741 if (!async_extent->pages) {
742 int page_started = 0;
743 unsigned long nr_written = 0;
744
745 lock_extent(io_tree, async_extent->start,
746 async_extent->start +
747 async_extent->ram_size - 1);
748
749 /* allocate blocks */
750 ret = cow_file_range(inode, async_cow->locked_page,
751 async_extent->start,
752 async_extent->start +
753 async_extent->ram_size - 1,
754 async_extent->start +
755 async_extent->ram_size - 1,
756 &page_started, &nr_written, 0,
757 NULL);
758
759 /* JDM XXX */
760
761 /*
762 * if page_started, cow_file_range inserted an
763 * inline extent and took care of all the unlocking
764 * and IO for us. Otherwise, we need to submit
765 * all those pages down to the drive.
766 */
767 if (!page_started && !ret)
768 extent_write_locked_range(io_tree,
769 inode, async_extent->start,
770 async_extent->start +
771 async_extent->ram_size - 1,
772 btrfs_get_extent,
773 WB_SYNC_ALL);
774 else if (ret)
775 unlock_page(async_cow->locked_page);
776 kfree(async_extent);
777 cond_resched();
778 continue;
779 }
780
781 lock_extent(io_tree, async_extent->start,
782 async_extent->start + async_extent->ram_size - 1);
783
784 ret = btrfs_reserve_extent(root, async_extent->ram_size,
785 async_extent->compressed_size,
786 async_extent->compressed_size,
787 0, alloc_hint, &ins, 1, 1);
788 if (ret) {
789 free_async_extent_pages(async_extent);
790
791 if (ret == -ENOSPC) {
792 unlock_extent(io_tree, async_extent->start,
793 async_extent->start +
794 async_extent->ram_size - 1);
795
796 /*
797 * we need to redirty the pages if we decide to
798 * fallback to uncompressed IO, otherwise we
799 * will not submit these pages down to lower
800 * layers.
801 */
802 extent_range_redirty_for_io(inode,
803 async_extent->start,
804 async_extent->start +
805 async_extent->ram_size - 1);
806
807 goto retry;
808 }
809 goto out_free;
810 }
811 /*
812 * here we're doing allocation and writeback of the
813 * compressed pages
814 */
815 em = create_io_em(inode, async_extent->start,
816 async_extent->ram_size, /* len */
817 async_extent->start, /* orig_start */
818 ins.objectid, /* block_start */
819 ins.offset, /* block_len */
820 ins.offset, /* orig_block_len */
821 async_extent->ram_size, /* ram_bytes */
822 async_extent->compress_type,
823 BTRFS_ORDERED_COMPRESSED);
824 if (IS_ERR(em))
825 /* ret value is not necessary due to void function */
826 goto out_free_reserve;
827 free_extent_map(em);
828
829 ret = btrfs_add_ordered_extent_compress(inode,
830 async_extent->start,
831 ins.objectid,
832 async_extent->ram_size,
833 ins.offset,
834 BTRFS_ORDERED_COMPRESSED,
835 async_extent->compress_type);
836 if (ret) {
837 btrfs_drop_extent_cache(BTRFS_I(inode),
838 async_extent->start,
839 async_extent->start +
840 async_extent->ram_size - 1, 0);
841 goto out_free_reserve;
842 }
843 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
844
845 /*
846 * clear dirty, set writeback and unlock the pages.
847 */
848 extent_clear_unlock_delalloc(inode, async_extent->start,
849 async_extent->start +
850 async_extent->ram_size - 1,
851 async_extent->start +
852 async_extent->ram_size - 1,
853 NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
854 PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
855 PAGE_SET_WRITEBACK);
856 if (btrfs_submit_compressed_write(inode,
857 async_extent->start,
858 async_extent->ram_size,
859 ins.objectid,
860 ins.offset, async_extent->pages,
861 async_extent->nr_pages,
862 async_cow->write_flags)) {
863 struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
864 struct page *p = async_extent->pages[0];
865 const u64 start = async_extent->start;
866 const u64 end = start + async_extent->ram_size - 1;
867
868 p->mapping = inode->i_mapping;
869 tree->ops->writepage_end_io_hook(p, start, end,
870 NULL, 0);
871 p->mapping = NULL;
872 extent_clear_unlock_delalloc(inode, start, end, end,
873 NULL, 0,
874 PAGE_END_WRITEBACK |
875 PAGE_SET_ERROR);
876 free_async_extent_pages(async_extent);
877 }
878 alloc_hint = ins.objectid + ins.offset;
879 kfree(async_extent);
880 cond_resched();
881 }
882 return;
883 out_free_reserve:
884 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
885 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
886 out_free:
887 extent_clear_unlock_delalloc(inode, async_extent->start,
888 async_extent->start +
889 async_extent->ram_size - 1,
890 async_extent->start +
891 async_extent->ram_size - 1,
892 NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
893 EXTENT_DELALLOC_NEW |
894 EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
895 PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
896 PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
897 PAGE_SET_ERROR);
898 free_async_extent_pages(async_extent);
899 kfree(async_extent);
900 goto again;
901 }
902
903 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
904 u64 num_bytes)
905 {
906 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
907 struct extent_map *em;
908 u64 alloc_hint = 0;
909
910 read_lock(&em_tree->lock);
911 em = search_extent_mapping(em_tree, start, num_bytes);
912 if (em) {
913 /*
914 * if block start isn't an actual block number then find the
915 * first block in this inode and use that as a hint. If that
916 * block is also bogus then just don't worry about it.
917 */
918 if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
919 free_extent_map(em);
920 em = search_extent_mapping(em_tree, 0, 0);
921 if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
922 alloc_hint = em->block_start;
923 if (em)
924 free_extent_map(em);
925 } else {
926 alloc_hint = em->block_start;
927 free_extent_map(em);
928 }
929 }
930 read_unlock(&em_tree->lock);
931
932 return alloc_hint;
933 }
934
935 /*
936 * when extent_io.c finds a delayed allocation range in the file,
937 * the call backs end up in this code. The basic idea is to
938 * allocate extents on disk for the range, and create ordered data structs
939 * in ram to track those extents.
940 *
941 * locked_page is the page that writepage had locked already. We use
942 * it to make sure we don't do extra locks or unlocks.
943 *
944 * *page_started is set to one if we unlock locked_page and do everything
945 * required to start IO on it. It may be clean and already done with
946 * IO when we return.
947 */
948 static noinline int cow_file_range(struct inode *inode,
949 struct page *locked_page,
950 u64 start, u64 end, u64 delalloc_end,
951 int *page_started, unsigned long *nr_written,
952 int unlock, struct btrfs_dedupe_hash *hash)
953 {
954 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
955 struct btrfs_root *root = BTRFS_I(inode)->root;
956 u64 alloc_hint = 0;
957 u64 num_bytes;
958 unsigned long ram_size;
959 u64 disk_num_bytes;
960 u64 cur_alloc_size = 0;
961 u64 blocksize = fs_info->sectorsize;
962 struct btrfs_key ins;
963 struct extent_map *em;
964 unsigned clear_bits;
965 unsigned long page_ops;
966 bool extent_reserved = false;
967 int ret = 0;
968
969 if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
970 WARN_ON_ONCE(1);
971 ret = -EINVAL;
972 goto out_unlock;
973 }
974
975 num_bytes = ALIGN(end - start + 1, blocksize);
976 num_bytes = max(blocksize, num_bytes);
977 disk_num_bytes = num_bytes;
978
979 inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K);
980
981 if (start == 0) {
982 /* lets try to make an inline extent */
983 ret = cow_file_range_inline(root, inode, start, end, 0,
984 BTRFS_COMPRESS_NONE, NULL);
985 if (ret == 0) {
986 /*
987 * We use DO_ACCOUNTING here because we need the
988 * delalloc_release_metadata to be run _after_ we drop
989 * our outstanding extent for clearing delalloc for this
990 * range.
991 */
992 extent_clear_unlock_delalloc(inode, start, end,
993 delalloc_end, NULL,
994 EXTENT_LOCKED | EXTENT_DELALLOC |
995 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
996 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
997 PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
998 PAGE_END_WRITEBACK);
999 *nr_written = *nr_written +
1000 (end - start + PAGE_SIZE) / PAGE_SIZE;
1001 *page_started = 1;
1002 goto out;
1003 } else if (ret < 0) {
1004 goto out_unlock;
1005 }
1006 }
1007
1008 BUG_ON(disk_num_bytes >
1009 btrfs_super_total_bytes(fs_info->super_copy));
1010
1011 alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
1012 btrfs_drop_extent_cache(BTRFS_I(inode), start,
1013 start + num_bytes - 1, 0);
1014
1015 while (disk_num_bytes > 0) {
1016 cur_alloc_size = disk_num_bytes;
1017 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
1018 fs_info->sectorsize, 0, alloc_hint,
1019 &ins, 1, 1);
1020 if (ret < 0)
1021 goto out_unlock;
1022 cur_alloc_size = ins.offset;
1023 extent_reserved = true;
1024
1025 ram_size = ins.offset;
1026 em = create_io_em(inode, start, ins.offset, /* len */
1027 start, /* orig_start */
1028 ins.objectid, /* block_start */
1029 ins.offset, /* block_len */
1030 ins.offset, /* orig_block_len */
1031 ram_size, /* ram_bytes */
1032 BTRFS_COMPRESS_NONE, /* compress_type */
1033 BTRFS_ORDERED_REGULAR /* type */);
1034 if (IS_ERR(em))
1035 goto out_reserve;
1036 free_extent_map(em);
1037
1038 ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1039 ram_size, cur_alloc_size, 0);
1040 if (ret)
1041 goto out_drop_extent_cache;
1042
1043 if (root->root_key.objectid ==
1044 BTRFS_DATA_RELOC_TREE_OBJECTID) {
1045 ret = btrfs_reloc_clone_csums(inode, start,
1046 cur_alloc_size);
1047 /*
1048 * Only drop cache here, and process as normal.
1049 *
1050 * We must not allow extent_clear_unlock_delalloc()
1051 * at out_unlock label to free meta of this ordered
1052 * extent, as its meta should be freed by
1053 * btrfs_finish_ordered_io().
1054 *
1055 * So we must continue until @start is increased to
1056 * skip current ordered extent.
1057 */
1058 if (ret)
1059 btrfs_drop_extent_cache(BTRFS_I(inode), start,
1060 start + ram_size - 1, 0);
1061 }
1062
1063 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1064
1065 /* we're not doing compressed IO, don't unlock the first
1066 * page (which the caller expects to stay locked), don't
1067 * clear any dirty bits and don't set any writeback bits
1068 *
1069 * Do set the Private2 bit so we know this page was properly
1070 * setup for writepage
1071 */
1072 page_ops = unlock ? PAGE_UNLOCK : 0;
1073 page_ops |= PAGE_SET_PRIVATE2;
1074
1075 extent_clear_unlock_delalloc(inode, start,
1076 start + ram_size - 1,
1077 delalloc_end, locked_page,
1078 EXTENT_LOCKED | EXTENT_DELALLOC,
1079 page_ops);
1080 if (disk_num_bytes < cur_alloc_size)
1081 disk_num_bytes = 0;
1082 else
1083 disk_num_bytes -= cur_alloc_size;
1084 num_bytes -= cur_alloc_size;
1085 alloc_hint = ins.objectid + ins.offset;
1086 start += cur_alloc_size;
1087 extent_reserved = false;
1088
1089 /*
1090 * btrfs_reloc_clone_csums() error, since start is increased
1091 * extent_clear_unlock_delalloc() at out_unlock label won't
1092 * free metadata of current ordered extent, we're OK to exit.
1093 */
1094 if (ret)
1095 goto out_unlock;
1096 }
1097 out:
1098 return ret;
1099
1100 out_drop_extent_cache:
1101 btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0);
1102 out_reserve:
1103 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1104 btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1105 out_unlock:
1106 clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
1107 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
1108 page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
1109 PAGE_END_WRITEBACK;
1110 /*
1111 * If we reserved an extent for our delalloc range (or a subrange) and
1112 * failed to create the respective ordered extent, then it means that
1113 * when we reserved the extent we decremented the extent's size from
1114 * the data space_info's bytes_may_use counter and incremented the
1115 * space_info's bytes_reserved counter by the same amount. We must make
1116 * sure extent_clear_unlock_delalloc() does not try to decrement again
1117 * the data space_info's bytes_may_use counter, therefore we do not pass
1118 * it the flag EXTENT_CLEAR_DATA_RESV.
1119 */
1120 if (extent_reserved) {
1121 extent_clear_unlock_delalloc(inode, start,
1122 start + cur_alloc_size,
1123 start + cur_alloc_size,
1124 locked_page,
1125 clear_bits,
1126 page_ops);
1127 start += cur_alloc_size;
1128 if (start >= end)
1129 goto out;
1130 }
1131 extent_clear_unlock_delalloc(inode, start, end, delalloc_end,
1132 locked_page,
1133 clear_bits | EXTENT_CLEAR_DATA_RESV,
1134 page_ops);
1135 goto out;
1136 }
1137
1138 /*
1139 * work queue call back to started compression on a file and pages
1140 */
1141 static noinline void async_cow_start(struct btrfs_work *work)
1142 {
1143 struct async_cow *async_cow;
1144 int num_added = 0;
1145 async_cow = container_of(work, struct async_cow, work);
1146
1147 compress_file_range(async_cow->inode, async_cow->locked_page,
1148 async_cow->start, async_cow->end, async_cow,
1149 &num_added);
1150 if (num_added == 0) {
1151 btrfs_add_delayed_iput(async_cow->inode);
1152 async_cow->inode = NULL;
1153 }
1154 }
1155
1156 /*
1157 * work queue call back to submit previously compressed pages
1158 */
1159 static noinline void async_cow_submit(struct btrfs_work *work)
1160 {
1161 struct btrfs_fs_info *fs_info;
1162 struct async_cow *async_cow;
1163 struct btrfs_root *root;
1164 unsigned long nr_pages;
1165
1166 async_cow = container_of(work, struct async_cow, work);
1167
1168 root = async_cow->root;
1169 fs_info = root->fs_info;
1170 nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >>
1171 PAGE_SHIFT;
1172
1173 /*
1174 * atomic_sub_return implies a barrier for waitqueue_active
1175 */
1176 if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1177 5 * SZ_1M &&
1178 waitqueue_active(&fs_info->async_submit_wait))
1179 wake_up(&fs_info->async_submit_wait);
1180
1181 if (async_cow->inode)
1182 submit_compressed_extents(async_cow->inode, async_cow);
1183 }
1184
1185 static noinline void async_cow_free(struct btrfs_work *work)
1186 {
1187 struct async_cow *async_cow;
1188 async_cow = container_of(work, struct async_cow, work);
1189 if (async_cow->inode)
1190 btrfs_add_delayed_iput(async_cow->inode);
1191 kfree(async_cow);
1192 }
1193
1194 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1195 u64 start, u64 end, int *page_started,
1196 unsigned long *nr_written,
1197 unsigned int write_flags)
1198 {
1199 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1200 struct async_cow *async_cow;
1201 struct btrfs_root *root = BTRFS_I(inode)->root;
1202 unsigned long nr_pages;
1203 u64 cur_end;
1204
1205 clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
1206 1, 0, NULL, GFP_NOFS);
1207 while (start < end) {
1208 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1209 BUG_ON(!async_cow); /* -ENOMEM */
1210 async_cow->inode = igrab(inode);
1211 async_cow->root = root;
1212 async_cow->locked_page = locked_page;
1213 async_cow->start = start;
1214 async_cow->write_flags = write_flags;
1215
1216 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1217 !btrfs_test_opt(fs_info, FORCE_COMPRESS))
1218 cur_end = end;
1219 else
1220 cur_end = min(end, start + SZ_512K - 1);
1221
1222 async_cow->end = cur_end;
1223 INIT_LIST_HEAD(&async_cow->extents);
1224
1225 btrfs_init_work(&async_cow->work,
1226 btrfs_delalloc_helper,
1227 async_cow_start, async_cow_submit,
1228 async_cow_free);
1229
1230 nr_pages = (cur_end - start + PAGE_SIZE) >>
1231 PAGE_SHIFT;
1232 atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1233
1234 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work);
1235
1236 *nr_written += nr_pages;
1237 start = cur_end + 1;
1238 }
1239 *page_started = 1;
1240 return 0;
1241 }
1242
1243 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
1244 u64 bytenr, u64 num_bytes)
1245 {
1246 int ret;
1247 struct btrfs_ordered_sum *sums;
1248 LIST_HEAD(list);
1249
1250 ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
1251 bytenr + num_bytes - 1, &list, 0);
1252 if (ret == 0 && list_empty(&list))
1253 return 0;
1254
1255 while (!list_empty(&list)) {
1256 sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1257 list_del(&sums->list);
1258 kfree(sums);
1259 }
1260 return 1;
1261 }
1262
1263 /*
1264 * when nowcow writeback call back. This checks for snapshots or COW copies
1265 * of the extents that exist in the file, and COWs the file as required.
1266 *
1267 * If no cow copies or snapshots exist, we write directly to the existing
1268 * blocks on disk
1269 */
1270 static noinline int run_delalloc_nocow(struct inode *inode,
1271 struct page *locked_page,
1272 u64 start, u64 end, int *page_started, int force,
1273 unsigned long *nr_written)
1274 {
1275 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1276 struct btrfs_root *root = BTRFS_I(inode)->root;
1277 struct extent_buffer *leaf;
1278 struct btrfs_path *path;
1279 struct btrfs_file_extent_item *fi;
1280 struct btrfs_key found_key;
1281 struct extent_map *em;
1282 u64 cow_start;
1283 u64 cur_offset;
1284 u64 extent_end;
1285 u64 extent_offset;
1286 u64 disk_bytenr;
1287 u64 num_bytes;
1288 u64 disk_num_bytes;
1289 u64 ram_bytes;
1290 int extent_type;
1291 int ret, err;
1292 int type;
1293 int nocow;
1294 int check_prev = 1;
1295 bool nolock;
1296 u64 ino = btrfs_ino(BTRFS_I(inode));
1297
1298 path = btrfs_alloc_path();
1299 if (!path) {
1300 extent_clear_unlock_delalloc(inode, start, end, end,
1301 locked_page,
1302 EXTENT_LOCKED | EXTENT_DELALLOC |
1303 EXTENT_DO_ACCOUNTING |
1304 EXTENT_DEFRAG, PAGE_UNLOCK |
1305 PAGE_CLEAR_DIRTY |
1306 PAGE_SET_WRITEBACK |
1307 PAGE_END_WRITEBACK);
1308 return -ENOMEM;
1309 }
1310
1311 nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
1312
1313 cow_start = (u64)-1;
1314 cur_offset = start;
1315 while (1) {
1316 ret = btrfs_lookup_file_extent(NULL, root, path, ino,
1317 cur_offset, 0);
1318 if (ret < 0)
1319 goto error;
1320 if (ret > 0 && path->slots[0] > 0 && check_prev) {
1321 leaf = path->nodes[0];
1322 btrfs_item_key_to_cpu(leaf, &found_key,
1323 path->slots[0] - 1);
1324 if (found_key.objectid == ino &&
1325 found_key.type == BTRFS_EXTENT_DATA_KEY)
1326 path->slots[0]--;
1327 }
1328 check_prev = 0;
1329 next_slot:
1330 leaf = path->nodes[0];
1331 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1332 ret = btrfs_next_leaf(root, path);
1333 if (ret < 0)
1334 goto error;
1335 if (ret > 0)
1336 break;
1337 leaf = path->nodes[0];
1338 }
1339
1340 nocow = 0;
1341 disk_bytenr = 0;
1342 num_bytes = 0;
1343 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1344
1345 if (found_key.objectid > ino)
1346 break;
1347 if (WARN_ON_ONCE(found_key.objectid < ino) ||
1348 found_key.type < BTRFS_EXTENT_DATA_KEY) {
1349 path->slots[0]++;
1350 goto next_slot;
1351 }
1352 if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
1353 found_key.offset > end)
1354 break;
1355
1356 if (found_key.offset > cur_offset) {
1357 extent_end = found_key.offset;
1358 extent_type = 0;
1359 goto out_check;
1360 }
1361
1362 fi = btrfs_item_ptr(leaf, path->slots[0],
1363 struct btrfs_file_extent_item);
1364 extent_type = btrfs_file_extent_type(leaf, fi);
1365
1366 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1367 if (extent_type == BTRFS_FILE_EXTENT_REG ||
1368 extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1369 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1370 extent_offset = btrfs_file_extent_offset(leaf, fi);
1371 extent_end = found_key.offset +
1372 btrfs_file_extent_num_bytes(leaf, fi);
1373 disk_num_bytes =
1374 btrfs_file_extent_disk_num_bytes(leaf, fi);
1375 if (extent_end <= start) {
1376 path->slots[0]++;
1377 goto next_slot;
1378 }
1379 if (disk_bytenr == 0)
1380 goto out_check;
1381 if (btrfs_file_extent_compression(leaf, fi) ||
1382 btrfs_file_extent_encryption(leaf, fi) ||
1383 btrfs_file_extent_other_encoding(leaf, fi))
1384 goto out_check;
1385 if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1386 goto out_check;
1387 if (btrfs_extent_readonly(fs_info, disk_bytenr))
1388 goto out_check;
1389 if (btrfs_cross_ref_exist(root, ino,
1390 found_key.offset -
1391 extent_offset, disk_bytenr))
1392 goto out_check;
1393 disk_bytenr += extent_offset;
1394 disk_bytenr += cur_offset - found_key.offset;
1395 num_bytes = min(end + 1, extent_end) - cur_offset;
1396 /*
1397 * if there are pending snapshots for this root,
1398 * we fall into common COW way.
1399 */
1400 if (!nolock) {
1401 err = btrfs_start_write_no_snapshotting(root);
1402 if (!err)
1403 goto out_check;
1404 }
1405 /*
1406 * force cow if csum exists in the range.
1407 * this ensure that csum for a given extent are
1408 * either valid or do not exist.
1409 */
1410 if (csum_exist_in_range(fs_info, disk_bytenr,
1411 num_bytes)) {
1412 if (!nolock)
1413 btrfs_end_write_no_snapshotting(root);
1414 goto out_check;
1415 }
1416 if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) {
1417 if (!nolock)
1418 btrfs_end_write_no_snapshotting(root);
1419 goto out_check;
1420 }
1421 nocow = 1;
1422 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1423 extent_end = found_key.offset +
1424 btrfs_file_extent_inline_len(leaf,
1425 path->slots[0], fi);
1426 extent_end = ALIGN(extent_end,
1427 fs_info->sectorsize);
1428 } else {
1429 BUG_ON(1);
1430 }
1431 out_check:
1432 if (extent_end <= start) {
1433 path->slots[0]++;
1434 if (!nolock && nocow)
1435 btrfs_end_write_no_snapshotting(root);
1436 if (nocow)
1437 btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1438 goto next_slot;
1439 }
1440 if (!nocow) {
1441 if (cow_start == (u64)-1)
1442 cow_start = cur_offset;
1443 cur_offset = extent_end;
1444 if (cur_offset > end)
1445 break;
1446 path->slots[0]++;
1447 goto next_slot;
1448 }
1449
1450 btrfs_release_path(path);
1451 if (cow_start != (u64)-1) {
1452 ret = cow_file_range(inode, locked_page,
1453 cow_start, found_key.offset - 1,
1454 end, page_started, nr_written, 1,
1455 NULL);
1456 if (ret) {
1457 if (!nolock && nocow)
1458 btrfs_end_write_no_snapshotting(root);
1459 if (nocow)
1460 btrfs_dec_nocow_writers(fs_info,
1461 disk_bytenr);
1462 goto error;
1463 }
1464 cow_start = (u64)-1;
1465 }
1466
1467 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1468 u64 orig_start = found_key.offset - extent_offset;
1469
1470 em = create_io_em(inode, cur_offset, num_bytes,
1471 orig_start,
1472 disk_bytenr, /* block_start */
1473 num_bytes, /* block_len */
1474 disk_num_bytes, /* orig_block_len */
1475 ram_bytes, BTRFS_COMPRESS_NONE,
1476 BTRFS_ORDERED_PREALLOC);
1477 if (IS_ERR(em)) {
1478 if (!nolock && nocow)
1479 btrfs_end_write_no_snapshotting(root);
1480 if (nocow)
1481 btrfs_dec_nocow_writers(fs_info,
1482 disk_bytenr);
1483 ret = PTR_ERR(em);
1484 goto error;
1485 }
1486 free_extent_map(em);
1487 }
1488
1489 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1490 type = BTRFS_ORDERED_PREALLOC;
1491 } else {
1492 type = BTRFS_ORDERED_NOCOW;
1493 }
1494
1495 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
1496 num_bytes, num_bytes, type);
1497 if (nocow)
1498 btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1499 BUG_ON(ret); /* -ENOMEM */
1500
1501 if (root->root_key.objectid ==
1502 BTRFS_DATA_RELOC_TREE_OBJECTID)
1503 /*
1504 * Error handled later, as we must prevent
1505 * extent_clear_unlock_delalloc() in error handler
1506 * from freeing metadata of created ordered extent.
1507 */
1508 ret = btrfs_reloc_clone_csums(inode, cur_offset,
1509 num_bytes);
1510
1511 extent_clear_unlock_delalloc(inode, cur_offset,
1512 cur_offset + num_bytes - 1, end,
1513 locked_page, EXTENT_LOCKED |
1514 EXTENT_DELALLOC |
1515 EXTENT_CLEAR_DATA_RESV,
1516 PAGE_UNLOCK | PAGE_SET_PRIVATE2);
1517
1518 if (!nolock && nocow)
1519 btrfs_end_write_no_snapshotting(root);
1520 cur_offset = extent_end;
1521
1522 /*
1523 * btrfs_reloc_clone_csums() error, now we're OK to call error
1524 * handler, as metadata for created ordered extent will only
1525 * be freed by btrfs_finish_ordered_io().
1526 */
1527 if (ret)
1528 goto error;
1529 if (cur_offset > end)
1530 break;
1531 }
1532 btrfs_release_path(path);
1533
1534 if (cur_offset <= end && cow_start == (u64)-1) {
1535 cow_start = cur_offset;
1536 cur_offset = end;
1537 }
1538
1539 if (cow_start != (u64)-1) {
1540 ret = cow_file_range(inode, locked_page, cow_start, end, end,
1541 page_started, nr_written, 1, NULL);
1542 if (ret)
1543 goto error;
1544 }
1545
1546 error:
1547 if (ret && cur_offset < end)
1548 extent_clear_unlock_delalloc(inode, cur_offset, end, end,
1549 locked_page, EXTENT_LOCKED |
1550 EXTENT_DELALLOC | EXTENT_DEFRAG |
1551 EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1552 PAGE_CLEAR_DIRTY |
1553 PAGE_SET_WRITEBACK |
1554 PAGE_END_WRITEBACK);
1555 btrfs_free_path(path);
1556 return ret;
1557 }
1558
1559 static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
1560 {
1561
1562 if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
1563 !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
1564 return 0;
1565
1566 /*
1567 * @defrag_bytes is a hint value, no spinlock held here,
1568 * if is not zero, it means the file is defragging.
1569 * Force cow if given extent needs to be defragged.
1570 */
1571 if (BTRFS_I(inode)->defrag_bytes &&
1572 test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
1573 EXTENT_DEFRAG, 0, NULL))
1574 return 1;
1575
1576 return 0;
1577 }
1578
1579 /*
1580 * extent_io.c call back to do delayed allocation processing
1581 */
1582 static int run_delalloc_range(void *private_data, struct page *locked_page,
1583 u64 start, u64 end, int *page_started,
1584 unsigned long *nr_written,
1585 struct writeback_control *wbc)
1586 {
1587 struct inode *inode = private_data;
1588 int ret;
1589 int force_cow = need_force_cow(inode, start, end);
1590 unsigned int write_flags = wbc_to_write_flags(wbc);
1591
1592 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1593 ret = run_delalloc_nocow(inode, locked_page, start, end,
1594 page_started, 1, nr_written);
1595 } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
1596 ret = run_delalloc_nocow(inode, locked_page, start, end,
1597 page_started, 0, nr_written);
1598 } else if (!inode_need_compress(inode, start, end)) {
1599 ret = cow_file_range(inode, locked_page, start, end, end,
1600 page_started, nr_written, 1, NULL);
1601 } else {
1602 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1603 &BTRFS_I(inode)->runtime_flags);
1604 ret = cow_file_range_async(inode, locked_page, start, end,
1605 page_started, nr_written,
1606 write_flags);
1607 }
1608 if (ret)
1609 btrfs_cleanup_ordered_extents(inode, start, end - start + 1);
1610 return ret;
1611 }
1612
1613 static void btrfs_split_extent_hook(void *private_data,
1614 struct extent_state *orig, u64 split)
1615 {
1616 struct inode *inode = private_data;
1617 u64 size;
1618
1619 /* not delalloc, ignore it */
1620 if (!(orig->state & EXTENT_DELALLOC))
1621 return;
1622
1623 size = orig->end - orig->start + 1;
1624 if (size > BTRFS_MAX_EXTENT_SIZE) {
1625 u32 num_extents;
1626 u64 new_size;
1627
1628 /*
1629 * See the explanation in btrfs_merge_extent_hook, the same
1630 * applies here, just in reverse.
1631 */
1632 new_size = orig->end - split + 1;
1633 num_extents = count_max_extents(new_size);
1634 new_size = split - orig->start;
1635 num_extents += count_max_extents(new_size);
1636 if (count_max_extents(size) >= num_extents)
1637 return;
1638 }
1639
1640 spin_lock(&BTRFS_I(inode)->lock);
1641 btrfs_mod_outstanding_extents(BTRFS_I(inode), 1);
1642 spin_unlock(&BTRFS_I(inode)->lock);
1643 }
1644
1645 /*
1646 * extent_io.c merge_extent_hook, used to track merged delayed allocation
1647 * extents so we can keep track of new extents that are just merged onto old
1648 * extents, such as when we are doing sequential writes, so we can properly
1649 * account for the metadata space we'll need.
1650 */
1651 static void btrfs_merge_extent_hook(void *private_data,
1652 struct extent_state *new,
1653 struct extent_state *other)
1654 {
1655 struct inode *inode = private_data;
1656 u64 new_size, old_size;
1657 u32 num_extents;
1658
1659 /* not delalloc, ignore it */
1660 if (!(other->state & EXTENT_DELALLOC))
1661 return;
1662
1663 if (new->start > other->start)
1664 new_size = new->end - other->start + 1;
1665 else
1666 new_size = other->end - new->start + 1;
1667
1668 /* we're not bigger than the max, unreserve the space and go */
1669 if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
1670 spin_lock(&BTRFS_I(inode)->lock);
1671 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1672 spin_unlock(&BTRFS_I(inode)->lock);
1673 return;
1674 }
1675
1676 /*
1677 * We have to add up either side to figure out how many extents were
1678 * accounted for before we merged into one big extent. If the number of
1679 * extents we accounted for is <= the amount we need for the new range
1680 * then we can return, otherwise drop. Think of it like this
1681 *
1682 * [ 4k][MAX_SIZE]
1683 *
1684 * So we've grown the extent by a MAX_SIZE extent, this would mean we
1685 * need 2 outstanding extents, on one side we have 1 and the other side
1686 * we have 1 so they are == and we can return. But in this case
1687 *
1688 * [MAX_SIZE+4k][MAX_SIZE+4k]
1689 *
1690 * Each range on their own accounts for 2 extents, but merged together
1691 * they are only 3 extents worth of accounting, so we need to drop in
1692 * this case.
1693 */
1694 old_size = other->end - other->start + 1;
1695 num_extents = count_max_extents(old_size);
1696 old_size = new->end - new->start + 1;
1697 num_extents += count_max_extents(old_size);
1698 if (count_max_extents(new_size) >= num_extents)
1699 return;
1700
1701 spin_lock(&BTRFS_I(inode)->lock);
1702 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1703 spin_unlock(&BTRFS_I(inode)->lock);
1704 }
1705
1706 static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
1707 struct inode *inode)
1708 {
1709 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1710
1711 spin_lock(&root->delalloc_lock);
1712 if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1713 list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1714 &root->delalloc_inodes);
1715 set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1716 &BTRFS_I(inode)->runtime_flags);
1717 root->nr_delalloc_inodes++;
1718 if (root->nr_delalloc_inodes == 1) {
1719 spin_lock(&fs_info->delalloc_root_lock);
1720 BUG_ON(!list_empty(&root->delalloc_root));
1721 list_add_tail(&root->delalloc_root,
1722 &fs_info->delalloc_roots);
1723 spin_unlock(&fs_info->delalloc_root_lock);
1724 }
1725 }
1726 spin_unlock(&root->delalloc_lock);
1727 }
1728
1729 static void btrfs_del_delalloc_inode(struct btrfs_root *root,
1730 struct btrfs_inode *inode)
1731 {
1732 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1733
1734 spin_lock(&root->delalloc_lock);
1735 if (!list_empty(&inode->delalloc_inodes)) {
1736 list_del_init(&inode->delalloc_inodes);
1737 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1738 &inode->runtime_flags);
1739 root->nr_delalloc_inodes--;
1740 if (!root->nr_delalloc_inodes) {
1741 spin_lock(&fs_info->delalloc_root_lock);
1742 BUG_ON(list_empty(&root->delalloc_root));
1743 list_del_init(&root->delalloc_root);
1744 spin_unlock(&fs_info->delalloc_root_lock);
1745 }
1746 }
1747 spin_unlock(&root->delalloc_lock);
1748 }
1749
1750 /*
1751 * extent_io.c set_bit_hook, used to track delayed allocation
1752 * bytes in this file, and to maintain the list of inodes that
1753 * have pending delalloc work to be done.
1754 */
1755 static void btrfs_set_bit_hook(void *private_data,
1756 struct extent_state *state, unsigned *bits)
1757 {
1758 struct inode *inode = private_data;
1759
1760 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1761
1762 if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
1763 WARN_ON(1);
1764 /*
1765 * set_bit and clear bit hooks normally require _irqsave/restore
1766 * but in this case, we are only testing for the DELALLOC
1767 * bit, which is only set or cleared with irqs on
1768 */
1769 if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1770 struct btrfs_root *root = BTRFS_I(inode)->root;
1771 u64 len = state->end + 1 - state->start;
1772 u32 num_extents = count_max_extents(len);
1773 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
1774
1775 spin_lock(&BTRFS_I(inode)->lock);
1776 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents);
1777 spin_unlock(&BTRFS_I(inode)->lock);
1778
1779 /* For sanity tests */
1780 if (btrfs_is_testing(fs_info))
1781 return;
1782
1783 percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
1784 fs_info->delalloc_batch);
1785 spin_lock(&BTRFS_I(inode)->lock);
1786 BTRFS_I(inode)->delalloc_bytes += len;
1787 if (*bits & EXTENT_DEFRAG)
1788 BTRFS_I(inode)->defrag_bytes += len;
1789 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1790 &BTRFS_I(inode)->runtime_flags))
1791 btrfs_add_delalloc_inodes(root, inode);
1792 spin_unlock(&BTRFS_I(inode)->lock);
1793 }
1794
1795 if (!(state->state & EXTENT_DELALLOC_NEW) &&
1796 (*bits & EXTENT_DELALLOC_NEW)) {
1797 spin_lock(&BTRFS_I(inode)->lock);
1798 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
1799 state->start;
1800 spin_unlock(&BTRFS_I(inode)->lock);
1801 }
1802 }
1803
1804 /*
1805 * extent_io.c clear_bit_hook, see set_bit_hook for why
1806 */
1807 static void btrfs_clear_bit_hook(void *private_data,
1808 struct extent_state *state,
1809 unsigned *bits)
1810 {
1811 struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data);
1812 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1813 u64 len = state->end + 1 - state->start;
1814 u32 num_extents = count_max_extents(len);
1815
1816 if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) {
1817 spin_lock(&inode->lock);
1818 inode->defrag_bytes -= len;
1819 spin_unlock(&inode->lock);
1820 }
1821
1822 /*
1823 * set_bit and clear bit hooks normally require _irqsave/restore
1824 * but in this case, we are only testing for the DELALLOC
1825 * bit, which is only set or cleared with irqs on
1826 */
1827 if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1828 struct btrfs_root *root = inode->root;
1829 bool do_list = !btrfs_is_free_space_inode(inode);
1830
1831 spin_lock(&inode->lock);
1832 btrfs_mod_outstanding_extents(inode, -num_extents);
1833 spin_unlock(&inode->lock);
1834
1835 /*
1836 * We don't reserve metadata space for space cache inodes so we
1837 * don't need to call dellalloc_release_metadata if there is an
1838 * error.
1839 */
1840 if (*bits & EXTENT_CLEAR_META_RESV &&
1841 root != fs_info->tree_root)
1842 btrfs_delalloc_release_metadata(inode, len);
1843
1844 /* For sanity tests. */
1845 if (btrfs_is_testing(fs_info))
1846 return;
1847
1848 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
1849 do_list && !(state->state & EXTENT_NORESERVE) &&
1850 (*bits & EXTENT_CLEAR_DATA_RESV))
1851 btrfs_free_reserved_data_space_noquota(
1852 &inode->vfs_inode,
1853 state->start, len);
1854
1855 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
1856 fs_info->delalloc_batch);
1857 spin_lock(&inode->lock);
1858 inode->delalloc_bytes -= len;
1859 if (do_list && inode->delalloc_bytes == 0 &&
1860 test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1861 &inode->runtime_flags))
1862 btrfs_del_delalloc_inode(root, inode);
1863 spin_unlock(&inode->lock);
1864 }
1865
1866 if ((state->state & EXTENT_DELALLOC_NEW) &&
1867 (*bits & EXTENT_DELALLOC_NEW)) {
1868 spin_lock(&inode->lock);
1869 ASSERT(inode->new_delalloc_bytes >= len);
1870 inode->new_delalloc_bytes -= len;
1871 spin_unlock(&inode->lock);
1872 }
1873 }
1874
1875 /*
1876 * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1877 * we don't create bios that span stripes or chunks
1878 *
1879 * return 1 if page cannot be merged to bio
1880 * return 0 if page can be merged to bio
1881 * return error otherwise
1882 */
1883 int btrfs_merge_bio_hook(struct page *page, unsigned long offset,
1884 size_t size, struct bio *bio,
1885 unsigned long bio_flags)
1886 {
1887 struct inode *inode = page->mapping->host;
1888 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1889 u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1890 u64 length = 0;
1891 u64 map_length;
1892 int ret;
1893
1894 if (bio_flags & EXTENT_BIO_COMPRESSED)
1895 return 0;
1896
1897 length = bio->bi_iter.bi_size;
1898 map_length = length;
1899 ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length,
1900 NULL, 0);
1901 if (ret < 0)
1902 return ret;
1903 if (map_length < length + size)
1904 return 1;
1905 return 0;
1906 }
1907
1908 /*
1909 * in order to insert checksums into the metadata in large chunks,
1910 * we wait until bio submission time. All the pages in the bio are
1911 * checksummed and sums are attached onto the ordered extent record.
1912 *
1913 * At IO completion time the cums attached on the ordered extent record
1914 * are inserted into the btree
1915 */
1916 static blk_status_t __btrfs_submit_bio_start(void *private_data, struct bio *bio,
1917 int mirror_num, unsigned long bio_flags,
1918 u64 bio_offset)
1919 {
1920 struct inode *inode = private_data;
1921 blk_status_t ret = 0;
1922
1923 ret = btrfs_csum_one_bio(inode, bio, 0, 0);
1924 BUG_ON(ret); /* -ENOMEM */
1925 return 0;
1926 }
1927
1928 /*
1929 * in order to insert checksums into the metadata in large chunks,
1930 * we wait until bio submission time. All the pages in the bio are
1931 * checksummed and sums are attached onto the ordered extent record.
1932 *
1933 * At IO completion time the cums attached on the ordered extent record
1934 * are inserted into the btree
1935 */
1936 static blk_status_t __btrfs_submit_bio_done(void *private_data, struct bio *bio,
1937 int mirror_num, unsigned long bio_flags,
1938 u64 bio_offset)
1939 {
1940 struct inode *inode = private_data;
1941 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1942 blk_status_t ret;
1943
1944 ret = btrfs_map_bio(fs_info, bio, mirror_num, 1);
1945 if (ret) {
1946 bio->bi_status = ret;
1947 bio_endio(bio);
1948 }
1949 return ret;
1950 }
1951
1952 /*
1953 * extent_io.c submission hook. This does the right thing for csum calculation
1954 * on write, or reading the csums from the tree before a read
1955 */
1956 static blk_status_t btrfs_submit_bio_hook(void *private_data, struct bio *bio,
1957 int mirror_num, unsigned long bio_flags,
1958 u64 bio_offset)
1959 {
1960 struct inode *inode = private_data;
1961 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1962 struct btrfs_root *root = BTRFS_I(inode)->root;
1963 enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
1964 blk_status_t ret = 0;
1965 int skip_sum;
1966 int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1967
1968 skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1969
1970 if (btrfs_is_free_space_inode(BTRFS_I(inode)))
1971 metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
1972
1973 if (bio_op(bio) != REQ_OP_WRITE) {
1974 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
1975 if (ret)
1976 goto out;
1977
1978 if (bio_flags & EXTENT_BIO_COMPRESSED) {
1979 ret = btrfs_submit_compressed_read(inode, bio,
1980 mirror_num,
1981 bio_flags);
1982 goto out;
1983 } else if (!skip_sum) {
1984 ret = btrfs_lookup_bio_sums(inode, bio, NULL);
1985 if (ret)
1986 goto out;
1987 }
1988 goto mapit;
1989 } else if (async && !skip_sum) {
1990 /* csum items have already been cloned */
1991 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
1992 goto mapit;
1993 /* we're doing a write, do the async checksumming */
1994 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags,
1995 bio_offset, inode,
1996 __btrfs_submit_bio_start,
1997 __btrfs_submit_bio_done);
1998 goto out;
1999 } else if (!skip_sum) {
2000 ret = btrfs_csum_one_bio(inode, bio, 0, 0);
2001 if (ret)
2002 goto out;
2003 }
2004
2005 mapit:
2006 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
2007
2008 out:
2009 if (ret) {
2010 bio->bi_status = ret;
2011 bio_endio(bio);
2012 }
2013 return ret;
2014 }
2015
2016 /*
2017 * given a list of ordered sums record them in the inode. This happens
2018 * at IO completion time based on sums calculated at bio submission time.
2019 */
2020 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
2021 struct inode *inode, struct list_head *list)
2022 {
2023 struct btrfs_ordered_sum *sum;
2024
2025 list_for_each_entry(sum, list, list) {
2026 trans->adding_csums = 1;
2027 btrfs_csum_file_blocks(trans,
2028 BTRFS_I(inode)->root->fs_info->csum_root, sum);
2029 trans->adding_csums = 0;
2030 }
2031 return 0;
2032 }
2033
2034 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
2035 unsigned int extra_bits,
2036 struct extent_state **cached_state, int dedupe)
2037 {
2038 WARN_ON((end & (PAGE_SIZE - 1)) == 0);
2039 return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
2040 extra_bits, cached_state);
2041 }
2042
2043 /* see btrfs_writepage_start_hook for details on why this is required */
2044 struct btrfs_writepage_fixup {
2045 struct page *page;
2046 struct btrfs_work work;
2047 };
2048
2049 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2050 {
2051 struct btrfs_writepage_fixup *fixup;
2052 struct btrfs_ordered_extent *ordered;
2053 struct extent_state *cached_state = NULL;
2054 struct extent_changeset *data_reserved = NULL;
2055 struct page *page;
2056 struct inode *inode;
2057 u64 page_start;
2058 u64 page_end;
2059 int ret;
2060
2061 fixup = container_of(work, struct btrfs_writepage_fixup, work);
2062 page = fixup->page;
2063 again:
2064 lock_page(page);
2065 if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
2066 ClearPageChecked(page);
2067 goto out_page;
2068 }
2069
2070 inode = page->mapping->host;
2071 page_start = page_offset(page);
2072 page_end = page_offset(page) + PAGE_SIZE - 1;
2073
2074 lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
2075 &cached_state);
2076
2077 /* already ordered? We're done */
2078 if (PagePrivate2(page))
2079 goto out;
2080
2081 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
2082 PAGE_SIZE);
2083 if (ordered) {
2084 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
2085 page_end, &cached_state, GFP_NOFS);
2086 unlock_page(page);
2087 btrfs_start_ordered_extent(inode, ordered, 1);
2088 btrfs_put_ordered_extent(ordered);
2089 goto again;
2090 }
2091
2092 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2093 PAGE_SIZE);
2094 if (ret) {
2095 mapping_set_error(page->mapping, ret);
2096 end_extent_writepage(page, ret, page_start, page_end);
2097 ClearPageChecked(page);
2098 goto out;
2099 }
2100
2101 btrfs_set_extent_delalloc(inode, page_start, page_end, 0, &cached_state,
2102 0);
2103 ClearPageChecked(page);
2104 set_page_dirty(page);
2105 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
2106 out:
2107 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
2108 &cached_state, GFP_NOFS);
2109 out_page:
2110 unlock_page(page);
2111 put_page(page);
2112 kfree(fixup);
2113 extent_changeset_free(data_reserved);
2114 }
2115
2116 /*
2117 * There are a few paths in the higher layers of the kernel that directly
2118 * set the page dirty bit without asking the filesystem if it is a
2119 * good idea. This causes problems because we want to make sure COW
2120 * properly happens and the data=ordered rules are followed.
2121 *
2122 * In our case any range that doesn't have the ORDERED bit set
2123 * hasn't been properly setup for IO. We kick off an async process
2124 * to fix it up. The async helper will wait for ordered extents, set
2125 * the delalloc bit and make it safe to write the page.
2126 */
2127 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
2128 {
2129 struct inode *inode = page->mapping->host;
2130 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2131 struct btrfs_writepage_fixup *fixup;
2132
2133 /* this page is properly in the ordered list */
2134 if (TestClearPagePrivate2(page))
2135 return 0;
2136
2137 if (PageChecked(page))
2138 return -EAGAIN;
2139
2140 fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2141 if (!fixup)
2142 return -EAGAIN;
2143
2144 SetPageChecked(page);
2145 get_page(page);
2146 btrfs_init_work(&fixup->work, btrfs_fixup_helper,
2147 btrfs_writepage_fixup_worker, NULL, NULL);
2148 fixup->page = page;
2149 btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
2150 return -EBUSY;
2151 }
2152
2153 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2154 struct inode *inode, u64 file_pos,
2155 u64 disk_bytenr, u64 disk_num_bytes,
2156 u64 num_bytes, u64 ram_bytes,
2157 u8 compression, u8 encryption,
2158 u16 other_encoding, int extent_type)
2159 {
2160 struct btrfs_root *root = BTRFS_I(inode)->root;
2161 struct btrfs_file_extent_item *fi;
2162 struct btrfs_path *path;
2163 struct extent_buffer *leaf;
2164 struct btrfs_key ins;
2165 u64 qg_released;
2166 int extent_inserted = 0;
2167 int ret;
2168
2169 path = btrfs_alloc_path();
2170 if (!path)
2171 return -ENOMEM;
2172
2173 /*
2174 * we may be replacing one extent in the tree with another.
2175 * The new extent is pinned in the extent map, and we don't want
2176 * to drop it from the cache until it is completely in the btree.
2177 *
2178 * So, tell btrfs_drop_extents to leave this extent in the cache.
2179 * the caller is expected to unpin it and allow it to be merged
2180 * with the others.
2181 */
2182 ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
2183 file_pos + num_bytes, NULL, 0,
2184 1, sizeof(*fi), &extent_inserted);
2185 if (ret)
2186 goto out;
2187
2188 if (!extent_inserted) {
2189 ins.objectid = btrfs_ino(BTRFS_I(inode));
2190 ins.offset = file_pos;
2191 ins.type = BTRFS_EXTENT_DATA_KEY;
2192
2193 path->leave_spinning = 1;
2194 ret = btrfs_insert_empty_item(trans, root, path, &ins,
2195 sizeof(*fi));
2196 if (ret)
2197 goto out;
2198 }
2199 leaf = path->nodes[0];
2200 fi = btrfs_item_ptr(leaf, path->slots[0],
2201 struct btrfs_file_extent_item);
2202 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2203 btrfs_set_file_extent_type(leaf, fi, extent_type);
2204 btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
2205 btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
2206 btrfs_set_file_extent_offset(leaf, fi, 0);
2207 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2208 btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
2209 btrfs_set_file_extent_compression(leaf, fi, compression);
2210 btrfs_set_file_extent_encryption(leaf, fi, encryption);
2211 btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2212
2213 btrfs_mark_buffer_dirty(leaf);
2214 btrfs_release_path(path);
2215
2216 inode_add_bytes(inode, num_bytes);
2217
2218 ins.objectid = disk_bytenr;
2219 ins.offset = disk_num_bytes;
2220 ins.type = BTRFS_EXTENT_ITEM_KEY;
2221
2222 /*
2223 * Release the reserved range from inode dirty range map, as it is
2224 * already moved into delayed_ref_head
2225 */
2226 ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
2227 if (ret < 0)
2228 goto out;
2229 qg_released = ret;
2230 ret = btrfs_alloc_reserved_file_extent(trans, root,
2231 btrfs_ino(BTRFS_I(inode)),
2232 file_pos, qg_released, &ins);
2233 out:
2234 btrfs_free_path(path);
2235
2236 return ret;
2237 }
2238
2239 /* snapshot-aware defrag */
2240 struct sa_defrag_extent_backref {
2241 struct rb_node node;
2242 struct old_sa_defrag_extent *old;
2243 u64 root_id;
2244 u64 inum;
2245 u64 file_pos;
2246 u64 extent_offset;
2247 u64 num_bytes;
2248 u64 generation;
2249 };
2250
2251 struct old_sa_defrag_extent {
2252 struct list_head list;
2253 struct new_sa_defrag_extent *new;
2254
2255 u64 extent_offset;
2256 u64 bytenr;
2257 u64 offset;
2258 u64 len;
2259 int count;
2260 };
2261
2262 struct new_sa_defrag_extent {
2263 struct rb_root root;
2264 struct list_head head;
2265 struct btrfs_path *path;
2266 struct inode *inode;
2267 u64 file_pos;
2268 u64 len;
2269 u64 bytenr;
2270 u64 disk_len;
2271 u8 compress_type;
2272 };
2273
2274 static int backref_comp(struct sa_defrag_extent_backref *b1,
2275 struct sa_defrag_extent_backref *b2)
2276 {
2277 if (b1->root_id < b2->root_id)
2278 return -1;
2279 else if (b1->root_id > b2->root_id)
2280 return 1;
2281
2282 if (b1->inum < b2->inum)
2283 return -1;
2284 else if (b1->inum > b2->inum)
2285 return 1;
2286
2287 if (b1->file_pos < b2->file_pos)
2288 return -1;
2289 else if (b1->file_pos > b2->file_pos)
2290 return 1;
2291
2292 /*
2293 * [------------------------------] ===> (a range of space)
2294 * |<--->| |<---->| =============> (fs/file tree A)
2295 * |<---------------------------->| ===> (fs/file tree B)
2296 *
2297 * A range of space can refer to two file extents in one tree while
2298 * refer to only one file extent in another tree.
2299 *
2300 * So we may process a disk offset more than one time(two extents in A)
2301 * and locate at the same extent(one extent in B), then insert two same
2302 * backrefs(both refer to the extent in B).
2303 */
2304 return 0;
2305 }
2306
2307 static void backref_insert(struct rb_root *root,
2308 struct sa_defrag_extent_backref *backref)
2309 {
2310 struct rb_node **p = &root->rb_node;
2311 struct rb_node *parent = NULL;
2312 struct sa_defrag_extent_backref *entry;
2313 int ret;
2314
2315 while (*p) {
2316 parent = *p;
2317 entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2318
2319 ret = backref_comp(backref, entry);
2320 if (ret < 0)
2321 p = &(*p)->rb_left;
2322 else
2323 p = &(*p)->rb_right;
2324 }
2325
2326 rb_link_node(&backref->node, parent, p);
2327 rb_insert_color(&backref->node, root);
2328 }
2329
2330 /*
2331 * Note the backref might has changed, and in this case we just return 0.
2332 */
2333 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2334 void *ctx)
2335 {
2336 struct btrfs_file_extent_item *extent;
2337 struct old_sa_defrag_extent *old = ctx;
2338 struct new_sa_defrag_extent *new = old->new;
2339 struct btrfs_path *path = new->path;
2340 struct btrfs_key key;
2341 struct btrfs_root *root;
2342 struct sa_defrag_extent_backref *backref;
2343 struct extent_buffer *leaf;
2344 struct inode *inode = new->inode;
2345 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2346 int slot;
2347 int ret;
2348 u64 extent_offset;
2349 u64 num_bytes;
2350
2351 if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2352 inum == btrfs_ino(BTRFS_I(inode)))
2353 return 0;
2354
2355 key.objectid = root_id;
2356 key.type = BTRFS_ROOT_ITEM_KEY;
2357 key.offset = (u64)-1;
2358
2359 root = btrfs_read_fs_root_no_name(fs_info, &key);
2360 if (IS_ERR(root)) {
2361 if (PTR_ERR(root) == -ENOENT)
2362 return 0;
2363 WARN_ON(1);
2364 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
2365 inum, offset, root_id);
2366 return PTR_ERR(root);
2367 }
2368
2369 key.objectid = inum;
2370 key.type = BTRFS_EXTENT_DATA_KEY;
2371 if (offset > (u64)-1 << 32)
2372 key.offset = 0;
2373 else
2374 key.offset = offset;
2375
2376 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2377 if (WARN_ON(ret < 0))
2378 return ret;
2379 ret = 0;
2380
2381 while (1) {
2382 cond_resched();
2383
2384 leaf = path->nodes[0];
2385 slot = path->slots[0];
2386
2387 if (slot >= btrfs_header_nritems(leaf)) {
2388 ret = btrfs_next_leaf(root, path);
2389 if (ret < 0) {
2390 goto out;
2391 } else if (ret > 0) {
2392 ret = 0;
2393 goto out;
2394 }
2395 continue;
2396 }
2397
2398 path->slots[0]++;
2399
2400 btrfs_item_key_to_cpu(leaf, &key, slot);
2401
2402 if (key.objectid > inum)
2403 goto out;
2404
2405 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2406 continue;
2407
2408 extent = btrfs_item_ptr(leaf, slot,
2409 struct btrfs_file_extent_item);
2410
2411 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2412 continue;
2413
2414 /*
2415 * 'offset' refers to the exact key.offset,
2416 * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2417 * (key.offset - extent_offset).
2418 */
2419 if (key.offset != offset)
2420 continue;
2421
2422 extent_offset = btrfs_file_extent_offset(leaf, extent);
2423 num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2424
2425 if (extent_offset >= old->extent_offset + old->offset +
2426 old->len || extent_offset + num_bytes <=
2427 old->extent_offset + old->offset)
2428 continue;
2429 break;
2430 }
2431
2432 backref = kmalloc(sizeof(*backref), GFP_NOFS);
2433 if (!backref) {
2434 ret = -ENOENT;
2435 goto out;
2436 }
2437
2438 backref->root_id = root_id;
2439 backref->inum = inum;
2440 backref->file_pos = offset;
2441 backref->num_bytes = num_bytes;
2442 backref->extent_offset = extent_offset;
2443 backref->generation = btrfs_file_extent_generation(leaf, extent);
2444 backref->old = old;
2445 backref_insert(&new->root, backref);
2446 old->count++;
2447 out:
2448 btrfs_release_path(path);
2449 WARN_ON(ret);
2450 return ret;
2451 }
2452
2453 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2454 struct new_sa_defrag_extent *new)
2455 {
2456 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2457 struct old_sa_defrag_extent *old, *tmp;
2458 int ret;
2459
2460 new->path = path;
2461
2462 list_for_each_entry_safe(old, tmp, &new->head, list) {
2463 ret = iterate_inodes_from_logical(old->bytenr +
2464 old->extent_offset, fs_info,
2465 path, record_one_backref,
2466 old, false);
2467 if (ret < 0 && ret != -ENOENT)
2468 return false;
2469
2470 /* no backref to be processed for this extent */
2471 if (!old->count) {
2472 list_del(&old->list);
2473 kfree(old);
2474 }
2475 }
2476
2477 if (list_empty(&new->head))
2478 return false;
2479
2480 return true;
2481 }
2482
2483 static int relink_is_mergable(struct extent_buffer *leaf,
2484 struct btrfs_file_extent_item *fi,
2485 struct new_sa_defrag_extent *new)
2486 {
2487 if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
2488 return 0;
2489
2490 if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2491 return 0;
2492
2493 if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
2494 return 0;
2495
2496 if (btrfs_file_extent_encryption(leaf, fi) ||
2497 btrfs_file_extent_other_encoding(leaf, fi))
2498 return 0;
2499
2500 return 1;
2501 }
2502
2503 /*
2504 * Note the backref might has changed, and in this case we just return 0.
2505 */
2506 static noinline int relink_extent_backref(struct btrfs_path *path,
2507 struct sa_defrag_extent_backref *prev,
2508 struct sa_defrag_extent_backref *backref)
2509 {
2510 struct btrfs_file_extent_item *extent;
2511 struct btrfs_file_extent_item *item;
2512 struct btrfs_ordered_extent *ordered;
2513 struct btrfs_trans_handle *trans;
2514 struct btrfs_root *root;
2515 struct btrfs_key key;
2516 struct extent_buffer *leaf;
2517 struct old_sa_defrag_extent *old = backref->old;
2518 struct new_sa_defrag_extent *new = old->new;
2519 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2520 struct inode *inode;
2521 struct extent_state *cached = NULL;
2522 int ret = 0;
2523 u64 start;
2524 u64 len;
2525 u64 lock_start;
2526 u64 lock_end;
2527 bool merge = false;
2528 int index;
2529
2530 if (prev && prev->root_id == backref->root_id &&
2531 prev->inum == backref->inum &&
2532 prev->file_pos + prev->num_bytes == backref->file_pos)
2533 merge = true;
2534
2535 /* step 1: get root */
2536 key.objectid = backref->root_id;
2537 key.type = BTRFS_ROOT_ITEM_KEY;
2538 key.offset = (u64)-1;
2539
2540 index = srcu_read_lock(&fs_info->subvol_srcu);
2541
2542 root = btrfs_read_fs_root_no_name(fs_info, &key);
2543 if (IS_ERR(root)) {
2544 srcu_read_unlock(&fs_info->subvol_srcu, index);
2545 if (PTR_ERR(root) == -ENOENT)
2546 return 0;
2547 return PTR_ERR(root);
2548 }
2549
2550 if (btrfs_root_readonly(root)) {
2551 srcu_read_unlock(&fs_info->subvol_srcu, index);
2552 return 0;
2553 }
2554
2555 /* step 2: get inode */
2556 key.objectid = backref->inum;
2557 key.type = BTRFS_INODE_ITEM_KEY;
2558 key.offset = 0;
2559
2560 inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2561 if (IS_ERR(inode)) {
2562 srcu_read_unlock(&fs_info->subvol_srcu, index);
2563 return 0;
2564 }
2565
2566 srcu_read_unlock(&fs_info->subvol_srcu, index);
2567
2568 /* step 3: relink backref */
2569 lock_start = backref->file_pos;
2570 lock_end = backref->file_pos + backref->num_bytes - 1;
2571 lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2572 &cached);
2573
2574 ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2575 if (ordered) {
2576 btrfs_put_ordered_extent(ordered);
2577 goto out_unlock;
2578 }
2579
2580 trans = btrfs_join_transaction(root);
2581 if (IS_ERR(trans)) {
2582 ret = PTR_ERR(trans);
2583 goto out_unlock;
2584 }
2585
2586 key.objectid = backref->inum;
2587 key.type = BTRFS_EXTENT_DATA_KEY;
2588 key.offset = backref->file_pos;
2589
2590 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2591 if (ret < 0) {
2592 goto out_free_path;
2593 } else if (ret > 0) {
2594 ret = 0;
2595 goto out_free_path;
2596 }
2597
2598 extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2599 struct btrfs_file_extent_item);
2600
2601 if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2602 backref->generation)
2603 goto out_free_path;
2604
2605 btrfs_release_path(path);
2606
2607 start = backref->file_pos;
2608 if (backref->extent_offset < old->extent_offset + old->offset)
2609 start += old->extent_offset + old->offset -
2610 backref->extent_offset;
2611
2612 len = min(backref->extent_offset + backref->num_bytes,
2613 old->extent_offset + old->offset + old->len);
2614 len -= max(backref->extent_offset, old->extent_offset + old->offset);
2615
2616 ret = btrfs_drop_extents(trans, root, inode, start,
2617 start + len, 1);
2618 if (ret)
2619 goto out_free_path;
2620 again:
2621 key.objectid = btrfs_ino(BTRFS_I(inode));
2622 key.type = BTRFS_EXTENT_DATA_KEY;
2623 key.offset = start;
2624
2625 path->leave_spinning = 1;
2626 if (merge) {
2627 struct btrfs_file_extent_item *fi;
2628 u64 extent_len;
2629 struct btrfs_key found_key;
2630
2631 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2632 if (ret < 0)
2633 goto out_free_path;
2634
2635 path->slots[0]--;
2636 leaf = path->nodes[0];
2637 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2638
2639 fi = btrfs_item_ptr(leaf, path->slots[0],
2640 struct btrfs_file_extent_item);
2641 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2642
2643 if (extent_len + found_key.offset == start &&
2644 relink_is_mergable(leaf, fi, new)) {
2645 btrfs_set_file_extent_num_bytes(leaf, fi,
2646 extent_len + len);
2647 btrfs_mark_buffer_dirty(leaf);
2648 inode_add_bytes(inode, len);
2649
2650 ret = 1;
2651 goto out_free_path;
2652 } else {
2653 merge = false;
2654 btrfs_release_path(path);
2655 goto again;
2656 }
2657 }
2658
2659 ret = btrfs_insert_empty_item(trans, root, path, &key,
2660 sizeof(*extent));
2661 if (ret) {
2662 btrfs_abort_transaction(trans, ret);
2663 goto out_free_path;
2664 }
2665
2666 leaf = path->nodes[0];
2667 item = btrfs_item_ptr(leaf, path->slots[0],
2668 struct btrfs_file_extent_item);
2669 btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2670 btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2671 btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2672 btrfs_set_file_extent_num_bytes(leaf, item, len);
2673 btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2674 btrfs_set_file_extent_generation(leaf, item, trans->transid);
2675 btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2676 btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2677 btrfs_set_file_extent_encryption(leaf, item, 0);
2678 btrfs_set_file_extent_other_encoding(leaf, item, 0);
2679
2680 btrfs_mark_buffer_dirty(leaf);
2681 inode_add_bytes(inode, len);
2682 btrfs_release_path(path);
2683
2684 ret = btrfs_inc_extent_ref(trans, root, new->bytenr,
2685 new->disk_len, 0,
2686 backref->root_id, backref->inum,
2687 new->file_pos); /* start - extent_offset */
2688 if (ret) {
2689 btrfs_abort_transaction(trans, ret);
2690 goto out_free_path;
2691 }
2692
2693 ret = 1;
2694 out_free_path:
2695 btrfs_release_path(path);
2696 path->leave_spinning = 0;
2697 btrfs_end_transaction(trans);
2698 out_unlock:
2699 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2700 &cached, GFP_NOFS);
2701 iput(inode);
2702 return ret;
2703 }
2704
2705 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
2706 {
2707 struct old_sa_defrag_extent *old, *tmp;
2708
2709 if (!new)
2710 return;
2711
2712 list_for_each_entry_safe(old, tmp, &new->head, list) {
2713 kfree(old);
2714 }
2715 kfree(new);
2716 }
2717
2718 static void relink_file_extents(struct new_sa_defrag_extent *new)
2719 {
2720 struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2721 struct btrfs_path *path;
2722 struct sa_defrag_extent_backref *backref;
2723 struct sa_defrag_extent_backref *prev = NULL;
2724 struct inode *inode;
2725 struct btrfs_root *root;
2726 struct rb_node *node;
2727 int ret;
2728
2729 inode = new->inode;
2730 root = BTRFS_I(inode)->root;
2731
2732 path = btrfs_alloc_path();
2733 if (!path)
2734 return;
2735
2736 if (!record_extent_backrefs(path, new)) {
2737 btrfs_free_path(path);
2738 goto out;
2739 }
2740 btrfs_release_path(path);
2741
2742 while (1) {
2743 node = rb_first(&new->root);
2744 if (!node)
2745 break;
2746 rb_erase(node, &new->root);
2747
2748 backref = rb_entry(node, struct sa_defrag_extent_backref, node);
2749
2750 ret = relink_extent_backref(path, prev, backref);
2751 WARN_ON(ret < 0);
2752
2753 kfree(prev);
2754
2755 if (ret == 1)
2756 prev = backref;
2757 else
2758 prev = NULL;
2759 cond_resched();
2760 }
2761 kfree(prev);
2762
2763 btrfs_free_path(path);
2764 out:
2765 free_sa_defrag_extent(new);
2766
2767 atomic_dec(&fs_info->defrag_running);
2768 wake_up(&fs_info->transaction_wait);
2769 }
2770
2771 static struct new_sa_defrag_extent *
2772 record_old_file_extents(struct inode *inode,
2773 struct btrfs_ordered_extent *ordered)
2774 {
2775 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2776 struct btrfs_root *root = BTRFS_I(inode)->root;
2777 struct btrfs_path *path;
2778 struct btrfs_key key;
2779 struct old_sa_defrag_extent *old;
2780 struct new_sa_defrag_extent *new;
2781 int ret;
2782
2783 new = kmalloc(sizeof(*new), GFP_NOFS);
2784 if (!new)
2785 return NULL;
2786
2787 new->inode = inode;
2788 new->file_pos = ordered->file_offset;
2789 new->len = ordered->len;
2790 new->bytenr = ordered->start;
2791 new->disk_len = ordered->disk_len;
2792 new->compress_type = ordered->compress_type;
2793 new->root = RB_ROOT;
2794 INIT_LIST_HEAD(&new->head);
2795
2796 path = btrfs_alloc_path();
2797 if (!path)
2798 goto out_kfree;
2799
2800 key.objectid = btrfs_ino(BTRFS_I(inode));
2801 key.type = BTRFS_EXTENT_DATA_KEY;
2802 key.offset = new->file_pos;
2803
2804 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2805 if (ret < 0)
2806 goto out_free_path;
2807 if (ret > 0 && path->slots[0] > 0)
2808 path->slots[0]--;
2809
2810 /* find out all the old extents for the file range */
2811 while (1) {
2812 struct btrfs_file_extent_item *extent;
2813 struct extent_buffer *l;
2814 int slot;
2815 u64 num_bytes;
2816 u64 offset;
2817 u64 end;
2818 u64 disk_bytenr;
2819 u64 extent_offset;
2820
2821 l = path->nodes[0];
2822 slot = path->slots[0];
2823
2824 if (slot >= btrfs_header_nritems(l)) {
2825 ret = btrfs_next_leaf(root, path);
2826 if (ret < 0)
2827 goto out_free_path;
2828 else if (ret > 0)
2829 break;
2830 continue;
2831 }
2832
2833 btrfs_item_key_to_cpu(l, &key, slot);
2834
2835 if (key.objectid != btrfs_ino(BTRFS_I(inode)))
2836 break;
2837 if (key.type != BTRFS_EXTENT_DATA_KEY)
2838 break;
2839 if (key.offset >= new->file_pos + new->len)
2840 break;
2841
2842 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
2843
2844 num_bytes = btrfs_file_extent_num_bytes(l, extent);
2845 if (key.offset + num_bytes < new->file_pos)
2846 goto next;
2847
2848 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
2849 if (!disk_bytenr)
2850 goto next;
2851
2852 extent_offset = btrfs_file_extent_offset(l, extent);
2853
2854 old = kmalloc(sizeof(*old), GFP_NOFS);
2855 if (!old)
2856 goto out_free_path;
2857
2858 offset = max(new->file_pos, key.offset);
2859 end = min(new->file_pos + new->len, key.offset + num_bytes);
2860
2861 old->bytenr = disk_bytenr;
2862 old->extent_offset = extent_offset;
2863 old->offset = offset - key.offset;
2864 old->len = end - offset;
2865 old->new = new;
2866 old->count = 0;
2867 list_add_tail(&old->list, &new->head);
2868 next:
2869 path->slots[0]++;
2870 cond_resched();
2871 }
2872
2873 btrfs_free_path(path);
2874 atomic_inc(&fs_info->defrag_running);
2875
2876 return new;
2877
2878 out_free_path:
2879 btrfs_free_path(path);
2880 out_kfree:
2881 free_sa_defrag_extent(new);
2882 return NULL;
2883 }
2884
2885 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
2886 u64 start, u64 len)
2887 {
2888 struct btrfs_block_group_cache *cache;
2889
2890 cache = btrfs_lookup_block_group(fs_info, start);
2891 ASSERT(cache);
2892
2893 spin_lock(&cache->lock);
2894 cache->delalloc_bytes -= len;
2895 spin_unlock(&cache->lock);
2896
2897 btrfs_put_block_group(cache);
2898 }
2899
2900 /* as ordered data IO finishes, this gets called so we can finish
2901 * an ordered extent if the range of bytes in the file it covers are
2902 * fully written.
2903 */
2904 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2905 {
2906 struct inode *inode = ordered_extent->inode;
2907 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2908 struct btrfs_root *root = BTRFS_I(inode)->root;
2909 struct btrfs_trans_handle *trans = NULL;
2910 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2911 struct extent_state *cached_state = NULL;
2912 struct new_sa_defrag_extent *new = NULL;
2913 int compress_type = 0;
2914 int ret = 0;
2915 u64 logical_len = ordered_extent->len;
2916 bool nolock;
2917 bool truncated = false;
2918 bool range_locked = false;
2919 bool clear_new_delalloc_bytes = false;
2920
2921 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
2922 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
2923 !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags))
2924 clear_new_delalloc_bytes = true;
2925
2926 nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
2927
2928 if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
2929 ret = -EIO;
2930 goto out;
2931 }
2932
2933 btrfs_free_io_failure_record(BTRFS_I(inode),
2934 ordered_extent->file_offset,
2935 ordered_extent->file_offset +
2936 ordered_extent->len - 1);
2937
2938 if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
2939 truncated = true;
2940 logical_len = ordered_extent->truncated_len;
2941 /* Truncated the entire extent, don't bother adding */
2942 if (!logical_len)
2943 goto out;
2944 }
2945
2946 if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2947 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2948
2949 /*
2950 * For mwrite(mmap + memset to write) case, we still reserve
2951 * space for NOCOW range.
2952 * As NOCOW won't cause a new delayed ref, just free the space
2953 */
2954 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
2955 ordered_extent->len);
2956 btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2957 if (nolock)
2958 trans = btrfs_join_transaction_nolock(root);
2959 else
2960 trans = btrfs_join_transaction(root);
2961 if (IS_ERR(trans)) {
2962 ret = PTR_ERR(trans);
2963 trans = NULL;
2964 goto out;
2965 }
2966 trans->block_rsv = &BTRFS_I(inode)->block_rsv;
2967 ret = btrfs_update_inode_fallback(trans, root, inode);
2968 if (ret) /* -ENOMEM or corruption */
2969 btrfs_abort_transaction(trans, ret);
2970 goto out;
2971 }
2972
2973 range_locked = true;
2974 lock_extent_bits(io_tree, ordered_extent->file_offset,
2975 ordered_extent->file_offset + ordered_extent->len - 1,
2976 &cached_state);
2977
2978 ret = test_range_bit(io_tree, ordered_extent->file_offset,
2979 ordered_extent->file_offset + ordered_extent->len - 1,
2980 EXTENT_DEFRAG, 0, cached_state);
2981 if (ret) {
2982 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
2983 if (0 && last_snapshot >= BTRFS_I(inode)->generation)
2984 /* the inode is shared */
2985 new = record_old_file_extents(inode, ordered_extent);
2986
2987 clear_extent_bit(io_tree, ordered_extent->file_offset,
2988 ordered_extent->file_offset + ordered_extent->len - 1,
2989 EXTENT_DEFRAG, 0, 0, &cached_state, GFP_NOFS);
2990 }
2991
2992 if (nolock)
2993 trans = btrfs_join_transaction_nolock(root);
2994 else
2995 trans = btrfs_join_transaction(root);
2996 if (IS_ERR(trans)) {
2997 ret = PTR_ERR(trans);
2998 trans = NULL;
2999 goto out;
3000 }
3001
3002 trans->block_rsv = &BTRFS_I(inode)->block_rsv;
3003
3004 if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
3005 compress_type = ordered_extent->compress_type;
3006 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3007 BUG_ON(compress_type);
3008 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
3009 ordered_extent->file_offset,
3010 ordered_extent->file_offset +
3011 logical_len);
3012 } else {
3013 BUG_ON(root == fs_info->tree_root);
3014 ret = insert_reserved_file_extent(trans, inode,
3015 ordered_extent->file_offset,
3016 ordered_extent->start,
3017 ordered_extent->disk_len,
3018 logical_len, logical_len,
3019 compress_type, 0, 0,
3020 BTRFS_FILE_EXTENT_REG);
3021 if (!ret)
3022 btrfs_release_delalloc_bytes(fs_info,
3023 ordered_extent->start,
3024 ordered_extent->disk_len);
3025 }
3026 unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
3027 ordered_extent->file_offset, ordered_extent->len,
3028 trans->transid);
3029 if (ret < 0) {
3030 btrfs_abort_transaction(trans, ret);
3031 goto out;
3032 }
3033
3034 add_pending_csums(trans, inode, &ordered_extent->list);
3035
3036 btrfs_ordered_update_i_size(inode, 0, ordered_extent);
3037 ret = btrfs_update_inode_fallback(trans, root, inode);
3038 if (ret) { /* -ENOMEM or corruption */
3039 btrfs_abort_transaction(trans, ret);
3040 goto out;
3041 }
3042 ret = 0;
3043 out:
3044 if (range_locked || clear_new_delalloc_bytes) {
3045 unsigned int clear_bits = 0;
3046
3047 if (range_locked)
3048 clear_bits |= EXTENT_LOCKED;
3049 if (clear_new_delalloc_bytes)
3050 clear_bits |= EXTENT_DELALLOC_NEW;
3051 clear_extent_bit(&BTRFS_I(inode)->io_tree,
3052 ordered_extent->file_offset,
3053 ordered_extent->file_offset +
3054 ordered_extent->len - 1,
3055 clear_bits,
3056 (clear_bits & EXTENT_LOCKED) ? 1 : 0,
3057 0, &cached_state, GFP_NOFS);
3058 }
3059
3060 if (trans)
3061 btrfs_end_transaction(trans);
3062
3063 if (ret || truncated) {
3064 u64 start, end;
3065
3066 if (truncated)
3067 start = ordered_extent->file_offset + logical_len;
3068 else
3069 start = ordered_extent->file_offset;
3070 end = ordered_extent->file_offset + ordered_extent->len - 1;
3071 clear_extent_uptodate(io_tree, start, end, NULL, GFP_NOFS);
3072
3073 /* Drop the cache for the part of the extent we didn't write. */
3074 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
3075
3076 /*
3077 * If the ordered extent had an IOERR or something else went
3078 * wrong we need to return the space for this ordered extent
3079 * back to the allocator. We only free the extent in the
3080 * truncated case if we didn't write out the extent at all.
3081 */
3082 if ((ret || !logical_len) &&
3083 !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3084 !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
3085 btrfs_free_reserved_extent(fs_info,
3086 ordered_extent->start,
3087 ordered_extent->disk_len, 1);
3088 }
3089
3090
3091 /*
3092 * This needs to be done to make sure anybody waiting knows we are done
3093 * updating everything for this ordered extent.
3094 */
3095 btrfs_remove_ordered_extent(inode, ordered_extent);
3096
3097 /* for snapshot-aware defrag */
3098 if (new) {
3099 if (ret) {
3100 free_sa_defrag_extent(new);
3101 atomic_dec(&fs_info->defrag_running);
3102 } else {
3103 relink_file_extents(new);
3104 }
3105 }
3106
3107 /* once for us */
3108 btrfs_put_ordered_extent(ordered_extent);
3109 /* once for the tree */
3110 btrfs_put_ordered_extent(ordered_extent);
3111
3112 return ret;
3113 }
3114
3115 static void finish_ordered_fn(struct btrfs_work *work)
3116 {
3117 struct btrfs_ordered_extent *ordered_extent;
3118 ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
3119 btrfs_finish_ordered_io(ordered_extent);
3120 }
3121
3122 static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
3123 struct extent_state *state, int uptodate)
3124 {
3125 struct inode *inode = page->mapping->host;
3126 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3127 struct btrfs_ordered_extent *ordered_extent = NULL;
3128 struct btrfs_workqueue *wq;
3129 btrfs_work_func_t func;
3130
3131 trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
3132
3133 ClearPagePrivate2(page);
3134 if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
3135 end - start + 1, uptodate))
3136 return;
3137
3138 if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
3139 wq = fs_info->endio_freespace_worker;
3140 func = btrfs_freespace_write_helper;
3141 } else {
3142 wq = fs_info->endio_write_workers;
3143 func = btrfs_endio_write_helper;
3144 }
3145
3146 btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
3147 NULL);
3148 btrfs_queue_work(wq, &ordered_extent->work);
3149 }
3150
3151 static int __readpage_endio_check(struct inode *inode,
3152 struct btrfs_io_bio *io_bio,
3153 int icsum, struct page *page,
3154 int pgoff, u64 start, size_t len)
3155 {
3156 char *kaddr;
3157 u32 csum_expected;
3158 u32 csum = ~(u32)0;
3159
3160 csum_expected = *(((u32 *)io_bio->csum) + icsum);
3161
3162 kaddr = kmap_atomic(page);
3163 csum = btrfs_csum_data(kaddr + pgoff, csum, len);
3164 btrfs_csum_final(csum, (u8 *)&csum);
3165 if (csum != csum_expected)
3166 goto zeroit;
3167
3168 kunmap_atomic(kaddr);
3169 return 0;
3170 zeroit:
3171 btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
3172 io_bio->mirror_num);
3173 memset(kaddr + pgoff, 1, len);
3174 flush_dcache_page(page);
3175 kunmap_atomic(kaddr);
3176 return -EIO;
3177 }
3178
3179 /*
3180 * when reads are done, we need to check csums to verify the data is correct
3181 * if there's a match, we allow the bio to finish. If not, the code in
3182 * extent_io.c will try to find good copies for us.
3183 */
3184 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
3185 u64 phy_offset, struct page *page,
3186 u64 start, u64 end, int mirror)
3187 {
3188 size_t offset = start - page_offset(page);
3189 struct inode *inode = page->mapping->host;
3190 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3191 struct btrfs_root *root = BTRFS_I(inode)->root;
3192
3193 if (PageChecked(page)) {
3194 ClearPageChecked(page);
3195 return 0;
3196 }
3197
3198 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3199 return 0;
3200
3201 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3202 test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3203 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
3204 return 0;
3205 }
3206
3207 phy_offset >>= inode->i_sb->s_blocksize_bits;
3208 return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
3209 start, (size_t)(end - start + 1));
3210 }
3211
3212 void btrfs_add_delayed_iput(struct inode *inode)
3213 {
3214 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3215 struct btrfs_inode *binode = BTRFS_I(inode);
3216
3217 if (atomic_add_unless(&inode->i_count, -1, 1))
3218 return;
3219
3220 spin_lock(&fs_info->delayed_iput_lock);
3221 if (binode->delayed_iput_count == 0) {
3222 ASSERT(list_empty(&binode->delayed_iput));
3223 list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
3224 } else {
3225 binode->delayed_iput_count++;
3226 }
3227 spin_unlock(&fs_info->delayed_iput_lock);
3228 }
3229
3230 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3231 {
3232
3233 spin_lock(&fs_info->delayed_iput_lock);
3234 while (!list_empty(&fs_info->delayed_iputs)) {
3235 struct btrfs_inode *inode;
3236
3237 inode = list_first_entry(&fs_info->delayed_iputs,
3238 struct btrfs_inode, delayed_iput);
3239 if (inode->delayed_iput_count) {
3240 inode->delayed_iput_count--;
3241 list_move_tail(&inode->delayed_iput,
3242 &fs_info->delayed_iputs);
3243 } else {
3244 list_del_init(&inode->delayed_iput);
3245 }
3246 spin_unlock(&fs_info->delayed_iput_lock);
3247 iput(&inode->vfs_inode);
3248 spin_lock(&fs_info->delayed_iput_lock);
3249 }
3250 spin_unlock(&fs_info->delayed_iput_lock);
3251 }
3252
3253 /*
3254 * This is called in transaction commit time. If there are no orphan
3255 * files in the subvolume, it removes orphan item and frees block_rsv
3256 * structure.
3257 */
3258 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
3259 struct btrfs_root *root)
3260 {
3261 struct btrfs_fs_info *fs_info = root->fs_info;
3262 struct btrfs_block_rsv *block_rsv;
3263 int ret;
3264
3265 if (atomic_read(&root->orphan_inodes) ||
3266 root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
3267 return;
3268
3269 spin_lock(&root->orphan_lock);
3270 if (atomic_read(&root->orphan_inodes)) {
3271 spin_unlock(&root->orphan_lock);
3272 return;
3273 }
3274
3275 if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
3276 spin_unlock(&root->orphan_lock);
3277 return;
3278 }
3279
3280 block_rsv = root->orphan_block_rsv;
3281 root->orphan_block_rsv = NULL;
3282 spin_unlock(&root->orphan_lock);
3283
3284 if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
3285 btrfs_root_refs(&root->root_item) > 0) {
3286 ret = btrfs_del_orphan_item(trans, fs_info->tree_root,
3287 root->root_key.objectid);
3288 if (ret)
3289 btrfs_abort_transaction(trans, ret);
3290 else
3291 clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
3292 &root->state);
3293 }
3294
3295 if (block_rsv) {
3296 WARN_ON(block_rsv->size > 0);
3297 btrfs_free_block_rsv(fs_info, block_rsv);
3298 }
3299 }
3300
3301 /*
3302 * This creates an orphan entry for the given inode in case something goes
3303 * wrong in the middle of an unlink/truncate.
3304 *
3305 * NOTE: caller of this function should reserve 5 units of metadata for
3306 * this function.
3307 */
3308 int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3309 struct btrfs_inode *inode)
3310 {
3311 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
3312 struct btrfs_root *root = inode->root;
3313 struct btrfs_block_rsv *block_rsv = NULL;
3314 int reserve = 0;
3315 int insert = 0;
3316 int ret;
3317
3318 if (!root->orphan_block_rsv) {
3319 block_rsv = btrfs_alloc_block_rsv(fs_info,
3320 BTRFS_BLOCK_RSV_TEMP);
3321 if (!block_rsv)
3322 return -ENOMEM;
3323 }
3324
3325 spin_lock(&root->orphan_lock);
3326 if (!root->orphan_block_rsv) {
3327 root->orphan_block_rsv = block_rsv;
3328 } else if (block_rsv) {
3329 btrfs_free_block_rsv(fs_info, block_rsv);
3330 block_rsv = NULL;
3331 }
3332
3333 if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3334 &inode->runtime_flags)) {
3335 #if 0
3336 /*
3337 * For proper ENOSPC handling, we should do orphan
3338 * cleanup when mounting. But this introduces backward
3339 * compatibility issue.
3340 */
3341 if (!xchg(&root->orphan_item_inserted, 1))
3342 insert = 2;
3343 else
3344 insert = 1;
3345 #endif
3346 insert = 1;
3347 atomic_inc(&root->orphan_inodes);
3348 }
3349
3350 if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3351 &inode->runtime_flags))
3352 reserve = 1;
3353 spin_unlock(&root->orphan_lock);
3354
3355 /* grab metadata reservation from transaction handle */
3356 if (reserve) {
3357 ret = btrfs_orphan_reserve_metadata(trans, inode);
3358 ASSERT(!ret);
3359 if (ret) {
3360 atomic_dec(&root->orphan_inodes);
3361 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3362 &inode->runtime_flags);
3363 if (insert)
3364 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3365 &inode->runtime_flags);
3366 return ret;
3367 }
3368 }
3369
3370 /* insert an orphan item to track this unlinked/truncated file */
3371 if (insert >= 1) {
3372 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
3373 if (ret) {
3374 atomic_dec(&root->orphan_inodes);
3375 if (reserve) {
3376 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3377 &inode->runtime_flags);
3378 btrfs_orphan_release_metadata(inode);
3379 }
3380 if (ret != -EEXIST) {
3381 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3382 &inode->runtime_flags);
3383 btrfs_abort_transaction(trans, ret);
3384 return ret;
3385 }
3386 }
3387 ret = 0;
3388 }
3389
3390 /* insert an orphan item to track subvolume contains orphan files */
3391 if (insert >= 2) {
3392 ret = btrfs_insert_orphan_item(trans, fs_info->tree_root,
3393 root->root_key.objectid);
3394 if (ret && ret != -EEXIST) {
3395 btrfs_abort_transaction(trans, ret);
3396 return ret;
3397 }
3398 }
3399 return 0;
3400 }
3401
3402 /*
3403 * We have done the truncate/delete so we can go ahead and remove the orphan
3404 * item for this particular inode.
3405 */
3406 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3407 struct btrfs_inode *inode)
3408 {
3409 struct btrfs_root *root = inode->root;
3410 int delete_item = 0;
3411 int release_rsv = 0;
3412 int ret = 0;
3413
3414 spin_lock(&root->orphan_lock);
3415 if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3416 &inode->runtime_flags))
3417 delete_item = 1;
3418
3419 if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3420 &inode->runtime_flags))
3421 release_rsv = 1;
3422 spin_unlock(&root->orphan_lock);
3423
3424 if (delete_item) {
3425 atomic_dec(&root->orphan_inodes);
3426 if (trans)
3427 ret = btrfs_del_orphan_item(trans, root,
3428 btrfs_ino(inode));
3429 }
3430
3431 if (release_rsv)
3432 btrfs_orphan_release_metadata(inode);
3433
3434 return ret;
3435 }
3436
3437 /*
3438 * this cleans up any orphans that may be left on the list from the last use
3439 * of this root.
3440 */
3441 int btrfs_orphan_cleanup(struct btrfs_root *root)
3442 {
3443 struct btrfs_fs_info *fs_info = root->fs_info;
3444 struct btrfs_path *path;
3445 struct extent_buffer *leaf;
3446 struct btrfs_key key, found_key;
3447 struct btrfs_trans_handle *trans;
3448 struct inode *inode;
3449 u64 last_objectid = 0;
3450 int ret = 0, nr_unlink = 0, nr_truncate = 0;
3451
3452 if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3453 return 0;
3454
3455 path = btrfs_alloc_path();
3456 if (!path) {
3457 ret = -ENOMEM;
3458 goto out;
3459 }
3460 path->reada = READA_BACK;
3461
3462 key.objectid = BTRFS_ORPHAN_OBJECTID;
3463 key.type = BTRFS_ORPHAN_ITEM_KEY;
3464 key.offset = (u64)-1;
3465
3466 while (1) {
3467 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3468 if (ret < 0)
3469 goto out;
3470
3471 /*
3472 * if ret == 0 means we found what we were searching for, which
3473 * is weird, but possible, so only screw with path if we didn't
3474 * find the key and see if we have stuff that matches
3475 */
3476 if (ret > 0) {
3477 ret = 0;
3478 if (path->slots[0] == 0)
3479 break;
3480 path->slots[0]--;
3481 }
3482
3483 /* pull out the item */
3484 leaf = path->nodes[0];
3485 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3486
3487 /* make sure the item matches what we want */
3488 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3489 break;
3490 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3491 break;
3492
3493 /* release the path since we're done with it */
3494 btrfs_release_path(path);
3495
3496 /*
3497 * this is where we are basically btrfs_lookup, without the
3498 * crossing root thing. we store the inode number in the
3499 * offset of the orphan item.
3500 */
3501
3502 if (found_key.offset == last_objectid) {
3503 btrfs_err(fs_info,
3504 "Error removing orphan entry, stopping orphan cleanup");
3505 ret = -EINVAL;
3506 goto out;
3507 }
3508
3509 last_objectid = found_key.offset;
3510
3511 found_key.objectid = found_key.offset;
3512 found_key.type = BTRFS_INODE_ITEM_KEY;
3513 found_key.offset = 0;
3514 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL);
3515 ret = PTR_ERR_OR_ZERO(inode);
3516 if (ret && ret != -ENOENT)
3517 goto out;
3518
3519 if (ret == -ENOENT && root == fs_info->tree_root) {
3520 struct btrfs_root *dead_root;
3521 struct btrfs_fs_info *fs_info = root->fs_info;
3522 int is_dead_root = 0;
3523
3524 /*
3525 * this is an orphan in the tree root. Currently these
3526 * could come from 2 sources:
3527 * a) a snapshot deletion in progress
3528 * b) a free space cache inode
3529 * We need to distinguish those two, as the snapshot
3530 * orphan must not get deleted.
3531 * find_dead_roots already ran before us, so if this
3532 * is a snapshot deletion, we should find the root
3533 * in the dead_roots list
3534 */
3535 spin_lock(&fs_info->trans_lock);
3536 list_for_each_entry(dead_root, &fs_info->dead_roots,
3537 root_list) {
3538 if (dead_root->root_key.objectid ==
3539 found_key.objectid) {
3540 is_dead_root = 1;
3541 break;
3542 }
3543 }
3544 spin_unlock(&fs_info->trans_lock);
3545 if (is_dead_root) {
3546 /* prevent this orphan from being found again */
3547 key.offset = found_key.objectid - 1;
3548 continue;
3549 }
3550 }
3551 /*
3552 * Inode is already gone but the orphan item is still there,
3553 * kill the orphan item.
3554 */
3555 if (ret == -ENOENT) {
3556 trans = btrfs_start_transaction(root, 1);
3557 if (IS_ERR(trans)) {
3558 ret = PTR_ERR(trans);
3559 goto out;
3560 }
3561 btrfs_debug(fs_info, "auto deleting %Lu",
3562 found_key.objectid);
3563 ret = btrfs_del_orphan_item(trans, root,
3564 found_key.objectid);
3565 btrfs_end_transaction(trans);
3566 if (ret)
3567 goto out;
3568 continue;
3569 }
3570
3571 /*
3572 * add this inode to the orphan list so btrfs_orphan_del does
3573 * the proper thing when we hit it
3574 */
3575 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3576 &BTRFS_I(inode)->runtime_flags);
3577 atomic_inc(&root->orphan_inodes);
3578
3579 /* if we have links, this was a truncate, lets do that */
3580 if (inode->i_nlink) {
3581 if (WARN_ON(!S_ISREG(inode->i_mode))) {
3582 iput(inode);
3583 continue;
3584 }
3585 nr_truncate++;
3586
3587 /* 1 for the orphan item deletion. */
3588 trans = btrfs_start_transaction(root, 1);
3589 if (IS_ERR(trans)) {
3590 iput(inode);
3591 ret = PTR_ERR(trans);
3592 goto out;
3593 }
3594 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
3595 btrfs_end_transaction(trans);
3596 if (ret) {
3597 iput(inode);
3598 goto out;
3599 }
3600
3601 ret = btrfs_truncate(inode);
3602 if (ret)
3603 btrfs_orphan_del(NULL, BTRFS_I(inode));
3604 } else {
3605 nr_unlink++;
3606 }
3607
3608 /* this will do delete_inode and everything for us */
3609 iput(inode);
3610 if (ret)
3611 goto out;
3612 }
3613 /* release the path since we're done with it */
3614 btrfs_release_path(path);
3615
3616 root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
3617
3618 if (root->orphan_block_rsv)
3619 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv,
3620 (u64)-1);
3621
3622 if (root->orphan_block_rsv ||
3623 test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3624 trans = btrfs_join_transaction(root);
3625 if (!IS_ERR(trans))
3626 btrfs_end_transaction(trans);
3627 }
3628
3629 if (nr_unlink)
3630 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3631 if (nr_truncate)
3632 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate);
3633
3634 out:
3635 if (ret)
3636 btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3637 btrfs_free_path(path);
3638 return ret;
3639 }
3640
3641 /*
3642 * very simple check to peek ahead in the leaf looking for xattrs. If we
3643 * don't find any xattrs, we know there can't be any acls.
3644 *
3645 * slot is the slot the inode is in, objectid is the objectid of the inode
3646 */
3647 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3648 int slot, u64 objectid,
3649 int *first_xattr_slot)
3650 {
3651 u32 nritems = btrfs_header_nritems(leaf);
3652 struct btrfs_key found_key;
3653 static u64 xattr_access = 0;
3654 static u64 xattr_default = 0;
3655 int scanned = 0;
3656
3657 if (!xattr_access) {
3658 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3659 strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3660 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3661 strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3662 }
3663
3664 slot++;
3665 *first_xattr_slot = -1;
3666 while (slot < nritems) {
3667 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3668
3669 /* we found a different objectid, there must not be acls */
3670 if (found_key.objectid != objectid)
3671 return 0;
3672
3673 /* we found an xattr, assume we've got an acl */
3674 if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3675 if (*first_xattr_slot == -1)
3676 *first_xattr_slot = slot;
3677 if (found_key.offset == xattr_access ||
3678 found_key.offset == xattr_default)
3679 return 1;
3680 }
3681
3682 /*
3683 * we found a key greater than an xattr key, there can't
3684 * be any acls later on
3685 */
3686 if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3687 return 0;
3688
3689 slot++;
3690 scanned++;
3691
3692 /*
3693 * it goes inode, inode backrefs, xattrs, extents,
3694 * so if there are a ton of hard links to an inode there can
3695 * be a lot of backrefs. Don't waste time searching too hard,
3696 * this is just an optimization
3697 */
3698 if (scanned >= 8)
3699 break;
3700 }
3701 /* we hit the end of the leaf before we found an xattr or
3702 * something larger than an xattr. We have to assume the inode
3703 * has acls
3704 */
3705 if (*first_xattr_slot == -1)
3706 *first_xattr_slot = slot;
3707 return 1;
3708 }
3709
3710 /*
3711 * read an inode from the btree into the in-memory inode
3712 */
3713 static int btrfs_read_locked_inode(struct inode *inode)
3714 {
3715 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3716 struct btrfs_path *path;
3717 struct extent_buffer *leaf;
3718 struct btrfs_inode_item *inode_item;
3719 struct btrfs_root *root = BTRFS_I(inode)->root;
3720 struct btrfs_key location;
3721 unsigned long ptr;
3722 int maybe_acls;
3723 u32 rdev;
3724 int ret;
3725 bool filled = false;
3726 int first_xattr_slot;
3727
3728 ret = btrfs_fill_inode(inode, &rdev);
3729 if (!ret)
3730 filled = true;
3731
3732 path = btrfs_alloc_path();
3733 if (!path) {
3734 ret = -ENOMEM;
3735 goto make_bad;
3736 }
3737
3738 memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3739
3740 ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3741 if (ret) {
3742 if (ret > 0)
3743 ret = -ENOENT;
3744 goto make_bad;
3745 }
3746
3747 leaf = path->nodes[0];
3748
3749 if (filled)
3750 goto cache_index;
3751
3752 inode_item = btrfs_item_ptr(leaf, path->slots[0],
3753 struct btrfs_inode_item);
3754 inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3755 set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3756 i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3757 i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3758 btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
3759
3760 inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
3761 inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3762
3763 inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
3764 inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3765
3766 inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
3767 inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3768
3769 BTRFS_I(inode)->i_otime.tv_sec =
3770 btrfs_timespec_sec(leaf, &inode_item->otime);
3771 BTRFS_I(inode)->i_otime.tv_nsec =
3772 btrfs_timespec_nsec(leaf, &inode_item->otime);
3773
3774 inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3775 BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3776 BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3777
3778 inode->i_version = btrfs_inode_sequence(leaf, inode_item);
3779 inode->i_generation = BTRFS_I(inode)->generation;
3780 inode->i_rdev = 0;
3781 rdev = btrfs_inode_rdev(leaf, inode_item);
3782
3783 BTRFS_I(inode)->index_cnt = (u64)-1;
3784 BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
3785
3786 cache_index:
3787 /*
3788 * If we were modified in the current generation and evicted from memory
3789 * and then re-read we need to do a full sync since we don't have any
3790 * idea about which extents were modified before we were evicted from
3791 * cache.
3792 *
3793 * This is required for both inode re-read from disk and delayed inode
3794 * in delayed_nodes_tree.
3795 */
3796 if (BTRFS_I(inode)->last_trans == fs_info->generation)
3797 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3798 &BTRFS_I(inode)->runtime_flags);
3799
3800 /*
3801 * We don't persist the id of the transaction where an unlink operation
3802 * against the inode was last made. So here we assume the inode might
3803 * have been evicted, and therefore the exact value of last_unlink_trans
3804 * lost, and set it to last_trans to avoid metadata inconsistencies
3805 * between the inode and its parent if the inode is fsync'ed and the log
3806 * replayed. For example, in the scenario:
3807 *
3808 * touch mydir/foo
3809 * ln mydir/foo mydir/bar
3810 * sync
3811 * unlink mydir/bar
3812 * echo 2 > /proc/sys/vm/drop_caches # evicts inode
3813 * xfs_io -c fsync mydir/foo
3814 * <power failure>
3815 * mount fs, triggers fsync log replay
3816 *
3817 * We must make sure that when we fsync our inode foo we also log its
3818 * parent inode, otherwise after log replay the parent still has the
3819 * dentry with the "bar" name but our inode foo has a link count of 1
3820 * and doesn't have an inode ref with the name "bar" anymore.
3821 *
3822 * Setting last_unlink_trans to last_trans is a pessimistic approach,
3823 * but it guarantees correctness at the expense of occasional full
3824 * transaction commits on fsync if our inode is a directory, or if our
3825 * inode is not a directory, logging its parent unnecessarily.
3826 */
3827 BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
3828
3829 path->slots[0]++;
3830 if (inode->i_nlink != 1 ||
3831 path->slots[0] >= btrfs_header_nritems(leaf))
3832 goto cache_acl;
3833
3834 btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
3835 if (location.objectid != btrfs_ino(BTRFS_I(inode)))
3836 goto cache_acl;
3837
3838 ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
3839 if (location.type == BTRFS_INODE_REF_KEY) {
3840 struct btrfs_inode_ref *ref;
3841
3842 ref = (struct btrfs_inode_ref *)ptr;
3843 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
3844 } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
3845 struct btrfs_inode_extref *extref;
3846
3847 extref = (struct btrfs_inode_extref *)ptr;
3848 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
3849 extref);
3850 }
3851 cache_acl:
3852 /*
3853 * try to precache a NULL acl entry for files that don't have
3854 * any xattrs or acls
3855 */
3856 maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3857 btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
3858 if (first_xattr_slot != -1) {
3859 path->slots[0] = first_xattr_slot;
3860 ret = btrfs_load_inode_props(inode, path);
3861 if (ret)
3862 btrfs_err(fs_info,
3863 "error loading props for ino %llu (root %llu): %d",
3864 btrfs_ino(BTRFS_I(inode)),
3865 root->root_key.objectid, ret);
3866 }
3867 btrfs_free_path(path);
3868
3869 if (!maybe_acls)
3870 cache_no_acl(inode);
3871
3872 switch (inode->i_mode & S_IFMT) {
3873 case S_IFREG:
3874 inode->i_mapping->a_ops = &btrfs_aops;
3875 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
3876 inode->i_fop = &btrfs_file_operations;
3877 inode->i_op = &btrfs_file_inode_operations;
3878 break;
3879 case S_IFDIR:
3880 inode->i_fop = &btrfs_dir_file_operations;
3881 inode->i_op = &btrfs_dir_inode_operations;
3882 break;
3883 case S_IFLNK:
3884 inode->i_op = &btrfs_symlink_inode_operations;
3885 inode_nohighmem(inode);
3886 inode->i_mapping->a_ops = &btrfs_symlink_aops;
3887 break;
3888 default:
3889 inode->i_op = &btrfs_special_inode_operations;
3890 init_special_inode(inode, inode->i_mode, rdev);
3891 break;
3892 }
3893
3894 btrfs_update_iflags(inode);
3895 return 0;
3896
3897 make_bad:
3898 btrfs_free_path(path);
3899 make_bad_inode(inode);
3900 return ret;
3901 }
3902
3903 /*
3904 * given a leaf and an inode, copy the inode fields into the leaf
3905 */
3906 static void fill_inode_item(struct btrfs_trans_handle *trans,
3907 struct extent_buffer *leaf,
3908 struct btrfs_inode_item *item,
3909 struct inode *inode)
3910 {
3911 struct btrfs_map_token token;
3912
3913 btrfs_init_map_token(&token);
3914
3915 btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3916 btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3917 btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
3918 &token);
3919 btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3920 btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3921
3922 btrfs_set_token_timespec_sec(leaf, &item->atime,
3923 inode->i_atime.tv_sec, &token);
3924 btrfs_set_token_timespec_nsec(leaf, &item->atime,
3925 inode->i_atime.tv_nsec, &token);
3926
3927 btrfs_set_token_timespec_sec(leaf, &item->mtime,
3928 inode->i_mtime.tv_sec, &token);
3929 btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3930 inode->i_mtime.tv_nsec, &token);
3931
3932 btrfs_set_token_timespec_sec(leaf, &item->ctime,
3933 inode->i_ctime.tv_sec, &token);
3934 btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3935 inode->i_ctime.tv_nsec, &token);
3936
3937 btrfs_set_token_timespec_sec(leaf, &item->otime,
3938 BTRFS_I(inode)->i_otime.tv_sec, &token);
3939 btrfs_set_token_timespec_nsec(leaf, &item->otime,
3940 BTRFS_I(inode)->i_otime.tv_nsec, &token);
3941
3942 btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3943 &token);
3944 btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
3945 &token);
3946 btrfs_set_token_inode_sequence(leaf, item, inode->i_version, &token);
3947 btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3948 btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3949 btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3950 btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3951 }
3952
3953 /*
3954 * copy everything in the in-memory inode into the btree.
3955 */
3956 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
3957 struct btrfs_root *root, struct inode *inode)
3958 {
3959 struct btrfs_inode_item *inode_item;
3960 struct btrfs_path *path;
3961 struct extent_buffer *leaf;
3962 int ret;
3963
3964 path = btrfs_alloc_path();
3965 if (!path)
3966 return -ENOMEM;
3967
3968 path->leave_spinning = 1;
3969 ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
3970 1);
3971 if (ret) {
3972 if (ret > 0)
3973 ret = -ENOENT;
3974 goto failed;
3975 }
3976
3977 leaf = path->nodes[0];
3978 inode_item = btrfs_item_ptr(leaf, path->slots[0],
3979 struct btrfs_inode_item);
3980
3981 fill_inode_item(trans, leaf, inode_item, inode);
3982 btrfs_mark_buffer_dirty(leaf);
3983 btrfs_set_inode_last_trans(trans, inode);
3984 ret = 0;
3985 failed:
3986 btrfs_free_path(path);
3987 return ret;
3988 }
3989
3990 /*
3991 * copy everything in the in-memory inode into the btree.
3992 */
3993 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
3994 struct btrfs_root *root, struct inode *inode)
3995 {
3996 struct btrfs_fs_info *fs_info = root->fs_info;
3997 int ret;
3998
3999 /*
4000 * If the inode is a free space inode, we can deadlock during commit
4001 * if we put it into the delayed code.
4002 *
4003 * The data relocation inode should also be directly updated
4004 * without delay
4005 */
4006 if (!btrfs_is_free_space_inode(BTRFS_I(inode))
4007 && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
4008 && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
4009 btrfs_update_root_times(trans, root);
4010
4011 ret = btrfs_delayed_update_inode(trans, root, inode);
4012 if (!ret)
4013 btrfs_set_inode_last_trans(trans, inode);
4014 return ret;
4015 }
4016
4017 return btrfs_update_inode_item(trans, root, inode);
4018 }
4019
4020 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4021 struct btrfs_root *root,
4022 struct inode *inode)
4023 {
4024 int ret;
4025
4026 ret = btrfs_update_inode(trans, root, inode);
4027 if (ret == -ENOSPC)
4028 return btrfs_update_inode_item(trans, root, inode);
4029 return ret;
4030 }
4031
4032 /*
4033 * unlink helper that gets used here in inode.c and in the tree logging
4034 * recovery code. It remove a link in a directory with a given name, and
4035 * also drops the back refs in the inode to the directory
4036 */
4037 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4038 struct btrfs_root *root,
4039 struct btrfs_inode *dir,
4040 struct btrfs_inode *inode,
4041 const char *name, int name_len)
4042 {
4043 struct btrfs_fs_info *fs_info = root->fs_info;
4044 struct btrfs_path *path;
4045 int ret = 0;
4046 struct extent_buffer *leaf;
4047 struct btrfs_dir_item *di;
4048 struct btrfs_key key;
4049 u64 index;
4050 u64 ino = btrfs_ino(inode);
4051 u64 dir_ino = btrfs_ino(dir);
4052
4053 path = btrfs_alloc_path();
4054 if (!path) {
4055 ret = -ENOMEM;
4056 goto out;
4057 }
4058
4059 path->leave_spinning = 1;
4060 di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4061 name, name_len, -1);
4062 if (IS_ERR(di)) {
4063 ret = PTR_ERR(di);
4064 goto err;
4065 }
4066 if (!di) {
4067 ret = -ENOENT;
4068 goto err;
4069 }
4070 leaf = path->nodes[0];
4071 btrfs_dir_item_key_to_cpu(leaf, di, &key);
4072 ret = btrfs_delete_one_dir_name(trans, root, path, di);
4073 if (ret)
4074 goto err;
4075 btrfs_release_path(path);
4076
4077 /*
4078 * If we don't have dir index, we have to get it by looking up
4079 * the inode ref, since we get the inode ref, remove it directly,
4080 * it is unnecessary to do delayed deletion.
4081 *
4082 * But if we have dir index, needn't search inode ref to get it.
4083 * Since the inode ref is close to the inode item, it is better
4084 * that we delay to delete it, and just do this deletion when
4085 * we update the inode item.
4086 */
4087 if (inode->dir_index) {
4088 ret = btrfs_delayed_delete_inode_ref(inode);
4089 if (!ret) {
4090 index = inode->dir_index;
4091 goto skip_backref;
4092 }
4093 }
4094
4095 ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
4096 dir_ino, &index);
4097 if (ret) {
4098 btrfs_info(fs_info,
4099 "failed to delete reference to %.*s, inode %llu parent %llu",
4100 name_len, name, ino, dir_ino);
4101 btrfs_abort_transaction(trans, ret);
4102 goto err;
4103 }
4104 skip_backref:
4105 ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index);
4106 if (ret) {
4107 btrfs_abort_transaction(trans, ret);
4108 goto err;
4109 }
4110
4111 ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode,
4112 dir_ino);
4113 if (ret != 0 && ret != -ENOENT) {
4114 btrfs_abort_transaction(trans, ret);
4115 goto err;
4116 }
4117
4118 ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir,
4119 index);
4120 if (ret == -ENOENT)
4121 ret = 0;
4122 else if (ret)
4123 btrfs_abort_transaction(trans, ret);
4124 err:
4125 btrfs_free_path(path);
4126 if (ret)
4127 goto out;
4128
4129 btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
4130 inode_inc_iversion(&inode->vfs_inode);
4131 inode_inc_iversion(&dir->vfs_inode);
4132 inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
4133 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
4134 ret = btrfs_update_inode(trans, root, &dir->vfs_inode);
4135 out:
4136 return ret;
4137 }
4138
4139 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4140 struct btrfs_root *root,
4141 struct btrfs_inode *dir, struct btrfs_inode *inode,
4142 const char *name, int name_len)
4143 {
4144 int ret;
4145 ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
4146 if (!ret) {
4147 drop_nlink(&inode->vfs_inode);
4148 ret = btrfs_update_inode(trans, root, &inode->vfs_inode);
4149 }
4150 return ret;
4151 }
4152
4153 /*
4154 * helper to start transaction for unlink and rmdir.
4155 *
4156 * unlink and rmdir are special in btrfs, they do not always free space, so
4157 * if we cannot make our reservations the normal way try and see if there is
4158 * plenty of slack room in the global reserve to migrate, otherwise we cannot
4159 * allow the unlink to occur.
4160 */
4161 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
4162 {
4163 struct btrfs_root *root = BTRFS_I(dir)->root;
4164
4165 /*
4166 * 1 for the possible orphan item
4167 * 1 for the dir item
4168 * 1 for the dir index
4169 * 1 for the inode ref
4170 * 1 for the inode
4171 */
4172 return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
4173 }
4174
4175 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4176 {
4177 struct btrfs_root *root = BTRFS_I(dir)->root;
4178 struct btrfs_trans_handle *trans;
4179 struct inode *inode = d_inode(dentry);
4180 int ret;
4181
4182 trans = __unlink_start_trans(dir);
4183 if (IS_ERR(trans))
4184 return PTR_ERR(trans);
4185
4186 btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4187 0);
4188
4189 ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4190 BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4191 dentry->d_name.len);
4192 if (ret)
4193 goto out;
4194
4195 if (inode->i_nlink == 0) {
4196 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4197 if (ret)
4198 goto out;
4199 }
4200
4201 out:
4202 btrfs_end_transaction(trans);
4203 btrfs_btree_balance_dirty(root->fs_info);
4204 return ret;
4205 }
4206
4207 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4208 struct btrfs_root *root,
4209 struct inode *dir, u64 objectid,
4210 const char *name, int name_len)
4211 {
4212 struct btrfs_fs_info *fs_info = root->fs_info;
4213 struct btrfs_path *path;
4214 struct extent_buffer *leaf;
4215 struct btrfs_dir_item *di;
4216 struct btrfs_key key;
4217 u64 index;
4218 int ret;
4219 u64 dir_ino = btrfs_ino(BTRFS_I(dir));
4220
4221 path = btrfs_alloc_path();
4222 if (!path)
4223 return -ENOMEM;
4224
4225 di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4226 name, name_len, -1);
4227 if (IS_ERR_OR_NULL(di)) {
4228 if (!di)
4229 ret = -ENOENT;
4230 else
4231 ret = PTR_ERR(di);
4232 goto out;
4233 }
4234
4235 leaf = path->nodes[0];
4236 btrfs_dir_item_key_to_cpu(leaf, di, &key);
4237 WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4238 ret = btrfs_delete_one_dir_name(trans, root, path, di);
4239 if (ret) {
4240 btrfs_abort_transaction(trans, ret);
4241 goto out;
4242 }
4243 btrfs_release_path(path);
4244
4245 ret = btrfs_del_root_ref(trans, fs_info, objectid,
4246 root->root_key.objectid, dir_ino,
4247 &index, name, name_len);
4248 if (ret < 0) {
4249 if (ret != -ENOENT) {
4250 btrfs_abort_transaction(trans, ret);
4251 goto out;
4252 }
4253 di = btrfs_search_dir_index_item(root, path, dir_ino,
4254 name, name_len);
4255 if (IS_ERR_OR_NULL(di)) {
4256 if (!di)
4257 ret = -ENOENT;
4258 else
4259 ret = PTR_ERR(di);
4260 btrfs_abort_transaction(trans, ret);
4261 goto out;
4262 }
4263
4264 leaf = path->nodes[0];
4265 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4266 btrfs_release_path(path);
4267 index = key.offset;
4268 }
4269 btrfs_release_path(path);
4270
4271 ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index);
4272 if (ret) {
4273 btrfs_abort_transaction(trans, ret);
4274 goto out;
4275 }
4276
4277 btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
4278 inode_inc_iversion(dir);
4279 dir->i_mtime = dir->i_ctime = current_time(dir);
4280 ret = btrfs_update_inode_fallback(trans, root, dir);
4281 if (ret)
4282 btrfs_abort_transaction(trans, ret);
4283 out:
4284 btrfs_free_path(path);
4285 return ret;
4286 }
4287
4288 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4289 {
4290 struct inode *inode = d_inode(dentry);
4291 int err = 0;
4292 struct btrfs_root *root = BTRFS_I(dir)->root;
4293 struct btrfs_trans_handle *trans;
4294 u64 last_unlink_trans;
4295
4296 if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4297 return -ENOTEMPTY;
4298 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID)
4299 return -EPERM;
4300
4301 trans = __unlink_start_trans(dir);
4302 if (IS_ERR(trans))
4303 return PTR_ERR(trans);
4304
4305 if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4306 err = btrfs_unlink_subvol(trans, root, dir,
4307 BTRFS_I(inode)->location.objectid,
4308 dentry->d_name.name,
4309 dentry->d_name.len);
4310 goto out;
4311 }
4312
4313 err = btrfs_orphan_add(trans, BTRFS_I(inode));
4314 if (err)
4315 goto out;
4316
4317 last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
4318
4319 /* now the directory is empty */
4320 err = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4321 BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4322 dentry->d_name.len);
4323 if (!err) {
4324 btrfs_i_size_write(BTRFS_I(inode), 0);
4325 /*
4326 * Propagate the last_unlink_trans value of the deleted dir to
4327 * its parent directory. This is to prevent an unrecoverable
4328 * log tree in the case we do something like this:
4329 * 1) create dir foo
4330 * 2) create snapshot under dir foo
4331 * 3) delete the snapshot
4332 * 4) rmdir foo
4333 * 5) mkdir foo
4334 * 6) fsync foo or some file inside foo
4335 */
4336 if (last_unlink_trans >= trans->transid)
4337 BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
4338 }
4339 out:
4340 btrfs_end_transaction(trans);
4341 btrfs_btree_balance_dirty(root->fs_info);
4342
4343 return err;
4344 }
4345
4346 static int truncate_space_check(struct btrfs_trans_handle *trans,
4347 struct btrfs_root *root,
4348 u64 bytes_deleted)
4349 {
4350 struct btrfs_fs_info *fs_info = root->fs_info;
4351 int ret;
4352
4353 /*
4354 * This is only used to apply pressure to the enospc system, we don't
4355 * intend to use this reservation at all.
4356 */
4357 bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted);
4358 bytes_deleted *= fs_info->nodesize;
4359 ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv,
4360 bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
4361 if (!ret) {
4362 trace_btrfs_space_reservation(fs_info, "transaction",
4363 trans->transid,
4364 bytes_deleted, 1);
4365 trans->bytes_reserved += bytes_deleted;
4366 }
4367 return ret;
4368
4369 }
4370
4371 /*
4372 * Return this if we need to call truncate_block for the last bit of the
4373 * truncate.
4374 */
4375 #define NEED_TRUNCATE_BLOCK 1
4376
4377 /*
4378 * this can truncate away extent items, csum items and directory items.
4379 * It starts at a high offset and removes keys until it can't find
4380 * any higher than new_size
4381 *
4382 * csum items that cross the new i_size are truncated to the new size
4383 * as well.
4384 *
4385 * min_type is the minimum key type to truncate down to. If set to 0, this
4386 * will kill all the items on this inode, including the INODE_ITEM_KEY.
4387 */
4388 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
4389 struct btrfs_root *root,
4390 struct inode *inode,
4391 u64 new_size, u32 min_type)
4392 {
4393 struct btrfs_fs_info *fs_info = root->fs_info;
4394 struct btrfs_path *path;
4395 struct extent_buffer *leaf;
4396 struct btrfs_file_extent_item *fi;
4397 struct btrfs_key key;
4398 struct btrfs_key found_key;
4399 u64 extent_start = 0;
4400 u64 extent_num_bytes = 0;
4401 u64 extent_offset = 0;
4402 u64 item_end = 0;
4403 u64 last_size = new_size;
4404 u32 found_type = (u8)-1;
4405 int found_extent;
4406 int del_item;
4407 int pending_del_nr = 0;
4408 int pending_del_slot = 0;
4409 int extent_type = -1;
4410 int ret;
4411 int err = 0;
4412 u64 ino = btrfs_ino(BTRFS_I(inode));
4413 u64 bytes_deleted = 0;
4414 bool be_nice = false;
4415 bool should_throttle = false;
4416 bool should_end = false;
4417
4418 BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
4419
4420 /*
4421 * for non-free space inodes and ref cows, we want to back off from
4422 * time to time
4423 */
4424 if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
4425 test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4426 be_nice = true;
4427
4428 path = btrfs_alloc_path();
4429 if (!path)
4430 return -ENOMEM;
4431 path->reada = READA_BACK;
4432
4433 /*
4434 * We want to drop from the next block forward in case this new size is
4435 * not block aligned since we will be keeping the last block of the
4436 * extent just the way it is.
4437 */
4438 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4439 root == fs_info->tree_root)
4440 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
4441 fs_info->sectorsize),
4442 (u64)-1, 0);
4443
4444 /*
4445 * This function is also used to drop the items in the log tree before
4446 * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4447 * it is used to drop the loged items. So we shouldn't kill the delayed
4448 * items.
4449 */
4450 if (min_type == 0 && root == BTRFS_I(inode)->root)
4451 btrfs_kill_delayed_inode_items(BTRFS_I(inode));
4452
4453 key.objectid = ino;
4454 key.offset = (u64)-1;
4455 key.type = (u8)-1;
4456
4457 search_again:
4458 /*
4459 * with a 16K leaf size and 128MB extents, you can actually queue
4460 * up a huge file in a single leaf. Most of the time that
4461 * bytes_deleted is > 0, it will be huge by the time we get here
4462 */
4463 if (be_nice && bytes_deleted > SZ_32M) {
4464 if (btrfs_should_end_transaction(trans)) {
4465 err = -EAGAIN;
4466 goto error;
4467 }
4468 }
4469
4470
4471 path->leave_spinning = 1;
4472 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4473 if (ret < 0) {
4474 err = ret;
4475 goto out;
4476 }
4477
4478 if (ret > 0) {
4479 /* there are no items in the tree for us to truncate, we're
4480 * done
4481 */
4482 if (path->slots[0] == 0)
4483 goto out;
4484 path->slots[0]--;
4485 }
4486
4487 while (1) {
4488 fi = NULL;
4489 leaf = path->nodes[0];
4490 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4491 found_type = found_key.type;
4492
4493 if (found_key.objectid != ino)
4494 break;
4495
4496 if (found_type < min_type)
4497 break;
4498
4499 item_end = found_key.offset;
4500 if (found_type == BTRFS_EXTENT_DATA_KEY) {
4501 fi = btrfs_item_ptr(leaf, path->slots[0],
4502 struct btrfs_file_extent_item);
4503 extent_type = btrfs_file_extent_type(leaf, fi);
4504 if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4505 item_end +=
4506 btrfs_file_extent_num_bytes(leaf, fi);
4507
4508 trace_btrfs_truncate_show_fi_regular(
4509 BTRFS_I(inode), leaf, fi,
4510 found_key.offset);
4511 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4512 item_end += btrfs_file_extent_inline_len(leaf,
4513 path->slots[0], fi);
4514
4515 trace_btrfs_truncate_show_fi_inline(
4516 BTRFS_I(inode), leaf, fi, path->slots[0],
4517 found_key.offset);
4518 }
4519 item_end--;
4520 }
4521 if (found_type > min_type) {
4522 del_item = 1;
4523 } else {
4524 if (item_end < new_size)
4525 break;
4526 if (found_key.offset >= new_size)
4527 del_item = 1;
4528 else
4529 del_item = 0;
4530 }
4531 found_extent = 0;
4532 /* FIXME, shrink the extent if the ref count is only 1 */
4533 if (found_type != BTRFS_EXTENT_DATA_KEY)
4534 goto delete;
4535
4536 if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4537 u64 num_dec;
4538 extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4539 if (!del_item) {
4540 u64 orig_num_bytes =
4541 btrfs_file_extent_num_bytes(leaf, fi);
4542 extent_num_bytes = ALIGN(new_size -
4543 found_key.offset,
4544 fs_info->sectorsize);
4545 btrfs_set_file_extent_num_bytes(leaf, fi,
4546 extent_num_bytes);
4547 num_dec = (orig_num_bytes -
4548 extent_num_bytes);
4549 if (test_bit(BTRFS_ROOT_REF_COWS,
4550 &root->state) &&
4551 extent_start != 0)
4552 inode_sub_bytes(inode, num_dec);
4553 btrfs_mark_buffer_dirty(leaf);
4554 } else {
4555 extent_num_bytes =
4556 btrfs_file_extent_disk_num_bytes(leaf,
4557 fi);
4558 extent_offset = found_key.offset -
4559 btrfs_file_extent_offset(leaf, fi);
4560
4561 /* FIXME blocksize != 4096 */
4562 num_dec = btrfs_file_extent_num_bytes(leaf, fi);
4563 if (extent_start != 0) {
4564 found_extent = 1;
4565 if (test_bit(BTRFS_ROOT_REF_COWS,
4566 &root->state))
4567 inode_sub_bytes(inode, num_dec);
4568 }
4569 }
4570 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4571 /*
4572 * we can't truncate inline items that have had
4573 * special encodings
4574 */
4575 if (!del_item &&
4576 btrfs_file_extent_encryption(leaf, fi) == 0 &&
4577 btrfs_file_extent_other_encoding(leaf, fi) == 0 &&
4578 btrfs_file_extent_compression(leaf, fi) == 0) {
4579 u32 size = (u32)(new_size - found_key.offset);
4580
4581 btrfs_set_file_extent_ram_bytes(leaf, fi, size);
4582 size = btrfs_file_extent_calc_inline_size(size);
4583 btrfs_truncate_item(root->fs_info, path, size, 1);
4584 } else if (!del_item) {
4585 /*
4586 * We have to bail so the last_size is set to
4587 * just before this extent.
4588 */
4589 err = NEED_TRUNCATE_BLOCK;
4590 break;
4591 }
4592
4593 if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4594 inode_sub_bytes(inode, item_end + 1 - new_size);
4595 }
4596 delete:
4597 if (del_item)
4598 last_size = found_key.offset;
4599 else
4600 last_size = new_size;
4601 if (del_item) {
4602 if (!pending_del_nr) {
4603 /* no pending yet, add ourselves */
4604 pending_del_slot = path->slots[0];
4605 pending_del_nr = 1;
4606 } else if (pending_del_nr &&
4607 path->slots[0] + 1 == pending_del_slot) {
4608 /* hop on the pending chunk */
4609 pending_del_nr++;
4610 pending_del_slot = path->slots[0];
4611 } else {
4612 BUG();
4613 }
4614 } else {
4615 break;
4616 }
4617 should_throttle = false;
4618
4619 if (found_extent &&
4620 (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4621 root == fs_info->tree_root)) {
4622 btrfs_set_path_blocking(path);
4623 bytes_deleted += extent_num_bytes;
4624 ret = btrfs_free_extent(trans, root, extent_start,
4625 extent_num_bytes, 0,
4626 btrfs_header_owner(leaf),
4627 ino, extent_offset);
4628 BUG_ON(ret);
4629 if (btrfs_should_throttle_delayed_refs(trans, fs_info))
4630 btrfs_async_run_delayed_refs(fs_info,
4631 trans->delayed_ref_updates * 2,
4632 trans->transid, 0);
4633 if (be_nice) {
4634 if (truncate_space_check(trans, root,
4635 extent_num_bytes)) {
4636 should_end = true;
4637 }
4638 if (btrfs_should_throttle_delayed_refs(trans,
4639 fs_info))
4640 should_throttle = true;
4641 }
4642 }
4643
4644 if (found_type == BTRFS_INODE_ITEM_KEY)
4645 break;
4646
4647 if (path->slots[0] == 0 ||
4648 path->slots[0] != pending_del_slot ||
4649 should_throttle || should_end) {
4650 if (pending_del_nr) {
4651 ret = btrfs_del_items(trans, root, path,
4652 pending_del_slot,
4653 pending_del_nr);
4654 if (ret) {
4655 btrfs_abort_transaction(trans, ret);
4656 goto error;
4657 }
4658 pending_del_nr = 0;
4659 }
4660 btrfs_release_path(path);
4661 if (should_throttle) {
4662 unsigned long updates = trans->delayed_ref_updates;
4663 if (updates) {
4664 trans->delayed_ref_updates = 0;
4665 ret = btrfs_run_delayed_refs(trans,
4666 fs_info,
4667 updates * 2);
4668 if (ret && !err)
4669 err = ret;
4670 }
4671 }
4672 /*
4673 * if we failed to refill our space rsv, bail out
4674 * and let the transaction restart
4675 */
4676 if (should_end) {
4677 err = -EAGAIN;
4678 goto error;
4679 }
4680 goto search_again;
4681 } else {
4682 path->slots[0]--;
4683 }
4684 }
4685 out:
4686 if (pending_del_nr) {
4687 ret = btrfs_del_items(trans, root, path, pending_del_slot,
4688 pending_del_nr);
4689 if (ret)
4690 btrfs_abort_transaction(trans, ret);
4691 }
4692 error:
4693 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
4694 ASSERT(last_size >= new_size);
4695 if (!err && last_size > new_size)
4696 last_size = new_size;
4697 btrfs_ordered_update_i_size(inode, last_size, NULL);
4698 }
4699
4700 btrfs_free_path(path);
4701
4702 if (be_nice && bytes_deleted > SZ_32M) {
4703 unsigned long updates = trans->delayed_ref_updates;
4704 if (updates) {
4705 trans->delayed_ref_updates = 0;
4706 ret = btrfs_run_delayed_refs(trans, fs_info,
4707 updates * 2);
4708 if (ret && !err)
4709 err = ret;
4710 }
4711 }
4712 return err;
4713 }
4714
4715 /*
4716 * btrfs_truncate_block - read, zero a chunk and write a block
4717 * @inode - inode that we're zeroing
4718 * @from - the offset to start zeroing
4719 * @len - the length to zero, 0 to zero the entire range respective to the
4720 * offset
4721 * @front - zero up to the offset instead of from the offset on
4722 *
4723 * This will find the block for the "from" offset and cow the block and zero the
4724 * part we want to zero. This is used with truncate and hole punching.
4725 */
4726 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
4727 int front)
4728 {
4729 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4730 struct address_space *mapping = inode->i_mapping;
4731 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4732 struct btrfs_ordered_extent *ordered;
4733 struct extent_state *cached_state = NULL;
4734 struct extent_changeset *data_reserved = NULL;
4735 char *kaddr;
4736 u32 blocksize = fs_info->sectorsize;
4737 pgoff_t index = from >> PAGE_SHIFT;
4738 unsigned offset = from & (blocksize - 1);
4739 struct page *page;
4740 gfp_t mask = btrfs_alloc_write_mask(mapping);
4741 int ret = 0;
4742 u64 block_start;
4743 u64 block_end;
4744
4745 if ((offset & (blocksize - 1)) == 0 &&
4746 (!len || ((len & (blocksize - 1)) == 0)))
4747 goto out;
4748
4749 block_start = round_down(from, blocksize);
4750 block_end = block_start + blocksize - 1;
4751
4752 ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
4753 block_start, blocksize);
4754 if (ret)
4755 goto out;
4756
4757 again:
4758 page = find_or_create_page(mapping, index, mask);
4759 if (!page) {
4760 btrfs_delalloc_release_space(inode, data_reserved,
4761 block_start, blocksize);
4762 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize);
4763 ret = -ENOMEM;
4764 goto out;
4765 }
4766
4767 if (!PageUptodate(page)) {
4768 ret = btrfs_readpage(NULL, page);
4769 lock_page(page);
4770 if (page->mapping != mapping) {
4771 unlock_page(page);
4772 put_page(page);
4773 goto again;
4774 }
4775 if (!PageUptodate(page)) {
4776 ret = -EIO;
4777 goto out_unlock;
4778 }
4779 }
4780 wait_on_page_writeback(page);
4781
4782 lock_extent_bits(io_tree, block_start, block_end, &cached_state);
4783 set_page_extent_mapped(page);
4784
4785 ordered = btrfs_lookup_ordered_extent(inode, block_start);
4786 if (ordered) {
4787 unlock_extent_cached(io_tree, block_start, block_end,
4788 &cached_state, GFP_NOFS);
4789 unlock_page(page);
4790 put_page(page);
4791 btrfs_start_ordered_extent(inode, ordered, 1);
4792 btrfs_put_ordered_extent(ordered);
4793 goto again;
4794 }
4795
4796 clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
4797 EXTENT_DIRTY | EXTENT_DELALLOC |
4798 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4799 0, 0, &cached_state, GFP_NOFS);
4800
4801 ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0,
4802 &cached_state, 0);
4803 if (ret) {
4804 unlock_extent_cached(io_tree, block_start, block_end,
4805 &cached_state, GFP_NOFS);
4806 goto out_unlock;
4807 }
4808
4809 if (offset != blocksize) {
4810 if (!len)
4811 len = blocksize - offset;
4812 kaddr = kmap(page);
4813 if (front)
4814 memset(kaddr + (block_start - page_offset(page)),
4815 0, offset);
4816 else
4817 memset(kaddr + (block_start - page_offset(page)) + offset,
4818 0, len);
4819 flush_dcache_page(page);
4820 kunmap(page);
4821 }
4822 ClearPageChecked(page);
4823 set_page_dirty(page);
4824 unlock_extent_cached(io_tree, block_start, block_end, &cached_state,
4825 GFP_NOFS);
4826
4827 out_unlock:
4828 if (ret)
4829 btrfs_delalloc_release_space(inode, data_reserved, block_start,
4830 blocksize);
4831 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize);
4832 unlock_page(page);
4833 put_page(page);
4834 out:
4835 extent_changeset_free(data_reserved);
4836 return ret;
4837 }
4838
4839 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
4840 u64 offset, u64 len)
4841 {
4842 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4843 struct btrfs_trans_handle *trans;
4844 int ret;
4845
4846 /*
4847 * Still need to make sure the inode looks like it's been updated so
4848 * that any holes get logged if we fsync.
4849 */
4850 if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
4851 BTRFS_I(inode)->last_trans = fs_info->generation;
4852 BTRFS_I(inode)->last_sub_trans = root->log_transid;
4853 BTRFS_I(inode)->last_log_commit = root->last_log_commit;
4854 return 0;
4855 }
4856
4857 /*
4858 * 1 - for the one we're dropping
4859 * 1 - for the one we're adding
4860 * 1 - for updating the inode.
4861 */
4862 trans = btrfs_start_transaction(root, 3);
4863 if (IS_ERR(trans))
4864 return PTR_ERR(trans);
4865
4866 ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
4867 if (ret) {
4868 btrfs_abort_transaction(trans, ret);
4869 btrfs_end_transaction(trans);
4870 return ret;
4871 }
4872
4873 ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
4874 offset, 0, 0, len, 0, len, 0, 0, 0);
4875 if (ret)
4876 btrfs_abort_transaction(trans, ret);
4877 else
4878 btrfs_update_inode(trans, root, inode);
4879 btrfs_end_transaction(trans);
4880 return ret;
4881 }
4882
4883 /*
4884 * This function puts in dummy file extents for the area we're creating a hole
4885 * for. So if we are truncating this file to a larger size we need to insert
4886 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4887 * the range between oldsize and size
4888 */
4889 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
4890 {
4891 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4892 struct btrfs_root *root = BTRFS_I(inode)->root;
4893 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4894 struct extent_map *em = NULL;
4895 struct extent_state *cached_state = NULL;
4896 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
4897 u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
4898 u64 block_end = ALIGN(size, fs_info->sectorsize);
4899 u64 last_byte;
4900 u64 cur_offset;
4901 u64 hole_size;
4902 int err = 0;
4903
4904 /*
4905 * If our size started in the middle of a block we need to zero out the
4906 * rest of the block before we expand the i_size, otherwise we could
4907 * expose stale data.
4908 */
4909 err = btrfs_truncate_block(inode, oldsize, 0, 0);
4910 if (err)
4911 return err;
4912
4913 if (size <= hole_start)
4914 return 0;
4915
4916 while (1) {
4917 struct btrfs_ordered_extent *ordered;
4918
4919 lock_extent_bits(io_tree, hole_start, block_end - 1,
4920 &cached_state);
4921 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start,
4922 block_end - hole_start);
4923 if (!ordered)
4924 break;
4925 unlock_extent_cached(io_tree, hole_start, block_end - 1,
4926 &cached_state, GFP_NOFS);
4927 btrfs_start_ordered_extent(inode, ordered, 1);
4928 btrfs_put_ordered_extent(ordered);
4929 }
4930
4931 cur_offset = hole_start;
4932 while (1) {
4933 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
4934 block_end - cur_offset, 0);
4935 if (IS_ERR(em)) {
4936 err = PTR_ERR(em);
4937 em = NULL;
4938 break;
4939 }
4940 last_byte = min(extent_map_end(em), block_end);
4941 last_byte = ALIGN(last_byte, fs_info->sectorsize);
4942 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
4943 struct extent_map *hole_em;
4944 hole_size = last_byte - cur_offset;
4945
4946 err = maybe_insert_hole(root, inode, cur_offset,
4947 hole_size);
4948 if (err)
4949 break;
4950 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
4951 cur_offset + hole_size - 1, 0);
4952 hole_em = alloc_extent_map();
4953 if (!hole_em) {
4954 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4955 &BTRFS_I(inode)->runtime_flags);
4956 goto next;
4957 }
4958 hole_em->start = cur_offset;
4959 hole_em->len = hole_size;
4960 hole_em->orig_start = cur_offset;
4961
4962 hole_em->block_start = EXTENT_MAP_HOLE;
4963 hole_em->block_len = 0;
4964 hole_em->orig_block_len = 0;
4965 hole_em->ram_bytes = hole_size;
4966 hole_em->bdev = fs_info->fs_devices->latest_bdev;
4967 hole_em->compress_type = BTRFS_COMPRESS_NONE;
4968 hole_em->generation = fs_info->generation;
4969
4970 while (1) {
4971 write_lock(&em_tree->lock);
4972 err = add_extent_mapping(em_tree, hole_em, 1);
4973 write_unlock(&em_tree->lock);
4974 if (err != -EEXIST)
4975 break;
4976 btrfs_drop_extent_cache(BTRFS_I(inode),
4977 cur_offset,
4978 cur_offset +
4979 hole_size - 1, 0);
4980 }
4981 free_extent_map(hole_em);
4982 }
4983 next:
4984 free_extent_map(em);
4985 em = NULL;
4986 cur_offset = last_byte;
4987 if (cur_offset >= block_end)
4988 break;
4989 }
4990 free_extent_map(em);
4991 unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state,
4992 GFP_NOFS);
4993 return err;
4994 }
4995
4996 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
4997 {
4998 struct btrfs_root *root = BTRFS_I(inode)->root;
4999 struct btrfs_trans_handle *trans;
5000 loff_t oldsize = i_size_read(inode);
5001 loff_t newsize = attr->ia_size;
5002 int mask = attr->ia_valid;
5003 int ret;
5004
5005 /*
5006 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5007 * special case where we need to update the times despite not having
5008 * these flags set. For all other operations the VFS set these flags
5009 * explicitly if it wants a timestamp update.
5010 */
5011 if (newsize != oldsize) {
5012 inode_inc_iversion(inode);
5013 if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
5014 inode->i_ctime = inode->i_mtime =
5015 current_time(inode);
5016 }
5017
5018 if (newsize > oldsize) {
5019 /*
5020 * Don't do an expanding truncate while snapshotting is ongoing.
5021 * This is to ensure the snapshot captures a fully consistent
5022 * state of this file - if the snapshot captures this expanding
5023 * truncation, it must capture all writes that happened before
5024 * this truncation.
5025 */
5026 btrfs_wait_for_snapshot_creation(root);
5027 ret = btrfs_cont_expand(inode, oldsize, newsize);
5028 if (ret) {
5029 btrfs_end_write_no_snapshotting(root);
5030 return ret;
5031 }
5032
5033 trans = btrfs_start_transaction(root, 1);
5034 if (IS_ERR(trans)) {
5035 btrfs_end_write_no_snapshotting(root);
5036 return PTR_ERR(trans);
5037 }
5038
5039 i_size_write(inode, newsize);
5040 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
5041 pagecache_isize_extended(inode, oldsize, newsize);
5042 ret = btrfs_update_inode(trans, root, inode);
5043 btrfs_end_write_no_snapshotting(root);
5044 btrfs_end_transaction(trans);
5045 } else {
5046
5047 /*
5048 * We're truncating a file that used to have good data down to
5049 * zero. Make sure it gets into the ordered flush list so that
5050 * any new writes get down to disk quickly.
5051 */
5052 if (newsize == 0)
5053 set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
5054 &BTRFS_I(inode)->runtime_flags);
5055
5056 /*
5057 * 1 for the orphan item we're going to add
5058 * 1 for the orphan item deletion.
5059 */
5060 trans = btrfs_start_transaction(root, 2);
5061 if (IS_ERR(trans))
5062 return PTR_ERR(trans);
5063
5064 /*
5065 * We need to do this in case we fail at _any_ point during the
5066 * actual truncate. Once we do the truncate_setsize we could
5067 * invalidate pages which forces any outstanding ordered io to
5068 * be instantly completed which will give us extents that need
5069 * to be truncated. If we fail to get an orphan inode down we
5070 * could have left over extents that were never meant to live,
5071 * so we need to guarantee from this point on that everything
5072 * will be consistent.
5073 */
5074 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
5075 btrfs_end_transaction(trans);
5076 if (ret)
5077 return ret;
5078
5079 /* we don't support swapfiles, so vmtruncate shouldn't fail */
5080 truncate_setsize(inode, newsize);
5081
5082 /* Disable nonlocked read DIO to avoid the end less truncate */
5083 btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
5084 inode_dio_wait(inode);
5085 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
5086
5087 ret = btrfs_truncate(inode);
5088 if (ret && inode->i_nlink) {
5089 int err;
5090
5091 /* To get a stable disk_i_size */
5092 err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
5093 if (err) {
5094 btrfs_orphan_del(NULL, BTRFS_I(inode));
5095 return err;
5096 }
5097
5098 /*
5099 * failed to truncate, disk_i_size is only adjusted down
5100 * as we remove extents, so it should represent the true
5101 * size of the inode, so reset the in memory size and
5102 * delete our orphan entry.
5103 */
5104 trans = btrfs_join_transaction(root);
5105 if (IS_ERR(trans)) {
5106 btrfs_orphan_del(NULL, BTRFS_I(inode));
5107 return ret;
5108 }
5109 i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5110 err = btrfs_orphan_del(trans, BTRFS_I(inode));
5111 if (err)
5112 btrfs_abort_transaction(trans, err);
5113 btrfs_end_transaction(trans);
5114 }
5115 }
5116
5117 return ret;
5118 }
5119
5120 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
5121 {
5122 struct inode *inode = d_inode(dentry);
5123 struct btrfs_root *root = BTRFS_I(inode)->root;
5124 int err;
5125
5126 if (btrfs_root_readonly(root))
5127 return -EROFS;
5128
5129 err = setattr_prepare(dentry, attr);
5130 if (err)
5131 return err;
5132
5133 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5134 err = btrfs_setsize(inode, attr);
5135 if (err)
5136 return err;
5137 }
5138
5139 if (attr->ia_valid) {
5140 setattr_copy(inode, attr);
5141 inode_inc_iversion(inode);
5142 err = btrfs_dirty_inode(inode);
5143
5144 if (!err && attr->ia_valid & ATTR_MODE)
5145 err = posix_acl_chmod(inode, inode->i_mode);
5146 }
5147
5148 return err;
5149 }
5150
5151 /*
5152 * While truncating the inode pages during eviction, we get the VFS calling
5153 * btrfs_invalidatepage() against each page of the inode. This is slow because
5154 * the calls to btrfs_invalidatepage() result in a huge amount of calls to
5155 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
5156 * extent_state structures over and over, wasting lots of time.
5157 *
5158 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
5159 * those expensive operations on a per page basis and do only the ordered io
5160 * finishing, while we release here the extent_map and extent_state structures,
5161 * without the excessive merging and splitting.
5162 */
5163 static void evict_inode_truncate_pages(struct inode *inode)
5164 {
5165 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5166 struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
5167 struct rb_node *node;
5168
5169 ASSERT(inode->i_state & I_FREEING);
5170 truncate_inode_pages_final(&inode->i_data);
5171
5172 write_lock(&map_tree->lock);
5173 while (!RB_EMPTY_ROOT(&map_tree->map)) {
5174 struct extent_map *em;
5175
5176 node = rb_first(&map_tree->map);
5177 em = rb_entry(node, struct extent_map, rb_node);
5178 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
5179 clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
5180 remove_extent_mapping(map_tree, em);
5181 free_extent_map(em);
5182 if (need_resched()) {
5183 write_unlock(&map_tree->lock);
5184 cond_resched();
5185 write_lock(&map_tree->lock);
5186 }
5187 }
5188 write_unlock(&map_tree->lock);
5189
5190 /*
5191 * Keep looping until we have no more ranges in the io tree.
5192 * We can have ongoing bios started by readpages (called from readahead)
5193 * that have their endio callback (extent_io.c:end_bio_extent_readpage)
5194 * still in progress (unlocked the pages in the bio but did not yet
5195 * unlocked the ranges in the io tree). Therefore this means some
5196 * ranges can still be locked and eviction started because before
5197 * submitting those bios, which are executed by a separate task (work
5198 * queue kthread), inode references (inode->i_count) were not taken
5199 * (which would be dropped in the end io callback of each bio).
5200 * Therefore here we effectively end up waiting for those bios and
5201 * anyone else holding locked ranges without having bumped the inode's
5202 * reference count - if we don't do it, when they access the inode's
5203 * io_tree to unlock a range it may be too late, leading to an
5204 * use-after-free issue.
5205 */
5206 spin_lock(&io_tree->lock);
5207 while (!RB_EMPTY_ROOT(&io_tree->state)) {
5208 struct extent_state *state;
5209 struct extent_state *cached_state = NULL;
5210 u64 start;
5211 u64 end;
5212
5213 node = rb_first(&io_tree->state);
5214 state = rb_entry(node, struct extent_state, rb_node);
5215 start = state->start;
5216 end = state->end;
5217 spin_unlock(&io_tree->lock);
5218
5219 lock_extent_bits(io_tree, start, end, &cached_state);
5220
5221 /*
5222 * If still has DELALLOC flag, the extent didn't reach disk,
5223 * and its reserved space won't be freed by delayed_ref.
5224 * So we need to free its reserved space here.
5225 * (Refer to comment in btrfs_invalidatepage, case 2)
5226 *
5227 * Note, end is the bytenr of last byte, so we need + 1 here.
5228 */
5229 if (state->state & EXTENT_DELALLOC)
5230 btrfs_qgroup_free_data(inode, NULL, start, end - start + 1);
5231
5232 clear_extent_bit(io_tree, start, end,
5233 EXTENT_LOCKED | EXTENT_DIRTY |
5234 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
5235 EXTENT_DEFRAG, 1, 1,
5236 &cached_state, GFP_NOFS);
5237
5238 cond_resched();
5239 spin_lock(&io_tree->lock);
5240 }
5241 spin_unlock(&io_tree->lock);
5242 }
5243
5244 void btrfs_evict_inode(struct inode *inode)
5245 {
5246 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5247 struct btrfs_trans_handle *trans;
5248 struct btrfs_root *root = BTRFS_I(inode)->root;
5249 struct btrfs_block_rsv *rsv, *global_rsv;
5250 int steal_from_global = 0;
5251 u64 min_size;
5252 int ret;
5253
5254 trace_btrfs_inode_evict(inode);
5255
5256 if (!root) {
5257 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
5258 return;
5259 }
5260
5261 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
5262
5263 evict_inode_truncate_pages(inode);
5264
5265 if (inode->i_nlink &&
5266 ((btrfs_root_refs(&root->root_item) != 0 &&
5267 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
5268 btrfs_is_free_space_inode(BTRFS_I(inode))))
5269 goto no_delete;
5270
5271 if (is_bad_inode(inode)) {
5272 btrfs_orphan_del(NULL, BTRFS_I(inode));
5273 goto no_delete;
5274 }
5275 /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5276 if (!special_file(inode->i_mode))
5277 btrfs_wait_ordered_range(inode, 0, (u64)-1);
5278
5279 btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
5280
5281 if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
5282 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
5283 &BTRFS_I(inode)->runtime_flags));
5284 goto no_delete;
5285 }
5286
5287 if (inode->i_nlink > 0) {
5288 BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5289 root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5290 goto no_delete;
5291 }
5292
5293 ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5294 if (ret) {
5295 btrfs_orphan_del(NULL, BTRFS_I(inode));
5296 goto no_delete;
5297 }
5298
5299 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
5300 if (!rsv) {
5301 btrfs_orphan_del(NULL, BTRFS_I(inode));
5302 goto no_delete;
5303 }
5304 rsv->size = min_size;
5305 rsv->failfast = 1;
5306 global_rsv = &fs_info->global_block_rsv;
5307
5308 btrfs_i_size_write(BTRFS_I(inode), 0);
5309
5310 /*
5311 * This is a bit simpler than btrfs_truncate since we've already
5312 * reserved our space for our orphan item in the unlink, so we just
5313 * need to reserve some slack space in case we add bytes and update
5314 * inode item when doing the truncate.
5315 */
5316 while (1) {
5317 ret = btrfs_block_rsv_refill(root, rsv, min_size,
5318 BTRFS_RESERVE_FLUSH_LIMIT);
5319
5320 /*
5321 * Try and steal from the global reserve since we will
5322 * likely not use this space anyway, we want to try as
5323 * hard as possible to get this to work.
5324 */
5325 if (ret)
5326 steal_from_global++;
5327 else
5328 steal_from_global = 0;
5329 ret = 0;
5330
5331 /*
5332 * steal_from_global == 0: we reserved stuff, hooray!
5333 * steal_from_global == 1: we didn't reserve stuff, boo!
5334 * steal_from_global == 2: we've committed, still not a lot of
5335 * room but maybe we'll have room in the global reserve this
5336 * time.
5337 * steal_from_global == 3: abandon all hope!
5338 */
5339 if (steal_from_global > 2) {
5340 btrfs_warn(fs_info,
5341 "Could not get space for a delete, will truncate on mount %d",
5342 ret);
5343 btrfs_orphan_del(NULL, BTRFS_I(inode));
5344 btrfs_free_block_rsv(fs_info, rsv);
5345 goto no_delete;
5346 }
5347
5348 trans = btrfs_join_transaction(root);
5349 if (IS_ERR(trans)) {
5350 btrfs_orphan_del(NULL, BTRFS_I(inode));
5351 btrfs_free_block_rsv(fs_info, rsv);
5352 goto no_delete;
5353 }
5354
5355 /*
5356 * We can't just steal from the global reserve, we need to make
5357 * sure there is room to do it, if not we need to commit and try
5358 * again.
5359 */
5360 if (steal_from_global) {
5361 if (!btrfs_check_space_for_delayed_refs(trans, fs_info))
5362 ret = btrfs_block_rsv_migrate(global_rsv, rsv,
5363 min_size, 0);
5364 else
5365 ret = -ENOSPC;
5366 }
5367
5368 /*
5369 * Couldn't steal from the global reserve, we have too much
5370 * pending stuff built up, commit the transaction and try it
5371 * again.
5372 */
5373 if (ret) {
5374 ret = btrfs_commit_transaction(trans);
5375 if (ret) {
5376 btrfs_orphan_del(NULL, BTRFS_I(inode));
5377 btrfs_free_block_rsv(fs_info, rsv);
5378 goto no_delete;
5379 }
5380 continue;
5381 } else {
5382 steal_from_global = 0;
5383 }
5384
5385 trans->block_rsv = rsv;
5386
5387 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5388 if (ret != -ENOSPC && ret != -EAGAIN)
5389 break;
5390
5391 trans->block_rsv = &fs_info->trans_block_rsv;
5392 btrfs_end_transaction(trans);
5393 trans = NULL;
5394 btrfs_btree_balance_dirty(fs_info);
5395 }
5396
5397 btrfs_free_block_rsv(fs_info, rsv);
5398
5399 /*
5400 * Errors here aren't a big deal, it just means we leave orphan items
5401 * in the tree. They will be cleaned up on the next mount.
5402 */
5403 if (ret == 0) {
5404 trans->block_rsv = root->orphan_block_rsv;
5405 btrfs_orphan_del(trans, BTRFS_I(inode));
5406 } else {
5407 btrfs_orphan_del(NULL, BTRFS_I(inode));
5408 }
5409
5410 trans->block_rsv = &fs_info->trans_block_rsv;
5411 if (!(root == fs_info->tree_root ||
5412 root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5413 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode)));
5414
5415 btrfs_end_transaction(trans);
5416 btrfs_btree_balance_dirty(fs_info);
5417 no_delete:
5418 btrfs_remove_delayed_node(BTRFS_I(inode));
5419 clear_inode(inode);
5420 }
5421
5422 /*
5423 * this returns the key found in the dir entry in the location pointer.
5424 * If no dir entries were found, location->objectid is 0.
5425 */
5426 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
5427 struct btrfs_key *location)
5428 {
5429 const char *name = dentry->d_name.name;
5430 int namelen = dentry->d_name.len;
5431 struct btrfs_dir_item *di;
5432 struct btrfs_path *path;
5433 struct btrfs_root *root = BTRFS_I(dir)->root;
5434 int ret = 0;
5435
5436 path = btrfs_alloc_path();
5437 if (!path)
5438 return -ENOMEM;
5439
5440 di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)),
5441 name, namelen, 0);
5442 if (IS_ERR(di))
5443 ret = PTR_ERR(di);
5444
5445 if (IS_ERR_OR_NULL(di))
5446 goto out_err;
5447
5448 btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5449 if (location->type != BTRFS_INODE_ITEM_KEY &&
5450 location->type != BTRFS_ROOT_ITEM_KEY) {
5451 btrfs_warn(root->fs_info,
5452 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))",
5453 __func__, name, btrfs_ino(BTRFS_I(dir)),
5454 location->objectid, location->type, location->offset);
5455 goto out_err;
5456 }
5457 out:
5458 btrfs_free_path(path);
5459 return ret;
5460 out_err:
5461 location->objectid = 0;
5462 goto out;
5463 }
5464
5465 /*
5466 * when we hit a tree root in a directory, the btrfs part of the inode
5467 * needs to be changed to reflect the root directory of the tree root. This
5468 * is kind of like crossing a mount point.
5469 */
5470 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5471 struct inode *dir,
5472 struct dentry *dentry,
5473 struct btrfs_key *location,
5474 struct btrfs_root **sub_root)
5475 {
5476 struct btrfs_path *path;
5477 struct btrfs_root *new_root;
5478 struct btrfs_root_ref *ref;
5479 struct extent_buffer *leaf;
5480 struct btrfs_key key;
5481 int ret;
5482 int err = 0;
5483
5484 path = btrfs_alloc_path();
5485 if (!path) {
5486 err = -ENOMEM;
5487 goto out;
5488 }
5489
5490 err = -ENOENT;
5491 key.objectid = BTRFS_I(dir)->root->root_key.objectid;
5492 key.type = BTRFS_ROOT_REF_KEY;
5493 key.offset = location->objectid;
5494
5495 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5496 if (ret) {
5497 if (ret < 0)
5498 err = ret;
5499 goto out;
5500 }
5501
5502 leaf = path->nodes[0];
5503 ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5504 if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) ||
5505 btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
5506 goto out;
5507
5508 ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
5509 (unsigned long)(ref + 1),
5510 dentry->d_name.len);
5511 if (ret)
5512 goto out;
5513
5514 btrfs_release_path(path);
5515
5516 new_root = btrfs_read_fs_root_no_name(fs_info, location);
5517 if (IS_ERR(new_root)) {
5518 err = PTR_ERR(new_root);
5519 goto out;
5520 }
5521
5522 *sub_root = new_root;
5523 location->objectid = btrfs_root_dirid(&new_root->root_item);
5524 location->type = BTRFS_INODE_ITEM_KEY;
5525 location->offset = 0;
5526 err = 0;
5527 out:
5528 btrfs_free_path(path);
5529 return err;
5530 }
5531
5532 static void inode_tree_add(struct inode *inode)
5533 {
5534 struct btrfs_root *root = BTRFS_I(inode)->root;
5535 struct btrfs_inode *entry;
5536 struct rb_node **p;
5537 struct rb_node *parent;
5538 struct rb_node *new = &BTRFS_I(inode)->rb_node;
5539 u64 ino = btrfs_ino(BTRFS_I(inode));
5540
5541 if (inode_unhashed(inode))
5542 return;
5543 parent = NULL;
5544 spin_lock(&root->inode_lock);
5545 p = &root->inode_tree.rb_node;
5546 while (*p) {
5547 parent = *p;
5548 entry = rb_entry(parent, struct btrfs_inode, rb_node);
5549
5550 if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5551 p = &parent->rb_left;
5552 else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5553 p = &parent->rb_right;
5554 else {
5555 WARN_ON(!(entry->vfs_inode.i_state &
5556 (I_WILL_FREE | I_FREEING)));
5557 rb_replace_node(parent, new, &root->inode_tree);
5558 RB_CLEAR_NODE(parent);
5559 spin_unlock(&root->inode_lock);
5560 return;
5561 }
5562 }
5563 rb_link_node(new, parent, p);
5564 rb_insert_color(new, &root->inode_tree);
5565 spin_unlock(&root->inode_lock);
5566 }
5567
5568 static void inode_tree_del(struct inode *inode)
5569 {
5570 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5571 struct btrfs_root *root = BTRFS_I(inode)->root;
5572 int empty = 0;
5573
5574 spin_lock(&root->inode_lock);
5575 if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
5576 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
5577 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5578 empty = RB_EMPTY_ROOT(&root->inode_tree);
5579 }
5580 spin_unlock(&root->inode_lock);
5581
5582 if (empty && btrfs_root_refs(&root->root_item) == 0) {
5583 synchronize_srcu(&fs_info->subvol_srcu);
5584 spin_lock(&root->inode_lock);
5585 empty = RB_EMPTY_ROOT(&root->inode_tree);
5586 spin_unlock(&root->inode_lock);
5587 if (empty)
5588 btrfs_add_dead_root(root);
5589 }
5590 }
5591
5592 void btrfs_invalidate_inodes(struct btrfs_root *root)
5593 {
5594 struct btrfs_fs_info *fs_info = root->fs_info;
5595 struct rb_node *node;
5596 struct rb_node *prev;
5597 struct btrfs_inode *entry;
5598 struct inode *inode;
5599 u64 objectid = 0;
5600
5601 if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
5602 WARN_ON(btrfs_root_refs(&root->root_item) != 0);
5603
5604 spin_lock(&root->inode_lock);
5605 again:
5606 node = root->inode_tree.rb_node;
5607 prev = NULL;
5608 while (node) {
5609 prev = node;
5610 entry = rb_entry(node, struct btrfs_inode, rb_node);
5611
5612 if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5613 node = node->rb_left;
5614 else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5615 node = node->rb_right;
5616 else
5617 break;
5618 }
5619 if (!node) {
5620 while (prev) {
5621 entry = rb_entry(prev, struct btrfs_inode, rb_node);
5622 if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) {
5623 node = prev;
5624 break;
5625 }
5626 prev = rb_next(prev);
5627 }
5628 }
5629 while (node) {
5630 entry = rb_entry(node, struct btrfs_inode, rb_node);
5631 objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1;
5632 inode = igrab(&entry->vfs_inode);
5633 if (inode) {
5634 spin_unlock(&root->inode_lock);
5635 if (atomic_read(&inode->i_count) > 1)
5636 d_prune_aliases(inode);
5637 /*
5638 * btrfs_drop_inode will have it removed from
5639 * the inode cache when its usage count
5640 * hits zero.
5641 */
5642 iput(inode);
5643 cond_resched();
5644 spin_lock(&root->inode_lock);
5645 goto again;
5646 }
5647
5648 if (cond_resched_lock(&root->inode_lock))
5649 goto again;
5650
5651 node = rb_next(node);
5652 }
5653 spin_unlock(&root->inode_lock);
5654 }
5655
5656 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5657 {
5658 struct btrfs_iget_args *args = p;
5659 inode->i_ino = args->location->objectid;
5660 memcpy(&BTRFS_I(inode)->location, args->location,
5661 sizeof(*args->location));
5662 BTRFS_I(inode)->root = args->root;
5663 return 0;
5664 }
5665
5666 static int btrfs_find_actor(struct inode *inode, void *opaque)
5667 {
5668 struct btrfs_iget_args *args = opaque;
5669 return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5670 args->root == BTRFS_I(inode)->root;
5671 }
5672
5673 static struct inode *btrfs_iget_locked(struct super_block *s,
5674 struct btrfs_key *location,
5675 struct btrfs_root *root)
5676 {
5677 struct inode *inode;
5678 struct btrfs_iget_args args;
5679 unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5680
5681 args.location = location;
5682 args.root = root;
5683
5684 inode = iget5_locked(s, hashval, btrfs_find_actor,
5685 btrfs_init_locked_inode,
5686 (void *)&args);
5687 return inode;
5688 }
5689
5690 /* Get an inode object given its location and corresponding root.
5691 * Returns in *is_new if the inode was read from disk
5692 */
5693 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5694 struct btrfs_root *root, int *new)
5695 {
5696 struct inode *inode;
5697
5698 inode = btrfs_iget_locked(s, location, root);
5699 if (!inode)
5700 return ERR_PTR(-ENOMEM);
5701
5702 if (inode->i_state & I_NEW) {
5703 int ret;
5704
5705 ret = btrfs_read_locked_inode(inode);
5706 if (!is_bad_inode(inode)) {
5707 inode_tree_add(inode);
5708 unlock_new_inode(inode);
5709 if (new)
5710 *new = 1;
5711 } else {
5712 unlock_new_inode(inode);
5713 iput(inode);
5714 ASSERT(ret < 0);
5715 inode = ERR_PTR(ret < 0 ? ret : -ESTALE);
5716 }
5717 }
5718
5719 return inode;
5720 }
5721
5722 static struct inode *new_simple_dir(struct super_block *s,
5723 struct btrfs_key *key,
5724 struct btrfs_root *root)
5725 {
5726 struct inode *inode = new_inode(s);
5727
5728 if (!inode)
5729 return ERR_PTR(-ENOMEM);
5730
5731 BTRFS_I(inode)->root = root;
5732 memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
5733 set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5734
5735 inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
5736 inode->i_op = &btrfs_dir_ro_inode_operations;
5737 inode->i_opflags &= ~IOP_XATTR;
5738 inode->i_fop = &simple_dir_operations;
5739 inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5740 inode->i_mtime = current_time(inode);
5741 inode->i_atime = inode->i_mtime;
5742 inode->i_ctime = inode->i_mtime;
5743 BTRFS_I(inode)->i_otime = inode->i_mtime;
5744
5745 return inode;
5746 }
5747
5748 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5749 {
5750 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
5751 struct inode *inode;
5752 struct btrfs_root *root = BTRFS_I(dir)->root;
5753 struct btrfs_root *sub_root = root;
5754 struct btrfs_key location;
5755 int index;
5756 int ret = 0;
5757
5758 if (dentry->d_name.len > BTRFS_NAME_LEN)
5759 return ERR_PTR(-ENAMETOOLONG);
5760
5761 ret = btrfs_inode_by_name(dir, dentry, &location);
5762 if (ret < 0)
5763 return ERR_PTR(ret);
5764
5765 if (location.objectid == 0)
5766 return ERR_PTR(-ENOENT);
5767
5768 if (location.type == BTRFS_INODE_ITEM_KEY) {
5769 inode = btrfs_iget(dir->i_sb, &location, root, NULL);
5770 return inode;
5771 }
5772
5773 index = srcu_read_lock(&fs_info->subvol_srcu);
5774 ret = fixup_tree_root_location(fs_info, dir, dentry,
5775 &location, &sub_root);
5776 if (ret < 0) {
5777 if (ret != -ENOENT)
5778 inode = ERR_PTR(ret);
5779 else
5780 inode = new_simple_dir(dir->i_sb, &location, sub_root);
5781 } else {
5782 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
5783 }
5784 srcu_read_unlock(&fs_info->subvol_srcu, index);
5785
5786 if (!IS_ERR(inode) && root != sub_root) {
5787 down_read(&fs_info->cleanup_work_sem);
5788 if (!sb_rdonly(inode->i_sb))
5789 ret = btrfs_orphan_cleanup(sub_root);
5790 up_read(&fs_info->cleanup_work_sem);
5791 if (ret) {
5792 iput(inode);
5793 inode = ERR_PTR(ret);
5794 }
5795 }
5796
5797 return inode;
5798 }
5799
5800 static int btrfs_dentry_delete(const struct dentry *dentry)
5801 {
5802 struct btrfs_root *root;
5803 struct inode *inode = d_inode(dentry);
5804
5805 if (!inode && !IS_ROOT(dentry))
5806 inode = d_inode(dentry->d_parent);
5807
5808 if (inode) {
5809 root = BTRFS_I(inode)->root;
5810 if (btrfs_root_refs(&root->root_item) == 0)
5811 return 1;
5812
5813 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
5814 return 1;
5815 }
5816 return 0;
5817 }
5818
5819 static void btrfs_dentry_release(struct dentry *dentry)
5820 {
5821 kfree(dentry->d_fsdata);
5822 }
5823
5824 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5825 unsigned int flags)
5826 {
5827 struct inode *inode;
5828
5829 inode = btrfs_lookup_dentry(dir, dentry);
5830 if (IS_ERR(inode)) {
5831 if (PTR_ERR(inode) == -ENOENT)
5832 inode = NULL;
5833 else
5834 return ERR_CAST(inode);
5835 }
5836
5837 return d_splice_alias(inode, dentry);
5838 }
5839
5840 unsigned char btrfs_filetype_table[] = {
5841 DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
5842 };
5843
5844 /*
5845 * All this infrastructure exists because dir_emit can fault, and we are holding
5846 * the tree lock when doing readdir. For now just allocate a buffer and copy
5847 * our information into that, and then dir_emit from the buffer. This is
5848 * similar to what NFS does, only we don't keep the buffer around in pagecache
5849 * because I'm afraid I'll mess that up. Long term we need to make filldir do
5850 * copy_to_user_inatomic so we don't have to worry about page faulting under the
5851 * tree lock.
5852 */
5853 static int btrfs_opendir(struct inode *inode, struct file *file)
5854 {
5855 struct btrfs_file_private *private;
5856
5857 private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);
5858 if (!private)
5859 return -ENOMEM;
5860 private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
5861 if (!private->filldir_buf) {
5862 kfree(private);
5863 return -ENOMEM;
5864 }
5865 file->private_data = private;
5866 return 0;
5867 }
5868
5869 struct dir_entry {
5870 u64 ino;
5871 u64 offset;
5872 unsigned type;
5873 int name_len;
5874 };
5875
5876 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)
5877 {
5878 while (entries--) {
5879 struct dir_entry *entry = addr;
5880 char *name = (char *)(entry + 1);
5881
5882 ctx->pos = entry->offset;
5883 if (!dir_emit(ctx, name, entry->name_len, entry->ino,
5884 entry->type))
5885 return 1;
5886 addr += sizeof(struct dir_entry) + entry->name_len;
5887 ctx->pos++;
5888 }
5889 return 0;
5890 }
5891
5892 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
5893 {
5894 struct inode *inode = file_inode(file);
5895 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5896 struct btrfs_root *root = BTRFS_I(inode)->root;
5897 struct btrfs_file_private *private = file->private_data;
5898 struct btrfs_dir_item *di;
5899 struct btrfs_key key;
5900 struct btrfs_key found_key;
5901 struct btrfs_path *path;
5902 void *addr;
5903 struct list_head ins_list;
5904 struct list_head del_list;
5905 int ret;
5906 struct extent_buffer *leaf;
5907 int slot;
5908 char *name_ptr;
5909 int name_len;
5910 int entries = 0;
5911 int total_len = 0;
5912 bool put = false;
5913 struct btrfs_key location;
5914
5915 if (!dir_emit_dots(file, ctx))
5916 return 0;
5917
5918 path = btrfs_alloc_path();
5919 if (!path)
5920 return -ENOMEM;
5921
5922 addr = private->filldir_buf;
5923 path->reada = READA_FORWARD;
5924
5925 INIT_LIST_HEAD(&ins_list);
5926 INIT_LIST_HEAD(&del_list);
5927 put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
5928
5929 again:
5930 key.type = BTRFS_DIR_INDEX_KEY;
5931 key.offset = ctx->pos;
5932 key.objectid = btrfs_ino(BTRFS_I(inode));
5933
5934 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5935 if (ret < 0)
5936 goto err;
5937
5938 while (1) {
5939 struct dir_entry *entry;
5940
5941 leaf = path->nodes[0];
5942 slot = path->slots[0];
5943 if (slot >= btrfs_header_nritems(leaf)) {
5944 ret = btrfs_next_leaf(root, path);
5945 if (ret < 0)
5946 goto err;
5947 else if (ret > 0)
5948 break;
5949 continue;
5950 }
5951
5952 btrfs_item_key_to_cpu(leaf, &found_key, slot);
5953
5954 if (found_key.objectid != key.objectid)
5955 break;
5956 if (found_key.type != BTRFS_DIR_INDEX_KEY)
5957 break;
5958 if (found_key.offset < ctx->pos)
5959 goto next;
5960 if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
5961 goto next;
5962 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
5963 if (verify_dir_item(fs_info, leaf, slot, di))
5964 goto next;
5965
5966 name_len = btrfs_dir_name_len(leaf, di);
5967 if ((total_len + sizeof(struct dir_entry) + name_len) >=
5968 PAGE_SIZE) {
5969 btrfs_release_path(path);
5970 ret = btrfs_filldir(private->filldir_buf, entries, ctx);
5971 if (ret)
5972 goto nopos;
5973 addr = private->filldir_buf;
5974 entries = 0;
5975 total_len = 0;
5976 goto again;
5977 }
5978
5979 entry = addr;
5980 entry->name_len = name_len;
5981 name_ptr = (char *)(entry + 1);
5982 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
5983 name_len);
5984 entry->type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
5985 btrfs_dir_item_key_to_cpu(leaf, di, &location);
5986 entry->ino = location.objectid;
5987 entry->offset = found_key.offset;
5988 entries++;
5989 addr += sizeof(struct dir_entry) + name_len;
5990 total_len += sizeof(struct dir_entry) + name_len;
5991 next:
5992 path->slots[0]++;
5993 }
5994 btrfs_release_path(path);
5995
5996 ret = btrfs_filldir(private->filldir_buf, entries, ctx);
5997 if (ret)
5998 goto nopos;
5999
6000 ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
6001 if (ret)
6002 goto nopos;
6003
6004 /*
6005 * Stop new entries from being returned after we return the last
6006 * entry.
6007 *
6008 * New directory entries are assigned a strictly increasing
6009 * offset. This means that new entries created during readdir
6010 * are *guaranteed* to be seen in the future by that readdir.
6011 * This has broken buggy programs which operate on names as
6012 * they're returned by readdir. Until we re-use freed offsets
6013 * we have this hack to stop new entries from being returned
6014 * under the assumption that they'll never reach this huge
6015 * offset.
6016 *
6017 * This is being careful not to overflow 32bit loff_t unless the
6018 * last entry requires it because doing so has broken 32bit apps
6019 * in the past.
6020 */
6021 if (ctx->pos >= INT_MAX)
6022 ctx->pos = LLONG_MAX;
6023 else
6024 ctx->pos = INT_MAX;
6025 nopos:
6026 ret = 0;
6027 err:
6028 if (put)
6029 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
6030 btrfs_free_path(path);
6031 return ret;
6032 }
6033
6034 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
6035 {
6036 struct btrfs_root *root = BTRFS_I(inode)->root;
6037 struct btrfs_trans_handle *trans;
6038 int ret = 0;
6039 bool nolock = false;
6040
6041 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6042 return 0;
6043
6044 if (btrfs_fs_closing(root->fs_info) &&
6045 btrfs_is_free_space_inode(BTRFS_I(inode)))
6046 nolock = true;
6047
6048 if (wbc->sync_mode == WB_SYNC_ALL) {
6049 if (nolock)
6050 trans = btrfs_join_transaction_nolock(root);
6051 else
6052 trans = btrfs_join_transaction(root);
6053 if (IS_ERR(trans))
6054 return PTR_ERR(trans);
6055 ret = btrfs_commit_transaction(trans);
6056 }
6057 return ret;
6058 }
6059
6060 /*
6061 * This is somewhat expensive, updating the tree every time the
6062 * inode changes. But, it is most likely to find the inode in cache.
6063 * FIXME, needs more benchmarking...there are no reasons other than performance
6064 * to keep or drop this code.
6065 */
6066 static int btrfs_dirty_inode(struct inode *inode)
6067 {
6068 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6069 struct btrfs_root *root = BTRFS_I(inode)->root;
6070 struct btrfs_trans_handle *trans;
6071 int ret;
6072
6073 if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6074 return 0;
6075
6076 trans = btrfs_join_transaction(root);
6077 if (IS_ERR(trans))
6078 return PTR_ERR(trans);
6079
6080 ret = btrfs_update_inode(trans, root, inode);
6081 if (ret && ret == -ENOSPC) {
6082 /* whoops, lets try again with the full transaction */
6083 btrfs_end_transaction(trans);
6084 trans = btrfs_start_transaction(root, 1);
6085 if (IS_ERR(trans))
6086 return PTR_ERR(trans);
6087
6088 ret = btrfs_update_inode(trans, root, inode);
6089 }
6090 btrfs_end_transaction(trans);
6091 if (BTRFS_I(inode)->delayed_node)
6092 btrfs_balance_delayed_items(fs_info);
6093
6094 return ret;
6095 }
6096
6097 /*
6098 * This is a copy of file_update_time. We need this so we can return error on
6099 * ENOSPC for updating the inode in the case of file write and mmap writes.
6100 */
6101 static int btrfs_update_time(struct inode *inode, struct timespec *now,
6102 int flags)
6103 {
6104 struct btrfs_root *root = BTRFS_I(inode)->root;
6105
6106 if (btrfs_root_readonly(root))
6107 return -EROFS;
6108
6109 if (flags & S_VERSION)
6110 inode_inc_iversion(inode);
6111 if (flags & S_CTIME)
6112 inode->i_ctime = *now;
6113 if (flags & S_MTIME)
6114 inode->i_mtime = *now;
6115 if (flags & S_ATIME)
6116 inode->i_atime = *now;
6117 return btrfs_dirty_inode(inode);
6118 }
6119
6120 /*
6121 * find the highest existing sequence number in a directory
6122 * and then set the in-memory index_cnt variable to reflect
6123 * free sequence numbers
6124 */
6125 static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
6126 {
6127 struct btrfs_root *root = inode->root;
6128 struct btrfs_key key, found_key;
6129 struct btrfs_path *path;
6130 struct extent_buffer *leaf;
6131 int ret;
6132
6133 key.objectid = btrfs_ino(inode);
6134 key.type = BTRFS_DIR_INDEX_KEY;
6135 key.offset = (u64)-1;
6136
6137 path = btrfs_alloc_path();
6138 if (!path)
6139 return -ENOMEM;
6140
6141 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6142 if (ret < 0)
6143 goto out;
6144 /* FIXME: we should be able to handle this */
6145 if (ret == 0)
6146 goto out;
6147 ret = 0;
6148
6149 /*
6150 * MAGIC NUMBER EXPLANATION:
6151 * since we search a directory based on f_pos we have to start at 2
6152 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
6153 * else has to start at 2
6154 */
6155 if (path->slots[0] == 0) {
6156 inode->index_cnt = 2;
6157 goto out;
6158 }
6159
6160 path->slots[0]--;
6161
6162 leaf = path->nodes[0];
6163 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6164
6165 if (found_key.objectid != btrfs_ino(inode) ||
6166 found_key.type != BTRFS_DIR_INDEX_KEY) {
6167 inode->index_cnt = 2;
6168 goto out;
6169 }
6170
6171 inode->index_cnt = found_key.offset + 1;
6172 out:
6173 btrfs_free_path(path);
6174 return ret;
6175 }
6176
6177 /*
6178 * helper to find a free sequence number in a given directory. This current
6179 * code is very simple, later versions will do smarter things in the btree
6180 */
6181 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
6182 {
6183 int ret = 0;
6184
6185 if (dir->index_cnt == (u64)-1) {
6186 ret = btrfs_inode_delayed_dir_index_count(dir);
6187 if (ret) {
6188 ret = btrfs_set_inode_index_count(dir);
6189 if (ret)
6190 return ret;
6191 }
6192 }
6193
6194 *index = dir->index_cnt;
6195 dir->index_cnt++;
6196
6197 return ret;
6198 }
6199
6200 static int btrfs_insert_inode_locked(struct inode *inode)
6201 {
6202 struct btrfs_iget_args args;
6203 args.location = &BTRFS_I(inode)->location;
6204 args.root = BTRFS_I(inode)->root;
6205
6206 return insert_inode_locked4(inode,
6207 btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6208 btrfs_find_actor, &args);
6209 }
6210
6211 /*
6212 * Inherit flags from the parent inode.
6213 *
6214 * Currently only the compression flags and the cow flags are inherited.
6215 */
6216 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir)
6217 {
6218 unsigned int flags;
6219
6220 if (!dir)
6221 return;
6222
6223 flags = BTRFS_I(dir)->flags;
6224
6225 if (flags & BTRFS_INODE_NOCOMPRESS) {
6226 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS;
6227 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
6228 } else if (flags & BTRFS_INODE_COMPRESS) {
6229 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS;
6230 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS;
6231 }
6232
6233 if (flags & BTRFS_INODE_NODATACOW) {
6234 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW;
6235 if (S_ISREG(inode->i_mode))
6236 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6237 }
6238
6239 btrfs_update_iflags(inode);
6240 }
6241
6242 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
6243 struct btrfs_root *root,
6244 struct inode *dir,
6245 const char *name, int name_len,
6246 u64 ref_objectid, u64 objectid,
6247 umode_t mode, u64 *index)
6248 {
6249 struct btrfs_fs_info *fs_info = root->fs_info;
6250 struct inode *inode;
6251 struct btrfs_inode_item *inode_item;
6252 struct btrfs_key *location;
6253 struct btrfs_path *path;
6254 struct btrfs_inode_ref *ref;
6255 struct btrfs_key key[2];
6256 u32 sizes[2];
6257 int nitems = name ? 2 : 1;
6258 unsigned long ptr;
6259 int ret;
6260
6261 path = btrfs_alloc_path();
6262 if (!path)
6263 return ERR_PTR(-ENOMEM);
6264
6265 inode = new_inode(fs_info->sb);
6266 if (!inode) {
6267 btrfs_free_path(path);
6268 return ERR_PTR(-ENOMEM);
6269 }
6270
6271 /*
6272 * O_TMPFILE, set link count to 0, so that after this point,
6273 * we fill in an inode item with the correct link count.
6274 */
6275 if (!name)
6276 set_nlink(inode, 0);
6277
6278 /*
6279 * we have to initialize this early, so we can reclaim the inode
6280 * number if we fail afterwards in this function.
6281 */
6282 inode->i_ino = objectid;
6283
6284 if (dir && name) {
6285 trace_btrfs_inode_request(dir);
6286
6287 ret = btrfs_set_inode_index(BTRFS_I(dir), index);
6288 if (ret) {
6289 btrfs_free_path(path);
6290 iput(inode);
6291 return ERR_PTR(ret);
6292 }
6293 } else if (dir) {
6294 *index = 0;
6295 }
6296 /*
6297 * index_cnt is ignored for everything but a dir,
6298 * btrfs_get_inode_index_count has an explanation for the magic
6299 * number
6300 */
6301 BTRFS_I(inode)->index_cnt = 2;
6302 BTRFS_I(inode)->dir_index = *index;
6303 BTRFS_I(inode)->root = root;
6304 BTRFS_I(inode)->generation = trans->transid;
6305 inode->i_generation = BTRFS_I(inode)->generation;
6306
6307 /*
6308 * We could have gotten an inode number from somebody who was fsynced
6309 * and then removed in this same transaction, so let's just set full
6310 * sync since it will be a full sync anyway and this will blow away the
6311 * old info in the log.
6312 */
6313 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
6314
6315 key[0].objectid = objectid;
6316 key[0].type = BTRFS_INODE_ITEM_KEY;
6317 key[0].offset = 0;
6318
6319 sizes[0] = sizeof(struct btrfs_inode_item);
6320
6321 if (name) {
6322 /*
6323 * Start new inodes with an inode_ref. This is slightly more
6324 * efficient for small numbers of hard links since they will
6325 * be packed into one item. Extended refs will kick in if we
6326 * add more hard links than can fit in the ref item.
6327 */
6328 key[1].objectid = objectid;
6329 key[1].type = BTRFS_INODE_REF_KEY;
6330 key[1].offset = ref_objectid;
6331
6332 sizes[1] = name_len + sizeof(*ref);
6333 }
6334
6335 location = &BTRFS_I(inode)->location;
6336 location->objectid = objectid;
6337 location->offset = 0;
6338 location->type = BTRFS_INODE_ITEM_KEY;
6339
6340 ret = btrfs_insert_inode_locked(inode);
6341 if (ret < 0)
6342 goto fail;
6343
6344 path->leave_spinning = 1;
6345 ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6346 if (ret != 0)
6347 goto fail_unlock;
6348
6349 inode_init_owner(inode, dir, mode);
6350 inode_set_bytes(inode, 0);
6351
6352 inode->i_mtime = current_time(inode);
6353 inode->i_atime = inode->i_mtime;
6354 inode->i_ctime = inode->i_mtime;
6355 BTRFS_I(inode)->i_otime = inode->i_mtime;
6356
6357 inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6358 struct btrfs_inode_item);
6359 memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6360 sizeof(*inode_item));
6361 fill_inode_item(trans, path->nodes[0], inode_item, inode);
6362
6363 if (name) {
6364 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6365 struct btrfs_inode_ref);
6366 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
6367 btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
6368 ptr = (unsigned long)(ref + 1);
6369 write_extent_buffer(path->nodes[0], name, ptr, name_len);
6370 }
6371
6372 btrfs_mark_buffer_dirty(path->nodes[0]);
6373 btrfs_free_path(path);
6374
6375 btrfs_inherit_iflags(inode, dir);
6376
6377 if (S_ISREG(mode)) {
6378 if (btrfs_test_opt(fs_info, NODATASUM))
6379 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6380 if (btrfs_test_opt(fs_info, NODATACOW))
6381 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6382 BTRFS_INODE_NODATASUM;
6383 }
6384
6385 inode_tree_add(inode);
6386
6387 trace_btrfs_inode_new(inode);
6388 btrfs_set_inode_last_trans(trans, inode);
6389
6390 btrfs_update_root_times(trans, root);
6391
6392 ret = btrfs_inode_inherit_props(trans, inode, dir);
6393 if (ret)
6394 btrfs_err(fs_info,
6395 "error inheriting props for ino %llu (root %llu): %d",
6396 btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret);
6397
6398 return inode;
6399
6400 fail_unlock:
6401 unlock_new_inode(inode);
6402 fail:
6403 if (dir && name)
6404 BTRFS_I(dir)->index_cnt--;
6405 btrfs_free_path(path);
6406 iput(inode);
6407 return ERR_PTR(ret);
6408 }
6409
6410 static inline u8 btrfs_inode_type(struct inode *inode)
6411 {
6412 return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
6413 }
6414
6415 /*
6416 * utility function to add 'inode' into 'parent_inode' with
6417 * a give name and a given sequence number.
6418 * if 'add_backref' is true, also insert a backref from the
6419 * inode to the parent directory.
6420 */
6421 int btrfs_add_link(struct btrfs_trans_handle *trans,
6422 struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
6423 const char *name, int name_len, int add_backref, u64 index)
6424 {
6425 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6426 int ret = 0;
6427 struct btrfs_key key;
6428 struct btrfs_root *root = parent_inode->root;
6429 u64 ino = btrfs_ino(inode);
6430 u64 parent_ino = btrfs_ino(parent_inode);
6431
6432 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6433 memcpy(&key, &inode->root->root_key, sizeof(key));
6434 } else {
6435 key.objectid = ino;
6436 key.type = BTRFS_INODE_ITEM_KEY;
6437 key.offset = 0;
6438 }
6439
6440 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6441 ret = btrfs_add_root_ref(trans, fs_info, key.objectid,
6442 root->root_key.objectid, parent_ino,
6443 index, name, name_len);
6444 } else if (add_backref) {
6445 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
6446 parent_ino, index);
6447 }
6448
6449 /* Nothing to clean up yet */
6450 if (ret)
6451 return ret;
6452
6453 ret = btrfs_insert_dir_item(trans, root, name, name_len,
6454 parent_inode, &key,
6455 btrfs_inode_type(&inode->vfs_inode), index);
6456 if (ret == -EEXIST || ret == -EOVERFLOW)
6457 goto fail_dir_item;
6458 else if (ret) {
6459 btrfs_abort_transaction(trans, ret);
6460 return ret;
6461 }
6462
6463 btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
6464 name_len * 2);
6465 inode_inc_iversion(&parent_inode->vfs_inode);
6466 parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime =
6467 current_time(&parent_inode->vfs_inode);
6468 ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
6469 if (ret)
6470 btrfs_abort_transaction(trans, ret);
6471 return ret;
6472
6473 fail_dir_item:
6474 if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6475 u64 local_index;
6476 int err;
6477 err = btrfs_del_root_ref(trans, fs_info, key.objectid,
6478 root->root_key.objectid, parent_ino,
6479 &local_index, name, name_len);
6480
6481 } else if (add_backref) {
6482 u64 local_index;
6483 int err;
6484
6485 err = btrfs_del_inode_ref(trans, root, name, name_len,
6486 ino, parent_ino, &local_index);
6487 }
6488 return ret;
6489 }
6490
6491 static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6492 struct btrfs_inode *dir, struct dentry *dentry,
6493 struct btrfs_inode *inode, int backref, u64 index)
6494 {
6495 int err = btrfs_add_link(trans, dir, inode,
6496 dentry->d_name.name, dentry->d_name.len,
6497 backref, index);
6498 if (err > 0)
6499 err = -EEXIST;
6500 return err;
6501 }
6502
6503 static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
6504 umode_t mode, dev_t rdev)
6505 {
6506 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6507 struct btrfs_trans_handle *trans;
6508 struct btrfs_root *root = BTRFS_I(dir)->root;
6509 struct inode *inode = NULL;
6510 int err;
6511 int drop_inode = 0;
6512 u64 objectid;
6513 u64 index = 0;
6514
6515 /*
6516 * 2 for inode item and ref
6517 * 2 for dir items
6518 * 1 for xattr if selinux is on
6519 */
6520 trans = btrfs_start_transaction(root, 5);
6521 if (IS_ERR(trans))
6522 return PTR_ERR(trans);
6523
6524 err = btrfs_find_free_ino(root, &objectid);
6525 if (err)
6526 goto out_unlock;
6527
6528 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6529 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6530 mode, &index);
6531 if (IS_ERR(inode)) {
6532 err = PTR_ERR(inode);
6533 goto out_unlock;
6534 }
6535
6536 /*
6537 * If the active LSM wants to access the inode during
6538 * d_instantiate it needs these. Smack checks to see
6539 * if the filesystem supports xattrs by looking at the
6540 * ops vector.
6541 */
6542 inode->i_op = &btrfs_special_inode_operations;
6543 init_special_inode(inode, inode->i_mode, rdev);
6544
6545 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6546 if (err)
6547 goto out_unlock_inode;
6548
6549 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6550 0, index);
6551 if (err) {
6552 goto out_unlock_inode;
6553 } else {
6554 btrfs_update_inode(trans, root, inode);
6555 unlock_new_inode(inode);
6556 d_instantiate(dentry, inode);
6557 }
6558
6559 out_unlock:
6560 btrfs_end_transaction(trans);
6561 btrfs_balance_delayed_items(fs_info);
6562 btrfs_btree_balance_dirty(fs_info);
6563 if (drop_inode) {
6564 inode_dec_link_count(inode);
6565 iput(inode);
6566 }
6567 return err;
6568
6569 out_unlock_inode:
6570 drop_inode = 1;
6571 unlock_new_inode(inode);
6572 goto out_unlock;
6573
6574 }
6575
6576 static int btrfs_create(struct inode *dir, struct dentry *dentry,
6577 umode_t mode, bool excl)
6578 {
6579 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6580 struct btrfs_trans_handle *trans;
6581 struct btrfs_root *root = BTRFS_I(dir)->root;
6582 struct inode *inode = NULL;
6583 int drop_inode_on_err = 0;
6584 int err;
6585 u64 objectid;
6586 u64 index = 0;
6587
6588 /*
6589 * 2 for inode item and ref
6590 * 2 for dir items
6591 * 1 for xattr if selinux is on
6592 */
6593 trans = btrfs_start_transaction(root, 5);
6594 if (IS_ERR(trans))
6595 return PTR_ERR(trans);
6596
6597 err = btrfs_find_free_ino(root, &objectid);
6598 if (err)
6599 goto out_unlock;
6600
6601 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6602 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6603 mode, &index);
6604 if (IS_ERR(inode)) {
6605 err = PTR_ERR(inode);
6606 goto out_unlock;
6607 }
6608 drop_inode_on_err = 1;
6609 /*
6610 * If the active LSM wants to access the inode during
6611 * d_instantiate it needs these. Smack checks to see
6612 * if the filesystem supports xattrs by looking at the
6613 * ops vector.
6614 */
6615 inode->i_fop = &btrfs_file_operations;
6616 inode->i_op = &btrfs_file_inode_operations;
6617 inode->i_mapping->a_ops = &btrfs_aops;
6618
6619 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6620 if (err)
6621 goto out_unlock_inode;
6622
6623 err = btrfs_update_inode(trans, root, inode);
6624 if (err)
6625 goto out_unlock_inode;
6626
6627 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6628 0, index);
6629 if (err)
6630 goto out_unlock_inode;
6631
6632 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6633 unlock_new_inode(inode);
6634 d_instantiate(dentry, inode);
6635
6636 out_unlock:
6637 btrfs_end_transaction(trans);
6638 if (err && drop_inode_on_err) {
6639 inode_dec_link_count(inode);
6640 iput(inode);
6641 }
6642 btrfs_balance_delayed_items(fs_info);
6643 btrfs_btree_balance_dirty(fs_info);
6644 return err;
6645
6646 out_unlock_inode:
6647 unlock_new_inode(inode);
6648 goto out_unlock;
6649
6650 }
6651
6652 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6653 struct dentry *dentry)
6654 {
6655 struct btrfs_trans_handle *trans = NULL;
6656 struct btrfs_root *root = BTRFS_I(dir)->root;
6657 struct inode *inode = d_inode(old_dentry);
6658 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6659 u64 index;
6660 int err;
6661 int drop_inode = 0;
6662
6663 /* do not allow sys_link's with other subvols of the same device */
6664 if (root->objectid != BTRFS_I(inode)->root->objectid)
6665 return -EXDEV;
6666
6667 if (inode->i_nlink >= BTRFS_LINK_MAX)
6668 return -EMLINK;
6669
6670 err = btrfs_set_inode_index(BTRFS_I(dir), &index);
6671 if (err)
6672 goto fail;
6673
6674 /*
6675 * 2 items for inode and inode ref
6676 * 2 items for dir items
6677 * 1 item for parent inode
6678 */
6679 trans = btrfs_start_transaction(root, 5);
6680 if (IS_ERR(trans)) {
6681 err = PTR_ERR(trans);
6682 trans = NULL;
6683 goto fail;
6684 }
6685
6686 /* There are several dir indexes for this inode, clear the cache. */
6687 BTRFS_I(inode)->dir_index = 0ULL;
6688 inc_nlink(inode);
6689 inode_inc_iversion(inode);
6690 inode->i_ctime = current_time(inode);
6691 ihold(inode);
6692 set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6693
6694 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6695 1, index);
6696
6697 if (err) {
6698 drop_inode = 1;
6699 } else {
6700 struct dentry *parent = dentry->d_parent;
6701 err = btrfs_update_inode(trans, root, inode);
6702 if (err)
6703 goto fail;
6704 if (inode->i_nlink == 1) {
6705 /*
6706 * If new hard link count is 1, it's a file created
6707 * with open(2) O_TMPFILE flag.
6708 */
6709 err = btrfs_orphan_del(trans, BTRFS_I(inode));
6710 if (err)
6711 goto fail;
6712 }
6713 d_instantiate(dentry, inode);
6714 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
6715 }
6716
6717 btrfs_balance_delayed_items(fs_info);
6718 fail:
6719 if (trans)
6720 btrfs_end_transaction(trans);
6721 if (drop_inode) {
6722 inode_dec_link_count(inode);
6723 iput(inode);
6724 }
6725 btrfs_btree_balance_dirty(fs_info);
6726 return err;
6727 }
6728
6729 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
6730 {
6731 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6732 struct inode *inode = NULL;
6733 struct btrfs_trans_handle *trans;
6734 struct btrfs_root *root = BTRFS_I(dir)->root;
6735 int err = 0;
6736 int drop_on_err = 0;
6737 u64 objectid = 0;
6738 u64 index = 0;
6739
6740 /*
6741 * 2 items for inode and ref
6742 * 2 items for dir items
6743 * 1 for xattr if selinux is on
6744 */
6745 trans = btrfs_start_transaction(root, 5);
6746 if (IS_ERR(trans))
6747 return PTR_ERR(trans);
6748
6749 err = btrfs_find_free_ino(root, &objectid);
6750 if (err)
6751 goto out_fail;
6752
6753 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6754 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6755 S_IFDIR | mode, &index);
6756 if (IS_ERR(inode)) {
6757 err = PTR_ERR(inode);
6758 goto out_fail;
6759 }
6760
6761 drop_on_err = 1;
6762 /* these must be set before we unlock the inode */
6763 inode->i_op = &btrfs_dir_inode_operations;
6764 inode->i_fop = &btrfs_dir_file_operations;
6765
6766 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6767 if (err)
6768 goto out_fail_inode;
6769
6770 btrfs_i_size_write(BTRFS_I(inode), 0);
6771 err = btrfs_update_inode(trans, root, inode);
6772 if (err)
6773 goto out_fail_inode;
6774
6775 err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
6776 dentry->d_name.name,
6777 dentry->d_name.len, 0, index);
6778 if (err)
6779 goto out_fail_inode;
6780
6781 d_instantiate(dentry, inode);
6782 /*
6783 * mkdir is special. We're unlocking after we call d_instantiate
6784 * to avoid a race with nfsd calling d_instantiate.
6785 */
6786 unlock_new_inode(inode);
6787 drop_on_err = 0;
6788
6789 out_fail:
6790 btrfs_end_transaction(trans);
6791 if (drop_on_err) {
6792 inode_dec_link_count(inode);
6793 iput(inode);
6794 }
6795 btrfs_balance_delayed_items(fs_info);
6796 btrfs_btree_balance_dirty(fs_info);
6797 return err;
6798
6799 out_fail_inode:
6800 unlock_new_inode(inode);
6801 goto out_fail;
6802 }
6803
6804 /* Find next extent map of a given extent map, caller needs to ensure locks */
6805 static struct extent_map *next_extent_map(struct extent_map *em)
6806 {
6807 struct rb_node *next;
6808
6809 next = rb_next(&em->rb_node);
6810 if (!next)
6811 return NULL;
6812 return container_of(next, struct extent_map, rb_node);
6813 }
6814
6815 static struct extent_map *prev_extent_map(struct extent_map *em)
6816 {
6817 struct rb_node *prev;
6818
6819 prev = rb_prev(&em->rb_node);
6820 if (!prev)
6821 return NULL;
6822 return container_of(prev, struct extent_map, rb_node);
6823 }
6824
6825 /* helper for btfs_get_extent. Given an existing extent in the tree,
6826 * the existing extent is the nearest extent to map_start,
6827 * and an extent that you want to insert, deal with overlap and insert
6828 * the best fitted new extent into the tree.
6829 */
6830 static int merge_extent_mapping(struct extent_map_tree *em_tree,
6831 struct extent_map *existing,
6832 struct extent_map *em,
6833 u64 map_start)
6834 {
6835 struct extent_map *prev;
6836 struct extent_map *next;
6837 u64 start;
6838 u64 end;
6839 u64 start_diff;
6840
6841 BUG_ON(map_start < em->start || map_start >= extent_map_end(em));
6842
6843 if (existing->start > map_start) {
6844 next = existing;
6845 prev = prev_extent_map(next);
6846 } else {
6847 prev = existing;
6848 next = next_extent_map(prev);
6849 }
6850
6851 start = prev ? extent_map_end(prev) : em->start;
6852 start = max_t(u64, start, em->start);
6853 end = next ? next->start : extent_map_end(em);
6854 end = min_t(u64, end, extent_map_end(em));
6855 start_diff = start - em->start;
6856 em->start = start;
6857 em->len = end - start;
6858 if (em->block_start < EXTENT_MAP_LAST_BYTE &&
6859 !test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
6860 em->block_start += start_diff;
6861 em->block_len -= start_diff;
6862 }
6863 return add_extent_mapping(em_tree, em, 0);
6864 }
6865
6866 static noinline int uncompress_inline(struct btrfs_path *path,
6867 struct page *page,
6868 size_t pg_offset, u64 extent_offset,
6869 struct btrfs_file_extent_item *item)
6870 {
6871 int ret;
6872 struct extent_buffer *leaf = path->nodes[0];
6873 char *tmp;
6874 size_t max_size;
6875 unsigned long inline_size;
6876 unsigned long ptr;
6877 int compress_type;
6878
6879 WARN_ON(pg_offset != 0);
6880 compress_type = btrfs_file_extent_compression(leaf, item);
6881 max_size = btrfs_file_extent_ram_bytes(leaf, item);
6882 inline_size = btrfs_file_extent_inline_item_len(leaf,
6883 btrfs_item_nr(path->slots[0]));
6884 tmp = kmalloc(inline_size, GFP_NOFS);
6885 if (!tmp)
6886 return -ENOMEM;
6887 ptr = btrfs_file_extent_inline_start(item);
6888
6889 read_extent_buffer(leaf, tmp, ptr, inline_size);
6890
6891 max_size = min_t(unsigned long, PAGE_SIZE, max_size);
6892 ret = btrfs_decompress(compress_type, tmp, page,
6893 extent_offset, inline_size, max_size);
6894
6895 /*
6896 * decompression code contains a memset to fill in any space between the end
6897 * of the uncompressed data and the end of max_size in case the decompressed
6898 * data ends up shorter than ram_bytes. That doesn't cover the hole between
6899 * the end of an inline extent and the beginning of the next block, so we
6900 * cover that region here.
6901 */
6902
6903 if (max_size + pg_offset < PAGE_SIZE) {
6904 char *map = kmap(page);
6905 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset);
6906 kunmap(page);
6907 }
6908 kfree(tmp);
6909 return ret;
6910 }
6911
6912 /*
6913 * a bit scary, this does extent mapping from logical file offset to the disk.
6914 * the ugly parts come from merging extents from the disk with the in-ram
6915 * representation. This gets more complex because of the data=ordered code,
6916 * where the in-ram extents might be locked pending data=ordered completion.
6917 *
6918 * This also copies inline extents directly into the page.
6919 */
6920 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
6921 struct page *page,
6922 size_t pg_offset, u64 start, u64 len,
6923 int create)
6924 {
6925 struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6926 int ret;
6927 int err = 0;
6928 u64 extent_start = 0;
6929 u64 extent_end = 0;
6930 u64 objectid = btrfs_ino(inode);
6931 u32 found_type;
6932 struct btrfs_path *path = NULL;
6933 struct btrfs_root *root = inode->root;
6934 struct btrfs_file_extent_item *item;
6935 struct extent_buffer *leaf;
6936 struct btrfs_key found_key;
6937 struct extent_map *em = NULL;
6938 struct extent_map_tree *em_tree = &inode->extent_tree;
6939 struct extent_io_tree *io_tree = &inode->io_tree;
6940 struct btrfs_trans_handle *trans = NULL;
6941 const bool new_inline = !page || create;
6942
6943 again:
6944 read_lock(&em_tree->lock);
6945 em = lookup_extent_mapping(em_tree, start, len);
6946 if (em)
6947 em->bdev = fs_info->fs_devices->latest_bdev;
6948 read_unlock(&em_tree->lock);
6949
6950 if (em) {
6951 if (em->start > start || em->start + em->len <= start)
6952 free_extent_map(em);
6953 else if (em->block_start == EXTENT_MAP_INLINE && page)
6954 free_extent_map(em);
6955 else
6956 goto out;
6957 }
6958 em = alloc_extent_map();
6959 if (!em) {
6960 err = -ENOMEM;
6961 goto out;
6962 }
6963 em->bdev = fs_info->fs_devices->latest_bdev;
6964 em->start = EXTENT_MAP_HOLE;
6965 em->orig_start = EXTENT_MAP_HOLE;
6966 em->len = (u64)-1;
6967 em->block_len = (u64)-1;
6968
6969 if (!path) {
6970 path = btrfs_alloc_path();
6971 if (!path) {
6972 err = -ENOMEM;
6973 goto out;
6974 }
6975 /*
6976 * Chances are we'll be called again, so go ahead and do
6977 * readahead
6978 */
6979 path->reada = READA_FORWARD;
6980 }
6981
6982 ret = btrfs_lookup_file_extent(trans, root, path,
6983 objectid, start, trans != NULL);
6984 if (ret < 0) {
6985 err = ret;
6986 goto out;
6987 }
6988
6989 if (ret != 0) {
6990 if (path->slots[0] == 0)
6991 goto not_found;
6992 path->slots[0]--;
6993 }
6994
6995 leaf = path->nodes[0];
6996 item = btrfs_item_ptr(leaf, path->slots[0],
6997 struct btrfs_file_extent_item);
6998 /* are we inside the extent that was found? */
6999 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7000 found_type = found_key.type;
7001 if (found_key.objectid != objectid ||
7002 found_type != BTRFS_EXTENT_DATA_KEY) {
7003 /*
7004 * If we backup past the first extent we want to move forward
7005 * and see if there is an extent in front of us, otherwise we'll
7006 * say there is a hole for our whole search range which can
7007 * cause problems.
7008 */
7009 extent_end = start;
7010 goto next;
7011 }
7012
7013 found_type = btrfs_file_extent_type(leaf, item);
7014 extent_start = found_key.offset;
7015 if (found_type == BTRFS_FILE_EXTENT_REG ||
7016 found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7017 extent_end = extent_start +
7018 btrfs_file_extent_num_bytes(leaf, item);
7019
7020 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
7021 extent_start);
7022 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
7023 size_t size;
7024 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
7025 extent_end = ALIGN(extent_start + size,
7026 fs_info->sectorsize);
7027
7028 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
7029 path->slots[0],
7030 extent_start);
7031 }
7032 next:
7033 if (start >= extent_end) {
7034 path->slots[0]++;
7035 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
7036 ret = btrfs_next_leaf(root, path);
7037 if (ret < 0) {
7038 err = ret;
7039 goto out;
7040 }
7041 if (ret > 0)
7042 goto not_found;
7043 leaf = path->nodes[0];
7044 }
7045 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7046 if (found_key.objectid != objectid ||
7047 found_key.type != BTRFS_EXTENT_DATA_KEY)
7048 goto not_found;
7049 if (start + len <= found_key.offset)
7050 goto not_found;
7051 if (start > found_key.offset)
7052 goto next;
7053 em->start = start;
7054 em->orig_start = start;
7055 em->len = found_key.offset - start;
7056 goto not_found_em;
7057 }
7058
7059 btrfs_extent_item_to_extent_map(inode, path, item,
7060 new_inline, em);
7061
7062 if (found_type == BTRFS_FILE_EXTENT_REG ||
7063 found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7064 goto insert;
7065 } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
7066 unsigned long ptr;
7067 char *map;
7068 size_t size;
7069 size_t extent_offset;
7070 size_t copy_size;
7071
7072 if (new_inline)
7073 goto out;
7074
7075 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
7076 extent_offset = page_offset(page) + pg_offset - extent_start;
7077 copy_size = min_t(u64, PAGE_SIZE - pg_offset,
7078 size - extent_offset);
7079 em->start = extent_start + extent_offset;
7080 em->len = ALIGN(copy_size, fs_info->sectorsize);
7081 em->orig_block_len = em->len;
7082 em->orig_start = em->start;
7083 ptr = btrfs_file_extent_inline_start(item) + extent_offset;
7084 if (create == 0 && !PageUptodate(page)) {
7085 if (btrfs_file_extent_compression(leaf, item) !=
7086 BTRFS_COMPRESS_NONE) {
7087 ret = uncompress_inline(path, page, pg_offset,
7088 extent_offset, item);
7089 if (ret) {
7090 err = ret;
7091 goto out;
7092 }
7093 } else {
7094 map = kmap(page);
7095 read_extent_buffer(leaf, map + pg_offset, ptr,
7096 copy_size);
7097 if (pg_offset + copy_size < PAGE_SIZE) {
7098 memset(map + pg_offset + copy_size, 0,
7099 PAGE_SIZE - pg_offset -
7100 copy_size);
7101 }
7102 kunmap(page);
7103 }
7104 flush_dcache_page(page);
7105 } else if (create && PageUptodate(page)) {
7106 BUG();
7107 if (!trans) {
7108 kunmap(page);
7109 free_extent_map(em);
7110 em = NULL;
7111
7112 btrfs_release_path(path);
7113 trans = btrfs_join_transaction(root);
7114
7115 if (IS_ERR(trans))
7116 return ERR_CAST(trans);
7117 goto again;
7118 }
7119 map = kmap(page);
7120 write_extent_buffer(leaf, map + pg_offset, ptr,
7121 copy_size);
7122 kunmap(page);
7123 btrfs_mark_buffer_dirty(leaf);
7124 }
7125 set_extent_uptodate(io_tree, em->start,
7126 extent_map_end(em) - 1, NULL, GFP_NOFS);
7127 goto insert;
7128 }
7129 not_found:
7130 em->start = start;
7131 em->orig_start = start;
7132 em->len = len;
7133 not_found_em:
7134 em->block_start = EXTENT_MAP_HOLE;
7135 set_bit(EXTENT_FLAG_VACANCY, &em->flags);
7136 insert:
7137 btrfs_release_path(path);
7138 if (em->start > start || extent_map_end(em) <= start) {
7139 btrfs_err(fs_info,
7140 "bad extent! em: [%llu %llu] passed [%llu %llu]",
7141 em->start, em->len, start, len);
7142 err = -EIO;
7143 goto out;
7144 }
7145
7146 err = 0;
7147 write_lock(&em_tree->lock);
7148 ret = add_extent_mapping(em_tree, em, 0);
7149 /* it is possible that someone inserted the extent into the tree
7150 * while we had the lock dropped. It is also possible that
7151 * an overlapping map exists in the tree
7152 */
7153 if (ret == -EEXIST) {
7154 struct extent_map *existing;
7155
7156 ret = 0;
7157
7158 existing = search_extent_mapping(em_tree, start, len);
7159 /*
7160 * existing will always be non-NULL, since there must be
7161 * extent causing the -EEXIST.
7162 */
7163 if (existing->start == em->start &&
7164 extent_map_end(existing) >= extent_map_end(em) &&
7165 em->block_start == existing->block_start) {
7166 /*
7167 * The existing extent map already encompasses the
7168 * entire extent map we tried to add.
7169 */
7170 free_extent_map(em);
7171 em = existing;
7172 err = 0;
7173
7174 } else if (start >= extent_map_end(existing) ||
7175 start <= existing->start) {
7176 /*
7177 * The existing extent map is the one nearest to
7178 * the [start, start + len) range which overlaps
7179 */
7180 err = merge_extent_mapping(em_tree, existing,
7181 em, start);
7182 free_extent_map(existing);
7183 if (err) {
7184 free_extent_map(em);
7185 em = NULL;
7186 }
7187 } else {
7188 free_extent_map(em);
7189 em = existing;
7190 err = 0;
7191 }
7192 }
7193 write_unlock(&em_tree->lock);
7194 out:
7195
7196 trace_btrfs_get_extent(root, inode, em);
7197
7198 btrfs_free_path(path);
7199 if (trans) {
7200 ret = btrfs_end_transaction(trans);
7201 if (!err)
7202 err = ret;
7203 }
7204 if (err) {
7205 free_extent_map(em);
7206 return ERR_PTR(err);
7207 }
7208 BUG_ON(!em); /* Error is always set */
7209 return em;
7210 }
7211
7212 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
7213 struct page *page,
7214 size_t pg_offset, u64 start, u64 len,
7215 int create)
7216 {
7217 struct extent_map *em;
7218 struct extent_map *hole_em = NULL;
7219 u64 range_start = start;
7220 u64 end;
7221 u64 found;
7222 u64 found_end;
7223 int err = 0;
7224
7225 em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
7226 if (IS_ERR(em))
7227 return em;
7228 /*
7229 * If our em maps to:
7230 * - a hole or
7231 * - a pre-alloc extent,
7232 * there might actually be delalloc bytes behind it.
7233 */
7234 if (em->block_start != EXTENT_MAP_HOLE &&
7235 !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7236 return em;
7237 else
7238 hole_em = em;
7239
7240 /* check to see if we've wrapped (len == -1 or similar) */
7241 end = start + len;
7242 if (end < start)
7243 end = (u64)-1;
7244 else
7245 end -= 1;
7246
7247 em = NULL;
7248
7249 /* ok, we didn't find anything, lets look for delalloc */
7250 found = count_range_bits(&inode->io_tree, &range_start,
7251 end, len, EXTENT_DELALLOC, 1);
7252 found_end = range_start + found;
7253 if (found_end < range_start)
7254 found_end = (u64)-1;
7255
7256 /*
7257 * we didn't find anything useful, return
7258 * the original results from get_extent()
7259 */
7260 if (range_start > end || found_end <= start) {
7261 em = hole_em;
7262 hole_em = NULL;
7263 goto out;
7264 }
7265
7266 /* adjust the range_start to make sure it doesn't
7267 * go backwards from the start they passed in
7268 */
7269 range_start = max(start, range_start);
7270 found = found_end - range_start;
7271
7272 if (found > 0) {
7273 u64 hole_start = start;
7274 u64 hole_len = len;
7275
7276 em = alloc_extent_map();
7277 if (!em) {
7278 err = -ENOMEM;
7279 goto out;
7280 }
7281 /*
7282 * when btrfs_get_extent can't find anything it
7283 * returns one huge hole
7284 *
7285 * make sure what it found really fits our range, and
7286 * adjust to make sure it is based on the start from
7287 * the caller
7288 */
7289 if (hole_em) {
7290 u64 calc_end = extent_map_end(hole_em);
7291
7292 if (calc_end <= start || (hole_em->start > end)) {
7293 free_extent_map(hole_em);
7294 hole_em = NULL;
7295 } else {
7296 hole_start = max(hole_em->start, start);
7297 hole_len = calc_end - hole_start;
7298 }
7299 }
7300 em->bdev = NULL;
7301 if (hole_em && range_start > hole_start) {
7302 /* our hole starts before our delalloc, so we
7303 * have to return just the parts of the hole
7304 * that go until the delalloc starts
7305 */
7306 em->len = min(hole_len,
7307 range_start - hole_start);
7308 em->start = hole_start;
7309 em->orig_start = hole_start;
7310 /*
7311 * don't adjust block start at all,
7312 * it is fixed at EXTENT_MAP_HOLE
7313 */
7314 em->block_start = hole_em->block_start;
7315 em->block_len = hole_len;
7316 if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
7317 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7318 } else {
7319 em->start = range_start;
7320 em->len = found;
7321 em->orig_start = range_start;
7322 em->block_start = EXTENT_MAP_DELALLOC;
7323 em->block_len = found;
7324 }
7325 } else if (hole_em) {
7326 return hole_em;
7327 }
7328 out:
7329
7330 free_extent_map(hole_em);
7331 if (err) {
7332 free_extent_map(em);
7333 return ERR_PTR(err);
7334 }
7335 return em;
7336 }
7337
7338 static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
7339 const u64 start,
7340 const u64 len,
7341 const u64 orig_start,
7342 const u64 block_start,
7343 const u64 block_len,
7344 const u64 orig_block_len,
7345 const u64 ram_bytes,
7346 const int type)
7347 {
7348 struct extent_map *em = NULL;
7349 int ret;
7350
7351 if (type != BTRFS_ORDERED_NOCOW) {
7352 em = create_io_em(inode, start, len, orig_start,
7353 block_start, block_len, orig_block_len,
7354 ram_bytes,
7355 BTRFS_COMPRESS_NONE, /* compress_type */
7356 type);
7357 if (IS_ERR(em))
7358 goto out;
7359 }
7360 ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
7361 len, block_len, type);
7362 if (ret) {
7363 if (em) {
7364 free_extent_map(em);
7365 btrfs_drop_extent_cache(BTRFS_I(inode), start,
7366 start + len - 1, 0);
7367 }
7368 em = ERR_PTR(ret);
7369 }
7370 out:
7371
7372 return em;
7373 }
7374
7375 static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
7376 u64 start, u64 len)
7377 {
7378 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7379 struct btrfs_root *root = BTRFS_I(inode)->root;
7380 struct extent_map *em;
7381 struct btrfs_key ins;
7382 u64 alloc_hint;
7383 int ret;
7384
7385 alloc_hint = get_extent_allocation_hint(inode, start, len);
7386 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
7387 0, alloc_hint, &ins, 1, 1);
7388 if (ret)
7389 return ERR_PTR(ret);
7390
7391 em = btrfs_create_dio_extent(inode, start, ins.offset, start,
7392 ins.objectid, ins.offset, ins.offset,
7393 ins.offset, BTRFS_ORDERED_REGULAR);
7394 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
7395 if (IS_ERR(em))
7396 btrfs_free_reserved_extent(fs_info, ins.objectid,
7397 ins.offset, 1);
7398
7399 return em;
7400 }
7401
7402 /*
7403 * returns 1 when the nocow is safe, < 1 on error, 0 if the
7404 * block must be cow'd
7405 */
7406 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7407 u64 *orig_start, u64 *orig_block_len,
7408 u64 *ram_bytes)
7409 {
7410 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7411 struct btrfs_path *path;
7412 int ret;
7413 struct extent_buffer *leaf;
7414 struct btrfs_root *root = BTRFS_I(inode)->root;
7415 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7416 struct btrfs_file_extent_item *fi;
7417 struct btrfs_key key;
7418 u64 disk_bytenr;
7419 u64 backref_offset;
7420 u64 extent_end;
7421 u64 num_bytes;
7422 int slot;
7423 int found_type;
7424 bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7425
7426 path = btrfs_alloc_path();
7427 if (!path)
7428 return -ENOMEM;
7429
7430 ret = btrfs_lookup_file_extent(NULL, root, path,
7431 btrfs_ino(BTRFS_I(inode)), offset, 0);
7432 if (ret < 0)
7433 goto out;
7434
7435 slot = path->slots[0];
7436 if (ret == 1) {
7437 if (slot == 0) {
7438 /* can't find the item, must cow */
7439 ret = 0;
7440 goto out;
7441 }
7442 slot--;
7443 }
7444 ret = 0;
7445 leaf = path->nodes[0];
7446 btrfs_item_key_to_cpu(leaf, &key, slot);
7447 if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
7448 key.type != BTRFS_EXTENT_DATA_KEY) {
7449 /* not our file or wrong item type, must cow */
7450 goto out;
7451 }
7452
7453 if (key.offset > offset) {
7454 /* Wrong offset, must cow */
7455 goto out;
7456 }
7457
7458 fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
7459 found_type = btrfs_file_extent_type(leaf, fi);
7460 if (found_type != BTRFS_FILE_EXTENT_REG &&
7461 found_type != BTRFS_FILE_EXTENT_PREALLOC) {
7462 /* not a regular extent, must cow */
7463 goto out;
7464 }
7465
7466 if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
7467 goto out;
7468
7469 extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
7470 if (extent_end <= offset)
7471 goto out;
7472
7473 disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7474 if (disk_bytenr == 0)
7475 goto out;
7476
7477 if (btrfs_file_extent_compression(leaf, fi) ||
7478 btrfs_file_extent_encryption(leaf, fi) ||
7479 btrfs_file_extent_other_encoding(leaf, fi))
7480 goto out;
7481
7482 backref_offset = btrfs_file_extent_offset(leaf, fi);
7483
7484 if (orig_start) {
7485 *orig_start = key.offset - backref_offset;
7486 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
7487 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
7488 }
7489
7490 if (btrfs_extent_readonly(fs_info, disk_bytenr))
7491 goto out;
7492
7493 num_bytes = min(offset + *len, extent_end) - offset;
7494 if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7495 u64 range_end;
7496
7497 range_end = round_up(offset + num_bytes,
7498 root->fs_info->sectorsize) - 1;
7499 ret = test_range_bit(io_tree, offset, range_end,
7500 EXTENT_DELALLOC, 0, NULL);
7501 if (ret) {
7502 ret = -EAGAIN;
7503 goto out;
7504 }
7505 }
7506
7507 btrfs_release_path(path);
7508
7509 /*
7510 * look for other files referencing this extent, if we
7511 * find any we must cow
7512 */
7513
7514 ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)),
7515 key.offset - backref_offset, disk_bytenr);
7516 if (ret) {
7517 ret = 0;
7518 goto out;
7519 }
7520
7521 /*
7522 * adjust disk_bytenr and num_bytes to cover just the bytes
7523 * in this extent we are about to write. If there
7524 * are any csums in that range we have to cow in order
7525 * to keep the csums correct
7526 */
7527 disk_bytenr += backref_offset;
7528 disk_bytenr += offset - key.offset;
7529 if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
7530 goto out;
7531 /*
7532 * all of the above have passed, it is safe to overwrite this extent
7533 * without cow
7534 */
7535 *len = num_bytes;
7536 ret = 1;
7537 out:
7538 btrfs_free_path(path);
7539 return ret;
7540 }
7541
7542 bool btrfs_page_exists_in_range(struct inode *inode, loff_t start, loff_t end)
7543 {
7544 struct radix_tree_root *root = &inode->i_mapping->page_tree;
7545 bool found = false;
7546 void **pagep = NULL;
7547 struct page *page = NULL;
7548 unsigned long start_idx;
7549 unsigned long end_idx;
7550
7551 start_idx = start >> PAGE_SHIFT;
7552
7553 /*
7554 * end is the last byte in the last page. end == start is legal
7555 */
7556 end_idx = end >> PAGE_SHIFT;
7557
7558 rcu_read_lock();
7559
7560 /* Most of the code in this while loop is lifted from
7561 * find_get_page. It's been modified to begin searching from a
7562 * page and return just the first page found in that range. If the
7563 * found idx is less than or equal to the end idx then we know that
7564 * a page exists. If no pages are found or if those pages are
7565 * outside of the range then we're fine (yay!) */
7566 while (page == NULL &&
7567 radix_tree_gang_lookup_slot(root, &pagep, NULL, start_idx, 1)) {
7568 page = radix_tree_deref_slot(pagep);
7569 if (unlikely(!page))
7570 break;
7571
7572 if (radix_tree_exception(page)) {
7573 if (radix_tree_deref_retry(page)) {
7574 page = NULL;
7575 continue;
7576 }
7577 /*
7578 * Otherwise, shmem/tmpfs must be storing a swap entry
7579 * here as an exceptional entry: so return it without
7580 * attempting to raise page count.
7581 */
7582 page = NULL;
7583 break; /* TODO: Is this relevant for this use case? */
7584 }
7585
7586 if (!page_cache_get_speculative(page)) {
7587 page = NULL;
7588 continue;
7589 }
7590
7591 /*
7592 * Has the page moved?
7593 * This is part of the lockless pagecache protocol. See
7594 * include/linux/pagemap.h for details.
7595 */
7596 if (unlikely(page != *pagep)) {
7597 put_page(page);
7598 page = NULL;
7599 }
7600 }
7601
7602 if (page) {
7603 if (page->index <= end_idx)
7604 found = true;
7605 put_page(page);
7606 }
7607
7608 rcu_read_unlock();
7609 return found;
7610 }
7611
7612 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7613 struct extent_state **cached_state, int writing)
7614 {
7615 struct btrfs_ordered_extent *ordered;
7616 int ret = 0;
7617
7618 while (1) {
7619 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7620 cached_state);
7621 /*
7622 * We're concerned with the entire range that we're going to be
7623 * doing DIO to, so we need to make sure there's no ordered
7624 * extents in this range.
7625 */
7626 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
7627 lockend - lockstart + 1);
7628
7629 /*
7630 * We need to make sure there are no buffered pages in this
7631 * range either, we could have raced between the invalidate in
7632 * generic_file_direct_write and locking the extent. The
7633 * invalidate needs to happen so that reads after a write do not
7634 * get stale data.
7635 */
7636 if (!ordered &&
7637 (!writing ||
7638 !btrfs_page_exists_in_range(inode, lockstart, lockend)))
7639 break;
7640
7641 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7642 cached_state, GFP_NOFS);
7643
7644 if (ordered) {
7645 /*
7646 * If we are doing a DIO read and the ordered extent we
7647 * found is for a buffered write, we can not wait for it
7648 * to complete and retry, because if we do so we can
7649 * deadlock with concurrent buffered writes on page
7650 * locks. This happens only if our DIO read covers more
7651 * than one extent map, if at this point has already
7652 * created an ordered extent for a previous extent map
7653 * and locked its range in the inode's io tree, and a
7654 * concurrent write against that previous extent map's
7655 * range and this range started (we unlock the ranges
7656 * in the io tree only when the bios complete and
7657 * buffered writes always lock pages before attempting
7658 * to lock range in the io tree).
7659 */
7660 if (writing ||
7661 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
7662 btrfs_start_ordered_extent(inode, ordered, 1);
7663 else
7664 ret = -ENOTBLK;
7665 btrfs_put_ordered_extent(ordered);
7666 } else {
7667 /*
7668 * We could trigger writeback for this range (and wait
7669 * for it to complete) and then invalidate the pages for
7670 * this range (through invalidate_inode_pages2_range()),
7671 * but that can lead us to a deadlock with a concurrent
7672 * call to readpages() (a buffered read or a defrag call
7673 * triggered a readahead) on a page lock due to an
7674 * ordered dio extent we created before but did not have
7675 * yet a corresponding bio submitted (whence it can not
7676 * complete), which makes readpages() wait for that
7677 * ordered extent to complete while holding a lock on
7678 * that page.
7679 */
7680 ret = -ENOTBLK;
7681 }
7682
7683 if (ret)
7684 break;
7685
7686 cond_resched();
7687 }
7688
7689 return ret;
7690 }
7691
7692 /* The callers of this must take lock_extent() */
7693 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
7694 u64 orig_start, u64 block_start,
7695 u64 block_len, u64 orig_block_len,
7696 u64 ram_bytes, int compress_type,
7697 int type)
7698 {
7699 struct extent_map_tree *em_tree;
7700 struct extent_map *em;
7701 struct btrfs_root *root = BTRFS_I(inode)->root;
7702 int ret;
7703
7704 ASSERT(type == BTRFS_ORDERED_PREALLOC ||
7705 type == BTRFS_ORDERED_COMPRESSED ||
7706 type == BTRFS_ORDERED_NOCOW ||
7707 type == BTRFS_ORDERED_REGULAR);
7708
7709 em_tree = &BTRFS_I(inode)->extent_tree;
7710 em = alloc_extent_map();
7711 if (!em)
7712 return ERR_PTR(-ENOMEM);
7713
7714 em->start = start;
7715 em->orig_start = orig_start;
7716 em->len = len;
7717 em->block_len = block_len;
7718 em->block_start = block_start;
7719 em->bdev = root->fs_info->fs_devices->latest_bdev;
7720 em->orig_block_len = orig_block_len;
7721 em->ram_bytes = ram_bytes;
7722 em->generation = -1;
7723 set_bit(EXTENT_FLAG_PINNED, &em->flags);
7724 if (type == BTRFS_ORDERED_PREALLOC) {
7725 set_bit(EXTENT_FLAG_FILLING, &em->flags);
7726 } else if (type == BTRFS_ORDERED_COMPRESSED) {
7727 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
7728 em->compress_type = compress_type;
7729 }
7730
7731 do {
7732 btrfs_drop_extent_cache(BTRFS_I(inode), em->start,
7733 em->start + em->len - 1, 0);
7734 write_lock(&em_tree->lock);
7735 ret = add_extent_mapping(em_tree, em, 1);
7736 write_unlock(&em_tree->lock);
7737 /*
7738 * The caller has taken lock_extent(), who could race with us
7739 * to add em?
7740 */
7741 } while (ret == -EEXIST);
7742
7743 if (ret) {
7744 free_extent_map(em);
7745 return ERR_PTR(ret);
7746 }
7747
7748 /* em got 2 refs now, callers needs to do free_extent_map once. */
7749 return em;
7750 }
7751
7752 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
7753 struct buffer_head *bh_result, int create)
7754 {
7755 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7756 struct extent_map *em;
7757 struct extent_state *cached_state = NULL;
7758 struct btrfs_dio_data *dio_data = NULL;
7759 u64 start = iblock << inode->i_blkbits;
7760 u64 lockstart, lockend;
7761 u64 len = bh_result->b_size;
7762 int unlock_bits = EXTENT_LOCKED;
7763 int ret = 0;
7764
7765 if (create)
7766 unlock_bits |= EXTENT_DIRTY;
7767 else
7768 len = min_t(u64, len, fs_info->sectorsize);
7769
7770 lockstart = start;
7771 lockend = start + len - 1;
7772
7773 if (current->journal_info) {
7774 /*
7775 * Need to pull our outstanding extents and set journal_info to NULL so
7776 * that anything that needs to check if there's a transaction doesn't get
7777 * confused.
7778 */
7779 dio_data = current->journal_info;
7780 current->journal_info = NULL;
7781 }
7782
7783 /*
7784 * If this errors out it's because we couldn't invalidate pagecache for
7785 * this range and we need to fallback to buffered.
7786 */
7787 if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
7788 create)) {
7789 ret = -ENOTBLK;
7790 goto err;
7791 }
7792
7793 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
7794 if (IS_ERR(em)) {
7795 ret = PTR_ERR(em);
7796 goto unlock_err;
7797 }
7798
7799 /*
7800 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7801 * io. INLINE is special, and we could probably kludge it in here, but
7802 * it's still buffered so for safety lets just fall back to the generic
7803 * buffered path.
7804 *
7805 * For COMPRESSED we _have_ to read the entire extent in so we can
7806 * decompress it, so there will be buffering required no matter what we
7807 * do, so go ahead and fallback to buffered.
7808 *
7809 * We return -ENOTBLK because that's what makes DIO go ahead and go back
7810 * to buffered IO. Don't blame me, this is the price we pay for using
7811 * the generic code.
7812 */
7813 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
7814 em->block_start == EXTENT_MAP_INLINE) {
7815 free_extent_map(em);
7816 ret = -ENOTBLK;
7817 goto unlock_err;
7818 }
7819
7820 /* Just a good old fashioned hole, return */
7821 if (!create && (em->block_start == EXTENT_MAP_HOLE ||
7822 test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
7823 free_extent_map(em);
7824 goto unlock_err;
7825 }
7826
7827 /*
7828 * We don't allocate a new extent in the following cases
7829 *
7830 * 1) The inode is marked as NODATACOW. In this case we'll just use the
7831 * existing extent.
7832 * 2) The extent is marked as PREALLOC. We're good to go here and can
7833 * just use the extent.
7834 *
7835 */
7836 if (!create) {
7837 len = min(len, em->len - (start - em->start));
7838 lockstart = start + len;
7839 goto unlock;
7840 }
7841
7842 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
7843 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7844 em->block_start != EXTENT_MAP_HOLE)) {
7845 int type;
7846 u64 block_start, orig_start, orig_block_len, ram_bytes;
7847
7848 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7849 type = BTRFS_ORDERED_PREALLOC;
7850 else
7851 type = BTRFS_ORDERED_NOCOW;
7852 len = min(len, em->len - (start - em->start));
7853 block_start = em->block_start + (start - em->start);
7854
7855 if (can_nocow_extent(inode, start, &len, &orig_start,
7856 &orig_block_len, &ram_bytes) == 1 &&
7857 btrfs_inc_nocow_writers(fs_info, block_start)) {
7858 struct extent_map *em2;
7859
7860 em2 = btrfs_create_dio_extent(inode, start, len,
7861 orig_start, block_start,
7862 len, orig_block_len,
7863 ram_bytes, type);
7864 btrfs_dec_nocow_writers(fs_info, block_start);
7865 if (type == BTRFS_ORDERED_PREALLOC) {
7866 free_extent_map(em);
7867 em = em2;
7868 }
7869 if (em2 && IS_ERR(em2)) {
7870 ret = PTR_ERR(em2);
7871 goto unlock_err;
7872 }
7873 /*
7874 * For inode marked NODATACOW or extent marked PREALLOC,
7875 * use the existing or preallocated extent, so does not
7876 * need to adjust btrfs_space_info's bytes_may_use.
7877 */
7878 btrfs_free_reserved_data_space_noquota(inode,
7879 start, len);
7880 goto unlock;
7881 }
7882 }
7883
7884 /*
7885 * this will cow the extent, reset the len in case we changed
7886 * it above
7887 */
7888 len = bh_result->b_size;
7889 free_extent_map(em);
7890 em = btrfs_new_extent_direct(inode, start, len);
7891 if (IS_ERR(em)) {
7892 ret = PTR_ERR(em);
7893 goto unlock_err;
7894 }
7895 len = min(len, em->len - (start - em->start));
7896 unlock:
7897 bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7898 inode->i_blkbits;
7899 bh_result->b_size = len;
7900 bh_result->b_bdev = em->bdev;
7901 set_buffer_mapped(bh_result);
7902 if (create) {
7903 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7904 set_buffer_new(bh_result);
7905
7906 /*
7907 * Need to update the i_size under the extent lock so buffered
7908 * readers will get the updated i_size when we unlock.
7909 */
7910 if (!dio_data->overwrite && start + len > i_size_read(inode))
7911 i_size_write(inode, start + len);
7912
7913 WARN_ON(dio_data->reserve < len);
7914 dio_data->reserve -= len;
7915 dio_data->unsubmitted_oe_range_end = start + len;
7916 current->journal_info = dio_data;
7917 }
7918
7919 /*
7920 * In the case of write we need to clear and unlock the entire range,
7921 * in the case of read we need to unlock only the end area that we
7922 * aren't using if there is any left over space.
7923 */
7924 if (lockstart < lockend) {
7925 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
7926 lockend, unlock_bits, 1, 0,
7927 &cached_state, GFP_NOFS);
7928 } else {
7929 free_extent_state(cached_state);
7930 }
7931
7932 free_extent_map(em);
7933
7934 return 0;
7935
7936 unlock_err:
7937 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7938 unlock_bits, 1, 0, &cached_state, GFP_NOFS);
7939 err:
7940 if (dio_data)
7941 current->journal_info = dio_data;
7942 return ret;
7943 }
7944
7945 static inline blk_status_t submit_dio_repair_bio(struct inode *inode,
7946 struct bio *bio,
7947 int mirror_num)
7948 {
7949 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7950 blk_status_t ret;
7951
7952 BUG_ON(bio_op(bio) == REQ_OP_WRITE);
7953
7954 bio_get(bio);
7955
7956 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR);
7957 if (ret)
7958 goto err;
7959
7960 ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
7961 err:
7962 bio_put(bio);
7963 return ret;
7964 }
7965
7966 static int btrfs_check_dio_repairable(struct inode *inode,
7967 struct bio *failed_bio,
7968 struct io_failure_record *failrec,
7969 int failed_mirror)
7970 {
7971 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7972 int num_copies;
7973
7974 num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
7975 if (num_copies == 1) {
7976 /*
7977 * we only have a single copy of the data, so don't bother with
7978 * all the retry and error correction code that follows. no
7979 * matter what the error is, it is very likely to persist.
7980 */
7981 btrfs_debug(fs_info,
7982 "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
7983 num_copies, failrec->this_mirror, failed_mirror);
7984 return 0;
7985 }
7986
7987 failrec->failed_mirror = failed_mirror;
7988 failrec->this_mirror++;
7989 if (failrec->this_mirror == failed_mirror)
7990 failrec->this_mirror++;
7991
7992 if (failrec->this_mirror > num_copies) {
7993 btrfs_debug(fs_info,
7994 "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
7995 num_copies, failrec->this_mirror, failed_mirror);
7996 return 0;
7997 }
7998
7999 return 1;
8000 }
8001
8002 static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio,
8003 struct page *page, unsigned int pgoff,
8004 u64 start, u64 end, int failed_mirror,
8005 bio_end_io_t *repair_endio, void *repair_arg)
8006 {
8007 struct io_failure_record *failrec;
8008 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
8009 struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
8010 struct bio *bio;
8011 int isector;
8012 unsigned int read_mode = 0;
8013 int segs;
8014 int ret;
8015 blk_status_t status;
8016
8017 BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
8018
8019 ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
8020 if (ret)
8021 return errno_to_blk_status(ret);
8022
8023 ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
8024 failed_mirror);
8025 if (!ret) {
8026 free_io_failure(failure_tree, io_tree, failrec);
8027 return BLK_STS_IOERR;
8028 }
8029
8030 segs = bio_segments(failed_bio);
8031 if (segs > 1 ||
8032 (failed_bio->bi_io_vec->bv_len > btrfs_inode_sectorsize(inode)))
8033 read_mode |= REQ_FAILFAST_DEV;
8034
8035 isector = start - btrfs_io_bio(failed_bio)->logical;
8036 isector >>= inode->i_sb->s_blocksize_bits;
8037 bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
8038 pgoff, isector, repair_endio, repair_arg);
8039 bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
8040
8041 btrfs_debug(BTRFS_I(inode)->root->fs_info,
8042 "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d",
8043 read_mode, failrec->this_mirror, failrec->in_validation);
8044
8045 status = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
8046 if (status) {
8047 free_io_failure(failure_tree, io_tree, failrec);
8048 bio_put(bio);
8049 }
8050
8051 return status;
8052 }
8053
8054 struct btrfs_retry_complete {
8055 struct completion done;
8056 struct inode *inode;
8057 u64 start;
8058 int uptodate;
8059 };
8060
8061 static void btrfs_retry_endio_nocsum(struct bio *bio)
8062 {
8063 struct btrfs_retry_complete *done = bio->bi_private;
8064 struct inode *inode = done->inode;
8065 struct bio_vec *bvec;
8066 struct extent_io_tree *io_tree, *failure_tree;
8067 int i;
8068
8069 if (bio->bi_status)
8070 goto end;
8071
8072 ASSERT(bio->bi_vcnt == 1);
8073 io_tree = &BTRFS_I(inode)->io_tree;
8074 failure_tree = &BTRFS_I(inode)->io_failure_tree;
8075 ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(inode));
8076
8077 done->uptodate = 1;
8078 ASSERT(!bio_flagged(bio, BIO_CLONED));
8079 bio_for_each_segment_all(bvec, bio, i)
8080 clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
8081 io_tree, done->start, bvec->bv_page,
8082 btrfs_ino(BTRFS_I(inode)), 0);
8083 end:
8084 complete(&done->done);
8085 bio_put(bio);
8086 }
8087
8088 static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode,
8089 struct btrfs_io_bio *io_bio)
8090 {
8091 struct btrfs_fs_info *fs_info;
8092 struct bio_vec bvec;
8093 struct bvec_iter iter;
8094 struct btrfs_retry_complete done;
8095 u64 start;
8096 unsigned int pgoff;
8097 u32 sectorsize;
8098 int nr_sectors;
8099 blk_status_t ret;
8100 blk_status_t err = BLK_STS_OK;
8101
8102 fs_info = BTRFS_I(inode)->root->fs_info;
8103 sectorsize = fs_info->sectorsize;
8104
8105 start = io_bio->logical;
8106 done.inode = inode;
8107 io_bio->bio.bi_iter = io_bio->iter;
8108
8109 bio_for_each_segment(bvec, &io_bio->bio, iter) {
8110 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
8111 pgoff = bvec.bv_offset;
8112
8113 next_block_or_try_again:
8114 done.uptodate = 0;
8115 done.start = start;
8116 init_completion(&done.done);
8117
8118 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
8119 pgoff, start, start + sectorsize - 1,
8120 io_bio->mirror_num,
8121 btrfs_retry_endio_nocsum, &done);
8122 if (ret) {
8123 err = ret;
8124 goto next;
8125 }
8126
8127 wait_for_completion_io(&done.done);
8128
8129 if (!done.uptodate) {
8130 /* We might have another mirror, so try again */
8131 goto next_block_or_try_again;
8132 }
8133
8134 next:
8135 start += sectorsize;
8136
8137 nr_sectors--;
8138 if (nr_sectors) {
8139 pgoff += sectorsize;
8140 ASSERT(pgoff < PAGE_SIZE);
8141 goto next_block_or_try_again;
8142 }
8143 }
8144
8145 return err;
8146 }
8147
8148 static void btrfs_retry_endio(struct bio *bio)
8149 {
8150 struct btrfs_retry_complete *done = bio->bi_private;
8151 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8152 struct extent_io_tree *io_tree, *failure_tree;
8153 struct inode *inode = done->inode;
8154 struct bio_vec *bvec;
8155 int uptodate;
8156 int ret;
8157 int i;
8158
8159 if (bio->bi_status)
8160 goto end;
8161
8162 uptodate = 1;
8163
8164 ASSERT(bio->bi_vcnt == 1);
8165 ASSERT(bio->bi_io_vec->bv_len == btrfs_inode_sectorsize(done->inode));
8166
8167 io_tree = &BTRFS_I(inode)->io_tree;
8168 failure_tree = &BTRFS_I(inode)->io_failure_tree;
8169
8170 ASSERT(!bio_flagged(bio, BIO_CLONED));
8171 bio_for_each_segment_all(bvec, bio, i) {
8172 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
8173 bvec->bv_offset, done->start,
8174 bvec->bv_len);
8175 if (!ret)
8176 clean_io_failure(BTRFS_I(inode)->root->fs_info,
8177 failure_tree, io_tree, done->start,
8178 bvec->bv_page,
8179 btrfs_ino(BTRFS_I(inode)),
8180 bvec->bv_offset);
8181 else
8182 uptodate = 0;
8183 }
8184
8185 done->uptodate = uptodate;
8186 end:
8187 complete(&done->done);
8188 bio_put(bio);
8189 }
8190
8191 static blk_status_t __btrfs_subio_endio_read(struct inode *inode,
8192 struct btrfs_io_bio *io_bio, blk_status_t err)
8193 {
8194 struct btrfs_fs_info *fs_info;
8195 struct bio_vec bvec;
8196 struct bvec_iter iter;
8197 struct btrfs_retry_complete done;
8198 u64 start;
8199 u64 offset = 0;
8200 u32 sectorsize;
8201 int nr_sectors;
8202 unsigned int pgoff;
8203 int csum_pos;
8204 bool uptodate = (err == 0);
8205 int ret;
8206 blk_status_t status;
8207
8208 fs_info = BTRFS_I(inode)->root->fs_info;
8209 sectorsize = fs_info->sectorsize;
8210
8211 err = BLK_STS_OK;
8212 start = io_bio->logical;
8213 done.inode = inode;
8214 io_bio->bio.bi_iter = io_bio->iter;
8215
8216 bio_for_each_segment(bvec, &io_bio->bio, iter) {
8217 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
8218
8219 pgoff = bvec.bv_offset;
8220 next_block:
8221 if (uptodate) {
8222 csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
8223 ret = __readpage_endio_check(inode, io_bio, csum_pos,
8224 bvec.bv_page, pgoff, start, sectorsize);
8225 if (likely(!ret))
8226 goto next;
8227 }
8228 try_again:
8229 done.uptodate = 0;
8230 done.start = start;
8231 init_completion(&done.done);
8232
8233 status = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
8234 pgoff, start, start + sectorsize - 1,
8235 io_bio->mirror_num, btrfs_retry_endio,
8236 &done);
8237 if (status) {
8238 err = status;
8239 goto next;
8240 }
8241
8242 wait_for_completion_io(&done.done);
8243
8244 if (!done.uptodate) {
8245 /* We might have another mirror, so try again */
8246 goto try_again;
8247 }
8248 next:
8249 offset += sectorsize;
8250 start += sectorsize;
8251
8252 ASSERT(nr_sectors);
8253
8254 nr_sectors--;
8255 if (nr_sectors) {
8256 pgoff += sectorsize;
8257 ASSERT(pgoff < PAGE_SIZE);
8258 goto next_block;
8259 }
8260 }
8261
8262 return err;
8263 }
8264
8265 static blk_status_t btrfs_subio_endio_read(struct inode *inode,
8266 struct btrfs_io_bio *io_bio, blk_status_t err)
8267 {
8268 bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8269
8270 if (skip_csum) {
8271 if (unlikely(err))
8272 return __btrfs_correct_data_nocsum(inode, io_bio);
8273 else
8274 return BLK_STS_OK;
8275 } else {
8276 return __btrfs_subio_endio_read(inode, io_bio, err);
8277 }
8278 }
8279
8280 static void btrfs_endio_direct_read(struct bio *bio)
8281 {
8282 struct btrfs_dio_private *dip = bio->bi_private;
8283 struct inode *inode = dip->inode;
8284 struct bio *dio_bio;
8285 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8286 blk_status_t err = bio->bi_status;
8287
8288 if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
8289 err = btrfs_subio_endio_read(inode, io_bio, err);
8290
8291 unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
8292 dip->logical_offset + dip->bytes - 1);
8293 dio_bio = dip->dio_bio;
8294
8295 kfree(dip);
8296
8297 dio_bio->bi_status = err;
8298 dio_end_io(dio_bio);
8299
8300 if (io_bio->end_io)
8301 io_bio->end_io(io_bio, blk_status_to_errno(err));
8302 bio_put(bio);
8303 }
8304
8305 static void __endio_write_update_ordered(struct inode *inode,
8306 const u64 offset, const u64 bytes,
8307 const bool uptodate)
8308 {
8309 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8310 struct btrfs_ordered_extent *ordered = NULL;
8311 struct btrfs_workqueue *wq;
8312 btrfs_work_func_t func;
8313 u64 ordered_offset = offset;
8314 u64 ordered_bytes = bytes;
8315 u64 last_offset;
8316 int ret;
8317
8318 if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
8319 wq = fs_info->endio_freespace_worker;
8320 func = btrfs_freespace_write_helper;
8321 } else {
8322 wq = fs_info->endio_write_workers;
8323 func = btrfs_endio_write_helper;
8324 }
8325
8326 again:
8327 last_offset = ordered_offset;
8328 ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
8329 &ordered_offset,
8330 ordered_bytes,
8331 uptodate);
8332 if (!ret)
8333 goto out_test;
8334
8335 btrfs_init_work(&ordered->work, func, finish_ordered_fn, NULL, NULL);
8336 btrfs_queue_work(wq, &ordered->work);
8337 out_test:
8338 /*
8339 * If btrfs_dec_test_ordered_pending does not find any ordered extent
8340 * in the range, we can exit.
8341 */
8342 if (ordered_offset == last_offset)
8343 return;
8344 /*
8345 * our bio might span multiple ordered extents. If we haven't
8346 * completed the accounting for the whole dio, go back and try again
8347 */
8348 if (ordered_offset < offset + bytes) {
8349 ordered_bytes = offset + bytes - ordered_offset;
8350 ordered = NULL;
8351 goto again;
8352 }
8353 }
8354
8355 static void btrfs_endio_direct_write(struct bio *bio)
8356 {
8357 struct btrfs_dio_private *dip = bio->bi_private;
8358 struct bio *dio_bio = dip->dio_bio;
8359
8360 __endio_write_update_ordered(dip->inode, dip->logical_offset,
8361 dip->bytes, !bio->bi_status);
8362
8363 kfree(dip);
8364
8365 dio_bio->bi_status = bio->bi_status;
8366 dio_end_io(dio_bio);
8367 bio_put(bio);
8368 }
8369
8370 static blk_status_t __btrfs_submit_bio_start_direct_io(void *private_data,
8371 struct bio *bio, int mirror_num,
8372 unsigned long bio_flags, u64 offset)
8373 {
8374 struct inode *inode = private_data;
8375 blk_status_t ret;
8376 ret = btrfs_csum_one_bio(inode, bio, offset, 1);
8377 BUG_ON(ret); /* -ENOMEM */
8378 return 0;
8379 }
8380
8381 static void btrfs_end_dio_bio(struct bio *bio)
8382 {
8383 struct btrfs_dio_private *dip = bio->bi_private;
8384 blk_status_t err = bio->bi_status;
8385
8386 if (err)
8387 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
8388 "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
8389 btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio),
8390 bio->bi_opf,
8391 (unsigned long long)bio->bi_iter.bi_sector,
8392 bio->bi_iter.bi_size, err);
8393
8394 if (dip->subio_endio)
8395 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
8396
8397 if (err) {
8398 dip->errors = 1;
8399
8400 /*
8401 * before atomic variable goto zero, we must make sure
8402 * dip->errors is perceived to be set.
8403 */
8404 smp_mb__before_atomic();
8405 }
8406
8407 /* if there are more bios still pending for this dio, just exit */
8408 if (!atomic_dec_and_test(&dip->pending_bios))
8409 goto out;
8410
8411 if (dip->errors) {
8412 bio_io_error(dip->orig_bio);
8413 } else {
8414 dip->dio_bio->bi_status = BLK_STS_OK;
8415 bio_endio(dip->orig_bio);
8416 }
8417 out:
8418 bio_put(bio);
8419 }
8420
8421 static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode,
8422 struct btrfs_dio_private *dip,
8423 struct bio *bio,
8424 u64 file_offset)
8425 {
8426 struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8427 struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
8428 blk_status_t ret;
8429
8430 /*
8431 * We load all the csum data we need when we submit
8432 * the first bio to reduce the csum tree search and
8433 * contention.
8434 */
8435 if (dip->logical_offset == file_offset) {
8436 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio,
8437 file_offset);
8438 if (ret)
8439 return ret;
8440 }
8441
8442 if (bio == dip->orig_bio)
8443 return 0;
8444
8445 file_offset -= dip->logical_offset;
8446 file_offset >>= inode->i_sb->s_blocksize_bits;
8447 io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset);
8448
8449 return 0;
8450 }
8451
8452 static inline blk_status_t
8453 __btrfs_submit_dio_bio(struct bio *bio, struct inode *inode, u64 file_offset,
8454 int async_submit)
8455 {
8456 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8457 struct btrfs_dio_private *dip = bio->bi_private;
8458 bool write = bio_op(bio) == REQ_OP_WRITE;
8459 blk_status_t ret;
8460
8461 if (async_submit)
8462 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
8463
8464 bio_get(bio);
8465
8466 if (!write) {
8467 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
8468 if (ret)
8469 goto err;
8470 }
8471
8472 if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
8473 goto map;
8474
8475 if (write && async_submit) {
8476 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0,
8477 file_offset, inode,
8478 __btrfs_submit_bio_start_direct_io,
8479 __btrfs_submit_bio_done);
8480 goto err;
8481 } else if (write) {
8482 /*
8483 * If we aren't doing async submit, calculate the csum of the
8484 * bio now.
8485 */
8486 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1);
8487 if (ret)
8488 goto err;
8489 } else {
8490 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio,
8491 file_offset);
8492 if (ret)
8493 goto err;
8494 }
8495 map:
8496 ret = btrfs_map_bio(fs_info, bio, 0, 0);
8497 err:
8498 bio_put(bio);
8499 return ret;
8500 }
8501
8502 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip)
8503 {
8504 struct inode *inode = dip->inode;
8505 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8506 struct bio *bio;
8507 struct bio *orig_bio = dip->orig_bio;
8508 u64 start_sector = orig_bio->bi_iter.bi_sector;
8509 u64 file_offset = dip->logical_offset;
8510 u64 map_length;
8511 int async_submit = 0;
8512 u64 submit_len;
8513 int clone_offset = 0;
8514 int clone_len;
8515 int ret;
8516 blk_status_t status;
8517
8518 map_length = orig_bio->bi_iter.bi_size;
8519 submit_len = map_length;
8520 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9,
8521 &map_length, NULL, 0);
8522 if (ret)
8523 return -EIO;
8524
8525 if (map_length >= submit_len) {
8526 bio = orig_bio;
8527 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
8528 goto submit;
8529 }
8530
8531 /* async crcs make it difficult to collect full stripe writes. */
8532 if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK)
8533 async_submit = 0;
8534 else
8535 async_submit = 1;
8536
8537 /* bio split */
8538 ASSERT(map_length <= INT_MAX);
8539 atomic_inc(&dip->pending_bios);
8540 do {
8541 clone_len = min_t(int, submit_len, map_length);
8542
8543 /*
8544 * This will never fail as it's passing GPF_NOFS and
8545 * the allocation is backed by btrfs_bioset.
8546 */
8547 bio = btrfs_bio_clone_partial(orig_bio, clone_offset,
8548 clone_len);
8549 bio->bi_private = dip;
8550 bio->bi_end_io = btrfs_end_dio_bio;
8551 btrfs_io_bio(bio)->logical = file_offset;
8552
8553 ASSERT(submit_len >= clone_len);
8554 submit_len -= clone_len;
8555 if (submit_len == 0)
8556 break;
8557
8558 /*
8559 * Increase the count before we submit the bio so we know
8560 * the end IO handler won't happen before we increase the
8561 * count. Otherwise, the dip might get freed before we're
8562 * done setting it up.
8563 */
8564 atomic_inc(&dip->pending_bios);
8565
8566 status = __btrfs_submit_dio_bio(bio, inode, file_offset,
8567 async_submit);
8568 if (status) {
8569 bio_put(bio);
8570 atomic_dec(&dip->pending_bios);
8571 goto out_err;
8572 }
8573
8574 clone_offset += clone_len;
8575 start_sector += clone_len >> 9;
8576 file_offset += clone_len;
8577
8578 map_length = submit_len;
8579 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio),
8580 start_sector << 9, &map_length, NULL, 0);
8581 if (ret)
8582 goto out_err;
8583 } while (submit_len > 0);
8584
8585 submit:
8586 status = __btrfs_submit_dio_bio(bio, inode, file_offset, async_submit);
8587 if (!status)
8588 return 0;
8589
8590 bio_put(bio);
8591 out_err:
8592 dip->errors = 1;
8593 /*
8594 * before atomic variable goto zero, we must
8595 * make sure dip->errors is perceived to be set.
8596 */
8597 smp_mb__before_atomic();
8598 if (atomic_dec_and_test(&dip->pending_bios))
8599 bio_io_error(dip->orig_bio);
8600
8601 /* bio_end_io() will handle error, so we needn't return it */
8602 return 0;
8603 }
8604
8605 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
8606 loff_t file_offset)
8607 {
8608 struct btrfs_dio_private *dip = NULL;
8609 struct bio *bio = NULL;
8610 struct btrfs_io_bio *io_bio;
8611 bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8612 int ret = 0;
8613
8614 bio = btrfs_bio_clone(dio_bio);
8615
8616 dip = kzalloc(sizeof(*dip), GFP_NOFS);
8617 if (!dip) {
8618 ret = -ENOMEM;
8619 goto free_ordered;
8620 }
8621
8622 dip->private = dio_bio->bi_private;
8623 dip->inode = inode;
8624 dip->logical_offset = file_offset;
8625 dip->bytes = dio_bio->bi_iter.bi_size;
8626 dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8627 bio->bi_private = dip;
8628 dip->orig_bio = bio;
8629 dip->dio_bio = dio_bio;
8630 atomic_set(&dip->pending_bios, 0);
8631 io_bio = btrfs_io_bio(bio);
8632 io_bio->logical = file_offset;
8633
8634 if (write) {
8635 bio->bi_end_io = btrfs_endio_direct_write;
8636 } else {
8637 bio->bi_end_io = btrfs_endio_direct_read;
8638 dip->subio_endio = btrfs_subio_endio_read;
8639 }
8640
8641 /*
8642 * Reset the range for unsubmitted ordered extents (to a 0 length range)
8643 * even if we fail to submit a bio, because in such case we do the
8644 * corresponding error handling below and it must not be done a second
8645 * time by btrfs_direct_IO().
8646 */
8647 if (write) {
8648 struct btrfs_dio_data *dio_data = current->journal_info;
8649
8650 dio_data->unsubmitted_oe_range_end = dip->logical_offset +
8651 dip->bytes;
8652 dio_data->unsubmitted_oe_range_start =
8653 dio_data->unsubmitted_oe_range_end;
8654 }
8655
8656 ret = btrfs_submit_direct_hook(dip);
8657 if (!ret)
8658 return;
8659
8660 if (io_bio->end_io)
8661 io_bio->end_io(io_bio, ret);
8662
8663 free_ordered:
8664 /*
8665 * If we arrived here it means either we failed to submit the dip
8666 * or we either failed to clone the dio_bio or failed to allocate the
8667 * dip. If we cloned the dio_bio and allocated the dip, we can just
8668 * call bio_endio against our io_bio so that we get proper resource
8669 * cleanup if we fail to submit the dip, otherwise, we must do the
8670 * same as btrfs_endio_direct_[write|read] because we can't call these
8671 * callbacks - they require an allocated dip and a clone of dio_bio.
8672 */
8673 if (bio && dip) {
8674 bio_io_error(bio);
8675 /*
8676 * The end io callbacks free our dip, do the final put on bio
8677 * and all the cleanup and final put for dio_bio (through
8678 * dio_end_io()).
8679 */
8680 dip = NULL;
8681 bio = NULL;
8682 } else {
8683 if (write)
8684 __endio_write_update_ordered(inode,
8685 file_offset,
8686 dio_bio->bi_iter.bi_size,
8687 false);
8688 else
8689 unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8690 file_offset + dio_bio->bi_iter.bi_size - 1);
8691
8692 dio_bio->bi_status = BLK_STS_IOERR;
8693 /*
8694 * Releases and cleans up our dio_bio, no need to bio_put()
8695 * nor bio_endio()/bio_io_error() against dio_bio.
8696 */
8697 dio_end_io(dio_bio);
8698 }
8699 if (bio)
8700 bio_put(bio);
8701 kfree(dip);
8702 }
8703
8704 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
8705 const struct iov_iter *iter, loff_t offset)
8706 {
8707 int seg;
8708 int i;
8709 unsigned int blocksize_mask = fs_info->sectorsize - 1;
8710 ssize_t retval = -EINVAL;
8711
8712 if (offset & blocksize_mask)
8713 goto out;
8714
8715 if (iov_iter_alignment(iter) & blocksize_mask)
8716 goto out;
8717
8718 /* If this is a write we don't need to check anymore */
8719 if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
8720 return 0;
8721 /*
8722 * Check to make sure we don't have duplicate iov_base's in this
8723 * iovec, if so return EINVAL, otherwise we'll get csum errors
8724 * when reading back.
8725 */
8726 for (seg = 0; seg < iter->nr_segs; seg++) {
8727 for (i = seg + 1; i < iter->nr_segs; i++) {
8728 if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8729 goto out;
8730 }
8731 }
8732 retval = 0;
8733 out:
8734 return retval;
8735 }
8736
8737 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
8738 {
8739 struct file *file = iocb->ki_filp;
8740 struct inode *inode = file->f_mapping->host;
8741 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8742 struct btrfs_dio_data dio_data = { 0 };
8743 struct extent_changeset *data_reserved = NULL;
8744 loff_t offset = iocb->ki_pos;
8745 size_t count = 0;
8746 int flags = 0;
8747 bool wakeup = true;
8748 bool relock = false;
8749 ssize_t ret;
8750
8751 if (check_direct_IO(fs_info, iter, offset))
8752 return 0;
8753
8754 inode_dio_begin(inode);
8755
8756 /*
8757 * The generic stuff only does filemap_write_and_wait_range, which
8758 * isn't enough if we've written compressed pages to this area, so
8759 * we need to flush the dirty pages again to make absolutely sure
8760 * that any outstanding dirty pages are on disk.
8761 */
8762 count = iov_iter_count(iter);
8763 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8764 &BTRFS_I(inode)->runtime_flags))
8765 filemap_fdatawrite_range(inode->i_mapping, offset,
8766 offset + count - 1);
8767
8768 if (iov_iter_rw(iter) == WRITE) {
8769 /*
8770 * If the write DIO is beyond the EOF, we need update
8771 * the isize, but it is protected by i_mutex. So we can
8772 * not unlock the i_mutex at this case.
8773 */
8774 if (offset + count <= inode->i_size) {
8775 dio_data.overwrite = 1;
8776 inode_unlock(inode);
8777 relock = true;
8778 } else if (iocb->ki_flags & IOCB_NOWAIT) {
8779 ret = -EAGAIN;
8780 goto out;
8781 }
8782 ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
8783 offset, count);
8784 if (ret)
8785 goto out;
8786
8787 /*
8788 * We need to know how many extents we reserved so that we can
8789 * do the accounting properly if we go over the number we
8790 * originally calculated. Abuse current->journal_info for this.
8791 */
8792 dio_data.reserve = round_up(count,
8793 fs_info->sectorsize);
8794 dio_data.unsubmitted_oe_range_start = (u64)offset;
8795 dio_data.unsubmitted_oe_range_end = (u64)offset;
8796 current->journal_info = &dio_data;
8797 down_read(&BTRFS_I(inode)->dio_sem);
8798 } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8799 &BTRFS_I(inode)->runtime_flags)) {
8800 inode_dio_end(inode);
8801 flags = DIO_LOCKING | DIO_SKIP_HOLES;
8802 wakeup = false;
8803 }
8804
8805 ret = __blockdev_direct_IO(iocb, inode,
8806 fs_info->fs_devices->latest_bdev,
8807 iter, btrfs_get_blocks_direct, NULL,
8808 btrfs_submit_direct, flags);
8809 if (iov_iter_rw(iter) == WRITE) {
8810 up_read(&BTRFS_I(inode)->dio_sem);
8811 current->journal_info = NULL;
8812 if (ret < 0 && ret != -EIOCBQUEUED) {
8813 if (dio_data.reserve)
8814 btrfs_delalloc_release_space(inode, data_reserved,
8815 offset, dio_data.reserve);
8816 /*
8817 * On error we might have left some ordered extents
8818 * without submitting corresponding bios for them, so
8819 * cleanup them up to avoid other tasks getting them
8820 * and waiting for them to complete forever.
8821 */
8822 if (dio_data.unsubmitted_oe_range_start <
8823 dio_data.unsubmitted_oe_range_end)
8824 __endio_write_update_ordered(inode,
8825 dio_data.unsubmitted_oe_range_start,
8826 dio_data.unsubmitted_oe_range_end -
8827 dio_data.unsubmitted_oe_range_start,
8828 false);
8829 } else if (ret >= 0 && (size_t)ret < count)
8830 btrfs_delalloc_release_space(inode, data_reserved,
8831 offset, count - (size_t)ret);
8832 btrfs_delalloc_release_extents(BTRFS_I(inode), count);
8833 }
8834 out:
8835 if (wakeup)
8836 inode_dio_end(inode);
8837 if (relock)
8838 inode_lock(inode);
8839
8840 extent_changeset_free(data_reserved);
8841 return ret;
8842 }
8843
8844 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC)
8845
8846 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
8847 __u64 start, __u64 len)
8848 {
8849 int ret;
8850
8851 ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
8852 if (ret)
8853 return ret;
8854
8855 return extent_fiemap(inode, fieinfo, start, len, btrfs_get_extent_fiemap);
8856 }
8857
8858 int btrfs_readpage(struct file *file, struct page *page)
8859 {
8860 struct extent_io_tree *tree;
8861 tree = &BTRFS_I(page->mapping->host)->io_tree;
8862 return extent_read_full_page(tree, page, btrfs_get_extent, 0);
8863 }
8864
8865 static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
8866 {
8867 struct extent_io_tree *tree;
8868 struct inode *inode = page->mapping->host;
8869 int ret;
8870
8871 if (current->flags & PF_MEMALLOC) {
8872 redirty_page_for_writepage(wbc, page);
8873 unlock_page(page);
8874 return 0;
8875 }
8876
8877 /*
8878 * If we are under memory pressure we will call this directly from the
8879 * VM, we need to make sure we have the inode referenced for the ordered
8880 * extent. If not just return like we didn't do anything.
8881 */
8882 if (!igrab(inode)) {
8883 redirty_page_for_writepage(wbc, page);
8884 return AOP_WRITEPAGE_ACTIVATE;
8885 }
8886 tree = &BTRFS_I(page->mapping->host)->io_tree;
8887 ret = extent_write_full_page(tree, page, btrfs_get_extent, wbc);
8888 btrfs_add_delayed_iput(inode);
8889 return ret;
8890 }
8891
8892 static int btrfs_writepages(struct address_space *mapping,
8893 struct writeback_control *wbc)
8894 {
8895 struct extent_io_tree *tree;
8896
8897 tree = &BTRFS_I(mapping->host)->io_tree;
8898 return extent_writepages(tree, mapping, btrfs_get_extent, wbc);
8899 }
8900
8901 static int
8902 btrfs_readpages(struct file *file, struct address_space *mapping,
8903 struct list_head *pages, unsigned nr_pages)
8904 {
8905 struct extent_io_tree *tree;
8906 tree = &BTRFS_I(mapping->host)->io_tree;
8907 return extent_readpages(tree, mapping, pages, nr_pages,
8908 btrfs_get_extent);
8909 }
8910 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8911 {
8912 struct extent_io_tree *tree;
8913 struct extent_map_tree *map;
8914 int ret;
8915
8916 tree = &BTRFS_I(page->mapping->host)->io_tree;
8917 map = &BTRFS_I(page->mapping->host)->extent_tree;
8918 ret = try_release_extent_mapping(map, tree, page, gfp_flags);
8919 if (ret == 1) {
8920 ClearPagePrivate(page);
8921 set_page_private(page, 0);
8922 put_page(page);
8923 }
8924 return ret;
8925 }
8926
8927 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8928 {
8929 if (PageWriteback(page) || PageDirty(page))
8930 return 0;
8931 return __btrfs_releasepage(page, gfp_flags);
8932 }
8933
8934 static void btrfs_invalidatepage(struct page *page, unsigned int offset,
8935 unsigned int length)
8936 {
8937 struct inode *inode = page->mapping->host;
8938 struct extent_io_tree *tree;
8939 struct btrfs_ordered_extent *ordered;
8940 struct extent_state *cached_state = NULL;
8941 u64 page_start = page_offset(page);
8942 u64 page_end = page_start + PAGE_SIZE - 1;
8943 u64 start;
8944 u64 end;
8945 int inode_evicting = inode->i_state & I_FREEING;
8946
8947 /*
8948 * we have the page locked, so new writeback can't start,
8949 * and the dirty bit won't be cleared while we are here.
8950 *
8951 * Wait for IO on this page so that we can safely clear
8952 * the PagePrivate2 bit and do ordered accounting
8953 */
8954 wait_on_page_writeback(page);
8955
8956 tree = &BTRFS_I(inode)->io_tree;
8957 if (offset) {
8958 btrfs_releasepage(page, GFP_NOFS);
8959 return;
8960 }
8961
8962 if (!inode_evicting)
8963 lock_extent_bits(tree, page_start, page_end, &cached_state);
8964 again:
8965 start = page_start;
8966 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
8967 page_end - start + 1);
8968 if (ordered) {
8969 end = min(page_end, ordered->file_offset + ordered->len - 1);
8970 /*
8971 * IO on this page will never be started, so we need
8972 * to account for any ordered extents now
8973 */
8974 if (!inode_evicting)
8975 clear_extent_bit(tree, start, end,
8976 EXTENT_DIRTY | EXTENT_DELALLOC |
8977 EXTENT_DELALLOC_NEW |
8978 EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
8979 EXTENT_DEFRAG, 1, 0, &cached_state,
8980 GFP_NOFS);
8981 /*
8982 * whoever cleared the private bit is responsible
8983 * for the finish_ordered_io
8984 */
8985 if (TestClearPagePrivate2(page)) {
8986 struct btrfs_ordered_inode_tree *tree;
8987 u64 new_len;
8988
8989 tree = &BTRFS_I(inode)->ordered_tree;
8990
8991 spin_lock_irq(&tree->lock);
8992 set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
8993 new_len = start - ordered->file_offset;
8994 if (new_len < ordered->truncated_len)
8995 ordered->truncated_len = new_len;
8996 spin_unlock_irq(&tree->lock);
8997
8998 if (btrfs_dec_test_ordered_pending(inode, &ordered,
8999 start,
9000 end - start + 1, 1))
9001 btrfs_finish_ordered_io(ordered);
9002 }
9003 btrfs_put_ordered_extent(ordered);
9004 if (!inode_evicting) {
9005 cached_state = NULL;
9006 lock_extent_bits(tree, start, end,
9007 &cached_state);
9008 }
9009
9010 start = end + 1;
9011 if (start < page_end)
9012 goto again;
9013 }
9014
9015 /*
9016 * Qgroup reserved space handler
9017 * Page here will be either
9018 * 1) Already written to disk
9019 * In this case, its reserved space is released from data rsv map
9020 * and will be freed by delayed_ref handler finally.
9021 * So even we call qgroup_free_data(), it won't decrease reserved
9022 * space.
9023 * 2) Not written to disk
9024 * This means the reserved space should be freed here. However,
9025 * if a truncate invalidates the page (by clearing PageDirty)
9026 * and the page is accounted for while allocating extent
9027 * in btrfs_check_data_free_space() we let delayed_ref to
9028 * free the entire extent.
9029 */
9030 if (PageDirty(page))
9031 btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE);
9032 if (!inode_evicting) {
9033 clear_extent_bit(tree, page_start, page_end,
9034 EXTENT_LOCKED | EXTENT_DIRTY |
9035 EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
9036 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
9037 &cached_state, GFP_NOFS);
9038
9039 __btrfs_releasepage(page, GFP_NOFS);
9040 }
9041
9042 ClearPageChecked(page);
9043 if (PagePrivate(page)) {
9044 ClearPagePrivate(page);
9045 set_page_private(page, 0);
9046 put_page(page);
9047 }
9048 }
9049
9050 /*
9051 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
9052 * called from a page fault handler when a page is first dirtied. Hence we must
9053 * be careful to check for EOF conditions here. We set the page up correctly
9054 * for a written page which means we get ENOSPC checking when writing into
9055 * holes and correct delalloc and unwritten extent mapping on filesystems that
9056 * support these features.
9057 *
9058 * We are not allowed to take the i_mutex here so we have to play games to
9059 * protect against truncate races as the page could now be beyond EOF. Because
9060 * vmtruncate() writes the inode size before removing pages, once we have the
9061 * page lock we can determine safely if the page is beyond EOF. If it is not
9062 * beyond EOF, then the page is guaranteed safe against truncation until we
9063 * unlock the page.
9064 */
9065 int btrfs_page_mkwrite(struct vm_fault *vmf)
9066 {
9067 struct page *page = vmf->page;
9068 struct inode *inode = file_inode(vmf->vma->vm_file);
9069 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9070 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
9071 struct btrfs_ordered_extent *ordered;
9072 struct extent_state *cached_state = NULL;
9073 struct extent_changeset *data_reserved = NULL;
9074 char *kaddr;
9075 unsigned long zero_start;
9076 loff_t size;
9077 int ret;
9078 int reserved = 0;
9079 u64 reserved_space;
9080 u64 page_start;
9081 u64 page_end;
9082 u64 end;
9083
9084 reserved_space = PAGE_SIZE;
9085
9086 sb_start_pagefault(inode->i_sb);
9087 page_start = page_offset(page);
9088 page_end = page_start + PAGE_SIZE - 1;
9089 end = page_end;
9090
9091 /*
9092 * Reserving delalloc space after obtaining the page lock can lead to
9093 * deadlock. For example, if a dirty page is locked by this function
9094 * and the call to btrfs_delalloc_reserve_space() ends up triggering
9095 * dirty page write out, then the btrfs_writepage() function could
9096 * end up waiting indefinitely to get a lock on the page currently
9097 * being processed by btrfs_page_mkwrite() function.
9098 */
9099 ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
9100 reserved_space);
9101 if (!ret) {
9102 ret = file_update_time(vmf->vma->vm_file);
9103 reserved = 1;
9104 }
9105 if (ret) {
9106 if (ret == -ENOMEM)
9107 ret = VM_FAULT_OOM;
9108 else /* -ENOSPC, -EIO, etc */
9109 ret = VM_FAULT_SIGBUS;
9110 if (reserved)
9111 goto out;
9112 goto out_noreserve;
9113 }
9114
9115 ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
9116 again:
9117 lock_page(page);
9118 size = i_size_read(inode);
9119
9120 if ((page->mapping != inode->i_mapping) ||
9121 (page_start >= size)) {
9122 /* page got truncated out from underneath us */
9123 goto out_unlock;
9124 }
9125 wait_on_page_writeback(page);
9126
9127 lock_extent_bits(io_tree, page_start, page_end, &cached_state);
9128 set_page_extent_mapped(page);
9129
9130 /*
9131 * we can't set the delalloc bits if there are pending ordered
9132 * extents. Drop our locks and wait for them to finish
9133 */
9134 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
9135 PAGE_SIZE);
9136 if (ordered) {
9137 unlock_extent_cached(io_tree, page_start, page_end,
9138 &cached_state, GFP_NOFS);
9139 unlock_page(page);
9140 btrfs_start_ordered_extent(inode, ordered, 1);
9141 btrfs_put_ordered_extent(ordered);
9142 goto again;
9143 }
9144
9145 if (page->index == ((size - 1) >> PAGE_SHIFT)) {
9146 reserved_space = round_up(size - page_start,
9147 fs_info->sectorsize);
9148 if (reserved_space < PAGE_SIZE) {
9149 end = page_start + reserved_space - 1;
9150 btrfs_delalloc_release_space(inode, data_reserved,
9151 page_start, PAGE_SIZE - reserved_space);
9152 }
9153 }
9154
9155 /*
9156 * page_mkwrite gets called when the page is firstly dirtied after it's
9157 * faulted in, but write(2) could also dirty a page and set delalloc
9158 * bits, thus in this case for space account reason, we still need to
9159 * clear any delalloc bits within this page range since we have to
9160 * reserve data&meta space before lock_page() (see above comments).
9161 */
9162 clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
9163 EXTENT_DIRTY | EXTENT_DELALLOC |
9164 EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
9165 0, 0, &cached_state, GFP_NOFS);
9166
9167 ret = btrfs_set_extent_delalloc(inode, page_start, end, 0,
9168 &cached_state, 0);
9169 if (ret) {
9170 unlock_extent_cached(io_tree, page_start, page_end,
9171 &cached_state, GFP_NOFS);
9172 ret = VM_FAULT_SIGBUS;
9173 goto out_unlock;
9174 }
9175 ret = 0;
9176
9177 /* page is wholly or partially inside EOF */
9178 if (page_start + PAGE_SIZE > size)
9179 zero_start = size & ~PAGE_MASK;
9180 else
9181 zero_start = PAGE_SIZE;
9182
9183 if (zero_start != PAGE_SIZE) {
9184 kaddr = kmap(page);
9185 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
9186 flush_dcache_page(page);
9187 kunmap(page);
9188 }
9189 ClearPageChecked(page);
9190 set_page_dirty(page);
9191 SetPageUptodate(page);
9192
9193 BTRFS_I(inode)->last_trans = fs_info->generation;
9194 BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid;
9195 BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit;
9196
9197 unlock_extent_cached(io_tree, page_start, page_end, &cached_state, GFP_NOFS);
9198
9199 out_unlock:
9200 if (!ret) {
9201 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
9202 sb_end_pagefault(inode->i_sb);
9203 extent_changeset_free(data_reserved);
9204 return VM_FAULT_LOCKED;
9205 }
9206 unlock_page(page);
9207 out:
9208 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
9209 btrfs_delalloc_release_space(inode, data_reserved, page_start,
9210 reserved_space);
9211 out_noreserve:
9212 sb_end_pagefault(inode->i_sb);
9213 extent_changeset_free(data_reserved);
9214 return ret;
9215 }
9216
9217 static int btrfs_truncate(struct inode *inode)
9218 {
9219 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9220 struct btrfs_root *root = BTRFS_I(inode)->root;
9221 struct btrfs_block_rsv *rsv;
9222 int ret = 0;
9223 int err = 0;
9224 struct btrfs_trans_handle *trans;
9225 u64 mask = fs_info->sectorsize - 1;
9226 u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
9227
9228 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
9229 (u64)-1);
9230 if (ret)
9231 return ret;
9232
9233 /*
9234 * Yes ladies and gentlemen, this is indeed ugly. The fact is we have
9235 * 3 things going on here
9236 *
9237 * 1) We need to reserve space for our orphan item and the space to
9238 * delete our orphan item. Lord knows we don't want to have a dangling
9239 * orphan item because we didn't reserve space to remove it.
9240 *
9241 * 2) We need to reserve space to update our inode.
9242 *
9243 * 3) We need to have something to cache all the space that is going to
9244 * be free'd up by the truncate operation, but also have some slack
9245 * space reserved in case it uses space during the truncate (thank you
9246 * very much snapshotting).
9247 *
9248 * And we need these to all be separate. The fact is we can use a lot of
9249 * space doing the truncate, and we have no earthly idea how much space
9250 * we will use, so we need the truncate reservation to be separate so it
9251 * doesn't end up using space reserved for updating the inode or
9252 * removing the orphan item. We also need to be able to stop the
9253 * transaction and start a new one, which means we need to be able to
9254 * update the inode several times, and we have no idea of knowing how
9255 * many times that will be, so we can't just reserve 1 item for the
9256 * entirety of the operation, so that has to be done separately as well.
9257 * Then there is the orphan item, which does indeed need to be held on
9258 * to for the whole operation, and we need nobody to touch this reserved
9259 * space except the orphan code.
9260 *
9261 * So that leaves us with
9262 *
9263 * 1) root->orphan_block_rsv - for the orphan deletion.
9264 * 2) rsv - for the truncate reservation, which we will steal from the
9265 * transaction reservation.
9266 * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
9267 * updating the inode.
9268 */
9269 rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
9270 if (!rsv)
9271 return -ENOMEM;
9272 rsv->size = min_size;
9273 rsv->failfast = 1;
9274
9275 /*
9276 * 1 for the truncate slack space
9277 * 1 for updating the inode.
9278 */
9279 trans = btrfs_start_transaction(root, 2);
9280 if (IS_ERR(trans)) {
9281 err = PTR_ERR(trans);
9282 goto out;
9283 }
9284
9285 /* Migrate the slack space for the truncate to our reserve */
9286 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
9287 min_size, 0);
9288 BUG_ON(ret);
9289
9290 /*
9291 * So if we truncate and then write and fsync we normally would just
9292 * write the extents that changed, which is a problem if we need to
9293 * first truncate that entire inode. So set this flag so we write out
9294 * all of the extents in the inode to the sync log so we're completely
9295 * safe.
9296 */
9297 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
9298 trans->block_rsv = rsv;
9299
9300 while (1) {
9301 ret = btrfs_truncate_inode_items(trans, root, inode,
9302 inode->i_size,
9303 BTRFS_EXTENT_DATA_KEY);
9304 trans->block_rsv = &fs_info->trans_block_rsv;
9305 if (ret != -ENOSPC && ret != -EAGAIN) {
9306 err = ret;
9307 break;
9308 }
9309
9310 ret = btrfs_update_inode(trans, root, inode);
9311 if (ret) {
9312 err = ret;
9313 break;
9314 }
9315
9316 btrfs_end_transaction(trans);
9317 btrfs_btree_balance_dirty(fs_info);
9318
9319 trans = btrfs_start_transaction(root, 2);
9320 if (IS_ERR(trans)) {
9321 ret = err = PTR_ERR(trans);
9322 trans = NULL;
9323 break;
9324 }
9325
9326 btrfs_block_rsv_release(fs_info, rsv, -1);
9327 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
9328 rsv, min_size, 0);
9329 BUG_ON(ret); /* shouldn't happen */
9330 trans->block_rsv = rsv;
9331 }
9332
9333 /*
9334 * We can't call btrfs_truncate_block inside a trans handle as we could
9335 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know
9336 * we've truncated everything except the last little bit, and can do
9337 * btrfs_truncate_block and then update the disk_i_size.
9338 */
9339 if (ret == NEED_TRUNCATE_BLOCK) {
9340 btrfs_end_transaction(trans);
9341 btrfs_btree_balance_dirty(fs_info);
9342
9343 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0);
9344 if (ret)
9345 goto out;
9346 trans = btrfs_start_transaction(root, 1);
9347 if (IS_ERR(trans)) {
9348 ret = PTR_ERR(trans);
9349 goto out;
9350 }
9351 btrfs_ordered_update_i_size(inode, inode->i_size, NULL);
9352 }
9353
9354 if (ret == 0 && inode->i_nlink > 0) {
9355 trans->block_rsv = root->orphan_block_rsv;
9356 ret = btrfs_orphan_del(trans, BTRFS_I(inode));
9357 if (ret)
9358 err = ret;
9359 }
9360
9361 if (trans) {
9362 trans->block_rsv = &fs_info->trans_block_rsv;
9363 ret = btrfs_update_inode(trans, root, inode);
9364 if (ret && !err)
9365 err = ret;
9366
9367 ret = btrfs_end_transaction(trans);
9368 btrfs_btree_balance_dirty(fs_info);
9369 }
9370 out:
9371 btrfs_free_block_rsv(fs_info, rsv);
9372
9373 if (ret && !err)
9374 err = ret;
9375
9376 return err;
9377 }
9378
9379 /*
9380 * create a new subvolume directory/inode (helper for the ioctl).
9381 */
9382 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
9383 struct btrfs_root *new_root,
9384 struct btrfs_root *parent_root,
9385 u64 new_dirid)
9386 {
9387 struct inode *inode;
9388 int err;
9389 u64 index = 0;
9390
9391 inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
9392 new_dirid, new_dirid,
9393 S_IFDIR | (~current_umask() & S_IRWXUGO),
9394 &index);
9395 if (IS_ERR(inode))
9396 return PTR_ERR(inode);
9397 inode->i_op = &btrfs_dir_inode_operations;
9398 inode->i_fop = &btrfs_dir_file_operations;
9399
9400 set_nlink(inode, 1);
9401 btrfs_i_size_write(BTRFS_I(inode), 0);
9402 unlock_new_inode(inode);
9403
9404 err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
9405 if (err)
9406 btrfs_err(new_root->fs_info,
9407 "error inheriting subvolume %llu properties: %d",
9408 new_root->root_key.objectid, err);
9409
9410 err = btrfs_update_inode(trans, new_root, inode);
9411
9412 iput(inode);
9413 return err;
9414 }
9415
9416 struct inode *btrfs_alloc_inode(struct super_block *sb)
9417 {
9418 struct btrfs_fs_info *fs_info = btrfs_sb(sb);
9419 struct btrfs_inode *ei;
9420 struct inode *inode;
9421
9422 ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_NOFS);
9423 if (!ei)
9424 return NULL;
9425
9426 ei->root = NULL;
9427 ei->generation = 0;
9428 ei->last_trans = 0;
9429 ei->last_sub_trans = 0;
9430 ei->logged_trans = 0;
9431 ei->delalloc_bytes = 0;
9432 ei->new_delalloc_bytes = 0;
9433 ei->defrag_bytes = 0;
9434 ei->disk_i_size = 0;
9435 ei->flags = 0;
9436 ei->csum_bytes = 0;
9437 ei->index_cnt = (u64)-1;
9438 ei->dir_index = 0;
9439 ei->last_unlink_trans = 0;
9440 ei->last_log_commit = 0;
9441 ei->delayed_iput_count = 0;
9442
9443 spin_lock_init(&ei->lock);
9444 ei->outstanding_extents = 0;
9445 if (sb->s_magic != BTRFS_TEST_MAGIC)
9446 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv,
9447 BTRFS_BLOCK_RSV_DELALLOC);
9448 ei->runtime_flags = 0;
9449 ei->prop_compress = BTRFS_COMPRESS_NONE;
9450 ei->defrag_compress = BTRFS_COMPRESS_NONE;
9451
9452 ei->delayed_node = NULL;
9453
9454 ei->i_otime.tv_sec = 0;
9455 ei->i_otime.tv_nsec = 0;
9456
9457 inode = &ei->vfs_inode;
9458 extent_map_tree_init(&ei->extent_tree);
9459 extent_io_tree_init(&ei->io_tree, inode);
9460 extent_io_tree_init(&ei->io_failure_tree, inode);
9461 ei->io_tree.track_uptodate = 1;
9462 ei->io_failure_tree.track_uptodate = 1;
9463 atomic_set(&ei->sync_writers, 0);
9464 mutex_init(&ei->log_mutex);
9465 mutex_init(&ei->delalloc_mutex);
9466 btrfs_ordered_inode_tree_init(&ei->ordered_tree);
9467 INIT_LIST_HEAD(&ei->delalloc_inodes);
9468 INIT_LIST_HEAD(&ei->delayed_iput);
9469 RB_CLEAR_NODE(&ei->rb_node);
9470 init_rwsem(&ei->dio_sem);
9471
9472 return inode;
9473 }
9474
9475 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
9476 void btrfs_test_destroy_inode(struct inode *inode)
9477 {
9478 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9479 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9480 }
9481 #endif
9482
9483 static void btrfs_i_callback(struct rcu_head *head)
9484 {
9485 struct inode *inode = container_of(head, struct inode, i_rcu);
9486 kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9487 }
9488
9489 void btrfs_destroy_inode(struct inode *inode)
9490 {
9491 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9492 struct btrfs_ordered_extent *ordered;
9493 struct btrfs_root *root = BTRFS_I(inode)->root;
9494
9495 WARN_ON(!hlist_empty(&inode->i_dentry));
9496 WARN_ON(inode->i_data.nrpages);
9497 WARN_ON(BTRFS_I(inode)->block_rsv.reserved);
9498 WARN_ON(BTRFS_I(inode)->block_rsv.size);
9499 WARN_ON(BTRFS_I(inode)->outstanding_extents);
9500 WARN_ON(BTRFS_I(inode)->delalloc_bytes);
9501 WARN_ON(BTRFS_I(inode)->new_delalloc_bytes);
9502 WARN_ON(BTRFS_I(inode)->csum_bytes);
9503 WARN_ON(BTRFS_I(inode)->defrag_bytes);
9504
9505 /*
9506 * This can happen where we create an inode, but somebody else also
9507 * created the same inode and we need to destroy the one we already
9508 * created.
9509 */
9510 if (!root)
9511 goto free;
9512
9513 if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
9514 &BTRFS_I(inode)->runtime_flags)) {
9515 btrfs_info(fs_info, "inode %llu still on the orphan list",
9516 btrfs_ino(BTRFS_I(inode)));
9517 atomic_dec(&root->orphan_inodes);
9518 }
9519
9520 while (1) {
9521 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
9522 if (!ordered)
9523 break;
9524 else {
9525 btrfs_err(fs_info,
9526 "found ordered extent %llu %llu on inode cleanup",
9527 ordered->file_offset, ordered->len);
9528 btrfs_remove_ordered_extent(inode, ordered);
9529 btrfs_put_ordered_extent(ordered);
9530 btrfs_put_ordered_extent(ordered);
9531 }
9532 }
9533 btrfs_qgroup_check_reserved_leak(inode);
9534 inode_tree_del(inode);
9535 btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9536 free:
9537 call_rcu(&inode->i_rcu, btrfs_i_callback);
9538 }
9539
9540 int btrfs_drop_inode(struct inode *inode)
9541 {
9542 struct btrfs_root *root = BTRFS_I(inode)->root;
9543
9544 if (root == NULL)
9545 return 1;
9546
9547 /* the snap/subvol tree is on deleting */
9548 if (btrfs_root_refs(&root->root_item) == 0)
9549 return 1;
9550 else
9551 return generic_drop_inode(inode);
9552 }
9553
9554 static void init_once(void *foo)
9555 {
9556 struct btrfs_inode *ei = (struct btrfs_inode *) foo;
9557
9558 inode_init_once(&ei->vfs_inode);
9559 }
9560
9561 void btrfs_destroy_cachep(void)
9562 {
9563 /*
9564 * Make sure all delayed rcu free inodes are flushed before we
9565 * destroy cache.
9566 */
9567 rcu_barrier();
9568 kmem_cache_destroy(btrfs_inode_cachep);
9569 kmem_cache_destroy(btrfs_trans_handle_cachep);
9570 kmem_cache_destroy(btrfs_path_cachep);
9571 kmem_cache_destroy(btrfs_free_space_cachep);
9572 }
9573
9574 int btrfs_init_cachep(void)
9575 {
9576 btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
9577 sizeof(struct btrfs_inode), 0,
9578 SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
9579 init_once);
9580 if (!btrfs_inode_cachep)
9581 goto fail;
9582
9583 btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
9584 sizeof(struct btrfs_trans_handle), 0,
9585 SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
9586 if (!btrfs_trans_handle_cachep)
9587 goto fail;
9588
9589 btrfs_path_cachep = kmem_cache_create("btrfs_path",
9590 sizeof(struct btrfs_path), 0,
9591 SLAB_MEM_SPREAD, NULL);
9592 if (!btrfs_path_cachep)
9593 goto fail;
9594
9595 btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
9596 sizeof(struct btrfs_free_space), 0,
9597 SLAB_MEM_SPREAD, NULL);
9598 if (!btrfs_free_space_cachep)
9599 goto fail;
9600
9601 return 0;
9602 fail:
9603 btrfs_destroy_cachep();
9604 return -ENOMEM;
9605 }
9606
9607 static int btrfs_getattr(const struct path *path, struct kstat *stat,
9608 u32 request_mask, unsigned int flags)
9609 {
9610 u64 delalloc_bytes;
9611 struct inode *inode = d_inode(path->dentry);
9612 u32 blocksize = inode->i_sb->s_blocksize;
9613 u32 bi_flags = BTRFS_I(inode)->flags;
9614
9615 stat->result_mask |= STATX_BTIME;
9616 stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec;
9617 stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec;
9618 if (bi_flags & BTRFS_INODE_APPEND)
9619 stat->attributes |= STATX_ATTR_APPEND;
9620 if (bi_flags & BTRFS_INODE_COMPRESS)
9621 stat->attributes |= STATX_ATTR_COMPRESSED;
9622 if (bi_flags & BTRFS_INODE_IMMUTABLE)
9623 stat->attributes |= STATX_ATTR_IMMUTABLE;
9624 if (bi_flags & BTRFS_INODE_NODUMP)
9625 stat->attributes |= STATX_ATTR_NODUMP;
9626
9627 stat->attributes_mask |= (STATX_ATTR_APPEND |
9628 STATX_ATTR_COMPRESSED |
9629 STATX_ATTR_IMMUTABLE |
9630 STATX_ATTR_NODUMP);
9631
9632 generic_fillattr(inode, stat);
9633 stat->dev = BTRFS_I(inode)->root->anon_dev;
9634
9635 spin_lock(&BTRFS_I(inode)->lock);
9636 delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
9637 spin_unlock(&BTRFS_I(inode)->lock);
9638 stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
9639 ALIGN(delalloc_bytes, blocksize)) >> 9;
9640 return 0;
9641 }
9642
9643 static int btrfs_rename_exchange(struct inode *old_dir,
9644 struct dentry *old_dentry,
9645 struct inode *new_dir,
9646 struct dentry *new_dentry)
9647 {
9648 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9649 struct btrfs_trans_handle *trans;
9650 struct btrfs_root *root = BTRFS_I(old_dir)->root;
9651 struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9652 struct inode *new_inode = new_dentry->d_inode;
9653 struct inode *old_inode = old_dentry->d_inode;
9654 struct timespec ctime = current_time(old_inode);
9655 struct dentry *parent;
9656 u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9657 u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
9658 u64 old_idx = 0;
9659 u64 new_idx = 0;
9660 u64 root_objectid;
9661 int ret;
9662 bool root_log_pinned = false;
9663 bool dest_log_pinned = false;
9664
9665 /* we only allow rename subvolume link between subvolumes */
9666 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9667 return -EXDEV;
9668
9669 /* close the race window with snapshot create/destroy ioctl */
9670 if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9671 down_read(&fs_info->subvol_sem);
9672 if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9673 down_read(&fs_info->subvol_sem);
9674
9675 /*
9676 * We want to reserve the absolute worst case amount of items. So if
9677 * both inodes are subvols and we need to unlink them then that would
9678 * require 4 item modifications, but if they are both normal inodes it
9679 * would require 5 item modifications, so we'll assume their normal
9680 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items
9681 * should cover the worst case number of items we'll modify.
9682 */
9683 trans = btrfs_start_transaction(root, 12);
9684 if (IS_ERR(trans)) {
9685 ret = PTR_ERR(trans);
9686 goto out_notrans;
9687 }
9688
9689 /*
9690 * We need to find a free sequence number both in the source and
9691 * in the destination directory for the exchange.
9692 */
9693 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
9694 if (ret)
9695 goto out_fail;
9696 ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
9697 if (ret)
9698 goto out_fail;
9699
9700 BTRFS_I(old_inode)->dir_index = 0ULL;
9701 BTRFS_I(new_inode)->dir_index = 0ULL;
9702
9703 /* Reference for the source. */
9704 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9705 /* force full log commit if subvolume involved. */
9706 btrfs_set_log_full_commit(fs_info, trans);
9707 } else {
9708 btrfs_pin_log_trans(root);
9709 root_log_pinned = true;
9710 ret = btrfs_insert_inode_ref(trans, dest,
9711 new_dentry->d_name.name,
9712 new_dentry->d_name.len,
9713 old_ino,
9714 btrfs_ino(BTRFS_I(new_dir)),
9715 old_idx);
9716 if (ret)
9717 goto out_fail;
9718 }
9719
9720 /* And now for the dest. */
9721 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9722 /* force full log commit if subvolume involved. */
9723 btrfs_set_log_full_commit(fs_info, trans);
9724 } else {
9725 btrfs_pin_log_trans(dest);
9726 dest_log_pinned = true;
9727 ret = btrfs_insert_inode_ref(trans, root,
9728 old_dentry->d_name.name,
9729 old_dentry->d_name.len,
9730 new_ino,
9731 btrfs_ino(BTRFS_I(old_dir)),
9732 new_idx);
9733 if (ret)
9734 goto out_fail;
9735 }
9736
9737 /* Update inode version and ctime/mtime. */
9738 inode_inc_iversion(old_dir);
9739 inode_inc_iversion(new_dir);
9740 inode_inc_iversion(old_inode);
9741 inode_inc_iversion(new_inode);
9742 old_dir->i_ctime = old_dir->i_mtime = ctime;
9743 new_dir->i_ctime = new_dir->i_mtime = ctime;
9744 old_inode->i_ctime = ctime;
9745 new_inode->i_ctime = ctime;
9746
9747 if (old_dentry->d_parent != new_dentry->d_parent) {
9748 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
9749 BTRFS_I(old_inode), 1);
9750 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
9751 BTRFS_I(new_inode), 1);
9752 }
9753
9754 /* src is a subvolume */
9755 if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9756 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
9757 ret = btrfs_unlink_subvol(trans, root, old_dir,
9758 root_objectid,
9759 old_dentry->d_name.name,
9760 old_dentry->d_name.len);
9761 } else { /* src is an inode */
9762 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
9763 BTRFS_I(old_dentry->d_inode),
9764 old_dentry->d_name.name,
9765 old_dentry->d_name.len);
9766 if (!ret)
9767 ret = btrfs_update_inode(trans, root, old_inode);
9768 }
9769 if (ret) {
9770 btrfs_abort_transaction(trans, ret);
9771 goto out_fail;
9772 }
9773
9774 /* dest is a subvolume */
9775 if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9776 root_objectid = BTRFS_I(new_inode)->root->root_key.objectid;
9777 ret = btrfs_unlink_subvol(trans, dest, new_dir,
9778 root_objectid,
9779 new_dentry->d_name.name,
9780 new_dentry->d_name.len);
9781 } else { /* dest is an inode */
9782 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
9783 BTRFS_I(new_dentry->d_inode),
9784 new_dentry->d_name.name,
9785 new_dentry->d_name.len);
9786 if (!ret)
9787 ret = btrfs_update_inode(trans, dest, new_inode);
9788 }
9789 if (ret) {
9790 btrfs_abort_transaction(trans, ret);
9791 goto out_fail;
9792 }
9793
9794 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
9795 new_dentry->d_name.name,
9796 new_dentry->d_name.len, 0, old_idx);
9797 if (ret) {
9798 btrfs_abort_transaction(trans, ret);
9799 goto out_fail;
9800 }
9801
9802 ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
9803 old_dentry->d_name.name,
9804 old_dentry->d_name.len, 0, new_idx);
9805 if (ret) {
9806 btrfs_abort_transaction(trans, ret);
9807 goto out_fail;
9808 }
9809
9810 if (old_inode->i_nlink == 1)
9811 BTRFS_I(old_inode)->dir_index = old_idx;
9812 if (new_inode->i_nlink == 1)
9813 BTRFS_I(new_inode)->dir_index = new_idx;
9814
9815 if (root_log_pinned) {
9816 parent = new_dentry->d_parent;
9817 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
9818 parent);
9819 btrfs_end_log_trans(root);
9820 root_log_pinned = false;
9821 }
9822 if (dest_log_pinned) {
9823 parent = old_dentry->d_parent;
9824 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir),
9825 parent);
9826 btrfs_end_log_trans(dest);
9827 dest_log_pinned = false;
9828 }
9829 out_fail:
9830 /*
9831 * If we have pinned a log and an error happened, we unpin tasks
9832 * trying to sync the log and force them to fallback to a transaction
9833 * commit if the log currently contains any of the inodes involved in
9834 * this rename operation (to ensure we do not persist a log with an
9835 * inconsistent state for any of these inodes or leading to any
9836 * inconsistencies when replayed). If the transaction was aborted, the
9837 * abortion reason is propagated to userspace when attempting to commit
9838 * the transaction. If the log does not contain any of these inodes, we
9839 * allow the tasks to sync it.
9840 */
9841 if (ret && (root_log_pinned || dest_log_pinned)) {
9842 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
9843 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
9844 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
9845 (new_inode &&
9846 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
9847 btrfs_set_log_full_commit(fs_info, trans);
9848
9849 if (root_log_pinned) {
9850 btrfs_end_log_trans(root);
9851 root_log_pinned = false;
9852 }
9853 if (dest_log_pinned) {
9854 btrfs_end_log_trans(dest);
9855 dest_log_pinned = false;
9856 }
9857 }
9858 ret = btrfs_end_transaction(trans);
9859 out_notrans:
9860 if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9861 up_read(&fs_info->subvol_sem);
9862 if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9863 up_read(&fs_info->subvol_sem);
9864
9865 return ret;
9866 }
9867
9868 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
9869 struct btrfs_root *root,
9870 struct inode *dir,
9871 struct dentry *dentry)
9872 {
9873 int ret;
9874 struct inode *inode;
9875 u64 objectid;
9876 u64 index;
9877
9878 ret = btrfs_find_free_ino(root, &objectid);
9879 if (ret)
9880 return ret;
9881
9882 inode = btrfs_new_inode(trans, root, dir,
9883 dentry->d_name.name,
9884 dentry->d_name.len,
9885 btrfs_ino(BTRFS_I(dir)),
9886 objectid,
9887 S_IFCHR | WHITEOUT_MODE,
9888 &index);
9889
9890 if (IS_ERR(inode)) {
9891 ret = PTR_ERR(inode);
9892 return ret;
9893 }
9894
9895 inode->i_op = &btrfs_special_inode_operations;
9896 init_special_inode(inode, inode->i_mode,
9897 WHITEOUT_DEV);
9898
9899 ret = btrfs_init_inode_security(trans, inode, dir,
9900 &dentry->d_name);
9901 if (ret)
9902 goto out;
9903
9904 ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
9905 BTRFS_I(inode), 0, index);
9906 if (ret)
9907 goto out;
9908
9909 ret = btrfs_update_inode(trans, root, inode);
9910 out:
9911 unlock_new_inode(inode);
9912 if (ret)
9913 inode_dec_link_count(inode);
9914 iput(inode);
9915
9916 return ret;
9917 }
9918
9919 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
9920 struct inode *new_dir, struct dentry *new_dentry,
9921 unsigned int flags)
9922 {
9923 struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9924 struct btrfs_trans_handle *trans;
9925 unsigned int trans_num_items;
9926 struct btrfs_root *root = BTRFS_I(old_dir)->root;
9927 struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9928 struct inode *new_inode = d_inode(new_dentry);
9929 struct inode *old_inode = d_inode(old_dentry);
9930 u64 index = 0;
9931 u64 root_objectid;
9932 int ret;
9933 u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9934 bool log_pinned = false;
9935
9936 if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
9937 return -EPERM;
9938
9939 /* we only allow rename subvolume link between subvolumes */
9940 if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9941 return -EXDEV;
9942
9943 if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
9944 (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
9945 return -ENOTEMPTY;
9946
9947 if (S_ISDIR(old_inode->i_mode) && new_inode &&
9948 new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
9949 return -ENOTEMPTY;
9950
9951
9952 /* check for collisions, even if the name isn't there */
9953 ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
9954 new_dentry->d_name.name,
9955 new_dentry->d_name.len);
9956
9957 if (ret) {
9958 if (ret == -EEXIST) {
9959 /* we shouldn't get
9960 * eexist without a new_inode */
9961 if (WARN_ON(!new_inode)) {
9962 return ret;
9963 }
9964 } else {
9965 /* maybe -EOVERFLOW */
9966 return ret;
9967 }
9968 }
9969 ret = 0;
9970
9971 /*
9972 * we're using rename to replace one file with another. Start IO on it
9973 * now so we don't add too much work to the end of the transaction
9974 */
9975 if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
9976 filemap_flush(old_inode->i_mapping);
9977
9978 /* close the racy window with snapshot create/destroy ioctl */
9979 if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9980 down_read(&fs_info->subvol_sem);
9981 /*
9982 * We want to reserve the absolute worst case amount of items. So if
9983 * both inodes are subvols and we need to unlink them then that would
9984 * require 4 item modifications, but if they are both normal inodes it
9985 * would require 5 item modifications, so we'll assume they are normal
9986 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9987 * should cover the worst case number of items we'll modify.
9988 * If our rename has the whiteout flag, we need more 5 units for the
9989 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
9990 * when selinux is enabled).
9991 */
9992 trans_num_items = 11;
9993 if (flags & RENAME_WHITEOUT)
9994 trans_num_items += 5;
9995 trans = btrfs_start_transaction(root, trans_num_items);
9996 if (IS_ERR(trans)) {
9997 ret = PTR_ERR(trans);
9998 goto out_notrans;
9999 }
10000
10001 if (dest != root)
10002 btrfs_record_root_in_trans(trans, dest);
10003
10004 ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
10005 if (ret)
10006 goto out_fail;
10007
10008 BTRFS_I(old_inode)->dir_index = 0ULL;
10009 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
10010 /* force full log commit if subvolume involved. */
10011 btrfs_set_log_full_commit(fs_info, trans);
10012 } else {
10013 btrfs_pin_log_trans(root);
10014 log_pinned = true;
10015 ret = btrfs_insert_inode_ref(trans, dest,
10016 new_dentry->d_name.name,
10017 new_dentry->d_name.len,
10018 old_ino,
10019 btrfs_ino(BTRFS_I(new_dir)), index);
10020 if (ret)
10021 goto out_fail;
10022 }
10023
10024 inode_inc_iversion(old_dir);
10025 inode_inc_iversion(new_dir);
10026 inode_inc_iversion(old_inode);
10027 old_dir->i_ctime = old_dir->i_mtime =
10028 new_dir->i_ctime = new_dir->i_mtime =
10029 old_inode->i_ctime = current_time(old_dir);
10030
10031 if (old_dentry->d_parent != new_dentry->d_parent)
10032 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
10033 BTRFS_I(old_inode), 1);
10034
10035 if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
10036 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
10037 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid,
10038 old_dentry->d_name.name,
10039 old_dentry->d_name.len);
10040 } else {
10041 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
10042 BTRFS_I(d_inode(old_dentry)),
10043 old_dentry->d_name.name,
10044 old_dentry->d_name.len);
10045 if (!ret)
10046 ret = btrfs_update_inode(trans, root, old_inode);
10047 }
10048 if (ret) {
10049 btrfs_abort_transaction(trans, ret);
10050 goto out_fail;
10051 }
10052
10053 if (new_inode) {
10054 inode_inc_iversion(new_inode);
10055 new_inode->i_ctime = current_time(new_inode);
10056 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
10057 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
10058 root_objectid = BTRFS_I(new_inode)->location.objectid;
10059 ret = btrfs_unlink_subvol(trans, dest, new_dir,
10060 root_objectid,
10061 new_dentry->d_name.name,
10062 new_dentry->d_name.len);
10063 BUG_ON(new_inode->i_nlink == 0);
10064 } else {
10065 ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
10066 BTRFS_I(d_inode(new_dentry)),
10067 new_dentry->d_name.name,
10068 new_dentry->d_name.len);
10069 }
10070 if (!ret && new_inode->i_nlink == 0)
10071 ret = btrfs_orphan_add(trans,
10072 BTRFS_I(d_inode(new_dentry)));
10073 if (ret) {
10074 btrfs_abort_transaction(trans, ret);
10075 goto out_fail;
10076 }
10077 }
10078
10079 ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
10080 new_dentry->d_name.name,
10081 new_dentry->d_name.len, 0, index);
10082 if (ret) {
10083 btrfs_abort_transaction(trans, ret);
10084 goto out_fail;
10085 }
10086
10087 if (old_inode->i_nlink == 1)
10088 BTRFS_I(old_inode)->dir_index = index;
10089
10090 if (log_pinned) {
10091 struct dentry *parent = new_dentry->d_parent;
10092
10093 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
10094 parent);
10095 btrfs_end_log_trans(root);
10096 log_pinned = false;
10097 }
10098
10099 if (flags & RENAME_WHITEOUT) {
10100 ret = btrfs_whiteout_for_rename(trans, root, old_dir,
10101 old_dentry);
10102
10103 if (ret) {
10104 btrfs_abort_transaction(trans, ret);
10105 goto out_fail;
10106 }
10107 }
10108 out_fail:
10109 /*
10110 * If we have pinned the log and an error happened, we unpin tasks
10111 * trying to sync the log and force them to fallback to a transaction
10112 * commit if the log currently contains any of the inodes involved in
10113 * this rename operation (to ensure we do not persist a log with an
10114 * inconsistent state for any of these inodes or leading to any
10115 * inconsistencies when replayed). If the transaction was aborted, the
10116 * abortion reason is propagated to userspace when attempting to commit
10117 * the transaction. If the log does not contain any of these inodes, we
10118 * allow the tasks to sync it.
10119 */
10120 if (ret && log_pinned) {
10121 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
10122 btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
10123 btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
10124 (new_inode &&
10125 btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
10126 btrfs_set_log_full_commit(fs_info, trans);
10127
10128 btrfs_end_log_trans(root);
10129 log_pinned = false;
10130 }
10131 btrfs_end_transaction(trans);
10132 out_notrans:
10133 if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
10134 up_read(&fs_info->subvol_sem);
10135
10136 return ret;
10137 }
10138
10139 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
10140 struct inode *new_dir, struct dentry *new_dentry,
10141 unsigned int flags)
10142 {
10143 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
10144 return -EINVAL;
10145
10146 if (flags & RENAME_EXCHANGE)
10147 return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
10148 new_dentry);
10149
10150 return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
10151 }
10152
10153 static void btrfs_run_delalloc_work(struct btrfs_work *work)
10154 {
10155 struct btrfs_delalloc_work *delalloc_work;
10156 struct inode *inode;
10157
10158 delalloc_work = container_of(work, struct btrfs_delalloc_work,
10159 work);
10160 inode = delalloc_work->inode;
10161 filemap_flush(inode->i_mapping);
10162 if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
10163 &BTRFS_I(inode)->runtime_flags))
10164 filemap_flush(inode->i_mapping);
10165
10166 if (delalloc_work->delay_iput)
10167 btrfs_add_delayed_iput(inode);
10168 else
10169 iput(inode);
10170 complete(&delalloc_work->completion);
10171 }
10172
10173 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode,
10174 int delay_iput)
10175 {
10176 struct btrfs_delalloc_work *work;
10177
10178 work = kmalloc(sizeof(*work), GFP_NOFS);
10179 if (!work)
10180 return NULL;
10181
10182 init_completion(&work->completion);
10183 INIT_LIST_HEAD(&work->list);
10184 work->inode = inode;
10185 work->delay_iput = delay_iput;
10186 WARN_ON_ONCE(!inode);
10187 btrfs_init_work(&work->work, btrfs_flush_delalloc_helper,
10188 btrfs_run_delalloc_work, NULL, NULL);
10189
10190 return work;
10191 }
10192
10193 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work)
10194 {
10195 wait_for_completion(&work->completion);
10196 kfree(work);
10197 }
10198
10199 /*
10200 * some fairly slow code that needs optimization. This walks the list
10201 * of all the inodes with pending delalloc and forces them to disk.
10202 */
10203 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput,
10204 int nr)
10205 {
10206 struct btrfs_inode *binode;
10207 struct inode *inode;
10208 struct btrfs_delalloc_work *work, *next;
10209 struct list_head works;
10210 struct list_head splice;
10211 int ret = 0;
10212
10213 INIT_LIST_HEAD(&works);
10214 INIT_LIST_HEAD(&splice);
10215
10216 mutex_lock(&root->delalloc_mutex);
10217 spin_lock(&root->delalloc_lock);
10218 list_splice_init(&root->delalloc_inodes, &splice);
10219 while (!list_empty(&splice)) {
10220 binode = list_entry(splice.next, struct btrfs_inode,
10221 delalloc_inodes);
10222
10223 list_move_tail(&binode->delalloc_inodes,
10224 &root->delalloc_inodes);
10225 inode = igrab(&binode->vfs_inode);
10226 if (!inode) {
10227 cond_resched_lock(&root->delalloc_lock);
10228 continue;
10229 }
10230 spin_unlock(&root->delalloc_lock);
10231
10232 work = btrfs_alloc_delalloc_work(inode, delay_iput);
10233 if (!work) {
10234 if (delay_iput)
10235 btrfs_add_delayed_iput(inode);
10236 else
10237 iput(inode);
10238 ret = -ENOMEM;
10239 goto out;
10240 }
10241 list_add_tail(&work->list, &works);
10242 btrfs_queue_work(root->fs_info->flush_workers,
10243 &work->work);
10244 ret++;
10245 if (nr != -1 && ret >= nr)
10246 goto out;
10247 cond_resched();
10248 spin_lock(&root->delalloc_lock);
10249 }
10250 spin_unlock(&root->delalloc_lock);
10251
10252 out:
10253 list_for_each_entry_safe(work, next, &works, list) {
10254 list_del_init(&work->list);
10255 btrfs_wait_and_free_delalloc_work(work);
10256 }
10257
10258 if (!list_empty_careful(&splice)) {
10259 spin_lock(&root->delalloc_lock);
10260 list_splice_tail(&splice, &root->delalloc_inodes);
10261 spin_unlock(&root->delalloc_lock);
10262 }
10263 mutex_unlock(&root->delalloc_mutex);
10264 return ret;
10265 }
10266
10267 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput)
10268 {
10269 struct btrfs_fs_info *fs_info = root->fs_info;
10270 int ret;
10271
10272 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10273 return -EROFS;
10274
10275 ret = __start_delalloc_inodes(root, delay_iput, -1);
10276 if (ret > 0)
10277 ret = 0;
10278 return ret;
10279 }
10280
10281 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput,
10282 int nr)
10283 {
10284 struct btrfs_root *root;
10285 struct list_head splice;
10286 int ret;
10287
10288 if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10289 return -EROFS;
10290
10291 INIT_LIST_HEAD(&splice);
10292
10293 mutex_lock(&fs_info->delalloc_root_mutex);
10294 spin_lock(&fs_info->delalloc_root_lock);
10295 list_splice_init(&fs_info->delalloc_roots, &splice);
10296 while (!list_empty(&splice) && nr) {
10297 root = list_first_entry(&splice, struct btrfs_root,
10298 delalloc_root);
10299 root = btrfs_grab_fs_root(root);
10300 BUG_ON(!root);
10301 list_move_tail(&root->delalloc_root,
10302 &fs_info->delalloc_roots);
10303 spin_unlock(&fs_info->delalloc_root_lock);
10304
10305 ret = __start_delalloc_inodes(root, delay_iput, nr);
10306 btrfs_put_fs_root(root);
10307 if (ret < 0)
10308 goto out;
10309
10310 if (nr != -1) {
10311 nr -= ret;
10312 WARN_ON(nr < 0);
10313 }
10314 spin_lock(&fs_info->delalloc_root_lock);
10315 }
10316 spin_unlock(&fs_info->delalloc_root_lock);
10317
10318 ret = 0;
10319 out:
10320 if (!list_empty_careful(&splice)) {
10321 spin_lock(&fs_info->delalloc_root_lock);
10322 list_splice_tail(&splice, &fs_info->delalloc_roots);
10323 spin_unlock(&fs_info->delalloc_root_lock);
10324 }
10325 mutex_unlock(&fs_info->delalloc_root_mutex);
10326 return ret;
10327 }
10328
10329 static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
10330 const char *symname)
10331 {
10332 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10333 struct btrfs_trans_handle *trans;
10334 struct btrfs_root *root = BTRFS_I(dir)->root;
10335 struct btrfs_path *path;
10336 struct btrfs_key key;
10337 struct inode *inode = NULL;
10338 int err;
10339 int drop_inode = 0;
10340 u64 objectid;
10341 u64 index = 0;
10342 int name_len;
10343 int datasize;
10344 unsigned long ptr;
10345 struct btrfs_file_extent_item *ei;
10346 struct extent_buffer *leaf;
10347
10348 name_len = strlen(symname);
10349 if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
10350 return -ENAMETOOLONG;
10351
10352 /*
10353 * 2 items for inode item and ref
10354 * 2 items for dir items
10355 * 1 item for updating parent inode item
10356 * 1 item for the inline extent item
10357 * 1 item for xattr if selinux is on
10358 */
10359 trans = btrfs_start_transaction(root, 7);
10360 if (IS_ERR(trans))
10361 return PTR_ERR(trans);
10362
10363 err = btrfs_find_free_ino(root, &objectid);
10364 if (err)
10365 goto out_unlock;
10366
10367 inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
10368 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)),
10369 objectid, S_IFLNK|S_IRWXUGO, &index);
10370 if (IS_ERR(inode)) {
10371 err = PTR_ERR(inode);
10372 goto out_unlock;
10373 }
10374
10375 /*
10376 * If the active LSM wants to access the inode during
10377 * d_instantiate it needs these. Smack checks to see
10378 * if the filesystem supports xattrs by looking at the
10379 * ops vector.
10380 */
10381 inode->i_fop = &btrfs_file_operations;
10382 inode->i_op = &btrfs_file_inode_operations;
10383 inode->i_mapping->a_ops = &btrfs_aops;
10384 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10385
10386 err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
10387 if (err)
10388 goto out_unlock_inode;
10389
10390 path = btrfs_alloc_path();
10391 if (!path) {
10392 err = -ENOMEM;
10393 goto out_unlock_inode;
10394 }
10395 key.objectid = btrfs_ino(BTRFS_I(inode));
10396 key.offset = 0;
10397 key.type = BTRFS_EXTENT_DATA_KEY;
10398 datasize = btrfs_file_extent_calc_inline_size(name_len);
10399 err = btrfs_insert_empty_item(trans, root, path, &key,
10400 datasize);
10401 if (err) {
10402 btrfs_free_path(path);
10403 goto out_unlock_inode;
10404 }
10405 leaf = path->nodes[0];
10406 ei = btrfs_item_ptr(leaf, path->slots[0],
10407 struct btrfs_file_extent_item);
10408 btrfs_set_file_extent_generation(leaf, ei, trans->transid);
10409 btrfs_set_file_extent_type(leaf, ei,
10410 BTRFS_FILE_EXTENT_INLINE);
10411 btrfs_set_file_extent_encryption(leaf, ei, 0);
10412 btrfs_set_file_extent_compression(leaf, ei, 0);
10413 btrfs_set_file_extent_other_encoding(leaf, ei, 0);
10414 btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
10415
10416 ptr = btrfs_file_extent_inline_start(ei);
10417 write_extent_buffer(leaf, symname, ptr, name_len);
10418 btrfs_mark_buffer_dirty(leaf);
10419 btrfs_free_path(path);
10420
10421 inode->i_op = &btrfs_symlink_inode_operations;
10422 inode_nohighmem(inode);
10423 inode->i_mapping->a_ops = &btrfs_symlink_aops;
10424 inode_set_bytes(inode, name_len);
10425 btrfs_i_size_write(BTRFS_I(inode), name_len);
10426 err = btrfs_update_inode(trans, root, inode);
10427 /*
10428 * Last step, add directory indexes for our symlink inode. This is the
10429 * last step to avoid extra cleanup of these indexes if an error happens
10430 * elsewhere above.
10431 */
10432 if (!err)
10433 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
10434 BTRFS_I(inode), 0, index);
10435 if (err) {
10436 drop_inode = 1;
10437 goto out_unlock_inode;
10438 }
10439
10440 unlock_new_inode(inode);
10441 d_instantiate(dentry, inode);
10442
10443 out_unlock:
10444 btrfs_end_transaction(trans);
10445 if (drop_inode) {
10446 inode_dec_link_count(inode);
10447 iput(inode);
10448 }
10449 btrfs_btree_balance_dirty(fs_info);
10450 return err;
10451
10452 out_unlock_inode:
10453 drop_inode = 1;
10454 unlock_new_inode(inode);
10455 goto out_unlock;
10456 }
10457
10458 static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
10459 u64 start, u64 num_bytes, u64 min_size,
10460 loff_t actual_len, u64 *alloc_hint,
10461 struct btrfs_trans_handle *trans)
10462 {
10463 struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
10464 struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
10465 struct extent_map *em;
10466 struct btrfs_root *root = BTRFS_I(inode)->root;
10467 struct btrfs_key ins;
10468 u64 cur_offset = start;
10469 u64 i_size;
10470 u64 cur_bytes;
10471 u64 last_alloc = (u64)-1;
10472 int ret = 0;
10473 bool own_trans = true;
10474 u64 end = start + num_bytes - 1;
10475
10476 if (trans)
10477 own_trans = false;
10478 while (num_bytes > 0) {
10479 if (own_trans) {
10480 trans = btrfs_start_transaction(root, 3);
10481 if (IS_ERR(trans)) {
10482 ret = PTR_ERR(trans);
10483 break;
10484 }
10485 }
10486
10487 cur_bytes = min_t(u64, num_bytes, SZ_256M);
10488 cur_bytes = max(cur_bytes, min_size);
10489 /*
10490 * If we are severely fragmented we could end up with really
10491 * small allocations, so if the allocator is returning small
10492 * chunks lets make its job easier by only searching for those
10493 * sized chunks.
10494 */
10495 cur_bytes = min(cur_bytes, last_alloc);
10496 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
10497 min_size, 0, *alloc_hint, &ins, 1, 0);
10498 if (ret) {
10499 if (own_trans)
10500 btrfs_end_transaction(trans);
10501 break;
10502 }
10503 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
10504
10505 last_alloc = ins.offset;
10506 ret = insert_reserved_file_extent(trans, inode,
10507 cur_offset, ins.objectid,
10508 ins.offset, ins.offset,
10509 ins.offset, 0, 0, 0,
10510 BTRFS_FILE_EXTENT_PREALLOC);
10511 if (ret) {
10512 btrfs_free_reserved_extent(fs_info, ins.objectid,
10513 ins.offset, 0);
10514 btrfs_abort_transaction(trans, ret);
10515 if (own_trans)
10516 btrfs_end_transaction(trans);
10517 break;
10518 }
10519
10520 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10521 cur_offset + ins.offset -1, 0);
10522
10523 em = alloc_extent_map();
10524 if (!em) {
10525 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
10526 &BTRFS_I(inode)->runtime_flags);
10527 goto next;
10528 }
10529
10530 em->start = cur_offset;
10531 em->orig_start = cur_offset;
10532 em->len = ins.offset;
10533 em->block_start = ins.objectid;
10534 em->block_len = ins.offset;
10535 em->orig_block_len = ins.offset;
10536 em->ram_bytes = ins.offset;
10537 em->bdev = fs_info->fs_devices->latest_bdev;
10538 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
10539 em->generation = trans->transid;
10540
10541 while (1) {
10542 write_lock(&em_tree->lock);
10543 ret = add_extent_mapping(em_tree, em, 1);
10544 write_unlock(&em_tree->lock);
10545 if (ret != -EEXIST)
10546 break;
10547 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10548 cur_offset + ins.offset - 1,
10549 0);
10550 }
10551 free_extent_map(em);
10552 next:
10553 num_bytes -= ins.offset;
10554 cur_offset += ins.offset;
10555 *alloc_hint = ins.objectid + ins.offset;
10556
10557 inode_inc_iversion(inode);
10558 inode->i_ctime = current_time(inode);
10559 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
10560 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
10561 (actual_len > inode->i_size) &&
10562 (cur_offset > inode->i_size)) {
10563 if (cur_offset > actual_len)
10564 i_size = actual_len;
10565 else
10566 i_size = cur_offset;
10567 i_size_write(inode, i_size);
10568 btrfs_ordered_update_i_size(inode, i_size, NULL);
10569 }
10570
10571 ret = btrfs_update_inode(trans, root, inode);
10572
10573 if (ret) {
10574 btrfs_abort_transaction(trans, ret);
10575 if (own_trans)
10576 btrfs_end_transaction(trans);
10577 break;
10578 }
10579
10580 if (own_trans)
10581 btrfs_end_transaction(trans);
10582 }
10583 if (cur_offset < end)
10584 btrfs_free_reserved_data_space(inode, NULL, cur_offset,
10585 end - cur_offset + 1);
10586 return ret;
10587 }
10588
10589 int btrfs_prealloc_file_range(struct inode *inode, int mode,
10590 u64 start, u64 num_bytes, u64 min_size,
10591 loff_t actual_len, u64 *alloc_hint)
10592 {
10593 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10594 min_size, actual_len, alloc_hint,
10595 NULL);
10596 }
10597
10598 int btrfs_prealloc_file_range_trans(struct inode *inode,
10599 struct btrfs_trans_handle *trans, int mode,
10600 u64 start, u64 num_bytes, u64 min_size,
10601 loff_t actual_len, u64 *alloc_hint)
10602 {
10603 return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10604 min_size, actual_len, alloc_hint, trans);
10605 }
10606
10607 static int btrfs_set_page_dirty(struct page *page)
10608 {
10609 return __set_page_dirty_nobuffers(page);
10610 }
10611
10612 static int btrfs_permission(struct inode *inode, int mask)
10613 {
10614 struct btrfs_root *root = BTRFS_I(inode)->root;
10615 umode_t mode = inode->i_mode;
10616
10617 if (mask & MAY_WRITE &&
10618 (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
10619 if (btrfs_root_readonly(root))
10620 return -EROFS;
10621 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
10622 return -EACCES;
10623 }
10624 return generic_permission(inode, mask);
10625 }
10626
10627 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
10628 {
10629 struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10630 struct btrfs_trans_handle *trans;
10631 struct btrfs_root *root = BTRFS_I(dir)->root;
10632 struct inode *inode = NULL;
10633 u64 objectid;
10634 u64 index;
10635 int ret = 0;
10636
10637 /*
10638 * 5 units required for adding orphan entry
10639 */
10640 trans = btrfs_start_transaction(root, 5);
10641 if (IS_ERR(trans))
10642 return PTR_ERR(trans);
10643
10644 ret = btrfs_find_free_ino(root, &objectid);
10645 if (ret)
10646 goto out;
10647
10648 inode = btrfs_new_inode(trans, root, dir, NULL, 0,
10649 btrfs_ino(BTRFS_I(dir)), objectid, mode, &index);
10650 if (IS_ERR(inode)) {
10651 ret = PTR_ERR(inode);
10652 inode = NULL;
10653 goto out;
10654 }
10655
10656 inode->i_fop = &btrfs_file_operations;
10657 inode->i_op = &btrfs_file_inode_operations;
10658
10659 inode->i_mapping->a_ops = &btrfs_aops;
10660 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10661
10662 ret = btrfs_init_inode_security(trans, inode, dir, NULL);
10663 if (ret)
10664 goto out_inode;
10665
10666 ret = btrfs_update_inode(trans, root, inode);
10667 if (ret)
10668 goto out_inode;
10669 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
10670 if (ret)
10671 goto out_inode;
10672
10673 /*
10674 * We set number of links to 0 in btrfs_new_inode(), and here we set
10675 * it to 1 because d_tmpfile() will issue a warning if the count is 0,
10676 * through:
10677 *
10678 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
10679 */
10680 set_nlink(inode, 1);
10681 unlock_new_inode(inode);
10682 d_tmpfile(dentry, inode);
10683 mark_inode_dirty(inode);
10684
10685 out:
10686 btrfs_end_transaction(trans);
10687 if (ret)
10688 iput(inode);
10689 btrfs_balance_delayed_items(fs_info);
10690 btrfs_btree_balance_dirty(fs_info);
10691 return ret;
10692
10693 out_inode:
10694 unlock_new_inode(inode);
10695 goto out;
10696
10697 }
10698
10699 __attribute__((const))
10700 static int btrfs_readpage_io_failed_hook(struct page *page, int failed_mirror)
10701 {
10702 return -EAGAIN;
10703 }
10704
10705 static struct btrfs_fs_info *iotree_fs_info(void *private_data)
10706 {
10707 struct inode *inode = private_data;
10708 return btrfs_sb(inode->i_sb);
10709 }
10710
10711 static void btrfs_check_extent_io_range(void *private_data, const char *caller,
10712 u64 start, u64 end)
10713 {
10714 struct inode *inode = private_data;
10715 u64 isize;
10716
10717 isize = i_size_read(inode);
10718 if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
10719 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
10720 "%s: ino %llu isize %llu odd range [%llu,%llu]",
10721 caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
10722 }
10723 }
10724
10725 void btrfs_set_range_writeback(void *private_data, u64 start, u64 end)
10726 {
10727 struct inode *inode = private_data;
10728 unsigned long index = start >> PAGE_SHIFT;
10729 unsigned long end_index = end >> PAGE_SHIFT;
10730 struct page *page;
10731
10732 while (index <= end_index) {
10733 page = find_get_page(inode->i_mapping, index);
10734 ASSERT(page); /* Pages should be in the extent_io_tree */
10735 set_page_writeback(page);
10736 put_page(page);
10737 index++;
10738 }
10739 }
10740
10741 static const struct inode_operations btrfs_dir_inode_operations = {
10742 .getattr = btrfs_getattr,
10743 .lookup = btrfs_lookup,
10744 .create = btrfs_create,
10745 .unlink = btrfs_unlink,
10746 .link = btrfs_link,
10747 .mkdir = btrfs_mkdir,
10748 .rmdir = btrfs_rmdir,
10749 .rename = btrfs_rename2,
10750 .symlink = btrfs_symlink,
10751 .setattr = btrfs_setattr,
10752 .mknod = btrfs_mknod,
10753 .listxattr = btrfs_listxattr,
10754 .permission = btrfs_permission,
10755 .get_acl = btrfs_get_acl,
10756 .set_acl = btrfs_set_acl,
10757 .update_time = btrfs_update_time,
10758 .tmpfile = btrfs_tmpfile,
10759 };
10760 static const struct inode_operations btrfs_dir_ro_inode_operations = {
10761 .lookup = btrfs_lookup,
10762 .permission = btrfs_permission,
10763 .update_time = btrfs_update_time,
10764 };
10765
10766 static const struct file_operations btrfs_dir_file_operations = {
10767 .llseek = generic_file_llseek,
10768 .read = generic_read_dir,
10769 .iterate_shared = btrfs_real_readdir,
10770 .open = btrfs_opendir,
10771 .unlocked_ioctl = btrfs_ioctl,
10772 #ifdef CONFIG_COMPAT
10773 .compat_ioctl = btrfs_compat_ioctl,
10774 #endif
10775 .release = btrfs_release_file,
10776 .fsync = btrfs_sync_file,
10777 };
10778
10779 static const struct extent_io_ops btrfs_extent_io_ops = {
10780 /* mandatory callbacks */
10781 .submit_bio_hook = btrfs_submit_bio_hook,
10782 .readpage_end_io_hook = btrfs_readpage_end_io_hook,
10783 .merge_bio_hook = btrfs_merge_bio_hook,
10784 .readpage_io_failed_hook = btrfs_readpage_io_failed_hook,
10785 .tree_fs_info = iotree_fs_info,
10786 .set_range_writeback = btrfs_set_range_writeback,
10787
10788 /* optional callbacks */
10789 .fill_delalloc = run_delalloc_range,
10790 .writepage_end_io_hook = btrfs_writepage_end_io_hook,
10791 .writepage_start_hook = btrfs_writepage_start_hook,
10792 .set_bit_hook = btrfs_set_bit_hook,
10793 .clear_bit_hook = btrfs_clear_bit_hook,
10794 .merge_extent_hook = btrfs_merge_extent_hook,
10795 .split_extent_hook = btrfs_split_extent_hook,
10796 .check_extent_io_range = btrfs_check_extent_io_range,
10797 };
10798
10799 /*
10800 * btrfs doesn't support the bmap operation because swapfiles
10801 * use bmap to make a mapping of extents in the file. They assume
10802 * these extents won't change over the life of the file and they
10803 * use the bmap result to do IO directly to the drive.
10804 *
10805 * the btrfs bmap call would return logical addresses that aren't
10806 * suitable for IO and they also will change frequently as COW
10807 * operations happen. So, swapfile + btrfs == corruption.
10808 *
10809 * For now we're avoiding this by dropping bmap.
10810 */
10811 static const struct address_space_operations btrfs_aops = {
10812 .readpage = btrfs_readpage,
10813 .writepage = btrfs_writepage,
10814 .writepages = btrfs_writepages,
10815 .readpages = btrfs_readpages,
10816 .direct_IO = btrfs_direct_IO,
10817 .invalidatepage = btrfs_invalidatepage,
10818 .releasepage = btrfs_releasepage,
10819 .set_page_dirty = btrfs_set_page_dirty,
10820 .error_remove_page = generic_error_remove_page,
10821 };
10822
10823 static const struct address_space_operations btrfs_symlink_aops = {
10824 .readpage = btrfs_readpage,
10825 .writepage = btrfs_writepage,
10826 .invalidatepage = btrfs_invalidatepage,
10827 .releasepage = btrfs_releasepage,
10828 };
10829
10830 static const struct inode_operations btrfs_file_inode_operations = {
10831 .getattr = btrfs_getattr,
10832 .setattr = btrfs_setattr,
10833 .listxattr = btrfs_listxattr,
10834 .permission = btrfs_permission,
10835 .fiemap = btrfs_fiemap,
10836 .get_acl = btrfs_get_acl,
10837 .set_acl = btrfs_set_acl,
10838 .update_time = btrfs_update_time,
10839 };
10840 static const struct inode_operations btrfs_special_inode_operations = {
10841 .getattr = btrfs_getattr,
10842 .setattr = btrfs_setattr,
10843 .permission = btrfs_permission,
10844 .listxattr = btrfs_listxattr,
10845 .get_acl = btrfs_get_acl,
10846 .set_acl = btrfs_set_acl,
10847 .update_time = btrfs_update_time,
10848 };
10849 static const struct inode_operations btrfs_symlink_inode_operations = {
10850 .get_link = page_get_link,
10851 .getattr = btrfs_getattr,
10852 .setattr = btrfs_setattr,
10853 .permission = btrfs_permission,
10854 .listxattr = btrfs_listxattr,
10855 .update_time = btrfs_update_time,
10856 };
10857
10858 const struct dentry_operations btrfs_dentry_operations = {
10859 .d_delete = btrfs_dentry_delete,
10860 .d_release = btrfs_dentry_release,
10861 };
Linux with added FPC-III support