| blob | 30982bbd31c30c2b154836b0f51b94c37e22923c |
1 /*
2 * Copyright (C) 2007 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
19 #include <linux/fs.h>
20 #include <linux/pagemap.h>
21 #include <linux/highmem.h>
22 #include <linux/time.h>
23 #include <linux/init.h>
24 #include <linux/string.h>
25 #include <linux/backing-dev.h>
26 #include <linux/mpage.h>
27 #include <linux/aio.h>
28 #include <linux/falloc.h>
29 #include <linux/swap.h>
30 #include <linux/writeback.h>
31 #include <linux/statfs.h>
32 #include <linux/compat.h>
33 #include <linux/slab.h>
34 #include <linux/btrfs.h>
46 /*
47 * when auto defrag is enabled we
48 * queue up these defrag structs to remember which
49 * inodes need defragging passes
50 */
53 /* objectid */
54 u64 ino;
55 /*
56 * transid where the defrag was added, we search for
57 * extents newer than this
58 */
59 u64 transid;
61 /* root objectid */
62 u64 root;
64 /* last offset we were able to defrag */
65 u64 last_offset;
67 /* if we've wrapped around back to zero once already */
69 };
73 {
82 else
84 }
86 /* pop a record for an inode into the defrag tree. The lock
87 * must be held already
88 *
89 * If you're inserting a record for an older transid than an
90 * existing record, the transid already in the tree is lowered
91 *
92 * If an existing record is found the defrag item you
93 * pass in is freed
94 */
97 {
115 /* if we're reinserting an entry for
116 * an old defrag run, make sure to
117 * lower the transid of our existing record
118 */
124 }
125 }
130 }
133 {
141 }
143 /*
144 * insert a defrag record for this inode if auto defrag is
145 * enabled
146 */
149 {
152 u64 transid;
163 else
176 /*
177 * If we set IN_DEFRAG flag and evict the inode from memory,
178 * and then re-read this inode, this new inode doesn't have
179 * IN_DEFRAG flag. At the case, we may find the existed defrag.
180 */
186 }
189 }
191 /*
192 * Requeue the defrag object. If there is a defrag object that points to
193 * the same inode in the tree, we will merge them together (by
194 * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
195 */
198 {
205 /*
206 * Here we don't check the IN_DEFRAG flag, because we need merge
207 * them together.
208 */
215 out:
217 }
219 /*
220 * pick the defragable inode that we want, if it doesn't exist, we will get
221 * the next one.
222 */
225 {
246 else
248 }
254 else
256 }
257 out:
262 }
265 {
280 }
283 }
285 }
287 #define BTRFS_DEFRAG_BATCH 1024
291 {
300 /* get the inode */
311 }
320 }
323 /* do a chunk of defrag */
331 BTRFS_DEFRAG_BATCH);
333 /*
334 * if we filled the whole defrag batch, there
335 * must be more work to do. Queue this defrag
336 * again
337 */
342 /*
343 * we didn't fill our defrag batch, but
344 * we didn't start at zero. Make sure we loop
345 * around to the start of the file.
346 */
352 }
356 cleanup:
360 }
362 /*
363 * run through the list of inodes in the FS that need
364 * defragging
365 */
367 {
374 /* Pause the auto defragger. */
382 /* find an inode to defrag */
384 first_ino);
392 }
393 }
399 }
402 /*
403 * during unmount, we use the transaction_wait queue to
404 * wait for the defragger to stop
405 */
408 }
410 /* simple helper to fault in pages and copy. This should go away
411 * and be replaced with calls into generic code.
412 */
417 {
427 /*
428 * Copy data from userspace to the current page
429 */
432 /* Flush processor's dcache for this page */
435 /*
436 * if we get a partial write, we can end up with
437 * partially up to date pages. These add
438 * a lot of complexity, so make sure they don't
439 * happen by forcing this copy to be retried.
440 *
441 * The rest of the btrfs_file_write code will fall
442 * back to page at a time copies after we return 0.
443 */
451 /* Return to btrfs_file_write_iter to fault page */
458 pg++;
460 }
461 }
463 }
465 /*
466 * unlocks pages after btrfs_file_write is done with them
467 */
469 {
472 /* page checked is some magic around finding pages that
473 * have been modified without going through btrfs_set_page_dirty
474 * clear it here. There should be no need to mark the pages
475 * accessed as prepare_pages should have marked them accessed
476 * in prepare_pages via find_or_create_page()
477 */
481 }
482 }
484 /*
485 * after copy_from_user, pages need to be dirtied and we need to make
486 * sure holes are created between the current EOF and the start of
487 * any next extents (if required).
488 *
489 * this also makes the decision about creating an inline extent vs
490 * doing real data extents, marking pages dirty and delalloc as required.
491 */
496 {
499 u64 num_bytes;
500 u64 start_pos;
501 u64 end_of_last_block;
510 cached);
519 }
521 /*
522 * we've only changed i_size in ram, and we haven't updated
523 * the disk i_size. There is no need to log the inode
524 * at this time.
525 */
529 }
531 /*
532 * this drops all the extents in the cache that intersect the range
533 * [start, end]. Existing extents are split as required.
534 */
537 {
543 u64 gen;
554 }
571 }
579 }
586 }
604 else
615 }
625 }
650 }
657 }
661 modified);
664 modified);
666 }
669 }
670 next:
675 /* once for us */
677 /* once for the tree*/
679 }
684 }
686 /*
687 * this is very complex, but the basic idea is to drop all extents
688 * in the range start - end. hint_block is filled in with a block number
689 * that would be a good hint to the block allocator for this file.
690 *
691 * If an extent intersects the range but is not entirely inside the range
692 * it is either truncated or split. Anything entirely inside the range
693 * is deleted from the tree.
694 */
700 u32 extent_item_size,
702 {
743 }
745 leafs_visited++;
746 next_slot:
756 }
757 leafs_visited++;
760 }
785 }
787 /*
788 * Don't skip extent items representing 0 byte lengths. They
789 * used to be created (bug) if while punching holes we hit
790 * -ENOSPC condition. So if we find one here, just ensure we
791 * delete it, otherwise we would insert a new file extent item
792 * with the same key (offset) as that 0 bytes length file
793 * extent item in the call to setup_items_for_insert() later
794 * in this function.
795 */
802 }
810 }
812 /*
813 * | - range to drop - |
814 * | -------- extent -------- |
815 */
821 }
830 }
856 }
858 }
859 /*
860 * | ---- range to drop ----- |
861 * | -------- extent -------- |
862 */
867 }
881 }
884 /*
885 * | ---- range to drop ----- |
886 * | -------- extent -------- |
887 */
893 }
905 }
907 /*
908 * | ---- range to drop ----- |
909 * | ------ extent ------ |
910 */
912 delete_extent_item:
918 del_nr++;
919 }
936 }
944 }
947 del_nr);
951 }
958 }
961 }
964 /*
965 * Set path->slots[0] to first slot, so that after the delete
966 * if items are move off from our leaf to its immediate left or
967 * right neighbor leafs, we end up with a correct and adjusted
968 * path->slots[0] for our insertion (if replace_extent != 0).
969 */
974 }
977 /*
978 * If btrfs_del_items() was called, it might have deleted a leaf, in
979 * which case it unlocked our path, so check path->locks[0] matches a
980 * write lock.
981 */
997 }
1000 extent_item_size,
1004 }
1011 }
1016 {
1027 }
1032 {
1035 u64 extent_end;
1060 }
1062 /*
1063 * Mark extent in the range start - end as written.
1064 *
1065 * This changes extent type from 'pre-allocated' to 'regular'. If only
1066 * part of extent is marked as written, the extent will be split into
1067 * two or three.
1068 */
1071 {
1078 u64 bytenr;
1079 u64 num_bytes;
1080 u64 extent_end;
1081 u64 orig_offset;
1082 u64 other_start;
1083 u64 other_end;
1084 u64 split;
1094 again:
1113 BTRFS_FILE_EXTENT_PREALLOC);
1146 }
1147 }
1175 }
1176 }
1187 }
1191 }
1220 }
1222 }
1232 }
1235 del_nr++;
1240 }
1249 }
1252 del_nr++;
1257 }
1262 BTRFS_FILE_EXTENT_REG);
1269 BTRFS_FILE_EXTENT_REG);
1279 }
1280 }
1281 out:
1284 }
1286 /*
1287 * on error we return an unlocked page and the error value
1288 * on success we return a locked page and 0
1289 */
1292 {
1304 }
1305 }
1307 }
1309 /*
1310 * this just gets pages into the page cache and locks them down.
1311 */
1315 {
1329 }
1333 force_uptodate);
1341 }
1343 }
1346 fail:
1350 faili--;
1351 }
1354 }
1356 /*
1357 * This function locks the extent and properly waits for data=ordered extents
1358 * to finish before allowing the pages to be modified if need.
1359 *
1360 * The return value:
1361 * 1 - the extent is locked
1362 * 0 - the extent is not locked, and everything is OK
1363 * -EAGAIN - need re-prepare the pages
1364 * the other < 0 number - Something wrong happens
1365 */
1371 {
1372 u64 start_pos;
1373 u64 last_pos;
1395 }
1399 }
1410 }
1417 }
1420 }
1424 {
1428 u64 num_bytes;
1444 }
1448 }
1458 }
1463 }
1467 loff_t pos)
1468 {
1474 u64 lockstart;
1475 u64 lockend;
1498 offset);
1500 PAGE_CACHE_SIZE);
1507 /*
1508 * Fault pages before locking them in prepare_pages
1509 * to avoid recursive lock
1510 */
1514 }
1520 BTRFS_INODE_PREALLOC))) {
1524 /*
1525 * our prealloc extent may be smaller than
1526 * write_bytes, so scale down.
1527 */
1529 PAGE_CACHE_SIZE);
1534 }
1535 }
1544 reserve_bytes);
1545 else
1548 }
1552 again:
1553 /*
1554 * This is going to setup the pages array with the number of
1555 * pages we want, so we don't really need to worry about the
1556 * contents of pages from loop to loop
1557 */
1560 force_page_uptodate);
1574 }
1579 /*
1580 * if we have trouble faulting in the pages, fall
1581 * back to one page at a time
1582 */
1592 PAGE_CACHE_SIZE);
1593 }
1595 /*
1596 * If we had a short copy we need to release the excess delaloc
1597 * bytes we reserved. We need to increment outstanding_extents
1598 * because btrfs_delalloc_release_space will decrement it, but
1599 * we still have an outstanding extent for the chunk we actually
1600 * managed to copy.
1601 */
1604 PAGE_CACHE_SHIFT;
1609 }
1612 release_bytes);
1613 else
1615 release_bytes);
1616 }
1623 NULL);
1627 GFP_NOFS);
1631 }
1646 }
1658 }
1668 }
1669 }
1672 }
1676 loff_t pos)
1677 {
1680 ssize_t written;
1681 ssize_t written_buffered;
1682 loff_t endbyte;
1695 }
1696 /*
1697 * Ensure all data is persisted. We want the next direct IO read to be
1698 * able to read what was just written.
1699 */
1711 out:
1713 }
1716 {
1731 }
1735 {
1739 u64 start_pos;
1740 u64 end_pos;
1754 }
1759 }
1767 }
1769 /*
1770 * If BTRFS flips readonly due to some impossible error
1771 * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
1772 * although we have opened a file as writable, we have
1773 * to stop this write operation to ensure FS consistency.
1774 */
1779 }
1781 /*
1782 * We reserve space for updating the inode when we reserve space for the
1783 * extent we are going to write, so we will enospc out there. We don't
1784 * need to start yet another transaction to update the inode as we will
1785 * update the inode when we finish writing whatever data we write.
1786 */
1791 /* Expand hole size to cover write data, preventing empty gap */
1797 }
1798 }
1809 }
1813 /*
1814 * We also have to set last_sub_trans to the current log transid,
1815 * otherwise subsequent syncs to a file that's been synced in this
1816 * transaction will appear to have already occured.
1817 */
1823 }
1827 out:
1830 }
1833 {
1836 /*
1837 * ordered_data_close is set by settattr when we are about to truncate
1838 * a file from a non-zero size to a zero size. This tries to
1839 * flush down new bytes that may have been written if the
1840 * application were using truncate to replace a file in place.
1841 */
1846 }
1849 {
1857 }
1859 /*
1860 * fsync call for both files and directories. This logs the inode into
1861 * the tree log instead of forcing full commits whenever possible.
1862 *
1863 * It needs to call filemap_fdatawait so that all ordered extent updates are
1864 * in the metadata btree are up to date for copying to the log.
1865 *
1866 * It drops the inode mutex before doing the tree log commit. This is an
1867 * important optimization for directories because holding the mutex prevents
1868 * new operations on the dir while we write to disk.
1869 */
1871 {
1882 /*
1883 * We write the dirty pages in the range and wait until they complete
1884 * out of the ->i_mutex. If so, we can flush the dirty pages by
1885 * multi-task, and make the performance up. See
1886 * btrfs_wait_ordered_range for an explanation of the ASYNC check.
1887 */
1896 /*
1897 * We might have have had more pages made dirty after calling
1898 * start_ordered_ops and before acquiring the inode's i_mutex.
1899 */
1901 /*
1902 * For a full sync, we need to make sure any ordered operations
1903 * start and finish before we start logging the inode, so that
1904 * all extents are persisted and the respective file extent
1905 * items are in the fs/subvol btree.
1906 */
1909 /*
1910 * Start any new ordered operations before starting to log the
1911 * inode. We will wait for them to finish in btrfs_sync_log().
1912 *
1913 * Right before acquiring the inode's mutex, we might have new
1914 * writes dirtying pages, which won't immediately start the
1915 * respective ordered operations - that is done through the
1916 * fill_delalloc callbacks invoked from the writepage and
1917 * writepages address space operations. So make sure we start
1918 * all ordered operations before starting to log our inode. Not
1919 * doing this means that while logging the inode, writeback
1920 * could start and invoke writepage/writepages, which would call
1921 * the fill_delalloc callbacks (cow_file_range,
1922 * submit_compressed_extents). These callbacks add first an
1923 * extent map to the modified list of extents and then create
1924 * the respective ordered operation, which means in
1925 * tree-log.c:btrfs_log_inode() we might capture all existing
1926 * ordered operations (with btrfs_get_logged_extents()) before
1927 * the fill_delalloc callback adds its ordered operation, and by
1928 * the time we visit the modified list of extent maps (with
1929 * btrfs_log_changed_extents()), we see and process the extent
1930 * map they created. We then use the extent map to construct a
1931 * file extent item for logging without waiting for the
1932 * respective ordered operation to finish - this file extent
1933 * item points to a disk location that might not have yet been
1934 * written to, containing random data - so after a crash a log
1935 * replay will make our inode have file extent items that point
1936 * to disk locations containing invalid data, as we returned
1937 * success to userspace without waiting for the respective
1938 * ordered operation to finish, because it wasn't captured by
1939 * btrfs_get_logged_extents().
1940 */
1942 }
1946 }
1949 /*
1950 * If the last transaction that changed this file was before the current
1951 * transaction and we have the full sync flag set in our inode, we can
1952 * bail out now without any syncing.
1953 *
1954 * Note that we can't bail out if the full sync flag isn't set. This is
1955 * because when the full sync flag is set we start all ordered extents
1956 * and wait for them to fully complete - when they complete they update
1957 * the inode's last_trans field through:
1958 *
1959 * btrfs_finish_ordered_io() ->
1960 * btrfs_update_inode_fallback() ->
1961 * btrfs_update_inode() ->
1962 * btrfs_set_inode_last_trans()
1963 *
1964 * So we are sure that last_trans is up to date and can do this check to
1965 * bail out safely. For the fast path, when the full sync flag is not
1966 * set in our inode, we can not do it because we start only our ordered
1967 * extents and don't wait for them to complete (that is when
1968 * btrfs_finish_ordered_io runs), so here at this point their last_trans
1969 * value might be less than or equals to fs_info->last_trans_committed,
1970 * and setting a speculative last_trans for an inode when a buffered
1971 * write is made (such as fs_info->generation + 1 for example) would not
1972 * be reliable since after setting the value and before fsync is called
1973 * any number of transactions can start and commit (transaction kthread
1974 * commits the current transaction periodically), and a transaction
1975 * commit does not start nor waits for ordered extents to complete.
1976 */
1981 /*
1982 * We'v had everything committed since the last time we were
1983 * modified so clear this flag in case it was set for whatever
1984 * reason, it's no longer relevant.
1985 */
1990 }
1992 /*
1993 * ok we haven't committed the transaction yet, lets do a commit
1994 */
1998 /*
1999 * We use start here because we will need to wait on the IO to complete
2000 * in btrfs_sync_log, which could require joining a transaction (for
2001 * example checking cross references in the nocow path). If we use join
2002 * here we could get into a situation where we're waiting on IO to
2003 * happen that is blocked on a transaction trying to commit. With start
2004 * we inc the extwriter counter, so we wait for all extwriters to exit
2005 * before we start blocking join'ers. This comment is to keep somebody
2006 * from thinking they are super smart and changing this to
2007 * btrfs_join_transaction *cough*Josef*cough*.
2008 */
2014 }
2021 /* Fallthrough and commit/free transaction. */
2023 }
2025 /* we've logged all the items and now have a consistent
2026 * version of the file in the log. It is possible that
2027 * someone will come in and modify the file, but that's
2028 * fine because the log is consistent on disk, and we
2029 * have references to all of the file's extents
2030 *
2031 * It is possible that someone will come in and log the
2032 * file again, but that will end up using the synchronization
2033 * inside btrfs_sync_log to keep things safe.
2034 */
2037 /*
2038 * If any of the ordered extents had an error, just return it to user
2039 * space, so that the application knows some writes didn't succeed and
2040 * can take proper action (retry for e.g.). Blindly committing the
2041 * transaction in this case, would fool userspace that everything was
2042 * successful. And we also want to make sure our log doesn't contain
2043 * file extent items pointing to extents that weren't fully written to -
2044 * just like in the non fast fsync path, where we check for the ordered
2045 * operation's error flag before writing to the log tree and return -EIO
2046 * if any of them had this flag set (btrfs_wait_ordered_range) -
2047 * therefore we need to check for errors in the ordered operations,
2048 * which are indicated by ctx.io_err.
2049 */
2054 }
2062 }
2063 }
2070 }
2071 }
2075 }
2076 out:
2078 }
2084 };
2087 {
2097 }
2101 {
2126 }
2130 {
2153 u64 num_bytes;
2165 }
2168 u64 num_bytes;
2175 offset;
2181 }
2190 out:
2221 }
2224 }
2226 /*
2227 * Find a hole extent on given inode and change start/len to the end of hole
2228 * extent.(hole/vacuum extent whose em->start <= start &&
2229 * em->start + em->len > start)
2230 * When a hole extent is found, return 1 and modify start/len.
2231 */
2233 {
2241 else
2244 }
2246 /* Hole or vacuum extent(only exists in no-hole mode) */
2252 }
2255 }
2258 {
2264 u64 lockstart;
2265 u64 lockend;
2266 u64 tail_start;
2267 u64 tail_len;
2269 u64 cur_offset;
2271 u64 drop_end;
2277 u64 ino_size;
2291 /* Already in a large hole */
2294 }
2302 /*
2303 * We needn't truncate any page which is beyond the end of the file
2304 * because we are sure there is no data there.
2305 */
2306 /*
2307 * Only do this if we are in the same page and we aren't doing the
2308 * entire page.
2309 */
2316 }
2318 }
2320 /* zero back part of the first page */
2327 }
2328 }
2330 /* Check the aligned pages after the first unaligned page,
2331 * if offset != orig_start, which means the first unaligned page
2332 * including serveral following pages are already in holes,
2333 * the extra check can be skipped */
2335 /* after truncate page, check hole again */
2344 }
2346 }
2348 /* Check the tail unaligned part is in a hole */
2356 /* zero the front end of the last page */
2363 }
2364 }
2365 }
2370 }
2381 /*
2382 * We need to make sure we have no ordered extents in this range
2383 * and nobody raced in and read a page in this range, if we did
2384 * we need to try again.
2385 */
2393 }
2403 }
2404 }
2410 }
2416 }
2420 /*
2421 * 1 - update the inode
2422 * 1 - removing the extents in the range
2423 * 1 - adding the hole extent if no_holes isn't set
2424 */
2430 }
2433 min_size);
2450 drop_end);
2454 }
2455 }
2463 }
2473 }
2486 }
2487 }
2492 }
2495 /*
2496 * Don't insert file hole extent item if it's for a range beyond eof
2497 * (because it's useless) or if it represents a 0 bytes range (when
2498 * cur_offset == drop_end).
2499 */
2505 }
2506 }
2508 out_trans:
2520 out_free:
2523 out:
2526 out_only_mutex:
2528 /*
2529 * If we only end up zeroing part of a page, we still need to
2530 * update the inode item, so that all the time fields are
2531 * updated as well as the necessary btrfs inode in memory fields
2532 * for detecting, at fsync time, if the inode isn't yet in the
2533 * log tree or it's there but not up to date.
2534 */
2541 }
2542 }
2547 }
2551 {
2555 u64 cur_offset;
2556 u64 last_byte;
2557 u64 alloc_start;
2558 u64 alloc_end;
2560 u64 locked_end;
2568 /* Make sure we aren't being give some crap mode */
2575 /*
2576 * Make sure we have enough space before we do the
2577 * allocation.
2578 */
2586 }
2595 alloc_start);
2599 /*
2600 * If we are fallocating from the end of the file onward we
2601 * need to zero out the end of the page if i_size lands in the
2602 * middle of a page.
2603 */
2607 }
2609 /*
2610 * wait for ordered IO before we have any locks. We'll loop again
2611 * below with the locks held.
2612 */
2622 /* the extent lock is ordered inside the running
2623 * transaction
2624 */
2636 /*
2637 * we can't wait on the range with the transaction
2638 * running or with the extent lock held
2639 */
2648 }
2649 }
2653 u64 actual_end;
2660 else
2663 }
2680 }
2683 /*
2684 * We didn't need to allocate any more space, but we
2685 * still extended the size of the file so we need to
2686 * update i_size.
2687 */
2691 }
2698 }
2699 }
2702 out:
2706 out_reserve_fail:
2707 /* Let go of our reservation. */
2710 }
2713 {
2717 u64 lockstart;
2718 u64 lockend;
2719 u64 start;
2720 u64 len;
2726 /*
2727 * *offset can be negative, in this case we start finding DATA/HOLE from
2728 * the very start of the file.
2729 */
2736 lockend--;
2748 }
2763 }
2768 else
2770 }
2774 }
2777 {
2792 }
2798 }
2799 }
2802 out:
2805 }
2820 #ifdef CONFIG_COMPAT
2822 #endif
2823 };
2826 {
2829 }
2832 {
2836 NULL);
2841 }
2844 {
2847 /*
2848 * So with compression we will find and lock a dirty page and clear the
2849 * first one as dirty, setup an async extent, and immediately return
2850 * with the entire range locked but with nobody actually marked with
2851 * writeback. So we can't just filemap_write_and_wait_range() and
2852 * expect it to work since it will just kick off a thread to do the
2853 * actual work. So we need to call filemap_fdatawrite_range _again_
2854 * since it will wait on the page lock, which won't be unlocked until
2855 * after the pages have been marked as writeback and so we're good to go
2856 * from there. We have to do this otherwise we'll miss the ordered
2857 * extents and that results in badness. Please Josef, do not think you
2858 * know better and pull this out at some point in the future, it is
2859 * right and you are wrong.
2860 */
2867 }