| blob | b823fac91c9289bc67d3bb5191f4ce96e38294ac |
1 /*
2 * Copyright (C) 2007 Oracle. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
19 #include <linux/fs.h>
20 #include <linux/pagemap.h>
21 #include <linux/highmem.h>
22 #include <linux/time.h>
23 #include <linux/init.h>
24 #include <linux/string.h>
25 #include <linux/backing-dev.h>
26 #include <linux/mpage.h>
27 #include <linux/falloc.h>
28 #include <linux/swap.h>
29 #include <linux/writeback.h>
30 #include <linux/statfs.h>
31 #include <linux/compat.h>
32 #include <linux/slab.h>
33 #include <linux/btrfs.h>
34 #include <linux/uio.h>
46 /*
47 * when auto defrag is enabled we
48 * queue up these defrag structs to remember which
49 * inodes need defragging passes
50 */
53 /* objectid */
54 u64 ino;
55 /*
56 * transid where the defrag was added, we search for
57 * extents newer than this
58 */
59 u64 transid;
61 /* root objectid */
62 u64 root;
64 /* last offset we were able to defrag */
65 u64 last_offset;
67 /* if we've wrapped around back to zero once already */
69 };
73 {
82 else
84 }
86 /* pop a record for an inode into the defrag tree. The lock
87 * must be held already
88 *
89 * If you're inserting a record for an older transid than an
90 * existing record, the transid already in the tree is lowered
91 *
92 * If an existing record is found the defrag item you
93 * pass in is freed
94 */
97 {
115 /* if we're reinserting an entry for
116 * an old defrag run, make sure to
117 * lower the transid of our existing record
118 */
124 }
125 }
130 }
133 {
141 }
143 /*
144 * insert a defrag record for this inode if auto defrag is
145 * enabled
146 */
149 {
152 u64 transid;
163 else
176 /*
177 * If we set IN_DEFRAG flag and evict the inode from memory,
178 * and then re-read this inode, this new inode doesn't have
179 * IN_DEFRAG flag. At the case, we may find the existed defrag.
180 */
186 }
189 }
191 /*
192 * Requeue the defrag object. If there is a defrag object that points to
193 * the same inode in the tree, we will merge them together (by
194 * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
195 */
198 {
205 /*
206 * Here we don't check the IN_DEFRAG flag, because we need merge
207 * them together.
208 */
215 out:
217 }
219 /*
220 * pick the defragable inode that we want, if it doesn't exist, we will get
221 * the next one.
222 */
225 {
246 else
248 }
254 else
256 }
257 out:
262 }
265 {
279 }
281 }
283 #define BTRFS_DEFRAG_BATCH 1024
287 {
296 /* get the inode */
307 }
316 }
319 /* do a chunk of defrag */
327 BTRFS_DEFRAG_BATCH);
329 /*
330 * if we filled the whole defrag batch, there
331 * must be more work to do. Queue this defrag
332 * again
333 */
338 /*
339 * we didn't fill our defrag batch, but
340 * we didn't start at zero. Make sure we loop
341 * around to the start of the file.
342 */
348 }
352 cleanup:
356 }
358 /*
359 * run through the list of inodes in the FS that need
360 * defragging
361 */
363 {
370 /* Pause the auto defragger. */
378 /* find an inode to defrag */
380 first_ino);
388 }
389 }
395 }
398 /*
399 * during unmount, we use the transaction_wait queue to
400 * wait for the defragger to stop
401 */
404 }
406 /* simple helper to fault in pages and copy. This should go away
407 * and be replaced with calls into generic code.
408 */
413 {
423 /*
424 * Copy data from userspace to the current page
425 */
428 /* Flush processor's dcache for this page */
431 /*
432 * if we get a partial write, we can end up with
433 * partially up to date pages. These add
434 * a lot of complexity, so make sure they don't
435 * happen by forcing this copy to be retried.
436 *
437 * The rest of the btrfs_file_write code will fall
438 * back to page at a time copies after we return 0.
439 */
447 /* Return to btrfs_file_write_iter to fault page */
454 pg++;
456 }
457 }
459 }
461 /*
462 * unlocks pages after btrfs_file_write is done with them
463 */
465 {
468 /* page checked is some magic around finding pages that
469 * have been modified without going through btrfs_set_page_dirty
470 * clear it here. There should be no need to mark the pages
471 * accessed as prepare_pages should have marked them accessed
472 * in prepare_pages via find_or_create_page()
473 */
477 }
478 }
480 /*
481 * after copy_from_user, pages need to be dirtied and we need to make
482 * sure holes are created between the current EOF and the start of
483 * any next extents (if required).
484 *
485 * this also makes the decision about creating an inline extent vs
486 * doing real data extents, marking pages dirty and delalloc as required.
487 */
492 {
495 u64 num_bytes;
496 u64 start_pos;
497 u64 end_of_last_block;
506 cached);
515 }
517 /*
518 * we've only changed i_size in ram, and we haven't updated
519 * the disk i_size. There is no need to log the inode
520 * at this time.
521 */
525 }
527 /*
528 * this drops all the extents in the cache that intersect the range
529 * [start, end]. Existing extents are split as required.
530 */
533 {
539 u64 gen;
550 }
567 }
575 }
582 }
600 else
611 }
621 }
646 }
653 }
657 modified);
660 modified);
662 }
665 }
666 next:
671 /* once for us */
673 /* once for the tree*/
675 }
680 }
682 /*
683 * this is very complex, but the basic idea is to drop all extents
684 * in the range start - end. hint_block is filled in with a block number
685 * that would be a good hint to the block allocator for this file.
686 *
687 * If an extent intersects the range but is not entirely inside the range
688 * it is either truncated or split. Anything entirely inside the range
689 * is deleted from the tree.
690 */
696 u32 extent_item_size,
698 {
739 }
741 leafs_visited++;
742 next_slot:
752 }
753 leafs_visited++;
756 }
781 }
783 /*
784 * Don't skip extent items representing 0 byte lengths. They
785 * used to be created (bug) if while punching holes we hit
786 * -ENOSPC condition. So if we find one here, just ensure we
787 * delete it, otherwise we would insert a new file extent item
788 * with the same key (offset) as that 0 bytes length file
789 * extent item in the call to setup_items_for_insert() later
790 * in this function.
791 */
798 }
806 }
808 /*
809 * | - range to drop - |
810 * | -------- extent -------- |
811 */
817 }
826 }
852 }
854 }
855 /*
856 * | ---- range to drop ----- |
857 * | -------- extent -------- |
858 */
863 }
877 }
880 /*
881 * | ---- range to drop ----- |
882 * | -------- extent -------- |
883 */
889 }
901 }
903 /*
904 * | ---- range to drop ----- |
905 * | ------ extent ------ |
906 */
908 delete_extent_item:
914 del_nr++;
915 }
932 }
940 }
943 del_nr);
947 }
954 }
957 }
960 /*
961 * Set path->slots[0] to first slot, so that after the delete
962 * if items are move off from our leaf to its immediate left or
963 * right neighbor leafs, we end up with a correct and adjusted
964 * path->slots[0] for our insertion (if replace_extent != 0).
965 */
970 }
973 /*
974 * If btrfs_del_items() was called, it might have deleted a leaf, in
975 * which case it unlocked our path, so check path->locks[0] matches a
976 * write lock.
977 */
993 }
996 extent_item_size,
1000 }
1007 }
1012 {
1023 }
1028 {
1031 u64 extent_end;
1056 }
1058 /*
1059 * Mark extent in the range start - end as written.
1060 *
1061 * This changes extent type from 'pre-allocated' to 'regular'. If only
1062 * part of extent is marked as written, the extent will be split into
1063 * two or three.
1064 */
1067 {
1074 u64 bytenr;
1075 u64 num_bytes;
1076 u64 extent_end;
1077 u64 orig_offset;
1078 u64 other_start;
1079 u64 other_end;
1080 u64 split;
1090 again:
1109 BTRFS_FILE_EXTENT_PREALLOC);
1142 }
1143 }
1171 }
1172 }
1183 }
1187 }
1216 }
1218 }
1228 }
1231 del_nr++;
1236 }
1245 }
1248 del_nr++;
1253 }
1258 BTRFS_FILE_EXTENT_REG);
1265 BTRFS_FILE_EXTENT_REG);
1275 }
1276 }
1277 out:
1280 }
1282 /*
1283 * on error we return an unlocked page and the error value
1284 * on success we return a locked page and 0
1285 */
1288 {
1300 }
1301 }
1303 }
1305 /*
1306 * this just gets pages into the page cache and locks them down.
1307 */
1311 {
1325 }
1329 force_uptodate);
1337 }
1339 }
1342 fail:
1346 faili--;
1347 }
1350 }
1352 /*
1353 * This function locks the extent and properly waits for data=ordered extents
1354 * to finish before allowing the pages to be modified if need.
1355 *
1356 * The return value:
1357 * 1 - the extent is locked
1358 * 0 - the extent is not locked, and everything is OK
1359 * -EAGAIN - need re-prepare the pages
1360 * the other < 0 number - Something wrong happens
1361 */
1367 {
1368 u64 start_pos;
1369 u64 last_pos;
1391 }
1395 }
1406 }
1413 }
1416 }
1420 {
1424 u64 num_bytes;
1440 }
1444 }
1454 }
1459 }
1463 loff_t pos)
1464 {
1470 u64 lockstart;
1471 u64 lockend;
1494 offset);
1496 PAGE_CACHE_SIZE);
1503 /*
1504 * Fault pages before locking them in prepare_pages
1505 * to avoid recursive lock
1506 */
1510 }
1516 BTRFS_INODE_PREALLOC))) {
1520 /*
1521 * our prealloc extent may be smaller than
1522 * write_bytes, so scale down.
1523 */
1525 PAGE_CACHE_SIZE);
1530 }
1531 }
1540 reserve_bytes);
1541 else
1544 }
1548 again:
1549 /*
1550 * This is going to setup the pages array with the number of
1551 * pages we want, so we don't really need to worry about the
1552 * contents of pages from loop to loop
1553 */
1556 force_page_uptodate);
1570 }
1575 /*
1576 * if we have trouble faulting in the pages, fall
1577 * back to one page at a time
1578 */
1588 PAGE_CACHE_SIZE);
1589 }
1591 /*
1592 * If we had a short copy we need to release the excess delaloc
1593 * bytes we reserved. We need to increment outstanding_extents
1594 * because btrfs_delalloc_release_space will decrement it, but
1595 * we still have an outstanding extent for the chunk we actually
1596 * managed to copy.
1597 */
1600 PAGE_CACHE_SHIFT;
1605 }
1608 release_bytes);
1609 else
1611 release_bytes);
1612 }
1619 NULL);
1623 GFP_NOFS);
1627 }
1642 }
1654 }
1664 }
1665 }
1668 }
1672 loff_t pos)
1673 {
1676 ssize_t written;
1677 ssize_t written_buffered;
1678 loff_t endbyte;
1691 }
1692 /*
1693 * Ensure all data is persisted. We want the next direct IO read to be
1694 * able to read what was just written.
1695 */
1707 out:
1709 }
1712 {
1727 }
1731 {
1735 u64 start_pos;
1736 u64 end_pos;
1739 ssize_t err;
1740 loff_t pos;
1748 }
1755 }
1757 /*
1758 * If BTRFS flips readonly due to some impossible error
1759 * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
1760 * although we have opened a file as writable, we have
1761 * to stop this write operation to ensure FS consistency.
1762 */
1767 }
1769 /*
1770 * We reserve space for updating the inode when we reserve space for the
1771 * extent we are going to write, so we will enospc out there. We don't
1772 * need to start yet another transaction to update the inode as we will
1773 * update the inode when we finish writing whatever data we write.
1774 */
1781 /* Expand hole size to cover write data, preventing empty gap */
1787 }
1788 }
1799 }
1803 /*
1804 * We also have to set last_sub_trans to the current log transid,
1805 * otherwise subsequent syncs to a file that's been synced in this
1806 * transaction will appear to have already occured.
1807 */
1815 }
1819 out:
1822 }
1825 {
1828 /*
1829 * ordered_data_close is set by settattr when we are about to truncate
1830 * a file from a non-zero size to a zero size. This tries to
1831 * flush down new bytes that may have been written if the
1832 * application were using truncate to replace a file in place.
1833 */
1838 }
1841 {
1849 }
1851 /*
1852 * fsync call for both files and directories. This logs the inode into
1853 * the tree log instead of forcing full commits whenever possible.
1854 *
1855 * It needs to call filemap_fdatawait so that all ordered extent updates are
1856 * in the metadata btree are up to date for copying to the log.
1857 *
1858 * It drops the inode mutex before doing the tree log commit. This is an
1859 * important optimization for directories because holding the mutex prevents
1860 * new operations on the dir while we write to disk.
1861 */
1863 {
1875 /*
1876 * We write the dirty pages in the range and wait until they complete
1877 * out of the ->i_mutex. If so, we can flush the dirty pages by
1878 * multi-task, and make the performance up. See
1879 * btrfs_wait_ordered_range for an explanation of the ASYNC check.
1880 */
1889 /*
1890 * We might have have had more pages made dirty after calling
1891 * start_ordered_ops and before acquiring the inode's i_mutex.
1892 */
1894 /*
1895 * For a full sync, we need to make sure any ordered operations
1896 * start and finish before we start logging the inode, so that
1897 * all extents are persisted and the respective file extent
1898 * items are in the fs/subvol btree.
1899 */
1902 /*
1903 * Start any new ordered operations before starting to log the
1904 * inode. We will wait for them to finish in btrfs_sync_log().
1905 *
1906 * Right before acquiring the inode's mutex, we might have new
1907 * writes dirtying pages, which won't immediately start the
1908 * respective ordered operations - that is done through the
1909 * fill_delalloc callbacks invoked from the writepage and
1910 * writepages address space operations. So make sure we start
1911 * all ordered operations before starting to log our inode. Not
1912 * doing this means that while logging the inode, writeback
1913 * could start and invoke writepage/writepages, which would call
1914 * the fill_delalloc callbacks (cow_file_range,
1915 * submit_compressed_extents). These callbacks add first an
1916 * extent map to the modified list of extents and then create
1917 * the respective ordered operation, which means in
1918 * tree-log.c:btrfs_log_inode() we might capture all existing
1919 * ordered operations (with btrfs_get_logged_extents()) before
1920 * the fill_delalloc callback adds its ordered operation, and by
1921 * the time we visit the modified list of extent maps (with
1922 * btrfs_log_changed_extents()), we see and process the extent
1923 * map they created. We then use the extent map to construct a
1924 * file extent item for logging without waiting for the
1925 * respective ordered operation to finish - this file extent
1926 * item points to a disk location that might not have yet been
1927 * written to, containing random data - so after a crash a log
1928 * replay will make our inode have file extent items that point
1929 * to disk locations containing invalid data, as we returned
1930 * success to userspace without waiting for the respective
1931 * ordered operation to finish, because it wasn't captured by
1932 * btrfs_get_logged_extents().
1933 */
1935 }
1939 }
1942 /*
1943 * If the last transaction that changed this file was before the current
1944 * transaction and we have the full sync flag set in our inode, we can
1945 * bail out now without any syncing.
1946 *
1947 * Note that we can't bail out if the full sync flag isn't set. This is
1948 * because when the full sync flag is set we start all ordered extents
1949 * and wait for them to fully complete - when they complete they update
1950 * the inode's last_trans field through:
1951 *
1952 * btrfs_finish_ordered_io() ->
1953 * btrfs_update_inode_fallback() ->
1954 * btrfs_update_inode() ->
1955 * btrfs_set_inode_last_trans()
1956 *
1957 * So we are sure that last_trans is up to date and can do this check to
1958 * bail out safely. For the fast path, when the full sync flag is not
1959 * set in our inode, we can not do it because we start only our ordered
1960 * extents and don't wait for them to complete (that is when
1961 * btrfs_finish_ordered_io runs), so here at this point their last_trans
1962 * value might be less than or equals to fs_info->last_trans_committed,
1963 * and setting a speculative last_trans for an inode when a buffered
1964 * write is made (such as fs_info->generation + 1 for example) would not
1965 * be reliable since after setting the value and before fsync is called
1966 * any number of transactions can start and commit (transaction kthread
1967 * commits the current transaction periodically), and a transaction
1968 * commit does not start nor waits for ordered extents to complete.
1969 */
1976 /*
1977 * We'v had everything committed since the last time we were
1978 * modified so clear this flag in case it was set for whatever
1979 * reason, it's no longer relevant.
1980 */
1985 }
1987 /*
1988 * ok we haven't committed the transaction yet, lets do a commit
1989 */
1993 /*
1994 * We use start here because we will need to wait on the IO to complete
1995 * in btrfs_sync_log, which could require joining a transaction (for
1996 * example checking cross references in the nocow path). If we use join
1997 * here we could get into a situation where we're waiting on IO to
1998 * happen that is blocked on a transaction trying to commit. With start
1999 * we inc the extwriter counter, so we wait for all extwriters to exit
2000 * before we start blocking join'ers. This comment is to keep somebody
2001 * from thinking they are super smart and changing this to
2002 * btrfs_join_transaction *cough*Josef*cough*.
2003 */
2009 }
2016 /* Fallthrough and commit/free transaction. */
2018 }
2020 /* we've logged all the items and now have a consistent
2021 * version of the file in the log. It is possible that
2022 * someone will come in and modify the file, but that's
2023 * fine because the log is consistent on disk, and we
2024 * have references to all of the file's extents
2025 *
2026 * It is possible that someone will come in and log the
2027 * file again, but that will end up using the synchronization
2028 * inside btrfs_sync_log to keep things safe.
2029 */
2032 /*
2033 * If any of the ordered extents had an error, just return it to user
2034 * space, so that the application knows some writes didn't succeed and
2035 * can take proper action (retry for e.g.). Blindly committing the
2036 * transaction in this case, would fool userspace that everything was
2037 * successful. And we also want to make sure our log doesn't contain
2038 * file extent items pointing to extents that weren't fully written to -
2039 * just like in the non fast fsync path, where we check for the ordered
2040 * operation's error flag before writing to the log tree and return -EIO
2041 * if any of them had this flag set (btrfs_wait_ordered_range) -
2042 * therefore we need to check for errors in the ordered operations,
2043 * which are indicated by ctx.io_err.
2044 */
2049 }
2057 }
2058 }
2065 }
2066 }
2070 }
2071 out:
2073 }
2079 };
2082 {
2092 }
2096 {
2121 }
2125 {
2148 u64 num_bytes;
2160 }
2163 u64 num_bytes;
2170 offset;
2176 }
2185 out:
2216 }
2219 }
2221 /*
2222 * Find a hole extent on given inode and change start/len to the end of hole
2223 * extent.(hole/vacuum extent whose em->start <= start &&
2224 * em->start + em->len > start)
2225 * When a hole extent is found, return 1 and modify start/len.
2226 */
2228 {
2236 else
2239 }
2241 /* Hole or vacuum extent(only exists in no-hole mode) */
2247 }
2250 }
2253 {
2259 u64 lockstart;
2260 u64 lockend;
2261 u64 tail_start;
2262 u64 tail_len;
2264 u64 cur_offset;
2266 u64 drop_end;
2272 u64 ino_size;
2286 /* Already in a large hole */
2289 }
2297 /*
2298 * We needn't truncate any page which is beyond the end of the file
2299 * because we are sure there is no data there.
2300 */
2301 /*
2302 * Only do this if we are in the same page and we aren't doing the
2303 * entire page.
2304 */
2311 }
2313 }
2315 /* zero back part of the first page */
2322 }
2323 }
2325 /* Check the aligned pages after the first unaligned page,
2326 * if offset != orig_start, which means the first unaligned page
2327 * including serveral following pages are already in holes,
2328 * the extra check can be skipped */
2330 /* after truncate page, check hole again */
2339 }
2341 }
2343 /* Check the tail unaligned part is in a hole */
2351 /* zero the front end of the last page */
2358 }
2359 }
2360 }
2365 }
2376 /*
2377 * We need to make sure we have no ordered extents in this range
2378 * and nobody raced in and read a page in this range, if we did
2379 * we need to try again.
2380 */
2388 }
2398 }
2399 }
2405 }
2411 }
2415 /*
2416 * 1 - update the inode
2417 * 1 - removing the extents in the range
2418 * 1 - adding the hole extent if no_holes isn't set
2419 */
2425 }
2428 min_size);
2445 drop_end);
2449 }
2450 }
2458 }
2468 }
2481 }
2482 }
2487 }
2490 /*
2491 * Don't insert file hole extent item if it's for a range beyond eof
2492 * (because it's useless) or if it represents a 0 bytes range (when
2493 * cur_offset == drop_end).
2494 */
2500 }
2501 }
2503 out_trans:
2515 out_free:
2518 out:
2521 out_only_mutex:
2523 /*
2524 * If we only end up zeroing part of a page, we still need to
2525 * update the inode item, so that all the time fields are
2526 * updated as well as the necessary btrfs inode in memory fields
2527 * for detecting, at fsync time, if the inode isn't yet in the
2528 * log tree or it's there but not up to date.
2529 */
2536 }
2537 }
2542 }
2546 {
2549 u64 cur_offset;
2550 u64 last_byte;
2551 u64 alloc_start;
2552 u64 alloc_end;
2554 u64 locked_end;
2562 /* Make sure we aren't being give some crap mode */
2569 /*
2570 * Make sure we have enough space before we do the
2571 * allocation.
2572 */
2584 alloc_start);
2588 /*
2589 * If we are fallocating from the end of the file onward we
2590 * need to zero out the end of the page if i_size lands in the
2591 * middle of a page.
2592 */
2596 }
2598 /*
2599 * wait for ordered IO before we have any locks. We'll loop again
2600 * below with the locks held.
2601 */
2611 /* the extent lock is ordered inside the running
2612 * transaction
2613 */
2625 /*
2626 * we can't wait on the range with the transaction
2627 * running or with the extent lock held
2628 */
2637 }
2638 }
2642 u64 actual_end;
2649 else
2652 }
2670 /*
2671 * We didn't need to allocate any more space, but we
2672 * still extended the size of the file so we need to
2673 * update i_size and the inode item.
2674 */
2682 NULL);
2686 else
2688 root);
2689 }
2690 }
2699 }
2700 }
2703 out:
2705 /* Let go of our reservation. */
2708 }
2711 {
2715 u64 lockstart;
2716 u64 lockend;
2717 u64 start;
2718 u64 len;
2724 /*
2725 * *offset can be negative, in this case we start finding DATA/HOLE from
2726 * the very start of the file.
2727 */
2734 lockend--;
2746 }
2761 }
2766 else
2768 }
2772 }
2775 {
2790 }
2796 }
2797 }
2800 out:
2803 }
2816 #ifdef CONFIG_COMPAT
2818 #endif
2819 };
2822 {
2825 }
2828 {
2832 NULL);
2837 }
2840 {
2843 /*
2844 * So with compression we will find and lock a dirty page and clear the
2845 * first one as dirty, setup an async extent, and immediately return
2846 * with the entire range locked but with nobody actually marked with
2847 * writeback. So we can't just filemap_write_and_wait_range() and
2848 * expect it to work since it will just kick off a thread to do the
2849 * actual work. So we need to call filemap_fdatawrite_range _again_
2850 * since it will wait on the page lock, which won't be unlocked until
2851 * after the pages have been marked as writeback and so we're good to go
2852 * from there. We have to do this otherwise we'll miss the ordered
2853 * extents and that results in badness. Please Josef, do not think you
2854 * know better and pull this out at some point in the future, it is
2855 * right and you are wrong.
2856 */
2863 }