| blob | 0e41459b8de66781490fcf7192cd1e84e298a582 |
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
6 #include <linux/fs.h>
7 #include <linux/pagemap.h>
8 #include <linux/time.h>
9 #include <linux/init.h>
10 #include <linux/string.h>
11 #include <linux/backing-dev.h>
12 #include <linux/falloc.h>
13 #include <linux/writeback.h>
14 #include <linux/compat.h>
15 #include <linux/slab.h>
16 #include <linux/btrfs.h>
17 #include <linux/uio.h>
18 #include <linux/iversion.h>
33 /*
34 * when auto defrag is enabled we
35 * queue up these defrag structs to remember which
36 * inodes need defragging passes
37 */
40 /* objectid */
41 u64 ino;
42 /*
43 * transid where the defrag was added, we search for
44 * extents newer than this
45 */
46 u64 transid;
48 /* root objectid */
49 u64 root;
51 /* last offset we were able to defrag */
52 u64 last_offset;
54 /* if we've wrapped around back to zero once already */
56 };
60 {
69 else
71 }
73 /* pop a record for an inode into the defrag tree. The lock
74 * must be held already
75 *
76 * If you're inserting a record for an older transid than an
77 * existing record, the transid already in the tree is lowered
78 *
79 * If an existing record is found the defrag item you
80 * pass in is freed
81 */
84 {
102 /* if we're reinserting an entry for
103 * an old defrag run, make sure to
104 * lower the transid of our existing record
105 */
111 }
112 }
117 }
120 {
128 }
130 /*
131 * insert a defrag record for this inode if auto defrag is
132 * enabled
133 */
136 {
140 u64 transid;
151 else
164 /*
165 * If we set IN_DEFRAG flag and evict the inode from memory,
166 * and then re-read this inode, this new inode doesn't have
167 * IN_DEFRAG flag. At the case, we may find the existed defrag.
168 */
174 }
177 }
179 /*
180 * Requeue the defrag object. If there is a defrag object that points to
181 * the same inode in the tree, we will merge them together (by
182 * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
183 */
186 {
193 /*
194 * Here we don't check the IN_DEFRAG flag, because we need merge
195 * them together.
196 */
203 out:
205 }
207 /*
208 * pick the defragable inode that we want, if it doesn't exist, we will get
209 * the next one.
210 */
213 {
234 else
236 }
242 else
244 }
245 out:
250 }
253 {
267 }
269 }
271 #define BTRFS_DEFRAG_BATCH 1024
275 {
282 /* get the inode */
287 }
294 }
296 /* do a chunk of defrag */
304 BTRFS_DEFRAG_BATCH);
306 /*
307 * if we filled the whole defrag batch, there
308 * must be more work to do. Queue this defrag
309 * again
310 */
315 /*
316 * we didn't fill our defrag batch, but
317 * we didn't start at zero. Make sure we loop
318 * around to the start of the file.
319 */
325 }
329 cleanup:
332 }
334 /*
335 * run through the list of inodes in the FS that need
336 * defragging
337 */
339 {
346 /* Pause the auto defragger. */
354 /* find an inode to defrag */
356 first_ino);
364 }
365 }
371 }
374 /*
375 * during unmount, we use the transaction_wait queue to
376 * wait for the defragger to stop
377 */
380 }
382 /* simple helper to fault in pages and copy. This should go away
383 * and be replaced with calls into generic code.
384 */
388 {
398 /*
399 * Copy data from userspace to the current page
400 */
403 /* Flush processor's dcache for this page */
406 /*
407 * if we get a partial write, we can end up with
408 * partially up to date pages. These add
409 * a lot of complexity, so make sure they don't
410 * happen by forcing this copy to be retried.
411 *
412 * The rest of the btrfs_file_write code will fall
413 * back to page at a time copies after we return 0.
414 */
422 /* Return to btrfs_file_write_iter to fault page */
429 pg++;
431 }
432 }
434 }
436 /*
437 * unlocks pages after btrfs_file_write is done with them
438 */
440 {
443 /* page checked is some magic around finding pages that
444 * have been modified without going through btrfs_set_page_dirty
445 * clear it here. There should be no need to mark the pages
446 * accessed as prepare_pages should have marked them accessed
447 * in prepare_pages via find_or_create_page()
448 */
452 }
453 }
455 /*
456 * after copy_from_user, pages need to be dirtied and we need to make
457 * sure holes are created between the current EOF and the start of
458 * any next extents (if required).
459 *
460 * this also makes the decision about creating an inline extent vs
461 * doing real data extents, marking pages dirty and delalloc as required.
462 */
466 {
470 u64 num_bytes;
471 u64 start_pos;
472 u64 end_of_last_block;
489 /*
490 * The pages may have already been dirty, clear out old accounting so
491 * we can set things up properly
492 */
507 }
509 /*
510 * we've only changed i_size in ram, and we haven't updated
511 * the disk i_size. There is no need to log the inode
512 * at this time.
513 */
517 }
519 /*
520 * this drops all the extents in the cache that intersect the range
521 * [start, end]. Existing extents are split as required.
522 */
525 {
531 u64 gen;
542 }
559 }
567 }
574 }
592 else
603 }
612 }
636 }
643 }
647 modified);
650 modified);
652 }
655 }
656 next:
661 /* once for us */
663 /* once for the tree*/
665 }
670 }
672 /*
673 * this is very complex, but the basic idea is to drop all extents
674 * in the range start - end. hint_block is filled in with a block number
675 * that would be a good hint to the block allocator for this file.
676 *
677 * If an extent intersects the range but is not entirely inside the range
678 * it is either truncated or split. Anything entirely inside the range
679 * is deleted from the tree.
680 *
681 * Note: the VFS' inode number of bytes is not updated, it's up to the caller
682 * to deal with that. We set the field 'bytes_found' of the arguments structure
683 * with the number of allocated bytes found in the target range, so that the
684 * caller can update the inode's number of bytes in an atomic way when
685 * replacing extents in a range to avoid races with stat(2).
686 */
690 {
718 /* Must always have a path if ->replace_extent is true */
726 }
727 }
749 }
751 leafs_visited++;
752 next_slot:
762 }
763 leafs_visited++;
766 }
777 }
796 /* can't happen */
798 }
800 /*
801 * Don't skip extent items representing 0 byte lengths. They
802 * used to be created (bug) if while punching holes we hit
803 * -ENOSPC condition. So if we find one here, just ensure we
804 * delete it, otherwise we would insert a new file extent item
805 * with the same key (offset) as that 0 bytes length file
806 * extent item in the call to setup_items_for_insert() later
807 * in this function.
808 */
812 }
817 }
825 }
827 /*
828 * | - range to drop - |
829 * | -------- extent -------- |
830 */
836 }
845 }
866 BTRFS_ADD_DELAYED_REF,
874 }
876 }
877 /*
878 * From here on out we will have actually dropped something, so
879 * last_end can be updated.
880 */
883 /*
884 * | ---- range to drop ----- |
885 * | -------- extent -------- |
886 */
891 }
905 }
908 /*
909 * | ---- range to drop ----- |
910 * | -------- extent -------- |
911 */
917 }
929 }
931 /*
932 * | ---- range to drop ----- |
933 * | ------ extent ------ |
934 */
936 delete_extent_item:
942 del_nr++;
943 }
952 BTRFS_DROP_DELAYED_REF,
961 }
969 }
972 del_nr);
976 }
983 }
986 }
989 /*
990 * Set path->slots[0] to first slot, so that after the delete
991 * if items are move off from our leaf to its immediate left or
992 * right neighbor leafs, we end up with a correct and adjusted
993 * path->slots[0] for our insertion (if args->replace_extent).
994 */
999 }
1002 /*
1003 * If btrfs_del_items() was called, it might have deleted a leaf, in
1004 * which case it unlocked our path, so check path->locks[0] matches a
1005 * write lock.
1006 */
1021 }
1025 }
1031 out:
1035 }
1040 {
1043 u64 extent_end;
1068 }
1070 /*
1071 * Mark extent in the range start - end as written.
1072 *
1073 * This changes extent type from 'pre-allocated' to 'regular'. If only
1074 * part of extent is marked as written, the extent will be split into
1075 * two or three.
1076 */
1079 {
1088 u64 bytenr;
1089 u64 num_bytes;
1090 u64 extent_end;
1091 u64 orig_offset;
1092 u64 other_start;
1093 u64 other_end;
1094 u64 split;
1104 again:
1124 }
1131 }
1137 }
1168 }
1169 }
1197 }
1198 }
1209 }
1213 }
1234 orig_offset);
1239 }
1248 }
1251 }
1253 }
1266 }
1269 del_nr++;
1274 }
1275 }
1284 }
1287 del_nr++;
1292 }
1293 }
1298 BTRFS_FILE_EXTENT_REG);
1305 BTRFS_FILE_EXTENT_REG);
1315 }
1316 }
1317 out:
1320 }
1322 /*
1323 * on error we return an unlocked page and the error value
1324 * on success we return a locked page and 0
1325 */
1329 {
1341 }
1345 }
1346 }
1348 }
1350 /*
1351 * this just gets pages into the page cache and locks them down.
1352 */
1356 {
1364 again:
1371 }
1375 force_uptodate);
1384 }
1387 }
1389 }
1392 fail:
1396 faili--;
1397 }
1400 }
1402 /*
1403 * This function locks the extent and properly waits for data=ordered extents
1404 * to finish before allowing the pages to be modified if need.
1405 *
1406 * The return value:
1407 * 1 - the extent is locked
1408 * 0 - the extent is not locked, and everything is OK
1409 * -EAGAIN - need re-prepare the pages
1410 * the other < 0 number - Something wrong happens
1411 */
1418 {
1420 u64 start_pos;
1421 u64 last_pos;
1432 cached_state);
1443 }
1447 }
1454 }
1456 /*
1457 * It's possible the pages are dirty right now, but we don't want
1458 * to clean them yet because copy_from_user may catch a page fault
1459 * and we might have to fall back to one page at a time. If that
1460 * happens, we'll unlock these pages and we'd have a window where
1461 * reclaim could sneak in and drop the once-dirty page on the floor
1462 * without writing it.
1463 *
1464 * We have the pages locked and the extent range locked, so there's
1465 * no way someone can start IO on any dirty pages in this range.
1466 *
1467 * We'll call btrfs_dirty_pages() later on, and that will flip around
1468 * delalloc bits and dirty the pages as required.
1469 */
1473 }
1476 }
1480 {
1484 u64 num_bytes;
1505 num_bytes);
1510 }
1514 }
1525 }
1526 out_unlock:
1530 }
1534 {
1536 }
1538 /*
1539 * Check if we can do nocow write into the range [@pos, @pos + @write_bytes)
1540 *
1541 * @pos: File offset
1542 * @write_bytes: The length to write, will be updated to the nocow writeable
1543 * range
1544 *
1545 * This function will flush ordered extents in the range to ensure proper
1546 * nocow checks.
1547 *
1548 * Return:
1549 * >0 and update @write_bytes if we can do nocow write
1550 * 0 if we can't do nocow write
1551 * -EAGAIN if we can't get the needed lock or there are ordered extents
1552 * for * (nowait == true) case
1553 * <0 if other error happened
1554 *
1555 * NOTE: Callers need to release the lock by btrfs_check_nocow_unlock().
1556 */
1559 {
1561 }
1564 {
1566 }
1569 {
1584 }
1588 {
1594 loff_t oldsize;
1595 loff_t start_pos;
1600 /* We will allocate space in case nodatacow is not set, so bail */
1603 /*
1604 * There are holes in the range or parts of the range that must
1605 * be COWed (shared extents, RO block groups, etc), so just bail
1606 * out.
1607 */
1610 }
1617 /*
1618 * We reserve space for updating the inode when we reserve space for the
1619 * extent we are going to write, so we will enospc out there. We don't
1620 * need to start yet another transaction to update the inode as we will
1621 * update the inode when we finish writing whatever data we write.
1622 */
1628 /* Expand hole size to cover write data, preventing empty gap */
1635 }
1636 }
1639 }
1643 {
1645 loff_t pos;
1651 u64 lockstart;
1652 u64 lockend;
1655 ssize_t ret;
1685 }
1693 offset);
1702 /*
1703 * Fault pages before locking them in prepare_pages
1704 * to avoid recursive lock
1705 */
1709 }
1717 write_bytes);
1719 /*
1720 * If we don't have to COW at the offset, reserve
1721 * metadata only. write_bytes may get smaller than
1722 * requested here.
1723 */
1727 else
1729 }
1737 reserve_bytes);
1742 write_bytes);
1743 else
1746 }
1749 again:
1750 /*
1751 * This is going to setup the pages array with the number of
1752 * pages we want, so we don't really need to worry about the
1753 * contents of pages from loop to loop
1754 */
1757 force_page_uptodate);
1760 reserve_bytes);
1762 }
1772 reserve_bytes);
1775 }
1784 /*
1785 * if we have trouble faulting in the pages, fall
1786 * back to one page at a time
1787 */
1798 PAGE_SIZE);
1799 }
1802 /* release everything except the sectors we dirtied */
1808 u64 __pos;
1816 }
1817 }
1826 /*
1827 * If we have not locked the extent range, because the range's
1828 * start offset is >= i_size, we might still have a non-NULL
1829 * cached extent state, acquired while marking the extent range
1830 * as delalloc through btrfs_dirty_pages(). Therefore free any
1831 * possible cached extent state to avoid a memory leak.
1832 */
1836 else
1843 }
1857 }
1868 data_reserved,
1871 }
1872 }
1878 }
1879 out:
1882 }
1886 {
1896 }
1899 {
1903 loff_t pos;
1905 ssize_t written_buffered;
1906 loff_t endbyte;
1907 ssize_t err;
1914 /* If the write DIO is within EOF, use a shared lock */
1918 relock:
1927 }
1933 }
1936 /*
1937 * Re-check since file size may have changed just before taking the
1938 * lock or pos may have changed because of O_APPEND in generic_write_check()
1939 */
1945 }
1950 }
1963 }
1968 }
1970 buffered:
1976 }
1977 /*
1978 * Ensure all data is persisted. We want the next direct IO read to be
1979 * able to read what was just written.
1980 */
1992 out:
1994 }
1998 {
2006 /*
2007 * If the fs flips readonly due to some impossible error, although we
2008 * have opened a file as writable, we have to stop this write operation
2009 * to ensure consistency.
2010 */
2023 else
2026 /*
2027 * We also have to set last_sub_trans to the current log transid,
2028 * otherwise subsequent syncs to a file that's been synced in this
2029 * transaction will appear to have already occurred.
2030 */
2042 }
2045 {
2053 /*
2054 * Set by setattr when we are about to truncate a file from a non-zero
2055 * size to a zero size. This tries to flush down new bytes that may
2056 * have been written if the application were using truncate to replace
2057 * a file in place.
2058 */
2063 }
2066 {
2070 /*
2071 * This is only called in fsync, which would do synchronous writes, so
2072 * a plug can merge adjacent IOs as much as possible. Esp. in case of
2073 * multiple disks using raid profile, a large IO can be split to
2074 * several segments of stripe length (currently 64K).
2075 */
2083 }
2085 /*
2086 * fsync call for both files and directories. This logs the inode into
2087 * the tree log instead of forcing full commits whenever possible.
2088 *
2089 * It needs to call filemap_fdatawait so that all ordered extent updates are
2090 * in the metadata btree are up to date for copying to the log.
2091 *
2092 * It drops the inode mutex before doing the tree log commit. This is an
2093 * important optimization for directories because holding the mutex prevents
2094 * new operations on the dir while we write to disk.
2095 */
2097 {
2105 u64 len;
2112 /*
2113 * Always set the range to a full range, otherwise we can get into
2114 * several problems, from missing file extent items to represent holes
2115 * when not using the NO_HOLES feature, to log tree corruption due to
2116 * races between hole detection during logging and completion of ordered
2117 * extents outside the range, to missing checksums due to ordered extents
2118 * for which we flushed only a subset of their pages.
2119 */
2124 /*
2125 * We write the dirty pages in the range and wait until they complete
2126 * out of the ->i_mutex. If so, we can flush the dirty pages by
2127 * multi-task, and make the performance up. See
2128 * btrfs_wait_ordered_range for an explanation of the ASYNC check.
2129 */
2138 /*
2139 * Always check for the full sync flag while holding the inode's lock,
2140 * to avoid races with other tasks. The flag must be either set all the
2141 * time during logging or always off all the time while logging.
2142 */
2146 /*
2147 * Before we acquired the inode's lock, someone may have dirtied more
2148 * pages in the target range. We need to make sure that writeback for
2149 * any such pages does not start while we are logging the inode, because
2150 * if it does, any of the following might happen when we are not doing a
2151 * full inode sync:
2152 *
2153 * 1) We log an extent after its writeback finishes but before its
2154 * checksums are added to the csum tree, leading to -EIO errors
2155 * when attempting to read the extent after a log replay.
2156 *
2157 * 2) We can end up logging an extent before its writeback finishes.
2158 * Therefore after the log replay we will have a file extent item
2159 * pointing to an unwritten extent (and no data checksums as well).
2160 *
2161 * So trigger writeback for any eventual new dirty pages and then we
2162 * wait for all ordered extents to complete below.
2163 */
2168 }
2170 /*
2171 * We have to do this here to avoid the priority inversion of waiting on
2172 * IO of a lower priority task while holding a transaction open.
2173 *
2174 * For a full fsync we wait for the ordered extents to complete while
2175 * for a fast fsync we wait just for writeback to complete, and then
2176 * attach the ordered extents to the transaction so that a transaction
2177 * commit waits for their completion, to avoid data loss if we fsync,
2178 * the current transaction commits before the ordered extents complete
2179 * and a power failure happens right after that.
2180 */
2184 /*
2185 * Get our ordered extents as soon as possible to avoid doing
2186 * checksum lookups in the csum tree, and use instead the
2187 * checksums attached to the ordered extents.
2188 */
2192 }
2199 /*
2200 * If we are doing a fast fsync we can not bail out if the inode's
2201 * last_trans is <= then the last committed transaction, because we only
2202 * update the last_trans of the inode during ordered extent completion,
2203 * and for a fast fsync we don't wait for that, we only wait for the
2204 * writeback to complete.
2205 */
2210 /*
2211 * We've had everything committed since the last time we were
2212 * modified so clear this flag in case it was set for whatever
2213 * reason, it's no longer relevant.
2214 */
2217 /*
2218 * An ordered extent might have started before and completed
2219 * already with io errors, in which case the inode was not
2220 * updated and we end up here. So check the inode's mapping
2221 * for any errors that might have happened since we last
2222 * checked called fsync.
2223 */
2226 }
2228 /*
2229 * We use start here because we will need to wait on the IO to complete
2230 * in btrfs_sync_log, which could require joining a transaction (for
2231 * example checking cross references in the nocow path). If we use join
2232 * here we could get into a situation where we're waiting on IO to
2233 * happen that is blocked on a transaction trying to commit. With start
2234 * we inc the extwriter counter, so we wait for all extwriters to exit
2235 * before we start blocking joiners. This comment is to keep somebody
2236 * from thinking they are super smart and changing this to
2237 * btrfs_join_transaction *cough*Josef*cough*.
2238 */
2243 }
2248 /* Fallthrough and commit/free transaction. */
2250 }
2252 /* we've logged all the items and now have a consistent
2253 * version of the file in the log. It is possible that
2254 * someone will come in and modify the file, but that's
2255 * fine because the log is consistent on disk, and we
2256 * have references to all of the file's extents
2257 *
2258 * It is possible that someone will come in and log the
2259 * file again, but that will end up using the synchronization
2260 * inside btrfs_sync_log to keep things safe.
2261 */
2270 }
2271 }
2277 }
2278 }
2282 }
2283 out:
2290 out_release_extents:
2294 }
2300 };
2303 {
2313 }
2317 {
2342 }
2347 {
2366 /*
2367 * We should have dropped this offset, so if we find it then
2368 * something has gone horribly wrong.
2369 */
2373 }
2377 u64 num_bytes;
2389 }
2392 u64 num_bytes;
2399 offset;
2405 }
2413 out:
2442 }
2445 }
2447 /*
2448 * Find a hole extent on given inode and change start/len to the end of hole
2449 * extent.(hole/vacuum extent whose em->start <= start &&
2450 * em->start + em->len > start)
2451 * When a hole extent is found, return 1 and modify start/len.
2452 */
2454 {
2465 /* Hole or vacuum extent(only exists in no-hole mode) */
2471 }
2474 }
2480 {
2488 cached_state);
2490 lockend);
2492 /*
2493 * We need to make sure we have no ordered extents in this range
2494 * and nobody raced in and read a page in this range, if we did
2495 * we need to try again.
2496 */
2505 }
2514 }
2516 }
2524 {
2541 }
2565 replace_len);
2569 /* If it's a hole, nothing more needs to be done. */
2573 }
2587 u64 ref_offset;
2596 }
2601 }
2603 /*
2604 * The respective range must have been previously locked, as well as the inode.
2605 * The end offset is inclusive (last byte of the range).
2606 * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
2607 * the file range with an extent.
2608 * When not punching a hole, we don't want to end up in a state where we dropped
2609 * extents without inserting a new one, so we must abort the transaction to avoid
2610 * a corruption.
2611 */
2616 {
2625 u64 cur_offset;
2636 }
2640 /*
2641 * 1 - update the inode
2642 * 1 - removing the extents in the range
2643 * 1 - adding the hole extent if no_holes isn't set or if we are
2644 * replacing the range with a new extent
2645 */
2648 else
2656 }
2670 /* If we are punching a hole decrement the inode's byte count */
2675 /*
2676 * When cloning we want to avoid transaction aborts when
2677 * nothing was done and we are attempting to clone parts
2678 * of inline extents, in such cases -EOPNOTSUPP is
2679 * returned by __btrfs_drop_extents() without having
2680 * changed anything in the file.
2681 */
2686 }
2695 /*
2696 * If we failed then we didn't insert our hole
2697 * entries for the area we dropped, so now the
2698 * fs is corrupted, so we must abort the
2699 * transaction.
2700 */
2703 }
2705 /*
2706 * We are past the i_size here, but since we didn't
2707 * insert holes we need to clear the mapped area so we
2708 * know to not set disk_i_size in this area until a new
2709 * file extent is inserted here.
2710 */
2712 cur_offset,
2715 /*
2716 * We couldn't clear our area, so we could
2717 * presumably adjust up and corrupt the fs, so
2718 * we need to abort.
2719 */
2722 }
2723 }
2736 }
2740 }
2756 }
2771 }
2772 }
2773 }
2775 /*
2776 * If we were cloning, force the next fsync to be a full one since we
2777 * we replaced (or just dropped in the case of cloning holes when
2778 * NO_HOLES is enabled) extents and extent maps.
2779 * This is for the sake of simplicity, and cloning into files larger
2780 * than 16Mb would force the full fsync any way (when
2781 * try_release_extent_mapping() is invoked during page cache truncation.
2782 */
2791 /*
2792 * If we are using the NO_HOLES feature we might have had already an
2793 * hole that overlaps a part of the region [lockstart, lockend] and
2794 * ends at (or beyond) lockend. Since we have no file extent items to
2795 * represent holes, drop_end can be less than lockend and so we must
2796 * make sure we have an extent map representing the existing hole (the
2797 * call to __btrfs_drop_extents() might have dropped the existing extent
2798 * map representing the existing hole), otherwise the fast fsync path
2799 * will not record the existence of the hole region
2800 * [existing_hole_start, lockend].
2801 */
2804 /*
2805 * Don't insert file hole extent item if it's for a range beyond eof
2806 * (because it's useless) or if it represents a 0 bytes range (when
2807 * cur_offset == drop_end).
2808 */
2814 /* Same comment as above. */
2817 }
2819 /* See the comment in the loop above for the reasoning here. */
2825 }
2827 }
2835 }
2836 }
2838 out_trans:
2845 else
2847 out_free:
2849 out:
2851 }
2854 {
2860 u64 lockstart;
2861 u64 lockend;
2862 u64 tail_start;
2863 u64 tail_len;
2867 u64 ino_size;
2881 /* Already in a large hole */
2884 }
2891 /*
2892 * We needn't truncate any block which is beyond the end of the file
2893 * because we are sure there is no data there.
2894 */
2895 /*
2896 * Only do this if we are in the same block and we aren't doing the
2897 * entire block.
2898 */
2906 }
2908 }
2910 /* zero back part of the first block */
2917 }
2918 }
2920 /* Check the aligned pages after the first unaligned page,
2921 * if offset != orig_start, which means the first unaligned page
2922 * including several following pages are already in holes,
2923 * the extra check can be skipped */
2925 /* after truncate page, check hole again */
2934 }
2936 }
2938 /* Check the tail unaligned part is in a hole */
2946 /* zero the front end of the last page */
2954 }
2955 }
2956 }
2961 }
2972 }
2987 out:
2990 out_only_mutex:
2992 /*
2993 * If we only end up zeroing part of a page, we still need to
2994 * update the inode item, so that all the time fields are
2995 * updated as well as the necessary btrfs inode in memory fields
2996 * for detecting, at fsync time, if the inode isn't yet in the
2997 * log tree or it's there but not up to date.
2998 */
3014 }
3015 }
3018 }
3020 /* Helper structure to record which range is already reserved */
3023 u64 start;
3024 u64 len;
3025 };
3027 /*
3028 * Helper function to add falloc range
3029 *
3030 * Caller should have locked the larger range of extent containing
3031 * [start, len)
3032 */
3034 {
3041 /*
3042 * As fallocate iterate by bytenr order, we only need to check
3043 * the last range.
3044 */
3049 }
3050 insert:
3058 }
3063 {
3083 }
3086 RANGE_BOUNDARY_WRITTEN_EXTENT,
3087 RANGE_BOUNDARY_PREALLOC_EXTENT,
3088 RANGE_BOUNDARY_HOLE,
3089 };
3092 u64 offset)
3093 {
3107 else
3112 }
3115 loff_t offset,
3116 loff_t len,
3118 {
3137 }
3139 /*
3140 * Avoid hole punching and extent allocation for some cases. More cases
3141 * could be considered, but these are unlikely common and we keep things
3142 * as simple as possible for now. Also, intentionally, if the target
3143 * range contains one or more prealloc extents together with regular
3144 * extents and holes, we drop all the existing extents and allocate a
3145 * new prealloc extent, so that we get a larger contiguous disk extent.
3146 */
3152 /*
3153 * The whole range is already a prealloc extent,
3154 * do nothing except updating the inode's i_size if
3155 * needed.
3156 */
3159 mode);
3161 }
3162 /*
3163 * Part of the range is already a prealloc extent, so operate
3164 * only on the remaining part of the range.
3165 */
3171 }
3177 sectorsize);
3181 }
3186 mode);
3188 }
3196 mode);
3198 }
3203 }
3208 /*
3209 * For unaligned ranges, check the pages at the boundaries, they might
3210 * map to an extent, in which case we need to partially zero them, or
3211 * they might map to a hole, in which case we need our allocation range
3212 * to cover them.
3213 */
3216 offset);
3228 }
3229 }
3246 }
3247 }
3249 reserve_space:
3257 bytes_to_reserve);
3275 /* btrfs_prealloc_file_range releases reserved space on error */
3279 }
3280 }
3282 out:
3289 }
3293 {
3300 u64 cur_offset;
3301 u64 last_byte;
3302 u64 alloc_start;
3303 u64 alloc_end;
3305 u64 locked_end;
3311 /* Do not allow fallocate in ZONED mode */
3319 /* Make sure we aren't being give some crap mode */
3321 FALLOC_FL_ZERO_RANGE))
3327 /*
3328 * Only trigger disk allocation, don't trigger qgroup reserve
3329 *
3330 * For qgroup space, it will be checked later.
3331 */
3337 }
3345 }
3347 /*
3348 * TODO: Move these two operations after we have checked
3349 * accurate reserved space, or fallocate can still fail but
3350 * with page truncated or size expanded.
3351 *
3352 * But that's a minor problem and won't do much harm BTW.
3353 */
3356 alloc_start);
3360 /*
3361 * If we are fallocating from the end of the file onward we
3362 * need to zero out the end of the block if i_size lands in the
3363 * middle of a block.
3364 */
3368 }
3370 /*
3371 * wait for ordered IO before we have any locks. We'll loop again
3372 * below with the locks held.
3373 */
3383 }
3389 /* the extent lock is ordered inside the running
3390 * transaction
3391 */
3395 locked_end);
3404 /*
3405 * we can't wait on the range with the transaction
3406 * running or with the extent lock held
3407 */
3416 }
3417 }
3419 /* First, check if we exceed the qgroup limit */
3427 }
3439 }
3447 }
3449 /*
3450 * Do not need to reserve unwritten extent for this
3451 * range, free reserved data space first, otherwise
3452 * it'll result in false ENOSPC error.
3453 */
3457 }
3460 }
3462 /*
3463 * If ret is still 0, means we're OK to fallocate.
3464 * Or just cleanup the list and exit.
3465 */
3472 else
3478 }
3482 /*
3483 * We didn't need to allocate any more space, but we still extended the
3484 * size of the file so we need to update i_size and the inode item.
3485 */
3487 out_unlock:
3490 out:
3492 /* Let go of our reservation. */
3498 }
3502 {
3507 u64 lockstart;
3508 u64 lockend;
3509 u64 start;
3510 u64 len;
3516 /*
3517 * offset can be negative, in this case we start finding DATA/HOLE from
3518 * the very start of the file.
3519 */
3526 lockend--;
3538 }
3553 }
3562 else
3564 }
3567 }
3570 {
3582 }
3588 }
3591 {
3594 }
3598 {
3614 }
3617 {
3619 ssize_t ret;
3629 }
3632 {
3640 }
3643 }
3657 #ifdef CONFIG_COMPAT
3659 #endif
3661 };
3664 {
3666 }
3669 {
3672 SLAB_MEM_SPREAD,
3673 NULL);
3678 }
3681 {
3684 /*
3685 * So with compression we will find and lock a dirty page and clear the
3686 * first one as dirty, setup an async extent, and immediately return
3687 * with the entire range locked but with nobody actually marked with
3688 * writeback. So we can't just filemap_write_and_wait_range() and
3689 * expect it to work since it will just kick off a thread to do the
3690 * actual work. So we need to call filemap_fdatawrite_range _again_
3691 * since it will wait on the page lock, which won't be unlocked until
3692 * after the pages have been marked as writeback and so we're good to go
3693 * from there. We have to do this otherwise we'll miss the ordered
3694 * extents and that results in badness. Please Josef, do not think you
3695 * know better and pull this out at some point in the future, it is
3696 * right and you are wrong.
3697 */
3704 }