| blob | 14e27473c5bcea8d1959ac1814babe022c7f5cf0 |
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
6 #include <linux/fs.h>
7 #include <linux/pagemap.h>
8 #include <linux/time.h>
9 #include <linux/init.h>
10 #include <linux/string.h>
11 #include <linux/backing-dev.h>
12 #include <linux/falloc.h>
13 #include <linux/writeback.h>
14 #include <linux/compat.h>
15 #include <linux/slab.h>
16 #include <linux/btrfs.h>
17 #include <linux/uio.h>
18 #include <linux/iversion.h>
19 #include <linux/fsverity.h>
40 /*
41 * Helper to fault in page and copy. This should go away and be replaced with
42 * calls into generic code.
43 */
46 {
53 /*
54 * Copy data from userspace to the current page
55 */
58 /* Flush processor's dcache for this page */
61 /*
62 * if we get a partial write, we can end up with
63 * partially up to date page. These add
64 * a lot of complexity, so make sure they don't
65 * happen by forcing this copy to be retried.
66 *
67 * The rest of the btrfs_file_write code will fall
68 * back to page at a time copies after we return 0.
69 */
74 }
77 }
82 }
84 }
86 /*
87 * Unlock folio after btrfs_file_write() is done with it.
88 */
91 {
96 /*
97 * Folio checked is some magic around finding folios that have been
98 * modified without going through btrfs_dirty_folio(). Clear it here.
99 * There should be no need to mark the pages accessed as
100 * prepare_one_folio() should have marked them accessed in
101 * prepare_one_folio() via find_or_create_page()
102 */
106 }
108 /*
109 * After btrfs_copy_from_user(), update the following things for delalloc:
110 * - Mark newly dirtied folio as DELALLOC in the io tree.
111 * Used to advise which range is to be written back.
112 * - Mark modified folio as Uptodate/Dirty and not needing COW fixup
113 * - Update inode size for past EOF write
114 */
117 {
120 u64 num_bytes;
121 u64 start_pos;
122 u64 end_of_last_block;
142 /*
143 * The pages may have already been dirty, clear out old accounting so
144 * we can set things up properly
145 */
148 cached);
159 /*
160 * we've only changed i_size in ram, and we haven't updated
161 * the disk i_size. There is no need to log the inode
162 * at this time.
163 */
167 }
169 /*
170 * this is very complex, but the basic idea is to drop all extents
171 * in the range start - end. hint_block is filled in with a block number
172 * that would be a good hint to the block allocator for this file.
173 *
174 * If an extent intersects the range but is not entirely inside the range
175 * it is either truncated or split. Anything entirely inside the range
176 * is deleted from the tree.
177 *
178 * Note: the VFS' inode number of bytes is not updated, it's up to the caller
179 * to deal with that. We set the field 'bytes_found' of the arguments structure
180 * with the number of allocated bytes found in the target range, so that the
181 * caller can update the inode's number of bytes in an atomic way when
182 * replacing extents in a range to avoid races with stat(2).
183 */
187 {
213 /* Must always have a path if ->replace_extent is true */
221 }
222 }
243 }
245 next_slot:
255 }
258 }
269 }
288 /* can't happen */
290 }
292 /*
293 * Don't skip extent items representing 0 byte lengths. They
294 * used to be created (bug) if while punching holes we hit
295 * -ENOSPC condition. So if we find one here, just ensure we
296 * delete it, otherwise we would insert a new file extent item
297 * with the same key (offset) as that 0 bytes length file
298 * extent item in the call to setup_items_for_insert() later
299 * in this function.
300 */
304 }
309 }
317 }
319 /*
320 * | - range to drop - |
321 * | -------- extent -------- |
322 */
328 }
337 }
364 };
372 }
373 }
375 }
376 /*
377 * From here on out we will have actually dropped something, so
378 * last_end can be updated.
379 */
382 /*
383 * | ---- range to drop ----- |
384 * | -------- extent -------- |
385 */
390 }
404 }
407 /*
408 * | ---- range to drop ----- |
409 * | -------- extent -------- |
410 */
416 }
428 }
430 /*
431 * | ---- range to drop ----- |
432 * | ------ extent ------ |
433 */
435 delete_extent_item:
441 del_nr++;
442 }
457 };
465 }
467 }
475 }
478 del_nr);
482 }
489 }
492 }
495 /*
496 * Set path->slots[0] to first slot, so that after the delete
497 * if items are move off from our leaf to its immediate left or
498 * right neighbor leafs, we end up with a correct and adjusted
499 * path->slots[0] for our insertion (if args->replace_extent).
500 */
505 }
508 /*
509 * If btrfs_del_items() was called, it might have deleted a leaf, in
510 * which case it unlocked our path, so check path->locks[0] matches a
511 * write lock.
512 */
527 }
531 }
537 out:
541 }
546 {
549 u64 extent_end;
574 }
576 /*
577 * Mark extent in the range start - end as written.
578 *
579 * This changes extent type from 'pre-allocated' to 'regular'. If only
580 * part of extent is marked as written, the extent will be split into
581 * two or three.
582 */
585 {
593 u64 bytenr;
594 u64 num_bytes;
595 u64 extent_end;
596 u64 orig_offset;
597 u64 other_start;
598 u64 other_end;
599 u64 split;
609 again:
629 }
636 }
642 }
673 }
674 }
702 }
703 }
714 }
718 }
747 }
756 }
759 }
761 }
779 }
782 del_nr++;
787 }
788 }
797 }
800 del_nr++;
805 }
806 }
811 BTRFS_FILE_EXTENT_REG);
818 BTRFS_FILE_EXTENT_REG);
828 }
829 }
830 out:
833 }
835 /*
836 * On error return an unlocked folio and the error value
837 * On success return a locked folio and 0
838 */
841 {
861 }
863 /*
864 * Since btrfs_read_folio() will unlock the folio before it returns,
865 * there is a window where btrfs_release_folio() can be called to
866 * release the page. Here we check both inode mapping and page
867 * private to make sure the page was not released.
868 *
869 * The private flag check is essential for subpage as we need to store
870 * extra bitmap using folio private.
871 */
875 }
877 }
880 {
881 gfp_t gfp;
887 }
890 }
892 /*
893 * Get folio into the page cache and lock it.
894 */
898 {
905 again:
910 else
913 }
915 /* Only support page sized folio yet. */
922 }
925 /* The folio is already unlocked. */
930 }
932 }
935 }
937 /*
938 * Locks the extent and properly waits for data=ordered extents to finish
939 * before allowing the folios to be modified if need.
940 *
941 * Return:
942 * 1 - the extent is locked
943 * 0 - the extent is not locked, and everything is OK
944 * -EAGAIN - need to prepare the folios again
945 */
951 {
953 u64 start_pos;
954 u64 last_pos;
965 cached_state)) {
969 }
972 }
980 cached_state);
986 }
993 }
995 /*
996 * We should be called after prepare_one_folio() which should have locked
997 * all pages in the range.
998 */
1002 }
1004 /*
1005 * Check if we can do nocow write into the range [@pos, @pos + @write_bytes)
1006 *
1007 * @pos: File offset.
1008 * @write_bytes: The length to write, will be updated to the nocow writeable
1009 * range.
1010 *
1011 * This function will flush ordered extents in the range to ensure proper
1012 * nocow checks.
1013 *
1014 * Return:
1015 * > 0 If we can nocow, and updates @write_bytes.
1016 * 0 If we can't do a nocow write.
1017 * -EAGAIN If we can't do a nocow write because snapshoting of the inode's
1018 * root is in progress.
1019 * < 0 If an error happened.
1020 *
1021 * NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0.
1022 */
1025 {
1030 u64 num_bytes;
1049 }
1053 }
1058 else
1064 }
1067 {
1069 }
1072 {
1078 loff_t oldsize;
1079 loff_t start_pos;
1081 /*
1082 * Quickly bail out on NOWAIT writes if we don't have the nodatacow or
1083 * prealloc flags, as without those flags we always have to COW. We will
1084 * later check if we can really COW into the target range (using
1085 * can_nocow_extent() at btrfs_get_blocks_direct_write()).
1086 */
1095 /*
1096 * We reserve space for updating the inode when we reserve space for the
1097 * extent we are going to write, so we will enospc out there. We don't
1098 * need to start yet another transaction to update the inode as we will
1099 * update the inode when we finish writing whatever data we write.
1100 */
1104 }
1109 /* Expand hole size to cover write data, preventing empty gap */
1115 }
1118 }
1121 {
1123 loff_t pos;
1128 u64 lockstart;
1129 u64 lockend;
1131 ssize_t ret;
1167 /*
1168 * Fault pages before locking them in prepare_one_folio()
1169 * to avoid recursive lock
1170 */
1174 }
1189 }
1191 /*
1192 * If we don't have to COW at the offset, reserve
1193 * metadata only. write_bytes may get smaller than
1194 * requested here.
1195 */
1205 }
1211 reserve_bytes,
1217 write_bytes);
1218 else
1224 }
1227 again:
1232 }
1238 reserve_bytes);
1240 }
1250 reserve_bytes);
1253 }
1267 }
1270 /* release everything except the sectors we dirtied */
1281 }
1282 }
1290 /*
1291 * If we have not locked the extent range, because the range's
1292 * start offset is >= i_size, we might still have a non-NULL
1293 * cached extent state, acquired while marking the extent range
1294 * as delalloc through btrfs_dirty_page(). Therefore free any
1295 * possible cached extent state to avoid a memory leak.
1296 */
1300 else
1307 }
1319 }
1328 data_reserved,
1331 }
1332 }
1338 }
1339 out:
1342 }
1346 {
1349 loff_t count;
1350 ssize_t ret;
1356 /*
1357 * The write got truncated by generic_write_checks_count(). We
1358 * can't do a partial encoded write.
1359 */
1361 }
1370 out:
1373 }
1377 {
1382 /*
1383 * If the fs flips readonly due to some impossible error, although we
1384 * have opened a file as writable, we have to stop this write operation
1385 * to ensure consistency.
1386 */
1402 }
1410 }
1413 }
1416 {
1418 }
1421 {
1429 }
1431 /*
1432 * Set by setattr when we are about to truncate a file from a non-zero
1433 * size to a zero size. This tries to flush down new bytes that may
1434 * have been written if the application were using truncate to replace
1435 * a file in place.
1436 */
1441 }
1444 {
1448 /*
1449 * This is only called in fsync, which would do synchronous writes, so
1450 * a plug can merge adjacent IOs as much as possible. Esp. in case of
1451 * multiple disks using raid profile, a large IO can be split to
1452 * several segments of stripe length (currently 64K).
1453 */
1459 }
1462 {
1470 /*
1471 * If we are doing a fast fsync we can not bail out if the inode's
1472 * last_trans is <= then the last committed transaction, because we only
1473 * update the last_trans of the inode during ordered extent completion,
1474 * and for a fast fsync we don't wait for that, we only wait for the
1475 * writeback to complete.
1476 */
1483 }
1485 /*
1486 * fsync call for both files and directories. This logs the inode into
1487 * the tree log instead of forcing full commits whenever possible.
1488 *
1489 * It needs to call filemap_fdatawait so that all ordered extent updates are
1490 * in the metadata btree are up to date for copying to the log.
1491 *
1492 * It drops the inode mutex before doing the tree log commit. This is an
1493 * important optimization for directories because holding the mutex prevents
1494 * new operations on the dir while we write to disk.
1495 */
1497 {
1505 u64 len;
1513 }
1519 /*
1520 * Always set the range to a full range, otherwise we can get into
1521 * several problems, from missing file extent items to represent holes
1522 * when not using the NO_HOLES feature, to log tree corruption due to
1523 * races between hole detection during logging and completion of ordered
1524 * extents outside the range, to missing checksums due to ordered extents
1525 * for which we flushed only a subset of their pages.
1526 */
1531 /*
1532 * We write the dirty pages in the range and wait until they complete
1533 * out of the ->i_mutex. If so, we can flush the dirty pages by
1534 * multi-task, and make the performance up. See
1535 * btrfs_wait_ordered_range for an explanation of the ASYNC check.
1536 */
1543 else
1548 /*
1549 * Before we acquired the inode's lock and the mmap lock, someone may
1550 * have dirtied more pages in the target range. We need to make sure
1551 * that writeback for any such pages does not start while we are logging
1552 * the inode, because if it does, any of the following might happen when
1553 * we are not doing a full inode sync:
1554 *
1555 * 1) We log an extent after its writeback finishes but before its
1556 * checksums are added to the csum tree, leading to -EIO errors
1557 * when attempting to read the extent after a log replay.
1558 *
1559 * 2) We can end up logging an extent before its writeback finishes.
1560 * Therefore after the log replay we will have a file extent item
1561 * pointing to an unwritten extent (and no data checksums as well).
1562 *
1563 * So trigger writeback for any eventual new dirty pages and then we
1564 * wait for all ordered extents to complete below.
1565 */
1570 else
1573 }
1575 /*
1576 * Always check for the full sync flag while holding the inode's lock,
1577 * to avoid races with other tasks. The flag must be either set all the
1578 * time during logging or always off all the time while logging.
1579 * We check the flag here after starting delalloc above, because when
1580 * running delalloc the full sync flag may be set if we need to drop
1581 * extra extent map ranges due to temporary memory allocation failures.
1582 */
1585 /*
1586 * We have to do this here to avoid the priority inversion of waiting on
1587 * IO of a lower priority task while holding a transaction open.
1588 *
1589 * For a full fsync we wait for the ordered extents to complete while
1590 * for a fast fsync we wait just for writeback to complete, and then
1591 * attach the ordered extents to the transaction so that a transaction
1592 * commit waits for their completion, to avoid data loss if we fsync,
1593 * the current transaction commits before the ordered extents complete
1594 * and a power failure happens right after that.
1595 *
1596 * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the
1597 * logical address recorded in the ordered extent may change. We need
1598 * to wait for the IO to stabilize the logical address.
1599 */
1604 /*
1605 * Get our ordered extents as soon as possible to avoid doing
1606 * checksum lookups in the csum tree, and use instead the
1607 * checksums attached to the ordered extents.
1608 */
1614 /*
1615 * Check and clear the BTRFS_INODE_COW_WRITE_ERROR now after
1616 * starting and waiting for writeback, because for buffered IO
1617 * it may have been set during the end IO callback
1618 * (end_bbio_data_write() -> btrfs_finish_ordered_extent()) in
1619 * case an error happened and we need to wait for ordered
1620 * extents to complete so that any extent maps that point to
1621 * unwritten locations are dropped and we don't log them.
1622 */
1625 }
1633 /*
1634 * We've had everything committed since the last time we were
1635 * modified so clear this flag in case it was set for whatever
1636 * reason, it's no longer relevant.
1637 */
1639 /*
1640 * An ordered extent might have started before and completed
1641 * already with io errors, in which case the inode was not
1642 * updated and we end up here. So check the inode's mapping
1643 * for any errors that might have happened since we last
1644 * checked called fsync.
1645 */
1648 }
1652 /*
1653 * We use start here because we will need to wait on the IO to complete
1654 * in btrfs_sync_log, which could require joining a transaction (for
1655 * example checking cross references in the nocow path). If we use join
1656 * here we could get into a situation where we're waiting on IO to
1657 * happen that is blocked on a transaction trying to commit. With start
1658 * we inc the extwriter counter, so we wait for all extwriters to exit
1659 * before we start blocking joiners. This comment is to keep somebody
1660 * from thinking they are super smart and changing this to
1661 * btrfs_join_transaction *cough*Josef*cough*.
1662 */
1667 }
1671 /*
1672 * Scratch eb no longer needed, release before syncing log or commit
1673 * transaction, to avoid holding unnecessary memory during such long
1674 * operations.
1675 */
1679 }
1682 /* Fallthrough and commit/free transaction. */
1684 }
1686 /* we've logged all the items and now have a consistent
1687 * version of the file in the log. It is possible that
1688 * someone will come in and modify the file, but that's
1689 * fine because the log is consistent on disk, and we
1690 * have references to all of the file's extents
1691 *
1692 * It is possible that someone will come in and log the
1693 * file again, but that will end up using the synchronization
1694 * inside btrfs_sync_log to keep things safe.
1695 */
1698 else
1704 }
1706 /* We successfully logged the inode, attempt to sync the log. */
1712 }
1713 }
1715 /*
1716 * At this point we need to commit the transaction because we had
1717 * btrfs_need_log_full_commit() or some other error.
1718 *
1719 * If we didn't do a full sync we have to stop the trans handle, wait on
1720 * the ordered extents, start it again and commit the transaction. If
1721 * we attempt to wait on the ordered extents here we could deadlock with
1722 * something like fallocate() that is holding the extent lock trying to
1723 * start a transaction while some other thread is trying to commit the
1724 * transaction while we (fsync) are currently holding the transaction
1725 * open.
1726 */
1735 /*
1736 * This is safe to use here because we're only interested in
1737 * making sure the transaction that had the ordered extents is
1738 * committed. We aren't waiting on anything past this point,
1739 * we're purely getting the transaction and committing it.
1740 */
1745 /*
1746 * We committed the transaction and there's no currently
1747 * running transaction, this means everything we care
1748 * about made it to disk and we are done.
1749 */
1753 }
1754 }
1757 out:
1766 out_release_extents:
1770 else
1773 }
1775 /*
1776 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
1777 * called from a page fault handler when a page is first dirtied. Hence we must
1778 * be careful to check for EOF conditions here. We set the page up correctly
1779 * for a written page which means we get ENOSPC checking when writing into
1780 * holes and correct delalloc and unwritten extent mapping on filesystems that
1781 * support these features.
1782 *
1783 * We are not allowed to take the i_mutex here so we have to play games to
1784 * protect against truncate races as the page could now be beyond EOF. Because
1785 * truncate_setsize() writes the inode size before removing pages, once we have
1786 * the page lock we can determine safely if the page is beyond EOF. If it is not
1787 * beyond EOF, then the page is guaranteed safe against truncation until we
1788 * unlock the page.
1789 */
1791 {
1801 loff_t size;
1802 vm_fault_t ret;
1805 u64 reserved_space;
1806 u64 page_start;
1807 u64 page_end;
1808 u64 end;
1819 /*
1820 * Reserving delalloc space after obtaining the page lock can lead to
1821 * deadlock. For example, if a dirty page is locked by this function
1822 * and the call to btrfs_delalloc_reserve_space() ends up triggering
1823 * dirty page write out, then the btrfs_writepages() function could
1824 * end up waiting indefinitely to get a lock on the page currently
1825 * being processed by btrfs_page_mkwrite() function.
1826 */
1832 }
1838 }
1840 /* Make the VM retry the fault. */
1842 again:
1849 /* Page got truncated out from underneath us. */
1851 }
1860 }
1862 /*
1863 * We can't set the delalloc bits if there are pending ordered
1864 * extents. Drop our locks and wait for them to finish.
1865 */
1874 }
1883 }
1884 }
1886 /*
1887 * page_mkwrite gets called when the page is firstly dirtied after it's
1888 * faulted in, but write(2) could also dirty a page and set delalloc
1889 * bits, thus in this case for space account reason, we still need to
1890 * clear any delalloc bits within this page range since we have to
1891 * reserve data&meta space before lock_page() (see above comments).
1892 */
1903 }
1905 /* Page is wholly or partially inside EOF. */
1908 else
1928 out_unlock:
1931 out:
1935 out_noreserve:
1939 }
1945 };
1948 {
1958 }
1962 {
1987 }
1992 {
2010 /*
2011 * We should have dropped this offset, so if we find it then
2012 * something has gone horribly wrong.
2013 */
2017 }
2021 u64 num_bytes;
2034 }
2037 u64 num_bytes;
2044 offset;
2051 }
2059 out:
2079 }
2082 }
2084 /*
2085 * Find a hole extent on given inode and change start/len to the end of hole
2086 * extent.(hole/vacuum extent whose em->start <= start &&
2087 * em->start + em->len > start)
2088 * When a hole extent is found, return 1 and modify start/len.
2089 */
2091 {
2102 /* Hole or vacuum extent(only exists in no-hole mode) */
2108 }
2111 }
2117 {
2118 /*
2119 * For subpage case, if the range is not at page boundary, we could
2120 * have pages at the leading/tailing part of the range.
2121 * This could lead to dead loop since filemap_range_has_page()
2122 * will always return true.
2123 * So here we need to do extra page alignment for
2124 * filemap_range_has_page().
2125 */
2133 cached_state);
2134 /*
2135 * We can't have ordered extents in the range, nor dirty/writeback
2136 * pages, because we have locked the inode's VFS lock in exclusive
2137 * mode, we have locked the inode's i_mmap_lock in exclusive mode,
2138 * we have flushed all delalloc in the range and we have waited
2139 * for any ordered extents in the range to complete.
2140 * We can race with anyone reading pages from this range, so after
2141 * locking the range check if we have pages in the range, and if
2142 * we do, unlock the range and retry.
2143 */
2145 page_lockend))
2149 cached_state);
2150 }
2153 }
2161 {
2177 }
2201 replace_len);
2205 /* If it's a hole, nothing more needs to be done. */
2209 }
2229 };
2230 u64 ref_offset;
2235 }
2240 }
2242 /*
2243 * The respective range must have been previously locked, as well as the inode.
2244 * The end offset is inclusive (last byte of the range).
2245 * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
2246 * the file range with an extent.
2247 * When not punching a hole, we don't want to end up in a state where we dropped
2248 * extents without inserting a new one, so we must abort the transaction to avoid
2249 * a corruption.
2250 */
2256 {
2265 u64 cur_offset;
2276 }
2280 /*
2281 * 1 - update the inode
2282 * 1 - removing the extents in the range
2283 * 1 - adding the hole extent if no_holes isn't set or if we are
2284 * replacing the range with a new extent
2285 */
2288 else
2296 }
2311 /* If we are punching a hole decrement the inode's byte count */
2316 /*
2317 * The only time we don't want to abort is if we are
2318 * attempting to clone a partial inline extent, in which
2319 * case we'll get EOPNOTSUPP. However if we aren't
2320 * clone we need to abort no matter what, because if we
2321 * got EOPNOTSUPP via prealloc then we messed up and
2322 * need to abort.
2323 */
2329 }
2338 /*
2339 * If we failed then we didn't insert our hole
2340 * entries for the area we dropped, so now the
2341 * fs is corrupted, so we must abort the
2342 * transaction.
2343 */
2346 }
2348 /*
2349 * We are past the i_size here, but since we didn't
2350 * insert holes we need to clear the mapped area so we
2351 * know to not set disk_i_size in this area until a new
2352 * file extent is inserted here.
2353 */
2355 cur_offset,
2358 /*
2359 * We couldn't clear our area, so we could
2360 * presumably adjust up and corrupt the fs, so
2361 * we need to abort.
2362 */
2365 }
2366 }
2379 }
2383 }
2385 /*
2386 * We are releasing our handle on the transaction, balance the
2387 * dirty pages of the btree inode and flush delayed items, and
2388 * then get a new transaction handle, which may now point to a
2389 * new transaction in case someone else may have committed the
2390 * transaction we used to replace/drop file extent items. So
2391 * bump the inode's iversion and update mtime and ctime except
2392 * if we are called from a dedupe context. This is because a
2393 * power failure/crash may happen after the transaction is
2394 * committed and before we finish replacing/dropping all the
2395 * file extent items we need.
2396 */
2415 }
2432 }
2433 }
2434 }
2436 /*
2437 * If we were cloning, force the next fsync to be a full one since we
2438 * we replaced (or just dropped in the case of cloning holes when
2439 * NO_HOLES is enabled) file extent items and did not setup new extent
2440 * maps for the replacement extents (or holes).
2441 */
2449 /*
2450 * If we are using the NO_HOLES feature we might have had already an
2451 * hole that overlaps a part of the region [lockstart, lockend] and
2452 * ends at (or beyond) lockend. Since we have no file extent items to
2453 * represent holes, drop_end can be less than lockend and so we must
2454 * make sure we have an extent map representing the existing hole (the
2455 * call to __btrfs_drop_extents() might have dropped the existing extent
2456 * map representing the existing hole), otherwise the fast fsync path
2457 * will not record the existence of the hole region
2458 * [existing_hole_start, lockend].
2459 */
2462 /*
2463 * Don't insert file hole extent item if it's for a range beyond eof
2464 * (because it's useless) or if it represents a 0 bytes range (when
2465 * cur_offset == drop_end).
2466 */
2472 /* Same comment as above. */
2475 }
2477 /* See the comment in the loop above for the reasoning here. */
2483 }
2485 }
2493 }
2494 }
2496 out_trans:
2503 else
2505 out_free:
2507 out:
2509 }
2512 {
2519 u64 lockstart;
2520 u64 lockend;
2521 u64 tail_start;
2522 u64 tail_len;
2526 u64 ino_size;
2541 /* Already in a large hole */
2544 }
2554 /*
2555 * We needn't truncate any block which is beyond the end of the file
2556 * because we are sure there is no data there.
2557 */
2558 /*
2559 * Only do this if we are in the same block and we aren't doing the
2560 * entire block.
2561 */
2569 }
2571 }
2573 /* zero back part of the first block */
2580 }
2581 }
2583 /* Check the aligned pages after the first unaligned page,
2584 * if offset != orig_start, which means the first unaligned page
2585 * including several following pages are already in holes,
2586 * the extra check can be skipped */
2588 /* after truncate page, check hole again */
2597 }
2599 }
2601 /* Check the tail unaligned part is in a hole */
2609 /* zero the front end of the last page */
2617 }
2618 }
2619 }
2624 }
2632 }
2647 out:
2650 out_only_mutex:
2652 /*
2653 * If we only end up zeroing part of a page, we still need to
2654 * update the inode item, so that all the time fields are
2655 * updated as well as the necessary btrfs inode in memory fields
2656 * for detecting, at fsync time, if the inode isn't yet in the
2657 * log tree or it's there but not up to date.
2658 */
2673 }
2674 }
2677 }
2679 /* Helper structure to record which range is already reserved */
2682 u64 start;
2683 u64 len;
2684 };
2686 /*
2687 * Helper function to add falloc range
2688 *
2689 * Caller should have locked the larger range of extent containing
2690 * [start, len)
2691 */
2693 {
2697 /*
2698 * As fallocate iterates by bytenr order, we only need to check
2699 * the last range.
2700 */
2705 }
2706 }
2715 }
2720 {
2740 }
2743 RANGE_BOUNDARY_WRITTEN_EXTENT,
2744 RANGE_BOUNDARY_PREALLOC_EXTENT,
2745 RANGE_BOUNDARY_HOLE,
2746 };
2749 u64 offset)
2750 {
2764 else
2769 }
2772 loff_t offset,
2773 loff_t len,
2775 {
2792 }
2794 /*
2795 * Avoid hole punching and extent allocation for some cases. More cases
2796 * could be considered, but these are unlikely common and we keep things
2797 * as simple as possible for now. Also, intentionally, if the target
2798 * range contains one or more prealloc extents together with regular
2799 * extents and holes, we drop all the existing extents and allocate a
2800 * new prealloc extent, so that we get a larger contiguous disk extent.
2801 */
2806 /*
2807 * The whole range is already a prealloc extent,
2808 * do nothing except updating the inode's i_size if
2809 * needed.
2810 */
2813 mode);
2815 }
2816 /*
2817 * Part of the range is already a prealloc extent, so operate
2818 * only on the remaining part of the range.
2819 */
2825 }
2834 }
2839 mode);
2841 }
2849 mode);
2851 }
2856 }
2861 /*
2862 * For unaligned ranges, check the pages at the boundaries, they might
2863 * map to an extent, in which case we need to partially zero them, or
2864 * they might map to a hole, in which case we need our allocation range
2865 * to cover them.
2866 */
2869 offset);
2881 }
2882 }
2899 }
2900 }
2902 reserve_space:
2910 bytes_to_reserve);
2922 }
2929 /* btrfs_prealloc_file_range releases reserved space on error */
2933 }
2934 }
2936 out:
2943 }
2947 {
2954 u64 cur_offset;
2955 u64 last_byte;
2956 u64 alloc_start;
2957 u64 alloc_end;
2959 u64 locked_end;
2968 /* Do not allow fallocate in ZONED mode */
2976 /* Make sure we aren't being give some crap mode */
2978 FALLOC_FL_ZERO_RANGE))
2990 }
2996 /*
2997 * TODO: Move these two operations after we have checked
2998 * accurate reserved space, or fallocate can still fail but
2999 * with page truncated or size expanded.
3000 *
3001 * But that's a minor problem and won't do much harm BTW.
3002 */
3005 alloc_start);
3009 /*
3010 * If we are fallocating from the end of the file onward we
3011 * need to zero out the end of the block if i_size lands in the
3012 * middle of a block.
3013 */
3017 }
3019 /*
3020 * We have locked the inode at the VFS level (in exclusive mode) and we
3021 * have locked the i_mmap_lock lock (in exclusive mode). Now before
3022 * locking the file range, flush all dealloc in the range and wait for
3023 * all ordered extents in the range to complete. After this we can lock
3024 * the file range and, due to the previous locking we did, we know there
3025 * can't be more delalloc or ordered extents in the range.
3026 */
3036 }
3044 /* First, check if we exceed the qgroup limit */
3051 }
3064 }
3070 }
3073 }
3076 }
3079 /*
3080 * We are safe to reserve space here as we can't have delalloc
3081 * in the range, see above.
3082 */
3084 data_space_needed);
3087 }
3089 /*
3090 * If ret is still 0, means we're OK to fallocate.
3091 * Or just cleanup the list and exit.
3092 */
3099 /*
3100 * btrfs_prealloc_file_range() releases space even
3101 * if it returns an error.
3102 */
3115 }
3118 }
3122 /*
3123 * We didn't need to allocate any more space, but we still extended the
3124 * size of the file so we need to update i_size and the inode item.
3125 */
3127 out_unlock:
3130 out:
3134 }
3136 /*
3137 * Helper for btrfs_find_delalloc_in_range(). Find a subrange in a given range
3138 * that has unflushed and/or flushing delalloc. There might be other adjacent
3139 * subranges after the one it found, so btrfs_find_delalloc_in_range() keeps
3140 * looping while it gets adjacent subranges, and merging them together.
3141 */
3146 {
3150 u64 oe_start;
3151 u64 oe_end;
3153 /*
3154 * Search the io tree first for EXTENT_DELALLOC. If we find any, it
3155 * means we have delalloc (dirty pages) for which writeback has not
3156 * started yet.
3157 */
3166 cached_state);
3169 }
3170 }
3173 /*
3174 * If delalloc was found then *delalloc_start_ret has a sector size
3175 * aligned value (rounded down).
3176 */
3180 /* Delalloc for the whole range, nothing more to do. */
3183 /* Else trim our search range for ordered extents. */
3186 }
3188 /* No delalloc, future calls don't need to search again. */
3190 }
3192 /*
3193 * Now also check if there's any ordered extent in the range.
3194 * We do this because:
3195 *
3196 * 1) When delalloc is flushed, the file range is locked, we clear the
3197 * EXTENT_DELALLOC bit from the io tree and create an extent map and
3198 * an ordered extent for the write. So we might just have been called
3199 * after delalloc is flushed and before the ordered extent completes
3200 * and inserts the new file extent item in the subvolume's btree;
3201 *
3202 * 2) We may have an ordered extent created by flushing delalloc for a
3203 * subrange that starts before the subrange we found marked with
3204 * EXTENT_DELALLOC in the io tree.
3205 *
3206 * We could also use the extent map tree to find such delalloc that is
3207 * being flushed, but using the ordered extents tree is more efficient
3208 * because it's usually much smaller as ordered extents are removed from
3209 * the tree once they complete. With the extent maps, we mau have them
3210 * in the extent map tree for a very long time, and they were either
3211 * created by previous writes or loaded by read operations.
3212 */
3217 /* The ordered extent may span beyond our search range. */
3223 /* Don't have unflushed delalloc, return the ordered extent range. */
3228 }
3230 /*
3231 * We have both unflushed delalloc (io_tree) and an ordered extent.
3232 * If the ranges are adjacent returned a combined range, otherwise
3233 * return the leftmost range.
3234 */
3241 }
3244 }
3246 /*
3247 * Check if there's delalloc in a given range.
3248 *
3249 * @inode: The inode.
3250 * @start: The start offset of the range. It does not need to be
3251 * sector size aligned.
3252 * @end: The end offset (inclusive value) of the search range.
3253 * It does not need to be sector size aligned.
3254 * @cached_state: Extent state record used for speeding up delalloc
3255 * searches in the inode's io_tree. Can be NULL.
3256 * @delalloc_start_ret: Output argument, set to the start offset of the
3257 * subrange found with delalloc (may not be sector size
3258 * aligned).
3259 * @delalloc_end_ret: Output argument, set to he end offset (inclusive value)
3260 * of the subrange found with delalloc.
3261 *
3262 * Returns true if a subrange with delalloc is found within the given range, and
3263 * if so it sets @delalloc_start_ret and @delalloc_end_ret with the start and
3264 * end offsets of the subrange.
3265 */
3269 {
3276 u64 delalloc_start;
3277 u64 delalloc_end;
3288 /* First subrange found. */
3293 /* Subrange adjacent to the previous one, merge them. */
3296 /* Subrange not adjacent to the previous one, exit. */
3298 }
3303 }
3306 }
3308 /*
3309 * Check if there's a hole or delalloc range in a range representing a hole (or
3310 * prealloc extent) found in the inode's subvolume btree.
3311 *
3312 * @inode: The inode.
3313 * @whence: Seek mode (SEEK_DATA or SEEK_HOLE).
3314 * @start: Start offset of the hole region. It does not need to be sector
3315 * size aligned.
3316 * @end: End offset (inclusive value) of the hole region. It does not
3317 * need to be sector size aligned.
3318 * @start_ret: Return parameter, used to set the start of the subrange in the
3319 * hole that matches the search criteria (seek mode), if such
3320 * subrange is found (return value of the function is true).
3321 * The value returned here may not be sector size aligned.
3322 *
3323 * Returns true if a subrange matching the given seek mode is found, and if one
3324 * is found, it updates @start_ret with the start of the subrange.
3325 */
3329 {
3330 u64 delalloc_start;
3331 u64 delalloc_end;
3339 }
3342 /*
3343 * We found delalloc but it starts after out start offset. So we
3344 * have a hole between our start offset and the delalloc start.
3345 */
3349 }
3350 /*
3351 * Delalloc range starts at our start offset.
3352 * If the delalloc range's length is smaller than our range,
3353 * then it means we have a hole that starts where the delalloc
3354 * subrange ends.
3355 */
3359 }
3361 /* There's delalloc for the whole range. */
3363 }
3368 }
3370 /*
3371 * No delalloc in the range and we are seeking for data. The caller has
3372 * to iterate to the next extent item in the subvolume btree.
3373 */
3375 }
3378 {
3389 u64 last_extent_end;
3390 u64 lockstart;
3391 u64 lockend;
3392 u64 start;
3399 /*
3400 * Quick path. If the inode has no prealloc extents and its number of
3401 * bytes used matches its i_size, then it can not have holes.
3402 */
3413 /*
3414 * Not allocated by us, don't use it as its cached state is used
3415 * by the task that allocated it and we don't want neither to
3416 * mess with it nor get incorrect results because it reflects an
3417 * invalid state for the current task.
3418 */
3422 /*
3423 * No worries if memory allocation failed.
3424 * The private structure is used only for speeding up multiple
3425 * lseek SEEK_HOLE/DATA calls to a file when there's delalloc,
3426 * so everything will still be correct.
3427 */
3436 else
3443 }
3444 }
3445 }
3449 else
3452 /*
3453 * offset can be negative, in this case we start finding DATA/HOLE from
3454 * the very start of the file.
3455 */
3462 lockend--;
3484 }
3489 u64 extent_end;
3490 u8 type;
3500 }
3508 /*
3509 * In the first iteration we may have a slot that points to an
3510 * extent that ends before our start offset, so skip it.
3511 */
3515 }
3517 /* We have an implicit hole, NO_HOLES feature is likely set. */
3520 u64 found_start;
3522 /*
3523 * First iteration, @start matches @offset and it's
3524 * within the hole.
3525 */
3530 delalloc_cached_state,
3531 search_start,
3537 }
3538 /*
3539 * Didn't find data or a hole (due to delalloc) in the
3540 * implicit hole range, so need to analyze the extent.
3541 */
3542 }
3548 /*
3549 * Can't access the extent's disk_bytenr field if this is an
3550 * inline extent, since at that offset, it's where the extent
3551 * data starts.
3552 */
3556 /*
3557 * Explicit hole or prealloc extent, search for delalloc.
3558 * A prealloc extent is treated like a hole.
3559 */
3561 u64 found_start;
3563 /*
3564 * First iteration, @start matches @offset and it's
3565 * within the hole.
3566 */
3571 delalloc_cached_state,
3572 search_start,
3578 }
3579 /*
3580 * Didn't find data or a hole (due to delalloc) in the
3581 * implicit hole range, so need to analyze the next
3582 * extent item.
3583 */
3585 /*
3586 * Found a regular or inline extent.
3587 * If we are seeking for data, adjust the start offset
3588 * and stop, we're done.
3589 */
3594 }
3595 /*
3596 * Else, we are seeking for a hole, check the next file
3597 * extent item.
3598 */
3599 }
3607 }
3609 }
3611 /* We have an implicit hole from the last extent found up to i_size. */
3618 }
3620 out:
3631 }
3634 {
3646 }
3652 }
3655 {
3664 }
3667 {
3675 }
3678 }
3693 #ifdef CONFIG_COMPAT
3695 #endif
3699 };
3702 {
3706 /*
3707 * So with compression we will find and lock a dirty page and clear the
3708 * first one as dirty, setup an async extent, and immediately return
3709 * with the entire range locked but with nobody actually marked with
3710 * writeback. So we can't just filemap_write_and_wait_range() and
3711 * expect it to work since it will just kick off a thread to do the
3712 * actual work. So we need to call filemap_fdatawrite_range _again_
3713 * since it will wait on the page lock, which won't be unlocked until
3714 * after the pages have been marked as writeback and so we're good to go
3715 * from there. We have to do this otherwise we'll miss the ordered
3716 * extents and that results in badness. Please Josef, do not think you
3717 * know better and pull this out at some point in the future, it is
3718 * right and you are wrong.
3719 */
3725 }