| blob | 588c353d296935d6cc100e5d48d1a1d8671b3d8c |
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
6 #include <linux/fs.h>
7 #include <linux/pagemap.h>
8 #include <linux/time.h>
9 #include <linux/init.h>
10 #include <linux/string.h>
11 #include <linux/backing-dev.h>
12 #include <linux/falloc.h>
13 #include <linux/writeback.h>
14 #include <linux/compat.h>
15 #include <linux/slab.h>
16 #include <linux/btrfs.h>
17 #include <linux/uio.h>
18 #include <linux/iversion.h>
19 #include <linux/fsverity.h>
40 /*
41 * Helper to fault in page and copy. This should go away and be replaced with
42 * calls into generic code.
43 */
46 {
53 /*
54 * Copy data from userspace to the current page
55 */
58 /* Flush processor's dcache for this page */
61 /*
62 * if we get a partial write, we can end up with
63 * partially up to date page. These add
64 * a lot of complexity, so make sure they don't
65 * happen by forcing this copy to be retried.
66 *
67 * The rest of the btrfs_file_write code will fall
68 * back to page at a time copies after we return 0.
69 */
74 }
77 }
82 }
84 }
86 /*
87 * Unlock folio after btrfs_file_write() is done with it.
88 */
91 {
96 /*
97 * Folio checked is some magic around finding folios that have been
98 * modified without going through btrfs_dirty_folio(). Clear it here.
99 * There should be no need to mark the pages accessed as
100 * prepare_one_folio() should have marked them accessed in
101 * prepare_one_folio() via find_or_create_page()
102 */
106 }
108 /*
109 * After btrfs_copy_from_user(), update the following things for delalloc:
110 * - Mark newly dirtied folio as DELALLOC in the io tree.
111 * Used to advise which range is to be written back.
112 * - Mark modified folio as Uptodate/Dirty and not needing COW fixup
113 * - Update inode size for past EOF write
114 */
117 {
120 u64 num_bytes;
121 u64 start_pos;
122 u64 end_of_last_block;
142 /*
143 * The pages may have already been dirty, clear out old accounting so
144 * we can set things up properly
145 */
148 cached);
159 /*
160 * we've only changed i_size in ram, and we haven't updated
161 * the disk i_size. There is no need to log the inode
162 * at this time.
163 */
167 }
169 /*
170 * this is very complex, but the basic idea is to drop all extents
171 * in the range start - end. hint_block is filled in with a block number
172 * that would be a good hint to the block allocator for this file.
173 *
174 * If an extent intersects the range but is not entirely inside the range
175 * it is either truncated or split. Anything entirely inside the range
176 * is deleted from the tree.
177 *
178 * Note: the VFS' inode number of bytes is not updated, it's up to the caller
179 * to deal with that. We set the field 'bytes_found' of the arguments structure
180 * with the number of allocated bytes found in the target range, so that the
181 * caller can update the inode's number of bytes in an atomic way when
182 * replacing extents in a range to avoid races with stat(2).
183 */
187 {
213 /* Must always have a path if ->replace_extent is true */
221 }
222 }
243 }
245 next_slot:
255 }
258 }
269 }
288 /* can't happen */
290 }
292 /*
293 * Don't skip extent items representing 0 byte lengths. They
294 * used to be created (bug) if while punching holes we hit
295 * -ENOSPC condition. So if we find one here, just ensure we
296 * delete it, otherwise we would insert a new file extent item
297 * with the same key (offset) as that 0 bytes length file
298 * extent item in the call to setup_items_for_insert() later
299 * in this function.
300 */
304 }
309 }
317 }
319 /*
320 * | - range to drop - |
321 * | -------- extent -------- |
322 */
328 }
337 }
364 };
372 }
373 }
375 }
376 /*
377 * From here on out we will have actually dropped something, so
378 * last_end can be updated.
379 */
382 /*
383 * | ---- range to drop ----- |
384 * | -------- extent -------- |
385 */
390 }
404 }
407 /*
408 * | ---- range to drop ----- |
409 * | -------- extent -------- |
410 */
416 }
428 }
430 /*
431 * | ---- range to drop ----- |
432 * | ------ extent ------ |
433 */
435 delete_extent_item:
441 del_nr++;
442 }
457 };
465 }
467 }
475 }
478 del_nr);
482 }
489 }
492 }
495 /*
496 * Set path->slots[0] to first slot, so that after the delete
497 * if items are move off from our leaf to its immediate left or
498 * right neighbor leafs, we end up with a correct and adjusted
499 * path->slots[0] for our insertion (if args->replace_extent).
500 */
505 }
508 /*
509 * If btrfs_del_items() was called, it might have deleted a leaf, in
510 * which case it unlocked our path, so check path->locks[0] matches a
511 * write lock.
512 */
527 }
531 }
537 out:
541 }
546 {
549 u64 extent_end;
574 }
576 /*
577 * Mark extent in the range start - end as written.
578 *
579 * This changes extent type from 'pre-allocated' to 'regular'. If only
580 * part of extent is marked as written, the extent will be split into
581 * two or three.
582 */
585 {
593 u64 bytenr;
594 u64 num_bytes;
595 u64 extent_end;
596 u64 orig_offset;
597 u64 other_start;
598 u64 other_end;
599 u64 split;
609 again:
629 }
636 }
642 }
673 }
674 }
702 }
703 }
714 }
718 }
747 }
756 }
759 }
761 }
779 }
782 del_nr++;
787 }
788 }
797 }
800 del_nr++;
805 }
806 }
811 BTRFS_FILE_EXTENT_REG);
818 BTRFS_FILE_EXTENT_REG);
828 }
829 }
830 out:
833 }
835 /*
836 * On error return an unlocked folio and the error value
837 * On success return a locked folio and 0
838 */
841 {
861 }
863 /*
864 * Since btrfs_read_folio() will unlock the folio before it returns,
865 * there is a window where btrfs_release_folio() can be called to
866 * release the page. Here we check both inode mapping and page
867 * private to make sure the page was not released.
868 *
869 * The private flag check is essential for subpage as we need to store
870 * extra bitmap using folio private.
871 */
875 }
877 }
880 {
881 gfp_t gfp;
887 }
890 }
892 /*
893 * Get folio into the page cache and lock it.
894 */
898 {
905 again:
910 else
913 }
914 /* Only support page sized folio yet. */
921 }
924 /* The folio is already unlocked. */
929 }
931 }
934 }
936 /*
937 * Locks the extent and properly waits for data=ordered extents to finish
938 * before allowing the folios to be modified if need.
939 *
940 * Return:
941 * 1 - the extent is locked
942 * 0 - the extent is not locked, and everything is OK
943 * -EAGAIN - need to prepare the folios again
944 */
950 {
952 u64 start_pos;
953 u64 last_pos;
964 cached_state)) {
968 }
971 }
979 cached_state);
985 }
992 }
994 /*
995 * We should be called after prepare_one_folio() which should have locked
996 * all pages in the range.
997 */
1001 }
1003 /*
1004 * Check if we can do nocow write into the range [@pos, @pos + @write_bytes)
1005 *
1006 * @pos: File offset.
1007 * @write_bytes: The length to write, will be updated to the nocow writeable
1008 * range.
1009 *
1010 * This function will flush ordered extents in the range to ensure proper
1011 * nocow checks.
1012 *
1013 * Return:
1014 * > 0 If we can nocow, and updates @write_bytes.
1015 * 0 If we can't do a nocow write.
1016 * -EAGAIN If we can't do a nocow write because snapshoting of the inode's
1017 * root is in progress.
1018 * < 0 If an error happened.
1019 *
1020 * NOTE: Callers need to call btrfs_check_nocow_unlock() if we return > 0.
1021 */
1024 {
1029 u64 num_bytes;
1048 }
1052 }
1057 else
1063 }
1066 {
1068 }
1071 {
1077 loff_t oldsize;
1078 loff_t start_pos;
1080 /*
1081 * Quickly bail out on NOWAIT writes if we don't have the nodatacow or
1082 * prealloc flags, as without those flags we always have to COW. We will
1083 * later check if we can really COW into the target range (using
1084 * can_nocow_extent() at btrfs_get_blocks_direct_write()).
1085 */
1094 /*
1095 * We reserve space for updating the inode when we reserve space for the
1096 * extent we are going to write, so we will enospc out there. We don't
1097 * need to start yet another transaction to update the inode as we will
1098 * update the inode when we finish writing whatever data we write.
1099 */
1103 }
1108 /* Expand hole size to cover write data, preventing empty gap */
1114 }
1117 }
1120 {
1122 loff_t pos;
1127 u64 lockstart;
1128 u64 lockend;
1130 ssize_t ret;
1166 /*
1167 * Fault pages before locking them in prepare_one_folio()
1168 * to avoid recursive lock
1169 */
1173 }
1188 }
1190 /*
1191 * If we don't have to COW at the offset, reserve
1192 * metadata only. write_bytes may get smaller than
1193 * requested here.
1194 */
1204 }
1210 reserve_bytes,
1216 write_bytes);
1217 else
1223 }
1226 again:
1231 }
1237 reserve_bytes);
1239 }
1249 reserve_bytes);
1252 }
1266 }
1269 /* release everything except the sectors we dirtied */
1280 }
1281 }
1289 /*
1290 * If we have not locked the extent range, because the range's
1291 * start offset is >= i_size, we might still have a non-NULL
1292 * cached extent state, acquired while marking the extent range
1293 * as delalloc through btrfs_dirty_page(). Therefore free any
1294 * possible cached extent state to avoid a memory leak.
1295 */
1299 else
1306 }
1318 }
1327 data_reserved,
1330 }
1331 }
1337 }
1338 out:
1341 }
1345 {
1348 loff_t count;
1349 ssize_t ret;
1355 /*
1356 * The write got truncated by generic_write_checks_count(). We
1357 * can't do a partial encoded write.
1358 */
1360 }
1369 out:
1372 }
1376 {
1381 /*
1382 * If the fs flips readonly due to some impossible error, although we
1383 * have opened a file as writable, we have to stop this write operation
1384 * to ensure consistency.
1385 */
1401 }
1409 }
1412 }
1415 {
1417 }
1420 {
1428 }
1430 /*
1431 * Set by setattr when we are about to truncate a file from a non-zero
1432 * size to a zero size. This tries to flush down new bytes that may
1433 * have been written if the application were using truncate to replace
1434 * a file in place.
1435 */
1440 }
1443 {
1447 /*
1448 * This is only called in fsync, which would do synchronous writes, so
1449 * a plug can merge adjacent IOs as much as possible. Esp. in case of
1450 * multiple disks using raid profile, a large IO can be split to
1451 * several segments of stripe length (currently 64K).
1452 */
1458 }
1461 {
1469 /*
1470 * If we are doing a fast fsync we can not bail out if the inode's
1471 * last_trans is <= then the last committed transaction, because we only
1472 * update the last_trans of the inode during ordered extent completion,
1473 * and for a fast fsync we don't wait for that, we only wait for the
1474 * writeback to complete.
1475 */
1482 }
1484 /*
1485 * fsync call for both files and directories. This logs the inode into
1486 * the tree log instead of forcing full commits whenever possible.
1487 *
1488 * It needs to call filemap_fdatawait so that all ordered extent updates are
1489 * in the metadata btree are up to date for copying to the log.
1490 *
1491 * It drops the inode mutex before doing the tree log commit. This is an
1492 * important optimization for directories because holding the mutex prevents
1493 * new operations on the dir while we write to disk.
1494 */
1496 {
1504 u64 len;
1512 }
1518 /*
1519 * Always set the range to a full range, otherwise we can get into
1520 * several problems, from missing file extent items to represent holes
1521 * when not using the NO_HOLES feature, to log tree corruption due to
1522 * races between hole detection during logging and completion of ordered
1523 * extents outside the range, to missing checksums due to ordered extents
1524 * for which we flushed only a subset of their pages.
1525 */
1530 /*
1531 * We write the dirty pages in the range and wait until they complete
1532 * out of the ->i_mutex. If so, we can flush the dirty pages by
1533 * multi-task, and make the performance up. See
1534 * btrfs_wait_ordered_range for an explanation of the ASYNC check.
1535 */
1542 else
1547 /*
1548 * Before we acquired the inode's lock and the mmap lock, someone may
1549 * have dirtied more pages in the target range. We need to make sure
1550 * that writeback for any such pages does not start while we are logging
1551 * the inode, because if it does, any of the following might happen when
1552 * we are not doing a full inode sync:
1553 *
1554 * 1) We log an extent after its writeback finishes but before its
1555 * checksums are added to the csum tree, leading to -EIO errors
1556 * when attempting to read the extent after a log replay.
1557 *
1558 * 2) We can end up logging an extent before its writeback finishes.
1559 * Therefore after the log replay we will have a file extent item
1560 * pointing to an unwritten extent (and no data checksums as well).
1561 *
1562 * So trigger writeback for any eventual new dirty pages and then we
1563 * wait for all ordered extents to complete below.
1564 */
1569 else
1572 }
1574 /*
1575 * Always check for the full sync flag while holding the inode's lock,
1576 * to avoid races with other tasks. The flag must be either set all the
1577 * time during logging or always off all the time while logging.
1578 * We check the flag here after starting delalloc above, because when
1579 * running delalloc the full sync flag may be set if we need to drop
1580 * extra extent map ranges due to temporary memory allocation failures.
1581 */
1584 /*
1585 * We have to do this here to avoid the priority inversion of waiting on
1586 * IO of a lower priority task while holding a transaction open.
1587 *
1588 * For a full fsync we wait for the ordered extents to complete while
1589 * for a fast fsync we wait just for writeback to complete, and then
1590 * attach the ordered extents to the transaction so that a transaction
1591 * commit waits for their completion, to avoid data loss if we fsync,
1592 * the current transaction commits before the ordered extents complete
1593 * and a power failure happens right after that.
1594 *
1595 * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the
1596 * logical address recorded in the ordered extent may change. We need
1597 * to wait for the IO to stabilize the logical address.
1598 */
1603 /*
1604 * Get our ordered extents as soon as possible to avoid doing
1605 * checksum lookups in the csum tree, and use instead the
1606 * checksums attached to the ordered extents.
1607 */
1613 /*
1614 * Check and clear the BTRFS_INODE_COW_WRITE_ERROR now after
1615 * starting and waiting for writeback, because for buffered IO
1616 * it may have been set during the end IO callback
1617 * (end_bbio_data_write() -> btrfs_finish_ordered_extent()) in
1618 * case an error happened and we need to wait for ordered
1619 * extents to complete so that any extent maps that point to
1620 * unwritten locations are dropped and we don't log them.
1621 */
1624 }
1632 /*
1633 * We've had everything committed since the last time we were
1634 * modified so clear this flag in case it was set for whatever
1635 * reason, it's no longer relevant.
1636 */
1638 /*
1639 * An ordered extent might have started before and completed
1640 * already with io errors, in which case the inode was not
1641 * updated and we end up here. So check the inode's mapping
1642 * for any errors that might have happened since we last
1643 * checked called fsync.
1644 */
1647 }
1651 /*
1652 * We use start here because we will need to wait on the IO to complete
1653 * in btrfs_sync_log, which could require joining a transaction (for
1654 * example checking cross references in the nocow path). If we use join
1655 * here we could get into a situation where we're waiting on IO to
1656 * happen that is blocked on a transaction trying to commit. With start
1657 * we inc the extwriter counter, so we wait for all extwriters to exit
1658 * before we start blocking joiners. This comment is to keep somebody
1659 * from thinking they are super smart and changing this to
1660 * btrfs_join_transaction *cough*Josef*cough*.
1661 */
1666 }
1670 /*
1671 * Scratch eb no longer needed, release before syncing log or commit
1672 * transaction, to avoid holding unnecessary memory during such long
1673 * operations.
1674 */
1678 }
1681 /* Fallthrough and commit/free transaction. */
1683 }
1685 /* we've logged all the items and now have a consistent
1686 * version of the file in the log. It is possible that
1687 * someone will come in and modify the file, but that's
1688 * fine because the log is consistent on disk, and we
1689 * have references to all of the file's extents
1690 *
1691 * It is possible that someone will come in and log the
1692 * file again, but that will end up using the synchronization
1693 * inside btrfs_sync_log to keep things safe.
1694 */
1697 else
1703 }
1705 /* We successfully logged the inode, attempt to sync the log. */
1711 }
1712 }
1714 /*
1715 * At this point we need to commit the transaction because we had
1716 * btrfs_need_log_full_commit() or some other error.
1717 *
1718 * If we didn't do a full sync we have to stop the trans handle, wait on
1719 * the ordered extents, start it again and commit the transaction. If
1720 * we attempt to wait on the ordered extents here we could deadlock with
1721 * something like fallocate() that is holding the extent lock trying to
1722 * start a transaction while some other thread is trying to commit the
1723 * transaction while we (fsync) are currently holding the transaction
1724 * open.
1725 */
1734 /*
1735 * This is safe to use here because we're only interested in
1736 * making sure the transaction that had the ordered extents is
1737 * committed. We aren't waiting on anything past this point,
1738 * we're purely getting the transaction and committing it.
1739 */
1744 /*
1745 * We committed the transaction and there's no currently
1746 * running transaction, this means everything we care
1747 * about made it to disk and we are done.
1748 */
1752 }
1753 }
1756 out:
1765 out_release_extents:
1769 else
1772 }
1774 /*
1775 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
1776 * called from a page fault handler when a page is first dirtied. Hence we must
1777 * be careful to check for EOF conditions here. We set the page up correctly
1778 * for a written page which means we get ENOSPC checking when writing into
1779 * holes and correct delalloc and unwritten extent mapping on filesystems that
1780 * support these features.
1781 *
1782 * We are not allowed to take the i_mutex here so we have to play games to
1783 * protect against truncate races as the page could now be beyond EOF. Because
1784 * truncate_setsize() writes the inode size before removing pages, once we have
1785 * the page lock we can determine safely if the page is beyond EOF. If it is not
1786 * beyond EOF, then the page is guaranteed safe against truncation until we
1787 * unlock the page.
1788 */
1790 {
1800 loff_t size;
1801 vm_fault_t ret;
1804 u64 reserved_space;
1805 u64 page_start;
1806 u64 page_end;
1807 u64 end;
1818 /*
1819 * Reserving delalloc space after obtaining the page lock can lead to
1820 * deadlock. For example, if a dirty page is locked by this function
1821 * and the call to btrfs_delalloc_reserve_space() ends up triggering
1822 * dirty page write out, then the btrfs_writepages() function could
1823 * end up waiting indefinitely to get a lock on the page currently
1824 * being processed by btrfs_page_mkwrite() function.
1825 */
1831 }
1837 }
1839 /* Make the VM retry the fault. */
1841 again:
1848 /* Page got truncated out from underneath us. */
1850 }
1859 }
1861 /*
1862 * We can't set the delalloc bits if there are pending ordered
1863 * extents. Drop our locks and wait for them to finish.
1864 */
1873 }
1882 }
1883 }
1885 /*
1886 * page_mkwrite gets called when the page is firstly dirtied after it's
1887 * faulted in, but write(2) could also dirty a page and set delalloc
1888 * bits, thus in this case for space account reason, we still need to
1889 * clear any delalloc bits within this page range since we have to
1890 * reserve data&meta space before lock_page() (see above comments).
1891 */
1902 }
1904 /* Page is wholly or partially inside EOF. */
1907 else
1927 out_unlock:
1930 out:
1934 out_noreserve:
1938 }
1944 };
1947 {
1957 }
1961 {
1986 }
1991 {
2009 /*
2010 * We should have dropped this offset, so if we find it then
2011 * something has gone horribly wrong.
2012 */
2016 }
2020 u64 num_bytes;
2033 }
2036 u64 num_bytes;
2043 offset;
2050 }
2058 out:
2078 }
2081 }
2083 /*
2084 * Find a hole extent on given inode and change start/len to the end of hole
2085 * extent.(hole/vacuum extent whose em->start <= start &&
2086 * em->start + em->len > start)
2087 * When a hole extent is found, return 1 and modify start/len.
2088 */
2090 {
2101 /* Hole or vacuum extent(only exists in no-hole mode) */
2107 }
2110 }
2116 {
2117 /*
2118 * For subpage case, if the range is not at page boundary, we could
2119 * have pages at the leading/tailing part of the range.
2120 * This could lead to dead loop since filemap_range_has_page()
2121 * will always return true.
2122 * So here we need to do extra page alignment for
2123 * filemap_range_has_page().
2124 */
2132 cached_state);
2133 /*
2134 * We can't have ordered extents in the range, nor dirty/writeback
2135 * pages, because we have locked the inode's VFS lock in exclusive
2136 * mode, we have locked the inode's i_mmap_lock in exclusive mode,
2137 * we have flushed all delalloc in the range and we have waited
2138 * for any ordered extents in the range to complete.
2139 * We can race with anyone reading pages from this range, so after
2140 * locking the range check if we have pages in the range, and if
2141 * we do, unlock the range and retry.
2142 */
2144 page_lockend))
2148 cached_state);
2149 }
2152 }
2160 {
2176 }
2200 replace_len);
2204 /* If it's a hole, nothing more needs to be done. */
2208 }
2228 };
2229 u64 ref_offset;
2234 }
2239 }
2241 /*
2242 * The respective range must have been previously locked, as well as the inode.
2243 * The end offset is inclusive (last byte of the range).
2244 * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
2245 * the file range with an extent.
2246 * When not punching a hole, we don't want to end up in a state where we dropped
2247 * extents without inserting a new one, so we must abort the transaction to avoid
2248 * a corruption.
2249 */
2255 {
2264 u64 cur_offset;
2275 }
2279 /*
2280 * 1 - update the inode
2281 * 1 - removing the extents in the range
2282 * 1 - adding the hole extent if no_holes isn't set or if we are
2283 * replacing the range with a new extent
2284 */
2287 else
2295 }
2310 /* If we are punching a hole decrement the inode's byte count */
2315 /*
2316 * The only time we don't want to abort is if we are
2317 * attempting to clone a partial inline extent, in which
2318 * case we'll get EOPNOTSUPP. However if we aren't
2319 * clone we need to abort no matter what, because if we
2320 * got EOPNOTSUPP via prealloc then we messed up and
2321 * need to abort.
2322 */
2328 }
2337 /*
2338 * If we failed then we didn't insert our hole
2339 * entries for the area we dropped, so now the
2340 * fs is corrupted, so we must abort the
2341 * transaction.
2342 */
2345 }
2347 /*
2348 * We are past the i_size here, but since we didn't
2349 * insert holes we need to clear the mapped area so we
2350 * know to not set disk_i_size in this area until a new
2351 * file extent is inserted here.
2352 */
2354 cur_offset,
2357 /*
2358 * We couldn't clear our area, so we could
2359 * presumably adjust up and corrupt the fs, so
2360 * we need to abort.
2361 */
2364 }
2365 }
2378 }
2382 }
2384 /*
2385 * We are releasing our handle on the transaction, balance the
2386 * dirty pages of the btree inode and flush delayed items, and
2387 * then get a new transaction handle, which may now point to a
2388 * new transaction in case someone else may have committed the
2389 * transaction we used to replace/drop file extent items. So
2390 * bump the inode's iversion and update mtime and ctime except
2391 * if we are called from a dedupe context. This is because a
2392 * power failure/crash may happen after the transaction is
2393 * committed and before we finish replacing/dropping all the
2394 * file extent items we need.
2395 */
2414 }
2431 }
2432 }
2433 }
2435 /*
2436 * If we were cloning, force the next fsync to be a full one since we
2437 * we replaced (or just dropped in the case of cloning holes when
2438 * NO_HOLES is enabled) file extent items and did not setup new extent
2439 * maps for the replacement extents (or holes).
2440 */
2448 /*
2449 * If we are using the NO_HOLES feature we might have had already an
2450 * hole that overlaps a part of the region [lockstart, lockend] and
2451 * ends at (or beyond) lockend. Since we have no file extent items to
2452 * represent holes, drop_end can be less than lockend and so we must
2453 * make sure we have an extent map representing the existing hole (the
2454 * call to __btrfs_drop_extents() might have dropped the existing extent
2455 * map representing the existing hole), otherwise the fast fsync path
2456 * will not record the existence of the hole region
2457 * [existing_hole_start, lockend].
2458 */
2461 /*
2462 * Don't insert file hole extent item if it's for a range beyond eof
2463 * (because it's useless) or if it represents a 0 bytes range (when
2464 * cur_offset == drop_end).
2465 */
2471 /* Same comment as above. */
2474 }
2476 /* See the comment in the loop above for the reasoning here. */
2482 }
2484 }
2492 }
2493 }
2495 out_trans:
2502 else
2504 out_free:
2506 out:
2508 }
2511 {
2518 u64 lockstart;
2519 u64 lockend;
2520 u64 tail_start;
2521 u64 tail_len;
2525 u64 ino_size;
2540 /* Already in a large hole */
2543 }
2553 /*
2554 * We needn't truncate any block which is beyond the end of the file
2555 * because we are sure there is no data there.
2556 */
2557 /*
2558 * Only do this if we are in the same block and we aren't doing the
2559 * entire block.
2560 */
2568 }
2570 }
2572 /* zero back part of the first block */
2579 }
2580 }
2582 /* Check the aligned pages after the first unaligned page,
2583 * if offset != orig_start, which means the first unaligned page
2584 * including several following pages are already in holes,
2585 * the extra check can be skipped */
2587 /* after truncate page, check hole again */
2596 }
2598 }
2600 /* Check the tail unaligned part is in a hole */
2608 /* zero the front end of the last page */
2616 }
2617 }
2618 }
2623 }
2631 }
2646 out:
2649 out_only_mutex:
2651 /*
2652 * If we only end up zeroing part of a page, we still need to
2653 * update the inode item, so that all the time fields are
2654 * updated as well as the necessary btrfs inode in memory fields
2655 * for detecting, at fsync time, if the inode isn't yet in the
2656 * log tree or it's there but not up to date.
2657 */
2672 }
2673 }
2676 }
2678 /* Helper structure to record which range is already reserved */
2681 u64 start;
2682 u64 len;
2683 };
2685 /*
2686 * Helper function to add falloc range
2687 *
2688 * Caller should have locked the larger range of extent containing
2689 * [start, len)
2690 */
2692 {
2696 /*
2697 * As fallocate iterates by bytenr order, we only need to check
2698 * the last range.
2699 */
2704 }
2705 }
2714 }
2719 {
2739 }
2742 RANGE_BOUNDARY_WRITTEN_EXTENT,
2743 RANGE_BOUNDARY_PREALLOC_EXTENT,
2744 RANGE_BOUNDARY_HOLE,
2745 };
2748 u64 offset)
2749 {
2763 else
2768 }
2771 loff_t offset,
2772 loff_t len,
2774 {
2791 }
2793 /*
2794 * Avoid hole punching and extent allocation for some cases. More cases
2795 * could be considered, but these are unlikely common and we keep things
2796 * as simple as possible for now. Also, intentionally, if the target
2797 * range contains one or more prealloc extents together with regular
2798 * extents and holes, we drop all the existing extents and allocate a
2799 * new prealloc extent, so that we get a larger contiguous disk extent.
2800 */
2805 /*
2806 * The whole range is already a prealloc extent,
2807 * do nothing except updating the inode's i_size if
2808 * needed.
2809 */
2812 mode);
2814 }
2815 /*
2816 * Part of the range is already a prealloc extent, so operate
2817 * only on the remaining part of the range.
2818 */
2824 }
2833 }
2838 mode);
2840 }
2848 mode);
2850 }
2855 }
2860 /*
2861 * For unaligned ranges, check the pages at the boundaries, they might
2862 * map to an extent, in which case we need to partially zero them, or
2863 * they might map to a hole, in which case we need our allocation range
2864 * to cover them.
2865 */
2868 offset);
2880 }
2881 }
2898 }
2899 }
2901 reserve_space:
2909 bytes_to_reserve);
2921 }
2928 /* btrfs_prealloc_file_range releases reserved space on error */
2932 }
2933 }
2935 out:
2942 }
2946 {
2953 u64 cur_offset;
2954 u64 last_byte;
2955 u64 alloc_start;
2956 u64 alloc_end;
2958 u64 locked_end;
2967 /* Do not allow fallocate in ZONED mode */
2975 /* Make sure we aren't being give some crap mode */
2977 FALLOC_FL_ZERO_RANGE))
2989 }
2995 /*
2996 * TODO: Move these two operations after we have checked
2997 * accurate reserved space, or fallocate can still fail but
2998 * with page truncated or size expanded.
2999 *
3000 * But that's a minor problem and won't do much harm BTW.
3001 */
3004 alloc_start);
3008 /*
3009 * If we are fallocating from the end of the file onward we
3010 * need to zero out the end of the block if i_size lands in the
3011 * middle of a block.
3012 */
3016 }
3018 /*
3019 * We have locked the inode at the VFS level (in exclusive mode) and we
3020 * have locked the i_mmap_lock lock (in exclusive mode). Now before
3021 * locking the file range, flush all dealloc in the range and wait for
3022 * all ordered extents in the range to complete. After this we can lock
3023 * the file range and, due to the previous locking we did, we know there
3024 * can't be more delalloc or ordered extents in the range.
3025 */
3035 }
3043 /* First, check if we exceed the qgroup limit */
3050 }
3063 }
3069 }
3072 }
3075 }
3078 /*
3079 * We are safe to reserve space here as we can't have delalloc
3080 * in the range, see above.
3081 */
3083 data_space_needed);
3086 }
3088 /*
3089 * If ret is still 0, means we're OK to fallocate.
3090 * Or just cleanup the list and exit.
3091 */
3098 /*
3099 * btrfs_prealloc_file_range() releases space even
3100 * if it returns an error.
3101 */
3114 }
3117 }
3121 /*
3122 * We didn't need to allocate any more space, but we still extended the
3123 * size of the file so we need to update i_size and the inode item.
3124 */
3126 out_unlock:
3129 out:
3133 }
3135 /*
3136 * Helper for btrfs_find_delalloc_in_range(). Find a subrange in a given range
3137 * that has unflushed and/or flushing delalloc. There might be other adjacent
3138 * subranges after the one it found, so btrfs_find_delalloc_in_range() keeps
3139 * looping while it gets adjacent subranges, and merging them together.
3140 */
3145 {
3149 u64 oe_start;
3150 u64 oe_end;
3152 /*
3153 * Search the io tree first for EXTENT_DELALLOC. If we find any, it
3154 * means we have delalloc (dirty pages) for which writeback has not
3155 * started yet.
3156 */
3165 cached_state);
3168 }
3169 }
3172 /*
3173 * If delalloc was found then *delalloc_start_ret has a sector size
3174 * aligned value (rounded down).
3175 */
3179 /* Delalloc for the whole range, nothing more to do. */
3182 /* Else trim our search range for ordered extents. */
3185 }
3187 /* No delalloc, future calls don't need to search again. */
3189 }
3191 /*
3192 * Now also check if there's any ordered extent in the range.
3193 * We do this because:
3194 *
3195 * 1) When delalloc is flushed, the file range is locked, we clear the
3196 * EXTENT_DELALLOC bit from the io tree and create an extent map and
3197 * an ordered extent for the write. So we might just have been called
3198 * after delalloc is flushed and before the ordered extent completes
3199 * and inserts the new file extent item in the subvolume's btree;
3200 *
3201 * 2) We may have an ordered extent created by flushing delalloc for a
3202 * subrange that starts before the subrange we found marked with
3203 * EXTENT_DELALLOC in the io tree.
3204 *
3205 * We could also use the extent map tree to find such delalloc that is
3206 * being flushed, but using the ordered extents tree is more efficient
3207 * because it's usually much smaller as ordered extents are removed from
3208 * the tree once they complete. With the extent maps, we mau have them
3209 * in the extent map tree for a very long time, and they were either
3210 * created by previous writes or loaded by read operations.
3211 */
3216 /* The ordered extent may span beyond our search range. */
3222 /* Don't have unflushed delalloc, return the ordered extent range. */
3227 }
3229 /*
3230 * We have both unflushed delalloc (io_tree) and an ordered extent.
3231 * If the ranges are adjacent returned a combined range, otherwise
3232 * return the leftmost range.
3233 */
3240 }
3243 }
3245 /*
3246 * Check if there's delalloc in a given range.
3247 *
3248 * @inode: The inode.
3249 * @start: The start offset of the range. It does not need to be
3250 * sector size aligned.
3251 * @end: The end offset (inclusive value) of the search range.
3252 * It does not need to be sector size aligned.
3253 * @cached_state: Extent state record used for speeding up delalloc
3254 * searches in the inode's io_tree. Can be NULL.
3255 * @delalloc_start_ret: Output argument, set to the start offset of the
3256 * subrange found with delalloc (may not be sector size
3257 * aligned).
3258 * @delalloc_end_ret: Output argument, set to he end offset (inclusive value)
3259 * of the subrange found with delalloc.
3260 *
3261 * Returns true if a subrange with delalloc is found within the given range, and
3262 * if so it sets @delalloc_start_ret and @delalloc_end_ret with the start and
3263 * end offsets of the subrange.
3264 */
3268 {
3275 u64 delalloc_start;
3276 u64 delalloc_end;
3287 /* First subrange found. */
3292 /* Subrange adjacent to the previous one, merge them. */
3295 /* Subrange not adjacent to the previous one, exit. */
3297 }
3302 }
3305 }
3307 /*
3308 * Check if there's a hole or delalloc range in a range representing a hole (or
3309 * prealloc extent) found in the inode's subvolume btree.
3310 *
3311 * @inode: The inode.
3312 * @whence: Seek mode (SEEK_DATA or SEEK_HOLE).
3313 * @start: Start offset of the hole region. It does not need to be sector
3314 * size aligned.
3315 * @end: End offset (inclusive value) of the hole region. It does not
3316 * need to be sector size aligned.
3317 * @start_ret: Return parameter, used to set the start of the subrange in the
3318 * hole that matches the search criteria (seek mode), if such
3319 * subrange is found (return value of the function is true).
3320 * The value returned here may not be sector size aligned.
3321 *
3322 * Returns true if a subrange matching the given seek mode is found, and if one
3323 * is found, it updates @start_ret with the start of the subrange.
3324 */
3328 {
3329 u64 delalloc_start;
3330 u64 delalloc_end;
3338 }
3341 /*
3342 * We found delalloc but it starts after out start offset. So we
3343 * have a hole between our start offset and the delalloc start.
3344 */
3348 }
3349 /*
3350 * Delalloc range starts at our start offset.
3351 * If the delalloc range's length is smaller than our range,
3352 * then it means we have a hole that starts where the delalloc
3353 * subrange ends.
3354 */
3358 }
3360 /* There's delalloc for the whole range. */
3362 }
3367 }
3369 /*
3370 * No delalloc in the range and we are seeking for data. The caller has
3371 * to iterate to the next extent item in the subvolume btree.
3372 */
3374 }
3377 {
3388 u64 last_extent_end;
3389 u64 lockstart;
3390 u64 lockend;
3391 u64 start;
3398 /*
3399 * Quick path. If the inode has no prealloc extents and its number of
3400 * bytes used matches its i_size, then it can not have holes.
3401 */
3412 /*
3413 * Not allocated by us, don't use it as its cached state is used
3414 * by the task that allocated it and we don't want neither to
3415 * mess with it nor get incorrect results because it reflects an
3416 * invalid state for the current task.
3417 */
3421 /*
3422 * No worries if memory allocation failed.
3423 * The private structure is used only for speeding up multiple
3424 * lseek SEEK_HOLE/DATA calls to a file when there's delalloc,
3425 * so everything will still be correct.
3426 */
3435 else
3442 }
3443 }
3444 }
3448 else
3451 /*
3452 * offset can be negative, in this case we start finding DATA/HOLE from
3453 * the very start of the file.
3454 */
3461 lockend--;
3483 }
3488 u64 extent_end;
3489 u8 type;
3499 }
3507 /*
3508 * In the first iteration we may have a slot that points to an
3509 * extent that ends before our start offset, so skip it.
3510 */
3514 }
3516 /* We have an implicit hole, NO_HOLES feature is likely set. */
3519 u64 found_start;
3521 /*
3522 * First iteration, @start matches @offset and it's
3523 * within the hole.
3524 */
3529 delalloc_cached_state,
3530 search_start,
3536 }
3537 /*
3538 * Didn't find data or a hole (due to delalloc) in the
3539 * implicit hole range, so need to analyze the extent.
3540 */
3541 }
3547 /*
3548 * Can't access the extent's disk_bytenr field if this is an
3549 * inline extent, since at that offset, it's where the extent
3550 * data starts.
3551 */
3555 /*
3556 * Explicit hole or prealloc extent, search for delalloc.
3557 * A prealloc extent is treated like a hole.
3558 */
3560 u64 found_start;
3562 /*
3563 * First iteration, @start matches @offset and it's
3564 * within the hole.
3565 */
3570 delalloc_cached_state,
3571 search_start,
3577 }
3578 /*
3579 * Didn't find data or a hole (due to delalloc) in the
3580 * implicit hole range, so need to analyze the next
3581 * extent item.
3582 */
3584 /*
3585 * Found a regular or inline extent.
3586 * If we are seeking for data, adjust the start offset
3587 * and stop, we're done.
3588 */
3593 }
3594 /*
3595 * Else, we are seeking for a hole, check the next file
3596 * extent item.
3597 */
3598 }
3606 }
3608 }
3610 /* We have an implicit hole from the last extent found up to i_size. */
3617 }
3619 out:
3630 }
3633 {
3645 }
3651 }
3654 {
3663 }
3666 {
3674 }
3677 }
3692 #ifdef CONFIG_COMPAT
3694 #endif
3698 };
3701 {
3705 /*
3706 * So with compression we will find and lock a dirty page and clear the
3707 * first one as dirty, setup an async extent, and immediately return
3708 * with the entire range locked but with nobody actually marked with
3709 * writeback. So we can't just filemap_write_and_wait_range() and
3710 * expect it to work since it will just kick off a thread to do the
3711 * actual work. So we need to call filemap_fdatawrite_range _again_
3712 * since it will wait on the page lock, which won't be unlocked until
3713 * after the pages have been marked as writeback and so we're good to go
3714 * from there. We have to do this otherwise we'll miss the ordered
3715 * extents and that results in badness. Please Josef, do not think you
3716 * know better and pull this out at some point in the future, it is
3717 * right and you are wrong.
3718 */
3724 }