| blob | 320d1062068d36efa08869fd6f557255cf38b4cc |
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
6 #include <linux/kernel.h>
7 #include <linux/bio.h>
8 #include <linux/buffer_head.h>
9 #include <linux/file.h>
10 #include <linux/fs.h>
11 #include <linux/pagemap.h>
12 #include <linux/highmem.h>
13 #include <linux/time.h>
14 #include <linux/init.h>
15 #include <linux/string.h>
16 #include <linux/backing-dev.h>
17 #include <linux/writeback.h>
18 #include <linux/compat.h>
19 #include <linux/xattr.h>
20 #include <linux/posix_acl.h>
21 #include <linux/falloc.h>
22 #include <linux/slab.h>
23 #include <linux/ratelimit.h>
24 #include <linux/btrfs.h>
25 #include <linux/blkdev.h>
26 #include <linux/posix_acl_xattr.h>
27 #include <linux/uio.h>
28 #include <linux/magic.h>
29 #include <linux/iversion.h>
30 #include <linux/swap.h>
31 #include <linux/migrate.h>
32 #include <linux/sched/mm.h>
33 #include <asm/unaligned.h>
56 };
59 u64 reserve;
60 u64 unsubmitted_oe_range_start;
61 u64 unsubmitted_oe_range_end;
63 };
96 /*
97 * Cleanup all submitted ordered extents in specified range to handle errors
98 * from the btrfs_run_delalloc_range() callback.
99 *
100 * NOTE: caller must ensure that when an error happens, it can not call
101 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
102 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
103 * to be released, which we want to happen only when finishing the ordered
104 * extent (btrfs_finish_ordered_io()).
105 */
109 {
119 index++;
124 }
126 /*
127 * In case this page belongs to the delalloc range being instantiated
128 * then skip it, since the first page of a range is going to be
129 * properly cleaned up by the caller of run_delalloc_range
130 */
134 }
137 }
141 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
143 {
145 }
146 #endif
151 {
158 }
160 /*
161 * this does all the hard work for inserting an inline extent into
162 * the btree. The caller should have done a btrfs_drop_extents so that
163 * no overlapping inline items exist in the btree
164 */
171 {
200 datasize);
203 }
220 PAGE_SIZE);
226 i++;
229 }
231 compress_type);
241 }
245 /*
246 * We align size to sectorsize for inline extents just for simplicity
247 * sake.
248 */
254 /*
255 * we're an inline extent, so nobody can
256 * extend the file past i_size without locking
257 * a page we already have locked.
258 *
259 * We must do any isize and inode updates
260 * before we unlock the pages. Otherwise we
261 * could end up racing with unlink.
262 */
266 fail:
268 }
271 /*
272 * conditionally insert an inline extent into the file. This
273 * does the checks required to make sure the data is small enough
274 * to fit as an inline extent.
275 */
280 {
292 u32 extent_item_size;
305 }
315 }
320 compressed_size);
321 else
323 inline_len);
331 }
345 }
349 out:
350 /*
351 * Don't forget to free the reserved space, as for inlined extent
352 * it won't count as data extent, free them directly here.
353 * And at reserve time, it's always aligned to page size, so
354 * just free one page here.
355 */
360 }
363 u64 start;
364 u64 ram_size;
365 u64 compressed_size;
370 };
375 u64 start;
376 u64 end;
382 };
385 /* Number of chunks in flight; must be first in the structure */
386 atomic_t num_chunks;
388 };
392 u64 compressed_size,
396 {
409 }
411 /*
412 * Check if the inode has flags compatible with compression
413 */
415 {
420 }
422 /*
423 * Check if the inode needs to be submitted to compression, based on mount
424 * options, defragmentation, properties or heuristics.
425 */
427 {
435 }
436 /* force compress */
439 /* defrag ioctl */
442 /* bad compression ratios */
450 }
454 {
455 /* If this is a small write inside eof, kick off a defrag */
459 }
461 /*
462 * we create compressed extents in two phases. The first
463 * phase compresses a range of pages that have already been
464 * locked (both pages and state bits are locked).
465 *
466 * This is done inside an ordered work queue, and the compression
467 * is spread across many cpus. The actual IO submission is step
468 * two, and the ordered work queue takes care of making sure that
469 * happens in the same order things were put onto the queue by
470 * writepages and friends.
471 *
472 * If this code finds it can't get good compression, it puts an
473 * entry onto the work queue to write the uncompressed bytes. This
474 * makes sure that both compressed inodes and uncompressed inodes
475 * are written in the same order that the flusher thread sent them
476 * down.
477 */
479 {
485 u64 actual_end;
486 u64 i_size;
499 SZ_16K);
501 /*
502 * We need to save i_size before now because it could change in between
503 * us evaluating the size and assigning it. This is because we lock and
504 * unlock the page in truncate and fallocate, and then modify the i_size
505 * later on.
506 *
507 * The barriers are to emulate READ_ONCE, remove that once i_size_read
508 * does that for us.
509 */
514 again:
521 /*
522 * we don't want to send crud past the end of i_size through
523 * compression, that's just a waste of CPU time. So, if the
524 * end of the file is before the start of our current
525 * requested range of bytes, we bail out to the uncompressed
526 * cleanup code that can deal with all of this.
527 *
528 * It isn't really the fastest way to fix things, but this is a
529 * very uncommon corner.
530 */
536 /*
537 * skip compression for a small file range(<=blocksize) that
538 * isn't an inline extent, since it doesn't save disk space at all.
539 */
545 BTRFS_MAX_UNCOMPRESSED);
549 /*
550 * we do compression for mount -o compress and when the
551 * inode has not been flagged as nocompress. This flag can
552 * change at any time if we discover bad compression ratios.
553 */
558 /* just bail out to the uncompressed code */
561 }
568 /*
569 * we need to call clear_page_dirty_for_io on each
570 * page in the range. Otherwise applications with the file
571 * mmap'd can wander in and change the page contents while
572 * we are compressing them.
573 *
574 * If the compression fails for any reason, we set the pages
575 * dirty again later on.
576 *
577 * Note that the remaining part is redirtied, the start pointer
578 * has moved, the end is the original one.
579 */
583 }
585 /* Compression level is applied here and only here */
589 pages,
599 /* zero the tail end of the last page, we might be
600 * sending it down to disk
601 */
607 }
609 }
610 }
611 cont:
613 /* lets try to make an inline extent */
615 /* we didn't compress the entire range, try
616 * to make an uncompressed inline extent.
617 */
621 /* try making a compressed inline extent */
623 total_compressed,
625 }
629 EXTENT_DO_ACCOUNTING;
634 /*
635 * inline extent creation worked or returned error,
636 * we don't need to create any more async work items.
637 * Unlock and free up our temp pages.
638 *
639 * We use DO_ACCOUNTING here because we need the
640 * delalloc_release_metadata to be done _after_ we drop
641 * our outstanding extent for clearing delalloc for this
642 * range.
643 */
645 clear_flags,
646 PAGE_UNLOCK |
647 PAGE_CLEAR_DIRTY |
648 PAGE_SET_WRITEBACK |
649 page_error_op |
650 PAGE_END_WRITEBACK);
655 }
659 }
660 }
663 /*
664 * we aren't doing an inline extent round the compressed size
665 * up to a block size boundary so the allocator does sane
666 * things
667 */
670 /*
671 * one last check to make sure the compression is really a
672 * win, compare the page count read with the blocks on disk,
673 * compression must free at least one sector size
674 */
677 compressed_extents++;
679 /*
680 * The async work queues will take care of doing actual
681 * allocation on disk for these compressed pages, and
682 * will submit them to the elevator.
683 */
686 compress_type);
693 }
695 }
696 }
698 /*
699 * the compression code ran but failed to make things smaller,
700 * free any pages it allocated and our page pointer array
701 */
705 }
711 /* flag the file so we don't compress in the future */
715 }
716 }
717 cleanup_and_bail_uncompressed:
718 /*
719 * No compression, but we still need to write the pages in the file
720 * we've been given so far. redirty the locked page if it corresponds
721 * to our extent and set things up for the async work queue to run
722 * cow_file_range to do the normal delalloc dance.
723 */
728 /* unlocked later on in the async handlers */
729 }
734 BTRFS_COMPRESS_NONE);
735 compressed_extents++;
738 }
741 {
750 }
754 }
756 /*
757 * phase two of compressed writeback. This is the ordered portion
758 * of the code, which only gets called in the order the work was
759 * queued. We walk all the async extents created by compress_file_range
760 * and send them down to the disk.
761 */
763 {
774 again:
780 retry:
783 /* did the compression code fall back to uncompressed IO? */
788 /* allocate blocks */
795 /* JDM XXX */
797 /*
798 * if page_started, cow_file_range inserted an
799 * inline extent and took care of all the unlocking
800 * and IO for us. Otherwise, we need to submit
801 * all those pages down to the drive.
802 */
808 WB_SYNC_ALL);
814 }
828 /*
829 * we need to redirty the pages if we decide to
830 * fallback to uncompressed IO, otherwise we
831 * will not submit these pages down to lower
832 * layers.
833 */
840 }
842 }
843 /*
844 * here we're doing allocation and writeback of the
845 * compressed pages
846 */
855 BTRFS_ORDERED_COMPRESSED);
857 /* ret value is not necessary due to void function */
866 BTRFS_ORDERED_COMPRESSED,
874 }
877 /*
878 * clear dirty, set writeback and unlock the pages.
879 */
885 PAGE_SET_WRITEBACK);
904 PAGE_END_WRITEBACK |
905 PAGE_SET_ERROR);
907 }
911 }
913 out_free_reserve:
916 out_free:
921 EXTENT_DELALLOC_NEW |
925 PAGE_SET_ERROR);
929 }
932 u64 num_bytes)
933 {
941 /*
942 * if block start isn't an actual block number then find the
943 * first block in this inode and use that as a hint. If that
944 * block is also bogus then just don't worry about it.
945 */
956 }
957 }
961 }
963 /*
964 * when extent_io.c finds a delayed allocation range in the file,
965 * the call backs end up in this code. The basic idea is to
966 * allocate extents on disk for the range, and create ordered data structs
967 * in ram to track those extents.
968 *
969 * locked_page is the page that writepage had locked already. We use
970 * it to make sure we don't do extra locks or unlocks.
971 *
972 * *page_started is set to one if we unlock locked_page and do everything
973 * required to start IO on it. It may be clean and already done with
974 * IO when we return.
975 */
980 {
984 u64 num_bytes;
999 }
1008 /* lets try to make an inline extent */
1012 /*
1013 * We use DO_ACCOUNTING here because we need the
1014 * delalloc_release_metadata to be run _after_ we drop
1015 * our outstanding extent for clearing delalloc for this
1016 * range.
1017 */
1023 PAGE_END_WRITEBACK);
1030 }
1031 }
1059 }
1068 BTRFS_DATA_RELOC_TREE_OBJECTID) {
1070 cur_alloc_size);
1071 /*
1072 * Only drop cache here, and process as normal.
1073 *
1074 * We must not allow extent_clear_unlock_delalloc()
1075 * at out_unlock label to free meta of this ordered
1076 * extent, as its meta should be freed by
1077 * btrfs_finish_ordered_io().
1078 *
1079 * So we must continue until @start is increased to
1080 * skip current ordered extent.
1081 */
1085 }
1089 /* we're not doing compressed IO, don't unlock the first
1090 * page (which the caller expects to stay locked), don't
1091 * clear any dirty bits and don't set any writeback bits
1092 *
1093 * Do set the Private2 bit so we know this page was properly
1094 * setup for writepage
1095 */
1101 locked_page,
1103 page_ops);
1106 else
1112 /*
1113 * btrfs_reloc_clone_csums() error, since start is increased
1114 * extent_clear_unlock_delalloc() at out_unlock label won't
1115 * free metadata of current ordered extent, we're OK to exit.
1116 */
1119 }
1120 out:
1123 out_drop_extent_cache:
1125 out_reserve:
1128 out_unlock:
1132 PAGE_END_WRITEBACK;
1133 /*
1134 * If we reserved an extent for our delalloc range (or a subrange) and
1135 * failed to create the respective ordered extent, then it means that
1136 * when we reserved the extent we decremented the extent's size from
1137 * the data space_info's bytes_may_use counter and incremented the
1138 * space_info's bytes_reserved counter by the same amount. We must make
1139 * sure extent_clear_unlock_delalloc() does not try to decrement again
1140 * the data space_info's bytes_may_use counter, therefore we do not pass
1141 * it the flag EXTENT_CLEAR_DATA_RESV.
1142 */
1146 locked_page,
1147 clear_bits,
1148 page_ops);
1152 }
1155 page_ops);
1157 }
1159 /*
1160 * work queue call back to started compression on a file and pages
1161 */
1163 {
1173 }
1174 }
1176 /*
1177 * work queue call back to submit previously compressed pages
1178 */
1180 {
1182 work);
1187 PAGE_SHIFT;
1189 /* atomic_sub_return implies a barrier */
1194 /*
1195 * ->inode could be NULL if async_chunk_start has failed to compress,
1196 * in which case we don't have anything to submit, yet we need to
1197 * always adjust ->async_delalloc_pages as its paired with the init
1198 * happening in cow_file_range_async
1199 */
1202 }
1205 {
1213 /*
1214 * Since the pointer to 'pending' is at the beginning of the array of
1215 * async_chunk's, freeing it ensures the whole array has been freed.
1216 */
1219 }
1226 {
1232 u64 cur_end;
1247 }
1256 EXTENT_DO_ACCOUNTING;
1259 PAGE_SET_ERROR;
1264 }
1272 else
1275 /*
1276 * igrab is called higher up in the call chain, take only the
1277 * lightweight reference for the callback lifetime
1278 */
1287 /*
1288 * The locked_page comes all the way from writepage and its
1289 * the original page we were actually given. As we spread
1290 * this large delalloc region across multiple async_chunk
1291 * structs, only the first struct needs a pointer to locked_page
1292 *
1293 * This way we don't need racey decisions about who is supposed
1294 * to unlock it.
1295 */
1297 /*
1298 * Depending on the compressibility, the pages might or
1299 * might not go through async. We want all of them to
1300 * be accounted against wbc once. Let's do it here
1301 * before the paths diverge. wbc accounting is used
1302 * only for foreign writeback detection and doesn't
1303 * need full accuracy. Just account the whole thing
1304 * against the first page.
1305 */
1312 }
1319 }
1331 }
1334 }
1338 {
1352 }
1356 }
1358 /*
1359 * when nowcow writeback call back. This checks for snapshots or COW copies
1360 * of the extents that exist in the file, and COWs the file as required.
1361 *
1362 * If no cow copies or snapshots exist, we write directly to the existing
1363 * blocks on disk
1364 */
1370 {
1387 EXTENT_DO_ACCOUNTING |
1389 PAGE_CLEAR_DIRTY |
1390 PAGE_SET_WRITEBACK |
1391 PAGE_END_WRITEBACK);
1393 }
1399 u64 extent_end;
1400 u64 extent_offset;
1402 u64 disk_num_bytes;
1403 u64 ram_bytes;
1413 /*
1414 * If there is no extent for our range when doing the initial
1415 * search, then go back to the previous slot as it will be the
1416 * one containing the search offset
1417 */
1425 }
1427 next_slot:
1428 /* Go to next leaf if we have exhausted the current one */
1436 }
1440 }
1444 /* Didn't find anything for our INO */
1447 /*
1448 * Keep searching until we find an EXTENT_ITEM or there are no
1449 * more extents for this inode
1450 */
1455 }
1457 /* Found key is not EXTENT_DATA_KEY or starts after req range */
1462 /*
1463 * If the found extent starts after requested offset, then
1464 * adjust extent_end to be right before this extent begins
1465 */
1470 }
1472 /*
1473 * Found extent which begins before our range and potentially
1474 * intersect it
1475 */
1487 disk_num_bytes =
1489 /*
1490 * If the extent we got ends before our current offset,
1491 * skip to the next extent.
1492 */
1496 }
1497 /* Skip holes */
1500 /* Skip compressed/encrypted/encoded extents */
1505 /*
1506 * If extent is created before the last volume's snapshot
1507 * this implies the extent is shared, hence we can't do
1508 * nocow. This is the same check as in
1509 * btrfs_cross_ref_exist but without calling
1510 * btrfs_search_slot.
1511 */
1518 /* If extent is RO, we must COW it */
1525 /*
1526 * ret could be -EIO if the above fails to read
1527 * metadata.
1528 */
1533 }
1537 }
1541 /*
1542 * If there are pending snapshots for this root, we
1543 * fall into common COW way
1544 */
1547 /*
1548 * force cow if csum exists in the range.
1549 * this ensure that csum for a given extent are
1550 * either valid or do not exist.
1551 */
1553 num_bytes);
1555 /*
1556 * ret could be -EIO if the above fails to read
1557 * metadata.
1558 */
1563 }
1566 }
1573 /* Skip extents outside of our requested range */
1577 }
1579 /* If this triggers then we have a memory corruption */
1581 }
1582 out_check:
1583 /*
1584 * If nocow is false then record the beginning of the range
1585 * that needs to be COWed
1586 */
1595 }
1599 /*
1600 * COW range from cow_start to found_key.offset - 1. As the key
1601 * will contain the beginning of the first extent that can be
1602 * NOCOW, following one which needs to be COW'ed
1603 */
1611 disk_bytenr);
1613 }
1615 }
1622 orig_start,
1627 BTRFS_ORDERED_PREALLOC);
1631 disk_bytenr);
1634 }
1638 num_bytes,
1639 BTRFS_ORDERED_PREALLOC);
1642 cur_offset,
1646 }
1650 num_bytes,
1651 BTRFS_ORDERED_NOCOW);
1654 }
1661 BTRFS_DATA_RELOC_TREE_OBJECTID)
1662 /*
1663 * Error handled later, as we must prevent
1664 * extent_clear_unlock_delalloc() in error handler
1665 * from freeing metadata of created ordered extent.
1666 */
1668 num_bytes);
1673 EXTENT_DELALLOC |
1674 EXTENT_CLEAR_DATA_RESV,
1679 /*
1680 * btrfs_reloc_clone_csums() error, now we're OK to call error
1681 * handler, as metadata for created ordered extent will only
1682 * be freed by btrfs_finish_ordered_io().
1683 */
1688 }
1700 }
1702 error:
1711 PAGE_CLEAR_DIRTY |
1712 PAGE_SET_WRITEBACK |
1713 PAGE_END_WRITEBACK);
1716 }
1719 {
1725 /*
1726 * @defrag_bytes is a hint value, no spinlock held here,
1727 * if is not zero, it means the file is defragging.
1728 * Force cow if given extent needs to be defragged.
1729 */
1736 }
1738 /*
1739 * Function to process delayed allocation (create CoW) for ranges which are
1740 * being touched for the first time.
1741 */
1745 {
1764 }
1769 }
1773 {
1774 u64 size;
1776 /* not delalloc, ignore it */
1782 u32 num_extents;
1783 u64 new_size;
1785 /*
1786 * See the explanation in btrfs_merge_delalloc_extent, the same
1787 * applies here, just in reverse.
1788 */
1795 }
1800 }
1802 /*
1803 * Handle merged delayed allocation extents so we can keep track of new extents
1804 * that are just merged onto old extents, such as when we are doing sequential
1805 * writes, so we can properly account for the metadata space we'll need.
1806 */
1809 {
1811 u32 num_extents;
1813 /* not delalloc, ignore it */
1819 else
1822 /* we're not bigger than the max, unreserve the space and go */
1828 }
1830 /*
1831 * We have to add up either side to figure out how many extents were
1832 * accounted for before we merged into one big extent. If the number of
1833 * extents we accounted for is <= the amount we need for the new range
1834 * then we can return, otherwise drop. Think of it like this
1835 *
1836 * [ 4k][MAX_SIZE]
1837 *
1838 * So we've grown the extent by a MAX_SIZE extent, this would mean we
1839 * need 2 outstanding extents, on one side we have 1 and the other side
1840 * we have 1 so they are == and we can return. But in this case
1841 *
1842 * [MAX_SIZE+4k][MAX_SIZE+4k]
1843 *
1844 * Each range on their own accounts for 2 extents, but merged together
1845 * they are only 3 extents worth of accounting, so we need to drop in
1846 * this case.
1847 */
1858 }
1862 {
1878 }
1879 }
1881 }
1886 {
1900 }
1901 }
1902 }
1906 {
1910 }
1912 /*
1913 * Properly track delayed allocation bytes in the inode and to maintain the
1914 * list of inodes that have pending delalloc work to be done.
1915 */
1918 {
1923 /*
1924 * set_bit and clear bit hooks normally require _irqsave/restore
1925 * but in this case, we are only testing for the DELALLOC
1926 * bit, which is only set or cleared with irqs on
1927 */
1938 /* For sanity tests */
1952 }
1960 }
1961 }
1963 /*
1964 * Once a range is no longer delalloc this function ensures that proper
1965 * accounting happens.
1966 */
1969 {
1979 }
1981 /*
1982 * set_bit and clear bit hooks normally require _irqsave/restore
1983 * but in this case, we are only testing for the DELALLOC
1984 * bit, which is only set or cleared with irqs on
1985 */
1994 /*
1995 * We don't reserve metadata space for space cache inodes so we
1996 * don't need to call delalloc_release_metadata if there is an
1997 * error.
1998 */
2003 /* For sanity tests. */
2023 }
2031 }
2032 }
2034 /*
2035 * btrfs_bio_fits_in_stripe - Checks whether the size of the given bio will fit
2036 * in a chunk's stripe. This function ensures that bios do not span a
2037 * stripe/chunk
2038 *
2039 * @page - The page we are about to add to the bio
2040 * @size - size we want to add to the bio
2041 * @bio - bio we want to ensure is smaller than a stripe
2042 * @bio_flags - flags of the bio
2043 *
2044 * return 1 if page cannot be added to the bio
2045 * return 0 if page can be added to the bio
2046 * return error otherwise
2047 */
2050 {
2055 u64 map_length;
2072 }
2074 /*
2075 * in order to insert checksums into the metadata in large chunks,
2076 * we wait until bio submission time. All the pages in the bio are
2077 * checksummed and sums are attached onto the ordered extent record.
2078 *
2079 * At IO completion time the cums attached on the ordered extent record
2080 * are inserted into the btree
2081 */
2083 u64 bio_offset)
2084 {
2091 }
2093 /*
2094 * extent_io.c submission hook. This does the right thing for csum calculation
2095 * on write, or reading the csums from the tree before a read.
2096 *
2097 * Rules about async/sync submit,
2098 * a) read: sync submit
2099 *
2100 * b) write without checksum: sync submit
2101 *
2102 * c) write with checksum:
2103 * c-1) if bio is issued by fsync: sync submit
2104 * (sync_writers != 0)
2105 *
2106 * c-2) if root is reloc root: sync submit
2107 * (only in case of buffered IO)
2108 *
2109 * c-3) otherwise: async submit
2110 */
2115 {
2135 mirror_num,
2136 bio_flags);
2142 }
2145 /* csum items have already been cloned */
2148 /* we're doing a write, do the async checksumming */
2156 }
2158 mapit:
2161 out:
2165 }
2167 }
2169 /*
2170 * given a list of ordered sums record them in the inode. This happens
2171 * at IO completion time based on sums calculated at bio submission time.
2172 */
2175 {
2186 }
2188 }
2193 {
2197 }
2199 /* see btrfs_writepage_start_hook for details on why this is required */
2204 };
2207 {
2214 u64 page_start;
2215 u64 page_end;
2225 /*
2226 * This is similar to page_mkwrite, we need to reserve the space before
2227 * we take the page lock.
2228 */
2230 PAGE_SIZE);
2231 again:
2234 /*
2235 * Before we queued this fixup, we took a reference on the page.
2236 * page->mapping may go NULL, but it shouldn't be moved to a different
2237 * address space.
2238 */
2240 /*
2241 * Unfortunately this is a little tricky, either
2242 *
2243 * 1) We got here and our page had already been dealt with and
2244 * we reserved our space, thus ret == 0, so we need to just
2245 * drop our space reservation and bail. This can happen the
2246 * first time we come into the fixup worker, or could happen
2247 * while waiting for the ordered extent.
2248 * 2) Our page was already dealt with, but we happened to get an
2249 * ENOSPC above from the btrfs_delalloc_reserve_space. In
2250 * this case we obviously don't have anything to release, but
2251 * because the page was already dealt with we don't want to
2252 * mark the page with an error, so make sure we're resetting
2253 * ret to 0. This is why we have this check _before_ the ret
2254 * check, because we do not want to have a surprise ENOSPC
2255 * when the page was already properly dealt with.
2256 */
2259 PAGE_SIZE);
2263 }
2266 }
2268 /*
2269 * We can't mess with the page state unless it is locked, so now that
2270 * it is locked bail if we failed to make our space reservation.
2271 */
2278 /* already ordered? We're done */
2283 PAGE_SIZE);
2291 }
2298 /*
2299 * Everything went as planned, we're now the owner of a dirty page with
2300 * delayed allocation bits set and space reserved for our COW
2301 * destination.
2302 *
2303 * The page was dirty when we started, nothing should have cleaned it.
2304 */
2307 out_reserved:
2314 out_page:
2316 /*
2317 * We hit ENOSPC or other errors. Update the mapping and page
2318 * to reflect the errors and clean the page.
2319 */
2324 }
2330 /*
2331 * As a precaution, do a delayed iput in case it would be the last iput
2332 * that could need flushing space. Recursing back to fixup worker would
2333 * deadlock.
2334 */
2336 }
2338 /*
2339 * There are a few paths in the higher layers of the kernel that directly
2340 * set the page dirty bit without asking the filesystem if it is a
2341 * good idea. This causes problems because we want to make sure COW
2342 * properly happens and the data=ordered rules are followed.
2343 *
2344 * In our case any range that doesn't have the ORDERED bit set
2345 * hasn't been properly setup for IO. We kick off an async process
2346 * to fix it up. The async helper will wait for ordered extents, set
2347 * the delalloc bit and make it safe to write the page.
2348 */
2350 {
2355 /* this page is properly in the ordered list */
2359 /*
2360 * PageChecked is set below when we create a fixup worker for this page,
2361 * don't try to create another one if we're already PageChecked()
2362 *
2363 * The extent_io writepage code will redirty the page if we send back
2364 * EAGAIN.
2365 */
2373 /*
2374 * We are already holding a reference to this inode from
2375 * write_cache_pages. We need to hold it because the space reservation
2376 * takes place outside of the page lock, and we can't trust
2377 * page->mapping outside of the page lock.
2378 */
2388 }
2396 {
2402 u64 qg_released;
2410 /*
2411 * we may be replacing one extent in the tree with another.
2412 * The new extent is pinned in the extent map, and we don't want
2413 * to drop it from the cache until it is completely in the btree.
2414 *
2415 * So, tell btrfs_drop_extents to leave this extent in the cache.
2416 * the caller is expected to unpin it and allow it to be merged
2417 * with the others.
2418 */
2435 }
2460 ram_bytes);
2464 /*
2465 * Release the reserved range from inode dirty range map, as it is
2466 * already moved into delayed_ref_head
2467 */
2475 out:
2479 }
2483 {
2494 }
2496 /* as ordered data IO finishes, this gets called so we can finish
2497 * an ordered extent if the range of bytes in the file it covers are
2498 * fully written.
2499 */
2501 {
2532 }
2539 /* Truncated the entire extent, don't bother adding */
2542 }
2547 /*
2548 * For mwrite(mmap + memset to write) case, we still reserve
2549 * space for NOCOW range.
2550 * As NOCOW won't cause a new delayed ref, just free the space
2551 */
2557 else
2563 }
2569 }
2576 else
2582 }
2595 logical_len);
2603 BTRFS_FILE_EXTENT_REG);
2609 }
2610 }
2617 }
2623 }
2630 }
2632 out:
2652 /* Drop the cache for the part of the extent we didn't write. */
2655 /*
2656 * If the ordered extent had an IOERR or something else went
2657 * wrong we need to return the space for this ordered extent
2658 * back to the allocator. We only free the extent in the
2659 * truncated case if we didn't write out the extent at all.
2660 *
2661 * If we made it past insert_reserved_file_extent before we
2662 * errored out then we don't need to do this as the accounting
2663 * has already been done.
2664 */
2666 clear_reserved_extent &&
2669 /*
2670 * Discard the range before returning it back to the
2671 * free space pool
2672 */
2677 NULL);
2681 }
2682 }
2684 /*
2685 * This needs to be done to make sure anybody waiting knows we are done
2686 * updating everything for this ordered extent.
2687 */
2690 /* once for us */
2692 /* once for the tree */
2696 }
2699 {
2703 }
2707 {
2722 else
2727 }
2733 {
2755 zeroit:
2762 }
2764 /*
2765 * when reads are done, we need to check csums to verify the data is correct
2766 * if there's a match, we allow the bio to finish. If not, the code in
2767 * extent_io.c will try to find good copies for us.
2768 */
2772 {
2781 }
2790 }
2795 }
2797 /*
2798 * btrfs_add_delayed_iput - perform a delayed iput on @inode
2799 *
2800 * @inode: The inode we want to perform iput on
2801 *
2802 * This function uses the generic vfs_inode::i_count to track whether we should
2803 * just decrement it (in case it's > 1) or if this is the last iput then link
2804 * the inode to the delayed iput machinery. Delayed iputs are processed at
2805 * transaction commit time/superblock commit/cleaner kthread.
2806 */
2808 {
2822 }
2826 {
2833 }
2837 {
2843 }
2844 }
2847 {
2856 }
2858 }
2860 /**
2861 * btrfs_wait_on_delayed_iputs - wait on the delayed iputs to be done running
2862 * @fs_info - the fs_info for this fs
2863 * @return - EINTR if we were killed, 0 if nothing's pending
2864 *
2865 * This will wait on any delayed iputs that are currently running with KILLABLE
2866 * set. Once they are all done running we will return, unless we are killed in
2867 * which case we return EINTR. This helps in user operations like fallocate etc
2868 * that might get blocked on the iputs.
2869 */
2871 {
2877 }
2879 /*
2880 * This creates an orphan entry for the given inode in case something goes wrong
2881 * in the middle of an unlink.
2882 */
2885 {
2892 }
2895 }
2897 /*
2898 * We have done the delete so we can go ahead and remove the orphan item for
2899 * this particular inode.
2900 */
2903 {
2905 }
2907 /*
2908 * this cleans up any orphans that may be left on the list from the last use
2909 * of this root.
2910 */
2912 {
2929 }
2941 /*
2942 * if ret == 0 means we found what we were searching for, which
2943 * is weird, but possible, so only screw with path if we didn't
2944 * find the key and see if we have stuff that matches
2945 */
2951 }
2953 /* pull out the item */
2957 /* make sure the item matches what we want */
2963 /* release the path since we're done with it */
2966 /*
2967 * this is where we are basically btrfs_lookup, without the
2968 * crossing root thing. we store the inode number in the
2969 * offset of the orphan item.
2970 */
2977 }
2994 /*
2995 * this is an orphan in the tree root. Currently these
2996 * could come from 2 sources:
2997 * a) a snapshot deletion in progress
2998 * b) a free space cache inode
2999 * We need to distinguish those two, as the snapshot
3000 * orphan must not get deleted.
3001 * find_dead_roots already ran before us, so if this
3002 * is a snapshot deletion, we should find the root
3003 * in the dead_roots list
3004 */
3007 root_list) {
3012 }
3013 }
3016 /* prevent this orphan from being found again */
3019 }
3021 }
3023 /*
3024 * If we have an inode with links, there are a couple of
3025 * possibilities. Old kernels (before v3.12) used to create an
3026 * orphan item for truncate indicating that there were possibly
3027 * extent items past i_size that needed to be deleted. In v3.12,
3028 * truncate was changed to update i_size in sync with the extent
3029 * items, but the (useless) orphan item was still created. Since
3030 * v4.18, we don't create the orphan item for truncate at all.
3031 *
3032 * So, this item could mean that we need to do a truncate, but
3033 * only if this filesystem was last used on a pre-v3.12 kernel
3034 * and was not cleanly unmounted. The odds of that are quite
3035 * slim, and it's a pain to do the truncate now, so just delete
3036 * the orphan item.
3037 *
3038 * It's also possible that this orphan item was supposed to be
3039 * deleted but wasn't. The inode number may have been reused,
3040 * but either way, we can delete the orphan item.
3041 */
3049 }
3058 }
3060 nr_unlink++;
3062 /* this will do delete_inode and everything for us */
3064 }
3065 /* release the path since we're done with it */
3074 }
3079 out:
3084 }
3086 /*
3087 * very simple check to peek ahead in the leaf looking for xattrs. If we
3088 * don't find any xattrs, we know there can't be any acls.
3089 *
3090 * slot is the slot the inode is in, objectid is the objectid of the inode
3091 */
3095 {
3107 }
3109 slot++;
3114 /* we found a different objectid, there must not be acls */
3118 /* we found an xattr, assume we've got an acl */
3125 }
3127 /*
3128 * we found a key greater than an xattr key, there can't
3129 * be any acls later on
3130 */
3134 slot++;
3135 scanned++;
3137 /*
3138 * it goes inode, inode backrefs, xattrs, extents,
3139 * so if there are a ton of hard links to an inode there can
3140 * be a lot of backrefs. Don't waste time searching too hard,
3141 * this is just an optimization
3142 */
3145 }
3146 /* we hit the end of the leaf before we found an xattr or
3147 * something larger than an xattr. We have to assume the inode
3148 * has acls
3149 */
3153 }
3155 /*
3156 * read an inode from the btree into the in-memory inode
3157 */
3160 {
3169 u32 rdev;
3182 }
3191 }
3235 cache_index:
3236 /*
3237 * If we were modified in the current generation and evicted from memory
3238 * and then re-read we need to do a full sync since we don't have any
3239 * idea about which extents were modified before we were evicted from
3240 * cache.
3241 *
3242 * This is required for both inode re-read from disk and delayed inode
3243 * in delayed_nodes_tree.
3244 */
3249 /*
3250 * We don't persist the id of the transaction where an unlink operation
3251 * against the inode was last made. So here we assume the inode might
3252 * have been evicted, and therefore the exact value of last_unlink_trans
3253 * lost, and set it to last_trans to avoid metadata inconsistencies
3254 * between the inode and its parent if the inode is fsync'ed and the log
3255 * replayed. For example, in the scenario:
3256 *
3257 * touch mydir/foo
3258 * ln mydir/foo mydir/bar
3259 * sync
3260 * unlink mydir/bar
3261 * echo 2 > /proc/sys/vm/drop_caches # evicts inode
3262 * xfs_io -c fsync mydir/foo
3263 * <power failure>
3264 * mount fs, triggers fsync log replay
3265 *
3266 * We must make sure that when we fsync our inode foo we also log its
3267 * parent inode, otherwise after log replay the parent still has the
3268 * dentry with the "bar" name but our inode foo has a link count of 1
3269 * and doesn't have an inode ref with the name "bar" anymore.
3270 *
3271 * Setting last_unlink_trans to last_trans is a pessimistic approach,
3272 * but it guarantees correctness at the expense of occasional full
3273 * transaction commits on fsync if our inode is a directory, or if our
3274 * inode is not a directory, logging its parent unnecessarily.
3275 */
3298 extref);
3299 }
3300 cache_acl:
3301 /*
3302 * try to precache a NULL acl entry for files that don't have
3303 * any xattrs or acls
3304 */
3315 }
3342 }
3346 }
3348 /*
3349 * given a leaf and an inode, copy the inode fields into the leaf
3350 */
3355 {
3397 }
3399 /*
3400 * copy everything in the in-memory inode into the btree.
3401 */
3404 {
3421 }
3431 failed:
3434 }
3436 /*
3437 * copy everything in the in-memory inode into the btree.
3438 */
3441 {
3445 /*
3446 * If the inode is a free space inode, we can deadlock during commit
3447 * if we put it into the delayed code.
3448 *
3449 * The data relocation inode should also be directly updated
3450 * without delay
3451 */
3461 }
3464 }
3469 {
3476 }
3478 /*
3479 * unlink helper that gets used here in inode.c and in the tree logging
3480 * recovery code. It remove a link in a directory with a given name, and
3481 * also drops the back refs in the inode to the directory
3482 */
3488 {
3493 u64 index;
3501 }
3509 }
3515 /*
3516 * If we don't have dir index, we have to get it by looking up
3517 * the inode ref, since we get the inode ref, remove it directly,
3518 * it is unnecessary to do delayed deletion.
3519 *
3520 * But if we have dir index, needn't search inode ref to get it.
3521 * Since the inode ref is close to the inode item, it is better
3522 * that we delay to delete it, and just do this deletion when
3523 * we update the inode item.
3524 */
3530 }
3531 }
3541 }
3542 skip_backref:
3547 }
3550 dir_ino);
3554 }
3557 index);
3563 /*
3564 * If we have a pending delayed iput we could end up with the final iput
3565 * being run in btrfs-cleaner context. If we have enough of these built
3566 * up we can end up burning a lot of time in btrfs-cleaner without any
3567 * way to throttle the unlinks. Since we're currently holding a ref on
3568 * the inode we can run the delayed iput here without any issues as the
3569 * final iput won't be done until after we drop the ref we're currently
3570 * holding.
3571 */
3573 err:
3584 out:
3586 }
3592 {
3598 }
3600 }
3602 /*
3603 * helper to start transaction for unlink and rmdir.
3604 *
3605 * unlink and rmdir are special in btrfs, they do not always free space, so
3606 * if we cannot make our reservations the normal way try and see if there is
3607 * plenty of slack room in the global reserve to migrate, otherwise we cannot
3608 * allow the unlink to occur.
3609 */
3611 {
3614 /*
3615 * 1 for the possible orphan item
3616 * 1 for the dir item
3617 * 1 for the dir index
3618 * 1 for the inode ref
3619 * 1 for the inode
3620 */
3622 }
3625 {
3648 }
3650 out:
3654 }
3658 {
3667 u64 index;
3669 u64 objectid;
3679 }
3690 }
3699 }
3702 /*
3703 * This is a placeholder inode for a subvolume we didn't have a
3704 * reference to at the time of the snapshot creation. In the meantime
3705 * we could have renamed the real subvol link into our snapshot, so
3706 * depending on btrfs_del_root_ref to return -ENOENT here is incorret.
3707 * Instead simply lookup the dir_index_item for this entry so we can
3708 * remove it. Otherwise we know we have a ref to the root and we can
3709 * call btrfs_del_root_ref, and it _shouldn't_ fail.
3710 */
3717 else
3721 }
3734 }
3735 }
3741 }
3749 out:
3752 }
3754 /*
3755 * Helper to check if the subvolume references other subvolumes or if it's
3756 * default.
3757 */
3759 {
3764 u64 dir_id;
3771 /* Make sure this root isn't set as the default subvol */
3783 }
3785 }
3803 }
3804 out:
3807 }
3809 /* Delete all dentries for inodes belonging to the root */
3811 {
3823 again:
3834 else
3836 }
3843 }
3845 }
3846 }
3855 /*
3856 * btrfs_drop_inode will have it removed from the inode
3857 * cache when its usage count hits zero.
3858 */
3863 }
3869 }
3871 }
3874 {
3881 u64 root_flags;
3885 /*
3886 * Don't allow to delete a subvolume with send in progress. This is
3887 * inside the inode lock so the error handling that has to drop the bit
3888 * again is not run concurrently.
3889 */
3897 }
3910 /*
3911 * One for dir inode,
3912 * two for dir entries,
3913 * two for root ref/backref.
3914 */
3923 }
3934 }
3951 }
3952 }
3955 BTRFS_UUID_KEY_SUBVOL,
3961 }
3965 BTRFS_UUID_KEY_RECEIVED_SUBVOL,
3971 }
3972 }
3974 out_end_trans:
3981 out_release:
3983 out_up_write:
3996 /* the last ref */
4000 }
4001 }
4004 }
4007 {
4012 u64 last_unlink_trans;
4026 }
4034 /* now the directory is empty */
4040 /*
4041 * Propagate the last_unlink_trans value of the deleted dir to
4042 * its parent directory. This is to prevent an unrecoverable
4043 * log tree in the case we do something like this:
4044 * 1) create dir foo
4045 * 2) create snapshot under dir foo
4046 * 3) delete the snapshot
4047 * 4) rmdir foo
4048 * 5) mkdir foo
4049 * 6) fsync foo or some file inside foo
4050 */
4053 }
4054 out:
4059 }
4061 /*
4062 * Return this if we need to call truncate_block for the last bit of the
4063 * truncate.
4064 */
4065 #define NEED_TRUNCATE_BLOCK 1
4067 /*
4068 * this can truncate away extent items, csum items and directory items.
4069 * It starts at a high offset and removes keys until it can't find
4070 * any higher than new_size
4071 *
4072 * csum items that cross the new i_size are truncated to the new size
4073 * as well.
4074 *
4075 * min_type is the minimum key type to truncate down to. If set to 0, this
4076 * will kill all the items on this inode, including the INODE_ITEM_KEY.
4077 */
4082 {
4110 /*
4111 * for non-free space inodes and ref cows, we want to back off from
4112 * time to time
4113 */
4127 /*
4128 * We want to drop from the next block forward in case this new size is
4129 * not block aligned since we will be keeping the last block of the
4130 * extent just the way it is.
4131 */
4138 /*
4139 * This function is also used to drop the items in the log tree before
4140 * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4141 * it is used to drop the logged items. So we shouldn't kill the delayed
4142 * items.
4143 */
4151 search_again:
4152 /*
4153 * with a 16K leaf size and 128MB extents, you can actually queue
4154 * up a huge file in a single leaf. Most of the time that
4155 * bytes_deleted is > 0, it will be huge by the time we get here
4156 */
4161 }
4169 /* there are no items in the tree for us to truncate, we're
4170 * done
4171 */
4175 }
4197 item_end +=
4205 fi);
4210 }
4211 item_end--;
4212 }
4220 else
4222 }
4224 /* FIXME, shrink the extent if the ref count is only 1 */
4229 u64 num_dec;
4234 u64 orig_num_bytes =
4241 extent_num_bytes);
4243 extent_num_bytes);
4250 extent_num_bytes =
4252 fi);
4256 /* FIXME blocksize != 4096 */
4263 }
4264 }
4267 /*
4268 * we can't truncate inline items that have had
4269 * special encodings
4270 */
4281 /*
4282 * We have to bail so the last_size is set to
4283 * just before this extent.
4284 */
4288 /*
4289 * Inline extents are special, we just treat
4290 * them as a full sector worth in the file
4291 * extent tree just for simplicity sake.
4292 */
4294 }
4298 }
4300 /*
4301 * We use btrfs_truncate_inode_items() to clean up log trees for
4302 * multiple fsyncs, and in this case we don't want to clear the
4303 * file extent range because it's just the log.
4304 */
4311 }
4312 }
4316 else
4320 /* no pending yet, add ourselves */
4325 /* hop on the pending chunk */
4326 pending_del_nr++;
4330 }
4333 }
4352 }
4356 }
4357 }
4364 should_throttle) {
4367 pending_del_slot,
4368 pending_del_nr);
4372 }
4374 }
4377 /*
4378 * We can generate a lot of delayed refs, so we need to
4379 * throttle every once and a while and make sure we're
4380 * adding enough space to keep up with the work we are
4381 * generating. Since we hold a transaction here we
4382 * can't flush, and we don't want to FLUSH_LIMIT because
4383 * we could have generated too many delayed refs to
4384 * actually allocate, so just bail if we're short and
4385 * let the normal reservation dance happen higher up.
4386 */
4389 BTRFS_RESERVE_NO_FLUSH);
4393 }
4394 }
4398 }
4399 }
4400 out:
4405 pending_del_nr);
4409 }
4410 }
4418 }
4422 }
4424 /*
4425 * btrfs_truncate_block - read, zero a chunk and write a block
4426 * @inode - inode that we're zeroing
4427 * @from - the offset to start zeroing
4428 * @len - the length to zero, 0 to zero the entire range respective to the
4429 * offset
4430 * @front - zero up to the offset instead of from the offset on
4431 *
4432 * This will find the block for the "from" offset and cow the block and zero the
4433 * part we want to zero. This is used with truncate and hole punching.
4434 */
4437 {
4451 u64 block_start;
4452 u64 block_end;
4466 again:
4474 }
4483 }
4487 }
4488 }
4503 }
4515 }
4524 else
4529 }
4534 out_unlock:
4541 out:
4544 }
4548 {
4553 /*
4554 * Still need to make sure the inode looks like it's been updated so
4555 * that any holes get logged if we fsync.
4556 */
4562 }
4564 /*
4565 * 1 - for the one we're dropping
4566 * 1 - for the one we're adding
4567 * 1 - for updating the inode.
4568 */
4578 }
4584 else
4588 }
4590 /*
4591 * This function puts in dummy file extents for the area we're creating a hole
4592 * for. So if we are truncating this file to a larger size we need to insert
4593 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4594 * the range between oldsize and size
4595 */
4597 {
4606 u64 last_byte;
4607 u64 cur_offset;
4608 u64 hole_size;
4611 /*
4612 * If our size started in the middle of a block we need to zero out the
4613 * rest of the block before we expand the i_size, otherwise we could
4614 * expose stale data.
4615 */
4633 }
4642 hole_size);
4658 }
4677 cur_offset,
4678 cur_offset +
4680 }
4687 }
4688 next:
4694 }
4698 }
4701 {
4709 /*
4710 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
4711 * special case where we need to update the times despite not having
4712 * these flags set. For all other operations the VFS set these flags
4713 * explicitly if it wants a timestamp update.
4714 */
4720 }
4723 /*
4724 * Don't do an expanding truncate while snapshotting is ongoing.
4725 * This is to ensure the snapshot captures a fully consistent
4726 * state of this file - if the snapshot captures this expanding
4727 * truncation, it must capture all writes that happened before
4728 * this truncation.
4729 */
4735 }
4741 }
4751 /*
4752 * We're truncating a file that used to have good data down to
4753 * zero. Make sure it gets into the ordered flush list so that
4754 * any new writes get down to disk quickly.
4755 */
4762 /* Disable nonlocked read DIO to avoid the endless truncate */
4771 /*
4772 * Truncate failed, so fix up the in-memory size. We
4773 * adjusted disk_i_size down as we removed extents, so
4774 * wait for disk_i_size to be stable and then update the
4775 * in-memory size to match.
4776 */
4781 }
4782 }
4785 }
4788 {
4804 }
4813 }
4816 }
4818 /*
4819 * While truncating the inode pages during eviction, we get the VFS calling
4820 * btrfs_invalidatepage() against each page of the inode. This is slow because
4821 * the calls to btrfs_invalidatepage() result in a huge amount of calls to
4822 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
4823 * extent_state structures over and over, wasting lots of time.
4824 *
4825 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
4826 * those expensive operations on a per page basis and do only the ordered io
4827 * finishing, while we release here the extent_map and extent_state structures,
4828 * without the excessive merging and splitting.
4829 */
4831 {
4853 }
4854 }
4857 /*
4858 * Keep looping until we have no more ranges in the io tree.
4859 * We can have ongoing bios started by readpages (called from readahead)
4860 * that have their endio callback (extent_io.c:end_bio_extent_readpage)
4861 * still in progress (unlocked the pages in the bio but did not yet
4862 * unlocked the ranges in the io tree). Therefore this means some
4863 * ranges can still be locked and eviction started because before
4864 * submitting those bios, which are executed by a separate task (work
4865 * queue kthread), inode references (inode->i_count) were not taken
4866 * (which would be dropped in the end io callback of each bio).
4867 * Therefore here we effectively end up waiting for those bios and
4868 * anyone else holding locked ranges without having bumped the inode's
4869 * reference count - if we don't do it, when they access the inode's
4870 * io_tree to unlock a range it may be too late, leading to an
4871 * use-after-free issue.
4872 */
4877 u64 start;
4878 u64 end;
4890 /*
4891 * If still has DELALLOC flag, the extent didn't reach disk,
4892 * and its reserved space won't be freed by delayed_ref.
4893 * So we need to free its reserved space here.
4894 * (Refer to comment in btrfs_invalidatepage, case 2)
4895 *
4896 * Note, end is the bytenr of last byte, so we need + 1 here.
4897 */
4908 }
4910 }
4914 {
4921 /*
4922 * Eviction should be taking place at some place safe because of our
4923 * delayed iputs. However the normal flushing code will run delayed
4924 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.
4925 *
4926 * We reserve the delayed_refs_extra here again because we can't use
4927 * btrfs_start_transaction(root, 0) for the same deadlocky reason as
4928 * above. We reserve our extra bit here because we generate a ton of
4929 * delayed refs activity by truncating.
4930 *
4931 * If we cannot make our reservation we'll attempt to steal from the
4932 * global reserve, because we really want to be able to free up space.
4933 */
4935 BTRFS_RESERVE_FLUSH_EVICT);
4937 /*
4938 * Try to steal from the global reserve if there is space for
4939 * it.
4940 */
4946 }
4948 }
4959 }
4961 }
4964 {
4976 }
4998 }
5027 }
5029 /*
5030 * Errors here aren't a big deal, it just means we leave orphan items in
5031 * the tree. They will be cleaned up on the next mount. If the inode
5032 * number gets reused, cleanup deletes the orphan item without doing
5033 * anything, and unlink reuses the existing orphan item.
5034 *
5035 * If it turns out that we are dropping too many of these, we might want
5036 * to add a mechanism for retrying these after a commit.
5037 */
5044 }
5050 free_rsv:
5052 no_delete:
5053 /*
5054 * If we didn't successfully delete, the orphan item will still be in
5055 * the tree and we'll retry on the next mount. Again, we might also want
5056 * to retry these periodically in the future.
5057 */
5060 }
5062 /*
5063 * Return the key found in the dir entry in the location pointer, fill @type
5064 * with BTRFS_FT_*, and return 0.
5065 *
5066 * If no dir entries were found, returns -ENOENT.
5067 * If found a corrupted location in dir entry, returns -EUCLEAN.
5068 */
5071 {
5088 }
5098 }
5101 out:
5104 }
5106 /*
5107 * when we hit a tree root in a directory, the btrfs part of the inode
5108 * needs to be changed to reflect the root directory of the tree root. This
5109 * is kind of like crossing a mount point.
5110 */
5116 {
5129 }
5141 }
5161 }
5168 out:
5171 }
5174 {
5202 }
5203 }
5207 }
5210 {
5219 }
5228 }
5229 }
5233 {
5241 }
5244 {
5248 }
5253 {
5262 btrfs_init_locked_inode,
5265 }
5267 /*
5268 * Get an inode object given its location and corresponding root.
5269 * Path can be preallocated to prevent recursing back to iget through
5270 * allocator. NULL is also valid but may require an additional allocation
5271 * later.
5272 */
5275 {
5291 /*
5292 * ret > 0 can come from btrfs_search_slot called by
5293 * btrfs_read_locked_inode, this means the inode item
5294 * was not found.
5295 */
5299 }
5300 }
5303 }
5307 {
5309 }
5314 {
5325 /*
5326 * We only need lookup, the rest is read-only and there's no inode
5327 * associated with the dentry
5328 */
5339 }
5342 {
5343 /*
5344 * Compile-time asserts that generic FT_* types still match
5345 * BTRFS_FT_* types
5346 */
5357 }
5360 {
5381 /* Do extra check against inode mode with di_type */
5386 di_type);
5389 }
5391 }
5398 else
5402 }
5414 }
5415 }
5418 }
5421 {
5435 }
5437 }
5441 {
5447 }
5449 /*
5450 * All this infrastructure exists because dir_emit can fault, and we are holding
5451 * the tree lock when doing readdir. For now just allocate a buffer and copy
5452 * our information into that, and then dir_emit from the buffer. This is
5453 * similar to what NFS does, only we don't keep the buffer around in pagecache
5454 * because I'm afraid I'll mess that up. Long term we need to make filldir do
5455 * copy_to_user_inatomic so we don't have to worry about page faulting under the
5456 * tree lock.
5457 */
5459 {
5469 }
5472 }
5475 u64 ino;
5476 u64 offset;
5479 };
5482 {
5495 }
5497 }
5500 {
5535 again:
5556 }
5571 PAGE_SIZE) {
5580 }
5586 name_len);
5592 entries++;
5595 next:
5597 }
5608 /*
5609 * Stop new entries from being returned after we return the last
5610 * entry.
5611 *
5612 * New directory entries are assigned a strictly increasing
5613 * offset. This means that new entries created during readdir
5614 * are *guaranteed* to be seen in the future by that readdir.
5615 * This has broken buggy programs which operate on names as
5616 * they're returned by readdir. Until we re-use freed offsets
5617 * we have this hack to stop new entries from being returned
5618 * under the assumption that they'll never reach this huge
5619 * offset.
5620 *
5621 * This is being careful not to overflow 32bit loff_t unless the
5622 * last entry requires it because doing so has broken 32bit apps
5623 * in the past.
5624 */
5627 else
5629 nopos:
5631 err:
5636 }
5638 /*
5639 * This is somewhat expensive, updating the tree every time the
5640 * inode changes. But, it is most likely to find the inode in cache.
5641 * FIXME, needs more benchmarking...there are no reasons other than performance
5642 * to keep or drop this code.
5643 */
5645 {
5660 /* whoops, lets try again with the full transaction */
5667 }
5673 }
5675 /*
5676 * This is a copy of file_update_time. We need this so we can return error on
5677 * ENOSPC for updating the inode in the case of file write and mmap writes.
5678 */
5681 {
5697 }
5699 /*
5700 * find the highest existing sequence number in a directory
5701 * and then set the in-memory index_cnt variable to reflect
5702 * free sequence numbers
5703 */
5705 {
5723 /* FIXME: we should be able to handle this */
5728 /*
5729 * MAGIC NUMBER EXPLANATION:
5730 * since we search a directory based on f_pos we have to start at 2
5731 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
5732 * else has to start at 2
5733 */
5737 }
5748 }
5751 out:
5754 }
5756 /*
5757 * helper to find a free sequence number in a given directory. This current
5758 * code is very simple, later versions will do smarter things in the btree
5759 */
5761 {
5770 }
5771 }
5777 }
5780 {
5788 }
5790 /*
5791 * Inherit flags from the parent inode.
5792 *
5793 * Currently only the compression flags and the cow flags are inherited.
5794 */
5796 {
5810 }
5816 }
5819 }
5827 {
5851 }
5853 /*
5854 * O_TMPFILE, set link count to 0, so that after this point,
5855 * we fill in an inode item with the correct link count.
5856 */
5860 /*
5861 * we have to initialize this early, so we can reclaim the inode
5862 * number if we fail afterwards in this function.
5863 */
5874 }
5877 }
5878 /*
5879 * index_cnt is ignored for everything but a dir,
5880 * btrfs_set_inode_index_count has an explanation for the magic
5881 * number
5882 */
5889 /*
5890 * We could have gotten an inode number from somebody who was fsynced
5891 * and then removed in this same transaction, so let's just set full
5892 * sync since it will be a full sync anyway and this will blow away the
5893 * old info in the log.
5894 */
5904 /*
5905 * Start new inodes with an inode_ref. This is slightly more
5906 * efficient for small numbers of hard links since they will
5907 * be packed into one item. Extended refs will kick in if we
5908 * add more hard links than can fit in the ref item.
5909 */
5915 }
5926 }
5954 }
5966 BTRFS_INODE_NODATASUM;
5967 }
5984 fail_unlock:
5986 fail:
5991 }
5993 /*
5994 * utility function to add 'inode' into 'parent_inode' with
5995 * a give name and a given sequence number.
5996 * if 'add_backref' is true, also insert a backref from the
5997 * inode to the parent directory.
5998 */
6002 {
6015 }
6024 }
6026 /* Nothing to clean up yet */
6037 }
6042 /*
6043 * If we are replaying a log tree, we do not want to update the mtime
6044 * and ctime of the parent directory with the current time, since the
6045 * log replay procedure is responsible for setting them to their correct
6046 * values (the ones it had when the fsync was done).
6047 */
6053 }
6059 fail_dir_item:
6061 u64 local_index;
6069 u64 local_index;
6076 }
6078 /* Return the original error code */
6080 }
6085 {
6092 }
6096 {
6102 u64 objectid;
6105 /*
6106 * 2 for inode item and ref
6107 * 2 for dir items
6108 * 1 for xattr if selinux is on
6109 */
6125 }
6127 /*
6128 * If the active LSM wants to access the inode during
6129 * d_instantiate it needs these. Smack checks to see
6130 * if the filesystem supports xattrs by looking at the
6131 * ops vector.
6132 */
6148 out_unlock:
6154 }
6156 }
6160 {
6166 u64 objectid;
6169 /*
6170 * 2 for inode item and ref
6171 * 2 for dir items
6172 * 1 for xattr if selinux is on
6173 */
6189 }
6190 /*
6191 * If the active LSM wants to access the inode during
6192 * d_instantiate it needs these. Smack checks to see
6193 * if the filesystem supports xattrs by looking at the
6194 * ops vector.
6195 */
6216 out_unlock:
6221 }
6224 }
6228 {
6233 u64 index;
6237 /* do not allow sys_link's with other subvols of the same device */
6248 /*
6249 * 2 items for inode and inode ref
6250 * 2 items for dir items
6251 * 1 item for parent inode
6252 * 1 item for orphan item deletion if O_TMPFILE
6253 */
6259 }
6261 /* There are several dir indexes for this inode, clear the cache. */
6282 /*
6283 * If new hard link count is 1, it's a file created
6284 * with open(2) O_TMPFILE flag.
6285 */
6289 }
6296 }
6297 }
6299 fail:
6305 }
6308 }
6311 {
6320 /*
6321 * 2 items for inode and ref
6322 * 2 items for dir items
6323 * 1 for xattr if selinux is on
6324 */
6340 }
6342 /* these must be set before we unlock the inode */
6363 out_fail:
6368 }
6371 }
6377 {
6402 /*
6403 * decompression code contains a memset to fill in any space between the end
6404 * of the uncompressed data and the end of max_size in case the decompressed
6405 * data ends up shorter than ram_bytes. That doesn't cover the hole between
6406 * the end of an inline extent and the beginning of the next block, so we
6407 * cover that region here.
6408 */
6414 }
6417 }
6419 /**
6420 * btrfs_get_extent - Lookup the first extent overlapping a range in a file.
6421 * @inode: file to search in
6422 * @page: page to read extent data into if the extent is inline
6423 * @pg_offset: offset into @page to copy to
6424 * @start: file offset
6425 * @len: length of range starting at @start
6426 *
6427 * This returns the first &struct extent_map which overlaps with the given
6428 * range, reading it from the B-tree and caching it if necessary. Note that
6429 * there may be more extents which overlap the given range after the returned
6430 * extent_map.
6431 *
6432 * If @page is not NULL and the extent is inline, this also reads the extent
6433 * data directly into the page and marks the extent up to date in the io_tree.
6434 *
6435 * Return: ERR_PTR on error, non-NULL extent_map on success.
6436 */
6440 {
6466 else
6468 }
6473 }
6483 }
6485 /* Chances are we'll be called again, so go ahead and do readahead */
6488 /*
6489 * Unless we're going to uncompress the inline extent, no sleep would
6490 * happen.
6491 */
6502 }
6510 /*
6511 * If we backup past the first extent we want to move forward
6512 * and see if there is an extent in front of us, otherwise we'll
6513 * say there is a hole for our whole search range which can
6514 * cause problems.
6515 */
6518 }
6525 /* Only regular file could have regular/prealloc extent */
6532 }
6534 extent_start);
6538 extent_start);
6539 }
6540 next:
6550 }
6552 }
6562 /* New extent overlaps with existing one */
6568 }
6598 BTRFS_COMPRESS_NONE) {
6604 }
6608 copy_size);
6612 copy_size);
6613 }
6615 }
6617 }
6621 }
6622 not_found:
6627 insert:
6635 }
6641 out:
6649 }
6652 }
6656 {
6660 u64 end;
6661 u64 delalloc_len;
6662 u64 delalloc_end;
6668 /*
6669 * If our em maps to:
6670 * - a hole or
6671 * - a pre-alloc extent,
6672 * there might actually be delalloc bytes behind it.
6673 */
6677 else
6680 /* check to see if we've wrapped (len == -1 or similar) */
6684 else
6689 /* ok, we didn't find anything, lets look for delalloc */
6696 /*
6697 * We didn't find anything useful, return the original results from
6698 * get_extent()
6699 */
6704 }
6706 /*
6707 * Adjust the delalloc_start to make sure it doesn't go backwards from
6708 * the start they passed in
6709 */
6714 u64 hole_start;
6715 u64 hole_len;
6722 }
6725 /*
6726 * When btrfs_get_extent can't find anything it returns one
6727 * huge hole
6728 *
6729 * Make sure what it found really fits our range, and adjust to
6730 * make sure it is based on the start from the caller
6731 */
6738 }
6741 /*
6742 * Our hole starts before our delalloc, so we have to
6743 * return just the parts of the hole that go until the
6744 * delalloc starts
6745 */
6749 /*
6750 * Don't adjust block start at all, it is fixed at
6751 * EXTENT_MAP_HOLE
6752 */
6758 /*
6759 * Hole is out of passed range or it starts after
6760 * delalloc range
6761 */
6767 }
6770 }
6771 out:
6777 }
6779 }
6790 {
6797 ram_bytes,
6799 type);
6802 }
6810 }
6812 }
6813 out:
6816 }
6820 {
6825 u64 alloc_hint;
6843 }
6845 /*
6846 * returns 1 when the nocow is safe, < 1 on error, 0 if the
6847 * block must be cow'd
6848 */
6852 {
6861 u64 disk_bytenr;
6862 u64 backref_offset;
6863 u64 extent_end;
6864 u64 num_bytes;
6881 /* can't find the item, must cow */
6884 }
6885 slot--;
6886 }
6892 /* not our file or wrong item type, must cow */
6894 }
6897 /* Wrong offset, must cow */
6899 }
6905 /* not a regular extent, must cow */
6907 }
6925 /*
6926 * Do the same check as in btrfs_cross_ref_exist but without the
6927 * unnecessary search.
6928 */
6939 }
6946 u64 range_end;
6955 }
6956 }
6960 /*
6961 * look for other files referencing this extent, if we
6962 * find any we must cow
6963 */
6970 }
6972 /*
6973 * adjust disk_bytenr and num_bytes to cover just the bytes
6974 * in this extent we are about to write. If there
6975 * are any csums in that range we have to cow in order
6976 * to keep the csums correct
6977 */
6982 /*
6983 * all of the above have passed, it is safe to overwrite this extent
6984 * without cow
6985 */
6988 out:
6991 }
6995 {
7001 cached_state);
7002 /*
7003 * We're concerned with the entire range that we're going to be
7004 * doing DIO to, so we need to make sure there's no ordered
7005 * extents in this range.
7006 */
7010 /*
7011 * We need to make sure there are no buffered pages in this
7012 * range either, we could have raced between the invalidate in
7013 * generic_file_direct_write and locking the extent. The
7014 * invalidate needs to happen so that reads after a write do not
7015 * get stale data.
7016 */
7023 cached_state);
7026 /*
7027 * If we are doing a DIO read and the ordered extent we
7028 * found is for a buffered write, we can not wait for it
7029 * to complete and retry, because if we do so we can
7030 * deadlock with concurrent buffered writes on page
7031 * locks. This happens only if our DIO read covers more
7032 * than one extent map, if at this point has already
7033 * created an ordered extent for a previous extent map
7034 * and locked its range in the inode's io tree, and a
7035 * concurrent write against that previous extent map's
7036 * range and this range started (we unlock the ranges
7037 * in the io tree only when the bios complete and
7038 * buffered writes always lock pages before attempting
7039 * to lock range in the io tree).
7040 */
7044 else
7048 /*
7049 * We could trigger writeback for this range (and wait
7050 * for it to complete) and then invalidate the pages for
7051 * this range (through invalidate_inode_pages2_range()),
7052 * but that can lead us to a deadlock with a concurrent
7053 * call to readpages() (a buffered read or a defrag call
7054 * triggered a readahead) on a page lock due to an
7055 * ordered dio extent we created before but did not have
7056 * yet a corresponding bio submitted (whence it can not
7057 * complete), which makes readpages() wait for that
7058 * ordered extent to complete while holding a lock on
7059 * that page.
7060 */
7062 }
7068 }
7071 }
7073 /* The callers of this must take lock_extent() */
7079 {
7108 }
7116 /*
7117 * The caller has taken lock_extent(), who could race with us
7118 * to add em?
7119 */
7125 }
7127 /* em got 2 refs now, callers needs to do free_extent_map once. */
7129 }
7136 {
7152 }
7159 {
7164 /*
7165 * We don't allocate a new extent in the following cases
7166 *
7167 * 1) The inode is marked as NODATACOW. In this case we'll just use the
7168 * existing extent.
7169 * 2) The extent is marked as PREALLOC. We're good to go here and can
7170 * just use the extent.
7171 *
7172 */
7181 else
7199 }
7204 }
7205 /*
7206 * For inode marked NODATACOW or extent marked PREALLOC,
7207 * use the existing or preallocated extent, so does not
7208 * need to adjust btrfs_space_info's bytes_may_use.
7209 */
7211 len);
7213 }
7214 }
7216 /* this will cow the extent */
7223 }
7227 skip_cow:
7237 /*
7238 * Need to update the i_size under the extent lock so buffered
7239 * readers will get the updated i_size when we unlock.
7240 */
7248 out:
7250 }
7254 {
7271 /*
7272 * Need to pull our outstanding extents and set journal_info to NULL so
7273 * that anything that needs to check if there's a transaction doesn't get
7274 * confused.
7275 */
7278 }
7280 /*
7281 * If this errors out it's because we couldn't invalidate pagecache for
7282 * this range and we need to fallback to buffered.
7283 */
7285 create)) {
7288 }
7294 }
7296 /*
7297 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7298 * io. INLINE is special, and we could probably kludge it in here, but
7299 * it's still buffered so for safety lets just fall back to the generic
7300 * buffered path.
7301 *
7302 * For COMPRESSED we _have_ to read the entire extent in so we can
7303 * decompress it, so there will be buffering required no matter what we
7304 * do, so go ahead and fallback to buffered.
7305 *
7306 * We return -ENOTBLK because that's what makes DIO go ahead and go back
7307 * to buffered IO. Don't blame me, this is the price we pay for using
7308 * the generic code.
7309 */
7315 }
7328 /* Can be negative only if we read from a hole */
7333 }
7334 /*
7335 * We need to unlock only the end area that we aren't using.
7336 * The rest is going to be unlocked by the endio routine.
7337 */
7344 }
7345 }
7351 unlock_err:
7354 err:
7358 }
7363 {
7365 blk_status_t ret;
7376 }
7382 {
7388 /*
7389 * we only have a single copy of the data, so don't bother with
7390 * all the retry and error correction code that follows. no
7391 * matter what the error is, it is very likely to persist.
7392 */
7397 }
7409 }
7412 }
7418 {
7427 blk_status_t status;
7437 failed_mirror);
7441 }
7463 }
7466 }
7471 u64 start;
7473 };
7476 {
7497 end:
7500 }
7504 {
7509 u64 start;
7511 u32 sectorsize;
7513 blk_status_t ret;
7527 next_block_or_try_again:
7539 }
7544 /* We might have another mirror, so try again */
7546 }
7548 next:
7551 nr_sectors--;
7556 }
7557 }
7560 }
7563 {
7596 else
7598 i++;
7599 }
7602 end:
7605 }
7609 {
7614 u64 start;
7616 u32 sectorsize;
7622 blk_status_t status;
7636 next_block:
7643 }
7644 try_again:
7656 }
7661 /* We might have another mirror, so try again */
7663 }
7664 next:
7670 nr_sectors--;
7675 }
7676 }
7679 }
7683 {
7689 else
7693 }
7694 }
7697 {
7717 }
7722 {
7728 u64 last_offset;
7732 else
7739 ordered_bytes,
7740 uptodate)) {
7742 NULL);
7744 }
7745 /*
7746 * If btrfs_dec_test_ordered_pending does not find any ordered
7747 * extent in the range, we can exit.
7748 */
7751 /*
7752 * Our bio might span multiple ordered extents. In this case
7753 * we keep going until we have accounted the whole dio.
7754 */
7758 }
7759 }
7760 }
7763 {
7775 }
7779 {
7781 blk_status_t ret;
7785 }
7788 {
7804 /*
7805 * We want to perceive the errors flag being set before
7806 * decrementing the reference count. We don't need a barrier
7807 * since atomic operations with a return value are fully
7808 * ordered as per atomic_t.txt
7809 */
7811 }
7813 /* if there are more bios still pending for this dio, just exit */
7822 }
7823 out:
7825 }
7830 u64 file_offset)
7831 {
7834 u16 csum_size;
7835 blk_status_t ret;
7837 /*
7838 * We load all the csum data we need when we submit
7839 * the first bio to reduce the csum tree search and
7840 * contention.
7841 */
7844 NULL);
7847 }
7858 }
7862 {
7866 blk_status_t ret;
7868 /* Check btrfs_submit_bio_hook() for rules about async submit. */
7876 }
7884 btrfs_submit_bio_start_direct_io);
7887 /*
7888 * If we aren't doing async submit, calculate the csum of the
7889 * bio now.
7890 */
7896 file_offset);
7899 }
7900 map:
7902 err:
7904 }
7907 {
7915 u64 submit_len;
7919 blk_status_t status;
7932 }
7934 /* async crcs make it difficult to collect full stripe writes. */
7937 else
7940 /* bio split */
7946 /*
7947 * This will never fail as it's passing GPF_NOFS and
7948 * the allocation is backed by btrfs_bioset.
7949 */
7951 clone_len);
7961 /*
7962 * Increase the count before we submit the bio so we know
7963 * the end IO handler won't happen before we increase the
7964 * count. Otherwise, the dip might get freed before we're
7965 * done setting it up.
7966 */
7970 async_submit);
7975 }
7987 submit:
7993 out_err:
7995 /*
7996 * Before atomic variable goto zero, we must make sure dip->errors is
7997 * perceived to be set. This ordering is ensured by the fact that an
7998 * atomic operations with a return value are fully ordered as per
7999 * atomic_t.txt
8000 */
8004 /* bio_end_io() will handle error, so we needn't return it */
8006 }
8009 loff_t file_offset)
8010 {
8023 }
8042 }
8044 /*
8045 * Reset the range for unsubmitted ordered extents (to a 0 length range)
8046 * even if we fail to submit a bio, because in such case we do the
8047 * corresponding error handling below and it must not be done a second
8048 * time by btrfs_direct_IO().
8049 */
8057 }
8065 free_ordered:
8066 /*
8067 * If we arrived here it means either we failed to submit the dip
8068 * or we either failed to clone the dio_bio or failed to allocate the
8069 * dip. If we cloned the dio_bio and allocated the dip, we can just
8070 * call bio_endio against our io_bio so that we get proper resource
8071 * cleanup if we fail to submit the dip, otherwise, we must do the
8072 * same as btrfs_endio_direct_[write|read] because we can't call these
8073 * callbacks - they require an allocated dip and a clone of dio_bio.
8074 */
8077 /*
8078 * The end io callbacks free our dip, do the final put on bio
8079 * and all the cleanup and final put for dio_bio (through
8080 * dio_end_io()).
8081 */
8087 file_offset,
8090 else
8095 /*
8096 * Releases and cleans up our dio_bio, no need to bio_put()
8097 * nor bio_endio()/bio_io_error() against dio_bio.
8098 */
8100 }
8104 }
8108 {
8120 /* If this is a write we don't need to check anymore */
8123 /*
8124 * Check to make sure we don't have duplicate iov_base's in this
8125 * iovec, if so return EINVAL, otherwise we'll get csum errors
8126 * when reading back.
8127 */
8132 }
8133 }
8135 out:
8137 }
8140 {
8151 ssize_t ret;
8158 /*
8159 * The generic stuff only does filemap_write_and_wait_range, which
8160 * isn't enough if we've written compressed pages to this area, so
8161 * we need to flush the dirty pages again to make absolutely sure
8162 * that any outstanding dirty pages are on disk.
8163 */
8171 /*
8172 * If the write DIO is beyond the EOF, we need update
8173 * the isize, but it is protected by i_mutex. So we can
8174 * not unlock the i_mutex at this case.
8175 */
8183 }
8189 /*
8190 * We need to know how many extents we reserved so that we can
8191 * do the accounting properly if we go over the number we
8192 * originally calculated. Abuse current->journal_info for this.
8193 */
8205 }
8218 /*
8219 * On error we might have left some ordered extents
8220 * without submitting corresponding bios for them, so
8221 * cleanup them up to avoid other tasks getting them
8222 * and waiting for them to complete forever.
8223 */
8235 }
8236 out:
8244 }
8246 #define BTRFS_FIEMAP_FLAGS (FIEMAP_FLAG_SYNC)
8250 {
8258 }
8261 {
8263 }
8266 {
8274 }
8276 /*
8277 * If we are under memory pressure we will call this directly from the
8278 * VM, we need to make sure we have the inode referenced for the ordered
8279 * extent. If not just return like we didn't do anything.
8280 */
8284 }
8288 }
8292 {
8294 }
8296 static int
8299 {
8301 }
8304 {
8310 }
8312 }
8315 {
8319 }
8321 #ifdef CONFIG_MIGRATION
8325 {
8339 }
8344 }
8348 else
8351 }
8352 #endif
8356 {
8363 u64 start;
8364 u64 end;
8367 /*
8368 * we have the page locked, so new writeback can't start,
8369 * and the dirty bit won't be cleared while we are here.
8370 *
8371 * Wait for IO on this page so that we can safely clear
8372 * the PagePrivate2 bit and do ordered accounting
8373 */
8380 }
8384 again:
8391 /*
8392 * IO on this page will never be started, so we need
8393 * to account for any ordered extents now
8394 */
8400 /*
8401 * whoever cleared the private bit is responsible
8402 * for the finish_ordered_io
8403 */
8406 u64 new_len;
8418 start,
8421 }
8427 }
8432 }
8434 /*
8435 * Qgroup reserved space handler
8436 * Page here will be either
8437 * 1) Already written to disk
8438 * In this case, its reserved space is released from data rsv map
8439 * and will be freed by delayed_ref handler finally.
8440 * So even we call qgroup_free_data(), it won't decrease reserved
8441 * space.
8442 * 2) Not written to disk
8443 * This means the reserved space should be freed here. However,
8444 * if a truncate invalidates the page (by clearing PageDirty)
8445 * and the page is accounted for while allocating extent
8446 * in btrfs_check_data_free_space() we let delayed_ref to
8447 * free the entire extent.
8448 */
8458 }
8465 }
8466 }
8468 /*
8469 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
8470 * called from a page fault handler when a page is first dirtied. Hence we must
8471 * be careful to check for EOF conditions here. We set the page up correctly
8472 * for a written page which means we get ENOSPC checking when writing into
8473 * holes and correct delalloc and unwritten extent mapping on filesystems that
8474 * support these features.
8475 *
8476 * We are not allowed to take the i_mutex here so we have to play games to
8477 * protect against truncate races as the page could now be beyond EOF. Because
8478 * truncate_setsize() writes the inode size before removing pages, once we have
8479 * the page lock we can determine safely if the page is beyond EOF. If it is not
8480 * beyond EOF, then the page is guaranteed safe against truncation until we
8481 * unlock the page.
8482 */
8484 {
8494 loff_t size;
8495 vm_fault_t ret;
8498 u64 reserved_space;
8499 u64 page_start;
8500 u64 page_end;
8501 u64 end;
8510 /*
8511 * Reserving delalloc space after obtaining the page lock can lead to
8512 * deadlock. For example, if a dirty page is locked by this function
8513 * and the call to btrfs_delalloc_reserve_space() ends up triggering
8514 * dirty page write out, then the btrfs_writepage() function could
8515 * end up waiting indefinitely to get a lock on the page currently
8516 * being processed by btrfs_page_mkwrite() function.
8517 */
8519 reserved_space);
8523 }
8529 }
8532 again:
8538 /* page got truncated out from underneath us */
8540 }
8546 /*
8547 * we can't set the delalloc bits if there are pending ordered
8548 * extents. Drop our locks and wait for them to finish
8549 */
8551 PAGE_SIZE);
8559 }
8569 }
8570 }
8572 /*
8573 * page_mkwrite gets called when the page is firstly dirtied after it's
8574 * faulted in, but write(2) could also dirty a page and set delalloc
8575 * bits, thus in this case for space account reason, we still need to
8576 * clear any delalloc bits within this page range since we have to
8577 * reserve data&meta space before lock_page() (see above comments).
8578 */
8590 }
8592 /* page is wholly or partially inside EOF */
8595 else
8603 }
8619 out_unlock:
8621 out:
8625 out_noreserve:
8629 }
8632 {
8646 }
8648 /*
8649 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of
8650 * things going on here:
8651 *
8652 * 1) We need to reserve space to update our inode.
8653 *
8654 * 2) We need to have something to cache all the space that is going to
8655 * be free'd up by the truncate operation, but also have some slack
8656 * space reserved in case it uses space during the truncate (thank you
8657 * very much snapshotting).
8658 *
8659 * And we need these to be separate. The fact is we can use a lot of
8660 * space doing the truncate, and we have no earthly idea how much space
8661 * we will use, so we need the truncate reservation to be separate so it
8662 * doesn't end up using space reserved for updating the inode. We also
8663 * need to be able to stop the transaction and start a new one, which
8664 * means we need to be able to update the inode several times, and we
8665 * have no idea of knowing how many times that will be, so we can't just
8666 * reserve 1 item for the entirety of the operation, so that has to be
8667 * done separately as well.
8668 *
8669 * So that leaves us with
8670 *
8671 * 1) rsv - for the truncate reservation, which we will steal from the
8672 * transaction reservation.
8673 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for
8674 * updating the inode.
8675 */
8682 /*
8683 * 1 for the truncate slack space
8684 * 1 for updating the inode.
8685 */
8690 }
8692 /* Migrate the slack space for the truncate to our reserve */
8697 /*
8698 * So if we truncate and then write and fsync we normally would just
8699 * write the extents that changed, which is a problem if we need to
8700 * first truncate that entire inode. So set this flag so we write out
8701 * all of the extents in the inode to the sync log so we're completely
8702 * safe.
8703 */
8710 BTRFS_EXTENT_DATA_KEY);
8727 }
8734 }
8736 /*
8737 * We can't call btrfs_truncate_block inside a trans handle as we could
8738 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know
8739 * we've truncated everything except the last little bit, and can do
8740 * btrfs_truncate_block and then update the disk_i_size.
8741 */
8753 }
8755 }
8769 }
8770 out:
8774 }
8776 /*
8777 * create a new subvolume directory/inode (helper for the ioctl).
8778 */
8782 u64 new_dirid)
8783 {
8811 }
8814 {
8843 BTRFS_BLOCK_RSV_DELALLOC);
8871 }
8873 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
8875 {
8878 }
8879 #endif
8882 {
8884 }
8887 {
8902 /*
8903 * This can happen where we create an inode, but somebody else also
8904 * created the same inode and we need to destroy the one we already
8905 * created.
8906 */
8921 }
8922 }
8928 }
8931 {
8937 /* the snap/subvol tree is on deleting */
8940 else
8942 }
8945 {
8949 }
8952 {
8953 /*
8954 * Make sure all delayed rcu free inodes are flushed before we
8955 * destroy cache.
8956 */
8963 }
8966 {
8970 init_once);
8999 fail:
9002 }
9006 {
9007 u64 delalloc_bytes;
9025 STATX_ATTR_COMPRESSED |
9026 STATX_ATTR_IMMUTABLE |
9027 STATX_ATTR_NODUMP);
9038 }
9044 {
9066 /* we only allow rename subvolume link between subvolumes */
9073 /* close the race window with snapshot create/destroy ioctl */
9078 /*
9079 * We want to reserve the absolute worst case amount of items. So if
9080 * both inodes are subvols and we need to unlink them then that would
9081 * require 4 item modifications, but if they are both normal inodes it
9082 * would require 5 item modifications, so we'll assume their normal
9083 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items
9084 * should cover the worst case number of items we'll modify.
9085 */
9090 }
9095 /*
9096 * We need to find a free sequence number both in the source and
9097 * in the destination directory for the exchange.
9098 */
9109 /* Reference for the source. */
9111 /* force full log commit if subvolume involved. */
9119 old_ino,
9121 old_idx);
9124 }
9126 /* And now for the dest. */
9128 /* force full log commit if subvolume involved. */
9136 new_ino,
9138 new_idx);
9141 }
9143 /* Update inode version and ctime/mtime. */
9158 }
9160 /* src is a subvolume */
9170 }
9174 }
9176 /* dest is a subvolume */
9186 }
9190 }
9198 }
9206 }
9225 }
9237 }
9240 }
9241 out_fail:
9242 /*
9243 * If we have pinned a log and an error happened, we unpin tasks
9244 * trying to sync the log and force them to fallback to a transaction
9245 * commit if the log currently contains any of the inodes involved in
9246 * this rename operation (to ensure we do not persist a log with an
9247 * inconsistent state for any of these inodes or leading to any
9248 * inconsistencies when replayed). If the transaction was aborted, the
9249 * abortion reason is propagated to userspace when attempting to commit
9250 * the transaction. If the log does not contain any of these inodes, we
9251 * allow the tasks to sync it.
9252 */
9264 }
9268 }
9269 }
9275 }
9281 }
9283 /*
9284 * We may have set commit_transaction when logging the new name
9285 * in the destination root, in which case we left the source
9286 * root context in the list of log contextes. So make sure we
9287 * remove it to avoid invalid memory accesses, since the context
9288 * was allocated in our stack frame.
9289 */
9294 }
9301 }
9302 out_notrans:
9311 }
9317 {
9320 u64 objectid;
9321 u64 index;
9331 objectid,
9338 }
9342 WHITEOUT_DEV);
9355 out:
9362 }
9367 {
9386 /* we only allow rename subvolume link between subvolumes */
9399 /* check for collisions, even if the name isn't there */
9406 /* we shouldn't get
9407 * eexist without a new_inode */
9410 }
9412 /* maybe -EOVERFLOW */
9414 }
9415 }
9418 /*
9419 * we're using rename to replace one file with another. Start IO on it
9420 * now so we don't add too much work to the end of the transaction
9421 */
9425 /* close the racy window with snapshot create/destroy ioctl */
9428 /*
9429 * We want to reserve the absolute worst case amount of items. So if
9430 * both inodes are subvols and we need to unlink them then that would
9431 * require 4 item modifications, but if they are both normal inodes it
9432 * would require 5 item modifications, so we'll assume they are normal
9433 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9434 * should cover the worst case number of items we'll modify.
9435 * If our rename has the whiteout flag, we need more 5 units for the
9436 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
9437 * when selinux is enabled).
9438 */
9446 }
9457 /* force full log commit if subvolume involved. */
9465 old_ino,
9469 }
9491 }
9495 }
9501 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
9509 }
9516 }
9517 }
9525 }
9544 }
9548 old_dentry);
9553 }
9554 }
9555 out_fail:
9556 /*
9557 * If we have pinned the log and an error happened, we unpin tasks
9558 * trying to sync the log and force them to fallback to a transaction
9559 * commit if the log currently contains any of the inodes involved in
9560 * this rename operation (to ensure we do not persist a log with an
9561 * inconsistent state for any of these inodes or leading to any
9562 * inconsistencies when replayed). If the transaction was aborted, the
9563 * abortion reason is propagated to userspace when attempting to commit
9564 * the transaction. If the log does not contain any of these inodes, we
9565 * allow the tasks to sync it.
9566 */
9577 }
9586 }
9594 }
9595 out_notrans:
9600 }
9605 {
9611 new_dentry);
9614 }
9621 };
9624 {
9629 work);
9638 }
9641 {
9654 }
9656 /*
9657 * some fairly slow code that needs optimization. This walks the list
9658 * of all the inodes with pending delalloc and forces them to disk.
9659 */
9661 {
9677 delalloc_inodes);
9685 }
9696 }
9700 ret++;
9705 }
9708 out:
9713 }
9719 }
9722 }
9725 {
9736 }
9739 {
9754 delalloc_root);
9769 }
9771 }
9775 out:
9780 }
9783 }
9787 {
9795 u64 objectid;
9807 /*
9808 * 2 items for inode item and ref
9809 * 2 items for dir items
9810 * 1 item for updating parent inode item
9811 * 1 item for the inline extent item
9812 * 1 item for xattr if selinux is on
9813 */
9829 }
9831 /*
9832 * If the active LSM wants to access the inode during
9833 * d_instantiate it needs these. Smack checks to see
9834 * if the filesystem supports xattrs by looking at the
9835 * ops vector.
9836 */
9850 }
9856 datasize);
9860 }
9866 BTRFS_FILE_EXTENT_INLINE);
9882 /*
9883 * Last step, add directory indexes for our symlink inode. This is the
9884 * last step to avoid extra cleanup of these indexes if an error happens
9885 * elsewhere above.
9886 */
9895 out_unlock:
9900 }
9903 }
9909 {
9917 u64 i_size;
9918 u64 cur_bytes;
9932 }
9933 }
9937 /*
9938 * If we are severely fragmented we could end up with really
9939 * small allocations, so if the allocator is returning small
9940 * chunks lets make its job easier by only searching for those
9941 * sized chunks.
9942 */
9950 }
9952 /*
9953 * We've reserved this space, and thus converted it from
9954 * ->bytes_may_use to ->bytes_reserved. Any error that happens
9955 * from here on out we will only need to clear our reservation
9956 * for the remaining unreserved area, so advance our
9957 * clear_offset by our extent size.
9958 */
9967 BTRFS_FILE_EXTENT_PREALLOC);
9975 }
9985 }
10006 }
10008 next:
10021 else
10025 }
10034 }
10038 }
10043 }
10048 {
10051 NULL);
10052 }
10058 {
10061 }
10064 {
10066 }
10069 {
10079 }
10081 }
10084 {
10089 u64 objectid;
10090 u64 index;
10093 /*
10094 * 5 units required for adding orphan entry
10095 */
10110 }
10129 /*
10130 * We set number of links to 0 in btrfs_new_inode(), and here we set
10131 * it to 1 because d_tmpfile() will issue a warning if the count is 0,
10132 * through:
10133 *
10134 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
10135 */
10140 out:
10146 }
10149 {
10160 index++;
10161 }
10162 }
10164 #ifdef CONFIG_SWAP
10165 /*
10166 * Add an entry indicating a block group or device which is pinned by a
10167 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a
10168 * negative errno on failure.
10169 */
10172 {
10200 }
10201 }
10206 }
10208 /* Free all of the entries pinned by this swapfile. */
10210 {
10225 }
10227 }
10229 }
10232 u64 start;
10233 u64 block_start;
10234 u64 block_len;
10235 u64 lowest_ppage;
10236 u64 highest_ppage;
10239 };
10243 {
10258 first_ppage_reported++;
10270 }
10273 {
10278 }
10282 {
10291 };
10293 u64 isize;
10294 u64 start;
10296 /*
10297 * If the swap file was just created, make sure delalloc is done. If the
10298 * file changes again after this, the user is doing something stupid and
10299 * we don't really care.
10300 */
10305 /*
10306 * The inode is locked, so these flags won't change after we check them.
10307 */
10311 }
10315 }
10319 }
10321 /*
10322 * Balance or device remove/replace/resize can move stuff around from
10323 * under us. The EXCL_OP flag makes sure they aren't running/won't run
10324 * concurrently while we are mapping the swap extents, and
10325 * fs_info->swapfile_pins prevents them from running while the swap file
10326 * is active and moving the extents. Note that this also prevents a
10327 * concurrent device add which isn't actually necessary, but it's not
10328 * really worth the trouble to allow it.
10329 */
10334 }
10335 /*
10336 * Snapshots can create extents which require COW even if NODATACOW is
10337 * set. We use this counter to prevent snapshots. We must increment it
10338 * before walking the extents because we don't want a concurrent
10339 * snapshot to run after we've already checked the extents.
10340 */
10356 }
10362 }
10364 /*
10365 * It's unlikely we'll ever actually find ourselves
10366 * here, as a file small enough to fit inline won't be
10367 * big enough to store more than the swap header, but in
10368 * case something changes in the future, let's catch it
10369 * here rather than later.
10370 */
10374 }
10379 }
10396 }
10402 }
10409 }
10422 }
10436 }
10443 else
10445 }
10455 }
10459 }
10462 }
10467 out:
10488 }
10489 #else
10491 {
10492 }
10496 {
10498 }
10499 #endif
10519 };
10527 #ifdef CONFIG_COMPAT
10529 #endif
10532 };
10535 /* mandatory callbacks */
10538 };
10540 /*
10541 * btrfs doesn't support the bmap operation because swapfiles
10542 * use bmap to make a mapping of extents in the file. They assume
10543 * these extents won't change over the life of the file and they
10544 * use the bmap result to do IO directly to the drive.
10545 *
10546 * the btrfs bmap call would return logical addresses that aren't
10547 * suitable for IO and they also will change frequently as COW
10548 * operations happen. So, swapfile + btrfs == corruption.
10549 *
10550 * For now we're avoiding this by dropping bmap.
10551 */
10560 #ifdef CONFIG_MIGRATION
10562 #endif
10567 };
10578 };
10587 };
10595 };
10599 };