| blob | 18d384f4af54e1d9c606bfeec8fff79688669a7d |
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2007 Oracle. All rights reserved.
4 */
6 #include <linux/kernel.h>
7 #include <linux/bio.h>
8 #include <linux/buffer_head.h>
9 #include <linux/file.h>
10 #include <linux/fs.h>
11 #include <linux/pagemap.h>
12 #include <linux/highmem.h>
13 #include <linux/time.h>
14 #include <linux/init.h>
15 #include <linux/string.h>
16 #include <linux/backing-dev.h>
17 #include <linux/writeback.h>
18 #include <linux/compat.h>
19 #include <linux/xattr.h>
20 #include <linux/posix_acl.h>
21 #include <linux/falloc.h>
22 #include <linux/slab.h>
23 #include <linux/ratelimit.h>
24 #include <linux/btrfs.h>
25 #include <linux/blkdev.h>
26 #include <linux/posix_acl_xattr.h>
27 #include <linux/uio.h>
28 #include <linux/magic.h>
29 #include <linux/iversion.h>
30 #include <linux/swap.h>
31 #include <linux/migrate.h>
32 #include <linux/sched/mm.h>
33 #include <asm/unaligned.h>
55 u64 ino;
57 };
60 u64 reserve;
61 u64 unsubmitted_oe_range_start;
62 u64 unsubmitted_oe_range_end;
64 };
97 /*
98 * Cleanup all submitted ordered extents in specified range to handle errors
99 * from the btrfs_run_delalloc_range() callback.
100 *
101 * NOTE: caller must ensure that when an error happens, it can not call
102 * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
103 * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
104 * to be released, which we want to happen only when finishing the ordered
105 * extent (btrfs_finish_ordered_io()).
106 */
110 {
120 index++;
125 }
127 /*
128 * In case this page belongs to the delalloc range being instantiated
129 * then skip it, since the first page of a range is going to be
130 * properly cleaned up by the caller of run_delalloc_range
131 */
135 }
138 }
142 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
144 {
146 }
147 #endif
152 {
159 }
161 /*
162 * this does all the hard work for inserting an inline extent into
163 * the btree. The caller should have done a btrfs_drop_extents so that
164 * no overlapping inline items exist in the btree
165 */
172 {
201 datasize);
204 }
221 PAGE_SIZE);
227 i++;
230 }
232 compress_type);
242 }
246 /*
247 * We align size to sectorsize for inline extents just for simplicity
248 * sake.
249 */
255 /*
256 * we're an inline extent, so nobody can
257 * extend the file past i_size without locking
258 * a page we already have locked.
259 *
260 * We must do any isize and inode updates
261 * before we unlock the pages. Otherwise we
262 * could end up racing with unlink.
263 */
267 fail:
269 }
272 /*
273 * conditionally insert an inline extent into the file. This
274 * does the checks required to make sure the data is small enough
275 * to fit as an inline extent.
276 */
281 {
293 u32 extent_item_size;
306 }
316 }
321 compressed_size);
322 else
324 inline_len);
332 }
346 }
350 out:
351 /*
352 * Don't forget to free the reserved space, as for inlined extent
353 * it won't count as data extent, free them directly here.
354 * And at reserve time, it's always aligned to page size, so
355 * just free one page here.
356 */
361 }
364 u64 start;
365 u64 ram_size;
366 u64 compressed_size;
371 };
376 u64 start;
377 u64 end;
383 };
386 /* Number of chunks in flight; must be first in the structure */
387 atomic_t num_chunks;
389 };
393 u64 compressed_size,
397 {
410 }
412 /*
413 * Check if the inode has flags compatible with compression
414 */
416 {
421 }
423 /*
424 * Check if the inode needs to be submitted to compression, based on mount
425 * options, defragmentation, properties or heuristics.
426 */
428 {
436 }
437 /* force compress */
440 /* defrag ioctl */
443 /* bad compression ratios */
451 }
455 {
456 /* If this is a small write inside eof, kick off a defrag */
460 }
462 /*
463 * we create compressed extents in two phases. The first
464 * phase compresses a range of pages that have already been
465 * locked (both pages and state bits are locked).
466 *
467 * This is done inside an ordered work queue, and the compression
468 * is spread across many cpus. The actual IO submission is step
469 * two, and the ordered work queue takes care of making sure that
470 * happens in the same order things were put onto the queue by
471 * writepages and friends.
472 *
473 * If this code finds it can't get good compression, it puts an
474 * entry onto the work queue to write the uncompressed bytes. This
475 * makes sure that both compressed inodes and uncompressed inodes
476 * are written in the same order that the flusher thread sent them
477 * down.
478 */
480 {
486 u64 actual_end;
487 u64 i_size;
500 SZ_16K);
502 /*
503 * We need to save i_size before now because it could change in between
504 * us evaluating the size and assigning it. This is because we lock and
505 * unlock the page in truncate and fallocate, and then modify the i_size
506 * later on.
507 *
508 * The barriers are to emulate READ_ONCE, remove that once i_size_read
509 * does that for us.
510 */
515 again:
522 /*
523 * we don't want to send crud past the end of i_size through
524 * compression, that's just a waste of CPU time. So, if the
525 * end of the file is before the start of our current
526 * requested range of bytes, we bail out to the uncompressed
527 * cleanup code that can deal with all of this.
528 *
529 * It isn't really the fastest way to fix things, but this is a
530 * very uncommon corner.
531 */
537 /*
538 * skip compression for a small file range(<=blocksize) that
539 * isn't an inline extent, since it doesn't save disk space at all.
540 */
546 BTRFS_MAX_UNCOMPRESSED);
550 /*
551 * we do compression for mount -o compress and when the
552 * inode has not been flagged as nocompress. This flag can
553 * change at any time if we discover bad compression ratios.
554 */
559 /* just bail out to the uncompressed code */
562 }
569 /*
570 * we need to call clear_page_dirty_for_io on each
571 * page in the range. Otherwise applications with the file
572 * mmap'd can wander in and change the page contents while
573 * we are compressing them.
574 *
575 * If the compression fails for any reason, we set the pages
576 * dirty again later on.
577 *
578 * Note that the remaining part is redirtied, the start pointer
579 * has moved, the end is the original one.
580 */
584 }
586 /* Compression level is applied here and only here */
590 pages,
600 /* zero the tail end of the last page, we might be
601 * sending it down to disk
602 */
608 }
610 }
611 }
612 cont:
614 /* lets try to make an inline extent */
616 /* we didn't compress the entire range, try
617 * to make an uncompressed inline extent.
618 */
622 /* try making a compressed inline extent */
624 total_compressed,
626 }
630 EXTENT_DO_ACCOUNTING;
635 /*
636 * inline extent creation worked or returned error,
637 * we don't need to create any more async work items.
638 * Unlock and free up our temp pages.
639 *
640 * We use DO_ACCOUNTING here because we need the
641 * delalloc_release_metadata to be done _after_ we drop
642 * our outstanding extent for clearing delalloc for this
643 * range.
644 */
646 clear_flags,
647 PAGE_UNLOCK |
648 PAGE_CLEAR_DIRTY |
649 PAGE_SET_WRITEBACK |
650 page_error_op |
651 PAGE_END_WRITEBACK);
656 }
660 }
661 }
664 /*
665 * we aren't doing an inline extent round the compressed size
666 * up to a block size boundary so the allocator does sane
667 * things
668 */
671 /*
672 * one last check to make sure the compression is really a
673 * win, compare the page count read with the blocks on disk,
674 * compression must free at least one sector size
675 */
678 compressed_extents++;
680 /*
681 * The async work queues will take care of doing actual
682 * allocation on disk for these compressed pages, and
683 * will submit them to the elevator.
684 */
687 compress_type);
694 }
696 }
697 }
699 /*
700 * the compression code ran but failed to make things smaller,
701 * free any pages it allocated and our page pointer array
702 */
706 }
712 /* flag the file so we don't compress in the future */
716 }
717 }
718 cleanup_and_bail_uncompressed:
719 /*
720 * No compression, but we still need to write the pages in the file
721 * we've been given so far. redirty the locked page if it corresponds
722 * to our extent and set things up for the async work queue to run
723 * cow_file_range to do the normal delalloc dance.
724 */
729 /* unlocked later on in the async handlers */
730 }
735 BTRFS_COMPRESS_NONE);
736 compressed_extents++;
739 }
742 {
751 }
755 }
757 /*
758 * phase two of compressed writeback. This is the ordered portion
759 * of the code, which only gets called in the order the work was
760 * queued. We walk all the async extents created by compress_file_range
761 * and send them down to the disk.
762 */
764 {
775 again:
781 retry:
784 /* did the compression code fall back to uncompressed IO? */
789 /* allocate blocks */
796 /* JDM XXX */
798 /*
799 * if page_started, cow_file_range inserted an
800 * inline extent and took care of all the unlocking
801 * and IO for us. Otherwise, we need to submit
802 * all those pages down to the drive.
803 */
809 WB_SYNC_ALL);
815 }
829 /*
830 * we need to redirty the pages if we decide to
831 * fallback to uncompressed IO, otherwise we
832 * will not submit these pages down to lower
833 * layers.
834 */
841 }
843 }
844 /*
845 * here we're doing allocation and writeback of the
846 * compressed pages
847 */
856 BTRFS_ORDERED_COMPRESSED);
858 /* ret value is not necessary due to void function */
867 BTRFS_ORDERED_COMPRESSED,
875 }
878 /*
879 * clear dirty, set writeback and unlock the pages.
880 */
886 PAGE_SET_WRITEBACK);
905 PAGE_END_WRITEBACK |
906 PAGE_SET_ERROR);
908 }
912 }
914 out_free_reserve:
917 out_free:
922 EXTENT_DELALLOC_NEW |
926 PAGE_SET_ERROR);
930 }
933 u64 num_bytes)
934 {
942 /*
943 * if block start isn't an actual block number then find the
944 * first block in this inode and use that as a hint. If that
945 * block is also bogus then just don't worry about it.
946 */
957 }
958 }
962 }
964 /*
965 * when extent_io.c finds a delayed allocation range in the file,
966 * the call backs end up in this code. The basic idea is to
967 * allocate extents on disk for the range, and create ordered data structs
968 * in ram to track those extents.
969 *
970 * locked_page is the page that writepage had locked already. We use
971 * it to make sure we don't do extra locks or unlocks.
972 *
973 * *page_started is set to one if we unlock locked_page and do everything
974 * required to start IO on it. It may be clean and already done with
975 * IO when we return.
976 */
981 {
985 u64 num_bytes;
988 u64 min_alloc_size;
1001 }
1010 /* lets try to make an inline extent */
1014 /*
1015 * We use DO_ACCOUNTING here because we need the
1016 * delalloc_release_metadata to be run _after_ we drop
1017 * our outstanding extent for clearing delalloc for this
1018 * range.
1019 */
1025 PAGE_END_WRITEBACK);
1032 }
1033 }
1039 /*
1040 * Relocation relies on the relocated extents to have exactly the same
1041 * size as the original extents. Normally writeback for relocation data
1042 * extents follows a NOCOW path because relocation preallocates the
1043 * extents. However, due to an operation such as scrub turning a block
1044 * group to RO mode, it may fallback to COW mode, so we must make sure
1045 * an extent allocated during COW has exactly the requested size and can
1046 * not be split into smaller extents, otherwise relocation breaks and
1047 * fails during the stage where it updates the bytenr of file extent
1048 * items.
1049 */
1052 else
1077 }
1086 BTRFS_DATA_RELOC_TREE_OBJECTID) {
1088 cur_alloc_size);
1089 /*
1090 * Only drop cache here, and process as normal.
1091 *
1092 * We must not allow extent_clear_unlock_delalloc()
1093 * at out_unlock label to free meta of this ordered
1094 * extent, as its meta should be freed by
1095 * btrfs_finish_ordered_io().
1096 *
1097 * So we must continue until @start is increased to
1098 * skip current ordered extent.
1099 */
1103 }
1107 /* we're not doing compressed IO, don't unlock the first
1108 * page (which the caller expects to stay locked), don't
1109 * clear any dirty bits and don't set any writeback bits
1110 *
1111 * Do set the Private2 bit so we know this page was properly
1112 * setup for writepage
1113 */
1119 locked_page,
1121 page_ops);
1124 else
1130 /*
1131 * btrfs_reloc_clone_csums() error, since start is increased
1132 * extent_clear_unlock_delalloc() at out_unlock label won't
1133 * free metadata of current ordered extent, we're OK to exit.
1134 */
1137 }
1138 out:
1141 out_drop_extent_cache:
1143 out_reserve:
1146 out_unlock:
1150 PAGE_END_WRITEBACK;
1151 /*
1152 * If we reserved an extent for our delalloc range (or a subrange) and
1153 * failed to create the respective ordered extent, then it means that
1154 * when we reserved the extent we decremented the extent's size from
1155 * the data space_info's bytes_may_use counter and incremented the
1156 * space_info's bytes_reserved counter by the same amount. We must make
1157 * sure extent_clear_unlock_delalloc() does not try to decrement again
1158 * the data space_info's bytes_may_use counter, therefore we do not pass
1159 * it the flag EXTENT_CLEAR_DATA_RESV.
1160 */
1164 locked_page,
1165 clear_bits,
1166 page_ops);
1170 }
1173 page_ops);
1175 }
1177 /*
1178 * work queue call back to started compression on a file and pages
1179 */
1181 {
1191 }
1192 }
1194 /*
1195 * work queue call back to submit previously compressed pages
1196 */
1198 {
1200 work);
1205 PAGE_SHIFT;
1207 /* atomic_sub_return implies a barrier */
1212 /*
1213 * ->inode could be NULL if async_chunk_start has failed to compress,
1214 * in which case we don't have anything to submit, yet we need to
1215 * always adjust ->async_delalloc_pages as its paired with the init
1216 * happening in cow_file_range_async
1217 */
1220 }
1223 {
1231 /*
1232 * Since the pointer to 'pending' is at the beginning of the array of
1233 * async_chunk's, freeing it ensures the whole array has been freed.
1234 */
1237 }
1244 {
1250 u64 cur_end;
1265 }
1274 EXTENT_DO_ACCOUNTING;
1277 PAGE_SET_ERROR;
1282 }
1290 else
1293 /*
1294 * igrab is called higher up in the call chain, take only the
1295 * lightweight reference for the callback lifetime
1296 */
1305 /*
1306 * The locked_page comes all the way from writepage and its
1307 * the original page we were actually given. As we spread
1308 * this large delalloc region across multiple async_chunk
1309 * structs, only the first struct needs a pointer to locked_page
1310 *
1311 * This way we don't need racey decisions about who is supposed
1312 * to unlock it.
1313 */
1315 /*
1316 * Depending on the compressibility, the pages might or
1317 * might not go through async. We want all of them to
1318 * be accounted against wbc once. Let's do it here
1319 * before the paths diverge. wbc accounting is used
1320 * only for foreign writeback detection and doesn't
1321 * need full accuracy. Just account the whole thing
1322 * against the first page.
1323 */
1330 }
1337 }
1349 }
1352 }
1356 {
1370 }
1374 }
1379 {
1382 BTRFS_DATA_RELOC_TREE_OBJECTID);
1386 u64 count;
1388 /*
1389 * If EXTENT_NORESERVE is set it means that when the buffered write was
1390 * made we had not enough available data space and therefore we did not
1391 * reserve data space for it, since we though we could do NOCOW for the
1392 * respective file range (either there is prealloc extent or the inode
1393 * has the NOCOW bit set).
1394 *
1395 * However when we need to fallback to COW mode (because for example the
1396 * block group for the corresponding extent was turned to RO mode by a
1397 * scrub or relocation) we need to do the following:
1398 *
1399 * 1) We increment the bytes_may_use counter of the data space info.
1400 * If COW succeeds, it allocates a new data extent and after doing
1401 * that it decrements the space info's bytes_may_use counter and
1402 * increments its bytes_reserved counter by the same amount (we do
1403 * this at btrfs_add_reserved_bytes()). So we need to increment the
1404 * bytes_may_use counter to compensate (when space is reserved at
1405 * buffered write time, the bytes_may_use counter is incremented);
1406 *
1407 * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so
1408 * that if the COW path fails for any reason, it decrements (through
1409 * extent_clear_unlock_delalloc()) the bytes_may_use counter of the
1410 * data space info, which we incremented in the step above.
1411 *
1412 * If we need to fallback to cow and the inode corresponds to a free
1413 * space cache inode or an inode of the data relocation tree, we must
1414 * also increment bytes_may_use of the data space_info for the same
1415 * reason. Space caches and relocated data extents always get a prealloc
1416 * extent for them, however scrub or balance may have set the block
1417 * group that contains that extent to RO mode and therefore force COW
1418 * when starting writeback.
1419 */
1437 }
1441 }
1443 /*
1444 * when nowcow writeback call back. This checks for snapshots or COW copies
1445 * of the extents that exist in the file, and COWs the file as required.
1446 *
1447 * If no cow copies or snapshots exist, we write directly to the existing
1448 * blocks on disk
1449 */
1455 {
1472 EXTENT_DO_ACCOUNTING |
1474 PAGE_CLEAR_DIRTY |
1475 PAGE_SET_WRITEBACK |
1476 PAGE_END_WRITEBACK);
1478 }
1484 u64 extent_end;
1485 u64 extent_offset;
1487 u64 disk_num_bytes;
1488 u64 ram_bytes;
1498 /*
1499 * If there is no extent for our range when doing the initial
1500 * search, then go back to the previous slot as it will be the
1501 * one containing the search offset
1502 */
1510 }
1512 next_slot:
1513 /* Go to next leaf if we have exhausted the current one */
1521 }
1525 }
1529 /* Didn't find anything for our INO */
1532 /*
1533 * Keep searching until we find an EXTENT_ITEM or there are no
1534 * more extents for this inode
1535 */
1540 }
1542 /* Found key is not EXTENT_DATA_KEY or starts after req range */
1547 /*
1548 * If the found extent starts after requested offset, then
1549 * adjust extent_end to be right before this extent begins
1550 */
1555 }
1557 /*
1558 * Found extent which begins before our range and potentially
1559 * intersect it
1560 */
1572 disk_num_bytes =
1574 /*
1575 * If the extent we got ends before our current offset,
1576 * skip to the next extent.
1577 */
1581 }
1582 /* Skip holes */
1585 /* Skip compressed/encrypted/encoded extents */
1590 /*
1591 * If extent is created before the last volume's snapshot
1592 * this implies the extent is shared, hence we can't do
1593 * nocow. This is the same check as in
1594 * btrfs_cross_ref_exist but without calling
1595 * btrfs_search_slot.
1596 */
1603 /* If extent is RO, we must COW it */
1610 /*
1611 * ret could be -EIO if the above fails to read
1612 * metadata.
1613 */
1618 }
1622 }
1626 /*
1627 * If there are pending snapshots for this root, we
1628 * fall into common COW way
1629 */
1632 /*
1633 * force cow if csum exists in the range.
1634 * this ensure that csum for a given extent are
1635 * either valid or do not exist.
1636 */
1638 num_bytes);
1640 /*
1641 * ret could be -EIO if the above fails to read
1642 * metadata.
1643 */
1648 }
1651 }
1658 /* Skip extents outside of our requested range */
1662 }
1664 /* If this triggers then we have a memory corruption */
1666 }
1667 out_check:
1668 /*
1669 * If nocow is false then record the beginning of the range
1670 * that needs to be COWed
1671 */
1680 }
1684 /*
1685 * COW range from cow_start to found_key.offset - 1. As the key
1686 * will contain the beginning of the first extent that can be
1687 * NOCOW, following one which needs to be COW'ed
1688 */
1696 disk_bytenr);
1698 }
1700 }
1707 orig_start,
1712 BTRFS_ORDERED_PREALLOC);
1716 disk_bytenr);
1719 }
1723 num_bytes,
1724 BTRFS_ORDERED_PREALLOC);
1727 cur_offset,
1731 }
1735 num_bytes,
1736 BTRFS_ORDERED_NOCOW);
1739 }
1746 BTRFS_DATA_RELOC_TREE_OBJECTID)
1747 /*
1748 * Error handled later, as we must prevent
1749 * extent_clear_unlock_delalloc() in error handler
1750 * from freeing metadata of created ordered extent.
1751 */
1753 num_bytes);
1758 EXTENT_DELALLOC |
1759 EXTENT_CLEAR_DATA_RESV,
1764 /*
1765 * btrfs_reloc_clone_csums() error, now we're OK to call error
1766 * handler, as metadata for created ordered extent will only
1767 * be freed by btrfs_finish_ordered_io().
1768 */
1773 }
1785 }
1787 error:
1796 PAGE_CLEAR_DIRTY |
1797 PAGE_SET_WRITEBACK |
1798 PAGE_END_WRITEBACK);
1801 }
1804 {
1810 /*
1811 * @defrag_bytes is a hint value, no spinlock held here,
1812 * if is not zero, it means the file is defragging.
1813 * Force cow if given extent needs to be defragged.
1814 */
1821 }
1823 /*
1824 * Function to process delayed allocation (create CoW) for ranges which are
1825 * being touched for the first time.
1826 */
1830 {
1849 }
1854 }
1858 {
1859 u64 size;
1861 /* not delalloc, ignore it */
1867 u32 num_extents;
1868 u64 new_size;
1870 /*
1871 * See the explanation in btrfs_merge_delalloc_extent, the same
1872 * applies here, just in reverse.
1873 */
1880 }
1885 }
1887 /*
1888 * Handle merged delayed allocation extents so we can keep track of new extents
1889 * that are just merged onto old extents, such as when we are doing sequential
1890 * writes, so we can properly account for the metadata space we'll need.
1891 */
1894 {
1896 u32 num_extents;
1898 /* not delalloc, ignore it */
1904 else
1907 /* we're not bigger than the max, unreserve the space and go */
1913 }
1915 /*
1916 * We have to add up either side to figure out how many extents were
1917 * accounted for before we merged into one big extent. If the number of
1918 * extents we accounted for is <= the amount we need for the new range
1919 * then we can return, otherwise drop. Think of it like this
1920 *
1921 * [ 4k][MAX_SIZE]
1922 *
1923 * So we've grown the extent by a MAX_SIZE extent, this would mean we
1924 * need 2 outstanding extents, on one side we have 1 and the other side
1925 * we have 1 so they are == and we can return. But in this case
1926 *
1927 * [MAX_SIZE+4k][MAX_SIZE+4k]
1928 *
1929 * Each range on their own accounts for 2 extents, but merged together
1930 * they are only 3 extents worth of accounting, so we need to drop in
1931 * this case.
1932 */
1943 }
1947 {
1963 }
1964 }
1966 }
1971 {
1985 }
1986 }
1987 }
1991 {
1995 }
1997 /*
1998 * Properly track delayed allocation bytes in the inode and to maintain the
1999 * list of inodes that have pending delalloc work to be done.
2000 */
2003 {
2008 /*
2009 * set_bit and clear bit hooks normally require _irqsave/restore
2010 * but in this case, we are only testing for the DELALLOC
2011 * bit, which is only set or cleared with irqs on
2012 */
2023 /* For sanity tests */
2037 }
2045 }
2046 }
2048 /*
2049 * Once a range is no longer delalloc this function ensures that proper
2050 * accounting happens.
2051 */
2054 {
2064 }
2066 /*
2067 * set_bit and clear bit hooks normally require _irqsave/restore
2068 * but in this case, we are only testing for the DELALLOC
2069 * bit, which is only set or cleared with irqs on
2070 */
2079 /*
2080 * We don't reserve metadata space for space cache inodes so we
2081 * don't need to call delalloc_release_metadata if there is an
2082 * error.
2083 */
2088 /* For sanity tests. */
2108 }
2116 }
2117 }
2119 /*
2120 * btrfs_bio_fits_in_stripe - Checks whether the size of the given bio will fit
2121 * in a chunk's stripe. This function ensures that bios do not span a
2122 * stripe/chunk
2123 *
2124 * @page - The page we are about to add to the bio
2125 * @size - size we want to add to the bio
2126 * @bio - bio we want to ensure is smaller than a stripe
2127 * @bio_flags - flags of the bio
2128 *
2129 * return 1 if page cannot be added to the bio
2130 * return 0 if page can be added to the bio
2131 * return error otherwise
2132 */
2135 {
2140 u64 map_length;
2157 }
2159 /*
2160 * in order to insert checksums into the metadata in large chunks,
2161 * we wait until bio submission time. All the pages in the bio are
2162 * checksummed and sums are attached onto the ordered extent record.
2163 *
2164 * At IO completion time the cums attached on the ordered extent record
2165 * are inserted into the btree
2166 */
2168 u64 bio_offset)
2169 {
2176 }
2178 /*
2179 * extent_io.c submission hook. This does the right thing for csum calculation
2180 * on write, or reading the csums from the tree before a read.
2181 *
2182 * Rules about async/sync submit,
2183 * a) read: sync submit
2184 *
2185 * b) write without checksum: sync submit
2186 *
2187 * c) write with checksum:
2188 * c-1) if bio is issued by fsync: sync submit
2189 * (sync_writers != 0)
2190 *
2191 * c-2) if root is reloc root: sync submit
2192 * (only in case of buffered IO)
2193 *
2194 * c-3) otherwise: async submit
2195 */
2200 {
2220 mirror_num,
2221 bio_flags);
2227 }
2230 /* csum items have already been cloned */
2233 /* we're doing a write, do the async checksumming */
2241 }
2243 mapit:
2246 out:
2250 }
2252 }
2254 /*
2255 * given a list of ordered sums record them in the inode. This happens
2256 * at IO completion time based on sums calculated at bio submission time.
2257 */
2260 {
2271 }
2273 }
2278 {
2282 }
2284 /* see btrfs_writepage_start_hook for details on why this is required */
2289 };
2292 {
2299 u64 page_start;
2300 u64 page_end;
2310 /*
2311 * This is similar to page_mkwrite, we need to reserve the space before
2312 * we take the page lock.
2313 */
2315 PAGE_SIZE);
2316 again:
2319 /*
2320 * Before we queued this fixup, we took a reference on the page.
2321 * page->mapping may go NULL, but it shouldn't be moved to a different
2322 * address space.
2323 */
2325 /*
2326 * Unfortunately this is a little tricky, either
2327 *
2328 * 1) We got here and our page had already been dealt with and
2329 * we reserved our space, thus ret == 0, so we need to just
2330 * drop our space reservation and bail. This can happen the
2331 * first time we come into the fixup worker, or could happen
2332 * while waiting for the ordered extent.
2333 * 2) Our page was already dealt with, but we happened to get an
2334 * ENOSPC above from the btrfs_delalloc_reserve_space. In
2335 * this case we obviously don't have anything to release, but
2336 * because the page was already dealt with we don't want to
2337 * mark the page with an error, so make sure we're resetting
2338 * ret to 0. This is why we have this check _before_ the ret
2339 * check, because we do not want to have a surprise ENOSPC
2340 * when the page was already properly dealt with.
2341 */
2344 PAGE_SIZE);
2348 }
2351 }
2353 /*
2354 * We can't mess with the page state unless it is locked, so now that
2355 * it is locked bail if we failed to make our space reservation.
2356 */
2363 /* already ordered? We're done */
2368 PAGE_SIZE);
2376 }
2383 /*
2384 * Everything went as planned, we're now the owner of a dirty page with
2385 * delayed allocation bits set and space reserved for our COW
2386 * destination.
2387 *
2388 * The page was dirty when we started, nothing should have cleaned it.
2389 */
2392 out_reserved:
2399 out_page:
2401 /*
2402 * We hit ENOSPC or other errors. Update the mapping and page
2403 * to reflect the errors and clean the page.
2404 */
2409 }
2415 /*
2416 * As a precaution, do a delayed iput in case it would be the last iput
2417 * that could need flushing space. Recursing back to fixup worker would
2418 * deadlock.
2419 */
2421 }
2423 /*
2424 * There are a few paths in the higher layers of the kernel that directly
2425 * set the page dirty bit without asking the filesystem if it is a
2426 * good idea. This causes problems because we want to make sure COW
2427 * properly happens and the data=ordered rules are followed.
2428 *
2429 * In our case any range that doesn't have the ORDERED bit set
2430 * hasn't been properly setup for IO. We kick off an async process
2431 * to fix it up. The async helper will wait for ordered extents, set
2432 * the delalloc bit and make it safe to write the page.
2433 */
2435 {
2440 /* this page is properly in the ordered list */
2444 /*
2445 * PageChecked is set below when we create a fixup worker for this page,
2446 * don't try to create another one if we're already PageChecked()
2447 *
2448 * The extent_io writepage code will redirty the page if we send back
2449 * EAGAIN.
2450 */
2458 /*
2459 * We are already holding a reference to this inode from
2460 * write_cache_pages. We need to hold it because the space reservation
2461 * takes place outside of the page lock, and we can't trust
2462 * page->mapping outside of the page lock.
2463 */
2473 }
2481 {
2487 u64 qg_released;
2495 /*
2496 * we may be replacing one extent in the tree with another.
2497 * The new extent is pinned in the extent map, and we don't want
2498 * to drop it from the cache until it is completely in the btree.
2499 *
2500 * So, tell btrfs_drop_extents to leave this extent in the cache.
2501 * the caller is expected to unpin it and allow it to be merged
2502 * with the others.
2503 */
2520 }
2545 ram_bytes);
2549 /*
2550 * Release the reserved range from inode dirty range map, as it is
2551 * already moved into delayed_ref_head
2552 */
2560 out:
2564 }
2568 {
2579 }
2581 /* as ordered data IO finishes, this gets called so we can finish
2582 * an ordered extent if the range of bytes in the file it covers are
2583 * fully written.
2584 */
2586 {
2617 }
2624 /* Truncated the entire extent, don't bother adding */
2627 }
2632 /*
2633 * For mwrite(mmap + memset to write) case, we still reserve
2634 * space for NOCOW range.
2635 * As NOCOW won't cause a new delayed ref, just free the space
2636 */
2642 else
2648 }
2654 }
2661 else
2667 }
2680 logical_len);
2688 BTRFS_FILE_EXTENT_REG);
2694 }
2695 }
2702 }
2708 }
2715 }
2717 out:
2737 /* Drop the cache for the part of the extent we didn't write. */
2740 /*
2741 * If the ordered extent had an IOERR or something else went
2742 * wrong we need to return the space for this ordered extent
2743 * back to the allocator. We only free the extent in the
2744 * truncated case if we didn't write out the extent at all.
2745 *
2746 * If we made it past insert_reserved_file_extent before we
2747 * errored out then we don't need to do this as the accounting
2748 * has already been done.
2749 */
2751 clear_reserved_extent &&
2754 /*
2755 * Discard the range before returning it back to the
2756 * free space pool
2757 */
2762 NULL);
2766 }
2767 }
2769 /*
2770 * This needs to be done to make sure anybody waiting knows we are done
2771 * updating everything for this ordered extent.
2772 */
2775 /* once for us */
2777 /* once for the tree */
2781 }
2784 {
2788 }
2792 {
2807 else
2812 }
2817 {
2837 zeroit:
2844 }
2846 /*
2847 * when reads are done, we need to check csums to verify the data is correct
2848 * if there's a match, we allow the bio to finish. If not, the code in
2849 * extent_io.c will try to find good copies for us.
2850 */
2854 {
2863 }
2872 }
2877 }
2879 /*
2880 * btrfs_add_delayed_iput - perform a delayed iput on @inode
2881 *
2882 * @inode: The inode we want to perform iput on
2883 *
2884 * This function uses the generic vfs_inode::i_count to track whether we should
2885 * just decrement it (in case it's > 1) or if this is the last iput then link
2886 * the inode to the delayed iput machinery. Delayed iputs are processed at
2887 * transaction commit time/superblock commit/cleaner kthread.
2888 */
2890 {
2904 }
2908 {
2915 }
2919 {
2925 }
2926 }
2929 {
2938 }
2940 }
2942 /**
2943 * btrfs_wait_on_delayed_iputs - wait on the delayed iputs to be done running
2944 * @fs_info - the fs_info for this fs
2945 * @return - EINTR if we were killed, 0 if nothing's pending
2946 *
2947 * This will wait on any delayed iputs that are currently running with KILLABLE
2948 * set. Once they are all done running we will return, unless we are killed in
2949 * which case we return EINTR. This helps in user operations like fallocate etc
2950 * that might get blocked on the iputs.
2951 */
2953 {
2959 }
2961 /*
2962 * This creates an orphan entry for the given inode in case something goes wrong
2963 * in the middle of an unlink.
2964 */
2967 {
2974 }
2977 }
2979 /*
2980 * We have done the delete so we can go ahead and remove the orphan item for
2981 * this particular inode.
2982 */
2985 {
2987 }
2989 /*
2990 * this cleans up any orphans that may be left on the list from the last use
2991 * of this root.
2992 */
2994 {
3011 }
3023 /*
3024 * if ret == 0 means we found what we were searching for, which
3025 * is weird, but possible, so only screw with path if we didn't
3026 * find the key and see if we have stuff that matches
3027 */
3033 }
3035 /* pull out the item */
3039 /* make sure the item matches what we want */
3045 /* release the path since we're done with it */
3048 /*
3049 * this is where we are basically btrfs_lookup, without the
3050 * crossing root thing. we store the inode number in the
3051 * offset of the orphan item.
3052 */
3059 }
3076 /*
3077 * this is an orphan in the tree root. Currently these
3078 * could come from 2 sources:
3079 * a) a snapshot deletion in progress
3080 * b) a free space cache inode
3081 * We need to distinguish those two, as the snapshot
3082 * orphan must not get deleted.
3083 * find_dead_roots already ran before us, so if this
3084 * is a snapshot deletion, we should find the root
3085 * in the fs_roots radix tree.
3086 */
3096 /* prevent this orphan from being found again */
3099 }
3101 }
3103 /*
3104 * If we have an inode with links, there are a couple of
3105 * possibilities. Old kernels (before v3.12) used to create an
3106 * orphan item for truncate indicating that there were possibly
3107 * extent items past i_size that needed to be deleted. In v3.12,
3108 * truncate was changed to update i_size in sync with the extent
3109 * items, but the (useless) orphan item was still created. Since
3110 * v4.18, we don't create the orphan item for truncate at all.
3111 *
3112 * So, this item could mean that we need to do a truncate, but
3113 * only if this filesystem was last used on a pre-v3.12 kernel
3114 * and was not cleanly unmounted. The odds of that are quite
3115 * slim, and it's a pain to do the truncate now, so just delete
3116 * the orphan item.
3117 *
3118 * It's also possible that this orphan item was supposed to be
3119 * deleted but wasn't. The inode number may have been reused,
3120 * but either way, we can delete the orphan item.
3121 */
3129 }
3138 }
3140 nr_unlink++;
3142 /* this will do delete_inode and everything for us */
3144 }
3145 /* release the path since we're done with it */
3154 }
3159 out:
3164 }
3166 /*
3167 * very simple check to peek ahead in the leaf looking for xattrs. If we
3168 * don't find any xattrs, we know there can't be any acls.
3169 *
3170 * slot is the slot the inode is in, objectid is the objectid of the inode
3171 */
3175 {
3187 }
3189 slot++;
3194 /* we found a different objectid, there must not be acls */
3198 /* we found an xattr, assume we've got an acl */
3205 }
3207 /*
3208 * we found a key greater than an xattr key, there can't
3209 * be any acls later on
3210 */
3214 slot++;
3215 scanned++;
3217 /*
3218 * it goes inode, inode backrefs, xattrs, extents,
3219 * so if there are a ton of hard links to an inode there can
3220 * be a lot of backrefs. Don't waste time searching too hard,
3221 * this is just an optimization
3222 */
3225 }
3226 /* we hit the end of the leaf before we found an xattr or
3227 * something larger than an xattr. We have to assume the inode
3228 * has acls
3229 */
3233 }
3235 /*
3236 * read an inode from the btree into the in-memory inode
3237 */
3240 {
3249 u32 rdev;
3262 }
3271 }
3315 cache_index:
3316 /*
3317 * If we were modified in the current generation and evicted from memory
3318 * and then re-read we need to do a full sync since we don't have any
3319 * idea about which extents were modified before we were evicted from
3320 * cache.
3321 *
3322 * This is required for both inode re-read from disk and delayed inode
3323 * in delayed_nodes_tree.
3324 */
3329 /*
3330 * We don't persist the id of the transaction where an unlink operation
3331 * against the inode was last made. So here we assume the inode might
3332 * have been evicted, and therefore the exact value of last_unlink_trans
3333 * lost, and set it to last_trans to avoid metadata inconsistencies
3334 * between the inode and its parent if the inode is fsync'ed and the log
3335 * replayed. For example, in the scenario:
3336 *
3337 * touch mydir/foo
3338 * ln mydir/foo mydir/bar
3339 * sync
3340 * unlink mydir/bar
3341 * echo 2 > /proc/sys/vm/drop_caches # evicts inode
3342 * xfs_io -c fsync mydir/foo
3343 * <power failure>
3344 * mount fs, triggers fsync log replay
3345 *
3346 * We must make sure that when we fsync our inode foo we also log its
3347 * parent inode, otherwise after log replay the parent still has the
3348 * dentry with the "bar" name but our inode foo has a link count of 1
3349 * and doesn't have an inode ref with the name "bar" anymore.
3350 *
3351 * Setting last_unlink_trans to last_trans is a pessimistic approach,
3352 * but it guarantees correctness at the expense of occasional full
3353 * transaction commits on fsync if our inode is a directory, or if our
3354 * inode is not a directory, logging its parent unnecessarily.
3355 */
3378 extref);
3379 }
3380 cache_acl:
3381 /*
3382 * try to precache a NULL acl entry for files that don't have
3383 * any xattrs or acls
3384 */
3395 }
3422 }
3426 }
3428 /*
3429 * given a leaf and an inode, copy the inode fields into the leaf
3430 */
3435 {
3474 }
3476 /*
3477 * copy everything in the in-memory inode into the btree.
3478 */
3481 {
3498 }
3508 failed:
3511 }
3513 /*
3514 * copy everything in the in-memory inode into the btree.
3515 */
3518 {
3522 /*
3523 * If the inode is a free space inode, we can deadlock during commit
3524 * if we put it into the delayed code.
3525 *
3526 * The data relocation inode should also be directly updated
3527 * without delay
3528 */
3538 }
3541 }
3546 {
3553 }
3555 /*
3556 * unlink helper that gets used here in inode.c and in the tree logging
3557 * recovery code. It remove a link in a directory with a given name, and
3558 * also drops the back refs in the inode to the directory
3559 */
3565 {
3570 u64 index;
3578 }
3586 }
3592 /*
3593 * If we don't have dir index, we have to get it by looking up
3594 * the inode ref, since we get the inode ref, remove it directly,
3595 * it is unnecessary to do delayed deletion.
3596 *
3597 * But if we have dir index, needn't search inode ref to get it.
3598 * Since the inode ref is close to the inode item, it is better
3599 * that we delay to delete it, and just do this deletion when
3600 * we update the inode item.
3601 */
3607 }
3608 }
3618 }
3619 skip_backref:
3624 }
3627 dir_ino);
3631 }
3634 index);
3640 /*
3641 * If we have a pending delayed iput we could end up with the final iput
3642 * being run in btrfs-cleaner context. If we have enough of these built
3643 * up we can end up burning a lot of time in btrfs-cleaner without any
3644 * way to throttle the unlinks. Since we're currently holding a ref on
3645 * the inode we can run the delayed iput here without any issues as the
3646 * final iput won't be done until after we drop the ref we're currently
3647 * holding.
3648 */
3650 err:
3661 out:
3663 }
3669 {
3675 }
3677 }
3679 /*
3680 * helper to start transaction for unlink and rmdir.
3681 *
3682 * unlink and rmdir are special in btrfs, they do not always free space, so
3683 * if we cannot make our reservations the normal way try and see if there is
3684 * plenty of slack room in the global reserve to migrate, otherwise we cannot
3685 * allow the unlink to occur.
3686 */
3688 {
3691 /*
3692 * 1 for the possible orphan item
3693 * 1 for the dir item
3694 * 1 for the dir index
3695 * 1 for the inode ref
3696 * 1 for the inode
3697 */
3699 }
3702 {
3725 }
3727 out:
3731 }
3735 {
3744 u64 index;
3746 u64 objectid;
3756 }
3767 }
3776 }
3779 /*
3780 * This is a placeholder inode for a subvolume we didn't have a
3781 * reference to at the time of the snapshot creation. In the meantime
3782 * we could have renamed the real subvol link into our snapshot, so
3783 * depending on btrfs_del_root_ref to return -ENOENT here is incorret.
3784 * Instead simply lookup the dir_index_item for this entry so we can
3785 * remove it. Otherwise we know we have a ref to the root and we can
3786 * call btrfs_del_root_ref, and it _shouldn't_ fail.
3787 */
3794 else
3798 }
3811 }
3812 }
3818 }
3826 out:
3829 }
3831 /*
3832 * Helper to check if the subvolume references other subvolumes or if it's
3833 * default.
3834 */
3836 {
3841 u64 dir_id;
3848 /* Make sure this root isn't set as the default subvol */
3860 }
3862 }
3880 }
3881 out:
3884 }
3886 /* Delete all dentries for inodes belonging to the root */
3888 {
3900 again:
3911 else
3913 }
3920 }
3922 }
3923 }
3932 /*
3933 * btrfs_drop_inode will have it removed from the inode
3934 * cache when its usage count hits zero.
3935 */
3940 }
3946 }
3948 }
3951 {
3958 u64 root_flags;
3962 /*
3963 * Don't allow to delete a subvolume with send in progress. This is
3964 * inside the inode lock so the error handling that has to drop the bit
3965 * again is not run concurrently.
3966 */
3974 }
3987 /*
3988 * One for dir inode,
3989 * two for dir entries,
3990 * two for root ref/backref.
3991 */
4000 }
4011 }
4028 }
4029 }
4032 BTRFS_UUID_KEY_SUBVOL,
4038 }
4042 BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4048 }
4049 }
4051 out_end_trans:
4058 out_release:
4060 out_up_write:
4073 /* the last ref */
4077 }
4078 }
4081 }
4084 {
4089 u64 last_unlink_trans;
4103 }
4111 /* now the directory is empty */
4117 /*
4118 * Propagate the last_unlink_trans value of the deleted dir to
4119 * its parent directory. This is to prevent an unrecoverable
4120 * log tree in the case we do something like this:
4121 * 1) create dir foo
4122 * 2) create snapshot under dir foo
4123 * 3) delete the snapshot
4124 * 4) rmdir foo
4125 * 5) mkdir foo
4126 * 6) fsync foo or some file inside foo
4127 */
4130 }
4131 out:
4136 }
4138 /*
4139 * Return this if we need to call truncate_block for the last bit of the
4140 * truncate.
4141 */
4142 #define NEED_TRUNCATE_BLOCK 1
4144 /*
4145 * this can truncate away extent items, csum items and directory items.
4146 * It starts at a high offset and removes keys until it can't find
4147 * any higher than new_size
4148 *
4149 * csum items that cross the new i_size are truncated to the new size
4150 * as well.
4151 *
4152 * min_type is the minimum key type to truncate down to. If set to 0, this
4153 * will kill all the items on this inode, including the INODE_ITEM_KEY.
4154 */
4159 {
4187 /*
4188 * For non-free space inodes and non-shareable roots, we want to back
4189 * off from time to time. This means all inodes in subvolume roots,
4190 * reloc roots, and data reloc roots.
4191 */
4205 /*
4206 * We want to drop from the next block forward in case this
4207 * new size is not block aligned since we will be keeping the
4208 * last block of the extent just the way it is.
4209 */
4213 }
4215 /*
4216 * This function is also used to drop the items in the log tree before
4217 * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4218 * it is used to drop the logged items. So we shouldn't kill the delayed
4219 * items.
4220 */
4228 search_again:
4229 /*
4230 * with a 16K leaf size and 128MB extents, you can actually queue
4231 * up a huge file in a single leaf. Most of the time that
4232 * bytes_deleted is > 0, it will be huge by the time we get here
4233 */
4238 }
4246 /* there are no items in the tree for us to truncate, we're
4247 * done
4248 */
4252 }
4274 item_end +=
4282 fi);
4287 }
4288 item_end--;
4289 }
4297 else
4299 }
4301 /* FIXME, shrink the extent if the ref count is only 1 */
4306 u64 num_dec;
4311 u64 orig_num_bytes =
4318 extent_num_bytes);
4320 extent_num_bytes);
4327 extent_num_bytes =
4329 fi);
4333 /* FIXME blocksize != 4096 */
4340 }
4341 }
4344 /*
4345 * we can't truncate inline items that have had
4346 * special encodings
4347 */
4358 /*
4359 * We have to bail so the last_size is set to
4360 * just before this extent.
4361 */
4365 /*
4366 * Inline extents are special, we just treat
4367 * them as a full sector worth in the file
4368 * extent tree just for simplicity sake.
4369 */
4371 }
4375 }
4377 /*
4378 * We use btrfs_truncate_inode_items() to clean up log trees for
4379 * multiple fsyncs, and in this case we don't want to clear the
4380 * file extent range because it's just the log.
4381 */
4388 }
4389 }
4393 else
4397 /* no pending yet, add ourselves */
4402 /* hop on the pending chunk */
4403 pending_del_nr++;
4407 }
4410 }
4428 }
4432 }
4433 }
4440 should_throttle) {
4443 pending_del_slot,
4444 pending_del_nr);
4448 }
4450 }
4453 /*
4454 * We can generate a lot of delayed refs, so we need to
4455 * throttle every once and a while and make sure we're
4456 * adding enough space to keep up with the work we are
4457 * generating. Since we hold a transaction here we
4458 * can't flush, and we don't want to FLUSH_LIMIT because
4459 * we could have generated too many delayed refs to
4460 * actually allocate, so just bail if we're short and
4461 * let the normal reservation dance happen higher up.
4462 */
4465 BTRFS_RESERVE_NO_FLUSH);
4469 }
4470 }
4474 }
4475 }
4476 out:
4481 pending_del_nr);
4485 }
4486 }
4494 }
4498 }
4500 /*
4501 * btrfs_truncate_block - read, zero a chunk and write a block
4502 * @inode - inode that we're zeroing
4503 * @from - the offset to start zeroing
4504 * @len - the length to zero, 0 to zero the entire range respective to the
4505 * offset
4506 * @front - zero up to the offset instead of from the offset on
4507 *
4508 * This will find the block for the "from" offset and cow the block and zero the
4509 * part we want to zero. This is used with truncate and hole punching.
4510 */
4513 {
4527 u64 block_start;
4528 u64 block_end;
4542 again:
4550 }
4559 }
4563 }
4564 }
4579 }
4591 }
4600 else
4605 }
4610 out_unlock:
4617 out:
4620 }
4624 {
4629 /*
4630 * Still need to make sure the inode looks like it's been updated so
4631 * that any holes get logged if we fsync.
4632 */
4638 }
4640 /*
4641 * 1 - for the one we're dropping
4642 * 1 - for the one we're adding
4643 * 1 - for updating the inode.
4644 */
4654 }
4660 else
4664 }
4666 /*
4667 * This function puts in dummy file extents for the area we're creating a hole
4668 * for. So if we are truncating this file to a larger size we need to insert
4669 * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4670 * the range between oldsize and size
4671 */
4673 {
4682 u64 last_byte;
4683 u64 cur_offset;
4684 u64 hole_size;
4687 /*
4688 * If our size started in the middle of a block we need to zero out the
4689 * rest of the block before we expand the i_size, otherwise we could
4690 * expose stale data.
4691 */
4709 }
4718 hole_size);
4734 }
4753 cur_offset,
4754 cur_offset +
4756 }
4763 }
4764 next:
4770 }
4774 }
4777 {
4785 /*
4786 * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
4787 * special case where we need to update the times despite not having
4788 * these flags set. For all other operations the VFS set these flags
4789 * explicitly if it wants a timestamp update.
4790 */
4796 }
4799 /*
4800 * Don't do an expanding truncate while snapshotting is ongoing.
4801 * This is to ensure the snapshot captures a fully consistent
4802 * state of this file - if the snapshot captures this expanding
4803 * truncation, it must capture all writes that happened before
4804 * this truncation.
4805 */
4811 }
4817 }
4827 /*
4828 * We're truncating a file that used to have good data down to
4829 * zero. Make sure it gets into the ordered flush list so that
4830 * any new writes get down to disk quickly.
4831 */
4838 /* Disable nonlocked read DIO to avoid the endless truncate */
4847 /*
4848 * Truncate failed, so fix up the in-memory size. We
4849 * adjusted disk_i_size down as we removed extents, so
4850 * wait for disk_i_size to be stable and then update the
4851 * in-memory size to match.
4852 */
4857 }
4858 }
4861 }
4864 {
4880 }
4889 }
4892 }
4894 /*
4895 * While truncating the inode pages during eviction, we get the VFS calling
4896 * btrfs_invalidatepage() against each page of the inode. This is slow because
4897 * the calls to btrfs_invalidatepage() result in a huge amount of calls to
4898 * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
4899 * extent_state structures over and over, wasting lots of time.
4900 *
4901 * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
4902 * those expensive operations on a per page basis and do only the ordered io
4903 * finishing, while we release here the extent_map and extent_state structures,
4904 * without the excessive merging and splitting.
4905 */
4907 {
4929 }
4930 }
4933 /*
4934 * Keep looping until we have no more ranges in the io tree.
4935 * We can have ongoing bios started by readahead that have
4936 * their endio callback (extent_io.c:end_bio_extent_readpage)
4937 * still in progress (unlocked the pages in the bio but did not yet
4938 * unlocked the ranges in the io tree). Therefore this means some
4939 * ranges can still be locked and eviction started because before
4940 * submitting those bios, which are executed by a separate task (work
4941 * queue kthread), inode references (inode->i_count) were not taken
4942 * (which would be dropped in the end io callback of each bio).
4943 * Therefore here we effectively end up waiting for those bios and
4944 * anyone else holding locked ranges without having bumped the inode's
4945 * reference count - if we don't do it, when they access the inode's
4946 * io_tree to unlock a range it may be too late, leading to an
4947 * use-after-free issue.
4948 */
4953 u64 start;
4954 u64 end;
4966 /*
4967 * If still has DELALLOC flag, the extent didn't reach disk,
4968 * and its reserved space won't be freed by delayed_ref.
4969 * So we need to free its reserved space here.
4970 * (Refer to comment in btrfs_invalidatepage, case 2)
4971 *
4972 * Note, end is the bytenr of last byte, so we need + 1 here.
4973 */
4984 }
4986 }
4990 {
4997 /*
4998 * Eviction should be taking place at some place safe because of our
4999 * delayed iputs. However the normal flushing code will run delayed
5000 * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.
5001 *
5002 * We reserve the delayed_refs_extra here again because we can't use
5003 * btrfs_start_transaction(root, 0) for the same deadlocky reason as
5004 * above. We reserve our extra bit here because we generate a ton of
5005 * delayed refs activity by truncating.
5006 *
5007 * If we cannot make our reservation we'll attempt to steal from the
5008 * global reserve, because we really want to be able to free up space.
5009 */
5011 BTRFS_RESERVE_FLUSH_EVICT);
5013 /*
5014 * Try to steal from the global reserve if there is space for
5015 * it.
5016 */
5022 }
5024 }
5035 }
5037 }
5040 {
5052 }
5074 }
5103 }
5105 /*
5106 * Errors here aren't a big deal, it just means we leave orphan items in
5107 * the tree. They will be cleaned up on the next mount. If the inode
5108 * number gets reused, cleanup deletes the orphan item without doing
5109 * anything, and unlink reuses the existing orphan item.
5110 *
5111 * If it turns out that we are dropping too many of these, we might want
5112 * to add a mechanism for retrying these after a commit.
5113 */
5120 }
5126 free_rsv:
5128 no_delete:
5129 /*
5130 * If we didn't successfully delete, the orphan item will still be in
5131 * the tree and we'll retry on the next mount. Again, we might also want
5132 * to retry these periodically in the future.
5133 */
5136 }
5138 /*
5139 * Return the key found in the dir entry in the location pointer, fill @type
5140 * with BTRFS_FT_*, and return 0.
5141 *
5142 * If no dir entries were found, returns -ENOENT.
5143 * If found a corrupted location in dir entry, returns -EUCLEAN.
5144 */
5147 {
5164 }
5174 }
5177 out:
5180 }
5182 /*
5183 * when we hit a tree root in a directory, the btrfs part of the inode
5184 * needs to be changed to reflect the root directory of the tree root. This
5185 * is kind of like crossing a mount point.
5186 */
5192 {
5205 }
5217 }
5237 }
5244 out:
5247 }
5250 {
5278 }
5279 }
5283 }
5286 {
5295 }
5304 }
5305 }
5309 {
5319 }
5322 {
5327 }
5331 {
5340 btrfs_init_locked_inode,
5343 }
5345 /*
5346 * Get an inode object given its inode number and corresponding root.
5347 * Path can be preallocated to prevent recursing back to iget through
5348 * allocator. NULL is also valid but may require an additional allocation
5349 * later.
5350 */
5353 {
5369 /*
5370 * ret > 0 can come from btrfs_search_slot called by
5371 * btrfs_read_locked_inode, this means the inode item
5372 * was not found.
5373 */
5377 }
5378 }
5381 }
5384 {
5386 }
5391 {
5402 /*
5403 * We only need lookup, the rest is read-only and there's no inode
5404 * associated with the dentry
5405 */
5416 }
5419 {
5420 /*
5421 * Compile-time asserts that generic FT_* types still match
5422 * BTRFS_FT_* types
5423 */
5434 }
5437 {
5458 /* Do extra check against inode mode with di_type */
5463 di_type);
5466 }
5468 }
5475 else
5479 }
5491 }
5492 }
5495 }
5498 {
5512 }
5514 }
5518 {
5524 }
5526 /*
5527 * All this infrastructure exists because dir_emit can fault, and we are holding
5528 * the tree lock when doing readdir. For now just allocate a buffer and copy
5529 * our information into that, and then dir_emit from the buffer. This is
5530 * similar to what NFS does, only we don't keep the buffer around in pagecache
5531 * because I'm afraid I'll mess that up. Long term we need to make filldir do
5532 * copy_to_user_inatomic so we don't have to worry about page faulting under the
5533 * tree lock.
5534 */
5536 {
5546 }
5549 }
5552 u64 ino;
5553 u64 offset;
5556 };
5559 {
5572 }
5574 }
5577 {
5612 again:
5633 }
5648 PAGE_SIZE) {
5657 }
5663 name_len);
5669 entries++;
5672 next:
5674 }
5685 /*
5686 * Stop new entries from being returned after we return the last
5687 * entry.
5688 *
5689 * New directory entries are assigned a strictly increasing
5690 * offset. This means that new entries created during readdir
5691 * are *guaranteed* to be seen in the future by that readdir.
5692 * This has broken buggy programs which operate on names as
5693 * they're returned by readdir. Until we re-use freed offsets
5694 * we have this hack to stop new entries from being returned
5695 * under the assumption that they'll never reach this huge
5696 * offset.
5697 *
5698 * This is being careful not to overflow 32bit loff_t unless the
5699 * last entry requires it because doing so has broken 32bit apps
5700 * in the past.
5701 */
5704 else
5706 nopos:
5708 err:
5713 }
5715 /*
5716 * This is somewhat expensive, updating the tree every time the
5717 * inode changes. But, it is most likely to find the inode in cache.
5718 * FIXME, needs more benchmarking...there are no reasons other than performance
5719 * to keep or drop this code.
5720 */
5722 {
5737 /* whoops, lets try again with the full transaction */
5744 }
5750 }
5752 /*
5753 * This is a copy of file_update_time. We need this so we can return error on
5754 * ENOSPC for updating the inode in the case of file write and mmap writes.
5755 */
5758 {
5774 }
5776 /*
5777 * find the highest existing sequence number in a directory
5778 * and then set the in-memory index_cnt variable to reflect
5779 * free sequence numbers
5780 */
5782 {
5800 /* FIXME: we should be able to handle this */
5805 /*
5806 * MAGIC NUMBER EXPLANATION:
5807 * since we search a directory based on f_pos we have to start at 2
5808 * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
5809 * else has to start at 2
5810 */
5814 }
5825 }
5828 out:
5831 }
5833 /*
5834 * helper to find a free sequence number in a given directory. This current
5835 * code is very simple, later versions will do smarter things in the btree
5836 */
5838 {
5847 }
5848 }
5854 }
5857 {
5866 }
5868 /*
5869 * Inherit flags from the parent inode.
5870 *
5871 * Currently only the compression flags and the cow flags are inherited.
5872 */
5874 {
5888 }
5894 }
5897 }
5905 {
5929 }
5931 /*
5932 * O_TMPFILE, set link count to 0, so that after this point,
5933 * we fill in an inode item with the correct link count.
5934 */
5938 /*
5939 * we have to initialize this early, so we can reclaim the inode
5940 * number if we fail afterwards in this function.
5941 */
5952 }
5955 }
5956 /*
5957 * index_cnt is ignored for everything but a dir,
5958 * btrfs_set_inode_index_count has an explanation for the magic
5959 * number
5960 */
5967 /*
5968 * We could have gotten an inode number from somebody who was fsynced
5969 * and then removed in this same transaction, so let's just set full
5970 * sync since it will be a full sync anyway and this will blow away the
5971 * old info in the log.
5972 */
5982 /*
5983 * Start new inodes with an inode_ref. This is slightly more
5984 * efficient for small numbers of hard links since they will
5985 * be packed into one item. Extended refs will kick in if we
5986 * add more hard links than can fit in the ref item.
5987 */
5993 }
6004 }
6032 }
6044 BTRFS_INODE_NODATASUM;
6045 }
6062 fail_unlock:
6064 fail:
6069 }
6071 /*
6072 * utility function to add 'inode' into 'parent_inode' with
6073 * a give name and a given sequence number.
6074 * if 'add_backref' is true, also insert a backref from the
6075 * inode to the parent directory.
6076 */
6080 {
6093 }
6102 }
6104 /* Nothing to clean up yet */
6115 }
6120 /*
6121 * If we are replaying a log tree, we do not want to update the mtime
6122 * and ctime of the parent directory with the current time, since the
6123 * log replay procedure is responsible for setting them to their correct
6124 * values (the ones it had when the fsync was done).
6125 */
6131 }
6137 fail_dir_item:
6139 u64 local_index;
6147 u64 local_index;
6154 }
6156 /* Return the original error code */
6158 }
6163 {
6170 }
6174 {
6180 u64 objectid;
6183 /*
6184 * 2 for inode item and ref
6185 * 2 for dir items
6186 * 1 for xattr if selinux is on
6187 */
6203 }
6205 /*
6206 * If the active LSM wants to access the inode during
6207 * d_instantiate it needs these. Smack checks to see
6208 * if the filesystem supports xattrs by looking at the
6209 * ops vector.
6210 */
6226 out_unlock:
6232 }
6234 }
6238 {
6244 u64 objectid;
6247 /*
6248 * 2 for inode item and ref
6249 * 2 for dir items
6250 * 1 for xattr if selinux is on
6251 */
6267 }
6268 /*
6269 * If the active LSM wants to access the inode during
6270 * d_instantiate it needs these. Smack checks to see
6271 * if the filesystem supports xattrs by looking at the
6272 * ops vector.
6273 */
6294 out_unlock:
6299 }
6302 }
6306 {
6311 u64 index;
6315 /* do not allow sys_link's with other subvols of the same device */
6326 /*
6327 * 2 items for inode and inode ref
6328 * 2 items for dir items
6329 * 1 item for parent inode
6330 * 1 item for orphan item deletion if O_TMPFILE
6331 */
6337 }
6339 /* There are several dir indexes for this inode, clear the cache. */
6360 /*
6361 * If new hard link count is 1, it's a file created
6362 * with open(2) O_TMPFILE flag.
6363 */
6367 }
6374 }
6375 }
6377 fail:
6383 }
6386 }
6389 {
6398 /*
6399 * 2 items for inode and ref
6400 * 2 items for dir items
6401 * 1 for xattr if selinux is on
6402 */
6418 }
6420 /* these must be set before we unlock the inode */
6441 out_fail:
6446 }
6449 }
6455 {
6480 /*
6481 * decompression code contains a memset to fill in any space between the end
6482 * of the uncompressed data and the end of max_size in case the decompressed
6483 * data ends up shorter than ram_bytes. That doesn't cover the hole between
6484 * the end of an inline extent and the beginning of the next block, so we
6485 * cover that region here.
6486 */
6492 }
6495 }
6497 /**
6498 * btrfs_get_extent - Lookup the first extent overlapping a range in a file.
6499 * @inode: file to search in
6500 * @page: page to read extent data into if the extent is inline
6501 * @pg_offset: offset into @page to copy to
6502 * @start: file offset
6503 * @len: length of range starting at @start
6504 *
6505 * This returns the first &struct extent_map which overlaps with the given
6506 * range, reading it from the B-tree and caching it if necessary. Note that
6507 * there may be more extents which overlap the given range after the returned
6508 * extent_map.
6509 *
6510 * If @page is not NULL and the extent is inline, this also reads the extent
6511 * data directly into the page and marks the extent up to date in the io_tree.
6512 *
6513 * Return: ERR_PTR on error, non-NULL extent_map on success.
6514 */
6518 {
6544 else
6546 }
6551 }
6561 }
6563 /* Chances are we'll be called again, so go ahead and do readahead */
6566 /*
6567 * Unless we're going to uncompress the inline extent, no sleep would
6568 * happen.
6569 */
6580 }
6588 /*
6589 * If we backup past the first extent we want to move forward
6590 * and see if there is an extent in front of us, otherwise we'll
6591 * say there is a hole for our whole search range which can
6592 * cause problems.
6593 */
6596 }
6603 /* Only regular file could have regular/prealloc extent */
6610 }
6612 extent_start);
6616 extent_start);
6617 }
6618 next:
6628 }
6630 }
6640 /* New extent overlaps with existing one */
6646 }
6676 BTRFS_COMPRESS_NONE) {
6682 }
6686 copy_size);
6690 copy_size);
6691 }
6693 }
6695 }
6699 }
6700 not_found:
6705 insert:
6713 }
6719 out:
6727 }
6730 }
6734 {
6738 u64 end;
6739 u64 delalloc_len;
6740 u64 delalloc_end;
6746 /*
6747 * If our em maps to:
6748 * - a hole or
6749 * - a pre-alloc extent,
6750 * there might actually be delalloc bytes behind it.
6751 */
6755 else
6758 /* check to see if we've wrapped (len == -1 or similar) */
6762 else
6767 /* ok, we didn't find anything, lets look for delalloc */
6774 /*
6775 * We didn't find anything useful, return the original results from
6776 * get_extent()
6777 */
6782 }
6784 /*
6785 * Adjust the delalloc_start to make sure it doesn't go backwards from
6786 * the start they passed in
6787 */
6792 u64 hole_start;
6793 u64 hole_len;
6800 }
6803 /*
6804 * When btrfs_get_extent can't find anything it returns one
6805 * huge hole
6806 *
6807 * Make sure what it found really fits our range, and adjust to
6808 * make sure it is based on the start from the caller
6809 */
6816 }
6819 /*
6820 * Our hole starts before our delalloc, so we have to
6821 * return just the parts of the hole that go until the
6822 * delalloc starts
6823 */
6827 /*
6828 * Don't adjust block start at all, it is fixed at
6829 * EXTENT_MAP_HOLE
6830 */
6836 /*
6837 * Hole is out of passed range or it starts after
6838 * delalloc range
6839 */
6845 }
6848 }
6849 out:
6855 }
6857 }
6868 {
6875 ram_bytes,
6877 type);
6880 }
6888 }
6890 }
6891 out:
6894 }
6898 {
6903 u64 alloc_hint;
6921 }
6923 /*
6924 * returns 1 when the nocow is safe, < 1 on error, 0 if the
6925 * block must be cow'd
6926 */
6930 {
6939 u64 disk_bytenr;
6940 u64 backref_offset;
6941 u64 extent_end;
6942 u64 num_bytes;
6959 /* can't find the item, must cow */
6962 }
6963 slot--;
6964 }
6970 /* not our file or wrong item type, must cow */
6972 }
6975 /* Wrong offset, must cow */
6977 }
6983 /* not a regular extent, must cow */
6985 }
7003 /*
7004 * Do the same check as in btrfs_cross_ref_exist but without the
7005 * unnecessary search.
7006 */
7017 }
7024 u64 range_end;
7033 }
7034 }
7038 /*
7039 * look for other files referencing this extent, if we
7040 * find any we must cow
7041 */
7048 }
7050 /*
7051 * adjust disk_bytenr and num_bytes to cover just the bytes
7052 * in this extent we are about to write. If there
7053 * are any csums in that range we have to cow in order
7054 * to keep the csums correct
7055 */
7060 /*
7061 * all of the above have passed, it is safe to overwrite this extent
7062 * without cow
7063 */
7066 out:
7069 }
7073 {
7079 cached_state);
7080 /*
7081 * We're concerned with the entire range that we're going to be
7082 * doing DIO to, so we need to make sure there's no ordered
7083 * extents in this range.
7084 */
7088 /*
7089 * We need to make sure there are no buffered pages in this
7090 * range either, we could have raced between the invalidate in
7091 * generic_file_direct_write and locking the extent. The
7092 * invalidate needs to happen so that reads after a write do not
7093 * get stale data.
7094 */
7101 cached_state);
7104 /*
7105 * If we are doing a DIO read and the ordered extent we
7106 * found is for a buffered write, we can not wait for it
7107 * to complete and retry, because if we do so we can
7108 * deadlock with concurrent buffered writes on page
7109 * locks. This happens only if our DIO read covers more
7110 * than one extent map, if at this point has already
7111 * created an ordered extent for a previous extent map
7112 * and locked its range in the inode's io tree, and a
7113 * concurrent write against that previous extent map's
7114 * range and this range started (we unlock the ranges
7115 * in the io tree only when the bios complete and
7116 * buffered writes always lock pages before attempting
7117 * to lock range in the io tree).
7118 */
7122 else
7126 /*
7127 * We could trigger writeback for this range (and wait
7128 * for it to complete) and then invalidate the pages for
7129 * this range (through invalidate_inode_pages2_range()),
7130 * but that can lead us to a deadlock with a concurrent
7131 * call to readahead (a buffered read or a defrag call
7132 * triggered a readahead) on a page lock due to an
7133 * ordered dio extent we created before but did not have
7134 * yet a corresponding bio submitted (whence it can not
7135 * complete), which makes readahead wait for that
7136 * ordered extent to complete while holding a lock on
7137 * that page.
7138 */
7140 }
7146 }
7149 }
7151 /* The callers of this must take lock_extent() */
7157 {
7186 }
7194 /*
7195 * The caller has taken lock_extent(), who could race with us
7196 * to add em?
7197 */
7203 }
7205 /* em got 2 refs now, callers needs to do free_extent_map once. */
7207 }
7214 {
7230 }
7237 {
7242 /*
7243 * We don't allocate a new extent in the following cases
7244 *
7245 * 1) The inode is marked as NODATACOW. In this case we'll just use the
7246 * existing extent.
7247 * 2) The extent is marked as PREALLOC. We're good to go here and can
7248 * just use the extent.
7249 *
7250 */
7259 else
7277 }
7282 }
7283 /*
7284 * For inode marked NODATACOW or extent marked PREALLOC,
7285 * use the existing or preallocated extent, so does not
7286 * need to adjust btrfs_space_info's bytes_may_use.
7287 */
7289 len);
7291 }
7292 }
7294 /* this will cow the extent */
7301 }
7305 skip_cow:
7315 /*
7316 * Need to update the i_size under the extent lock so buffered
7317 * readers will get the updated i_size when we unlock.
7318 */
7326 out:
7328 }
7332 {
7349 /*
7350 * Need to pull our outstanding extents and set journal_info to NULL so
7351 * that anything that needs to check if there's a transaction doesn't get
7352 * confused.
7353 */
7356 }
7358 /*
7359 * If this errors out it's because we couldn't invalidate pagecache for
7360 * this range and we need to fallback to buffered.
7361 */
7363 create)) {
7366 }
7372 }
7374 /*
7375 * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7376 * io. INLINE is special, and we could probably kludge it in here, but
7377 * it's still buffered so for safety lets just fall back to the generic
7378 * buffered path.
7379 *
7380 * For COMPRESSED we _have_ to read the entire extent in so we can
7381 * decompress it, so there will be buffering required no matter what we
7382 * do, so go ahead and fallback to buffered.
7383 *
7384 * We return -ENOTBLK because that's what makes DIO go ahead and go back
7385 * to buffered IO. Don't blame me, this is the price we pay for using
7386 * the generic code.
7387 */
7393 }
7406 /* Can be negative only if we read from a hole */
7411 }
7412 /*
7413 * We need to unlock only the end area that we aren't using.
7414 * The rest is going to be unlocked by the endio routine.
7415 */
7422 }
7423 }
7429 unlock_err:
7432 err:
7436 }
7439 {
7440 /*
7441 * This implies a barrier so that stores to dio_bio->bi_status before
7442 * this and loads of dio_bio->bi_status after this are fully ordered.
7443 */
7455 }
7459 }
7464 {
7467 blk_status_t ret;
7480 }
7485 {
7511 pgoff);
7513 blk_status_t status;
7519 start,
7522 submit_dio_repair_bio);
7525 }
7527 icsum++;
7529 }
7530 }
7532 }
7537 {
7543 u64 last_offset;
7547 else
7554 ordered_bytes,
7555 uptodate)) {
7557 NULL);
7559 }
7560 /*
7561 * If btrfs_dec_test_ordered_pending does not find any ordered
7562 * extent in the range, we can exit.
7563 */
7566 /*
7567 * Our bio might span multiple ordered extents. In this case
7568 * we keep going until we have accounted the whole dio.
7569 */
7573 }
7574 }
7575 }
7579 {
7581 blk_status_t ret;
7585 }
7588 {
7603 }
7610 }
7614 {
7618 blk_status_t ret;
7620 /* Check btrfs_submit_bio_hook() for rules about async submit. */
7628 }
7636 btrfs_submit_bio_start_direct_io);
7639 /*
7640 * If we aren't doing async submit, calculate the csum of the
7641 * bio now.
7642 */
7647 u64 csum_offset;
7653 }
7654 map:
7656 err:
7658 }
7660 /*
7661 * If this succeeds, the btrfs_dio_private is responsible for cleaning up locked
7662 * or ordered extents whether or not we submit any bios.
7663 */
7666 loff_t file_offset)
7667 {
7681 }
7697 /*
7698 * Setting range start and end to the same value means that
7699 * no cleanup will happen in btrfs_direct_IO
7700 */
7705 }
7707 }
7710 loff_t file_offset)
7711 {
7716 BTRFS_BLOCK_GROUP_RAID56_MASK);
7719 u64 start_sector;
7721 u64 submit_len;
7725 blk_status_t status;
7733 }
7737 }
7740 /*
7741 * Load the csums up front to reduce csum tree searches and
7742 * contention when submitting bios.
7743 */
7748 }
7760 }
7765 /*
7766 * This will never fail as it's passing GPF_NOFS and
7767 * the allocation is backed by btrfs_bioset.
7768 */
7777 /*
7778 * Increase the count before we submit the bio so we know
7779 * the end IO handler won't happen before we increase the
7780 * count. Otherwise, the dip might get freed before we're
7781 * done setting it up.
7782 *
7783 * We transfer the initial reference to the last bio, so we
7784 * don't need to increment the reference count for the last one.
7785 */
7788 /*
7789 * If we are submitting more than one bio, submit them
7790 * all asynchronously. The exception is RAID 5 or 6, as
7791 * asynchronous checksums make it difficult to collect
7792 * full stripe writes.
7793 */
7796 }
7799 async_submit);
7805 }
7813 out_err:
7816 }
7820 {
7832 /* If this is a write we don't need to check anymore */
7835 /*
7836 * Check to make sure we don't have duplicate iov_base's in this
7837 * iovec, if so return EINVAL, otherwise we'll get csum errors
7838 * when reading back.
7839 */
7844 }
7845 }
7847 out:
7849 }
7852 {
7863 ssize_t ret;
7870 /*
7871 * The generic stuff only does filemap_write_and_wait_range, which
7872 * isn't enough if we've written compressed pages to this area, so
7873 * we need to flush the dirty pages again to make absolutely sure
7874 * that any outstanding dirty pages are on disk.
7875 */
7883 /*
7884 * If the write DIO is beyond the EOF, we need update
7885 * the isize, but it is protected by i_mutex. So we can
7886 * not unlock the i_mutex at this case.
7887 */
7892 }
7898 /*
7899 * We need to know how many extents we reserved so that we can
7900 * do the accounting properly if we go over the number we
7901 * originally calculated. Abuse current->journal_info for this.
7902 */
7914 }
7927 /*
7928 * On error we might have left some ordered extents
7929 * without submitting corresponding bios for them, so
7930 * cleanup them up to avoid other tasks getting them
7931 * and waiting for them to complete forever.
7932 */
7944 }
7945 out:
7953 }
7957 {
7965 }
7968 {
7970 }
7973 {
7981 }
7983 /*
7984 * If we are under memory pressure we will call this directly from the
7985 * VM, we need to make sure we have the inode referenced for the ordered
7986 * extent. If not just return like we didn't do anything.
7987 */
7991 }
7995 }
7999 {
8001 }
8004 {
8006 }
8009 {
8014 }
8017 {
8021 }
8023 #ifdef CONFIG_MIGRATION
8027 {
8040 }
8044 else
8047 }
8048 #endif
8052 {
8059 u64 start;
8060 u64 end;
8063 /*
8064 * we have the page locked, so new writeback can't start,
8065 * and the dirty bit won't be cleared while we are here.
8066 *
8067 * Wait for IO on this page so that we can safely clear
8068 * the PagePrivate2 bit and do ordered accounting
8069 */
8076 }
8080 again:
8087 /*
8088 * IO on this page will never be started, so we need
8089 * to account for any ordered extents now
8090 */
8096 /*
8097 * whoever cleared the private bit is responsible
8098 * for the finish_ordered_io
8099 */
8102 u64 new_len;
8114 start,
8117 }
8123 }
8128 }
8130 /*
8131 * Qgroup reserved space handler
8132 * Page here will be either
8133 * 1) Already written to disk
8134 * In this case, its reserved space is released from data rsv map
8135 * and will be freed by delayed_ref handler finally.
8136 * So even we call qgroup_free_data(), it won't decrease reserved
8137 * space.
8138 * 2) Not written to disk
8139 * This means the reserved space should be freed here. However,
8140 * if a truncate invalidates the page (by clearing PageDirty)
8141 * and the page is accounted for while allocating extent
8142 * in btrfs_check_data_free_space() we let delayed_ref to
8143 * free the entire extent.
8144 */
8154 }
8158 }
8160 /*
8161 * btrfs_page_mkwrite() is not allowed to change the file size as it gets
8162 * called from a page fault handler when a page is first dirtied. Hence we must
8163 * be careful to check for EOF conditions here. We set the page up correctly
8164 * for a written page which means we get ENOSPC checking when writing into
8165 * holes and correct delalloc and unwritten extent mapping on filesystems that
8166 * support these features.
8167 *
8168 * We are not allowed to take the i_mutex here so we have to play games to
8169 * protect against truncate races as the page could now be beyond EOF. Because
8170 * truncate_setsize() writes the inode size before removing pages, once we have
8171 * the page lock we can determine safely if the page is beyond EOF. If it is not
8172 * beyond EOF, then the page is guaranteed safe against truncation until we
8173 * unlock the page.
8174 */
8176 {
8186 loff_t size;
8187 vm_fault_t ret;
8190 u64 reserved_space;
8191 u64 page_start;
8192 u64 page_end;
8193 u64 end;
8202 /*
8203 * Reserving delalloc space after obtaining the page lock can lead to
8204 * deadlock. For example, if a dirty page is locked by this function
8205 * and the call to btrfs_delalloc_reserve_space() ends up triggering
8206 * dirty page write out, then the btrfs_writepage() function could
8207 * end up waiting indefinitely to get a lock on the page currently
8208 * being processed by btrfs_page_mkwrite() function.
8209 */
8211 reserved_space);
8215 }
8221 }
8224 again:
8230 /* page got truncated out from underneath us */
8232 }
8238 /*
8239 * we can't set the delalloc bits if there are pending ordered
8240 * extents. Drop our locks and wait for them to finish
8241 */
8243 PAGE_SIZE);
8251 }
8261 }
8262 }
8264 /*
8265 * page_mkwrite gets called when the page is firstly dirtied after it's
8266 * faulted in, but write(2) could also dirty a page and set delalloc
8267 * bits, thus in this case for space account reason, we still need to
8268 * clear any delalloc bits within this page range since we have to
8269 * reserve data&meta space before lock_page() (see above comments).
8270 */
8282 }
8284 /* page is wholly or partially inside EOF */
8287 else
8295 }
8311 out_unlock:
8313 out:
8317 out_noreserve:
8321 }
8324 {
8338 }
8340 /*
8341 * Yes ladies and gentlemen, this is indeed ugly. We have a couple of
8342 * things going on here:
8343 *
8344 * 1) We need to reserve space to update our inode.
8345 *
8346 * 2) We need to have something to cache all the space that is going to
8347 * be free'd up by the truncate operation, but also have some slack
8348 * space reserved in case it uses space during the truncate (thank you
8349 * very much snapshotting).
8350 *
8351 * And we need these to be separate. The fact is we can use a lot of
8352 * space doing the truncate, and we have no earthly idea how much space
8353 * we will use, so we need the truncate reservation to be separate so it
8354 * doesn't end up using space reserved for updating the inode. We also
8355 * need to be able to stop the transaction and start a new one, which
8356 * means we need to be able to update the inode several times, and we
8357 * have no idea of knowing how many times that will be, so we can't just
8358 * reserve 1 item for the entirety of the operation, so that has to be
8359 * done separately as well.
8360 *
8361 * So that leaves us with
8362 *
8363 * 1) rsv - for the truncate reservation, which we will steal from the
8364 * transaction reservation.
8365 * 2) fs_info->trans_block_rsv - this will have 1 items worth left for
8366 * updating the inode.
8367 */
8374 /*
8375 * 1 for the truncate slack space
8376 * 1 for updating the inode.
8377 */
8382 }
8384 /* Migrate the slack space for the truncate to our reserve */
8389 /*
8390 * So if we truncate and then write and fsync we normally would just
8391 * write the extents that changed, which is a problem if we need to
8392 * first truncate that entire inode. So set this flag so we write out
8393 * all of the extents in the inode to the sync log so we're completely
8394 * safe.
8395 */
8402 BTRFS_EXTENT_DATA_KEY);
8419 }
8426 }
8428 /*
8429 * We can't call btrfs_truncate_block inside a trans handle as we could
8430 * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know
8431 * we've truncated everything except the last little bit, and can do
8432 * btrfs_truncate_block and then update the disk_i_size.
8433 */
8445 }
8447 }
8461 }
8462 out:
8466 }
8468 /*
8469 * create a new subvolume directory/inode (helper for the ioctl).
8470 */
8474 u64 new_dirid)
8475 {
8503 }
8506 {
8535 BTRFS_BLOCK_RSV_DELALLOC);
8563 }
8565 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
8567 {
8570 }
8571 #endif
8574 {
8576 }
8579 {
8594 /*
8595 * This can happen where we create an inode, but somebody else also
8596 * created the same inode and we need to destroy the one we already
8597 * created.
8598 */
8613 }
8614 }
8620 }
8623 {
8629 /* the snap/subvol tree is on deleting */
8632 else
8634 }
8637 {
8641 }
8644 {
8645 /*
8646 * Make sure all delayed rcu free inodes are flushed before we
8647 * destroy cache.
8648 */
8655 }
8658 {
8662 init_once);
8691 fail:
8694 }
8698 {
8699 u64 delalloc_bytes;
8717 STATX_ATTR_COMPRESSED |
8718 STATX_ATTR_IMMUTABLE |
8719 STATX_ATTR_NODUMP);
8730 }
8736 {
8758 /* we only allow rename subvolume link between subvolumes */
8765 /* close the race window with snapshot create/destroy ioctl */
8770 /*
8771 * We want to reserve the absolute worst case amount of items. So if
8772 * both inodes are subvols and we need to unlink them then that would
8773 * require 4 item modifications, but if they are both normal inodes it
8774 * would require 5 item modifications, so we'll assume their normal
8775 * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items
8776 * should cover the worst case number of items we'll modify.
8777 */
8782 }
8787 /*
8788 * We need to find a free sequence number both in the source and
8789 * in the destination directory for the exchange.
8790 */
8801 /* Reference for the source. */
8803 /* force full log commit if subvolume involved. */
8811 old_ino,
8813 old_idx);
8816 }
8818 /* And now for the dest. */
8820 /* force full log commit if subvolume involved. */
8828 new_ino,
8830 new_idx);
8833 }
8835 /* Update inode version and ctime/mtime. */
8850 }
8852 /* src is a subvolume */
8862 }
8866 }
8868 /* dest is a subvolume */
8878 }
8882 }
8890 }
8898 }
8917 }
8929 }
8932 }
8933 out_fail:
8934 /*
8935 * If we have pinned a log and an error happened, we unpin tasks
8936 * trying to sync the log and force them to fallback to a transaction
8937 * commit if the log currently contains any of the inodes involved in
8938 * this rename operation (to ensure we do not persist a log with an
8939 * inconsistent state for any of these inodes or leading to any
8940 * inconsistencies when replayed). If the transaction was aborted, the
8941 * abortion reason is propagated to userspace when attempting to commit
8942 * the transaction. If the log does not contain any of these inodes, we
8943 * allow the tasks to sync it.
8944 */
8956 }
8960 }
8961 }
8967 }
8973 }
8975 /*
8976 * We may have set commit_transaction when logging the new name
8977 * in the destination root, in which case we left the source
8978 * root context in the list of log contextes. So make sure we
8979 * remove it to avoid invalid memory accesses, since the context
8980 * was allocated in our stack frame.
8981 */
8986 }
8993 }
8994 out_notrans:
9003 }
9009 {
9012 u64 objectid;
9013 u64 index;
9023 objectid,
9030 }
9034 WHITEOUT_DEV);
9047 out:
9054 }
9059 {
9078 /* we only allow rename subvolume link between subvolumes */
9091 /* check for collisions, even if the name isn't there */
9098 /* we shouldn't get
9099 * eexist without a new_inode */
9102 }
9104 /* maybe -EOVERFLOW */
9106 }
9107 }
9110 /*
9111 * we're using rename to replace one file with another. Start IO on it
9112 * now so we don't add too much work to the end of the transaction
9113 */
9117 /* close the racy window with snapshot create/destroy ioctl */
9120 /*
9121 * We want to reserve the absolute worst case amount of items. So if
9122 * both inodes are subvols and we need to unlink them then that would
9123 * require 4 item modifications, but if they are both normal inodes it
9124 * would require 5 item modifications, so we'll assume they are normal
9125 * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9126 * should cover the worst case number of items we'll modify.
9127 * If our rename has the whiteout flag, we need more 5 units for the
9128 * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
9129 * when selinux is enabled).
9130 */
9138 }
9149 /* force full log commit if subvolume involved. */
9157 old_ino,
9161 }
9183 }
9187 }
9193 BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
9201 }
9208 }
9209 }
9217 }
9236 }
9240 old_dentry);
9245 }
9246 }
9247 out_fail:
9248 /*
9249 * If we have pinned the log and an error happened, we unpin tasks
9250 * trying to sync the log and force them to fallback to a transaction
9251 * commit if the log currently contains any of the inodes involved in
9252 * this rename operation (to ensure we do not persist a log with an
9253 * inconsistent state for any of these inodes or leading to any
9254 * inconsistencies when replayed). If the transaction was aborted, the
9255 * abortion reason is propagated to userspace when attempting to commit
9256 * the transaction. If the log does not contain any of these inodes, we
9257 * allow the tasks to sync it.
9258 */
9269 }
9278 }
9286 }
9287 out_notrans:
9292 }
9297 {
9303 new_dentry);
9306 }
9313 };
9316 {
9321 work);
9330 }
9333 {
9346 }
9348 /*
9349 * some fairly slow code that needs optimization. This walks the list
9350 * of all the inodes with pending delalloc and forces them to disk.
9351 */
9353 {
9369 delalloc_inodes);
9377 }
9388 }
9392 ret++;
9397 }
9400 out:
9405 }
9411 }
9414 }
9417 {
9428 }
9431 {
9446 delalloc_root);
9461 }
9463 }
9467 out:
9472 }
9475 }
9479 {
9487 u64 objectid;
9499 /*
9500 * 2 items for inode item and ref
9501 * 2 items for dir items
9502 * 1 item for updating parent inode item
9503 * 1 item for the inline extent item
9504 * 1 item for xattr if selinux is on
9505 */
9521 }
9523 /*
9524 * If the active LSM wants to access the inode during
9525 * d_instantiate it needs these. Smack checks to see
9526 * if the filesystem supports xattrs by looking at the
9527 * ops vector.
9528 */
9542 }
9548 datasize);
9552 }
9558 BTRFS_FILE_EXTENT_INLINE);
9574 /*
9575 * Last step, add directory indexes for our symlink inode. This is the
9576 * last step to avoid extra cleanup of these indexes if an error happens
9577 * elsewhere above.
9578 */
9587 out_unlock:
9592 }
9595 }
9601 {
9609 u64 i_size;
9610 u64 cur_bytes;
9624 }
9625 }
9629 /*
9630 * If we are severely fragmented we could end up with really
9631 * small allocations, so if the allocator is returning small
9632 * chunks lets make its job easier by only searching for those
9633 * sized chunks.
9634 */
9642 }
9644 /*
9645 * We've reserved this space, and thus converted it from
9646 * ->bytes_may_use to ->bytes_reserved. Any error that happens
9647 * from here on out we will only need to clear our reservation
9648 * for the remaining unreserved area, so advance our
9649 * clear_offset by our extent size.
9650 */
9659 BTRFS_FILE_EXTENT_PREALLOC);
9667 }
9677 }
9698 }
9700 next:
9713 else
9717 }
9726 }
9730 }
9735 }
9740 {
9743 NULL);
9744 }
9750 {
9753 }
9756 {
9758 }
9761 {
9771 }
9773 }
9776 {
9781 u64 objectid;
9782 u64 index;
9785 /*
9786 * 5 units required for adding orphan entry
9787 */
9802 }
9821 /*
9822 * We set number of links to 0 in btrfs_new_inode(), and here we set
9823 * it to 1 because d_tmpfile() will issue a warning if the count is 0,
9824 * through:
9825 *
9826 * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
9827 */
9832 out:
9838 }
9841 {
9852 index++;
9853 }
9854 }
9856 #ifdef CONFIG_SWAP
9857 /*
9858 * Add an entry indicating a block group or device which is pinned by a
9859 * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a
9860 * negative errno on failure.
9861 */
9864 {
9892 }
9893 }
9898 }
9900 /* Free all of the entries pinned by this swapfile. */
9902 {
9917 }
9919 }
9921 }
9924 u64 start;
9925 u64 block_start;
9926 u64 block_len;
9927 u64 lowest_ppage;
9928 u64 highest_ppage;
9931 };
9935 {
9950 first_ppage_reported++;
9962 }
9965 {
9970 }
9974 {
9983 };
9985 u64 isize;
9986 u64 start;
9988 /*
9989 * If the swap file was just created, make sure delalloc is done. If the
9990 * file changes again after this, the user is doing something stupid and
9991 * we don't really care.
9992 */
9997 /*
9998 * The inode is locked, so these flags won't change after we check them.
9999 */
10003 }
10007 }
10011 }
10013 /*
10014 * Balance or device remove/replace/resize can move stuff around from
10015 * under us. The EXCL_OP flag makes sure they aren't running/won't run
10016 * concurrently while we are mapping the swap extents, and
10017 * fs_info->swapfile_pins prevents them from running while the swap file
10018 * is active and moving the extents. Note that this also prevents a
10019 * concurrent device add which isn't actually necessary, but it's not
10020 * really worth the trouble to allow it.
10021 */
10026 }
10027 /*
10028 * Snapshots can create extents which require COW even if NODATACOW is
10029 * set. We use this counter to prevent snapshots. We must increment it
10030 * before walking the extents because we don't want a concurrent
10031 * snapshot to run after we've already checked the extents.
10032 */
10048 }
10054 }
10056 /*
10057 * It's unlikely we'll ever actually find ourselves
10058 * here, as a file small enough to fit inline won't be
10059 * big enough to store more than the swap header, but in
10060 * case something changes in the future, let's catch it
10061 * here rather than later.
10062 */
10066 }
10071 }
10088 }
10094 }
10101 }
10114 }
10128 }
10135 else
10137 }
10147 }
10151 }
10154 }
10159 out:
10180 }
10181 #else
10183 {
10184 }
10188 {
10190 }
10191 #endif
10211 };
10219 #ifdef CONFIG_COMPAT
10221 #endif
10224 };
10227 /* mandatory callbacks */
10230 };
10232 /*
10233 * btrfs doesn't support the bmap operation because swapfiles
10234 * use bmap to make a mapping of extents in the file. They assume
10235 * these extents won't change over the life of the file and they
10236 * use the bmap result to do IO directly to the drive.
10237 *
10238 * the btrfs bmap call would return logical addresses that aren't
10239 * suitable for IO and they also will change frequently as COW
10240 * operations happen. So, swapfile + btrfs == corruption.
10241 *
10242 * For now we're avoiding this by dropping bmap.
10243 */
10252 #ifdef CONFIG_MIGRATION
10254 #endif
10259 };
10270 };
10279 };
10287 };
10291 };