| blob | 981a1fc30eaa2c291c97958ce088e3ce96ab4353 |
1 /*
2 * linux/fs/ext4/inode.c
3 *
4 * Copyright (C) 1992, 1993, 1994, 1995
5 * Remy Card (card@masi.ibp.fr)
6 * Laboratoire MASI - Institut Blaise Pascal
7 * Universite Pierre et Marie Curie (Paris VI)
8 *
9 * from
10 *
11 * linux/fs/minix/inode.c
12 *
13 * Copyright (C) 1991, 1992 Linus Torvalds
14 *
15 * 64-bit file support on 64-bit platforms by Jakub Jelinek
16 * (jj@sunsite.ms.mff.cuni.cz)
17 *
18 * Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
19 */
21 #include <linux/fs.h>
22 #include <linux/time.h>
23 #include <linux/highuid.h>
24 #include <linux/pagemap.h>
25 #include <linux/dax.h>
26 #include <linux/quotaops.h>
27 #include <linux/string.h>
28 #include <linux/buffer_head.h>
29 #include <linux/writeback.h>
30 #include <linux/pagevec.h>
31 #include <linux/mpage.h>
32 #include <linux/namei.h>
33 #include <linux/uio.h>
34 #include <linux/bio.h>
35 #include <linux/workqueue.h>
36 #include <linux/kernel.h>
37 #include <linux/printk.h>
38 #include <linux/slab.h>
39 #include <linux/bitops.h>
46 #include <trace/events/ext4.h>
48 #define MPAGE_DA_EXTENT_TAIL 0x01
52 {
54 __u16 csum_lo;
56 __u32 csum;
64 }
75 }
79 {
92 else
96 }
100 {
101 __u32 csum;
113 }
116 loff_t new_size)
117 {
119 /*
120 * If jinode is zero, then we never opened the file for
121 * writing, so there's no need to call
122 * jbd2_journal_begin_ordered_truncate() since there's no
123 * outstanding writes we need to flush.
124 */
129 new_size);
130 }
139 /*
140 * Test whether an inode is a fast symlink.
141 */
143 {
151 }
153 /*
154 * Restart the transaction associated with *handle. This does a commit,
155 * so before we call here everything must be consistently dirtied against
156 * this transaction.
157 */
160 {
163 /*
164 * Drop i_data_sem to avoid deadlock with ext4_map_blocks. At this
165 * moment, get_block can be called only for blocks inside i_size since
166 * page cache has been already dropped and writes are blocked by
167 * i_mutex. So we can safely drop the i_data_sem here.
168 */
177 }
179 /*
180 * Called at the last iput() if i_nlink is zero.
181 */
183 {
190 /*
191 * When journalling data dirty buffers are tracked only in the
192 * journal. So although mm thinks everything is clean and
193 * ready for reaping the inode might still have some pages to
194 * write in the running transaction or waiting to be
195 * checkpointed. Thus calling jbd2_journal_invalidatepage()
196 * (via truncate_inode_pages()) to discard these buffers can
197 * cause data loss. Also even if we did not discard these
198 * buffers, we would have no way to find them after the inode
199 * is reaped and thus user could see stale data if he tries to
200 * read them before the transaction is checkpointed. So be
201 * careful and force everything to disk here... We use
202 * ei->i_datasync_tid to store the newest transaction
203 * containing inode's data.
204 *
205 * Note that directories do not have this problem because they
206 * don't use page cache.
207 */
216 }
220 }
230 /*
231 * Protect us against freezing - iput() caller didn't have to have any
232 * protection against it
233 */
239 /*
240 * If we're going to skip the normal cleanup, we still need to
241 * make sure that the in-core orphan linked list is properly
242 * cleaned up.
243 */
247 }
257 }
261 /*
262 * ext4_ext_truncate() doesn't reserve any slop when it
263 * restarts journal transactions; therefore there may not be
264 * enough credits left in the handle to remove the inode from
265 * the orphan list and set the dtime field.
266 */
274 stop_handle:
279 }
280 }
282 /*
283 * Kill off the orphan record which ext4_truncate created.
284 * AKPM: I think this can be inside the above `if'.
285 * Note that ext4_orphan_del() has to be able to cope with the
286 * deletion of a non-existent orphan - this is because we don't
287 * know if ext4_truncate() actually created an orphan record.
288 * (Well, we could do this if we need to, but heck - it works)
289 */
293 /*
294 * One subtle ordering requirement: if anything has gone wrong
295 * (transaction abort, IO errors, whatever), then we can still
296 * do these next steps (the fs will already have been marked as
297 * having errors), but we can't free the inode if the mark_dirty
298 * fails.
299 */
301 /* If that failed, just do the required in-core inode clear. */
303 else
308 no_delete:
310 }
312 #ifdef CONFIG_QUOTA
314 {
316 }
317 #endif
319 /*
320 * Called with i_data_sem down, which is important since we can call
321 * ext4_discard_preallocations() from here.
322 */
325 {
338 }
340 /* Update per-inode reservations */
346 /* Update quota subsystem for data blocks */
350 /*
351 * We did fallocate with an offset that is already delayed
352 * allocated. So on delayed allocated writeback we should
353 * not re-claim the quota for fallocated blocks.
354 */
356 }
358 /*
359 * If we have done all the pending block allocations and if
360 * there aren't any writers on the inode, we can discard the
361 * inode's preallocations.
362 */
366 }
371 {
375 "lblock %lu mapped to illegal pblock "
379 }
381 }
384 ext4_lblk_t len)
385 {
396 }
398 #define check_block_validity(inode, map) \
399 __check_block_validity((inode), __func__, __LINE__, (map))
401 #ifdef ES_AGGRESSIVE_TEST
407 {
411 /*
412 * There is a race window that the result is not the same.
413 * e.g. xfstests #223 when dioread_nolock enables. The reason
414 * is that we lookup a block mapping in extent status tree with
415 * out taking i_data_sem. So at the time the unwritten extent
416 * could be converted.
417 */
421 EXT4_GET_BLOCKS_KEEP_SIZE);
424 EXT4_GET_BLOCKS_KEEP_SIZE);
425 }
428 /*
429 * We don't check m_len because extent will be collpased in status
430 * tree. So the m_len might not equal.
431 */
436 "es_cached ex [%d/%d/%llu/%x] != "
442 }
443 }
446 /*
447 * The ext4_map_blocks() function tries to look up the requested blocks,
448 * and returns if the blocks are already mapped.
449 *
450 * Otherwise it takes the write lock of the i_data_sem and allocate blocks
451 * and store the allocated blocks in the result buffer head and mark it
452 * mapped.
453 *
454 * If file type is extents based, it will call ext4_ext_map_blocks(),
455 * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
456 * based files
457 *
458 * On success, it returns the number of blocks being mapped or allocated. if
459 * create==0 and the blocks are pre-allocated and unwritten, the resulting @map
460 * is marked as unwritten. If the create == 1, it will mark @map as mapped.
461 *
462 * It returns 0 if plain look up failed (blocks have not been allocated), in
463 * that case, @map is returned as unmapped but we still do fill map->m_len to
464 * indicate the length of a hole starting at map->m_lblk.
465 *
466 * It returns the error in case of allocation failure.
467 */
470 {
474 #ifdef ES_AGGRESSIVE_TEST
478 #endif
485 /*
486 * ext4_map_blocks returns an int, and m_len is an unsigned int
487 */
491 /* We can handle the block number less than EXT_MAX_BLOCKS */
495 /* Lookup extent status tree firstly */
515 }
516 #ifdef ES_AGGRESSIVE_TEST
519 #endif
521 }
523 /*
524 * Try to see if we can get the block without requesting a new
525 * file system block.
526 */
530 EXT4_GET_BLOCKS_KEEP_SIZE);
533 EXT4_GET_BLOCKS_KEEP_SIZE);
534 }
540 "ES len assertion failed for inode "
544 }
557 }
560 found:
565 }
567 /* If it is only a block(s) look up */
571 /*
572 * Returns if the blocks have already allocated
573 *
574 * Note that if blocks have been preallocated
575 * ext4_ext_get_block() returns the create = 0
576 * with buffer head unmapped.
577 */
579 /*
580 * If we need to convert extent to unwritten
581 * we continue and do the actual work in
582 * ext4_ext_map_blocks()
583 */
587 /*
588 * Here we clear m_flags because after allocating an new extent,
589 * it will be set again.
590 */
593 /*
594 * New blocks allocate and/or writing to unwritten extent
595 * will possibly result in updating i_data, so we take
596 * the write lock of i_data_sem, and call get_block()
597 * with create == 1 flag.
598 */
601 /*
602 * We need to check for EXT4 here because migrate
603 * could have changed the inode type in between
604 */
611 /*
612 * We allocated new blocks which will result in
613 * i_data's format changing. Force the migrate
614 * to fail by clearing migrate flags
615 */
617 }
619 /*
620 * Update reserved blocks/metadata blocks after successful
621 * block allocation which had been deferred till now. We don't
622 * support fallocate for non extent files. So we can update
623 * reserve space here.
624 */
628 }
635 "ES len assertion failed for inode "
639 }
641 /*
642 * We have to zeroout blocks before inserting them into extent
643 * status tree. Otherwise someone could look them up there and
644 * use them before they are really zeroed.
645 */
654 }
655 }
657 /*
658 * If the extent has been zeroed out, we don't need to update
659 * extent status tree.
660 */
665 }
678 }
679 }
681 out_sem:
687 }
689 }
691 /*
692 * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
693 * we have to be careful as someone else may be manipulating b_state as well.
694 */
696 {
702 /* Dummy buffer_head? Set non-atomically. */
706 }
707 /*
708 * Someone else may be modifying b_state. Be careful! This is ugly but
709 * once we get rid of using bh as a container for mapping information
710 * to pass to / from get_block functions, this can go away.
711 */
717 }
721 {
732 flags);
738 }
740 }
744 {
747 }
749 /*
750 * Get block function used when preparing for buffered write if we require
751 * creating an unwritten extent if blocks haven't been allocated. The extent
752 * will be converted to written after the IO is complete.
753 */
756 {
760 EXT4_GET_BLOCKS_IO_CREATE_EXT);
761 }
763 /* Maximum number of blocks we map for direct IO at once. */
764 #define DIO_MAX_BLOCKS 4096
766 /*
767 * Get blocks function for the cases that need to start a transaction -
768 * generally difference cases of direct IO and DAX IO. It also handles retries
769 * in case of ENOSPC.
770 */
773 {
779 /* Trim mapping request to maximum we can map at once for DIO */
784 retry:
795 }
797 /* Get block function for DIO reads and writes to inodes without extents */
800 {
801 /* We don't expect handle for direct IO */
807 }
809 /*
810 * Get block function for AIO DIO writes when we create unwritten extent if
811 * blocks are not allocated yet. The extent will be converted to written
812 * after IO is complete.
813 */
816 {
819 /* We don't expect handle for direct IO */
823 EXT4_GET_BLOCKS_IO_CREATE_EXT);
825 /*
826 * When doing DIO using unwritten extents, we need io_end to convert
827 * unwritten extents to written on IO completion. We allocate io_end
828 * once we spot unwritten extent and store it in b_private. Generic
829 * DIO code keeps b_private set and furthermore passes the value to
830 * our completion callback in 'private' argument.
831 */
841 }
843 }
846 }
848 /*
849 * Get block function for non-AIO DIO writes when we create unwritten extent if
850 * blocks are not allocated yet. The extent will be converted to written
851 * after IO is complete from ext4_ext_direct_IO() function.
852 */
855 {
858 /* We don't expect handle for direct IO */
862 EXT4_GET_BLOCKS_IO_CREATE_EXT);
864 /*
865 * Mark inode as having pending DIO writes to unwritten extents.
866 * ext4_ext_direct_IO() checks this flag and converts extents to
867 * written.
868 */
873 }
877 {
882 /* We don't expect handle for direct IO */
886 /*
887 * Blocks should have been preallocated! ext4_file_write_iter() checks
888 * that.
889 */
893 }
896 /*
897 * `handle' can be NULL if create is zero
898 */
901 {
925 /*
926 * Now that we do not always journal data, we should
927 * keep in mind whether this should always journal the
928 * new buffer as metadata. For now, regular file
929 * writes use ext4_get_block instead, so it's not a
930 * problem.
931 */
938 }
942 }
951 errout:
954 }
958 {
972 }
981 {
997 }
1001 }
1003 }
1005 /*
1006 * To preserve ordering, it is essential that the hole instantiation and
1007 * the data write be encapsulated in a single transaction. We cannot
1008 * close off a transaction and start a new one between the ext4_get_block()
1009 * and the commit_write(). So doing the jbd2_journal_start at the start of
1010 * prepare_write() is the right place.
1011 *
1012 * Also, this function can nest inside ext4_writepage(). In that case, we
1013 * *know* that ext4_writepage() has generated enough buffer credits to do the
1014 * whole page. So we won't block on the journal in that case, which is good,
1015 * because the caller may be PF_MEMALLOC.
1016 *
1017 * By accident, ext4 can be reentered when a transaction is open via
1018 * quota file writes. If we were to commit the transaction while thus
1019 * reentered, there can be a deadlock - we would be holding a quota
1020 * lock, and the commit would never complete if another thread had a
1021 * transaction open and was blocking on the quota lock - a ranking
1022 * violation.
1023 *
1024 * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
1025 * will _not_ run commit under these circumstances because handle->h_ref
1026 * is elevated. We'll still have enough credits for the tiny quotafile
1027 * write.
1028 */
1031 {
1037 /*
1038 * __block_write_begin() could have dirtied some buffers. Clean
1039 * the dirty bit as jbd2_journal_get_write_access() could complain
1040 * otherwise about fs integrity issues. Setting of the dirty bit
1041 * by __block_write_begin() isn't a real problem here as we clear
1042 * the bit before releasing a page lock and thus writeback cannot
1043 * ever write the buffer.
1044 */
1052 }
1054 #ifdef CONFIG_EXT4_FS_ENCRYPTION
1057 {
1062 sector_t block;
1087 }
1089 }
1105 }
1110 }
1111 }
1116 }
1124 }
1125 }
1126 /*
1127 * If we issued read requests, let them complete.
1128 */
1133 }
1139 }
1140 #endif
1145 {
1151 pgoff_t index;
1155 /*
1156 * Reserve one block more for addition to orphan list in case
1157 * we allocate blocks but write fails for some reason
1158 */
1171 }
1173 /*
1174 * grab_cache_page_write_begin() can take a long time if the
1175 * system is thrashing due to memory pressure, or if the page
1176 * is being written back. So grab it first before we start
1177 * the transaction handle. This also allows us to allocate
1178 * the page (if needed) without using GFP_NOFS.
1179 */
1180 retry_grab:
1186 retry_journal:
1191 }
1195 /* The page got truncated from under us */
1200 }
1201 /* In case writeback began while the page was unlocked */
1204 #ifdef CONFIG_EXT4_FS_ENCRYPTION
1207 ext4_get_block_unwritten);
1208 else
1210 ext4_get_block);
1211 #else
1214 ext4_get_block_unwritten);
1215 else
1217 #endif
1221 do_journal_get_write_access);
1222 }
1226 /*
1227 * __block_write_begin may have instantiated a few blocks
1228 * outside i_size. Trim these off again. Don't need
1229 * i_size_read because we hold i_mutex.
1230 *
1231 * Add inode to orphan list in case we crash before
1232 * truncate finishes
1233 */
1240 /*
1241 * If truncate failed early the inode might
1242 * still be on the orphan list; we need to
1243 * make sure the inode is removed from the
1244 * orphan list in that case.
1245 */
1248 }
1255 }
1258 }
1260 /* For write_end() in data=journal mode */
1262 {
1271 }
1273 /*
1274 * We need to pick up the new inode size which generic_commit_write gave us
1275 * `file' can be NULL - eg, when called from page_symlink().
1276 *
1277 * ext4 never places buffers on inode->i_mapping->private_list. metadata
1278 * buffers are managed internally.
1279 */
1284 {
1298 }
1299 }
1310 /*
1311 * it's important to update i_size while still holding page lock:
1312 * page writeout could otherwise come in and zero beyond i_size.
1313 */
1320 /*
1321 * Don't mark the inode dirty under page lock. First, it unnecessarily
1322 * makes the holding time of page lock longer. Second, it forces lock
1323 * ordering of page lock and transaction start for journaling
1324 * filesystems.
1325 */
1330 /* if we have allocated more blocks and copied
1331 * less. We will have blocks allocated outside
1332 * inode->i_size. So truncate them
1333 */
1335 errout:
1342 /*
1343 * If truncate failed early the inode might still be
1344 * on the orphan list; we need to make sure the inode
1345 * is removed from the orphan list in that case.
1346 */
1349 }
1352 }
1354 /*
1355 * This is a private version of page_zero_new_buffers() which doesn't
1356 * set the buffer to be dirty, since in data=journalled mode we need
1357 * to call ext4_handle_dirty_metadata() instead.
1358 */
1360 {
1377 }
1379 }
1380 }
1384 }
1390 {
1413 }
1419 }
1433 }
1436 /* if we have allocated more blocks and copied
1437 * less. We will have blocks allocated outside
1438 * inode->i_size. So truncate them
1439 */
1447 /*
1448 * If truncate failed early the inode might still be
1449 * on the orphan list; we need to make sure the inode
1450 * is removed from the orphan list in that case.
1451 */
1454 }
1457 }
1459 /*
1460 * Reserve space for a single cluster
1461 */
1463 {
1468 /*
1469 * We will charge metadata quota at writeout time; this saves
1470 * us from metadata over-estimation, though we may go over by
1471 * a small amount in the end. Here we just reserve for data.
1472 */
1482 }
1488 }
1491 {
1502 /*
1503 * if there aren't enough reserved blocks, then the
1504 * counter is messed up somewhere. Since this
1505 * function is called from invalidate page, it's
1506 * harmless to return without any action.
1507 */
1509 "ino %lu, to_free %d with only %d reserved "
1514 }
1517 /* update fs dirty data blocks counter */
1523 }
1528 {
1536 ext4_fsblk_t lblk;
1549 to_release++;
1550 contiguous_blks++;
1556 contiguous_blks;
1559 }
1567 }
1569 /* If we have released all the blocks belonging to a cluster, then we
1570 * need to release the reserved space for that cluster. */
1579 num_clusters--;
1580 }
1581 }
1583 /*
1584 * Delayed allocation stuff
1585 */
1594 /*
1595 * Extent to map - this can be after first_page because that can be
1596 * fully mapped. We somewhat abuse m_flags to store whether the extent
1597 * is delalloc or unwritten.
1598 */
1601 };
1605 {
1612 /* This is necessary when next_page == 0. */
1623 }
1639 }
1641 }
1644 }
1645 }
1648 {
1667 }
1670 {
1672 }
1674 /*
1675 * This function is grabs code from the very beginning of
1676 * ext4_map_blocks, but assumes that the caller is from delayed write
1677 * time. This function looks up the requested blocks and sets the
1678 * buffer delay bit under the protection of i_data_sem.
1679 */
1683 {
1687 #ifdef ES_AGGRESSIVE_TEST
1691 #endif
1701 /* Lookup extent status tree firstly */
1707 }
1709 /*
1710 * Delayed extent could be allocated by fallocate.
1711 * So we need to check it.
1712 */
1718 }
1729 else
1732 #ifdef ES_AGGRESSIVE_TEST
1734 #endif
1736 }
1738 /*
1739 * Try to see if we can get the block without requesting a new
1740 * file system block.
1741 */
1747 else
1750 add_delayed:
1753 /*
1754 * XXX: __block_prepare_write() unmaps passed block,
1755 * is it OK?
1756 */
1757 /*
1758 * If the block was allocated from previously allocated cluster,
1759 * then we don't need to reserve it again. However we still need
1760 * to reserve metadata for every block we're going to write.
1761 */
1766 /* not enough space to reserve */
1769 }
1770 }
1777 }
1788 "ES len assertion failed for inode "
1792 }
1800 }
1802 out_unlock:
1806 }
1808 /*
1809 * This is a special get_block_t callback which is used by
1810 * ext4_da_write_begin(). It will either return mapped block or
1811 * reserve space for a single block.
1812 *
1813 * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1814 * We also have b_blocknr = -1 and b_bdev initialized properly
1815 *
1816 * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1817 * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1818 * initialized properly.
1819 */
1822 {
1832 /*
1833 * first, we need to know whether the block is allocated already
1834 * preallocated blocks are unmapped but should treated
1835 * the same as allocated blocks.
1836 */
1845 /* A delayed write to unwritten bh should be marked
1846 * new and mapped. Mapped ensures that we don't do
1847 * get_block multiple times when we write to the same
1848 * offset and new ensures that we do proper zero out
1849 * for partial write.
1850 */
1853 }
1855 }
1858 {
1861 }
1864 {
1867 }
1871 {
1893 }
1896 }
1897 /*
1898 * We need to release the page lock before we start the
1899 * journal, so grab a reference so the page won't disappear
1900 * out from under us.
1901 */
1911 }
1917 /* The page got truncated from under us */
1921 }
1931 do_journal_get_write_access);
1934 write_end_fn);
1935 }
1947 out:
1949 out_no_pagelock:
1952 }
1954 /*
1955 * Note that we don't need to start a transaction unless we're journaling data
1956 * because we should have holes filled from ext4_page_mkwrite(). We even don't
1957 * need to file the inode to the transaction's list in ordered mode because if
1958 * we are writing back data added by write(), the inode is already there and if
1959 * we are writing back data modified via mmap(), no one guarantees in which
1960 * transaction the data will hit the disk. In case we are journaling data, we
1961 * cannot start transaction directly because transaction start ranks above page
1962 * lock so we have to do some magic.
1963 *
1964 * This function can get called via...
1965 * - ext4_writepages after taking page lock (have journal handle)
1966 * - journal_submit_inode_data_buffers (no journal handle)
1967 * - shrink_page_list via the kswapd/direct reclaim (no journal handle)
1968 * - grab_page_cache when doing write_begin (have journal handle)
1969 *
1970 * We don't do any block allocation in this function. If we have page with
1971 * multiple blocks we need to write those buffer_heads that are mapped. This
1972 * is important for mmaped based write. So if we do with blocksize 1K
1973 * truncate(f, 1024);
1974 * a = mmap(f, 0, 4096);
1975 * a[0] = 'a';
1976 * truncate(f, 4096);
1977 * we have in the page first buffer_head mapped via page_mkwrite call back
1978 * but other buffer_heads would be unmapped but dirty (dirty done via the
1979 * do_wp_page). So writepage should write the first block. If we modify
1980 * the mmap area beyond 1024 we will again get a page_fault and the
1981 * page_mkwrite callback will do the block allocation and mark the
1982 * buffer_heads mapped.
1983 *
1984 * We redirty the page if we have any buffer_heads that is either delay or
1985 * unwritten in the page.
1986 *
1987 * We can get recursively called as show below.
1988 *
1989 * ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
1990 * ext4_writepage()
1991 *
1992 * But since we don't do any block allocation we should not deadlock.
1993 * Page also have the dirty flag cleared so we don't get recurive page_lock.
1994 */
1997 {
1999 loff_t size;
2010 else
2014 /*
2015 * We cannot do block allocation or other extent handling in this
2016 * function. If there are buffers needing that, we have to redirty
2017 * the page. But we may reach here when we do a journal commit via
2018 * journal_submit_inode_data_buffers() and in that case we must write
2019 * allocated buffers to achieve data=ordered mode guarantees.
2020 *
2021 * Also, if there is only one buffer per page (the fs block
2022 * size == the page size), if one buffer needs block
2023 * allocation or needs to modify the extent tree to clear the
2024 * unwritten flag, we know that the page can't be written at
2025 * all, so we might as well refuse the write immediately.
2026 * Unfortunately if the block size != page size, we can't as
2027 * easily detect this case using ext4_walk_page_buffers(), but
2028 * for the extremely common case, this is an optimization that
2029 * skips a useless round trip through ext4_bio_write_page().
2030 */
2032 ext4_bh_delay_or_unwritten)) {
2036 /*
2037 * For memory cleaning there's no point in writing only
2038 * some buffers. So just bail out. Warn if we came here
2039 * from direct reclaim.
2040 */
2045 }
2047 }
2050 /*
2051 * It's mmapped pagecache. Add buffers and journal it. There
2052 * doesn't seem much point in redirtying the page here.
2053 */
2062 }
2065 /* Drop io_end reference we got from init */
2068 }
2071 {
2079 else
2088 }
2090 #define BH_FLAGS ((1 << BH_Unwritten) | (1 << BH_Delay))
2092 /*
2093 * mballoc gives us at most this number of blocks...
2094 * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
2095 * The rest of mballoc seems to handle chunks up to full group size.
2096 */
2097 #define MAX_WRITEPAGES_EXTENT_LEN 2048
2099 /*
2100 * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2101 *
2102 * @mpd - extent of blocks
2103 * @lblk - logical number of the block in the file
2104 * @bh - buffer head we want to add to the extent
2105 *
2106 * The function is used to collect contig. blocks in the same state. If the
2107 * buffer doesn't require mapping for writeback and we haven't started the
2108 * extent of buffers to map yet, the function returns 'true' immediately - the
2109 * caller can write the buffer right away. Otherwise the function returns true
2110 * if the block has been added to the extent, false if the block couldn't be
2111 * added.
2112 */
2115 {
2118 /* Buffer that doesn't need mapping for writeback? */
2121 /* So far no extent to map => we write the buffer right away */
2125 }
2127 /* First block in the extent? */
2133 }
2135 /* Don't go larger than mballoc is willing to allocate */
2139 /* Can we merge the block to our big extent? */
2144 }
2146 }
2148 /*
2149 * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2150 *
2151 * @mpd - extent of blocks for mapping
2152 * @head - the first buffer in the page
2153 * @bh - buffer we should start processing from
2154 * @lblk - logical number of the block in the file corresponding to @bh
2155 *
2156 * Walk through page buffers from @bh upto @head (exclusive) and either submit
2157 * the page for IO if all buffers in this page were mapped and there's no
2158 * accumulated extent of buffers to map or add buffers in the page to the
2159 * extent of buffers to map. The function returns 1 if the caller can continue
2160 * by processing the next page, 0 if it should stop adding buffers to the
2161 * extent to map because we cannot extend it anymore. It can also return value
2162 * < 0 in case of error during IO submission.
2163 */
2167 ext4_lblk_t lblk)
2168 {
2178 /* Found extent to map? */
2181 /* Everything mapped so far and we hit EOF */
2183 }
2185 /* So far everything mapped? Submit the page for IO. */
2190 }
2192 }
2194 /*
2195 * mpage_map_buffers - update buffers corresponding to changed extent and
2196 * submit fully mapped pages for IO
2197 *
2198 * @mpd - description of extent to map, on return next extent to map
2199 *
2200 * Scan buffers corresponding to changed extent (we expect corresponding pages
2201 * to be already locked) and update buffer state according to new extent state.
2202 * We map delalloc buffers to their physical location, clear unwritten bits,
2203 * and mark buffers as uninit when we perform writes to unwritten extents
2204 * and do extent conversion after IO is finished. If the last page is not fully
2205 * mapped, we update @map to the next extent in the last page that needs
2206 * mapping. Otherwise we submit the page for IO.
2207 */
2209 {
2216 ext4_lblk_t lblk;
2217 sector_t pblock;
2228 PAGEVEC_SIZE);
2236 /* Up to 'end' pages must be contiguous */
2243 /*
2244 * Buffer after end of mapped extent.
2245 * Find next buffer in the page to map.
2246 */
2249 /*
2250 * FIXME: If dioread_nolock supports
2251 * blocksize < pagesize, we need to make
2252 * sure we add size mapped so far to
2253 * io_end->size as the following call
2254 * can submit the page for IO.
2255 */
2262 }
2266 }
2270 /*
2271 * FIXME: This is going to break if dioread_nolock
2272 * supports blocksize < pagesize as we will try to
2273 * convert potentially unmapped parts of inode.
2274 */
2276 /* Page fully mapped - let IO run! */
2281 }
2282 start++;
2283 }
2285 }
2286 /* Extent fully mapped and matches with page boundary. We are done. */
2290 }
2293 {
2300 /*
2301 * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
2302 * to convert an unwritten extent to be initialized (in the case
2303 * where we have written into one or more preallocated blocks). It is
2304 * possible that we're going to need more metadata blocks than
2305 * previously reserved. However we must not fail because we're in
2306 * writeback and there is nothing we can do about it so it might result
2307 * in data loss. So use reserved blocks to allocate metadata if
2308 * possible.
2309 *
2310 * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
2311 * the blocks in question are delalloc blocks. This indicates
2312 * that the blocks and quotas has already been checked when
2313 * the data was copied into the page cache.
2314 */
2316 EXT4_GET_BLOCKS_METADATA_NOFAIL;
2331 }
2333 }
2342 }
2344 }
2346 /*
2347 * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2348 * mpd->len and submit pages underlying it for IO
2349 *
2350 * @handle - handle for journal operations
2351 * @mpd - extent to map
2352 * @give_up_on_write - we set this to true iff there is a fatal error and there
2353 * is no hope of writing the data. The caller should discard
2354 * dirty pages to avoid infinite loops.
2355 *
2356 * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2357 * delayed, blocks are allocated, if it is unwritten, we may need to convert
2358 * them to initialized or split the described range from larger unwritten
2359 * extent. Note that we need not map all the described range since allocation
2360 * can return less blocks or the range is covered by more unwritten extents. We
2361 * cannot map more because we are limited by reserved transaction credits. On
2362 * the other hand we always make sure that the last touched page is fully
2363 * mapped so that it can be written out (and thus forward progress is
2364 * guaranteed). After mapping we submit all mapped pages for IO.
2365 */
2369 {
2373 loff_t disksize;
2385 /*
2386 * Let the uper layers retry transient errors.
2387 * In the case of ENOSPC, if ext4_count_free_blocks()
2388 * is non-zero, a commit should free up blocks.
2389 */
2395 }
2397 "Delayed block allocation failed for "
2398 "inode %lu at logical offset %llu with"
2404 "This should not happen!! Data will "
2408 invalidate_dirty_pages:
2411 }
2413 /*
2414 * Update buffer state, submit mapped pages, and get us new
2415 * extent to map
2416 */
2422 update_disksize:
2423 /*
2424 * Update on-disk size after IO is submitted. Races with
2425 * truncate are avoided by checking i_size under i_data_sem.
2426 */
2430 loff_t i_size;
2446 }
2448 }
2450 /*
2451 * Calculate the total number of credits to reserve for one writepages
2452 * iteration. This is called from ext4_writepages(). We map an extent of
2453 * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
2454 * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
2455 * bpp - 1 blocks in bpp different extents.
2456 */
2458 {
2463 }
2465 /*
2466 * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2467 * and underlying extent to map
2468 *
2469 * @mpd - where to look for pages
2470 *
2471 * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2472 * IO immediately. When we find a page which isn't mapped we start accumulating
2473 * extent of buffers underlying these pages that needs mapping (formed by
2474 * either delayed or unwritten buffers). We also lock the pages containing
2475 * these buffers. The extent found is returned in @mpd structure (starting at
2476 * mpd->lblk with length mpd->len blocks).
2477 *
2478 * Note that this function can attach bios to one io_end structure which are
2479 * neither logically nor physically contiguous. Although it may seem as an
2480 * unnecessary complication, it is actually inevitable in blocksize < pagesize
2481 * case as we need to track IO to all buffers underlying a page in one io_end.
2482 */
2484 {
2494 ext4_lblk_t lblk;
2499 else
2514 /*
2515 * At this point, the page may be truncated or
2516 * invalidated (changing page->mapping to NULL), or
2517 * even swizzled back from swapper_space to tmpfs file
2518 * mapping. However, page->index will not change
2519 * because we have a reference on the page.
2520 */
2524 /*
2525 * Accumulated enough dirty pages? This doesn't apply
2526 * to WB_SYNC_ALL mode. For integrity sync we have to
2527 * keep going because someone may be concurrently
2528 * dirtying pages, and we might have synced a lot of
2529 * newly appeared dirty pages, but have not synced all
2530 * of the old dirty pages.
2531 */
2535 /* If we can't merge this page, we are done. */
2540 /*
2541 * If the page is no longer dirty, or its mapping no
2542 * longer corresponds to inode we are writing (which
2543 * means it has been truncated or invalidated), or the
2544 * page is already under writeback and we are not doing
2545 * a data integrity writeback, skip the page
2546 */
2553 }
2561 /* Add all dirty buffers to mpd */
2569 left--;
2570 }
2573 }
2575 out:
2578 }
2582 {
2587 }
2591 {
2609 wbc);
2611 /*
2612 * No pages to write? This is mainly a kludge to avoid starting
2613 * a transaction for special inodes like journal inode on last iput()
2614 * because that could violate lock ordering on umount
2615 */
2626 }
2628 /*
2629 * If the filesystem has aborted, it is read-only, so return
2630 * right away instead of dumping stack traces later on that
2631 * will obscure the real source of the problem. We test
2632 * EXT4_MF_FS_ABORTED instead of sb->s_flag's MS_RDONLY because
2633 * the latter could be true if the filesystem is mounted
2634 * read-only, and in that case, ext4_writepages should
2635 * *never* be called, so if that ever happens, we would want
2636 * the stack trace.
2637 */
2641 }
2644 /*
2645 * We may need to convert up to one extent per block in
2646 * the page and we may dirty the inode.
2647 */
2649 }
2651 /*
2652 * If we have inline data and arrive here, it means that
2653 * we will soon create the block for the 1st page, so
2654 * we'd better clear the inline data here.
2655 */
2657 /* Just inode will be modified... */
2662 }
2664 EXT4_STATE_MAY_INLINE_DATA));
2667 }
2681 }
2686 retry:
2692 /* For each extent of pages we use new io_end */
2697 }
2699 /*
2700 * We have two constraints: We find one extent to map and we
2701 * must always write out whole page (makes a difference when
2702 * blocksize < pagesize) so that we don't block on IO when we
2703 * try to write out the rest of the page. Journalled mode is
2704 * not supported by delalloc.
2705 */
2709 /* start a new transaction */
2717 /* Release allocated io_end */
2720 }
2729 /*
2730 * We scanned the whole range (or exhausted
2731 * nr_to_write), submitted what was mapped and
2732 * didn't find anything needing mapping. We are
2733 * done.
2734 */
2736 }
2737 }
2739 /* Submit prepared bio */
2741 /* Unlock pages we didn't use */
2743 /* Drop our io_end reference we got from init */
2747 /*
2748 * Commit the transaction which would
2749 * free blocks released in the transaction
2750 * and try again
2751 */
2755 }
2756 /* Fatal error - ENOMEM, EIO... */
2759 }
2766 }
2768 /* Update index */
2770 /*
2771 * Set the writeback_index so that range_cyclic
2772 * mode will write it back later
2773 */
2776 out_writepages:
2780 }
2783 {
2787 /*
2788 * switch to non delalloc mode if we are running low
2789 * on free block. The free block accounting via percpu
2790 * counters can get slightly wrong with percpu_counter_batch getting
2791 * accumulated on each CPU without updating global counters
2792 * Delalloc need an accurate free block accounting. So switch
2793 * to non delalloc when we are near to error range.
2794 */
2795 free_clusters =
2797 dirty_clusters =
2799 /*
2800 * Start pushing delalloc when 1/2 of free blocks are dirty.
2801 */
2807 /*
2808 * free block count is less than 150% of dirty blocks
2809 * or free blocks is less than watermark
2810 */
2812 }
2814 }
2816 /* We always reserve for an inode update; the superblock could be there too */
2818 {
2825 /* We might need to update the superblock to set LARGE_FILE */
2827 }
2832 {
2835 pgoff_t index;
2845 }
2857 }
2859 /*
2860 * grab_cache_page_write_begin() can take a long time if the
2861 * system is thrashing due to memory pressure, or if the page
2862 * is being written back. So grab it first before we start
2863 * the transaction handle. This also allows us to allocate
2864 * the page (if needed) without using GFP_NOFS.
2865 */
2866 retry_grab:
2872 /*
2873 * With delayed allocation, we don't log the i_disksize update
2874 * if there is delayed block allocation. But we still need
2875 * to journalling the i_disksize update if writes to the end
2876 * of file which has an already mapped buffer.
2877 */
2878 retry_journal:
2884 }
2888 /* The page got truncated from under us */
2893 }
2894 /* In case writeback began while the page was unlocked */
2897 #ifdef CONFIG_EXT4_FS_ENCRYPTION
2899 ext4_da_get_block_prep);
2900 #else
2902 #endif
2906 /*
2907 * block_write_begin may have instantiated a few blocks
2908 * outside i_size. Trim these off again. Don't need
2909 * i_size_read because we hold i_mutex.
2910 */
2920 }
2924 }
2926 /*
2927 * Check if we should update i_disksize
2928 * when write to the end of file but not require block allocation
2929 */
2932 {
2947 }
2953 {
2957 loff_t new_i_size;
2969 /*
2970 * generic_write_end() will run mark_inode_dirty() if i_size
2971 * changes. So let's piggyback the i_disksize mark_inode_dirty
2972 * into that.
2973 */
2979 /* We need to mark inode dirty even if
2980 * new_i_size is less that inode->i_size
2981 * bu greater than i_disksize.(hint delalloc)
2982 */
2984 }
2985 }
2991 page);
2992 else
3004 }
3008 {
3009 /*
3010 * Drop reserved blocks
3011 */
3018 out:
3022 }
3024 /*
3025 * Force all delayed allocation blocks to be allocated for a given inode.
3026 */
3028 {
3034 /*
3035 * We do something simple for now. The filemap_flush() will
3036 * also start triggering a write of the data blocks, which is
3037 * not strictly speaking necessary (and for users of
3038 * laptop_mode, not even desirable). However, to do otherwise
3039 * would require replicating code paths in:
3040 *
3041 * ext4_writepages() ->
3042 * write_cache_pages() ---> (via passed in callback function)
3043 * __mpage_da_writepage() -->
3044 * mpage_add_bh_to_extent()
3045 * mpage_da_map_blocks()
3046 *
3047 * The problem is that write_cache_pages(), located in
3048 * mm/page-writeback.c, marks pages clean in preparation for
3049 * doing I/O, which is not desirable if we're not planning on
3050 * doing I/O at all.
3051 *
3052 * We could call write_cache_pages(), and then redirty all of
3053 * the pages by calling redirty_page_for_writepage() but that
3054 * would be ugly in the extreme. So instead we would need to
3055 * replicate parts of the code in the above functions,
3056 * simplifying them because we wouldn't actually intend to
3057 * write out the pages, but rather only collect contiguous
3058 * logical block extents, call the multi-block allocator, and
3059 * then update the buffer heads with the block allocations.
3060 *
3061 * For now, though, we'll cheat by calling filemap_flush(),
3062 * which will map the blocks, and start the I/O, but not
3063 * actually wait for the I/O to complete.
3064 */
3066 }
3068 /*
3069 * bmap() is special. It gets used by applications such as lilo and by
3070 * the swapper to find the on-disk block of a specific piece of data.
3071 *
3072 * Naturally, this is dangerous if the block concerned is still in the
3073 * journal. If somebody makes a swapfile on an ext4 data-journaling
3074 * filesystem and enables swap, then they may get a nasty shock when the
3075 * data getting swapped to that swapfile suddenly gets overwritten by
3076 * the original zero's written out previously to the journal and
3077 * awaiting writeback in the kernel's buffer cache.
3078 *
3079 * So, if we see any bmap calls here on a modified, data-journaled file,
3080 * take extra steps to flush any blocks which might be in the cache.
3081 */
3083 {
3088 /*
3089 * We can get here for an inline file via the FIBMAP ioctl
3090 */
3096 /*
3097 * With delalloc we want to sync the file
3098 * so that we can make sure we allocate
3099 * blocks for file
3100 */
3102 }
3106 /*
3107 * This is a REALLY heavyweight approach, but the use of
3108 * bmap on dirty files is expected to be extremely rare:
3109 * only if we run lilo or swapon on a freshly made file
3110 * do we expect this to happen.
3111 *
3112 * (bmap requires CAP_SYS_RAWIO so this does not
3113 * represent an unprivileged user DOS attack --- we'd be
3114 * in trouble if mortal users could trigger this path at
3115 * will.)
3116 *
3117 * NB. EXT4_STATE_JDATA is not set on files other than
3118 * regular files. If somebody wants to bmap a directory
3119 * or symlink and gets confused because the buffer
3120 * hasn't yet been flushed to disk, they deserve
3121 * everything they get.
3122 */
3132 }
3135 }
3138 {
3151 }
3153 static int
3156 {
3159 /* If the file has inline data, no need to do readpages. */
3164 }
3168 {
3171 /* No journalling happens on data buffers when this function is used */
3175 }
3180 {
3185 /*
3186 * If it's a full truncate we just forget about the pending dirtying
3187 */
3192 }
3194 /* Wrapper for aops... */
3198 {
3200 }
3203 {
3208 /* Page has dirty journalled data -> cannot release */
3213 else
3215 }
3217 #ifdef CONFIG_FS_DAX
3220 {
3238 }
3239 }
3246 }
3252 /*
3253 * We are protected by i_mmap_sem so we know block cannot go
3254 * away from under us even though we dropped i_data_sem.
3255 * Convert extent to written and write zeros there.
3256 *
3257 * Note: We may get here even when create == 0.
3258 */
3263 }
3272 }
3273 out:
3277 /*
3278 * At least for now we have to clear BH_New so that DAX code
3279 * doesn't attempt to zero blocks again in a racy way.
3280 */
3285 }
3287 }
3288 #endif
3292 {
3295 /* if not async direct IO just return */
3303 /*
3304 * Error during AIO DIO. We cannot convert unwritten extents as the
3305 * data was not written. Just clear the unwritten flag and drop io_end.
3306 */
3310 }
3316 }
3318 /*
3319 * For ext4 extent files, ext4 will do direct-io write to holes,
3320 * preallocated extents, and those write extend the file, no need to
3321 * fall back to buffered IO.
3322 *
3323 * For holes, we fallocate those blocks, mark them as unwritten
3324 * If those blocks were preallocated, we mark sure they are split, but
3325 * still keep the range to write as unwritten.
3326 *
3327 * The unwritten extents will be converted to written when DIO is completed.
3328 * For async direct IO, since the IO may still pending when return, we
3329 * set up an end_io call back function, which will do the conversion
3330 * when async direct IO completed.
3331 *
3332 * If the O_DIRECT write will extend the file then add this inode to the
3333 * orphan list. So recovery will truncate it back to the original size
3334 * if the machine crashes during the write.
3335 *
3336 */
3338 loff_t offset)
3339 {
3342 ssize_t ret;
3349 /* Use the old path for reads and writes beyond i_size. */
3355 /*
3356 * Make all waiters for direct IO properly wait also for extent
3357 * conversion. This also disallows race between truncate() and
3358 * overwrite DIO as i_dio_count needs to be incremented under i_mutex.
3359 */
3363 /* If we do a overwrite dio, i_mutex locking can be released */
3369 /*
3370 * We could direct write to holes and fallocate.
3371 *
3372 * Allocated blocks to fill the hole are marked as unwritten to prevent
3373 * parallel buffered read to expose the stale data before DIO complete
3374 * the data IO.
3375 *
3376 * As to previously fallocated extents, ext4 get_block will just simply
3377 * mark the buffer mapped but still keep the extents unwritten.
3378 *
3379 * For non AIO case, we will convert those unwritten extents to written
3380 * after return back from blockdev_direct_IO. That way we save us from
3381 * allocating io_end structure and also the overhead of offloading
3382 * the extent convertion to a workqueue.
3383 *
3384 * For async DIO, the conversion needs to be deferred when the
3385 * IO is completed. The ext4 end_io callback function will be
3386 * called to take care of the conversion work. Here for async
3387 * case, we allocate an io_end structure to hook to the iocb.
3388 */
3398 }
3399 #ifdef CONFIG_EXT4_FS_ENCRYPTION
3401 #endif
3405 else
3408 get_block_func,
3412 EXT4_STATE_DIO_UNWRITTEN)) {
3414 /*
3415 * for non AIO case, since the IO is already
3416 * completed, we could do the conversion right here
3417 */
3423 }
3427 /* take i_mutex locking again if we do a ovewrite dio */
3432 }
3435 loff_t offset)
3436 {
3440 ssize_t ret;
3442 #ifdef CONFIG_EXT4_FS_ENCRYPTION
3445 #endif
3447 /*
3448 * If we are doing data journalling we don't support O_DIRECT
3449 */
3453 /* Let buffer I/O handle the inline data case. */
3460 else
3464 }
3466 /*
3467 * Pages can be marked dirty completely asynchronously from ext4's journalling
3468 * activity. By filemap_sync_pte(), try_to_unmap_one(), etc. We cannot do
3469 * much here because ->set_page_dirty is called under VFS locks. The page is
3470 * not necessarily locked.
3471 *
3472 * We cannot just dirty the page and leave attached buffers clean, because the
3473 * buffers' dirty state is "definitive". We cannot just set the buffers dirty
3474 * or jbddirty because all the journalling code will explode.
3475 *
3476 * So what we do is to mark the page "pending dirty" and next time writepage
3477 * is called, propagate that into the buffers appropriately.
3478 */
3480 {
3483 }
3499 };
3515 };
3531 };
3534 {
3547 }
3550 else
3552 }
3556 {
3560 ext4_lblk_t iblock;
3578 /* Find the buffer that contains "offset" */
3583 iblock++;
3585 }
3589 }
3593 /* unmapped? It's a hole - nothing to do */
3597 }
3598 }
3600 /* Ok, it's mapped. Make sure it's up-to-date */
3608 /* Uhhuh. Read error. Complain and punt. */
3613 /* We expect the key to be set. */
3617 }
3618 }
3624 }
3635 }
3637 unlock:
3641 }
3643 /*
3644 * ext4_block_zero_page_range() zeros out a mapping of length 'length'
3645 * starting from file offset 'from'. The range to be zero'd must
3646 * be contained with in one block. If the specified range exceeds
3647 * the end of the block it will be shortened to end of the block
3648 * that cooresponds to 'from'
3649 */
3652 {
3658 /*
3659 * correct length if it does not fall between
3660 * 'from' and the end of the block
3661 */
3668 }
3670 /*
3671 * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
3672 * up to the end of the block which corresponds to `from'.
3673 * This required during truncate. We need to physically zero the tail end
3674 * of that block so it doesn't yield old data if the file is later grown.
3675 */
3678 {
3688 }
3692 {
3706 /* Handle partial zero within the single block */
3712 }
3713 /* Handle partial zero out on the start of the range */
3719 }
3720 /* Handle partial zero out on the end of the range */
3726 }
3729 {
3737 }
3739 /*
3740 * We have to make sure i_disksize gets properly updated before we truncate
3741 * page cache due to hole punching or zero range. Otherwise i_disksize update
3742 * can get lost as it may have been postponed to submission of writeback but
3743 * that will never happen after we truncate page cache.
3744 */
3746 loff_t len)
3747 {
3766 }
3768 /*
3769 * ext4_punch_hole: punches a hole in a file by releaseing the blocks
3770 * associated with the given offset and length
3771 *
3772 * @inode: File inode
3773 * @offset: The offset where the hole will begin
3774 * @len: The length of the hole
3775 *
3776 * Returns: 0 on success or negative on failure
3777 */
3780 {
3794 /*
3795 * Write out all dirty pages to avoid race conditions
3796 * Then release them.
3797 */
3803 }
3807 /* No need to punch hole beyond i_size */
3811 /*
3812 * If the hole extends beyond i_size, set the hole
3813 * to end after the page that contains i_size
3814 */
3818 offset;
3819 }
3823 /*
3824 * Attach jinode to inode for jbd2 if we do any zeroing of
3825 * partial block
3826 */
3831 }
3833 /* Wait all existing dio workers, newcomers will block on i_mutex */
3837 /*
3838 * Prevent page faults from reinstantiating pages we have released from
3839 * page cache.
3840 */
3845 /* Now release the pages and zero block aligned part of pages*/
3851 last_block_offset);
3852 }
3856 else
3863 }
3866 length);
3874 /* If there are no blocks to remove, return now */
3886 }
3891 else
3893 stop_block);
3901 out_stop:
3903 out_dio:
3906 out_mutex:
3909 }
3912 {
3925 }
3929 }
3934 }
3936 /*
3937 * ext4_truncate()
3938 *
3939 * We block out ext4_get_block() block instantiations across the entire
3940 * transaction, and VFS/VM ensures that ext4_truncate() cannot run
3941 * simultaneously on behalf of the same inode.
3942 *
3943 * As we work through the truncate and commit bits of it to the journal there
3944 * is one core, guiding principle: the file's tree must always be consistent on
3945 * disk. We must be able to restart the truncate after a crash.
3946 *
3947 * The file's tree may be transiently inconsistent in memory (although it
3948 * probably isn't), but whenever we close off and commit a journal transaction,
3949 * the contents of (the filesystem + the journal) must be consistent and
3950 * restartable. It's pretty simple, really: bottom up, right to left (although
3951 * left-to-right works OK too).
3952 *
3953 * Note that at recovery time, journal replay occurs *before* the restart of
3954 * truncate against the orphan inode list.
3955 *
3956 * The committed inode has the new, desired i_size (which is the same as
3957 * i_disksize in this case). After a crash, ext4_orphan_cleanup() will see
3958 * that this inode's truncate did not complete and it will again call
3959 * ext4_truncate() to have another go. So there will be instantiated blocks
3960 * to the right of the truncation point in a crashed ext4 filesystem. But
3961 * that's fine - as long as they are linked from the inode, the post-crash
3962 * ext4_truncate() run will find them and release them.
3963 */
3965 {
3971 /*
3972 * There is a possibility that we're either freeing the inode
3973 * or it's a completely new inode. In those cases we might not
3974 * have i_mutex locked because it's not necessary.
3975 */
3994 }
3996 /* If we zero-out tail of the page, we have to create jinode for jbd2 */
4000 }
4004 else
4011 }
4016 /*
4017 * We add the inode to the orphan list, so that if this
4018 * truncate spans multiple transactions, and we crash, we will
4019 * resume the truncate when the filesystem recovers. It also
4020 * marks the inode dirty, to catch the new size.
4021 *
4022 * Implication: the file must always be in a sane, consistent
4023 * truncatable state while each transaction commits.
4024 */
4034 else
4042 out_stop:
4043 /*
4044 * If this was a simple ftruncate() and the file will remain alive,
4045 * then we need to clear up the orphan record which we created above.
4046 * However, if this was a real unlink then we were called by
4047 * ext4_evict_inode(), and we allow that function to clean up the
4048 * orphan info for us.
4049 */
4058 }
4060 /*
4061 * ext4_get_inode_loc returns with an extra refcount against the inode's
4062 * underlying buffer_head on success. If 'in_mem' is true, we have all
4063 * data in memory that is needed to recreate the on-disk version of this
4064 * inode.
4065 */
4068 {
4072 ext4_fsblk_t block;
4084 /*
4085 * Figure out the offset within the block group inode table
4086 */
4099 /*
4100 * If the buffer has the write error flag, we have failed
4101 * to write out another inode in the same block. In this
4102 * case, we don't have to read the block because we may
4103 * read the old inode data successfully.
4104 */
4109 /* someone brought it uptodate while we waited */
4112 }
4114 /*
4115 * If we have all information of the inode in memory and this
4116 * is the only valid inode in the block, we need not read the
4117 * block.
4118 */
4125 /* Is the inode bitmap in cache? */
4130 /*
4131 * If the inode bitmap isn't in cache then the
4132 * optimisation may end up performing two reads instead
4133 * of one, so skip it.
4134 */
4138 }
4144 }
4147 /* all other inodes are free, so skip I/O */
4152 }
4153 }
4155 make_io:
4156 /*
4157 * If we need to do any I/O, try to pre-readahead extra
4158 * blocks from the inode table.
4159 */
4166 /* s_inode_readahead_blks is always a power of 2 */
4179 }
4181 /*
4182 * There are other valid inodes in the buffer, this inode
4183 * has in-inode xattrs, or we don't have this inode in memory.
4184 * Read the block from disk.
4185 */
4196 }
4197 }
4198 has_buffer:
4201 }
4204 {
4205 /* We have all inode data except xattrs in memory here. */
4208 }
4211 {
4229 }
4231 /* Propagate flags from i_flags to EXT4_I(inode)->i_flags */
4233 {
4242 EXT4_DIRSYNC_FL);
4254 }
4258 {
4259 blkcnt_t i_blocks ;
4264 /* we are using combined 48 bit field */
4268 /* i_blocks represent file system block size */
4272 }
4275 }
4276 }
4281 {
4289 }
4292 {
4297 }
4300 {
4308 uid_t i_uid;
4309 gid_t i_gid;
4310 projid_t i_projid;
4335 }
4339 /* Precompute checksum seed for inode metadata */
4342 __u32 csum;
4349 }
4355 }
4364 else
4370 }
4380 /* We now have enough fields to check if the inode was active or not.
4381 * This is needed because nfsd might try to access dead inodes
4382 * the test is that same one that e2fsck uses
4383 * NeilBrown 1999oct15
4384 */
4389 /* this inode is deleted */
4392 }
4393 /* The only unlinked inodes we let through here have
4394 * valid i_mode and are being read by the orphan
4395 * recovery code: that's fine, we're about to complete
4396 * the process of deleting those.
4397 * OR it is the EXT4_BOOT_LOADER_INO which is
4398 * not initialized on a new filesystem. */
4399 }
4408 #ifdef CONFIG_QUOTA
4410 #endif
4414 /*
4415 * NOTE! The in-memory inode i_data array is in little-endian order
4416 * even on big-endian machines: we do NOT byteswap the block numbers!
4417 */
4422 /*
4423 * Set transaction id's of transactions that have to be committed
4424 * to finish f[data]sync. We set them to currently running transaction
4425 * as we cannot be sure that the inode or some of its metadata isn't
4426 * part of the transaction - the inode could have been reclaimed and
4427 * now it is reread from disk.
4428 */
4431 tid_t tid;
4436 else
4440 else
4445 }
4449 /* The extra space is currently unused. Use it. */
4451 EXT4_GOOD_OLD_INODE_SIZE;
4454 }
4455 }
4468 }
4469 }
4483 /* Validate extent which is part of inode */
4488 /* Validate block references which are part of inode */
4490 }
4491 }
4514 }
4522 else
4531 }
4537 bad_inode:
4541 }
4544 {
4548 }
4553 {
4559 /*
4560 * i_blocks can be represented in a 32 bit variable
4561 * as multiple of 512 bytes
4562 */
4567 }
4572 /*
4573 * i_blocks can be represented in a 48 bit variable
4574 * as multiple of 512 bytes
4575 */
4581 /* i_block is stored in file system block size */
4585 }
4587 }
4592 };
4596 {
4621 }
4624 }
4626 /*
4627 * Opportunistically update the other time fields for other inodes in
4628 * the same inode table block.
4629 */
4632 {
4639 /*
4640 * Calculate the first inode in the inode table block. Inode
4641 * numbers are one-based. That is, the first inode in a block
4642 * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
4643 */
4650 }
4651 }
4653 /*
4654 * Post the struct inode info into an on-disk inode location in the
4655 * buffer-cache. This gobbles the caller's reference to the
4656 * buffer_head in the inode location struct.
4657 *
4658 * The caller must have write access to iloc->bh.
4659 */
4663 {
4670 uid_t i_uid;
4671 gid_t i_gid;
4672 projid_t i_projid;
4676 /* For fields not tracked in the in-memory inode,
4677 * initialise them to zero for new inodes. */
4689 /*
4690 * Fix up interoperability with old kernels. Otherwise, old inodes get
4691 * re-used with the upper 16 bits of the uid/gid intact
4692 */
4701 }
4707 }
4719 }
4729 }
4735 }
4747 }
4751 }
4761 }
4762 }
4765 EXT4_FEATURE_RO_COMPAT_PROJECT) &&
4792 }
4794 out_brelse:
4798 }
4800 /*
4801 * ext4_write_inode()
4802 *
4803 * We are called from a few places:
4804 *
4805 * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
4806 * Here, there will be no transaction running. We wait for any running
4807 * transaction to commit.
4808 *
4809 * - Within flush work (sys_sync(), kupdate and such).
4810 * We wait on commit, if told to.
4811 *
4812 * - Within iput_final() -> write_inode_now()
4813 * We wait on commit, if told to.
4814 *
4815 * In all cases it is actually safe for us to return without doing anything,
4816 * because the inode has been copied into a raw inode buffer in
4817 * ext4_mark_inode_dirty(). This is a correctness thing for WB_SYNC_ALL
4818 * writeback.
4819 *
4820 * Note that we are absolutely dependent upon all inode dirtiers doing the
4821 * right thing: they *must* call mark_inode_dirty() after dirtying info in
4822 * which we are interested.
4823 *
4824 * It would be a bug for them to not do this. The code:
4825 *
4826 * mark_inode_dirty(inode)
4827 * stuff();
4828 * inode->i_size = expr;
4829 *
4830 * is in error because write_inode() could occur while `stuff()' is running,
4831 * and the new i_size will be lost. Plus the inode will no longer be on the
4832 * superblock's dirty inode list.
4833 */
4835 {
4846 }
4848 /*
4849 * No need to force transaction in WB_SYNC_NONE mode. Also
4850 * ext4_sync_fs() will force the commit after everything is
4851 * written.
4852 */
4863 /*
4864 * sync(2) will flush the whole buffer cache. No need to do
4865 * it here separately for each inode.
4866 */
4873 }
4875 }
4877 }
4879 /*
4880 * In data=journal mode ext4_journalled_invalidatepage() may fail to invalidate
4881 * buffers that are attached to a page stradding i_size and are undergoing
4882 * commit. In that case we have to wait for commit to finish and try again.
4883 */
4885 {
4893 /*
4894 * All buffers in the last page remain valid? Then there's nothing to
4895 * do. We do the check mainly to optimize the common PAGE_SIZE ==
4896 * blocksize case
4897 */
4918 }
4919 }
4921 /*
4922 * ext4_setattr()
4923 *
4924 * Called from notify_change.
4925 *
4926 * We want to trap VFS attempts to truncate the file as soon as
4927 * possible. In particular, we want to make sure that when the VFS
4928 * shrinks i_size, we put the inode on the orphan list and modify
4929 * i_disksize immediately, so that during the subsequent flushing of
4930 * dirty pages and freeing of disk blocks, we can guarantee that any
4931 * commit will leave the blocks being flushed in an unused state on
4932 * disk. (On recovery, the inode will get truncated and the blocks will
4933 * be freed, so we have a strong guarantee that no future commit will
4934 * leave these blocks visible to the user.)
4935 *
4936 * Another thing we have to assure is that if we are in ordered mode
4937 * and inode is still attached to the committing transaction, we must
4938 * we start writeout of all the dirty pages which are being truncated.
4939 * This way we are sure that all the data written in the previous
4940 * transaction are already on disk (truncate waits for pages under
4941 * writeback).
4942 *
4943 * Called with inode->i_mutex down.
4944 */
4946 {
4960 }
4965 /* (user+group)*(old+new) structure, inode write (sb,
4966 * inode block, ? - but truncate inode update has it) */
4973 }
4978 }
4979 /* Update corresponding info in inode so that everything is in
4980 * one transaction */
4987 }
4999 }
5012 }
5018 }
5022 }
5023 /*
5024 * Update c/mtime on truncate up, ext4_truncate() will
5025 * update c/mtime in shrink case below
5026 */
5030 }
5036 /*
5037 * We have to update i_size under i_data_sem together
5038 * with i_disksize to avoid races with writeback code
5039 * running ext4_wb_update_i_disksize().
5040 */
5049 }
5050 }
5054 /*
5055 * Blocks are going to be removed from the inode. Wait
5056 * for dio in flight. Temporarily disable
5057 * dioread_nolock to prevent livelock.
5058 */
5066 }
5068 /*
5069 * Truncate pagecache after we've waited for commit
5070 * in data=journal mode to make pages freeable.
5071 */
5076 }
5081 }
5083 /*
5084 * If the call to ext4_truncate failed to get a transaction handle at
5085 * all, we need to clean up the in-core orphan list manually.
5086 */
5093 err_out:
5098 }
5102 {
5109 /*
5110 * If there is inline data in the inode, the inode will normally not
5111 * have data blocks allocated (it may have an external xattr block).
5112 * Report at least one sector for such files, so tools like tar, rsync,
5113 * others doen't incorrectly think the file is completely sparse.
5114 */
5118 /*
5119 * We can't update i_blocks if the block allocation is delayed
5120 * otherwise in the case of system crash before the real block
5121 * allocation is done, we will have i_blocks inconsistent with
5122 * on-disk file blocks.
5123 * We always keep i_blocks updated together with real
5124 * allocation. But to not confuse with user, stat
5125 * will return the blocks that include the delayed allocation
5126 * blocks for this file.
5127 */
5132 }
5136 {
5140 }
5142 /*
5143 * Account for index blocks, block groups bitmaps and block group
5144 * descriptor blocks if modify datablocks and index blocks
5145 * worse case, the indexs blocks spread over different block groups
5146 *
5147 * If datablocks are discontiguous, they are possible to spread over
5148 * different block groups too. If they are contiguous, with flexbg,
5149 * they could still across block group boundary.
5150 *
5151 * Also account for superblock, inode, quota and xattr blocks
5152 */
5155 {
5161 /*
5162 * How many index blocks need to touch to map @lblocks logical blocks
5163 * to @pextents physical extents?
5164 */
5169 /*
5170 * Now let's see how many group bitmaps and group descriptors need
5171 * to account
5172 */
5180 /* bitmaps and block group descriptor blocks */
5183 /* Blocks for super block, inode, quota and xattr blocks */
5187 }
5189 /*
5190 * Calculate the total number of credits to reserve to fit
5191 * the modification of a single pages into a single transaction,
5192 * which may include multiple chunks of block allocations.
5193 *
5194 * This could be called via ext4_write_begin()
5195 *
5196 * We need to consider the worse case, when
5197 * one new block per extent.
5198 */
5200 {
5206 /* Account for data blocks for journalled mode */
5210 }
5212 /*
5213 * Calculate the journal credits for a chunk of data modification.
5214 *
5215 * This is called from DIO, fallocate or whoever calling
5216 * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
5217 *
5218 * journal buffers for data blocks are not included here, as DIO
5219 * and fallocate do no need to journal data buffers.
5220 */
5222 {
5224 }
5226 /*
5227 * The caller must have previously called ext4_reserve_inode_write().
5228 * Give this, we know that the caller already has write access to iloc->bh.
5229 */
5232 {
5238 /* the do_update_inode consumes one bh->b_count */
5241 /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
5245 }
5247 /*
5248 * On success, We end up with an outstanding reference count against
5249 * iloc->bh. This _must_ be cleaned up later.
5250 */
5252 int
5255 {
5265 }
5266 }
5269 }
5271 /*
5272 * Expand an inode by new_extra_isize bytes.
5273 * Returns 0 on success or negative error number on failure.
5274 */
5279 {
5290 /* No extended attributes present */
5294 new_extra_isize);
5297 }
5299 /* try to expand with EAs present */
5302 }
5304 /*
5305 * What we do here is to mark the in-core inode as clean with respect to inode
5306 * dirtiness (it may still be data-dirty).
5307 * This means that the in-core inode may be reaped by prune_icache
5308 * without having to perform any I/O. This is a very good thing,
5309 * because *any* task may call prune_icache - even ones which
5310 * have a transaction open against a different journal.
5311 *
5312 * Is this cheating? Not really. Sure, we haven't written the
5313 * inode out, but prune_icache isn't a user-visible syncing function.
5314 * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
5315 * we start and wait on commits.
5316 */
5318 {
5332 /*
5333 * We need extra buffer credits since we may write into EA block
5334 * with this same handle. If journal_extend fails, then it will
5335 * only result in a minor loss of functionality for that inode.
5336 * If this is felt to be critical, then e2fsck should be run to
5337 * force a large enough s_min_extra_isize.
5338 */
5346 EXT4_STATE_NO_EXPAND);
5350 "Unable to expand inode %lu. Delete"
5353 mnt_count =
5355 }
5356 }
5357 }
5358 }
5360 }
5362 /*
5363 * ext4_dirty_inode() is called from __mark_inode_dirty()
5364 *
5365 * We're really interested in the case where a file is being extended.
5366 * i_size has been changed by generic_commit_write() and we thus need
5367 * to include the updated inode in the current transaction.
5368 *
5369 * Also, dquot_alloc_block() will always dirty the inode when blocks
5370 * are allocated to the file.
5371 *
5372 * If the inode is marked synchronous, we don't honour that here - doing
5373 * so would cause a commit on atime updates, which we don't bother doing.
5374 * We handle synchronous inodes at the highest possible level.
5375 *
5376 * If only the I_DIRTY_TIME flag is set, we can skip everything. If
5377 * I_DIRTY_TIME and I_DIRTY_SYNC is set, the only inode fields we need
5378 * to copy into the on-disk inode structure are the timestamp files.
5379 */
5381 {
5393 out:
5395 }
5397 #if 0
5398 /*
5399 * Bind an inode's backing buffer_head into this transaction, to prevent
5400 * it from being flushed to disk early. Unlike
5401 * ext4_reserve_inode_write, this leaves behind no bh reference and
5402 * returns no iloc structure, so the caller needs to repeat the iloc
5403 * lookup to mark the inode dirty later.
5404 */
5406 {
5417 NULL,
5420 }
5421 }
5424 }
5425 #endif
5428 {
5433 /*
5434 * We have to be very careful here: changing a data block's
5435 * journaling status dynamically is dangerous. If we write a
5436 * data block to the journal, change the status and then delete
5437 * that block, we risk forgetting to revoke the old log record
5438 * from the journal and so a subsequent replay can corrupt data.
5439 * So, first we make sure that the journal is empty and that
5440 * nobody is changing anything.
5441 */
5448 /* We have to allocate physical blocks for delalloc blocks
5449 * before flushing journal. otherwise delalloc blocks can not
5450 * be allocated any more. even more truncate on delalloc blocks
5451 * could trigger BUG by flushing delalloc blocks in journal.
5452 * There is no delalloc block in non-journal data mode.
5453 */
5458 }
5460 /* Wait for all existing dio workers */
5466 /*
5467 * OK, there are no updates running now, and all cached data is
5468 * synced to disk. We are now in a completely consistent state
5469 * which doesn't have anything in the journal, and we know that
5470 * no filesystem updates are running, so it is safe to modify
5471 * the inode's in-core data-journaling state flag now.
5472 */
5482 }
5484 }
5490 /* Finally we can mark the inode as dirty. */
5502 }
5505 {
5507 }
5510 {
5512 loff_t size;
5526 /* Delalloc case is easy... */
5532 ext4_da_get_block_prep);
5536 }
5540 /* Page got truncated from under us? */
5545 }
5549 else
5551 /*
5552 * Return if we have all the buffers mapped. This avoids the need to do
5553 * journal_start/journal_stop which can block and take a long time
5554 */
5558 ext4_bh_unmapped)) {
5559 /* Wait so that we don't change page under IO */
5563 }
5564 }
5566 /* OK, we need to fill the hole... */
5569 else
5571 retry_alloc:
5577 }
5586 }
5588 }
5592 out_ret:
5594 out:
5598 }
5601 {
5610 }
5612 /*
5613 * Find the first extent at or after @lblk in an inode that is not a hole.
5614 * Search for @map_len blocks at most. The extent is returned in @result.
5615 *
5616 * The function returns 1 if we found an extent. The function returns 0 in
5617 * case there is no extent at or after @lblk and in that case also sets
5618 * @result->es_len to 0. In case of error, the error code is returned.
5619 */
5622 {
5630 /*
5631 * For non-extent based files this loop may iterate several times since
5632 * we do not determine full hole size.
5633 */
5638 /* There's extent covering m_lblk? Just return it. */
5647 else
5651 }
5655 /* Is delalloc data before next block in extent tree? */
5668 }
5669 /* There's a hole at m_lblk, advance us after it */
5674 }
5677 }