1 // SPDX-License-Identifier: LGPL-2.1
3 * Copyright (c) 2008,2009 NEC Software Tohoku, Ltd.
4 * Written by Takashi Sato <t-sato@yk.jp.nec.com>
5 * Akira Fujita <a-fujita@rs.jp.nec.com>
9 #include <linux/quotaops.h>
10 #include <linux/slab.h>
11 #include "ext4_jbd2.h"
13 #include "ext4_extents.h"
16 * get_ext_path - Find an extent path for designated logical block number.
18 * @inode: an inode which is searched
19 * @lblock: logical block number to find an extent path
20 * @path: pointer to an extent path pointer (for output)
22 * ext4_find_extent wrapper. Return 0 on success, or a negative error value
26 get_ext_path(struct inode
*inode
, ext4_lblk_t lblock
,
27 struct ext4_ext_path
**ppath
)
29 struct ext4_ext_path
*path
;
31 path
= ext4_find_extent(inode
, lblock
, ppath
, EXT4_EX_NOCACHE
);
34 if (path
[ext_depth(inode
)].p_ext
== NULL
) {
35 ext4_ext_drop_refs(path
);
45 * ext4_double_down_write_data_sem - Acquire two inodes' write lock
48 * Acquire write lock of i_data_sem of the two inodes
51 ext4_double_down_write_data_sem(struct inode
*first
, struct inode
*second
)
54 down_write(&EXT4_I(first
)->i_data_sem
);
55 down_write_nested(&EXT4_I(second
)->i_data_sem
, I_DATA_SEM_OTHER
);
57 down_write(&EXT4_I(second
)->i_data_sem
);
58 down_write_nested(&EXT4_I(first
)->i_data_sem
, I_DATA_SEM_OTHER
);
64 * ext4_double_up_write_data_sem - Release two inodes' write lock of i_data_sem
66 * @orig_inode: original inode structure to be released its lock first
67 * @donor_inode: donor inode structure to be released its lock second
68 * Release write lock of i_data_sem of two inodes (orig and donor).
71 ext4_double_up_write_data_sem(struct inode
*orig_inode
,
72 struct inode
*donor_inode
)
74 up_write(&EXT4_I(orig_inode
)->i_data_sem
);
75 up_write(&EXT4_I(donor_inode
)->i_data_sem
);
79 * mext_check_coverage - Check that all extents in range has the same type
81 * @inode: inode in question
82 * @from: block offset of inode
83 * @count: block count to be checked
84 * @unwritten: extents expected to be unwritten
85 * @err: pointer to save error value
87 * Return 1 if all extents in range has expected type, and zero otherwise.
90 mext_check_coverage(struct inode
*inode
, ext4_lblk_t from
, ext4_lblk_t count
,
91 int unwritten
, int *err
)
93 struct ext4_ext_path
*path
= NULL
;
94 struct ext4_extent
*ext
;
96 ext4_lblk_t last
= from
+ count
;
98 *err
= get_ext_path(inode
, from
, &path
);
101 ext
= path
[ext_depth(inode
)].p_ext
;
102 if (unwritten
!= ext4_ext_is_unwritten(ext
))
104 from
+= ext4_ext_get_actual_len(ext
);
105 ext4_ext_drop_refs(path
);
109 ext4_ext_drop_refs(path
);
115 * mext_page_double_lock - Grab and lock pages on both @inode1 and @inode2
117 * @inode1: the inode structure
118 * @inode2: the inode structure
119 * @index1: page index
120 * @index2: page index
121 * @page: result page vector
123 * Grab two locked pages for inode's by inode order
126 mext_page_double_lock(struct inode
*inode1
, struct inode
*inode2
,
127 pgoff_t index1
, pgoff_t index2
, struct page
*page
[2])
129 struct address_space
*mapping
[2];
130 unsigned fl
= AOP_FLAG_NOFS
;
132 BUG_ON(!inode1
|| !inode2
);
133 if (inode1
< inode2
) {
134 mapping
[0] = inode1
->i_mapping
;
135 mapping
[1] = inode2
->i_mapping
;
137 pgoff_t tmp
= index1
;
140 mapping
[0] = inode2
->i_mapping
;
141 mapping
[1] = inode1
->i_mapping
;
144 page
[0] = grab_cache_page_write_begin(mapping
[0], index1
, fl
);
148 page
[1] = grab_cache_page_write_begin(mapping
[1], index2
, fl
);
150 unlock_page(page
[0]);
155 * grab_cache_page_write_begin() may not wait on page's writeback if
156 * BDI not demand that. But it is reasonable to be very conservative
157 * here and explicitly wait on page's writeback
159 wait_on_page_writeback(page
[0]);
160 wait_on_page_writeback(page
[1]);
162 swap(page
[0], page
[1]);
167 /* Force page buffers uptodate w/o dropping page's lock */
169 mext_page_mkuptodate(struct page
*page
, unsigned from
, unsigned to
)
171 struct inode
*inode
= page
->mapping
->host
;
173 struct buffer_head
*bh
, *head
, *arr
[MAX_BUF_PER_PAGE
];
174 unsigned int blocksize
, block_start
, block_end
;
175 int i
, err
, nr
= 0, partial
= 0;
176 BUG_ON(!PageLocked(page
));
177 BUG_ON(PageWriteback(page
));
179 if (PageUptodate(page
))
182 blocksize
= i_blocksize(inode
);
183 if (!page_has_buffers(page
))
184 create_empty_buffers(page
, blocksize
, 0);
186 head
= page_buffers(page
);
187 block
= (sector_t
)page
->index
<< (PAGE_SHIFT
- inode
->i_blkbits
);
188 for (bh
= head
, block_start
= 0; bh
!= head
|| !block_start
;
189 block
++, block_start
= block_end
, bh
= bh
->b_this_page
) {
190 block_end
= block_start
+ blocksize
;
191 if (block_end
<= from
|| block_start
>= to
) {
192 if (!buffer_uptodate(bh
))
196 if (buffer_uptodate(bh
))
198 if (!buffer_mapped(bh
)) {
199 err
= ext4_get_block(inode
, block
, bh
, 0);
204 if (!buffer_mapped(bh
)) {
205 zero_user(page
, block_start
, blocksize
);
206 set_buffer_uptodate(bh
);
210 BUG_ON(nr
>= MAX_BUF_PER_PAGE
);
217 for (i
= 0; i
< nr
; i
++) {
219 if (!bh_uptodate_or_lock(bh
)) {
220 err
= bh_submit_read(bh
);
227 SetPageUptodate(page
);
232 * move_extent_per_page - Move extent data per page
234 * @o_filp: file structure of original file
235 * @donor_inode: donor inode
236 * @orig_page_offset: page index on original file
237 * @donor_page_offset: page index on donor file
238 * @data_offset_in_page: block index where data swapping starts
239 * @block_len_in_page: the number of blocks to be swapped
240 * @unwritten: orig extent is unwritten or not
241 * @err: pointer to save return value
243 * Save the data in original inode blocks and replace original inode extents
244 * with donor inode extents by calling ext4_swap_extents().
245 * Finally, write out the saved data in new original inode blocks. Return
246 * replaced block count.
249 move_extent_per_page(struct file
*o_filp
, struct inode
*donor_inode
,
250 pgoff_t orig_page_offset
, pgoff_t donor_page_offset
,
251 int data_offset_in_page
,
252 int block_len_in_page
, int unwritten
, int *err
)
254 struct inode
*orig_inode
= file_inode(o_filp
);
255 struct page
*pagep
[2] = {NULL
, NULL
};
257 ext4_lblk_t orig_blk_offset
, donor_blk_offset
;
258 unsigned long blocksize
= orig_inode
->i_sb
->s_blocksize
;
259 unsigned int tmp_data_size
, data_size
, replaced_size
;
260 int i
, err2
, jblocks
, retries
= 0;
261 int replaced_count
= 0;
262 int from
= data_offset_in_page
<< orig_inode
->i_blkbits
;
263 int blocks_per_page
= PAGE_SIZE
>> orig_inode
->i_blkbits
;
264 struct super_block
*sb
= orig_inode
->i_sb
;
265 struct buffer_head
*bh
= NULL
;
268 * It needs twice the amount of ordinary journal buffers because
269 * inode and donor_inode may change each different metadata blocks.
273 jblocks
= ext4_writepage_trans_blocks(orig_inode
) * 2;
274 handle
= ext4_journal_start(orig_inode
, EXT4_HT_MOVE_EXTENTS
, jblocks
);
275 if (IS_ERR(handle
)) {
276 *err
= PTR_ERR(handle
);
280 orig_blk_offset
= orig_page_offset
* blocks_per_page
+
283 donor_blk_offset
= donor_page_offset
* blocks_per_page
+
286 /* Calculate data_size */
287 if ((orig_blk_offset
+ block_len_in_page
- 1) ==
288 ((orig_inode
->i_size
- 1) >> orig_inode
->i_blkbits
)) {
289 /* Replace the last block */
290 tmp_data_size
= orig_inode
->i_size
& (blocksize
- 1);
292 * If data_size equal zero, it shows data_size is multiples of
293 * blocksize. So we set appropriate value.
295 if (tmp_data_size
== 0)
296 tmp_data_size
= blocksize
;
298 data_size
= tmp_data_size
+
299 ((block_len_in_page
- 1) << orig_inode
->i_blkbits
);
301 data_size
= block_len_in_page
<< orig_inode
->i_blkbits
;
303 replaced_size
= data_size
;
305 *err
= mext_page_double_lock(orig_inode
, donor_inode
, orig_page_offset
,
306 donor_page_offset
, pagep
);
307 if (unlikely(*err
< 0))
310 * If orig extent was unwritten it can become initialized
311 * at any time after i_data_sem was dropped, in order to
312 * serialize with delalloc we have recheck extent while we
313 * hold page's lock, if it is still the case data copy is not
314 * necessary, just swap data blocks between orig and donor.
317 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
318 /* If any of extents in range became initialized we have to
319 * fallback to data copying */
320 unwritten
= mext_check_coverage(orig_inode
, orig_blk_offset
,
321 block_len_in_page
, 1, err
);
325 unwritten
&= mext_check_coverage(donor_inode
, donor_blk_offset
,
326 block_len_in_page
, 1, err
);
331 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
334 if ((page_has_private(pagep
[0]) &&
335 !try_to_release_page(pagep
[0], 0)) ||
336 (page_has_private(pagep
[1]) &&
337 !try_to_release_page(pagep
[1], 0))) {
341 replaced_count
= ext4_swap_extents(handle
, orig_inode
,
342 donor_inode
, orig_blk_offset
,
344 block_len_in_page
, 1, err
);
346 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
350 *err
= mext_page_mkuptodate(pagep
[0], from
, from
+ replaced_size
);
354 /* At this point all buffers in range are uptodate, old mapping layout
355 * is no longer required, try to drop it now. */
356 if ((page_has_private(pagep
[0]) && !try_to_release_page(pagep
[0], 0)) ||
357 (page_has_private(pagep
[1]) && !try_to_release_page(pagep
[1], 0))) {
361 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
362 replaced_count
= ext4_swap_extents(handle
, orig_inode
, donor_inode
,
363 orig_blk_offset
, donor_blk_offset
,
364 block_len_in_page
, 1, err
);
365 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
367 if (replaced_count
) {
368 block_len_in_page
= replaced_count
;
370 block_len_in_page
<< orig_inode
->i_blkbits
;
374 /* Perform all necessary steps similar write_begin()/write_end()
375 * but keeping in mind that i_size will not change */
376 if (!page_has_buffers(pagep
[0]))
377 create_empty_buffers(pagep
[0], 1 << orig_inode
->i_blkbits
, 0);
378 bh
= page_buffers(pagep
[0]);
379 for (i
= 0; i
< data_offset_in_page
; i
++)
380 bh
= bh
->b_this_page
;
381 for (i
= 0; i
< block_len_in_page
; i
++) {
382 *err
= ext4_get_block(orig_inode
, orig_blk_offset
+ i
, bh
, 0);
385 bh
= bh
->b_this_page
;
388 *err
= block_commit_write(pagep
[0], from
, from
+ replaced_size
);
390 if (unlikely(*err
< 0))
391 goto repair_branches
;
393 /* Even in case of data=writeback it is reasonable to pin
394 * inode to transaction, to prevent unexpected data loss */
395 *err
= ext4_jbd2_inode_add_write(handle
, orig_inode
);
398 unlock_page(pagep
[0]);
400 unlock_page(pagep
[1]);
403 ext4_journal_stop(handle
);
404 if (*err
== -ENOSPC
&&
405 ext4_should_retry_alloc(sb
, &retries
))
407 /* Buffer was busy because probably is pinned to journal transaction,
408 * force transaction commit may help to free it. */
409 if (*err
== -EBUSY
&& retries
++ < 4 && EXT4_SB(sb
)->s_journal
&&
410 jbd2_journal_force_commit_nested(EXT4_SB(sb
)->s_journal
))
412 return replaced_count
;
416 * This should never ever happen!
417 * Extents are swapped already, but we are not able to copy data.
418 * Try to swap extents to it's original places
420 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
421 replaced_count
= ext4_swap_extents(handle
, donor_inode
, orig_inode
,
422 orig_blk_offset
, donor_blk_offset
,
423 block_len_in_page
, 0, &err2
);
424 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
425 if (replaced_count
!= block_len_in_page
) {
426 EXT4_ERROR_INODE_BLOCK(orig_inode
, (sector_t
)(orig_blk_offset
),
427 "Unable to copy data block,"
428 " data will be lost.");
436 * mext_check_arguments - Check whether move extent can be done
438 * @orig_inode: original inode
439 * @donor_inode: donor inode
440 * @orig_start: logical start offset in block for orig
441 * @donor_start: logical start offset in block for donor
442 * @len: the number of blocks to be moved
444 * Check the arguments of ext4_move_extents() whether the files can be
445 * exchanged with each other.
446 * Return 0 on success, or a negative error value on failure.
449 mext_check_arguments(struct inode
*orig_inode
,
450 struct inode
*donor_inode
, __u64 orig_start
,
451 __u64 donor_start
, __u64
*len
)
453 __u64 orig_eof
, donor_eof
;
454 unsigned int blkbits
= orig_inode
->i_blkbits
;
455 unsigned int blocksize
= 1 << blkbits
;
457 orig_eof
= (i_size_read(orig_inode
) + blocksize
- 1) >> blkbits
;
458 donor_eof
= (i_size_read(donor_inode
) + blocksize
- 1) >> blkbits
;
461 if (donor_inode
->i_mode
& (S_ISUID
|S_ISGID
)) {
462 ext4_debug("ext4 move extent: suid or sgid is set"
463 " to donor file [ino:orig %lu, donor %lu]\n",
464 orig_inode
->i_ino
, donor_inode
->i_ino
);
468 if (IS_IMMUTABLE(donor_inode
) || IS_APPEND(donor_inode
))
471 /* Ext4 move extent does not support swapfile */
472 if (IS_SWAPFILE(orig_inode
) || IS_SWAPFILE(donor_inode
)) {
473 ext4_debug("ext4 move extent: The argument files should "
474 "not be swapfile [ino:orig %lu, donor %lu]\n",
475 orig_inode
->i_ino
, donor_inode
->i_ino
);
479 if (ext4_is_quota_file(orig_inode
) && ext4_is_quota_file(donor_inode
)) {
480 ext4_debug("ext4 move extent: The argument files should "
481 "not be quota files [ino:orig %lu, donor %lu]\n",
482 orig_inode
->i_ino
, donor_inode
->i_ino
);
486 /* Ext4 move extent supports only extent based file */
487 if (!(ext4_test_inode_flag(orig_inode
, EXT4_INODE_EXTENTS
))) {
488 ext4_debug("ext4 move extent: orig file is not extents "
489 "based file [ino:orig %lu]\n", orig_inode
->i_ino
);
491 } else if (!(ext4_test_inode_flag(donor_inode
, EXT4_INODE_EXTENTS
))) {
492 ext4_debug("ext4 move extent: donor file is not extents "
493 "based file [ino:donor %lu]\n", donor_inode
->i_ino
);
497 if ((!orig_inode
->i_size
) || (!donor_inode
->i_size
)) {
498 ext4_debug("ext4 move extent: File size is 0 byte\n");
502 /* Start offset should be same */
503 if ((orig_start
& ~(PAGE_MASK
>> orig_inode
->i_blkbits
)) !=
504 (donor_start
& ~(PAGE_MASK
>> orig_inode
->i_blkbits
))) {
505 ext4_debug("ext4 move extent: orig and donor's start "
506 "offsets are not aligned [ino:orig %lu, donor %lu]\n",
507 orig_inode
->i_ino
, donor_inode
->i_ino
);
511 if ((orig_start
>= EXT_MAX_BLOCKS
) ||
512 (donor_start
>= EXT_MAX_BLOCKS
) ||
513 (*len
> EXT_MAX_BLOCKS
) ||
514 (donor_start
+ *len
>= EXT_MAX_BLOCKS
) ||
515 (orig_start
+ *len
>= EXT_MAX_BLOCKS
)) {
516 ext4_debug("ext4 move extent: Can't handle over [%u] blocks "
517 "[ino:orig %lu, donor %lu]\n", EXT_MAX_BLOCKS
,
518 orig_inode
->i_ino
, donor_inode
->i_ino
);
521 if (orig_eof
< orig_start
+ *len
- 1)
522 *len
= orig_eof
- orig_start
;
523 if (donor_eof
< donor_start
+ *len
- 1)
524 *len
= donor_eof
- donor_start
;
526 ext4_debug("ext4 move extent: len should not be 0 "
527 "[ino:orig %lu, donor %lu]\n", orig_inode
->i_ino
,
536 * ext4_move_extents - Exchange the specified range of a file
538 * @o_filp: file structure of the original file
539 * @d_filp: file structure of the donor file
540 * @orig_blk: start offset in block for orig
541 * @donor_blk: start offset in block for donor
542 * @len: the number of blocks to be moved
543 * @moved_len: moved block length
545 * This function returns 0 and moved block length is set in moved_len
546 * if succeed, otherwise returns error value.
550 ext4_move_extents(struct file
*o_filp
, struct file
*d_filp
, __u64 orig_blk
,
551 __u64 donor_blk
, __u64 len
, __u64
*moved_len
)
553 struct inode
*orig_inode
= file_inode(o_filp
);
554 struct inode
*donor_inode
= file_inode(d_filp
);
555 struct ext4_ext_path
*path
= NULL
;
556 int blocks_per_page
= PAGE_SIZE
>> orig_inode
->i_blkbits
;
557 ext4_lblk_t o_end
, o_start
= orig_blk
;
558 ext4_lblk_t d_start
= donor_blk
;
561 if (orig_inode
->i_sb
!= donor_inode
->i_sb
) {
562 ext4_debug("ext4 move extent: The argument files "
563 "should be in same FS [ino:orig %lu, donor %lu]\n",
564 orig_inode
->i_ino
, donor_inode
->i_ino
);
568 /* orig and donor should be different inodes */
569 if (orig_inode
== donor_inode
) {
570 ext4_debug("ext4 move extent: The argument files should not "
571 "be same inode [ino:orig %lu, donor %lu]\n",
572 orig_inode
->i_ino
, donor_inode
->i_ino
);
576 /* Regular file check */
577 if (!S_ISREG(orig_inode
->i_mode
) || !S_ISREG(donor_inode
->i_mode
)) {
578 ext4_debug("ext4 move extent: The argument files should be "
579 "regular file [ino:orig %lu, donor %lu]\n",
580 orig_inode
->i_ino
, donor_inode
->i_ino
);
584 /* TODO: it's not obvious how to swap blocks for inodes with full
585 journaling enabled */
586 if (ext4_should_journal_data(orig_inode
) ||
587 ext4_should_journal_data(donor_inode
)) {
588 ext4_msg(orig_inode
->i_sb
, KERN_ERR
,
589 "Online defrag not supported with data journaling");
593 if (ext4_encrypted_inode(orig_inode
) ||
594 ext4_encrypted_inode(donor_inode
)) {
595 ext4_msg(orig_inode
->i_sb
, KERN_ERR
,
596 "Online defrag not supported for encrypted files");
600 /* Protect orig and donor inodes against a truncate */
601 lock_two_nondirectories(orig_inode
, donor_inode
);
603 /* Wait for all existing dio workers */
604 ext4_inode_block_unlocked_dio(orig_inode
);
605 ext4_inode_block_unlocked_dio(donor_inode
);
606 inode_dio_wait(orig_inode
);
607 inode_dio_wait(donor_inode
);
609 /* Protect extent tree against block allocations via delalloc */
610 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
611 /* Check the filesystem environment whether move_extent can be done */
612 ret
= mext_check_arguments(orig_inode
, donor_inode
, orig_blk
,
616 o_end
= o_start
+ len
;
618 while (o_start
< o_end
) {
619 struct ext4_extent
*ex
;
620 ext4_lblk_t cur_blk
, next_blk
;
621 pgoff_t orig_page_index
, donor_page_index
;
623 int unwritten
, cur_len
;
625 ret
= get_ext_path(orig_inode
, o_start
, &path
);
628 ex
= path
[path
->p_depth
].p_ext
;
629 next_blk
= ext4_ext_next_allocated_block(path
);
630 cur_blk
= le32_to_cpu(ex
->ee_block
);
631 cur_len
= ext4_ext_get_actual_len(ex
);
632 /* Check hole before the start pos */
633 if (cur_blk
+ cur_len
- 1 < o_start
) {
634 if (next_blk
== EXT_MAX_BLOCKS
) {
639 d_start
+= next_blk
- o_start
;
642 /* Check hole after the start pos */
643 } else if (cur_blk
> o_start
) {
645 d_start
+= cur_blk
- o_start
;
647 /* Extent inside requested range ?*/
648 if (cur_blk
>= o_end
)
650 } else { /* in_range(o_start, o_blk, o_len) */
651 cur_len
+= cur_blk
- o_start
;
653 unwritten
= ext4_ext_is_unwritten(ex
);
654 if (o_end
- o_start
< cur_len
)
655 cur_len
= o_end
- o_start
;
657 orig_page_index
= o_start
>> (PAGE_SHIFT
-
658 orig_inode
->i_blkbits
);
659 donor_page_index
= d_start
>> (PAGE_SHIFT
-
660 donor_inode
->i_blkbits
);
661 offset_in_page
= o_start
% blocks_per_page
;
662 if (cur_len
> blocks_per_page
- offset_in_page
)
663 cur_len
= blocks_per_page
- offset_in_page
;
665 * Up semaphore to avoid following problems:
666 * a. transaction deadlock among ext4_journal_start,
667 * ->write_begin via pagefault, and jbd2_journal_commit
668 * b. racing with ->readpage, ->write_begin, and ext4_get_block
669 * in move_extent_per_page
671 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
672 /* Swap original branches with new branches */
673 move_extent_per_page(o_filp
, donor_inode
,
674 orig_page_index
, donor_page_index
,
675 offset_in_page
, cur_len
,
677 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
683 *moved_len
= o_start
- orig_blk
;
684 if (*moved_len
> len
)
689 ext4_discard_preallocations(orig_inode
);
690 ext4_discard_preallocations(donor_inode
);
693 ext4_ext_drop_refs(path
);
695 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
696 ext4_inode_resume_unlocked_dio(orig_inode
);
697 ext4_inode_resume_unlocked_dio(donor_inode
);
698 unlock_two_nondirectories(orig_inode
, donor_inode
);