2 * Copyright (c) 2008,2009 NEC Software Tohoku, Ltd.
3 * Written by Takashi Sato <t-sato@yk.jp.nec.com>
4 * Akira Fujita <a-fujita@rs.jp.nec.com>
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of version 2.1 of the GNU Lesser General Public License
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
17 #include <linux/quotaops.h>
18 #include <linux/slab.h>
19 #include "ext4_jbd2.h"
21 #include "ext4_extents.h"
24 * get_ext_path - Find an extent path for designated logical block number.
26 * @inode: an inode which is searched
27 * @lblock: logical block number to find an extent path
28 * @path: pointer to an extent path pointer (for output)
30 * ext4_find_extent wrapper. Return 0 on success, or a negative error value
34 get_ext_path(struct inode
*inode
, ext4_lblk_t lblock
,
35 struct ext4_ext_path
**ppath
)
37 struct ext4_ext_path
*path
;
39 path
= ext4_find_extent(inode
, lblock
, ppath
, EXT4_EX_NOCACHE
);
42 if (path
[ext_depth(inode
)].p_ext
== NULL
) {
43 ext4_ext_drop_refs(path
);
53 * ext4_double_down_write_data_sem - Acquire two inodes' write lock
56 * Acquire write lock of i_data_sem of the two inodes
59 ext4_double_down_write_data_sem(struct inode
*first
, struct inode
*second
)
62 down_write(&EXT4_I(first
)->i_data_sem
);
63 down_write_nested(&EXT4_I(second
)->i_data_sem
, SINGLE_DEPTH_NESTING
);
65 down_write(&EXT4_I(second
)->i_data_sem
);
66 down_write_nested(&EXT4_I(first
)->i_data_sem
, SINGLE_DEPTH_NESTING
);
72 * ext4_double_up_write_data_sem - Release two inodes' write lock of i_data_sem
74 * @orig_inode: original inode structure to be released its lock first
75 * @donor_inode: donor inode structure to be released its lock second
76 * Release write lock of i_data_sem of two inodes (orig and donor).
79 ext4_double_up_write_data_sem(struct inode
*orig_inode
,
80 struct inode
*donor_inode
)
82 up_write(&EXT4_I(orig_inode
)->i_data_sem
);
83 up_write(&EXT4_I(donor_inode
)->i_data_sem
);
87 * mext_check_coverage - Check that all extents in range has the same type
89 * @inode: inode in question
90 * @from: block offset of inode
91 * @count: block count to be checked
92 * @unwritten: extents expected to be unwritten
93 * @err: pointer to save error value
95 * Return 1 if all extents in range has expected type, and zero otherwise.
98 mext_check_coverage(struct inode
*inode
, ext4_lblk_t from
, ext4_lblk_t count
,
99 int unwritten
, int *err
)
101 struct ext4_ext_path
*path
= NULL
;
102 struct ext4_extent
*ext
;
104 ext4_lblk_t last
= from
+ count
;
105 while (from
< last
) {
106 *err
= get_ext_path(inode
, from
, &path
);
109 ext
= path
[ext_depth(inode
)].p_ext
;
110 if (unwritten
!= ext4_ext_is_unwritten(ext
))
112 from
+= ext4_ext_get_actual_len(ext
);
113 ext4_ext_drop_refs(path
);
117 ext4_ext_drop_refs(path
);
123 * mext_page_double_lock - Grab and lock pages on both @inode1 and @inode2
125 * @inode1: the inode structure
126 * @inode2: the inode structure
127 * @index1: page index
128 * @index2: page index
129 * @page: result page vector
131 * Grab two locked pages for inode's by inode order
134 mext_page_double_lock(struct inode
*inode1
, struct inode
*inode2
,
135 pgoff_t index1
, pgoff_t index2
, struct page
*page
[2])
137 struct address_space
*mapping
[2];
138 unsigned fl
= AOP_FLAG_NOFS
;
140 BUG_ON(!inode1
|| !inode2
);
141 if (inode1
< inode2
) {
142 mapping
[0] = inode1
->i_mapping
;
143 mapping
[1] = inode2
->i_mapping
;
145 pgoff_t tmp
= index1
;
148 mapping
[0] = inode2
->i_mapping
;
149 mapping
[1] = inode1
->i_mapping
;
152 page
[0] = grab_cache_page_write_begin(mapping
[0], index1
, fl
);
156 page
[1] = grab_cache_page_write_begin(mapping
[1], index2
, fl
);
158 unlock_page(page
[0]);
159 page_cache_release(page
[0]);
163 * grab_cache_page_write_begin() may not wait on page's writeback if
164 * BDI not demand that. But it is reasonable to be very conservative
165 * here and explicitly wait on page's writeback
167 wait_on_page_writeback(page
[0]);
168 wait_on_page_writeback(page
[1]);
169 if (inode1
> inode2
) {
178 /* Force page buffers uptodate w/o dropping page's lock */
180 mext_page_mkuptodate(struct page
*page
, unsigned from
, unsigned to
)
182 struct inode
*inode
= page
->mapping
->host
;
184 struct buffer_head
*bh
, *head
, *arr
[MAX_BUF_PER_PAGE
];
185 unsigned int blocksize
, block_start
, block_end
;
186 int i
, err
, nr
= 0, partial
= 0;
187 BUG_ON(!PageLocked(page
));
188 BUG_ON(PageWriteback(page
));
190 if (PageUptodate(page
))
193 blocksize
= 1 << inode
->i_blkbits
;
194 if (!page_has_buffers(page
))
195 create_empty_buffers(page
, blocksize
, 0);
197 head
= page_buffers(page
);
198 block
= (sector_t
)page
->index
<< (PAGE_CACHE_SHIFT
- inode
->i_blkbits
);
199 for (bh
= head
, block_start
= 0; bh
!= head
|| !block_start
;
200 block
++, block_start
= block_end
, bh
= bh
->b_this_page
) {
201 block_end
= block_start
+ blocksize
;
202 if (block_end
<= from
|| block_start
>= to
) {
203 if (!buffer_uptodate(bh
))
207 if (buffer_uptodate(bh
))
209 if (!buffer_mapped(bh
)) {
210 err
= ext4_get_block(inode
, block
, bh
, 0);
215 if (!buffer_mapped(bh
)) {
216 zero_user(page
, block_start
, blocksize
);
217 set_buffer_uptodate(bh
);
221 BUG_ON(nr
>= MAX_BUF_PER_PAGE
);
228 for (i
= 0; i
< nr
; i
++) {
230 if (!bh_uptodate_or_lock(bh
)) {
231 err
= bh_submit_read(bh
);
238 SetPageUptodate(page
);
243 * move_extent_per_page - Move extent data per page
245 * @o_filp: file structure of original file
246 * @donor_inode: donor inode
247 * @orig_page_offset: page index on original file
248 * @donor_page_offset: page index on donor file
249 * @data_offset_in_page: block index where data swapping starts
250 * @block_len_in_page: the number of blocks to be swapped
251 * @unwritten: orig extent is unwritten or not
252 * @err: pointer to save return value
254 * Save the data in original inode blocks and replace original inode extents
255 * with donor inode extents by calling ext4_swap_extents().
256 * Finally, write out the saved data in new original inode blocks. Return
257 * replaced block count.
260 move_extent_per_page(struct file
*o_filp
, struct inode
*donor_inode
,
261 pgoff_t orig_page_offset
, pgoff_t donor_page_offset
,
262 int data_offset_in_page
,
263 int block_len_in_page
, int unwritten
, int *err
)
265 struct inode
*orig_inode
= file_inode(o_filp
);
266 struct page
*pagep
[2] = {NULL
, NULL
};
268 ext4_lblk_t orig_blk_offset
, donor_blk_offset
;
269 unsigned long blocksize
= orig_inode
->i_sb
->s_blocksize
;
270 unsigned int w_flags
= 0;
271 unsigned int tmp_data_size
, data_size
, replaced_size
;
272 int err2
, jblocks
, retries
= 0;
273 int replaced_count
= 0;
274 int from
= data_offset_in_page
<< orig_inode
->i_blkbits
;
275 int blocks_per_page
= PAGE_CACHE_SIZE
>> orig_inode
->i_blkbits
;
278 * It needs twice the amount of ordinary journal buffers because
279 * inode and donor_inode may change each different metadata blocks.
283 jblocks
= ext4_writepage_trans_blocks(orig_inode
) * 2;
284 handle
= ext4_journal_start(orig_inode
, EXT4_HT_MOVE_EXTENTS
, jblocks
);
285 if (IS_ERR(handle
)) {
286 *err
= PTR_ERR(handle
);
290 if (segment_eq(get_fs(), KERNEL_DS
))
291 w_flags
|= AOP_FLAG_UNINTERRUPTIBLE
;
293 orig_blk_offset
= orig_page_offset
* blocks_per_page
+
296 donor_blk_offset
= donor_page_offset
* blocks_per_page
+
299 /* Calculate data_size */
300 if ((orig_blk_offset
+ block_len_in_page
- 1) ==
301 ((orig_inode
->i_size
- 1) >> orig_inode
->i_blkbits
)) {
302 /* Replace the last block */
303 tmp_data_size
= orig_inode
->i_size
& (blocksize
- 1);
305 * If data_size equal zero, it shows data_size is multiples of
306 * blocksize. So we set appropriate value.
308 if (tmp_data_size
== 0)
309 tmp_data_size
= blocksize
;
311 data_size
= tmp_data_size
+
312 ((block_len_in_page
- 1) << orig_inode
->i_blkbits
);
314 data_size
= block_len_in_page
<< orig_inode
->i_blkbits
;
316 replaced_size
= data_size
;
318 *err
= mext_page_double_lock(orig_inode
, donor_inode
, orig_page_offset
,
319 donor_page_offset
, pagep
);
320 if (unlikely(*err
< 0))
323 * If orig extent was unwritten it can become initialized
324 * at any time after i_data_sem was dropped, in order to
325 * serialize with delalloc we have recheck extent while we
326 * hold page's lock, if it is still the case data copy is not
327 * necessary, just swap data blocks between orig and donor.
330 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
331 /* If any of extents in range became initialized we have to
332 * fallback to data copying */
333 unwritten
= mext_check_coverage(orig_inode
, orig_blk_offset
,
334 block_len_in_page
, 1, err
);
338 unwritten
&= mext_check_coverage(donor_inode
, donor_blk_offset
,
339 block_len_in_page
, 1, err
);
344 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
347 if ((page_has_private(pagep
[0]) &&
348 !try_to_release_page(pagep
[0], 0)) ||
349 (page_has_private(pagep
[1]) &&
350 !try_to_release_page(pagep
[1], 0))) {
354 replaced_count
= ext4_swap_extents(handle
, orig_inode
,
355 donor_inode
, orig_blk_offset
,
357 block_len_in_page
, 1, err
);
359 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
363 *err
= mext_page_mkuptodate(pagep
[0], from
, from
+ replaced_size
);
367 /* At this point all buffers in range are uptodate, old mapping layout
368 * is no longer required, try to drop it now. */
369 if ((page_has_private(pagep
[0]) && !try_to_release_page(pagep
[0], 0)) ||
370 (page_has_private(pagep
[1]) && !try_to_release_page(pagep
[1], 0))) {
374 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
375 replaced_count
= ext4_swap_extents(handle
, orig_inode
, donor_inode
,
376 orig_blk_offset
, donor_blk_offset
,
377 block_len_in_page
, 1, err
);
378 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
380 if (replaced_count
) {
381 block_len_in_page
= replaced_count
;
383 block_len_in_page
<< orig_inode
->i_blkbits
;
387 /* Perform all necessary steps similar write_begin()/write_end()
388 * but keeping in mind that i_size will not change */
389 *err
= __block_write_begin(pagep
[0], from
, replaced_size
,
392 *err
= block_commit_write(pagep
[0], from
, from
+ replaced_size
);
394 if (unlikely(*err
< 0))
395 goto repair_branches
;
397 /* Even in case of data=writeback it is reasonable to pin
398 * inode to transaction, to prevent unexpected data loss */
399 *err
= ext4_jbd2_file_inode(handle
, orig_inode
);
402 unlock_page(pagep
[0]);
403 page_cache_release(pagep
[0]);
404 unlock_page(pagep
[1]);
405 page_cache_release(pagep
[1]);
407 ext4_journal_stop(handle
);
408 /* Buffer was busy because probably is pinned to journal transaction,
409 * force transaction commit may help to free it. */
410 if (*err
== -EBUSY
&& ext4_should_retry_alloc(orig_inode
->i_sb
,
413 return replaced_count
;
417 * This should never ever happen!
418 * Extents are swapped already, but we are not able to copy data.
419 * Try to swap extents to it's original places
421 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
422 replaced_count
= ext4_swap_extents(handle
, donor_inode
, orig_inode
,
423 orig_blk_offset
, donor_blk_offset
,
424 block_len_in_page
, 0, &err2
);
425 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
426 if (replaced_count
!= block_len_in_page
) {
427 EXT4_ERROR_INODE_BLOCK(orig_inode
, (sector_t
)(orig_blk_offset
),
428 "Unable to copy data block,"
429 " data will be lost.");
437 * mext_check_arguments - Check whether move extent can be done
439 * @orig_inode: original inode
440 * @donor_inode: donor inode
441 * @orig_start: logical start offset in block for orig
442 * @donor_start: logical start offset in block for donor
443 * @len: the number of blocks to be moved
445 * Check the arguments of ext4_move_extents() whether the files can be
446 * exchanged with each other.
447 * Return 0 on success, or a negative error value on failure.
450 mext_check_arguments(struct inode
*orig_inode
,
451 struct inode
*donor_inode
, __u64 orig_start
,
452 __u64 donor_start
, __u64
*len
)
454 __u64 orig_eof
, donor_eof
;
455 unsigned int blkbits
= orig_inode
->i_blkbits
;
456 unsigned int blocksize
= 1 << blkbits
;
458 orig_eof
= (i_size_read(orig_inode
) + blocksize
- 1) >> blkbits
;
459 donor_eof
= (i_size_read(donor_inode
) + blocksize
- 1) >> blkbits
;
462 if (donor_inode
->i_mode
& (S_ISUID
|S_ISGID
)) {
463 ext4_debug("ext4 move extent: suid or sgid is set"
464 " to donor file [ino:orig %lu, donor %lu]\n",
465 orig_inode
->i_ino
, donor_inode
->i_ino
);
469 if (IS_IMMUTABLE(donor_inode
) || IS_APPEND(donor_inode
))
472 /* Ext4 move extent does not support swapfile */
473 if (IS_SWAPFILE(orig_inode
) || IS_SWAPFILE(donor_inode
)) {
474 ext4_debug("ext4 move extent: The argument files should "
475 "not be swapfile [ino:orig %lu, donor %lu]\n",
476 orig_inode
->i_ino
, donor_inode
->i_ino
);
480 /* Ext4 move extent supports only extent based file */
481 if (!(ext4_test_inode_flag(orig_inode
, EXT4_INODE_EXTENTS
))) {
482 ext4_debug("ext4 move extent: orig file is not extents "
483 "based file [ino:orig %lu]\n", orig_inode
->i_ino
);
485 } else if (!(ext4_test_inode_flag(donor_inode
, EXT4_INODE_EXTENTS
))) {
486 ext4_debug("ext4 move extent: donor file is not extents "
487 "based file [ino:donor %lu]\n", donor_inode
->i_ino
);
491 if ((!orig_inode
->i_size
) || (!donor_inode
->i_size
)) {
492 ext4_debug("ext4 move extent: File size is 0 byte\n");
496 /* Start offset should be same */
497 if ((orig_start
& ~(PAGE_MASK
>> orig_inode
->i_blkbits
)) !=
498 (donor_start
& ~(PAGE_MASK
>> orig_inode
->i_blkbits
))) {
499 ext4_debug("ext4 move extent: orig and donor's start "
500 "offset are not alligned [ino:orig %lu, donor %lu]\n",
501 orig_inode
->i_ino
, donor_inode
->i_ino
);
505 if ((orig_start
>= EXT_MAX_BLOCKS
) ||
506 (donor_start
>= EXT_MAX_BLOCKS
) ||
507 (*len
> EXT_MAX_BLOCKS
) ||
508 (donor_start
+ *len
>= EXT_MAX_BLOCKS
) ||
509 (orig_start
+ *len
>= EXT_MAX_BLOCKS
)) {
510 ext4_debug("ext4 move extent: Can't handle over [%u] blocks "
511 "[ino:orig %lu, donor %lu]\n", EXT_MAX_BLOCKS
,
512 orig_inode
->i_ino
, donor_inode
->i_ino
);
515 if (orig_eof
< orig_start
+ *len
- 1)
516 *len
= orig_eof
- orig_start
;
517 if (donor_eof
< donor_start
+ *len
- 1)
518 *len
= donor_eof
- donor_start
;
520 ext4_debug("ext4 move extent: len should not be 0 "
521 "[ino:orig %lu, donor %lu]\n", orig_inode
->i_ino
,
530 * ext4_move_extents - Exchange the specified range of a file
532 * @o_filp: file structure of the original file
533 * @d_filp: file structure of the donor file
534 * @orig_blk: start offset in block for orig
535 * @donor_blk: start offset in block for donor
536 * @len: the number of blocks to be moved
537 * @moved_len: moved block length
539 * This function returns 0 and moved block length is set in moved_len
540 * if succeed, otherwise returns error value.
544 ext4_move_extents(struct file
*o_filp
, struct file
*d_filp
, __u64 orig_blk
,
545 __u64 donor_blk
, __u64 len
, __u64
*moved_len
)
547 struct inode
*orig_inode
= file_inode(o_filp
);
548 struct inode
*donor_inode
= file_inode(d_filp
);
549 struct ext4_ext_path
*path
= NULL
;
550 int blocks_per_page
= PAGE_CACHE_SIZE
>> orig_inode
->i_blkbits
;
551 ext4_lblk_t o_end
, o_start
= orig_blk
;
552 ext4_lblk_t d_start
= donor_blk
;
555 if (orig_inode
->i_sb
!= donor_inode
->i_sb
) {
556 ext4_debug("ext4 move extent: The argument files "
557 "should be in same FS [ino:orig %lu, donor %lu]\n",
558 orig_inode
->i_ino
, donor_inode
->i_ino
);
562 /* orig and donor should be different inodes */
563 if (orig_inode
== donor_inode
) {
564 ext4_debug("ext4 move extent: The argument files should not "
565 "be same inode [ino:orig %lu, donor %lu]\n",
566 orig_inode
->i_ino
, donor_inode
->i_ino
);
570 /* Regular file check */
571 if (!S_ISREG(orig_inode
->i_mode
) || !S_ISREG(donor_inode
->i_mode
)) {
572 ext4_debug("ext4 move extent: The argument files should be "
573 "regular file [ino:orig %lu, donor %lu]\n",
574 orig_inode
->i_ino
, donor_inode
->i_ino
);
577 /* TODO: This is non obvious task to swap blocks for inodes with full
579 if (ext4_should_journal_data(orig_inode
) ||
580 ext4_should_journal_data(donor_inode
)) {
583 /* Protect orig and donor inodes against a truncate */
584 lock_two_nondirectories(orig_inode
, donor_inode
);
586 /* Wait for all existing dio workers */
587 ext4_inode_block_unlocked_dio(orig_inode
);
588 ext4_inode_block_unlocked_dio(donor_inode
);
589 inode_dio_wait(orig_inode
);
590 inode_dio_wait(donor_inode
);
592 /* Protect extent tree against block allocations via delalloc */
593 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
594 /* Check the filesystem environment whether move_extent can be done */
595 ret
= mext_check_arguments(orig_inode
, donor_inode
, orig_blk
,
599 o_end
= o_start
+ len
;
601 while (o_start
< o_end
) {
602 struct ext4_extent
*ex
;
603 ext4_lblk_t cur_blk
, next_blk
;
604 pgoff_t orig_page_index
, donor_page_index
;
606 int unwritten
, cur_len
;
608 ret
= get_ext_path(orig_inode
, o_start
, &path
);
611 ex
= path
[path
->p_depth
].p_ext
;
612 next_blk
= ext4_ext_next_allocated_block(path
);
613 cur_blk
= le32_to_cpu(ex
->ee_block
);
614 cur_len
= ext4_ext_get_actual_len(ex
);
615 /* Check hole before the start pos */
616 if (cur_blk
+ cur_len
- 1 < o_start
) {
617 if (next_blk
== EXT_MAX_BLOCKS
) {
622 d_start
+= next_blk
- o_start
;
625 /* Check hole after the start pos */
626 } else if (cur_blk
> o_start
) {
628 d_start
+= cur_blk
- o_start
;
630 /* Extent inside requested range ?*/
631 if (cur_blk
>= o_end
)
633 } else { /* in_range(o_start, o_blk, o_len) */
634 cur_len
+= cur_blk
- o_start
;
636 unwritten
= ext4_ext_is_unwritten(ex
);
637 if (o_end
- o_start
< cur_len
)
638 cur_len
= o_end
- o_start
;
640 orig_page_index
= o_start
>> (PAGE_CACHE_SHIFT
-
641 orig_inode
->i_blkbits
);
642 donor_page_index
= d_start
>> (PAGE_CACHE_SHIFT
-
643 donor_inode
->i_blkbits
);
644 offset_in_page
= o_start
% blocks_per_page
;
645 if (cur_len
> blocks_per_page
- offset_in_page
)
646 cur_len
= blocks_per_page
- offset_in_page
;
648 * Up semaphore to avoid following problems:
649 * a. transaction deadlock among ext4_journal_start,
650 * ->write_begin via pagefault, and jbd2_journal_commit
651 * b. racing with ->readpage, ->write_begin, and ext4_get_block
652 * in move_extent_per_page
654 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
655 /* Swap original branches with new branches */
656 move_extent_per_page(o_filp
, donor_inode
,
657 orig_page_index
, donor_page_index
,
658 offset_in_page
, cur_len
,
660 ext4_double_down_write_data_sem(orig_inode
, donor_inode
);
666 *moved_len
= o_start
- orig_blk
;
667 if (*moved_len
> len
)
672 ext4_discard_preallocations(orig_inode
);
673 ext4_discard_preallocations(donor_inode
);
676 ext4_ext_drop_refs(path
);
678 ext4_double_up_write_data_sem(orig_inode
, donor_inode
);
679 ext4_inode_resume_unlocked_dio(orig_inode
);
680 ext4_inode_resume_unlocked_dio(donor_inode
);
681 unlock_two_nondirectories(orig_inode
, donor_inode
);