1 // SPDX-License-Identifier: GPL-2.0
5 * Copyright (C) 1992, 1993, 1994, 1995
6 * Remy Card (card@masi.ibp.fr)
7 * Laboratoire MASI - Institut Blaise Pascal
8 * Universite Pierre et Marie Curie (Paris VI)
12 * linux/fs/minix/file.c
14 * Copyright (C) 1991, 1992 Linus Torvalds
16 * ext4 fs regular file handling primitives
18 * 64-bit file support on 64-bit platforms by Jakub Jelinek
19 * (jj@sunsite.ms.mff.cuni.cz)
22 #include <linux/time.h>
24 #include <linux/iomap.h>
25 #include <linux/mount.h>
26 #include <linux/path.h>
27 #include <linux/dax.h>
28 #include <linux/quotaops.h>
29 #include <linux/pagevec.h>
30 #include <linux/uio.h>
31 #include <linux/mman.h>
33 #include "ext4_jbd2.h"
38 static ssize_t
ext4_dax_read_iter(struct kiocb
*iocb
, struct iov_iter
*to
)
40 struct inode
*inode
= file_inode(iocb
->ki_filp
);
43 if (iocb
->ki_flags
& IOCB_NOWAIT
) {
44 if (!inode_trylock_shared(inode
))
47 inode_lock_shared(inode
);
50 * Recheck under inode lock - at this point we are sure it cannot
54 inode_unlock_shared(inode
);
55 /* Fallback to buffered IO in case we cannot support DAX */
56 return generic_file_read_iter(iocb
, to
);
58 ret
= dax_iomap_rw(iocb
, to
, &ext4_iomap_ops
);
59 inode_unlock_shared(inode
);
61 file_accessed(iocb
->ki_filp
);
66 static ssize_t
ext4_file_read_iter(struct kiocb
*iocb
, struct iov_iter
*to
)
68 if (unlikely(ext4_forced_shutdown(EXT4_SB(file_inode(iocb
->ki_filp
)->i_sb
))))
71 if (!iov_iter_count(to
))
72 return 0; /* skip atime */
75 if (IS_DAX(file_inode(iocb
->ki_filp
)))
76 return ext4_dax_read_iter(iocb
, to
);
78 return generic_file_read_iter(iocb
, to
);
82 * Called when an inode is released. Note that this is different
83 * from ext4_file_open: open gets called at every open, but release
84 * gets called only when /all/ the files are closed.
86 static int ext4_release_file(struct inode
*inode
, struct file
*filp
)
88 if (ext4_test_inode_state(inode
, EXT4_STATE_DA_ALLOC_CLOSE
)) {
89 ext4_alloc_da_blocks(inode
);
90 ext4_clear_inode_state(inode
, EXT4_STATE_DA_ALLOC_CLOSE
);
92 /* if we are the last writer on the inode, drop the block reservation */
93 if ((filp
->f_mode
& FMODE_WRITE
) &&
94 (atomic_read(&inode
->i_writecount
) == 1) &&
95 !EXT4_I(inode
)->i_reserved_data_blocks
)
97 down_write(&EXT4_I(inode
)->i_data_sem
);
98 ext4_discard_preallocations(inode
);
99 up_write(&EXT4_I(inode
)->i_data_sem
);
101 if (is_dx(inode
) && filp
->private_data
)
102 ext4_htree_free_dir_info(filp
->private_data
);
107 static void ext4_unwritten_wait(struct inode
*inode
)
109 wait_queue_head_t
*wq
= ext4_ioend_wq(inode
);
111 wait_event(*wq
, (atomic_read(&EXT4_I(inode
)->i_unwritten
) == 0));
115 * This tests whether the IO in question is block-aligned or not.
116 * Ext4 utilizes unwritten extents when hole-filling during direct IO, and they
117 * are converted to written only after the IO is complete. Until they are
118 * mapped, these blocks appear as holes, so dio_zero_block() will assume that
119 * it needs to zero out portions of the start and/or end block. If 2 AIO
120 * threads are at work on the same unwritten block, they must be synchronized
121 * or one thread will zero the other's data, causing corruption.
124 ext4_unaligned_aio(struct inode
*inode
, struct iov_iter
*from
, loff_t pos
)
126 struct super_block
*sb
= inode
->i_sb
;
127 int blockmask
= sb
->s_blocksize
- 1;
129 if (pos
>= ALIGN(i_size_read(inode
), sb
->s_blocksize
))
132 if ((pos
| iov_iter_alignment(from
)) & blockmask
)
138 /* Is IO overwriting allocated and initialized blocks? */
139 static bool ext4_overwrite_io(struct inode
*inode
, loff_t pos
, loff_t len
)
141 struct ext4_map_blocks map
;
142 unsigned int blkbits
= inode
->i_blkbits
;
145 if (pos
+ len
> i_size_read(inode
))
148 map
.m_lblk
= pos
>> blkbits
;
149 map
.m_len
= EXT4_MAX_BLOCKS(len
, pos
, blkbits
);
152 err
= ext4_map_blocks(NULL
, inode
, &map
, 0);
154 * 'err==len' means that all of the blocks have been preallocated,
155 * regardless of whether they have been initialized or not. To exclude
156 * unwritten extents, we need to check m_flags.
158 return err
== blklen
&& (map
.m_flags
& EXT4_MAP_MAPPED
);
161 static ssize_t
ext4_write_checks(struct kiocb
*iocb
, struct iov_iter
*from
)
163 struct inode
*inode
= file_inode(iocb
->ki_filp
);
166 ret
= generic_write_checks(iocb
, from
);
170 if (unlikely(IS_IMMUTABLE(inode
)))
174 * If we have encountered a bitmap-format file, the size limit
175 * is smaller than s_maxbytes, which is for extent-mapped files.
177 if (!(ext4_test_inode_flag(inode
, EXT4_INODE_EXTENTS
))) {
178 struct ext4_sb_info
*sbi
= EXT4_SB(inode
->i_sb
);
180 if (iocb
->ki_pos
>= sbi
->s_bitmap_maxbytes
)
182 iov_iter_truncate(from
, sbi
->s_bitmap_maxbytes
- iocb
->ki_pos
);
184 return iov_iter_count(from
);
189 ext4_dax_write_iter(struct kiocb
*iocb
, struct iov_iter
*from
)
191 struct inode
*inode
= file_inode(iocb
->ki_filp
);
194 if (iocb
->ki_flags
& IOCB_NOWAIT
) {
195 if (!inode_trylock(inode
))
200 ret
= ext4_write_checks(iocb
, from
);
203 ret
= file_remove_privs(iocb
->ki_filp
);
206 ret
= file_update_time(iocb
->ki_filp
);
210 ret
= dax_iomap_rw(iocb
, from
, &ext4_iomap_ops
);
214 ret
= generic_write_sync(iocb
, ret
);
220 ext4_file_write_iter(struct kiocb
*iocb
, struct iov_iter
*from
)
222 struct inode
*inode
= file_inode(iocb
->ki_filp
);
223 int o_direct
= iocb
->ki_flags
& IOCB_DIRECT
;
224 int unaligned_aio
= 0;
228 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode
->i_sb
))))
233 return ext4_dax_write_iter(iocb
, from
);
235 if (!o_direct
&& (iocb
->ki_flags
& IOCB_NOWAIT
))
238 if (!inode_trylock(inode
)) {
239 if (iocb
->ki_flags
& IOCB_NOWAIT
)
244 ret
= ext4_write_checks(iocb
, from
);
249 * Unaligned direct AIO must be serialized among each other as zeroing
250 * of partial blocks of two competing unaligned AIOs can result in data
253 if (o_direct
&& ext4_test_inode_flag(inode
, EXT4_INODE_EXTENTS
) &&
254 !is_sync_kiocb(iocb
) &&
255 ext4_unaligned_aio(inode
, from
, iocb
->ki_pos
)) {
257 ext4_unwritten_wait(inode
);
260 iocb
->private = &overwrite
;
261 /* Check whether we do a DIO overwrite or not */
262 if (o_direct
&& !unaligned_aio
) {
263 if (ext4_overwrite_io(inode
, iocb
->ki_pos
, iov_iter_count(from
))) {
264 if (ext4_should_dioread_nolock(inode
))
266 } else if (iocb
->ki_flags
& IOCB_NOWAIT
) {
272 ret
= __generic_file_write_iter(iocb
, from
);
274 * Unaligned direct AIO must be the only IO in flight. Otherwise
275 * overlapping aligned IO after unaligned might result in data
278 if (ret
== -EIOCBQUEUED
&& unaligned_aio
)
279 ext4_unwritten_wait(inode
);
283 ret
= generic_write_sync(iocb
, ret
);
293 static vm_fault_t
ext4_dax_huge_fault(struct vm_fault
*vmf
,
294 enum page_entry_size pe_size
)
299 handle_t
*handle
= NULL
;
300 struct inode
*inode
= file_inode(vmf
->vma
->vm_file
);
301 struct super_block
*sb
= inode
->i_sb
;
304 * We have to distinguish real writes from writes which will result in a
305 * COW page; COW writes should *not* poke the journal (the file will not
306 * be changed). Doing so would cause unintended failures when mounted
309 * We check for VM_SHARED rather than vmf->cow_page since the latter is
310 * unset for pe_size != PE_SIZE_PTE (i.e. only in do_cow_fault); for
311 * other sizes, dax_iomap_fault will handle splitting / fallback so that
312 * we eventually come back with a COW page.
314 bool write
= (vmf
->flags
& FAULT_FLAG_WRITE
) &&
315 (vmf
->vma
->vm_flags
& VM_SHARED
);
319 sb_start_pagefault(sb
);
320 file_update_time(vmf
->vma
->vm_file
);
321 down_read(&EXT4_I(inode
)->i_mmap_sem
);
323 handle
= ext4_journal_start_sb(sb
, EXT4_HT_WRITE_PAGE
,
324 EXT4_DATA_TRANS_BLOCKS(sb
));
325 if (IS_ERR(handle
)) {
326 up_read(&EXT4_I(inode
)->i_mmap_sem
);
327 sb_end_pagefault(sb
);
328 return VM_FAULT_SIGBUS
;
331 down_read(&EXT4_I(inode
)->i_mmap_sem
);
333 result
= dax_iomap_fault(vmf
, pe_size
, &pfn
, &error
, &ext4_iomap_ops
);
335 ext4_journal_stop(handle
);
337 if ((result
& VM_FAULT_ERROR
) && error
== -ENOSPC
&&
338 ext4_should_retry_alloc(sb
, &retries
))
340 /* Handling synchronous page fault? */
341 if (result
& VM_FAULT_NEEDDSYNC
)
342 result
= dax_finish_sync_fault(vmf
, pe_size
, pfn
);
343 up_read(&EXT4_I(inode
)->i_mmap_sem
);
344 sb_end_pagefault(sb
);
346 up_read(&EXT4_I(inode
)->i_mmap_sem
);
352 static vm_fault_t
ext4_dax_fault(struct vm_fault
*vmf
)
354 return ext4_dax_huge_fault(vmf
, PE_SIZE_PTE
);
357 static const struct vm_operations_struct ext4_dax_vm_ops
= {
358 .fault
= ext4_dax_fault
,
359 .huge_fault
= ext4_dax_huge_fault
,
360 .page_mkwrite
= ext4_dax_fault
,
361 .pfn_mkwrite
= ext4_dax_fault
,
364 #define ext4_dax_vm_ops ext4_file_vm_ops
367 static const struct vm_operations_struct ext4_file_vm_ops
= {
368 .fault
= ext4_filemap_fault
,
369 .map_pages
= filemap_map_pages
,
370 .page_mkwrite
= ext4_page_mkwrite
,
373 static int ext4_file_mmap(struct file
*file
, struct vm_area_struct
*vma
)
375 struct inode
*inode
= file
->f_mapping
->host
;
377 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode
->i_sb
))))
381 * We don't support synchronous mappings for non-DAX files. At least
382 * until someone comes with a sensible use case.
384 if (!IS_DAX(file_inode(file
)) && (vma
->vm_flags
& VM_SYNC
))
388 if (IS_DAX(file_inode(file
))) {
389 vma
->vm_ops
= &ext4_dax_vm_ops
;
390 vma
->vm_flags
|= VM_HUGEPAGE
;
392 vma
->vm_ops
= &ext4_file_vm_ops
;
397 static int ext4_sample_last_mounted(struct super_block
*sb
,
398 struct vfsmount
*mnt
)
400 struct ext4_sb_info
*sbi
= EXT4_SB(sb
);
406 if (likely(sbi
->s_mount_flags
& EXT4_MF_MNTDIR_SAMPLED
))
409 if (sb_rdonly(sb
) || !sb_start_intwrite_trylock(sb
))
412 sbi
->s_mount_flags
|= EXT4_MF_MNTDIR_SAMPLED
;
414 * Sample where the filesystem has been mounted and
415 * store it in the superblock for sysadmin convenience
416 * when trying to sort through large numbers of block
417 * devices or filesystem images.
419 memset(buf
, 0, sizeof(buf
));
421 path
.dentry
= mnt
->mnt_root
;
422 cp
= d_path(&path
, buf
, sizeof(buf
));
427 handle
= ext4_journal_start_sb(sb
, EXT4_HT_MISC
, 1);
428 err
= PTR_ERR(handle
);
431 BUFFER_TRACE(sbi
->s_sbh
, "get_write_access");
432 err
= ext4_journal_get_write_access(handle
, sbi
->s_sbh
);
435 strlcpy(sbi
->s_es
->s_last_mounted
, cp
,
436 sizeof(sbi
->s_es
->s_last_mounted
));
437 ext4_handle_dirty_super(handle
, sb
);
439 ext4_journal_stop(handle
);
445 static int ext4_file_open(struct inode
* inode
, struct file
* filp
)
449 if (unlikely(ext4_forced_shutdown(EXT4_SB(inode
->i_sb
))))
452 ret
= ext4_sample_last_mounted(inode
->i_sb
, filp
->f_path
.mnt
);
456 ret
= fscrypt_file_open(inode
, filp
);
461 * Set up the jbd2_inode if we are opening the inode for
462 * writing and the journal is present
464 if (filp
->f_mode
& FMODE_WRITE
) {
465 ret
= ext4_inode_attach_jinode(inode
);
470 filp
->f_mode
|= FMODE_NOWAIT
;
471 return dquot_file_open(inode
, filp
);
475 * ext4_llseek() handles both block-mapped and extent-mapped maxbytes values
476 * by calling generic_file_llseek_size() with the appropriate maxbytes
479 loff_t
ext4_llseek(struct file
*file
, loff_t offset
, int whence
)
481 struct inode
*inode
= file
->f_mapping
->host
;
484 if (!(ext4_test_inode_flag(inode
, EXT4_INODE_EXTENTS
)))
485 maxbytes
= EXT4_SB(inode
->i_sb
)->s_bitmap_maxbytes
;
487 maxbytes
= inode
->i_sb
->s_maxbytes
;
491 return generic_file_llseek_size(file
, offset
, whence
,
492 maxbytes
, i_size_read(inode
));
494 inode_lock_shared(inode
);
495 offset
= iomap_seek_hole(inode
, offset
, &ext4_iomap_ops
);
496 inode_unlock_shared(inode
);
499 inode_lock_shared(inode
);
500 offset
= iomap_seek_data(inode
, offset
, &ext4_iomap_ops
);
501 inode_unlock_shared(inode
);
507 return vfs_setpos(file
, offset
, maxbytes
);
510 const struct file_operations ext4_file_operations
= {
511 .llseek
= ext4_llseek
,
512 .read_iter
= ext4_file_read_iter
,
513 .write_iter
= ext4_file_write_iter
,
514 .unlocked_ioctl
= ext4_ioctl
,
516 .compat_ioctl
= ext4_compat_ioctl
,
518 .mmap
= ext4_file_mmap
,
519 .mmap_supported_flags
= MAP_SYNC
,
520 .open
= ext4_file_open
,
521 .release
= ext4_release_file
,
522 .fsync
= ext4_sync_file
,
523 .get_unmapped_area
= thp_get_unmapped_area
,
524 .splice_read
= generic_file_splice_read
,
525 .splice_write
= iter_file_splice_write
,
526 .fallocate
= ext4_fallocate
,
529 const struct inode_operations ext4_file_inode_operations
= {
530 .setattr
= ext4_setattr
,
531 .getattr
= ext4_file_getattr
,
532 .listxattr
= ext4_listxattr
,
533 .get_acl
= ext4_get_acl
,
534 .set_acl
= ext4_set_acl
,
535 .fiemap
= ext4_fiemap
,