6 * Partial copy of Linus' read cache modifications to fs/nfs/file.c
7 * modified for async RPC by okir@monad.swb.de
10 #include <linux/time.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/fcntl.h>
14 #include <linux/stat.h>
16 #include <linux/slab.h>
17 #include <linux/pagemap.h>
18 #include <linux/sunrpc/clnt.h>
19 #include <linux/nfs_fs.h>
20 #include <linux/nfs_page.h>
21 #include <linux/module.h>
29 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
31 static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops
;
32 static const struct nfs_rw_ops nfs_rw_read_ops
;
34 static struct kmem_cache
*nfs_rdata_cachep
;
36 static struct nfs_pgio_header
*nfs_readhdr_alloc(void)
38 return kmem_cache_zalloc(nfs_rdata_cachep
, GFP_KERNEL
);
41 static void nfs_readhdr_free(struct nfs_pgio_header
*rhdr
)
43 kmem_cache_free(nfs_rdata_cachep
, rhdr
);
47 int nfs_return_empty_page(struct page
*page
)
49 zero_user(page
, 0, PAGE_CACHE_SIZE
);
50 SetPageUptodate(page
);
55 void nfs_pageio_init_read(struct nfs_pageio_descriptor
*pgio
,
56 struct inode
*inode
, bool force_mds
,
57 const struct nfs_pgio_completion_ops
*compl_ops
)
59 struct nfs_server
*server
= NFS_SERVER(inode
);
60 const struct nfs_pageio_ops
*pg_ops
= &nfs_pgio_rw_ops
;
62 #ifdef CONFIG_NFS_V4_1
63 if (server
->pnfs_curr_ld
&& !force_mds
)
64 pg_ops
= server
->pnfs_curr_ld
->pg_read_ops
;
66 nfs_pageio_init(pgio
, inode
, pg_ops
, compl_ops
, &nfs_rw_read_ops
,
69 EXPORT_SYMBOL_GPL(nfs_pageio_init_read
);
71 void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor
*pgio
)
73 struct nfs_pgio_mirror
*mirror
;
75 if (pgio
->pg_ops
&& pgio
->pg_ops
->pg_cleanup
)
76 pgio
->pg_ops
->pg_cleanup(pgio
);
78 pgio
->pg_ops
= &nfs_pgio_rw_ops
;
80 /* read path should never have more than one mirror */
81 WARN_ON_ONCE(pgio
->pg_mirror_count
!= 1);
83 mirror
= &pgio
->pg_mirrors
[0];
84 mirror
->pg_bsize
= NFS_SERVER(pgio
->pg_inode
)->rsize
;
86 EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds
);
88 int nfs_readpage_async(struct nfs_open_context
*ctx
, struct inode
*inode
,
93 struct nfs_pageio_descriptor pgio
;
94 struct nfs_pgio_mirror
*pgm
;
96 len
= nfs_page_length(page
);
98 return nfs_return_empty_page(page
);
99 new = nfs_create_request(ctx
, page
, NULL
, 0, len
);
104 if (len
< PAGE_CACHE_SIZE
)
105 zero_user_segment(page
, len
, PAGE_CACHE_SIZE
);
107 nfs_pageio_init_read(&pgio
, inode
, false,
108 &nfs_async_read_completion_ops
);
109 nfs_pageio_add_request(&pgio
, new);
110 nfs_pageio_complete(&pgio
);
112 /* It doesn't make sense to do mirrored reads! */
113 WARN_ON_ONCE(pgio
.pg_mirror_count
!= 1);
115 pgm
= &pgio
.pg_mirrors
[0];
116 NFS_I(inode
)->read_io
+= pgm
->pg_bytes_written
;
118 return pgio
.pg_error
< 0 ? pgio
.pg_error
: 0;
121 static void nfs_readpage_release(struct nfs_page
*req
)
123 struct inode
*inode
= d_inode(req
->wb_context
->dentry
);
125 dprintk("NFS: read done (%s/%llu %d@%lld)\n", inode
->i_sb
->s_id
,
126 (unsigned long long)NFS_FILEID(inode
), req
->wb_bytes
,
127 (long long)req_offset(req
));
129 if (nfs_page_group_sync_on_bit(req
, PG_UNLOCKPAGE
)) {
130 if (PageUptodate(req
->wb_page
))
131 nfs_readpage_to_fscache(inode
, req
->wb_page
, 0);
133 unlock_page(req
->wb_page
);
135 nfs_release_request(req
);
138 static void nfs_page_group_set_uptodate(struct nfs_page
*req
)
140 if (nfs_page_group_sync_on_bit(req
, PG_UPTODATE
))
141 SetPageUptodate(req
->wb_page
);
144 static void nfs_read_completion(struct nfs_pgio_header
*hdr
)
146 unsigned long bytes
= 0;
148 if (test_bit(NFS_IOHDR_REDO
, &hdr
->flags
))
150 while (!list_empty(&hdr
->pages
)) {
151 struct nfs_page
*req
= nfs_list_entry(hdr
->pages
.next
);
152 struct page
*page
= req
->wb_page
;
153 unsigned long start
= req
->wb_pgbase
;
154 unsigned long end
= req
->wb_pgbase
+ req
->wb_bytes
;
156 if (test_bit(NFS_IOHDR_EOF
, &hdr
->flags
)) {
157 /* note: regions of the page not covered by a
158 * request are zeroed in nfs_readpage_async /
159 * readpage_async_filler */
160 if (bytes
> hdr
->good_bytes
) {
161 /* nothing in this request was good, so zero
162 * the full extent of the request */
163 zero_user_segment(page
, start
, end
);
165 } else if (hdr
->good_bytes
- bytes
< req
->wb_bytes
) {
166 /* part of this request has good bytes, but
167 * not all. zero the bad bytes */
168 start
+= hdr
->good_bytes
- bytes
;
169 WARN_ON(start
< req
->wb_pgbase
);
170 zero_user_segment(page
, start
, end
);
173 bytes
+= req
->wb_bytes
;
174 if (test_bit(NFS_IOHDR_ERROR
, &hdr
->flags
)) {
175 if (bytes
<= hdr
->good_bytes
)
176 nfs_page_group_set_uptodate(req
);
178 nfs_page_group_set_uptodate(req
);
179 nfs_list_remove_request(req
);
180 nfs_readpage_release(req
);
186 static void nfs_initiate_read(struct nfs_pgio_header
*hdr
,
187 struct rpc_message
*msg
,
188 const struct nfs_rpc_ops
*rpc_ops
,
189 struct rpc_task_setup
*task_setup_data
, int how
)
191 struct inode
*inode
= hdr
->inode
;
192 int swap_flags
= IS_SWAPFILE(inode
) ? NFS_RPC_SWAPFLAGS
: 0;
194 task_setup_data
->flags
|= swap_flags
;
195 rpc_ops
->read_setup(hdr
, msg
);
199 nfs_async_read_error(struct list_head
*head
)
201 struct nfs_page
*req
;
203 while (!list_empty(head
)) {
204 req
= nfs_list_entry(head
->next
);
205 nfs_list_remove_request(req
);
206 nfs_readpage_release(req
);
210 static const struct nfs_pgio_completion_ops nfs_async_read_completion_ops
= {
211 .error_cleanup
= nfs_async_read_error
,
212 .completion
= nfs_read_completion
,
216 * This is the callback from RPC telling us whether a reply was
217 * received or some error occurred (timeout or socket shutdown).
219 static int nfs_readpage_done(struct rpc_task
*task
,
220 struct nfs_pgio_header
*hdr
,
223 int status
= NFS_PROTO(inode
)->read_done(task
, hdr
);
227 nfs_add_stats(inode
, NFSIOS_SERVERREADBYTES
, hdr
->res
.count
);
229 if (task
->tk_status
== -ESTALE
) {
230 set_bit(NFS_INO_STALE
, &NFS_I(inode
)->flags
);
231 nfs_mark_for_revalidate(inode
);
236 static void nfs_readpage_retry(struct rpc_task
*task
,
237 struct nfs_pgio_header
*hdr
)
239 struct nfs_pgio_args
*argp
= &hdr
->args
;
240 struct nfs_pgio_res
*resp
= &hdr
->res
;
242 /* This is a short read! */
243 nfs_inc_stats(hdr
->inode
, NFSIOS_SHORTREAD
);
244 /* Has the server at least made some progress? */
245 if (resp
->count
== 0) {
246 nfs_set_pgio_error(hdr
, -EIO
, argp
->offset
);
250 /* For non rpc-based layout drivers, retry-through-MDS */
252 hdr
->pnfs_error
= -EAGAIN
;
256 /* Yes, so retry the read at the end of the hdr */
257 hdr
->mds_offset
+= resp
->count
;
258 argp
->offset
+= resp
->count
;
259 argp
->pgbase
+= resp
->count
;
260 argp
->count
-= resp
->count
;
261 rpc_restart_call_prepare(task
);
264 static void nfs_readpage_result(struct rpc_task
*task
,
265 struct nfs_pgio_header
*hdr
)
270 bound
= hdr
->args
.offset
+ hdr
->res
.count
;
271 spin_lock(&hdr
->lock
);
272 if (bound
< hdr
->io_start
+ hdr
->good_bytes
) {
273 set_bit(NFS_IOHDR_EOF
, &hdr
->flags
);
274 clear_bit(NFS_IOHDR_ERROR
, &hdr
->flags
);
275 hdr
->good_bytes
= bound
- hdr
->io_start
;
277 spin_unlock(&hdr
->lock
);
278 } else if (hdr
->res
.count
< hdr
->args
.count
)
279 nfs_readpage_retry(task
, hdr
);
283 * Read a page over NFS.
284 * We read the page synchronously in the following case:
285 * - The error flag is set for this page. This happens only when a
286 * previous async read operation failed.
288 int nfs_readpage(struct file
*file
, struct page
*page
)
290 struct nfs_open_context
*ctx
;
291 struct inode
*inode
= page_file_mapping(page
)->host
;
294 dprintk("NFS: nfs_readpage (%p %ld@%lu)\n",
295 page
, PAGE_CACHE_SIZE
, page_file_index(page
));
296 nfs_inc_stats(inode
, NFSIOS_VFSREADPAGE
);
297 nfs_add_stats(inode
, NFSIOS_READPAGES
, 1);
300 * Try to flush any pending writes to the file..
302 * NOTE! Because we own the page lock, there cannot
303 * be any new pending writes generated at this point
304 * for this page (other pages can be written to).
306 error
= nfs_wb_page(inode
, page
);
309 if (PageUptodate(page
))
313 if (NFS_STALE(inode
))
318 ctx
= nfs_find_open_context(inode
, NULL
, FMODE_READ
);
322 ctx
= get_nfs_open_context(nfs_file_open_context(file
));
324 if (!IS_SYNC(inode
)) {
325 error
= nfs_readpage_from_fscache(ctx
, inode
, page
);
330 error
= nfs_readpage_async(ctx
, inode
, page
);
333 put_nfs_open_context(ctx
);
340 struct nfs_readdesc
{
341 struct nfs_pageio_descriptor
*pgio
;
342 struct nfs_open_context
*ctx
;
346 readpage_async_filler(void *data
, struct page
*page
)
348 struct nfs_readdesc
*desc
= (struct nfs_readdesc
*)data
;
349 struct nfs_page
*new;
353 len
= nfs_page_length(page
);
355 return nfs_return_empty_page(page
);
357 new = nfs_create_request(desc
->ctx
, page
, NULL
, 0, len
);
361 if (len
< PAGE_CACHE_SIZE
)
362 zero_user_segment(page
, len
, PAGE_CACHE_SIZE
);
363 if (!nfs_pageio_add_request(desc
->pgio
, new)) {
364 error
= desc
->pgio
->pg_error
;
369 error
= PTR_ERR(new);
375 int nfs_readpages(struct file
*filp
, struct address_space
*mapping
,
376 struct list_head
*pages
, unsigned nr_pages
)
378 struct nfs_pageio_descriptor pgio
;
379 struct nfs_pgio_mirror
*pgm
;
380 struct nfs_readdesc desc
= {
383 struct inode
*inode
= mapping
->host
;
384 unsigned long npages
;
387 dprintk("NFS: nfs_readpages (%s/%Lu %d)\n",
389 (unsigned long long)NFS_FILEID(inode
),
391 nfs_inc_stats(inode
, NFSIOS_VFSREADPAGES
);
393 if (NFS_STALE(inode
))
397 desc
.ctx
= nfs_find_open_context(inode
, NULL
, FMODE_READ
);
398 if (desc
.ctx
== NULL
)
401 desc
.ctx
= get_nfs_open_context(nfs_file_open_context(filp
));
403 /* attempt to read as many of the pages as possible from the cache
404 * - this returns -ENOBUFS immediately if the cookie is negative
406 ret
= nfs_readpages_from_fscache(desc
.ctx
, inode
, mapping
,
409 goto read_complete
; /* all pages were read */
411 nfs_pageio_init_read(&pgio
, inode
, false,
412 &nfs_async_read_completion_ops
);
414 ret
= read_cache_pages(mapping
, pages
, readpage_async_filler
, &desc
);
415 nfs_pageio_complete(&pgio
);
417 /* It doesn't make sense to do mirrored reads! */
418 WARN_ON_ONCE(pgio
.pg_mirror_count
!= 1);
420 pgm
= &pgio
.pg_mirrors
[0];
421 NFS_I(inode
)->read_io
+= pgm
->pg_bytes_written
;
422 npages
= (pgm
->pg_bytes_written
+ PAGE_CACHE_SIZE
- 1) >>
424 nfs_add_stats(inode
, NFSIOS_READPAGES
, npages
);
426 put_nfs_open_context(desc
.ctx
);
431 int __init
nfs_init_readpagecache(void)
433 nfs_rdata_cachep
= kmem_cache_create("nfs_read_data",
434 sizeof(struct nfs_pgio_header
),
435 0, SLAB_HWCACHE_ALIGN
,
437 if (nfs_rdata_cachep
== NULL
)
443 void nfs_destroy_readpagecache(void)
445 kmem_cache_destroy(nfs_rdata_cachep
);
448 static const struct nfs_rw_ops nfs_rw_read_ops
= {
449 .rw_mode
= FMODE_READ
,
450 .rw_alloc_header
= nfs_readhdr_alloc
,
451 .rw_free_header
= nfs_readhdr_free
,
452 .rw_done
= nfs_readpage_done
,
453 .rw_result
= nfs_readpage_result
,
454 .rw_initiate
= nfs_initiate_read
,