?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
net: Clone skb before setting peeked flag
[linux/fpc-iii.git] / fs / fscache / page.c
blobde33b3fccca650da99ee850597d5aef437b3a6fe
1 /* Cache page management and data I/O routines
2 *
3 * Copyright (C) 2004-2008 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12 #define FSCACHE_DEBUG_LEVEL PAGE
13 #include <linux/module.h>
14 #include <linux/fscache-cache.h>
15 #include <linux/buffer_head.h>
16 #include <linux/pagevec.h>
17 #include <linux/slab.h>
18 #include "internal.h"
19
20 /*
21 * check to see if a page is being written to the cache
22 */
23 bool __fscache_check_page_write(struct fscache_cookie *cookie, struct page *page)
24 {
25 void *val;
26
27 rcu_read_lock();
28 val = radix_tree_lookup(&cookie->stores, page->index);
29 rcu_read_unlock();
30
31 return val != NULL;
32 }
33 EXPORT_SYMBOL(__fscache_check_page_write);
34
35 /*
36 * wait for a page to finish being written to the cache
37 */
38 void __fscache_wait_on_page_write(struct fscache_cookie *cookie, struct page *page)
39 {
40 wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
41
42 wait_event(*wq, !__fscache_check_page_write(cookie, page));
43 }
44 EXPORT_SYMBOL(__fscache_wait_on_page_write);
45
46 /*
47 * wait for a page to finish being written to the cache. Put a timeout here
48 * since we might be called recursively via parent fs.
49 */
50 static
51 bool release_page_wait_timeout(struct fscache_cookie *cookie, struct page *page)
52 {
53 wait_queue_head_t *wq = bit_waitqueue(&cookie->flags, 0);
54
55 return wait_event_timeout(*wq, !__fscache_check_page_write(cookie, page),
56 HZ);
57 }
58
59 /*
60 * decide whether a page can be released, possibly by cancelling a store to it
61 * - we're allowed to sleep if __GFP_WAIT is flagged
62 */
63 bool __fscache_maybe_release_page(struct fscache_cookie *cookie,
64 struct page *page,
65 gfp_t gfp)
66 {
67 struct page *xpage;
68 void *val;
69
70 _enter("%p,%p,%x", cookie, page, gfp);
71
72 try_again:
73 rcu_read_lock();
74 val = radix_tree_lookup(&cookie->stores, page->index);
75 if (!val) {
76 rcu_read_unlock();
77 fscache_stat(&fscache_n_store_vmscan_not_storing);
78 __fscache_uncache_page(cookie, page);
79 return true;
80 }
81
82 /* see if the page is actually undergoing storage - if so we can't get
83 * rid of it till the cache has finished with it */
84 if (radix_tree_tag_get(&cookie->stores, page->index,
85 FSCACHE_COOKIE_STORING_TAG)) {
86 rcu_read_unlock();
87 goto page_busy;
88 }
89
90 /* the page is pending storage, so we attempt to cancel the store and
91 * discard the store request so that the page can be reclaimed */
92 spin_lock(&cookie->stores_lock);
93 rcu_read_unlock();
94
95 if (radix_tree_tag_get(&cookie->stores, page->index,
96 FSCACHE_COOKIE_STORING_TAG)) {
97 /* the page started to undergo storage whilst we were looking,
98 * so now we can only wait or return */
99 spin_unlock(&cookie->stores_lock);
100 goto page_busy;
101 }
102
103 xpage = radix_tree_delete(&cookie->stores, page->index);
104 spin_unlock(&cookie->stores_lock);
105
106 if (xpage) {
107 fscache_stat(&fscache_n_store_vmscan_cancelled);
108 fscache_stat(&fscache_n_store_radix_deletes);
109 ASSERTCMP(xpage, ==, page);
110 } else {
111 fscache_stat(&fscache_n_store_vmscan_gone);
112 }
113
114 wake_up_bit(&cookie->flags, 0);
115 if (xpage)
116 page_cache_release(xpage);
117 __fscache_uncache_page(cookie, page);
118 return true;
119
120 page_busy:
121 /* We will wait here if we're allowed to, but that could deadlock the
122 * allocator as the work threads writing to the cache may all end up
123 * sleeping on memory allocation, so we may need to impose a timeout
124 * too. */
125 if (!(gfp & __GFP_WAIT) || !(gfp & __GFP_FS)) {
126 fscache_stat(&fscache_n_store_vmscan_busy);
127 return false;
128 }
129
130 fscache_stat(&fscache_n_store_vmscan_wait);
131 if (!release_page_wait_timeout(cookie, page))
132 _debug("fscache writeout timeout page: %p{%lx}",
133 page, page->index);
134
135 gfp &= ~__GFP_WAIT;
136 goto try_again;
137 }
138 EXPORT_SYMBOL(__fscache_maybe_release_page);
139
140 /*
141 * note that a page has finished being written to the cache
142 */
143 static void fscache_end_page_write(struct fscache_object *object,
144 struct page *page)
145 {
146 struct fscache_cookie *cookie;
147 struct page *xpage = NULL;
148
149 spin_lock(&object->lock);
150 cookie = object->cookie;
151 if (cookie) {
152 /* delete the page from the tree if it is now no longer
153 * pending */
154 spin_lock(&cookie->stores_lock);
155 radix_tree_tag_clear(&cookie->stores, page->index,
156 FSCACHE_COOKIE_STORING_TAG);
157 if (!radix_tree_tag_get(&cookie->stores, page->index,
158 FSCACHE_COOKIE_PENDING_TAG)) {
159 fscache_stat(&fscache_n_store_radix_deletes);
160 xpage = radix_tree_delete(&cookie->stores, page->index);
161 }
162 spin_unlock(&cookie->stores_lock);
163 wake_up_bit(&cookie->flags, 0);
164 }
165 spin_unlock(&object->lock);
166 if (xpage)
167 page_cache_release(xpage);
168 }
169
170 /*
171 * actually apply the changed attributes to a cache object
172 */
173 static void fscache_attr_changed_op(struct fscache_operation *op)
174 {
175 struct fscache_object *object = op->object;
176 int ret;
177
178 _enter("{OBJ%x OP%x}", object->debug_id, op->debug_id);
179
180 fscache_stat(&fscache_n_attr_changed_calls);
181
182 if (fscache_object_is_active(object)) {
183 fscache_stat(&fscache_n_cop_attr_changed);
184 ret = object->cache->ops->attr_changed(object);
185 fscache_stat_d(&fscache_n_cop_attr_changed);
186 if (ret < 0)
187 fscache_abort_object(object);
188 }
189
190 fscache_op_complete(op, true);
191 _leave("");
192 }
193
194 /*
195 * notification that the attributes on an object have changed
196 */
197 int __fscache_attr_changed(struct fscache_cookie *cookie)
198 {
199 struct fscache_operation *op;
200 struct fscache_object *object;
201 bool wake_cookie = false;
202
203 _enter("%p", cookie);
204
205 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
206
207 fscache_stat(&fscache_n_attr_changed);
208
209 op = kzalloc(sizeof(*op), GFP_KERNEL);
210 if (!op) {
211 fscache_stat(&fscache_n_attr_changed_nomem);
212 _leave(" = -ENOMEM");
213 return -ENOMEM;
214 }
215
216 fscache_operation_init(op, fscache_attr_changed_op, NULL);
217 op->flags = FSCACHE_OP_ASYNC |
218 (1 << FSCACHE_OP_EXCLUSIVE) |
219 (1 << FSCACHE_OP_UNUSE_COOKIE);
220
221 spin_lock(&cookie->lock);
222
223 if (!fscache_cookie_enabled(cookie) ||
224 hlist_empty(&cookie->backing_objects))
225 goto nobufs;
226 object = hlist_entry(cookie->backing_objects.first,
227 struct fscache_object, cookie_link);
228
229 __fscache_use_cookie(cookie);
230 if (fscache_submit_exclusive_op(object, op) < 0)
231 goto nobufs_dec;
232 spin_unlock(&cookie->lock);
233 fscache_stat(&fscache_n_attr_changed_ok);
234 fscache_put_operation(op);
235 _leave(" = 0");
236 return 0;
237
238 nobufs_dec:
239 wake_cookie = __fscache_unuse_cookie(cookie);
240 nobufs:
241 spin_unlock(&cookie->lock);
242 kfree(op);
243 if (wake_cookie)
244 __fscache_wake_unused_cookie(cookie);
245 fscache_stat(&fscache_n_attr_changed_nobufs);
246 _leave(" = %d", -ENOBUFS);
247 return -ENOBUFS;
248 }
249 EXPORT_SYMBOL(__fscache_attr_changed);
250
251 /*
252 * release a retrieval op reference
253 */
254 static void fscache_release_retrieval_op(struct fscache_operation *_op)
255 {
256 struct fscache_retrieval *op =
257 container_of(_op, struct fscache_retrieval, op);
258
259 _enter("{OP%x}", op->op.debug_id);
260
261 ASSERTCMP(atomic_read(&op->n_pages), ==, 0);
262
263 fscache_hist(fscache_retrieval_histogram, op->start_time);
264 if (op->context)
265 fscache_put_context(op->op.object->cookie, op->context);
266
267 _leave("");
268 }
269
270 /*
271 * allocate a retrieval op
272 */
273 static struct fscache_retrieval *fscache_alloc_retrieval(
274 struct fscache_cookie *cookie,
275 struct address_space *mapping,
276 fscache_rw_complete_t end_io_func,
277 void *context)
278 {
279 struct fscache_retrieval *op;
280
281 /* allocate a retrieval operation and attempt to submit it */
282 op = kzalloc(sizeof(*op), GFP_NOIO);
283 if (!op) {
284 fscache_stat(&fscache_n_retrievals_nomem);
285 return NULL;
286 }
287
288 fscache_operation_init(&op->op, NULL, fscache_release_retrieval_op);
289 op->op.flags = FSCACHE_OP_MYTHREAD |
290 (1UL << FSCACHE_OP_WAITING) |
291 (1UL << FSCACHE_OP_UNUSE_COOKIE);
292 op->mapping = mapping;
293 op->end_io_func = end_io_func;
294 op->context = context;
295 op->start_time = jiffies;
296 INIT_LIST_HEAD(&op->to_do);
297 return op;
298 }
299
300 /*
301 * wait for a deferred lookup to complete
302 */
303 int fscache_wait_for_deferred_lookup(struct fscache_cookie *cookie)
304 {
305 unsigned long jif;
306
307 _enter("");
308
309 if (!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags)) {
310 _leave(" = 0 [imm]");
311 return 0;
312 }
313
314 fscache_stat(&fscache_n_retrievals_wait);
315
316 jif = jiffies;
317 if (wait_on_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP,
318 TASK_INTERRUPTIBLE) != 0) {
319 fscache_stat(&fscache_n_retrievals_intr);
320 _leave(" = -ERESTARTSYS");
321 return -ERESTARTSYS;
322 }
323
324 ASSERT(!test_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags));
325
326 smp_rmb();
327 fscache_hist(fscache_retrieval_delay_histogram, jif);
328 _leave(" = 0 [dly]");
329 return 0;
330 }
331
332 /*
333 * Handle cancellation of a pending retrieval op
334 */
335 static void fscache_do_cancel_retrieval(struct fscache_operation *_op)
336 {
337 struct fscache_retrieval *op =
338 container_of(_op, struct fscache_retrieval, op);
339
340 atomic_set(&op->n_pages, 0);
341 }
342
343 /*
344 * wait for an object to become active (or dead)
345 */
346 int fscache_wait_for_operation_activation(struct fscache_object *object,
347 struct fscache_operation *op,
348 atomic_t *stat_op_waits,
349 atomic_t *stat_object_dead,
350 void (*do_cancel)(struct fscache_operation *))
351 {
352 int ret;
353
354 if (!test_bit(FSCACHE_OP_WAITING, &op->flags))
355 goto check_if_dead;
356
357 _debug(">>> WT");
358 if (stat_op_waits)
359 fscache_stat(stat_op_waits);
360 if (wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
361 TASK_INTERRUPTIBLE) != 0) {
362 ret = fscache_cancel_op(op, do_cancel);
363 if (ret == 0)
364 return -ERESTARTSYS;
365
366 /* it's been removed from the pending queue by another party,
367 * so we should get to run shortly */
368 wait_on_bit(&op->flags, FSCACHE_OP_WAITING,
369 TASK_UNINTERRUPTIBLE);
370 }
371 _debug("<<< GO");
372
373 check_if_dead:
374 if (op->state == FSCACHE_OP_ST_CANCELLED) {
375 if (stat_object_dead)
376 fscache_stat(stat_object_dead);
377 _leave(" = -ENOBUFS [cancelled]");
378 return -ENOBUFS;
379 }
380 if (unlikely(fscache_object_is_dead(object))) {
381 pr_err("%s() = -ENOBUFS [obj dead %d]\n", __func__, op->state);
382 fscache_cancel_op(op, do_cancel);
383 if (stat_object_dead)
384 fscache_stat(stat_object_dead);
385 return -ENOBUFS;
386 }
387 return 0;
388 }
389
390 /*
391 * read a page from the cache or allocate a block in which to store it
392 * - we return:
393 * -ENOMEM - out of memory, nothing done
394 * -ERESTARTSYS - interrupted
395 * -ENOBUFS - no backing object available in which to cache the block
396 * -ENODATA - no data available in the backing object for this block
397 * 0 - dispatched a read - it'll call end_io_func() when finished
398 */
399 int __fscache_read_or_alloc_page(struct fscache_cookie *cookie,
400 struct page *page,
401 fscache_rw_complete_t end_io_func,
402 void *context,
403 gfp_t gfp)
404 {
405 struct fscache_retrieval *op;
406 struct fscache_object *object;
407 bool wake_cookie = false;
408 int ret;
409
410 _enter("%p,%p,,,", cookie, page);
411
412 fscache_stat(&fscache_n_retrievals);
413
414 if (hlist_empty(&cookie->backing_objects))
415 goto nobufs;
416
417 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
418 _leave(" = -ENOBUFS [invalidating]");
419 return -ENOBUFS;
420 }
421
422 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
423 ASSERTCMP(page, !=, NULL);
424
425 if (fscache_wait_for_deferred_lookup(cookie) < 0)
426 return -ERESTARTSYS;
427
428 op = fscache_alloc_retrieval(cookie, page->mapping,
429 end_io_func, context);
430 if (!op) {
431 _leave(" = -ENOMEM");
432 return -ENOMEM;
433 }
434 atomic_set(&op->n_pages, 1);
435
436 spin_lock(&cookie->lock);
437
438 if (!fscache_cookie_enabled(cookie) ||
439 hlist_empty(&cookie->backing_objects))
440 goto nobufs_unlock;
441 object = hlist_entry(cookie->backing_objects.first,
442 struct fscache_object, cookie_link);
443
444 ASSERT(test_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags));
445
446 __fscache_use_cookie(cookie);
447 atomic_inc(&object->n_reads);
448 __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
449
450 if (fscache_submit_op(object, &op->op) < 0)
451 goto nobufs_unlock_dec;
452 spin_unlock(&cookie->lock);
453
454 fscache_stat(&fscache_n_retrieval_ops);
455
456 /* pin the netfs read context in case we need to do the actual netfs
457 * read because we've encountered a cache read failure */
458 fscache_get_context(object->cookie, op->context);
459
460 /* we wait for the operation to become active, and then process it
461 * *here*, in this thread, and not in the thread pool */
462 ret = fscache_wait_for_operation_activation(
463 object, &op->op,
464 __fscache_stat(&fscache_n_retrieval_op_waits),
465 __fscache_stat(&fscache_n_retrievals_object_dead),
466 fscache_do_cancel_retrieval);
467 if (ret < 0)
468 goto error;
469
470 /* ask the cache to honour the operation */
471 if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
472 fscache_stat(&fscache_n_cop_allocate_page);
473 ret = object->cache->ops->allocate_page(op, page, gfp);
474 fscache_stat_d(&fscache_n_cop_allocate_page);
475 if (ret == 0)
476 ret = -ENODATA;
477 } else {
478 fscache_stat(&fscache_n_cop_read_or_alloc_page);
479 ret = object->cache->ops->read_or_alloc_page(op, page, gfp);
480 fscache_stat_d(&fscache_n_cop_read_or_alloc_page);
481 }
482
483 error:
484 if (ret == -ENOMEM)
485 fscache_stat(&fscache_n_retrievals_nomem);
486 else if (ret == -ERESTARTSYS)
487 fscache_stat(&fscache_n_retrievals_intr);
488 else if (ret == -ENODATA)
489 fscache_stat(&fscache_n_retrievals_nodata);
490 else if (ret < 0)
491 fscache_stat(&fscache_n_retrievals_nobufs);
492 else
493 fscache_stat(&fscache_n_retrievals_ok);
494
495 fscache_put_retrieval(op);
496 _leave(" = %d", ret);
497 return ret;
498
499 nobufs_unlock_dec:
500 atomic_dec(&object->n_reads);
501 wake_cookie = __fscache_unuse_cookie(cookie);
502 nobufs_unlock:
503 spin_unlock(&cookie->lock);
504 if (wake_cookie)
505 __fscache_wake_unused_cookie(cookie);
506 kfree(op);
507 nobufs:
508 fscache_stat(&fscache_n_retrievals_nobufs);
509 _leave(" = -ENOBUFS");
510 return -ENOBUFS;
511 }
512 EXPORT_SYMBOL(__fscache_read_or_alloc_page);
513
514 /*
515 * read a list of page from the cache or allocate a block in which to store
516 * them
517 * - we return:
518 * -ENOMEM - out of memory, some pages may be being read
519 * -ERESTARTSYS - interrupted, some pages may be being read
520 * -ENOBUFS - no backing object or space available in which to cache any
521 * pages not being read
522 * -ENODATA - no data available in the backing object for some or all of
523 * the pages
524 * 0 - dispatched a read on all pages
525 *
526 * end_io_func() will be called for each page read from the cache as it is
527 * finishes being read
528 *
529 * any pages for which a read is dispatched will be removed from pages and
530 * nr_pages
531 */
532 int __fscache_read_or_alloc_pages(struct fscache_cookie *cookie,
533 struct address_space *mapping,
534 struct list_head *pages,
535 unsigned *nr_pages,
536 fscache_rw_complete_t end_io_func,
537 void *context,
538 gfp_t gfp)
539 {
540 struct fscache_retrieval *op;
541 struct fscache_object *object;
542 bool wake_cookie = false;
543 int ret;
544
545 _enter("%p,,%d,,,", cookie, *nr_pages);
546
547 fscache_stat(&fscache_n_retrievals);
548
549 if (hlist_empty(&cookie->backing_objects))
550 goto nobufs;
551
552 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
553 _leave(" = -ENOBUFS [invalidating]");
554 return -ENOBUFS;
555 }
556
557 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
558 ASSERTCMP(*nr_pages, >, 0);
559 ASSERT(!list_empty(pages));
560
561 if (fscache_wait_for_deferred_lookup(cookie) < 0)
562 return -ERESTARTSYS;
563
564 op = fscache_alloc_retrieval(cookie, mapping, end_io_func, context);
565 if (!op)
566 return -ENOMEM;
567 atomic_set(&op->n_pages, *nr_pages);
568
569 spin_lock(&cookie->lock);
570
571 if (!fscache_cookie_enabled(cookie) ||
572 hlist_empty(&cookie->backing_objects))
573 goto nobufs_unlock;
574 object = hlist_entry(cookie->backing_objects.first,
575 struct fscache_object, cookie_link);
576
577 __fscache_use_cookie(cookie);
578 atomic_inc(&object->n_reads);
579 __set_bit(FSCACHE_OP_DEC_READ_CNT, &op->op.flags);
580
581 if (fscache_submit_op(object, &op->op) < 0)
582 goto nobufs_unlock_dec;
583 spin_unlock(&cookie->lock);
584
585 fscache_stat(&fscache_n_retrieval_ops);
586
587 /* pin the netfs read context in case we need to do the actual netfs
588 * read because we've encountered a cache read failure */
589 fscache_get_context(object->cookie, op->context);
590
591 /* we wait for the operation to become active, and then process it
592 * *here*, in this thread, and not in the thread pool */
593 ret = fscache_wait_for_operation_activation(
594 object, &op->op,
595 __fscache_stat(&fscache_n_retrieval_op_waits),
596 __fscache_stat(&fscache_n_retrievals_object_dead),
597 fscache_do_cancel_retrieval);
598 if (ret < 0)
599 goto error;
600
601 /* ask the cache to honour the operation */
602 if (test_bit(FSCACHE_COOKIE_NO_DATA_YET, &object->cookie->flags)) {
603 fscache_stat(&fscache_n_cop_allocate_pages);
604 ret = object->cache->ops->allocate_pages(
605 op, pages, nr_pages, gfp);
606 fscache_stat_d(&fscache_n_cop_allocate_pages);
607 } else {
608 fscache_stat(&fscache_n_cop_read_or_alloc_pages);
609 ret = object->cache->ops->read_or_alloc_pages(
610 op, pages, nr_pages, gfp);
611 fscache_stat_d(&fscache_n_cop_read_or_alloc_pages);
612 }
613
614 error:
615 if (ret == -ENOMEM)
616 fscache_stat(&fscache_n_retrievals_nomem);
617 else if (ret == -ERESTARTSYS)
618 fscache_stat(&fscache_n_retrievals_intr);
619 else if (ret == -ENODATA)
620 fscache_stat(&fscache_n_retrievals_nodata);
621 else if (ret < 0)
622 fscache_stat(&fscache_n_retrievals_nobufs);
623 else
624 fscache_stat(&fscache_n_retrievals_ok);
625
626 fscache_put_retrieval(op);
627 _leave(" = %d", ret);
628 return ret;
629
630 nobufs_unlock_dec:
631 atomic_dec(&object->n_reads);
632 wake_cookie = __fscache_unuse_cookie(cookie);
633 nobufs_unlock:
634 spin_unlock(&cookie->lock);
635 kfree(op);
636 if (wake_cookie)
637 __fscache_wake_unused_cookie(cookie);
638 nobufs:
639 fscache_stat(&fscache_n_retrievals_nobufs);
640 _leave(" = -ENOBUFS");
641 return -ENOBUFS;
642 }
643 EXPORT_SYMBOL(__fscache_read_or_alloc_pages);
644
645 /*
646 * allocate a block in the cache on which to store a page
647 * - we return:
648 * -ENOMEM - out of memory, nothing done
649 * -ERESTARTSYS - interrupted
650 * -ENOBUFS - no backing object available in which to cache the block
651 * 0 - block allocated
652 */
653 int __fscache_alloc_page(struct fscache_cookie *cookie,
654 struct page *page,
655 gfp_t gfp)
656 {
657 struct fscache_retrieval *op;
658 struct fscache_object *object;
659 bool wake_cookie = false;
660 int ret;
661
662 _enter("%p,%p,,,", cookie, page);
663
664 fscache_stat(&fscache_n_allocs);
665
666 if (hlist_empty(&cookie->backing_objects))
667 goto nobufs;
668
669 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
670 ASSERTCMP(page, !=, NULL);
671
672 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
673 _leave(" = -ENOBUFS [invalidating]");
674 return -ENOBUFS;
675 }
676
677 if (fscache_wait_for_deferred_lookup(cookie) < 0)
678 return -ERESTARTSYS;
679
680 op = fscache_alloc_retrieval(cookie, page->mapping, NULL, NULL);
681 if (!op)
682 return -ENOMEM;
683 atomic_set(&op->n_pages, 1);
684
685 spin_lock(&cookie->lock);
686
687 if (!fscache_cookie_enabled(cookie) ||
688 hlist_empty(&cookie->backing_objects))
689 goto nobufs_unlock;
690 object = hlist_entry(cookie->backing_objects.first,
691 struct fscache_object, cookie_link);
692
693 __fscache_use_cookie(cookie);
694 if (fscache_submit_op(object, &op->op) < 0)
695 goto nobufs_unlock_dec;
696 spin_unlock(&cookie->lock);
697
698 fscache_stat(&fscache_n_alloc_ops);
699
700 ret = fscache_wait_for_operation_activation(
701 object, &op->op,
702 __fscache_stat(&fscache_n_alloc_op_waits),
703 __fscache_stat(&fscache_n_allocs_object_dead),
704 fscache_do_cancel_retrieval);
705 if (ret < 0)
706 goto error;
707
708 /* ask the cache to honour the operation */
709 fscache_stat(&fscache_n_cop_allocate_page);
710 ret = object->cache->ops->allocate_page(op, page, gfp);
711 fscache_stat_d(&fscache_n_cop_allocate_page);
712
713 error:
714 if (ret == -ERESTARTSYS)
715 fscache_stat(&fscache_n_allocs_intr);
716 else if (ret < 0)
717 fscache_stat(&fscache_n_allocs_nobufs);
718 else
719 fscache_stat(&fscache_n_allocs_ok);
720
721 fscache_put_retrieval(op);
722 _leave(" = %d", ret);
723 return ret;
724
725 nobufs_unlock_dec:
726 wake_cookie = __fscache_unuse_cookie(cookie);
727 nobufs_unlock:
728 spin_unlock(&cookie->lock);
729 kfree(op);
730 if (wake_cookie)
731 __fscache_wake_unused_cookie(cookie);
732 nobufs:
733 fscache_stat(&fscache_n_allocs_nobufs);
734 _leave(" = -ENOBUFS");
735 return -ENOBUFS;
736 }
737 EXPORT_SYMBOL(__fscache_alloc_page);
738
739 /*
740 * Unmark pages allocate in the readahead code path (via:
741 * fscache_readpages_or_alloc) after delegating to the base filesystem
742 */
743 void __fscache_readpages_cancel(struct fscache_cookie *cookie,
744 struct list_head *pages)
745 {
746 struct page *page;
747
748 list_for_each_entry(page, pages, lru) {
749 if (PageFsCache(page))
750 __fscache_uncache_page(cookie, page);
751 }
752 }
753 EXPORT_SYMBOL(__fscache_readpages_cancel);
754
755 /*
756 * release a write op reference
757 */
758 static void fscache_release_write_op(struct fscache_operation *_op)
759 {
760 _enter("{OP%x}", _op->debug_id);
761 }
762
763 /*
764 * perform the background storage of a page into the cache
765 */
766 static void fscache_write_op(struct fscache_operation *_op)
767 {
768 struct fscache_storage *op =
769 container_of(_op, struct fscache_storage, op);
770 struct fscache_object *object = op->op.object;
771 struct fscache_cookie *cookie;
772 struct page *page;
773 unsigned n;
774 void *results[1];
775 int ret;
776
777 _enter("{OP%x,%d}", op->op.debug_id, atomic_read(&op->op.usage));
778
779 spin_lock(&object->lock);
780 cookie = object->cookie;
781
782 if (!fscache_object_is_active(object)) {
783 /* If we get here, then the on-disk cache object likely longer
784 * exists, so we should just cancel this write operation.
785 */
786 spin_unlock(&object->lock);
787 fscache_op_complete(&op->op, false);
788 _leave(" [inactive]");
789 return;
790 }
791
792 if (!cookie) {
793 /* If we get here, then the cookie belonging to the object was
794 * detached, probably by the cookie being withdrawn due to
795 * memory pressure, which means that the pages we might write
796 * to the cache from no longer exist - therefore, we can just
797 * cancel this write operation.
798 */
799 spin_unlock(&object->lock);
800 fscache_op_complete(&op->op, false);
801 _leave(" [cancel] op{f=%lx s=%u} obj{s=%s f=%lx}",
802 _op->flags, _op->state, object->state->short_name,
803 object->flags);
804 return;
805 }
806
807 spin_lock(&cookie->stores_lock);
808
809 fscache_stat(&fscache_n_store_calls);
810
811 /* find a page to store */
812 page = NULL;
813 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0, 1,
814 FSCACHE_COOKIE_PENDING_TAG);
815 if (n != 1)
816 goto superseded;
817 page = results[0];
818 _debug("gang %d [%lx]", n, page->index);
819 if (page->index > op->store_limit) {
820 fscache_stat(&fscache_n_store_pages_over_limit);
821 goto superseded;
822 }
823
824 radix_tree_tag_set(&cookie->stores, page->index,
825 FSCACHE_COOKIE_STORING_TAG);
826 radix_tree_tag_clear(&cookie->stores, page->index,
827 FSCACHE_COOKIE_PENDING_TAG);
828
829 spin_unlock(&cookie->stores_lock);
830 spin_unlock(&object->lock);
831
832 fscache_stat(&fscache_n_store_pages);
833 fscache_stat(&fscache_n_cop_write_page);
834 ret = object->cache->ops->write_page(op, page);
835 fscache_stat_d(&fscache_n_cop_write_page);
836 fscache_end_page_write(object, page);
837 if (ret < 0) {
838 fscache_abort_object(object);
839 fscache_op_complete(&op->op, true);
840 } else {
841 fscache_enqueue_operation(&op->op);
842 }
843
844 _leave("");
845 return;
846
847 superseded:
848 /* this writer is going away and there aren't any more things to
849 * write */
850 _debug("cease");
851 spin_unlock(&cookie->stores_lock);
852 clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);
853 spin_unlock(&object->lock);
854 fscache_op_complete(&op->op, true);
855 _leave("");
856 }
857
858 /*
859 * Clear the pages pending writing for invalidation
860 */
861 void fscache_invalidate_writes(struct fscache_cookie *cookie)
862 {
863 struct page *page;
864 void *results[16];
865 int n, i;
866
867 _enter("");
868
869 for (;;) {
870 spin_lock(&cookie->stores_lock);
871 n = radix_tree_gang_lookup_tag(&cookie->stores, results, 0,
872 ARRAY_SIZE(results),
873 FSCACHE_COOKIE_PENDING_TAG);
874 if (n == 0) {
875 spin_unlock(&cookie->stores_lock);
876 break;
877 }
878
879 for (i = n - 1; i >= 0; i--) {
880 page = results[i];
881 radix_tree_delete(&cookie->stores, page->index);
882 }
883
884 spin_unlock(&cookie->stores_lock);
885
886 for (i = n - 1; i >= 0; i--)
887 page_cache_release(results[i]);
888 }
889
890 _leave("");
891 }
892
893 /*
894 * request a page be stored in the cache
895 * - returns:
896 * -ENOMEM - out of memory, nothing done
897 * -ENOBUFS - no backing object available in which to cache the page
898 * 0 - dispatched a write - it'll call end_io_func() when finished
899 *
900 * if the cookie still has a backing object at this point, that object can be
901 * in one of a few states with respect to storage processing:
902 *
903 * (1) negative lookup, object not yet created (FSCACHE_COOKIE_CREATING is
904 * set)
905 *
906 * (a) no writes yet
907 *
908 * (b) writes deferred till post-creation (mark page for writing and
909 * return immediately)
910 *
911 * (2) negative lookup, object created, initial fill being made from netfs
912 *
913 * (a) fill point not yet reached this page (mark page for writing and
914 * return)
915 *
916 * (b) fill point passed this page (queue op to store this page)
917 *
918 * (3) object extant (queue op to store this page)
919 *
920 * any other state is invalid
921 */
922 int __fscache_write_page(struct fscache_cookie *cookie,
923 struct page *page,
924 gfp_t gfp)
925 {
926 struct fscache_storage *op;
927 struct fscache_object *object;
928 bool wake_cookie = false;
929 int ret;
930
931 _enter("%p,%x,", cookie, (u32) page->flags);
932
933 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
934 ASSERT(PageFsCache(page));
935
936 fscache_stat(&fscache_n_stores);
937
938 if (test_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags)) {
939 _leave(" = -ENOBUFS [invalidating]");
940 return -ENOBUFS;
941 }
942
943 op = kzalloc(sizeof(*op), GFP_NOIO | __GFP_NOMEMALLOC | __GFP_NORETRY);
944 if (!op)
945 goto nomem;
946
947 fscache_operation_init(&op->op, fscache_write_op,
948 fscache_release_write_op);
949 op->op.flags = FSCACHE_OP_ASYNC |
950 (1 << FSCACHE_OP_WAITING) |
951 (1 << FSCACHE_OP_UNUSE_COOKIE);
952
953 ret = radix_tree_maybe_preload(gfp & ~__GFP_HIGHMEM);
954 if (ret < 0)
955 goto nomem_free;
956
957 ret = -ENOBUFS;
958 spin_lock(&cookie->lock);
959
960 if (!fscache_cookie_enabled(cookie) ||
961 hlist_empty(&cookie->backing_objects))
962 goto nobufs;
963 object = hlist_entry(cookie->backing_objects.first,
964 struct fscache_object, cookie_link);
965 if (test_bit(FSCACHE_IOERROR, &object->cache->flags))
966 goto nobufs;
967
968 /* add the page to the pending-storage radix tree on the backing
969 * object */
970 spin_lock(&object->lock);
971 spin_lock(&cookie->stores_lock);
972
973 _debug("store limit %llx", (unsigned long long) object->store_limit);
974
975 ret = radix_tree_insert(&cookie->stores, page->index, page);
976 if (ret < 0) {
977 if (ret == -EEXIST)
978 goto already_queued;
979 _debug("insert failed %d", ret);
980 goto nobufs_unlock_obj;
981 }
982
983 radix_tree_tag_set(&cookie->stores, page->index,
984 FSCACHE_COOKIE_PENDING_TAG);
985 page_cache_get(page);
986
987 /* we only want one writer at a time, but we do need to queue new
988 * writers after exclusive ops */
989 if (test_and_set_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags))
990 goto already_pending;
991
992 spin_unlock(&cookie->stores_lock);
993 spin_unlock(&object->lock);
994
995 op->op.debug_id = atomic_inc_return(&fscache_op_debug_id);
996 op->store_limit = object->store_limit;
997
998 __fscache_use_cookie(cookie);
999 if (fscache_submit_op(object, &op->op) < 0)
1000 goto submit_failed;
1001
1002 spin_unlock(&cookie->lock);
1003 radix_tree_preload_end();
1004 fscache_stat(&fscache_n_store_ops);
1005 fscache_stat(&fscache_n_stores_ok);
1006
1007 /* the work queue now carries its own ref on the object */
1008 fscache_put_operation(&op->op);
1009 _leave(" = 0");
1010 return 0;
1011
1012 already_queued:
1013 fscache_stat(&fscache_n_stores_again);
1014 already_pending:
1015 spin_unlock(&cookie->stores_lock);
1016 spin_unlock(&object->lock);
1017 spin_unlock(&cookie->lock);
1018 radix_tree_preload_end();
1019 kfree(op);
1020 fscache_stat(&fscache_n_stores_ok);
1021 _leave(" = 0");
1022 return 0;
1023
1024 submit_failed:
1025 spin_lock(&cookie->stores_lock);
1026 radix_tree_delete(&cookie->stores, page->index);
1027 spin_unlock(&cookie->stores_lock);
1028 wake_cookie = __fscache_unuse_cookie(cookie);
1029 page_cache_release(page);
1030 ret = -ENOBUFS;
1031 goto nobufs;
1032
1033 nobufs_unlock_obj:
1034 spin_unlock(&cookie->stores_lock);
1035 spin_unlock(&object->lock);
1036 nobufs:
1037 spin_unlock(&cookie->lock);
1038 radix_tree_preload_end();
1039 kfree(op);
1040 if (wake_cookie)
1041 __fscache_wake_unused_cookie(cookie);
1042 fscache_stat(&fscache_n_stores_nobufs);
1043 _leave(" = -ENOBUFS");
1044 return -ENOBUFS;
1045
1046 nomem_free:
1047 kfree(op);
1048 nomem:
1049 fscache_stat(&fscache_n_stores_oom);
1050 _leave(" = -ENOMEM");
1051 return -ENOMEM;
1052 }
1053 EXPORT_SYMBOL(__fscache_write_page);
1054
1055 /*
1056 * remove a page from the cache
1057 */
1058 void __fscache_uncache_page(struct fscache_cookie *cookie, struct page *page)
1059 {
1060 struct fscache_object *object;
1061
1062 _enter(",%p", page);
1063
1064 ASSERTCMP(cookie->def->type, !=, FSCACHE_COOKIE_TYPE_INDEX);
1065 ASSERTCMP(page, !=, NULL);
1066
1067 fscache_stat(&fscache_n_uncaches);
1068
1069 /* cache withdrawal may beat us to it */
1070 if (!PageFsCache(page))
1071 goto done;
1072
1073 /* get the object */
1074 spin_lock(&cookie->lock);
1075
1076 if (hlist_empty(&cookie->backing_objects)) {
1077 ClearPageFsCache(page);
1078 goto done_unlock;
1079 }
1080
1081 object = hlist_entry(cookie->backing_objects.first,
1082 struct fscache_object, cookie_link);
1083
1084 /* there might now be stuff on disk we could read */
1085 clear_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);
1086
1087 /* only invoke the cache backend if we managed to mark the page
1088 * uncached here; this deals with synchronisation vs withdrawal */
1089 if (TestClearPageFsCache(page) &&
1090 object->cache->ops->uncache_page) {
1091 /* the cache backend releases the cookie lock */
1092 fscache_stat(&fscache_n_cop_uncache_page);
1093 object->cache->ops->uncache_page(object, page);
1094 fscache_stat_d(&fscache_n_cop_uncache_page);
1095 goto done;
1096 }
1097
1098 done_unlock:
1099 spin_unlock(&cookie->lock);
1100 done:
1101 _leave("");
1102 }
1103 EXPORT_SYMBOL(__fscache_uncache_page);
1104
1105 /**
1106 * fscache_mark_page_cached - Mark a page as being cached
1107 * @op: The retrieval op pages are being marked for
1108 * @page: The page to be marked
1109 *
1110 * Mark a netfs page as being cached. After this is called, the netfs
1111 * must call fscache_uncache_page() to remove the mark.
1112 */
1113 void fscache_mark_page_cached(struct fscache_retrieval *op, struct page *page)
1114 {
1115 struct fscache_cookie *cookie = op->op.object->cookie;
1116
1117 #ifdef CONFIG_FSCACHE_STATS
1118 atomic_inc(&fscache_n_marks);
1119 #endif
1120
1121 _debug("- mark %p{%lx}", page, page->index);
1122 if (TestSetPageFsCache(page)) {
1123 static bool once_only;
1124 if (!once_only) {
1125 once_only = true;
1126 pr_warn("Cookie type %s marked page %lx multiple times\n",
1127 cookie->def->name, page->index);
1128 }
1129 }
1130
1131 if (cookie->def->mark_page_cached)
1132 cookie->def->mark_page_cached(cookie->netfs_data,
1133 op->mapping, page);
1134 }
1135 EXPORT_SYMBOL(fscache_mark_page_cached);
1136
1137 /**
1138 * fscache_mark_pages_cached - Mark pages as being cached
1139 * @op: The retrieval op pages are being marked for
1140 * @pagevec: The pages to be marked
1141 *
1142 * Mark a bunch of netfs pages as being cached. After this is called,
1143 * the netfs must call fscache_uncache_page() to remove the mark.
1144 */
1145 void fscache_mark_pages_cached(struct fscache_retrieval *op,
1146 struct pagevec *pagevec)
1147 {
1148 unsigned long loop;
1149
1150 for (loop = 0; loop < pagevec->nr; loop++)
1151 fscache_mark_page_cached(op, pagevec->pages[loop]);
1152
1153 pagevec_reinit(pagevec);
1154 }
1155 EXPORT_SYMBOL(fscache_mark_pages_cached);
1156
1157 /*
1158 * Uncache all the pages in an inode that are marked PG_fscache, assuming them
1159 * to be associated with the given cookie.
1160 */
1161 void __fscache_uncache_all_inode_pages(struct fscache_cookie *cookie,
1162 struct inode *inode)
1163 {
1164 struct address_space *mapping = inode->i_mapping;
1165 struct pagevec pvec;
1166 pgoff_t next;
1167 int i;
1168
1169 _enter("%p,%p", cookie, inode);
1170
1171 if (!mapping || mapping->nrpages == 0) {
1172 _leave(" [no pages]");
1173 return;
1174 }
1175
1176 pagevec_init(&pvec, 0);
1177 next = 0;
1178 do {
1179 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE))
1180 break;
1181 for (i = 0; i < pagevec_count(&pvec); i++) {
1182 struct page *page = pvec.pages[i];
1183 next = page->index;
1184 if (PageFsCache(page)) {
1185 __fscache_wait_on_page_write(cookie, page);
1186 __fscache_uncache_page(cookie, page);
1187 }
1188 }
1189 pagevec_release(&pvec);
1190 cond_resched();
1191 } while (++next);
1192
1193 _leave("");
1194 }
1195 EXPORT_SYMBOL(__fscache_uncache_all_inode_pages);
Linux with added FPC-III support