| blob | 1cf7a53a02771eb1ebdc49564d6f42aef76a449c |
1 /*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
7 /*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/export.h>
33 #include <linux/backing-dev.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/task_io_accounting_ops.h>
39 #include <linux/bio.h>
40 #include <linux/notifier.h>
41 #include <linux/cpu.h>
42 #include <linux/bitops.h>
43 #include <linux/mpage.h>
44 #include <linux/bit_spinlock.h>
45 #include <trace/events/block.h>
52 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
55 {
58 }
62 {
65 }
69 {
71 }
75 {
79 }
82 /*
83 * Returns if the page has dirty or writeback buffers. If all the buffers
84 * are unlocked and clean then the PageDirty information is stale. If
85 * any of the pages are locked, it is assumed they are locked for IO.
86 */
89 {
113 }
116 /*
117 * Block until a buffer comes unlocked. This doesn't stop it
118 * from becoming locked again - you have to lock it yourself
119 * if you want to preserve its state.
120 */
122 {
124 }
127 static void
129 {
133 }
136 {
144 }
146 /*
147 * End-of-IO handler helper function which does not touch the bh after
148 * unlocking it.
149 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
150 * a race there is benign: unlock_buffer() only use the bh's address for
151 * hashing after unlocking the buffer, so it doesn't actually touch the bh
152 * itself.
153 */
155 {
159 /* This happens, due to failed READA attempts. */
161 }
163 }
165 /*
166 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
167 * unlock the buffer. This is what ll_rw_block uses too.
168 */
170 {
173 }
177 {
184 }
187 }
190 /*
191 * Various filesystems appear to want __find_get_block to be non-blocking.
192 * But it's the page lock which protects the buffers. To get around this,
193 * we get exclusion from try_to_free_buffers with the blockdev mapping's
194 * private_lock.
195 *
196 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
197 * may be quite high. This code could TryLock the page, and if that
198 * succeeds, there is no need to take private_lock. (But if
199 * private_lock is contended then so is mapping->tree_lock).
200 */
203 {
207 pgoff_t index;
230 }
234 /* we might be here because some of the buffers on this page are
235 * not mapped. This is due to various races between
236 * file io on the block device and getblk. It gets dealt with
237 * elsewhere, don't buffer_error if we had some unmapped buffers
238 */
250 }
251 out_unlock:
254 out:
256 }
258 /*
259 * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
260 */
262 {
276 }
277 }
279 /*
280 * I/O completion handler for block_read_full_page() - pages
281 * which come unlocked at the end of I/O.
282 */
284 {
300 }
302 /*
303 * Be _very_ careful from here on. Bad things can happen if
304 * two buffer heads end IO at almost the same time and both
305 * decide that the page is now completely done.
306 */
319 }
325 /*
326 * If none of the buffers had errors and they are all
327 * uptodate then we can set the page uptodate.
328 */
334 still_busy:
338 }
340 /*
341 * Completion handler for block_write_full_page() - pages which are unlocked
342 * during I/O, and which have PageWriteback cleared upon I/O completion.
343 */
345 {
362 }
375 }
377 }
383 still_busy:
387 }
390 /*
391 * If a page's buffers are under async readin (end_buffer_async_read
392 * completion) then there is a possibility that another thread of
393 * control could lock one of the buffers after it has completed
394 * but while some of the other buffers have not completed. This
395 * locked buffer would confuse end_buffer_async_read() into not unlocking
396 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
397 * that this buffer is not under async I/O.
398 *
399 * The page comes unlocked when it has no locked buffer_async buffers
400 * left.
401 *
402 * PageLocked prevents anyone starting new async I/O reads any of
403 * the buffers.
404 *
405 * PageWriteback is used to prevent simultaneous writeout of the same
406 * page.
407 *
408 * PageLocked prevents anyone from starting writeback of a page which is
409 * under read I/O (PageWriteback is only ever set against a locked page).
410 */
412 {
415 }
419 {
422 }
425 {
427 }
431 /*
432 * fs/buffer.c contains helper functions for buffer-backed address space's
433 * fsync functions. A common requirement for buffer-based filesystems is
434 * that certain data from the backing blockdev needs to be written out for
435 * a successful fsync(). For example, ext2 indirect blocks need to be
436 * written back and waited upon before fsync() returns.
437 *
438 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
439 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
440 * management of a list of dependent buffers at ->i_mapping->private_list.
441 *
442 * Locking is a little subtle: try_to_free_buffers() will remove buffers
443 * from their controlling inode's queue when they are being freed. But
444 * try_to_free_buffers() will be operating against the *blockdev* mapping
445 * at the time, not against the S_ISREG file which depends on those buffers.
446 * So the locking for private_list is via the private_lock in the address_space
447 * which backs the buffers. Which is different from the address_space
448 * against which the buffers are listed. So for a particular address_space,
449 * mapping->private_lock does *not* protect mapping->private_list! In fact,
450 * mapping->private_list will always be protected by the backing blockdev's
451 * ->private_lock.
452 *
453 * Which introduces a requirement: all buffers on an address_space's
454 * ->private_list must be from the same address_space: the blockdev's.
455 *
456 * address_spaces which do not place buffers at ->private_list via these
457 * utility functions are free to use private_lock and private_list for
458 * whatever they want. The only requirement is that list_empty(private_list)
459 * be true at clear_inode() time.
460 *
461 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
462 * filesystems should do that. invalidate_inode_buffers() should just go
463 * BUG_ON(!list_empty).
464 *
465 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
466 * take an address_space, not an inode. And it should be called
467 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
468 * queued up.
469 *
470 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
471 * list if it is already on a list. Because if the buffer is on a list,
472 * it *must* already be on the right one. If not, the filesystem is being
473 * silly. This will save a ton of locking. But first we have to ensure
474 * that buffers are taken *off* the old inode's list when they are freed
475 * (presumably in truncate). That requires careful auditing of all
476 * filesystems (do it inside bforget()). It could also be done by bringing
477 * b_inode back.
478 */
480 /*
481 * The buffer's backing address_space's private_lock must be held
482 */
484 {
490 }
493 {
495 }
497 /*
498 * osync is designed to support O_SYNC io. It waits synchronously for
499 * all already-submitted IO to complete, but does not queue any new
500 * writes to the disk.
501 *
502 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
503 * you dirty the buffers, and then use osync_inode_buffers to wait for
504 * completion. Any other dirty buffers which are not yet queued for
505 * write will not be flushed to disk by the osync.
506 */
508 {
514 repeat:
526 }
527 }
530 }
533 {
538 }
541 {
545 }
547 /**
548 * emergency_thaw_all -- forcibly thaw every frozen filesystem
549 *
550 * Used for emergency unfreeze of all filesystems via SysRq
551 */
553 {
560 }
561 }
563 /**
564 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
565 * @mapping: the mapping which wants those buffers written
566 *
567 * Starts I/O against the buffers at mapping->private_list, and waits upon
568 * that I/O.
569 *
570 * Basically, this is a convenience function for fsync().
571 * @mapping is a file or directory which needs those buffers to be written for
572 * a successful fsync().
573 */
575 {
583 }
586 /*
587 * Called when we've recently written block `bblock', and it is known that
588 * `bblock' was for a buffer_boundary() buffer. This means that the block at
589 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
590 * dirty, schedule it for IO. So that indirects merge nicely with their data.
591 */
594 {
600 }
601 }
604 {
613 }
620 }
621 }
624 /*
625 * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
626 * dirty.
627 *
628 * If warn is true, then emit a warning if the page is not uptodate and has
629 * not been truncated.
630 *
631 * The caller must hold mem_cgroup_begin_page_stat() lock.
632 */
635 {
644 }
646 }
648 /*
649 * Add a page to the dirty page list.
650 *
651 * It is a sad fact of life that this function is called from several places
652 * deeply under spinlocking. It may not sleep.
653 *
654 * If the page has buffers, the uptodate buffers are set dirty, to preserve
655 * dirty-state coherency between the page and the buffers. It the page does
656 * not have buffers then when they are later attached they will all be set
657 * dirty.
658 *
659 * The buffers are dirtied before the page is dirtied. There's a small race
660 * window in which a writepage caller may see the page cleanness but not the
661 * buffer dirtiness. That's fine. If this code were to set the page dirty
662 * before the buffers, a concurrent writepage caller could clear the page dirty
663 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
664 * page on the dirty page list.
665 *
666 * We use private_lock to lock against try_to_free_buffers while using the
667 * page's buffer list. Also use this to protect against clean buffers being
668 * added to the page after it was set dirty.
669 *
670 * FIXME: may need to call ->reservepage here as well. That's rather up to the
671 * address_space though.
672 */
674 {
691 }
692 /*
693 * Use mem_group_begin_page_stat() to keep PageDirty synchronized with
694 * per-memcg dirty page counters.
695 */
709 }
712 /*
713 * Write out and wait upon a list of buffers.
714 *
715 * We have conflicting pressures: we want to make sure that all
716 * initially dirty buffers get waited on, but that any subsequently
717 * dirtied buffers don't. After all, we don't want fsync to last
718 * forever if somebody is actively writing to the file.
719 *
720 * Do this in two main stages: first we copy dirty buffers to a
721 * temporary inode list, queueing the writes as we go. Then we clean
722 * up, waiting for those writes to complete.
723 *
724 * During this second stage, any subsequent updates to the file may end
725 * up refiling the buffer on the original inode's dirty list again, so
726 * there is a chance we will end up with a buffer queued for write but
727 * not yet completed on that list. So, as a final cleanup we go through
728 * the osync code to catch these locked, dirty buffers without requeuing
729 * any newly dirty buffers for write.
730 */
732 {
747 /* Avoid race with mark_buffer_dirty_inode() which does
748 * a lockless check and we rely on seeing the dirty bit */
756 /*
757 * Ensure any pending I/O completes so that
758 * write_dirty_buffer() actually writes the
759 * current contents - it is a noop if I/O is
760 * still in flight on potentially older
761 * contents.
762 */
765 /*
766 * Kick off IO for the previous mapping. Note
767 * that we will not run the very last mapping,
768 * wait_on_buffer() will do that for us
769 * through sync_buffer().
770 */
773 }
774 }
775 }
786 /* Avoid race with mark_buffer_dirty_inode() which does
787 * a lockless check and we rely on seeing the dirty bit */
793 }
800 }
806 else
808 }
810 /*
811 * Invalidate any and all dirty buffers on a given inode. We are
812 * probably unmounting the fs, but that doesn't mean we have already
813 * done a sync(). Just drop the buffers from the inode list.
814 *
815 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
816 * assumes that all the buffers are against the blockdev. Not true
817 * for reiserfs.
818 */
820 {
830 }
831 }
834 /*
835 * Remove any clean buffers from the inode's buffer list. This is called
836 * when we're trying to free the inode itself. Those buffers can pin it.
837 *
838 * Returns true if all buffers were removed.
839 */
841 {
855 }
857 }
859 }
861 }
863 /*
864 * Create the appropriate buffers when given a page for data area and
865 * the size of each buffer.. Use the bh->b_this_page linked list to
866 * follow the buffers created. Return NULL if unable to create more
867 * buffers.
868 *
869 * The retry flag is used to differentiate async IO (paging, swapping)
870 * which may not fail from ordinary buffer allocations.
871 */
874 {
878 try_again:
892 /* Link the buffer to its page */
894 }
896 /*
897 * In case anything failed, we just free everything we got.
898 */
899 no_grow:
906 }
908 /*
909 * Return failure for non-async IO requests. Async IO requests
910 * are not allowed to fail, so we have to wait until buffer heads
911 * become available. But we don't want tasks sleeping with
912 * partially complete buffers, so all were released above.
913 */
917 /* We're _really_ low on memory. Now we just
918 * wait for old buffer heads to become free due to
919 * finishing IO. Since this is an async request and
920 * the reserve list is empty, we're sure there are
921 * async buffer heads in use.
922 */
925 }
930 {
940 }
943 {
950 }
952 }
954 /*
955 * Initialise the state of a blockdev page's buffers.
956 */
957 static sector_t
960 {
975 }
976 block++;
980 /*
981 * Caller needs to validate requested block against end of device.
982 */
984 }
986 /*
987 * Create the page-cache page that contains the requested block.
988 *
989 * This is used purely for blockdev mappings.
990 */
991 static int
994 {
998 sector_t end_block;
1000 gfp_t gfp_mask;
1004 /*
1005 * XXX: __getblk_slow() can not really deal with failure and
1006 * will endlessly loop on improvised global reclaim. Prefer
1007 * looping in the allocator rather than here, at least that
1008 * code knows what it's doing.
1009 */
1023 size);
1025 }
1028 }
1030 /*
1031 * Allocate some buffers for this page
1032 */
1037 /*
1038 * Link the page to the buffers and initialise them. Take the
1039 * lock to be atomic wrt __find_get_block(), which does not
1040 * run under the page lock.
1041 */
1045 size);
1047 done:
1049 failed:
1053 }
1055 /*
1056 * Create buffers for the specified block device block's page. If
1057 * that page was dirty, the buffers are set dirty also.
1058 */
1059 static int
1061 {
1062 pgoff_t index;
1067 sizebits++;
1072 /*
1073 * Check for a block which wants to lie outside our maximum possible
1074 * pagecache index. (this comparison is done using sector_t types).
1075 */
1084 }
1086 /* Create a page with the proper size buffers.. */
1088 }
1093 {
1094 /* Size must be multiple of hard sectorsize */
1098 size);
1104 }
1119 }
1120 }
1123 /*
1124 * The relationship between dirty buffers and dirty pages:
1125 *
1126 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1127 * the page is tagged dirty in its radix tree.
1128 *
1129 * At all times, the dirtiness of the buffers represents the dirtiness of
1130 * subsections of the page. If the page has buffers, the page dirty bit is
1131 * merely a hint about the true dirty state.
1132 *
1133 * When a page is set dirty in its entirety, all its buffers are marked dirty
1134 * (if the page has buffers).
1135 *
1136 * When a buffer is marked dirty, its page is dirtied, but the page's other
1137 * buffers are not.
1138 *
1139 * Also. When blockdev buffers are explicitly read with bread(), they
1140 * individually become uptodate. But their backing page remains not
1141 * uptodate - even if all of its buffers are uptodate. A subsequent
1142 * block_read_full_page() against that page will discover all the uptodate
1143 * buffers, will set the page uptodate and will perform no I/O.
1144 */
1146 /**
1147 * mark_buffer_dirty - mark a buffer_head as needing writeout
1148 * @bh: the buffer_head to mark dirty
1149 *
1150 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1151 * backing page dirty, then tag the page as dirty in its address_space's radix
1152 * tree and then attach the address_space's inode to its superblock's dirty
1153 * inode list.
1154 *
1155 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1156 * mapping->tree_lock and mapping->host->i_lock.
1157 */
1159 {
1164 /*
1165 * Very *carefully* optimize the it-is-already-dirty case.
1166 *
1167 * Don't let the final "is it dirty" escape to before we
1168 * perhaps modified the buffer.
1169 */
1174 }
1186 }
1190 }
1191 }
1194 /*
1195 * Decrement a buffer_head's reference count. If all buffers against a page
1196 * have zero reference count, are clean and unlocked, and if the page is clean
1197 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1198 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1199 * a page but it ends up not being freed, and buffers may later be reattached).
1200 */
1202 {
1206 }
1208 }
1211 /*
1212 * bforget() is like brelse(), except it discards any
1213 * potentially dirty data.
1214 */
1216 {
1225 }
1227 }
1231 {
1243 }
1246 }
1248 /*
1249 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1250 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1251 * refcount elevated by one when they're in an LRU. A buffer can only appear
1252 * once in a particular CPU's LRU. A single buffer can be present in multiple
1253 * CPU's LRUs at the same time.
1254 *
1255 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1256 * sb_find_get_block().
1257 *
1258 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1259 * a local interrupt disable for that.
1260 */
1262 #define BH_LRU_SIZE 16
1266 };
1270 #ifdef CONFIG_SMP
1271 #define bh_lru_lock() local_irq_disable()
1272 #define bh_lru_unlock() local_irq_enable()
1273 #else
1274 #define bh_lru_lock() preempt_disable()
1275 #define bh_lru_unlock() preempt_enable()
1276 #endif
1279 {
1280 #ifdef irqs_disabled
1282 #endif
1283 }
1285 /*
1286 * The LRU management algorithm is dopey-but-simple. Sorry.
1287 */
1289 {
1313 }
1314 }
1315 }
1319 }
1324 }
1326 /*
1327 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1328 */
1331 {
1346 i--;
1347 }
1349 }
1353 }
1354 }
1357 }
1359 /*
1360 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1361 * it in the LRU and mark it as accessed. If it is not present then return
1362 * NULL
1363 */
1366 {
1370 /* __find_get_block_slow will mark the page accessed */
1378 }
1381 /*
1382 * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1383 * which corresponds to the passed block_device, block and size. The
1384 * returned buffer has its reference count incremented.
1385 *
1386 * __getblk_gfp() will lock up the machine if grow_dev_page's
1387 * try_to_free_buffers() attempt is failing. FIXME, perhaps?
1388 */
1392 {
1399 }
1402 /*
1403 * Do async read-ahead on a buffer..
1404 */
1406 {
1411 }
1412 }
1415 /**
1416 * __bread_gfp() - reads a specified block and returns the bh
1417 * @bdev: the block_device to read from
1418 * @block: number of block
1419 * @size: size (in bytes) to read
1420 * @gfp: page allocation flag
1421 *
1422 * Reads a specified block, and returns buffer head that contains it.
1423 * The page cache can be allocated from non-movable area
1424 * not to prevent page migration if you set gfp to zero.
1425 * It returns NULL if the block was unreadable.
1426 */
1430 {
1436 }
1439 /*
1440 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1441 * This doesn't race because it runs in each cpu either in irq
1442 * or with preempt disabled.
1443 */
1445 {
1452 }
1454 }
1457 {
1464 }
1467 }
1470 {
1472 }
1477 {
1481 /*
1482 * This catches illegal uses and preserves the offset:
1483 */
1485 else
1487 }
1490 /*
1491 * Called when truncating a buffer on a page completely.
1492 */
1494 /* Bits that are cleared during an invalidate */
1495 #define BUFFER_FLAGS_DISCARD \
1496 (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1497 1 << BH_Delay | 1 << BH_Unwritten)
1500 {
1513 }
1515 }
1517 /**
1518 * block_invalidatepage - invalidate part or all of a buffer-backed page
1519 *
1520 * @page: the page which is affected
1521 * @offset: start of the range to invalidate
1522 * @length: length of the range to invalidate
1523 *
1524 * block_invalidatepage() is called when all or part of the page has become
1525 * invalidated by a truncate operation.
1526 *
1527 * block_invalidatepage() does not have to release all buffers, but it must
1528 * ensure that no dirty buffer is left outside @offset and that no I/O
1529 * is underway against any of the blocks which are outside the truncation
1530 * point. Because the caller is about to free (and possibly reuse) those
1531 * blocks on-disk.
1532 */
1535 {
1544 /*
1545 * Check for overflow
1546 */
1555 /*
1556 * Are we still fully in range ?
1557 */
1561 /*
1562 * is this block fully invalidated?
1563 */
1570 /*
1571 * We release buffers only if the entire page is being invalidated.
1572 * The get_block cached value has been unconditionally invalidated,
1573 * so real IO is not possible anymore.
1574 */
1577 out:
1579 }
1583 /*
1584 * We attach and possibly dirty the buffers atomically wrt
1585 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1586 * is already excluded via the page lock.
1587 */
1590 {
1612 }
1615 }
1618 /*
1619 * We are taking a block for data and we don't want any output from any
1620 * buffer-cache aliases starting from return from that function and
1621 * until the moment when something will explicitly mark the buffer
1622 * dirty (hopefully that will not happen until we will free that block ;-)
1623 * We don't even need to mark it not-uptodate - nobody can expect
1624 * anything from a newly allocated buffer anyway. We used to used
1625 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1626 * don't want to mark the alias unmapped, for example - it would confuse
1627 * anyone who might pick it with bread() afterwards...
1628 *
1629 * Also.. Note that bforget() doesn't lock the buffer. So there can
1630 * be writeout I/O going on against recently-freed buffers. We don't
1631 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1632 * only if we really need to. That happens here.
1633 */
1635 {
1646 }
1647 }
1650 /*
1651 * Size is a power-of-two in the range 512..PAGE_SIZE,
1652 * and the case we care about most is PAGE_SIZE.
1653 *
1654 * So this *could* possibly be written with those
1655 * constraints in mind (relevant mostly if some
1656 * architecture has a slow bit-scan instruction)
1657 */
1659 {
1661 }
1663 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1664 {
1670 }
1672 /*
1673 * NOTE! All mapped/uptodate combinations are valid:
1674 *
1675 * Mapped Uptodate Meaning
1676 *
1677 * No No "unknown" - must do get_block()
1678 * No Yes "hole" - zero-filled
1679 * Yes No "allocated" - allocated on disk, not read in
1680 * Yes Yes "valid" - allocated and up-to-date in memory.
1681 *
1682 * "Dirty" is valid only with the last case (mapped+uptodate).
1683 */
1685 /*
1686 * While block_write_full_page is writing back the dirty buffers under
1687 * the page lock, whoever dirtied the buffers may decide to clean them
1688 * again at any time. We handle that by only looking at the buffer
1689 * state inside lock_buffer().
1690 *
1691 * If block_write_full_page() is called for regular writeback
1692 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1693 * locked buffer. This only can happen if someone has written the buffer
1694 * directly, with submit_bh(). At the address_space level PageWriteback
1695 * prevents this contention from occurring.
1696 *
1697 * If block_write_full_page() is called with wbc->sync_mode ==
1698 * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this
1699 * causes the writes to be flagged as synchronous writes.
1700 */
1704 {
1706 sector_t block;
1707 sector_t last_block;
1716 /*
1717 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1718 * here, and the (potentially unmapped) buffers may become dirty at
1719 * any time. If a buffer becomes dirty here after we've inspected it
1720 * then we just miss that fact, and the page stays dirty.
1721 *
1722 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1723 * handle that here by just cleaning them.
1724 */
1733 /*
1734 * Get all the dirty buffers mapped to disk addresses and
1735 * handle any aliases from the underlying blockdev's mapping.
1736 */
1739 /*
1740 * mapped buffers outside i_size will occur, because
1741 * this page can be outside i_size when there is a
1742 * truncate in progress.
1743 */
1744 /*
1745 * The buffer was zeroed by block_write_full_page()
1746 */
1757 /* blockdev mappings never come here */
1761 }
1762 }
1764 block++;
1770 /*
1771 * If it's a fully non-blocking write attempt and we cannot
1772 * lock the buffer then redirty the page. Note that this can
1773 * potentially cause a busy-wait loop from writeback threads
1774 * and kswapd activity, but those code paths have their own
1775 * higher-level throttling.
1776 */
1782 }
1787 }
1790 /*
1791 * The page and its buffers are protected by PageWriteback(), so we can
1792 * drop the bh refcounts early.
1793 */
1801 nr_underway++;
1802 }
1808 done:
1810 /*
1811 * The page was marked dirty, but the buffers were
1812 * clean. Someone wrote them back by hand with
1813 * ll_rw_block/submit_bh. A rare case.
1814 */
1817 /*
1818 * The page and buffer_heads can be released at any time from
1819 * here on.
1820 */
1821 }
1824 recover:
1825 /*
1826 * ENOSPC, or some other error. We may already have added some
1827 * blocks to the file, so we need to write these out to avoid
1828 * exposing stale data.
1829 * The page is currently locked and not marked for writeback
1830 */
1832 /* Recovery: lock and submit the mapped buffers */
1839 /*
1840 * The buffer may have been set dirty during
1841 * attachment to a dirty page.
1842 */
1844 }
1855 nr_underway++;
1856 }
1861 }
1863 /*
1864 * If a page has any new buffers, zero them out here, and mark them uptodate
1865 * and dirty so they'll be written out (in order to prevent uninitialised
1866 * block data from leaking). And clear the new bit.
1867 */
1869 {
1892 }
1896 }
1897 }
1902 }
1907 {
1912 sector_t block;
1935 }
1937 }
1953 }
1959 }
1960 }
1965 }
1971 }
1972 }
1973 /*
1974 * If we issued read requests - let them complete.
1975 */
1980 }
1984 }
1989 {
2007 }
2014 /*
2015 * If this is a partial write which happened to make all buffers
2016 * uptodate then we can optimize away a bogus readpage() for
2017 * the next read(). Here we 'discover' whether the page went
2018 * uptodate as a result of this (potentially partial) write.
2019 */
2023 }
2025 /*
2026 * block_write_begin takes care of the basic task of block allocation and
2027 * bringing partial write blocks uptodate first.
2028 *
2029 * The filesystem needs to handle block truncation upon failure.
2030 */
2033 {
2047 }
2051 }
2057 {
2064 /*
2065 * The buffers that were written will now be uptodate, so we
2066 * don't have to worry about a readpage reading them and
2067 * overwriting a partial write. However if we have encountered
2068 * a short write and only partially written into a buffer, it
2069 * will not be marked uptodate, so a readpage might come in and
2070 * destroy our partial write.
2071 *
2072 * Do the simplest thing, and just treat any short write to a
2073 * non uptodate page as a zero-length write, and force the
2074 * caller to redo the whole thing.
2075 */
2080 }
2083 /* This could be a short (even 0-length) commit */
2087 }
2093 {
2100 /*
2101 * No need to use i_size_read() here, the i_size
2102 * cannot change under us because we hold i_mutex.
2103 *
2104 * But it's important to update i_size while still holding page lock:
2105 * page writeout could otherwise come in and zero beyond i_size.
2106 */
2110 }
2117 /*
2118 * Don't mark the inode dirty under page lock. First, it unnecessarily
2119 * makes the holding time of page lock longer. Second, it forces lock
2120 * ordering of page lock and transaction start for journaling
2121 * filesystems.
2122 */
2127 }
2130 /*
2131 * block_is_partially_uptodate checks whether buffers within a page are
2132 * uptodate or not.
2133 *
2134 * Returns true if all buffers which correspond to a file portion
2135 * we want to read are uptodate.
2136 */
2139 {
2163 }
2166 }
2172 }
2175 /*
2176 * Generic "read page" function for block devices that have the normal
2177 * get_block functionality. This is most of the block device filesystems.
2178 * Reads the page asynchronously --- the unlock_buffer() and
2179 * set/clear_buffer_uptodate() functions propagate buffer state into the
2180 * page struct once IO has completed.
2181 */
2183 {
2214 }
2220 }
2221 /*
2222 * get_block() might have updated the buffer
2223 * synchronously
2224 */
2227 }
2235 /*
2236 * All buffers are uptodate - we can set the page uptodate
2237 * as well. But not if get_block() returned an error.
2238 */
2243 }
2245 /* Stage two: lock the buffers */
2250 }
2252 /*
2253 * Stage 3: start the IO. Check for uptodateness
2254 * inside the buffer lock in case another process reading
2255 * the underlying blockdev brought it uptodate (the sct fix).
2256 */
2261 else
2263 }
2265 }
2268 /* utility function for filesystems that need to do work on expanding
2269 * truncates. Uses filesystem pagecache writes to allow the filesystem to
2270 * deal with the hole.
2271 */
2273 {
2292 out:
2294 }
2299 {
2305 loff_t curpos;
2317 }
2321 AOP_FLAG_UNINTERRUPTIBLE,
2338 }
2339 }
2341 /* page covers the boundary, find the boundary offset */
2344 /* if we will expand the thing last block will be filled */
2347 }
2351 }
2355 AOP_FLAG_UNINTERRUPTIBLE,
2366 }
2367 out:
2369 }
2371 /*
2372 * For moronic filesystems that do not allow holes in file.
2373 * We may have to extend the file.
2374 */
2379 {
2393 }
2396 }
2400 {
2404 }
2407 /*
2408 * block_page_mkwrite() is not allowed to change the file size as it gets
2409 * called from a page fault handler when a page is first dirtied. Hence we must
2410 * be careful to check for EOF conditions here. We set the page up correctly
2411 * for a written page which means we get ENOSPC checking when writing into
2412 * holes and correct delalloc and unwritten extent mapping on filesystems that
2413 * support these features.
2414 *
2415 * We are not allowed to take the i_mutex here so we have to play games to
2416 * protect against truncate races as the page could now be beyond EOF. Because
2417 * truncate writes the inode size before removing pages, once we have the
2418 * page lock we can determine safely if the page is beyond EOF. If it is not
2419 * beyond EOF, then the page is guaranteed safe against truncation until we
2420 * unlock the page.
2421 *
2422 * Direct callers of this function should protect against filesystem freezing
2423 * using sb_start_write() - sb_end_write() functions.
2424 */
2426 get_block_t get_block)
2427 {
2431 loff_t size;
2438 /* We overload EFAULT to mean page got truncated */
2441 }
2443 /* page is wholly or partially inside EOF */
2446 else
2458 out_unlock:
2461 }
2465 get_block_t get_block)
2466 {
2472 /*
2473 * Update file times before taking page lock. We may end up failing the
2474 * fault so this update may be superfluous but who really cares...
2475 */
2481 }
2484 /*
2485 * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2486 * immediately, while under the page lock. So it needs a special end_io
2487 * handler which does not touch the bh after unlocking it.
2488 */
2490 {
2492 }
2494 /*
2495 * Attach the singly-linked list of buffers created by nobh_write_begin, to
2496 * the page (converting it to circular linked list and taking care of page
2497 * dirty races).
2498 */
2500 {
2516 }
2518 /*
2519 * On entry, the page is fully not uptodate.
2520 * On exit the page is fully uptodate in the areas outside (from,to)
2521 * The filesystem needs to handle block truncation upon failure.
2522 */
2527 {
2533 pgoff_t index;
2537 sector_t block_in_file;
2557 }
2562 /*
2563 * Allocate buffers so that we can keep track of state, and potentially
2564 * attach them to the page if an error occurs. In the common case of
2565 * no error, they will just be freed again without ever being attached
2566 * to the page (which is all OK, because we're under the page lock).
2567 *
2568 * Be careful: the buffer linked list is a NULL terminated one, rather
2569 * than the circular one we're used to.
2570 */
2575 }
2579 /*
2580 * We loop across all blocks in the page, whether or not they are
2581 * part of the affected region. This is so we can discover if the
2582 * page is fully mapped-to-disk.
2583 */
2605 }
2610 }
2617 nr_reads++;
2618 }
2619 }
2622 /*
2623 * The page is locked, so these buffers are protected from
2624 * any VM or truncate activity. Hence we don't need to care
2625 * for the buffer_head refcounts.
2626 */
2631 }
2634 }
2643 failed:
2645 /*
2646 * Error recovery is a bit difficult. We need to zero out blocks that
2647 * were newly allocated, and dirty them to ensure they get written out.
2648 * Buffers need to be attached to the page at this point, otherwise
2649 * the handling of potential IO errors during writeout would be hard
2650 * (could try doing synchronous writeout, but what if that fails too?)
2651 */
2655 out_release:
2661 }
2667 {
2684 }
2693 }
2696 }
2699 /*
2700 * nobh_writepage() - based on block_full_write_page() except
2701 * that it tries to operate without attaching bufferheads to
2702 * the page.
2703 */
2706 {
2713 /* Is the page fully inside i_size? */
2717 /* Is the page fully outside i_size? (truncate in progress) */
2720 /*
2721 * The page may have dirty, unmapped buffers. For example,
2722 * they may have been added in ext3_writepage(). Make them
2723 * freeable here, so the page does not leak.
2724 */
2725 #if 0
2726 /* Not really sure about this - do we need this ? */
2729 #endif
2732 }
2734 /*
2735 * The page straddles i_size. It must be zeroed out on each and every
2736 * writepage invocation because it may be mmapped. "A file is mapped
2737 * in multiples of the page size. For a file that is not a multiple of
2738 * the page size, the remaining memory is zeroed when mapped, and
2739 * writes to that region are not written out to the file."
2740 */
2742 out:
2746 end_buffer_async_write);
2748 }
2753 {
2757 sector_t iblock;
2767 /* Block boundary? Nothing to do */
2780 has_buffers:
2784 }
2786 /* Find the buffer that contains "offset" */
2789 iblock++;
2791 }
2798 /* unmapped? It's a hole - nothing to do */
2802 /* Ok, it's mapped. Make sure it's up-to-date */
2808 }
2813 }
2816 }
2821 unlock:
2824 out:
2826 }
2831 {
2835 sector_t iblock;
2845 /* Block boundary? Nothing to do */
2860 /* Find the buffer that contains "offset" */
2865 iblock++;
2867 }
2875 /* unmapped? It's a hole - nothing to do */
2878 }
2880 /* Ok, it's mapped. Make sure it's up-to-date */
2888 /* Uhhuh. Read error. Complain and punt. */
2891 }
2897 unlock:
2900 out:
2902 }
2905 /*
2906 * The generic ->writepage function for buffer-backed address_spaces
2907 */
2910 {
2916 /* Is the page fully inside i_size? */
2919 end_buffer_async_write);
2921 /* Is the page fully outside i_size? (truncate in progress) */
2924 /*
2925 * The page may have dirty, unmapped buffers. For example,
2926 * they may have been added in ext3_writepage(). Make them
2927 * freeable here, so the page does not leak.
2928 */
2932 }
2934 /*
2935 * The page straddles i_size. It must be zeroed out on each and every
2936 * writepage invocation because it may be mmapped. "A file is mapped
2937 * in multiples of the page size. For a file that is not a multiple of
2938 * the page size, the remaining memory is zeroed when mapped, and
2939 * writes to that region are not written out to the file."
2940 */
2943 end_buffer_async_write);
2944 }
2949 {
2957 }
2961 {
2969 }
2971 /*
2972 * This allows us to do IO even on the odd last sectors
2973 * of a device, even if the block size is some multiple
2974 * of the physical sector size.
2975 *
2976 * We'll just truncate the bio to the size of the device,
2977 * and clear the end of the buffer head manually.
2978 *
2979 * Truly out-of-range accesses will turn into actual IO
2980 * errors, this only handles the "we need to be able to
2981 * do IO at the final sector" case.
2982 */
2984 {
2985 sector_t maxsector;
2993 /*
2994 * If the *whole* IO is past the end of the device,
2995 * let it through, and the IO layer will turn it into
2996 * an EIO.
2997 */
3005 /* Uhhuh. We've got a bio that straddles the device size! */
3008 /* Truncate the bio.. */
3012 /* ..and clear the end of the buffer for reads */
3015 truncated_bytes);
3016 }
3017 }
3021 {
3030 /*
3031 * Only clear out a write error when rewriting
3032 */
3036 /*
3037 * from here on down, it's all bio -- do the initial mapping,
3038 * submit_bio -> generic_make_request may further map this bio around
3039 */
3045 }
3060 /* Take care of bh's that straddle the end of the device */
3070 }
3073 {
3075 }
3079 {
3081 }
3084 /**
3085 * ll_rw_block: low-level access to block devices (DEPRECATED)
3086 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
3087 * @nr: number of &struct buffer_heads in the array
3088 * @bhs: array of pointers to &struct buffer_head
3089 *
3090 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3091 * requests an I/O operation on them, either a %READ or a %WRITE. The third
3092 * %READA option is described in the documentation for generic_make_request()
3093 * which ll_rw_block() calls.
3094 *
3095 * This function drops any buffer that it cannot get a lock on (with the
3096 * BH_Lock state bit), any buffer that appears to be clean when doing a write
3097 * request, and any buffer that appears to be up-to-date when doing read
3098 * request. Further it marks as clean buffers that are processed for
3099 * writing (the buffer cache won't assume that they are actually clean
3100 * until the buffer gets unlocked).
3101 *
3102 * ll_rw_block sets b_end_io to simple completion handler that marks
3103 * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3104 * any waiters.
3105 *
3106 * All of the buffers must be for the same device, and must also be a
3107 * multiple of the current approved size for the device.
3108 */
3110 {
3124 }
3131 }
3132 }
3134 }
3135 }
3139 {
3144 }
3148 }
3151 /*
3152 * For a data-integrity writeout, we need to wait upon any in-progress I/O
3153 * and then start new I/O and then wait upon it. The caller must have a ref on
3154 * the buffer_head.
3155 */
3157 {
3171 }
3173 }
3177 {
3179 }
3182 /*
3183 * try_to_free_buffers() checks if all the buffers on this particular page
3184 * are unused, and releases them if so.
3185 *
3186 * Exclusion against try_to_free_buffers may be obtained by either
3187 * locking the page or by holding its mapping's private_lock.
3188 *
3189 * If the page is dirty but all the buffers are clean then we need to
3190 * be sure to mark the page clean as well. This is because the page
3191 * may be against a block device, and a later reattachment of buffers
3192 * to a dirty page will set *all* buffers dirty. Which would corrupt
3193 * filesystem data on the same device.
3194 *
3195 * The same applies to regular filesystem pages: if all the buffers are
3196 * clean then we set the page clean and proceed. To do that, we require
3197 * total exclusion from __set_page_dirty_buffers(). That is obtained with
3198 * private_lock.
3199 *
3200 * try_to_free_buffers() is non-blocking.
3201 */
3203 {
3206 }
3208 static int
3210 {
3233 failed:
3235 }
3238 {
3250 }
3255 /*
3256 * If the filesystem writes its buffers by hand (eg ext3)
3257 * then we can have clean buffers against a dirty page. We
3258 * clean the page here; otherwise the VM will never notice
3259 * that the filesystem did any IO at all.
3260 *
3261 * Also, during truncate, discard_buffer will have marked all
3262 * the page's buffers clean. We discover that here and clean
3263 * the page also.
3264 *
3265 * private_lock must be held over this entire operation in order
3266 * to synchronise against __set_page_dirty_buffers and prevent the
3267 * dirty bit from being lost.
3268 */
3272 out:
3281 }
3283 }
3286 /*
3287 * There are no bdflush tunables left. But distributions are
3288 * still running obsolete flush daemons, so we terminate them here.
3289 *
3290 * Use of bdflush() is deprecated and will be removed in a future kernel.
3291 * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3292 */
3294 {
3301 msg_count++;
3303 "warning: process `%s' used the obsolete bdflush"
3306 }
3311 }
3313 /*
3314 * Buffer-head allocation
3315 */
3318 /*
3319 * Once the number of bh's in the machine exceeds this level, we start
3320 * stripping them in writeback.
3321 */
3329 };
3334 {
3344 }
3347 {
3355 }
3357 }
3361 {
3368 }
3372 {
3379 }
3382 }
3386 {
3390 }
3392 /**
3393 * bh_uptodate_or_lock - Test whether the buffer is uptodate
3394 * @bh: struct buffer_head
3395 *
3396 * Return true if the buffer is up-to-date and false,
3397 * with the buffer locked, if not.
3398 */
3400 {
3406 }
3408 }
3411 /**
3412 * bh_submit_read - Submit a locked buffer for reading
3413 * @bh: struct buffer_head
3414 *
3415 * Returns zero on success and -EIO on error.
3416 */
3418 {
3424 }
3433 }
3437 {
3443 SLAB_MEM_SPREAD),
3444 NULL);
3446 /*
3447 * Limit the bh occupancy to 10% of ZONE_NORMAL
3448 */
3452 }