| blob | 8f05111bbb8bd36391ae01184ee5dcc9a6cfb40e |
1 /*
2 * linux/fs/buffer.c
3 *
4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
5 */
7 /*
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9 *
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12 *
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
15 *
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17 *
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19 */
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/mm.h>
25 #include <linux/percpu.h>
26 #include <linux/slab.h>
27 #include <linux/capability.h>
28 #include <linux/blkdev.h>
29 #include <linux/file.h>
30 #include <linux/quotaops.h>
31 #include <linux/highmem.h>
32 #include <linux/export.h>
33 #include <linux/writeback.h>
34 #include <linux/hash.h>
35 #include <linux/suspend.h>
36 #include <linux/buffer_head.h>
37 #include <linux/task_io_accounting_ops.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
43 #include <linux/bit_spinlock.h>
44 #include <trace/events/block.h>
48 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
51 {
54 }
58 {
61 }
65 {
67 }
71 {
75 }
78 /*
79 * Returns if the page has dirty or writeback buffers. If all the buffers
80 * are unlocked and clean then the PageDirty information is stale. If
81 * any of the pages are locked, it is assumed they are locked for IO.
82 */
85 {
109 }
112 /*
113 * Block until a buffer comes unlocked. This doesn't stop it
114 * from becoming locked again - you have to lock it yourself
115 * if you want to preserve its state.
116 */
118 {
120 }
123 static void
125 {
129 }
133 {
137 }
141 {
146 }
148 /*
149 * End-of-IO handler helper function which does not touch the bh after
150 * unlocking it.
151 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
152 * a race there is benign: unlock_buffer() only use the bh's address for
153 * hashing after unlocking the buffer, so it doesn't actually touch the bh
154 * itself.
155 */
157 {
161 /* This happens, due to failed READA attempts. */
163 }
165 }
167 /*
168 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
169 * unlock the buffer. This is what ll_rw_block uses too.
170 */
172 {
175 }
179 {
190 }
193 }
196 }
199 /*
200 * Various filesystems appear to want __find_get_block to be non-blocking.
201 * But it's the page lock which protects the buffers. To get around this,
202 * we get exclusion from try_to_free_buffers with the blockdev mapping's
203 * private_lock.
204 *
205 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
206 * may be quite high. This code could TryLock the page, and if that
207 * succeeds, there is no need to take private_lock. (But if
208 * private_lock is contended then so is mapping->tree_lock).
209 */
212 {
216 pgoff_t index;
239 }
243 /* we might be here because some of the buffers on this page are
244 * not mapped. This is due to various races between
245 * file io on the block device and getblk. It gets dealt with
246 * elsewhere, don't buffer_error if we had some unmapped buffers
247 */
259 }
260 out_unlock:
263 out:
265 }
267 /*
268 * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
269 */
271 {
285 }
286 }
288 /*
289 * I/O completion handler for block_read_full_page() - pages
290 * which come unlocked at the end of I/O.
291 */
293 {
310 }
312 /*
313 * Be _very_ careful from here on. Bad things can happen if
314 * two buffer heads end IO at almost the same time and both
315 * decide that the page is now completely done.
316 */
329 }
335 /*
336 * If none of the buffers had errors and they are all
337 * uptodate then we can set the page uptodate.
338 */
344 still_busy:
348 }
350 /*
351 * Completion handler for block_write_full_page() - pages which are unlocked
352 * during I/O, and which have PageWriteback cleared upon I/O completion.
353 */
355 {
373 }
378 }
391 }
393 }
399 still_busy:
403 }
406 /*
407 * If a page's buffers are under async readin (end_buffer_async_read
408 * completion) then there is a possibility that another thread of
409 * control could lock one of the buffers after it has completed
410 * but while some of the other buffers have not completed. This
411 * locked buffer would confuse end_buffer_async_read() into not unlocking
412 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
413 * that this buffer is not under async I/O.
414 *
415 * The page comes unlocked when it has no locked buffer_async buffers
416 * left.
417 *
418 * PageLocked prevents anyone starting new async I/O reads any of
419 * the buffers.
420 *
421 * PageWriteback is used to prevent simultaneous writeout of the same
422 * page.
423 *
424 * PageLocked prevents anyone from starting writeback of a page which is
425 * under read I/O (PageWriteback is only ever set against a locked page).
426 */
428 {
431 }
435 {
438 }
441 {
443 }
447 /*
448 * fs/buffer.c contains helper functions for buffer-backed address space's
449 * fsync functions. A common requirement for buffer-based filesystems is
450 * that certain data from the backing blockdev needs to be written out for
451 * a successful fsync(). For example, ext2 indirect blocks need to be
452 * written back and waited upon before fsync() returns.
453 *
454 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
455 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
456 * management of a list of dependent buffers at ->i_mapping->private_list.
457 *
458 * Locking is a little subtle: try_to_free_buffers() will remove buffers
459 * from their controlling inode's queue when they are being freed. But
460 * try_to_free_buffers() will be operating against the *blockdev* mapping
461 * at the time, not against the S_ISREG file which depends on those buffers.
462 * So the locking for private_list is via the private_lock in the address_space
463 * which backs the buffers. Which is different from the address_space
464 * against which the buffers are listed. So for a particular address_space,
465 * mapping->private_lock does *not* protect mapping->private_list! In fact,
466 * mapping->private_list will always be protected by the backing blockdev's
467 * ->private_lock.
468 *
469 * Which introduces a requirement: all buffers on an address_space's
470 * ->private_list must be from the same address_space: the blockdev's.
471 *
472 * address_spaces which do not place buffers at ->private_list via these
473 * utility functions are free to use private_lock and private_list for
474 * whatever they want. The only requirement is that list_empty(private_list)
475 * be true at clear_inode() time.
476 *
477 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
478 * filesystems should do that. invalidate_inode_buffers() should just go
479 * BUG_ON(!list_empty).
480 *
481 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
482 * take an address_space, not an inode. And it should be called
483 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
484 * queued up.
485 *
486 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
487 * list if it is already on a list. Because if the buffer is on a list,
488 * it *must* already be on the right one. If not, the filesystem is being
489 * silly. This will save a ton of locking. But first we have to ensure
490 * that buffers are taken *off* the old inode's list when they are freed
491 * (presumably in truncate). That requires careful auditing of all
492 * filesystems (do it inside bforget()). It could also be done by bringing
493 * b_inode back.
494 */
496 /*
497 * The buffer's backing address_space's private_lock must be held
498 */
500 {
506 }
509 {
511 }
513 /*
514 * osync is designed to support O_SYNC io. It waits synchronously for
515 * all already-submitted IO to complete, but does not queue any new
516 * writes to the disk.
517 *
518 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
519 * you dirty the buffers, and then use osync_inode_buffers to wait for
520 * completion. Any other dirty buffers which are not yet queued for
521 * write will not be flushed to disk by the osync.
522 */
524 {
530 repeat:
542 }
543 }
546 }
549 {
554 }
557 {
561 }
563 /**
564 * emergency_thaw_all -- forcibly thaw every frozen filesystem
565 *
566 * Used for emergency unfreeze of all filesystems via SysRq
567 */
569 {
576 }
577 }
579 /**
580 * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
581 * @mapping: the mapping which wants those buffers written
582 *
583 * Starts I/O against the buffers at mapping->private_list, and waits upon
584 * that I/O.
585 *
586 * Basically, this is a convenience function for fsync().
587 * @mapping is a file or directory which needs those buffers to be written for
588 * a successful fsync().
589 */
591 {
599 }
602 /*
603 * Called when we've recently written block `bblock', and it is known that
604 * `bblock' was for a buffer_boundary() buffer. This means that the block at
605 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
606 * dirty, schedule it for IO. So that indirects merge nicely with their data.
607 */
610 {
616 }
617 }
620 {
629 }
636 }
637 }
640 /*
641 * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
642 * dirty.
643 *
644 * If warn is true, then emit a warning if the page is not uptodate and has
645 * not been truncated.
646 */
649 {
658 }
661 }
663 /*
664 * Add a page to the dirty page list.
665 *
666 * It is a sad fact of life that this function is called from several places
667 * deeply under spinlocking. It may not sleep.
668 *
669 * If the page has buffers, the uptodate buffers are set dirty, to preserve
670 * dirty-state coherency between the page and the buffers. It the page does
671 * not have buffers then when they are later attached they will all be set
672 * dirty.
673 *
674 * The buffers are dirtied before the page is dirtied. There's a small race
675 * window in which a writepage caller may see the page cleanness but not the
676 * buffer dirtiness. That's fine. If this code were to set the page dirty
677 * before the buffers, a concurrent writepage caller could clear the page dirty
678 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
679 * page on the dirty page list.
680 *
681 * We use private_lock to lock against try_to_free_buffers while using the
682 * page's buffer list. Also use this to protect against clean buffers being
683 * added to the page after it was set dirty.
684 *
685 * FIXME: may need to call ->reservepage here as well. That's rather up to the
686 * address_space though.
687 */
689 {
705 }
712 }
715 /*
716 * Write out and wait upon a list of buffers.
717 *
718 * We have conflicting pressures: we want to make sure that all
719 * initially dirty buffers get waited on, but that any subsequently
720 * dirtied buffers don't. After all, we don't want fsync to last
721 * forever if somebody is actively writing to the file.
722 *
723 * Do this in two main stages: first we copy dirty buffers to a
724 * temporary inode list, queueing the writes as we go. Then we clean
725 * up, waiting for those writes to complete.
726 *
727 * During this second stage, any subsequent updates to the file may end
728 * up refiling the buffer on the original inode's dirty list again, so
729 * there is a chance we will end up with a buffer queued for write but
730 * not yet completed on that list. So, as a final cleanup we go through
731 * the osync code to catch these locked, dirty buffers without requeuing
732 * any newly dirty buffers for write.
733 */
735 {
750 /* Avoid race with mark_buffer_dirty_inode() which does
751 * a lockless check and we rely on seeing the dirty bit */
759 /*
760 * Ensure any pending I/O completes so that
761 * write_dirty_buffer() actually writes the
762 * current contents - it is a noop if I/O is
763 * still in flight on potentially older
764 * contents.
765 */
768 /*
769 * Kick off IO for the previous mapping. Note
770 * that we will not run the very last mapping,
771 * wait_on_buffer() will do that for us
772 * through sync_buffer().
773 */
776 }
777 }
778 }
789 /* Avoid race with mark_buffer_dirty_inode() which does
790 * a lockless check and we rely on seeing the dirty bit */
796 }
803 }
809 else
811 }
813 /*
814 * Invalidate any and all dirty buffers on a given inode. We are
815 * probably unmounting the fs, but that doesn't mean we have already
816 * done a sync(). Just drop the buffers from the inode list.
817 *
818 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
819 * assumes that all the buffers are against the blockdev. Not true
820 * for reiserfs.
821 */
823 {
833 }
834 }
837 /*
838 * Remove any clean buffers from the inode's buffer list. This is called
839 * when we're trying to free the inode itself. Those buffers can pin it.
840 *
841 * Returns true if all buffers were removed.
842 */
844 {
858 }
860 }
862 }
864 }
866 /*
867 * Create the appropriate buffers when given a page for data area and
868 * the size of each buffer.. Use the bh->b_this_page linked list to
869 * follow the buffers created. Return NULL if unable to create more
870 * buffers.
871 *
872 * The retry flag is used to differentiate async IO (paging, swapping)
873 * which may not fail from ordinary buffer allocations.
874 */
877 {
881 try_again:
895 /* Link the buffer to its page */
897 }
899 /*
900 * In case anything failed, we just free everything we got.
901 */
902 no_grow:
909 }
911 /*
912 * Return failure for non-async IO requests. Async IO requests
913 * are not allowed to fail, so we have to wait until buffer heads
914 * become available. But we don't want tasks sleeping with
915 * partially complete buffers, so all were released above.
916 */
920 /* We're _really_ low on memory. Now we just
921 * wait for old buffer heads to become free due to
922 * finishing IO. Since this is an async request and
923 * the reserve list is empty, we're sure there are
924 * async buffer heads in use.
925 */
928 }
933 {
943 }
946 {
953 }
955 }
957 /*
958 * Initialise the state of a blockdev page's buffers.
959 */
960 static sector_t
963 {
978 }
979 block++;
983 /*
984 * Caller needs to validate requested block against end of device.
985 */
987 }
989 /*
990 * Create the page-cache page that contains the requested block.
991 *
992 * This is used purely for blockdev mappings.
993 */
994 static int
997 {
1001 sector_t end_block;
1003 gfp_t gfp_mask;
1007 /*
1008 * XXX: __getblk_slow() can not really deal with failure and
1009 * will endlessly loop on improvised global reclaim. Prefer
1010 * looping in the allocator rather than here, at least that
1011 * code knows what it's doing.
1012 */
1027 }
1030 }
1032 /*
1033 * Allocate some buffers for this page
1034 */
1039 /*
1040 * Link the page to the buffers and initialise them. Take the
1041 * lock to be atomic wrt __find_get_block(), which does not
1042 * run under the page lock.
1043 */
1048 done:
1050 failed:
1054 }
1056 /*
1057 * Create buffers for the specified block device block's page. If
1058 * that page was dirty, the buffers are set dirty also.
1059 */
1060 static int
1062 {
1063 pgoff_t index;
1068 sizebits++;
1073 /*
1074 * Check for a block which wants to lie outside our maximum possible
1075 * pagecache index. (this comparison is done using sector_t types).
1076 */
1085 }
1087 /* Create a page with the proper size buffers.. */
1089 }
1093 {
1094 /* Size must be multiple of hard sectorsize */
1098 size);
1104 }
1119 }
1120 }
1122 /*
1123 * The relationship between dirty buffers and dirty pages:
1124 *
1125 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1126 * the page is tagged dirty in its radix tree.
1127 *
1128 * At all times, the dirtiness of the buffers represents the dirtiness of
1129 * subsections of the page. If the page has buffers, the page dirty bit is
1130 * merely a hint about the true dirty state.
1131 *
1132 * When a page is set dirty in its entirety, all its buffers are marked dirty
1133 * (if the page has buffers).
1134 *
1135 * When a buffer is marked dirty, its page is dirtied, but the page's other
1136 * buffers are not.
1137 *
1138 * Also. When blockdev buffers are explicitly read with bread(), they
1139 * individually become uptodate. But their backing page remains not
1140 * uptodate - even if all of its buffers are uptodate. A subsequent
1141 * block_read_full_page() against that page will discover all the uptodate
1142 * buffers, will set the page uptodate and will perform no I/O.
1143 */
1145 /**
1146 * mark_buffer_dirty - mark a buffer_head as needing writeout
1147 * @bh: the buffer_head to mark dirty
1148 *
1149 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1150 * backing page dirty, then tag the page as dirty in its address_space's radix
1151 * tree and then attach the address_space's inode to its superblock's dirty
1152 * inode list.
1153 *
1154 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1155 * mapping->tree_lock and mapping->host->i_lock.
1156 */
1158 {
1163 /*
1164 * Very *carefully* optimize the it-is-already-dirty case.
1165 *
1166 * Don't let the final "is it dirty" escape to before we
1167 * perhaps modified the buffer.
1168 */
1173 }
1181 }
1182 }
1183 }
1186 /*
1187 * Decrement a buffer_head's reference count. If all buffers against a page
1188 * have zero reference count, are clean and unlocked, and if the page is clean
1189 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1190 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1191 * a page but it ends up not being freed, and buffers may later be reattached).
1192 */
1194 {
1198 }
1200 }
1203 /*
1204 * bforget() is like brelse(), except it discards any
1205 * potentially dirty data.
1206 */
1208 {
1217 }
1219 }
1223 {
1235 }
1238 }
1240 /*
1241 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1242 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1243 * refcount elevated by one when they're in an LRU. A buffer can only appear
1244 * once in a particular CPU's LRU. A single buffer can be present in multiple
1245 * CPU's LRUs at the same time.
1246 *
1247 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1248 * sb_find_get_block().
1249 *
1250 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1251 * a local interrupt disable for that.
1252 */
1254 #define BH_LRU_SIZE 8
1258 };
1262 #ifdef CONFIG_SMP
1263 #define bh_lru_lock() local_irq_disable()
1264 #define bh_lru_unlock() local_irq_enable()
1265 #else
1266 #define bh_lru_lock() preempt_disable()
1267 #define bh_lru_unlock() preempt_enable()
1268 #endif
1271 {
1272 #ifdef irqs_disabled
1274 #endif
1275 }
1277 /*
1278 * The LRU management algorithm is dopey-but-simple. Sorry.
1279 */
1281 {
1305 }
1306 }
1307 }
1311 }
1316 }
1318 /*
1319 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1320 */
1323 {
1338 i--;
1339 }
1341 }
1345 }
1346 }
1349 }
1351 /*
1352 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1353 * it in the LRU and mark it as accessed. If it is not present then return
1354 * NULL
1355 */
1358 {
1362 /* __find_get_block_slow will mark the page accessed */
1370 }
1373 /*
1374 * __getblk will locate (and, if necessary, create) the buffer_head
1375 * which corresponds to the passed block_device, block and size. The
1376 * returned buffer has its reference count incremented.
1377 *
1378 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1379 * attempt is failing. FIXME, perhaps?
1380 */
1383 {
1390 }
1393 /*
1394 * Do async read-ahead on a buffer..
1395 */
1397 {
1402 }
1403 }
1406 /**
1407 * __bread() - reads a specified block and returns the bh
1408 * @bdev: the block_device to read from
1409 * @block: number of block
1410 * @size: size (in bytes) to read
1411 *
1412 * Reads a specified block, and returns buffer head that contains it.
1413 * It returns NULL if the block was unreadable.
1414 */
1417 {
1423 }
1426 /*
1427 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1428 * This doesn't race because it runs in each cpu either in irq
1429 * or with preempt disabled.
1430 */
1432 {
1439 }
1441 }
1444 {
1451 }
1454 }
1457 {
1459 }
1464 {
1468 /*
1469 * This catches illegal uses and preserves the offset:
1470 */
1472 else
1474 }
1477 /*
1478 * Called when truncating a buffer on a page completely.
1479 */
1481 /* Bits that are cleared during an invalidate */
1482 #define BUFFER_FLAGS_DISCARD \
1483 (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1484 1 << BH_Delay | 1 << BH_Unwritten)
1487 {
1500 }
1502 }
1504 /**
1505 * block_invalidatepage - invalidate part or all of a buffer-backed page
1506 *
1507 * @page: the page which is affected
1508 * @offset: start of the range to invalidate
1509 * @length: length of the range to invalidate
1510 *
1511 * block_invalidatepage() is called when all or part of the page has become
1512 * invalidated by a truncate operation.
1513 *
1514 * block_invalidatepage() does not have to release all buffers, but it must
1515 * ensure that no dirty buffer is left outside @offset and that no I/O
1516 * is underway against any of the blocks which are outside the truncation
1517 * point. Because the caller is about to free (and possibly reuse) those
1518 * blocks on-disk.
1519 */
1522 {
1531 /*
1532 * Check for overflow
1533 */
1542 /*
1543 * Are we still fully in range ?
1544 */
1548 /*
1549 * is this block fully invalidated?
1550 */
1557 /*
1558 * We release buffers only if the entire page is being invalidated.
1559 * The get_block cached value has been unconditionally invalidated,
1560 * so real IO is not possible anymore.
1561 */
1564 out:
1566 }
1570 /*
1571 * We attach and possibly dirty the buffers atomically wrt
1572 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1573 * is already excluded via the page lock.
1574 */
1577 {
1599 }
1602 }
1605 /*
1606 * We are taking a block for data and we don't want any output from any
1607 * buffer-cache aliases starting from return from that function and
1608 * until the moment when something will explicitly mark the buffer
1609 * dirty (hopefully that will not happen until we will free that block ;-)
1610 * We don't even need to mark it not-uptodate - nobody can expect
1611 * anything from a newly allocated buffer anyway. We used to used
1612 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1613 * don't want to mark the alias unmapped, for example - it would confuse
1614 * anyone who might pick it with bread() afterwards...
1615 *
1616 * Also.. Note that bforget() doesn't lock the buffer. So there can
1617 * be writeout I/O going on against recently-freed buffers. We don't
1618 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1619 * only if we really need to. That happens here.
1620 */
1622 {
1633 }
1634 }
1637 /*
1638 * Size is a power-of-two in the range 512..PAGE_SIZE,
1639 * and the case we care about most is PAGE_SIZE.
1640 *
1641 * So this *could* possibly be written with those
1642 * constraints in mind (relevant mostly if some
1643 * architecture has a slow bit-scan instruction)
1644 */
1646 {
1648 }
1650 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1651 {
1657 }
1659 /*
1660 * NOTE! All mapped/uptodate combinations are valid:
1661 *
1662 * Mapped Uptodate Meaning
1663 *
1664 * No No "unknown" - must do get_block()
1665 * No Yes "hole" - zero-filled
1666 * Yes No "allocated" - allocated on disk, not read in
1667 * Yes Yes "valid" - allocated and up-to-date in memory.
1668 *
1669 * "Dirty" is valid only with the last case (mapped+uptodate).
1670 */
1672 /*
1673 * While block_write_full_page is writing back the dirty buffers under
1674 * the page lock, whoever dirtied the buffers may decide to clean them
1675 * again at any time. We handle that by only looking at the buffer
1676 * state inside lock_buffer().
1677 *
1678 * If block_write_full_page() is called for regular writeback
1679 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1680 * locked buffer. This only can happen if someone has written the buffer
1681 * directly, with submit_bh(). At the address_space level PageWriteback
1682 * prevents this contention from occurring.
1683 *
1684 * If block_write_full_page() is called with wbc->sync_mode ==
1685 * WB_SYNC_ALL, the writes are posted using WRITE_SYNC; this
1686 * causes the writes to be flagged as synchronous writes.
1687 */
1691 {
1693 sector_t block;
1694 sector_t last_block;
1704 /*
1705 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1706 * here, and the (potentially unmapped) buffers may become dirty at
1707 * any time. If a buffer becomes dirty here after we've inspected it
1708 * then we just miss that fact, and the page stays dirty.
1709 *
1710 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1711 * handle that here by just cleaning them.
1712 */
1721 /*
1722 * Get all the dirty buffers mapped to disk addresses and
1723 * handle any aliases from the underlying blockdev's mapping.
1724 */
1727 /*
1728 * mapped buffers outside i_size will occur, because
1729 * this page can be outside i_size when there is a
1730 * truncate in progress.
1731 */
1732 /*
1733 * The buffer was zeroed by block_write_full_page()
1734 */
1745 /* blockdev mappings never come here */
1749 }
1750 }
1752 block++;
1758 /*
1759 * If it's a fully non-blocking write attempt and we cannot
1760 * lock the buffer then redirty the page. Note that this can
1761 * potentially cause a busy-wait loop from writeback threads
1762 * and kswapd activity, but those code paths have their own
1763 * higher-level throttling.
1764 */
1770 }
1775 }
1778 /*
1779 * The page and its buffers are protected by PageWriteback(), so we can
1780 * drop the bh refcounts early.
1781 */
1789 nr_underway++;
1790 }
1796 done:
1798 /*
1799 * The page was marked dirty, but the buffers were
1800 * clean. Someone wrote them back by hand with
1801 * ll_rw_block/submit_bh. A rare case.
1802 */
1805 /*
1806 * The page and buffer_heads can be released at any time from
1807 * here on.
1808 */
1809 }
1812 recover:
1813 /*
1814 * ENOSPC, or some other error. We may already have added some
1815 * blocks to the file, so we need to write these out to avoid
1816 * exposing stale data.
1817 * The page is currently locked and not marked for writeback
1818 */
1820 /* Recovery: lock and submit the mapped buffers */
1827 /*
1828 * The buffer may have been set dirty during
1829 * attachment to a dirty page.
1830 */
1832 }
1843 nr_underway++;
1844 }
1849 }
1851 /*
1852 * If a page has any new buffers, zero them out here, and mark them uptodate
1853 * and dirty so they'll be written out (in order to prevent uninitialised
1854 * block data from leaking). And clear the new bit.
1855 */
1857 {
1880 }
1884 }
1885 }
1890 }
1895 {
1900 sector_t block;
1923 }
1925 }
1941 }
1947 }
1948 }
1953 }
1959 }
1960 }
1961 /*
1962 * If we issued read requests - let them complete.
1963 */
1968 }
1972 }
1977 {
1995 }
2002 /*
2003 * If this is a partial write which happened to make all buffers
2004 * uptodate then we can optimize away a bogus readpage() for
2005 * the next read(). Here we 'discover' whether the page went
2006 * uptodate as a result of this (potentially partial) write.
2007 */
2011 }
2013 /*
2014 * block_write_begin takes care of the basic task of block allocation and
2015 * bringing partial write blocks uptodate first.
2016 *
2017 * The filesystem needs to handle block truncation upon failure.
2018 */
2021 {
2035 }
2039 }
2045 {
2052 /*
2053 * The buffers that were written will now be uptodate, so we
2054 * don't have to worry about a readpage reading them and
2055 * overwriting a partial write. However if we have encountered
2056 * a short write and only partially written into a buffer, it
2057 * will not be marked uptodate, so a readpage might come in and
2058 * destroy our partial write.
2059 *
2060 * Do the simplest thing, and just treat any short write to a
2061 * non uptodate page as a zero-length write, and force the
2062 * caller to redo the whole thing.
2063 */
2068 }
2071 /* This could be a short (even 0-length) commit */
2075 }
2081 {
2087 /*
2088 * No need to use i_size_read() here, the i_size
2089 * cannot change under us because we hold i_mutex.
2090 *
2091 * But it's important to update i_size while still holding page lock:
2092 * page writeout could otherwise come in and zero beyond i_size.
2093 */
2097 }
2102 /*
2103 * Don't mark the inode dirty under page lock. First, it unnecessarily
2104 * makes the holding time of page lock longer. Second, it forces lock
2105 * ordering of page lock and transaction start for journaling
2106 * filesystems.
2107 */
2112 }
2115 /*
2116 * block_is_partially_uptodate checks whether buffers within a page are
2117 * uptodate or not.
2118 *
2119 * Returns true if all buffers which correspond to a file portion
2120 * we want to read are uptodate.
2121 */
2124 {
2148 }
2151 }
2157 }
2160 /*
2161 * Generic "read page" function for block devices that have the normal
2162 * get_block functionality. This is most of the block device filesystems.
2163 * Reads the page asynchronously --- the unlock_buffer() and
2164 * set/clear_buffer_uptodate() functions propagate buffer state into the
2165 * page struct once IO has completed.
2166 */
2168 {
2199 }
2205 }
2206 /*
2207 * get_block() might have updated the buffer
2208 * synchronously
2209 */
2212 }
2220 /*
2221 * All buffers are uptodate - we can set the page uptodate
2222 * as well. But not if get_block() returned an error.
2223 */
2228 }
2230 /* Stage two: lock the buffers */
2235 }
2237 /*
2238 * Stage 3: start the IO. Check for uptodateness
2239 * inside the buffer lock in case another process reading
2240 * the underlying blockdev brought it uptodate (the sct fix).
2241 */
2246 else
2248 }
2250 }
2253 /* utility function for filesystems that need to do work on expanding
2254 * truncates. Uses filesystem pagecache writes to allow the filesystem to
2255 * deal with the hole.
2256 */
2258 {
2277 out:
2279 }
2284 {
2290 loff_t curpos;
2302 }
2306 AOP_FLAG_UNINTERRUPTIBLE,
2319 }
2321 /* page covers the boundary, find the boundary offset */
2324 /* if we will expand the thing last block will be filled */
2327 }
2331 }
2335 AOP_FLAG_UNINTERRUPTIBLE,
2346 }
2347 out:
2349 }
2351 /*
2352 * For moronic filesystems that do not allow holes in file.
2353 * We may have to extend the file.
2354 */
2359 {
2373 }
2376 }
2380 {
2384 }
2387 /*
2388 * block_page_mkwrite() is not allowed to change the file size as it gets
2389 * called from a page fault handler when a page is first dirtied. Hence we must
2390 * be careful to check for EOF conditions here. We set the page up correctly
2391 * for a written page which means we get ENOSPC checking when writing into
2392 * holes and correct delalloc and unwritten extent mapping on filesystems that
2393 * support these features.
2394 *
2395 * We are not allowed to take the i_mutex here so we have to play games to
2396 * protect against truncate races as the page could now be beyond EOF. Because
2397 * truncate writes the inode size before removing pages, once we have the
2398 * page lock we can determine safely if the page is beyond EOF. If it is not
2399 * beyond EOF, then the page is guaranteed safe against truncation until we
2400 * unlock the page.
2401 *
2402 * Direct callers of this function should protect against filesystem freezing
2403 * using sb_start_write() - sb_end_write() functions.
2404 */
2406 get_block_t get_block)
2407 {
2411 loff_t size;
2418 /* We overload EFAULT to mean page got truncated */
2421 }
2423 /* page is wholly or partially inside EOF */
2426 else
2438 out_unlock:
2441 }
2445 get_block_t get_block)
2446 {
2452 /*
2453 * Update file times before taking page lock. We may end up failing the
2454 * fault so this update may be superfluous but who really cares...
2455 */
2461 }
2464 /*
2465 * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2466 * immediately, while under the page lock. So it needs a special end_io
2467 * handler which does not touch the bh after unlocking it.
2468 */
2470 {
2472 }
2474 /*
2475 * Attach the singly-linked list of buffers created by nobh_write_begin, to
2476 * the page (converting it to circular linked list and taking care of page
2477 * dirty races).
2478 */
2480 {
2496 }
2498 /*
2499 * On entry, the page is fully not uptodate.
2500 * On exit the page is fully uptodate in the areas outside (from,to)
2501 * The filesystem needs to handle block truncation upon failure.
2502 */
2507 {
2513 pgoff_t index;
2517 sector_t block_in_file;
2537 }
2542 /*
2543 * Allocate buffers so that we can keep track of state, and potentially
2544 * attach them to the page if an error occurs. In the common case of
2545 * no error, they will just be freed again without ever being attached
2546 * to the page (which is all OK, because we're under the page lock).
2547 *
2548 * Be careful: the buffer linked list is a NULL terminated one, rather
2549 * than the circular one we're used to.
2550 */
2555 }
2559 /*
2560 * We loop across all blocks in the page, whether or not they are
2561 * part of the affected region. This is so we can discover if the
2562 * page is fully mapped-to-disk.
2563 */
2585 }
2590 }
2597 nr_reads++;
2598 }
2599 }
2602 /*
2603 * The page is locked, so these buffers are protected from
2604 * any VM or truncate activity. Hence we don't need to care
2605 * for the buffer_head refcounts.
2606 */
2611 }
2614 }
2623 failed:
2625 /*
2626 * Error recovery is a bit difficult. We need to zero out blocks that
2627 * were newly allocated, and dirty them to ensure they get written out.
2628 * Buffers need to be attached to the page at this point, otherwise
2629 * the handling of potential IO errors during writeout would be hard
2630 * (could try doing synchronous writeout, but what if that fails too?)
2631 */
2635 out_release:
2641 }
2647 {
2664 }
2673 }
2676 }
2679 /*
2680 * nobh_writepage() - based on block_full_write_page() except
2681 * that it tries to operate without attaching bufferheads to
2682 * the page.
2683 */
2686 {
2693 /* Is the page fully inside i_size? */
2697 /* Is the page fully outside i_size? (truncate in progress) */
2700 /*
2701 * The page may have dirty, unmapped buffers. For example,
2702 * they may have been added in ext3_writepage(). Make them
2703 * freeable here, so the page does not leak.
2704 */
2705 #if 0
2706 /* Not really sure about this - do we need this ? */
2709 #endif
2712 }
2714 /*
2715 * The page straddles i_size. It must be zeroed out on each and every
2716 * writepage invocation because it may be mmapped. "A file is mapped
2717 * in multiples of the page size. For a file that is not a multiple of
2718 * the page size, the remaining memory is zeroed when mapped, and
2719 * writes to that region are not written out to the file."
2720 */
2722 out:
2726 end_buffer_async_write);
2728 }
2733 {
2737 sector_t iblock;
2747 /* Block boundary? Nothing to do */
2760 has_buffers:
2764 }
2766 /* Find the buffer that contains "offset" */
2769 iblock++;
2771 }
2778 /* unmapped? It's a hole - nothing to do */
2782 /* Ok, it's mapped. Make sure it's up-to-date */
2788 }
2793 }
2796 }
2801 unlock:
2804 out:
2806 }
2811 {
2815 sector_t iblock;
2825 /* Block boundary? Nothing to do */
2840 /* Find the buffer that contains "offset" */
2845 iblock++;
2847 }
2855 /* unmapped? It's a hole - nothing to do */
2858 }
2860 /* Ok, it's mapped. Make sure it's up-to-date */
2868 /* Uhhuh. Read error. Complain and punt. */
2871 }
2877 unlock:
2880 out:
2882 }
2885 /*
2886 * The generic ->writepage function for buffer-backed address_spaces
2887 */
2890 {
2896 /* Is the page fully inside i_size? */
2899 end_buffer_async_write);
2901 /* Is the page fully outside i_size? (truncate in progress) */
2904 /*
2905 * The page may have dirty, unmapped buffers. For example,
2906 * they may have been added in ext3_writepage(). Make them
2907 * freeable here, so the page does not leak.
2908 */
2912 }
2914 /*
2915 * The page straddles i_size. It must be zeroed out on each and every
2916 * writepage invocation because it may be mmapped. "A file is mapped
2917 * in multiples of the page size. For a file that is not a multiple of
2918 * the page size, the remaining memory is zeroed when mapped, and
2919 * writes to that region are not written out to the file."
2920 */
2923 end_buffer_async_write);
2924 }
2929 {
2937 }
2941 {
2946 }
2953 }
2955 /*
2956 * This allows us to do IO even on the odd last sectors
2957 * of a device, even if the bh block size is some multiple
2958 * of the physical sector size.
2959 *
2960 * We'll just truncate the bio to the size of the device,
2961 * and clear the end of the buffer head manually.
2962 *
2963 * Truly out-of-range accesses will turn into actual IO
2964 * errors, this only handles the "we need to be able to
2965 * do IO at the final sector" case.
2966 */
2968 {
2969 sector_t maxsector;
2976 /*
2977 * If the *whole* IO is past the end of the device,
2978 * let it through, and the IO layer will turn it into
2979 * an EIO.
2980 */
2989 /* Uhhuh. We've got a bh that straddles the device size! */
2992 /* Truncate the bio.. */
2996 /* ..and clear the end of the buffer for reads */
3002 }
3003 }
3006 {
3016 /*
3017 * Only clear out a write error when rewriting
3018 */
3022 /*
3023 * from here on down, it's all bio -- do the initial mapping,
3024 * submit_bio -> generic_make_request may further map this bio around
3025 */
3041 /* Take care of bh's that straddle the end of the device */
3057 }
3061 {
3063 }
3066 /**
3067 * ll_rw_block: low-level access to block devices (DEPRECATED)
3068 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
3069 * @nr: number of &struct buffer_heads in the array
3070 * @bhs: array of pointers to &struct buffer_head
3071 *
3072 * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3073 * requests an I/O operation on them, either a %READ or a %WRITE. The third
3074 * %READA option is described in the documentation for generic_make_request()
3075 * which ll_rw_block() calls.
3076 *
3077 * This function drops any buffer that it cannot get a lock on (with the
3078 * BH_Lock state bit), any buffer that appears to be clean when doing a write
3079 * request, and any buffer that appears to be up-to-date when doing read
3080 * request. Further it marks as clean buffers that are processed for
3081 * writing (the buffer cache won't assume that they are actually clean
3082 * until the buffer gets unlocked).
3083 *
3084 * ll_rw_block sets b_end_io to simple completion handler that marks
3085 * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3086 * any waiters.
3087 *
3088 * All of the buffers must be for the same device, and must also be a
3089 * multiple of the current approved size for the device.
3090 */
3092 {
3106 }
3113 }
3114 }
3116 }
3117 }
3121 {
3126 }
3130 }
3133 /*
3134 * For a data-integrity writeout, we need to wait upon any in-progress I/O
3135 * and then start new I/O and then wait upon it. The caller must have a ref on
3136 * the buffer_head.
3137 */
3139 {
3153 }
3155 }
3159 {
3161 }
3164 /*
3165 * try_to_free_buffers() checks if all the buffers on this particular page
3166 * are unused, and releases them if so.
3167 *
3168 * Exclusion against try_to_free_buffers may be obtained by either
3169 * locking the page or by holding its mapping's private_lock.
3170 *
3171 * If the page is dirty but all the buffers are clean then we need to
3172 * be sure to mark the page clean as well. This is because the page
3173 * may be against a block device, and a later reattachment of buffers
3174 * to a dirty page will set *all* buffers dirty. Which would corrupt
3175 * filesystem data on the same device.
3176 *
3177 * The same applies to regular filesystem pages: if all the buffers are
3178 * clean then we set the page clean and proceed. To do that, we require
3179 * total exclusion from __set_page_dirty_buffers(). That is obtained with
3180 * private_lock.
3181 *
3182 * try_to_free_buffers() is non-blocking.
3183 */
3185 {
3188 }
3190 static int
3192 {
3215 failed:
3217 }
3220 {
3232 }
3237 /*
3238 * If the filesystem writes its buffers by hand (eg ext3)
3239 * then we can have clean buffers against a dirty page. We
3240 * clean the page here; otherwise the VM will never notice
3241 * that the filesystem did any IO at all.
3242 *
3243 * Also, during truncate, discard_buffer will have marked all
3244 * the page's buffers clean. We discover that here and clean
3245 * the page also.
3246 *
3247 * private_lock must be held over this entire operation in order
3248 * to synchronise against __set_page_dirty_buffers and prevent the
3249 * dirty bit from being lost.
3250 */
3254 out:
3263 }
3265 }
3268 /*
3269 * There are no bdflush tunables left. But distributions are
3270 * still running obsolete flush daemons, so we terminate them here.
3271 *
3272 * Use of bdflush() is deprecated and will be removed in a future kernel.
3273 * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3274 */
3276 {
3283 msg_count++;
3285 "warning: process `%s' used the obsolete bdflush"
3288 }
3293 }
3295 /*
3296 * Buffer-head allocation
3297 */
3300 /*
3301 * Once the number of bh's in the machine exceeds this level, we start
3302 * stripping them in writeback.
3303 */
3311 };
3316 {
3326 }
3329 {
3337 }
3339 }
3343 {
3350 }
3354 {
3361 }
3364 }
3368 {
3372 }
3374 /**
3375 * bh_uptodate_or_lock - Test whether the buffer is uptodate
3376 * @bh: struct buffer_head
3377 *
3378 * Return true if the buffer is up-to-date and false,
3379 * with the buffer locked, if not.
3380 */
3382 {
3388 }
3390 }
3393 /**
3394 * bh_submit_read - Submit a locked buffer for reading
3395 * @bh: struct buffer_head
3396 *
3397 * Returns zero on success and -EIO on error.
3398 */
3400 {
3406 }
3415 }
3419 {
3425 SLAB_MEM_SPREAD),
3426 NULL);
3428 /*
3429 * Limit the bh occupancy to 10% of ZONE_NORMAL
3430 */
3434 }