2 * Copyright (c) 2000-2002,2005 Silicon Graphics, Inc.
3 * Copyright (c) 2008 Dave Chinner
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License as
8 * published by the Free Software Foundation.
10 * This program is distributed in the hope that it would be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "xfs_types.h"
24 #include "xfs_trans.h"
27 #include "xfs_mount.h"
28 #include "xfs_trans_priv.h"
29 #include "xfs_trace.h"
30 #include "xfs_error.h"
34 * Check that the list is sorted as it should be.
41 xfs_log_item_t
*prev_lip
;
43 if (list_empty(&ailp
->xa_ail
))
47 * Check the next and previous entries are valid.
49 ASSERT((lip
->li_flags
& XFS_LI_IN_AIL
) != 0);
50 prev_lip
= list_entry(lip
->li_ail
.prev
, xfs_log_item_t
, li_ail
);
51 if (&prev_lip
->li_ail
!= &ailp
->xa_ail
)
52 ASSERT(XFS_LSN_CMP(prev_lip
->li_lsn
, lip
->li_lsn
) <= 0);
54 prev_lip
= list_entry(lip
->li_ail
.next
, xfs_log_item_t
, li_ail
);
55 if (&prev_lip
->li_ail
!= &ailp
->xa_ail
)
56 ASSERT(XFS_LSN_CMP(prev_lip
->li_lsn
, lip
->li_lsn
) >= 0);
59 #ifdef XFS_TRANS_DEBUG
61 * Walk the list checking lsn ordering, and that every entry has the
62 * XFS_LI_IN_AIL flag set. This is really expensive, so only do it
63 * when specifically debugging the transaction subsystem.
65 prev_lip
= list_entry(&ailp
->xa_ail
, xfs_log_item_t
, li_ail
);
66 list_for_each_entry(lip
, &ailp
->xa_ail
, li_ail
) {
67 if (&prev_lip
->li_ail
!= &ailp
->xa_ail
)
68 ASSERT(XFS_LSN_CMP(prev_lip
->li_lsn
, lip
->li_lsn
) <= 0);
69 ASSERT((lip
->li_flags
& XFS_LI_IN_AIL
) != 0);
72 #endif /* XFS_TRANS_DEBUG */
75 #define xfs_ail_check(a,l)
79 * Return a pointer to the first item in the AIL. If the AIL is empty, then
82 static xfs_log_item_t
*
86 if (list_empty(&ailp
->xa_ail
))
89 return list_first_entry(&ailp
->xa_ail
, xfs_log_item_t
, li_ail
);
93 * Return a pointer to the last item in the AIL. If the AIL is empty, then
96 static xfs_log_item_t
*
100 if (list_empty(&ailp
->xa_ail
))
103 return list_entry(ailp
->xa_ail
.prev
, xfs_log_item_t
, li_ail
);
107 * Return a pointer to the item which follows the given item in the AIL. If
108 * the given item is the last item in the list, then return NULL.
110 static xfs_log_item_t
*
112 struct xfs_ail
*ailp
,
115 if (lip
->li_ail
.next
== &ailp
->xa_ail
)
118 return list_first_entry(&lip
->li_ail
, xfs_log_item_t
, li_ail
);
122 * This is called by the log manager code to determine the LSN of the tail of
123 * the log. This is exactly the LSN of the first item in the AIL. If the AIL
124 * is empty, then this function returns 0.
126 * We need the AIL lock in order to get a coherent read of the lsn of the last
131 struct xfs_ail
*ailp
)
136 spin_lock(&ailp
->xa_lock
);
137 lip
= xfs_ail_min(ailp
);
140 spin_unlock(&ailp
->xa_lock
);
146 * Return the maximum lsn held in the AIL, or zero if the AIL is empty.
150 struct xfs_ail
*ailp
)
155 spin_lock(&ailp
->xa_lock
);
156 lip
= xfs_ail_max(ailp
);
159 spin_unlock(&ailp
->xa_lock
);
165 * The cursor keeps track of where our current traversal is up to by tracking
166 * the next item in the list for us. However, for this to be safe, removing an
167 * object from the AIL needs to invalidate any cursor that points to it. hence
168 * the traversal cursor needs to be linked to the struct xfs_ail so that
169 * deletion can search all the active cursors for invalidation.
172 xfs_trans_ail_cursor_init(
173 struct xfs_ail
*ailp
,
174 struct xfs_ail_cursor
*cur
)
177 list_add_tail(&cur
->list
, &ailp
->xa_cursors
);
181 * Get the next item in the traversal and advance the cursor. If the cursor
182 * was invalidated (indicated by a lip of 1), restart the traversal.
184 struct xfs_log_item
*
185 xfs_trans_ail_cursor_next(
186 struct xfs_ail
*ailp
,
187 struct xfs_ail_cursor
*cur
)
189 struct xfs_log_item
*lip
= cur
->item
;
191 if ((__psint_t
)lip
& 1)
192 lip
= xfs_ail_min(ailp
);
194 cur
->item
= xfs_ail_next(ailp
, lip
);
199 * When the traversal is complete, we need to remove the cursor from the list
200 * of traversing cursors.
203 xfs_trans_ail_cursor_done(
204 struct xfs_ail
*ailp
,
205 struct xfs_ail_cursor
*cur
)
208 list_del_init(&cur
->list
);
212 * Invalidate any cursor that is pointing to this item. This is called when an
213 * item is removed from the AIL. Any cursor pointing to this object is now
214 * invalid and the traversal needs to be terminated so it doesn't reference a
215 * freed object. We set the low bit of the cursor item pointer so we can
216 * distinguish between an invalidation and the end of the list when getting the
217 * next item from the cursor.
220 xfs_trans_ail_cursor_clear(
221 struct xfs_ail
*ailp
,
222 struct xfs_log_item
*lip
)
224 struct xfs_ail_cursor
*cur
;
226 list_for_each_entry(cur
, &ailp
->xa_cursors
, list
) {
227 if (cur
->item
== lip
)
228 cur
->item
= (struct xfs_log_item
*)
229 ((__psint_t
)cur
->item
| 1);
234 * Find the first item in the AIL with the given @lsn by searching in ascending
235 * LSN order and initialise the cursor to point to the next item for a
236 * ascending traversal. Pass a @lsn of zero to initialise the cursor to the
237 * first item in the AIL. Returns NULL if the list is empty.
240 xfs_trans_ail_cursor_first(
241 struct xfs_ail
*ailp
,
242 struct xfs_ail_cursor
*cur
,
247 xfs_trans_ail_cursor_init(ailp
, cur
);
250 lip
= xfs_ail_min(ailp
);
254 list_for_each_entry(lip
, &ailp
->xa_ail
, li_ail
) {
255 if (XFS_LSN_CMP(lip
->li_lsn
, lsn
) >= 0)
262 cur
->item
= xfs_ail_next(ailp
, lip
);
266 static struct xfs_log_item
*
267 __xfs_trans_ail_cursor_last(
268 struct xfs_ail
*ailp
,
273 list_for_each_entry_reverse(lip
, &ailp
->xa_ail
, li_ail
) {
274 if (XFS_LSN_CMP(lip
->li_lsn
, lsn
) <= 0)
281 * Find the last item in the AIL with the given @lsn by searching in descending
282 * LSN order and initialise the cursor to point to that item. If there is no
283 * item with the value of @lsn, then it sets the cursor to the last item with an
284 * LSN lower than @lsn. Returns NULL if the list is empty.
286 struct xfs_log_item
*
287 xfs_trans_ail_cursor_last(
288 struct xfs_ail
*ailp
,
289 struct xfs_ail_cursor
*cur
,
292 xfs_trans_ail_cursor_init(ailp
, cur
);
293 cur
->item
= __xfs_trans_ail_cursor_last(ailp
, lsn
);
298 * Splice the log item list into the AIL at the given LSN. We splice to the
299 * tail of the given LSN to maintain insert order for push traversals. The
300 * cursor is optional, allowing repeated updates to the same LSN to avoid
301 * repeated traversals. This should not be called with an empty list.
305 struct xfs_ail
*ailp
,
306 struct xfs_ail_cursor
*cur
,
307 struct list_head
*list
,
310 struct xfs_log_item
*lip
;
312 ASSERT(!list_empty(list
));
315 * Use the cursor to determine the insertion point if one is
316 * provided. If not, or if the one we got is not valid,
317 * find the place in the AIL where the items belong.
319 lip
= cur
? cur
->item
: NULL
;
320 if (!lip
|| (__psint_t
) lip
& 1)
321 lip
= __xfs_trans_ail_cursor_last(ailp
, lsn
);
324 * If a cursor is provided, we know we're processing the AIL
325 * in lsn order, and future items to be spliced in will
326 * follow the last one being inserted now. Update the
327 * cursor to point to that last item, now while we have a
328 * reliable pointer to it.
331 cur
->item
= list_entry(list
->prev
, struct xfs_log_item
, li_ail
);
334 * Finally perform the splice. Unless the AIL was empty,
335 * lip points to the item in the AIL _after_ which the new
336 * items should go. If lip is null the AIL was empty, so
337 * the new items go at the head of the AIL.
340 list_splice(list
, &lip
->li_ail
);
342 list_splice(list
, &ailp
->xa_ail
);
346 * Delete the given item from the AIL. Return a pointer to the item.
350 struct xfs_ail
*ailp
,
353 xfs_ail_check(ailp
, lip
);
354 list_del(&lip
->li_ail
);
355 xfs_trans_ail_cursor_clear(ailp
, lip
);
360 struct xfs_ail
*ailp
)
362 xfs_mount_t
*mp
= ailp
->xa_mount
;
363 struct xfs_ail_cursor cur
;
370 int push_xfsbufd
= 0;
373 * If last time we ran we encountered pinned items, force the log first
374 * and wait for it before pushing again.
376 spin_lock(&ailp
->xa_lock
);
377 if (ailp
->xa_last_pushed_lsn
== 0 && ailp
->xa_log_flush
&&
378 !list_empty(&ailp
->xa_ail
)) {
379 ailp
->xa_log_flush
= 0;
380 spin_unlock(&ailp
->xa_lock
);
381 XFS_STATS_INC(xs_push_ail_flush
);
382 xfs_log_force(mp
, XFS_LOG_SYNC
);
383 spin_lock(&ailp
->xa_lock
);
386 target
= ailp
->xa_target
;
387 lip
= xfs_trans_ail_cursor_first(ailp
, &cur
, ailp
->xa_last_pushed_lsn
);
388 if (!lip
|| XFS_FORCED_SHUTDOWN(mp
)) {
390 * AIL is empty or our push has reached the end.
392 xfs_trans_ail_cursor_done(ailp
, &cur
);
393 spin_unlock(&ailp
->xa_lock
);
397 XFS_STATS_INC(xs_push_ail
);
400 * While the item we are looking at is below the given threshold
401 * try to flush it out. We'd like not to stop until we've at least
402 * tried to push on everything in the AIL with an LSN less than
403 * the given threshold.
405 * However, we will stop after a certain number of pushes and wait
406 * for a reduced timeout to fire before pushing further. This
407 * prevents use from spinning when we can't do anything or there is
408 * lots of contention on the AIL lists.
411 while ((XFS_LSN_CMP(lip
->li_lsn
, target
) <= 0)) {
414 * If we can lock the item without sleeping, unlock the AIL
415 * lock and flush the item. Then re-grab the AIL lock so we
416 * can look for the next item on the AIL. List changes are
417 * handled by the AIL lookup functions internally
419 * If we can't lock the item, either its holder will flush it
420 * or it is already being flushed or it is being relogged. In
421 * any of these case it is being taken care of and we can just
422 * skip to the next item in the list.
424 lock_result
= IOP_TRYLOCK(lip
);
425 spin_unlock(&ailp
->xa_lock
);
426 switch (lock_result
) {
427 case XFS_ITEM_SUCCESS
:
428 XFS_STATS_INC(xs_push_ail_success
);
429 trace_xfs_ail_push(lip
);
432 ailp
->xa_last_pushed_lsn
= lsn
;
435 case XFS_ITEM_PUSHBUF
:
436 XFS_STATS_INC(xs_push_ail_pushbuf
);
437 trace_xfs_ail_pushbuf(lip
);
439 if (!IOP_PUSHBUF(lip
)) {
440 trace_xfs_ail_pushbuf_pinned(lip
);
442 ailp
->xa_log_flush
++;
444 ailp
->xa_last_pushed_lsn
= lsn
;
449 case XFS_ITEM_PINNED
:
450 XFS_STATS_INC(xs_push_ail_pinned
);
451 trace_xfs_ail_pinned(lip
);
454 ailp
->xa_log_flush
++;
457 case XFS_ITEM_LOCKED
:
458 XFS_STATS_INC(xs_push_ail_locked
);
459 trace_xfs_ail_locked(lip
);
468 spin_lock(&ailp
->xa_lock
);
469 /* should we bother continuing? */
470 if (XFS_FORCED_SHUTDOWN(mp
))
477 * Are there too many items we can't do anything with?
478 * If we we are skipping too many items because we can't flush
479 * them or they are already being flushed, we back off and
480 * given them time to complete whatever operation is being
481 * done. i.e. remove pressure from the AIL while we can't make
482 * progress so traversals don't slow down further inserts and
483 * removals to/from the AIL.
485 * The value of 100 is an arbitrary magic number based on
491 lip
= xfs_trans_ail_cursor_next(ailp
, &cur
);
496 xfs_trans_ail_cursor_done(ailp
, &cur
);
497 spin_unlock(&ailp
->xa_lock
);
500 /* we've got delayed write buffers to flush */
501 wake_up_process(mp
->m_ddev_targp
->bt_task
);
504 /* assume we have more work to do in a short while */
507 /* We're past our target or empty, so idle */
508 ailp
->xa_last_pushed_lsn
= 0;
509 ailp
->xa_log_flush
= 0;
512 } else if (XFS_LSN_CMP(lsn
, target
) >= 0) {
514 * We reached the target so wait a bit longer for I/O to
515 * complete and remove pushed items from the AIL before we
516 * start the next scan from the start of the AIL.
519 ailp
->xa_last_pushed_lsn
= 0;
520 } else if ((stuck
* 100) / count
> 90) {
522 * Either there is a lot of contention on the AIL or we
523 * are stuck due to operations in progress. "Stuck" in this
524 * case is defined as >90% of the items we tried to push
527 * Backoff a bit more to allow some I/O to complete before
528 * restarting from the start of the AIL. This prevents us
529 * from spinning on the same items, and if they are pinned will
530 * all the restart to issue a log force to unpin the stuck
534 ailp
->xa_last_pushed_lsn
= 0;
544 struct xfs_ail
*ailp
= data
;
545 long tout
= 0; /* milliseconds */
547 while (!kthread_should_stop()) {
548 if (tout
&& tout
<= 20)
549 __set_current_state(TASK_KILLABLE
);
551 __set_current_state(TASK_INTERRUPTIBLE
);
552 schedule_timeout(tout
?
553 msecs_to_jiffies(tout
) : MAX_SCHEDULE_TIMEOUT
);
557 tout
= xfsaild_push(ailp
);
564 * This routine is called to move the tail of the AIL forward. It does this by
565 * trying to flush items in the AIL whose lsns are below the given
568 * The push is run asynchronously in a workqueue, which means the caller needs
569 * to handle waiting on the async flush for space to become available.
570 * We don't want to interrupt any push that is in progress, hence we only queue
571 * work if we set the pushing bit approriately.
573 * We do this unlocked - we only need to know whether there is anything in the
574 * AIL at the time we are called. We don't need to access the contents of
575 * any of the objects, so the lock is not needed.
579 struct xfs_ail
*ailp
,
580 xfs_lsn_t threshold_lsn
)
584 lip
= xfs_ail_min(ailp
);
585 if (!lip
|| XFS_FORCED_SHUTDOWN(ailp
->xa_mount
) ||
586 XFS_LSN_CMP(threshold_lsn
, ailp
->xa_target
) <= 0)
590 * Ensure that the new target is noticed in push code before it clears
591 * the XFS_AIL_PUSHING_BIT.
594 xfs_trans_ail_copy_lsn(ailp
, &ailp
->xa_target
, &threshold_lsn
);
597 wake_up_process(ailp
->xa_task
);
601 * Push out all items in the AIL immediately
605 struct xfs_ail
*ailp
)
607 xfs_lsn_t threshold_lsn
= xfs_ail_max_lsn(ailp
);
610 xfs_ail_push(ailp
, threshold_lsn
);
614 * xfs_trans_ail_update - bulk AIL insertion operation.
616 * @xfs_trans_ail_update takes an array of log items that all need to be
617 * positioned at the same LSN in the AIL. If an item is not in the AIL, it will
618 * be added. Otherwise, it will be repositioned by removing it and re-adding
619 * it to the AIL. If we move the first item in the AIL, update the log tail to
620 * match the new minimum LSN in the AIL.
622 * This function takes the AIL lock once to execute the update operations on
623 * all the items in the array, and as such should not be called with the AIL
624 * lock held. As a result, once we have the AIL lock, we need to check each log
625 * item LSN to confirm it needs to be moved forward in the AIL.
627 * To optimise the insert operation, we delete all the items from the AIL in
628 * the first pass, moving them into a temporary list, then splice the temporary
629 * list into the correct position in the AIL. This avoids needing to do an
630 * insert operation on every item.
632 * This function must be called with the AIL lock held. The lock is dropped
636 xfs_trans_ail_update_bulk(
637 struct xfs_ail
*ailp
,
638 struct xfs_ail_cursor
*cur
,
639 struct xfs_log_item
**log_items
,
641 xfs_lsn_t lsn
) __releases(ailp
->xa_lock
)
643 xfs_log_item_t
*mlip
;
644 int mlip_changed
= 0;
648 ASSERT(nr_items
> 0); /* Not required, but true. */
649 mlip
= xfs_ail_min(ailp
);
651 for (i
= 0; i
< nr_items
; i
++) {
652 struct xfs_log_item
*lip
= log_items
[i
];
653 if (lip
->li_flags
& XFS_LI_IN_AIL
) {
654 /* check if we really need to move the item */
655 if (XFS_LSN_CMP(lsn
, lip
->li_lsn
) <= 0)
658 xfs_ail_delete(ailp
, lip
);
662 lip
->li_flags
|= XFS_LI_IN_AIL
;
665 list_add(&lip
->li_ail
, &tmp
);
668 if (!list_empty(&tmp
))
669 xfs_ail_splice(ailp
, cur
, &tmp
, lsn
);
670 spin_unlock(&ailp
->xa_lock
);
672 if (mlip_changed
&& !XFS_FORCED_SHUTDOWN(ailp
->xa_mount
)) {
673 xlog_assign_tail_lsn(ailp
->xa_mount
);
674 xfs_log_space_wake(ailp
->xa_mount
);
679 * xfs_trans_ail_delete_bulk - remove multiple log items from the AIL
681 * @xfs_trans_ail_delete_bulk takes an array of log items that all need to
682 * removed from the AIL. The caller is already holding the AIL lock, and done
683 * all the checks necessary to ensure the items passed in via @log_items are
684 * ready for deletion. This includes checking that the items are in the AIL.
686 * For each log item to be removed, unlink it from the AIL, clear the IN_AIL
687 * flag from the item and reset the item's lsn to 0. If we remove the first
688 * item in the AIL, update the log tail to match the new minimum LSN in the
691 * This function will not drop the AIL lock until all items are removed from
692 * the AIL to minimise the amount of lock traffic on the AIL. This does not
693 * greatly increase the AIL hold time, but does significantly reduce the amount
694 * of traffic on the lock, especially during IO completion.
696 * This function must be called with the AIL lock held. The lock is dropped
700 xfs_trans_ail_delete_bulk(
701 struct xfs_ail
*ailp
,
702 struct xfs_log_item
**log_items
,
703 int nr_items
) __releases(ailp
->xa_lock
)
705 xfs_log_item_t
*mlip
;
706 int mlip_changed
= 0;
709 mlip
= xfs_ail_min(ailp
);
711 for (i
= 0; i
< nr_items
; i
++) {
712 struct xfs_log_item
*lip
= log_items
[i
];
713 if (!(lip
->li_flags
& XFS_LI_IN_AIL
)) {
714 struct xfs_mount
*mp
= ailp
->xa_mount
;
716 spin_unlock(&ailp
->xa_lock
);
717 if (!XFS_FORCED_SHUTDOWN(mp
)) {
718 xfs_alert_tag(mp
, XFS_PTAG_AILDELETE
,
719 "%s: attempting to delete a log item that is not in the AIL",
721 xfs_force_shutdown(mp
, SHUTDOWN_CORRUPT_INCORE
);
726 xfs_ail_delete(ailp
, lip
);
727 lip
->li_flags
&= ~XFS_LI_IN_AIL
;
732 spin_unlock(&ailp
->xa_lock
);
734 if (mlip_changed
&& !XFS_FORCED_SHUTDOWN(ailp
->xa_mount
)) {
735 xlog_assign_tail_lsn(ailp
->xa_mount
);
736 xfs_log_space_wake(ailp
->xa_mount
);
741 * The active item list (AIL) is a doubly linked list of log
742 * items sorted by ascending lsn. The base of the list is
743 * a forw/back pointer pair embedded in the xfs mount structure.
744 * The base is initialized with both pointers pointing to the
745 * base. This case always needs to be distinguished, because
746 * the base has no lsn to look at. We almost always insert
747 * at the end of the list, so on inserts we search from the
748 * end of the list to find where the new item belongs.
752 * Initialize the doubly linked list to point only to itself.
758 struct xfs_ail
*ailp
;
760 ailp
= kmem_zalloc(sizeof(struct xfs_ail
), KM_MAYFAIL
);
765 INIT_LIST_HEAD(&ailp
->xa_ail
);
766 INIT_LIST_HEAD(&ailp
->xa_cursors
);
767 spin_lock_init(&ailp
->xa_lock
);
769 ailp
->xa_task
= kthread_run(xfsaild
, ailp
, "xfsaild/%s",
770 ailp
->xa_mount
->m_fsname
);
771 if (IS_ERR(ailp
->xa_task
))
783 xfs_trans_ail_destroy(
786 struct xfs_ail
*ailp
= mp
->m_ail
;
788 kthread_stop(ailp
->xa_task
);