fs/btrfs/locking.c

   1 /*
   2  * Copyright (C) 2008 Oracle.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18 #include <linux/sched.h>
  19 #include <linux/gfp.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/spinlock.h>
  22 #include <linux/page-flags.h>
  23 #include <asm/bug.h>
  24 #include "ctree.h"
  25 #include "extent_io.h"
  26 #include "locking.h"
  27
  28 static inline void spin_nested(struct extent_buffer *eb)
  29 {
  30         spin_lock(&eb->lock);
  31 }
  32
  33 /*
  34  * Setting a lock to blocking will drop the spinlock and set the
  35  * flag that forces other procs who want the lock to wait.  After
  36  * this you can safely schedule with the lock held.
  37  */
  38 void btrfs_set_lock_blocking(struct extent_buffer *eb)
  39 {
  40         if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
  41                 set_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
  42                 spin_unlock(&eb->lock);
  43         }
  44         /* exit with the spin lock released and the bit set */
  45 }
  46
  47 /*
  48  * clearing the blocking flag will take the spinlock again.
  49  * After this you can't safely schedule
  50  */
  51 void btrfs_clear_lock_blocking(struct extent_buffer *eb)
  52 {
  53         if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
  54                 spin_nested(eb);
  55                 clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags);
  56                 smp_mb__after_clear_bit();
  57         }
  58         /* exit with the spin lock held */
  59 }
  60
  61 /*
  62  * unfortunately, many of the places that currently set a lock to blocking
  63  * don't end up blocking for very long, and often they don't block
  64  * at all.  For a dbench 50 run, if we don't spin on the blocking bit
  65  * at all, the context switch rate can jump up to 400,000/sec or more.
  66  *
  67  * So, we're still stuck with this crummy spin on the blocking bit,
  68  * at least until the most common causes of the short blocks
  69  * can be dealt with.
  70  */
  71 static int btrfs_spin_on_block(struct extent_buffer *eb)
  72 {
  73         int i;
  74
  75         for (i = 0; i < 512; i++) {
  76                 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
  77                         return 1;
  78                 if (need_resched())
  79                         break;
  80                 cpu_relax();
  81         }
  82         return 0;
  83 }
  84
  85 /*
  86  * This is somewhat different from trylock.  It will take the
  87  * spinlock but if it finds the lock is set to blocking, it will
  88  * return without the lock held.
  89  *
  90  * returns 1 if it was able to take the lock and zero otherwise
  91  *
  92  * After this call, scheduling is not safe without first calling
  93  * btrfs_set_lock_blocking()
  94  */
  95 int btrfs_try_spin_lock(struct extent_buffer *eb)
  96 {
  97         int i;
  98
  99         if (btrfs_spin_on_block(eb)) {
 100                 spin_nested(eb);
 101                 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
 102                         return 1;
 103                 spin_unlock(&eb->lock);
 104         }
 105         /* spin for a bit on the BLOCKING flag */
 106         for (i = 0; i < 2; i++) {
 107                 cpu_relax();
 108                 if (!btrfs_spin_on_block(eb))
 109                         break;
 110
 111                 spin_nested(eb);
 112                 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
 113                         return 1;
 114                 spin_unlock(&eb->lock);
 115         }
 116         return 0;
 117 }
 118
 119 /*
 120  * the autoremove wake function will return 0 if it tried to wake up
 121  * a process that was already awake, which means that process won't
 122  * count as an exclusive wakeup.  The waitq code will continue waking
 123  * procs until it finds one that was actually sleeping.
 124  *
 125  * For btrfs, this isn't quite what we want.  We want a single proc
 126  * to be notified that the lock is ready for taking.  If that proc
 127  * already happen to be awake, great, it will loop around and try for
 128  * the lock.
 129  *
 130  * So, btrfs_wake_function always returns 1, even when the proc that we
 131  * tried to wake up was already awake.
 132  */
 133 static int btrfs_wake_function(wait_queue_t *wait, unsigned mode,
 134                                int sync, void *key)
 135 {
 136         autoremove_wake_function(wait, mode, sync, key);
 137         return 1;
 138 }
 139
 140 /*
 141  * returns with the extent buffer spinlocked.
 142  *
 143  * This will spin and/or wait as required to take the lock, and then
 144  * return with the spinlock held.
 145  *
 146  * After this call, scheduling is not safe without first calling
 147  * btrfs_set_lock_blocking()
 148  */
 149 int btrfs_tree_lock(struct extent_buffer *eb)
 150 {
 151         DEFINE_WAIT(wait);
 152         wait.func = btrfs_wake_function;
 153
 154         if (!btrfs_spin_on_block(eb))
 155                 goto sleep;
 156
 157         while(1) {
 158                 spin_nested(eb);
 159
 160                 /* nobody is blocking, exit with the spinlock held */
 161                 if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
 162                         return 0;
 163
 164                 /*
 165                  * we have the spinlock, but the real owner is blocking.
 166                  * wait for them
 167                  */
 168                 spin_unlock(&eb->lock);
 169
 170                 /*
 171                  * spin for a bit, and if the blocking flag goes away,
 172                  * loop around
 173                  */
 174                 cpu_relax();
 175                 if (btrfs_spin_on_block(eb))
 176                         continue;
 177 sleep:
 178                 prepare_to_wait_exclusive(&eb->lock_wq, &wait,
 179                                           TASK_UNINTERRUPTIBLE);
 180
 181                 if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
 182                         schedule();
 183
 184                 finish_wait(&eb->lock_wq, &wait);
 185         }
 186         return 0;
 187 }
 188
 189 /*
 190  * Very quick trylock, this does not spin or schedule.  It returns
 191  * 1 with the spinlock held if it was able to take the lock, or it
 192  * returns zero if it was unable to take the lock.
 193  *
 194  * After this call, scheduling is not safe without first calling
 195  * btrfs_set_lock_blocking()
 196  */
 197 int btrfs_try_tree_lock(struct extent_buffer *eb)
 198 {
 199         if (spin_trylock(&eb->lock)) {
 200                 if (test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags)) {
 201                         /*
 202                          * we've got the spinlock, but the real owner is
 203                          * blocking.  Drop the spinlock and return failure
 204                          */
 205                         spin_unlock(&eb->lock);
 206                         return 0;
 207                 }
 208                 return 1;
 209         }
 210         /* someone else has the spinlock giveup */
 211         return 0;
 212 }
 213
 214 int btrfs_tree_unlock(struct extent_buffer *eb)
 215 {
 216         /*
 217          * if we were a blocking owner, we don't have the spinlock held
 218          * just clear the bit and look for waiters
 219          */
 220         if (test_and_clear_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
 221                 smp_mb__after_clear_bit();
 222         else
 223                 spin_unlock(&eb->lock);
 224
 225         if (waitqueue_active(&eb->lock_wq))
 226                 wake_up(&eb->lock_wq);
 227         return 0;
 228 }
 229
 230 void btrfs_assert_tree_locked(struct extent_buffer *eb)
 231 {
 232         if (!test_bit(EXTENT_BUFFER_BLOCKING, &eb->bflags))
 233                 assert_spin_locked(&eb->lock);
 234 }