lib/refcount.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Variant of atomic_t specialized for reference counts.
   4  *
   5  * The interface matches the atomic_t interface (to aid in porting) but only
   6  * provides the few functions one should use for reference counting.
   7  *
   8  * It differs in that the counter saturates at UINT_MAX and will not move once
   9  * there. This avoids wrapping the counter and causing 'spurious'
  10  * use-after-free issues.
  11  *
  12  * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
  13  * and provide only what is strictly required for refcounts.
  14  *
  15  * The increments are fully relaxed; these will not provide ordering. The
  16  * rationale is that whatever is used to obtain the object we're increasing the
  17  * reference count on will provide the ordering. For locked data structures,
  18  * its the lock acquire, for RCU/lockless data structures its the dependent
  19  * load.
  20  *
  21  * Do note that inc_not_zero() provides a control dependency which will order
  22  * future stores against the inc, this ensures we'll never modify the object
  23  * if we did not in fact acquire a reference.
  24  *
  25  * The decrements will provide release order, such that all the prior loads and
  26  * stores will be issued before, it also provides a control dependency, which
  27  * will order us against the subsequent free().
  28  *
  29  * The control dependency is against the load of the cmpxchg (ll/sc) that
  30  * succeeded. This means the stores aren't fully ordered, but this is fine
  31  * because the 1->0 transition indicates no concurrency.
  32  *
  33  * Note that the allocator is responsible for ordering things between free()
  34  * and alloc().
  35  *
  36  */
  37
  38 #include <linux/refcount.h>
  39 #include <linux/bug.h>
  40
  41 #ifdef CONFIG_REFCOUNT_FULL
  42
  43 /**
  44  * refcount_add_not_zero - add a value to a refcount unless it is 0
  45  * @i: the value to add to the refcount
  46  * @r: the refcount
  47  *
  48  * Will saturate at UINT_MAX and WARN.
  49  *
  50  * Provides no memory ordering, it is assumed the caller has guaranteed the
  51  * object memory to be stable (RCU, etc.). It does provide a control dependency
  52  * and thereby orders future stores. See the comment on top.
  53  *
  54  * Use of this function is not recommended for the normal reference counting
  55  * use case in which references are taken and released one at a time.  In these
  56  * cases, refcount_inc(), or one of its variants, should instead be used to
  57  * increment a reference count.
  58  *
  59  * Return: false if the passed refcount is 0, true otherwise
  60  */
  61 bool refcount_add_not_zero(unsigned int i, refcount_t *r)
  62 {
  63         unsigned int new, val = atomic_read(&r->refs);
  64
  65         do {
  66                 if (!val)
  67                         return false;
  68
  69                 if (unlikely(val == UINT_MAX))
  70                         return true;
  71
  72                 new = val + i;
  73                 if (new < val)
  74                         new = UINT_MAX;
  75
  76         } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
  77
  78         WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
  79
  80         return true;
  81 }
  82 EXPORT_SYMBOL(refcount_add_not_zero);
  83
  84 /**
  85  * refcount_add - add a value to a refcount
  86  * @i: the value to add to the refcount
  87  * @r: the refcount
  88  *
  89  * Similar to atomic_add(), but will saturate at UINT_MAX and WARN.
  90  *
  91  * Provides no memory ordering, it is assumed the caller has guaranteed the
  92  * object memory to be stable (RCU, etc.). It does provide a control dependency
  93  * and thereby orders future stores. See the comment on top.
  94  *
  95  * Use of this function is not recommended for the normal reference counting
  96  * use case in which references are taken and released one at a time.  In these
  97  * cases, refcount_inc(), or one of its variants, should instead be used to
  98  * increment a reference count.
  99  */
 100 void refcount_add(unsigned int i, refcount_t *r)
 101 {
 102         WARN_ONCE(!refcount_add_not_zero(i, r), "refcount_t: addition on 0; use-after-free.\n");
 103 }
 104 EXPORT_SYMBOL(refcount_add);
 105
 106 /**
 107  * refcount_inc_not_zero - increment a refcount unless it is 0
 108  * @r: the refcount to increment
 109  *
 110  * Similar to atomic_inc_not_zero(), but will saturate at UINT_MAX and WARN.
 111  *
 112  * Provides no memory ordering, it is assumed the caller has guaranteed the
 113  * object memory to be stable (RCU, etc.). It does provide a control dependency
 114  * and thereby orders future stores. See the comment on top.
 115  *
 116  * Return: true if the increment was successful, false otherwise
 117  */
 118 bool refcount_inc_not_zero(refcount_t *r)
 119 {
 120         unsigned int new, val = atomic_read(&r->refs);
 121
 122         do {
 123                 new = val + 1;
 124
 125                 if (!val)
 126                         return false;
 127
 128                 if (unlikely(!new))
 129                         return true;
 130
 131         } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
 132
 133         WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
 134
 135         return true;
 136 }
 137 EXPORT_SYMBOL(refcount_inc_not_zero);
 138
 139 /**
 140  * refcount_inc - increment a refcount
 141  * @r: the refcount to increment
 142  *
 143  * Similar to atomic_inc(), but will saturate at UINT_MAX and WARN.
 144  *
 145  * Provides no memory ordering, it is assumed the caller already has a
 146  * reference on the object.
 147  *
 148  * Will WARN if the refcount is 0, as this represents a possible use-after-free
 149  * condition.
 150  */
 151 void refcount_inc(refcount_t *r)
 152 {
 153         WARN_ONCE(!refcount_inc_not_zero(r), "refcount_t: increment on 0; use-after-free.\n");
 154 }
 155 EXPORT_SYMBOL(refcount_inc);
 156
 157 /**
 158  * refcount_sub_and_test - subtract from a refcount and test if it is 0
 159  * @i: amount to subtract from the refcount
 160  * @r: the refcount
 161  *
 162  * Similar to atomic_dec_and_test(), but it will WARN, return false and
 163  * ultimately leak on underflow and will fail to decrement when saturated
 164  * at UINT_MAX.
 165  *
 166  * Provides release memory ordering, such that prior loads and stores are done
 167  * before, and provides a control dependency such that free() must come after.
 168  * See the comment on top.
 169  *
 170  * Use of this function is not recommended for the normal reference counting
 171  * use case in which references are taken and released one at a time.  In these
 172  * cases, refcount_dec(), or one of its variants, should instead be used to
 173  * decrement a reference count.
 174  *
 175  * Return: true if the resulting refcount is 0, false otherwise
 176  */
 177 bool refcount_sub_and_test(unsigned int i, refcount_t *r)
 178 {
 179         unsigned int new, val = atomic_read(&r->refs);
 180
 181         do {
 182                 if (unlikely(val == UINT_MAX))
 183                         return false;
 184
 185                 new = val - i;
 186                 if (new > val) {
 187                         WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
 188                         return false;
 189                 }
 190
 191         } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
 192
 193         return !new;
 194 }
 195 EXPORT_SYMBOL(refcount_sub_and_test);
 196
 197 /**
 198  * refcount_dec_and_test - decrement a refcount and test if it is 0
 199  * @r: the refcount
 200  *
 201  * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
 202  * decrement when saturated at UINT_MAX.
 203  *
 204  * Provides release memory ordering, such that prior loads and stores are done
 205  * before, and provides a control dependency such that free() must come after.
 206  * See the comment on top.
 207  *
 208  * Return: true if the resulting refcount is 0, false otherwise
 209  */
 210 bool refcount_dec_and_test(refcount_t *r)
 211 {
 212         return refcount_sub_and_test(1, r);
 213 }
 214 EXPORT_SYMBOL(refcount_dec_and_test);
 215
 216 /**
 217  * refcount_dec - decrement a refcount
 218  * @r: the refcount
 219  *
 220  * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
 221  * when saturated at UINT_MAX.
 222  *
 223  * Provides release memory ordering, such that prior loads and stores are done
 224  * before.
 225  */
 226 void refcount_dec(refcount_t *r)
 227 {
 228         WARN_ONCE(refcount_dec_and_test(r), "refcount_t: decrement hit 0; leaking memory.\n");
 229 }
 230 EXPORT_SYMBOL(refcount_dec);
 231 #endif /* CONFIG_REFCOUNT_FULL */
 232
 233 /**
 234  * refcount_dec_if_one - decrement a refcount if it is 1
 235  * @r: the refcount
 236  *
 237  * No atomic_t counterpart, it attempts a 1 -> 0 transition and returns the
 238  * success thereof.
 239  *
 240  * Like all decrement operations, it provides release memory order and provides
 241  * a control dependency.
 242  *
 243  * It can be used like a try-delete operator; this explicit case is provided
 244  * and not cmpxchg in generic, because that would allow implementing unsafe
 245  * operations.
 246  *
 247  * Return: true if the resulting refcount is 0, false otherwise
 248  */
 249 bool refcount_dec_if_one(refcount_t *r)
 250 {
 251         int val = 1;
 252
 253         return atomic_try_cmpxchg_release(&r->refs, &val, 0);
 254 }
 255 EXPORT_SYMBOL(refcount_dec_if_one);
 256
 257 /**
 258  * refcount_dec_not_one - decrement a refcount if it is not 1
 259  * @r: the refcount
 260  *
 261  * No atomic_t counterpart, it decrements unless the value is 1, in which case
 262  * it will return false.
 263  *
 264  * Was often done like: atomic_add_unless(&var, -1, 1)
 265  *
 266  * Return: true if the decrement operation was successful, false otherwise
 267  */
 268 bool refcount_dec_not_one(refcount_t *r)
 269 {
 270         unsigned int new, val = atomic_read(&r->refs);
 271
 272         do {
 273                 if (unlikely(val == UINT_MAX))
 274                         return true;
 275
 276                 if (val == 1)
 277                         return false;
 278
 279                 new = val - 1;
 280                 if (new > val) {
 281                         WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
 282                         return true;
 283                 }
 284
 285         } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
 286
 287         return true;
 288 }
 289 EXPORT_SYMBOL(refcount_dec_not_one);
 290
 291 /**
 292  * refcount_dec_and_mutex_lock - return holding mutex if able to decrement
 293  *                               refcount to 0
 294  * @r: the refcount
 295  * @lock: the mutex to be locked
 296  *
 297  * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail
 298  * to decrement when saturated at UINT_MAX.
 299  *
 300  * Provides release memory ordering, such that prior loads and stores are done
 301  * before, and provides a control dependency such that free() must come after.
 302  * See the comment on top.
 303  *
 304  * Return: true and hold mutex if able to decrement refcount to 0, false
 305  *         otherwise
 306  */
 307 bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock)
 308 {
 309         if (refcount_dec_not_one(r))
 310                 return false;
 311
 312         mutex_lock(lock);
 313         if (!refcount_dec_and_test(r)) {
 314                 mutex_unlock(lock);
 315                 return false;
 316         }
 317
 318         return true;
 319 }
 320 EXPORT_SYMBOL(refcount_dec_and_mutex_lock);
 321
 322 /**
 323  * refcount_dec_and_lock - return holding spinlock if able to decrement
 324  *                         refcount to 0
 325  * @r: the refcount
 326  * @lock: the spinlock to be locked
 327  *
 328  * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to
 329  * decrement when saturated at UINT_MAX.
 330  *
 331  * Provides release memory ordering, such that prior loads and stores are done
 332  * before, and provides a control dependency such that free() must come after.
 333  * See the comment on top.
 334  *
 335  * Return: true and hold spinlock if able to decrement refcount to 0, false
 336  *         otherwise
 337  */
 338 bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock)
 339 {
 340         if (refcount_dec_not_one(r))
 341                 return false;
 342
 343         spin_lock(lock);
 344         if (!refcount_dec_and_test(r)) {
 345                 spin_unlock(lock);
 346                 return false;
 347         }
 348
 349         return true;
 350 }
 351 EXPORT_SYMBOL(refcount_dec_and_lock);
 352
 353 /**
 354  * refcount_dec_and_lock_irqsave - return holding spinlock with disabled
 355  *                                 interrupts if able to decrement refcount to 0
 356  * @r: the refcount
 357  * @lock: the spinlock to be locked
 358  * @flags: saved IRQ-flags if the is acquired
 359  *
 360  * Same as refcount_dec_and_lock() above except that the spinlock is acquired
 361  * with disabled interupts.
 362  *
 363  * Return: true and hold spinlock if able to decrement refcount to 0, false
 364  *         otherwise
 365  */
 366 bool refcount_dec_and_lock_irqsave(refcount_t *r, spinlock_t *lock,
 367                                    unsigned long *flags)
 368 {
 369         if (refcount_dec_not_one(r))
 370                 return false;
 371
 372         spin_lock_irqsave(lock, *flags);
 373         if (!refcount_dec_and_test(r)) {
 374                 spin_unlock_irqrestore(lock, *flags);
 375                 return false;
 376         }
 377
 378         return true;
 379 }
 380 EXPORT_SYMBOL(refcount_dec_and_lock_irqsave);