module/zfs/refcount.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012, 2021 by Delphix. All rights reserved.
  24  */
  25
  26 #include <sys/zfs_context.h>
  27 #include <sys/zfs_refcount.h>
  28
  29 #ifdef  ZFS_DEBUG
  30 /*
  31  * Reference count tracking is disabled by default.  It's memory requirements
  32  * are reasonable, however as implemented it consumes a significant amount of
  33  * cpu time.  Until its performance is improved it should be manually enabled.
  34  */
  35 int reference_tracking_enable = B_FALSE;
  36 static uint_t reference_history = 3; /* tunable */
  37
  38 static kmem_cache_t *reference_cache;
  39 static kmem_cache_t *reference_history_cache;
  40
  41 void
  42 zfs_refcount_init(void)
  43 {
  44         reference_cache = kmem_cache_create("reference_cache",
  45             sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
  46
  47         reference_history_cache = kmem_cache_create("reference_history_cache",
  48             sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
  49 }
  50
  51 void
  52 zfs_refcount_fini(void)
  53 {
  54         kmem_cache_destroy(reference_cache);
  55         kmem_cache_destroy(reference_history_cache);
  56 }
  57
  58 void
  59 zfs_refcount_create(zfs_refcount_t *rc)
  60 {
  61         mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);
  62         list_create(&rc->rc_list, sizeof (reference_t),
  63             offsetof(reference_t, ref_link));
  64         list_create(&rc->rc_removed, sizeof (reference_t),
  65             offsetof(reference_t, ref_link));
  66         rc->rc_count = 0;
  67         rc->rc_removed_count = 0;
  68         rc->rc_tracked = reference_tracking_enable;
  69 }
  70
  71 void
  72 zfs_refcount_create_tracked(zfs_refcount_t *rc)
  73 {
  74         zfs_refcount_create(rc);
  75         rc->rc_tracked = B_TRUE;
  76 }
  77
  78 void
  79 zfs_refcount_create_untracked(zfs_refcount_t *rc)
  80 {
  81         zfs_refcount_create(rc);
  82         rc->rc_tracked = B_FALSE;
  83 }
  84
  85 void
  86 zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number)
  87 {
  88         reference_t *ref;
  89
  90         ASSERT3U(rc->rc_count, ==, number);
  91         while ((ref = list_head(&rc->rc_list))) {
  92                 list_remove(&rc->rc_list, ref);
  93                 kmem_cache_free(reference_cache, ref);
  94         }
  95         list_destroy(&rc->rc_list);
  96
  97         while ((ref = list_head(&rc->rc_removed))) {
  98                 list_remove(&rc->rc_removed, ref);
  99                 kmem_cache_free(reference_history_cache, ref->ref_removed);
 100                 kmem_cache_free(reference_cache, ref);
 101         }
 102         list_destroy(&rc->rc_removed);
 103         mutex_destroy(&rc->rc_mtx);
 104 }
 105
 106 void
 107 zfs_refcount_destroy(zfs_refcount_t *rc)
 108 {
 109         zfs_refcount_destroy_many(rc, 0);
 110 }
 111
 112 int
 113 zfs_refcount_is_zero(zfs_refcount_t *rc)
 114 {
 115         return (zfs_refcount_count(rc) == 0);
 116 }
 117
 118 int64_t
 119 zfs_refcount_count(zfs_refcount_t *rc)
 120 {
 121         return (atomic_load_64(&rc->rc_count));
 122 }
 123
 124 int64_t
 125 zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder)
 126 {
 127         reference_t *ref = NULL;
 128         int64_t count;
 129
 130         if (!rc->rc_tracked) {
 131                 count = atomic_add_64_nv(&(rc)->rc_count, number);
 132                 ASSERT3U(count, >=, number);
 133                 return (count);
 134         }
 135
 136         ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
 137         ref->ref_holder = holder;
 138         ref->ref_number = number;
 139         mutex_enter(&rc->rc_mtx);
 140         list_insert_head(&rc->rc_list, ref);
 141         rc->rc_count += number;
 142         count = rc->rc_count;
 143         mutex_exit(&rc->rc_mtx);
 144
 145         return (count);
 146 }
 147
 148 int64_t
 149 zfs_refcount_add(zfs_refcount_t *rc, const void *holder)
 150 {
 151         return (zfs_refcount_add_many(rc, 1, holder));
 152 }
 153
 154 int64_t
 155 zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
 156     const void *holder)
 157 {
 158         reference_t *ref;
 159         int64_t count;
 160
 161         if (!rc->rc_tracked) {
 162                 count = atomic_add_64_nv(&(rc)->rc_count, -number);
 163                 ASSERT3S(count, >=, 0);
 164                 return (count);
 165         }
 166
 167         mutex_enter(&rc->rc_mtx);
 168         ASSERT3U(rc->rc_count, >=, number);
 169         for (ref = list_head(&rc->rc_list); ref;
 170             ref = list_next(&rc->rc_list, ref)) {
 171                 if (ref->ref_holder == holder && ref->ref_number == number) {
 172                         list_remove(&rc->rc_list, ref);
 173                         if (reference_history > 0) {
 174                                 ref->ref_removed =
 175                                     kmem_cache_alloc(reference_history_cache,
 176                                     KM_SLEEP);
 177                                 list_insert_head(&rc->rc_removed, ref);
 178                                 rc->rc_removed_count++;
 179                                 if (rc->rc_removed_count > reference_history) {
 180                                         ref = list_tail(&rc->rc_removed);
 181                                         list_remove(&rc->rc_removed, ref);
 182                                         kmem_cache_free(reference_history_cache,
 183                                             ref->ref_removed);
 184                                         kmem_cache_free(reference_cache, ref);
 185                                         rc->rc_removed_count--;
 186                                 }
 187                         } else {
 188                                 kmem_cache_free(reference_cache, ref);
 189                         }
 190                         rc->rc_count -= number;
 191                         count = rc->rc_count;
 192                         mutex_exit(&rc->rc_mtx);
 193                         return (count);
 194                 }
 195         }
 196         panic("No such hold %p on refcount %llx", holder,
 197             (u_longlong_t)(uintptr_t)rc);
 198         return (-1);
 199 }
 200
 201 int64_t
 202 zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)
 203 {
 204         return (zfs_refcount_remove_many(rc, 1, holder));
 205 }
 206
 207 void
 208 zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
 209 {
 210         int64_t count, removed_count;
 211         list_t list, removed;
 212
 213         list_create(&list, sizeof (reference_t),
 214             offsetof(reference_t, ref_link));
 215         list_create(&removed, sizeof (reference_t),
 216             offsetof(reference_t, ref_link));
 217
 218         mutex_enter(&src->rc_mtx);
 219         count = src->rc_count;
 220         removed_count = src->rc_removed_count;
 221         src->rc_count = 0;
 222         src->rc_removed_count = 0;
 223         list_move_tail(&list, &src->rc_list);
 224         list_move_tail(&removed, &src->rc_removed);
 225         mutex_exit(&src->rc_mtx);
 226
 227         mutex_enter(&dst->rc_mtx);
 228         dst->rc_count += count;
 229         dst->rc_removed_count += removed_count;
 230         list_move_tail(&dst->rc_list, &list);
 231         list_move_tail(&dst->rc_removed, &removed);
 232         mutex_exit(&dst->rc_mtx);
 233
 234         list_destroy(&list);
 235         list_destroy(&removed);
 236 }
 237
 238 void
 239 zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,
 240     const void *current_holder, const void *new_holder)
 241 {
 242         reference_t *ref;
 243         boolean_t found = B_FALSE;
 244
 245         if (!rc->rc_tracked)
 246                 return;
 247
 248         mutex_enter(&rc->rc_mtx);
 249         for (ref = list_head(&rc->rc_list); ref;
 250             ref = list_next(&rc->rc_list, ref)) {
 251                 if (ref->ref_holder == current_holder &&
 252                     ref->ref_number == number) {
 253                         ref->ref_holder = new_holder;
 254                         found = B_TRUE;
 255                         break;
 256                 }
 257         }
 258         ASSERT(found);
 259         mutex_exit(&rc->rc_mtx);
 260 }
 261
 262 void
 263 zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder,
 264     const void *new_holder)
 265 {
 266         return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder,
 267             new_holder));
 268 }
 269
 270 /*
 271  * If tracking is enabled, return true if a reference exists that matches
 272  * the "holder" tag. If tracking is disabled, then return true if a reference
 273  * might be held.
 274  */
 275 boolean_t
 276 zfs_refcount_held(zfs_refcount_t *rc, const void *holder)
 277 {
 278         reference_t *ref;
 279
 280         if (!rc->rc_tracked)
 281                 return (zfs_refcount_count(rc) > 0);
 282
 283         mutex_enter(&rc->rc_mtx);
 284         for (ref = list_head(&rc->rc_list); ref;
 285             ref = list_next(&rc->rc_list, ref)) {
 286                 if (ref->ref_holder == holder) {
 287                         mutex_exit(&rc->rc_mtx);
 288                         return (B_TRUE);
 289                 }
 290         }
 291         mutex_exit(&rc->rc_mtx);
 292         return (B_FALSE);
 293 }
 294
 295 /*
 296  * If tracking is enabled, return true if a reference does not exist that
 297  * matches the "holder" tag. If tracking is disabled, always return true
 298  * since the reference might not be held.
 299  */
 300 boolean_t
 301 zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder)
 302 {
 303         reference_t *ref;
 304
 305         if (!rc->rc_tracked)
 306                 return (B_TRUE);
 307
 308         mutex_enter(&rc->rc_mtx);
 309         for (ref = list_head(&rc->rc_list); ref;
 310             ref = list_next(&rc->rc_list, ref)) {
 311                 if (ref->ref_holder == holder) {
 312                         mutex_exit(&rc->rc_mtx);
 313                         return (B_FALSE);
 314                 }
 315         }
 316         mutex_exit(&rc->rc_mtx);
 317         return (B_TRUE);
 318 }
 319
 320 EXPORT_SYMBOL(zfs_refcount_create);
 321 EXPORT_SYMBOL(zfs_refcount_destroy);
 322 EXPORT_SYMBOL(zfs_refcount_is_zero);
 323 EXPORT_SYMBOL(zfs_refcount_count);
 324 EXPORT_SYMBOL(zfs_refcount_add);
 325 EXPORT_SYMBOL(zfs_refcount_remove);
 326 EXPORT_SYMBOL(zfs_refcount_held);
 327
 328 /* BEGIN CSTYLED */
 329 ZFS_MODULE_PARAM(zfs, , reference_tracking_enable, INT, ZMOD_RW,
 330         "Track reference holders to refcount_t objects");
 331
 332 ZFS_MODULE_PARAM(zfs, , reference_history, UINT, ZMOD_RW,
 333         "Maximum reference holders being tracked");
 334 /* END CSTYLED */
 335 #endif  /* ZFS_DEBUG */