include/linux/percpu_counter.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_PERCPU_COUNTER_H
   3 #define _LINUX_PERCPU_COUNTER_H
   4 /*
   5  * A simple "approximate counter" for use in ext2 and ext3 superblocks.
   6  *
   7  * WARNING: these things are HUGE.  4 kbytes per counter on 32-way P4.
   8  */
   9
  10 #include <linux/spinlock.h>
  11 #include <linux/smp.h>
  12 #include <linux/list.h>
  13 #include <linux/threads.h>
  14 #include <linux/percpu.h>
  15 #include <linux/types.h>
  16
  17 /* percpu_counter batch for local add or sub */
  18 #define PERCPU_COUNTER_LOCAL_BATCH      INT_MAX
  19
  20 #ifdef CONFIG_SMP
  21
  22 struct percpu_counter {
  23         raw_spinlock_t lock;
  24         s64 count;
  25 #ifdef CONFIG_HOTPLUG_CPU
  26         struct list_head list;  /* All percpu_counters are on a list */
  27 #endif
  28         s32 __percpu *counters;
  29 };
  30
  31 extern int percpu_counter_batch;
  32
  33 int __percpu_counter_init_many(struct percpu_counter *fbc, s64 amount,
  34                                gfp_t gfp, u32 nr_counters,
  35                                struct lock_class_key *key);
  36
  37 #define percpu_counter_init_many(fbc, value, gfp, nr_counters)          \
  38         ({                                                              \
  39                 static struct lock_class_key __key;                     \
  40                                                                         \
  41                 __percpu_counter_init_many(fbc, value, gfp, nr_counters,\
  42                                            &__key);                     \
  43         })
  44
  45
  46 #define percpu_counter_init(fbc, value, gfp)                            \
  47         percpu_counter_init_many(fbc, value, gfp, 1)
  48
  49 void percpu_counter_destroy_many(struct percpu_counter *fbc, u32 nr_counters);
  50 static inline void percpu_counter_destroy(struct percpu_counter *fbc)
  51 {
  52         percpu_counter_destroy_many(fbc, 1);
  53 }
  54
  55 void percpu_counter_set(struct percpu_counter *fbc, s64 amount);
  56 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount,
  57                               s32 batch);
  58 s64 __percpu_counter_sum(struct percpu_counter *fbc);
  59 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch);
  60 bool __percpu_counter_limited_add(struct percpu_counter *fbc, s64 limit,
  61                                   s64 amount, s32 batch);
  62 void percpu_counter_sync(struct percpu_counter *fbc);
  63
  64 static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
  65 {
  66         return __percpu_counter_compare(fbc, rhs, percpu_counter_batch);
  67 }
  68
  69 static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount)
  70 {
  71         percpu_counter_add_batch(fbc, amount, percpu_counter_batch);
  72 }
  73
  74 static inline bool
  75 percpu_counter_limited_add(struct percpu_counter *fbc, s64 limit, s64 amount)
  76 {
  77         return __percpu_counter_limited_add(fbc, limit, amount,
  78                                             percpu_counter_batch);
  79 }
  80
  81 /*
  82  * With percpu_counter_add_local() and percpu_counter_sub_local(), counts
  83  * are accumulated in local per cpu counter and not in fbc->count until
  84  * local count overflows PERCPU_COUNTER_LOCAL_BATCH. This makes counter
  85  * write efficient.
  86  * But percpu_counter_sum(), instead of percpu_counter_read(), needs to be
  87  * used to add up the counts from each CPU to account for all the local
  88  * counts. So percpu_counter_add_local() and percpu_counter_sub_local()
  89  * should be used when a counter is updated frequently and read rarely.
  90  */
  91 static inline void
  92 percpu_counter_add_local(struct percpu_counter *fbc, s64 amount)
  93 {
  94         percpu_counter_add_batch(fbc, amount, PERCPU_COUNTER_LOCAL_BATCH);
  95 }
  96
  97 static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
  98 {
  99         s64 ret = __percpu_counter_sum(fbc);
 100         return ret < 0 ? 0 : ret;
 101 }
 102
 103 static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
 104 {
 105         return __percpu_counter_sum(fbc);
 106 }
 107
 108 static inline s64 percpu_counter_read(struct percpu_counter *fbc)
 109 {
 110         return fbc->count;
 111 }
 112
 113 /*
 114  * It is possible for the percpu_counter_read() to return a small negative
 115  * number for some counter which should never be negative.
 116  *
 117  */
 118 static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
 119 {
 120         /* Prevent reloads of fbc->count */
 121         s64 ret = READ_ONCE(fbc->count);
 122
 123         if (ret >= 0)
 124                 return ret;
 125         return 0;
 126 }
 127
 128 static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
 129 {
 130         return (fbc->counters != NULL);
 131 }
 132
 133 #else /* !CONFIG_SMP */
 134
 135 struct percpu_counter {
 136         s64 count;
 137 };
 138
 139 static inline int percpu_counter_init_many(struct percpu_counter *fbc,
 140                                            s64 amount, gfp_t gfp,
 141                                            u32 nr_counters)
 142 {
 143         u32 i;
 144
 145         for (i = 0; i < nr_counters; i++)
 146                 fbc[i].count = amount;
 147
 148         return 0;
 149 }
 150
 151 static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount,
 152                                       gfp_t gfp)
 153 {
 154         return percpu_counter_init_many(fbc, amount, gfp, 1);
 155 }
 156
 157 static inline void percpu_counter_destroy_many(struct percpu_counter *fbc,
 158                                                u32 nr_counters)
 159 {
 160 }
 161
 162 static inline void percpu_counter_destroy(struct percpu_counter *fbc)
 163 {
 164 }
 165
 166 static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
 167 {
 168         fbc->count = amount;
 169 }
 170
 171 static inline int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs)
 172 {
 173         if (fbc->count > rhs)
 174                 return 1;
 175         else if (fbc->count < rhs)
 176                 return -1;
 177         else
 178                 return 0;
 179 }
 180
 181 static inline int
 182 __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)
 183 {
 184         return percpu_counter_compare(fbc, rhs);
 185 }
 186
 187 static inline void
 188 percpu_counter_add(struct percpu_counter *fbc, s64 amount)
 189 {
 190         unsigned long flags;
 191
 192         local_irq_save(flags);
 193         fbc->count += amount;
 194         local_irq_restore(flags);
 195 }
 196
 197 static inline bool
 198 percpu_counter_limited_add(struct percpu_counter *fbc, s64 limit, s64 amount)
 199 {
 200         unsigned long flags;
 201         bool good = false;
 202         s64 count;
 203
 204         if (amount == 0)
 205                 return true;
 206
 207         local_irq_save(flags);
 208         count = fbc->count + amount;
 209         if ((amount > 0 && count <= limit) ||
 210             (amount < 0 && count >= limit)) {
 211                 fbc->count = count;
 212                 good = true;
 213         }
 214         local_irq_restore(flags);
 215         return good;
 216 }
 217
 218 /* non-SMP percpu_counter_add_local is the same with percpu_counter_add */
 219 static inline void
 220 percpu_counter_add_local(struct percpu_counter *fbc, s64 amount)
 221 {
 222         percpu_counter_add(fbc, amount);
 223 }
 224
 225 static inline void
 226 percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)
 227 {
 228         percpu_counter_add(fbc, amount);
 229 }
 230
 231 static inline s64 percpu_counter_read(struct percpu_counter *fbc)
 232 {
 233         return fbc->count;
 234 }
 235
 236 /*
 237  * percpu_counter is intended to track positive numbers. In the UP case the
 238  * number should never be negative.
 239  */
 240 static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
 241 {
 242         return fbc->count;
 243 }
 244
 245 static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
 246 {
 247         return percpu_counter_read_positive(fbc);
 248 }
 249
 250 static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
 251 {
 252         return percpu_counter_read(fbc);
 253 }
 254
 255 static inline bool percpu_counter_initialized(struct percpu_counter *fbc)
 256 {
 257         return true;
 258 }
 259
 260 static inline void percpu_counter_sync(struct percpu_counter *fbc)
 261 {
 262 }
 263 #endif  /* CONFIG_SMP */
 264
 265 static inline void percpu_counter_inc(struct percpu_counter *fbc)
 266 {
 267         percpu_counter_add(fbc, 1);
 268 }
 269
 270 static inline void percpu_counter_dec(struct percpu_counter *fbc)
 271 {
 272         percpu_counter_add(fbc, -1);
 273 }
 274
 275 static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount)
 276 {
 277         percpu_counter_add(fbc, -amount);
 278 }
 279
 280 static inline void
 281 percpu_counter_sub_local(struct percpu_counter *fbc, s64 amount)
 282 {
 283         percpu_counter_add_local(fbc, -amount);
 284 }
 285
 286 #endif /* _LINUX_PERCPU_COUNTER_H */