include/linux/vmstat.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_VMSTAT_H
   3 #define _LINUX_VMSTAT_H
   4
   5 #include <linux/types.h>
   6 #include <linux/percpu.h>
   7 #include <linux/mmzone.h>
   8 #include <linux/vm_event_item.h>
   9 #include <linux/atomic.h>
  10 #include <linux/static_key.h>
  11
  12 extern int sysctl_stat_interval;
  13
  14 #ifdef CONFIG_NUMA
  15 #define ENABLE_NUMA_STAT   1
  16 #define DISABLE_NUMA_STAT   0
  17 extern int sysctl_vm_numa_stat;
  18 DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
  19 extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
  20                 int write, void __user *buffer, size_t *length, loff_t *ppos);
  21 #endif
  22
  23 #ifdef CONFIG_VM_EVENT_COUNTERS
  24 /*
  25  * Light weight per cpu counter implementation.
  26  *
  27  * Counters should only be incremented and no critical kernel component
  28  * should rely on the counter values.
  29  *
  30  * Counters are handled completely inline. On many platforms the code
  31  * generated will simply be the increment of a global address.
  32  */
  33
  34 struct vm_event_state {
  35         unsigned long event[NR_VM_EVENT_ITEMS];
  36 };
  37
  38 DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
  39
  40 /*
  41  * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
  42  * local_irq_disable overhead.
  43  */
  44 static inline void __count_vm_event(enum vm_event_item item)
  45 {
  46         raw_cpu_inc(vm_event_states.event[item]);
  47 }
  48
  49 static inline void count_vm_event(enum vm_event_item item)
  50 {
  51         this_cpu_inc(vm_event_states.event[item]);
  52 }
  53
  54 static inline void __count_vm_events(enum vm_event_item item, long delta)
  55 {
  56         raw_cpu_add(vm_event_states.event[item], delta);
  57 }
  58
  59 static inline void count_vm_events(enum vm_event_item item, long delta)
  60 {
  61         this_cpu_add(vm_event_states.event[item], delta);
  62 }
  63
  64 extern void all_vm_events(unsigned long *);
  65
  66 extern void vm_events_fold_cpu(int cpu);
  67
  68 #else
  69
  70 /* Disable counters */
  71 static inline void count_vm_event(enum vm_event_item item)
  72 {
  73 }
  74 static inline void count_vm_events(enum vm_event_item item, long delta)
  75 {
  76 }
  77 static inline void __count_vm_event(enum vm_event_item item)
  78 {
  79 }
  80 static inline void __count_vm_events(enum vm_event_item item, long delta)
  81 {
  82 }
  83 static inline void all_vm_events(unsigned long *ret)
  84 {
  85 }
  86 static inline void vm_events_fold_cpu(int cpu)
  87 {
  88 }
  89
  90 #endif /* CONFIG_VM_EVENT_COUNTERS */
  91
  92 #ifdef CONFIG_NUMA_BALANCING
  93 #define count_vm_numa_event(x)     count_vm_event(x)
  94 #define count_vm_numa_events(x, y) count_vm_events(x, y)
  95 #else
  96 #define count_vm_numa_event(x) do {} while (0)
  97 #define count_vm_numa_events(x, y) do { (void)(y); } while (0)
  98 #endif /* CONFIG_NUMA_BALANCING */
  99
 100 #ifdef CONFIG_DEBUG_TLBFLUSH
 101 #define count_vm_tlb_event(x)      count_vm_event(x)
 102 #define count_vm_tlb_events(x, y)  count_vm_events(x, y)
 103 #else
 104 #define count_vm_tlb_event(x)     do {} while (0)
 105 #define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
 106 #endif
 107
 108 #ifdef CONFIG_DEBUG_VM_VMACACHE
 109 #define count_vm_vmacache_event(x) count_vm_event(x)
 110 #else
 111 #define count_vm_vmacache_event(x) do {} while (0)
 112 #endif
 113
 114 #define __count_zid_vm_events(item, zid, delta) \
 115         __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
 116
 117 /*
 118  * Zone and node-based page accounting with per cpu differentials.
 119  */
 120 extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
 121 extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
 122 extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
 123
 124 #ifdef CONFIG_NUMA
 125 static inline void zone_numa_state_add(long x, struct zone *zone,
 126                                  enum numa_stat_item item)
 127 {
 128         atomic_long_add(x, &zone->vm_numa_stat[item]);
 129         atomic_long_add(x, &vm_numa_stat[item]);
 130 }
 131
 132 static inline unsigned long global_numa_state(enum numa_stat_item item)
 133 {
 134         long x = atomic_long_read(&vm_numa_stat[item]);
 135
 136         return x;
 137 }
 138
 139 static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
 140                                         enum numa_stat_item item)
 141 {
 142         long x = atomic_long_read(&zone->vm_numa_stat[item]);
 143         int cpu;
 144
 145         for_each_online_cpu(cpu)
 146                 x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
 147
 148         return x;
 149 }
 150 #endif /* CONFIG_NUMA */
 151
 152 static inline void zone_page_state_add(long x, struct zone *zone,
 153                                  enum zone_stat_item item)
 154 {
 155         atomic_long_add(x, &zone->vm_stat[item]);
 156         atomic_long_add(x, &vm_zone_stat[item]);
 157 }
 158
 159 static inline void node_page_state_add(long x, struct pglist_data *pgdat,
 160                                  enum node_stat_item item)
 161 {
 162         atomic_long_add(x, &pgdat->vm_stat[item]);
 163         atomic_long_add(x, &vm_node_stat[item]);
 164 }
 165
 166 static inline unsigned long global_zone_page_state(enum zone_stat_item item)
 167 {
 168         long x = atomic_long_read(&vm_zone_stat[item]);
 169 #ifdef CONFIG_SMP
 170         if (x < 0)
 171                 x = 0;
 172 #endif
 173         return x;
 174 }
 175
 176 static inline unsigned long global_node_page_state(enum node_stat_item item)
 177 {
 178         long x = atomic_long_read(&vm_node_stat[item]);
 179 #ifdef CONFIG_SMP
 180         if (x < 0)
 181                 x = 0;
 182 #endif
 183         return x;
 184 }
 185
 186 static inline unsigned long zone_page_state(struct zone *zone,
 187                                         enum zone_stat_item item)
 188 {
 189         long x = atomic_long_read(&zone->vm_stat[item]);
 190 #ifdef CONFIG_SMP
 191         if (x < 0)
 192                 x = 0;
 193 #endif
 194         return x;
 195 }
 196
 197 /*
 198  * More accurate version that also considers the currently pending
 199  * deltas. For that we need to loop over all cpus to find the current
 200  * deltas. There is no synchronization so the result cannot be
 201  * exactly accurate either.
 202  */
 203 static inline unsigned long zone_page_state_snapshot(struct zone *zone,
 204                                         enum zone_stat_item item)
 205 {
 206         long x = atomic_long_read(&zone->vm_stat[item]);
 207
 208 #ifdef CONFIG_SMP
 209         int cpu;
 210         for_each_online_cpu(cpu)
 211                 x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
 212
 213         if (x < 0)
 214                 x = 0;
 215 #endif
 216         return x;
 217 }
 218
 219 #ifdef CONFIG_NUMA
 220 extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
 221 extern unsigned long sum_zone_node_page_state(int node,
 222                                               enum zone_stat_item item);
 223 extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
 224 extern unsigned long node_page_state(struct pglist_data *pgdat,
 225                                                 enum node_stat_item item);
 226 #else
 227 #define sum_zone_node_page_state(node, item) global_zone_page_state(item)
 228 #define node_page_state(node, item) global_node_page_state(item)
 229 #endif /* CONFIG_NUMA */
 230
 231 #define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
 232 #define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
 233 #define add_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, __d)
 234 #define sub_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, -(__d))
 235
 236 #ifdef CONFIG_SMP
 237 void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
 238 void __inc_zone_page_state(struct page *, enum zone_stat_item);
 239 void __dec_zone_page_state(struct page *, enum zone_stat_item);
 240
 241 void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
 242 void __inc_node_page_state(struct page *, enum node_stat_item);
 243 void __dec_node_page_state(struct page *, enum node_stat_item);
 244
 245 void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
 246 void inc_zone_page_state(struct page *, enum zone_stat_item);
 247 void dec_zone_page_state(struct page *, enum zone_stat_item);
 248
 249 void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
 250 void inc_node_page_state(struct page *, enum node_stat_item);
 251 void dec_node_page_state(struct page *, enum node_stat_item);
 252
 253 extern void inc_node_state(struct pglist_data *, enum node_stat_item);
 254 extern void __inc_zone_state(struct zone *, enum zone_stat_item);
 255 extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
 256 extern void dec_zone_state(struct zone *, enum zone_stat_item);
 257 extern void __dec_zone_state(struct zone *, enum zone_stat_item);
 258 extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
 259
 260 void quiet_vmstat(void);
 261 void cpu_vm_stats_fold(int cpu);
 262 void refresh_zone_stat_thresholds(void);
 263
 264 struct ctl_table;
 265 int vmstat_refresh(struct ctl_table *, int write,
 266                    void __user *buffer, size_t *lenp, loff_t *ppos);
 267
 268 void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
 269
 270 int calculate_pressure_threshold(struct zone *zone);
 271 int calculate_normal_threshold(struct zone *zone);
 272 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
 273                                 int (*calculate_pressure)(struct zone *));
 274 #else /* CONFIG_SMP */
 275
 276 /*
 277  * We do not maintain differentials in a single processor configuration.
 278  * The functions directly modify the zone and global counters.
 279  */
 280 static inline void __mod_zone_page_state(struct zone *zone,
 281                         enum zone_stat_item item, long delta)
 282 {
 283         zone_page_state_add(delta, zone, item);
 284 }
 285
 286 static inline void __mod_node_page_state(struct pglist_data *pgdat,
 287                         enum node_stat_item item, int delta)
 288 {
 289         node_page_state_add(delta, pgdat, item);
 290 }
 291
 292 static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
 293 {
 294         atomic_long_inc(&zone->vm_stat[item]);
 295         atomic_long_inc(&vm_zone_stat[item]);
 296 }
 297
 298 static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
 299 {
 300         atomic_long_inc(&pgdat->vm_stat[item]);
 301         atomic_long_inc(&vm_node_stat[item]);
 302 }
 303
 304 static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
 305 {
 306         atomic_long_dec(&zone->vm_stat[item]);
 307         atomic_long_dec(&vm_zone_stat[item]);
 308 }
 309
 310 static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
 311 {
 312         atomic_long_dec(&pgdat->vm_stat[item]);
 313         atomic_long_dec(&vm_node_stat[item]);
 314 }
 315
 316 static inline void __inc_zone_page_state(struct page *page,
 317                         enum zone_stat_item item)
 318 {
 319         __inc_zone_state(page_zone(page), item);
 320 }
 321
 322 static inline void __inc_node_page_state(struct page *page,
 323                         enum node_stat_item item)
 324 {
 325         __inc_node_state(page_pgdat(page), item);
 326 }
 327
 328
 329 static inline void __dec_zone_page_state(struct page *page,
 330                         enum zone_stat_item item)
 331 {
 332         __dec_zone_state(page_zone(page), item);
 333 }
 334
 335 static inline void __dec_node_page_state(struct page *page,
 336                         enum node_stat_item item)
 337 {
 338         __dec_node_state(page_pgdat(page), item);
 339 }
 340
 341
 342 /*
 343  * We only use atomic operations to update counters. So there is no need to
 344  * disable interrupts.
 345  */
 346 #define inc_zone_page_state __inc_zone_page_state
 347 #define dec_zone_page_state __dec_zone_page_state
 348 #define mod_zone_page_state __mod_zone_page_state
 349
 350 #define inc_node_page_state __inc_node_page_state
 351 #define dec_node_page_state __dec_node_page_state
 352 #define mod_node_page_state __mod_node_page_state
 353
 354 #define inc_zone_state __inc_zone_state
 355 #define inc_node_state __inc_node_state
 356 #define dec_zone_state __dec_zone_state
 357
 358 #define set_pgdat_percpu_threshold(pgdat, callback) { }
 359
 360 static inline void refresh_zone_stat_thresholds(void) { }
 361 static inline void cpu_vm_stats_fold(int cpu) { }
 362 static inline void quiet_vmstat(void) { }
 363
 364 static inline void drain_zonestat(struct zone *zone,
 365                         struct per_cpu_pageset *pset) { }
 366 #endif          /* CONFIG_SMP */
 367
 368 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
 369                                              int migratetype)
 370 {
 371         __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
 372         if (is_migrate_cma(migratetype))
 373                 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
 374 }
 375
 376 extern const char * const vmstat_text[];
 377
 378 #endif /* _LINUX_VMSTAT_H */