mm/slab.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef MM_SLAB_H
   3 #define MM_SLAB_H
   4
   5 #include <linux/reciprocal_div.h>
   6 #include <linux/list_lru.h>
   7 #include <linux/local_lock.h>
   8 #include <linux/random.h>
   9 #include <linux/kobject.h>
  10 #include <linux/sched/mm.h>
  11 #include <linux/memcontrol.h>
  12 #include <linux/kfence.h>
  13 #include <linux/kasan.h>
  14
  15 /*
  16  * Internal slab definitions
  17  */
  18
  19 #ifdef CONFIG_64BIT
  20 # ifdef system_has_cmpxchg128
  21 # define system_has_freelist_aba()      system_has_cmpxchg128()
  22 # define try_cmpxchg_freelist           try_cmpxchg128
  23 # endif
  24 #define this_cpu_try_cmpxchg_freelist   this_cpu_try_cmpxchg128
  25 typedef u128 freelist_full_t;
  26 #else /* CONFIG_64BIT */
  27 # ifdef system_has_cmpxchg64
  28 # define system_has_freelist_aba()      system_has_cmpxchg64()
  29 # define try_cmpxchg_freelist           try_cmpxchg64
  30 # endif
  31 #define this_cpu_try_cmpxchg_freelist   this_cpu_try_cmpxchg64
  32 typedef u64 freelist_full_t;
  33 #endif /* CONFIG_64BIT */
  34
  35 #if defined(system_has_freelist_aba) && !defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
  36 #undef system_has_freelist_aba
  37 #endif
  38
  39 /*
  40  * Freelist pointer and counter to cmpxchg together, avoids the typical ABA
  41  * problems with cmpxchg of just a pointer.
  42  */
  43 typedef union {
  44         struct {
  45                 void *freelist;
  46                 unsigned long counter;
  47         };
  48         freelist_full_t full;
  49 } freelist_aba_t;
  50
  51 /* Reuses the bits in struct page */
  52 struct slab {
  53         unsigned long __page_flags;
  54
  55         struct kmem_cache *slab_cache;
  56         union {
  57                 struct {
  58                         union {
  59                                 struct list_head slab_list;
  60 #ifdef CONFIG_SLUB_CPU_PARTIAL
  61                                 struct {
  62                                         struct slab *next;
  63                                         int slabs;      /* Nr of slabs left */
  64                                 };
  65 #endif
  66                         };
  67                         /* Double-word boundary */
  68                         union {
  69                                 struct {
  70                                         void *freelist;         /* first free object */
  71                                         union {
  72                                                 unsigned long counters;
  73                                                 struct {
  74                                                         unsigned inuse:16;
  75                                                         unsigned objects:15;
  76                                                         /*
  77                                                          * If slab debugging is enabled then the
  78                                                          * frozen bit can be reused to indicate
  79                                                          * that the slab was corrupted
  80                                                          */
  81                                                         unsigned frozen:1;
  82                                                 };
  83                                         };
  84                                 };
  85 #ifdef system_has_freelist_aba
  86                                 freelist_aba_t freelist_counter;
  87 #endif
  88                         };
  89                 };
  90                 struct rcu_head rcu_head;
  91         };
  92
  93         unsigned int __page_type;
  94         atomic_t __page_refcount;
  95 #ifdef CONFIG_SLAB_OBJ_EXT
  96         unsigned long obj_exts;
  97 #endif
  98 };
  99
 100 #define SLAB_MATCH(pg, sl)                                              \
 101         static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))
 102 SLAB_MATCH(flags, __page_flags);
 103 SLAB_MATCH(compound_head, slab_cache);  /* Ensure bit 0 is clear */
 104 SLAB_MATCH(_refcount, __page_refcount);
 105 #ifdef CONFIG_MEMCG
 106 SLAB_MATCH(memcg_data, obj_exts);
 107 #elif defined(CONFIG_SLAB_OBJ_EXT)
 108 SLAB_MATCH(_unused_slab_obj_exts, obj_exts);
 109 #endif
 110 #undef SLAB_MATCH
 111 static_assert(sizeof(struct slab) <= sizeof(struct page));
 112 #if defined(system_has_freelist_aba)
 113 static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(freelist_aba_t)));
 114 #endif
 115
 116 /**
 117  * folio_slab - Converts from folio to slab.
 118  * @folio: The folio.
 119  *
 120  * Currently struct slab is a different representation of a folio where
 121  * folio_test_slab() is true.
 122  *
 123  * Return: The slab which contains this folio.
 124  */
 125 #define folio_slab(folio)       (_Generic((folio),                      \
 126         const struct folio *:   (const struct slab *)(folio),           \
 127         struct folio *:         (struct slab *)(folio)))
 128
 129 /**
 130  * slab_folio - The folio allocated for a slab
 131  * @slab: The slab.
 132  *
 133  * Slabs are allocated as folios that contain the individual objects and are
 134  * using some fields in the first struct page of the folio - those fields are
 135  * now accessed by struct slab. It is occasionally necessary to convert back to
 136  * a folio in order to communicate with the rest of the mm.  Please use this
 137  * helper function instead of casting yourself, as the implementation may change
 138  * in the future.
 139  */
 140 #define slab_folio(s)           (_Generic((s),                          \
 141         const struct slab *:    (const struct folio *)s,                \
 142         struct slab *:          (struct folio *)s))
 143
 144 /**
 145  * page_slab - Converts from first struct page to slab.
 146  * @p: The first (either head of compound or single) page of slab.
 147  *
 148  * A temporary wrapper to convert struct page to struct slab in situations where
 149  * we know the page is the compound head, or single order-0 page.
 150  *
 151  * Long-term ideally everything would work with struct slab directly or go
 152  * through folio to struct slab.
 153  *
 154  * Return: The slab which contains this page
 155  */
 156 #define page_slab(p)            (_Generic((p),                          \
 157         const struct page *:    (const struct slab *)(p),               \
 158         struct page *:          (struct slab *)(p)))
 159
 160 /**
 161  * slab_page - The first struct page allocated for a slab
 162  * @slab: The slab.
 163  *
 164  * A convenience wrapper for converting slab to the first struct page of the
 165  * underlying folio, to communicate with code not yet converted to folio or
 166  * struct slab.
 167  */
 168 #define slab_page(s) folio_page(slab_folio(s), 0)
 169
 170 /*
 171  * If network-based swap is enabled, sl*b must keep track of whether pages
 172  * were allocated from pfmemalloc reserves.
 173  */
 174 static inline bool slab_test_pfmemalloc(const struct slab *slab)
 175 {
 176         return folio_test_active(slab_folio(slab));
 177 }
 178
 179 static inline void slab_set_pfmemalloc(struct slab *slab)
 180 {
 181         folio_set_active(slab_folio(slab));
 182 }
 183
 184 static inline void slab_clear_pfmemalloc(struct slab *slab)
 185 {
 186         folio_clear_active(slab_folio(slab));
 187 }
 188
 189 static inline void __slab_clear_pfmemalloc(struct slab *slab)
 190 {
 191         __folio_clear_active(slab_folio(slab));
 192 }
 193
 194 static inline void *slab_address(const struct slab *slab)
 195 {
 196         return folio_address(slab_folio(slab));
 197 }
 198
 199 static inline int slab_nid(const struct slab *slab)
 200 {
 201         return folio_nid(slab_folio(slab));
 202 }
 203
 204 static inline pg_data_t *slab_pgdat(const struct slab *slab)
 205 {
 206         return folio_pgdat(slab_folio(slab));
 207 }
 208
 209 static inline struct slab *virt_to_slab(const void *addr)
 210 {
 211         struct folio *folio = virt_to_folio(addr);
 212
 213         if (!folio_test_slab(folio))
 214                 return NULL;
 215
 216         return folio_slab(folio);
 217 }
 218
 219 static inline int slab_order(const struct slab *slab)
 220 {
 221         return folio_order(slab_folio(slab));
 222 }
 223
 224 static inline size_t slab_size(const struct slab *slab)
 225 {
 226         return PAGE_SIZE << slab_order(slab);
 227 }
 228
 229 #ifdef CONFIG_SLUB_CPU_PARTIAL
 230 #define slub_percpu_partial(c)                  ((c)->partial)
 231
 232 #define slub_set_percpu_partial(c, p)           \
 233 ({                                              \
 234         slub_percpu_partial(c) = (p)->next;     \
 235 })
 236
 237 #define slub_percpu_partial_read_once(c)        READ_ONCE(slub_percpu_partial(c))
 238 #else
 239 #define slub_percpu_partial(c)                  NULL
 240
 241 #define slub_set_percpu_partial(c, p)
 242
 243 #define slub_percpu_partial_read_once(c)        NULL
 244 #endif // CONFIG_SLUB_CPU_PARTIAL
 245
 246 /*
 247  * Word size structure that can be atomically updated or read and that
 248  * contains both the order and the number of objects that a slab of the
 249  * given order would contain.
 250  */
 251 struct kmem_cache_order_objects {
 252         unsigned int x;
 253 };
 254
 255 /*
 256  * Slab cache management.
 257  */
 258 struct kmem_cache {
 259 #ifndef CONFIG_SLUB_TINY
 260         struct kmem_cache_cpu __percpu *cpu_slab;
 261 #endif
 262         /* Used for retrieving partial slabs, etc. */
 263         slab_flags_t flags;
 264         unsigned long min_partial;
 265         unsigned int size;              /* Object size including metadata */
 266         unsigned int object_size;       /* Object size without metadata */
 267         struct reciprocal_value reciprocal_size;
 268         unsigned int offset;            /* Free pointer offset */
 269 #ifdef CONFIG_SLUB_CPU_PARTIAL
 270         /* Number of per cpu partial objects to keep around */
 271         unsigned int cpu_partial;
 272         /* Number of per cpu partial slabs to keep around */
 273         unsigned int cpu_partial_slabs;
 274 #endif
 275         struct kmem_cache_order_objects oo;
 276
 277         /* Allocation and freeing of slabs */
 278         struct kmem_cache_order_objects min;
 279         gfp_t allocflags;               /* gfp flags to use on each alloc */
 280         int refcount;                   /* Refcount for slab cache destroy */
 281         void (*ctor)(void *object);     /* Object constructor */
 282         unsigned int inuse;             /* Offset to metadata */
 283         unsigned int align;             /* Alignment */
 284         unsigned int red_left_pad;      /* Left redzone padding size */
 285         const char *name;               /* Name (only for display!) */
 286         struct list_head list;          /* List of slab caches */
 287 #ifdef CONFIG_SYSFS
 288         struct kobject kobj;            /* For sysfs */
 289 #endif
 290 #ifdef CONFIG_SLAB_FREELIST_HARDENED
 291         unsigned long random;
 292 #endif
 293
 294 #ifdef CONFIG_NUMA
 295         /*
 296          * Defragmentation by allocating from a remote node.
 297          */
 298         unsigned int remote_node_defrag_ratio;
 299 #endif
 300
 301 #ifdef CONFIG_SLAB_FREELIST_RANDOM
 302         unsigned int *random_seq;
 303 #endif
 304
 305 #ifdef CONFIG_KASAN_GENERIC
 306         struct kasan_cache kasan_info;
 307 #endif
 308
 309 #ifdef CONFIG_HARDENED_USERCOPY
 310         unsigned int useroffset;        /* Usercopy region offset */
 311         unsigned int usersize;          /* Usercopy region size */
 312 #endif
 313
 314         struct kmem_cache_node *node[MAX_NUMNODES];
 315 };
 316
 317 #if defined(CONFIG_SYSFS) && !defined(CONFIG_SLUB_TINY)
 318 #define SLAB_SUPPORTS_SYSFS 1
 319 void sysfs_slab_unlink(struct kmem_cache *s);
 320 void sysfs_slab_release(struct kmem_cache *s);
 321 #else
 322 static inline void sysfs_slab_unlink(struct kmem_cache *s) { }
 323 static inline void sysfs_slab_release(struct kmem_cache *s) { }
 324 #endif
 325
 326 void *fixup_red_left(struct kmem_cache *s, void *p);
 327
 328 static inline void *nearest_obj(struct kmem_cache *cache,
 329                                 const struct slab *slab, void *x)
 330 {
 331         void *object = x - (x - slab_address(slab)) % cache->size;
 332         void *last_object = slab_address(slab) +
 333                 (slab->objects - 1) * cache->size;
 334         void *result = (unlikely(object > last_object)) ? last_object : object;
 335
 336         result = fixup_red_left(cache, result);
 337         return result;
 338 }
 339
 340 /* Determine object index from a given position */
 341 static inline unsigned int __obj_to_index(const struct kmem_cache *cache,
 342                                           void *addr, void *obj)
 343 {
 344         return reciprocal_divide(kasan_reset_tag(obj) - addr,
 345                                  cache->reciprocal_size);
 346 }
 347
 348 static inline unsigned int obj_to_index(const struct kmem_cache *cache,
 349                                         const struct slab *slab, void *obj)
 350 {
 351         if (is_kfence_address(obj))
 352                 return 0;
 353         return __obj_to_index(cache, slab_address(slab), obj);
 354 }
 355
 356 static inline int objs_per_slab(const struct kmem_cache *cache,
 357                                 const struct slab *slab)
 358 {
 359         return slab->objects;
 360 }
 361
 362 /*
 363  * State of the slab allocator.
 364  *
 365  * This is used to describe the states of the allocator during bootup.
 366  * Allocators use this to gradually bootstrap themselves. Most allocators
 367  * have the problem that the structures used for managing slab caches are
 368  * allocated from slab caches themselves.
 369  */
 370 enum slab_state {
 371         DOWN,                   /* No slab functionality yet */
 372         PARTIAL,                /* SLUB: kmem_cache_node available */
 373         UP,                     /* Slab caches usable but not all extras yet */
 374         FULL                    /* Everything is working */
 375 };
 376
 377 extern enum slab_state slab_state;
 378
 379 /* The slab cache mutex protects the management structures during changes */
 380 extern struct mutex slab_mutex;
 381
 382 /* The list of all slab caches on the system */
 383 extern struct list_head slab_caches;
 384
 385 /* The slab cache that manages slab cache information */
 386 extern struct kmem_cache *kmem_cache;
 387
 388 /* A table of kmalloc cache names and sizes */
 389 extern const struct kmalloc_info_struct {
 390         const char *name[NR_KMALLOC_TYPES];
 391         unsigned int size;
 392 } kmalloc_info[];
 393
 394 /* Kmalloc array related functions */
 395 void setup_kmalloc_cache_index_table(void);
 396 void create_kmalloc_caches(void);
 397
 398 extern u8 kmalloc_size_index[24];
 399
 400 static inline unsigned int size_index_elem(unsigned int bytes)
 401 {
 402         return (bytes - 1) / 8;
 403 }
 404
 405 /*
 406  * Find the kmem_cache structure that serves a given size of
 407  * allocation
 408  *
 409  * This assumes size is larger than zero and not larger than
 410  * KMALLOC_MAX_CACHE_SIZE and the caller must check that.
 411  */
 412 static inline struct kmem_cache *
 413 kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, unsigned long caller)
 414 {
 415         unsigned int index;
 416
 417         if (!b)
 418                 b = &kmalloc_caches[kmalloc_type(flags, caller)];
 419         if (size <= 192)
 420                 index = kmalloc_size_index[size_index_elem(size)];
 421         else
 422                 index = fls(size - 1);
 423
 424         return (*b)[index];
 425 }
 426
 427 gfp_t kmalloc_fix_flags(gfp_t flags);
 428
 429 /* Functions provided by the slab allocators */
 430 int do_kmem_cache_create(struct kmem_cache *s, const char *name,
 431                          unsigned int size, struct kmem_cache_args *args,
 432                          slab_flags_t flags);
 433
 434 void __init kmem_cache_init(void);
 435 extern void create_boot_cache(struct kmem_cache *, const char *name,
 436                         unsigned int size, slab_flags_t flags,
 437                         unsigned int useroffset, unsigned int usersize);
 438
 439 int slab_unmergeable(struct kmem_cache *s);
 440 struct kmem_cache *find_mergeable(unsigned size, unsigned align,
 441                 slab_flags_t flags, const char *name, void (*ctor)(void *));
 442 struct kmem_cache *
 443 __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
 444                    slab_flags_t flags, void (*ctor)(void *));
 445
 446 slab_flags_t kmem_cache_flags(slab_flags_t flags, const char *name);
 447
 448 static inline bool is_kmalloc_cache(struct kmem_cache *s)
 449 {
 450         return (s->flags & SLAB_KMALLOC);
 451 }
 452
 453 static inline bool is_kmalloc_normal(struct kmem_cache *s)
 454 {
 455         if (!is_kmalloc_cache(s))
 456                 return false;
 457         return !(s->flags & (SLAB_CACHE_DMA|SLAB_ACCOUNT|SLAB_RECLAIM_ACCOUNT));
 458 }
 459
 460 /* Legal flag mask for kmem_cache_create(), for various configurations */
 461 #define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
 462                          SLAB_CACHE_DMA32 | SLAB_PANIC | \
 463                          SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS )
 464
 465 #ifdef CONFIG_SLUB_DEBUG
 466 #define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
 467                           SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
 468 #else
 469 #define SLAB_DEBUG_FLAGS (0)
 470 #endif
 471
 472 #define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
 473                           SLAB_TEMPORARY | SLAB_ACCOUNT | \
 474                           SLAB_NO_USER_FLAGS | SLAB_KMALLOC | SLAB_NO_MERGE)
 475
 476 /* Common flags available with current configuration */
 477 #define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
 478
 479 /* Common flags permitted for kmem_cache_create */
 480 #define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \
 481                               SLAB_RED_ZONE | \
 482                               SLAB_POISON | \
 483                               SLAB_STORE_USER | \
 484                               SLAB_TRACE | \
 485                               SLAB_CONSISTENCY_CHECKS | \
 486                               SLAB_NOLEAKTRACE | \
 487                               SLAB_RECLAIM_ACCOUNT | \
 488                               SLAB_TEMPORARY | \
 489                               SLAB_ACCOUNT | \
 490                               SLAB_KMALLOC | \
 491                               SLAB_NO_MERGE | \
 492                               SLAB_NO_USER_FLAGS)
 493
 494 bool __kmem_cache_empty(struct kmem_cache *);
 495 int __kmem_cache_shutdown(struct kmem_cache *);
 496 void __kmem_cache_release(struct kmem_cache *);
 497 int __kmem_cache_shrink(struct kmem_cache *);
 498 void slab_kmem_cache_release(struct kmem_cache *);
 499
 500 struct seq_file;
 501 struct file;
 502
 503 struct slabinfo {
 504         unsigned long active_objs;
 505         unsigned long num_objs;
 506         unsigned long active_slabs;
 507         unsigned long num_slabs;
 508         unsigned long shared_avail;
 509         unsigned int limit;
 510         unsigned int batchcount;
 511         unsigned int shared;
 512         unsigned int objects_per_slab;
 513         unsigned int cache_order;
 514 };
 515
 516 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
 517
 518 #ifdef CONFIG_SLUB_DEBUG
 519 #ifdef CONFIG_SLUB_DEBUG_ON
 520 DECLARE_STATIC_KEY_TRUE(slub_debug_enabled);
 521 #else
 522 DECLARE_STATIC_KEY_FALSE(slub_debug_enabled);
 523 #endif
 524 extern void print_tracking(struct kmem_cache *s, void *object);
 525 long validate_slab_cache(struct kmem_cache *s);
 526 static inline bool __slub_debug_enabled(void)
 527 {
 528         return static_branch_unlikely(&slub_debug_enabled);
 529 }
 530 #else
 531 static inline void print_tracking(struct kmem_cache *s, void *object)
 532 {
 533 }
 534 static inline bool __slub_debug_enabled(void)
 535 {
 536         return false;
 537 }
 538 #endif
 539
 540 /*
 541  * Returns true if any of the specified slab_debug flags is enabled for the
 542  * cache. Use only for flags parsed by setup_slub_debug() as it also enables
 543  * the static key.
 544  */
 545 static inline bool kmem_cache_debug_flags(struct kmem_cache *s, slab_flags_t flags)
 546 {
 547         if (IS_ENABLED(CONFIG_SLUB_DEBUG))
 548                 VM_WARN_ON_ONCE(!(flags & SLAB_DEBUG_FLAGS));
 549         if (__slub_debug_enabled())
 550                 return s->flags & flags;
 551         return false;
 552 }
 553
 554 #if IS_ENABLED(CONFIG_SLUB_DEBUG) && IS_ENABLED(CONFIG_KUNIT)
 555 bool slab_in_kunit_test(void);
 556 #else
 557 static inline bool slab_in_kunit_test(void) { return false; }
 558 #endif
 559
 560 #ifdef CONFIG_SLAB_OBJ_EXT
 561
 562 /*
 563  * slab_obj_exts - get the pointer to the slab object extension vector
 564  * associated with a slab.
 565  * @slab: a pointer to the slab struct
 566  *
 567  * Returns a pointer to the object extension vector associated with the slab,
 568  * or NULL if no such vector has been associated yet.
 569  */
 570 static inline struct slabobj_ext *slab_obj_exts(struct slab *slab)
 571 {
 572         unsigned long obj_exts = READ_ONCE(slab->obj_exts);
 573
 574 #ifdef CONFIG_MEMCG
 575         VM_BUG_ON_PAGE(obj_exts && !(obj_exts & MEMCG_DATA_OBJEXTS),
 576                                                         slab_page(slab));
 577         VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab));
 578 #endif
 579         return (struct slabobj_ext *)(obj_exts & ~OBJEXTS_FLAGS_MASK);
 580 }
 581
 582 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
 583                         gfp_t gfp, bool new_slab);
 584
 585 #else /* CONFIG_SLAB_OBJ_EXT */
 586
 587 static inline struct slabobj_ext *slab_obj_exts(struct slab *slab)
 588 {
 589         return NULL;
 590 }
 591
 592 #endif /* CONFIG_SLAB_OBJ_EXT */
 593
 594 static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
 595 {
 596         return (s->flags & SLAB_RECLAIM_ACCOUNT) ?
 597                 NR_SLAB_RECLAIMABLE_B : NR_SLAB_UNRECLAIMABLE_B;
 598 }
 599
 600 #ifdef CONFIG_MEMCG
 601 bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
 602                                   gfp_t flags, size_t size, void **p);
 603 void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
 604                             void **p, int objects, struct slabobj_ext *obj_exts);
 605 #endif
 606
 607 size_t __ksize(const void *objp);
 608
 609 static inline size_t slab_ksize(const struct kmem_cache *s)
 610 {
 611 #ifdef CONFIG_SLUB_DEBUG
 612         /*
 613          * Debugging requires use of the padding between object
 614          * and whatever may come after it.
 615          */
 616         if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
 617                 return s->object_size;
 618 #endif
 619         if (s->flags & SLAB_KASAN)
 620                 return s->object_size;
 621         /*
 622          * If we have the need to store the freelist pointer
 623          * back there or track user information then we can
 624          * only use the space before that information.
 625          */
 626         if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
 627                 return s->inuse;
 628         /*
 629          * Else we can use all the padding etc for the allocation
 630          */
 631         return s->size;
 632 }
 633
 634 #ifdef CONFIG_SLUB_DEBUG
 635 void dump_unreclaimable_slab(void);
 636 #else
 637 static inline void dump_unreclaimable_slab(void)
 638 {
 639 }
 640 #endif
 641
 642 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);
 643
 644 #ifdef CONFIG_SLAB_FREELIST_RANDOM
 645 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
 646                         gfp_t gfp);
 647 void cache_random_seq_destroy(struct kmem_cache *cachep);
 648 #else
 649 static inline int cache_random_seq_create(struct kmem_cache *cachep,
 650                                         unsigned int count, gfp_t gfp)
 651 {
 652         return 0;
 653 }
 654 static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
 655 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
 656
 657 static inline bool slab_want_init_on_alloc(gfp_t flags, struct kmem_cache *c)
 658 {
 659         if (static_branch_maybe(CONFIG_INIT_ON_ALLOC_DEFAULT_ON,
 660                                 &init_on_alloc)) {
 661                 if (c->ctor)
 662                         return false;
 663                 if (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON))
 664                         return flags & __GFP_ZERO;
 665                 return true;
 666         }
 667         return flags & __GFP_ZERO;
 668 }
 669
 670 static inline bool slab_want_init_on_free(struct kmem_cache *c)
 671 {
 672         if (static_branch_maybe(CONFIG_INIT_ON_FREE_DEFAULT_ON,
 673                                 &init_on_free))
 674                 return !(c->ctor ||
 675                          (c->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)));
 676         return false;
 677 }
 678
 679 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
 680 void debugfs_slab_release(struct kmem_cache *);
 681 #else
 682 static inline void debugfs_slab_release(struct kmem_cache *s) { }
 683 #endif
 684
 685 #ifdef CONFIG_PRINTK
 686 #define KS_ADDRS_COUNT 16
 687 struct kmem_obj_info {
 688         void *kp_ptr;
 689         struct slab *kp_slab;
 690         void *kp_objp;
 691         unsigned long kp_data_offset;
 692         struct kmem_cache *kp_slab_cache;
 693         void *kp_ret;
 694         void *kp_stack[KS_ADDRS_COUNT];
 695         void *kp_free_stack[KS_ADDRS_COUNT];
 696 };
 697 void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);
 698 #endif
 699
 700 void __check_heap_object(const void *ptr, unsigned long n,
 701                          const struct slab *slab, bool to_user);
 702
 703 static inline bool slub_debug_orig_size(struct kmem_cache *s)
 704 {
 705         return (kmem_cache_debug_flags(s, SLAB_STORE_USER) &&
 706                         (s->flags & SLAB_KMALLOC));
 707 }
 708
 709 #ifdef CONFIG_SLUB_DEBUG
 710 void skip_orig_size_check(struct kmem_cache *s, const void *object);
 711 #endif
 712
 713 #endif /* MM_SLAB_H */