include/linux/percpu.h

   1 #ifndef __LINUX_PERCPU_H
   2 #define __LINUX_PERCPU_H
   3
   4 #include <linux/preempt.h>
   5 #include <linux/slab.h> /* For kmalloc() */
   6 #include <linux/smp.h>
   7 #include <linux/string.h> /* For memset() */
   8 #include <linux/cpumask.h>
   9
  10 #include <asm/percpu.h>
  11
  12 #ifndef PER_CPU_ATTRIBUTES
  13 #define PER_CPU_ATTRIBUTES
  14 #endif
  15
  16 #ifdef CONFIG_SMP
  17 #define DEFINE_PER_CPU(type, name)                                      \
  18         __attribute__((__section__(".data.percpu")))                    \
  19         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  20
  21 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
  22         __attribute__((__section__(".data.percpu.shared_aligned")))     \
  23         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name             \
  24         ____cacheline_aligned_in_smp
  25 #else
  26 #define DEFINE_PER_CPU(type, name)                                      \
  27         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  28
  29 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                     \
  30         DEFINE_PER_CPU(type, name)
  31 #endif
  32
  33 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
  34 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
  35
  36 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  37 #ifndef PERCPU_ENOUGH_ROOM
  38 #ifdef CONFIG_MODULES
  39 #define PERCPU_MODULE_RESERVE   8192
  40 #else
  41 #define PERCPU_MODULE_RESERVE   0
  42 #endif
  43
  44 #define PERCPU_ENOUGH_ROOM                                              \
  45         (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
  46 #endif  /* PERCPU_ENOUGH_ROOM */
  47
  48 /*
  49  * Must be an lvalue. Since @var must be a simple identifier,
  50  * we force a syntax error here if it isn't.
  51  */
  52 #define get_cpu_var(var) (*({                           \
  53         extern int simple_identifier_##var(void);       \
  54         preempt_disable();                              \
  55         &__get_cpu_var(var); }))
  56 #define put_cpu_var(var) preempt_enable()
  57
  58 #ifdef CONFIG_SMP
  59
  60 struct percpu_data {
  61         void *ptrs[NR_CPUS];
  62 };
  63
  64 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  65 /*
  66  * Use this to get to a cpu's version of the per-cpu object dynamically
  67  * allocated. Non-atomic access to the current CPU's version should
  68  * probably be combined with get_cpu()/put_cpu().
  69  */
  70 #define percpu_ptr(ptr, cpu)                              \
  71 ({                                                        \
  72         struct percpu_data *__p = __percpu_disguise(ptr); \
  73         (__typeof__(ptr))__p->ptrs[(cpu)];                \
  74 })
  75
  76 extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu);
  77 extern void percpu_depopulate(void *__pdata, int cpu);
  78 extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  79                                   cpumask_t *mask);
  80 extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask);
  81 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
  82 extern void percpu_free(void *__pdata);
  83
  84 #else /* CONFIG_SMP */
  85
  86 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  87
  88 static inline void percpu_depopulate(void *__pdata, int cpu)
  89 {
  90 }
  91
  92 static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
  93 {
  94 }
  95
  96 static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp,
  97                                     int cpu)
  98 {
  99         return percpu_ptr(__pdata, cpu);
 100 }
 101
 102 static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
 103                                          cpumask_t *mask)
 104 {
 105         return 0;
 106 }
 107
 108 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
 109 {
 110         return kzalloc(size, gfp);
 111 }
 112
 113 static inline void percpu_free(void *__pdata)
 114 {
 115         kfree(__pdata);
 116 }
 117
 118 #endif /* CONFIG_SMP */
 119
 120 #define percpu_populate_mask(__pdata, size, gfp, mask) \
 121         __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
 122 #define percpu_depopulate_mask(__pdata, mask) \
 123         __percpu_depopulate_mask((__pdata), &(mask))
 124 #define percpu_alloc_mask(size, gfp, mask) \
 125         __percpu_alloc_mask((size), (gfp), &(mask))
 126
 127 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
 128
 129 /* (legacy) interface for use without CPU hotplug handling */
 130
 131 #define __alloc_percpu(size)    percpu_alloc_mask((size), GFP_KERNEL, \
 132                                                   cpu_possible_map)
 133 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type))
 134 #define free_percpu(ptr)        percpu_free((ptr))
 135 #define per_cpu_ptr(ptr, cpu)   percpu_ptr((ptr), (cpu))
 136
 137 #endif /* __LINUX_PERCPU_H */