include/linux/percpu.h

   1 #ifndef __LINUX_PERCPU_H
   2 #define __LINUX_PERCPU_H
   3
   4 #include <linux/preempt.h>
   5 #include <linux/slab.h> /* For kmalloc() */
   6 #include <linux/smp.h>
   7 #include <linux/cpumask.h>
   8
   9 #include <asm/percpu.h>
  10
  11 #ifdef CONFIG_SMP
  12 #define DEFINE_PER_CPU(type, name)                                      \
  13         __attribute__((__section__(".data.percpu")))                    \
  14         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  15
  16 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
  17         __attribute__((__section__(".data.percpu.shared_aligned")))     \
  18         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name             \
  19         ____cacheline_aligned_in_smp
  20 #else
  21 #define DEFINE_PER_CPU(type, name)                                      \
  22         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  23
  24 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                     \
  25         DEFINE_PER_CPU(type, name)
  26 #endif
  27
  28 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
  29 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
  30
  31 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  32 #ifndef PERCPU_ENOUGH_ROOM
  33 #ifdef CONFIG_MODULES
  34 #define PERCPU_MODULE_RESERVE   8192
  35 #else
  36 #define PERCPU_MODULE_RESERVE   0
  37 #endif
  38
  39 #define PERCPU_ENOUGH_ROOM                                              \
  40         (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
  41 #endif  /* PERCPU_ENOUGH_ROOM */
  42
  43 /*
  44  * Must be an lvalue. Since @var must be a simple identifier,
  45  * we force a syntax error here if it isn't.
  46  */
  47 #define get_cpu_var(var) (*({                           \
  48         extern int simple_identifier_##var(void);       \
  49         preempt_disable();                              \
  50         &__get_cpu_var(var); }))
  51 #define put_cpu_var(var) preempt_enable()
  52
  53 #ifdef CONFIG_SMP
  54
  55 struct percpu_data {
  56         void *ptrs[1];
  57 };
  58
  59 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  60 /*
  61  * Use this to get to a cpu's version of the per-cpu object dynamically
  62  * allocated. Non-atomic access to the current CPU's version should
  63  * probably be combined with get_cpu()/put_cpu().
  64  */
  65 #define percpu_ptr(ptr, cpu)                              \
  66 ({                                                        \
  67         struct percpu_data *__p = __percpu_disguise(ptr); \
  68         (__typeof__(ptr))__p->ptrs[(cpu)];                \
  69 })
  70
  71 extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu);
  72 extern void percpu_depopulate(void *__pdata, int cpu);
  73 extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  74                                   cpumask_t *mask);
  75 extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask);
  76 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
  77 extern void percpu_free(void *__pdata);
  78
  79 #else /* CONFIG_SMP */
  80
  81 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
  82
  83 static inline void percpu_depopulate(void *__pdata, int cpu)
  84 {
  85 }
  86
  87 static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
  88 {
  89 }
  90
  91 static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp,
  92                                     int cpu)
  93 {
  94         return percpu_ptr(__pdata, cpu);
  95 }
  96
  97 static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
  98                                          cpumask_t *mask)
  99 {
 100         return 0;
 101 }
 102
 103 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
 104 {
 105         return kzalloc(size, gfp);
 106 }
 107
 108 static inline void percpu_free(void *__pdata)
 109 {
 110         kfree(__pdata);
 111 }
 112
 113 #endif /* CONFIG_SMP */
 114
 115 #define percpu_populate_mask(__pdata, size, gfp, mask) \
 116         __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
 117 #define percpu_depopulate_mask(__pdata, mask) \
 118         __percpu_depopulate_mask((__pdata), &(mask))
 119 #define percpu_alloc_mask(size, gfp, mask) \
 120         __percpu_alloc_mask((size), (gfp), &(mask))
 121
 122 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
 123
 124 /* (legacy) interface for use without CPU hotplug handling */
 125
 126 #define __alloc_percpu(size)    percpu_alloc_mask((size), GFP_KERNEL, \
 127                                                   cpu_possible_map)
 128 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type))
 129 #define free_percpu(ptr)        percpu_free((ptr))
 130 #define per_cpu_ptr(ptr, cpu)   percpu_ptr((ptr), (cpu))
 131
 132 #endif /* __LINUX_PERCPU_H */