1 #ifndef __LINUX_PERCPU_H 2 #define __LINUX_PERCPU_H 3 4 #include <linux/preempt.h> 5 #include <linux/slab.h> /* For kmalloc() */ 6 #include <linux/smp.h> 7 #include <linux/cpumask.h> 8 9 #include <asm/percpu.h> 10 11 #ifdef CONFIG_SMP 12 #define DEFINE_PER_CPU(type, name) \ 13 __attribute__((__section__(".data.percpu"))) \ 14 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name 15 16 #ifdef MODULE 17 #define SHARED_ALIGNED_SECTION ".data.percpu" 18 #else 19 #define SHARED_ALIGNED_SECTION ".data.percpu.shared_aligned" 20 #endif 21 22 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ 23 __attribute__((__section__(SHARED_ALIGNED_SECTION))) \ 24 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \ 25 ____cacheline_aligned_in_smp 26 #else 27 #define DEFINE_PER_CPU(type, name) \ 28 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name 29 30 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ 31 DEFINE_PER_CPU(type, name) 32 #endif 33 34 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) 35 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) 36 37 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ 38 #ifndef PERCPU_ENOUGH_ROOM 39 #ifdef CONFIG_MODULES 40 #define PERCPU_MODULE_RESERVE 8192 41 #else 42 #define PERCPU_MODULE_RESERVE 0 43 #endif 44 45 #define PERCPU_ENOUGH_ROOM \ 46 (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) 47 #endif /* PERCPU_ENOUGH_ROOM */ 48 49 /* 50 * Must be an lvalue. Since @var must be a simple identifier, 51 * we force a syntax error here if it isn't. 52 */ 53 #define get_cpu_var(var) (*({ \ 54 extern int simple_identifier_##var(void); \ 55 preempt_disable(); \ 56 &__get_cpu_var(var); })) 57 #define put_cpu_var(var) preempt_enable() 58 59 #ifdef CONFIG_SMP 60 61 struct percpu_data { 62 void *ptrs[1]; 63 }; 64 65 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata) 66 /* 67 * Use this to get to a cpu's version of the per-cpu object dynamically 68 * allocated. Non-atomic access to the current CPU's version should 69 * probably be combined with get_cpu()/put_cpu(). 70 */ 71 #define percpu_ptr(ptr, cpu) \ 72 ({ \ 73 struct percpu_data *__p = __percpu_disguise(ptr); \ 74 (__typeof__(ptr))__p->ptrs[(cpu)]; \ 75 }) 76 77 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask); 78 extern void percpu_free(void *__pdata); 79 80 #else /* CONFIG_SMP */ 81 82 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) 83 84 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) 85 { 86 return kzalloc(size, gfp); 87 } 88 89 static inline void percpu_free(void *__pdata) 90 { 91 kfree(__pdata); 92 } 93 94 #endif /* CONFIG_SMP */ 95 96 #define percpu_alloc_mask(size, gfp, mask) \ 97 __percpu_alloc_mask((size), (gfp), &(mask)) 98 99 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map) 100 101 /* (legacy) interface for use without CPU hotplug handling */ 102 103 #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \ 104 cpu_possible_map) 105 #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type)) 106 #define free_percpu(ptr) percpu_free((ptr)) 107 #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu)) 108 109 #endif /* __LINUX_PERCPU_H */ 110