1 #ifndef __LINUX_PERCPU_H 2 #define __LINUX_PERCPU_H 3 4 #include <linux/preempt.h> 5 #include <linux/slab.h> /* For kmalloc() */ 6 #include <linux/smp.h> 7 #include <linux/cpumask.h> 8 9 #include <asm/percpu.h> 10 11 #ifdef CONFIG_SMP 12 #define DEFINE_PER_CPU(type, name) \ 13 __attribute__((__section__(".data.percpu"))) \ 14 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name 15 16 #ifdef MODULE 17 #define SHARED_ALIGNED_SECTION ".data.percpu" 18 #else 19 #define SHARED_ALIGNED_SECTION ".data.percpu.shared_aligned" 20 #endif 21 22 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ 23 __attribute__((__section__(SHARED_ALIGNED_SECTION))) \ 24 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \ 25 ____cacheline_aligned_in_smp 26 27 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ 28 __attribute__((__section__(".data.percpu.page_aligned"))) \ 29 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name 30 #else 31 #define DEFINE_PER_CPU(type, name) \ 32 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name 33 34 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ 35 DEFINE_PER_CPU(type, name) 36 37 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ 38 DEFINE_PER_CPU(type, name) 39 #endif 40 41 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) 42 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) 43 44 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ 45 #ifndef PERCPU_ENOUGH_ROOM 46 #ifdef CONFIG_MODULES 47 #define PERCPU_MODULE_RESERVE 8192 48 #else 49 #define PERCPU_MODULE_RESERVE 0 50 #endif 51 52 #define PERCPU_ENOUGH_ROOM \ 53 (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) 54 #endif /* PERCPU_ENOUGH_ROOM */ 55 56 /* 57 * Must be an lvalue. Since @var must be a simple identifier, 58 * we force a syntax error here if it isn't. 59 */ 60 #define get_cpu_var(var) (*({ \ 61 extern int simple_identifier_##var(void); \ 62 preempt_disable(); \ 63 &__get_cpu_var(var); })) 64 #define put_cpu_var(var) preempt_enable() 65 66 #ifdef CONFIG_SMP 67 68 struct percpu_data { 69 void *ptrs[1]; 70 }; 71 72 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata) 73 /* 74 * Use this to get to a cpu's version of the per-cpu object dynamically 75 * allocated. Non-atomic access to the current CPU's version should 76 * probably be combined with get_cpu()/put_cpu(). 77 */ 78 #define percpu_ptr(ptr, cpu) \ 79 ({ \ 80 struct percpu_data *__p = __percpu_disguise(ptr); \ 81 (__typeof__(ptr))__p->ptrs[(cpu)]; \ 82 }) 83 84 extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask); 85 extern void percpu_free(void *__pdata); 86 87 #else /* CONFIG_SMP */ 88 89 #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) 90 91 static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) 92 { 93 return kzalloc(size, gfp); 94 } 95 96 static inline void percpu_free(void *__pdata) 97 { 98 kfree(__pdata); 99 } 100 101 #endif /* CONFIG_SMP */ 102 103 #define percpu_alloc_mask(size, gfp, mask) \ 104 __percpu_alloc_mask((size), (gfp), &(mask)) 105 106 #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map) 107 108 /* (legacy) interface for use without CPU hotplug handling */ 109 110 #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \ 111 cpu_possible_map) 112 #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type)) 113 #define free_percpu(ptr) percpu_free((ptr)) 114 #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu)) 115 116 #endif /* __LINUX_PERCPU_H */ 117