xref: /linux-6.15/include/linux/percpu_counter.h (revision ea319518) 
1 #ifndef _LINUX_PERCPU_COUNTER_H
2 #define _LINUX_PERCPU_COUNTER_H
3 /*
4  * A simple "approximate counter" for use in ext2 and ext3 superblocks.
5  *
6  * WARNING: these things are HUGE.  4 kbytes per counter on 32-way P4.
7  */
8 
9 #include <linux/spinlock.h>
10 #include <linux/smp.h>
11 #include <linux/list.h>
12 #include <linux/threads.h>
13 #include <linux/percpu.h>
14 #include <linux/types.h>
15 
16 #ifdef CONFIG_SMP
17 
18 struct percpu_counter {
19 	spinlock_t lock;
20 	s64 count;
21 #ifdef CONFIG_HOTPLUG_CPU
22 	struct list_head list;	/* All percpu_counters are on a list */
23 #endif
24 	s32 *counters;
25 };
26 
27 #if NR_CPUS >= 16
28 #define FBC_BATCH	(NR_CPUS*2)
29 #else
30 #define FBC_BATCH	(NR_CPUS*4)
31 #endif
32 
33 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount,
34 			  struct lock_class_key *key);
35 
36 #define percpu_counter_init(fbc, value)					\
37 	({								\
38 		static struct lock_class_key __key;			\
39 									\
40 		__percpu_counter_init(fbc, value, &__key);		\
41 	})
42 
43 void percpu_counter_destroy(struct percpu_counter *fbc);
44 void percpu_counter_set(struct percpu_counter *fbc, s64 amount);
45 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch);
46 s64 __percpu_counter_sum(struct percpu_counter *fbc);
47 
48 static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount)
49 {
50 	__percpu_counter_add(fbc, amount, FBC_BATCH);
51 }
52 
53 static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
54 {
55 	s64 ret = __percpu_counter_sum(fbc);
56 	return ret < 0 ? 0 : ret;
57 }
58 
59 static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
60 {
61 	return __percpu_counter_sum(fbc);
62 }
63 
64 static inline s64 percpu_counter_read(struct percpu_counter *fbc)
65 {
66 	return fbc->count;
67 }
68 
69 /*
70  * It is possible for the percpu_counter_read() to return a small negative
71  * number for some counter which should never be negative.
72  *
73  */
74 static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
75 {
76 	s64 ret = fbc->count;
77 
78 	barrier();		/* Prevent reloads of fbc->count */
79 	if (ret >= 0)
80 		return ret;
81 	return 1;
82 }
83 
84 #else
85 
86 struct percpu_counter {
87 	s64 count;
88 };
89 
90 static inline int percpu_counter_init(struct percpu_counter *fbc, s64 amount)
91 {
92 	fbc->count = amount;
93 	return 0;
94 }
95 
96 static inline void percpu_counter_destroy(struct percpu_counter *fbc)
97 {
98 }
99 
100 static inline void percpu_counter_set(struct percpu_counter *fbc, s64 amount)
101 {
102 	fbc->count = amount;
103 }
104 
105 #define __percpu_counter_add(fbc, amount, batch) \
106 	percpu_counter_add(fbc, amount)
107 
108 static inline void
109 percpu_counter_add(struct percpu_counter *fbc, s64 amount)
110 {
111 	preempt_disable();
112 	fbc->count += amount;
113 	preempt_enable();
114 }
115 
116 static inline s64 percpu_counter_read(struct percpu_counter *fbc)
117 {
118 	return fbc->count;
119 }
120 
121 static inline s64 percpu_counter_read_positive(struct percpu_counter *fbc)
122 {
123 	return fbc->count;
124 }
125 
126 static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc)
127 {
128 	return percpu_counter_read_positive(fbc);
129 }
130 
131 static inline s64 percpu_counter_sum(struct percpu_counter *fbc)
132 {
133 	return percpu_counter_read(fbc);
134 }
135 
136 #endif	/* CONFIG_SMP */
137 
138 static inline void percpu_counter_inc(struct percpu_counter *fbc)
139 {
140 	percpu_counter_add(fbc, 1);
141 }
142 
143 static inline void percpu_counter_dec(struct percpu_counter *fbc)
144 {
145 	percpu_counter_add(fbc, -1);
146 }
147 
148 static inline void percpu_counter_sub(struct percpu_counter *fbc, s64 amount)
149 {
150 	percpu_counter_add(fbc, -amount);
151 }
152 
153 #endif /* _LINUX_PERCPU_COUNTER_H */
154