1// SPDX-License-Identifier: GPL-2.0+
2//
3// An earlier version of this file appeared in the companion webpage for
4// "Frightening small children and disconcerting grown-ups: Concurrency
5// in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern,
6// which appeared in ASPLOS 2018.
7
8// ONCE
9READ_ONCE(X) __load{once}(X)
10WRITE_ONCE(X,V) { __store{once}(X,V); }
11
12// Release Acquire and friends
13smp_store_release(X,V) { __store{release}(*X,V); }
14smp_load_acquire(X) __load{acquire}(*X)
15rcu_assign_pointer(X,V) { __store{release}(X,V); }
16rcu_dereference(X) __load{once}(X)
17smp_store_mb(X,V) { __store{once}(X,V); __fence{mb}; }
18
19// Fences
20smp_mb() { __fence{mb}; }
21smp_rmb() { __fence{rmb}; }
22smp_wmb() { __fence{wmb}; }
23smp_mb__before_atomic() { __fence{before-atomic}; }
24smp_mb__after_atomic() { __fence{after-atomic}; }
25smp_mb__after_spinlock() { __fence{after-spinlock}; }
26smp_mb__after_unlock_lock() { __fence{after-unlock-lock}; }
27smp_mb__after_srcu_read_unlock() { __fence{after-srcu-read-unlock}; }
28barrier() { __fence{barrier}; }
29
30// Exchange
31xchg(X,V)  __xchg{mb}(X,V)
32xchg_relaxed(X,V) __xchg{once}(X,V)
33xchg_release(X,V) __xchg{release}(X,V)
34xchg_acquire(X,V) __xchg{acquire}(X,V)
35cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W)
36cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W)
37cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W)
38cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W)
39
40// Spinlocks
41spin_lock(X) { __lock(X); }
42spin_unlock(X) { __unlock(X); }
43spin_trylock(X) __trylock(X)
44spin_is_locked(X) __islocked(X)
45
46// RCU
47rcu_read_lock() { __fence{rcu-lock}; }
48rcu_read_unlock() { __fence{rcu-unlock}; }
49synchronize_rcu() { __fence{sync-rcu}; }
50synchronize_rcu_expedited() { __fence{sync-rcu}; }
51
52// SRCU
53srcu_read_lock(X) __load{srcu-lock}(*X)
54srcu_read_unlock(X,Y) { __store{srcu-unlock}(*X,Y); }
55srcu_down_read(X) __load{srcu-lock}(*X)
56srcu_up_read(X,Y) { __store{srcu-unlock}(*X,Y); }
57synchronize_srcu(X)  { __srcu{sync-srcu}(X); }
58synchronize_srcu_expedited(X)  { __srcu{sync-srcu}(X); }
59
60// Atomic
61atomic_read(X) READ_ONCE(*X)
62atomic_set(X,V) { WRITE_ONCE(*X,V); }
63atomic_read_acquire(X) smp_load_acquire(X)
64atomic_set_release(X,V) { smp_store_release(X,V); }
65
66atomic_add(V,X) { __atomic_op(X,+,V); }
67atomic_sub(V,X) { __atomic_op(X,-,V); }
68atomic_and(V,X) { __atomic_op(X,&,V); }
69atomic_or(V,X)  { __atomic_op(X,|,V); }
70atomic_xor(V,X) { __atomic_op(X,^,V); }
71atomic_inc(X)   { __atomic_op(X,+,1); }
72atomic_dec(X)   { __atomic_op(X,-,1); }
73atomic_andnot(V,X) { __atomic_op(X,&~,V); }
74
75atomic_add_return(V,X) __atomic_op_return{mb}(X,+,V)
76atomic_add_return_relaxed(V,X) __atomic_op_return{once}(X,+,V)
77atomic_add_return_acquire(V,X) __atomic_op_return{acquire}(X,+,V)
78atomic_add_return_release(V,X) __atomic_op_return{release}(X,+,V)
79atomic_fetch_add(V,X) __atomic_fetch_op{mb}(X,+,V)
80atomic_fetch_add_relaxed(V,X) __atomic_fetch_op{once}(X,+,V)
81atomic_fetch_add_acquire(V,X) __atomic_fetch_op{acquire}(X,+,V)
82atomic_fetch_add_release(V,X) __atomic_fetch_op{release}(X,+,V)
83
84atomic_fetch_and(V,X) __atomic_fetch_op{mb}(X,&,V)
85atomic_fetch_and_relaxed(V,X) __atomic_fetch_op{once}(X,&,V)
86atomic_fetch_and_acquire(V,X) __atomic_fetch_op{acquire}(X,&,V)
87atomic_fetch_and_release(V,X) __atomic_fetch_op{release}(X,&,V)
88
89atomic_fetch_or(V,X) __atomic_fetch_op{mb}(X,|,V)
90atomic_fetch_or_relaxed(V,X) __atomic_fetch_op{once}(X,|,V)
91atomic_fetch_or_acquire(V,X) __atomic_fetch_op{acquire}(X,|,V)
92atomic_fetch_or_release(V,X) __atomic_fetch_op{release}(X,|,V)
93
94atomic_fetch_xor(V,X) __atomic_fetch_op{mb}(X,^,V)
95atomic_fetch_xor_relaxed(V,X) __atomic_fetch_op{once}(X,^,V)
96atomic_fetch_xor_acquire(V,X) __atomic_fetch_op{acquire}(X,^,V)
97atomic_fetch_xor_release(V,X) __atomic_fetch_op{release}(X,^,V)
98
99atomic_inc_return(X) __atomic_op_return{mb}(X,+,1)
100atomic_inc_return_relaxed(X) __atomic_op_return{once}(X,+,1)
101atomic_inc_return_acquire(X) __atomic_op_return{acquire}(X,+,1)
102atomic_inc_return_release(X) __atomic_op_return{release}(X,+,1)
103atomic_fetch_inc(X) __atomic_fetch_op{mb}(X,+,1)
104atomic_fetch_inc_relaxed(X) __atomic_fetch_op{once}(X,+,1)
105atomic_fetch_inc_acquire(X) __atomic_fetch_op{acquire}(X,+,1)
106atomic_fetch_inc_release(X) __atomic_fetch_op{release}(X,+,1)
107
108atomic_sub_return(V,X) __atomic_op_return{mb}(X,-,V)
109atomic_sub_return_relaxed(V,X) __atomic_op_return{once}(X,-,V)
110atomic_sub_return_acquire(V,X) __atomic_op_return{acquire}(X,-,V)
111atomic_sub_return_release(V,X) __atomic_op_return{release}(X,-,V)
112atomic_fetch_sub(V,X) __atomic_fetch_op{mb}(X,-,V)
113atomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{once}(X,-,V)
114atomic_fetch_sub_acquire(V,X) __atomic_fetch_op{acquire}(X,-,V)
115atomic_fetch_sub_release(V,X) __atomic_fetch_op{release}(X,-,V)
116
117atomic_dec_return(X) __atomic_op_return{mb}(X,-,1)
118atomic_dec_return_relaxed(X) __atomic_op_return{once}(X,-,1)
119atomic_dec_return_acquire(X) __atomic_op_return{acquire}(X,-,1)
120atomic_dec_return_release(X) __atomic_op_return{release}(X,-,1)
121atomic_fetch_dec(X) __atomic_fetch_op{mb}(X,-,1)
122atomic_fetch_dec_relaxed(X) __atomic_fetch_op{once}(X,-,1)
123atomic_fetch_dec_acquire(X) __atomic_fetch_op{acquire}(X,-,1)
124atomic_fetch_dec_release(X) __atomic_fetch_op{release}(X,-,1)
125
126atomic_xchg(X,V) __xchg{mb}(X,V)
127atomic_xchg_relaxed(X,V) __xchg{once}(X,V)
128atomic_xchg_release(X,V) __xchg{release}(X,V)
129atomic_xchg_acquire(X,V) __xchg{acquire}(X,V)
130atomic_cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W)
131atomic_cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W)
132atomic_cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W)
133atomic_cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W)
134
135atomic_sub_and_test(V,X) __atomic_op_return{mb}(X,-,V) == 0
136atomic_dec_and_test(X)  __atomic_op_return{mb}(X,-,1) == 0
137atomic_inc_and_test(X)  __atomic_op_return{mb}(X,+,1) == 0
138atomic_add_negative(V,X) __atomic_op_return{mb}(X,+,V) < 0
139atomic_add_negative_relaxed(V,X) __atomic_op_return{once}(X,+,V) < 0
140atomic_add_negative_acquire(V,X) __atomic_op_return{acquire}(X,+,V) < 0
141atomic_add_negative_release(V,X) __atomic_op_return{release}(X,+,V) < 0
142
143atomic_fetch_andnot(V,X) __atomic_fetch_op{mb}(X,&~,V)
144atomic_fetch_andnot_acquire(V,X) __atomic_fetch_op{acquire}(X,&~,V)
145atomic_fetch_andnot_release(V,X) __atomic_fetch_op{release}(X,&~,V)
146atomic_fetch_andnot_relaxed(V,X) __atomic_fetch_op{once}(X,&~,V)
147