1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2015 Cavium, Inc
3 * Copyright(c) 2020 Arm Limited
4 */
5
6 #ifndef _RTE_ATOMIC_ARM64_H_
7 #define _RTE_ATOMIC_ARM64_H_
8
9 #ifndef RTE_FORCE_INTRINSICS
10 # error Platform must be built with RTE_FORCE_INTRINSICS
11 #endif
12
13 #ifdef __cplusplus
14 extern "C" {
15 #endif
16
17 #include "generic/rte_atomic.h"
18 #include <rte_branch_prediction.h>
19 #include <rte_compat.h>
20 #include <rte_debug.h>
21
22 #define rte_mb() asm volatile("dmb osh" : : : "memory")
23
24 #define rte_wmb() asm volatile("dmb oshst" : : : "memory")
25
26 #define rte_rmb() asm volatile("dmb oshld" : : : "memory")
27
28 #define rte_smp_mb() asm volatile("dmb ish" : : : "memory")
29
30 #define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory")
31
32 #define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory")
33
34 #define rte_io_mb() rte_mb()
35
36 #define rte_io_wmb() rte_wmb()
37
38 #define rte_io_rmb() rte_rmb()
39
40 static __rte_always_inline void
rte_atomic_thread_fence(int memorder)41 rte_atomic_thread_fence(int memorder)
42 {
43 __atomic_thread_fence(memorder);
44 }
45
46 /*------------------------ 128 bit atomic operations -------------------------*/
47
48 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
49
50 #if defined(RTE_CC_CLANG)
51 #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \
52 static __rte_noinline void \
53 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \
54 { \
55 /* caspX instructions register pair must start from even-numbered
56 * register at operand 1.
57 * So, specify registers for local variables here.
58 */ \
59 register uint64_t x0 __asm("x0") = (uint64_t)old->val[0]; \
60 register uint64_t x1 __asm("x1") = (uint64_t)old->val[1]; \
61 register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \
62 register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \
63 asm volatile( \
64 ".arch armv8-a+lse\n" \
65 op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \
66 : [old0] "+r" (x0), \
67 [old1] "+r" (x1) \
68 : [upd0] "r" (x2), \
69 [upd1] "r" (x3), \
70 [dst] "r" (dst) \
71 : "memory"); \
72 old->val[0] = x0; \
73 old->val[1] = x1; \
74 }
75 #else
76 #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \
77 static __rte_always_inline void \
78 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \
79 { \
80 asm volatile( \
81 op_string " %[old], %H[old], %[upd], %H[upd], [%[dst]]" \
82 : [old] "+r"(old->int128) \
83 : [upd] "r"(updated.int128), [dst] "r"(dst) \
84 : "memory"); \
85 }
86 #endif
87
88 __ATOMIC128_CAS_OP(__cas_128_relaxed, "casp")
89 __ATOMIC128_CAS_OP(__cas_128_acquire, "caspa")
90 __ATOMIC128_CAS_OP(__cas_128_release, "caspl")
91 __ATOMIC128_CAS_OP(__cas_128_acq_rel, "caspal")
92
93 #undef __ATOMIC128_CAS_OP
94
95 #endif
96
97 __rte_experimental
98 static inline int
rte_atomic128_cmp_exchange(rte_int128_t * dst,rte_int128_t * exp,const rte_int128_t * src,unsigned int weak,int success,int failure)99 rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp,
100 const rte_int128_t *src, unsigned int weak, int success,
101 int failure)
102 {
103 /* Always do strong CAS */
104 RTE_SET_USED(weak);
105 /* Ignore memory ordering for failure, memory order for
106 * success must be stronger or equal
107 */
108 RTE_SET_USED(failure);
109 /* Find invalid memory order */
110 RTE_ASSERT(success == __ATOMIC_RELAXED ||
111 success == __ATOMIC_ACQUIRE ||
112 success == __ATOMIC_RELEASE ||
113 success == __ATOMIC_ACQ_REL ||
114 success == __ATOMIC_SEQ_CST);
115
116 rte_int128_t expected = *exp;
117 rte_int128_t desired = *src;
118 rte_int128_t old;
119
120 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
121 if (success == __ATOMIC_RELAXED)
122 __cas_128_relaxed(dst, exp, desired);
123 else if (success == __ATOMIC_ACQUIRE)
124 __cas_128_acquire(dst, exp, desired);
125 else if (success == __ATOMIC_RELEASE)
126 __cas_128_release(dst, exp, desired);
127 else
128 __cas_128_acq_rel(dst, exp, desired);
129 old = *exp;
130 #else
131 #define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE)
132 #define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \
133 (mo) == __ATOMIC_SEQ_CST)
134
135 int ldx_mo = __HAS_ACQ(success) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED;
136 int stx_mo = __HAS_RLS(success) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED;
137
138 #undef __HAS_ACQ
139 #undef __HAS_RLS
140
141 uint32_t ret = 1;
142
143 /* ldx128 can not guarantee atomic,
144 * Must write back src or old to verify atomicity of ldx128;
145 */
146 do {
147
148 #define __LOAD_128(op_string, src, dst) { \
149 asm volatile( \
150 op_string " %0, %1, %2" \
151 : "=&r" (dst.val[0]), \
152 "=&r" (dst.val[1]) \
153 : "Q" (src->val[0]) \
154 : "memory"); }
155
156 if (ldx_mo == __ATOMIC_RELAXED)
157 __LOAD_128("ldxp", dst, old)
158 else
159 __LOAD_128("ldaxp", dst, old)
160
161 #undef __LOAD_128
162
163 #define __STORE_128(op_string, dst, src, ret) { \
164 asm volatile( \
165 op_string " %w0, %1, %2, %3" \
166 : "=&r" (ret) \
167 : "r" (src.val[0]), \
168 "r" (src.val[1]), \
169 "Q" (dst->val[0]) \
170 : "memory"); }
171
172 if (likely(old.int128 == expected.int128)) {
173 if (stx_mo == __ATOMIC_RELAXED)
174 __STORE_128("stxp", dst, desired, ret)
175 else
176 __STORE_128("stlxp", dst, desired, ret)
177 } else {
178 /* In the failure case (since 'weak' is ignored and only
179 * weak == 0 is implemented), expected should contain
180 * the atomically read value of dst. This means, 'old'
181 * needs to be stored back to ensure it was read
182 * atomically.
183 */
184 if (stx_mo == __ATOMIC_RELAXED)
185 __STORE_128("stxp", dst, old, ret)
186 else
187 __STORE_128("stlxp", dst, old, ret)
188 }
189
190 #undef __STORE_128
191
192 } while (unlikely(ret));
193
194 /* Unconditionally updating the value of exp removes an 'if' statement.
195 * The value of exp should already be in register if not in the cache.
196 */
197 *exp = old;
198 #endif
199
200 return (old.int128 == expected.int128);
201 }
202
203 #ifdef __cplusplus
204 }
205 #endif
206
207 #endif /* _RTE_ATOMIC_ARM64_H_ */
208