1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015 Cavium, Inc 3 * Copyright(c) 2020 Arm Limited 4 */ 5 6 #ifndef _RTE_ATOMIC_ARM64_H_ 7 #define _RTE_ATOMIC_ARM64_H_ 8 9 #ifndef RTE_FORCE_INTRINSICS 10 # error Platform must be built with RTE_FORCE_INTRINSICS 11 #endif 12 13 #ifdef __cplusplus 14 extern "C" { 15 #endif 16 17 #include "generic/rte_atomic.h" 18 #include <rte_branch_prediction.h> 19 #include <rte_compat.h> 20 #include <rte_debug.h> 21 22 #define rte_mb() asm volatile("dmb osh" : : : "memory") 23 24 #define rte_wmb() asm volatile("dmb oshst" : : : "memory") 25 26 #define rte_rmb() asm volatile("dmb oshld" : : : "memory") 27 28 #define rte_smp_mb() asm volatile("dmb ish" : : : "memory") 29 30 #define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory") 31 32 #define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory") 33 34 #define rte_io_mb() rte_mb() 35 36 #define rte_io_wmb() rte_wmb() 37 38 #define rte_io_rmb() rte_rmb() 39 40 static __rte_always_inline void 41 rte_atomic_thread_fence(int memorder) 42 { 43 __atomic_thread_fence(memorder); 44 } 45 46 /*------------------------ 128 bit atomic operations -------------------------*/ 47 48 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) 49 50 #if defined(RTE_CC_CLANG) 51 #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \ 52 static __rte_noinline void \ 53 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ 54 { \ 55 /* caspX instructions register pair must start from even-numbered 56 * register at operand 1. 57 * So, specify registers for local variables here. 58 */ \ 59 register uint64_t x0 __asm("x0") = (uint64_t)old->val[0]; \ 60 register uint64_t x1 __asm("x1") = (uint64_t)old->val[1]; \ 61 register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ 62 register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ 63 asm volatile( \ 64 ".arch armv8-a+lse\n" \ 65 op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \ 66 : [old0] "+r" (x0), \ 67 [old1] "+r" (x1) \ 68 : [upd0] "r" (x2), \ 69 [upd1] "r" (x3), \ 70 [dst] "r" (dst) \ 71 : "memory"); \ 72 old->val[0] = x0; \ 73 old->val[1] = x1; \ 74 } 75 #else 76 #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \ 77 static __rte_always_inline void \ 78 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ 79 { \ 80 asm volatile( \ 81 op_string " %[old], %H[old], %[upd], %H[upd], [%[dst]]" \ 82 : [old] "+r"(old->int128) \ 83 : [upd] "r"(updated.int128), [dst] "r"(dst) \ 84 : "memory"); \ 85 } 86 #endif 87 88 __ATOMIC128_CAS_OP(__cas_128_relaxed, "casp") 89 __ATOMIC128_CAS_OP(__cas_128_acquire, "caspa") 90 __ATOMIC128_CAS_OP(__cas_128_release, "caspl") 91 __ATOMIC128_CAS_OP(__cas_128_acq_rel, "caspal") 92 93 #undef __ATOMIC128_CAS_OP 94 95 #endif 96 97 __rte_experimental 98 static inline int 99 rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp, 100 const rte_int128_t *src, unsigned int weak, int success, 101 int failure) 102 { 103 /* Always do strong CAS */ 104 RTE_SET_USED(weak); 105 /* Ignore memory ordering for failure, memory order for 106 * success must be stronger or equal 107 */ 108 RTE_SET_USED(failure); 109 /* Find invalid memory order */ 110 RTE_ASSERT(success == __ATOMIC_RELAXED || 111 success == __ATOMIC_ACQUIRE || 112 success == __ATOMIC_RELEASE || 113 success == __ATOMIC_ACQ_REL || 114 success == __ATOMIC_SEQ_CST); 115 116 rte_int128_t expected = *exp; 117 rte_int128_t desired = *src; 118 rte_int128_t old; 119 120 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) 121 if (success == __ATOMIC_RELAXED) 122 __cas_128_relaxed(dst, exp, desired); 123 else if (success == __ATOMIC_ACQUIRE) 124 __cas_128_acquire(dst, exp, desired); 125 else if (success == __ATOMIC_RELEASE) 126 __cas_128_release(dst, exp, desired); 127 else 128 __cas_128_acq_rel(dst, exp, desired); 129 old = *exp; 130 #else 131 #define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) 132 #define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \ 133 (mo) == __ATOMIC_SEQ_CST) 134 135 int ldx_mo = __HAS_ACQ(success) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED; 136 int stx_mo = __HAS_RLS(success) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED; 137 138 #undef __HAS_ACQ 139 #undef __HAS_RLS 140 141 uint32_t ret = 1; 142 143 /* ldx128 can not guarantee atomic, 144 * Must write back src or old to verify atomicity of ldx128; 145 */ 146 do { 147 148 #define __LOAD_128(op_string, src, dst) { \ 149 asm volatile( \ 150 op_string " %0, %1, %2" \ 151 : "=&r" (dst.val[0]), \ 152 "=&r" (dst.val[1]) \ 153 : "Q" (src->val[0]) \ 154 : "memory"); } 155 156 if (ldx_mo == __ATOMIC_RELAXED) 157 __LOAD_128("ldxp", dst, old) 158 else 159 __LOAD_128("ldaxp", dst, old) 160 161 #undef __LOAD_128 162 163 #define __STORE_128(op_string, dst, src, ret) { \ 164 asm volatile( \ 165 op_string " %w0, %1, %2, %3" \ 166 : "=&r" (ret) \ 167 : "r" (src.val[0]), \ 168 "r" (src.val[1]), \ 169 "Q" (dst->val[0]) \ 170 : "memory"); } 171 172 if (likely(old.int128 == expected.int128)) { 173 if (stx_mo == __ATOMIC_RELAXED) 174 __STORE_128("stxp", dst, desired, ret) 175 else 176 __STORE_128("stlxp", dst, desired, ret) 177 } else { 178 /* In the failure case (since 'weak' is ignored and only 179 * weak == 0 is implemented), expected should contain 180 * the atomically read value of dst. This means, 'old' 181 * needs to be stored back to ensure it was read 182 * atomically. 183 */ 184 if (stx_mo == __ATOMIC_RELAXED) 185 __STORE_128("stxp", dst, old, ret) 186 else 187 __STORE_128("stlxp", dst, old, ret) 188 } 189 190 #undef __STORE_128 191 192 } while (unlikely(ret)); 193 194 /* Unconditionally updating the value of exp removes an 'if' statement. 195 * The value of exp should already be in register if not in the cache. 196 */ 197 *exp = old; 198 #endif 199 200 return (old.int128 == expected.int128); 201 } 202 203 #ifdef __cplusplus 204 } 205 #endif 206 207 #endif /* _RTE_ATOMIC_ARM64_H_ */ 208