1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2015 Cavium, Inc 3 * Copyright(c) 2020 Arm Limited 4 */ 5 6 #ifndef _RTE_ATOMIC_ARM64_H_ 7 #define _RTE_ATOMIC_ARM64_H_ 8 9 #ifndef RTE_FORCE_INTRINSICS 10 # error Platform must be built with RTE_FORCE_INTRINSICS 11 #endif 12 13 #ifdef __cplusplus 14 extern "C" { 15 #endif 16 17 #include "generic/rte_atomic.h" 18 #include <rte_branch_prediction.h> 19 #include <rte_compat.h> 20 #include <rte_debug.h> 21 22 #define rte_mb() asm volatile("dmb osh" : : : "memory") 23 24 #define rte_wmb() asm volatile("dmb oshst" : : : "memory") 25 26 #define rte_rmb() asm volatile("dmb oshld" : : : "memory") 27 28 #define rte_smp_mb() asm volatile("dmb ish" : : : "memory") 29 30 #define rte_smp_wmb() asm volatile("dmb ishst" : : : "memory") 31 32 #define rte_smp_rmb() asm volatile("dmb ishld" : : : "memory") 33 34 #define rte_io_mb() rte_mb() 35 36 #define rte_io_wmb() rte_wmb() 37 38 #define rte_io_rmb() rte_rmb() 39 40 static __rte_always_inline void 41 rte_atomic_thread_fence(int memorder) 42 { 43 __atomic_thread_fence(memorder); 44 } 45 46 /*------------------------ 128 bit atomic operations -------------------------*/ 47 48 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) 49 #if defined(RTE_CC_CLANG) 50 #define __LSE_PREAMBLE ".arch armv8-a+lse\n" 51 #else 52 #define __LSE_PREAMBLE "" 53 #endif 54 55 #define __ATOMIC128_CAS_OP(cas_op_name, op_string) \ 56 static __rte_noinline void \ 57 cas_op_name(rte_int128_t *dst, rte_int128_t *old, rte_int128_t updated) \ 58 { \ 59 /* caspX instructions register pair must start from even-numbered 60 * register at operand 1. 61 * So, specify registers for local variables here. 62 */ \ 63 register uint64_t x0 __asm("x0") = (uint64_t)old->val[0]; \ 64 register uint64_t x1 __asm("x1") = (uint64_t)old->val[1]; \ 65 register uint64_t x2 __asm("x2") = (uint64_t)updated.val[0]; \ 66 register uint64_t x3 __asm("x3") = (uint64_t)updated.val[1]; \ 67 asm volatile( \ 68 __LSE_PREAMBLE \ 69 op_string " %[old0], %[old1], %[upd0], %[upd1], [%[dst]]" \ 70 : [old0] "+r" (x0), \ 71 [old1] "+r" (x1) \ 72 : [upd0] "r" (x2), \ 73 [upd1] "r" (x3), \ 74 [dst] "r" (dst) \ 75 : "memory"); \ 76 old->val[0] = x0; \ 77 old->val[1] = x1; \ 78 } 79 80 __ATOMIC128_CAS_OP(__cas_128_relaxed, "casp") 81 __ATOMIC128_CAS_OP(__cas_128_acquire, "caspa") 82 __ATOMIC128_CAS_OP(__cas_128_release, "caspl") 83 __ATOMIC128_CAS_OP(__cas_128_acq_rel, "caspal") 84 85 #undef __LSE_PREAMBLE 86 #undef __ATOMIC128_CAS_OP 87 88 #endif 89 90 __rte_experimental 91 static inline int 92 rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp, 93 const rte_int128_t *src, unsigned int weak, int success, 94 int failure) 95 { 96 /* Always do strong CAS */ 97 RTE_SET_USED(weak); 98 /* Ignore memory ordering for failure, memory order for 99 * success must be stronger or equal 100 */ 101 RTE_SET_USED(failure); 102 /* Find invalid memory order */ 103 RTE_ASSERT(success == __ATOMIC_RELAXED || 104 success == __ATOMIC_ACQUIRE || 105 success == __ATOMIC_RELEASE || 106 success == __ATOMIC_ACQ_REL || 107 success == __ATOMIC_SEQ_CST); 108 109 rte_int128_t expected = *exp; 110 rte_int128_t desired = *src; 111 rte_int128_t old; 112 113 #if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) 114 if (success == __ATOMIC_RELAXED) 115 __cas_128_relaxed(dst, exp, desired); 116 else if (success == __ATOMIC_ACQUIRE) 117 __cas_128_acquire(dst, exp, desired); 118 else if (success == __ATOMIC_RELEASE) 119 __cas_128_release(dst, exp, desired); 120 else 121 __cas_128_acq_rel(dst, exp, desired); 122 old = *exp; 123 #else 124 #define __HAS_ACQ(mo) ((mo) != __ATOMIC_RELAXED && (mo) != __ATOMIC_RELEASE) 125 #define __HAS_RLS(mo) ((mo) == __ATOMIC_RELEASE || (mo) == __ATOMIC_ACQ_REL || \ 126 (mo) == __ATOMIC_SEQ_CST) 127 128 int ldx_mo = __HAS_ACQ(success) ? __ATOMIC_ACQUIRE : __ATOMIC_RELAXED; 129 int stx_mo = __HAS_RLS(success) ? __ATOMIC_RELEASE : __ATOMIC_RELAXED; 130 131 #undef __HAS_ACQ 132 #undef __HAS_RLS 133 134 uint32_t ret = 1; 135 136 /* ldx128 can not guarantee atomic, 137 * Must write back src or old to verify atomicity of ldx128; 138 */ 139 do { 140 141 #define __LOAD_128(op_string, src, dst) { \ 142 asm volatile( \ 143 op_string " %0, %1, %2" \ 144 : "=&r" (dst.val[0]), \ 145 "=&r" (dst.val[1]) \ 146 : "Q" (src->val[0]) \ 147 : "memory"); } 148 149 if (ldx_mo == __ATOMIC_RELAXED) 150 __LOAD_128("ldxp", dst, old) 151 else 152 __LOAD_128("ldaxp", dst, old) 153 154 #undef __LOAD_128 155 156 #define __STORE_128(op_string, dst, src, ret) { \ 157 asm volatile( \ 158 op_string " %w0, %1, %2, %3" \ 159 : "=&r" (ret) \ 160 : "r" (src.val[0]), \ 161 "r" (src.val[1]), \ 162 "Q" (dst->val[0]) \ 163 : "memory"); } 164 165 if (likely(old.int128 == expected.int128)) { 166 if (stx_mo == __ATOMIC_RELAXED) 167 __STORE_128("stxp", dst, desired, ret) 168 else 169 __STORE_128("stlxp", dst, desired, ret) 170 } else { 171 /* In the failure case (since 'weak' is ignored and only 172 * weak == 0 is implemented), expected should contain 173 * the atomically read value of dst. This means, 'old' 174 * needs to be stored back to ensure it was read 175 * atomically. 176 */ 177 if (stx_mo == __ATOMIC_RELAXED) 178 __STORE_128("stxp", dst, old, ret) 179 else 180 __STORE_128("stlxp", dst, old, ret) 181 } 182 183 #undef __STORE_128 184 185 } while (unlikely(ret)); 186 187 /* Unconditionally updating the value of exp removes an 'if' statement. 188 * The value of exp should already be in register if not in the cache. 189 */ 190 *exp = old; 191 #endif 192 193 return (old.int128 == expected.int128); 194 } 195 196 #ifdef __cplusplus 197 } 198 #endif 199 200 #endif /* _RTE_ATOMIC_ARM64_H_ */ 201