1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 * Copyright(c) 2019 Arm Limited 4 */ 5 6 #ifndef _RTE_PAUSE_ARM64_H_ 7 #define _RTE_PAUSE_ARM64_H_ 8 9 #ifdef __cplusplus 10 extern "C" { 11 #endif 12 13 #include <rte_common.h> 14 15 #ifdef RTE_ARM_USE_WFE 16 #define RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED 17 #endif 18 19 #include "generic/rte_pause.h" 20 21 static inline void rte_pause(void) 22 { 23 asm volatile("yield" ::: "memory"); 24 } 25 26 #ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED 27 28 /* Send an event to quit WFE. */ 29 #define __RTE_ARM_SEVL() { asm volatile("sevl" : : : "memory"); } 30 31 /* Put processor into low power WFE(Wait For Event) state. */ 32 #define __RTE_ARM_WFE() { asm volatile("wfe" : : : "memory"); } 33 34 /* 35 * Atomic exclusive load from addr, it returns the 16-bit content of 36 * *addr while making it 'monitored', when it is written by someone 37 * else, the 'monitored' state is cleared and an event is generated 38 * implicitly to exit WFE. 39 */ 40 #define __RTE_ARM_LOAD_EXC_16(src, dst, memorder) { \ 41 if (memorder == __ATOMIC_RELAXED) { \ 42 asm volatile("ldxrh %w[tmp], [%x[addr]]" \ 43 : [tmp] "=&r" (dst) \ 44 : [addr] "r" (src) \ 45 : "memory"); \ 46 } else { \ 47 asm volatile("ldaxrh %w[tmp], [%x[addr]]" \ 48 : [tmp] "=&r" (dst) \ 49 : [addr] "r" (src) \ 50 : "memory"); \ 51 } } 52 53 /* 54 * Atomic exclusive load from addr, it returns the 32-bit content of 55 * *addr while making it 'monitored', when it is written by someone 56 * else, the 'monitored' state is cleared and an event is generated 57 * implicitly to exit WFE. 58 */ 59 #define __RTE_ARM_LOAD_EXC_32(src, dst, memorder) { \ 60 if (memorder == __ATOMIC_RELAXED) { \ 61 asm volatile("ldxr %w[tmp], [%x[addr]]" \ 62 : [tmp] "=&r" (dst) \ 63 : [addr] "r" (src) \ 64 : "memory"); \ 65 } else { \ 66 asm volatile("ldaxr %w[tmp], [%x[addr]]" \ 67 : [tmp] "=&r" (dst) \ 68 : [addr] "r" (src) \ 69 : "memory"); \ 70 } } 71 72 /* 73 * Atomic exclusive load from addr, it returns the 64-bit content of 74 * *addr while making it 'monitored', when it is written by someone 75 * else, the 'monitored' state is cleared and an event is generated 76 * implicitly to exit WFE. 77 */ 78 #define __RTE_ARM_LOAD_EXC_64(src, dst, memorder) { \ 79 if (memorder == __ATOMIC_RELAXED) { \ 80 asm volatile("ldxr %x[tmp], [%x[addr]]" \ 81 : [tmp] "=&r" (dst) \ 82 : [addr] "r" (src) \ 83 : "memory"); \ 84 } else { \ 85 asm volatile("ldaxr %x[tmp], [%x[addr]]" \ 86 : [tmp] "=&r" (dst) \ 87 : [addr] "r" (src) \ 88 : "memory"); \ 89 } } 90 91 /* 92 * Atomic exclusive load from addr, it returns the 128-bit content of 93 * *addr while making it 'monitored', when it is written by someone 94 * else, the 'monitored' state is cleared and an event is generated 95 * implicitly to exit WFE. 96 */ 97 #define __RTE_ARM_LOAD_EXC_128(src, dst, memorder) { \ 98 volatile rte_int128_t *dst_128 = (volatile rte_int128_t *)&dst; \ 99 if (memorder == __ATOMIC_RELAXED) { \ 100 asm volatile("ldxp %x[tmp0], %x[tmp1], [%x[addr]]" \ 101 : [tmp0] "=&r" (dst_128->val[0]), \ 102 [tmp1] "=&r" (dst_128->val[1]) \ 103 : [addr] "r" (src) \ 104 : "memory"); \ 105 } else { \ 106 asm volatile("ldaxp %x[tmp0], %x[tmp1], [%x[addr]]" \ 107 : [tmp0] "=&r" (dst_128->val[0]), \ 108 [tmp1] "=&r" (dst_128->val[1]) \ 109 : [addr] "r" (src) \ 110 : "memory"); \ 111 } } \ 112 113 #define __RTE_ARM_LOAD_EXC(src, dst, memorder, size) { \ 114 RTE_BUILD_BUG_ON(size != 16 && size != 32 && \ 115 size != 64 && size != 128); \ 116 if (size == 16) \ 117 __RTE_ARM_LOAD_EXC_16(src, dst, memorder) \ 118 else if (size == 32) \ 119 __RTE_ARM_LOAD_EXC_32(src, dst, memorder) \ 120 else if (size == 64) \ 121 __RTE_ARM_LOAD_EXC_64(src, dst, memorder) \ 122 else if (size == 128) \ 123 __RTE_ARM_LOAD_EXC_128(src, dst, memorder) \ 124 } 125 126 static __rte_always_inline void 127 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, 128 int memorder) 129 { 130 uint16_t value; 131 132 RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && 133 memorder != __ATOMIC_RELAXED); 134 135 __RTE_ARM_LOAD_EXC_16(addr, value, memorder) 136 if (value != expected) { 137 __RTE_ARM_SEVL() 138 do { 139 __RTE_ARM_WFE() 140 __RTE_ARM_LOAD_EXC_16(addr, value, memorder) 141 } while (value != expected); 142 } 143 } 144 145 static __rte_always_inline void 146 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, 147 int memorder) 148 { 149 uint32_t value; 150 151 RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && 152 memorder != __ATOMIC_RELAXED); 153 154 __RTE_ARM_LOAD_EXC_32(addr, value, memorder) 155 if (value != expected) { 156 __RTE_ARM_SEVL() 157 do { 158 __RTE_ARM_WFE() 159 __RTE_ARM_LOAD_EXC_32(addr, value, memorder) 160 } while (value != expected); 161 } 162 } 163 164 static __rte_always_inline void 165 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, 166 int memorder) 167 { 168 uint64_t value; 169 170 RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && 171 memorder != __ATOMIC_RELAXED); 172 173 __RTE_ARM_LOAD_EXC_64(addr, value, memorder) 174 if (value != expected) { 175 __RTE_ARM_SEVL() 176 do { 177 __RTE_ARM_WFE() 178 __RTE_ARM_LOAD_EXC_64(addr, value, memorder) 179 } while (value != expected); 180 } 181 } 182 183 #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \ 184 RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \ 185 RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && \ 186 memorder != __ATOMIC_RELAXED); \ 187 const uint32_t size = sizeof(*(addr)) << 3; \ 188 typeof(*(addr)) expected_value = (expected); \ 189 typeof(*(addr)) value; \ 190 __RTE_ARM_LOAD_EXC((addr), value, memorder, size) \ 191 if (!((value & (mask)) cond expected_value)) { \ 192 __RTE_ARM_SEVL() \ 193 do { \ 194 __RTE_ARM_WFE() \ 195 __RTE_ARM_LOAD_EXC((addr), value, memorder, size) \ 196 } while (!((value & (mask)) cond expected_value)); \ 197 } \ 198 } while (0) 199 200 #endif /* RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED */ 201 202 #ifdef __cplusplus 203 } 204 #endif 205 206 #endif /* _RTE_PAUSE_ARM64_H_ */ 207