1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 * Copyright(c) 2019 Arm Limited 4 */ 5 6 #ifndef _RTE_PAUSE_ARM64_H_ 7 #define _RTE_PAUSE_ARM64_H_ 8 9 #ifdef __cplusplus 10 extern "C" { 11 #endif 12 13 #include <rte_common.h> 14 15 #ifdef RTE_ARM_USE_WFE 16 #define RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED 17 #endif 18 19 #include "generic/rte_pause.h" 20 21 static inline void rte_pause(void) 22 { 23 asm volatile("yield" ::: "memory"); 24 } 25 26 #ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED 27 28 /* Send an event to quit WFE. */ 29 #define __SEVL() { asm volatile("sevl" : : : "memory"); } 30 31 /* Put processor into low power WFE(Wait For Event) state. */ 32 #define __WFE() { asm volatile("wfe" : : : "memory"); } 33 34 static __rte_always_inline void 35 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected, 36 int memorder) 37 { 38 uint16_t value; 39 40 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); 41 42 /* 43 * Atomic exclusive load from addr, it returns the 16-bit content of 44 * *addr while making it 'monitored',when it is written by someone 45 * else, the 'monitored' state is cleared and a event is generated 46 * implicitly to exit WFE. 47 */ 48 #define __LOAD_EXC_16(src, dst, memorder) { \ 49 if (memorder == __ATOMIC_RELAXED) { \ 50 asm volatile("ldxrh %w[tmp], [%x[addr]]" \ 51 : [tmp] "=&r" (dst) \ 52 : [addr] "r"(src) \ 53 : "memory"); \ 54 } else { \ 55 asm volatile("ldaxrh %w[tmp], [%x[addr]]" \ 56 : [tmp] "=&r" (dst) \ 57 : [addr] "r"(src) \ 58 : "memory"); \ 59 } } 60 61 __LOAD_EXC_16(addr, value, memorder) 62 if (value != expected) { 63 __SEVL() 64 do { 65 __WFE() 66 __LOAD_EXC_16(addr, value, memorder) 67 } while (value != expected); 68 } 69 #undef __LOAD_EXC_16 70 } 71 72 static __rte_always_inline void 73 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected, 74 int memorder) 75 { 76 uint32_t value; 77 78 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); 79 80 /* 81 * Atomic exclusive load from addr, it returns the 32-bit content of 82 * *addr while making it 'monitored',when it is written by someone 83 * else, the 'monitored' state is cleared and a event is generated 84 * implicitly to exit WFE. 85 */ 86 #define __LOAD_EXC_32(src, dst, memorder) { \ 87 if (memorder == __ATOMIC_RELAXED) { \ 88 asm volatile("ldxr %w[tmp], [%x[addr]]" \ 89 : [tmp] "=&r" (dst) \ 90 : [addr] "r"(src) \ 91 : "memory"); \ 92 } else { \ 93 asm volatile("ldaxr %w[tmp], [%x[addr]]" \ 94 : [tmp] "=&r" (dst) \ 95 : [addr] "r"(src) \ 96 : "memory"); \ 97 } } 98 99 __LOAD_EXC_32(addr, value, memorder) 100 if (value != expected) { 101 __SEVL() 102 do { 103 __WFE() 104 __LOAD_EXC_32(addr, value, memorder) 105 } while (value != expected); 106 } 107 #undef __LOAD_EXC_32 108 } 109 110 static __rte_always_inline void 111 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected, 112 int memorder) 113 { 114 uint64_t value; 115 116 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED); 117 118 /* 119 * Atomic exclusive load from addr, it returns the 64-bit content of 120 * *addr while making it 'monitored',when it is written by someone 121 * else, the 'monitored' state is cleared and a event is generated 122 * implicitly to exit WFE. 123 */ 124 #define __LOAD_EXC_64(src, dst, memorder) { \ 125 if (memorder == __ATOMIC_RELAXED) { \ 126 asm volatile("ldxr %x[tmp], [%x[addr]]" \ 127 : [tmp] "=&r" (dst) \ 128 : [addr] "r"(src) \ 129 : "memory"); \ 130 } else { \ 131 asm volatile("ldaxr %x[tmp], [%x[addr]]" \ 132 : [tmp] "=&r" (dst) \ 133 : [addr] "r"(src) \ 134 : "memory"); \ 135 } } 136 137 __LOAD_EXC_64(addr, value, memorder) 138 if (value != expected) { 139 __SEVL() 140 do { 141 __WFE() 142 __LOAD_EXC_64(addr, value, memorder) 143 } while (value != expected); 144 } 145 } 146 #undef __LOAD_EXC_64 147 148 #undef __SEVL 149 #undef __WFE 150 151 #endif 152 153 #ifdef __cplusplus 154 } 155 #endif 156 157 #endif /* _RTE_PAUSE_ARM64_H_ */ 158