1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
3 * Copyright(c) 2019 Arm Limited
4 */
5
6 #ifndef _RTE_PAUSE_ARM64_H_
7 #define _RTE_PAUSE_ARM64_H_
8
9 #ifdef __cplusplus
10 extern "C" {
11 #endif
12
13 #include <rte_common.h>
14
15 #ifdef RTE_ARM_USE_WFE
16 #define RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
17 #endif
18
19 #include "generic/rte_pause.h"
20
rte_pause(void)21 static inline void rte_pause(void)
22 {
23 asm volatile("yield" ::: "memory");
24 }
25
26 #ifdef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
27
28 /* Send an event to quit WFE. */
29 #define __SEVL() { asm volatile("sevl" : : : "memory"); }
30
31 /* Put processor into low power WFE(Wait For Event) state. */
32 #define __WFE() { asm volatile("wfe" : : : "memory"); }
33
34 static __rte_always_inline void
rte_wait_until_equal_16(volatile uint16_t * addr,uint16_t expected,int memorder)35 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
36 int memorder)
37 {
38 uint16_t value;
39
40 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
41
42 /*
43 * Atomic exclusive load from addr, it returns the 16-bit content of
44 * *addr while making it 'monitored',when it is written by someone
45 * else, the 'monitored' state is cleared and a event is generated
46 * implicitly to exit WFE.
47 */
48 #define __LOAD_EXC_16(src, dst, memorder) { \
49 if (memorder == __ATOMIC_RELAXED) { \
50 asm volatile("ldxrh %w[tmp], [%x[addr]]" \
51 : [tmp] "=&r" (dst) \
52 : [addr] "r"(src) \
53 : "memory"); \
54 } else { \
55 asm volatile("ldaxrh %w[tmp], [%x[addr]]" \
56 : [tmp] "=&r" (dst) \
57 : [addr] "r"(src) \
58 : "memory"); \
59 } }
60
61 __LOAD_EXC_16(addr, value, memorder)
62 if (value != expected) {
63 __SEVL()
64 do {
65 __WFE()
66 __LOAD_EXC_16(addr, value, memorder)
67 } while (value != expected);
68 }
69 #undef __LOAD_EXC_16
70 }
71
72 static __rte_always_inline void
rte_wait_until_equal_32(volatile uint32_t * addr,uint32_t expected,int memorder)73 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
74 int memorder)
75 {
76 uint32_t value;
77
78 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
79
80 /*
81 * Atomic exclusive load from addr, it returns the 32-bit content of
82 * *addr while making it 'monitored',when it is written by someone
83 * else, the 'monitored' state is cleared and a event is generated
84 * implicitly to exit WFE.
85 */
86 #define __LOAD_EXC_32(src, dst, memorder) { \
87 if (memorder == __ATOMIC_RELAXED) { \
88 asm volatile("ldxr %w[tmp], [%x[addr]]" \
89 : [tmp] "=&r" (dst) \
90 : [addr] "r"(src) \
91 : "memory"); \
92 } else { \
93 asm volatile("ldaxr %w[tmp], [%x[addr]]" \
94 : [tmp] "=&r" (dst) \
95 : [addr] "r"(src) \
96 : "memory"); \
97 } }
98
99 __LOAD_EXC_32(addr, value, memorder)
100 if (value != expected) {
101 __SEVL()
102 do {
103 __WFE()
104 __LOAD_EXC_32(addr, value, memorder)
105 } while (value != expected);
106 }
107 #undef __LOAD_EXC_32
108 }
109
110 static __rte_always_inline void
rte_wait_until_equal_64(volatile uint64_t * addr,uint64_t expected,int memorder)111 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
112 int memorder)
113 {
114 uint64_t value;
115
116 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
117
118 /*
119 * Atomic exclusive load from addr, it returns the 64-bit content of
120 * *addr while making it 'monitored',when it is written by someone
121 * else, the 'monitored' state is cleared and a event is generated
122 * implicitly to exit WFE.
123 */
124 #define __LOAD_EXC_64(src, dst, memorder) { \
125 if (memorder == __ATOMIC_RELAXED) { \
126 asm volatile("ldxr %x[tmp], [%x[addr]]" \
127 : [tmp] "=&r" (dst) \
128 : [addr] "r"(src) \
129 : "memory"); \
130 } else { \
131 asm volatile("ldaxr %x[tmp], [%x[addr]]" \
132 : [tmp] "=&r" (dst) \
133 : [addr] "r"(src) \
134 : "memory"); \
135 } }
136
137 __LOAD_EXC_64(addr, value, memorder)
138 if (value != expected) {
139 __SEVL()
140 do {
141 __WFE()
142 __LOAD_EXC_64(addr, value, memorder)
143 } while (value != expected);
144 }
145 }
146 #undef __LOAD_EXC_64
147
148 #undef __SEVL
149 #undef __WFE
150
151 #endif
152
153 #ifdef __cplusplus
154 }
155 #endif
156
157 #endif /* _RTE_PAUSE_ARM64_H_ */
158