1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
3 * Copyright(c) 2019 Arm Limited
4 */
5
6 #ifndef _RTE_PAUSE_H_
7 #define _RTE_PAUSE_H_
8
9 /**
10 * @file
11 *
12 * CPU pause operation.
13 *
14 */
15
16 #include <stdint.h>
17 #include <assert.h>
18 #include <rte_common.h>
19 #include <rte_atomic.h>
20 #include <rte_compat.h>
21
22 /**
23 * Pause CPU execution for a short while
24 *
25 * This call is intended for tight loops which poll a shared resource or wait
26 * for an event. A short pause within the loop may reduce the power consumption.
27 */
28 static inline void rte_pause(void);
29
30 /**
31 * Wait for *addr to be updated with a 16-bit expected value, with a relaxed
32 * memory ordering model meaning the loads around this API can be reordered.
33 *
34 * @param addr
35 * A pointer to the memory location.
36 * @param expected
37 * A 16-bit expected value to be in the memory location.
38 * @param memorder
39 * Two different memory orders that can be specified:
40 * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
41 * C++11 memory orders with the same names, see the C++11 standard or
42 * the GCC wiki on atomic synchronization for detailed definition.
43 */
44 static __rte_always_inline void
45 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
46 int memorder);
47
48 /**
49 * Wait for *addr to be updated with a 32-bit expected value, with a relaxed
50 * memory ordering model meaning the loads around this API can be reordered.
51 *
52 * @param addr
53 * A pointer to the memory location.
54 * @param expected
55 * A 32-bit expected value to be in the memory location.
56 * @param memorder
57 * Two different memory orders that can be specified:
58 * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
59 * C++11 memory orders with the same names, see the C++11 standard or
60 * the GCC wiki on atomic synchronization for detailed definition.
61 */
62 static __rte_always_inline void
63 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
64 int memorder);
65
66 /**
67 * Wait for *addr to be updated with a 64-bit expected value, with a relaxed
68 * memory ordering model meaning the loads around this API can be reordered.
69 *
70 * @param addr
71 * A pointer to the memory location.
72 * @param expected
73 * A 64-bit expected value to be in the memory location.
74 * @param memorder
75 * Two different memory orders that can be specified:
76 * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
77 * C++11 memory orders with the same names, see the C++11 standard or
78 * the GCC wiki on atomic synchronization for detailed definition.
79 */
80 static __rte_always_inline void
81 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
82 int memorder);
83
84 #ifndef RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED
85 static __rte_always_inline void
rte_wait_until_equal_16(volatile uint16_t * addr,uint16_t expected,int memorder)86 rte_wait_until_equal_16(volatile uint16_t *addr, uint16_t expected,
87 int memorder)
88 {
89 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
90
91 while (__atomic_load_n(addr, memorder) != expected)
92 rte_pause();
93 }
94
95 static __rte_always_inline void
rte_wait_until_equal_32(volatile uint32_t * addr,uint32_t expected,int memorder)96 rte_wait_until_equal_32(volatile uint32_t *addr, uint32_t expected,
97 int memorder)
98 {
99 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
100
101 while (__atomic_load_n(addr, memorder) != expected)
102 rte_pause();
103 }
104
105 static __rte_always_inline void
rte_wait_until_equal_64(volatile uint64_t * addr,uint64_t expected,int memorder)106 rte_wait_until_equal_64(volatile uint64_t *addr, uint64_t expected,
107 int memorder)
108 {
109 assert(memorder == __ATOMIC_ACQUIRE || memorder == __ATOMIC_RELAXED);
110
111 while (__atomic_load_n(addr, memorder) != expected)
112 rte_pause();
113 }
114
115 /*
116 * Wait until *addr & mask makes the condition true. With a relaxed memory
117 * ordering model, the loads around this helper can be reordered.
118 *
119 * @param addr
120 * A pointer to the memory location.
121 * @param mask
122 * A mask of value bits in interest.
123 * @param cond
124 * A symbol representing the condition.
125 * @param expected
126 * An expected value to be in the memory location.
127 * @param memorder
128 * Two different memory orders that can be specified:
129 * __ATOMIC_ACQUIRE and __ATOMIC_RELAXED. These map to
130 * C++11 memory orders with the same names, see the C++11 standard or
131 * the GCC wiki on atomic synchronization for detailed definition.
132 */
133 #define RTE_WAIT_UNTIL_MASKED(addr, mask, cond, expected, memorder) do { \
134 RTE_BUILD_BUG_ON(!__builtin_constant_p(memorder)); \
135 RTE_BUILD_BUG_ON(memorder != __ATOMIC_ACQUIRE && \
136 memorder != __ATOMIC_RELAXED); \
137 typeof(*(addr)) expected_value = (expected); \
138 while (!((__atomic_load_n((addr), (memorder)) & (mask)) cond \
139 expected_value)) \
140 rte_pause(); \
141 } while (0)
142 #endif /* ! RTE_WAIT_UNTIL_EQUAL_ARCH_DEFINED */
143
144 #endif /* _RTE_PAUSE_H_ */
145