1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2014 Intel Corporation 3 */ 4 5 #ifndef _RTE_SPINLOCK_H_ 6 #define _RTE_SPINLOCK_H_ 7 8 /** 9 * @file 10 * 11 * RTE Spinlocks 12 * 13 * This file defines an API for read-write locks, which are implemented 14 * in an architecture-specific way. This kind of lock simply waits in 15 * a loop repeatedly checking until the lock becomes available. 16 * 17 * All locks must be initialised before use, and only initialised once. 18 * 19 */ 20 21 #include <rte_lcore.h> 22 #ifdef RTE_FORCE_INTRINSICS 23 #include <rte_common.h> 24 #endif 25 #include <rte_pause.h> 26 27 /** 28 * The rte_spinlock_t type. 29 */ 30 typedef struct { 31 volatile int locked; /**< lock status 0 = unlocked, 1 = locked */ 32 } rte_spinlock_t; 33 34 /** 35 * A static spinlock initializer. 36 */ 37 #define RTE_SPINLOCK_INITIALIZER { 0 } 38 39 /** 40 * Initialize the spinlock to an unlocked state. 41 * 42 * @param sl 43 * A pointer to the spinlock. 44 */ 45 static inline void 46 rte_spinlock_init(rte_spinlock_t *sl) 47 { 48 sl->locked = 0; 49 } 50 51 /** 52 * Take the spinlock. 53 * 54 * @param sl 55 * A pointer to the spinlock. 56 */ 57 static inline void 58 rte_spinlock_lock(rte_spinlock_t *sl); 59 60 #ifdef RTE_FORCE_INTRINSICS 61 static inline void 62 rte_spinlock_lock(rte_spinlock_t *sl) 63 { 64 int exp = 0; 65 66 while (!__atomic_compare_exchange_n(&sl->locked, &exp, 1, 0, 67 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { 68 while (__atomic_load_n(&sl->locked, __ATOMIC_RELAXED)) 69 rte_pause(); 70 exp = 0; 71 } 72 } 73 #endif 74 75 /** 76 * Release the spinlock. 77 * 78 * @param sl 79 * A pointer to the spinlock. 80 */ 81 static inline void 82 rte_spinlock_unlock (rte_spinlock_t *sl); 83 84 #ifdef RTE_FORCE_INTRINSICS 85 static inline void 86 rte_spinlock_unlock (rte_spinlock_t *sl) 87 { 88 __atomic_store_n(&sl->locked, 0, __ATOMIC_RELEASE); 89 } 90 #endif 91 92 /** 93 * Try to take the lock. 94 * 95 * @param sl 96 * A pointer to the spinlock. 97 * @return 98 * 1 if the lock is successfully taken; 0 otherwise. 99 */ 100 static inline int 101 rte_spinlock_trylock (rte_spinlock_t *sl); 102 103 #ifdef RTE_FORCE_INTRINSICS 104 static inline int 105 rte_spinlock_trylock (rte_spinlock_t *sl) 106 { 107 int exp = 0; 108 return __atomic_compare_exchange_n(&sl->locked, &exp, 1, 109 0, /* disallow spurious failure */ 110 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); 111 } 112 #endif 113 114 /** 115 * Test if the lock is taken. 116 * 117 * @param sl 118 * A pointer to the spinlock. 119 * @return 120 * 1 if the lock is currently taken; 0 otherwise. 121 */ 122 static inline int rte_spinlock_is_locked (rte_spinlock_t *sl) 123 { 124 return __atomic_load_n(&sl->locked, __ATOMIC_ACQUIRE); 125 } 126 127 /** 128 * Test if hardware transactional memory (lock elision) is supported 129 * 130 * @return 131 * 1 if the hardware transactional memory is supported; 0 otherwise. 132 */ 133 static inline int rte_tm_supported(void); 134 135 /** 136 * Try to execute critical section in a hardware memory transaction, 137 * if it fails or not available take the spinlock. 138 * 139 * NOTE: An attempt to perform a HW I/O operation inside a hardware memory 140 * transaction always aborts the transaction since the CPU is not able to 141 * roll-back should the transaction fail. Therefore, hardware transactional 142 * locks are not advised to be used around rte_eth_rx_burst() and 143 * rte_eth_tx_burst() calls. 144 * 145 * @param sl 146 * A pointer to the spinlock. 147 */ 148 static inline void 149 rte_spinlock_lock_tm(rte_spinlock_t *sl); 150 151 /** 152 * Commit hardware memory transaction or release the spinlock if 153 * the spinlock is used as a fall-back 154 * 155 * @param sl 156 * A pointer to the spinlock. 157 */ 158 static inline void 159 rte_spinlock_unlock_tm(rte_spinlock_t *sl); 160 161 /** 162 * Try to execute critical section in a hardware memory transaction, 163 * if it fails or not available try to take the lock. 164 * 165 * NOTE: An attempt to perform a HW I/O operation inside a hardware memory 166 * transaction always aborts the transaction since the CPU is not able to 167 * roll-back should the transaction fail. Therefore, hardware transactional 168 * locks are not advised to be used around rte_eth_rx_burst() and 169 * rte_eth_tx_burst() calls. 170 * 171 * @param sl 172 * A pointer to the spinlock. 173 * @return 174 * 1 if the hardware memory transaction is successfully started 175 * or lock is successfully taken; 0 otherwise. 176 */ 177 static inline int 178 rte_spinlock_trylock_tm(rte_spinlock_t *sl); 179 180 /** 181 * The rte_spinlock_recursive_t type. 182 */ 183 typedef struct { 184 rte_spinlock_t sl; /**< the actual spinlock */ 185 volatile int user; /**< core id using lock, -1 for unused */ 186 volatile int count; /**< count of time this lock has been called */ 187 } rte_spinlock_recursive_t; 188 189 /** 190 * A static recursive spinlock initializer. 191 */ 192 #define RTE_SPINLOCK_RECURSIVE_INITIALIZER {RTE_SPINLOCK_INITIALIZER, -1, 0} 193 194 /** 195 * Initialize the recursive spinlock to an unlocked state. 196 * 197 * @param slr 198 * A pointer to the recursive spinlock. 199 */ 200 static inline void rte_spinlock_recursive_init(rte_spinlock_recursive_t *slr) 201 { 202 rte_spinlock_init(&slr->sl); 203 slr->user = -1; 204 slr->count = 0; 205 } 206 207 /** 208 * Take the recursive spinlock. 209 * 210 * @param slr 211 * A pointer to the recursive spinlock. 212 */ 213 static inline void rte_spinlock_recursive_lock(rte_spinlock_recursive_t *slr) 214 { 215 int id = rte_gettid(); 216 217 if (slr->user != id) { 218 rte_spinlock_lock(&slr->sl); 219 slr->user = id; 220 } 221 slr->count++; 222 } 223 /** 224 * Release the recursive spinlock. 225 * 226 * @param slr 227 * A pointer to the recursive spinlock. 228 */ 229 static inline void rte_spinlock_recursive_unlock(rte_spinlock_recursive_t *slr) 230 { 231 if (--(slr->count) == 0) { 232 slr->user = -1; 233 rte_spinlock_unlock(&slr->sl); 234 } 235 236 } 237 238 /** 239 * Try to take the recursive lock. 240 * 241 * @param slr 242 * A pointer to the recursive spinlock. 243 * @return 244 * 1 if the lock is successfully taken; 0 otherwise. 245 */ 246 static inline int rte_spinlock_recursive_trylock(rte_spinlock_recursive_t *slr) 247 { 248 int id = rte_gettid(); 249 250 if (slr->user != id) { 251 if (rte_spinlock_trylock(&slr->sl) == 0) 252 return 0; 253 slr->user = id; 254 } 255 slr->count++; 256 return 1; 257 } 258 259 260 /** 261 * Try to execute critical section in a hardware memory transaction, 262 * if it fails or not available take the recursive spinlocks 263 * 264 * NOTE: An attempt to perform a HW I/O operation inside a hardware memory 265 * transaction always aborts the transaction since the CPU is not able to 266 * roll-back should the transaction fail. Therefore, hardware transactional 267 * locks are not advised to be used around rte_eth_rx_burst() and 268 * rte_eth_tx_burst() calls. 269 * 270 * @param slr 271 * A pointer to the recursive spinlock. 272 */ 273 static inline void rte_spinlock_recursive_lock_tm( 274 rte_spinlock_recursive_t *slr); 275 276 /** 277 * Commit hardware memory transaction or release the recursive spinlock 278 * if the recursive spinlock is used as a fall-back 279 * 280 * @param slr 281 * A pointer to the recursive spinlock. 282 */ 283 static inline void rte_spinlock_recursive_unlock_tm( 284 rte_spinlock_recursive_t *slr); 285 286 /** 287 * Try to execute critical section in a hardware memory transaction, 288 * if it fails or not available try to take the recursive lock 289 * 290 * NOTE: An attempt to perform a HW I/O operation inside a hardware memory 291 * transaction always aborts the transaction since the CPU is not able to 292 * roll-back should the transaction fail. Therefore, hardware transactional 293 * locks are not advised to be used around rte_eth_rx_burst() and 294 * rte_eth_tx_burst() calls. 295 * 296 * @param slr 297 * A pointer to the recursive spinlock. 298 * @return 299 * 1 if the hardware memory transaction is successfully started 300 * or lock is successfully taken; 0 otherwise. 301 */ 302 static inline int rte_spinlock_recursive_trylock_tm( 303 rte_spinlock_recursive_t *slr); 304 305 #endif /* _RTE_SPINLOCK_H_ */ 306