1 /* 2 * SPDX-License-Identifier: BSD-3-Clause 3 * Inspired from FreeBSD src/sys/powerpc/include/atomic.h 4 * Copyright (c) 2008 Marcel Moolenaar 5 * Copyright (c) 2001 Benno Rice 6 * Copyright (c) 2001 David E. O'Brien 7 * Copyright (c) 1998 Doug Rabson 8 * All rights reserved. 9 */ 10 11 #ifndef _RTE_ATOMIC_PPC_64_H_ 12 #define _RTE_ATOMIC_PPC_64_H_ 13 14 #ifdef __cplusplus 15 extern "C" { 16 #endif 17 18 #include <stdint.h> 19 #include "generic/rte_atomic.h" 20 21 #define rte_mb() asm volatile("sync" : : : "memory") 22 23 #define rte_wmb() asm volatile("sync" : : : "memory") 24 25 #define rte_rmb() asm volatile("sync" : : : "memory") 26 27 #define rte_smp_mb() rte_mb() 28 29 #define rte_smp_wmb() rte_wmb() 30 31 #define rte_smp_rmb() rte_rmb() 32 33 #define rte_io_mb() rte_mb() 34 35 #define rte_io_wmb() rte_wmb() 36 37 #define rte_io_rmb() rte_rmb() 38 39 static __rte_always_inline void 40 rte_atomic_thread_fence(int memorder) 41 { 42 __atomic_thread_fence(memorder); 43 } 44 45 /*------------------------- 16 bit atomic operations -------------------------*/ 46 /* To be compatible with Power7, use GCC built-in functions for 16 bit 47 * operations */ 48 49 #ifndef RTE_FORCE_INTRINSICS 50 static inline int 51 rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src) 52 { 53 return __atomic_compare_exchange(dst, &exp, &src, 0, __ATOMIC_ACQUIRE, 54 __ATOMIC_ACQUIRE) ? 1 : 0; 55 } 56 57 static inline int rte_atomic16_test_and_set(rte_atomic16_t *v) 58 { 59 return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1); 60 } 61 62 static inline void 63 rte_atomic16_inc(rte_atomic16_t *v) 64 { 65 __atomic_add_fetch(&v->cnt, 1, __ATOMIC_ACQUIRE); 66 } 67 68 static inline void 69 rte_atomic16_dec(rte_atomic16_t *v) 70 { 71 __atomic_sub_fetch(&v->cnt, 1, __ATOMIC_ACQUIRE); 72 } 73 74 static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v) 75 { 76 return __atomic_add_fetch(&v->cnt, 1, __ATOMIC_ACQUIRE) == 0; 77 } 78 79 static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v) 80 { 81 return __atomic_sub_fetch(&v->cnt, 1, __ATOMIC_ACQUIRE) == 0; 82 } 83 84 static inline uint16_t 85 rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val) 86 { 87 return __atomic_exchange_2(dst, val, __ATOMIC_SEQ_CST); 88 } 89 90 /*------------------------- 32 bit atomic operations -------------------------*/ 91 92 static inline int 93 rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src) 94 { 95 unsigned int ret = 0; 96 97 asm volatile( 98 "\tlwsync\n" 99 "1:\tlwarx %[ret], 0, %[dst]\n" 100 "cmplw %[exp], %[ret]\n" 101 "bne 2f\n" 102 "stwcx. %[src], 0, %[dst]\n" 103 "bne- 1b\n" 104 "li %[ret], 1\n" 105 "b 3f\n" 106 "2:\n" 107 "stwcx. %[ret], 0, %[dst]\n" 108 "li %[ret], 0\n" 109 "3:\n" 110 "isync\n" 111 : [ret] "=&r" (ret), "=m" (*dst) 112 : [dst] "r" (dst), 113 [exp] "r" (exp), 114 [src] "r" (src), 115 "m" (*dst) 116 : "cc", "memory"); 117 118 return ret; 119 } 120 121 static inline int rte_atomic32_test_and_set(rte_atomic32_t *v) 122 { 123 return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1); 124 } 125 126 static inline void 127 rte_atomic32_inc(rte_atomic32_t *v) 128 { 129 int t; 130 131 asm volatile( 132 "1: lwarx %[t],0,%[cnt]\n" 133 "addic %[t],%[t],1\n" 134 "stwcx. %[t],0,%[cnt]\n" 135 "bne- 1b\n" 136 : [t] "=&r" (t), "=m" (v->cnt) 137 : [cnt] "r" (&v->cnt), "m" (v->cnt) 138 : "cc", "xer", "memory"); 139 } 140 141 static inline void 142 rte_atomic32_dec(rte_atomic32_t *v) 143 { 144 int t; 145 146 asm volatile( 147 "1: lwarx %[t],0,%[cnt]\n" 148 "addic %[t],%[t],-1\n" 149 "stwcx. %[t],0,%[cnt]\n" 150 "bne- 1b\n" 151 : [t] "=&r" (t), "=m" (v->cnt) 152 : [cnt] "r" (&v->cnt), "m" (v->cnt) 153 : "cc", "xer", "memory"); 154 } 155 156 static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v) 157 { 158 int ret; 159 160 asm volatile( 161 "\n\tlwsync\n" 162 "1: lwarx %[ret],0,%[cnt]\n" 163 "addic %[ret],%[ret],1\n" 164 "stwcx. %[ret],0,%[cnt]\n" 165 "bne- 1b\n" 166 "isync\n" 167 : [ret] "=&r" (ret) 168 : [cnt] "r" (&v->cnt) 169 : "cc", "xer", "memory"); 170 171 return ret == 0; 172 } 173 174 static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v) 175 { 176 int ret; 177 178 asm volatile( 179 "\n\tlwsync\n" 180 "1: lwarx %[ret],0,%[cnt]\n" 181 "addic %[ret],%[ret],-1\n" 182 "stwcx. %[ret],0,%[cnt]\n" 183 "bne- 1b\n" 184 "isync\n" 185 : [ret] "=&r" (ret) 186 : [cnt] "r" (&v->cnt) 187 : "cc", "xer", "memory"); 188 189 return ret == 0; 190 } 191 192 static inline uint32_t 193 rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val) 194 { 195 return __atomic_exchange_4(dst, val, __ATOMIC_SEQ_CST); 196 } 197 198 /*------------------------- 64 bit atomic operations -------------------------*/ 199 200 static inline int 201 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src) 202 { 203 unsigned int ret = 0; 204 205 asm volatile ( 206 "\tlwsync\n" 207 "1: ldarx %[ret], 0, %[dst]\n" 208 "cmpld %[exp], %[ret]\n" 209 "bne 2f\n" 210 "stdcx. %[src], 0, %[dst]\n" 211 "bne- 1b\n" 212 "li %[ret], 1\n" 213 "b 3f\n" 214 "2:\n" 215 "stdcx. %[ret], 0, %[dst]\n" 216 "li %[ret], 0\n" 217 "3:\n" 218 "isync\n" 219 : [ret] "=&r" (ret), "=m" (*dst) 220 : [dst] "r" (dst), 221 [exp] "r" (exp), 222 [src] "r" (src), 223 "m" (*dst) 224 : "cc", "memory"); 225 return ret; 226 } 227 228 static inline void 229 rte_atomic64_init(rte_atomic64_t *v) 230 { 231 v->cnt = 0; 232 } 233 234 static inline int64_t 235 rte_atomic64_read(rte_atomic64_t *v) 236 { 237 long ret; 238 239 asm volatile("ld%U1%X1 %[ret],%[cnt]" 240 : [ret] "=r"(ret) 241 : [cnt] "m"(v->cnt)); 242 243 return ret; 244 } 245 246 static inline void 247 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value) 248 { 249 asm volatile("std%U0%X0 %[new_value],%[cnt]" 250 : [cnt] "=m"(v->cnt) 251 : [new_value] "r"(new_value)); 252 } 253 254 static inline void 255 rte_atomic64_add(rte_atomic64_t *v, int64_t inc) 256 { 257 long t; 258 259 asm volatile( 260 "1: ldarx %[t],0,%[cnt]\n" 261 "add %[t],%[inc],%[t]\n" 262 "stdcx. %[t],0,%[cnt]\n" 263 "bne- 1b\n" 264 : [t] "=&r" (t), "=m" (v->cnt) 265 : [cnt] "r" (&v->cnt), [inc] "r" (inc), "m" (v->cnt) 266 : "cc", "memory"); 267 } 268 269 static inline void 270 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec) 271 { 272 long t; 273 274 asm volatile( 275 "1: ldarx %[t],0,%[cnt]\n" 276 "subf %[t],%[dec],%[t]\n" 277 "stdcx. %[t],0,%[cnt]\n" 278 "bne- 1b\n" 279 : [t] "=&r" (t), "+m" (v->cnt) 280 : [cnt] "r" (&v->cnt), [dec] "r" (dec), "m" (v->cnt) 281 : "cc", "memory"); 282 } 283 284 static inline void 285 rte_atomic64_inc(rte_atomic64_t *v) 286 { 287 long t; 288 289 asm volatile( 290 "1: ldarx %[t],0,%[cnt]\n" 291 "addic %[t],%[t],1\n" 292 "stdcx. %[t],0,%[cnt]\n" 293 "bne- 1b\n" 294 : [t] "=&r" (t), "+m" (v->cnt) 295 : [cnt] "r" (&v->cnt), "m" (v->cnt) 296 : "cc", "xer", "memory"); 297 } 298 299 static inline void 300 rte_atomic64_dec(rte_atomic64_t *v) 301 { 302 long t; 303 304 asm volatile( 305 "1: ldarx %[t],0,%[cnt]\n" 306 "addic %[t],%[t],-1\n" 307 "stdcx. %[t],0,%[cnt]\n" 308 "bne- 1b\n" 309 : [t] "=&r" (t), "+m" (v->cnt) 310 : [cnt] "r" (&v->cnt), "m" (v->cnt) 311 : "cc", "xer", "memory"); 312 } 313 314 static inline int64_t 315 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc) 316 { 317 long ret; 318 319 asm volatile( 320 "\n\tlwsync\n" 321 "1: ldarx %[ret],0,%[cnt]\n" 322 "add %[ret],%[inc],%[ret]\n" 323 "stdcx. %[ret],0,%[cnt]\n" 324 "bne- 1b\n" 325 "isync\n" 326 : [ret] "=&r" (ret) 327 : [inc] "r" (inc), [cnt] "r" (&v->cnt) 328 : "cc", "memory"); 329 330 return ret; 331 } 332 333 static inline int64_t 334 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec) 335 { 336 long ret; 337 338 asm volatile( 339 "\n\tlwsync\n" 340 "1: ldarx %[ret],0,%[cnt]\n" 341 "subf %[ret],%[dec],%[ret]\n" 342 "stdcx. %[ret],0,%[cnt]\n" 343 "bne- 1b\n" 344 "isync\n" 345 : [ret] "=&r" (ret) 346 : [dec] "r" (dec), [cnt] "r" (&v->cnt) 347 : "cc", "memory"); 348 349 return ret; 350 } 351 352 static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v) 353 { 354 long ret; 355 356 asm volatile( 357 "\n\tlwsync\n" 358 "1: ldarx %[ret],0,%[cnt]\n" 359 "addic %[ret],%[ret],1\n" 360 "stdcx. %[ret],0,%[cnt]\n" 361 "bne- 1b\n" 362 "isync\n" 363 : [ret] "=&r" (ret) 364 : [cnt] "r" (&v->cnt) 365 : "cc", "xer", "memory"); 366 367 return ret == 0; 368 } 369 370 static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v) 371 { 372 long ret; 373 374 asm volatile( 375 "\n\tlwsync\n" 376 "1: ldarx %[ret],0,%[cnt]\n" 377 "addic %[ret],%[ret],-1\n" 378 "stdcx. %[ret],0,%[cnt]\n" 379 "bne- 1b\n" 380 "isync\n" 381 : [ret] "=&r" (ret) 382 : [cnt] "r" (&v->cnt) 383 : "cc", "xer", "memory"); 384 385 return ret == 0; 386 } 387 388 static inline int rte_atomic64_test_and_set(rte_atomic64_t *v) 389 { 390 return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1); 391 } 392 /** 393 * Atomically set a 64-bit counter to 0. 394 * 395 * @param v 396 * A pointer to the atomic counter. 397 */ 398 static inline void rte_atomic64_clear(rte_atomic64_t *v) 399 { 400 v->cnt = 0; 401 } 402 403 static inline uint64_t 404 rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val) 405 { 406 return __atomic_exchange_8(dst, val, __ATOMIC_SEQ_CST); 407 } 408 409 #endif 410 411 #ifdef __cplusplus 412 } 413 #endif 414 415 #endif /* _RTE_ATOMIC_PPC_64_H_ */ 416