1 /* Atomic operations usable in machine independent code */ 2 #ifndef _LINUX_ATOMIC_H 3 #define _LINUX_ATOMIC_H 4 #include <asm/atomic.h> 5 #include <asm/barrier.h> 6 7 /* 8 * Relaxed variants of xchg, cmpxchg and some atomic operations. 9 * 10 * We support four variants: 11 * 12 * - Fully ordered: The default implementation, no suffix required. 13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix. 14 * - Release: Provides RELEASE semantics, _release suffix. 15 * - Relaxed: No ordering guarantees, _relaxed suffix. 16 * 17 * For compound atomics performing both a load and a store, ACQUIRE 18 * semantics apply only to the load and RELEASE semantics only to the 19 * store portion of the operation. Note that a failed cmpxchg_acquire 20 * does -not- imply any memory ordering constraints. 21 * 22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. 23 */ 24 25 #ifndef atomic_read_acquire 26 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) 27 #endif 28 29 #ifndef atomic_set_release 30 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) 31 #endif 32 33 /* 34 * The idea here is to build acquire/release variants by adding explicit 35 * barriers on top of the relaxed variant. In the case where the relaxed 36 * variant is already fully ordered, no additional barriers are needed. 37 */ 38 #define __atomic_op_acquire(op, args...) \ 39 ({ \ 40 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 41 smp_mb__after_atomic(); \ 42 __ret; \ 43 }) 44 45 #define __atomic_op_release(op, args...) \ 46 ({ \ 47 smp_mb__before_atomic(); \ 48 op##_relaxed(args); \ 49 }) 50 51 #define __atomic_op_fence(op, args...) \ 52 ({ \ 53 typeof(op##_relaxed(args)) __ret; \ 54 smp_mb__before_atomic(); \ 55 __ret = op##_relaxed(args); \ 56 smp_mb__after_atomic(); \ 57 __ret; \ 58 }) 59 60 /* atomic_add_return_relaxed */ 61 #ifndef atomic_add_return_relaxed 62 #define atomic_add_return_relaxed atomic_add_return 63 #define atomic_add_return_acquire atomic_add_return 64 #define atomic_add_return_release atomic_add_return 65 66 #else /* atomic_add_return_relaxed */ 67 68 #ifndef atomic_add_return_acquire 69 #define atomic_add_return_acquire(...) \ 70 __atomic_op_acquire(atomic_add_return, __VA_ARGS__) 71 #endif 72 73 #ifndef atomic_add_return_release 74 #define atomic_add_return_release(...) \ 75 __atomic_op_release(atomic_add_return, __VA_ARGS__) 76 #endif 77 78 #ifndef atomic_add_return 79 #define atomic_add_return(...) \ 80 __atomic_op_fence(atomic_add_return, __VA_ARGS__) 81 #endif 82 #endif /* atomic_add_return_relaxed */ 83 84 /* atomic_inc_return_relaxed */ 85 #ifndef atomic_inc_return_relaxed 86 #define atomic_inc_return_relaxed atomic_inc_return 87 #define atomic_inc_return_acquire atomic_inc_return 88 #define atomic_inc_return_release atomic_inc_return 89 90 #else /* atomic_inc_return_relaxed */ 91 92 #ifndef atomic_inc_return_acquire 93 #define atomic_inc_return_acquire(...) \ 94 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__) 95 #endif 96 97 #ifndef atomic_inc_return_release 98 #define atomic_inc_return_release(...) \ 99 __atomic_op_release(atomic_inc_return, __VA_ARGS__) 100 #endif 101 102 #ifndef atomic_inc_return 103 #define atomic_inc_return(...) \ 104 __atomic_op_fence(atomic_inc_return, __VA_ARGS__) 105 #endif 106 #endif /* atomic_inc_return_relaxed */ 107 108 /* atomic_sub_return_relaxed */ 109 #ifndef atomic_sub_return_relaxed 110 #define atomic_sub_return_relaxed atomic_sub_return 111 #define atomic_sub_return_acquire atomic_sub_return 112 #define atomic_sub_return_release atomic_sub_return 113 114 #else /* atomic_sub_return_relaxed */ 115 116 #ifndef atomic_sub_return_acquire 117 #define atomic_sub_return_acquire(...) \ 118 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__) 119 #endif 120 121 #ifndef atomic_sub_return_release 122 #define atomic_sub_return_release(...) \ 123 __atomic_op_release(atomic_sub_return, __VA_ARGS__) 124 #endif 125 126 #ifndef atomic_sub_return 127 #define atomic_sub_return(...) \ 128 __atomic_op_fence(atomic_sub_return, __VA_ARGS__) 129 #endif 130 #endif /* atomic_sub_return_relaxed */ 131 132 /* atomic_dec_return_relaxed */ 133 #ifndef atomic_dec_return_relaxed 134 #define atomic_dec_return_relaxed atomic_dec_return 135 #define atomic_dec_return_acquire atomic_dec_return 136 #define atomic_dec_return_release atomic_dec_return 137 138 #else /* atomic_dec_return_relaxed */ 139 140 #ifndef atomic_dec_return_acquire 141 #define atomic_dec_return_acquire(...) \ 142 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__) 143 #endif 144 145 #ifndef atomic_dec_return_release 146 #define atomic_dec_return_release(...) \ 147 __atomic_op_release(atomic_dec_return, __VA_ARGS__) 148 #endif 149 150 #ifndef atomic_dec_return 151 #define atomic_dec_return(...) \ 152 __atomic_op_fence(atomic_dec_return, __VA_ARGS__) 153 #endif 154 #endif /* atomic_dec_return_relaxed */ 155 156 /* atomic_xchg_relaxed */ 157 #ifndef atomic_xchg_relaxed 158 #define atomic_xchg_relaxed atomic_xchg 159 #define atomic_xchg_acquire atomic_xchg 160 #define atomic_xchg_release atomic_xchg 161 162 #else /* atomic_xchg_relaxed */ 163 164 #ifndef atomic_xchg_acquire 165 #define atomic_xchg_acquire(...) \ 166 __atomic_op_acquire(atomic_xchg, __VA_ARGS__) 167 #endif 168 169 #ifndef atomic_xchg_release 170 #define atomic_xchg_release(...) \ 171 __atomic_op_release(atomic_xchg, __VA_ARGS__) 172 #endif 173 174 #ifndef atomic_xchg 175 #define atomic_xchg(...) \ 176 __atomic_op_fence(atomic_xchg, __VA_ARGS__) 177 #endif 178 #endif /* atomic_xchg_relaxed */ 179 180 /* atomic_cmpxchg_relaxed */ 181 #ifndef atomic_cmpxchg_relaxed 182 #define atomic_cmpxchg_relaxed atomic_cmpxchg 183 #define atomic_cmpxchg_acquire atomic_cmpxchg 184 #define atomic_cmpxchg_release atomic_cmpxchg 185 186 #else /* atomic_cmpxchg_relaxed */ 187 188 #ifndef atomic_cmpxchg_acquire 189 #define atomic_cmpxchg_acquire(...) \ 190 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__) 191 #endif 192 193 #ifndef atomic_cmpxchg_release 194 #define atomic_cmpxchg_release(...) \ 195 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__) 196 #endif 197 198 #ifndef atomic_cmpxchg 199 #define atomic_cmpxchg(...) \ 200 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__) 201 #endif 202 #endif /* atomic_cmpxchg_relaxed */ 203 204 #ifndef atomic64_read_acquire 205 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter) 206 #endif 207 208 #ifndef atomic64_set_release 209 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i)) 210 #endif 211 212 /* atomic64_add_return_relaxed */ 213 #ifndef atomic64_add_return_relaxed 214 #define atomic64_add_return_relaxed atomic64_add_return 215 #define atomic64_add_return_acquire atomic64_add_return 216 #define atomic64_add_return_release atomic64_add_return 217 218 #else /* atomic64_add_return_relaxed */ 219 220 #ifndef atomic64_add_return_acquire 221 #define atomic64_add_return_acquire(...) \ 222 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__) 223 #endif 224 225 #ifndef atomic64_add_return_release 226 #define atomic64_add_return_release(...) \ 227 __atomic_op_release(atomic64_add_return, __VA_ARGS__) 228 #endif 229 230 #ifndef atomic64_add_return 231 #define atomic64_add_return(...) \ 232 __atomic_op_fence(atomic64_add_return, __VA_ARGS__) 233 #endif 234 #endif /* atomic64_add_return_relaxed */ 235 236 /* atomic64_inc_return_relaxed */ 237 #ifndef atomic64_inc_return_relaxed 238 #define atomic64_inc_return_relaxed atomic64_inc_return 239 #define atomic64_inc_return_acquire atomic64_inc_return 240 #define atomic64_inc_return_release atomic64_inc_return 241 242 #else /* atomic64_inc_return_relaxed */ 243 244 #ifndef atomic64_inc_return_acquire 245 #define atomic64_inc_return_acquire(...) \ 246 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__) 247 #endif 248 249 #ifndef atomic64_inc_return_release 250 #define atomic64_inc_return_release(...) \ 251 __atomic_op_release(atomic64_inc_return, __VA_ARGS__) 252 #endif 253 254 #ifndef atomic64_inc_return 255 #define atomic64_inc_return(...) \ 256 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__) 257 #endif 258 #endif /* atomic64_inc_return_relaxed */ 259 260 261 /* atomic64_sub_return_relaxed */ 262 #ifndef atomic64_sub_return_relaxed 263 #define atomic64_sub_return_relaxed atomic64_sub_return 264 #define atomic64_sub_return_acquire atomic64_sub_return 265 #define atomic64_sub_return_release atomic64_sub_return 266 267 #else /* atomic64_sub_return_relaxed */ 268 269 #ifndef atomic64_sub_return_acquire 270 #define atomic64_sub_return_acquire(...) \ 271 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__) 272 #endif 273 274 #ifndef atomic64_sub_return_release 275 #define atomic64_sub_return_release(...) \ 276 __atomic_op_release(atomic64_sub_return, __VA_ARGS__) 277 #endif 278 279 #ifndef atomic64_sub_return 280 #define atomic64_sub_return(...) \ 281 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__) 282 #endif 283 #endif /* atomic64_sub_return_relaxed */ 284 285 /* atomic64_dec_return_relaxed */ 286 #ifndef atomic64_dec_return_relaxed 287 #define atomic64_dec_return_relaxed atomic64_dec_return 288 #define atomic64_dec_return_acquire atomic64_dec_return 289 #define atomic64_dec_return_release atomic64_dec_return 290 291 #else /* atomic64_dec_return_relaxed */ 292 293 #ifndef atomic64_dec_return_acquire 294 #define atomic64_dec_return_acquire(...) \ 295 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__) 296 #endif 297 298 #ifndef atomic64_dec_return_release 299 #define atomic64_dec_return_release(...) \ 300 __atomic_op_release(atomic64_dec_return, __VA_ARGS__) 301 #endif 302 303 #ifndef atomic64_dec_return 304 #define atomic64_dec_return(...) \ 305 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__) 306 #endif 307 #endif /* atomic64_dec_return_relaxed */ 308 309 /* atomic64_xchg_relaxed */ 310 #ifndef atomic64_xchg_relaxed 311 #define atomic64_xchg_relaxed atomic64_xchg 312 #define atomic64_xchg_acquire atomic64_xchg 313 #define atomic64_xchg_release atomic64_xchg 314 315 #else /* atomic64_xchg_relaxed */ 316 317 #ifndef atomic64_xchg_acquire 318 #define atomic64_xchg_acquire(...) \ 319 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__) 320 #endif 321 322 #ifndef atomic64_xchg_release 323 #define atomic64_xchg_release(...) \ 324 __atomic_op_release(atomic64_xchg, __VA_ARGS__) 325 #endif 326 327 #ifndef atomic64_xchg 328 #define atomic64_xchg(...) \ 329 __atomic_op_fence(atomic64_xchg, __VA_ARGS__) 330 #endif 331 #endif /* atomic64_xchg_relaxed */ 332 333 /* atomic64_cmpxchg_relaxed */ 334 #ifndef atomic64_cmpxchg_relaxed 335 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg 336 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 337 #define atomic64_cmpxchg_release atomic64_cmpxchg 338 339 #else /* atomic64_cmpxchg_relaxed */ 340 341 #ifndef atomic64_cmpxchg_acquire 342 #define atomic64_cmpxchg_acquire(...) \ 343 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__) 344 #endif 345 346 #ifndef atomic64_cmpxchg_release 347 #define atomic64_cmpxchg_release(...) \ 348 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__) 349 #endif 350 351 #ifndef atomic64_cmpxchg 352 #define atomic64_cmpxchg(...) \ 353 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__) 354 #endif 355 #endif /* atomic64_cmpxchg_relaxed */ 356 357 /* cmpxchg_relaxed */ 358 #ifndef cmpxchg_relaxed 359 #define cmpxchg_relaxed cmpxchg 360 #define cmpxchg_acquire cmpxchg 361 #define cmpxchg_release cmpxchg 362 363 #else /* cmpxchg_relaxed */ 364 365 #ifndef cmpxchg_acquire 366 #define cmpxchg_acquire(...) \ 367 __atomic_op_acquire(cmpxchg, __VA_ARGS__) 368 #endif 369 370 #ifndef cmpxchg_release 371 #define cmpxchg_release(...) \ 372 __atomic_op_release(cmpxchg, __VA_ARGS__) 373 #endif 374 375 #ifndef cmpxchg 376 #define cmpxchg(...) \ 377 __atomic_op_fence(cmpxchg, __VA_ARGS__) 378 #endif 379 #endif /* cmpxchg_relaxed */ 380 381 /* cmpxchg64_relaxed */ 382 #ifndef cmpxchg64_relaxed 383 #define cmpxchg64_relaxed cmpxchg64 384 #define cmpxchg64_acquire cmpxchg64 385 #define cmpxchg64_release cmpxchg64 386 387 #else /* cmpxchg64_relaxed */ 388 389 #ifndef cmpxchg64_acquire 390 #define cmpxchg64_acquire(...) \ 391 __atomic_op_acquire(cmpxchg64, __VA_ARGS__) 392 #endif 393 394 #ifndef cmpxchg64_release 395 #define cmpxchg64_release(...) \ 396 __atomic_op_release(cmpxchg64, __VA_ARGS__) 397 #endif 398 399 #ifndef cmpxchg64 400 #define cmpxchg64(...) \ 401 __atomic_op_fence(cmpxchg64, __VA_ARGS__) 402 #endif 403 #endif /* cmpxchg64_relaxed */ 404 405 /* xchg_relaxed */ 406 #ifndef xchg_relaxed 407 #define xchg_relaxed xchg 408 #define xchg_acquire xchg 409 #define xchg_release xchg 410 411 #else /* xchg_relaxed */ 412 413 #ifndef xchg_acquire 414 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__) 415 #endif 416 417 #ifndef xchg_release 418 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__) 419 #endif 420 421 #ifndef xchg 422 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__) 423 #endif 424 #endif /* xchg_relaxed */ 425 426 /** 427 * atomic_add_unless - add unless the number is already a given value 428 * @v: pointer of type atomic_t 429 * @a: the amount to add to v... 430 * @u: ...unless v is equal to u. 431 * 432 * Atomically adds @a to @v, so long as @v was not already @u. 433 * Returns non-zero if @v was not @u, and zero otherwise. 434 */ 435 static inline int atomic_add_unless(atomic_t *v, int a, int u) 436 { 437 return __atomic_add_unless(v, a, u) != u; 438 } 439 440 /** 441 * atomic_inc_not_zero - increment unless the number is zero 442 * @v: pointer of type atomic_t 443 * 444 * Atomically increments @v by 1, so long as @v is non-zero. 445 * Returns non-zero if @v was non-zero, and zero otherwise. 446 */ 447 #ifndef atomic_inc_not_zero 448 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 449 #endif 450 451 #ifndef atomic_andnot 452 static inline void atomic_andnot(int i, atomic_t *v) 453 { 454 atomic_and(~i, v); 455 } 456 #endif 457 458 static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) 459 { 460 atomic_andnot(mask, v); 461 } 462 463 static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) 464 { 465 atomic_or(mask, v); 466 } 467 468 /** 469 * atomic_inc_not_zero_hint - increment if not null 470 * @v: pointer of type atomic_t 471 * @hint: probable value of the atomic before the increment 472 * 473 * This version of atomic_inc_not_zero() gives a hint of probable 474 * value of the atomic. This helps processor to not read the memory 475 * before doing the atomic read/modify/write cycle, lowering 476 * number of bus transactions on some arches. 477 * 478 * Returns: 0 if increment was not done, 1 otherwise. 479 */ 480 #ifndef atomic_inc_not_zero_hint 481 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint) 482 { 483 int val, c = hint; 484 485 /* sanity test, should be removed by compiler if hint is a constant */ 486 if (!hint) 487 return atomic_inc_not_zero(v); 488 489 do { 490 val = atomic_cmpxchg(v, c, c + 1); 491 if (val == c) 492 return 1; 493 c = val; 494 } while (c); 495 496 return 0; 497 } 498 #endif 499 500 #ifndef atomic_inc_unless_negative 501 static inline int atomic_inc_unless_negative(atomic_t *p) 502 { 503 int v, v1; 504 for (v = 0; v >= 0; v = v1) { 505 v1 = atomic_cmpxchg(p, v, v + 1); 506 if (likely(v1 == v)) 507 return 1; 508 } 509 return 0; 510 } 511 #endif 512 513 #ifndef atomic_dec_unless_positive 514 static inline int atomic_dec_unless_positive(atomic_t *p) 515 { 516 int v, v1; 517 for (v = 0; v <= 0; v = v1) { 518 v1 = atomic_cmpxchg(p, v, v - 1); 519 if (likely(v1 == v)) 520 return 1; 521 } 522 return 0; 523 } 524 #endif 525 526 /* 527 * atomic_dec_if_positive - decrement by 1 if old value positive 528 * @v: pointer of type atomic_t 529 * 530 * The function returns the old value of *v minus 1, even if 531 * the atomic variable, v, was not decremented. 532 */ 533 #ifndef atomic_dec_if_positive 534 static inline int atomic_dec_if_positive(atomic_t *v) 535 { 536 int c, old, dec; 537 c = atomic_read(v); 538 for (;;) { 539 dec = c - 1; 540 if (unlikely(dec < 0)) 541 break; 542 old = atomic_cmpxchg((v), c, dec); 543 if (likely(old == c)) 544 break; 545 c = old; 546 } 547 return dec; 548 } 549 #endif 550 551 #ifdef CONFIG_GENERIC_ATOMIC64 552 #include <asm-generic/atomic64.h> 553 #endif 554 555 #ifndef atomic64_andnot 556 static inline void atomic64_andnot(long long i, atomic64_t *v) 557 { 558 atomic64_and(~i, v); 559 } 560 #endif 561 562 #include <asm-generic/atomic-long.h> 563 564 #endif /* _LINUX_ATOMIC_H */ 565