1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Atomic operations usable in machine independent code */ 3 #ifndef _LINUX_ATOMIC_H 4 #define _LINUX_ATOMIC_H 5 #include <asm/atomic.h> 6 #include <asm/barrier.h> 7 8 /* 9 * Relaxed variants of xchg, cmpxchg and some atomic operations. 10 * 11 * We support four variants: 12 * 13 * - Fully ordered: The default implementation, no suffix required. 14 * - Acquire: Provides ACQUIRE semantics, _acquire suffix. 15 * - Release: Provides RELEASE semantics, _release suffix. 16 * - Relaxed: No ordering guarantees, _relaxed suffix. 17 * 18 * For compound atomics performing both a load and a store, ACQUIRE 19 * semantics apply only to the load and RELEASE semantics only to the 20 * store portion of the operation. Note that a failed cmpxchg_acquire 21 * does -not- imply any memory ordering constraints. 22 * 23 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. 24 */ 25 26 #ifndef atomic_read_acquire 27 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) 28 #endif 29 30 #ifndef atomic_set_release 31 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) 32 #endif 33 34 /* 35 * The idea here is to build acquire/release variants by adding explicit 36 * barriers on top of the relaxed variant. In the case where the relaxed 37 * variant is already fully ordered, no additional barriers are needed. 38 * 39 * Besides, if an arch has a special barrier for acquire/release, it could 40 * implement its own __atomic_op_* and use the same framework for building 41 * variants 42 * 43 * If an architecture overrides __atomic_op_acquire() it will probably want 44 * to define smp_mb__after_spinlock(). 45 */ 46 #ifndef __atomic_op_acquire 47 #define __atomic_op_acquire(op, args...) \ 48 ({ \ 49 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 50 smp_mb__after_atomic(); \ 51 __ret; \ 52 }) 53 #endif 54 55 #ifndef __atomic_op_release 56 #define __atomic_op_release(op, args...) \ 57 ({ \ 58 smp_mb__before_atomic(); \ 59 op##_relaxed(args); \ 60 }) 61 #endif 62 63 #ifndef __atomic_op_fence 64 #define __atomic_op_fence(op, args...) \ 65 ({ \ 66 typeof(op##_relaxed(args)) __ret; \ 67 smp_mb__before_atomic(); \ 68 __ret = op##_relaxed(args); \ 69 smp_mb__after_atomic(); \ 70 __ret; \ 71 }) 72 #endif 73 74 /* atomic_add_return_relaxed */ 75 #ifndef atomic_add_return_relaxed 76 #define atomic_add_return_relaxed atomic_add_return 77 #define atomic_add_return_acquire atomic_add_return 78 #define atomic_add_return_release atomic_add_return 79 80 #else /* atomic_add_return_relaxed */ 81 82 #ifndef atomic_add_return_acquire 83 #define atomic_add_return_acquire(...) \ 84 __atomic_op_acquire(atomic_add_return, __VA_ARGS__) 85 #endif 86 87 #ifndef atomic_add_return_release 88 #define atomic_add_return_release(...) \ 89 __atomic_op_release(atomic_add_return, __VA_ARGS__) 90 #endif 91 92 #ifndef atomic_add_return 93 #define atomic_add_return(...) \ 94 __atomic_op_fence(atomic_add_return, __VA_ARGS__) 95 #endif 96 #endif /* atomic_add_return_relaxed */ 97 98 /* atomic_inc_return_relaxed */ 99 #ifndef atomic_inc_return_relaxed 100 #define atomic_inc_return_relaxed atomic_inc_return 101 #define atomic_inc_return_acquire atomic_inc_return 102 #define atomic_inc_return_release atomic_inc_return 103 104 #else /* atomic_inc_return_relaxed */ 105 106 #ifndef atomic_inc_return_acquire 107 #define atomic_inc_return_acquire(...) \ 108 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__) 109 #endif 110 111 #ifndef atomic_inc_return_release 112 #define atomic_inc_return_release(...) \ 113 __atomic_op_release(atomic_inc_return, __VA_ARGS__) 114 #endif 115 116 #ifndef atomic_inc_return 117 #define atomic_inc_return(...) \ 118 __atomic_op_fence(atomic_inc_return, __VA_ARGS__) 119 #endif 120 #endif /* atomic_inc_return_relaxed */ 121 122 /* atomic_sub_return_relaxed */ 123 #ifndef atomic_sub_return_relaxed 124 #define atomic_sub_return_relaxed atomic_sub_return 125 #define atomic_sub_return_acquire atomic_sub_return 126 #define atomic_sub_return_release atomic_sub_return 127 128 #else /* atomic_sub_return_relaxed */ 129 130 #ifndef atomic_sub_return_acquire 131 #define atomic_sub_return_acquire(...) \ 132 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__) 133 #endif 134 135 #ifndef atomic_sub_return_release 136 #define atomic_sub_return_release(...) \ 137 __atomic_op_release(atomic_sub_return, __VA_ARGS__) 138 #endif 139 140 #ifndef atomic_sub_return 141 #define atomic_sub_return(...) \ 142 __atomic_op_fence(atomic_sub_return, __VA_ARGS__) 143 #endif 144 #endif /* atomic_sub_return_relaxed */ 145 146 /* atomic_dec_return_relaxed */ 147 #ifndef atomic_dec_return_relaxed 148 #define atomic_dec_return_relaxed atomic_dec_return 149 #define atomic_dec_return_acquire atomic_dec_return 150 #define atomic_dec_return_release atomic_dec_return 151 152 #else /* atomic_dec_return_relaxed */ 153 154 #ifndef atomic_dec_return_acquire 155 #define atomic_dec_return_acquire(...) \ 156 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__) 157 #endif 158 159 #ifndef atomic_dec_return_release 160 #define atomic_dec_return_release(...) \ 161 __atomic_op_release(atomic_dec_return, __VA_ARGS__) 162 #endif 163 164 #ifndef atomic_dec_return 165 #define atomic_dec_return(...) \ 166 __atomic_op_fence(atomic_dec_return, __VA_ARGS__) 167 #endif 168 #endif /* atomic_dec_return_relaxed */ 169 170 171 /* atomic_fetch_add_relaxed */ 172 #ifndef atomic_fetch_add_relaxed 173 #define atomic_fetch_add_relaxed atomic_fetch_add 174 #define atomic_fetch_add_acquire atomic_fetch_add 175 #define atomic_fetch_add_release atomic_fetch_add 176 177 #else /* atomic_fetch_add_relaxed */ 178 179 #ifndef atomic_fetch_add_acquire 180 #define atomic_fetch_add_acquire(...) \ 181 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__) 182 #endif 183 184 #ifndef atomic_fetch_add_release 185 #define atomic_fetch_add_release(...) \ 186 __atomic_op_release(atomic_fetch_add, __VA_ARGS__) 187 #endif 188 189 #ifndef atomic_fetch_add 190 #define atomic_fetch_add(...) \ 191 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__) 192 #endif 193 #endif /* atomic_fetch_add_relaxed */ 194 195 /* atomic_fetch_inc_relaxed */ 196 #ifndef atomic_fetch_inc_relaxed 197 198 #ifndef atomic_fetch_inc 199 #define atomic_fetch_inc(v) atomic_fetch_add(1, (v)) 200 #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v)) 201 #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v)) 202 #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v)) 203 #else /* atomic_fetch_inc */ 204 #define atomic_fetch_inc_relaxed atomic_fetch_inc 205 #define atomic_fetch_inc_acquire atomic_fetch_inc 206 #define atomic_fetch_inc_release atomic_fetch_inc 207 #endif /* atomic_fetch_inc */ 208 209 #else /* atomic_fetch_inc_relaxed */ 210 211 #ifndef atomic_fetch_inc_acquire 212 #define atomic_fetch_inc_acquire(...) \ 213 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__) 214 #endif 215 216 #ifndef atomic_fetch_inc_release 217 #define atomic_fetch_inc_release(...) \ 218 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__) 219 #endif 220 221 #ifndef atomic_fetch_inc 222 #define atomic_fetch_inc(...) \ 223 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__) 224 #endif 225 #endif /* atomic_fetch_inc_relaxed */ 226 227 /* atomic_fetch_sub_relaxed */ 228 #ifndef atomic_fetch_sub_relaxed 229 #define atomic_fetch_sub_relaxed atomic_fetch_sub 230 #define atomic_fetch_sub_acquire atomic_fetch_sub 231 #define atomic_fetch_sub_release atomic_fetch_sub 232 233 #else /* atomic_fetch_sub_relaxed */ 234 235 #ifndef atomic_fetch_sub_acquire 236 #define atomic_fetch_sub_acquire(...) \ 237 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__) 238 #endif 239 240 #ifndef atomic_fetch_sub_release 241 #define atomic_fetch_sub_release(...) \ 242 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__) 243 #endif 244 245 #ifndef atomic_fetch_sub 246 #define atomic_fetch_sub(...) \ 247 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__) 248 #endif 249 #endif /* atomic_fetch_sub_relaxed */ 250 251 /* atomic_fetch_dec_relaxed */ 252 #ifndef atomic_fetch_dec_relaxed 253 254 #ifndef atomic_fetch_dec 255 #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v)) 256 #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v)) 257 #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v)) 258 #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v)) 259 #else /* atomic_fetch_dec */ 260 #define atomic_fetch_dec_relaxed atomic_fetch_dec 261 #define atomic_fetch_dec_acquire atomic_fetch_dec 262 #define atomic_fetch_dec_release atomic_fetch_dec 263 #endif /* atomic_fetch_dec */ 264 265 #else /* atomic_fetch_dec_relaxed */ 266 267 #ifndef atomic_fetch_dec_acquire 268 #define atomic_fetch_dec_acquire(...) \ 269 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__) 270 #endif 271 272 #ifndef atomic_fetch_dec_release 273 #define atomic_fetch_dec_release(...) \ 274 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__) 275 #endif 276 277 #ifndef atomic_fetch_dec 278 #define atomic_fetch_dec(...) \ 279 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__) 280 #endif 281 #endif /* atomic_fetch_dec_relaxed */ 282 283 /* atomic_fetch_or_relaxed */ 284 #ifndef atomic_fetch_or_relaxed 285 #define atomic_fetch_or_relaxed atomic_fetch_or 286 #define atomic_fetch_or_acquire atomic_fetch_or 287 #define atomic_fetch_or_release atomic_fetch_or 288 289 #else /* atomic_fetch_or_relaxed */ 290 291 #ifndef atomic_fetch_or_acquire 292 #define atomic_fetch_or_acquire(...) \ 293 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__) 294 #endif 295 296 #ifndef atomic_fetch_or_release 297 #define atomic_fetch_or_release(...) \ 298 __atomic_op_release(atomic_fetch_or, __VA_ARGS__) 299 #endif 300 301 #ifndef atomic_fetch_or 302 #define atomic_fetch_or(...) \ 303 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__) 304 #endif 305 #endif /* atomic_fetch_or_relaxed */ 306 307 /* atomic_fetch_and_relaxed */ 308 #ifndef atomic_fetch_and_relaxed 309 #define atomic_fetch_and_relaxed atomic_fetch_and 310 #define atomic_fetch_and_acquire atomic_fetch_and 311 #define atomic_fetch_and_release atomic_fetch_and 312 313 #else /* atomic_fetch_and_relaxed */ 314 315 #ifndef atomic_fetch_and_acquire 316 #define atomic_fetch_and_acquire(...) \ 317 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__) 318 #endif 319 320 #ifndef atomic_fetch_and_release 321 #define atomic_fetch_and_release(...) \ 322 __atomic_op_release(atomic_fetch_and, __VA_ARGS__) 323 #endif 324 325 #ifndef atomic_fetch_and 326 #define atomic_fetch_and(...) \ 327 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__) 328 #endif 329 #endif /* atomic_fetch_and_relaxed */ 330 331 #ifdef atomic_andnot 332 /* atomic_fetch_andnot_relaxed */ 333 #ifndef atomic_fetch_andnot_relaxed 334 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot 335 #define atomic_fetch_andnot_acquire atomic_fetch_andnot 336 #define atomic_fetch_andnot_release atomic_fetch_andnot 337 338 #else /* atomic_fetch_andnot_relaxed */ 339 340 #ifndef atomic_fetch_andnot_acquire 341 #define atomic_fetch_andnot_acquire(...) \ 342 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__) 343 #endif 344 345 #ifndef atomic_fetch_andnot_release 346 #define atomic_fetch_andnot_release(...) \ 347 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__) 348 #endif 349 350 #ifndef atomic_fetch_andnot 351 #define atomic_fetch_andnot(...) \ 352 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__) 353 #endif 354 #endif /* atomic_fetch_andnot_relaxed */ 355 #endif /* atomic_andnot */ 356 357 /* atomic_fetch_xor_relaxed */ 358 #ifndef atomic_fetch_xor_relaxed 359 #define atomic_fetch_xor_relaxed atomic_fetch_xor 360 #define atomic_fetch_xor_acquire atomic_fetch_xor 361 #define atomic_fetch_xor_release atomic_fetch_xor 362 363 #else /* atomic_fetch_xor_relaxed */ 364 365 #ifndef atomic_fetch_xor_acquire 366 #define atomic_fetch_xor_acquire(...) \ 367 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__) 368 #endif 369 370 #ifndef atomic_fetch_xor_release 371 #define atomic_fetch_xor_release(...) \ 372 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__) 373 #endif 374 375 #ifndef atomic_fetch_xor 376 #define atomic_fetch_xor(...) \ 377 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__) 378 #endif 379 #endif /* atomic_fetch_xor_relaxed */ 380 381 382 /* atomic_xchg_relaxed */ 383 #ifndef atomic_xchg_relaxed 384 #define atomic_xchg_relaxed atomic_xchg 385 #define atomic_xchg_acquire atomic_xchg 386 #define atomic_xchg_release atomic_xchg 387 388 #else /* atomic_xchg_relaxed */ 389 390 #ifndef atomic_xchg_acquire 391 #define atomic_xchg_acquire(...) \ 392 __atomic_op_acquire(atomic_xchg, __VA_ARGS__) 393 #endif 394 395 #ifndef atomic_xchg_release 396 #define atomic_xchg_release(...) \ 397 __atomic_op_release(atomic_xchg, __VA_ARGS__) 398 #endif 399 400 #ifndef atomic_xchg 401 #define atomic_xchg(...) \ 402 __atomic_op_fence(atomic_xchg, __VA_ARGS__) 403 #endif 404 #endif /* atomic_xchg_relaxed */ 405 406 /* atomic_cmpxchg_relaxed */ 407 #ifndef atomic_cmpxchg_relaxed 408 #define atomic_cmpxchg_relaxed atomic_cmpxchg 409 #define atomic_cmpxchg_acquire atomic_cmpxchg 410 #define atomic_cmpxchg_release atomic_cmpxchg 411 412 #else /* atomic_cmpxchg_relaxed */ 413 414 #ifndef atomic_cmpxchg_acquire 415 #define atomic_cmpxchg_acquire(...) \ 416 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__) 417 #endif 418 419 #ifndef atomic_cmpxchg_release 420 #define atomic_cmpxchg_release(...) \ 421 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__) 422 #endif 423 424 #ifndef atomic_cmpxchg 425 #define atomic_cmpxchg(...) \ 426 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__) 427 #endif 428 #endif /* atomic_cmpxchg_relaxed */ 429 430 #ifndef atomic_try_cmpxchg 431 432 #define __atomic_try_cmpxchg(type, _p, _po, _n) \ 433 ({ \ 434 typeof(_po) __po = (_po); \ 435 typeof(*(_po)) __r, __o = *__po; \ 436 __r = atomic_cmpxchg##type((_p), __o, (_n)); \ 437 if (unlikely(__r != __o)) \ 438 *__po = __r; \ 439 likely(__r == __o); \ 440 }) 441 442 #define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n) 443 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n) 444 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n) 445 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n) 446 447 #else /* atomic_try_cmpxchg */ 448 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg 449 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg 450 #define atomic_try_cmpxchg_release atomic_try_cmpxchg 451 #endif /* atomic_try_cmpxchg */ 452 453 /* cmpxchg_relaxed */ 454 #ifndef cmpxchg_relaxed 455 #define cmpxchg_relaxed cmpxchg 456 #define cmpxchg_acquire cmpxchg 457 #define cmpxchg_release cmpxchg 458 459 #else /* cmpxchg_relaxed */ 460 461 #ifndef cmpxchg_acquire 462 #define cmpxchg_acquire(...) \ 463 __atomic_op_acquire(cmpxchg, __VA_ARGS__) 464 #endif 465 466 #ifndef cmpxchg_release 467 #define cmpxchg_release(...) \ 468 __atomic_op_release(cmpxchg, __VA_ARGS__) 469 #endif 470 471 #ifndef cmpxchg 472 #define cmpxchg(...) \ 473 __atomic_op_fence(cmpxchg, __VA_ARGS__) 474 #endif 475 #endif /* cmpxchg_relaxed */ 476 477 /* cmpxchg64_relaxed */ 478 #ifndef cmpxchg64_relaxed 479 #define cmpxchg64_relaxed cmpxchg64 480 #define cmpxchg64_acquire cmpxchg64 481 #define cmpxchg64_release cmpxchg64 482 483 #else /* cmpxchg64_relaxed */ 484 485 #ifndef cmpxchg64_acquire 486 #define cmpxchg64_acquire(...) \ 487 __atomic_op_acquire(cmpxchg64, __VA_ARGS__) 488 #endif 489 490 #ifndef cmpxchg64_release 491 #define cmpxchg64_release(...) \ 492 __atomic_op_release(cmpxchg64, __VA_ARGS__) 493 #endif 494 495 #ifndef cmpxchg64 496 #define cmpxchg64(...) \ 497 __atomic_op_fence(cmpxchg64, __VA_ARGS__) 498 #endif 499 #endif /* cmpxchg64_relaxed */ 500 501 /* xchg_relaxed */ 502 #ifndef xchg_relaxed 503 #define xchg_relaxed xchg 504 #define xchg_acquire xchg 505 #define xchg_release xchg 506 507 #else /* xchg_relaxed */ 508 509 #ifndef xchg_acquire 510 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__) 511 #endif 512 513 #ifndef xchg_release 514 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__) 515 #endif 516 517 #ifndef xchg 518 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__) 519 #endif 520 #endif /* xchg_relaxed */ 521 522 /** 523 * atomic_add_unless - add unless the number is already a given value 524 * @v: pointer of type atomic_t 525 * @a: the amount to add to v... 526 * @u: ...unless v is equal to u. 527 * 528 * Atomically adds @a to @v, so long as @v was not already @u. 529 * Returns non-zero if @v was not @u, and zero otherwise. 530 */ 531 static inline int atomic_add_unless(atomic_t *v, int a, int u) 532 { 533 return __atomic_add_unless(v, a, u) != u; 534 } 535 536 /** 537 * atomic_inc_not_zero - increment unless the number is zero 538 * @v: pointer of type atomic_t 539 * 540 * Atomically increments @v by 1, so long as @v is non-zero. 541 * Returns non-zero if @v was non-zero, and zero otherwise. 542 */ 543 #ifndef atomic_inc_not_zero 544 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 545 #endif 546 547 #ifndef atomic_andnot 548 static inline void atomic_andnot(int i, atomic_t *v) 549 { 550 atomic_and(~i, v); 551 } 552 553 static inline int atomic_fetch_andnot(int i, atomic_t *v) 554 { 555 return atomic_fetch_and(~i, v); 556 } 557 558 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) 559 { 560 return atomic_fetch_and_relaxed(~i, v); 561 } 562 563 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) 564 { 565 return atomic_fetch_and_acquire(~i, v); 566 } 567 568 static inline int atomic_fetch_andnot_release(int i, atomic_t *v) 569 { 570 return atomic_fetch_and_release(~i, v); 571 } 572 #endif 573 574 /** 575 * atomic_inc_not_zero_hint - increment if not null 576 * @v: pointer of type atomic_t 577 * @hint: probable value of the atomic before the increment 578 * 579 * This version of atomic_inc_not_zero() gives a hint of probable 580 * value of the atomic. This helps processor to not read the memory 581 * before doing the atomic read/modify/write cycle, lowering 582 * number of bus transactions on some arches. 583 * 584 * Returns: 0 if increment was not done, 1 otherwise. 585 */ 586 #ifndef atomic_inc_not_zero_hint 587 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint) 588 { 589 int val, c = hint; 590 591 /* sanity test, should be removed by compiler if hint is a constant */ 592 if (!hint) 593 return atomic_inc_not_zero(v); 594 595 do { 596 val = atomic_cmpxchg(v, c, c + 1); 597 if (val == c) 598 return 1; 599 c = val; 600 } while (c); 601 602 return 0; 603 } 604 #endif 605 606 #ifndef atomic_inc_unless_negative 607 static inline int atomic_inc_unless_negative(atomic_t *p) 608 { 609 int v, v1; 610 for (v = 0; v >= 0; v = v1) { 611 v1 = atomic_cmpxchg(p, v, v + 1); 612 if (likely(v1 == v)) 613 return 1; 614 } 615 return 0; 616 } 617 #endif 618 619 #ifndef atomic_dec_unless_positive 620 static inline int atomic_dec_unless_positive(atomic_t *p) 621 { 622 int v, v1; 623 for (v = 0; v <= 0; v = v1) { 624 v1 = atomic_cmpxchg(p, v, v - 1); 625 if (likely(v1 == v)) 626 return 1; 627 } 628 return 0; 629 } 630 #endif 631 632 /* 633 * atomic_dec_if_positive - decrement by 1 if old value positive 634 * @v: pointer of type atomic_t 635 * 636 * The function returns the old value of *v minus 1, even if 637 * the atomic variable, v, was not decremented. 638 */ 639 #ifndef atomic_dec_if_positive 640 static inline int atomic_dec_if_positive(atomic_t *v) 641 { 642 int c, old, dec; 643 c = atomic_read(v); 644 for (;;) { 645 dec = c - 1; 646 if (unlikely(dec < 0)) 647 break; 648 old = atomic_cmpxchg((v), c, dec); 649 if (likely(old == c)) 650 break; 651 c = old; 652 } 653 return dec; 654 } 655 #endif 656 657 #ifdef CONFIG_GENERIC_ATOMIC64 658 #include <asm-generic/atomic64.h> 659 #endif 660 661 #ifndef atomic64_read_acquire 662 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter) 663 #endif 664 665 #ifndef atomic64_set_release 666 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i)) 667 #endif 668 669 /* atomic64_add_return_relaxed */ 670 #ifndef atomic64_add_return_relaxed 671 #define atomic64_add_return_relaxed atomic64_add_return 672 #define atomic64_add_return_acquire atomic64_add_return 673 #define atomic64_add_return_release atomic64_add_return 674 675 #else /* atomic64_add_return_relaxed */ 676 677 #ifndef atomic64_add_return_acquire 678 #define atomic64_add_return_acquire(...) \ 679 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__) 680 #endif 681 682 #ifndef atomic64_add_return_release 683 #define atomic64_add_return_release(...) \ 684 __atomic_op_release(atomic64_add_return, __VA_ARGS__) 685 #endif 686 687 #ifndef atomic64_add_return 688 #define atomic64_add_return(...) \ 689 __atomic_op_fence(atomic64_add_return, __VA_ARGS__) 690 #endif 691 #endif /* atomic64_add_return_relaxed */ 692 693 /* atomic64_inc_return_relaxed */ 694 #ifndef atomic64_inc_return_relaxed 695 #define atomic64_inc_return_relaxed atomic64_inc_return 696 #define atomic64_inc_return_acquire atomic64_inc_return 697 #define atomic64_inc_return_release atomic64_inc_return 698 699 #else /* atomic64_inc_return_relaxed */ 700 701 #ifndef atomic64_inc_return_acquire 702 #define atomic64_inc_return_acquire(...) \ 703 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__) 704 #endif 705 706 #ifndef atomic64_inc_return_release 707 #define atomic64_inc_return_release(...) \ 708 __atomic_op_release(atomic64_inc_return, __VA_ARGS__) 709 #endif 710 711 #ifndef atomic64_inc_return 712 #define atomic64_inc_return(...) \ 713 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__) 714 #endif 715 #endif /* atomic64_inc_return_relaxed */ 716 717 718 /* atomic64_sub_return_relaxed */ 719 #ifndef atomic64_sub_return_relaxed 720 #define atomic64_sub_return_relaxed atomic64_sub_return 721 #define atomic64_sub_return_acquire atomic64_sub_return 722 #define atomic64_sub_return_release atomic64_sub_return 723 724 #else /* atomic64_sub_return_relaxed */ 725 726 #ifndef atomic64_sub_return_acquire 727 #define atomic64_sub_return_acquire(...) \ 728 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__) 729 #endif 730 731 #ifndef atomic64_sub_return_release 732 #define atomic64_sub_return_release(...) \ 733 __atomic_op_release(atomic64_sub_return, __VA_ARGS__) 734 #endif 735 736 #ifndef atomic64_sub_return 737 #define atomic64_sub_return(...) \ 738 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__) 739 #endif 740 #endif /* atomic64_sub_return_relaxed */ 741 742 /* atomic64_dec_return_relaxed */ 743 #ifndef atomic64_dec_return_relaxed 744 #define atomic64_dec_return_relaxed atomic64_dec_return 745 #define atomic64_dec_return_acquire atomic64_dec_return 746 #define atomic64_dec_return_release atomic64_dec_return 747 748 #else /* atomic64_dec_return_relaxed */ 749 750 #ifndef atomic64_dec_return_acquire 751 #define atomic64_dec_return_acquire(...) \ 752 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__) 753 #endif 754 755 #ifndef atomic64_dec_return_release 756 #define atomic64_dec_return_release(...) \ 757 __atomic_op_release(atomic64_dec_return, __VA_ARGS__) 758 #endif 759 760 #ifndef atomic64_dec_return 761 #define atomic64_dec_return(...) \ 762 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__) 763 #endif 764 #endif /* atomic64_dec_return_relaxed */ 765 766 767 /* atomic64_fetch_add_relaxed */ 768 #ifndef atomic64_fetch_add_relaxed 769 #define atomic64_fetch_add_relaxed atomic64_fetch_add 770 #define atomic64_fetch_add_acquire atomic64_fetch_add 771 #define atomic64_fetch_add_release atomic64_fetch_add 772 773 #else /* atomic64_fetch_add_relaxed */ 774 775 #ifndef atomic64_fetch_add_acquire 776 #define atomic64_fetch_add_acquire(...) \ 777 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__) 778 #endif 779 780 #ifndef atomic64_fetch_add_release 781 #define atomic64_fetch_add_release(...) \ 782 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__) 783 #endif 784 785 #ifndef atomic64_fetch_add 786 #define atomic64_fetch_add(...) \ 787 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__) 788 #endif 789 #endif /* atomic64_fetch_add_relaxed */ 790 791 /* atomic64_fetch_inc_relaxed */ 792 #ifndef atomic64_fetch_inc_relaxed 793 794 #ifndef atomic64_fetch_inc 795 #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v)) 796 #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v)) 797 #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v)) 798 #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v)) 799 #else /* atomic64_fetch_inc */ 800 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc 801 #define atomic64_fetch_inc_acquire atomic64_fetch_inc 802 #define atomic64_fetch_inc_release atomic64_fetch_inc 803 #endif /* atomic64_fetch_inc */ 804 805 #else /* atomic64_fetch_inc_relaxed */ 806 807 #ifndef atomic64_fetch_inc_acquire 808 #define atomic64_fetch_inc_acquire(...) \ 809 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__) 810 #endif 811 812 #ifndef atomic64_fetch_inc_release 813 #define atomic64_fetch_inc_release(...) \ 814 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__) 815 #endif 816 817 #ifndef atomic64_fetch_inc 818 #define atomic64_fetch_inc(...) \ 819 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__) 820 #endif 821 #endif /* atomic64_fetch_inc_relaxed */ 822 823 /* atomic64_fetch_sub_relaxed */ 824 #ifndef atomic64_fetch_sub_relaxed 825 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub 826 #define atomic64_fetch_sub_acquire atomic64_fetch_sub 827 #define atomic64_fetch_sub_release atomic64_fetch_sub 828 829 #else /* atomic64_fetch_sub_relaxed */ 830 831 #ifndef atomic64_fetch_sub_acquire 832 #define atomic64_fetch_sub_acquire(...) \ 833 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__) 834 #endif 835 836 #ifndef atomic64_fetch_sub_release 837 #define atomic64_fetch_sub_release(...) \ 838 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__) 839 #endif 840 841 #ifndef atomic64_fetch_sub 842 #define atomic64_fetch_sub(...) \ 843 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__) 844 #endif 845 #endif /* atomic64_fetch_sub_relaxed */ 846 847 /* atomic64_fetch_dec_relaxed */ 848 #ifndef atomic64_fetch_dec_relaxed 849 850 #ifndef atomic64_fetch_dec 851 #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v)) 852 #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v)) 853 #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v)) 854 #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v)) 855 #else /* atomic64_fetch_dec */ 856 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec 857 #define atomic64_fetch_dec_acquire atomic64_fetch_dec 858 #define atomic64_fetch_dec_release atomic64_fetch_dec 859 #endif /* atomic64_fetch_dec */ 860 861 #else /* atomic64_fetch_dec_relaxed */ 862 863 #ifndef atomic64_fetch_dec_acquire 864 #define atomic64_fetch_dec_acquire(...) \ 865 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__) 866 #endif 867 868 #ifndef atomic64_fetch_dec_release 869 #define atomic64_fetch_dec_release(...) \ 870 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__) 871 #endif 872 873 #ifndef atomic64_fetch_dec 874 #define atomic64_fetch_dec(...) \ 875 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__) 876 #endif 877 #endif /* atomic64_fetch_dec_relaxed */ 878 879 /* atomic64_fetch_or_relaxed */ 880 #ifndef atomic64_fetch_or_relaxed 881 #define atomic64_fetch_or_relaxed atomic64_fetch_or 882 #define atomic64_fetch_or_acquire atomic64_fetch_or 883 #define atomic64_fetch_or_release atomic64_fetch_or 884 885 #else /* atomic64_fetch_or_relaxed */ 886 887 #ifndef atomic64_fetch_or_acquire 888 #define atomic64_fetch_or_acquire(...) \ 889 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__) 890 #endif 891 892 #ifndef atomic64_fetch_or_release 893 #define atomic64_fetch_or_release(...) \ 894 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__) 895 #endif 896 897 #ifndef atomic64_fetch_or 898 #define atomic64_fetch_or(...) \ 899 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__) 900 #endif 901 #endif /* atomic64_fetch_or_relaxed */ 902 903 /* atomic64_fetch_and_relaxed */ 904 #ifndef atomic64_fetch_and_relaxed 905 #define atomic64_fetch_and_relaxed atomic64_fetch_and 906 #define atomic64_fetch_and_acquire atomic64_fetch_and 907 #define atomic64_fetch_and_release atomic64_fetch_and 908 909 #else /* atomic64_fetch_and_relaxed */ 910 911 #ifndef atomic64_fetch_and_acquire 912 #define atomic64_fetch_and_acquire(...) \ 913 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__) 914 #endif 915 916 #ifndef atomic64_fetch_and_release 917 #define atomic64_fetch_and_release(...) \ 918 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__) 919 #endif 920 921 #ifndef atomic64_fetch_and 922 #define atomic64_fetch_and(...) \ 923 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__) 924 #endif 925 #endif /* atomic64_fetch_and_relaxed */ 926 927 #ifdef atomic64_andnot 928 /* atomic64_fetch_andnot_relaxed */ 929 #ifndef atomic64_fetch_andnot_relaxed 930 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot 931 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot 932 #define atomic64_fetch_andnot_release atomic64_fetch_andnot 933 934 #else /* atomic64_fetch_andnot_relaxed */ 935 936 #ifndef atomic64_fetch_andnot_acquire 937 #define atomic64_fetch_andnot_acquire(...) \ 938 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__) 939 #endif 940 941 #ifndef atomic64_fetch_andnot_release 942 #define atomic64_fetch_andnot_release(...) \ 943 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__) 944 #endif 945 946 #ifndef atomic64_fetch_andnot 947 #define atomic64_fetch_andnot(...) \ 948 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__) 949 #endif 950 #endif /* atomic64_fetch_andnot_relaxed */ 951 #endif /* atomic64_andnot */ 952 953 /* atomic64_fetch_xor_relaxed */ 954 #ifndef atomic64_fetch_xor_relaxed 955 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor 956 #define atomic64_fetch_xor_acquire atomic64_fetch_xor 957 #define atomic64_fetch_xor_release atomic64_fetch_xor 958 959 #else /* atomic64_fetch_xor_relaxed */ 960 961 #ifndef atomic64_fetch_xor_acquire 962 #define atomic64_fetch_xor_acquire(...) \ 963 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__) 964 #endif 965 966 #ifndef atomic64_fetch_xor_release 967 #define atomic64_fetch_xor_release(...) \ 968 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__) 969 #endif 970 971 #ifndef atomic64_fetch_xor 972 #define atomic64_fetch_xor(...) \ 973 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__) 974 #endif 975 #endif /* atomic64_fetch_xor_relaxed */ 976 977 978 /* atomic64_xchg_relaxed */ 979 #ifndef atomic64_xchg_relaxed 980 #define atomic64_xchg_relaxed atomic64_xchg 981 #define atomic64_xchg_acquire atomic64_xchg 982 #define atomic64_xchg_release atomic64_xchg 983 984 #else /* atomic64_xchg_relaxed */ 985 986 #ifndef atomic64_xchg_acquire 987 #define atomic64_xchg_acquire(...) \ 988 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__) 989 #endif 990 991 #ifndef atomic64_xchg_release 992 #define atomic64_xchg_release(...) \ 993 __atomic_op_release(atomic64_xchg, __VA_ARGS__) 994 #endif 995 996 #ifndef atomic64_xchg 997 #define atomic64_xchg(...) \ 998 __atomic_op_fence(atomic64_xchg, __VA_ARGS__) 999 #endif 1000 #endif /* atomic64_xchg_relaxed */ 1001 1002 /* atomic64_cmpxchg_relaxed */ 1003 #ifndef atomic64_cmpxchg_relaxed 1004 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg 1005 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 1006 #define atomic64_cmpxchg_release atomic64_cmpxchg 1007 1008 #else /* atomic64_cmpxchg_relaxed */ 1009 1010 #ifndef atomic64_cmpxchg_acquire 1011 #define atomic64_cmpxchg_acquire(...) \ 1012 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__) 1013 #endif 1014 1015 #ifndef atomic64_cmpxchg_release 1016 #define atomic64_cmpxchg_release(...) \ 1017 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__) 1018 #endif 1019 1020 #ifndef atomic64_cmpxchg 1021 #define atomic64_cmpxchg(...) \ 1022 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__) 1023 #endif 1024 #endif /* atomic64_cmpxchg_relaxed */ 1025 1026 #ifndef atomic64_try_cmpxchg 1027 1028 #define __atomic64_try_cmpxchg(type, _p, _po, _n) \ 1029 ({ \ 1030 typeof(_po) __po = (_po); \ 1031 typeof(*(_po)) __r, __o = *__po; \ 1032 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \ 1033 if (unlikely(__r != __o)) \ 1034 *__po = __r; \ 1035 likely(__r == __o); \ 1036 }) 1037 1038 #define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n) 1039 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n) 1040 #define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n) 1041 #define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n) 1042 1043 #else /* atomic64_try_cmpxchg */ 1044 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg 1045 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg 1046 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg 1047 #endif /* atomic64_try_cmpxchg */ 1048 1049 #ifndef atomic64_andnot 1050 static inline void atomic64_andnot(long long i, atomic64_t *v) 1051 { 1052 atomic64_and(~i, v); 1053 } 1054 1055 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v) 1056 { 1057 return atomic64_fetch_and(~i, v); 1058 } 1059 1060 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v) 1061 { 1062 return atomic64_fetch_and_relaxed(~i, v); 1063 } 1064 1065 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v) 1066 { 1067 return atomic64_fetch_and_acquire(~i, v); 1068 } 1069 1070 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v) 1071 { 1072 return atomic64_fetch_and_release(~i, v); 1073 } 1074 #endif 1075 1076 #include <asm-generic/atomic-long.h> 1077 1078 #endif /* _LINUX_ATOMIC_H */ 1079