1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Atomic operations usable in machine independent code */ 3 #ifndef _LINUX_ATOMIC_H 4 #define _LINUX_ATOMIC_H 5 #include <linux/types.h> 6 7 #include <asm/atomic.h> 8 #include <asm/barrier.h> 9 10 /* 11 * Relaxed variants of xchg, cmpxchg and some atomic operations. 12 * 13 * We support four variants: 14 * 15 * - Fully ordered: The default implementation, no suffix required. 16 * - Acquire: Provides ACQUIRE semantics, _acquire suffix. 17 * - Release: Provides RELEASE semantics, _release suffix. 18 * - Relaxed: No ordering guarantees, _relaxed suffix. 19 * 20 * For compound atomics performing both a load and a store, ACQUIRE 21 * semantics apply only to the load and RELEASE semantics only to the 22 * store portion of the operation. Note that a failed cmpxchg_acquire 23 * does -not- imply any memory ordering constraints. 24 * 25 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. 26 */ 27 28 #ifndef atomic_read_acquire 29 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) 30 #endif 31 32 #ifndef atomic_set_release 33 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) 34 #endif 35 36 /* 37 * The idea here is to build acquire/release variants by adding explicit 38 * barriers on top of the relaxed variant. In the case where the relaxed 39 * variant is already fully ordered, no additional barriers are needed. 40 * 41 * Besides, if an arch has a special barrier for acquire/release, it could 42 * implement its own __atomic_op_* and use the same framework for building 43 * variants 44 * 45 * If an architecture overrides __atomic_op_acquire() it will probably want 46 * to define smp_mb__after_spinlock(). 47 */ 48 #ifndef __atomic_op_acquire 49 #define __atomic_op_acquire(op, args...) \ 50 ({ \ 51 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 52 smp_mb__after_atomic(); \ 53 __ret; \ 54 }) 55 #endif 56 57 #ifndef __atomic_op_release 58 #define __atomic_op_release(op, args...) \ 59 ({ \ 60 smp_mb__before_atomic(); \ 61 op##_relaxed(args); \ 62 }) 63 #endif 64 65 #ifndef __atomic_op_fence 66 #define __atomic_op_fence(op, args...) \ 67 ({ \ 68 typeof(op##_relaxed(args)) __ret; \ 69 smp_mb__before_atomic(); \ 70 __ret = op##_relaxed(args); \ 71 smp_mb__after_atomic(); \ 72 __ret; \ 73 }) 74 #endif 75 76 /* atomic_add_return_relaxed */ 77 #ifndef atomic_add_return_relaxed 78 #define atomic_add_return_relaxed atomic_add_return 79 #define atomic_add_return_acquire atomic_add_return 80 #define atomic_add_return_release atomic_add_return 81 82 #else /* atomic_add_return_relaxed */ 83 84 #ifndef atomic_add_return_acquire 85 #define atomic_add_return_acquire(...) \ 86 __atomic_op_acquire(atomic_add_return, __VA_ARGS__) 87 #endif 88 89 #ifndef atomic_add_return_release 90 #define atomic_add_return_release(...) \ 91 __atomic_op_release(atomic_add_return, __VA_ARGS__) 92 #endif 93 94 #ifndef atomic_add_return 95 #define atomic_add_return(...) \ 96 __atomic_op_fence(atomic_add_return, __VA_ARGS__) 97 #endif 98 #endif /* atomic_add_return_relaxed */ 99 100 /* atomic_inc_return_relaxed */ 101 #ifndef atomic_inc_return_relaxed 102 #define atomic_inc_return_relaxed atomic_inc_return 103 #define atomic_inc_return_acquire atomic_inc_return 104 #define atomic_inc_return_release atomic_inc_return 105 106 #else /* atomic_inc_return_relaxed */ 107 108 #ifndef atomic_inc_return_acquire 109 #define atomic_inc_return_acquire(...) \ 110 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__) 111 #endif 112 113 #ifndef atomic_inc_return_release 114 #define atomic_inc_return_release(...) \ 115 __atomic_op_release(atomic_inc_return, __VA_ARGS__) 116 #endif 117 118 #ifndef atomic_inc_return 119 #define atomic_inc_return(...) \ 120 __atomic_op_fence(atomic_inc_return, __VA_ARGS__) 121 #endif 122 #endif /* atomic_inc_return_relaxed */ 123 124 /* atomic_sub_return_relaxed */ 125 #ifndef atomic_sub_return_relaxed 126 #define atomic_sub_return_relaxed atomic_sub_return 127 #define atomic_sub_return_acquire atomic_sub_return 128 #define atomic_sub_return_release atomic_sub_return 129 130 #else /* atomic_sub_return_relaxed */ 131 132 #ifndef atomic_sub_return_acquire 133 #define atomic_sub_return_acquire(...) \ 134 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__) 135 #endif 136 137 #ifndef atomic_sub_return_release 138 #define atomic_sub_return_release(...) \ 139 __atomic_op_release(atomic_sub_return, __VA_ARGS__) 140 #endif 141 142 #ifndef atomic_sub_return 143 #define atomic_sub_return(...) \ 144 __atomic_op_fence(atomic_sub_return, __VA_ARGS__) 145 #endif 146 #endif /* atomic_sub_return_relaxed */ 147 148 /* atomic_dec_return_relaxed */ 149 #ifndef atomic_dec_return_relaxed 150 #define atomic_dec_return_relaxed atomic_dec_return 151 #define atomic_dec_return_acquire atomic_dec_return 152 #define atomic_dec_return_release atomic_dec_return 153 154 #else /* atomic_dec_return_relaxed */ 155 156 #ifndef atomic_dec_return_acquire 157 #define atomic_dec_return_acquire(...) \ 158 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__) 159 #endif 160 161 #ifndef atomic_dec_return_release 162 #define atomic_dec_return_release(...) \ 163 __atomic_op_release(atomic_dec_return, __VA_ARGS__) 164 #endif 165 166 #ifndef atomic_dec_return 167 #define atomic_dec_return(...) \ 168 __atomic_op_fence(atomic_dec_return, __VA_ARGS__) 169 #endif 170 #endif /* atomic_dec_return_relaxed */ 171 172 173 /* atomic_fetch_add_relaxed */ 174 #ifndef atomic_fetch_add_relaxed 175 #define atomic_fetch_add_relaxed atomic_fetch_add 176 #define atomic_fetch_add_acquire atomic_fetch_add 177 #define atomic_fetch_add_release atomic_fetch_add 178 179 #else /* atomic_fetch_add_relaxed */ 180 181 #ifndef atomic_fetch_add_acquire 182 #define atomic_fetch_add_acquire(...) \ 183 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__) 184 #endif 185 186 #ifndef atomic_fetch_add_release 187 #define atomic_fetch_add_release(...) \ 188 __atomic_op_release(atomic_fetch_add, __VA_ARGS__) 189 #endif 190 191 #ifndef atomic_fetch_add 192 #define atomic_fetch_add(...) \ 193 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__) 194 #endif 195 #endif /* atomic_fetch_add_relaxed */ 196 197 /* atomic_fetch_inc_relaxed */ 198 #ifndef atomic_fetch_inc_relaxed 199 200 #ifndef atomic_fetch_inc 201 #define atomic_fetch_inc(v) atomic_fetch_add(1, (v)) 202 #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v)) 203 #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v)) 204 #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v)) 205 #else /* atomic_fetch_inc */ 206 #define atomic_fetch_inc_relaxed atomic_fetch_inc 207 #define atomic_fetch_inc_acquire atomic_fetch_inc 208 #define atomic_fetch_inc_release atomic_fetch_inc 209 #endif /* atomic_fetch_inc */ 210 211 #else /* atomic_fetch_inc_relaxed */ 212 213 #ifndef atomic_fetch_inc_acquire 214 #define atomic_fetch_inc_acquire(...) \ 215 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__) 216 #endif 217 218 #ifndef atomic_fetch_inc_release 219 #define atomic_fetch_inc_release(...) \ 220 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__) 221 #endif 222 223 #ifndef atomic_fetch_inc 224 #define atomic_fetch_inc(...) \ 225 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__) 226 #endif 227 #endif /* atomic_fetch_inc_relaxed */ 228 229 /* atomic_fetch_sub_relaxed */ 230 #ifndef atomic_fetch_sub_relaxed 231 #define atomic_fetch_sub_relaxed atomic_fetch_sub 232 #define atomic_fetch_sub_acquire atomic_fetch_sub 233 #define atomic_fetch_sub_release atomic_fetch_sub 234 235 #else /* atomic_fetch_sub_relaxed */ 236 237 #ifndef atomic_fetch_sub_acquire 238 #define atomic_fetch_sub_acquire(...) \ 239 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__) 240 #endif 241 242 #ifndef atomic_fetch_sub_release 243 #define atomic_fetch_sub_release(...) \ 244 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__) 245 #endif 246 247 #ifndef atomic_fetch_sub 248 #define atomic_fetch_sub(...) \ 249 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__) 250 #endif 251 #endif /* atomic_fetch_sub_relaxed */ 252 253 /* atomic_fetch_dec_relaxed */ 254 #ifndef atomic_fetch_dec_relaxed 255 256 #ifndef atomic_fetch_dec 257 #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v)) 258 #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v)) 259 #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v)) 260 #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v)) 261 #else /* atomic_fetch_dec */ 262 #define atomic_fetch_dec_relaxed atomic_fetch_dec 263 #define atomic_fetch_dec_acquire atomic_fetch_dec 264 #define atomic_fetch_dec_release atomic_fetch_dec 265 #endif /* atomic_fetch_dec */ 266 267 #else /* atomic_fetch_dec_relaxed */ 268 269 #ifndef atomic_fetch_dec_acquire 270 #define atomic_fetch_dec_acquire(...) \ 271 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__) 272 #endif 273 274 #ifndef atomic_fetch_dec_release 275 #define atomic_fetch_dec_release(...) \ 276 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__) 277 #endif 278 279 #ifndef atomic_fetch_dec 280 #define atomic_fetch_dec(...) \ 281 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__) 282 #endif 283 #endif /* atomic_fetch_dec_relaxed */ 284 285 /* atomic_fetch_or_relaxed */ 286 #ifndef atomic_fetch_or_relaxed 287 #define atomic_fetch_or_relaxed atomic_fetch_or 288 #define atomic_fetch_or_acquire atomic_fetch_or 289 #define atomic_fetch_or_release atomic_fetch_or 290 291 #else /* atomic_fetch_or_relaxed */ 292 293 #ifndef atomic_fetch_or_acquire 294 #define atomic_fetch_or_acquire(...) \ 295 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__) 296 #endif 297 298 #ifndef atomic_fetch_or_release 299 #define atomic_fetch_or_release(...) \ 300 __atomic_op_release(atomic_fetch_or, __VA_ARGS__) 301 #endif 302 303 #ifndef atomic_fetch_or 304 #define atomic_fetch_or(...) \ 305 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__) 306 #endif 307 #endif /* atomic_fetch_or_relaxed */ 308 309 /* atomic_fetch_and_relaxed */ 310 #ifndef atomic_fetch_and_relaxed 311 #define atomic_fetch_and_relaxed atomic_fetch_and 312 #define atomic_fetch_and_acquire atomic_fetch_and 313 #define atomic_fetch_and_release atomic_fetch_and 314 315 #else /* atomic_fetch_and_relaxed */ 316 317 #ifndef atomic_fetch_and_acquire 318 #define atomic_fetch_and_acquire(...) \ 319 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__) 320 #endif 321 322 #ifndef atomic_fetch_and_release 323 #define atomic_fetch_and_release(...) \ 324 __atomic_op_release(atomic_fetch_and, __VA_ARGS__) 325 #endif 326 327 #ifndef atomic_fetch_and 328 #define atomic_fetch_and(...) \ 329 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__) 330 #endif 331 #endif /* atomic_fetch_and_relaxed */ 332 333 #ifdef atomic_andnot 334 /* atomic_fetch_andnot_relaxed */ 335 #ifndef atomic_fetch_andnot_relaxed 336 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot 337 #define atomic_fetch_andnot_acquire atomic_fetch_andnot 338 #define atomic_fetch_andnot_release atomic_fetch_andnot 339 340 #else /* atomic_fetch_andnot_relaxed */ 341 342 #ifndef atomic_fetch_andnot_acquire 343 #define atomic_fetch_andnot_acquire(...) \ 344 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__) 345 #endif 346 347 #ifndef atomic_fetch_andnot_release 348 #define atomic_fetch_andnot_release(...) \ 349 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__) 350 #endif 351 352 #ifndef atomic_fetch_andnot 353 #define atomic_fetch_andnot(...) \ 354 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__) 355 #endif 356 #endif /* atomic_fetch_andnot_relaxed */ 357 #endif /* atomic_andnot */ 358 359 /* atomic_fetch_xor_relaxed */ 360 #ifndef atomic_fetch_xor_relaxed 361 #define atomic_fetch_xor_relaxed atomic_fetch_xor 362 #define atomic_fetch_xor_acquire atomic_fetch_xor 363 #define atomic_fetch_xor_release atomic_fetch_xor 364 365 #else /* atomic_fetch_xor_relaxed */ 366 367 #ifndef atomic_fetch_xor_acquire 368 #define atomic_fetch_xor_acquire(...) \ 369 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__) 370 #endif 371 372 #ifndef atomic_fetch_xor_release 373 #define atomic_fetch_xor_release(...) \ 374 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__) 375 #endif 376 377 #ifndef atomic_fetch_xor 378 #define atomic_fetch_xor(...) \ 379 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__) 380 #endif 381 #endif /* atomic_fetch_xor_relaxed */ 382 383 384 /* atomic_xchg_relaxed */ 385 #ifndef atomic_xchg_relaxed 386 #define atomic_xchg_relaxed atomic_xchg 387 #define atomic_xchg_acquire atomic_xchg 388 #define atomic_xchg_release atomic_xchg 389 390 #else /* atomic_xchg_relaxed */ 391 392 #ifndef atomic_xchg_acquire 393 #define atomic_xchg_acquire(...) \ 394 __atomic_op_acquire(atomic_xchg, __VA_ARGS__) 395 #endif 396 397 #ifndef atomic_xchg_release 398 #define atomic_xchg_release(...) \ 399 __atomic_op_release(atomic_xchg, __VA_ARGS__) 400 #endif 401 402 #ifndef atomic_xchg 403 #define atomic_xchg(...) \ 404 __atomic_op_fence(atomic_xchg, __VA_ARGS__) 405 #endif 406 #endif /* atomic_xchg_relaxed */ 407 408 /* atomic_cmpxchg_relaxed */ 409 #ifndef atomic_cmpxchg_relaxed 410 #define atomic_cmpxchg_relaxed atomic_cmpxchg 411 #define atomic_cmpxchg_acquire atomic_cmpxchg 412 #define atomic_cmpxchg_release atomic_cmpxchg 413 414 #else /* atomic_cmpxchg_relaxed */ 415 416 #ifndef atomic_cmpxchg_acquire 417 #define atomic_cmpxchg_acquire(...) \ 418 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__) 419 #endif 420 421 #ifndef atomic_cmpxchg_release 422 #define atomic_cmpxchg_release(...) \ 423 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__) 424 #endif 425 426 #ifndef atomic_cmpxchg 427 #define atomic_cmpxchg(...) \ 428 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__) 429 #endif 430 #endif /* atomic_cmpxchg_relaxed */ 431 432 #ifndef atomic_try_cmpxchg 433 434 #define __atomic_try_cmpxchg(type, _p, _po, _n) \ 435 ({ \ 436 typeof(_po) __po = (_po); \ 437 typeof(*(_po)) __r, __o = *__po; \ 438 __r = atomic_cmpxchg##type((_p), __o, (_n)); \ 439 if (unlikely(__r != __o)) \ 440 *__po = __r; \ 441 likely(__r == __o); \ 442 }) 443 444 #define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n) 445 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n) 446 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n) 447 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n) 448 449 #else /* atomic_try_cmpxchg */ 450 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg 451 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg 452 #define atomic_try_cmpxchg_release atomic_try_cmpxchg 453 #endif /* atomic_try_cmpxchg */ 454 455 /* cmpxchg_relaxed */ 456 #ifndef cmpxchg_relaxed 457 #define cmpxchg_relaxed cmpxchg 458 #define cmpxchg_acquire cmpxchg 459 #define cmpxchg_release cmpxchg 460 461 #else /* cmpxchg_relaxed */ 462 463 #ifndef cmpxchg_acquire 464 #define cmpxchg_acquire(...) \ 465 __atomic_op_acquire(cmpxchg, __VA_ARGS__) 466 #endif 467 468 #ifndef cmpxchg_release 469 #define cmpxchg_release(...) \ 470 __atomic_op_release(cmpxchg, __VA_ARGS__) 471 #endif 472 473 #ifndef cmpxchg 474 #define cmpxchg(...) \ 475 __atomic_op_fence(cmpxchg, __VA_ARGS__) 476 #endif 477 #endif /* cmpxchg_relaxed */ 478 479 /* cmpxchg64_relaxed */ 480 #ifndef cmpxchg64_relaxed 481 #define cmpxchg64_relaxed cmpxchg64 482 #define cmpxchg64_acquire cmpxchg64 483 #define cmpxchg64_release cmpxchg64 484 485 #else /* cmpxchg64_relaxed */ 486 487 #ifndef cmpxchg64_acquire 488 #define cmpxchg64_acquire(...) \ 489 __atomic_op_acquire(cmpxchg64, __VA_ARGS__) 490 #endif 491 492 #ifndef cmpxchg64_release 493 #define cmpxchg64_release(...) \ 494 __atomic_op_release(cmpxchg64, __VA_ARGS__) 495 #endif 496 497 #ifndef cmpxchg64 498 #define cmpxchg64(...) \ 499 __atomic_op_fence(cmpxchg64, __VA_ARGS__) 500 #endif 501 #endif /* cmpxchg64_relaxed */ 502 503 /* xchg_relaxed */ 504 #ifndef xchg_relaxed 505 #define xchg_relaxed xchg 506 #define xchg_acquire xchg 507 #define xchg_release xchg 508 509 #else /* xchg_relaxed */ 510 511 #ifndef xchg_acquire 512 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__) 513 #endif 514 515 #ifndef xchg_release 516 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__) 517 #endif 518 519 #ifndef xchg 520 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__) 521 #endif 522 #endif /* xchg_relaxed */ 523 524 /** 525 * atomic_add_unless - add unless the number is already a given value 526 * @v: pointer of type atomic_t 527 * @a: the amount to add to v... 528 * @u: ...unless v is equal to u. 529 * 530 * Atomically adds @a to @v, if @v was not already @u. 531 * Returns true if the addition was done. 532 */ 533 static inline bool atomic_add_unless(atomic_t *v, int a, int u) 534 { 535 return atomic_fetch_add_unless(v, a, u) != u; 536 } 537 538 /** 539 * atomic_inc_not_zero - increment unless the number is zero 540 * @v: pointer of type atomic_t 541 * 542 * Atomically increments @v by 1, if @v is non-zero. 543 * Returns true if the increment was done. 544 */ 545 #ifndef atomic_inc_not_zero 546 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 547 #endif 548 549 #ifndef atomic_andnot 550 static inline void atomic_andnot(int i, atomic_t *v) 551 { 552 atomic_and(~i, v); 553 } 554 555 static inline int atomic_fetch_andnot(int i, atomic_t *v) 556 { 557 return atomic_fetch_and(~i, v); 558 } 559 560 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) 561 { 562 return atomic_fetch_and_relaxed(~i, v); 563 } 564 565 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) 566 { 567 return atomic_fetch_and_acquire(~i, v); 568 } 569 570 static inline int atomic_fetch_andnot_release(int i, atomic_t *v) 571 { 572 return atomic_fetch_and_release(~i, v); 573 } 574 #endif 575 576 #ifndef atomic_inc_unless_negative 577 static inline bool atomic_inc_unless_negative(atomic_t *p) 578 { 579 int v, v1; 580 for (v = 0; v >= 0; v = v1) { 581 v1 = atomic_cmpxchg(p, v, v + 1); 582 if (likely(v1 == v)) 583 return true; 584 } 585 return false; 586 } 587 #endif 588 589 #ifndef atomic_dec_unless_positive 590 static inline bool atomic_dec_unless_positive(atomic_t *p) 591 { 592 int v, v1; 593 for (v = 0; v <= 0; v = v1) { 594 v1 = atomic_cmpxchg(p, v, v - 1); 595 if (likely(v1 == v)) 596 return true; 597 } 598 return false; 599 } 600 #endif 601 602 /* 603 * atomic_dec_if_positive - decrement by 1 if old value positive 604 * @v: pointer of type atomic_t 605 * 606 * The function returns the old value of *v minus 1, even if 607 * the atomic variable, v, was not decremented. 608 */ 609 #ifndef atomic_dec_if_positive 610 static inline int atomic_dec_if_positive(atomic_t *v) 611 { 612 int c, old, dec; 613 c = atomic_read(v); 614 for (;;) { 615 dec = c - 1; 616 if (unlikely(dec < 0)) 617 break; 618 old = atomic_cmpxchg((v), c, dec); 619 if (likely(old == c)) 620 break; 621 c = old; 622 } 623 return dec; 624 } 625 #endif 626 627 #define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) 628 #define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) 629 630 #ifdef CONFIG_GENERIC_ATOMIC64 631 #include <asm-generic/atomic64.h> 632 #endif 633 634 #ifndef atomic64_read_acquire 635 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter) 636 #endif 637 638 #ifndef atomic64_set_release 639 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i)) 640 #endif 641 642 /* atomic64_add_return_relaxed */ 643 #ifndef atomic64_add_return_relaxed 644 #define atomic64_add_return_relaxed atomic64_add_return 645 #define atomic64_add_return_acquire atomic64_add_return 646 #define atomic64_add_return_release atomic64_add_return 647 648 #else /* atomic64_add_return_relaxed */ 649 650 #ifndef atomic64_add_return_acquire 651 #define atomic64_add_return_acquire(...) \ 652 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__) 653 #endif 654 655 #ifndef atomic64_add_return_release 656 #define atomic64_add_return_release(...) \ 657 __atomic_op_release(atomic64_add_return, __VA_ARGS__) 658 #endif 659 660 #ifndef atomic64_add_return 661 #define atomic64_add_return(...) \ 662 __atomic_op_fence(atomic64_add_return, __VA_ARGS__) 663 #endif 664 #endif /* atomic64_add_return_relaxed */ 665 666 /* atomic64_inc_return_relaxed */ 667 #ifndef atomic64_inc_return_relaxed 668 #define atomic64_inc_return_relaxed atomic64_inc_return 669 #define atomic64_inc_return_acquire atomic64_inc_return 670 #define atomic64_inc_return_release atomic64_inc_return 671 672 #else /* atomic64_inc_return_relaxed */ 673 674 #ifndef atomic64_inc_return_acquire 675 #define atomic64_inc_return_acquire(...) \ 676 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__) 677 #endif 678 679 #ifndef atomic64_inc_return_release 680 #define atomic64_inc_return_release(...) \ 681 __atomic_op_release(atomic64_inc_return, __VA_ARGS__) 682 #endif 683 684 #ifndef atomic64_inc_return 685 #define atomic64_inc_return(...) \ 686 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__) 687 #endif 688 #endif /* atomic64_inc_return_relaxed */ 689 690 691 /* atomic64_sub_return_relaxed */ 692 #ifndef atomic64_sub_return_relaxed 693 #define atomic64_sub_return_relaxed atomic64_sub_return 694 #define atomic64_sub_return_acquire atomic64_sub_return 695 #define atomic64_sub_return_release atomic64_sub_return 696 697 #else /* atomic64_sub_return_relaxed */ 698 699 #ifndef atomic64_sub_return_acquire 700 #define atomic64_sub_return_acquire(...) \ 701 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__) 702 #endif 703 704 #ifndef atomic64_sub_return_release 705 #define atomic64_sub_return_release(...) \ 706 __atomic_op_release(atomic64_sub_return, __VA_ARGS__) 707 #endif 708 709 #ifndef atomic64_sub_return 710 #define atomic64_sub_return(...) \ 711 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__) 712 #endif 713 #endif /* atomic64_sub_return_relaxed */ 714 715 /* atomic64_dec_return_relaxed */ 716 #ifndef atomic64_dec_return_relaxed 717 #define atomic64_dec_return_relaxed atomic64_dec_return 718 #define atomic64_dec_return_acquire atomic64_dec_return 719 #define atomic64_dec_return_release atomic64_dec_return 720 721 #else /* atomic64_dec_return_relaxed */ 722 723 #ifndef atomic64_dec_return_acquire 724 #define atomic64_dec_return_acquire(...) \ 725 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__) 726 #endif 727 728 #ifndef atomic64_dec_return_release 729 #define atomic64_dec_return_release(...) \ 730 __atomic_op_release(atomic64_dec_return, __VA_ARGS__) 731 #endif 732 733 #ifndef atomic64_dec_return 734 #define atomic64_dec_return(...) \ 735 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__) 736 #endif 737 #endif /* atomic64_dec_return_relaxed */ 738 739 740 /* atomic64_fetch_add_relaxed */ 741 #ifndef atomic64_fetch_add_relaxed 742 #define atomic64_fetch_add_relaxed atomic64_fetch_add 743 #define atomic64_fetch_add_acquire atomic64_fetch_add 744 #define atomic64_fetch_add_release atomic64_fetch_add 745 746 #else /* atomic64_fetch_add_relaxed */ 747 748 #ifndef atomic64_fetch_add_acquire 749 #define atomic64_fetch_add_acquire(...) \ 750 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__) 751 #endif 752 753 #ifndef atomic64_fetch_add_release 754 #define atomic64_fetch_add_release(...) \ 755 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__) 756 #endif 757 758 #ifndef atomic64_fetch_add 759 #define atomic64_fetch_add(...) \ 760 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__) 761 #endif 762 #endif /* atomic64_fetch_add_relaxed */ 763 764 /* atomic64_fetch_inc_relaxed */ 765 #ifndef atomic64_fetch_inc_relaxed 766 767 #ifndef atomic64_fetch_inc 768 #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v)) 769 #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v)) 770 #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v)) 771 #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v)) 772 #else /* atomic64_fetch_inc */ 773 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc 774 #define atomic64_fetch_inc_acquire atomic64_fetch_inc 775 #define atomic64_fetch_inc_release atomic64_fetch_inc 776 #endif /* atomic64_fetch_inc */ 777 778 #else /* atomic64_fetch_inc_relaxed */ 779 780 #ifndef atomic64_fetch_inc_acquire 781 #define atomic64_fetch_inc_acquire(...) \ 782 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__) 783 #endif 784 785 #ifndef atomic64_fetch_inc_release 786 #define atomic64_fetch_inc_release(...) \ 787 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__) 788 #endif 789 790 #ifndef atomic64_fetch_inc 791 #define atomic64_fetch_inc(...) \ 792 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__) 793 #endif 794 #endif /* atomic64_fetch_inc_relaxed */ 795 796 /* atomic64_fetch_sub_relaxed */ 797 #ifndef atomic64_fetch_sub_relaxed 798 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub 799 #define atomic64_fetch_sub_acquire atomic64_fetch_sub 800 #define atomic64_fetch_sub_release atomic64_fetch_sub 801 802 #else /* atomic64_fetch_sub_relaxed */ 803 804 #ifndef atomic64_fetch_sub_acquire 805 #define atomic64_fetch_sub_acquire(...) \ 806 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__) 807 #endif 808 809 #ifndef atomic64_fetch_sub_release 810 #define atomic64_fetch_sub_release(...) \ 811 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__) 812 #endif 813 814 #ifndef atomic64_fetch_sub 815 #define atomic64_fetch_sub(...) \ 816 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__) 817 #endif 818 #endif /* atomic64_fetch_sub_relaxed */ 819 820 /* atomic64_fetch_dec_relaxed */ 821 #ifndef atomic64_fetch_dec_relaxed 822 823 #ifndef atomic64_fetch_dec 824 #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v)) 825 #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v)) 826 #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v)) 827 #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v)) 828 #else /* atomic64_fetch_dec */ 829 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec 830 #define atomic64_fetch_dec_acquire atomic64_fetch_dec 831 #define atomic64_fetch_dec_release atomic64_fetch_dec 832 #endif /* atomic64_fetch_dec */ 833 834 #else /* atomic64_fetch_dec_relaxed */ 835 836 #ifndef atomic64_fetch_dec_acquire 837 #define atomic64_fetch_dec_acquire(...) \ 838 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__) 839 #endif 840 841 #ifndef atomic64_fetch_dec_release 842 #define atomic64_fetch_dec_release(...) \ 843 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__) 844 #endif 845 846 #ifndef atomic64_fetch_dec 847 #define atomic64_fetch_dec(...) \ 848 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__) 849 #endif 850 #endif /* atomic64_fetch_dec_relaxed */ 851 852 /* atomic64_fetch_or_relaxed */ 853 #ifndef atomic64_fetch_or_relaxed 854 #define atomic64_fetch_or_relaxed atomic64_fetch_or 855 #define atomic64_fetch_or_acquire atomic64_fetch_or 856 #define atomic64_fetch_or_release atomic64_fetch_or 857 858 #else /* atomic64_fetch_or_relaxed */ 859 860 #ifndef atomic64_fetch_or_acquire 861 #define atomic64_fetch_or_acquire(...) \ 862 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__) 863 #endif 864 865 #ifndef atomic64_fetch_or_release 866 #define atomic64_fetch_or_release(...) \ 867 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__) 868 #endif 869 870 #ifndef atomic64_fetch_or 871 #define atomic64_fetch_or(...) \ 872 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__) 873 #endif 874 #endif /* atomic64_fetch_or_relaxed */ 875 876 /* atomic64_fetch_and_relaxed */ 877 #ifndef atomic64_fetch_and_relaxed 878 #define atomic64_fetch_and_relaxed atomic64_fetch_and 879 #define atomic64_fetch_and_acquire atomic64_fetch_and 880 #define atomic64_fetch_and_release atomic64_fetch_and 881 882 #else /* atomic64_fetch_and_relaxed */ 883 884 #ifndef atomic64_fetch_and_acquire 885 #define atomic64_fetch_and_acquire(...) \ 886 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__) 887 #endif 888 889 #ifndef atomic64_fetch_and_release 890 #define atomic64_fetch_and_release(...) \ 891 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__) 892 #endif 893 894 #ifndef atomic64_fetch_and 895 #define atomic64_fetch_and(...) \ 896 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__) 897 #endif 898 #endif /* atomic64_fetch_and_relaxed */ 899 900 #ifdef atomic64_andnot 901 /* atomic64_fetch_andnot_relaxed */ 902 #ifndef atomic64_fetch_andnot_relaxed 903 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot 904 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot 905 #define atomic64_fetch_andnot_release atomic64_fetch_andnot 906 907 #else /* atomic64_fetch_andnot_relaxed */ 908 909 #ifndef atomic64_fetch_andnot_acquire 910 #define atomic64_fetch_andnot_acquire(...) \ 911 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__) 912 #endif 913 914 #ifndef atomic64_fetch_andnot_release 915 #define atomic64_fetch_andnot_release(...) \ 916 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__) 917 #endif 918 919 #ifndef atomic64_fetch_andnot 920 #define atomic64_fetch_andnot(...) \ 921 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__) 922 #endif 923 #endif /* atomic64_fetch_andnot_relaxed */ 924 #endif /* atomic64_andnot */ 925 926 /* atomic64_fetch_xor_relaxed */ 927 #ifndef atomic64_fetch_xor_relaxed 928 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor 929 #define atomic64_fetch_xor_acquire atomic64_fetch_xor 930 #define atomic64_fetch_xor_release atomic64_fetch_xor 931 932 #else /* atomic64_fetch_xor_relaxed */ 933 934 #ifndef atomic64_fetch_xor_acquire 935 #define atomic64_fetch_xor_acquire(...) \ 936 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__) 937 #endif 938 939 #ifndef atomic64_fetch_xor_release 940 #define atomic64_fetch_xor_release(...) \ 941 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__) 942 #endif 943 944 #ifndef atomic64_fetch_xor 945 #define atomic64_fetch_xor(...) \ 946 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__) 947 #endif 948 #endif /* atomic64_fetch_xor_relaxed */ 949 950 951 /* atomic64_xchg_relaxed */ 952 #ifndef atomic64_xchg_relaxed 953 #define atomic64_xchg_relaxed atomic64_xchg 954 #define atomic64_xchg_acquire atomic64_xchg 955 #define atomic64_xchg_release atomic64_xchg 956 957 #else /* atomic64_xchg_relaxed */ 958 959 #ifndef atomic64_xchg_acquire 960 #define atomic64_xchg_acquire(...) \ 961 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__) 962 #endif 963 964 #ifndef atomic64_xchg_release 965 #define atomic64_xchg_release(...) \ 966 __atomic_op_release(atomic64_xchg, __VA_ARGS__) 967 #endif 968 969 #ifndef atomic64_xchg 970 #define atomic64_xchg(...) \ 971 __atomic_op_fence(atomic64_xchg, __VA_ARGS__) 972 #endif 973 #endif /* atomic64_xchg_relaxed */ 974 975 /* atomic64_cmpxchg_relaxed */ 976 #ifndef atomic64_cmpxchg_relaxed 977 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg 978 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 979 #define atomic64_cmpxchg_release atomic64_cmpxchg 980 981 #else /* atomic64_cmpxchg_relaxed */ 982 983 #ifndef atomic64_cmpxchg_acquire 984 #define atomic64_cmpxchg_acquire(...) \ 985 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__) 986 #endif 987 988 #ifndef atomic64_cmpxchg_release 989 #define atomic64_cmpxchg_release(...) \ 990 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__) 991 #endif 992 993 #ifndef atomic64_cmpxchg 994 #define atomic64_cmpxchg(...) \ 995 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__) 996 #endif 997 #endif /* atomic64_cmpxchg_relaxed */ 998 999 #ifndef atomic64_try_cmpxchg 1000 1001 #define __atomic64_try_cmpxchg(type, _p, _po, _n) \ 1002 ({ \ 1003 typeof(_po) __po = (_po); \ 1004 typeof(*(_po)) __r, __o = *__po; \ 1005 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \ 1006 if (unlikely(__r != __o)) \ 1007 *__po = __r; \ 1008 likely(__r == __o); \ 1009 }) 1010 1011 #define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n) 1012 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n) 1013 #define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n) 1014 #define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n) 1015 1016 #else /* atomic64_try_cmpxchg */ 1017 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg 1018 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg 1019 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg 1020 #endif /* atomic64_try_cmpxchg */ 1021 1022 #ifndef atomic64_andnot 1023 static inline void atomic64_andnot(long long i, atomic64_t *v) 1024 { 1025 atomic64_and(~i, v); 1026 } 1027 1028 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v) 1029 { 1030 return atomic64_fetch_and(~i, v); 1031 } 1032 1033 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v) 1034 { 1035 return atomic64_fetch_and_relaxed(~i, v); 1036 } 1037 1038 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v) 1039 { 1040 return atomic64_fetch_and_acquire(~i, v); 1041 } 1042 1043 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v) 1044 { 1045 return atomic64_fetch_and_release(~i, v); 1046 } 1047 #endif 1048 1049 #define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) 1050 #define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) 1051 1052 #include <asm-generic/atomic-long.h> 1053 1054 #endif /* _LINUX_ATOMIC_H */ 1055