1 /* Atomic operations usable in machine independent code */ 2 #ifndef _LINUX_ATOMIC_H 3 #define _LINUX_ATOMIC_H 4 #include <asm/atomic.h> 5 #include <asm/barrier.h> 6 7 /* 8 * Relaxed variants of xchg, cmpxchg and some atomic operations. 9 * 10 * We support four variants: 11 * 12 * - Fully ordered: The default implementation, no suffix required. 13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix. 14 * - Release: Provides RELEASE semantics, _release suffix. 15 * - Relaxed: No ordering guarantees, _relaxed suffix. 16 * 17 * For compound atomics performing both a load and a store, ACQUIRE 18 * semantics apply only to the load and RELEASE semantics only to the 19 * store portion of the operation. Note that a failed cmpxchg_acquire 20 * does -not- imply any memory ordering constraints. 21 * 22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. 23 */ 24 25 #ifndef atomic_read_acquire 26 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) 27 #endif 28 29 #ifndef atomic_set_release 30 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) 31 #endif 32 33 /* 34 * The idea here is to build acquire/release variants by adding explicit 35 * barriers on top of the relaxed variant. In the case where the relaxed 36 * variant is already fully ordered, no additional barriers are needed. 37 * 38 * Besides, if an arch has a special barrier for acquire/release, it could 39 * implement its own __atomic_op_* and use the same framework for building 40 * variants 41 * 42 * If an architecture overrides __atomic_op_acquire() it will probably want 43 * to define smp_mb__after_spinlock(). 44 */ 45 #ifndef __atomic_op_acquire 46 #define __atomic_op_acquire(op, args...) \ 47 ({ \ 48 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 49 smp_mb__after_atomic(); \ 50 __ret; \ 51 }) 52 #endif 53 54 #ifndef __atomic_op_release 55 #define __atomic_op_release(op, args...) \ 56 ({ \ 57 smp_mb__before_atomic(); \ 58 op##_relaxed(args); \ 59 }) 60 #endif 61 62 #ifndef __atomic_op_fence 63 #define __atomic_op_fence(op, args...) \ 64 ({ \ 65 typeof(op##_relaxed(args)) __ret; \ 66 smp_mb__before_atomic(); \ 67 __ret = op##_relaxed(args); \ 68 smp_mb__after_atomic(); \ 69 __ret; \ 70 }) 71 #endif 72 73 /* atomic_add_return_relaxed */ 74 #ifndef atomic_add_return_relaxed 75 #define atomic_add_return_relaxed atomic_add_return 76 #define atomic_add_return_acquire atomic_add_return 77 #define atomic_add_return_release atomic_add_return 78 79 #else /* atomic_add_return_relaxed */ 80 81 #ifndef atomic_add_return_acquire 82 #define atomic_add_return_acquire(...) \ 83 __atomic_op_acquire(atomic_add_return, __VA_ARGS__) 84 #endif 85 86 #ifndef atomic_add_return_release 87 #define atomic_add_return_release(...) \ 88 __atomic_op_release(atomic_add_return, __VA_ARGS__) 89 #endif 90 91 #ifndef atomic_add_return 92 #define atomic_add_return(...) \ 93 __atomic_op_fence(atomic_add_return, __VA_ARGS__) 94 #endif 95 #endif /* atomic_add_return_relaxed */ 96 97 /* atomic_inc_return_relaxed */ 98 #ifndef atomic_inc_return_relaxed 99 #define atomic_inc_return_relaxed atomic_inc_return 100 #define atomic_inc_return_acquire atomic_inc_return 101 #define atomic_inc_return_release atomic_inc_return 102 103 #else /* atomic_inc_return_relaxed */ 104 105 #ifndef atomic_inc_return_acquire 106 #define atomic_inc_return_acquire(...) \ 107 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__) 108 #endif 109 110 #ifndef atomic_inc_return_release 111 #define atomic_inc_return_release(...) \ 112 __atomic_op_release(atomic_inc_return, __VA_ARGS__) 113 #endif 114 115 #ifndef atomic_inc_return 116 #define atomic_inc_return(...) \ 117 __atomic_op_fence(atomic_inc_return, __VA_ARGS__) 118 #endif 119 #endif /* atomic_inc_return_relaxed */ 120 121 /* atomic_sub_return_relaxed */ 122 #ifndef atomic_sub_return_relaxed 123 #define atomic_sub_return_relaxed atomic_sub_return 124 #define atomic_sub_return_acquire atomic_sub_return 125 #define atomic_sub_return_release atomic_sub_return 126 127 #else /* atomic_sub_return_relaxed */ 128 129 #ifndef atomic_sub_return_acquire 130 #define atomic_sub_return_acquire(...) \ 131 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__) 132 #endif 133 134 #ifndef atomic_sub_return_release 135 #define atomic_sub_return_release(...) \ 136 __atomic_op_release(atomic_sub_return, __VA_ARGS__) 137 #endif 138 139 #ifndef atomic_sub_return 140 #define atomic_sub_return(...) \ 141 __atomic_op_fence(atomic_sub_return, __VA_ARGS__) 142 #endif 143 #endif /* atomic_sub_return_relaxed */ 144 145 /* atomic_dec_return_relaxed */ 146 #ifndef atomic_dec_return_relaxed 147 #define atomic_dec_return_relaxed atomic_dec_return 148 #define atomic_dec_return_acquire atomic_dec_return 149 #define atomic_dec_return_release atomic_dec_return 150 151 #else /* atomic_dec_return_relaxed */ 152 153 #ifndef atomic_dec_return_acquire 154 #define atomic_dec_return_acquire(...) \ 155 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__) 156 #endif 157 158 #ifndef atomic_dec_return_release 159 #define atomic_dec_return_release(...) \ 160 __atomic_op_release(atomic_dec_return, __VA_ARGS__) 161 #endif 162 163 #ifndef atomic_dec_return 164 #define atomic_dec_return(...) \ 165 __atomic_op_fence(atomic_dec_return, __VA_ARGS__) 166 #endif 167 #endif /* atomic_dec_return_relaxed */ 168 169 170 /* atomic_fetch_add_relaxed */ 171 #ifndef atomic_fetch_add_relaxed 172 #define atomic_fetch_add_relaxed atomic_fetch_add 173 #define atomic_fetch_add_acquire atomic_fetch_add 174 #define atomic_fetch_add_release atomic_fetch_add 175 176 #else /* atomic_fetch_add_relaxed */ 177 178 #ifndef atomic_fetch_add_acquire 179 #define atomic_fetch_add_acquire(...) \ 180 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__) 181 #endif 182 183 #ifndef atomic_fetch_add_release 184 #define atomic_fetch_add_release(...) \ 185 __atomic_op_release(atomic_fetch_add, __VA_ARGS__) 186 #endif 187 188 #ifndef atomic_fetch_add 189 #define atomic_fetch_add(...) \ 190 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__) 191 #endif 192 #endif /* atomic_fetch_add_relaxed */ 193 194 /* atomic_fetch_inc_relaxed */ 195 #ifndef atomic_fetch_inc_relaxed 196 197 #ifndef atomic_fetch_inc 198 #define atomic_fetch_inc(v) atomic_fetch_add(1, (v)) 199 #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v)) 200 #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v)) 201 #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v)) 202 #else /* atomic_fetch_inc */ 203 #define atomic_fetch_inc_relaxed atomic_fetch_inc 204 #define atomic_fetch_inc_acquire atomic_fetch_inc 205 #define atomic_fetch_inc_release atomic_fetch_inc 206 #endif /* atomic_fetch_inc */ 207 208 #else /* atomic_fetch_inc_relaxed */ 209 210 #ifndef atomic_fetch_inc_acquire 211 #define atomic_fetch_inc_acquire(...) \ 212 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__) 213 #endif 214 215 #ifndef atomic_fetch_inc_release 216 #define atomic_fetch_inc_release(...) \ 217 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__) 218 #endif 219 220 #ifndef atomic_fetch_inc 221 #define atomic_fetch_inc(...) \ 222 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__) 223 #endif 224 #endif /* atomic_fetch_inc_relaxed */ 225 226 /* atomic_fetch_sub_relaxed */ 227 #ifndef atomic_fetch_sub_relaxed 228 #define atomic_fetch_sub_relaxed atomic_fetch_sub 229 #define atomic_fetch_sub_acquire atomic_fetch_sub 230 #define atomic_fetch_sub_release atomic_fetch_sub 231 232 #else /* atomic_fetch_sub_relaxed */ 233 234 #ifndef atomic_fetch_sub_acquire 235 #define atomic_fetch_sub_acquire(...) \ 236 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__) 237 #endif 238 239 #ifndef atomic_fetch_sub_release 240 #define atomic_fetch_sub_release(...) \ 241 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__) 242 #endif 243 244 #ifndef atomic_fetch_sub 245 #define atomic_fetch_sub(...) \ 246 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__) 247 #endif 248 #endif /* atomic_fetch_sub_relaxed */ 249 250 /* atomic_fetch_dec_relaxed */ 251 #ifndef atomic_fetch_dec_relaxed 252 253 #ifndef atomic_fetch_dec 254 #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v)) 255 #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v)) 256 #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v)) 257 #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v)) 258 #else /* atomic_fetch_dec */ 259 #define atomic_fetch_dec_relaxed atomic_fetch_dec 260 #define atomic_fetch_dec_acquire atomic_fetch_dec 261 #define atomic_fetch_dec_release atomic_fetch_dec 262 #endif /* atomic_fetch_dec */ 263 264 #else /* atomic_fetch_dec_relaxed */ 265 266 #ifndef atomic_fetch_dec_acquire 267 #define atomic_fetch_dec_acquire(...) \ 268 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__) 269 #endif 270 271 #ifndef atomic_fetch_dec_release 272 #define atomic_fetch_dec_release(...) \ 273 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__) 274 #endif 275 276 #ifndef atomic_fetch_dec 277 #define atomic_fetch_dec(...) \ 278 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__) 279 #endif 280 #endif /* atomic_fetch_dec_relaxed */ 281 282 /* atomic_fetch_or_relaxed */ 283 #ifndef atomic_fetch_or_relaxed 284 #define atomic_fetch_or_relaxed atomic_fetch_or 285 #define atomic_fetch_or_acquire atomic_fetch_or 286 #define atomic_fetch_or_release atomic_fetch_or 287 288 #else /* atomic_fetch_or_relaxed */ 289 290 #ifndef atomic_fetch_or_acquire 291 #define atomic_fetch_or_acquire(...) \ 292 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__) 293 #endif 294 295 #ifndef atomic_fetch_or_release 296 #define atomic_fetch_or_release(...) \ 297 __atomic_op_release(atomic_fetch_or, __VA_ARGS__) 298 #endif 299 300 #ifndef atomic_fetch_or 301 #define atomic_fetch_or(...) \ 302 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__) 303 #endif 304 #endif /* atomic_fetch_or_relaxed */ 305 306 /* atomic_fetch_and_relaxed */ 307 #ifndef atomic_fetch_and_relaxed 308 #define atomic_fetch_and_relaxed atomic_fetch_and 309 #define atomic_fetch_and_acquire atomic_fetch_and 310 #define atomic_fetch_and_release atomic_fetch_and 311 312 #else /* atomic_fetch_and_relaxed */ 313 314 #ifndef atomic_fetch_and_acquire 315 #define atomic_fetch_and_acquire(...) \ 316 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__) 317 #endif 318 319 #ifndef atomic_fetch_and_release 320 #define atomic_fetch_and_release(...) \ 321 __atomic_op_release(atomic_fetch_and, __VA_ARGS__) 322 #endif 323 324 #ifndef atomic_fetch_and 325 #define atomic_fetch_and(...) \ 326 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__) 327 #endif 328 #endif /* atomic_fetch_and_relaxed */ 329 330 #ifdef atomic_andnot 331 /* atomic_fetch_andnot_relaxed */ 332 #ifndef atomic_fetch_andnot_relaxed 333 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot 334 #define atomic_fetch_andnot_acquire atomic_fetch_andnot 335 #define atomic_fetch_andnot_release atomic_fetch_andnot 336 337 #else /* atomic_fetch_andnot_relaxed */ 338 339 #ifndef atomic_fetch_andnot_acquire 340 #define atomic_fetch_andnot_acquire(...) \ 341 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__) 342 #endif 343 344 #ifndef atomic_fetch_andnot_release 345 #define atomic_fetch_andnot_release(...) \ 346 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__) 347 #endif 348 349 #ifndef atomic_fetch_andnot 350 #define atomic_fetch_andnot(...) \ 351 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__) 352 #endif 353 #endif /* atomic_fetch_andnot_relaxed */ 354 #endif /* atomic_andnot */ 355 356 /* atomic_fetch_xor_relaxed */ 357 #ifndef atomic_fetch_xor_relaxed 358 #define atomic_fetch_xor_relaxed atomic_fetch_xor 359 #define atomic_fetch_xor_acquire atomic_fetch_xor 360 #define atomic_fetch_xor_release atomic_fetch_xor 361 362 #else /* atomic_fetch_xor_relaxed */ 363 364 #ifndef atomic_fetch_xor_acquire 365 #define atomic_fetch_xor_acquire(...) \ 366 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__) 367 #endif 368 369 #ifndef atomic_fetch_xor_release 370 #define atomic_fetch_xor_release(...) \ 371 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__) 372 #endif 373 374 #ifndef atomic_fetch_xor 375 #define atomic_fetch_xor(...) \ 376 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__) 377 #endif 378 #endif /* atomic_fetch_xor_relaxed */ 379 380 381 /* atomic_xchg_relaxed */ 382 #ifndef atomic_xchg_relaxed 383 #define atomic_xchg_relaxed atomic_xchg 384 #define atomic_xchg_acquire atomic_xchg 385 #define atomic_xchg_release atomic_xchg 386 387 #else /* atomic_xchg_relaxed */ 388 389 #ifndef atomic_xchg_acquire 390 #define atomic_xchg_acquire(...) \ 391 __atomic_op_acquire(atomic_xchg, __VA_ARGS__) 392 #endif 393 394 #ifndef atomic_xchg_release 395 #define atomic_xchg_release(...) \ 396 __atomic_op_release(atomic_xchg, __VA_ARGS__) 397 #endif 398 399 #ifndef atomic_xchg 400 #define atomic_xchg(...) \ 401 __atomic_op_fence(atomic_xchg, __VA_ARGS__) 402 #endif 403 #endif /* atomic_xchg_relaxed */ 404 405 /* atomic_cmpxchg_relaxed */ 406 #ifndef atomic_cmpxchg_relaxed 407 #define atomic_cmpxchg_relaxed atomic_cmpxchg 408 #define atomic_cmpxchg_acquire atomic_cmpxchg 409 #define atomic_cmpxchg_release atomic_cmpxchg 410 411 #else /* atomic_cmpxchg_relaxed */ 412 413 #ifndef atomic_cmpxchg_acquire 414 #define atomic_cmpxchg_acquire(...) \ 415 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__) 416 #endif 417 418 #ifndef atomic_cmpxchg_release 419 #define atomic_cmpxchg_release(...) \ 420 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__) 421 #endif 422 423 #ifndef atomic_cmpxchg 424 #define atomic_cmpxchg(...) \ 425 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__) 426 #endif 427 #endif /* atomic_cmpxchg_relaxed */ 428 429 #ifndef atomic_try_cmpxchg 430 431 #define __atomic_try_cmpxchg(type, _p, _po, _n) \ 432 ({ \ 433 typeof(_po) __po = (_po); \ 434 typeof(*(_po)) __r, __o = *__po; \ 435 __r = atomic_cmpxchg##type((_p), __o, (_n)); \ 436 if (unlikely(__r != __o)) \ 437 *__po = __r; \ 438 likely(__r == __o); \ 439 }) 440 441 #define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n) 442 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n) 443 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n) 444 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n) 445 446 #else /* atomic_try_cmpxchg */ 447 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg 448 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg 449 #define atomic_try_cmpxchg_release atomic_try_cmpxchg 450 #endif /* atomic_try_cmpxchg */ 451 452 /* cmpxchg_relaxed */ 453 #ifndef cmpxchg_relaxed 454 #define cmpxchg_relaxed cmpxchg 455 #define cmpxchg_acquire cmpxchg 456 #define cmpxchg_release cmpxchg 457 458 #else /* cmpxchg_relaxed */ 459 460 #ifndef cmpxchg_acquire 461 #define cmpxchg_acquire(...) \ 462 __atomic_op_acquire(cmpxchg, __VA_ARGS__) 463 #endif 464 465 #ifndef cmpxchg_release 466 #define cmpxchg_release(...) \ 467 __atomic_op_release(cmpxchg, __VA_ARGS__) 468 #endif 469 470 #ifndef cmpxchg 471 #define cmpxchg(...) \ 472 __atomic_op_fence(cmpxchg, __VA_ARGS__) 473 #endif 474 #endif /* cmpxchg_relaxed */ 475 476 /* cmpxchg64_relaxed */ 477 #ifndef cmpxchg64_relaxed 478 #define cmpxchg64_relaxed cmpxchg64 479 #define cmpxchg64_acquire cmpxchg64 480 #define cmpxchg64_release cmpxchg64 481 482 #else /* cmpxchg64_relaxed */ 483 484 #ifndef cmpxchg64_acquire 485 #define cmpxchg64_acquire(...) \ 486 __atomic_op_acquire(cmpxchg64, __VA_ARGS__) 487 #endif 488 489 #ifndef cmpxchg64_release 490 #define cmpxchg64_release(...) \ 491 __atomic_op_release(cmpxchg64, __VA_ARGS__) 492 #endif 493 494 #ifndef cmpxchg64 495 #define cmpxchg64(...) \ 496 __atomic_op_fence(cmpxchg64, __VA_ARGS__) 497 #endif 498 #endif /* cmpxchg64_relaxed */ 499 500 /* xchg_relaxed */ 501 #ifndef xchg_relaxed 502 #define xchg_relaxed xchg 503 #define xchg_acquire xchg 504 #define xchg_release xchg 505 506 #else /* xchg_relaxed */ 507 508 #ifndef xchg_acquire 509 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__) 510 #endif 511 512 #ifndef xchg_release 513 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__) 514 #endif 515 516 #ifndef xchg 517 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__) 518 #endif 519 #endif /* xchg_relaxed */ 520 521 /** 522 * atomic_add_unless - add unless the number is already a given value 523 * @v: pointer of type atomic_t 524 * @a: the amount to add to v... 525 * @u: ...unless v is equal to u. 526 * 527 * Atomically adds @a to @v, so long as @v was not already @u. 528 * Returns non-zero if @v was not @u, and zero otherwise. 529 */ 530 static inline int atomic_add_unless(atomic_t *v, int a, int u) 531 { 532 return __atomic_add_unless(v, a, u) != u; 533 } 534 535 /** 536 * atomic_inc_not_zero - increment unless the number is zero 537 * @v: pointer of type atomic_t 538 * 539 * Atomically increments @v by 1, so long as @v is non-zero. 540 * Returns non-zero if @v was non-zero, and zero otherwise. 541 */ 542 #ifndef atomic_inc_not_zero 543 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 544 #endif 545 546 #ifndef atomic_andnot 547 static inline void atomic_andnot(int i, atomic_t *v) 548 { 549 atomic_and(~i, v); 550 } 551 552 static inline int atomic_fetch_andnot(int i, atomic_t *v) 553 { 554 return atomic_fetch_and(~i, v); 555 } 556 557 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) 558 { 559 return atomic_fetch_and_relaxed(~i, v); 560 } 561 562 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) 563 { 564 return atomic_fetch_and_acquire(~i, v); 565 } 566 567 static inline int atomic_fetch_andnot_release(int i, atomic_t *v) 568 { 569 return atomic_fetch_and_release(~i, v); 570 } 571 #endif 572 573 /** 574 * atomic_inc_not_zero_hint - increment if not null 575 * @v: pointer of type atomic_t 576 * @hint: probable value of the atomic before the increment 577 * 578 * This version of atomic_inc_not_zero() gives a hint of probable 579 * value of the atomic. This helps processor to not read the memory 580 * before doing the atomic read/modify/write cycle, lowering 581 * number of bus transactions on some arches. 582 * 583 * Returns: 0 if increment was not done, 1 otherwise. 584 */ 585 #ifndef atomic_inc_not_zero_hint 586 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint) 587 { 588 int val, c = hint; 589 590 /* sanity test, should be removed by compiler if hint is a constant */ 591 if (!hint) 592 return atomic_inc_not_zero(v); 593 594 do { 595 val = atomic_cmpxchg(v, c, c + 1); 596 if (val == c) 597 return 1; 598 c = val; 599 } while (c); 600 601 return 0; 602 } 603 #endif 604 605 #ifndef atomic_inc_unless_negative 606 static inline int atomic_inc_unless_negative(atomic_t *p) 607 { 608 int v, v1; 609 for (v = 0; v >= 0; v = v1) { 610 v1 = atomic_cmpxchg(p, v, v + 1); 611 if (likely(v1 == v)) 612 return 1; 613 } 614 return 0; 615 } 616 #endif 617 618 #ifndef atomic_dec_unless_positive 619 static inline int atomic_dec_unless_positive(atomic_t *p) 620 { 621 int v, v1; 622 for (v = 0; v <= 0; v = v1) { 623 v1 = atomic_cmpxchg(p, v, v - 1); 624 if (likely(v1 == v)) 625 return 1; 626 } 627 return 0; 628 } 629 #endif 630 631 /* 632 * atomic_dec_if_positive - decrement by 1 if old value positive 633 * @v: pointer of type atomic_t 634 * 635 * The function returns the old value of *v minus 1, even if 636 * the atomic variable, v, was not decremented. 637 */ 638 #ifndef atomic_dec_if_positive 639 static inline int atomic_dec_if_positive(atomic_t *v) 640 { 641 int c, old, dec; 642 c = atomic_read(v); 643 for (;;) { 644 dec = c - 1; 645 if (unlikely(dec < 0)) 646 break; 647 old = atomic_cmpxchg((v), c, dec); 648 if (likely(old == c)) 649 break; 650 c = old; 651 } 652 return dec; 653 } 654 #endif 655 656 #ifdef CONFIG_GENERIC_ATOMIC64 657 #include <asm-generic/atomic64.h> 658 #endif 659 660 #ifndef atomic64_read_acquire 661 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter) 662 #endif 663 664 #ifndef atomic64_set_release 665 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i)) 666 #endif 667 668 /* atomic64_add_return_relaxed */ 669 #ifndef atomic64_add_return_relaxed 670 #define atomic64_add_return_relaxed atomic64_add_return 671 #define atomic64_add_return_acquire atomic64_add_return 672 #define atomic64_add_return_release atomic64_add_return 673 674 #else /* atomic64_add_return_relaxed */ 675 676 #ifndef atomic64_add_return_acquire 677 #define atomic64_add_return_acquire(...) \ 678 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__) 679 #endif 680 681 #ifndef atomic64_add_return_release 682 #define atomic64_add_return_release(...) \ 683 __atomic_op_release(atomic64_add_return, __VA_ARGS__) 684 #endif 685 686 #ifndef atomic64_add_return 687 #define atomic64_add_return(...) \ 688 __atomic_op_fence(atomic64_add_return, __VA_ARGS__) 689 #endif 690 #endif /* atomic64_add_return_relaxed */ 691 692 /* atomic64_inc_return_relaxed */ 693 #ifndef atomic64_inc_return_relaxed 694 #define atomic64_inc_return_relaxed atomic64_inc_return 695 #define atomic64_inc_return_acquire atomic64_inc_return 696 #define atomic64_inc_return_release atomic64_inc_return 697 698 #else /* atomic64_inc_return_relaxed */ 699 700 #ifndef atomic64_inc_return_acquire 701 #define atomic64_inc_return_acquire(...) \ 702 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__) 703 #endif 704 705 #ifndef atomic64_inc_return_release 706 #define atomic64_inc_return_release(...) \ 707 __atomic_op_release(atomic64_inc_return, __VA_ARGS__) 708 #endif 709 710 #ifndef atomic64_inc_return 711 #define atomic64_inc_return(...) \ 712 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__) 713 #endif 714 #endif /* atomic64_inc_return_relaxed */ 715 716 717 /* atomic64_sub_return_relaxed */ 718 #ifndef atomic64_sub_return_relaxed 719 #define atomic64_sub_return_relaxed atomic64_sub_return 720 #define atomic64_sub_return_acquire atomic64_sub_return 721 #define atomic64_sub_return_release atomic64_sub_return 722 723 #else /* atomic64_sub_return_relaxed */ 724 725 #ifndef atomic64_sub_return_acquire 726 #define atomic64_sub_return_acquire(...) \ 727 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__) 728 #endif 729 730 #ifndef atomic64_sub_return_release 731 #define atomic64_sub_return_release(...) \ 732 __atomic_op_release(atomic64_sub_return, __VA_ARGS__) 733 #endif 734 735 #ifndef atomic64_sub_return 736 #define atomic64_sub_return(...) \ 737 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__) 738 #endif 739 #endif /* atomic64_sub_return_relaxed */ 740 741 /* atomic64_dec_return_relaxed */ 742 #ifndef atomic64_dec_return_relaxed 743 #define atomic64_dec_return_relaxed atomic64_dec_return 744 #define atomic64_dec_return_acquire atomic64_dec_return 745 #define atomic64_dec_return_release atomic64_dec_return 746 747 #else /* atomic64_dec_return_relaxed */ 748 749 #ifndef atomic64_dec_return_acquire 750 #define atomic64_dec_return_acquire(...) \ 751 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__) 752 #endif 753 754 #ifndef atomic64_dec_return_release 755 #define atomic64_dec_return_release(...) \ 756 __atomic_op_release(atomic64_dec_return, __VA_ARGS__) 757 #endif 758 759 #ifndef atomic64_dec_return 760 #define atomic64_dec_return(...) \ 761 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__) 762 #endif 763 #endif /* atomic64_dec_return_relaxed */ 764 765 766 /* atomic64_fetch_add_relaxed */ 767 #ifndef atomic64_fetch_add_relaxed 768 #define atomic64_fetch_add_relaxed atomic64_fetch_add 769 #define atomic64_fetch_add_acquire atomic64_fetch_add 770 #define atomic64_fetch_add_release atomic64_fetch_add 771 772 #else /* atomic64_fetch_add_relaxed */ 773 774 #ifndef atomic64_fetch_add_acquire 775 #define atomic64_fetch_add_acquire(...) \ 776 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__) 777 #endif 778 779 #ifndef atomic64_fetch_add_release 780 #define atomic64_fetch_add_release(...) \ 781 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__) 782 #endif 783 784 #ifndef atomic64_fetch_add 785 #define atomic64_fetch_add(...) \ 786 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__) 787 #endif 788 #endif /* atomic64_fetch_add_relaxed */ 789 790 /* atomic64_fetch_inc_relaxed */ 791 #ifndef atomic64_fetch_inc_relaxed 792 793 #ifndef atomic64_fetch_inc 794 #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v)) 795 #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v)) 796 #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v)) 797 #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v)) 798 #else /* atomic64_fetch_inc */ 799 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc 800 #define atomic64_fetch_inc_acquire atomic64_fetch_inc 801 #define atomic64_fetch_inc_release atomic64_fetch_inc 802 #endif /* atomic64_fetch_inc */ 803 804 #else /* atomic64_fetch_inc_relaxed */ 805 806 #ifndef atomic64_fetch_inc_acquire 807 #define atomic64_fetch_inc_acquire(...) \ 808 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__) 809 #endif 810 811 #ifndef atomic64_fetch_inc_release 812 #define atomic64_fetch_inc_release(...) \ 813 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__) 814 #endif 815 816 #ifndef atomic64_fetch_inc 817 #define atomic64_fetch_inc(...) \ 818 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__) 819 #endif 820 #endif /* atomic64_fetch_inc_relaxed */ 821 822 /* atomic64_fetch_sub_relaxed */ 823 #ifndef atomic64_fetch_sub_relaxed 824 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub 825 #define atomic64_fetch_sub_acquire atomic64_fetch_sub 826 #define atomic64_fetch_sub_release atomic64_fetch_sub 827 828 #else /* atomic64_fetch_sub_relaxed */ 829 830 #ifndef atomic64_fetch_sub_acquire 831 #define atomic64_fetch_sub_acquire(...) \ 832 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__) 833 #endif 834 835 #ifndef atomic64_fetch_sub_release 836 #define atomic64_fetch_sub_release(...) \ 837 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__) 838 #endif 839 840 #ifndef atomic64_fetch_sub 841 #define atomic64_fetch_sub(...) \ 842 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__) 843 #endif 844 #endif /* atomic64_fetch_sub_relaxed */ 845 846 /* atomic64_fetch_dec_relaxed */ 847 #ifndef atomic64_fetch_dec_relaxed 848 849 #ifndef atomic64_fetch_dec 850 #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v)) 851 #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v)) 852 #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v)) 853 #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v)) 854 #else /* atomic64_fetch_dec */ 855 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec 856 #define atomic64_fetch_dec_acquire atomic64_fetch_dec 857 #define atomic64_fetch_dec_release atomic64_fetch_dec 858 #endif /* atomic64_fetch_dec */ 859 860 #else /* atomic64_fetch_dec_relaxed */ 861 862 #ifndef atomic64_fetch_dec_acquire 863 #define atomic64_fetch_dec_acquire(...) \ 864 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__) 865 #endif 866 867 #ifndef atomic64_fetch_dec_release 868 #define atomic64_fetch_dec_release(...) \ 869 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__) 870 #endif 871 872 #ifndef atomic64_fetch_dec 873 #define atomic64_fetch_dec(...) \ 874 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__) 875 #endif 876 #endif /* atomic64_fetch_dec_relaxed */ 877 878 /* atomic64_fetch_or_relaxed */ 879 #ifndef atomic64_fetch_or_relaxed 880 #define atomic64_fetch_or_relaxed atomic64_fetch_or 881 #define atomic64_fetch_or_acquire atomic64_fetch_or 882 #define atomic64_fetch_or_release atomic64_fetch_or 883 884 #else /* atomic64_fetch_or_relaxed */ 885 886 #ifndef atomic64_fetch_or_acquire 887 #define atomic64_fetch_or_acquire(...) \ 888 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__) 889 #endif 890 891 #ifndef atomic64_fetch_or_release 892 #define atomic64_fetch_or_release(...) \ 893 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__) 894 #endif 895 896 #ifndef atomic64_fetch_or 897 #define atomic64_fetch_or(...) \ 898 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__) 899 #endif 900 #endif /* atomic64_fetch_or_relaxed */ 901 902 /* atomic64_fetch_and_relaxed */ 903 #ifndef atomic64_fetch_and_relaxed 904 #define atomic64_fetch_and_relaxed atomic64_fetch_and 905 #define atomic64_fetch_and_acquire atomic64_fetch_and 906 #define atomic64_fetch_and_release atomic64_fetch_and 907 908 #else /* atomic64_fetch_and_relaxed */ 909 910 #ifndef atomic64_fetch_and_acquire 911 #define atomic64_fetch_and_acquire(...) \ 912 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__) 913 #endif 914 915 #ifndef atomic64_fetch_and_release 916 #define atomic64_fetch_and_release(...) \ 917 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__) 918 #endif 919 920 #ifndef atomic64_fetch_and 921 #define atomic64_fetch_and(...) \ 922 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__) 923 #endif 924 #endif /* atomic64_fetch_and_relaxed */ 925 926 #ifdef atomic64_andnot 927 /* atomic64_fetch_andnot_relaxed */ 928 #ifndef atomic64_fetch_andnot_relaxed 929 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot 930 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot 931 #define atomic64_fetch_andnot_release atomic64_fetch_andnot 932 933 #else /* atomic64_fetch_andnot_relaxed */ 934 935 #ifndef atomic64_fetch_andnot_acquire 936 #define atomic64_fetch_andnot_acquire(...) \ 937 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__) 938 #endif 939 940 #ifndef atomic64_fetch_andnot_release 941 #define atomic64_fetch_andnot_release(...) \ 942 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__) 943 #endif 944 945 #ifndef atomic64_fetch_andnot 946 #define atomic64_fetch_andnot(...) \ 947 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__) 948 #endif 949 #endif /* atomic64_fetch_andnot_relaxed */ 950 #endif /* atomic64_andnot */ 951 952 /* atomic64_fetch_xor_relaxed */ 953 #ifndef atomic64_fetch_xor_relaxed 954 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor 955 #define atomic64_fetch_xor_acquire atomic64_fetch_xor 956 #define atomic64_fetch_xor_release atomic64_fetch_xor 957 958 #else /* atomic64_fetch_xor_relaxed */ 959 960 #ifndef atomic64_fetch_xor_acquire 961 #define atomic64_fetch_xor_acquire(...) \ 962 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__) 963 #endif 964 965 #ifndef atomic64_fetch_xor_release 966 #define atomic64_fetch_xor_release(...) \ 967 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__) 968 #endif 969 970 #ifndef atomic64_fetch_xor 971 #define atomic64_fetch_xor(...) \ 972 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__) 973 #endif 974 #endif /* atomic64_fetch_xor_relaxed */ 975 976 977 /* atomic64_xchg_relaxed */ 978 #ifndef atomic64_xchg_relaxed 979 #define atomic64_xchg_relaxed atomic64_xchg 980 #define atomic64_xchg_acquire atomic64_xchg 981 #define atomic64_xchg_release atomic64_xchg 982 983 #else /* atomic64_xchg_relaxed */ 984 985 #ifndef atomic64_xchg_acquire 986 #define atomic64_xchg_acquire(...) \ 987 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__) 988 #endif 989 990 #ifndef atomic64_xchg_release 991 #define atomic64_xchg_release(...) \ 992 __atomic_op_release(atomic64_xchg, __VA_ARGS__) 993 #endif 994 995 #ifndef atomic64_xchg 996 #define atomic64_xchg(...) \ 997 __atomic_op_fence(atomic64_xchg, __VA_ARGS__) 998 #endif 999 #endif /* atomic64_xchg_relaxed */ 1000 1001 /* atomic64_cmpxchg_relaxed */ 1002 #ifndef atomic64_cmpxchg_relaxed 1003 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg 1004 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 1005 #define atomic64_cmpxchg_release atomic64_cmpxchg 1006 1007 #else /* atomic64_cmpxchg_relaxed */ 1008 1009 #ifndef atomic64_cmpxchg_acquire 1010 #define atomic64_cmpxchg_acquire(...) \ 1011 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__) 1012 #endif 1013 1014 #ifndef atomic64_cmpxchg_release 1015 #define atomic64_cmpxchg_release(...) \ 1016 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__) 1017 #endif 1018 1019 #ifndef atomic64_cmpxchg 1020 #define atomic64_cmpxchg(...) \ 1021 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__) 1022 #endif 1023 #endif /* atomic64_cmpxchg_relaxed */ 1024 1025 #ifndef atomic64_try_cmpxchg 1026 1027 #define __atomic64_try_cmpxchg(type, _p, _po, _n) \ 1028 ({ \ 1029 typeof(_po) __po = (_po); \ 1030 typeof(*(_po)) __r, __o = *__po; \ 1031 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \ 1032 if (unlikely(__r != __o)) \ 1033 *__po = __r; \ 1034 likely(__r == __o); \ 1035 }) 1036 1037 #define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n) 1038 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n) 1039 #define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n) 1040 #define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n) 1041 1042 #else /* atomic64_try_cmpxchg */ 1043 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg 1044 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg 1045 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg 1046 #endif /* atomic64_try_cmpxchg */ 1047 1048 #ifndef atomic64_andnot 1049 static inline void atomic64_andnot(long long i, atomic64_t *v) 1050 { 1051 atomic64_and(~i, v); 1052 } 1053 1054 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v) 1055 { 1056 return atomic64_fetch_and(~i, v); 1057 } 1058 1059 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v) 1060 { 1061 return atomic64_fetch_and_relaxed(~i, v); 1062 } 1063 1064 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v) 1065 { 1066 return atomic64_fetch_and_acquire(~i, v); 1067 } 1068 1069 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v) 1070 { 1071 return atomic64_fetch_and_release(~i, v); 1072 } 1073 #endif 1074 1075 #include <asm-generic/atomic-long.h> 1076 1077 #endif /* _LINUX_ATOMIC_H */ 1078