1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Atomic operations usable in machine independent code */ 3 #ifndef _LINUX_ATOMIC_H 4 #define _LINUX_ATOMIC_H 5 #include <linux/types.h> 6 7 #include <asm/atomic.h> 8 #include <asm/barrier.h> 9 10 /* 11 * Relaxed variants of xchg, cmpxchg and some atomic operations. 12 * 13 * We support four variants: 14 * 15 * - Fully ordered: The default implementation, no suffix required. 16 * - Acquire: Provides ACQUIRE semantics, _acquire suffix. 17 * - Release: Provides RELEASE semantics, _release suffix. 18 * - Relaxed: No ordering guarantees, _relaxed suffix. 19 * 20 * For compound atomics performing both a load and a store, ACQUIRE 21 * semantics apply only to the load and RELEASE semantics only to the 22 * store portion of the operation. Note that a failed cmpxchg_acquire 23 * does -not- imply any memory ordering constraints. 24 * 25 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. 26 */ 27 28 #ifndef atomic_read_acquire 29 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) 30 #endif 31 32 #ifndef atomic_set_release 33 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) 34 #endif 35 36 /* 37 * The idea here is to build acquire/release variants by adding explicit 38 * barriers on top of the relaxed variant. In the case where the relaxed 39 * variant is already fully ordered, no additional barriers are needed. 40 * 41 * If an architecture overrides __atomic_acquire_fence() it will probably 42 * want to define smp_mb__after_spinlock(). 43 */ 44 #ifndef __atomic_acquire_fence 45 #define __atomic_acquire_fence smp_mb__after_atomic 46 #endif 47 48 #ifndef __atomic_release_fence 49 #define __atomic_release_fence smp_mb__before_atomic 50 #endif 51 52 #ifndef __atomic_pre_full_fence 53 #define __atomic_pre_full_fence smp_mb__before_atomic 54 #endif 55 56 #ifndef __atomic_post_full_fence 57 #define __atomic_post_full_fence smp_mb__after_atomic 58 #endif 59 60 #define __atomic_op_acquire(op, args...) \ 61 ({ \ 62 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 63 __atomic_acquire_fence(); \ 64 __ret; \ 65 }) 66 67 #define __atomic_op_release(op, args...) \ 68 ({ \ 69 __atomic_release_fence(); \ 70 op##_relaxed(args); \ 71 }) 72 73 #define __atomic_op_fence(op, args...) \ 74 ({ \ 75 typeof(op##_relaxed(args)) __ret; \ 76 __atomic_pre_full_fence(); \ 77 __ret = op##_relaxed(args); \ 78 __atomic_post_full_fence(); \ 79 __ret; \ 80 }) 81 82 /* atomic_add_return_relaxed */ 83 #ifndef atomic_add_return_relaxed 84 #define atomic_add_return_relaxed atomic_add_return 85 #define atomic_add_return_acquire atomic_add_return 86 #define atomic_add_return_release atomic_add_return 87 88 #else /* atomic_add_return_relaxed */ 89 90 #ifndef atomic_add_return_acquire 91 #define atomic_add_return_acquire(...) \ 92 __atomic_op_acquire(atomic_add_return, __VA_ARGS__) 93 #endif 94 95 #ifndef atomic_add_return_release 96 #define atomic_add_return_release(...) \ 97 __atomic_op_release(atomic_add_return, __VA_ARGS__) 98 #endif 99 100 #ifndef atomic_add_return 101 #define atomic_add_return(...) \ 102 __atomic_op_fence(atomic_add_return, __VA_ARGS__) 103 #endif 104 #endif /* atomic_add_return_relaxed */ 105 106 #ifndef atomic_inc 107 #define atomic_inc(v) atomic_add(1, (v)) 108 #endif 109 110 /* atomic_inc_return_relaxed */ 111 #ifndef atomic_inc_return_relaxed 112 113 #ifndef atomic_inc_return 114 #define atomic_inc_return(v) atomic_add_return(1, (v)) 115 #define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v)) 116 #define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v)) 117 #define atomic_inc_return_release(v) atomic_add_return_release(1, (v)) 118 #else /* atomic_inc_return */ 119 #define atomic_inc_return_relaxed atomic_inc_return 120 #define atomic_inc_return_acquire atomic_inc_return 121 #define atomic_inc_return_release atomic_inc_return 122 #endif /* atomic_inc_return */ 123 124 #else /* atomic_inc_return_relaxed */ 125 126 #ifndef atomic_inc_return_acquire 127 #define atomic_inc_return_acquire(...) \ 128 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__) 129 #endif 130 131 #ifndef atomic_inc_return_release 132 #define atomic_inc_return_release(...) \ 133 __atomic_op_release(atomic_inc_return, __VA_ARGS__) 134 #endif 135 136 #ifndef atomic_inc_return 137 #define atomic_inc_return(...) \ 138 __atomic_op_fence(atomic_inc_return, __VA_ARGS__) 139 #endif 140 #endif /* atomic_inc_return_relaxed */ 141 142 /* atomic_sub_return_relaxed */ 143 #ifndef atomic_sub_return_relaxed 144 #define atomic_sub_return_relaxed atomic_sub_return 145 #define atomic_sub_return_acquire atomic_sub_return 146 #define atomic_sub_return_release atomic_sub_return 147 148 #else /* atomic_sub_return_relaxed */ 149 150 #ifndef atomic_sub_return_acquire 151 #define atomic_sub_return_acquire(...) \ 152 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__) 153 #endif 154 155 #ifndef atomic_sub_return_release 156 #define atomic_sub_return_release(...) \ 157 __atomic_op_release(atomic_sub_return, __VA_ARGS__) 158 #endif 159 160 #ifndef atomic_sub_return 161 #define atomic_sub_return(...) \ 162 __atomic_op_fence(atomic_sub_return, __VA_ARGS__) 163 #endif 164 #endif /* atomic_sub_return_relaxed */ 165 166 #ifndef atomic_dec 167 #define atomic_dec(v) atomic_sub(1, (v)) 168 #endif 169 170 /* atomic_dec_return_relaxed */ 171 #ifndef atomic_dec_return_relaxed 172 173 #ifndef atomic_dec_return 174 #define atomic_dec_return(v) atomic_sub_return(1, (v)) 175 #define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v)) 176 #define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v)) 177 #define atomic_dec_return_release(v) atomic_sub_return_release(1, (v)) 178 #else /* atomic_dec_return */ 179 #define atomic_dec_return_relaxed atomic_dec_return 180 #define atomic_dec_return_acquire atomic_dec_return 181 #define atomic_dec_return_release atomic_dec_return 182 #endif /* atomic_dec_return */ 183 184 #else /* atomic_dec_return_relaxed */ 185 186 #ifndef atomic_dec_return_acquire 187 #define atomic_dec_return_acquire(...) \ 188 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__) 189 #endif 190 191 #ifndef atomic_dec_return_release 192 #define atomic_dec_return_release(...) \ 193 __atomic_op_release(atomic_dec_return, __VA_ARGS__) 194 #endif 195 196 #ifndef atomic_dec_return 197 #define atomic_dec_return(...) \ 198 __atomic_op_fence(atomic_dec_return, __VA_ARGS__) 199 #endif 200 #endif /* atomic_dec_return_relaxed */ 201 202 203 /* atomic_fetch_add_relaxed */ 204 #ifndef atomic_fetch_add_relaxed 205 #define atomic_fetch_add_relaxed atomic_fetch_add 206 #define atomic_fetch_add_acquire atomic_fetch_add 207 #define atomic_fetch_add_release atomic_fetch_add 208 209 #else /* atomic_fetch_add_relaxed */ 210 211 #ifndef atomic_fetch_add_acquire 212 #define atomic_fetch_add_acquire(...) \ 213 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__) 214 #endif 215 216 #ifndef atomic_fetch_add_release 217 #define atomic_fetch_add_release(...) \ 218 __atomic_op_release(atomic_fetch_add, __VA_ARGS__) 219 #endif 220 221 #ifndef atomic_fetch_add 222 #define atomic_fetch_add(...) \ 223 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__) 224 #endif 225 #endif /* atomic_fetch_add_relaxed */ 226 227 /* atomic_fetch_inc_relaxed */ 228 #ifndef atomic_fetch_inc_relaxed 229 230 #ifndef atomic_fetch_inc 231 #define atomic_fetch_inc(v) atomic_fetch_add(1, (v)) 232 #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v)) 233 #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v)) 234 #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v)) 235 #else /* atomic_fetch_inc */ 236 #define atomic_fetch_inc_relaxed atomic_fetch_inc 237 #define atomic_fetch_inc_acquire atomic_fetch_inc 238 #define atomic_fetch_inc_release atomic_fetch_inc 239 #endif /* atomic_fetch_inc */ 240 241 #else /* atomic_fetch_inc_relaxed */ 242 243 #ifndef atomic_fetch_inc_acquire 244 #define atomic_fetch_inc_acquire(...) \ 245 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__) 246 #endif 247 248 #ifndef atomic_fetch_inc_release 249 #define atomic_fetch_inc_release(...) \ 250 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__) 251 #endif 252 253 #ifndef atomic_fetch_inc 254 #define atomic_fetch_inc(...) \ 255 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__) 256 #endif 257 #endif /* atomic_fetch_inc_relaxed */ 258 259 /* atomic_fetch_sub_relaxed */ 260 #ifndef atomic_fetch_sub_relaxed 261 #define atomic_fetch_sub_relaxed atomic_fetch_sub 262 #define atomic_fetch_sub_acquire atomic_fetch_sub 263 #define atomic_fetch_sub_release atomic_fetch_sub 264 265 #else /* atomic_fetch_sub_relaxed */ 266 267 #ifndef atomic_fetch_sub_acquire 268 #define atomic_fetch_sub_acquire(...) \ 269 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__) 270 #endif 271 272 #ifndef atomic_fetch_sub_release 273 #define atomic_fetch_sub_release(...) \ 274 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__) 275 #endif 276 277 #ifndef atomic_fetch_sub 278 #define atomic_fetch_sub(...) \ 279 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__) 280 #endif 281 #endif /* atomic_fetch_sub_relaxed */ 282 283 /* atomic_fetch_dec_relaxed */ 284 #ifndef atomic_fetch_dec_relaxed 285 286 #ifndef atomic_fetch_dec 287 #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v)) 288 #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v)) 289 #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v)) 290 #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v)) 291 #else /* atomic_fetch_dec */ 292 #define atomic_fetch_dec_relaxed atomic_fetch_dec 293 #define atomic_fetch_dec_acquire atomic_fetch_dec 294 #define atomic_fetch_dec_release atomic_fetch_dec 295 #endif /* atomic_fetch_dec */ 296 297 #else /* atomic_fetch_dec_relaxed */ 298 299 #ifndef atomic_fetch_dec_acquire 300 #define atomic_fetch_dec_acquire(...) \ 301 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__) 302 #endif 303 304 #ifndef atomic_fetch_dec_release 305 #define atomic_fetch_dec_release(...) \ 306 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__) 307 #endif 308 309 #ifndef atomic_fetch_dec 310 #define atomic_fetch_dec(...) \ 311 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__) 312 #endif 313 #endif /* atomic_fetch_dec_relaxed */ 314 315 /* atomic_fetch_or_relaxed */ 316 #ifndef atomic_fetch_or_relaxed 317 #define atomic_fetch_or_relaxed atomic_fetch_or 318 #define atomic_fetch_or_acquire atomic_fetch_or 319 #define atomic_fetch_or_release atomic_fetch_or 320 321 #else /* atomic_fetch_or_relaxed */ 322 323 #ifndef atomic_fetch_or_acquire 324 #define atomic_fetch_or_acquire(...) \ 325 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__) 326 #endif 327 328 #ifndef atomic_fetch_or_release 329 #define atomic_fetch_or_release(...) \ 330 __atomic_op_release(atomic_fetch_or, __VA_ARGS__) 331 #endif 332 333 #ifndef atomic_fetch_or 334 #define atomic_fetch_or(...) \ 335 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__) 336 #endif 337 #endif /* atomic_fetch_or_relaxed */ 338 339 /* atomic_fetch_and_relaxed */ 340 #ifndef atomic_fetch_and_relaxed 341 #define atomic_fetch_and_relaxed atomic_fetch_and 342 #define atomic_fetch_and_acquire atomic_fetch_and 343 #define atomic_fetch_and_release atomic_fetch_and 344 345 #else /* atomic_fetch_and_relaxed */ 346 347 #ifndef atomic_fetch_and_acquire 348 #define atomic_fetch_and_acquire(...) \ 349 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__) 350 #endif 351 352 #ifndef atomic_fetch_and_release 353 #define atomic_fetch_and_release(...) \ 354 __atomic_op_release(atomic_fetch_and, __VA_ARGS__) 355 #endif 356 357 #ifndef atomic_fetch_and 358 #define atomic_fetch_and(...) \ 359 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__) 360 #endif 361 #endif /* atomic_fetch_and_relaxed */ 362 363 #ifndef atomic_andnot 364 #define atomic_andnot(i, v) atomic_and(~(int)(i), (v)) 365 #endif 366 367 #ifndef atomic_fetch_andnot_relaxed 368 369 #ifndef atomic_fetch_andnot 370 #define atomic_fetch_andnot(i, v) atomic_fetch_and(~(int)(i), (v)) 371 #define atomic_fetch_andnot_relaxed(i, v) atomic_fetch_and_relaxed(~(int)(i), (v)) 372 #define atomic_fetch_andnot_acquire(i, v) atomic_fetch_and_acquire(~(int)(i), (v)) 373 #define atomic_fetch_andnot_release(i, v) atomic_fetch_and_release(~(int)(i), (v)) 374 #else /* atomic_fetch_andnot */ 375 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot 376 #define atomic_fetch_andnot_acquire atomic_fetch_andnot 377 #define atomic_fetch_andnot_release atomic_fetch_andnot 378 #endif /* atomic_fetch_andnot */ 379 380 #else /* atomic_fetch_andnot_relaxed */ 381 382 #ifndef atomic_fetch_andnot_acquire 383 #define atomic_fetch_andnot_acquire(...) \ 384 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__) 385 #endif 386 387 #ifndef atomic_fetch_andnot_release 388 #define atomic_fetch_andnot_release(...) \ 389 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__) 390 #endif 391 392 #ifndef atomic_fetch_andnot 393 #define atomic_fetch_andnot(...) \ 394 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__) 395 #endif 396 #endif /* atomic_fetch_andnot_relaxed */ 397 398 /* atomic_fetch_xor_relaxed */ 399 #ifndef atomic_fetch_xor_relaxed 400 #define atomic_fetch_xor_relaxed atomic_fetch_xor 401 #define atomic_fetch_xor_acquire atomic_fetch_xor 402 #define atomic_fetch_xor_release atomic_fetch_xor 403 404 #else /* atomic_fetch_xor_relaxed */ 405 406 #ifndef atomic_fetch_xor_acquire 407 #define atomic_fetch_xor_acquire(...) \ 408 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__) 409 #endif 410 411 #ifndef atomic_fetch_xor_release 412 #define atomic_fetch_xor_release(...) \ 413 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__) 414 #endif 415 416 #ifndef atomic_fetch_xor 417 #define atomic_fetch_xor(...) \ 418 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__) 419 #endif 420 #endif /* atomic_fetch_xor_relaxed */ 421 422 423 /* atomic_xchg_relaxed */ 424 #ifndef atomic_xchg_relaxed 425 #define atomic_xchg_relaxed atomic_xchg 426 #define atomic_xchg_acquire atomic_xchg 427 #define atomic_xchg_release atomic_xchg 428 429 #else /* atomic_xchg_relaxed */ 430 431 #ifndef atomic_xchg_acquire 432 #define atomic_xchg_acquire(...) \ 433 __atomic_op_acquire(atomic_xchg, __VA_ARGS__) 434 #endif 435 436 #ifndef atomic_xchg_release 437 #define atomic_xchg_release(...) \ 438 __atomic_op_release(atomic_xchg, __VA_ARGS__) 439 #endif 440 441 #ifndef atomic_xchg 442 #define atomic_xchg(...) \ 443 __atomic_op_fence(atomic_xchg, __VA_ARGS__) 444 #endif 445 #endif /* atomic_xchg_relaxed */ 446 447 /* atomic_cmpxchg_relaxed */ 448 #ifndef atomic_cmpxchg_relaxed 449 #define atomic_cmpxchg_relaxed atomic_cmpxchg 450 #define atomic_cmpxchg_acquire atomic_cmpxchg 451 #define atomic_cmpxchg_release atomic_cmpxchg 452 453 #else /* atomic_cmpxchg_relaxed */ 454 455 #ifndef atomic_cmpxchg_acquire 456 #define atomic_cmpxchg_acquire(...) \ 457 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__) 458 #endif 459 460 #ifndef atomic_cmpxchg_release 461 #define atomic_cmpxchg_release(...) \ 462 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__) 463 #endif 464 465 #ifndef atomic_cmpxchg 466 #define atomic_cmpxchg(...) \ 467 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__) 468 #endif 469 #endif /* atomic_cmpxchg_relaxed */ 470 471 #ifndef atomic_try_cmpxchg 472 473 #define __atomic_try_cmpxchg(type, _p, _po, _n) \ 474 ({ \ 475 typeof(_po) __po = (_po); \ 476 typeof(*(_po)) __r, __o = *__po; \ 477 __r = atomic_cmpxchg##type((_p), __o, (_n)); \ 478 if (unlikely(__r != __o)) \ 479 *__po = __r; \ 480 likely(__r == __o); \ 481 }) 482 483 #define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n) 484 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n) 485 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n) 486 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n) 487 488 #else /* atomic_try_cmpxchg */ 489 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg 490 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg 491 #define atomic_try_cmpxchg_release atomic_try_cmpxchg 492 #endif /* atomic_try_cmpxchg */ 493 494 /* cmpxchg_relaxed */ 495 #ifndef cmpxchg_relaxed 496 #define cmpxchg_relaxed cmpxchg 497 #define cmpxchg_acquire cmpxchg 498 #define cmpxchg_release cmpxchg 499 500 #else /* cmpxchg_relaxed */ 501 502 #ifndef cmpxchg_acquire 503 #define cmpxchg_acquire(...) \ 504 __atomic_op_acquire(cmpxchg, __VA_ARGS__) 505 #endif 506 507 #ifndef cmpxchg_release 508 #define cmpxchg_release(...) \ 509 __atomic_op_release(cmpxchg, __VA_ARGS__) 510 #endif 511 512 #ifndef cmpxchg 513 #define cmpxchg(...) \ 514 __atomic_op_fence(cmpxchg, __VA_ARGS__) 515 #endif 516 #endif /* cmpxchg_relaxed */ 517 518 /* cmpxchg64_relaxed */ 519 #ifndef cmpxchg64_relaxed 520 #define cmpxchg64_relaxed cmpxchg64 521 #define cmpxchg64_acquire cmpxchg64 522 #define cmpxchg64_release cmpxchg64 523 524 #else /* cmpxchg64_relaxed */ 525 526 #ifndef cmpxchg64_acquire 527 #define cmpxchg64_acquire(...) \ 528 __atomic_op_acquire(cmpxchg64, __VA_ARGS__) 529 #endif 530 531 #ifndef cmpxchg64_release 532 #define cmpxchg64_release(...) \ 533 __atomic_op_release(cmpxchg64, __VA_ARGS__) 534 #endif 535 536 #ifndef cmpxchg64 537 #define cmpxchg64(...) \ 538 __atomic_op_fence(cmpxchg64, __VA_ARGS__) 539 #endif 540 #endif /* cmpxchg64_relaxed */ 541 542 /* xchg_relaxed */ 543 #ifndef xchg_relaxed 544 #define xchg_relaxed xchg 545 #define xchg_acquire xchg 546 #define xchg_release xchg 547 548 #else /* xchg_relaxed */ 549 550 #ifndef xchg_acquire 551 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__) 552 #endif 553 554 #ifndef xchg_release 555 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__) 556 #endif 557 558 #ifndef xchg 559 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__) 560 #endif 561 #endif /* xchg_relaxed */ 562 563 /** 564 * atomic_fetch_add_unless - add unless the number is already a given value 565 * @v: pointer of type atomic_t 566 * @a: the amount to add to v... 567 * @u: ...unless v is equal to u. 568 * 569 * Atomically adds @a to @v, if @v was not already @u. 570 * Returns the original value of @v. 571 */ 572 #ifndef atomic_fetch_add_unless 573 static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) 574 { 575 int c = atomic_read(v); 576 577 do { 578 if (unlikely(c == u)) 579 break; 580 } while (!atomic_try_cmpxchg(v, &c, c + a)); 581 582 return c; 583 } 584 #endif 585 586 /** 587 * atomic_add_unless - add unless the number is already a given value 588 * @v: pointer of type atomic_t 589 * @a: the amount to add to v... 590 * @u: ...unless v is equal to u. 591 * 592 * Atomically adds @a to @v, if @v was not already @u. 593 * Returns true if the addition was done. 594 */ 595 static inline bool atomic_add_unless(atomic_t *v, int a, int u) 596 { 597 return atomic_fetch_add_unless(v, a, u) != u; 598 } 599 600 /** 601 * atomic_inc_not_zero - increment unless the number is zero 602 * @v: pointer of type atomic_t 603 * 604 * Atomically increments @v by 1, if @v is non-zero. 605 * Returns true if the increment was done. 606 */ 607 #ifndef atomic_inc_not_zero 608 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 609 #endif 610 611 /** 612 * atomic_inc_and_test - increment and test 613 * @v: pointer of type atomic_t 614 * 615 * Atomically increments @v by 1 616 * and returns true if the result is zero, or false for all 617 * other cases. 618 */ 619 #ifndef atomic_inc_and_test 620 static inline bool atomic_inc_and_test(atomic_t *v) 621 { 622 return atomic_inc_return(v) == 0; 623 } 624 #endif 625 626 /** 627 * atomic_dec_and_test - decrement and test 628 * @v: pointer of type atomic_t 629 * 630 * Atomically decrements @v by 1 and 631 * returns true if the result is 0, or false for all other 632 * cases. 633 */ 634 #ifndef atomic_dec_and_test 635 static inline bool atomic_dec_and_test(atomic_t *v) 636 { 637 return atomic_dec_return(v) == 0; 638 } 639 #endif 640 641 /** 642 * atomic_sub_and_test - subtract value from variable and test result 643 * @i: integer value to subtract 644 * @v: pointer of type atomic_t 645 * 646 * Atomically subtracts @i from @v and returns 647 * true if the result is zero, or false for all 648 * other cases. 649 */ 650 #ifndef atomic_sub_and_test 651 static inline bool atomic_sub_and_test(int i, atomic_t *v) 652 { 653 return atomic_sub_return(i, v) == 0; 654 } 655 #endif 656 657 /** 658 * atomic_add_negative - add and test if negative 659 * @i: integer value to add 660 * @v: pointer of type atomic_t 661 * 662 * Atomically adds @i to @v and returns true 663 * if the result is negative, or false when 664 * result is greater than or equal to zero. 665 */ 666 #ifndef atomic_add_negative 667 static inline bool atomic_add_negative(int i, atomic_t *v) 668 { 669 return atomic_add_return(i, v) < 0; 670 } 671 #endif 672 673 #ifndef atomic_inc_unless_negative 674 static inline bool atomic_inc_unless_negative(atomic_t *v) 675 { 676 int c = atomic_read(v); 677 678 do { 679 if (unlikely(c < 0)) 680 return false; 681 } while (!atomic_try_cmpxchg(v, &c, c + 1)); 682 683 return true; 684 } 685 #endif 686 687 #ifndef atomic_dec_unless_positive 688 static inline bool atomic_dec_unless_positive(atomic_t *v) 689 { 690 int c = atomic_read(v); 691 692 do { 693 if (unlikely(c > 0)) 694 return false; 695 } while (!atomic_try_cmpxchg(v, &c, c - 1)); 696 697 return true; 698 } 699 #endif 700 701 /* 702 * atomic_dec_if_positive - decrement by 1 if old value positive 703 * @v: pointer of type atomic_t 704 * 705 * The function returns the old value of *v minus 1, even if 706 * the atomic variable, v, was not decremented. 707 */ 708 #ifndef atomic_dec_if_positive 709 static inline int atomic_dec_if_positive(atomic_t *v) 710 { 711 int dec, c = atomic_read(v); 712 713 do { 714 dec = c - 1; 715 if (unlikely(dec < 0)) 716 break; 717 } while (!atomic_try_cmpxchg(v, &c, dec)); 718 719 return dec; 720 } 721 #endif 722 723 #define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) 724 #define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) 725 726 #ifdef CONFIG_GENERIC_ATOMIC64 727 #include <asm-generic/atomic64.h> 728 #endif 729 730 #ifndef atomic64_read_acquire 731 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter) 732 #endif 733 734 #ifndef atomic64_set_release 735 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i)) 736 #endif 737 738 /* atomic64_add_return_relaxed */ 739 #ifndef atomic64_add_return_relaxed 740 #define atomic64_add_return_relaxed atomic64_add_return 741 #define atomic64_add_return_acquire atomic64_add_return 742 #define atomic64_add_return_release atomic64_add_return 743 744 #else /* atomic64_add_return_relaxed */ 745 746 #ifndef atomic64_add_return_acquire 747 #define atomic64_add_return_acquire(...) \ 748 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__) 749 #endif 750 751 #ifndef atomic64_add_return_release 752 #define atomic64_add_return_release(...) \ 753 __atomic_op_release(atomic64_add_return, __VA_ARGS__) 754 #endif 755 756 #ifndef atomic64_add_return 757 #define atomic64_add_return(...) \ 758 __atomic_op_fence(atomic64_add_return, __VA_ARGS__) 759 #endif 760 #endif /* atomic64_add_return_relaxed */ 761 762 #ifndef atomic64_inc 763 #define atomic64_inc(v) atomic64_add(1, (v)) 764 #endif 765 766 /* atomic64_inc_return_relaxed */ 767 #ifndef atomic64_inc_return_relaxed 768 769 #ifndef atomic64_inc_return 770 #define atomic64_inc_return(v) atomic64_add_return(1, (v)) 771 #define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v)) 772 #define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v)) 773 #define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v)) 774 #else /* atomic64_inc_return */ 775 #define atomic64_inc_return_relaxed atomic64_inc_return 776 #define atomic64_inc_return_acquire atomic64_inc_return 777 #define atomic64_inc_return_release atomic64_inc_return 778 #endif /* atomic64_inc_return */ 779 780 #else /* atomic64_inc_return_relaxed */ 781 782 #ifndef atomic64_inc_return_acquire 783 #define atomic64_inc_return_acquire(...) \ 784 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__) 785 #endif 786 787 #ifndef atomic64_inc_return_release 788 #define atomic64_inc_return_release(...) \ 789 __atomic_op_release(atomic64_inc_return, __VA_ARGS__) 790 #endif 791 792 #ifndef atomic64_inc_return 793 #define atomic64_inc_return(...) \ 794 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__) 795 #endif 796 #endif /* atomic64_inc_return_relaxed */ 797 798 799 /* atomic64_sub_return_relaxed */ 800 #ifndef atomic64_sub_return_relaxed 801 #define atomic64_sub_return_relaxed atomic64_sub_return 802 #define atomic64_sub_return_acquire atomic64_sub_return 803 #define atomic64_sub_return_release atomic64_sub_return 804 805 #else /* atomic64_sub_return_relaxed */ 806 807 #ifndef atomic64_sub_return_acquire 808 #define atomic64_sub_return_acquire(...) \ 809 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__) 810 #endif 811 812 #ifndef atomic64_sub_return_release 813 #define atomic64_sub_return_release(...) \ 814 __atomic_op_release(atomic64_sub_return, __VA_ARGS__) 815 #endif 816 817 #ifndef atomic64_sub_return 818 #define atomic64_sub_return(...) \ 819 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__) 820 #endif 821 #endif /* atomic64_sub_return_relaxed */ 822 823 #ifndef atomic64_dec 824 #define atomic64_dec(v) atomic64_sub(1, (v)) 825 #endif 826 827 /* atomic64_dec_return_relaxed */ 828 #ifndef atomic64_dec_return_relaxed 829 830 #ifndef atomic64_dec_return 831 #define atomic64_dec_return(v) atomic64_sub_return(1, (v)) 832 #define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v)) 833 #define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v)) 834 #define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v)) 835 #else /* atomic64_dec_return */ 836 #define atomic64_dec_return_relaxed atomic64_dec_return 837 #define atomic64_dec_return_acquire atomic64_dec_return 838 #define atomic64_dec_return_release atomic64_dec_return 839 #endif /* atomic64_dec_return */ 840 841 #else /* atomic64_dec_return_relaxed */ 842 843 #ifndef atomic64_dec_return_acquire 844 #define atomic64_dec_return_acquire(...) \ 845 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__) 846 #endif 847 848 #ifndef atomic64_dec_return_release 849 #define atomic64_dec_return_release(...) \ 850 __atomic_op_release(atomic64_dec_return, __VA_ARGS__) 851 #endif 852 853 #ifndef atomic64_dec_return 854 #define atomic64_dec_return(...) \ 855 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__) 856 #endif 857 #endif /* atomic64_dec_return_relaxed */ 858 859 860 /* atomic64_fetch_add_relaxed */ 861 #ifndef atomic64_fetch_add_relaxed 862 #define atomic64_fetch_add_relaxed atomic64_fetch_add 863 #define atomic64_fetch_add_acquire atomic64_fetch_add 864 #define atomic64_fetch_add_release atomic64_fetch_add 865 866 #else /* atomic64_fetch_add_relaxed */ 867 868 #ifndef atomic64_fetch_add_acquire 869 #define atomic64_fetch_add_acquire(...) \ 870 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__) 871 #endif 872 873 #ifndef atomic64_fetch_add_release 874 #define atomic64_fetch_add_release(...) \ 875 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__) 876 #endif 877 878 #ifndef atomic64_fetch_add 879 #define atomic64_fetch_add(...) \ 880 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__) 881 #endif 882 #endif /* atomic64_fetch_add_relaxed */ 883 884 /* atomic64_fetch_inc_relaxed */ 885 #ifndef atomic64_fetch_inc_relaxed 886 887 #ifndef atomic64_fetch_inc 888 #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v)) 889 #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v)) 890 #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v)) 891 #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v)) 892 #else /* atomic64_fetch_inc */ 893 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc 894 #define atomic64_fetch_inc_acquire atomic64_fetch_inc 895 #define atomic64_fetch_inc_release atomic64_fetch_inc 896 #endif /* atomic64_fetch_inc */ 897 898 #else /* atomic64_fetch_inc_relaxed */ 899 900 #ifndef atomic64_fetch_inc_acquire 901 #define atomic64_fetch_inc_acquire(...) \ 902 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__) 903 #endif 904 905 #ifndef atomic64_fetch_inc_release 906 #define atomic64_fetch_inc_release(...) \ 907 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__) 908 #endif 909 910 #ifndef atomic64_fetch_inc 911 #define atomic64_fetch_inc(...) \ 912 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__) 913 #endif 914 #endif /* atomic64_fetch_inc_relaxed */ 915 916 /* atomic64_fetch_sub_relaxed */ 917 #ifndef atomic64_fetch_sub_relaxed 918 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub 919 #define atomic64_fetch_sub_acquire atomic64_fetch_sub 920 #define atomic64_fetch_sub_release atomic64_fetch_sub 921 922 #else /* atomic64_fetch_sub_relaxed */ 923 924 #ifndef atomic64_fetch_sub_acquire 925 #define atomic64_fetch_sub_acquire(...) \ 926 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__) 927 #endif 928 929 #ifndef atomic64_fetch_sub_release 930 #define atomic64_fetch_sub_release(...) \ 931 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__) 932 #endif 933 934 #ifndef atomic64_fetch_sub 935 #define atomic64_fetch_sub(...) \ 936 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__) 937 #endif 938 #endif /* atomic64_fetch_sub_relaxed */ 939 940 /* atomic64_fetch_dec_relaxed */ 941 #ifndef atomic64_fetch_dec_relaxed 942 943 #ifndef atomic64_fetch_dec 944 #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v)) 945 #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v)) 946 #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v)) 947 #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v)) 948 #else /* atomic64_fetch_dec */ 949 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec 950 #define atomic64_fetch_dec_acquire atomic64_fetch_dec 951 #define atomic64_fetch_dec_release atomic64_fetch_dec 952 #endif /* atomic64_fetch_dec */ 953 954 #else /* atomic64_fetch_dec_relaxed */ 955 956 #ifndef atomic64_fetch_dec_acquire 957 #define atomic64_fetch_dec_acquire(...) \ 958 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__) 959 #endif 960 961 #ifndef atomic64_fetch_dec_release 962 #define atomic64_fetch_dec_release(...) \ 963 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__) 964 #endif 965 966 #ifndef atomic64_fetch_dec 967 #define atomic64_fetch_dec(...) \ 968 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__) 969 #endif 970 #endif /* atomic64_fetch_dec_relaxed */ 971 972 /* atomic64_fetch_or_relaxed */ 973 #ifndef atomic64_fetch_or_relaxed 974 #define atomic64_fetch_or_relaxed atomic64_fetch_or 975 #define atomic64_fetch_or_acquire atomic64_fetch_or 976 #define atomic64_fetch_or_release atomic64_fetch_or 977 978 #else /* atomic64_fetch_or_relaxed */ 979 980 #ifndef atomic64_fetch_or_acquire 981 #define atomic64_fetch_or_acquire(...) \ 982 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__) 983 #endif 984 985 #ifndef atomic64_fetch_or_release 986 #define atomic64_fetch_or_release(...) \ 987 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__) 988 #endif 989 990 #ifndef atomic64_fetch_or 991 #define atomic64_fetch_or(...) \ 992 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__) 993 #endif 994 #endif /* atomic64_fetch_or_relaxed */ 995 996 /* atomic64_fetch_and_relaxed */ 997 #ifndef atomic64_fetch_and_relaxed 998 #define atomic64_fetch_and_relaxed atomic64_fetch_and 999 #define atomic64_fetch_and_acquire atomic64_fetch_and 1000 #define atomic64_fetch_and_release atomic64_fetch_and 1001 1002 #else /* atomic64_fetch_and_relaxed */ 1003 1004 #ifndef atomic64_fetch_and_acquire 1005 #define atomic64_fetch_and_acquire(...) \ 1006 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__) 1007 #endif 1008 1009 #ifndef atomic64_fetch_and_release 1010 #define atomic64_fetch_and_release(...) \ 1011 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__) 1012 #endif 1013 1014 #ifndef atomic64_fetch_and 1015 #define atomic64_fetch_and(...) \ 1016 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__) 1017 #endif 1018 #endif /* atomic64_fetch_and_relaxed */ 1019 1020 #ifndef atomic64_andnot 1021 #define atomic64_andnot(i, v) atomic64_and(~(long long)(i), (v)) 1022 #endif 1023 1024 #ifndef atomic64_fetch_andnot_relaxed 1025 1026 #ifndef atomic64_fetch_andnot 1027 #define atomic64_fetch_andnot(i, v) atomic64_fetch_and(~(long long)(i), (v)) 1028 #define atomic64_fetch_andnot_relaxed(i, v) atomic64_fetch_and_relaxed(~(long long)(i), (v)) 1029 #define atomic64_fetch_andnot_acquire(i, v) atomic64_fetch_and_acquire(~(long long)(i), (v)) 1030 #define atomic64_fetch_andnot_release(i, v) atomic64_fetch_and_release(~(long long)(i), (v)) 1031 #else /* atomic64_fetch_andnot */ 1032 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot 1033 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot 1034 #define atomic64_fetch_andnot_release atomic64_fetch_andnot 1035 #endif /* atomic64_fetch_andnot */ 1036 1037 #else /* atomic64_fetch_andnot_relaxed */ 1038 1039 #ifndef atomic64_fetch_andnot_acquire 1040 #define atomic64_fetch_andnot_acquire(...) \ 1041 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__) 1042 #endif 1043 1044 #ifndef atomic64_fetch_andnot_release 1045 #define atomic64_fetch_andnot_release(...) \ 1046 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__) 1047 #endif 1048 1049 #ifndef atomic64_fetch_andnot 1050 #define atomic64_fetch_andnot(...) \ 1051 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__) 1052 #endif 1053 #endif /* atomic64_fetch_andnot_relaxed */ 1054 1055 /* atomic64_fetch_xor_relaxed */ 1056 #ifndef atomic64_fetch_xor_relaxed 1057 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor 1058 #define atomic64_fetch_xor_acquire atomic64_fetch_xor 1059 #define atomic64_fetch_xor_release atomic64_fetch_xor 1060 1061 #else /* atomic64_fetch_xor_relaxed */ 1062 1063 #ifndef atomic64_fetch_xor_acquire 1064 #define atomic64_fetch_xor_acquire(...) \ 1065 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__) 1066 #endif 1067 1068 #ifndef atomic64_fetch_xor_release 1069 #define atomic64_fetch_xor_release(...) \ 1070 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__) 1071 #endif 1072 1073 #ifndef atomic64_fetch_xor 1074 #define atomic64_fetch_xor(...) \ 1075 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__) 1076 #endif 1077 #endif /* atomic64_fetch_xor_relaxed */ 1078 1079 1080 /* atomic64_xchg_relaxed */ 1081 #ifndef atomic64_xchg_relaxed 1082 #define atomic64_xchg_relaxed atomic64_xchg 1083 #define atomic64_xchg_acquire atomic64_xchg 1084 #define atomic64_xchg_release atomic64_xchg 1085 1086 #else /* atomic64_xchg_relaxed */ 1087 1088 #ifndef atomic64_xchg_acquire 1089 #define atomic64_xchg_acquire(...) \ 1090 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__) 1091 #endif 1092 1093 #ifndef atomic64_xchg_release 1094 #define atomic64_xchg_release(...) \ 1095 __atomic_op_release(atomic64_xchg, __VA_ARGS__) 1096 #endif 1097 1098 #ifndef atomic64_xchg 1099 #define atomic64_xchg(...) \ 1100 __atomic_op_fence(atomic64_xchg, __VA_ARGS__) 1101 #endif 1102 #endif /* atomic64_xchg_relaxed */ 1103 1104 /* atomic64_cmpxchg_relaxed */ 1105 #ifndef atomic64_cmpxchg_relaxed 1106 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg 1107 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 1108 #define atomic64_cmpxchg_release atomic64_cmpxchg 1109 1110 #else /* atomic64_cmpxchg_relaxed */ 1111 1112 #ifndef atomic64_cmpxchg_acquire 1113 #define atomic64_cmpxchg_acquire(...) \ 1114 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__) 1115 #endif 1116 1117 #ifndef atomic64_cmpxchg_release 1118 #define atomic64_cmpxchg_release(...) \ 1119 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__) 1120 #endif 1121 1122 #ifndef atomic64_cmpxchg 1123 #define atomic64_cmpxchg(...) \ 1124 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__) 1125 #endif 1126 #endif /* atomic64_cmpxchg_relaxed */ 1127 1128 #ifndef atomic64_try_cmpxchg 1129 1130 #define __atomic64_try_cmpxchg(type, _p, _po, _n) \ 1131 ({ \ 1132 typeof(_po) __po = (_po); \ 1133 typeof(*(_po)) __r, __o = *__po; \ 1134 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \ 1135 if (unlikely(__r != __o)) \ 1136 *__po = __r; \ 1137 likely(__r == __o); \ 1138 }) 1139 1140 #define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n) 1141 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n) 1142 #define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n) 1143 #define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n) 1144 1145 #else /* atomic64_try_cmpxchg */ 1146 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg 1147 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg 1148 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg 1149 #endif /* atomic64_try_cmpxchg */ 1150 1151 /** 1152 * atomic64_fetch_add_unless - add unless the number is already a given value 1153 * @v: pointer of type atomic64_t 1154 * @a: the amount to add to v... 1155 * @u: ...unless v is equal to u. 1156 * 1157 * Atomically adds @a to @v, if @v was not already @u. 1158 * Returns the original value of @v. 1159 */ 1160 #ifndef atomic64_fetch_add_unless 1161 static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, 1162 long long u) 1163 { 1164 long long c = atomic64_read(v); 1165 1166 do { 1167 if (unlikely(c == u)) 1168 break; 1169 } while (!atomic64_try_cmpxchg(v, &c, c + a)); 1170 1171 return c; 1172 } 1173 #endif 1174 1175 /** 1176 * atomic64_add_unless - add unless the number is already a given value 1177 * @v: pointer of type atomic_t 1178 * @a: the amount to add to v... 1179 * @u: ...unless v is equal to u. 1180 * 1181 * Atomically adds @a to @v, if @v was not already @u. 1182 * Returns true if the addition was done. 1183 */ 1184 static inline bool atomic64_add_unless(atomic64_t *v, long long a, long long u) 1185 { 1186 return atomic64_fetch_add_unless(v, a, u) != u; 1187 } 1188 1189 /** 1190 * atomic64_inc_not_zero - increment unless the number is zero 1191 * @v: pointer of type atomic64_t 1192 * 1193 * Atomically increments @v by 1, if @v is non-zero. 1194 * Returns true if the increment was done. 1195 */ 1196 #ifndef atomic64_inc_not_zero 1197 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 1198 #endif 1199 1200 /** 1201 * atomic64_inc_and_test - increment and test 1202 * @v: pointer of type atomic64_t 1203 * 1204 * Atomically increments @v by 1 1205 * and returns true if the result is zero, or false for all 1206 * other cases. 1207 */ 1208 #ifndef atomic64_inc_and_test 1209 static inline bool atomic64_inc_and_test(atomic64_t *v) 1210 { 1211 return atomic64_inc_return(v) == 0; 1212 } 1213 #endif 1214 1215 /** 1216 * atomic64_dec_and_test - decrement and test 1217 * @v: pointer of type atomic64_t 1218 * 1219 * Atomically decrements @v by 1 and 1220 * returns true if the result is 0, or false for all other 1221 * cases. 1222 */ 1223 #ifndef atomic64_dec_and_test 1224 static inline bool atomic64_dec_and_test(atomic64_t *v) 1225 { 1226 return atomic64_dec_return(v) == 0; 1227 } 1228 #endif 1229 1230 /** 1231 * atomic64_sub_and_test - subtract value from variable and test result 1232 * @i: integer value to subtract 1233 * @v: pointer of type atomic64_t 1234 * 1235 * Atomically subtracts @i from @v and returns 1236 * true if the result is zero, or false for all 1237 * other cases. 1238 */ 1239 #ifndef atomic64_sub_and_test 1240 static inline bool atomic64_sub_and_test(long long i, atomic64_t *v) 1241 { 1242 return atomic64_sub_return(i, v) == 0; 1243 } 1244 #endif 1245 1246 /** 1247 * atomic64_add_negative - add and test if negative 1248 * @i: integer value to add 1249 * @v: pointer of type atomic64_t 1250 * 1251 * Atomically adds @i to @v and returns true 1252 * if the result is negative, or false when 1253 * result is greater than or equal to zero. 1254 */ 1255 #ifndef atomic64_add_negative 1256 static inline bool atomic64_add_negative(long long i, atomic64_t *v) 1257 { 1258 return atomic64_add_return(i, v) < 0; 1259 } 1260 #endif 1261 1262 #ifndef atomic64_inc_unless_negative 1263 static inline bool atomic64_inc_unless_negative(atomic64_t *v) 1264 { 1265 long long c = atomic64_read(v); 1266 1267 do { 1268 if (unlikely(c < 0)) 1269 return false; 1270 } while (!atomic64_try_cmpxchg(v, &c, c + 1)); 1271 1272 return true; 1273 } 1274 #endif 1275 1276 #ifndef atomic64_dec_unless_positive 1277 static inline bool atomic64_dec_unless_positive(atomic64_t *v) 1278 { 1279 long long c = atomic64_read(v); 1280 1281 do { 1282 if (unlikely(c > 0)) 1283 return false; 1284 } while (!atomic64_try_cmpxchg(v, &c, c - 1)); 1285 1286 return true; 1287 } 1288 #endif 1289 1290 /* 1291 * atomic64_dec_if_positive - decrement by 1 if old value positive 1292 * @v: pointer of type atomic64_t 1293 * 1294 * The function returns the old value of *v minus 1, even if 1295 * the atomic64 variable, v, was not decremented. 1296 */ 1297 #ifndef atomic64_dec_if_positive 1298 static inline long long atomic64_dec_if_positive(atomic64_t *v) 1299 { 1300 long long dec, c = atomic64_read(v); 1301 1302 do { 1303 dec = c - 1; 1304 if (unlikely(dec < 0)) 1305 break; 1306 } while (!atomic64_try_cmpxchg(v, &c, dec)); 1307 1308 return dec; 1309 } 1310 #endif 1311 1312 #define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) 1313 #define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) 1314 1315 #include <asm-generic/atomic-long.h> 1316 1317 #endif /* _LINUX_ATOMIC_H */ 1318