1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* Atomic operations usable in machine independent code */ 3 #ifndef _LINUX_ATOMIC_H 4 #define _LINUX_ATOMIC_H 5 #include <linux/types.h> 6 7 #include <asm/atomic.h> 8 #include <asm/barrier.h> 9 10 /* 11 * Relaxed variants of xchg, cmpxchg and some atomic operations. 12 * 13 * We support four variants: 14 * 15 * - Fully ordered: The default implementation, no suffix required. 16 * - Acquire: Provides ACQUIRE semantics, _acquire suffix. 17 * - Release: Provides RELEASE semantics, _release suffix. 18 * - Relaxed: No ordering guarantees, _relaxed suffix. 19 * 20 * For compound atomics performing both a load and a store, ACQUIRE 21 * semantics apply only to the load and RELEASE semantics only to the 22 * store portion of the operation. Note that a failed cmpxchg_acquire 23 * does -not- imply any memory ordering constraints. 24 * 25 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. 26 */ 27 28 #ifndef atomic_read_acquire 29 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) 30 #endif 31 32 #ifndef atomic_set_release 33 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) 34 #endif 35 36 /* 37 * The idea here is to build acquire/release variants by adding explicit 38 * barriers on top of the relaxed variant. In the case where the relaxed 39 * variant is already fully ordered, no additional barriers are needed. 40 * 41 * Besides, if an arch has a special barrier for acquire/release, it could 42 * implement its own __atomic_op_* and use the same framework for building 43 * variants 44 * 45 * If an architecture overrides __atomic_op_acquire() it will probably want 46 * to define smp_mb__after_spinlock(). 47 */ 48 #ifndef __atomic_op_acquire 49 #define __atomic_op_acquire(op, args...) \ 50 ({ \ 51 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 52 smp_mb__after_atomic(); \ 53 __ret; \ 54 }) 55 #endif 56 57 #ifndef __atomic_op_release 58 #define __atomic_op_release(op, args...) \ 59 ({ \ 60 smp_mb__before_atomic(); \ 61 op##_relaxed(args); \ 62 }) 63 #endif 64 65 #ifndef __atomic_op_fence 66 #define __atomic_op_fence(op, args...) \ 67 ({ \ 68 typeof(op##_relaxed(args)) __ret; \ 69 smp_mb__before_atomic(); \ 70 __ret = op##_relaxed(args); \ 71 smp_mb__after_atomic(); \ 72 __ret; \ 73 }) 74 #endif 75 76 /* atomic_add_return_relaxed */ 77 #ifndef atomic_add_return_relaxed 78 #define atomic_add_return_relaxed atomic_add_return 79 #define atomic_add_return_acquire atomic_add_return 80 #define atomic_add_return_release atomic_add_return 81 82 #else /* atomic_add_return_relaxed */ 83 84 #ifndef atomic_add_return_acquire 85 #define atomic_add_return_acquire(...) \ 86 __atomic_op_acquire(atomic_add_return, __VA_ARGS__) 87 #endif 88 89 #ifndef atomic_add_return_release 90 #define atomic_add_return_release(...) \ 91 __atomic_op_release(atomic_add_return, __VA_ARGS__) 92 #endif 93 94 #ifndef atomic_add_return 95 #define atomic_add_return(...) \ 96 __atomic_op_fence(atomic_add_return, __VA_ARGS__) 97 #endif 98 #endif /* atomic_add_return_relaxed */ 99 100 #ifndef atomic_inc 101 #define atomic_inc(v) atomic_add(1, (v)) 102 #endif 103 104 /* atomic_inc_return_relaxed */ 105 #ifndef atomic_inc_return_relaxed 106 107 #ifndef atomic_inc_return 108 #define atomic_inc_return(v) atomic_add_return(1, (v)) 109 #define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v)) 110 #define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v)) 111 #define atomic_inc_return_release(v) atomic_add_return_release(1, (v)) 112 #else /* atomic_inc_return */ 113 #define atomic_inc_return_relaxed atomic_inc_return 114 #define atomic_inc_return_acquire atomic_inc_return 115 #define atomic_inc_return_release atomic_inc_return 116 #endif /* atomic_inc_return */ 117 118 #else /* atomic_inc_return_relaxed */ 119 120 #ifndef atomic_inc_return_acquire 121 #define atomic_inc_return_acquire(...) \ 122 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__) 123 #endif 124 125 #ifndef atomic_inc_return_release 126 #define atomic_inc_return_release(...) \ 127 __atomic_op_release(atomic_inc_return, __VA_ARGS__) 128 #endif 129 130 #ifndef atomic_inc_return 131 #define atomic_inc_return(...) \ 132 __atomic_op_fence(atomic_inc_return, __VA_ARGS__) 133 #endif 134 #endif /* atomic_inc_return_relaxed */ 135 136 /* atomic_sub_return_relaxed */ 137 #ifndef atomic_sub_return_relaxed 138 #define atomic_sub_return_relaxed atomic_sub_return 139 #define atomic_sub_return_acquire atomic_sub_return 140 #define atomic_sub_return_release atomic_sub_return 141 142 #else /* atomic_sub_return_relaxed */ 143 144 #ifndef atomic_sub_return_acquire 145 #define atomic_sub_return_acquire(...) \ 146 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__) 147 #endif 148 149 #ifndef atomic_sub_return_release 150 #define atomic_sub_return_release(...) \ 151 __atomic_op_release(atomic_sub_return, __VA_ARGS__) 152 #endif 153 154 #ifndef atomic_sub_return 155 #define atomic_sub_return(...) \ 156 __atomic_op_fence(atomic_sub_return, __VA_ARGS__) 157 #endif 158 #endif /* atomic_sub_return_relaxed */ 159 160 #ifndef atomic_dec 161 #define atomic_dec(v) atomic_sub(1, (v)) 162 #endif 163 164 /* atomic_dec_return_relaxed */ 165 #ifndef atomic_dec_return_relaxed 166 167 #ifndef atomic_dec_return 168 #define atomic_dec_return(v) atomic_sub_return(1, (v)) 169 #define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v)) 170 #define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v)) 171 #define atomic_dec_return_release(v) atomic_sub_return_release(1, (v)) 172 #else /* atomic_dec_return */ 173 #define atomic_dec_return_relaxed atomic_dec_return 174 #define atomic_dec_return_acquire atomic_dec_return 175 #define atomic_dec_return_release atomic_dec_return 176 #endif /* atomic_dec_return */ 177 178 #else /* atomic_dec_return_relaxed */ 179 180 #ifndef atomic_dec_return_acquire 181 #define atomic_dec_return_acquire(...) \ 182 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__) 183 #endif 184 185 #ifndef atomic_dec_return_release 186 #define atomic_dec_return_release(...) \ 187 __atomic_op_release(atomic_dec_return, __VA_ARGS__) 188 #endif 189 190 #ifndef atomic_dec_return 191 #define atomic_dec_return(...) \ 192 __atomic_op_fence(atomic_dec_return, __VA_ARGS__) 193 #endif 194 #endif /* atomic_dec_return_relaxed */ 195 196 197 /* atomic_fetch_add_relaxed */ 198 #ifndef atomic_fetch_add_relaxed 199 #define atomic_fetch_add_relaxed atomic_fetch_add 200 #define atomic_fetch_add_acquire atomic_fetch_add 201 #define atomic_fetch_add_release atomic_fetch_add 202 203 #else /* atomic_fetch_add_relaxed */ 204 205 #ifndef atomic_fetch_add_acquire 206 #define atomic_fetch_add_acquire(...) \ 207 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__) 208 #endif 209 210 #ifndef atomic_fetch_add_release 211 #define atomic_fetch_add_release(...) \ 212 __atomic_op_release(atomic_fetch_add, __VA_ARGS__) 213 #endif 214 215 #ifndef atomic_fetch_add 216 #define atomic_fetch_add(...) \ 217 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__) 218 #endif 219 #endif /* atomic_fetch_add_relaxed */ 220 221 /* atomic_fetch_inc_relaxed */ 222 #ifndef atomic_fetch_inc_relaxed 223 224 #ifndef atomic_fetch_inc 225 #define atomic_fetch_inc(v) atomic_fetch_add(1, (v)) 226 #define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v)) 227 #define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v)) 228 #define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v)) 229 #else /* atomic_fetch_inc */ 230 #define atomic_fetch_inc_relaxed atomic_fetch_inc 231 #define atomic_fetch_inc_acquire atomic_fetch_inc 232 #define atomic_fetch_inc_release atomic_fetch_inc 233 #endif /* atomic_fetch_inc */ 234 235 #else /* atomic_fetch_inc_relaxed */ 236 237 #ifndef atomic_fetch_inc_acquire 238 #define atomic_fetch_inc_acquire(...) \ 239 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__) 240 #endif 241 242 #ifndef atomic_fetch_inc_release 243 #define atomic_fetch_inc_release(...) \ 244 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__) 245 #endif 246 247 #ifndef atomic_fetch_inc 248 #define atomic_fetch_inc(...) \ 249 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__) 250 #endif 251 #endif /* atomic_fetch_inc_relaxed */ 252 253 /* atomic_fetch_sub_relaxed */ 254 #ifndef atomic_fetch_sub_relaxed 255 #define atomic_fetch_sub_relaxed atomic_fetch_sub 256 #define atomic_fetch_sub_acquire atomic_fetch_sub 257 #define atomic_fetch_sub_release atomic_fetch_sub 258 259 #else /* atomic_fetch_sub_relaxed */ 260 261 #ifndef atomic_fetch_sub_acquire 262 #define atomic_fetch_sub_acquire(...) \ 263 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__) 264 #endif 265 266 #ifndef atomic_fetch_sub_release 267 #define atomic_fetch_sub_release(...) \ 268 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__) 269 #endif 270 271 #ifndef atomic_fetch_sub 272 #define atomic_fetch_sub(...) \ 273 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__) 274 #endif 275 #endif /* atomic_fetch_sub_relaxed */ 276 277 /* atomic_fetch_dec_relaxed */ 278 #ifndef atomic_fetch_dec_relaxed 279 280 #ifndef atomic_fetch_dec 281 #define atomic_fetch_dec(v) atomic_fetch_sub(1, (v)) 282 #define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v)) 283 #define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v)) 284 #define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v)) 285 #else /* atomic_fetch_dec */ 286 #define atomic_fetch_dec_relaxed atomic_fetch_dec 287 #define atomic_fetch_dec_acquire atomic_fetch_dec 288 #define atomic_fetch_dec_release atomic_fetch_dec 289 #endif /* atomic_fetch_dec */ 290 291 #else /* atomic_fetch_dec_relaxed */ 292 293 #ifndef atomic_fetch_dec_acquire 294 #define atomic_fetch_dec_acquire(...) \ 295 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__) 296 #endif 297 298 #ifndef atomic_fetch_dec_release 299 #define atomic_fetch_dec_release(...) \ 300 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__) 301 #endif 302 303 #ifndef atomic_fetch_dec 304 #define atomic_fetch_dec(...) \ 305 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__) 306 #endif 307 #endif /* atomic_fetch_dec_relaxed */ 308 309 /* atomic_fetch_or_relaxed */ 310 #ifndef atomic_fetch_or_relaxed 311 #define atomic_fetch_or_relaxed atomic_fetch_or 312 #define atomic_fetch_or_acquire atomic_fetch_or 313 #define atomic_fetch_or_release atomic_fetch_or 314 315 #else /* atomic_fetch_or_relaxed */ 316 317 #ifndef atomic_fetch_or_acquire 318 #define atomic_fetch_or_acquire(...) \ 319 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__) 320 #endif 321 322 #ifndef atomic_fetch_or_release 323 #define atomic_fetch_or_release(...) \ 324 __atomic_op_release(atomic_fetch_or, __VA_ARGS__) 325 #endif 326 327 #ifndef atomic_fetch_or 328 #define atomic_fetch_or(...) \ 329 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__) 330 #endif 331 #endif /* atomic_fetch_or_relaxed */ 332 333 /* atomic_fetch_and_relaxed */ 334 #ifndef atomic_fetch_and_relaxed 335 #define atomic_fetch_and_relaxed atomic_fetch_and 336 #define atomic_fetch_and_acquire atomic_fetch_and 337 #define atomic_fetch_and_release atomic_fetch_and 338 339 #else /* atomic_fetch_and_relaxed */ 340 341 #ifndef atomic_fetch_and_acquire 342 #define atomic_fetch_and_acquire(...) \ 343 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__) 344 #endif 345 346 #ifndef atomic_fetch_and_release 347 #define atomic_fetch_and_release(...) \ 348 __atomic_op_release(atomic_fetch_and, __VA_ARGS__) 349 #endif 350 351 #ifndef atomic_fetch_and 352 #define atomic_fetch_and(...) \ 353 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__) 354 #endif 355 #endif /* atomic_fetch_and_relaxed */ 356 357 #ifdef atomic_andnot 358 /* atomic_fetch_andnot_relaxed */ 359 #ifndef atomic_fetch_andnot_relaxed 360 #define atomic_fetch_andnot_relaxed atomic_fetch_andnot 361 #define atomic_fetch_andnot_acquire atomic_fetch_andnot 362 #define atomic_fetch_andnot_release atomic_fetch_andnot 363 364 #else /* atomic_fetch_andnot_relaxed */ 365 366 #ifndef atomic_fetch_andnot_acquire 367 #define atomic_fetch_andnot_acquire(...) \ 368 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__) 369 #endif 370 371 #ifndef atomic_fetch_andnot_release 372 #define atomic_fetch_andnot_release(...) \ 373 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__) 374 #endif 375 376 #ifndef atomic_fetch_andnot 377 #define atomic_fetch_andnot(...) \ 378 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__) 379 #endif 380 #endif /* atomic_fetch_andnot_relaxed */ 381 #endif /* atomic_andnot */ 382 383 /* atomic_fetch_xor_relaxed */ 384 #ifndef atomic_fetch_xor_relaxed 385 #define atomic_fetch_xor_relaxed atomic_fetch_xor 386 #define atomic_fetch_xor_acquire atomic_fetch_xor 387 #define atomic_fetch_xor_release atomic_fetch_xor 388 389 #else /* atomic_fetch_xor_relaxed */ 390 391 #ifndef atomic_fetch_xor_acquire 392 #define atomic_fetch_xor_acquire(...) \ 393 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__) 394 #endif 395 396 #ifndef atomic_fetch_xor_release 397 #define atomic_fetch_xor_release(...) \ 398 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__) 399 #endif 400 401 #ifndef atomic_fetch_xor 402 #define atomic_fetch_xor(...) \ 403 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__) 404 #endif 405 #endif /* atomic_fetch_xor_relaxed */ 406 407 408 /* atomic_xchg_relaxed */ 409 #ifndef atomic_xchg_relaxed 410 #define atomic_xchg_relaxed atomic_xchg 411 #define atomic_xchg_acquire atomic_xchg 412 #define atomic_xchg_release atomic_xchg 413 414 #else /* atomic_xchg_relaxed */ 415 416 #ifndef atomic_xchg_acquire 417 #define atomic_xchg_acquire(...) \ 418 __atomic_op_acquire(atomic_xchg, __VA_ARGS__) 419 #endif 420 421 #ifndef atomic_xchg_release 422 #define atomic_xchg_release(...) \ 423 __atomic_op_release(atomic_xchg, __VA_ARGS__) 424 #endif 425 426 #ifndef atomic_xchg 427 #define atomic_xchg(...) \ 428 __atomic_op_fence(atomic_xchg, __VA_ARGS__) 429 #endif 430 #endif /* atomic_xchg_relaxed */ 431 432 /* atomic_cmpxchg_relaxed */ 433 #ifndef atomic_cmpxchg_relaxed 434 #define atomic_cmpxchg_relaxed atomic_cmpxchg 435 #define atomic_cmpxchg_acquire atomic_cmpxchg 436 #define atomic_cmpxchg_release atomic_cmpxchg 437 438 #else /* atomic_cmpxchg_relaxed */ 439 440 #ifndef atomic_cmpxchg_acquire 441 #define atomic_cmpxchg_acquire(...) \ 442 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__) 443 #endif 444 445 #ifndef atomic_cmpxchg_release 446 #define atomic_cmpxchg_release(...) \ 447 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__) 448 #endif 449 450 #ifndef atomic_cmpxchg 451 #define atomic_cmpxchg(...) \ 452 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__) 453 #endif 454 #endif /* atomic_cmpxchg_relaxed */ 455 456 #ifndef atomic_try_cmpxchg 457 458 #define __atomic_try_cmpxchg(type, _p, _po, _n) \ 459 ({ \ 460 typeof(_po) __po = (_po); \ 461 typeof(*(_po)) __r, __o = *__po; \ 462 __r = atomic_cmpxchg##type((_p), __o, (_n)); \ 463 if (unlikely(__r != __o)) \ 464 *__po = __r; \ 465 likely(__r == __o); \ 466 }) 467 468 #define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n) 469 #define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n) 470 #define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n) 471 #define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n) 472 473 #else /* atomic_try_cmpxchg */ 474 #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg 475 #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg 476 #define atomic_try_cmpxchg_release atomic_try_cmpxchg 477 #endif /* atomic_try_cmpxchg */ 478 479 /* cmpxchg_relaxed */ 480 #ifndef cmpxchg_relaxed 481 #define cmpxchg_relaxed cmpxchg 482 #define cmpxchg_acquire cmpxchg 483 #define cmpxchg_release cmpxchg 484 485 #else /* cmpxchg_relaxed */ 486 487 #ifndef cmpxchg_acquire 488 #define cmpxchg_acquire(...) \ 489 __atomic_op_acquire(cmpxchg, __VA_ARGS__) 490 #endif 491 492 #ifndef cmpxchg_release 493 #define cmpxchg_release(...) \ 494 __atomic_op_release(cmpxchg, __VA_ARGS__) 495 #endif 496 497 #ifndef cmpxchg 498 #define cmpxchg(...) \ 499 __atomic_op_fence(cmpxchg, __VA_ARGS__) 500 #endif 501 #endif /* cmpxchg_relaxed */ 502 503 /* cmpxchg64_relaxed */ 504 #ifndef cmpxchg64_relaxed 505 #define cmpxchg64_relaxed cmpxchg64 506 #define cmpxchg64_acquire cmpxchg64 507 #define cmpxchg64_release cmpxchg64 508 509 #else /* cmpxchg64_relaxed */ 510 511 #ifndef cmpxchg64_acquire 512 #define cmpxchg64_acquire(...) \ 513 __atomic_op_acquire(cmpxchg64, __VA_ARGS__) 514 #endif 515 516 #ifndef cmpxchg64_release 517 #define cmpxchg64_release(...) \ 518 __atomic_op_release(cmpxchg64, __VA_ARGS__) 519 #endif 520 521 #ifndef cmpxchg64 522 #define cmpxchg64(...) \ 523 __atomic_op_fence(cmpxchg64, __VA_ARGS__) 524 #endif 525 #endif /* cmpxchg64_relaxed */ 526 527 /* xchg_relaxed */ 528 #ifndef xchg_relaxed 529 #define xchg_relaxed xchg 530 #define xchg_acquire xchg 531 #define xchg_release xchg 532 533 #else /* xchg_relaxed */ 534 535 #ifndef xchg_acquire 536 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__) 537 #endif 538 539 #ifndef xchg_release 540 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__) 541 #endif 542 543 #ifndef xchg 544 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__) 545 #endif 546 #endif /* xchg_relaxed */ 547 548 /** 549 * atomic_fetch_add_unless - add unless the number is already a given value 550 * @v: pointer of type atomic_t 551 * @a: the amount to add to v... 552 * @u: ...unless v is equal to u. 553 * 554 * Atomically adds @a to @v, if @v was not already @u. 555 * Returns the original value of @v. 556 */ 557 #ifndef atomic_fetch_add_unless 558 static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) 559 { 560 int c = atomic_read(v); 561 562 do { 563 if (unlikely(c == u)) 564 break; 565 } while (!atomic_try_cmpxchg(v, &c, c + a)); 566 567 return c; 568 } 569 #endif 570 571 /** 572 * atomic_add_unless - add unless the number is already a given value 573 * @v: pointer of type atomic_t 574 * @a: the amount to add to v... 575 * @u: ...unless v is equal to u. 576 * 577 * Atomically adds @a to @v, if @v was not already @u. 578 * Returns true if the addition was done. 579 */ 580 static inline bool atomic_add_unless(atomic_t *v, int a, int u) 581 { 582 return atomic_fetch_add_unless(v, a, u) != u; 583 } 584 585 /** 586 * atomic_inc_not_zero - increment unless the number is zero 587 * @v: pointer of type atomic_t 588 * 589 * Atomically increments @v by 1, if @v is non-zero. 590 * Returns true if the increment was done. 591 */ 592 #ifndef atomic_inc_not_zero 593 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 594 #endif 595 596 /** 597 * atomic_inc_and_test - increment and test 598 * @v: pointer of type atomic_t 599 * 600 * Atomically increments @v by 1 601 * and returns true if the result is zero, or false for all 602 * other cases. 603 */ 604 #ifndef atomic_inc_and_test 605 static inline bool atomic_inc_and_test(atomic_t *v) 606 { 607 return atomic_inc_return(v) == 0; 608 } 609 #endif 610 611 /** 612 * atomic_dec_and_test - decrement and test 613 * @v: pointer of type atomic_t 614 * 615 * Atomically decrements @v by 1 and 616 * returns true if the result is 0, or false for all other 617 * cases. 618 */ 619 #ifndef atomic_dec_and_test 620 static inline bool atomic_dec_and_test(atomic_t *v) 621 { 622 return atomic_dec_return(v) == 0; 623 } 624 #endif 625 626 /** 627 * atomic_sub_and_test - subtract value from variable and test result 628 * @i: integer value to subtract 629 * @v: pointer of type atomic_t 630 * 631 * Atomically subtracts @i from @v and returns 632 * true if the result is zero, or false for all 633 * other cases. 634 */ 635 #ifndef atomic_sub_and_test 636 static inline bool atomic_sub_and_test(int i, atomic_t *v) 637 { 638 return atomic_sub_return(i, v) == 0; 639 } 640 #endif 641 642 /** 643 * atomic_add_negative - add and test if negative 644 * @i: integer value to add 645 * @v: pointer of type atomic_t 646 * 647 * Atomically adds @i to @v and returns true 648 * if the result is negative, or false when 649 * result is greater than or equal to zero. 650 */ 651 #ifndef atomic_add_negative 652 static inline bool atomic_add_negative(int i, atomic_t *v) 653 { 654 return atomic_add_return(i, v) < 0; 655 } 656 #endif 657 658 #ifndef atomic_andnot 659 static inline void atomic_andnot(int i, atomic_t *v) 660 { 661 atomic_and(~i, v); 662 } 663 664 static inline int atomic_fetch_andnot(int i, atomic_t *v) 665 { 666 return atomic_fetch_and(~i, v); 667 } 668 669 static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) 670 { 671 return atomic_fetch_and_relaxed(~i, v); 672 } 673 674 static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) 675 { 676 return atomic_fetch_and_acquire(~i, v); 677 } 678 679 static inline int atomic_fetch_andnot_release(int i, atomic_t *v) 680 { 681 return atomic_fetch_and_release(~i, v); 682 } 683 #endif 684 685 #ifndef atomic_inc_unless_negative 686 static inline bool atomic_inc_unless_negative(atomic_t *p) 687 { 688 int v, v1; 689 for (v = 0; v >= 0; v = v1) { 690 v1 = atomic_cmpxchg(p, v, v + 1); 691 if (likely(v1 == v)) 692 return true; 693 } 694 return false; 695 } 696 #endif 697 698 #ifndef atomic_dec_unless_positive 699 static inline bool atomic_dec_unless_positive(atomic_t *p) 700 { 701 int v, v1; 702 for (v = 0; v <= 0; v = v1) { 703 v1 = atomic_cmpxchg(p, v, v - 1); 704 if (likely(v1 == v)) 705 return true; 706 } 707 return false; 708 } 709 #endif 710 711 /* 712 * atomic_dec_if_positive - decrement by 1 if old value positive 713 * @v: pointer of type atomic_t 714 * 715 * The function returns the old value of *v minus 1, even if 716 * the atomic variable, v, was not decremented. 717 */ 718 #ifndef atomic_dec_if_positive 719 static inline int atomic_dec_if_positive(atomic_t *v) 720 { 721 int c, old, dec; 722 c = atomic_read(v); 723 for (;;) { 724 dec = c - 1; 725 if (unlikely(dec < 0)) 726 break; 727 old = atomic_cmpxchg((v), c, dec); 728 if (likely(old == c)) 729 break; 730 c = old; 731 } 732 return dec; 733 } 734 #endif 735 736 #define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) 737 #define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) 738 739 #ifdef CONFIG_GENERIC_ATOMIC64 740 #include <asm-generic/atomic64.h> 741 #endif 742 743 #ifndef atomic64_read_acquire 744 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter) 745 #endif 746 747 #ifndef atomic64_set_release 748 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i)) 749 #endif 750 751 /* atomic64_add_return_relaxed */ 752 #ifndef atomic64_add_return_relaxed 753 #define atomic64_add_return_relaxed atomic64_add_return 754 #define atomic64_add_return_acquire atomic64_add_return 755 #define atomic64_add_return_release atomic64_add_return 756 757 #else /* atomic64_add_return_relaxed */ 758 759 #ifndef atomic64_add_return_acquire 760 #define atomic64_add_return_acquire(...) \ 761 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__) 762 #endif 763 764 #ifndef atomic64_add_return_release 765 #define atomic64_add_return_release(...) \ 766 __atomic_op_release(atomic64_add_return, __VA_ARGS__) 767 #endif 768 769 #ifndef atomic64_add_return 770 #define atomic64_add_return(...) \ 771 __atomic_op_fence(atomic64_add_return, __VA_ARGS__) 772 #endif 773 #endif /* atomic64_add_return_relaxed */ 774 775 #ifndef atomic64_inc 776 #define atomic64_inc(v) atomic64_add(1, (v)) 777 #endif 778 779 /* atomic64_inc_return_relaxed */ 780 #ifndef atomic64_inc_return_relaxed 781 782 #ifndef atomic64_inc_return 783 #define atomic64_inc_return(v) atomic64_add_return(1, (v)) 784 #define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v)) 785 #define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v)) 786 #define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v)) 787 #else /* atomic64_inc_return */ 788 #define atomic64_inc_return_relaxed atomic64_inc_return 789 #define atomic64_inc_return_acquire atomic64_inc_return 790 #define atomic64_inc_return_release atomic64_inc_return 791 #endif /* atomic64_inc_return */ 792 793 #else /* atomic64_inc_return_relaxed */ 794 795 #ifndef atomic64_inc_return_acquire 796 #define atomic64_inc_return_acquire(...) \ 797 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__) 798 #endif 799 800 #ifndef atomic64_inc_return_release 801 #define atomic64_inc_return_release(...) \ 802 __atomic_op_release(atomic64_inc_return, __VA_ARGS__) 803 #endif 804 805 #ifndef atomic64_inc_return 806 #define atomic64_inc_return(...) \ 807 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__) 808 #endif 809 #endif /* atomic64_inc_return_relaxed */ 810 811 812 /* atomic64_sub_return_relaxed */ 813 #ifndef atomic64_sub_return_relaxed 814 #define atomic64_sub_return_relaxed atomic64_sub_return 815 #define atomic64_sub_return_acquire atomic64_sub_return 816 #define atomic64_sub_return_release atomic64_sub_return 817 818 #else /* atomic64_sub_return_relaxed */ 819 820 #ifndef atomic64_sub_return_acquire 821 #define atomic64_sub_return_acquire(...) \ 822 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__) 823 #endif 824 825 #ifndef atomic64_sub_return_release 826 #define atomic64_sub_return_release(...) \ 827 __atomic_op_release(atomic64_sub_return, __VA_ARGS__) 828 #endif 829 830 #ifndef atomic64_sub_return 831 #define atomic64_sub_return(...) \ 832 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__) 833 #endif 834 #endif /* atomic64_sub_return_relaxed */ 835 836 #ifndef atomic64_dec 837 #define atomic64_dec(v) atomic64_sub(1, (v)) 838 #endif 839 840 /* atomic64_dec_return_relaxed */ 841 #ifndef atomic64_dec_return_relaxed 842 843 #ifndef atomic64_dec_return 844 #define atomic64_dec_return(v) atomic64_sub_return(1, (v)) 845 #define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v)) 846 #define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v)) 847 #define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v)) 848 #else /* atomic64_dec_return */ 849 #define atomic64_dec_return_relaxed atomic64_dec_return 850 #define atomic64_dec_return_acquire atomic64_dec_return 851 #define atomic64_dec_return_release atomic64_dec_return 852 #endif /* atomic64_dec_return */ 853 854 #else /* atomic64_dec_return_relaxed */ 855 856 #ifndef atomic64_dec_return_acquire 857 #define atomic64_dec_return_acquire(...) \ 858 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__) 859 #endif 860 861 #ifndef atomic64_dec_return_release 862 #define atomic64_dec_return_release(...) \ 863 __atomic_op_release(atomic64_dec_return, __VA_ARGS__) 864 #endif 865 866 #ifndef atomic64_dec_return 867 #define atomic64_dec_return(...) \ 868 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__) 869 #endif 870 #endif /* atomic64_dec_return_relaxed */ 871 872 873 /* atomic64_fetch_add_relaxed */ 874 #ifndef atomic64_fetch_add_relaxed 875 #define atomic64_fetch_add_relaxed atomic64_fetch_add 876 #define atomic64_fetch_add_acquire atomic64_fetch_add 877 #define atomic64_fetch_add_release atomic64_fetch_add 878 879 #else /* atomic64_fetch_add_relaxed */ 880 881 #ifndef atomic64_fetch_add_acquire 882 #define atomic64_fetch_add_acquire(...) \ 883 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__) 884 #endif 885 886 #ifndef atomic64_fetch_add_release 887 #define atomic64_fetch_add_release(...) \ 888 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__) 889 #endif 890 891 #ifndef atomic64_fetch_add 892 #define atomic64_fetch_add(...) \ 893 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__) 894 #endif 895 #endif /* atomic64_fetch_add_relaxed */ 896 897 /* atomic64_fetch_inc_relaxed */ 898 #ifndef atomic64_fetch_inc_relaxed 899 900 #ifndef atomic64_fetch_inc 901 #define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v)) 902 #define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v)) 903 #define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v)) 904 #define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v)) 905 #else /* atomic64_fetch_inc */ 906 #define atomic64_fetch_inc_relaxed atomic64_fetch_inc 907 #define atomic64_fetch_inc_acquire atomic64_fetch_inc 908 #define atomic64_fetch_inc_release atomic64_fetch_inc 909 #endif /* atomic64_fetch_inc */ 910 911 #else /* atomic64_fetch_inc_relaxed */ 912 913 #ifndef atomic64_fetch_inc_acquire 914 #define atomic64_fetch_inc_acquire(...) \ 915 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__) 916 #endif 917 918 #ifndef atomic64_fetch_inc_release 919 #define atomic64_fetch_inc_release(...) \ 920 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__) 921 #endif 922 923 #ifndef atomic64_fetch_inc 924 #define atomic64_fetch_inc(...) \ 925 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__) 926 #endif 927 #endif /* atomic64_fetch_inc_relaxed */ 928 929 /* atomic64_fetch_sub_relaxed */ 930 #ifndef atomic64_fetch_sub_relaxed 931 #define atomic64_fetch_sub_relaxed atomic64_fetch_sub 932 #define atomic64_fetch_sub_acquire atomic64_fetch_sub 933 #define atomic64_fetch_sub_release atomic64_fetch_sub 934 935 #else /* atomic64_fetch_sub_relaxed */ 936 937 #ifndef atomic64_fetch_sub_acquire 938 #define atomic64_fetch_sub_acquire(...) \ 939 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__) 940 #endif 941 942 #ifndef atomic64_fetch_sub_release 943 #define atomic64_fetch_sub_release(...) \ 944 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__) 945 #endif 946 947 #ifndef atomic64_fetch_sub 948 #define atomic64_fetch_sub(...) \ 949 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__) 950 #endif 951 #endif /* atomic64_fetch_sub_relaxed */ 952 953 /* atomic64_fetch_dec_relaxed */ 954 #ifndef atomic64_fetch_dec_relaxed 955 956 #ifndef atomic64_fetch_dec 957 #define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v)) 958 #define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v)) 959 #define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v)) 960 #define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v)) 961 #else /* atomic64_fetch_dec */ 962 #define atomic64_fetch_dec_relaxed atomic64_fetch_dec 963 #define atomic64_fetch_dec_acquire atomic64_fetch_dec 964 #define atomic64_fetch_dec_release atomic64_fetch_dec 965 #endif /* atomic64_fetch_dec */ 966 967 #else /* atomic64_fetch_dec_relaxed */ 968 969 #ifndef atomic64_fetch_dec_acquire 970 #define atomic64_fetch_dec_acquire(...) \ 971 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__) 972 #endif 973 974 #ifndef atomic64_fetch_dec_release 975 #define atomic64_fetch_dec_release(...) \ 976 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__) 977 #endif 978 979 #ifndef atomic64_fetch_dec 980 #define atomic64_fetch_dec(...) \ 981 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__) 982 #endif 983 #endif /* atomic64_fetch_dec_relaxed */ 984 985 /* atomic64_fetch_or_relaxed */ 986 #ifndef atomic64_fetch_or_relaxed 987 #define atomic64_fetch_or_relaxed atomic64_fetch_or 988 #define atomic64_fetch_or_acquire atomic64_fetch_or 989 #define atomic64_fetch_or_release atomic64_fetch_or 990 991 #else /* atomic64_fetch_or_relaxed */ 992 993 #ifndef atomic64_fetch_or_acquire 994 #define atomic64_fetch_or_acquire(...) \ 995 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__) 996 #endif 997 998 #ifndef atomic64_fetch_or_release 999 #define atomic64_fetch_or_release(...) \ 1000 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__) 1001 #endif 1002 1003 #ifndef atomic64_fetch_or 1004 #define atomic64_fetch_or(...) \ 1005 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__) 1006 #endif 1007 #endif /* atomic64_fetch_or_relaxed */ 1008 1009 /* atomic64_fetch_and_relaxed */ 1010 #ifndef atomic64_fetch_and_relaxed 1011 #define atomic64_fetch_and_relaxed atomic64_fetch_and 1012 #define atomic64_fetch_and_acquire atomic64_fetch_and 1013 #define atomic64_fetch_and_release atomic64_fetch_and 1014 1015 #else /* atomic64_fetch_and_relaxed */ 1016 1017 #ifndef atomic64_fetch_and_acquire 1018 #define atomic64_fetch_and_acquire(...) \ 1019 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__) 1020 #endif 1021 1022 #ifndef atomic64_fetch_and_release 1023 #define atomic64_fetch_and_release(...) \ 1024 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__) 1025 #endif 1026 1027 #ifndef atomic64_fetch_and 1028 #define atomic64_fetch_and(...) \ 1029 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__) 1030 #endif 1031 #endif /* atomic64_fetch_and_relaxed */ 1032 1033 #ifdef atomic64_andnot 1034 /* atomic64_fetch_andnot_relaxed */ 1035 #ifndef atomic64_fetch_andnot_relaxed 1036 #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot 1037 #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot 1038 #define atomic64_fetch_andnot_release atomic64_fetch_andnot 1039 1040 #else /* atomic64_fetch_andnot_relaxed */ 1041 1042 #ifndef atomic64_fetch_andnot_acquire 1043 #define atomic64_fetch_andnot_acquire(...) \ 1044 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__) 1045 #endif 1046 1047 #ifndef atomic64_fetch_andnot_release 1048 #define atomic64_fetch_andnot_release(...) \ 1049 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__) 1050 #endif 1051 1052 #ifndef atomic64_fetch_andnot 1053 #define atomic64_fetch_andnot(...) \ 1054 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__) 1055 #endif 1056 #endif /* atomic64_fetch_andnot_relaxed */ 1057 #endif /* atomic64_andnot */ 1058 1059 /* atomic64_fetch_xor_relaxed */ 1060 #ifndef atomic64_fetch_xor_relaxed 1061 #define atomic64_fetch_xor_relaxed atomic64_fetch_xor 1062 #define atomic64_fetch_xor_acquire atomic64_fetch_xor 1063 #define atomic64_fetch_xor_release atomic64_fetch_xor 1064 1065 #else /* atomic64_fetch_xor_relaxed */ 1066 1067 #ifndef atomic64_fetch_xor_acquire 1068 #define atomic64_fetch_xor_acquire(...) \ 1069 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__) 1070 #endif 1071 1072 #ifndef atomic64_fetch_xor_release 1073 #define atomic64_fetch_xor_release(...) \ 1074 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__) 1075 #endif 1076 1077 #ifndef atomic64_fetch_xor 1078 #define atomic64_fetch_xor(...) \ 1079 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__) 1080 #endif 1081 #endif /* atomic64_fetch_xor_relaxed */ 1082 1083 1084 /* atomic64_xchg_relaxed */ 1085 #ifndef atomic64_xchg_relaxed 1086 #define atomic64_xchg_relaxed atomic64_xchg 1087 #define atomic64_xchg_acquire atomic64_xchg 1088 #define atomic64_xchg_release atomic64_xchg 1089 1090 #else /* atomic64_xchg_relaxed */ 1091 1092 #ifndef atomic64_xchg_acquire 1093 #define atomic64_xchg_acquire(...) \ 1094 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__) 1095 #endif 1096 1097 #ifndef atomic64_xchg_release 1098 #define atomic64_xchg_release(...) \ 1099 __atomic_op_release(atomic64_xchg, __VA_ARGS__) 1100 #endif 1101 1102 #ifndef atomic64_xchg 1103 #define atomic64_xchg(...) \ 1104 __atomic_op_fence(atomic64_xchg, __VA_ARGS__) 1105 #endif 1106 #endif /* atomic64_xchg_relaxed */ 1107 1108 /* atomic64_cmpxchg_relaxed */ 1109 #ifndef atomic64_cmpxchg_relaxed 1110 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg 1111 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 1112 #define atomic64_cmpxchg_release atomic64_cmpxchg 1113 1114 #else /* atomic64_cmpxchg_relaxed */ 1115 1116 #ifndef atomic64_cmpxchg_acquire 1117 #define atomic64_cmpxchg_acquire(...) \ 1118 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__) 1119 #endif 1120 1121 #ifndef atomic64_cmpxchg_release 1122 #define atomic64_cmpxchg_release(...) \ 1123 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__) 1124 #endif 1125 1126 #ifndef atomic64_cmpxchg 1127 #define atomic64_cmpxchg(...) \ 1128 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__) 1129 #endif 1130 #endif /* atomic64_cmpxchg_relaxed */ 1131 1132 #ifndef atomic64_try_cmpxchg 1133 1134 #define __atomic64_try_cmpxchg(type, _p, _po, _n) \ 1135 ({ \ 1136 typeof(_po) __po = (_po); \ 1137 typeof(*(_po)) __r, __o = *__po; \ 1138 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \ 1139 if (unlikely(__r != __o)) \ 1140 *__po = __r; \ 1141 likely(__r == __o); \ 1142 }) 1143 1144 #define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n) 1145 #define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n) 1146 #define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n) 1147 #define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n) 1148 1149 #else /* atomic64_try_cmpxchg */ 1150 #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg 1151 #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg 1152 #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg 1153 #endif /* atomic64_try_cmpxchg */ 1154 1155 /** 1156 * atomic64_fetch_add_unless - add unless the number is already a given value 1157 * @v: pointer of type atomic64_t 1158 * @a: the amount to add to v... 1159 * @u: ...unless v is equal to u. 1160 * 1161 * Atomically adds @a to @v, if @v was not already @u. 1162 * Returns the original value of @v. 1163 */ 1164 #ifndef atomic64_fetch_add_unless 1165 static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a, 1166 long long u) 1167 { 1168 long long c = atomic64_read(v); 1169 1170 do { 1171 if (unlikely(c == u)) 1172 break; 1173 } while (!atomic64_try_cmpxchg(v, &c, c + a)); 1174 1175 return c; 1176 } 1177 #endif 1178 1179 /** 1180 * atomic64_add_unless - add unless the number is already a given value 1181 * @v: pointer of type atomic_t 1182 * @a: the amount to add to v... 1183 * @u: ...unless v is equal to u. 1184 * 1185 * Atomically adds @a to @v, if @v was not already @u. 1186 * Returns true if the addition was done. 1187 */ 1188 static inline bool atomic64_add_unless(atomic64_t *v, long long a, long long u) 1189 { 1190 return atomic64_fetch_add_unless(v, a, u) != u; 1191 } 1192 1193 /** 1194 * atomic64_inc_not_zero - increment unless the number is zero 1195 * @v: pointer of type atomic64_t 1196 * 1197 * Atomically increments @v by 1, if @v is non-zero. 1198 * Returns true if the increment was done. 1199 */ 1200 #ifndef atomic64_inc_not_zero 1201 #define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0) 1202 #endif 1203 1204 /** 1205 * atomic64_inc_and_test - increment and test 1206 * @v: pointer of type atomic64_t 1207 * 1208 * Atomically increments @v by 1 1209 * and returns true if the result is zero, or false for all 1210 * other cases. 1211 */ 1212 #ifndef atomic64_inc_and_test 1213 static inline bool atomic64_inc_and_test(atomic64_t *v) 1214 { 1215 return atomic64_inc_return(v) == 0; 1216 } 1217 #endif 1218 1219 /** 1220 * atomic64_dec_and_test - decrement and test 1221 * @v: pointer of type atomic64_t 1222 * 1223 * Atomically decrements @v by 1 and 1224 * returns true if the result is 0, or false for all other 1225 * cases. 1226 */ 1227 #ifndef atomic64_dec_and_test 1228 static inline bool atomic64_dec_and_test(atomic64_t *v) 1229 { 1230 return atomic64_dec_return(v) == 0; 1231 } 1232 #endif 1233 1234 /** 1235 * atomic64_sub_and_test - subtract value from variable and test result 1236 * @i: integer value to subtract 1237 * @v: pointer of type atomic64_t 1238 * 1239 * Atomically subtracts @i from @v and returns 1240 * true if the result is zero, or false for all 1241 * other cases. 1242 */ 1243 #ifndef atomic64_sub_and_test 1244 static inline bool atomic64_sub_and_test(long long i, atomic64_t *v) 1245 { 1246 return atomic64_sub_return(i, v) == 0; 1247 } 1248 #endif 1249 1250 /** 1251 * atomic64_add_negative - add and test if negative 1252 * @i: integer value to add 1253 * @v: pointer of type atomic64_t 1254 * 1255 * Atomically adds @i to @v and returns true 1256 * if the result is negative, or false when 1257 * result is greater than or equal to zero. 1258 */ 1259 #ifndef atomic64_add_negative 1260 static inline bool atomic64_add_negative(long long i, atomic64_t *v) 1261 { 1262 return atomic64_add_return(i, v) < 0; 1263 } 1264 #endif 1265 1266 #ifndef atomic64_andnot 1267 static inline void atomic64_andnot(long long i, atomic64_t *v) 1268 { 1269 atomic64_and(~i, v); 1270 } 1271 1272 static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v) 1273 { 1274 return atomic64_fetch_and(~i, v); 1275 } 1276 1277 static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v) 1278 { 1279 return atomic64_fetch_and_relaxed(~i, v); 1280 } 1281 1282 static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v) 1283 { 1284 return atomic64_fetch_and_acquire(~i, v); 1285 } 1286 1287 static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v) 1288 { 1289 return atomic64_fetch_and_release(~i, v); 1290 } 1291 #endif 1292 1293 #define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) 1294 #define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) 1295 1296 #include <asm-generic/atomic-long.h> 1297 1298 #endif /* _LINUX_ATOMIC_H */ 1299