1 // SPDX-License-Identifier: GPL-2.0 2 3 // Generated by scripts/atomic/gen-atomic-fallback.sh 4 // DO NOT MODIFY THIS FILE DIRECTLY 5 6 #ifndef _LINUX_ATOMIC_FALLBACK_H 7 #define _LINUX_ATOMIC_FALLBACK_H 8 9 #include <linux/compiler.h> 10 11 #ifndef arch_xchg_relaxed 12 #define arch_xchg_acquire arch_xchg 13 #define arch_xchg_release arch_xchg 14 #define arch_xchg_relaxed arch_xchg 15 #else /* arch_xchg_relaxed */ 16 17 #ifndef arch_xchg_acquire 18 #define arch_xchg_acquire(...) \ 19 __atomic_op_acquire(arch_xchg, __VA_ARGS__) 20 #endif 21 22 #ifndef arch_xchg_release 23 #define arch_xchg_release(...) \ 24 __atomic_op_release(arch_xchg, __VA_ARGS__) 25 #endif 26 27 #ifndef arch_xchg 28 #define arch_xchg(...) \ 29 __atomic_op_fence(arch_xchg, __VA_ARGS__) 30 #endif 31 32 #endif /* arch_xchg_relaxed */ 33 34 #ifndef arch_cmpxchg_relaxed 35 #define arch_cmpxchg_acquire arch_cmpxchg 36 #define arch_cmpxchg_release arch_cmpxchg 37 #define arch_cmpxchg_relaxed arch_cmpxchg 38 #else /* arch_cmpxchg_relaxed */ 39 40 #ifndef arch_cmpxchg_acquire 41 #define arch_cmpxchg_acquire(...) \ 42 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) 43 #endif 44 45 #ifndef arch_cmpxchg_release 46 #define arch_cmpxchg_release(...) \ 47 __atomic_op_release(arch_cmpxchg, __VA_ARGS__) 48 #endif 49 50 #ifndef arch_cmpxchg 51 #define arch_cmpxchg(...) \ 52 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) 53 #endif 54 55 #endif /* arch_cmpxchg_relaxed */ 56 57 #ifndef arch_cmpxchg64_relaxed 58 #define arch_cmpxchg64_acquire arch_cmpxchg64 59 #define arch_cmpxchg64_release arch_cmpxchg64 60 #define arch_cmpxchg64_relaxed arch_cmpxchg64 61 #else /* arch_cmpxchg64_relaxed */ 62 63 #ifndef arch_cmpxchg64_acquire 64 #define arch_cmpxchg64_acquire(...) \ 65 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) 66 #endif 67 68 #ifndef arch_cmpxchg64_release 69 #define arch_cmpxchg64_release(...) \ 70 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) 71 #endif 72 73 #ifndef arch_cmpxchg64 74 #define arch_cmpxchg64(...) \ 75 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) 76 #endif 77 78 #endif /* arch_cmpxchg64_relaxed */ 79 80 #ifndef arch_try_cmpxchg_relaxed 81 #ifdef arch_try_cmpxchg 82 #define arch_try_cmpxchg_acquire arch_try_cmpxchg 83 #define arch_try_cmpxchg_release arch_try_cmpxchg 84 #define arch_try_cmpxchg_relaxed arch_try_cmpxchg 85 #endif /* arch_try_cmpxchg */ 86 87 #ifndef arch_try_cmpxchg 88 #define arch_try_cmpxchg(_ptr, _oldp, _new) \ 89 ({ \ 90 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 91 ___r = arch_cmpxchg((_ptr), ___o, (_new)); \ 92 if (unlikely(___r != ___o)) \ 93 *___op = ___r; \ 94 likely(___r == ___o); \ 95 }) 96 #endif /* arch_try_cmpxchg */ 97 98 #ifndef arch_try_cmpxchg_acquire 99 #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \ 100 ({ \ 101 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 102 ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \ 103 if (unlikely(___r != ___o)) \ 104 *___op = ___r; \ 105 likely(___r == ___o); \ 106 }) 107 #endif /* arch_try_cmpxchg_acquire */ 108 109 #ifndef arch_try_cmpxchg_release 110 #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \ 111 ({ \ 112 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 113 ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \ 114 if (unlikely(___r != ___o)) \ 115 *___op = ___r; \ 116 likely(___r == ___o); \ 117 }) 118 #endif /* arch_try_cmpxchg_release */ 119 120 #ifndef arch_try_cmpxchg_relaxed 121 #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ 122 ({ \ 123 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 124 ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \ 125 if (unlikely(___r != ___o)) \ 126 *___op = ___r; \ 127 likely(___r == ___o); \ 128 }) 129 #endif /* arch_try_cmpxchg_relaxed */ 130 131 #else /* arch_try_cmpxchg_relaxed */ 132 133 #ifndef arch_try_cmpxchg_acquire 134 #define arch_try_cmpxchg_acquire(...) \ 135 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) 136 #endif 137 138 #ifndef arch_try_cmpxchg_release 139 #define arch_try_cmpxchg_release(...) \ 140 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) 141 #endif 142 143 #ifndef arch_try_cmpxchg 144 #define arch_try_cmpxchg(...) \ 145 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) 146 #endif 147 148 #endif /* arch_try_cmpxchg_relaxed */ 149 150 #ifndef arch_atomic_read_acquire 151 static __always_inline int 152 arch_atomic_read_acquire(const atomic_t *v) 153 { 154 int ret; 155 156 if (__native_word(atomic_t)) { 157 ret = smp_load_acquire(&(v)->counter); 158 } else { 159 ret = arch_atomic_read(v); 160 __atomic_acquire_fence(); 161 } 162 163 return ret; 164 } 165 #define arch_atomic_read_acquire arch_atomic_read_acquire 166 #endif 167 168 #ifndef arch_atomic_set_release 169 static __always_inline void 170 arch_atomic_set_release(atomic_t *v, int i) 171 { 172 if (__native_word(atomic_t)) { 173 smp_store_release(&(v)->counter, i); 174 } else { 175 __atomic_release_fence(); 176 arch_atomic_set(v, i); 177 } 178 } 179 #define arch_atomic_set_release arch_atomic_set_release 180 #endif 181 182 #ifndef arch_atomic_add_return_relaxed 183 #define arch_atomic_add_return_acquire arch_atomic_add_return 184 #define arch_atomic_add_return_release arch_atomic_add_return 185 #define arch_atomic_add_return_relaxed arch_atomic_add_return 186 #else /* arch_atomic_add_return_relaxed */ 187 188 #ifndef arch_atomic_add_return_acquire 189 static __always_inline int 190 arch_atomic_add_return_acquire(int i, atomic_t *v) 191 { 192 int ret = arch_atomic_add_return_relaxed(i, v); 193 __atomic_acquire_fence(); 194 return ret; 195 } 196 #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire 197 #endif 198 199 #ifndef arch_atomic_add_return_release 200 static __always_inline int 201 arch_atomic_add_return_release(int i, atomic_t *v) 202 { 203 __atomic_release_fence(); 204 return arch_atomic_add_return_relaxed(i, v); 205 } 206 #define arch_atomic_add_return_release arch_atomic_add_return_release 207 #endif 208 209 #ifndef arch_atomic_add_return 210 static __always_inline int 211 arch_atomic_add_return(int i, atomic_t *v) 212 { 213 int ret; 214 __atomic_pre_full_fence(); 215 ret = arch_atomic_add_return_relaxed(i, v); 216 __atomic_post_full_fence(); 217 return ret; 218 } 219 #define arch_atomic_add_return arch_atomic_add_return 220 #endif 221 222 #endif /* arch_atomic_add_return_relaxed */ 223 224 #ifndef arch_atomic_fetch_add_relaxed 225 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add 226 #define arch_atomic_fetch_add_release arch_atomic_fetch_add 227 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add 228 #else /* arch_atomic_fetch_add_relaxed */ 229 230 #ifndef arch_atomic_fetch_add_acquire 231 static __always_inline int 232 arch_atomic_fetch_add_acquire(int i, atomic_t *v) 233 { 234 int ret = arch_atomic_fetch_add_relaxed(i, v); 235 __atomic_acquire_fence(); 236 return ret; 237 } 238 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire 239 #endif 240 241 #ifndef arch_atomic_fetch_add_release 242 static __always_inline int 243 arch_atomic_fetch_add_release(int i, atomic_t *v) 244 { 245 __atomic_release_fence(); 246 return arch_atomic_fetch_add_relaxed(i, v); 247 } 248 #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release 249 #endif 250 251 #ifndef arch_atomic_fetch_add 252 static __always_inline int 253 arch_atomic_fetch_add(int i, atomic_t *v) 254 { 255 int ret; 256 __atomic_pre_full_fence(); 257 ret = arch_atomic_fetch_add_relaxed(i, v); 258 __atomic_post_full_fence(); 259 return ret; 260 } 261 #define arch_atomic_fetch_add arch_atomic_fetch_add 262 #endif 263 264 #endif /* arch_atomic_fetch_add_relaxed */ 265 266 #ifndef arch_atomic_sub_return_relaxed 267 #define arch_atomic_sub_return_acquire arch_atomic_sub_return 268 #define arch_atomic_sub_return_release arch_atomic_sub_return 269 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return 270 #else /* arch_atomic_sub_return_relaxed */ 271 272 #ifndef arch_atomic_sub_return_acquire 273 static __always_inline int 274 arch_atomic_sub_return_acquire(int i, atomic_t *v) 275 { 276 int ret = arch_atomic_sub_return_relaxed(i, v); 277 __atomic_acquire_fence(); 278 return ret; 279 } 280 #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire 281 #endif 282 283 #ifndef arch_atomic_sub_return_release 284 static __always_inline int 285 arch_atomic_sub_return_release(int i, atomic_t *v) 286 { 287 __atomic_release_fence(); 288 return arch_atomic_sub_return_relaxed(i, v); 289 } 290 #define arch_atomic_sub_return_release arch_atomic_sub_return_release 291 #endif 292 293 #ifndef arch_atomic_sub_return 294 static __always_inline int 295 arch_atomic_sub_return(int i, atomic_t *v) 296 { 297 int ret; 298 __atomic_pre_full_fence(); 299 ret = arch_atomic_sub_return_relaxed(i, v); 300 __atomic_post_full_fence(); 301 return ret; 302 } 303 #define arch_atomic_sub_return arch_atomic_sub_return 304 #endif 305 306 #endif /* arch_atomic_sub_return_relaxed */ 307 308 #ifndef arch_atomic_fetch_sub_relaxed 309 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub 310 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub 311 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub 312 #else /* arch_atomic_fetch_sub_relaxed */ 313 314 #ifndef arch_atomic_fetch_sub_acquire 315 static __always_inline int 316 arch_atomic_fetch_sub_acquire(int i, atomic_t *v) 317 { 318 int ret = arch_atomic_fetch_sub_relaxed(i, v); 319 __atomic_acquire_fence(); 320 return ret; 321 } 322 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire 323 #endif 324 325 #ifndef arch_atomic_fetch_sub_release 326 static __always_inline int 327 arch_atomic_fetch_sub_release(int i, atomic_t *v) 328 { 329 __atomic_release_fence(); 330 return arch_atomic_fetch_sub_relaxed(i, v); 331 } 332 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release 333 #endif 334 335 #ifndef arch_atomic_fetch_sub 336 static __always_inline int 337 arch_atomic_fetch_sub(int i, atomic_t *v) 338 { 339 int ret; 340 __atomic_pre_full_fence(); 341 ret = arch_atomic_fetch_sub_relaxed(i, v); 342 __atomic_post_full_fence(); 343 return ret; 344 } 345 #define arch_atomic_fetch_sub arch_atomic_fetch_sub 346 #endif 347 348 #endif /* arch_atomic_fetch_sub_relaxed */ 349 350 #ifndef arch_atomic_inc 351 static __always_inline void 352 arch_atomic_inc(atomic_t *v) 353 { 354 arch_atomic_add(1, v); 355 } 356 #define arch_atomic_inc arch_atomic_inc 357 #endif 358 359 #ifndef arch_atomic_inc_return_relaxed 360 #ifdef arch_atomic_inc_return 361 #define arch_atomic_inc_return_acquire arch_atomic_inc_return 362 #define arch_atomic_inc_return_release arch_atomic_inc_return 363 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return 364 #endif /* arch_atomic_inc_return */ 365 366 #ifndef arch_atomic_inc_return 367 static __always_inline int 368 arch_atomic_inc_return(atomic_t *v) 369 { 370 return arch_atomic_add_return(1, v); 371 } 372 #define arch_atomic_inc_return arch_atomic_inc_return 373 #endif 374 375 #ifndef arch_atomic_inc_return_acquire 376 static __always_inline int 377 arch_atomic_inc_return_acquire(atomic_t *v) 378 { 379 return arch_atomic_add_return_acquire(1, v); 380 } 381 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 382 #endif 383 384 #ifndef arch_atomic_inc_return_release 385 static __always_inline int 386 arch_atomic_inc_return_release(atomic_t *v) 387 { 388 return arch_atomic_add_return_release(1, v); 389 } 390 #define arch_atomic_inc_return_release arch_atomic_inc_return_release 391 #endif 392 393 #ifndef arch_atomic_inc_return_relaxed 394 static __always_inline int 395 arch_atomic_inc_return_relaxed(atomic_t *v) 396 { 397 return arch_atomic_add_return_relaxed(1, v); 398 } 399 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed 400 #endif 401 402 #else /* arch_atomic_inc_return_relaxed */ 403 404 #ifndef arch_atomic_inc_return_acquire 405 static __always_inline int 406 arch_atomic_inc_return_acquire(atomic_t *v) 407 { 408 int ret = arch_atomic_inc_return_relaxed(v); 409 __atomic_acquire_fence(); 410 return ret; 411 } 412 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 413 #endif 414 415 #ifndef arch_atomic_inc_return_release 416 static __always_inline int 417 arch_atomic_inc_return_release(atomic_t *v) 418 { 419 __atomic_release_fence(); 420 return arch_atomic_inc_return_relaxed(v); 421 } 422 #define arch_atomic_inc_return_release arch_atomic_inc_return_release 423 #endif 424 425 #ifndef arch_atomic_inc_return 426 static __always_inline int 427 arch_atomic_inc_return(atomic_t *v) 428 { 429 int ret; 430 __atomic_pre_full_fence(); 431 ret = arch_atomic_inc_return_relaxed(v); 432 __atomic_post_full_fence(); 433 return ret; 434 } 435 #define arch_atomic_inc_return arch_atomic_inc_return 436 #endif 437 438 #endif /* arch_atomic_inc_return_relaxed */ 439 440 #ifndef arch_atomic_fetch_inc_relaxed 441 #ifdef arch_atomic_fetch_inc 442 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc 443 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc 444 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc 445 #endif /* arch_atomic_fetch_inc */ 446 447 #ifndef arch_atomic_fetch_inc 448 static __always_inline int 449 arch_atomic_fetch_inc(atomic_t *v) 450 { 451 return arch_atomic_fetch_add(1, v); 452 } 453 #define arch_atomic_fetch_inc arch_atomic_fetch_inc 454 #endif 455 456 #ifndef arch_atomic_fetch_inc_acquire 457 static __always_inline int 458 arch_atomic_fetch_inc_acquire(atomic_t *v) 459 { 460 return arch_atomic_fetch_add_acquire(1, v); 461 } 462 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 463 #endif 464 465 #ifndef arch_atomic_fetch_inc_release 466 static __always_inline int 467 arch_atomic_fetch_inc_release(atomic_t *v) 468 { 469 return arch_atomic_fetch_add_release(1, v); 470 } 471 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 472 #endif 473 474 #ifndef arch_atomic_fetch_inc_relaxed 475 static __always_inline int 476 arch_atomic_fetch_inc_relaxed(atomic_t *v) 477 { 478 return arch_atomic_fetch_add_relaxed(1, v); 479 } 480 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed 481 #endif 482 483 #else /* arch_atomic_fetch_inc_relaxed */ 484 485 #ifndef arch_atomic_fetch_inc_acquire 486 static __always_inline int 487 arch_atomic_fetch_inc_acquire(atomic_t *v) 488 { 489 int ret = arch_atomic_fetch_inc_relaxed(v); 490 __atomic_acquire_fence(); 491 return ret; 492 } 493 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 494 #endif 495 496 #ifndef arch_atomic_fetch_inc_release 497 static __always_inline int 498 arch_atomic_fetch_inc_release(atomic_t *v) 499 { 500 __atomic_release_fence(); 501 return arch_atomic_fetch_inc_relaxed(v); 502 } 503 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 504 #endif 505 506 #ifndef arch_atomic_fetch_inc 507 static __always_inline int 508 arch_atomic_fetch_inc(atomic_t *v) 509 { 510 int ret; 511 __atomic_pre_full_fence(); 512 ret = arch_atomic_fetch_inc_relaxed(v); 513 __atomic_post_full_fence(); 514 return ret; 515 } 516 #define arch_atomic_fetch_inc arch_atomic_fetch_inc 517 #endif 518 519 #endif /* arch_atomic_fetch_inc_relaxed */ 520 521 #ifndef arch_atomic_dec 522 static __always_inline void 523 arch_atomic_dec(atomic_t *v) 524 { 525 arch_atomic_sub(1, v); 526 } 527 #define arch_atomic_dec arch_atomic_dec 528 #endif 529 530 #ifndef arch_atomic_dec_return_relaxed 531 #ifdef arch_atomic_dec_return 532 #define arch_atomic_dec_return_acquire arch_atomic_dec_return 533 #define arch_atomic_dec_return_release arch_atomic_dec_return 534 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return 535 #endif /* arch_atomic_dec_return */ 536 537 #ifndef arch_atomic_dec_return 538 static __always_inline int 539 arch_atomic_dec_return(atomic_t *v) 540 { 541 return arch_atomic_sub_return(1, v); 542 } 543 #define arch_atomic_dec_return arch_atomic_dec_return 544 #endif 545 546 #ifndef arch_atomic_dec_return_acquire 547 static __always_inline int 548 arch_atomic_dec_return_acquire(atomic_t *v) 549 { 550 return arch_atomic_sub_return_acquire(1, v); 551 } 552 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 553 #endif 554 555 #ifndef arch_atomic_dec_return_release 556 static __always_inline int 557 arch_atomic_dec_return_release(atomic_t *v) 558 { 559 return arch_atomic_sub_return_release(1, v); 560 } 561 #define arch_atomic_dec_return_release arch_atomic_dec_return_release 562 #endif 563 564 #ifndef arch_atomic_dec_return_relaxed 565 static __always_inline int 566 arch_atomic_dec_return_relaxed(atomic_t *v) 567 { 568 return arch_atomic_sub_return_relaxed(1, v); 569 } 570 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed 571 #endif 572 573 #else /* arch_atomic_dec_return_relaxed */ 574 575 #ifndef arch_atomic_dec_return_acquire 576 static __always_inline int 577 arch_atomic_dec_return_acquire(atomic_t *v) 578 { 579 int ret = arch_atomic_dec_return_relaxed(v); 580 __atomic_acquire_fence(); 581 return ret; 582 } 583 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 584 #endif 585 586 #ifndef arch_atomic_dec_return_release 587 static __always_inline int 588 arch_atomic_dec_return_release(atomic_t *v) 589 { 590 __atomic_release_fence(); 591 return arch_atomic_dec_return_relaxed(v); 592 } 593 #define arch_atomic_dec_return_release arch_atomic_dec_return_release 594 #endif 595 596 #ifndef arch_atomic_dec_return 597 static __always_inline int 598 arch_atomic_dec_return(atomic_t *v) 599 { 600 int ret; 601 __atomic_pre_full_fence(); 602 ret = arch_atomic_dec_return_relaxed(v); 603 __atomic_post_full_fence(); 604 return ret; 605 } 606 #define arch_atomic_dec_return arch_atomic_dec_return 607 #endif 608 609 #endif /* arch_atomic_dec_return_relaxed */ 610 611 #ifndef arch_atomic_fetch_dec_relaxed 612 #ifdef arch_atomic_fetch_dec 613 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec 614 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec 615 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec 616 #endif /* arch_atomic_fetch_dec */ 617 618 #ifndef arch_atomic_fetch_dec 619 static __always_inline int 620 arch_atomic_fetch_dec(atomic_t *v) 621 { 622 return arch_atomic_fetch_sub(1, v); 623 } 624 #define arch_atomic_fetch_dec arch_atomic_fetch_dec 625 #endif 626 627 #ifndef arch_atomic_fetch_dec_acquire 628 static __always_inline int 629 arch_atomic_fetch_dec_acquire(atomic_t *v) 630 { 631 return arch_atomic_fetch_sub_acquire(1, v); 632 } 633 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 634 #endif 635 636 #ifndef arch_atomic_fetch_dec_release 637 static __always_inline int 638 arch_atomic_fetch_dec_release(atomic_t *v) 639 { 640 return arch_atomic_fetch_sub_release(1, v); 641 } 642 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 643 #endif 644 645 #ifndef arch_atomic_fetch_dec_relaxed 646 static __always_inline int 647 arch_atomic_fetch_dec_relaxed(atomic_t *v) 648 { 649 return arch_atomic_fetch_sub_relaxed(1, v); 650 } 651 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed 652 #endif 653 654 #else /* arch_atomic_fetch_dec_relaxed */ 655 656 #ifndef arch_atomic_fetch_dec_acquire 657 static __always_inline int 658 arch_atomic_fetch_dec_acquire(atomic_t *v) 659 { 660 int ret = arch_atomic_fetch_dec_relaxed(v); 661 __atomic_acquire_fence(); 662 return ret; 663 } 664 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 665 #endif 666 667 #ifndef arch_atomic_fetch_dec_release 668 static __always_inline int 669 arch_atomic_fetch_dec_release(atomic_t *v) 670 { 671 __atomic_release_fence(); 672 return arch_atomic_fetch_dec_relaxed(v); 673 } 674 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 675 #endif 676 677 #ifndef arch_atomic_fetch_dec 678 static __always_inline int 679 arch_atomic_fetch_dec(atomic_t *v) 680 { 681 int ret; 682 __atomic_pre_full_fence(); 683 ret = arch_atomic_fetch_dec_relaxed(v); 684 __atomic_post_full_fence(); 685 return ret; 686 } 687 #define arch_atomic_fetch_dec arch_atomic_fetch_dec 688 #endif 689 690 #endif /* arch_atomic_fetch_dec_relaxed */ 691 692 #ifndef arch_atomic_fetch_and_relaxed 693 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and 694 #define arch_atomic_fetch_and_release arch_atomic_fetch_and 695 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and 696 #else /* arch_atomic_fetch_and_relaxed */ 697 698 #ifndef arch_atomic_fetch_and_acquire 699 static __always_inline int 700 arch_atomic_fetch_and_acquire(int i, atomic_t *v) 701 { 702 int ret = arch_atomic_fetch_and_relaxed(i, v); 703 __atomic_acquire_fence(); 704 return ret; 705 } 706 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire 707 #endif 708 709 #ifndef arch_atomic_fetch_and_release 710 static __always_inline int 711 arch_atomic_fetch_and_release(int i, atomic_t *v) 712 { 713 __atomic_release_fence(); 714 return arch_atomic_fetch_and_relaxed(i, v); 715 } 716 #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release 717 #endif 718 719 #ifndef arch_atomic_fetch_and 720 static __always_inline int 721 arch_atomic_fetch_and(int i, atomic_t *v) 722 { 723 int ret; 724 __atomic_pre_full_fence(); 725 ret = arch_atomic_fetch_and_relaxed(i, v); 726 __atomic_post_full_fence(); 727 return ret; 728 } 729 #define arch_atomic_fetch_and arch_atomic_fetch_and 730 #endif 731 732 #endif /* arch_atomic_fetch_and_relaxed */ 733 734 #ifndef arch_atomic_andnot 735 static __always_inline void 736 arch_atomic_andnot(int i, atomic_t *v) 737 { 738 arch_atomic_and(~i, v); 739 } 740 #define arch_atomic_andnot arch_atomic_andnot 741 #endif 742 743 #ifndef arch_atomic_fetch_andnot_relaxed 744 #ifdef arch_atomic_fetch_andnot 745 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot 746 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot 747 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot 748 #endif /* arch_atomic_fetch_andnot */ 749 750 #ifndef arch_atomic_fetch_andnot 751 static __always_inline int 752 arch_atomic_fetch_andnot(int i, atomic_t *v) 753 { 754 return arch_atomic_fetch_and(~i, v); 755 } 756 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 757 #endif 758 759 #ifndef arch_atomic_fetch_andnot_acquire 760 static __always_inline int 761 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 762 { 763 return arch_atomic_fetch_and_acquire(~i, v); 764 } 765 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 766 #endif 767 768 #ifndef arch_atomic_fetch_andnot_release 769 static __always_inline int 770 arch_atomic_fetch_andnot_release(int i, atomic_t *v) 771 { 772 return arch_atomic_fetch_and_release(~i, v); 773 } 774 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 775 #endif 776 777 #ifndef arch_atomic_fetch_andnot_relaxed 778 static __always_inline int 779 arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 780 { 781 return arch_atomic_fetch_and_relaxed(~i, v); 782 } 783 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 784 #endif 785 786 #else /* arch_atomic_fetch_andnot_relaxed */ 787 788 #ifndef arch_atomic_fetch_andnot_acquire 789 static __always_inline int 790 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 791 { 792 int ret = arch_atomic_fetch_andnot_relaxed(i, v); 793 __atomic_acquire_fence(); 794 return ret; 795 } 796 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 797 #endif 798 799 #ifndef arch_atomic_fetch_andnot_release 800 static __always_inline int 801 arch_atomic_fetch_andnot_release(int i, atomic_t *v) 802 { 803 __atomic_release_fence(); 804 return arch_atomic_fetch_andnot_relaxed(i, v); 805 } 806 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 807 #endif 808 809 #ifndef arch_atomic_fetch_andnot 810 static __always_inline int 811 arch_atomic_fetch_andnot(int i, atomic_t *v) 812 { 813 int ret; 814 __atomic_pre_full_fence(); 815 ret = arch_atomic_fetch_andnot_relaxed(i, v); 816 __atomic_post_full_fence(); 817 return ret; 818 } 819 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 820 #endif 821 822 #endif /* arch_atomic_fetch_andnot_relaxed */ 823 824 #ifndef arch_atomic_fetch_or_relaxed 825 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or 826 #define arch_atomic_fetch_or_release arch_atomic_fetch_or 827 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or 828 #else /* arch_atomic_fetch_or_relaxed */ 829 830 #ifndef arch_atomic_fetch_or_acquire 831 static __always_inline int 832 arch_atomic_fetch_or_acquire(int i, atomic_t *v) 833 { 834 int ret = arch_atomic_fetch_or_relaxed(i, v); 835 __atomic_acquire_fence(); 836 return ret; 837 } 838 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire 839 #endif 840 841 #ifndef arch_atomic_fetch_or_release 842 static __always_inline int 843 arch_atomic_fetch_or_release(int i, atomic_t *v) 844 { 845 __atomic_release_fence(); 846 return arch_atomic_fetch_or_relaxed(i, v); 847 } 848 #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release 849 #endif 850 851 #ifndef arch_atomic_fetch_or 852 static __always_inline int 853 arch_atomic_fetch_or(int i, atomic_t *v) 854 { 855 int ret; 856 __atomic_pre_full_fence(); 857 ret = arch_atomic_fetch_or_relaxed(i, v); 858 __atomic_post_full_fence(); 859 return ret; 860 } 861 #define arch_atomic_fetch_or arch_atomic_fetch_or 862 #endif 863 864 #endif /* arch_atomic_fetch_or_relaxed */ 865 866 #ifndef arch_atomic_fetch_xor_relaxed 867 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor 868 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor 869 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor 870 #else /* arch_atomic_fetch_xor_relaxed */ 871 872 #ifndef arch_atomic_fetch_xor_acquire 873 static __always_inline int 874 arch_atomic_fetch_xor_acquire(int i, atomic_t *v) 875 { 876 int ret = arch_atomic_fetch_xor_relaxed(i, v); 877 __atomic_acquire_fence(); 878 return ret; 879 } 880 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire 881 #endif 882 883 #ifndef arch_atomic_fetch_xor_release 884 static __always_inline int 885 arch_atomic_fetch_xor_release(int i, atomic_t *v) 886 { 887 __atomic_release_fence(); 888 return arch_atomic_fetch_xor_relaxed(i, v); 889 } 890 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release 891 #endif 892 893 #ifndef arch_atomic_fetch_xor 894 static __always_inline int 895 arch_atomic_fetch_xor(int i, atomic_t *v) 896 { 897 int ret; 898 __atomic_pre_full_fence(); 899 ret = arch_atomic_fetch_xor_relaxed(i, v); 900 __atomic_post_full_fence(); 901 return ret; 902 } 903 #define arch_atomic_fetch_xor arch_atomic_fetch_xor 904 #endif 905 906 #endif /* arch_atomic_fetch_xor_relaxed */ 907 908 #ifndef arch_atomic_xchg_relaxed 909 #define arch_atomic_xchg_acquire arch_atomic_xchg 910 #define arch_atomic_xchg_release arch_atomic_xchg 911 #define arch_atomic_xchg_relaxed arch_atomic_xchg 912 #else /* arch_atomic_xchg_relaxed */ 913 914 #ifndef arch_atomic_xchg_acquire 915 static __always_inline int 916 arch_atomic_xchg_acquire(atomic_t *v, int i) 917 { 918 int ret = arch_atomic_xchg_relaxed(v, i); 919 __atomic_acquire_fence(); 920 return ret; 921 } 922 #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire 923 #endif 924 925 #ifndef arch_atomic_xchg_release 926 static __always_inline int 927 arch_atomic_xchg_release(atomic_t *v, int i) 928 { 929 __atomic_release_fence(); 930 return arch_atomic_xchg_relaxed(v, i); 931 } 932 #define arch_atomic_xchg_release arch_atomic_xchg_release 933 #endif 934 935 #ifndef arch_atomic_xchg 936 static __always_inline int 937 arch_atomic_xchg(atomic_t *v, int i) 938 { 939 int ret; 940 __atomic_pre_full_fence(); 941 ret = arch_atomic_xchg_relaxed(v, i); 942 __atomic_post_full_fence(); 943 return ret; 944 } 945 #define arch_atomic_xchg arch_atomic_xchg 946 #endif 947 948 #endif /* arch_atomic_xchg_relaxed */ 949 950 #ifndef arch_atomic_cmpxchg_relaxed 951 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg 952 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg 953 #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg 954 #else /* arch_atomic_cmpxchg_relaxed */ 955 956 #ifndef arch_atomic_cmpxchg_acquire 957 static __always_inline int 958 arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 959 { 960 int ret = arch_atomic_cmpxchg_relaxed(v, old, new); 961 __atomic_acquire_fence(); 962 return ret; 963 } 964 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire 965 #endif 966 967 #ifndef arch_atomic_cmpxchg_release 968 static __always_inline int 969 arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) 970 { 971 __atomic_release_fence(); 972 return arch_atomic_cmpxchg_relaxed(v, old, new); 973 } 974 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release 975 #endif 976 977 #ifndef arch_atomic_cmpxchg 978 static __always_inline int 979 arch_atomic_cmpxchg(atomic_t *v, int old, int new) 980 { 981 int ret; 982 __atomic_pre_full_fence(); 983 ret = arch_atomic_cmpxchg_relaxed(v, old, new); 984 __atomic_post_full_fence(); 985 return ret; 986 } 987 #define arch_atomic_cmpxchg arch_atomic_cmpxchg 988 #endif 989 990 #endif /* arch_atomic_cmpxchg_relaxed */ 991 992 #ifndef arch_atomic_try_cmpxchg_relaxed 993 #ifdef arch_atomic_try_cmpxchg 994 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg 995 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg 996 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg 997 #endif /* arch_atomic_try_cmpxchg */ 998 999 #ifndef arch_atomic_try_cmpxchg 1000 static __always_inline bool 1001 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1002 { 1003 int r, o = *old; 1004 r = arch_atomic_cmpxchg(v, o, new); 1005 if (unlikely(r != o)) 1006 *old = r; 1007 return likely(r == o); 1008 } 1009 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1010 #endif 1011 1012 #ifndef arch_atomic_try_cmpxchg_acquire 1013 static __always_inline bool 1014 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1015 { 1016 int r, o = *old; 1017 r = arch_atomic_cmpxchg_acquire(v, o, new); 1018 if (unlikely(r != o)) 1019 *old = r; 1020 return likely(r == o); 1021 } 1022 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1023 #endif 1024 1025 #ifndef arch_atomic_try_cmpxchg_release 1026 static __always_inline bool 1027 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1028 { 1029 int r, o = *old; 1030 r = arch_atomic_cmpxchg_release(v, o, new); 1031 if (unlikely(r != o)) 1032 *old = r; 1033 return likely(r == o); 1034 } 1035 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1036 #endif 1037 1038 #ifndef arch_atomic_try_cmpxchg_relaxed 1039 static __always_inline bool 1040 arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 1041 { 1042 int r, o = *old; 1043 r = arch_atomic_cmpxchg_relaxed(v, o, new); 1044 if (unlikely(r != o)) 1045 *old = r; 1046 return likely(r == o); 1047 } 1048 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed 1049 #endif 1050 1051 #else /* arch_atomic_try_cmpxchg_relaxed */ 1052 1053 #ifndef arch_atomic_try_cmpxchg_acquire 1054 static __always_inline bool 1055 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1056 { 1057 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1058 __atomic_acquire_fence(); 1059 return ret; 1060 } 1061 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1062 #endif 1063 1064 #ifndef arch_atomic_try_cmpxchg_release 1065 static __always_inline bool 1066 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1067 { 1068 __atomic_release_fence(); 1069 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 1070 } 1071 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1072 #endif 1073 1074 #ifndef arch_atomic_try_cmpxchg 1075 static __always_inline bool 1076 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1077 { 1078 bool ret; 1079 __atomic_pre_full_fence(); 1080 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1081 __atomic_post_full_fence(); 1082 return ret; 1083 } 1084 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1085 #endif 1086 1087 #endif /* arch_atomic_try_cmpxchg_relaxed */ 1088 1089 #ifndef arch_atomic_sub_and_test 1090 /** 1091 * arch_atomic_sub_and_test - subtract value from variable and test result 1092 * @i: integer value to subtract 1093 * @v: pointer of type atomic_t 1094 * 1095 * Atomically subtracts @i from @v and returns 1096 * true if the result is zero, or false for all 1097 * other cases. 1098 */ 1099 static __always_inline bool 1100 arch_atomic_sub_and_test(int i, atomic_t *v) 1101 { 1102 return arch_atomic_sub_return(i, v) == 0; 1103 } 1104 #define arch_atomic_sub_and_test arch_atomic_sub_and_test 1105 #endif 1106 1107 #ifndef arch_atomic_dec_and_test 1108 /** 1109 * arch_atomic_dec_and_test - decrement and test 1110 * @v: pointer of type atomic_t 1111 * 1112 * Atomically decrements @v by 1 and 1113 * returns true if the result is 0, or false for all other 1114 * cases. 1115 */ 1116 static __always_inline bool 1117 arch_atomic_dec_and_test(atomic_t *v) 1118 { 1119 return arch_atomic_dec_return(v) == 0; 1120 } 1121 #define arch_atomic_dec_and_test arch_atomic_dec_and_test 1122 #endif 1123 1124 #ifndef arch_atomic_inc_and_test 1125 /** 1126 * arch_atomic_inc_and_test - increment and test 1127 * @v: pointer of type atomic_t 1128 * 1129 * Atomically increments @v by 1 1130 * and returns true if the result is zero, or false for all 1131 * other cases. 1132 */ 1133 static __always_inline bool 1134 arch_atomic_inc_and_test(atomic_t *v) 1135 { 1136 return arch_atomic_inc_return(v) == 0; 1137 } 1138 #define arch_atomic_inc_and_test arch_atomic_inc_and_test 1139 #endif 1140 1141 #ifndef arch_atomic_add_negative 1142 /** 1143 * arch_atomic_add_negative - add and test if negative 1144 * @i: integer value to add 1145 * @v: pointer of type atomic_t 1146 * 1147 * Atomically adds @i to @v and returns true 1148 * if the result is negative, or false when 1149 * result is greater than or equal to zero. 1150 */ 1151 static __always_inline bool 1152 arch_atomic_add_negative(int i, atomic_t *v) 1153 { 1154 return arch_atomic_add_return(i, v) < 0; 1155 } 1156 #define arch_atomic_add_negative arch_atomic_add_negative 1157 #endif 1158 1159 #ifndef arch_atomic_fetch_add_unless 1160 /** 1161 * arch_atomic_fetch_add_unless - add unless the number is already a given value 1162 * @v: pointer of type atomic_t 1163 * @a: the amount to add to v... 1164 * @u: ...unless v is equal to u. 1165 * 1166 * Atomically adds @a to @v, so long as @v was not already @u. 1167 * Returns original value of @v 1168 */ 1169 static __always_inline int 1170 arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 1171 { 1172 int c = arch_atomic_read(v); 1173 1174 do { 1175 if (unlikely(c == u)) 1176 break; 1177 } while (!arch_atomic_try_cmpxchg(v, &c, c + a)); 1178 1179 return c; 1180 } 1181 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 1182 #endif 1183 1184 #ifndef arch_atomic_add_unless 1185 /** 1186 * arch_atomic_add_unless - add unless the number is already a given value 1187 * @v: pointer of type atomic_t 1188 * @a: the amount to add to v... 1189 * @u: ...unless v is equal to u. 1190 * 1191 * Atomically adds @a to @v, if @v was not already @u. 1192 * Returns true if the addition was done. 1193 */ 1194 static __always_inline bool 1195 arch_atomic_add_unless(atomic_t *v, int a, int u) 1196 { 1197 return arch_atomic_fetch_add_unless(v, a, u) != u; 1198 } 1199 #define arch_atomic_add_unless arch_atomic_add_unless 1200 #endif 1201 1202 #ifndef arch_atomic_inc_not_zero 1203 /** 1204 * arch_atomic_inc_not_zero - increment unless the number is zero 1205 * @v: pointer of type atomic_t 1206 * 1207 * Atomically increments @v by 1, if @v is non-zero. 1208 * Returns true if the increment was done. 1209 */ 1210 static __always_inline bool 1211 arch_atomic_inc_not_zero(atomic_t *v) 1212 { 1213 return arch_atomic_add_unless(v, 1, 0); 1214 } 1215 #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero 1216 #endif 1217 1218 #ifndef arch_atomic_inc_unless_negative 1219 static __always_inline bool 1220 arch_atomic_inc_unless_negative(atomic_t *v) 1221 { 1222 int c = arch_atomic_read(v); 1223 1224 do { 1225 if (unlikely(c < 0)) 1226 return false; 1227 } while (!arch_atomic_try_cmpxchg(v, &c, c + 1)); 1228 1229 return true; 1230 } 1231 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative 1232 #endif 1233 1234 #ifndef arch_atomic_dec_unless_positive 1235 static __always_inline bool 1236 arch_atomic_dec_unless_positive(atomic_t *v) 1237 { 1238 int c = arch_atomic_read(v); 1239 1240 do { 1241 if (unlikely(c > 0)) 1242 return false; 1243 } while (!arch_atomic_try_cmpxchg(v, &c, c - 1)); 1244 1245 return true; 1246 } 1247 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive 1248 #endif 1249 1250 #ifndef arch_atomic_dec_if_positive 1251 static __always_inline int 1252 arch_atomic_dec_if_positive(atomic_t *v) 1253 { 1254 int dec, c = arch_atomic_read(v); 1255 1256 do { 1257 dec = c - 1; 1258 if (unlikely(dec < 0)) 1259 break; 1260 } while (!arch_atomic_try_cmpxchg(v, &c, dec)); 1261 1262 return dec; 1263 } 1264 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 1265 #endif 1266 1267 #ifdef CONFIG_GENERIC_ATOMIC64 1268 #include <asm-generic/atomic64.h> 1269 #endif 1270 1271 #ifndef arch_atomic64_read_acquire 1272 static __always_inline s64 1273 arch_atomic64_read_acquire(const atomic64_t *v) 1274 { 1275 s64 ret; 1276 1277 if (__native_word(atomic64_t)) { 1278 ret = smp_load_acquire(&(v)->counter); 1279 } else { 1280 ret = arch_atomic64_read(v); 1281 __atomic_acquire_fence(); 1282 } 1283 1284 return ret; 1285 } 1286 #define arch_atomic64_read_acquire arch_atomic64_read_acquire 1287 #endif 1288 1289 #ifndef arch_atomic64_set_release 1290 static __always_inline void 1291 arch_atomic64_set_release(atomic64_t *v, s64 i) 1292 { 1293 if (__native_word(atomic64_t)) { 1294 smp_store_release(&(v)->counter, i); 1295 } else { 1296 __atomic_release_fence(); 1297 arch_atomic64_set(v, i); 1298 } 1299 } 1300 #define arch_atomic64_set_release arch_atomic64_set_release 1301 #endif 1302 1303 #ifndef arch_atomic64_add_return_relaxed 1304 #define arch_atomic64_add_return_acquire arch_atomic64_add_return 1305 #define arch_atomic64_add_return_release arch_atomic64_add_return 1306 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return 1307 #else /* arch_atomic64_add_return_relaxed */ 1308 1309 #ifndef arch_atomic64_add_return_acquire 1310 static __always_inline s64 1311 arch_atomic64_add_return_acquire(s64 i, atomic64_t *v) 1312 { 1313 s64 ret = arch_atomic64_add_return_relaxed(i, v); 1314 __atomic_acquire_fence(); 1315 return ret; 1316 } 1317 #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire 1318 #endif 1319 1320 #ifndef arch_atomic64_add_return_release 1321 static __always_inline s64 1322 arch_atomic64_add_return_release(s64 i, atomic64_t *v) 1323 { 1324 __atomic_release_fence(); 1325 return arch_atomic64_add_return_relaxed(i, v); 1326 } 1327 #define arch_atomic64_add_return_release arch_atomic64_add_return_release 1328 #endif 1329 1330 #ifndef arch_atomic64_add_return 1331 static __always_inline s64 1332 arch_atomic64_add_return(s64 i, atomic64_t *v) 1333 { 1334 s64 ret; 1335 __atomic_pre_full_fence(); 1336 ret = arch_atomic64_add_return_relaxed(i, v); 1337 __atomic_post_full_fence(); 1338 return ret; 1339 } 1340 #define arch_atomic64_add_return arch_atomic64_add_return 1341 #endif 1342 1343 #endif /* arch_atomic64_add_return_relaxed */ 1344 1345 #ifndef arch_atomic64_fetch_add_relaxed 1346 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add 1347 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add 1348 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add 1349 #else /* arch_atomic64_fetch_add_relaxed */ 1350 1351 #ifndef arch_atomic64_fetch_add_acquire 1352 static __always_inline s64 1353 arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 1354 { 1355 s64 ret = arch_atomic64_fetch_add_relaxed(i, v); 1356 __atomic_acquire_fence(); 1357 return ret; 1358 } 1359 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire 1360 #endif 1361 1362 #ifndef arch_atomic64_fetch_add_release 1363 static __always_inline s64 1364 arch_atomic64_fetch_add_release(s64 i, atomic64_t *v) 1365 { 1366 __atomic_release_fence(); 1367 return arch_atomic64_fetch_add_relaxed(i, v); 1368 } 1369 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release 1370 #endif 1371 1372 #ifndef arch_atomic64_fetch_add 1373 static __always_inline s64 1374 arch_atomic64_fetch_add(s64 i, atomic64_t *v) 1375 { 1376 s64 ret; 1377 __atomic_pre_full_fence(); 1378 ret = arch_atomic64_fetch_add_relaxed(i, v); 1379 __atomic_post_full_fence(); 1380 return ret; 1381 } 1382 #define arch_atomic64_fetch_add arch_atomic64_fetch_add 1383 #endif 1384 1385 #endif /* arch_atomic64_fetch_add_relaxed */ 1386 1387 #ifndef arch_atomic64_sub_return_relaxed 1388 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return 1389 #define arch_atomic64_sub_return_release arch_atomic64_sub_return 1390 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return 1391 #else /* arch_atomic64_sub_return_relaxed */ 1392 1393 #ifndef arch_atomic64_sub_return_acquire 1394 static __always_inline s64 1395 arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 1396 { 1397 s64 ret = arch_atomic64_sub_return_relaxed(i, v); 1398 __atomic_acquire_fence(); 1399 return ret; 1400 } 1401 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire 1402 #endif 1403 1404 #ifndef arch_atomic64_sub_return_release 1405 static __always_inline s64 1406 arch_atomic64_sub_return_release(s64 i, atomic64_t *v) 1407 { 1408 __atomic_release_fence(); 1409 return arch_atomic64_sub_return_relaxed(i, v); 1410 } 1411 #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release 1412 #endif 1413 1414 #ifndef arch_atomic64_sub_return 1415 static __always_inline s64 1416 arch_atomic64_sub_return(s64 i, atomic64_t *v) 1417 { 1418 s64 ret; 1419 __atomic_pre_full_fence(); 1420 ret = arch_atomic64_sub_return_relaxed(i, v); 1421 __atomic_post_full_fence(); 1422 return ret; 1423 } 1424 #define arch_atomic64_sub_return arch_atomic64_sub_return 1425 #endif 1426 1427 #endif /* arch_atomic64_sub_return_relaxed */ 1428 1429 #ifndef arch_atomic64_fetch_sub_relaxed 1430 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub 1431 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub 1432 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub 1433 #else /* arch_atomic64_fetch_sub_relaxed */ 1434 1435 #ifndef arch_atomic64_fetch_sub_acquire 1436 static __always_inline s64 1437 arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 1438 { 1439 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1440 __atomic_acquire_fence(); 1441 return ret; 1442 } 1443 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire 1444 #endif 1445 1446 #ifndef arch_atomic64_fetch_sub_release 1447 static __always_inline s64 1448 arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 1449 { 1450 __atomic_release_fence(); 1451 return arch_atomic64_fetch_sub_relaxed(i, v); 1452 } 1453 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release 1454 #endif 1455 1456 #ifndef arch_atomic64_fetch_sub 1457 static __always_inline s64 1458 arch_atomic64_fetch_sub(s64 i, atomic64_t *v) 1459 { 1460 s64 ret; 1461 __atomic_pre_full_fence(); 1462 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1463 __atomic_post_full_fence(); 1464 return ret; 1465 } 1466 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub 1467 #endif 1468 1469 #endif /* arch_atomic64_fetch_sub_relaxed */ 1470 1471 #ifndef arch_atomic64_inc 1472 static __always_inline void 1473 arch_atomic64_inc(atomic64_t *v) 1474 { 1475 arch_atomic64_add(1, v); 1476 } 1477 #define arch_atomic64_inc arch_atomic64_inc 1478 #endif 1479 1480 #ifndef arch_atomic64_inc_return_relaxed 1481 #ifdef arch_atomic64_inc_return 1482 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return 1483 #define arch_atomic64_inc_return_release arch_atomic64_inc_return 1484 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return 1485 #endif /* arch_atomic64_inc_return */ 1486 1487 #ifndef arch_atomic64_inc_return 1488 static __always_inline s64 1489 arch_atomic64_inc_return(atomic64_t *v) 1490 { 1491 return arch_atomic64_add_return(1, v); 1492 } 1493 #define arch_atomic64_inc_return arch_atomic64_inc_return 1494 #endif 1495 1496 #ifndef arch_atomic64_inc_return_acquire 1497 static __always_inline s64 1498 arch_atomic64_inc_return_acquire(atomic64_t *v) 1499 { 1500 return arch_atomic64_add_return_acquire(1, v); 1501 } 1502 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1503 #endif 1504 1505 #ifndef arch_atomic64_inc_return_release 1506 static __always_inline s64 1507 arch_atomic64_inc_return_release(atomic64_t *v) 1508 { 1509 return arch_atomic64_add_return_release(1, v); 1510 } 1511 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1512 #endif 1513 1514 #ifndef arch_atomic64_inc_return_relaxed 1515 static __always_inline s64 1516 arch_atomic64_inc_return_relaxed(atomic64_t *v) 1517 { 1518 return arch_atomic64_add_return_relaxed(1, v); 1519 } 1520 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed 1521 #endif 1522 1523 #else /* arch_atomic64_inc_return_relaxed */ 1524 1525 #ifndef arch_atomic64_inc_return_acquire 1526 static __always_inline s64 1527 arch_atomic64_inc_return_acquire(atomic64_t *v) 1528 { 1529 s64 ret = arch_atomic64_inc_return_relaxed(v); 1530 __atomic_acquire_fence(); 1531 return ret; 1532 } 1533 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1534 #endif 1535 1536 #ifndef arch_atomic64_inc_return_release 1537 static __always_inline s64 1538 arch_atomic64_inc_return_release(atomic64_t *v) 1539 { 1540 __atomic_release_fence(); 1541 return arch_atomic64_inc_return_relaxed(v); 1542 } 1543 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1544 #endif 1545 1546 #ifndef arch_atomic64_inc_return 1547 static __always_inline s64 1548 arch_atomic64_inc_return(atomic64_t *v) 1549 { 1550 s64 ret; 1551 __atomic_pre_full_fence(); 1552 ret = arch_atomic64_inc_return_relaxed(v); 1553 __atomic_post_full_fence(); 1554 return ret; 1555 } 1556 #define arch_atomic64_inc_return arch_atomic64_inc_return 1557 #endif 1558 1559 #endif /* arch_atomic64_inc_return_relaxed */ 1560 1561 #ifndef arch_atomic64_fetch_inc_relaxed 1562 #ifdef arch_atomic64_fetch_inc 1563 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc 1564 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc 1565 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc 1566 #endif /* arch_atomic64_fetch_inc */ 1567 1568 #ifndef arch_atomic64_fetch_inc 1569 static __always_inline s64 1570 arch_atomic64_fetch_inc(atomic64_t *v) 1571 { 1572 return arch_atomic64_fetch_add(1, v); 1573 } 1574 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 1575 #endif 1576 1577 #ifndef arch_atomic64_fetch_inc_acquire 1578 static __always_inline s64 1579 arch_atomic64_fetch_inc_acquire(atomic64_t *v) 1580 { 1581 return arch_atomic64_fetch_add_acquire(1, v); 1582 } 1583 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1584 #endif 1585 1586 #ifndef arch_atomic64_fetch_inc_release 1587 static __always_inline s64 1588 arch_atomic64_fetch_inc_release(atomic64_t *v) 1589 { 1590 return arch_atomic64_fetch_add_release(1, v); 1591 } 1592 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1593 #endif 1594 1595 #ifndef arch_atomic64_fetch_inc_relaxed 1596 static __always_inline s64 1597 arch_atomic64_fetch_inc_relaxed(atomic64_t *v) 1598 { 1599 return arch_atomic64_fetch_add_relaxed(1, v); 1600 } 1601 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed 1602 #endif 1603 1604 #else /* arch_atomic64_fetch_inc_relaxed */ 1605 1606 #ifndef arch_atomic64_fetch_inc_acquire 1607 static __always_inline s64 1608 arch_atomic64_fetch_inc_acquire(atomic64_t *v) 1609 { 1610 s64 ret = arch_atomic64_fetch_inc_relaxed(v); 1611 __atomic_acquire_fence(); 1612 return ret; 1613 } 1614 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1615 #endif 1616 1617 #ifndef arch_atomic64_fetch_inc_release 1618 static __always_inline s64 1619 arch_atomic64_fetch_inc_release(atomic64_t *v) 1620 { 1621 __atomic_release_fence(); 1622 return arch_atomic64_fetch_inc_relaxed(v); 1623 } 1624 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1625 #endif 1626 1627 #ifndef arch_atomic64_fetch_inc 1628 static __always_inline s64 1629 arch_atomic64_fetch_inc(atomic64_t *v) 1630 { 1631 s64 ret; 1632 __atomic_pre_full_fence(); 1633 ret = arch_atomic64_fetch_inc_relaxed(v); 1634 __atomic_post_full_fence(); 1635 return ret; 1636 } 1637 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 1638 #endif 1639 1640 #endif /* arch_atomic64_fetch_inc_relaxed */ 1641 1642 #ifndef arch_atomic64_dec 1643 static __always_inline void 1644 arch_atomic64_dec(atomic64_t *v) 1645 { 1646 arch_atomic64_sub(1, v); 1647 } 1648 #define arch_atomic64_dec arch_atomic64_dec 1649 #endif 1650 1651 #ifndef arch_atomic64_dec_return_relaxed 1652 #ifdef arch_atomic64_dec_return 1653 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return 1654 #define arch_atomic64_dec_return_release arch_atomic64_dec_return 1655 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return 1656 #endif /* arch_atomic64_dec_return */ 1657 1658 #ifndef arch_atomic64_dec_return 1659 static __always_inline s64 1660 arch_atomic64_dec_return(atomic64_t *v) 1661 { 1662 return arch_atomic64_sub_return(1, v); 1663 } 1664 #define arch_atomic64_dec_return arch_atomic64_dec_return 1665 #endif 1666 1667 #ifndef arch_atomic64_dec_return_acquire 1668 static __always_inline s64 1669 arch_atomic64_dec_return_acquire(atomic64_t *v) 1670 { 1671 return arch_atomic64_sub_return_acquire(1, v); 1672 } 1673 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 1674 #endif 1675 1676 #ifndef arch_atomic64_dec_return_release 1677 static __always_inline s64 1678 arch_atomic64_dec_return_release(atomic64_t *v) 1679 { 1680 return arch_atomic64_sub_return_release(1, v); 1681 } 1682 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 1683 #endif 1684 1685 #ifndef arch_atomic64_dec_return_relaxed 1686 static __always_inline s64 1687 arch_atomic64_dec_return_relaxed(atomic64_t *v) 1688 { 1689 return arch_atomic64_sub_return_relaxed(1, v); 1690 } 1691 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed 1692 #endif 1693 1694 #else /* arch_atomic64_dec_return_relaxed */ 1695 1696 #ifndef arch_atomic64_dec_return_acquire 1697 static __always_inline s64 1698 arch_atomic64_dec_return_acquire(atomic64_t *v) 1699 { 1700 s64 ret = arch_atomic64_dec_return_relaxed(v); 1701 __atomic_acquire_fence(); 1702 return ret; 1703 } 1704 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 1705 #endif 1706 1707 #ifndef arch_atomic64_dec_return_release 1708 static __always_inline s64 1709 arch_atomic64_dec_return_release(atomic64_t *v) 1710 { 1711 __atomic_release_fence(); 1712 return arch_atomic64_dec_return_relaxed(v); 1713 } 1714 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 1715 #endif 1716 1717 #ifndef arch_atomic64_dec_return 1718 static __always_inline s64 1719 arch_atomic64_dec_return(atomic64_t *v) 1720 { 1721 s64 ret; 1722 __atomic_pre_full_fence(); 1723 ret = arch_atomic64_dec_return_relaxed(v); 1724 __atomic_post_full_fence(); 1725 return ret; 1726 } 1727 #define arch_atomic64_dec_return arch_atomic64_dec_return 1728 #endif 1729 1730 #endif /* arch_atomic64_dec_return_relaxed */ 1731 1732 #ifndef arch_atomic64_fetch_dec_relaxed 1733 #ifdef arch_atomic64_fetch_dec 1734 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec 1735 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec 1736 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec 1737 #endif /* arch_atomic64_fetch_dec */ 1738 1739 #ifndef arch_atomic64_fetch_dec 1740 static __always_inline s64 1741 arch_atomic64_fetch_dec(atomic64_t *v) 1742 { 1743 return arch_atomic64_fetch_sub(1, v); 1744 } 1745 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 1746 #endif 1747 1748 #ifndef arch_atomic64_fetch_dec_acquire 1749 static __always_inline s64 1750 arch_atomic64_fetch_dec_acquire(atomic64_t *v) 1751 { 1752 return arch_atomic64_fetch_sub_acquire(1, v); 1753 } 1754 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 1755 #endif 1756 1757 #ifndef arch_atomic64_fetch_dec_release 1758 static __always_inline s64 1759 arch_atomic64_fetch_dec_release(atomic64_t *v) 1760 { 1761 return arch_atomic64_fetch_sub_release(1, v); 1762 } 1763 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 1764 #endif 1765 1766 #ifndef arch_atomic64_fetch_dec_relaxed 1767 static __always_inline s64 1768 arch_atomic64_fetch_dec_relaxed(atomic64_t *v) 1769 { 1770 return arch_atomic64_fetch_sub_relaxed(1, v); 1771 } 1772 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed 1773 #endif 1774 1775 #else /* arch_atomic64_fetch_dec_relaxed */ 1776 1777 #ifndef arch_atomic64_fetch_dec_acquire 1778 static __always_inline s64 1779 arch_atomic64_fetch_dec_acquire(atomic64_t *v) 1780 { 1781 s64 ret = arch_atomic64_fetch_dec_relaxed(v); 1782 __atomic_acquire_fence(); 1783 return ret; 1784 } 1785 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 1786 #endif 1787 1788 #ifndef arch_atomic64_fetch_dec_release 1789 static __always_inline s64 1790 arch_atomic64_fetch_dec_release(atomic64_t *v) 1791 { 1792 __atomic_release_fence(); 1793 return arch_atomic64_fetch_dec_relaxed(v); 1794 } 1795 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 1796 #endif 1797 1798 #ifndef arch_atomic64_fetch_dec 1799 static __always_inline s64 1800 arch_atomic64_fetch_dec(atomic64_t *v) 1801 { 1802 s64 ret; 1803 __atomic_pre_full_fence(); 1804 ret = arch_atomic64_fetch_dec_relaxed(v); 1805 __atomic_post_full_fence(); 1806 return ret; 1807 } 1808 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 1809 #endif 1810 1811 #endif /* arch_atomic64_fetch_dec_relaxed */ 1812 1813 #ifndef arch_atomic64_fetch_and_relaxed 1814 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and 1815 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and 1816 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and 1817 #else /* arch_atomic64_fetch_and_relaxed */ 1818 1819 #ifndef arch_atomic64_fetch_and_acquire 1820 static __always_inline s64 1821 arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 1822 { 1823 s64 ret = arch_atomic64_fetch_and_relaxed(i, v); 1824 __atomic_acquire_fence(); 1825 return ret; 1826 } 1827 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire 1828 #endif 1829 1830 #ifndef arch_atomic64_fetch_and_release 1831 static __always_inline s64 1832 arch_atomic64_fetch_and_release(s64 i, atomic64_t *v) 1833 { 1834 __atomic_release_fence(); 1835 return arch_atomic64_fetch_and_relaxed(i, v); 1836 } 1837 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release 1838 #endif 1839 1840 #ifndef arch_atomic64_fetch_and 1841 static __always_inline s64 1842 arch_atomic64_fetch_and(s64 i, atomic64_t *v) 1843 { 1844 s64 ret; 1845 __atomic_pre_full_fence(); 1846 ret = arch_atomic64_fetch_and_relaxed(i, v); 1847 __atomic_post_full_fence(); 1848 return ret; 1849 } 1850 #define arch_atomic64_fetch_and arch_atomic64_fetch_and 1851 #endif 1852 1853 #endif /* arch_atomic64_fetch_and_relaxed */ 1854 1855 #ifndef arch_atomic64_andnot 1856 static __always_inline void 1857 arch_atomic64_andnot(s64 i, atomic64_t *v) 1858 { 1859 arch_atomic64_and(~i, v); 1860 } 1861 #define arch_atomic64_andnot arch_atomic64_andnot 1862 #endif 1863 1864 #ifndef arch_atomic64_fetch_andnot_relaxed 1865 #ifdef arch_atomic64_fetch_andnot 1866 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot 1867 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot 1868 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot 1869 #endif /* arch_atomic64_fetch_andnot */ 1870 1871 #ifndef arch_atomic64_fetch_andnot 1872 static __always_inline s64 1873 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 1874 { 1875 return arch_atomic64_fetch_and(~i, v); 1876 } 1877 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 1878 #endif 1879 1880 #ifndef arch_atomic64_fetch_andnot_acquire 1881 static __always_inline s64 1882 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 1883 { 1884 return arch_atomic64_fetch_and_acquire(~i, v); 1885 } 1886 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 1887 #endif 1888 1889 #ifndef arch_atomic64_fetch_andnot_release 1890 static __always_inline s64 1891 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 1892 { 1893 return arch_atomic64_fetch_and_release(~i, v); 1894 } 1895 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 1896 #endif 1897 1898 #ifndef arch_atomic64_fetch_andnot_relaxed 1899 static __always_inline s64 1900 arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 1901 { 1902 return arch_atomic64_fetch_and_relaxed(~i, v); 1903 } 1904 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 1905 #endif 1906 1907 #else /* arch_atomic64_fetch_andnot_relaxed */ 1908 1909 #ifndef arch_atomic64_fetch_andnot_acquire 1910 static __always_inline s64 1911 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 1912 { 1913 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 1914 __atomic_acquire_fence(); 1915 return ret; 1916 } 1917 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 1918 #endif 1919 1920 #ifndef arch_atomic64_fetch_andnot_release 1921 static __always_inline s64 1922 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 1923 { 1924 __atomic_release_fence(); 1925 return arch_atomic64_fetch_andnot_relaxed(i, v); 1926 } 1927 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 1928 #endif 1929 1930 #ifndef arch_atomic64_fetch_andnot 1931 static __always_inline s64 1932 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 1933 { 1934 s64 ret; 1935 __atomic_pre_full_fence(); 1936 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 1937 __atomic_post_full_fence(); 1938 return ret; 1939 } 1940 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 1941 #endif 1942 1943 #endif /* arch_atomic64_fetch_andnot_relaxed */ 1944 1945 #ifndef arch_atomic64_fetch_or_relaxed 1946 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or 1947 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or 1948 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or 1949 #else /* arch_atomic64_fetch_or_relaxed */ 1950 1951 #ifndef arch_atomic64_fetch_or_acquire 1952 static __always_inline s64 1953 arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 1954 { 1955 s64 ret = arch_atomic64_fetch_or_relaxed(i, v); 1956 __atomic_acquire_fence(); 1957 return ret; 1958 } 1959 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire 1960 #endif 1961 1962 #ifndef arch_atomic64_fetch_or_release 1963 static __always_inline s64 1964 arch_atomic64_fetch_or_release(s64 i, atomic64_t *v) 1965 { 1966 __atomic_release_fence(); 1967 return arch_atomic64_fetch_or_relaxed(i, v); 1968 } 1969 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release 1970 #endif 1971 1972 #ifndef arch_atomic64_fetch_or 1973 static __always_inline s64 1974 arch_atomic64_fetch_or(s64 i, atomic64_t *v) 1975 { 1976 s64 ret; 1977 __atomic_pre_full_fence(); 1978 ret = arch_atomic64_fetch_or_relaxed(i, v); 1979 __atomic_post_full_fence(); 1980 return ret; 1981 } 1982 #define arch_atomic64_fetch_or arch_atomic64_fetch_or 1983 #endif 1984 1985 #endif /* arch_atomic64_fetch_or_relaxed */ 1986 1987 #ifndef arch_atomic64_fetch_xor_relaxed 1988 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor 1989 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor 1990 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor 1991 #else /* arch_atomic64_fetch_xor_relaxed */ 1992 1993 #ifndef arch_atomic64_fetch_xor_acquire 1994 static __always_inline s64 1995 arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 1996 { 1997 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); 1998 __atomic_acquire_fence(); 1999 return ret; 2000 } 2001 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire 2002 #endif 2003 2004 #ifndef arch_atomic64_fetch_xor_release 2005 static __always_inline s64 2006 arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 2007 { 2008 __atomic_release_fence(); 2009 return arch_atomic64_fetch_xor_relaxed(i, v); 2010 } 2011 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release 2012 #endif 2013 2014 #ifndef arch_atomic64_fetch_xor 2015 static __always_inline s64 2016 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) 2017 { 2018 s64 ret; 2019 __atomic_pre_full_fence(); 2020 ret = arch_atomic64_fetch_xor_relaxed(i, v); 2021 __atomic_post_full_fence(); 2022 return ret; 2023 } 2024 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 2025 #endif 2026 2027 #endif /* arch_atomic64_fetch_xor_relaxed */ 2028 2029 #ifndef arch_atomic64_xchg_relaxed 2030 #define arch_atomic64_xchg_acquire arch_atomic64_xchg 2031 #define arch_atomic64_xchg_release arch_atomic64_xchg 2032 #define arch_atomic64_xchg_relaxed arch_atomic64_xchg 2033 #else /* arch_atomic64_xchg_relaxed */ 2034 2035 #ifndef arch_atomic64_xchg_acquire 2036 static __always_inline s64 2037 arch_atomic64_xchg_acquire(atomic64_t *v, s64 i) 2038 { 2039 s64 ret = arch_atomic64_xchg_relaxed(v, i); 2040 __atomic_acquire_fence(); 2041 return ret; 2042 } 2043 #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire 2044 #endif 2045 2046 #ifndef arch_atomic64_xchg_release 2047 static __always_inline s64 2048 arch_atomic64_xchg_release(atomic64_t *v, s64 i) 2049 { 2050 __atomic_release_fence(); 2051 return arch_atomic64_xchg_relaxed(v, i); 2052 } 2053 #define arch_atomic64_xchg_release arch_atomic64_xchg_release 2054 #endif 2055 2056 #ifndef arch_atomic64_xchg 2057 static __always_inline s64 2058 arch_atomic64_xchg(atomic64_t *v, s64 i) 2059 { 2060 s64 ret; 2061 __atomic_pre_full_fence(); 2062 ret = arch_atomic64_xchg_relaxed(v, i); 2063 __atomic_post_full_fence(); 2064 return ret; 2065 } 2066 #define arch_atomic64_xchg arch_atomic64_xchg 2067 #endif 2068 2069 #endif /* arch_atomic64_xchg_relaxed */ 2070 2071 #ifndef arch_atomic64_cmpxchg_relaxed 2072 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg 2073 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg 2074 #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg 2075 #else /* arch_atomic64_cmpxchg_relaxed */ 2076 2077 #ifndef arch_atomic64_cmpxchg_acquire 2078 static __always_inline s64 2079 arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2080 { 2081 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2082 __atomic_acquire_fence(); 2083 return ret; 2084 } 2085 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire 2086 #endif 2087 2088 #ifndef arch_atomic64_cmpxchg_release 2089 static __always_inline s64 2090 arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2091 { 2092 __atomic_release_fence(); 2093 return arch_atomic64_cmpxchg_relaxed(v, old, new); 2094 } 2095 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release 2096 #endif 2097 2098 #ifndef arch_atomic64_cmpxchg 2099 static __always_inline s64 2100 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2101 { 2102 s64 ret; 2103 __atomic_pre_full_fence(); 2104 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2105 __atomic_post_full_fence(); 2106 return ret; 2107 } 2108 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg 2109 #endif 2110 2111 #endif /* arch_atomic64_cmpxchg_relaxed */ 2112 2113 #ifndef arch_atomic64_try_cmpxchg_relaxed 2114 #ifdef arch_atomic64_try_cmpxchg 2115 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg 2116 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg 2117 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg 2118 #endif /* arch_atomic64_try_cmpxchg */ 2119 2120 #ifndef arch_atomic64_try_cmpxchg 2121 static __always_inline bool 2122 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2123 { 2124 s64 r, o = *old; 2125 r = arch_atomic64_cmpxchg(v, o, new); 2126 if (unlikely(r != o)) 2127 *old = r; 2128 return likely(r == o); 2129 } 2130 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2131 #endif 2132 2133 #ifndef arch_atomic64_try_cmpxchg_acquire 2134 static __always_inline bool 2135 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2136 { 2137 s64 r, o = *old; 2138 r = arch_atomic64_cmpxchg_acquire(v, o, new); 2139 if (unlikely(r != o)) 2140 *old = r; 2141 return likely(r == o); 2142 } 2143 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2144 #endif 2145 2146 #ifndef arch_atomic64_try_cmpxchg_release 2147 static __always_inline bool 2148 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2149 { 2150 s64 r, o = *old; 2151 r = arch_atomic64_cmpxchg_release(v, o, new); 2152 if (unlikely(r != o)) 2153 *old = r; 2154 return likely(r == o); 2155 } 2156 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2157 #endif 2158 2159 #ifndef arch_atomic64_try_cmpxchg_relaxed 2160 static __always_inline bool 2161 arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 2162 { 2163 s64 r, o = *old; 2164 r = arch_atomic64_cmpxchg_relaxed(v, o, new); 2165 if (unlikely(r != o)) 2166 *old = r; 2167 return likely(r == o); 2168 } 2169 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed 2170 #endif 2171 2172 #else /* arch_atomic64_try_cmpxchg_relaxed */ 2173 2174 #ifndef arch_atomic64_try_cmpxchg_acquire 2175 static __always_inline bool 2176 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2177 { 2178 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2179 __atomic_acquire_fence(); 2180 return ret; 2181 } 2182 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2183 #endif 2184 2185 #ifndef arch_atomic64_try_cmpxchg_release 2186 static __always_inline bool 2187 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2188 { 2189 __atomic_release_fence(); 2190 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2191 } 2192 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2193 #endif 2194 2195 #ifndef arch_atomic64_try_cmpxchg 2196 static __always_inline bool 2197 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2198 { 2199 bool ret; 2200 __atomic_pre_full_fence(); 2201 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2202 __atomic_post_full_fence(); 2203 return ret; 2204 } 2205 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2206 #endif 2207 2208 #endif /* arch_atomic64_try_cmpxchg_relaxed */ 2209 2210 #ifndef arch_atomic64_sub_and_test 2211 /** 2212 * arch_atomic64_sub_and_test - subtract value from variable and test result 2213 * @i: integer value to subtract 2214 * @v: pointer of type atomic64_t 2215 * 2216 * Atomically subtracts @i from @v and returns 2217 * true if the result is zero, or false for all 2218 * other cases. 2219 */ 2220 static __always_inline bool 2221 arch_atomic64_sub_and_test(s64 i, atomic64_t *v) 2222 { 2223 return arch_atomic64_sub_return(i, v) == 0; 2224 } 2225 #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test 2226 #endif 2227 2228 #ifndef arch_atomic64_dec_and_test 2229 /** 2230 * arch_atomic64_dec_and_test - decrement and test 2231 * @v: pointer of type atomic64_t 2232 * 2233 * Atomically decrements @v by 1 and 2234 * returns true if the result is 0, or false for all other 2235 * cases. 2236 */ 2237 static __always_inline bool 2238 arch_atomic64_dec_and_test(atomic64_t *v) 2239 { 2240 return arch_atomic64_dec_return(v) == 0; 2241 } 2242 #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test 2243 #endif 2244 2245 #ifndef arch_atomic64_inc_and_test 2246 /** 2247 * arch_atomic64_inc_and_test - increment and test 2248 * @v: pointer of type atomic64_t 2249 * 2250 * Atomically increments @v by 1 2251 * and returns true if the result is zero, or false for all 2252 * other cases. 2253 */ 2254 static __always_inline bool 2255 arch_atomic64_inc_and_test(atomic64_t *v) 2256 { 2257 return arch_atomic64_inc_return(v) == 0; 2258 } 2259 #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test 2260 #endif 2261 2262 #ifndef arch_atomic64_add_negative 2263 /** 2264 * arch_atomic64_add_negative - add and test if negative 2265 * @i: integer value to add 2266 * @v: pointer of type atomic64_t 2267 * 2268 * Atomically adds @i to @v and returns true 2269 * if the result is negative, or false when 2270 * result is greater than or equal to zero. 2271 */ 2272 static __always_inline bool 2273 arch_atomic64_add_negative(s64 i, atomic64_t *v) 2274 { 2275 return arch_atomic64_add_return(i, v) < 0; 2276 } 2277 #define arch_atomic64_add_negative arch_atomic64_add_negative 2278 #endif 2279 2280 #ifndef arch_atomic64_fetch_add_unless 2281 /** 2282 * arch_atomic64_fetch_add_unless - add unless the number is already a given value 2283 * @v: pointer of type atomic64_t 2284 * @a: the amount to add to v... 2285 * @u: ...unless v is equal to u. 2286 * 2287 * Atomically adds @a to @v, so long as @v was not already @u. 2288 * Returns original value of @v 2289 */ 2290 static __always_inline s64 2291 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 2292 { 2293 s64 c = arch_atomic64_read(v); 2294 2295 do { 2296 if (unlikely(c == u)) 2297 break; 2298 } while (!arch_atomic64_try_cmpxchg(v, &c, c + a)); 2299 2300 return c; 2301 } 2302 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 2303 #endif 2304 2305 #ifndef arch_atomic64_add_unless 2306 /** 2307 * arch_atomic64_add_unless - add unless the number is already a given value 2308 * @v: pointer of type atomic64_t 2309 * @a: the amount to add to v... 2310 * @u: ...unless v is equal to u. 2311 * 2312 * Atomically adds @a to @v, if @v was not already @u. 2313 * Returns true if the addition was done. 2314 */ 2315 static __always_inline bool 2316 arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 2317 { 2318 return arch_atomic64_fetch_add_unless(v, a, u) != u; 2319 } 2320 #define arch_atomic64_add_unless arch_atomic64_add_unless 2321 #endif 2322 2323 #ifndef arch_atomic64_inc_not_zero 2324 /** 2325 * arch_atomic64_inc_not_zero - increment unless the number is zero 2326 * @v: pointer of type atomic64_t 2327 * 2328 * Atomically increments @v by 1, if @v is non-zero. 2329 * Returns true if the increment was done. 2330 */ 2331 static __always_inline bool 2332 arch_atomic64_inc_not_zero(atomic64_t *v) 2333 { 2334 return arch_atomic64_add_unless(v, 1, 0); 2335 } 2336 #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero 2337 #endif 2338 2339 #ifndef arch_atomic64_inc_unless_negative 2340 static __always_inline bool 2341 arch_atomic64_inc_unless_negative(atomic64_t *v) 2342 { 2343 s64 c = arch_atomic64_read(v); 2344 2345 do { 2346 if (unlikely(c < 0)) 2347 return false; 2348 } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1)); 2349 2350 return true; 2351 } 2352 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 2353 #endif 2354 2355 #ifndef arch_atomic64_dec_unless_positive 2356 static __always_inline bool 2357 arch_atomic64_dec_unless_positive(atomic64_t *v) 2358 { 2359 s64 c = arch_atomic64_read(v); 2360 2361 do { 2362 if (unlikely(c > 0)) 2363 return false; 2364 } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1)); 2365 2366 return true; 2367 } 2368 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 2369 #endif 2370 2371 #ifndef arch_atomic64_dec_if_positive 2372 static __always_inline s64 2373 arch_atomic64_dec_if_positive(atomic64_t *v) 2374 { 2375 s64 dec, c = arch_atomic64_read(v); 2376 2377 do { 2378 dec = c - 1; 2379 if (unlikely(dec < 0)) 2380 break; 2381 } while (!arch_atomic64_try_cmpxchg(v, &c, dec)); 2382 2383 return dec; 2384 } 2385 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 2386 #endif 2387 2388 #endif /* _LINUX_ATOMIC_FALLBACK_H */ 2389 // 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae 2390