1 // SPDX-License-Identifier: GPL-2.0 2 3 // Generated by scripts/atomic/gen-atomic-fallback.sh 4 // DO NOT MODIFY THIS FILE DIRECTLY 5 6 #ifndef _LINUX_ATOMIC_FALLBACK_H 7 #define _LINUX_ATOMIC_FALLBACK_H 8 9 #include <linux/compiler.h> 10 11 #ifndef arch_xchg_relaxed 12 #define arch_xchg_acquire arch_xchg 13 #define arch_xchg_release arch_xchg 14 #define arch_xchg_relaxed arch_xchg 15 #else /* arch_xchg_relaxed */ 16 17 #ifndef arch_xchg_acquire 18 #define arch_xchg_acquire(...) \ 19 __atomic_op_acquire(arch_xchg, __VA_ARGS__) 20 #endif 21 22 #ifndef arch_xchg_release 23 #define arch_xchg_release(...) \ 24 __atomic_op_release(arch_xchg, __VA_ARGS__) 25 #endif 26 27 #ifndef arch_xchg 28 #define arch_xchg(...) \ 29 __atomic_op_fence(arch_xchg, __VA_ARGS__) 30 #endif 31 32 #endif /* arch_xchg_relaxed */ 33 34 #ifndef arch_cmpxchg_relaxed 35 #define arch_cmpxchg_acquire arch_cmpxchg 36 #define arch_cmpxchg_release arch_cmpxchg 37 #define arch_cmpxchg_relaxed arch_cmpxchg 38 #else /* arch_cmpxchg_relaxed */ 39 40 #ifndef arch_cmpxchg_acquire 41 #define arch_cmpxchg_acquire(...) \ 42 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) 43 #endif 44 45 #ifndef arch_cmpxchg_release 46 #define arch_cmpxchg_release(...) \ 47 __atomic_op_release(arch_cmpxchg, __VA_ARGS__) 48 #endif 49 50 #ifndef arch_cmpxchg 51 #define arch_cmpxchg(...) \ 52 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) 53 #endif 54 55 #endif /* arch_cmpxchg_relaxed */ 56 57 #ifndef arch_cmpxchg64_relaxed 58 #define arch_cmpxchg64_acquire arch_cmpxchg64 59 #define arch_cmpxchg64_release arch_cmpxchg64 60 #define arch_cmpxchg64_relaxed arch_cmpxchg64 61 #else /* arch_cmpxchg64_relaxed */ 62 63 #ifndef arch_cmpxchg64_acquire 64 #define arch_cmpxchg64_acquire(...) \ 65 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) 66 #endif 67 68 #ifndef arch_cmpxchg64_release 69 #define arch_cmpxchg64_release(...) \ 70 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) 71 #endif 72 73 #ifndef arch_cmpxchg64 74 #define arch_cmpxchg64(...) \ 75 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) 76 #endif 77 78 #endif /* arch_cmpxchg64_relaxed */ 79 80 #ifndef arch_cmpxchg128_relaxed 81 #define arch_cmpxchg128_acquire arch_cmpxchg128 82 #define arch_cmpxchg128_release arch_cmpxchg128 83 #define arch_cmpxchg128_relaxed arch_cmpxchg128 84 #else /* arch_cmpxchg128_relaxed */ 85 86 #ifndef arch_cmpxchg128_acquire 87 #define arch_cmpxchg128_acquire(...) \ 88 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) 89 #endif 90 91 #ifndef arch_cmpxchg128_release 92 #define arch_cmpxchg128_release(...) \ 93 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) 94 #endif 95 96 #ifndef arch_cmpxchg128 97 #define arch_cmpxchg128(...) \ 98 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) 99 #endif 100 101 #endif /* arch_cmpxchg128_relaxed */ 102 103 #ifndef arch_try_cmpxchg_relaxed 104 #ifdef arch_try_cmpxchg 105 #define arch_try_cmpxchg_acquire arch_try_cmpxchg 106 #define arch_try_cmpxchg_release arch_try_cmpxchg 107 #define arch_try_cmpxchg_relaxed arch_try_cmpxchg 108 #endif /* arch_try_cmpxchg */ 109 110 #ifndef arch_try_cmpxchg 111 #define arch_try_cmpxchg(_ptr, _oldp, _new) \ 112 ({ \ 113 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 114 ___r = arch_cmpxchg((_ptr), ___o, (_new)); \ 115 if (unlikely(___r != ___o)) \ 116 *___op = ___r; \ 117 likely(___r == ___o); \ 118 }) 119 #endif /* arch_try_cmpxchg */ 120 121 #ifndef arch_try_cmpxchg_acquire 122 #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \ 123 ({ \ 124 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 125 ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \ 126 if (unlikely(___r != ___o)) \ 127 *___op = ___r; \ 128 likely(___r == ___o); \ 129 }) 130 #endif /* arch_try_cmpxchg_acquire */ 131 132 #ifndef arch_try_cmpxchg_release 133 #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \ 134 ({ \ 135 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 136 ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \ 137 if (unlikely(___r != ___o)) \ 138 *___op = ___r; \ 139 likely(___r == ___o); \ 140 }) 141 #endif /* arch_try_cmpxchg_release */ 142 143 #ifndef arch_try_cmpxchg_relaxed 144 #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ 145 ({ \ 146 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 147 ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \ 148 if (unlikely(___r != ___o)) \ 149 *___op = ___r; \ 150 likely(___r == ___o); \ 151 }) 152 #endif /* arch_try_cmpxchg_relaxed */ 153 154 #else /* arch_try_cmpxchg_relaxed */ 155 156 #ifndef arch_try_cmpxchg_acquire 157 #define arch_try_cmpxchg_acquire(...) \ 158 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) 159 #endif 160 161 #ifndef arch_try_cmpxchg_release 162 #define arch_try_cmpxchg_release(...) \ 163 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) 164 #endif 165 166 #ifndef arch_try_cmpxchg 167 #define arch_try_cmpxchg(...) \ 168 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) 169 #endif 170 171 #endif /* arch_try_cmpxchg_relaxed */ 172 173 #ifndef arch_try_cmpxchg64_relaxed 174 #ifdef arch_try_cmpxchg64 175 #define arch_try_cmpxchg64_acquire arch_try_cmpxchg64 176 #define arch_try_cmpxchg64_release arch_try_cmpxchg64 177 #define arch_try_cmpxchg64_relaxed arch_try_cmpxchg64 178 #endif /* arch_try_cmpxchg64 */ 179 180 #ifndef arch_try_cmpxchg64 181 #define arch_try_cmpxchg64(_ptr, _oldp, _new) \ 182 ({ \ 183 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 184 ___r = arch_cmpxchg64((_ptr), ___o, (_new)); \ 185 if (unlikely(___r != ___o)) \ 186 *___op = ___r; \ 187 likely(___r == ___o); \ 188 }) 189 #endif /* arch_try_cmpxchg64 */ 190 191 #ifndef arch_try_cmpxchg64_acquire 192 #define arch_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ 193 ({ \ 194 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 195 ___r = arch_cmpxchg64_acquire((_ptr), ___o, (_new)); \ 196 if (unlikely(___r != ___o)) \ 197 *___op = ___r; \ 198 likely(___r == ___o); \ 199 }) 200 #endif /* arch_try_cmpxchg64_acquire */ 201 202 #ifndef arch_try_cmpxchg64_release 203 #define arch_try_cmpxchg64_release(_ptr, _oldp, _new) \ 204 ({ \ 205 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 206 ___r = arch_cmpxchg64_release((_ptr), ___o, (_new)); \ 207 if (unlikely(___r != ___o)) \ 208 *___op = ___r; \ 209 likely(___r == ___o); \ 210 }) 211 #endif /* arch_try_cmpxchg64_release */ 212 213 #ifndef arch_try_cmpxchg64_relaxed 214 #define arch_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ 215 ({ \ 216 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 217 ___r = arch_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ 218 if (unlikely(___r != ___o)) \ 219 *___op = ___r; \ 220 likely(___r == ___o); \ 221 }) 222 #endif /* arch_try_cmpxchg64_relaxed */ 223 224 #else /* arch_try_cmpxchg64_relaxed */ 225 226 #ifndef arch_try_cmpxchg64_acquire 227 #define arch_try_cmpxchg64_acquire(...) \ 228 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) 229 #endif 230 231 #ifndef arch_try_cmpxchg64_release 232 #define arch_try_cmpxchg64_release(...) \ 233 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) 234 #endif 235 236 #ifndef arch_try_cmpxchg64 237 #define arch_try_cmpxchg64(...) \ 238 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) 239 #endif 240 241 #endif /* arch_try_cmpxchg64_relaxed */ 242 243 #ifndef arch_try_cmpxchg128_relaxed 244 #ifdef arch_try_cmpxchg128 245 #define arch_try_cmpxchg128_acquire arch_try_cmpxchg128 246 #define arch_try_cmpxchg128_release arch_try_cmpxchg128 247 #define arch_try_cmpxchg128_relaxed arch_try_cmpxchg128 248 #endif /* arch_try_cmpxchg128 */ 249 250 #ifndef arch_try_cmpxchg128 251 #define arch_try_cmpxchg128(_ptr, _oldp, _new) \ 252 ({ \ 253 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 254 ___r = arch_cmpxchg128((_ptr), ___o, (_new)); \ 255 if (unlikely(___r != ___o)) \ 256 *___op = ___r; \ 257 likely(___r == ___o); \ 258 }) 259 #endif /* arch_try_cmpxchg128 */ 260 261 #ifndef arch_try_cmpxchg128_acquire 262 #define arch_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ 263 ({ \ 264 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 265 ___r = arch_cmpxchg128_acquire((_ptr), ___o, (_new)); \ 266 if (unlikely(___r != ___o)) \ 267 *___op = ___r; \ 268 likely(___r == ___o); \ 269 }) 270 #endif /* arch_try_cmpxchg128_acquire */ 271 272 #ifndef arch_try_cmpxchg128_release 273 #define arch_try_cmpxchg128_release(_ptr, _oldp, _new) \ 274 ({ \ 275 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 276 ___r = arch_cmpxchg128_release((_ptr), ___o, (_new)); \ 277 if (unlikely(___r != ___o)) \ 278 *___op = ___r; \ 279 likely(___r == ___o); \ 280 }) 281 #endif /* arch_try_cmpxchg128_release */ 282 283 #ifndef arch_try_cmpxchg128_relaxed 284 #define arch_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ 285 ({ \ 286 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 287 ___r = arch_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ 288 if (unlikely(___r != ___o)) \ 289 *___op = ___r; \ 290 likely(___r == ___o); \ 291 }) 292 #endif /* arch_try_cmpxchg128_relaxed */ 293 294 #else /* arch_try_cmpxchg128_relaxed */ 295 296 #ifndef arch_try_cmpxchg128_acquire 297 #define arch_try_cmpxchg128_acquire(...) \ 298 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) 299 #endif 300 301 #ifndef arch_try_cmpxchg128_release 302 #define arch_try_cmpxchg128_release(...) \ 303 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) 304 #endif 305 306 #ifndef arch_try_cmpxchg128 307 #define arch_try_cmpxchg128(...) \ 308 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) 309 #endif 310 311 #endif /* arch_try_cmpxchg128_relaxed */ 312 313 #ifndef arch_try_cmpxchg_local 314 #define arch_try_cmpxchg_local(_ptr, _oldp, _new) \ 315 ({ \ 316 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 317 ___r = arch_cmpxchg_local((_ptr), ___o, (_new)); \ 318 if (unlikely(___r != ___o)) \ 319 *___op = ___r; \ 320 likely(___r == ___o); \ 321 }) 322 #endif /* arch_try_cmpxchg_local */ 323 324 #ifndef arch_try_cmpxchg64_local 325 #define arch_try_cmpxchg64_local(_ptr, _oldp, _new) \ 326 ({ \ 327 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 328 ___r = arch_cmpxchg64_local((_ptr), ___o, (_new)); \ 329 if (unlikely(___r != ___o)) \ 330 *___op = ___r; \ 331 likely(___r == ___o); \ 332 }) 333 #endif /* arch_try_cmpxchg64_local */ 334 335 #ifndef arch_atomic_read_acquire 336 static __always_inline int 337 arch_atomic_read_acquire(const atomic_t *v) 338 { 339 int ret; 340 341 if (__native_word(atomic_t)) { 342 ret = smp_load_acquire(&(v)->counter); 343 } else { 344 ret = arch_atomic_read(v); 345 __atomic_acquire_fence(); 346 } 347 348 return ret; 349 } 350 #define arch_atomic_read_acquire arch_atomic_read_acquire 351 #endif 352 353 #ifndef arch_atomic_set_release 354 static __always_inline void 355 arch_atomic_set_release(atomic_t *v, int i) 356 { 357 if (__native_word(atomic_t)) { 358 smp_store_release(&(v)->counter, i); 359 } else { 360 __atomic_release_fence(); 361 arch_atomic_set(v, i); 362 } 363 } 364 #define arch_atomic_set_release arch_atomic_set_release 365 #endif 366 367 #ifndef arch_atomic_add_return_relaxed 368 #define arch_atomic_add_return_acquire arch_atomic_add_return 369 #define arch_atomic_add_return_release arch_atomic_add_return 370 #define arch_atomic_add_return_relaxed arch_atomic_add_return 371 #else /* arch_atomic_add_return_relaxed */ 372 373 #ifndef arch_atomic_add_return_acquire 374 static __always_inline int 375 arch_atomic_add_return_acquire(int i, atomic_t *v) 376 { 377 int ret = arch_atomic_add_return_relaxed(i, v); 378 __atomic_acquire_fence(); 379 return ret; 380 } 381 #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire 382 #endif 383 384 #ifndef arch_atomic_add_return_release 385 static __always_inline int 386 arch_atomic_add_return_release(int i, atomic_t *v) 387 { 388 __atomic_release_fence(); 389 return arch_atomic_add_return_relaxed(i, v); 390 } 391 #define arch_atomic_add_return_release arch_atomic_add_return_release 392 #endif 393 394 #ifndef arch_atomic_add_return 395 static __always_inline int 396 arch_atomic_add_return(int i, atomic_t *v) 397 { 398 int ret; 399 __atomic_pre_full_fence(); 400 ret = arch_atomic_add_return_relaxed(i, v); 401 __atomic_post_full_fence(); 402 return ret; 403 } 404 #define arch_atomic_add_return arch_atomic_add_return 405 #endif 406 407 #endif /* arch_atomic_add_return_relaxed */ 408 409 #ifndef arch_atomic_fetch_add_relaxed 410 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add 411 #define arch_atomic_fetch_add_release arch_atomic_fetch_add 412 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add 413 #else /* arch_atomic_fetch_add_relaxed */ 414 415 #ifndef arch_atomic_fetch_add_acquire 416 static __always_inline int 417 arch_atomic_fetch_add_acquire(int i, atomic_t *v) 418 { 419 int ret = arch_atomic_fetch_add_relaxed(i, v); 420 __atomic_acquire_fence(); 421 return ret; 422 } 423 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire 424 #endif 425 426 #ifndef arch_atomic_fetch_add_release 427 static __always_inline int 428 arch_atomic_fetch_add_release(int i, atomic_t *v) 429 { 430 __atomic_release_fence(); 431 return arch_atomic_fetch_add_relaxed(i, v); 432 } 433 #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release 434 #endif 435 436 #ifndef arch_atomic_fetch_add 437 static __always_inline int 438 arch_atomic_fetch_add(int i, atomic_t *v) 439 { 440 int ret; 441 __atomic_pre_full_fence(); 442 ret = arch_atomic_fetch_add_relaxed(i, v); 443 __atomic_post_full_fence(); 444 return ret; 445 } 446 #define arch_atomic_fetch_add arch_atomic_fetch_add 447 #endif 448 449 #endif /* arch_atomic_fetch_add_relaxed */ 450 451 #ifndef arch_atomic_sub_return_relaxed 452 #define arch_atomic_sub_return_acquire arch_atomic_sub_return 453 #define arch_atomic_sub_return_release arch_atomic_sub_return 454 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return 455 #else /* arch_atomic_sub_return_relaxed */ 456 457 #ifndef arch_atomic_sub_return_acquire 458 static __always_inline int 459 arch_atomic_sub_return_acquire(int i, atomic_t *v) 460 { 461 int ret = arch_atomic_sub_return_relaxed(i, v); 462 __atomic_acquire_fence(); 463 return ret; 464 } 465 #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire 466 #endif 467 468 #ifndef arch_atomic_sub_return_release 469 static __always_inline int 470 arch_atomic_sub_return_release(int i, atomic_t *v) 471 { 472 __atomic_release_fence(); 473 return arch_atomic_sub_return_relaxed(i, v); 474 } 475 #define arch_atomic_sub_return_release arch_atomic_sub_return_release 476 #endif 477 478 #ifndef arch_atomic_sub_return 479 static __always_inline int 480 arch_atomic_sub_return(int i, atomic_t *v) 481 { 482 int ret; 483 __atomic_pre_full_fence(); 484 ret = arch_atomic_sub_return_relaxed(i, v); 485 __atomic_post_full_fence(); 486 return ret; 487 } 488 #define arch_atomic_sub_return arch_atomic_sub_return 489 #endif 490 491 #endif /* arch_atomic_sub_return_relaxed */ 492 493 #ifndef arch_atomic_fetch_sub_relaxed 494 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub 495 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub 496 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub 497 #else /* arch_atomic_fetch_sub_relaxed */ 498 499 #ifndef arch_atomic_fetch_sub_acquire 500 static __always_inline int 501 arch_atomic_fetch_sub_acquire(int i, atomic_t *v) 502 { 503 int ret = arch_atomic_fetch_sub_relaxed(i, v); 504 __atomic_acquire_fence(); 505 return ret; 506 } 507 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire 508 #endif 509 510 #ifndef arch_atomic_fetch_sub_release 511 static __always_inline int 512 arch_atomic_fetch_sub_release(int i, atomic_t *v) 513 { 514 __atomic_release_fence(); 515 return arch_atomic_fetch_sub_relaxed(i, v); 516 } 517 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release 518 #endif 519 520 #ifndef arch_atomic_fetch_sub 521 static __always_inline int 522 arch_atomic_fetch_sub(int i, atomic_t *v) 523 { 524 int ret; 525 __atomic_pre_full_fence(); 526 ret = arch_atomic_fetch_sub_relaxed(i, v); 527 __atomic_post_full_fence(); 528 return ret; 529 } 530 #define arch_atomic_fetch_sub arch_atomic_fetch_sub 531 #endif 532 533 #endif /* arch_atomic_fetch_sub_relaxed */ 534 535 #ifndef arch_atomic_inc 536 static __always_inline void 537 arch_atomic_inc(atomic_t *v) 538 { 539 arch_atomic_add(1, v); 540 } 541 #define arch_atomic_inc arch_atomic_inc 542 #endif 543 544 #ifndef arch_atomic_inc_return_relaxed 545 #ifdef arch_atomic_inc_return 546 #define arch_atomic_inc_return_acquire arch_atomic_inc_return 547 #define arch_atomic_inc_return_release arch_atomic_inc_return 548 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return 549 #endif /* arch_atomic_inc_return */ 550 551 #ifndef arch_atomic_inc_return 552 static __always_inline int 553 arch_atomic_inc_return(atomic_t *v) 554 { 555 return arch_atomic_add_return(1, v); 556 } 557 #define arch_atomic_inc_return arch_atomic_inc_return 558 #endif 559 560 #ifndef arch_atomic_inc_return_acquire 561 static __always_inline int 562 arch_atomic_inc_return_acquire(atomic_t *v) 563 { 564 return arch_atomic_add_return_acquire(1, v); 565 } 566 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 567 #endif 568 569 #ifndef arch_atomic_inc_return_release 570 static __always_inline int 571 arch_atomic_inc_return_release(atomic_t *v) 572 { 573 return arch_atomic_add_return_release(1, v); 574 } 575 #define arch_atomic_inc_return_release arch_atomic_inc_return_release 576 #endif 577 578 #ifndef arch_atomic_inc_return_relaxed 579 static __always_inline int 580 arch_atomic_inc_return_relaxed(atomic_t *v) 581 { 582 return arch_atomic_add_return_relaxed(1, v); 583 } 584 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed 585 #endif 586 587 #else /* arch_atomic_inc_return_relaxed */ 588 589 #ifndef arch_atomic_inc_return_acquire 590 static __always_inline int 591 arch_atomic_inc_return_acquire(atomic_t *v) 592 { 593 int ret = arch_atomic_inc_return_relaxed(v); 594 __atomic_acquire_fence(); 595 return ret; 596 } 597 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire 598 #endif 599 600 #ifndef arch_atomic_inc_return_release 601 static __always_inline int 602 arch_atomic_inc_return_release(atomic_t *v) 603 { 604 __atomic_release_fence(); 605 return arch_atomic_inc_return_relaxed(v); 606 } 607 #define arch_atomic_inc_return_release arch_atomic_inc_return_release 608 #endif 609 610 #ifndef arch_atomic_inc_return 611 static __always_inline int 612 arch_atomic_inc_return(atomic_t *v) 613 { 614 int ret; 615 __atomic_pre_full_fence(); 616 ret = arch_atomic_inc_return_relaxed(v); 617 __atomic_post_full_fence(); 618 return ret; 619 } 620 #define arch_atomic_inc_return arch_atomic_inc_return 621 #endif 622 623 #endif /* arch_atomic_inc_return_relaxed */ 624 625 #ifndef arch_atomic_fetch_inc_relaxed 626 #ifdef arch_atomic_fetch_inc 627 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc 628 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc 629 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc 630 #endif /* arch_atomic_fetch_inc */ 631 632 #ifndef arch_atomic_fetch_inc 633 static __always_inline int 634 arch_atomic_fetch_inc(atomic_t *v) 635 { 636 return arch_atomic_fetch_add(1, v); 637 } 638 #define arch_atomic_fetch_inc arch_atomic_fetch_inc 639 #endif 640 641 #ifndef arch_atomic_fetch_inc_acquire 642 static __always_inline int 643 arch_atomic_fetch_inc_acquire(atomic_t *v) 644 { 645 return arch_atomic_fetch_add_acquire(1, v); 646 } 647 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 648 #endif 649 650 #ifndef arch_atomic_fetch_inc_release 651 static __always_inline int 652 arch_atomic_fetch_inc_release(atomic_t *v) 653 { 654 return arch_atomic_fetch_add_release(1, v); 655 } 656 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 657 #endif 658 659 #ifndef arch_atomic_fetch_inc_relaxed 660 static __always_inline int 661 arch_atomic_fetch_inc_relaxed(atomic_t *v) 662 { 663 return arch_atomic_fetch_add_relaxed(1, v); 664 } 665 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed 666 #endif 667 668 #else /* arch_atomic_fetch_inc_relaxed */ 669 670 #ifndef arch_atomic_fetch_inc_acquire 671 static __always_inline int 672 arch_atomic_fetch_inc_acquire(atomic_t *v) 673 { 674 int ret = arch_atomic_fetch_inc_relaxed(v); 675 __atomic_acquire_fence(); 676 return ret; 677 } 678 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire 679 #endif 680 681 #ifndef arch_atomic_fetch_inc_release 682 static __always_inline int 683 arch_atomic_fetch_inc_release(atomic_t *v) 684 { 685 __atomic_release_fence(); 686 return arch_atomic_fetch_inc_relaxed(v); 687 } 688 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release 689 #endif 690 691 #ifndef arch_atomic_fetch_inc 692 static __always_inline int 693 arch_atomic_fetch_inc(atomic_t *v) 694 { 695 int ret; 696 __atomic_pre_full_fence(); 697 ret = arch_atomic_fetch_inc_relaxed(v); 698 __atomic_post_full_fence(); 699 return ret; 700 } 701 #define arch_atomic_fetch_inc arch_atomic_fetch_inc 702 #endif 703 704 #endif /* arch_atomic_fetch_inc_relaxed */ 705 706 #ifndef arch_atomic_dec 707 static __always_inline void 708 arch_atomic_dec(atomic_t *v) 709 { 710 arch_atomic_sub(1, v); 711 } 712 #define arch_atomic_dec arch_atomic_dec 713 #endif 714 715 #ifndef arch_atomic_dec_return_relaxed 716 #ifdef arch_atomic_dec_return 717 #define arch_atomic_dec_return_acquire arch_atomic_dec_return 718 #define arch_atomic_dec_return_release arch_atomic_dec_return 719 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return 720 #endif /* arch_atomic_dec_return */ 721 722 #ifndef arch_atomic_dec_return 723 static __always_inline int 724 arch_atomic_dec_return(atomic_t *v) 725 { 726 return arch_atomic_sub_return(1, v); 727 } 728 #define arch_atomic_dec_return arch_atomic_dec_return 729 #endif 730 731 #ifndef arch_atomic_dec_return_acquire 732 static __always_inline int 733 arch_atomic_dec_return_acquire(atomic_t *v) 734 { 735 return arch_atomic_sub_return_acquire(1, v); 736 } 737 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 738 #endif 739 740 #ifndef arch_atomic_dec_return_release 741 static __always_inline int 742 arch_atomic_dec_return_release(atomic_t *v) 743 { 744 return arch_atomic_sub_return_release(1, v); 745 } 746 #define arch_atomic_dec_return_release arch_atomic_dec_return_release 747 #endif 748 749 #ifndef arch_atomic_dec_return_relaxed 750 static __always_inline int 751 arch_atomic_dec_return_relaxed(atomic_t *v) 752 { 753 return arch_atomic_sub_return_relaxed(1, v); 754 } 755 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed 756 #endif 757 758 #else /* arch_atomic_dec_return_relaxed */ 759 760 #ifndef arch_atomic_dec_return_acquire 761 static __always_inline int 762 arch_atomic_dec_return_acquire(atomic_t *v) 763 { 764 int ret = arch_atomic_dec_return_relaxed(v); 765 __atomic_acquire_fence(); 766 return ret; 767 } 768 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire 769 #endif 770 771 #ifndef arch_atomic_dec_return_release 772 static __always_inline int 773 arch_atomic_dec_return_release(atomic_t *v) 774 { 775 __atomic_release_fence(); 776 return arch_atomic_dec_return_relaxed(v); 777 } 778 #define arch_atomic_dec_return_release arch_atomic_dec_return_release 779 #endif 780 781 #ifndef arch_atomic_dec_return 782 static __always_inline int 783 arch_atomic_dec_return(atomic_t *v) 784 { 785 int ret; 786 __atomic_pre_full_fence(); 787 ret = arch_atomic_dec_return_relaxed(v); 788 __atomic_post_full_fence(); 789 return ret; 790 } 791 #define arch_atomic_dec_return arch_atomic_dec_return 792 #endif 793 794 #endif /* arch_atomic_dec_return_relaxed */ 795 796 #ifndef arch_atomic_fetch_dec_relaxed 797 #ifdef arch_atomic_fetch_dec 798 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec 799 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec 800 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec 801 #endif /* arch_atomic_fetch_dec */ 802 803 #ifndef arch_atomic_fetch_dec 804 static __always_inline int 805 arch_atomic_fetch_dec(atomic_t *v) 806 { 807 return arch_atomic_fetch_sub(1, v); 808 } 809 #define arch_atomic_fetch_dec arch_atomic_fetch_dec 810 #endif 811 812 #ifndef arch_atomic_fetch_dec_acquire 813 static __always_inline int 814 arch_atomic_fetch_dec_acquire(atomic_t *v) 815 { 816 return arch_atomic_fetch_sub_acquire(1, v); 817 } 818 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 819 #endif 820 821 #ifndef arch_atomic_fetch_dec_release 822 static __always_inline int 823 arch_atomic_fetch_dec_release(atomic_t *v) 824 { 825 return arch_atomic_fetch_sub_release(1, v); 826 } 827 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 828 #endif 829 830 #ifndef arch_atomic_fetch_dec_relaxed 831 static __always_inline int 832 arch_atomic_fetch_dec_relaxed(atomic_t *v) 833 { 834 return arch_atomic_fetch_sub_relaxed(1, v); 835 } 836 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed 837 #endif 838 839 #else /* arch_atomic_fetch_dec_relaxed */ 840 841 #ifndef arch_atomic_fetch_dec_acquire 842 static __always_inline int 843 arch_atomic_fetch_dec_acquire(atomic_t *v) 844 { 845 int ret = arch_atomic_fetch_dec_relaxed(v); 846 __atomic_acquire_fence(); 847 return ret; 848 } 849 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire 850 #endif 851 852 #ifndef arch_atomic_fetch_dec_release 853 static __always_inline int 854 arch_atomic_fetch_dec_release(atomic_t *v) 855 { 856 __atomic_release_fence(); 857 return arch_atomic_fetch_dec_relaxed(v); 858 } 859 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release 860 #endif 861 862 #ifndef arch_atomic_fetch_dec 863 static __always_inline int 864 arch_atomic_fetch_dec(atomic_t *v) 865 { 866 int ret; 867 __atomic_pre_full_fence(); 868 ret = arch_atomic_fetch_dec_relaxed(v); 869 __atomic_post_full_fence(); 870 return ret; 871 } 872 #define arch_atomic_fetch_dec arch_atomic_fetch_dec 873 #endif 874 875 #endif /* arch_atomic_fetch_dec_relaxed */ 876 877 #ifndef arch_atomic_fetch_and_relaxed 878 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and 879 #define arch_atomic_fetch_and_release arch_atomic_fetch_and 880 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and 881 #else /* arch_atomic_fetch_and_relaxed */ 882 883 #ifndef arch_atomic_fetch_and_acquire 884 static __always_inline int 885 arch_atomic_fetch_and_acquire(int i, atomic_t *v) 886 { 887 int ret = arch_atomic_fetch_and_relaxed(i, v); 888 __atomic_acquire_fence(); 889 return ret; 890 } 891 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire 892 #endif 893 894 #ifndef arch_atomic_fetch_and_release 895 static __always_inline int 896 arch_atomic_fetch_and_release(int i, atomic_t *v) 897 { 898 __atomic_release_fence(); 899 return arch_atomic_fetch_and_relaxed(i, v); 900 } 901 #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release 902 #endif 903 904 #ifndef arch_atomic_fetch_and 905 static __always_inline int 906 arch_atomic_fetch_and(int i, atomic_t *v) 907 { 908 int ret; 909 __atomic_pre_full_fence(); 910 ret = arch_atomic_fetch_and_relaxed(i, v); 911 __atomic_post_full_fence(); 912 return ret; 913 } 914 #define arch_atomic_fetch_and arch_atomic_fetch_and 915 #endif 916 917 #endif /* arch_atomic_fetch_and_relaxed */ 918 919 #ifndef arch_atomic_andnot 920 static __always_inline void 921 arch_atomic_andnot(int i, atomic_t *v) 922 { 923 arch_atomic_and(~i, v); 924 } 925 #define arch_atomic_andnot arch_atomic_andnot 926 #endif 927 928 #ifndef arch_atomic_fetch_andnot_relaxed 929 #ifdef arch_atomic_fetch_andnot 930 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot 931 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot 932 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot 933 #endif /* arch_atomic_fetch_andnot */ 934 935 #ifndef arch_atomic_fetch_andnot 936 static __always_inline int 937 arch_atomic_fetch_andnot(int i, atomic_t *v) 938 { 939 return arch_atomic_fetch_and(~i, v); 940 } 941 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 942 #endif 943 944 #ifndef arch_atomic_fetch_andnot_acquire 945 static __always_inline int 946 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 947 { 948 return arch_atomic_fetch_and_acquire(~i, v); 949 } 950 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 951 #endif 952 953 #ifndef arch_atomic_fetch_andnot_release 954 static __always_inline int 955 arch_atomic_fetch_andnot_release(int i, atomic_t *v) 956 { 957 return arch_atomic_fetch_and_release(~i, v); 958 } 959 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 960 #endif 961 962 #ifndef arch_atomic_fetch_andnot_relaxed 963 static __always_inline int 964 arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 965 { 966 return arch_atomic_fetch_and_relaxed(~i, v); 967 } 968 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed 969 #endif 970 971 #else /* arch_atomic_fetch_andnot_relaxed */ 972 973 #ifndef arch_atomic_fetch_andnot_acquire 974 static __always_inline int 975 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) 976 { 977 int ret = arch_atomic_fetch_andnot_relaxed(i, v); 978 __atomic_acquire_fence(); 979 return ret; 980 } 981 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire 982 #endif 983 984 #ifndef arch_atomic_fetch_andnot_release 985 static __always_inline int 986 arch_atomic_fetch_andnot_release(int i, atomic_t *v) 987 { 988 __atomic_release_fence(); 989 return arch_atomic_fetch_andnot_relaxed(i, v); 990 } 991 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release 992 #endif 993 994 #ifndef arch_atomic_fetch_andnot 995 static __always_inline int 996 arch_atomic_fetch_andnot(int i, atomic_t *v) 997 { 998 int ret; 999 __atomic_pre_full_fence(); 1000 ret = arch_atomic_fetch_andnot_relaxed(i, v); 1001 __atomic_post_full_fence(); 1002 return ret; 1003 } 1004 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot 1005 #endif 1006 1007 #endif /* arch_atomic_fetch_andnot_relaxed */ 1008 1009 #ifndef arch_atomic_fetch_or_relaxed 1010 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or 1011 #define arch_atomic_fetch_or_release arch_atomic_fetch_or 1012 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or 1013 #else /* arch_atomic_fetch_or_relaxed */ 1014 1015 #ifndef arch_atomic_fetch_or_acquire 1016 static __always_inline int 1017 arch_atomic_fetch_or_acquire(int i, atomic_t *v) 1018 { 1019 int ret = arch_atomic_fetch_or_relaxed(i, v); 1020 __atomic_acquire_fence(); 1021 return ret; 1022 } 1023 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire 1024 #endif 1025 1026 #ifndef arch_atomic_fetch_or_release 1027 static __always_inline int 1028 arch_atomic_fetch_or_release(int i, atomic_t *v) 1029 { 1030 __atomic_release_fence(); 1031 return arch_atomic_fetch_or_relaxed(i, v); 1032 } 1033 #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release 1034 #endif 1035 1036 #ifndef arch_atomic_fetch_or 1037 static __always_inline int 1038 arch_atomic_fetch_or(int i, atomic_t *v) 1039 { 1040 int ret; 1041 __atomic_pre_full_fence(); 1042 ret = arch_atomic_fetch_or_relaxed(i, v); 1043 __atomic_post_full_fence(); 1044 return ret; 1045 } 1046 #define arch_atomic_fetch_or arch_atomic_fetch_or 1047 #endif 1048 1049 #endif /* arch_atomic_fetch_or_relaxed */ 1050 1051 #ifndef arch_atomic_fetch_xor_relaxed 1052 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor 1053 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor 1054 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor 1055 #else /* arch_atomic_fetch_xor_relaxed */ 1056 1057 #ifndef arch_atomic_fetch_xor_acquire 1058 static __always_inline int 1059 arch_atomic_fetch_xor_acquire(int i, atomic_t *v) 1060 { 1061 int ret = arch_atomic_fetch_xor_relaxed(i, v); 1062 __atomic_acquire_fence(); 1063 return ret; 1064 } 1065 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire 1066 #endif 1067 1068 #ifndef arch_atomic_fetch_xor_release 1069 static __always_inline int 1070 arch_atomic_fetch_xor_release(int i, atomic_t *v) 1071 { 1072 __atomic_release_fence(); 1073 return arch_atomic_fetch_xor_relaxed(i, v); 1074 } 1075 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release 1076 #endif 1077 1078 #ifndef arch_atomic_fetch_xor 1079 static __always_inline int 1080 arch_atomic_fetch_xor(int i, atomic_t *v) 1081 { 1082 int ret; 1083 __atomic_pre_full_fence(); 1084 ret = arch_atomic_fetch_xor_relaxed(i, v); 1085 __atomic_post_full_fence(); 1086 return ret; 1087 } 1088 #define arch_atomic_fetch_xor arch_atomic_fetch_xor 1089 #endif 1090 1091 #endif /* arch_atomic_fetch_xor_relaxed */ 1092 1093 #ifndef arch_atomic_xchg_relaxed 1094 #define arch_atomic_xchg_acquire arch_atomic_xchg 1095 #define arch_atomic_xchg_release arch_atomic_xchg 1096 #define arch_atomic_xchg_relaxed arch_atomic_xchg 1097 #else /* arch_atomic_xchg_relaxed */ 1098 1099 #ifndef arch_atomic_xchg_acquire 1100 static __always_inline int 1101 arch_atomic_xchg_acquire(atomic_t *v, int i) 1102 { 1103 int ret = arch_atomic_xchg_relaxed(v, i); 1104 __atomic_acquire_fence(); 1105 return ret; 1106 } 1107 #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire 1108 #endif 1109 1110 #ifndef arch_atomic_xchg_release 1111 static __always_inline int 1112 arch_atomic_xchg_release(atomic_t *v, int i) 1113 { 1114 __atomic_release_fence(); 1115 return arch_atomic_xchg_relaxed(v, i); 1116 } 1117 #define arch_atomic_xchg_release arch_atomic_xchg_release 1118 #endif 1119 1120 #ifndef arch_atomic_xchg 1121 static __always_inline int 1122 arch_atomic_xchg(atomic_t *v, int i) 1123 { 1124 int ret; 1125 __atomic_pre_full_fence(); 1126 ret = arch_atomic_xchg_relaxed(v, i); 1127 __atomic_post_full_fence(); 1128 return ret; 1129 } 1130 #define arch_atomic_xchg arch_atomic_xchg 1131 #endif 1132 1133 #endif /* arch_atomic_xchg_relaxed */ 1134 1135 #ifndef arch_atomic_cmpxchg_relaxed 1136 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg 1137 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg 1138 #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg 1139 #else /* arch_atomic_cmpxchg_relaxed */ 1140 1141 #ifndef arch_atomic_cmpxchg_acquire 1142 static __always_inline int 1143 arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 1144 { 1145 int ret = arch_atomic_cmpxchg_relaxed(v, old, new); 1146 __atomic_acquire_fence(); 1147 return ret; 1148 } 1149 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire 1150 #endif 1151 1152 #ifndef arch_atomic_cmpxchg_release 1153 static __always_inline int 1154 arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) 1155 { 1156 __atomic_release_fence(); 1157 return arch_atomic_cmpxchg_relaxed(v, old, new); 1158 } 1159 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release 1160 #endif 1161 1162 #ifndef arch_atomic_cmpxchg 1163 static __always_inline int 1164 arch_atomic_cmpxchg(atomic_t *v, int old, int new) 1165 { 1166 int ret; 1167 __atomic_pre_full_fence(); 1168 ret = arch_atomic_cmpxchg_relaxed(v, old, new); 1169 __atomic_post_full_fence(); 1170 return ret; 1171 } 1172 #define arch_atomic_cmpxchg arch_atomic_cmpxchg 1173 #endif 1174 1175 #endif /* arch_atomic_cmpxchg_relaxed */ 1176 1177 #ifndef arch_atomic_try_cmpxchg_relaxed 1178 #ifdef arch_atomic_try_cmpxchg 1179 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg 1180 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg 1181 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg 1182 #endif /* arch_atomic_try_cmpxchg */ 1183 1184 #ifndef arch_atomic_try_cmpxchg 1185 static __always_inline bool 1186 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1187 { 1188 int r, o = *old; 1189 r = arch_atomic_cmpxchg(v, o, new); 1190 if (unlikely(r != o)) 1191 *old = r; 1192 return likely(r == o); 1193 } 1194 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1195 #endif 1196 1197 #ifndef arch_atomic_try_cmpxchg_acquire 1198 static __always_inline bool 1199 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1200 { 1201 int r, o = *old; 1202 r = arch_atomic_cmpxchg_acquire(v, o, new); 1203 if (unlikely(r != o)) 1204 *old = r; 1205 return likely(r == o); 1206 } 1207 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1208 #endif 1209 1210 #ifndef arch_atomic_try_cmpxchg_release 1211 static __always_inline bool 1212 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1213 { 1214 int r, o = *old; 1215 r = arch_atomic_cmpxchg_release(v, o, new); 1216 if (unlikely(r != o)) 1217 *old = r; 1218 return likely(r == o); 1219 } 1220 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1221 #endif 1222 1223 #ifndef arch_atomic_try_cmpxchg_relaxed 1224 static __always_inline bool 1225 arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 1226 { 1227 int r, o = *old; 1228 r = arch_atomic_cmpxchg_relaxed(v, o, new); 1229 if (unlikely(r != o)) 1230 *old = r; 1231 return likely(r == o); 1232 } 1233 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed 1234 #endif 1235 1236 #else /* arch_atomic_try_cmpxchg_relaxed */ 1237 1238 #ifndef arch_atomic_try_cmpxchg_acquire 1239 static __always_inline bool 1240 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 1241 { 1242 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1243 __atomic_acquire_fence(); 1244 return ret; 1245 } 1246 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire 1247 #endif 1248 1249 #ifndef arch_atomic_try_cmpxchg_release 1250 static __always_inline bool 1251 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 1252 { 1253 __atomic_release_fence(); 1254 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 1255 } 1256 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release 1257 #endif 1258 1259 #ifndef arch_atomic_try_cmpxchg 1260 static __always_inline bool 1261 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 1262 { 1263 bool ret; 1264 __atomic_pre_full_fence(); 1265 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 1266 __atomic_post_full_fence(); 1267 return ret; 1268 } 1269 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg 1270 #endif 1271 1272 #endif /* arch_atomic_try_cmpxchg_relaxed */ 1273 1274 #ifndef arch_atomic_sub_and_test 1275 /** 1276 * arch_atomic_sub_and_test - subtract value from variable and test result 1277 * @i: integer value to subtract 1278 * @v: pointer of type atomic_t 1279 * 1280 * Atomically subtracts @i from @v and returns 1281 * true if the result is zero, or false for all 1282 * other cases. 1283 */ 1284 static __always_inline bool 1285 arch_atomic_sub_and_test(int i, atomic_t *v) 1286 { 1287 return arch_atomic_sub_return(i, v) == 0; 1288 } 1289 #define arch_atomic_sub_and_test arch_atomic_sub_and_test 1290 #endif 1291 1292 #ifndef arch_atomic_dec_and_test 1293 /** 1294 * arch_atomic_dec_and_test - decrement and test 1295 * @v: pointer of type atomic_t 1296 * 1297 * Atomically decrements @v by 1 and 1298 * returns true if the result is 0, or false for all other 1299 * cases. 1300 */ 1301 static __always_inline bool 1302 arch_atomic_dec_and_test(atomic_t *v) 1303 { 1304 return arch_atomic_dec_return(v) == 0; 1305 } 1306 #define arch_atomic_dec_and_test arch_atomic_dec_and_test 1307 #endif 1308 1309 #ifndef arch_atomic_inc_and_test 1310 /** 1311 * arch_atomic_inc_and_test - increment and test 1312 * @v: pointer of type atomic_t 1313 * 1314 * Atomically increments @v by 1 1315 * and returns true if the result is zero, or false for all 1316 * other cases. 1317 */ 1318 static __always_inline bool 1319 arch_atomic_inc_and_test(atomic_t *v) 1320 { 1321 return arch_atomic_inc_return(v) == 0; 1322 } 1323 #define arch_atomic_inc_and_test arch_atomic_inc_and_test 1324 #endif 1325 1326 #ifndef arch_atomic_add_negative_relaxed 1327 #ifdef arch_atomic_add_negative 1328 #define arch_atomic_add_negative_acquire arch_atomic_add_negative 1329 #define arch_atomic_add_negative_release arch_atomic_add_negative 1330 #define arch_atomic_add_negative_relaxed arch_atomic_add_negative 1331 #endif /* arch_atomic_add_negative */ 1332 1333 #ifndef arch_atomic_add_negative 1334 /** 1335 * arch_atomic_add_negative - Add and test if negative 1336 * @i: integer value to add 1337 * @v: pointer of type atomic_t 1338 * 1339 * Atomically adds @i to @v and returns true if the result is negative, 1340 * or false when the result is greater than or equal to zero. 1341 */ 1342 static __always_inline bool 1343 arch_atomic_add_negative(int i, atomic_t *v) 1344 { 1345 return arch_atomic_add_return(i, v) < 0; 1346 } 1347 #define arch_atomic_add_negative arch_atomic_add_negative 1348 #endif 1349 1350 #ifndef arch_atomic_add_negative_acquire 1351 /** 1352 * arch_atomic_add_negative_acquire - Add and test if negative 1353 * @i: integer value to add 1354 * @v: pointer of type atomic_t 1355 * 1356 * Atomically adds @i to @v and returns true if the result is negative, 1357 * or false when the result is greater than or equal to zero. 1358 */ 1359 static __always_inline bool 1360 arch_atomic_add_negative_acquire(int i, atomic_t *v) 1361 { 1362 return arch_atomic_add_return_acquire(i, v) < 0; 1363 } 1364 #define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire 1365 #endif 1366 1367 #ifndef arch_atomic_add_negative_release 1368 /** 1369 * arch_atomic_add_negative_release - Add and test if negative 1370 * @i: integer value to add 1371 * @v: pointer of type atomic_t 1372 * 1373 * Atomically adds @i to @v and returns true if the result is negative, 1374 * or false when the result is greater than or equal to zero. 1375 */ 1376 static __always_inline bool 1377 arch_atomic_add_negative_release(int i, atomic_t *v) 1378 { 1379 return arch_atomic_add_return_release(i, v) < 0; 1380 } 1381 #define arch_atomic_add_negative_release arch_atomic_add_negative_release 1382 #endif 1383 1384 #ifndef arch_atomic_add_negative_relaxed 1385 /** 1386 * arch_atomic_add_negative_relaxed - Add and test if negative 1387 * @i: integer value to add 1388 * @v: pointer of type atomic_t 1389 * 1390 * Atomically adds @i to @v and returns true if the result is negative, 1391 * or false when the result is greater than or equal to zero. 1392 */ 1393 static __always_inline bool 1394 arch_atomic_add_negative_relaxed(int i, atomic_t *v) 1395 { 1396 return arch_atomic_add_return_relaxed(i, v) < 0; 1397 } 1398 #define arch_atomic_add_negative_relaxed arch_atomic_add_negative_relaxed 1399 #endif 1400 1401 #else /* arch_atomic_add_negative_relaxed */ 1402 1403 #ifndef arch_atomic_add_negative_acquire 1404 static __always_inline bool 1405 arch_atomic_add_negative_acquire(int i, atomic_t *v) 1406 { 1407 bool ret = arch_atomic_add_negative_relaxed(i, v); 1408 __atomic_acquire_fence(); 1409 return ret; 1410 } 1411 #define arch_atomic_add_negative_acquire arch_atomic_add_negative_acquire 1412 #endif 1413 1414 #ifndef arch_atomic_add_negative_release 1415 static __always_inline bool 1416 arch_atomic_add_negative_release(int i, atomic_t *v) 1417 { 1418 __atomic_release_fence(); 1419 return arch_atomic_add_negative_relaxed(i, v); 1420 } 1421 #define arch_atomic_add_negative_release arch_atomic_add_negative_release 1422 #endif 1423 1424 #ifndef arch_atomic_add_negative 1425 static __always_inline bool 1426 arch_atomic_add_negative(int i, atomic_t *v) 1427 { 1428 bool ret; 1429 __atomic_pre_full_fence(); 1430 ret = arch_atomic_add_negative_relaxed(i, v); 1431 __atomic_post_full_fence(); 1432 return ret; 1433 } 1434 #define arch_atomic_add_negative arch_atomic_add_negative 1435 #endif 1436 1437 #endif /* arch_atomic_add_negative_relaxed */ 1438 1439 #ifndef arch_atomic_fetch_add_unless 1440 /** 1441 * arch_atomic_fetch_add_unless - add unless the number is already a given value 1442 * @v: pointer of type atomic_t 1443 * @a: the amount to add to v... 1444 * @u: ...unless v is equal to u. 1445 * 1446 * Atomically adds @a to @v, so long as @v was not already @u. 1447 * Returns original value of @v 1448 */ 1449 static __always_inline int 1450 arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) 1451 { 1452 int c = arch_atomic_read(v); 1453 1454 do { 1455 if (unlikely(c == u)) 1456 break; 1457 } while (!arch_atomic_try_cmpxchg(v, &c, c + a)); 1458 1459 return c; 1460 } 1461 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless 1462 #endif 1463 1464 #ifndef arch_atomic_add_unless 1465 /** 1466 * arch_atomic_add_unless - add unless the number is already a given value 1467 * @v: pointer of type atomic_t 1468 * @a: the amount to add to v... 1469 * @u: ...unless v is equal to u. 1470 * 1471 * Atomically adds @a to @v, if @v was not already @u. 1472 * Returns true if the addition was done. 1473 */ 1474 static __always_inline bool 1475 arch_atomic_add_unless(atomic_t *v, int a, int u) 1476 { 1477 return arch_atomic_fetch_add_unless(v, a, u) != u; 1478 } 1479 #define arch_atomic_add_unless arch_atomic_add_unless 1480 #endif 1481 1482 #ifndef arch_atomic_inc_not_zero 1483 /** 1484 * arch_atomic_inc_not_zero - increment unless the number is zero 1485 * @v: pointer of type atomic_t 1486 * 1487 * Atomically increments @v by 1, if @v is non-zero. 1488 * Returns true if the increment was done. 1489 */ 1490 static __always_inline bool 1491 arch_atomic_inc_not_zero(atomic_t *v) 1492 { 1493 return arch_atomic_add_unless(v, 1, 0); 1494 } 1495 #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero 1496 #endif 1497 1498 #ifndef arch_atomic_inc_unless_negative 1499 static __always_inline bool 1500 arch_atomic_inc_unless_negative(atomic_t *v) 1501 { 1502 int c = arch_atomic_read(v); 1503 1504 do { 1505 if (unlikely(c < 0)) 1506 return false; 1507 } while (!arch_atomic_try_cmpxchg(v, &c, c + 1)); 1508 1509 return true; 1510 } 1511 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative 1512 #endif 1513 1514 #ifndef arch_atomic_dec_unless_positive 1515 static __always_inline bool 1516 arch_atomic_dec_unless_positive(atomic_t *v) 1517 { 1518 int c = arch_atomic_read(v); 1519 1520 do { 1521 if (unlikely(c > 0)) 1522 return false; 1523 } while (!arch_atomic_try_cmpxchg(v, &c, c - 1)); 1524 1525 return true; 1526 } 1527 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive 1528 #endif 1529 1530 #ifndef arch_atomic_dec_if_positive 1531 static __always_inline int 1532 arch_atomic_dec_if_positive(atomic_t *v) 1533 { 1534 int dec, c = arch_atomic_read(v); 1535 1536 do { 1537 dec = c - 1; 1538 if (unlikely(dec < 0)) 1539 break; 1540 } while (!arch_atomic_try_cmpxchg(v, &c, dec)); 1541 1542 return dec; 1543 } 1544 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive 1545 #endif 1546 1547 #ifdef CONFIG_GENERIC_ATOMIC64 1548 #include <asm-generic/atomic64.h> 1549 #endif 1550 1551 #ifndef arch_atomic64_read_acquire 1552 static __always_inline s64 1553 arch_atomic64_read_acquire(const atomic64_t *v) 1554 { 1555 s64 ret; 1556 1557 if (__native_word(atomic64_t)) { 1558 ret = smp_load_acquire(&(v)->counter); 1559 } else { 1560 ret = arch_atomic64_read(v); 1561 __atomic_acquire_fence(); 1562 } 1563 1564 return ret; 1565 } 1566 #define arch_atomic64_read_acquire arch_atomic64_read_acquire 1567 #endif 1568 1569 #ifndef arch_atomic64_set_release 1570 static __always_inline void 1571 arch_atomic64_set_release(atomic64_t *v, s64 i) 1572 { 1573 if (__native_word(atomic64_t)) { 1574 smp_store_release(&(v)->counter, i); 1575 } else { 1576 __atomic_release_fence(); 1577 arch_atomic64_set(v, i); 1578 } 1579 } 1580 #define arch_atomic64_set_release arch_atomic64_set_release 1581 #endif 1582 1583 #ifndef arch_atomic64_add_return_relaxed 1584 #define arch_atomic64_add_return_acquire arch_atomic64_add_return 1585 #define arch_atomic64_add_return_release arch_atomic64_add_return 1586 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return 1587 #else /* arch_atomic64_add_return_relaxed */ 1588 1589 #ifndef arch_atomic64_add_return_acquire 1590 static __always_inline s64 1591 arch_atomic64_add_return_acquire(s64 i, atomic64_t *v) 1592 { 1593 s64 ret = arch_atomic64_add_return_relaxed(i, v); 1594 __atomic_acquire_fence(); 1595 return ret; 1596 } 1597 #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire 1598 #endif 1599 1600 #ifndef arch_atomic64_add_return_release 1601 static __always_inline s64 1602 arch_atomic64_add_return_release(s64 i, atomic64_t *v) 1603 { 1604 __atomic_release_fence(); 1605 return arch_atomic64_add_return_relaxed(i, v); 1606 } 1607 #define arch_atomic64_add_return_release arch_atomic64_add_return_release 1608 #endif 1609 1610 #ifndef arch_atomic64_add_return 1611 static __always_inline s64 1612 arch_atomic64_add_return(s64 i, atomic64_t *v) 1613 { 1614 s64 ret; 1615 __atomic_pre_full_fence(); 1616 ret = arch_atomic64_add_return_relaxed(i, v); 1617 __atomic_post_full_fence(); 1618 return ret; 1619 } 1620 #define arch_atomic64_add_return arch_atomic64_add_return 1621 #endif 1622 1623 #endif /* arch_atomic64_add_return_relaxed */ 1624 1625 #ifndef arch_atomic64_fetch_add_relaxed 1626 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add 1627 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add 1628 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add 1629 #else /* arch_atomic64_fetch_add_relaxed */ 1630 1631 #ifndef arch_atomic64_fetch_add_acquire 1632 static __always_inline s64 1633 arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 1634 { 1635 s64 ret = arch_atomic64_fetch_add_relaxed(i, v); 1636 __atomic_acquire_fence(); 1637 return ret; 1638 } 1639 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire 1640 #endif 1641 1642 #ifndef arch_atomic64_fetch_add_release 1643 static __always_inline s64 1644 arch_atomic64_fetch_add_release(s64 i, atomic64_t *v) 1645 { 1646 __atomic_release_fence(); 1647 return arch_atomic64_fetch_add_relaxed(i, v); 1648 } 1649 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release 1650 #endif 1651 1652 #ifndef arch_atomic64_fetch_add 1653 static __always_inline s64 1654 arch_atomic64_fetch_add(s64 i, atomic64_t *v) 1655 { 1656 s64 ret; 1657 __atomic_pre_full_fence(); 1658 ret = arch_atomic64_fetch_add_relaxed(i, v); 1659 __atomic_post_full_fence(); 1660 return ret; 1661 } 1662 #define arch_atomic64_fetch_add arch_atomic64_fetch_add 1663 #endif 1664 1665 #endif /* arch_atomic64_fetch_add_relaxed */ 1666 1667 #ifndef arch_atomic64_sub_return_relaxed 1668 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return 1669 #define arch_atomic64_sub_return_release arch_atomic64_sub_return 1670 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return 1671 #else /* arch_atomic64_sub_return_relaxed */ 1672 1673 #ifndef arch_atomic64_sub_return_acquire 1674 static __always_inline s64 1675 arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 1676 { 1677 s64 ret = arch_atomic64_sub_return_relaxed(i, v); 1678 __atomic_acquire_fence(); 1679 return ret; 1680 } 1681 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire 1682 #endif 1683 1684 #ifndef arch_atomic64_sub_return_release 1685 static __always_inline s64 1686 arch_atomic64_sub_return_release(s64 i, atomic64_t *v) 1687 { 1688 __atomic_release_fence(); 1689 return arch_atomic64_sub_return_relaxed(i, v); 1690 } 1691 #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release 1692 #endif 1693 1694 #ifndef arch_atomic64_sub_return 1695 static __always_inline s64 1696 arch_atomic64_sub_return(s64 i, atomic64_t *v) 1697 { 1698 s64 ret; 1699 __atomic_pre_full_fence(); 1700 ret = arch_atomic64_sub_return_relaxed(i, v); 1701 __atomic_post_full_fence(); 1702 return ret; 1703 } 1704 #define arch_atomic64_sub_return arch_atomic64_sub_return 1705 #endif 1706 1707 #endif /* arch_atomic64_sub_return_relaxed */ 1708 1709 #ifndef arch_atomic64_fetch_sub_relaxed 1710 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub 1711 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub 1712 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub 1713 #else /* arch_atomic64_fetch_sub_relaxed */ 1714 1715 #ifndef arch_atomic64_fetch_sub_acquire 1716 static __always_inline s64 1717 arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 1718 { 1719 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1720 __atomic_acquire_fence(); 1721 return ret; 1722 } 1723 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire 1724 #endif 1725 1726 #ifndef arch_atomic64_fetch_sub_release 1727 static __always_inline s64 1728 arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 1729 { 1730 __atomic_release_fence(); 1731 return arch_atomic64_fetch_sub_relaxed(i, v); 1732 } 1733 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release 1734 #endif 1735 1736 #ifndef arch_atomic64_fetch_sub 1737 static __always_inline s64 1738 arch_atomic64_fetch_sub(s64 i, atomic64_t *v) 1739 { 1740 s64 ret; 1741 __atomic_pre_full_fence(); 1742 ret = arch_atomic64_fetch_sub_relaxed(i, v); 1743 __atomic_post_full_fence(); 1744 return ret; 1745 } 1746 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub 1747 #endif 1748 1749 #endif /* arch_atomic64_fetch_sub_relaxed */ 1750 1751 #ifndef arch_atomic64_inc 1752 static __always_inline void 1753 arch_atomic64_inc(atomic64_t *v) 1754 { 1755 arch_atomic64_add(1, v); 1756 } 1757 #define arch_atomic64_inc arch_atomic64_inc 1758 #endif 1759 1760 #ifndef arch_atomic64_inc_return_relaxed 1761 #ifdef arch_atomic64_inc_return 1762 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return 1763 #define arch_atomic64_inc_return_release arch_atomic64_inc_return 1764 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return 1765 #endif /* arch_atomic64_inc_return */ 1766 1767 #ifndef arch_atomic64_inc_return 1768 static __always_inline s64 1769 arch_atomic64_inc_return(atomic64_t *v) 1770 { 1771 return arch_atomic64_add_return(1, v); 1772 } 1773 #define arch_atomic64_inc_return arch_atomic64_inc_return 1774 #endif 1775 1776 #ifndef arch_atomic64_inc_return_acquire 1777 static __always_inline s64 1778 arch_atomic64_inc_return_acquire(atomic64_t *v) 1779 { 1780 return arch_atomic64_add_return_acquire(1, v); 1781 } 1782 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1783 #endif 1784 1785 #ifndef arch_atomic64_inc_return_release 1786 static __always_inline s64 1787 arch_atomic64_inc_return_release(atomic64_t *v) 1788 { 1789 return arch_atomic64_add_return_release(1, v); 1790 } 1791 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1792 #endif 1793 1794 #ifndef arch_atomic64_inc_return_relaxed 1795 static __always_inline s64 1796 arch_atomic64_inc_return_relaxed(atomic64_t *v) 1797 { 1798 return arch_atomic64_add_return_relaxed(1, v); 1799 } 1800 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed 1801 #endif 1802 1803 #else /* arch_atomic64_inc_return_relaxed */ 1804 1805 #ifndef arch_atomic64_inc_return_acquire 1806 static __always_inline s64 1807 arch_atomic64_inc_return_acquire(atomic64_t *v) 1808 { 1809 s64 ret = arch_atomic64_inc_return_relaxed(v); 1810 __atomic_acquire_fence(); 1811 return ret; 1812 } 1813 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire 1814 #endif 1815 1816 #ifndef arch_atomic64_inc_return_release 1817 static __always_inline s64 1818 arch_atomic64_inc_return_release(atomic64_t *v) 1819 { 1820 __atomic_release_fence(); 1821 return arch_atomic64_inc_return_relaxed(v); 1822 } 1823 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release 1824 #endif 1825 1826 #ifndef arch_atomic64_inc_return 1827 static __always_inline s64 1828 arch_atomic64_inc_return(atomic64_t *v) 1829 { 1830 s64 ret; 1831 __atomic_pre_full_fence(); 1832 ret = arch_atomic64_inc_return_relaxed(v); 1833 __atomic_post_full_fence(); 1834 return ret; 1835 } 1836 #define arch_atomic64_inc_return arch_atomic64_inc_return 1837 #endif 1838 1839 #endif /* arch_atomic64_inc_return_relaxed */ 1840 1841 #ifndef arch_atomic64_fetch_inc_relaxed 1842 #ifdef arch_atomic64_fetch_inc 1843 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc 1844 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc 1845 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc 1846 #endif /* arch_atomic64_fetch_inc */ 1847 1848 #ifndef arch_atomic64_fetch_inc 1849 static __always_inline s64 1850 arch_atomic64_fetch_inc(atomic64_t *v) 1851 { 1852 return arch_atomic64_fetch_add(1, v); 1853 } 1854 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 1855 #endif 1856 1857 #ifndef arch_atomic64_fetch_inc_acquire 1858 static __always_inline s64 1859 arch_atomic64_fetch_inc_acquire(atomic64_t *v) 1860 { 1861 return arch_atomic64_fetch_add_acquire(1, v); 1862 } 1863 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1864 #endif 1865 1866 #ifndef arch_atomic64_fetch_inc_release 1867 static __always_inline s64 1868 arch_atomic64_fetch_inc_release(atomic64_t *v) 1869 { 1870 return arch_atomic64_fetch_add_release(1, v); 1871 } 1872 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1873 #endif 1874 1875 #ifndef arch_atomic64_fetch_inc_relaxed 1876 static __always_inline s64 1877 arch_atomic64_fetch_inc_relaxed(atomic64_t *v) 1878 { 1879 return arch_atomic64_fetch_add_relaxed(1, v); 1880 } 1881 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed 1882 #endif 1883 1884 #else /* arch_atomic64_fetch_inc_relaxed */ 1885 1886 #ifndef arch_atomic64_fetch_inc_acquire 1887 static __always_inline s64 1888 arch_atomic64_fetch_inc_acquire(atomic64_t *v) 1889 { 1890 s64 ret = arch_atomic64_fetch_inc_relaxed(v); 1891 __atomic_acquire_fence(); 1892 return ret; 1893 } 1894 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire 1895 #endif 1896 1897 #ifndef arch_atomic64_fetch_inc_release 1898 static __always_inline s64 1899 arch_atomic64_fetch_inc_release(atomic64_t *v) 1900 { 1901 __atomic_release_fence(); 1902 return arch_atomic64_fetch_inc_relaxed(v); 1903 } 1904 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release 1905 #endif 1906 1907 #ifndef arch_atomic64_fetch_inc 1908 static __always_inline s64 1909 arch_atomic64_fetch_inc(atomic64_t *v) 1910 { 1911 s64 ret; 1912 __atomic_pre_full_fence(); 1913 ret = arch_atomic64_fetch_inc_relaxed(v); 1914 __atomic_post_full_fence(); 1915 return ret; 1916 } 1917 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc 1918 #endif 1919 1920 #endif /* arch_atomic64_fetch_inc_relaxed */ 1921 1922 #ifndef arch_atomic64_dec 1923 static __always_inline void 1924 arch_atomic64_dec(atomic64_t *v) 1925 { 1926 arch_atomic64_sub(1, v); 1927 } 1928 #define arch_atomic64_dec arch_atomic64_dec 1929 #endif 1930 1931 #ifndef arch_atomic64_dec_return_relaxed 1932 #ifdef arch_atomic64_dec_return 1933 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return 1934 #define arch_atomic64_dec_return_release arch_atomic64_dec_return 1935 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return 1936 #endif /* arch_atomic64_dec_return */ 1937 1938 #ifndef arch_atomic64_dec_return 1939 static __always_inline s64 1940 arch_atomic64_dec_return(atomic64_t *v) 1941 { 1942 return arch_atomic64_sub_return(1, v); 1943 } 1944 #define arch_atomic64_dec_return arch_atomic64_dec_return 1945 #endif 1946 1947 #ifndef arch_atomic64_dec_return_acquire 1948 static __always_inline s64 1949 arch_atomic64_dec_return_acquire(atomic64_t *v) 1950 { 1951 return arch_atomic64_sub_return_acquire(1, v); 1952 } 1953 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 1954 #endif 1955 1956 #ifndef arch_atomic64_dec_return_release 1957 static __always_inline s64 1958 arch_atomic64_dec_return_release(atomic64_t *v) 1959 { 1960 return arch_atomic64_sub_return_release(1, v); 1961 } 1962 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 1963 #endif 1964 1965 #ifndef arch_atomic64_dec_return_relaxed 1966 static __always_inline s64 1967 arch_atomic64_dec_return_relaxed(atomic64_t *v) 1968 { 1969 return arch_atomic64_sub_return_relaxed(1, v); 1970 } 1971 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed 1972 #endif 1973 1974 #else /* arch_atomic64_dec_return_relaxed */ 1975 1976 #ifndef arch_atomic64_dec_return_acquire 1977 static __always_inline s64 1978 arch_atomic64_dec_return_acquire(atomic64_t *v) 1979 { 1980 s64 ret = arch_atomic64_dec_return_relaxed(v); 1981 __atomic_acquire_fence(); 1982 return ret; 1983 } 1984 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire 1985 #endif 1986 1987 #ifndef arch_atomic64_dec_return_release 1988 static __always_inline s64 1989 arch_atomic64_dec_return_release(atomic64_t *v) 1990 { 1991 __atomic_release_fence(); 1992 return arch_atomic64_dec_return_relaxed(v); 1993 } 1994 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release 1995 #endif 1996 1997 #ifndef arch_atomic64_dec_return 1998 static __always_inline s64 1999 arch_atomic64_dec_return(atomic64_t *v) 2000 { 2001 s64 ret; 2002 __atomic_pre_full_fence(); 2003 ret = arch_atomic64_dec_return_relaxed(v); 2004 __atomic_post_full_fence(); 2005 return ret; 2006 } 2007 #define arch_atomic64_dec_return arch_atomic64_dec_return 2008 #endif 2009 2010 #endif /* arch_atomic64_dec_return_relaxed */ 2011 2012 #ifndef arch_atomic64_fetch_dec_relaxed 2013 #ifdef arch_atomic64_fetch_dec 2014 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec 2015 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec 2016 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec 2017 #endif /* arch_atomic64_fetch_dec */ 2018 2019 #ifndef arch_atomic64_fetch_dec 2020 static __always_inline s64 2021 arch_atomic64_fetch_dec(atomic64_t *v) 2022 { 2023 return arch_atomic64_fetch_sub(1, v); 2024 } 2025 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 2026 #endif 2027 2028 #ifndef arch_atomic64_fetch_dec_acquire 2029 static __always_inline s64 2030 arch_atomic64_fetch_dec_acquire(atomic64_t *v) 2031 { 2032 return arch_atomic64_fetch_sub_acquire(1, v); 2033 } 2034 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 2035 #endif 2036 2037 #ifndef arch_atomic64_fetch_dec_release 2038 static __always_inline s64 2039 arch_atomic64_fetch_dec_release(atomic64_t *v) 2040 { 2041 return arch_atomic64_fetch_sub_release(1, v); 2042 } 2043 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 2044 #endif 2045 2046 #ifndef arch_atomic64_fetch_dec_relaxed 2047 static __always_inline s64 2048 arch_atomic64_fetch_dec_relaxed(atomic64_t *v) 2049 { 2050 return arch_atomic64_fetch_sub_relaxed(1, v); 2051 } 2052 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed 2053 #endif 2054 2055 #else /* arch_atomic64_fetch_dec_relaxed */ 2056 2057 #ifndef arch_atomic64_fetch_dec_acquire 2058 static __always_inline s64 2059 arch_atomic64_fetch_dec_acquire(atomic64_t *v) 2060 { 2061 s64 ret = arch_atomic64_fetch_dec_relaxed(v); 2062 __atomic_acquire_fence(); 2063 return ret; 2064 } 2065 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire 2066 #endif 2067 2068 #ifndef arch_atomic64_fetch_dec_release 2069 static __always_inline s64 2070 arch_atomic64_fetch_dec_release(atomic64_t *v) 2071 { 2072 __atomic_release_fence(); 2073 return arch_atomic64_fetch_dec_relaxed(v); 2074 } 2075 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release 2076 #endif 2077 2078 #ifndef arch_atomic64_fetch_dec 2079 static __always_inline s64 2080 arch_atomic64_fetch_dec(atomic64_t *v) 2081 { 2082 s64 ret; 2083 __atomic_pre_full_fence(); 2084 ret = arch_atomic64_fetch_dec_relaxed(v); 2085 __atomic_post_full_fence(); 2086 return ret; 2087 } 2088 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec 2089 #endif 2090 2091 #endif /* arch_atomic64_fetch_dec_relaxed */ 2092 2093 #ifndef arch_atomic64_fetch_and_relaxed 2094 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and 2095 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and 2096 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and 2097 #else /* arch_atomic64_fetch_and_relaxed */ 2098 2099 #ifndef arch_atomic64_fetch_and_acquire 2100 static __always_inline s64 2101 arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 2102 { 2103 s64 ret = arch_atomic64_fetch_and_relaxed(i, v); 2104 __atomic_acquire_fence(); 2105 return ret; 2106 } 2107 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire 2108 #endif 2109 2110 #ifndef arch_atomic64_fetch_and_release 2111 static __always_inline s64 2112 arch_atomic64_fetch_and_release(s64 i, atomic64_t *v) 2113 { 2114 __atomic_release_fence(); 2115 return arch_atomic64_fetch_and_relaxed(i, v); 2116 } 2117 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release 2118 #endif 2119 2120 #ifndef arch_atomic64_fetch_and 2121 static __always_inline s64 2122 arch_atomic64_fetch_and(s64 i, atomic64_t *v) 2123 { 2124 s64 ret; 2125 __atomic_pre_full_fence(); 2126 ret = arch_atomic64_fetch_and_relaxed(i, v); 2127 __atomic_post_full_fence(); 2128 return ret; 2129 } 2130 #define arch_atomic64_fetch_and arch_atomic64_fetch_and 2131 #endif 2132 2133 #endif /* arch_atomic64_fetch_and_relaxed */ 2134 2135 #ifndef arch_atomic64_andnot 2136 static __always_inline void 2137 arch_atomic64_andnot(s64 i, atomic64_t *v) 2138 { 2139 arch_atomic64_and(~i, v); 2140 } 2141 #define arch_atomic64_andnot arch_atomic64_andnot 2142 #endif 2143 2144 #ifndef arch_atomic64_fetch_andnot_relaxed 2145 #ifdef arch_atomic64_fetch_andnot 2146 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot 2147 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot 2148 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot 2149 #endif /* arch_atomic64_fetch_andnot */ 2150 2151 #ifndef arch_atomic64_fetch_andnot 2152 static __always_inline s64 2153 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 2154 { 2155 return arch_atomic64_fetch_and(~i, v); 2156 } 2157 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 2158 #endif 2159 2160 #ifndef arch_atomic64_fetch_andnot_acquire 2161 static __always_inline s64 2162 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 2163 { 2164 return arch_atomic64_fetch_and_acquire(~i, v); 2165 } 2166 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 2167 #endif 2168 2169 #ifndef arch_atomic64_fetch_andnot_release 2170 static __always_inline s64 2171 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 2172 { 2173 return arch_atomic64_fetch_and_release(~i, v); 2174 } 2175 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 2176 #endif 2177 2178 #ifndef arch_atomic64_fetch_andnot_relaxed 2179 static __always_inline s64 2180 arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 2181 { 2182 return arch_atomic64_fetch_and_relaxed(~i, v); 2183 } 2184 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 2185 #endif 2186 2187 #else /* arch_atomic64_fetch_andnot_relaxed */ 2188 2189 #ifndef arch_atomic64_fetch_andnot_acquire 2190 static __always_inline s64 2191 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 2192 { 2193 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 2194 __atomic_acquire_fence(); 2195 return ret; 2196 } 2197 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire 2198 #endif 2199 2200 #ifndef arch_atomic64_fetch_andnot_release 2201 static __always_inline s64 2202 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 2203 { 2204 __atomic_release_fence(); 2205 return arch_atomic64_fetch_andnot_relaxed(i, v); 2206 } 2207 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release 2208 #endif 2209 2210 #ifndef arch_atomic64_fetch_andnot 2211 static __always_inline s64 2212 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) 2213 { 2214 s64 ret; 2215 __atomic_pre_full_fence(); 2216 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 2217 __atomic_post_full_fence(); 2218 return ret; 2219 } 2220 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot 2221 #endif 2222 2223 #endif /* arch_atomic64_fetch_andnot_relaxed */ 2224 2225 #ifndef arch_atomic64_fetch_or_relaxed 2226 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or 2227 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or 2228 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or 2229 #else /* arch_atomic64_fetch_or_relaxed */ 2230 2231 #ifndef arch_atomic64_fetch_or_acquire 2232 static __always_inline s64 2233 arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 2234 { 2235 s64 ret = arch_atomic64_fetch_or_relaxed(i, v); 2236 __atomic_acquire_fence(); 2237 return ret; 2238 } 2239 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire 2240 #endif 2241 2242 #ifndef arch_atomic64_fetch_or_release 2243 static __always_inline s64 2244 arch_atomic64_fetch_or_release(s64 i, atomic64_t *v) 2245 { 2246 __atomic_release_fence(); 2247 return arch_atomic64_fetch_or_relaxed(i, v); 2248 } 2249 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release 2250 #endif 2251 2252 #ifndef arch_atomic64_fetch_or 2253 static __always_inline s64 2254 arch_atomic64_fetch_or(s64 i, atomic64_t *v) 2255 { 2256 s64 ret; 2257 __atomic_pre_full_fence(); 2258 ret = arch_atomic64_fetch_or_relaxed(i, v); 2259 __atomic_post_full_fence(); 2260 return ret; 2261 } 2262 #define arch_atomic64_fetch_or arch_atomic64_fetch_or 2263 #endif 2264 2265 #endif /* arch_atomic64_fetch_or_relaxed */ 2266 2267 #ifndef arch_atomic64_fetch_xor_relaxed 2268 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor 2269 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor 2270 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor 2271 #else /* arch_atomic64_fetch_xor_relaxed */ 2272 2273 #ifndef arch_atomic64_fetch_xor_acquire 2274 static __always_inline s64 2275 arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 2276 { 2277 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); 2278 __atomic_acquire_fence(); 2279 return ret; 2280 } 2281 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire 2282 #endif 2283 2284 #ifndef arch_atomic64_fetch_xor_release 2285 static __always_inline s64 2286 arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 2287 { 2288 __atomic_release_fence(); 2289 return arch_atomic64_fetch_xor_relaxed(i, v); 2290 } 2291 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release 2292 #endif 2293 2294 #ifndef arch_atomic64_fetch_xor 2295 static __always_inline s64 2296 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) 2297 { 2298 s64 ret; 2299 __atomic_pre_full_fence(); 2300 ret = arch_atomic64_fetch_xor_relaxed(i, v); 2301 __atomic_post_full_fence(); 2302 return ret; 2303 } 2304 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor 2305 #endif 2306 2307 #endif /* arch_atomic64_fetch_xor_relaxed */ 2308 2309 #ifndef arch_atomic64_xchg_relaxed 2310 #define arch_atomic64_xchg_acquire arch_atomic64_xchg 2311 #define arch_atomic64_xchg_release arch_atomic64_xchg 2312 #define arch_atomic64_xchg_relaxed arch_atomic64_xchg 2313 #else /* arch_atomic64_xchg_relaxed */ 2314 2315 #ifndef arch_atomic64_xchg_acquire 2316 static __always_inline s64 2317 arch_atomic64_xchg_acquire(atomic64_t *v, s64 i) 2318 { 2319 s64 ret = arch_atomic64_xchg_relaxed(v, i); 2320 __atomic_acquire_fence(); 2321 return ret; 2322 } 2323 #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire 2324 #endif 2325 2326 #ifndef arch_atomic64_xchg_release 2327 static __always_inline s64 2328 arch_atomic64_xchg_release(atomic64_t *v, s64 i) 2329 { 2330 __atomic_release_fence(); 2331 return arch_atomic64_xchg_relaxed(v, i); 2332 } 2333 #define arch_atomic64_xchg_release arch_atomic64_xchg_release 2334 #endif 2335 2336 #ifndef arch_atomic64_xchg 2337 static __always_inline s64 2338 arch_atomic64_xchg(atomic64_t *v, s64 i) 2339 { 2340 s64 ret; 2341 __atomic_pre_full_fence(); 2342 ret = arch_atomic64_xchg_relaxed(v, i); 2343 __atomic_post_full_fence(); 2344 return ret; 2345 } 2346 #define arch_atomic64_xchg arch_atomic64_xchg 2347 #endif 2348 2349 #endif /* arch_atomic64_xchg_relaxed */ 2350 2351 #ifndef arch_atomic64_cmpxchg_relaxed 2352 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg 2353 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg 2354 #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg 2355 #else /* arch_atomic64_cmpxchg_relaxed */ 2356 2357 #ifndef arch_atomic64_cmpxchg_acquire 2358 static __always_inline s64 2359 arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 2360 { 2361 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2362 __atomic_acquire_fence(); 2363 return ret; 2364 } 2365 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire 2366 #endif 2367 2368 #ifndef arch_atomic64_cmpxchg_release 2369 static __always_inline s64 2370 arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 2371 { 2372 __atomic_release_fence(); 2373 return arch_atomic64_cmpxchg_relaxed(v, old, new); 2374 } 2375 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release 2376 #endif 2377 2378 #ifndef arch_atomic64_cmpxchg 2379 static __always_inline s64 2380 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 2381 { 2382 s64 ret; 2383 __atomic_pre_full_fence(); 2384 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 2385 __atomic_post_full_fence(); 2386 return ret; 2387 } 2388 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg 2389 #endif 2390 2391 #endif /* arch_atomic64_cmpxchg_relaxed */ 2392 2393 #ifndef arch_atomic64_try_cmpxchg_relaxed 2394 #ifdef arch_atomic64_try_cmpxchg 2395 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg 2396 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg 2397 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg 2398 #endif /* arch_atomic64_try_cmpxchg */ 2399 2400 #ifndef arch_atomic64_try_cmpxchg 2401 static __always_inline bool 2402 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2403 { 2404 s64 r, o = *old; 2405 r = arch_atomic64_cmpxchg(v, o, new); 2406 if (unlikely(r != o)) 2407 *old = r; 2408 return likely(r == o); 2409 } 2410 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2411 #endif 2412 2413 #ifndef arch_atomic64_try_cmpxchg_acquire 2414 static __always_inline bool 2415 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2416 { 2417 s64 r, o = *old; 2418 r = arch_atomic64_cmpxchg_acquire(v, o, new); 2419 if (unlikely(r != o)) 2420 *old = r; 2421 return likely(r == o); 2422 } 2423 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2424 #endif 2425 2426 #ifndef arch_atomic64_try_cmpxchg_release 2427 static __always_inline bool 2428 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2429 { 2430 s64 r, o = *old; 2431 r = arch_atomic64_cmpxchg_release(v, o, new); 2432 if (unlikely(r != o)) 2433 *old = r; 2434 return likely(r == o); 2435 } 2436 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2437 #endif 2438 2439 #ifndef arch_atomic64_try_cmpxchg_relaxed 2440 static __always_inline bool 2441 arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 2442 { 2443 s64 r, o = *old; 2444 r = arch_atomic64_cmpxchg_relaxed(v, o, new); 2445 if (unlikely(r != o)) 2446 *old = r; 2447 return likely(r == o); 2448 } 2449 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed 2450 #endif 2451 2452 #else /* arch_atomic64_try_cmpxchg_relaxed */ 2453 2454 #ifndef arch_atomic64_try_cmpxchg_acquire 2455 static __always_inline bool 2456 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 2457 { 2458 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2459 __atomic_acquire_fence(); 2460 return ret; 2461 } 2462 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire 2463 #endif 2464 2465 #ifndef arch_atomic64_try_cmpxchg_release 2466 static __always_inline bool 2467 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 2468 { 2469 __atomic_release_fence(); 2470 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2471 } 2472 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release 2473 #endif 2474 2475 #ifndef arch_atomic64_try_cmpxchg 2476 static __always_inline bool 2477 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 2478 { 2479 bool ret; 2480 __atomic_pre_full_fence(); 2481 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 2482 __atomic_post_full_fence(); 2483 return ret; 2484 } 2485 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg 2486 #endif 2487 2488 #endif /* arch_atomic64_try_cmpxchg_relaxed */ 2489 2490 #ifndef arch_atomic64_sub_and_test 2491 /** 2492 * arch_atomic64_sub_and_test - subtract value from variable and test result 2493 * @i: integer value to subtract 2494 * @v: pointer of type atomic64_t 2495 * 2496 * Atomically subtracts @i from @v and returns 2497 * true if the result is zero, or false for all 2498 * other cases. 2499 */ 2500 static __always_inline bool 2501 arch_atomic64_sub_and_test(s64 i, atomic64_t *v) 2502 { 2503 return arch_atomic64_sub_return(i, v) == 0; 2504 } 2505 #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test 2506 #endif 2507 2508 #ifndef arch_atomic64_dec_and_test 2509 /** 2510 * arch_atomic64_dec_and_test - decrement and test 2511 * @v: pointer of type atomic64_t 2512 * 2513 * Atomically decrements @v by 1 and 2514 * returns true if the result is 0, or false for all other 2515 * cases. 2516 */ 2517 static __always_inline bool 2518 arch_atomic64_dec_and_test(atomic64_t *v) 2519 { 2520 return arch_atomic64_dec_return(v) == 0; 2521 } 2522 #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test 2523 #endif 2524 2525 #ifndef arch_atomic64_inc_and_test 2526 /** 2527 * arch_atomic64_inc_and_test - increment and test 2528 * @v: pointer of type atomic64_t 2529 * 2530 * Atomically increments @v by 1 2531 * and returns true if the result is zero, or false for all 2532 * other cases. 2533 */ 2534 static __always_inline bool 2535 arch_atomic64_inc_and_test(atomic64_t *v) 2536 { 2537 return arch_atomic64_inc_return(v) == 0; 2538 } 2539 #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test 2540 #endif 2541 2542 #ifndef arch_atomic64_add_negative_relaxed 2543 #ifdef arch_atomic64_add_negative 2544 #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative 2545 #define arch_atomic64_add_negative_release arch_atomic64_add_negative 2546 #define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative 2547 #endif /* arch_atomic64_add_negative */ 2548 2549 #ifndef arch_atomic64_add_negative 2550 /** 2551 * arch_atomic64_add_negative - Add and test if negative 2552 * @i: integer value to add 2553 * @v: pointer of type atomic64_t 2554 * 2555 * Atomically adds @i to @v and returns true if the result is negative, 2556 * or false when the result is greater than or equal to zero. 2557 */ 2558 static __always_inline bool 2559 arch_atomic64_add_negative(s64 i, atomic64_t *v) 2560 { 2561 return arch_atomic64_add_return(i, v) < 0; 2562 } 2563 #define arch_atomic64_add_negative arch_atomic64_add_negative 2564 #endif 2565 2566 #ifndef arch_atomic64_add_negative_acquire 2567 /** 2568 * arch_atomic64_add_negative_acquire - Add and test if negative 2569 * @i: integer value to add 2570 * @v: pointer of type atomic64_t 2571 * 2572 * Atomically adds @i to @v and returns true if the result is negative, 2573 * or false when the result is greater than or equal to zero. 2574 */ 2575 static __always_inline bool 2576 arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 2577 { 2578 return arch_atomic64_add_return_acquire(i, v) < 0; 2579 } 2580 #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire 2581 #endif 2582 2583 #ifndef arch_atomic64_add_negative_release 2584 /** 2585 * arch_atomic64_add_negative_release - Add and test if negative 2586 * @i: integer value to add 2587 * @v: pointer of type atomic64_t 2588 * 2589 * Atomically adds @i to @v and returns true if the result is negative, 2590 * or false when the result is greater than or equal to zero. 2591 */ 2592 static __always_inline bool 2593 arch_atomic64_add_negative_release(s64 i, atomic64_t *v) 2594 { 2595 return arch_atomic64_add_return_release(i, v) < 0; 2596 } 2597 #define arch_atomic64_add_negative_release arch_atomic64_add_negative_release 2598 #endif 2599 2600 #ifndef arch_atomic64_add_negative_relaxed 2601 /** 2602 * arch_atomic64_add_negative_relaxed - Add and test if negative 2603 * @i: integer value to add 2604 * @v: pointer of type atomic64_t 2605 * 2606 * Atomically adds @i to @v and returns true if the result is negative, 2607 * or false when the result is greater than or equal to zero. 2608 */ 2609 static __always_inline bool 2610 arch_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 2611 { 2612 return arch_atomic64_add_return_relaxed(i, v) < 0; 2613 } 2614 #define arch_atomic64_add_negative_relaxed arch_atomic64_add_negative_relaxed 2615 #endif 2616 2617 #else /* arch_atomic64_add_negative_relaxed */ 2618 2619 #ifndef arch_atomic64_add_negative_acquire 2620 static __always_inline bool 2621 arch_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 2622 { 2623 bool ret = arch_atomic64_add_negative_relaxed(i, v); 2624 __atomic_acquire_fence(); 2625 return ret; 2626 } 2627 #define arch_atomic64_add_negative_acquire arch_atomic64_add_negative_acquire 2628 #endif 2629 2630 #ifndef arch_atomic64_add_negative_release 2631 static __always_inline bool 2632 arch_atomic64_add_negative_release(s64 i, atomic64_t *v) 2633 { 2634 __atomic_release_fence(); 2635 return arch_atomic64_add_negative_relaxed(i, v); 2636 } 2637 #define arch_atomic64_add_negative_release arch_atomic64_add_negative_release 2638 #endif 2639 2640 #ifndef arch_atomic64_add_negative 2641 static __always_inline bool 2642 arch_atomic64_add_negative(s64 i, atomic64_t *v) 2643 { 2644 bool ret; 2645 __atomic_pre_full_fence(); 2646 ret = arch_atomic64_add_negative_relaxed(i, v); 2647 __atomic_post_full_fence(); 2648 return ret; 2649 } 2650 #define arch_atomic64_add_negative arch_atomic64_add_negative 2651 #endif 2652 2653 #endif /* arch_atomic64_add_negative_relaxed */ 2654 2655 #ifndef arch_atomic64_fetch_add_unless 2656 /** 2657 * arch_atomic64_fetch_add_unless - add unless the number is already a given value 2658 * @v: pointer of type atomic64_t 2659 * @a: the amount to add to v... 2660 * @u: ...unless v is equal to u. 2661 * 2662 * Atomically adds @a to @v, so long as @v was not already @u. 2663 * Returns original value of @v 2664 */ 2665 static __always_inline s64 2666 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 2667 { 2668 s64 c = arch_atomic64_read(v); 2669 2670 do { 2671 if (unlikely(c == u)) 2672 break; 2673 } while (!arch_atomic64_try_cmpxchg(v, &c, c + a)); 2674 2675 return c; 2676 } 2677 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 2678 #endif 2679 2680 #ifndef arch_atomic64_add_unless 2681 /** 2682 * arch_atomic64_add_unless - add unless the number is already a given value 2683 * @v: pointer of type atomic64_t 2684 * @a: the amount to add to v... 2685 * @u: ...unless v is equal to u. 2686 * 2687 * Atomically adds @a to @v, if @v was not already @u. 2688 * Returns true if the addition was done. 2689 */ 2690 static __always_inline bool 2691 arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 2692 { 2693 return arch_atomic64_fetch_add_unless(v, a, u) != u; 2694 } 2695 #define arch_atomic64_add_unless arch_atomic64_add_unless 2696 #endif 2697 2698 #ifndef arch_atomic64_inc_not_zero 2699 /** 2700 * arch_atomic64_inc_not_zero - increment unless the number is zero 2701 * @v: pointer of type atomic64_t 2702 * 2703 * Atomically increments @v by 1, if @v is non-zero. 2704 * Returns true if the increment was done. 2705 */ 2706 static __always_inline bool 2707 arch_atomic64_inc_not_zero(atomic64_t *v) 2708 { 2709 return arch_atomic64_add_unless(v, 1, 0); 2710 } 2711 #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero 2712 #endif 2713 2714 #ifndef arch_atomic64_inc_unless_negative 2715 static __always_inline bool 2716 arch_atomic64_inc_unless_negative(atomic64_t *v) 2717 { 2718 s64 c = arch_atomic64_read(v); 2719 2720 do { 2721 if (unlikely(c < 0)) 2722 return false; 2723 } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1)); 2724 2725 return true; 2726 } 2727 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative 2728 #endif 2729 2730 #ifndef arch_atomic64_dec_unless_positive 2731 static __always_inline bool 2732 arch_atomic64_dec_unless_positive(atomic64_t *v) 2733 { 2734 s64 c = arch_atomic64_read(v); 2735 2736 do { 2737 if (unlikely(c > 0)) 2738 return false; 2739 } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1)); 2740 2741 return true; 2742 } 2743 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive 2744 #endif 2745 2746 #ifndef arch_atomic64_dec_if_positive 2747 static __always_inline s64 2748 arch_atomic64_dec_if_positive(atomic64_t *v) 2749 { 2750 s64 dec, c = arch_atomic64_read(v); 2751 2752 do { 2753 dec = c - 1; 2754 if (unlikely(dec < 0)) 2755 break; 2756 } while (!arch_atomic64_try_cmpxchg(v, &c, dec)); 2757 2758 return dec; 2759 } 2760 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 2761 #endif 2762 2763 #endif /* _LINUX_ATOMIC_FALLBACK_H */ 2764 // 52dfc6fe4a2e7234bbd2aa3e16a377c1db793a53 2765