1 // SPDX-License-Identifier: GPL-2.0 2 3 // Generated by scripts/atomic/gen-atomic-fallback.sh 4 // DO NOT MODIFY THIS FILE DIRECTLY 5 6 #ifndef _LINUX_ATOMIC_FALLBACK_H 7 #define _LINUX_ATOMIC_FALLBACK_H 8 9 #include <linux/compiler.h> 10 11 #if defined(arch_xchg) 12 #define raw_xchg arch_xchg 13 #elif defined(arch_xchg_relaxed) 14 #define raw_xchg(...) \ 15 __atomic_op_fence(arch_xchg, __VA_ARGS__) 16 #else 17 extern void raw_xchg_not_implemented(void); 18 #define raw_xchg(...) raw_xchg_not_implemented() 19 #endif 20 21 #if defined(arch_xchg_acquire) 22 #define raw_xchg_acquire arch_xchg_acquire 23 #elif defined(arch_xchg_relaxed) 24 #define raw_xchg_acquire(...) \ 25 __atomic_op_acquire(arch_xchg, __VA_ARGS__) 26 #elif defined(arch_xchg) 27 #define raw_xchg_acquire arch_xchg 28 #else 29 extern void raw_xchg_acquire_not_implemented(void); 30 #define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented() 31 #endif 32 33 #if defined(arch_xchg_release) 34 #define raw_xchg_release arch_xchg_release 35 #elif defined(arch_xchg_relaxed) 36 #define raw_xchg_release(...) \ 37 __atomic_op_release(arch_xchg, __VA_ARGS__) 38 #elif defined(arch_xchg) 39 #define raw_xchg_release arch_xchg 40 #else 41 extern void raw_xchg_release_not_implemented(void); 42 #define raw_xchg_release(...) raw_xchg_release_not_implemented() 43 #endif 44 45 #if defined(arch_xchg_relaxed) 46 #define raw_xchg_relaxed arch_xchg_relaxed 47 #elif defined(arch_xchg) 48 #define raw_xchg_relaxed arch_xchg 49 #else 50 extern void raw_xchg_relaxed_not_implemented(void); 51 #define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented() 52 #endif 53 54 #if defined(arch_cmpxchg) 55 #define raw_cmpxchg arch_cmpxchg 56 #elif defined(arch_cmpxchg_relaxed) 57 #define raw_cmpxchg(...) \ 58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) 59 #else 60 extern void raw_cmpxchg_not_implemented(void); 61 #define raw_cmpxchg(...) raw_cmpxchg_not_implemented() 62 #endif 63 64 #if defined(arch_cmpxchg_acquire) 65 #define raw_cmpxchg_acquire arch_cmpxchg_acquire 66 #elif defined(arch_cmpxchg_relaxed) 67 #define raw_cmpxchg_acquire(...) \ 68 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) 69 #elif defined(arch_cmpxchg) 70 #define raw_cmpxchg_acquire arch_cmpxchg 71 #else 72 extern void raw_cmpxchg_acquire_not_implemented(void); 73 #define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented() 74 #endif 75 76 #if defined(arch_cmpxchg_release) 77 #define raw_cmpxchg_release arch_cmpxchg_release 78 #elif defined(arch_cmpxchg_relaxed) 79 #define raw_cmpxchg_release(...) \ 80 __atomic_op_release(arch_cmpxchg, __VA_ARGS__) 81 #elif defined(arch_cmpxchg) 82 #define raw_cmpxchg_release arch_cmpxchg 83 #else 84 extern void raw_cmpxchg_release_not_implemented(void); 85 #define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented() 86 #endif 87 88 #if defined(arch_cmpxchg_relaxed) 89 #define raw_cmpxchg_relaxed arch_cmpxchg_relaxed 90 #elif defined(arch_cmpxchg) 91 #define raw_cmpxchg_relaxed arch_cmpxchg 92 #else 93 extern void raw_cmpxchg_relaxed_not_implemented(void); 94 #define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented() 95 #endif 96 97 #if defined(arch_cmpxchg64) 98 #define raw_cmpxchg64 arch_cmpxchg64 99 #elif defined(arch_cmpxchg64_relaxed) 100 #define raw_cmpxchg64(...) \ 101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) 102 #else 103 extern void raw_cmpxchg64_not_implemented(void); 104 #define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented() 105 #endif 106 107 #if defined(arch_cmpxchg64_acquire) 108 #define raw_cmpxchg64_acquire arch_cmpxchg64_acquire 109 #elif defined(arch_cmpxchg64_relaxed) 110 #define raw_cmpxchg64_acquire(...) \ 111 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) 112 #elif defined(arch_cmpxchg64) 113 #define raw_cmpxchg64_acquire arch_cmpxchg64 114 #else 115 extern void raw_cmpxchg64_acquire_not_implemented(void); 116 #define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented() 117 #endif 118 119 #if defined(arch_cmpxchg64_release) 120 #define raw_cmpxchg64_release arch_cmpxchg64_release 121 #elif defined(arch_cmpxchg64_relaxed) 122 #define raw_cmpxchg64_release(...) \ 123 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) 124 #elif defined(arch_cmpxchg64) 125 #define raw_cmpxchg64_release arch_cmpxchg64 126 #else 127 extern void raw_cmpxchg64_release_not_implemented(void); 128 #define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented() 129 #endif 130 131 #if defined(arch_cmpxchg64_relaxed) 132 #define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed 133 #elif defined(arch_cmpxchg64) 134 #define raw_cmpxchg64_relaxed arch_cmpxchg64 135 #else 136 extern void raw_cmpxchg64_relaxed_not_implemented(void); 137 #define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented() 138 #endif 139 140 #if defined(arch_cmpxchg128) 141 #define raw_cmpxchg128 arch_cmpxchg128 142 #elif defined(arch_cmpxchg128_relaxed) 143 #define raw_cmpxchg128(...) \ 144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) 145 #else 146 extern void raw_cmpxchg128_not_implemented(void); 147 #define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented() 148 #endif 149 150 #if defined(arch_cmpxchg128_acquire) 151 #define raw_cmpxchg128_acquire arch_cmpxchg128_acquire 152 #elif defined(arch_cmpxchg128_relaxed) 153 #define raw_cmpxchg128_acquire(...) \ 154 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) 155 #elif defined(arch_cmpxchg128) 156 #define raw_cmpxchg128_acquire arch_cmpxchg128 157 #else 158 extern void raw_cmpxchg128_acquire_not_implemented(void); 159 #define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented() 160 #endif 161 162 #if defined(arch_cmpxchg128_release) 163 #define raw_cmpxchg128_release arch_cmpxchg128_release 164 #elif defined(arch_cmpxchg128_relaxed) 165 #define raw_cmpxchg128_release(...) \ 166 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) 167 #elif defined(arch_cmpxchg128) 168 #define raw_cmpxchg128_release arch_cmpxchg128 169 #else 170 extern void raw_cmpxchg128_release_not_implemented(void); 171 #define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented() 172 #endif 173 174 #if defined(arch_cmpxchg128_relaxed) 175 #define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed 176 #elif defined(arch_cmpxchg128) 177 #define raw_cmpxchg128_relaxed arch_cmpxchg128 178 #else 179 extern void raw_cmpxchg128_relaxed_not_implemented(void); 180 #define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented() 181 #endif 182 183 #if defined(arch_try_cmpxchg) 184 #define raw_try_cmpxchg arch_try_cmpxchg 185 #elif defined(arch_try_cmpxchg_relaxed) 186 #define raw_try_cmpxchg(...) \ 187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) 188 #else 189 #define raw_try_cmpxchg(_ptr, _oldp, _new) \ 190 ({ \ 191 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 192 ___r = raw_cmpxchg((_ptr), ___o, (_new)); \ 193 if (unlikely(___r != ___o)) \ 194 *___op = ___r; \ 195 likely(___r == ___o); \ 196 }) 197 #endif 198 199 #if defined(arch_try_cmpxchg_acquire) 200 #define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire 201 #elif defined(arch_try_cmpxchg_relaxed) 202 #define raw_try_cmpxchg_acquire(...) \ 203 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) 204 #elif defined(arch_try_cmpxchg) 205 #define raw_try_cmpxchg_acquire arch_try_cmpxchg 206 #else 207 #define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \ 208 ({ \ 209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 210 ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \ 211 if (unlikely(___r != ___o)) \ 212 *___op = ___r; \ 213 likely(___r == ___o); \ 214 }) 215 #endif 216 217 #if defined(arch_try_cmpxchg_release) 218 #define raw_try_cmpxchg_release arch_try_cmpxchg_release 219 #elif defined(arch_try_cmpxchg_relaxed) 220 #define raw_try_cmpxchg_release(...) \ 221 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) 222 #elif defined(arch_try_cmpxchg) 223 #define raw_try_cmpxchg_release arch_try_cmpxchg 224 #else 225 #define raw_try_cmpxchg_release(_ptr, _oldp, _new) \ 226 ({ \ 227 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 228 ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \ 229 if (unlikely(___r != ___o)) \ 230 *___op = ___r; \ 231 likely(___r == ___o); \ 232 }) 233 #endif 234 235 #if defined(arch_try_cmpxchg_relaxed) 236 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed 237 #elif defined(arch_try_cmpxchg) 238 #define raw_try_cmpxchg_relaxed arch_try_cmpxchg 239 #else 240 #define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ 241 ({ \ 242 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 243 ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \ 244 if (unlikely(___r != ___o)) \ 245 *___op = ___r; \ 246 likely(___r == ___o); \ 247 }) 248 #endif 249 250 #if defined(arch_try_cmpxchg64) 251 #define raw_try_cmpxchg64 arch_try_cmpxchg64 252 #elif defined(arch_try_cmpxchg64_relaxed) 253 #define raw_try_cmpxchg64(...) \ 254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) 255 #else 256 #define raw_try_cmpxchg64(_ptr, _oldp, _new) \ 257 ({ \ 258 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 259 ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \ 260 if (unlikely(___r != ___o)) \ 261 *___op = ___r; \ 262 likely(___r == ___o); \ 263 }) 264 #endif 265 266 #if defined(arch_try_cmpxchg64_acquire) 267 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire 268 #elif defined(arch_try_cmpxchg64_relaxed) 269 #define raw_try_cmpxchg64_acquire(...) \ 270 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) 271 #elif defined(arch_try_cmpxchg64) 272 #define raw_try_cmpxchg64_acquire arch_try_cmpxchg64 273 #else 274 #define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ 275 ({ \ 276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 277 ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \ 278 if (unlikely(___r != ___o)) \ 279 *___op = ___r; \ 280 likely(___r == ___o); \ 281 }) 282 #endif 283 284 #if defined(arch_try_cmpxchg64_release) 285 #define raw_try_cmpxchg64_release arch_try_cmpxchg64_release 286 #elif defined(arch_try_cmpxchg64_relaxed) 287 #define raw_try_cmpxchg64_release(...) \ 288 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) 289 #elif defined(arch_try_cmpxchg64) 290 #define raw_try_cmpxchg64_release arch_try_cmpxchg64 291 #else 292 #define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \ 293 ({ \ 294 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 295 ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \ 296 if (unlikely(___r != ___o)) \ 297 *___op = ___r; \ 298 likely(___r == ___o); \ 299 }) 300 #endif 301 302 #if defined(arch_try_cmpxchg64_relaxed) 303 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed 304 #elif defined(arch_try_cmpxchg64) 305 #define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64 306 #else 307 #define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ 308 ({ \ 309 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 310 ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ 311 if (unlikely(___r != ___o)) \ 312 *___op = ___r; \ 313 likely(___r == ___o); \ 314 }) 315 #endif 316 317 #if defined(arch_try_cmpxchg128) 318 #define raw_try_cmpxchg128 arch_try_cmpxchg128 319 #elif defined(arch_try_cmpxchg128_relaxed) 320 #define raw_try_cmpxchg128(...) \ 321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) 322 #else 323 #define raw_try_cmpxchg128(_ptr, _oldp, _new) \ 324 ({ \ 325 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 326 ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \ 327 if (unlikely(___r != ___o)) \ 328 *___op = ___r; \ 329 likely(___r == ___o); \ 330 }) 331 #endif 332 333 #if defined(arch_try_cmpxchg128_acquire) 334 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire 335 #elif defined(arch_try_cmpxchg128_relaxed) 336 #define raw_try_cmpxchg128_acquire(...) \ 337 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) 338 #elif defined(arch_try_cmpxchg128) 339 #define raw_try_cmpxchg128_acquire arch_try_cmpxchg128 340 #else 341 #define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ 342 ({ \ 343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 344 ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \ 345 if (unlikely(___r != ___o)) \ 346 *___op = ___r; \ 347 likely(___r == ___o); \ 348 }) 349 #endif 350 351 #if defined(arch_try_cmpxchg128_release) 352 #define raw_try_cmpxchg128_release arch_try_cmpxchg128_release 353 #elif defined(arch_try_cmpxchg128_relaxed) 354 #define raw_try_cmpxchg128_release(...) \ 355 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) 356 #elif defined(arch_try_cmpxchg128) 357 #define raw_try_cmpxchg128_release arch_try_cmpxchg128 358 #else 359 #define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \ 360 ({ \ 361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 362 ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \ 363 if (unlikely(___r != ___o)) \ 364 *___op = ___r; \ 365 likely(___r == ___o); \ 366 }) 367 #endif 368 369 #if defined(arch_try_cmpxchg128_relaxed) 370 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed 371 #elif defined(arch_try_cmpxchg128) 372 #define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128 373 #else 374 #define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ 375 ({ \ 376 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 377 ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ 378 if (unlikely(___r != ___o)) \ 379 *___op = ___r; \ 380 likely(___r == ___o); \ 381 }) 382 #endif 383 384 #define raw_cmpxchg_local arch_cmpxchg_local 385 386 #ifdef arch_try_cmpxchg_local 387 #define raw_try_cmpxchg_local arch_try_cmpxchg_local 388 #else 389 #define raw_try_cmpxchg_local(_ptr, _oldp, _new) \ 390 ({ \ 391 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 392 ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \ 393 if (unlikely(___r != ___o)) \ 394 *___op = ___r; \ 395 likely(___r == ___o); \ 396 }) 397 #endif 398 399 #define raw_cmpxchg64_local arch_cmpxchg64_local 400 401 #ifdef arch_try_cmpxchg64_local 402 #define raw_try_cmpxchg64_local arch_try_cmpxchg64_local 403 #else 404 #define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \ 405 ({ \ 406 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 407 ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \ 408 if (unlikely(___r != ___o)) \ 409 *___op = ___r; \ 410 likely(___r == ___o); \ 411 }) 412 #endif 413 414 #define raw_cmpxchg128_local arch_cmpxchg128_local 415 416 #ifdef arch_try_cmpxchg128_local 417 #define raw_try_cmpxchg128_local arch_try_cmpxchg128_local 418 #else 419 #define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \ 420 ({ \ 421 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 422 ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \ 423 if (unlikely(___r != ___o)) \ 424 *___op = ___r; \ 425 likely(___r == ___o); \ 426 }) 427 #endif 428 429 #define raw_sync_cmpxchg arch_sync_cmpxchg 430 431 /** 432 * raw_atomic_read() - atomic load with relaxed ordering 433 * @v: pointer to atomic_t 434 * 435 * Atomically loads the value of @v with relaxed ordering. 436 * 437 * Safe to use in noinstr code; prefer atomic_read() elsewhere. 438 * 439 * Return: The value loaded from @v. 440 */ 441 static __always_inline int 442 raw_atomic_read(const atomic_t *v) 443 { 444 return arch_atomic_read(v); 445 } 446 447 /** 448 * raw_atomic_read_acquire() - atomic load with acquire ordering 449 * @v: pointer to atomic_t 450 * 451 * Atomically loads the value of @v with acquire ordering. 452 * 453 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere. 454 * 455 * Return: The value loaded from @v. 456 */ 457 static __always_inline int 458 raw_atomic_read_acquire(const atomic_t *v) 459 { 460 #if defined(arch_atomic_read_acquire) 461 return arch_atomic_read_acquire(v); 462 #else 463 int ret; 464 465 if (__native_word(atomic_t)) { 466 ret = smp_load_acquire(&(v)->counter); 467 } else { 468 ret = raw_atomic_read(v); 469 __atomic_acquire_fence(); 470 } 471 472 return ret; 473 #endif 474 } 475 476 /** 477 * raw_atomic_set() - atomic set with relaxed ordering 478 * @v: pointer to atomic_t 479 * @i: int value to assign 480 * 481 * Atomically sets @v to @i with relaxed ordering. 482 * 483 * Safe to use in noinstr code; prefer atomic_set() elsewhere. 484 * 485 * Return: Nothing. 486 */ 487 static __always_inline void 488 raw_atomic_set(atomic_t *v, int i) 489 { 490 arch_atomic_set(v, i); 491 } 492 493 /** 494 * raw_atomic_set_release() - atomic set with release ordering 495 * @v: pointer to atomic_t 496 * @i: int value to assign 497 * 498 * Atomically sets @v to @i with release ordering. 499 * 500 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere. 501 * 502 * Return: Nothing. 503 */ 504 static __always_inline void 505 raw_atomic_set_release(atomic_t *v, int i) 506 { 507 #if defined(arch_atomic_set_release) 508 arch_atomic_set_release(v, i); 509 #else 510 if (__native_word(atomic_t)) { 511 smp_store_release(&(v)->counter, i); 512 } else { 513 __atomic_release_fence(); 514 raw_atomic_set(v, i); 515 } 516 #endif 517 } 518 519 /** 520 * raw_atomic_add() - atomic add with relaxed ordering 521 * @i: int value to add 522 * @v: pointer to atomic_t 523 * 524 * Atomically updates @v to (@v + @i) with relaxed ordering. 525 * 526 * Safe to use in noinstr code; prefer atomic_add() elsewhere. 527 * 528 * Return: Nothing. 529 */ 530 static __always_inline void 531 raw_atomic_add(int i, atomic_t *v) 532 { 533 arch_atomic_add(i, v); 534 } 535 536 /** 537 * raw_atomic_add_return() - atomic add with full ordering 538 * @i: int value to add 539 * @v: pointer to atomic_t 540 * 541 * Atomically updates @v to (@v + @i) with full ordering. 542 * 543 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere. 544 * 545 * Return: The updated value of @v. 546 */ 547 static __always_inline int 548 raw_atomic_add_return(int i, atomic_t *v) 549 { 550 #if defined(arch_atomic_add_return) 551 return arch_atomic_add_return(i, v); 552 #elif defined(arch_atomic_add_return_relaxed) 553 int ret; 554 __atomic_pre_full_fence(); 555 ret = arch_atomic_add_return_relaxed(i, v); 556 __atomic_post_full_fence(); 557 return ret; 558 #else 559 #error "Unable to define raw_atomic_add_return" 560 #endif 561 } 562 563 /** 564 * raw_atomic_add_return_acquire() - atomic add with acquire ordering 565 * @i: int value to add 566 * @v: pointer to atomic_t 567 * 568 * Atomically updates @v to (@v + @i) with acquire ordering. 569 * 570 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere. 571 * 572 * Return: The updated value of @v. 573 */ 574 static __always_inline int 575 raw_atomic_add_return_acquire(int i, atomic_t *v) 576 { 577 #if defined(arch_atomic_add_return_acquire) 578 return arch_atomic_add_return_acquire(i, v); 579 #elif defined(arch_atomic_add_return_relaxed) 580 int ret = arch_atomic_add_return_relaxed(i, v); 581 __atomic_acquire_fence(); 582 return ret; 583 #elif defined(arch_atomic_add_return) 584 return arch_atomic_add_return(i, v); 585 #else 586 #error "Unable to define raw_atomic_add_return_acquire" 587 #endif 588 } 589 590 /** 591 * raw_atomic_add_return_release() - atomic add with release ordering 592 * @i: int value to add 593 * @v: pointer to atomic_t 594 * 595 * Atomically updates @v to (@v + @i) with release ordering. 596 * 597 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere. 598 * 599 * Return: The updated value of @v. 600 */ 601 static __always_inline int 602 raw_atomic_add_return_release(int i, atomic_t *v) 603 { 604 #if defined(arch_atomic_add_return_release) 605 return arch_atomic_add_return_release(i, v); 606 #elif defined(arch_atomic_add_return_relaxed) 607 __atomic_release_fence(); 608 return arch_atomic_add_return_relaxed(i, v); 609 #elif defined(arch_atomic_add_return) 610 return arch_atomic_add_return(i, v); 611 #else 612 #error "Unable to define raw_atomic_add_return_release" 613 #endif 614 } 615 616 /** 617 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering 618 * @i: int value to add 619 * @v: pointer to atomic_t 620 * 621 * Atomically updates @v to (@v + @i) with relaxed ordering. 622 * 623 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere. 624 * 625 * Return: The updated value of @v. 626 */ 627 static __always_inline int 628 raw_atomic_add_return_relaxed(int i, atomic_t *v) 629 { 630 #if defined(arch_atomic_add_return_relaxed) 631 return arch_atomic_add_return_relaxed(i, v); 632 #elif defined(arch_atomic_add_return) 633 return arch_atomic_add_return(i, v); 634 #else 635 #error "Unable to define raw_atomic_add_return_relaxed" 636 #endif 637 } 638 639 /** 640 * raw_atomic_fetch_add() - atomic add with full ordering 641 * @i: int value to add 642 * @v: pointer to atomic_t 643 * 644 * Atomically updates @v to (@v + @i) with full ordering. 645 * 646 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere. 647 * 648 * Return: The original value of @v. 649 */ 650 static __always_inline int 651 raw_atomic_fetch_add(int i, atomic_t *v) 652 { 653 #if defined(arch_atomic_fetch_add) 654 return arch_atomic_fetch_add(i, v); 655 #elif defined(arch_atomic_fetch_add_relaxed) 656 int ret; 657 __atomic_pre_full_fence(); 658 ret = arch_atomic_fetch_add_relaxed(i, v); 659 __atomic_post_full_fence(); 660 return ret; 661 #else 662 #error "Unable to define raw_atomic_fetch_add" 663 #endif 664 } 665 666 /** 667 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering 668 * @i: int value to add 669 * @v: pointer to atomic_t 670 * 671 * Atomically updates @v to (@v + @i) with acquire ordering. 672 * 673 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere. 674 * 675 * Return: The original value of @v. 676 */ 677 static __always_inline int 678 raw_atomic_fetch_add_acquire(int i, atomic_t *v) 679 { 680 #if defined(arch_atomic_fetch_add_acquire) 681 return arch_atomic_fetch_add_acquire(i, v); 682 #elif defined(arch_atomic_fetch_add_relaxed) 683 int ret = arch_atomic_fetch_add_relaxed(i, v); 684 __atomic_acquire_fence(); 685 return ret; 686 #elif defined(arch_atomic_fetch_add) 687 return arch_atomic_fetch_add(i, v); 688 #else 689 #error "Unable to define raw_atomic_fetch_add_acquire" 690 #endif 691 } 692 693 /** 694 * raw_atomic_fetch_add_release() - atomic add with release ordering 695 * @i: int value to add 696 * @v: pointer to atomic_t 697 * 698 * Atomically updates @v to (@v + @i) with release ordering. 699 * 700 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere. 701 * 702 * Return: The original value of @v. 703 */ 704 static __always_inline int 705 raw_atomic_fetch_add_release(int i, atomic_t *v) 706 { 707 #if defined(arch_atomic_fetch_add_release) 708 return arch_atomic_fetch_add_release(i, v); 709 #elif defined(arch_atomic_fetch_add_relaxed) 710 __atomic_release_fence(); 711 return arch_atomic_fetch_add_relaxed(i, v); 712 #elif defined(arch_atomic_fetch_add) 713 return arch_atomic_fetch_add(i, v); 714 #else 715 #error "Unable to define raw_atomic_fetch_add_release" 716 #endif 717 } 718 719 /** 720 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering 721 * @i: int value to add 722 * @v: pointer to atomic_t 723 * 724 * Atomically updates @v to (@v + @i) with relaxed ordering. 725 * 726 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere. 727 * 728 * Return: The original value of @v. 729 */ 730 static __always_inline int 731 raw_atomic_fetch_add_relaxed(int i, atomic_t *v) 732 { 733 #if defined(arch_atomic_fetch_add_relaxed) 734 return arch_atomic_fetch_add_relaxed(i, v); 735 #elif defined(arch_atomic_fetch_add) 736 return arch_atomic_fetch_add(i, v); 737 #else 738 #error "Unable to define raw_atomic_fetch_add_relaxed" 739 #endif 740 } 741 742 /** 743 * raw_atomic_sub() - atomic subtract with relaxed ordering 744 * @i: int value to subtract 745 * @v: pointer to atomic_t 746 * 747 * Atomically updates @v to (@v - @i) with relaxed ordering. 748 * 749 * Safe to use in noinstr code; prefer atomic_sub() elsewhere. 750 * 751 * Return: Nothing. 752 */ 753 static __always_inline void 754 raw_atomic_sub(int i, atomic_t *v) 755 { 756 arch_atomic_sub(i, v); 757 } 758 759 /** 760 * raw_atomic_sub_return() - atomic subtract with full ordering 761 * @i: int value to subtract 762 * @v: pointer to atomic_t 763 * 764 * Atomically updates @v to (@v - @i) with full ordering. 765 * 766 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere. 767 * 768 * Return: The updated value of @v. 769 */ 770 static __always_inline int 771 raw_atomic_sub_return(int i, atomic_t *v) 772 { 773 #if defined(arch_atomic_sub_return) 774 return arch_atomic_sub_return(i, v); 775 #elif defined(arch_atomic_sub_return_relaxed) 776 int ret; 777 __atomic_pre_full_fence(); 778 ret = arch_atomic_sub_return_relaxed(i, v); 779 __atomic_post_full_fence(); 780 return ret; 781 #else 782 #error "Unable to define raw_atomic_sub_return" 783 #endif 784 } 785 786 /** 787 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering 788 * @i: int value to subtract 789 * @v: pointer to atomic_t 790 * 791 * Atomically updates @v to (@v - @i) with acquire ordering. 792 * 793 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere. 794 * 795 * Return: The updated value of @v. 796 */ 797 static __always_inline int 798 raw_atomic_sub_return_acquire(int i, atomic_t *v) 799 { 800 #if defined(arch_atomic_sub_return_acquire) 801 return arch_atomic_sub_return_acquire(i, v); 802 #elif defined(arch_atomic_sub_return_relaxed) 803 int ret = arch_atomic_sub_return_relaxed(i, v); 804 __atomic_acquire_fence(); 805 return ret; 806 #elif defined(arch_atomic_sub_return) 807 return arch_atomic_sub_return(i, v); 808 #else 809 #error "Unable to define raw_atomic_sub_return_acquire" 810 #endif 811 } 812 813 /** 814 * raw_atomic_sub_return_release() - atomic subtract with release ordering 815 * @i: int value to subtract 816 * @v: pointer to atomic_t 817 * 818 * Atomically updates @v to (@v - @i) with release ordering. 819 * 820 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere. 821 * 822 * Return: The updated value of @v. 823 */ 824 static __always_inline int 825 raw_atomic_sub_return_release(int i, atomic_t *v) 826 { 827 #if defined(arch_atomic_sub_return_release) 828 return arch_atomic_sub_return_release(i, v); 829 #elif defined(arch_atomic_sub_return_relaxed) 830 __atomic_release_fence(); 831 return arch_atomic_sub_return_relaxed(i, v); 832 #elif defined(arch_atomic_sub_return) 833 return arch_atomic_sub_return(i, v); 834 #else 835 #error "Unable to define raw_atomic_sub_return_release" 836 #endif 837 } 838 839 /** 840 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering 841 * @i: int value to subtract 842 * @v: pointer to atomic_t 843 * 844 * Atomically updates @v to (@v - @i) with relaxed ordering. 845 * 846 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere. 847 * 848 * Return: The updated value of @v. 849 */ 850 static __always_inline int 851 raw_atomic_sub_return_relaxed(int i, atomic_t *v) 852 { 853 #if defined(arch_atomic_sub_return_relaxed) 854 return arch_atomic_sub_return_relaxed(i, v); 855 #elif defined(arch_atomic_sub_return) 856 return arch_atomic_sub_return(i, v); 857 #else 858 #error "Unable to define raw_atomic_sub_return_relaxed" 859 #endif 860 } 861 862 /** 863 * raw_atomic_fetch_sub() - atomic subtract with full ordering 864 * @i: int value to subtract 865 * @v: pointer to atomic_t 866 * 867 * Atomically updates @v to (@v - @i) with full ordering. 868 * 869 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere. 870 * 871 * Return: The original value of @v. 872 */ 873 static __always_inline int 874 raw_atomic_fetch_sub(int i, atomic_t *v) 875 { 876 #if defined(arch_atomic_fetch_sub) 877 return arch_atomic_fetch_sub(i, v); 878 #elif defined(arch_atomic_fetch_sub_relaxed) 879 int ret; 880 __atomic_pre_full_fence(); 881 ret = arch_atomic_fetch_sub_relaxed(i, v); 882 __atomic_post_full_fence(); 883 return ret; 884 #else 885 #error "Unable to define raw_atomic_fetch_sub" 886 #endif 887 } 888 889 /** 890 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering 891 * @i: int value to subtract 892 * @v: pointer to atomic_t 893 * 894 * Atomically updates @v to (@v - @i) with acquire ordering. 895 * 896 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere. 897 * 898 * Return: The original value of @v. 899 */ 900 static __always_inline int 901 raw_atomic_fetch_sub_acquire(int i, atomic_t *v) 902 { 903 #if defined(arch_atomic_fetch_sub_acquire) 904 return arch_atomic_fetch_sub_acquire(i, v); 905 #elif defined(arch_atomic_fetch_sub_relaxed) 906 int ret = arch_atomic_fetch_sub_relaxed(i, v); 907 __atomic_acquire_fence(); 908 return ret; 909 #elif defined(arch_atomic_fetch_sub) 910 return arch_atomic_fetch_sub(i, v); 911 #else 912 #error "Unable to define raw_atomic_fetch_sub_acquire" 913 #endif 914 } 915 916 /** 917 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering 918 * @i: int value to subtract 919 * @v: pointer to atomic_t 920 * 921 * Atomically updates @v to (@v - @i) with release ordering. 922 * 923 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere. 924 * 925 * Return: The original value of @v. 926 */ 927 static __always_inline int 928 raw_atomic_fetch_sub_release(int i, atomic_t *v) 929 { 930 #if defined(arch_atomic_fetch_sub_release) 931 return arch_atomic_fetch_sub_release(i, v); 932 #elif defined(arch_atomic_fetch_sub_relaxed) 933 __atomic_release_fence(); 934 return arch_atomic_fetch_sub_relaxed(i, v); 935 #elif defined(arch_atomic_fetch_sub) 936 return arch_atomic_fetch_sub(i, v); 937 #else 938 #error "Unable to define raw_atomic_fetch_sub_release" 939 #endif 940 } 941 942 /** 943 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering 944 * @i: int value to subtract 945 * @v: pointer to atomic_t 946 * 947 * Atomically updates @v to (@v - @i) with relaxed ordering. 948 * 949 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere. 950 * 951 * Return: The original value of @v. 952 */ 953 static __always_inline int 954 raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) 955 { 956 #if defined(arch_atomic_fetch_sub_relaxed) 957 return arch_atomic_fetch_sub_relaxed(i, v); 958 #elif defined(arch_atomic_fetch_sub) 959 return arch_atomic_fetch_sub(i, v); 960 #else 961 #error "Unable to define raw_atomic_fetch_sub_relaxed" 962 #endif 963 } 964 965 /** 966 * raw_atomic_inc() - atomic increment with relaxed ordering 967 * @v: pointer to atomic_t 968 * 969 * Atomically updates @v to (@v + 1) with relaxed ordering. 970 * 971 * Safe to use in noinstr code; prefer atomic_inc() elsewhere. 972 * 973 * Return: Nothing. 974 */ 975 static __always_inline void 976 raw_atomic_inc(atomic_t *v) 977 { 978 #if defined(arch_atomic_inc) 979 arch_atomic_inc(v); 980 #else 981 raw_atomic_add(1, v); 982 #endif 983 } 984 985 /** 986 * raw_atomic_inc_return() - atomic increment with full ordering 987 * @v: pointer to atomic_t 988 * 989 * Atomically updates @v to (@v + 1) with full ordering. 990 * 991 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere. 992 * 993 * Return: The updated value of @v. 994 */ 995 static __always_inline int 996 raw_atomic_inc_return(atomic_t *v) 997 { 998 #if defined(arch_atomic_inc_return) 999 return arch_atomic_inc_return(v); 1000 #elif defined(arch_atomic_inc_return_relaxed) 1001 int ret; 1002 __atomic_pre_full_fence(); 1003 ret = arch_atomic_inc_return_relaxed(v); 1004 __atomic_post_full_fence(); 1005 return ret; 1006 #else 1007 return raw_atomic_add_return(1, v); 1008 #endif 1009 } 1010 1011 /** 1012 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering 1013 * @v: pointer to atomic_t 1014 * 1015 * Atomically updates @v to (@v + 1) with acquire ordering. 1016 * 1017 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere. 1018 * 1019 * Return: The updated value of @v. 1020 */ 1021 static __always_inline int 1022 raw_atomic_inc_return_acquire(atomic_t *v) 1023 { 1024 #if defined(arch_atomic_inc_return_acquire) 1025 return arch_atomic_inc_return_acquire(v); 1026 #elif defined(arch_atomic_inc_return_relaxed) 1027 int ret = arch_atomic_inc_return_relaxed(v); 1028 __atomic_acquire_fence(); 1029 return ret; 1030 #elif defined(arch_atomic_inc_return) 1031 return arch_atomic_inc_return(v); 1032 #else 1033 return raw_atomic_add_return_acquire(1, v); 1034 #endif 1035 } 1036 1037 /** 1038 * raw_atomic_inc_return_release() - atomic increment with release ordering 1039 * @v: pointer to atomic_t 1040 * 1041 * Atomically updates @v to (@v + 1) with release ordering. 1042 * 1043 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere. 1044 * 1045 * Return: The updated value of @v. 1046 */ 1047 static __always_inline int 1048 raw_atomic_inc_return_release(atomic_t *v) 1049 { 1050 #if defined(arch_atomic_inc_return_release) 1051 return arch_atomic_inc_return_release(v); 1052 #elif defined(arch_atomic_inc_return_relaxed) 1053 __atomic_release_fence(); 1054 return arch_atomic_inc_return_relaxed(v); 1055 #elif defined(arch_atomic_inc_return) 1056 return arch_atomic_inc_return(v); 1057 #else 1058 return raw_atomic_add_return_release(1, v); 1059 #endif 1060 } 1061 1062 /** 1063 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering 1064 * @v: pointer to atomic_t 1065 * 1066 * Atomically updates @v to (@v + 1) with relaxed ordering. 1067 * 1068 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere. 1069 * 1070 * Return: The updated value of @v. 1071 */ 1072 static __always_inline int 1073 raw_atomic_inc_return_relaxed(atomic_t *v) 1074 { 1075 #if defined(arch_atomic_inc_return_relaxed) 1076 return arch_atomic_inc_return_relaxed(v); 1077 #elif defined(arch_atomic_inc_return) 1078 return arch_atomic_inc_return(v); 1079 #else 1080 return raw_atomic_add_return_relaxed(1, v); 1081 #endif 1082 } 1083 1084 /** 1085 * raw_atomic_fetch_inc() - atomic increment with full ordering 1086 * @v: pointer to atomic_t 1087 * 1088 * Atomically updates @v to (@v + 1) with full ordering. 1089 * 1090 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere. 1091 * 1092 * Return: The original value of @v. 1093 */ 1094 static __always_inline int 1095 raw_atomic_fetch_inc(atomic_t *v) 1096 { 1097 #if defined(arch_atomic_fetch_inc) 1098 return arch_atomic_fetch_inc(v); 1099 #elif defined(arch_atomic_fetch_inc_relaxed) 1100 int ret; 1101 __atomic_pre_full_fence(); 1102 ret = arch_atomic_fetch_inc_relaxed(v); 1103 __atomic_post_full_fence(); 1104 return ret; 1105 #else 1106 return raw_atomic_fetch_add(1, v); 1107 #endif 1108 } 1109 1110 /** 1111 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering 1112 * @v: pointer to atomic_t 1113 * 1114 * Atomically updates @v to (@v + 1) with acquire ordering. 1115 * 1116 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere. 1117 * 1118 * Return: The original value of @v. 1119 */ 1120 static __always_inline int 1121 raw_atomic_fetch_inc_acquire(atomic_t *v) 1122 { 1123 #if defined(arch_atomic_fetch_inc_acquire) 1124 return arch_atomic_fetch_inc_acquire(v); 1125 #elif defined(arch_atomic_fetch_inc_relaxed) 1126 int ret = arch_atomic_fetch_inc_relaxed(v); 1127 __atomic_acquire_fence(); 1128 return ret; 1129 #elif defined(arch_atomic_fetch_inc) 1130 return arch_atomic_fetch_inc(v); 1131 #else 1132 return raw_atomic_fetch_add_acquire(1, v); 1133 #endif 1134 } 1135 1136 /** 1137 * raw_atomic_fetch_inc_release() - atomic increment with release ordering 1138 * @v: pointer to atomic_t 1139 * 1140 * Atomically updates @v to (@v + 1) with release ordering. 1141 * 1142 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere. 1143 * 1144 * Return: The original value of @v. 1145 */ 1146 static __always_inline int 1147 raw_atomic_fetch_inc_release(atomic_t *v) 1148 { 1149 #if defined(arch_atomic_fetch_inc_release) 1150 return arch_atomic_fetch_inc_release(v); 1151 #elif defined(arch_atomic_fetch_inc_relaxed) 1152 __atomic_release_fence(); 1153 return arch_atomic_fetch_inc_relaxed(v); 1154 #elif defined(arch_atomic_fetch_inc) 1155 return arch_atomic_fetch_inc(v); 1156 #else 1157 return raw_atomic_fetch_add_release(1, v); 1158 #endif 1159 } 1160 1161 /** 1162 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering 1163 * @v: pointer to atomic_t 1164 * 1165 * Atomically updates @v to (@v + 1) with relaxed ordering. 1166 * 1167 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere. 1168 * 1169 * Return: The original value of @v. 1170 */ 1171 static __always_inline int 1172 raw_atomic_fetch_inc_relaxed(atomic_t *v) 1173 { 1174 #if defined(arch_atomic_fetch_inc_relaxed) 1175 return arch_atomic_fetch_inc_relaxed(v); 1176 #elif defined(arch_atomic_fetch_inc) 1177 return arch_atomic_fetch_inc(v); 1178 #else 1179 return raw_atomic_fetch_add_relaxed(1, v); 1180 #endif 1181 } 1182 1183 /** 1184 * raw_atomic_dec() - atomic decrement with relaxed ordering 1185 * @v: pointer to atomic_t 1186 * 1187 * Atomically updates @v to (@v - 1) with relaxed ordering. 1188 * 1189 * Safe to use in noinstr code; prefer atomic_dec() elsewhere. 1190 * 1191 * Return: Nothing. 1192 */ 1193 static __always_inline void 1194 raw_atomic_dec(atomic_t *v) 1195 { 1196 #if defined(arch_atomic_dec) 1197 arch_atomic_dec(v); 1198 #else 1199 raw_atomic_sub(1, v); 1200 #endif 1201 } 1202 1203 /** 1204 * raw_atomic_dec_return() - atomic decrement with full ordering 1205 * @v: pointer to atomic_t 1206 * 1207 * Atomically updates @v to (@v - 1) with full ordering. 1208 * 1209 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere. 1210 * 1211 * Return: The updated value of @v. 1212 */ 1213 static __always_inline int 1214 raw_atomic_dec_return(atomic_t *v) 1215 { 1216 #if defined(arch_atomic_dec_return) 1217 return arch_atomic_dec_return(v); 1218 #elif defined(arch_atomic_dec_return_relaxed) 1219 int ret; 1220 __atomic_pre_full_fence(); 1221 ret = arch_atomic_dec_return_relaxed(v); 1222 __atomic_post_full_fence(); 1223 return ret; 1224 #else 1225 return raw_atomic_sub_return(1, v); 1226 #endif 1227 } 1228 1229 /** 1230 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering 1231 * @v: pointer to atomic_t 1232 * 1233 * Atomically updates @v to (@v - 1) with acquire ordering. 1234 * 1235 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere. 1236 * 1237 * Return: The updated value of @v. 1238 */ 1239 static __always_inline int 1240 raw_atomic_dec_return_acquire(atomic_t *v) 1241 { 1242 #if defined(arch_atomic_dec_return_acquire) 1243 return arch_atomic_dec_return_acquire(v); 1244 #elif defined(arch_atomic_dec_return_relaxed) 1245 int ret = arch_atomic_dec_return_relaxed(v); 1246 __atomic_acquire_fence(); 1247 return ret; 1248 #elif defined(arch_atomic_dec_return) 1249 return arch_atomic_dec_return(v); 1250 #else 1251 return raw_atomic_sub_return_acquire(1, v); 1252 #endif 1253 } 1254 1255 /** 1256 * raw_atomic_dec_return_release() - atomic decrement with release ordering 1257 * @v: pointer to atomic_t 1258 * 1259 * Atomically updates @v to (@v - 1) with release ordering. 1260 * 1261 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere. 1262 * 1263 * Return: The updated value of @v. 1264 */ 1265 static __always_inline int 1266 raw_atomic_dec_return_release(atomic_t *v) 1267 { 1268 #if defined(arch_atomic_dec_return_release) 1269 return arch_atomic_dec_return_release(v); 1270 #elif defined(arch_atomic_dec_return_relaxed) 1271 __atomic_release_fence(); 1272 return arch_atomic_dec_return_relaxed(v); 1273 #elif defined(arch_atomic_dec_return) 1274 return arch_atomic_dec_return(v); 1275 #else 1276 return raw_atomic_sub_return_release(1, v); 1277 #endif 1278 } 1279 1280 /** 1281 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering 1282 * @v: pointer to atomic_t 1283 * 1284 * Atomically updates @v to (@v - 1) with relaxed ordering. 1285 * 1286 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere. 1287 * 1288 * Return: The updated value of @v. 1289 */ 1290 static __always_inline int 1291 raw_atomic_dec_return_relaxed(atomic_t *v) 1292 { 1293 #if defined(arch_atomic_dec_return_relaxed) 1294 return arch_atomic_dec_return_relaxed(v); 1295 #elif defined(arch_atomic_dec_return) 1296 return arch_atomic_dec_return(v); 1297 #else 1298 return raw_atomic_sub_return_relaxed(1, v); 1299 #endif 1300 } 1301 1302 /** 1303 * raw_atomic_fetch_dec() - atomic decrement with full ordering 1304 * @v: pointer to atomic_t 1305 * 1306 * Atomically updates @v to (@v - 1) with full ordering. 1307 * 1308 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere. 1309 * 1310 * Return: The original value of @v. 1311 */ 1312 static __always_inline int 1313 raw_atomic_fetch_dec(atomic_t *v) 1314 { 1315 #if defined(arch_atomic_fetch_dec) 1316 return arch_atomic_fetch_dec(v); 1317 #elif defined(arch_atomic_fetch_dec_relaxed) 1318 int ret; 1319 __atomic_pre_full_fence(); 1320 ret = arch_atomic_fetch_dec_relaxed(v); 1321 __atomic_post_full_fence(); 1322 return ret; 1323 #else 1324 return raw_atomic_fetch_sub(1, v); 1325 #endif 1326 } 1327 1328 /** 1329 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering 1330 * @v: pointer to atomic_t 1331 * 1332 * Atomically updates @v to (@v - 1) with acquire ordering. 1333 * 1334 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere. 1335 * 1336 * Return: The original value of @v. 1337 */ 1338 static __always_inline int 1339 raw_atomic_fetch_dec_acquire(atomic_t *v) 1340 { 1341 #if defined(arch_atomic_fetch_dec_acquire) 1342 return arch_atomic_fetch_dec_acquire(v); 1343 #elif defined(arch_atomic_fetch_dec_relaxed) 1344 int ret = arch_atomic_fetch_dec_relaxed(v); 1345 __atomic_acquire_fence(); 1346 return ret; 1347 #elif defined(arch_atomic_fetch_dec) 1348 return arch_atomic_fetch_dec(v); 1349 #else 1350 return raw_atomic_fetch_sub_acquire(1, v); 1351 #endif 1352 } 1353 1354 /** 1355 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering 1356 * @v: pointer to atomic_t 1357 * 1358 * Atomically updates @v to (@v - 1) with release ordering. 1359 * 1360 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere. 1361 * 1362 * Return: The original value of @v. 1363 */ 1364 static __always_inline int 1365 raw_atomic_fetch_dec_release(atomic_t *v) 1366 { 1367 #if defined(arch_atomic_fetch_dec_release) 1368 return arch_atomic_fetch_dec_release(v); 1369 #elif defined(arch_atomic_fetch_dec_relaxed) 1370 __atomic_release_fence(); 1371 return arch_atomic_fetch_dec_relaxed(v); 1372 #elif defined(arch_atomic_fetch_dec) 1373 return arch_atomic_fetch_dec(v); 1374 #else 1375 return raw_atomic_fetch_sub_release(1, v); 1376 #endif 1377 } 1378 1379 /** 1380 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering 1381 * @v: pointer to atomic_t 1382 * 1383 * Atomically updates @v to (@v - 1) with relaxed ordering. 1384 * 1385 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere. 1386 * 1387 * Return: The original value of @v. 1388 */ 1389 static __always_inline int 1390 raw_atomic_fetch_dec_relaxed(atomic_t *v) 1391 { 1392 #if defined(arch_atomic_fetch_dec_relaxed) 1393 return arch_atomic_fetch_dec_relaxed(v); 1394 #elif defined(arch_atomic_fetch_dec) 1395 return arch_atomic_fetch_dec(v); 1396 #else 1397 return raw_atomic_fetch_sub_relaxed(1, v); 1398 #endif 1399 } 1400 1401 /** 1402 * raw_atomic_and() - atomic bitwise AND with relaxed ordering 1403 * @i: int value 1404 * @v: pointer to atomic_t 1405 * 1406 * Atomically updates @v to (@v & @i) with relaxed ordering. 1407 * 1408 * Safe to use in noinstr code; prefer atomic_and() elsewhere. 1409 * 1410 * Return: Nothing. 1411 */ 1412 static __always_inline void 1413 raw_atomic_and(int i, atomic_t *v) 1414 { 1415 arch_atomic_and(i, v); 1416 } 1417 1418 /** 1419 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering 1420 * @i: int value 1421 * @v: pointer to atomic_t 1422 * 1423 * Atomically updates @v to (@v & @i) with full ordering. 1424 * 1425 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere. 1426 * 1427 * Return: The original value of @v. 1428 */ 1429 static __always_inline int 1430 raw_atomic_fetch_and(int i, atomic_t *v) 1431 { 1432 #if defined(arch_atomic_fetch_and) 1433 return arch_atomic_fetch_and(i, v); 1434 #elif defined(arch_atomic_fetch_and_relaxed) 1435 int ret; 1436 __atomic_pre_full_fence(); 1437 ret = arch_atomic_fetch_and_relaxed(i, v); 1438 __atomic_post_full_fence(); 1439 return ret; 1440 #else 1441 #error "Unable to define raw_atomic_fetch_and" 1442 #endif 1443 } 1444 1445 /** 1446 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering 1447 * @i: int value 1448 * @v: pointer to atomic_t 1449 * 1450 * Atomically updates @v to (@v & @i) with acquire ordering. 1451 * 1452 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere. 1453 * 1454 * Return: The original value of @v. 1455 */ 1456 static __always_inline int 1457 raw_atomic_fetch_and_acquire(int i, atomic_t *v) 1458 { 1459 #if defined(arch_atomic_fetch_and_acquire) 1460 return arch_atomic_fetch_and_acquire(i, v); 1461 #elif defined(arch_atomic_fetch_and_relaxed) 1462 int ret = arch_atomic_fetch_and_relaxed(i, v); 1463 __atomic_acquire_fence(); 1464 return ret; 1465 #elif defined(arch_atomic_fetch_and) 1466 return arch_atomic_fetch_and(i, v); 1467 #else 1468 #error "Unable to define raw_atomic_fetch_and_acquire" 1469 #endif 1470 } 1471 1472 /** 1473 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering 1474 * @i: int value 1475 * @v: pointer to atomic_t 1476 * 1477 * Atomically updates @v to (@v & @i) with release ordering. 1478 * 1479 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere. 1480 * 1481 * Return: The original value of @v. 1482 */ 1483 static __always_inline int 1484 raw_atomic_fetch_and_release(int i, atomic_t *v) 1485 { 1486 #if defined(arch_atomic_fetch_and_release) 1487 return arch_atomic_fetch_and_release(i, v); 1488 #elif defined(arch_atomic_fetch_and_relaxed) 1489 __atomic_release_fence(); 1490 return arch_atomic_fetch_and_relaxed(i, v); 1491 #elif defined(arch_atomic_fetch_and) 1492 return arch_atomic_fetch_and(i, v); 1493 #else 1494 #error "Unable to define raw_atomic_fetch_and_release" 1495 #endif 1496 } 1497 1498 /** 1499 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering 1500 * @i: int value 1501 * @v: pointer to atomic_t 1502 * 1503 * Atomically updates @v to (@v & @i) with relaxed ordering. 1504 * 1505 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere. 1506 * 1507 * Return: The original value of @v. 1508 */ 1509 static __always_inline int 1510 raw_atomic_fetch_and_relaxed(int i, atomic_t *v) 1511 { 1512 #if defined(arch_atomic_fetch_and_relaxed) 1513 return arch_atomic_fetch_and_relaxed(i, v); 1514 #elif defined(arch_atomic_fetch_and) 1515 return arch_atomic_fetch_and(i, v); 1516 #else 1517 #error "Unable to define raw_atomic_fetch_and_relaxed" 1518 #endif 1519 } 1520 1521 /** 1522 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering 1523 * @i: int value 1524 * @v: pointer to atomic_t 1525 * 1526 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 1527 * 1528 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere. 1529 * 1530 * Return: Nothing. 1531 */ 1532 static __always_inline void 1533 raw_atomic_andnot(int i, atomic_t *v) 1534 { 1535 #if defined(arch_atomic_andnot) 1536 arch_atomic_andnot(i, v); 1537 #else 1538 raw_atomic_and(~i, v); 1539 #endif 1540 } 1541 1542 /** 1543 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering 1544 * @i: int value 1545 * @v: pointer to atomic_t 1546 * 1547 * Atomically updates @v to (@v & ~@i) with full ordering. 1548 * 1549 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere. 1550 * 1551 * Return: The original value of @v. 1552 */ 1553 static __always_inline int 1554 raw_atomic_fetch_andnot(int i, atomic_t *v) 1555 { 1556 #if defined(arch_atomic_fetch_andnot) 1557 return arch_atomic_fetch_andnot(i, v); 1558 #elif defined(arch_atomic_fetch_andnot_relaxed) 1559 int ret; 1560 __atomic_pre_full_fence(); 1561 ret = arch_atomic_fetch_andnot_relaxed(i, v); 1562 __atomic_post_full_fence(); 1563 return ret; 1564 #else 1565 return raw_atomic_fetch_and(~i, v); 1566 #endif 1567 } 1568 1569 /** 1570 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering 1571 * @i: int value 1572 * @v: pointer to atomic_t 1573 * 1574 * Atomically updates @v to (@v & ~@i) with acquire ordering. 1575 * 1576 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere. 1577 * 1578 * Return: The original value of @v. 1579 */ 1580 static __always_inline int 1581 raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) 1582 { 1583 #if defined(arch_atomic_fetch_andnot_acquire) 1584 return arch_atomic_fetch_andnot_acquire(i, v); 1585 #elif defined(arch_atomic_fetch_andnot_relaxed) 1586 int ret = arch_atomic_fetch_andnot_relaxed(i, v); 1587 __atomic_acquire_fence(); 1588 return ret; 1589 #elif defined(arch_atomic_fetch_andnot) 1590 return arch_atomic_fetch_andnot(i, v); 1591 #else 1592 return raw_atomic_fetch_and_acquire(~i, v); 1593 #endif 1594 } 1595 1596 /** 1597 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering 1598 * @i: int value 1599 * @v: pointer to atomic_t 1600 * 1601 * Atomically updates @v to (@v & ~@i) with release ordering. 1602 * 1603 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere. 1604 * 1605 * Return: The original value of @v. 1606 */ 1607 static __always_inline int 1608 raw_atomic_fetch_andnot_release(int i, atomic_t *v) 1609 { 1610 #if defined(arch_atomic_fetch_andnot_release) 1611 return arch_atomic_fetch_andnot_release(i, v); 1612 #elif defined(arch_atomic_fetch_andnot_relaxed) 1613 __atomic_release_fence(); 1614 return arch_atomic_fetch_andnot_relaxed(i, v); 1615 #elif defined(arch_atomic_fetch_andnot) 1616 return arch_atomic_fetch_andnot(i, v); 1617 #else 1618 return raw_atomic_fetch_and_release(~i, v); 1619 #endif 1620 } 1621 1622 /** 1623 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering 1624 * @i: int value 1625 * @v: pointer to atomic_t 1626 * 1627 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 1628 * 1629 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere. 1630 * 1631 * Return: The original value of @v. 1632 */ 1633 static __always_inline int 1634 raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 1635 { 1636 #if defined(arch_atomic_fetch_andnot_relaxed) 1637 return arch_atomic_fetch_andnot_relaxed(i, v); 1638 #elif defined(arch_atomic_fetch_andnot) 1639 return arch_atomic_fetch_andnot(i, v); 1640 #else 1641 return raw_atomic_fetch_and_relaxed(~i, v); 1642 #endif 1643 } 1644 1645 /** 1646 * raw_atomic_or() - atomic bitwise OR with relaxed ordering 1647 * @i: int value 1648 * @v: pointer to atomic_t 1649 * 1650 * Atomically updates @v to (@v | @i) with relaxed ordering. 1651 * 1652 * Safe to use in noinstr code; prefer atomic_or() elsewhere. 1653 * 1654 * Return: Nothing. 1655 */ 1656 static __always_inline void 1657 raw_atomic_or(int i, atomic_t *v) 1658 { 1659 arch_atomic_or(i, v); 1660 } 1661 1662 /** 1663 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering 1664 * @i: int value 1665 * @v: pointer to atomic_t 1666 * 1667 * Atomically updates @v to (@v | @i) with full ordering. 1668 * 1669 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere. 1670 * 1671 * Return: The original value of @v. 1672 */ 1673 static __always_inline int 1674 raw_atomic_fetch_or(int i, atomic_t *v) 1675 { 1676 #if defined(arch_atomic_fetch_or) 1677 return arch_atomic_fetch_or(i, v); 1678 #elif defined(arch_atomic_fetch_or_relaxed) 1679 int ret; 1680 __atomic_pre_full_fence(); 1681 ret = arch_atomic_fetch_or_relaxed(i, v); 1682 __atomic_post_full_fence(); 1683 return ret; 1684 #else 1685 #error "Unable to define raw_atomic_fetch_or" 1686 #endif 1687 } 1688 1689 /** 1690 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering 1691 * @i: int value 1692 * @v: pointer to atomic_t 1693 * 1694 * Atomically updates @v to (@v | @i) with acquire ordering. 1695 * 1696 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere. 1697 * 1698 * Return: The original value of @v. 1699 */ 1700 static __always_inline int 1701 raw_atomic_fetch_or_acquire(int i, atomic_t *v) 1702 { 1703 #if defined(arch_atomic_fetch_or_acquire) 1704 return arch_atomic_fetch_or_acquire(i, v); 1705 #elif defined(arch_atomic_fetch_or_relaxed) 1706 int ret = arch_atomic_fetch_or_relaxed(i, v); 1707 __atomic_acquire_fence(); 1708 return ret; 1709 #elif defined(arch_atomic_fetch_or) 1710 return arch_atomic_fetch_or(i, v); 1711 #else 1712 #error "Unable to define raw_atomic_fetch_or_acquire" 1713 #endif 1714 } 1715 1716 /** 1717 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering 1718 * @i: int value 1719 * @v: pointer to atomic_t 1720 * 1721 * Atomically updates @v to (@v | @i) with release ordering. 1722 * 1723 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere. 1724 * 1725 * Return: The original value of @v. 1726 */ 1727 static __always_inline int 1728 raw_atomic_fetch_or_release(int i, atomic_t *v) 1729 { 1730 #if defined(arch_atomic_fetch_or_release) 1731 return arch_atomic_fetch_or_release(i, v); 1732 #elif defined(arch_atomic_fetch_or_relaxed) 1733 __atomic_release_fence(); 1734 return arch_atomic_fetch_or_relaxed(i, v); 1735 #elif defined(arch_atomic_fetch_or) 1736 return arch_atomic_fetch_or(i, v); 1737 #else 1738 #error "Unable to define raw_atomic_fetch_or_release" 1739 #endif 1740 } 1741 1742 /** 1743 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering 1744 * @i: int value 1745 * @v: pointer to atomic_t 1746 * 1747 * Atomically updates @v to (@v | @i) with relaxed ordering. 1748 * 1749 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere. 1750 * 1751 * Return: The original value of @v. 1752 */ 1753 static __always_inline int 1754 raw_atomic_fetch_or_relaxed(int i, atomic_t *v) 1755 { 1756 #if defined(arch_atomic_fetch_or_relaxed) 1757 return arch_atomic_fetch_or_relaxed(i, v); 1758 #elif defined(arch_atomic_fetch_or) 1759 return arch_atomic_fetch_or(i, v); 1760 #else 1761 #error "Unable to define raw_atomic_fetch_or_relaxed" 1762 #endif 1763 } 1764 1765 /** 1766 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering 1767 * @i: int value 1768 * @v: pointer to atomic_t 1769 * 1770 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 1771 * 1772 * Safe to use in noinstr code; prefer atomic_xor() elsewhere. 1773 * 1774 * Return: Nothing. 1775 */ 1776 static __always_inline void 1777 raw_atomic_xor(int i, atomic_t *v) 1778 { 1779 arch_atomic_xor(i, v); 1780 } 1781 1782 /** 1783 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering 1784 * @i: int value 1785 * @v: pointer to atomic_t 1786 * 1787 * Atomically updates @v to (@v ^ @i) with full ordering. 1788 * 1789 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere. 1790 * 1791 * Return: The original value of @v. 1792 */ 1793 static __always_inline int 1794 raw_atomic_fetch_xor(int i, atomic_t *v) 1795 { 1796 #if defined(arch_atomic_fetch_xor) 1797 return arch_atomic_fetch_xor(i, v); 1798 #elif defined(arch_atomic_fetch_xor_relaxed) 1799 int ret; 1800 __atomic_pre_full_fence(); 1801 ret = arch_atomic_fetch_xor_relaxed(i, v); 1802 __atomic_post_full_fence(); 1803 return ret; 1804 #else 1805 #error "Unable to define raw_atomic_fetch_xor" 1806 #endif 1807 } 1808 1809 /** 1810 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering 1811 * @i: int value 1812 * @v: pointer to atomic_t 1813 * 1814 * Atomically updates @v to (@v ^ @i) with acquire ordering. 1815 * 1816 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere. 1817 * 1818 * Return: The original value of @v. 1819 */ 1820 static __always_inline int 1821 raw_atomic_fetch_xor_acquire(int i, atomic_t *v) 1822 { 1823 #if defined(arch_atomic_fetch_xor_acquire) 1824 return arch_atomic_fetch_xor_acquire(i, v); 1825 #elif defined(arch_atomic_fetch_xor_relaxed) 1826 int ret = arch_atomic_fetch_xor_relaxed(i, v); 1827 __atomic_acquire_fence(); 1828 return ret; 1829 #elif defined(arch_atomic_fetch_xor) 1830 return arch_atomic_fetch_xor(i, v); 1831 #else 1832 #error "Unable to define raw_atomic_fetch_xor_acquire" 1833 #endif 1834 } 1835 1836 /** 1837 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering 1838 * @i: int value 1839 * @v: pointer to atomic_t 1840 * 1841 * Atomically updates @v to (@v ^ @i) with release ordering. 1842 * 1843 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere. 1844 * 1845 * Return: The original value of @v. 1846 */ 1847 static __always_inline int 1848 raw_atomic_fetch_xor_release(int i, atomic_t *v) 1849 { 1850 #if defined(arch_atomic_fetch_xor_release) 1851 return arch_atomic_fetch_xor_release(i, v); 1852 #elif defined(arch_atomic_fetch_xor_relaxed) 1853 __atomic_release_fence(); 1854 return arch_atomic_fetch_xor_relaxed(i, v); 1855 #elif defined(arch_atomic_fetch_xor) 1856 return arch_atomic_fetch_xor(i, v); 1857 #else 1858 #error "Unable to define raw_atomic_fetch_xor_release" 1859 #endif 1860 } 1861 1862 /** 1863 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering 1864 * @i: int value 1865 * @v: pointer to atomic_t 1866 * 1867 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 1868 * 1869 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere. 1870 * 1871 * Return: The original value of @v. 1872 */ 1873 static __always_inline int 1874 raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) 1875 { 1876 #if defined(arch_atomic_fetch_xor_relaxed) 1877 return arch_atomic_fetch_xor_relaxed(i, v); 1878 #elif defined(arch_atomic_fetch_xor) 1879 return arch_atomic_fetch_xor(i, v); 1880 #else 1881 #error "Unable to define raw_atomic_fetch_xor_relaxed" 1882 #endif 1883 } 1884 1885 /** 1886 * raw_atomic_xchg() - atomic exchange with full ordering 1887 * @v: pointer to atomic_t 1888 * @new: int value to assign 1889 * 1890 * Atomically updates @v to @new with full ordering. 1891 * 1892 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere. 1893 * 1894 * Return: The original value of @v. 1895 */ 1896 static __always_inline int 1897 raw_atomic_xchg(atomic_t *v, int new) 1898 { 1899 #if defined(arch_atomic_xchg) 1900 return arch_atomic_xchg(v, new); 1901 #elif defined(arch_atomic_xchg_relaxed) 1902 int ret; 1903 __atomic_pre_full_fence(); 1904 ret = arch_atomic_xchg_relaxed(v, new); 1905 __atomic_post_full_fence(); 1906 return ret; 1907 #else 1908 return raw_xchg(&v->counter, new); 1909 #endif 1910 } 1911 1912 /** 1913 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering 1914 * @v: pointer to atomic_t 1915 * @new: int value to assign 1916 * 1917 * Atomically updates @v to @new with acquire ordering. 1918 * 1919 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere. 1920 * 1921 * Return: The original value of @v. 1922 */ 1923 static __always_inline int 1924 raw_atomic_xchg_acquire(atomic_t *v, int new) 1925 { 1926 #if defined(arch_atomic_xchg_acquire) 1927 return arch_atomic_xchg_acquire(v, new); 1928 #elif defined(arch_atomic_xchg_relaxed) 1929 int ret = arch_atomic_xchg_relaxed(v, new); 1930 __atomic_acquire_fence(); 1931 return ret; 1932 #elif defined(arch_atomic_xchg) 1933 return arch_atomic_xchg(v, new); 1934 #else 1935 return raw_xchg_acquire(&v->counter, new); 1936 #endif 1937 } 1938 1939 /** 1940 * raw_atomic_xchg_release() - atomic exchange with release ordering 1941 * @v: pointer to atomic_t 1942 * @new: int value to assign 1943 * 1944 * Atomically updates @v to @new with release ordering. 1945 * 1946 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere. 1947 * 1948 * Return: The original value of @v. 1949 */ 1950 static __always_inline int 1951 raw_atomic_xchg_release(atomic_t *v, int new) 1952 { 1953 #if defined(arch_atomic_xchg_release) 1954 return arch_atomic_xchg_release(v, new); 1955 #elif defined(arch_atomic_xchg_relaxed) 1956 __atomic_release_fence(); 1957 return arch_atomic_xchg_relaxed(v, new); 1958 #elif defined(arch_atomic_xchg) 1959 return arch_atomic_xchg(v, new); 1960 #else 1961 return raw_xchg_release(&v->counter, new); 1962 #endif 1963 } 1964 1965 /** 1966 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering 1967 * @v: pointer to atomic_t 1968 * @new: int value to assign 1969 * 1970 * Atomically updates @v to @new with relaxed ordering. 1971 * 1972 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere. 1973 * 1974 * Return: The original value of @v. 1975 */ 1976 static __always_inline int 1977 raw_atomic_xchg_relaxed(atomic_t *v, int new) 1978 { 1979 #if defined(arch_atomic_xchg_relaxed) 1980 return arch_atomic_xchg_relaxed(v, new); 1981 #elif defined(arch_atomic_xchg) 1982 return arch_atomic_xchg(v, new); 1983 #else 1984 return raw_xchg_relaxed(&v->counter, new); 1985 #endif 1986 } 1987 1988 /** 1989 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering 1990 * @v: pointer to atomic_t 1991 * @old: int value to compare with 1992 * @new: int value to assign 1993 * 1994 * If (@v == @old), atomically updates @v to @new with full ordering. 1995 * 1996 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere. 1997 * 1998 * Return: The original value of @v. 1999 */ 2000 static __always_inline int 2001 raw_atomic_cmpxchg(atomic_t *v, int old, int new) 2002 { 2003 #if defined(arch_atomic_cmpxchg) 2004 return arch_atomic_cmpxchg(v, old, new); 2005 #elif defined(arch_atomic_cmpxchg_relaxed) 2006 int ret; 2007 __atomic_pre_full_fence(); 2008 ret = arch_atomic_cmpxchg_relaxed(v, old, new); 2009 __atomic_post_full_fence(); 2010 return ret; 2011 #else 2012 return raw_cmpxchg(&v->counter, old, new); 2013 #endif 2014 } 2015 2016 /** 2017 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 2018 * @v: pointer to atomic_t 2019 * @old: int value to compare with 2020 * @new: int value to assign 2021 * 2022 * If (@v == @old), atomically updates @v to @new with acquire ordering. 2023 * 2024 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere. 2025 * 2026 * Return: The original value of @v. 2027 */ 2028 static __always_inline int 2029 raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 2030 { 2031 #if defined(arch_atomic_cmpxchg_acquire) 2032 return arch_atomic_cmpxchg_acquire(v, old, new); 2033 #elif defined(arch_atomic_cmpxchg_relaxed) 2034 int ret = arch_atomic_cmpxchg_relaxed(v, old, new); 2035 __atomic_acquire_fence(); 2036 return ret; 2037 #elif defined(arch_atomic_cmpxchg) 2038 return arch_atomic_cmpxchg(v, old, new); 2039 #else 2040 return raw_cmpxchg_acquire(&v->counter, old, new); 2041 #endif 2042 } 2043 2044 /** 2045 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering 2046 * @v: pointer to atomic_t 2047 * @old: int value to compare with 2048 * @new: int value to assign 2049 * 2050 * If (@v == @old), atomically updates @v to @new with release ordering. 2051 * 2052 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere. 2053 * 2054 * Return: The original value of @v. 2055 */ 2056 static __always_inline int 2057 raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) 2058 { 2059 #if defined(arch_atomic_cmpxchg_release) 2060 return arch_atomic_cmpxchg_release(v, old, new); 2061 #elif defined(arch_atomic_cmpxchg_relaxed) 2062 __atomic_release_fence(); 2063 return arch_atomic_cmpxchg_relaxed(v, old, new); 2064 #elif defined(arch_atomic_cmpxchg) 2065 return arch_atomic_cmpxchg(v, old, new); 2066 #else 2067 return raw_cmpxchg_release(&v->counter, old, new); 2068 #endif 2069 } 2070 2071 /** 2072 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 2073 * @v: pointer to atomic_t 2074 * @old: int value to compare with 2075 * @new: int value to assign 2076 * 2077 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 2078 * 2079 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere. 2080 * 2081 * Return: The original value of @v. 2082 */ 2083 static __always_inline int 2084 raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 2085 { 2086 #if defined(arch_atomic_cmpxchg_relaxed) 2087 return arch_atomic_cmpxchg_relaxed(v, old, new); 2088 #elif defined(arch_atomic_cmpxchg) 2089 return arch_atomic_cmpxchg(v, old, new); 2090 #else 2091 return raw_cmpxchg_relaxed(&v->counter, old, new); 2092 #endif 2093 } 2094 2095 /** 2096 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering 2097 * @v: pointer to atomic_t 2098 * @old: pointer to int value to compare with 2099 * @new: int value to assign 2100 * 2101 * If (@v == @old), atomically updates @v to @new with full ordering. 2102 * Otherwise, updates @old to the current value of @v. 2103 * 2104 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere. 2105 * 2106 * Return: @true if the exchange occured, @false otherwise. 2107 */ 2108 static __always_inline bool 2109 raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 2110 { 2111 #if defined(arch_atomic_try_cmpxchg) 2112 return arch_atomic_try_cmpxchg(v, old, new); 2113 #elif defined(arch_atomic_try_cmpxchg_relaxed) 2114 bool ret; 2115 __atomic_pre_full_fence(); 2116 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 2117 __atomic_post_full_fence(); 2118 return ret; 2119 #else 2120 int r, o = *old; 2121 r = raw_atomic_cmpxchg(v, o, new); 2122 if (unlikely(r != o)) 2123 *old = r; 2124 return likely(r == o); 2125 #endif 2126 } 2127 2128 /** 2129 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 2130 * @v: pointer to atomic_t 2131 * @old: pointer to int value to compare with 2132 * @new: int value to assign 2133 * 2134 * If (@v == @old), atomically updates @v to @new with acquire ordering. 2135 * Otherwise, updates @old to the current value of @v. 2136 * 2137 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere. 2138 * 2139 * Return: @true if the exchange occured, @false otherwise. 2140 */ 2141 static __always_inline bool 2142 raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 2143 { 2144 #if defined(arch_atomic_try_cmpxchg_acquire) 2145 return arch_atomic_try_cmpxchg_acquire(v, old, new); 2146 #elif defined(arch_atomic_try_cmpxchg_relaxed) 2147 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 2148 __atomic_acquire_fence(); 2149 return ret; 2150 #elif defined(arch_atomic_try_cmpxchg) 2151 return arch_atomic_try_cmpxchg(v, old, new); 2152 #else 2153 int r, o = *old; 2154 r = raw_atomic_cmpxchg_acquire(v, o, new); 2155 if (unlikely(r != o)) 2156 *old = r; 2157 return likely(r == o); 2158 #endif 2159 } 2160 2161 /** 2162 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering 2163 * @v: pointer to atomic_t 2164 * @old: pointer to int value to compare with 2165 * @new: int value to assign 2166 * 2167 * If (@v == @old), atomically updates @v to @new with release ordering. 2168 * Otherwise, updates @old to the current value of @v. 2169 * 2170 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere. 2171 * 2172 * Return: @true if the exchange occured, @false otherwise. 2173 */ 2174 static __always_inline bool 2175 raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 2176 { 2177 #if defined(arch_atomic_try_cmpxchg_release) 2178 return arch_atomic_try_cmpxchg_release(v, old, new); 2179 #elif defined(arch_atomic_try_cmpxchg_relaxed) 2180 __atomic_release_fence(); 2181 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 2182 #elif defined(arch_atomic_try_cmpxchg) 2183 return arch_atomic_try_cmpxchg(v, old, new); 2184 #else 2185 int r, o = *old; 2186 r = raw_atomic_cmpxchg_release(v, o, new); 2187 if (unlikely(r != o)) 2188 *old = r; 2189 return likely(r == o); 2190 #endif 2191 } 2192 2193 /** 2194 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 2195 * @v: pointer to atomic_t 2196 * @old: pointer to int value to compare with 2197 * @new: int value to assign 2198 * 2199 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 2200 * Otherwise, updates @old to the current value of @v. 2201 * 2202 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere. 2203 * 2204 * Return: @true if the exchange occured, @false otherwise. 2205 */ 2206 static __always_inline bool 2207 raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 2208 { 2209 #if defined(arch_atomic_try_cmpxchg_relaxed) 2210 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 2211 #elif defined(arch_atomic_try_cmpxchg) 2212 return arch_atomic_try_cmpxchg(v, old, new); 2213 #else 2214 int r, o = *old; 2215 r = raw_atomic_cmpxchg_relaxed(v, o, new); 2216 if (unlikely(r != o)) 2217 *old = r; 2218 return likely(r == o); 2219 #endif 2220 } 2221 2222 /** 2223 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering 2224 * @i: int value to add 2225 * @v: pointer to atomic_t 2226 * 2227 * Atomically updates @v to (@v - @i) with full ordering. 2228 * 2229 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere. 2230 * 2231 * Return: @true if the resulting value of @v is zero, @false otherwise. 2232 */ 2233 static __always_inline bool 2234 raw_atomic_sub_and_test(int i, atomic_t *v) 2235 { 2236 #if defined(arch_atomic_sub_and_test) 2237 return arch_atomic_sub_and_test(i, v); 2238 #else 2239 return raw_atomic_sub_return(i, v) == 0; 2240 #endif 2241 } 2242 2243 /** 2244 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering 2245 * @v: pointer to atomic_t 2246 * 2247 * Atomically updates @v to (@v - 1) with full ordering. 2248 * 2249 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere. 2250 * 2251 * Return: @true if the resulting value of @v is zero, @false otherwise. 2252 */ 2253 static __always_inline bool 2254 raw_atomic_dec_and_test(atomic_t *v) 2255 { 2256 #if defined(arch_atomic_dec_and_test) 2257 return arch_atomic_dec_and_test(v); 2258 #else 2259 return raw_atomic_dec_return(v) == 0; 2260 #endif 2261 } 2262 2263 /** 2264 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering 2265 * @v: pointer to atomic_t 2266 * 2267 * Atomically updates @v to (@v + 1) with full ordering. 2268 * 2269 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere. 2270 * 2271 * Return: @true if the resulting value of @v is zero, @false otherwise. 2272 */ 2273 static __always_inline bool 2274 raw_atomic_inc_and_test(atomic_t *v) 2275 { 2276 #if defined(arch_atomic_inc_and_test) 2277 return arch_atomic_inc_and_test(v); 2278 #else 2279 return raw_atomic_inc_return(v) == 0; 2280 #endif 2281 } 2282 2283 /** 2284 * raw_atomic_add_negative() - atomic add and test if negative with full ordering 2285 * @i: int value to add 2286 * @v: pointer to atomic_t 2287 * 2288 * Atomically updates @v to (@v + @i) with full ordering. 2289 * 2290 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere. 2291 * 2292 * Return: @true if the resulting value of @v is negative, @false otherwise. 2293 */ 2294 static __always_inline bool 2295 raw_atomic_add_negative(int i, atomic_t *v) 2296 { 2297 #if defined(arch_atomic_add_negative) 2298 return arch_atomic_add_negative(i, v); 2299 #elif defined(arch_atomic_add_negative_relaxed) 2300 bool ret; 2301 __atomic_pre_full_fence(); 2302 ret = arch_atomic_add_negative_relaxed(i, v); 2303 __atomic_post_full_fence(); 2304 return ret; 2305 #else 2306 return raw_atomic_add_return(i, v) < 0; 2307 #endif 2308 } 2309 2310 /** 2311 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering 2312 * @i: int value to add 2313 * @v: pointer to atomic_t 2314 * 2315 * Atomically updates @v to (@v + @i) with acquire ordering. 2316 * 2317 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere. 2318 * 2319 * Return: @true if the resulting value of @v is negative, @false otherwise. 2320 */ 2321 static __always_inline bool 2322 raw_atomic_add_negative_acquire(int i, atomic_t *v) 2323 { 2324 #if defined(arch_atomic_add_negative_acquire) 2325 return arch_atomic_add_negative_acquire(i, v); 2326 #elif defined(arch_atomic_add_negative_relaxed) 2327 bool ret = arch_atomic_add_negative_relaxed(i, v); 2328 __atomic_acquire_fence(); 2329 return ret; 2330 #elif defined(arch_atomic_add_negative) 2331 return arch_atomic_add_negative(i, v); 2332 #else 2333 return raw_atomic_add_return_acquire(i, v) < 0; 2334 #endif 2335 } 2336 2337 /** 2338 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering 2339 * @i: int value to add 2340 * @v: pointer to atomic_t 2341 * 2342 * Atomically updates @v to (@v + @i) with release ordering. 2343 * 2344 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere. 2345 * 2346 * Return: @true if the resulting value of @v is negative, @false otherwise. 2347 */ 2348 static __always_inline bool 2349 raw_atomic_add_negative_release(int i, atomic_t *v) 2350 { 2351 #if defined(arch_atomic_add_negative_release) 2352 return arch_atomic_add_negative_release(i, v); 2353 #elif defined(arch_atomic_add_negative_relaxed) 2354 __atomic_release_fence(); 2355 return arch_atomic_add_negative_relaxed(i, v); 2356 #elif defined(arch_atomic_add_negative) 2357 return arch_atomic_add_negative(i, v); 2358 #else 2359 return raw_atomic_add_return_release(i, v) < 0; 2360 #endif 2361 } 2362 2363 /** 2364 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering 2365 * @i: int value to add 2366 * @v: pointer to atomic_t 2367 * 2368 * Atomically updates @v to (@v + @i) with relaxed ordering. 2369 * 2370 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere. 2371 * 2372 * Return: @true if the resulting value of @v is negative, @false otherwise. 2373 */ 2374 static __always_inline bool 2375 raw_atomic_add_negative_relaxed(int i, atomic_t *v) 2376 { 2377 #if defined(arch_atomic_add_negative_relaxed) 2378 return arch_atomic_add_negative_relaxed(i, v); 2379 #elif defined(arch_atomic_add_negative) 2380 return arch_atomic_add_negative(i, v); 2381 #else 2382 return raw_atomic_add_return_relaxed(i, v) < 0; 2383 #endif 2384 } 2385 2386 /** 2387 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering 2388 * @v: pointer to atomic_t 2389 * @a: int value to add 2390 * @u: int value to compare with 2391 * 2392 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 2393 * 2394 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere. 2395 * 2396 * Return: The original value of @v. 2397 */ 2398 static __always_inline int 2399 raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) 2400 { 2401 #if defined(arch_atomic_fetch_add_unless) 2402 return arch_atomic_fetch_add_unless(v, a, u); 2403 #else 2404 int c = raw_atomic_read(v); 2405 2406 do { 2407 if (unlikely(c == u)) 2408 break; 2409 } while (!raw_atomic_try_cmpxchg(v, &c, c + a)); 2410 2411 return c; 2412 #endif 2413 } 2414 2415 /** 2416 * raw_atomic_add_unless() - atomic add unless value with full ordering 2417 * @v: pointer to atomic_t 2418 * @a: int value to add 2419 * @u: int value to compare with 2420 * 2421 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 2422 * 2423 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere. 2424 * 2425 * Return: @true if @v was updated, @false otherwise. 2426 */ 2427 static __always_inline bool 2428 raw_atomic_add_unless(atomic_t *v, int a, int u) 2429 { 2430 #if defined(arch_atomic_add_unless) 2431 return arch_atomic_add_unless(v, a, u); 2432 #else 2433 return raw_atomic_fetch_add_unless(v, a, u) != u; 2434 #endif 2435 } 2436 2437 /** 2438 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering 2439 * @v: pointer to atomic_t 2440 * 2441 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. 2442 * 2443 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere. 2444 * 2445 * Return: @true if @v was updated, @false otherwise. 2446 */ 2447 static __always_inline bool 2448 raw_atomic_inc_not_zero(atomic_t *v) 2449 { 2450 #if defined(arch_atomic_inc_not_zero) 2451 return arch_atomic_inc_not_zero(v); 2452 #else 2453 return raw_atomic_add_unless(v, 1, 0); 2454 #endif 2455 } 2456 2457 /** 2458 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering 2459 * @v: pointer to atomic_t 2460 * 2461 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. 2462 * 2463 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere. 2464 * 2465 * Return: @true if @v was updated, @false otherwise. 2466 */ 2467 static __always_inline bool 2468 raw_atomic_inc_unless_negative(atomic_t *v) 2469 { 2470 #if defined(arch_atomic_inc_unless_negative) 2471 return arch_atomic_inc_unless_negative(v); 2472 #else 2473 int c = raw_atomic_read(v); 2474 2475 do { 2476 if (unlikely(c < 0)) 2477 return false; 2478 } while (!raw_atomic_try_cmpxchg(v, &c, c + 1)); 2479 2480 return true; 2481 #endif 2482 } 2483 2484 /** 2485 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering 2486 * @v: pointer to atomic_t 2487 * 2488 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. 2489 * 2490 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere. 2491 * 2492 * Return: @true if @v was updated, @false otherwise. 2493 */ 2494 static __always_inline bool 2495 raw_atomic_dec_unless_positive(atomic_t *v) 2496 { 2497 #if defined(arch_atomic_dec_unless_positive) 2498 return arch_atomic_dec_unless_positive(v); 2499 #else 2500 int c = raw_atomic_read(v); 2501 2502 do { 2503 if (unlikely(c > 0)) 2504 return false; 2505 } while (!raw_atomic_try_cmpxchg(v, &c, c - 1)); 2506 2507 return true; 2508 #endif 2509 } 2510 2511 /** 2512 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering 2513 * @v: pointer to atomic_t 2514 * 2515 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. 2516 * 2517 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere. 2518 * 2519 * Return: The old value of (@v - 1), regardless of whether @v was updated. 2520 */ 2521 static __always_inline int 2522 raw_atomic_dec_if_positive(atomic_t *v) 2523 { 2524 #if defined(arch_atomic_dec_if_positive) 2525 return arch_atomic_dec_if_positive(v); 2526 #else 2527 int dec, c = raw_atomic_read(v); 2528 2529 do { 2530 dec = c - 1; 2531 if (unlikely(dec < 0)) 2532 break; 2533 } while (!raw_atomic_try_cmpxchg(v, &c, dec)); 2534 2535 return dec; 2536 #endif 2537 } 2538 2539 #ifdef CONFIG_GENERIC_ATOMIC64 2540 #include <asm-generic/atomic64.h> 2541 #endif 2542 2543 /** 2544 * raw_atomic64_read() - atomic load with relaxed ordering 2545 * @v: pointer to atomic64_t 2546 * 2547 * Atomically loads the value of @v with relaxed ordering. 2548 * 2549 * Safe to use in noinstr code; prefer atomic64_read() elsewhere. 2550 * 2551 * Return: The value loaded from @v. 2552 */ 2553 static __always_inline s64 2554 raw_atomic64_read(const atomic64_t *v) 2555 { 2556 return arch_atomic64_read(v); 2557 } 2558 2559 /** 2560 * raw_atomic64_read_acquire() - atomic load with acquire ordering 2561 * @v: pointer to atomic64_t 2562 * 2563 * Atomically loads the value of @v with acquire ordering. 2564 * 2565 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere. 2566 * 2567 * Return: The value loaded from @v. 2568 */ 2569 static __always_inline s64 2570 raw_atomic64_read_acquire(const atomic64_t *v) 2571 { 2572 #if defined(arch_atomic64_read_acquire) 2573 return arch_atomic64_read_acquire(v); 2574 #else 2575 s64 ret; 2576 2577 if (__native_word(atomic64_t)) { 2578 ret = smp_load_acquire(&(v)->counter); 2579 } else { 2580 ret = raw_atomic64_read(v); 2581 __atomic_acquire_fence(); 2582 } 2583 2584 return ret; 2585 #endif 2586 } 2587 2588 /** 2589 * raw_atomic64_set() - atomic set with relaxed ordering 2590 * @v: pointer to atomic64_t 2591 * @i: s64 value to assign 2592 * 2593 * Atomically sets @v to @i with relaxed ordering. 2594 * 2595 * Safe to use in noinstr code; prefer atomic64_set() elsewhere. 2596 * 2597 * Return: Nothing. 2598 */ 2599 static __always_inline void 2600 raw_atomic64_set(atomic64_t *v, s64 i) 2601 { 2602 arch_atomic64_set(v, i); 2603 } 2604 2605 /** 2606 * raw_atomic64_set_release() - atomic set with release ordering 2607 * @v: pointer to atomic64_t 2608 * @i: s64 value to assign 2609 * 2610 * Atomically sets @v to @i with release ordering. 2611 * 2612 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere. 2613 * 2614 * Return: Nothing. 2615 */ 2616 static __always_inline void 2617 raw_atomic64_set_release(atomic64_t *v, s64 i) 2618 { 2619 #if defined(arch_atomic64_set_release) 2620 arch_atomic64_set_release(v, i); 2621 #else 2622 if (__native_word(atomic64_t)) { 2623 smp_store_release(&(v)->counter, i); 2624 } else { 2625 __atomic_release_fence(); 2626 raw_atomic64_set(v, i); 2627 } 2628 #endif 2629 } 2630 2631 /** 2632 * raw_atomic64_add() - atomic add with relaxed ordering 2633 * @i: s64 value to add 2634 * @v: pointer to atomic64_t 2635 * 2636 * Atomically updates @v to (@v + @i) with relaxed ordering. 2637 * 2638 * Safe to use in noinstr code; prefer atomic64_add() elsewhere. 2639 * 2640 * Return: Nothing. 2641 */ 2642 static __always_inline void 2643 raw_atomic64_add(s64 i, atomic64_t *v) 2644 { 2645 arch_atomic64_add(i, v); 2646 } 2647 2648 /** 2649 * raw_atomic64_add_return() - atomic add with full ordering 2650 * @i: s64 value to add 2651 * @v: pointer to atomic64_t 2652 * 2653 * Atomically updates @v to (@v + @i) with full ordering. 2654 * 2655 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere. 2656 * 2657 * Return: The updated value of @v. 2658 */ 2659 static __always_inline s64 2660 raw_atomic64_add_return(s64 i, atomic64_t *v) 2661 { 2662 #if defined(arch_atomic64_add_return) 2663 return arch_atomic64_add_return(i, v); 2664 #elif defined(arch_atomic64_add_return_relaxed) 2665 s64 ret; 2666 __atomic_pre_full_fence(); 2667 ret = arch_atomic64_add_return_relaxed(i, v); 2668 __atomic_post_full_fence(); 2669 return ret; 2670 #else 2671 #error "Unable to define raw_atomic64_add_return" 2672 #endif 2673 } 2674 2675 /** 2676 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering 2677 * @i: s64 value to add 2678 * @v: pointer to atomic64_t 2679 * 2680 * Atomically updates @v to (@v + @i) with acquire ordering. 2681 * 2682 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere. 2683 * 2684 * Return: The updated value of @v. 2685 */ 2686 static __always_inline s64 2687 raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) 2688 { 2689 #if defined(arch_atomic64_add_return_acquire) 2690 return arch_atomic64_add_return_acquire(i, v); 2691 #elif defined(arch_atomic64_add_return_relaxed) 2692 s64 ret = arch_atomic64_add_return_relaxed(i, v); 2693 __atomic_acquire_fence(); 2694 return ret; 2695 #elif defined(arch_atomic64_add_return) 2696 return arch_atomic64_add_return(i, v); 2697 #else 2698 #error "Unable to define raw_atomic64_add_return_acquire" 2699 #endif 2700 } 2701 2702 /** 2703 * raw_atomic64_add_return_release() - atomic add with release ordering 2704 * @i: s64 value to add 2705 * @v: pointer to atomic64_t 2706 * 2707 * Atomically updates @v to (@v + @i) with release ordering. 2708 * 2709 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere. 2710 * 2711 * Return: The updated value of @v. 2712 */ 2713 static __always_inline s64 2714 raw_atomic64_add_return_release(s64 i, atomic64_t *v) 2715 { 2716 #if defined(arch_atomic64_add_return_release) 2717 return arch_atomic64_add_return_release(i, v); 2718 #elif defined(arch_atomic64_add_return_relaxed) 2719 __atomic_release_fence(); 2720 return arch_atomic64_add_return_relaxed(i, v); 2721 #elif defined(arch_atomic64_add_return) 2722 return arch_atomic64_add_return(i, v); 2723 #else 2724 #error "Unable to define raw_atomic64_add_return_release" 2725 #endif 2726 } 2727 2728 /** 2729 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering 2730 * @i: s64 value to add 2731 * @v: pointer to atomic64_t 2732 * 2733 * Atomically updates @v to (@v + @i) with relaxed ordering. 2734 * 2735 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere. 2736 * 2737 * Return: The updated value of @v. 2738 */ 2739 static __always_inline s64 2740 raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) 2741 { 2742 #if defined(arch_atomic64_add_return_relaxed) 2743 return arch_atomic64_add_return_relaxed(i, v); 2744 #elif defined(arch_atomic64_add_return) 2745 return arch_atomic64_add_return(i, v); 2746 #else 2747 #error "Unable to define raw_atomic64_add_return_relaxed" 2748 #endif 2749 } 2750 2751 /** 2752 * raw_atomic64_fetch_add() - atomic add with full ordering 2753 * @i: s64 value to add 2754 * @v: pointer to atomic64_t 2755 * 2756 * Atomically updates @v to (@v + @i) with full ordering. 2757 * 2758 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere. 2759 * 2760 * Return: The original value of @v. 2761 */ 2762 static __always_inline s64 2763 raw_atomic64_fetch_add(s64 i, atomic64_t *v) 2764 { 2765 #if defined(arch_atomic64_fetch_add) 2766 return arch_atomic64_fetch_add(i, v); 2767 #elif defined(arch_atomic64_fetch_add_relaxed) 2768 s64 ret; 2769 __atomic_pre_full_fence(); 2770 ret = arch_atomic64_fetch_add_relaxed(i, v); 2771 __atomic_post_full_fence(); 2772 return ret; 2773 #else 2774 #error "Unable to define raw_atomic64_fetch_add" 2775 #endif 2776 } 2777 2778 /** 2779 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering 2780 * @i: s64 value to add 2781 * @v: pointer to atomic64_t 2782 * 2783 * Atomically updates @v to (@v + @i) with acquire ordering. 2784 * 2785 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere. 2786 * 2787 * Return: The original value of @v. 2788 */ 2789 static __always_inline s64 2790 raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 2791 { 2792 #if defined(arch_atomic64_fetch_add_acquire) 2793 return arch_atomic64_fetch_add_acquire(i, v); 2794 #elif defined(arch_atomic64_fetch_add_relaxed) 2795 s64 ret = arch_atomic64_fetch_add_relaxed(i, v); 2796 __atomic_acquire_fence(); 2797 return ret; 2798 #elif defined(arch_atomic64_fetch_add) 2799 return arch_atomic64_fetch_add(i, v); 2800 #else 2801 #error "Unable to define raw_atomic64_fetch_add_acquire" 2802 #endif 2803 } 2804 2805 /** 2806 * raw_atomic64_fetch_add_release() - atomic add with release ordering 2807 * @i: s64 value to add 2808 * @v: pointer to atomic64_t 2809 * 2810 * Atomically updates @v to (@v + @i) with release ordering. 2811 * 2812 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere. 2813 * 2814 * Return: The original value of @v. 2815 */ 2816 static __always_inline s64 2817 raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) 2818 { 2819 #if defined(arch_atomic64_fetch_add_release) 2820 return arch_atomic64_fetch_add_release(i, v); 2821 #elif defined(arch_atomic64_fetch_add_relaxed) 2822 __atomic_release_fence(); 2823 return arch_atomic64_fetch_add_relaxed(i, v); 2824 #elif defined(arch_atomic64_fetch_add) 2825 return arch_atomic64_fetch_add(i, v); 2826 #else 2827 #error "Unable to define raw_atomic64_fetch_add_release" 2828 #endif 2829 } 2830 2831 /** 2832 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering 2833 * @i: s64 value to add 2834 * @v: pointer to atomic64_t 2835 * 2836 * Atomically updates @v to (@v + @i) with relaxed ordering. 2837 * 2838 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere. 2839 * 2840 * Return: The original value of @v. 2841 */ 2842 static __always_inline s64 2843 raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 2844 { 2845 #if defined(arch_atomic64_fetch_add_relaxed) 2846 return arch_atomic64_fetch_add_relaxed(i, v); 2847 #elif defined(arch_atomic64_fetch_add) 2848 return arch_atomic64_fetch_add(i, v); 2849 #else 2850 #error "Unable to define raw_atomic64_fetch_add_relaxed" 2851 #endif 2852 } 2853 2854 /** 2855 * raw_atomic64_sub() - atomic subtract with relaxed ordering 2856 * @i: s64 value to subtract 2857 * @v: pointer to atomic64_t 2858 * 2859 * Atomically updates @v to (@v - @i) with relaxed ordering. 2860 * 2861 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere. 2862 * 2863 * Return: Nothing. 2864 */ 2865 static __always_inline void 2866 raw_atomic64_sub(s64 i, atomic64_t *v) 2867 { 2868 arch_atomic64_sub(i, v); 2869 } 2870 2871 /** 2872 * raw_atomic64_sub_return() - atomic subtract with full ordering 2873 * @i: s64 value to subtract 2874 * @v: pointer to atomic64_t 2875 * 2876 * Atomically updates @v to (@v - @i) with full ordering. 2877 * 2878 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere. 2879 * 2880 * Return: The updated value of @v. 2881 */ 2882 static __always_inline s64 2883 raw_atomic64_sub_return(s64 i, atomic64_t *v) 2884 { 2885 #if defined(arch_atomic64_sub_return) 2886 return arch_atomic64_sub_return(i, v); 2887 #elif defined(arch_atomic64_sub_return_relaxed) 2888 s64 ret; 2889 __atomic_pre_full_fence(); 2890 ret = arch_atomic64_sub_return_relaxed(i, v); 2891 __atomic_post_full_fence(); 2892 return ret; 2893 #else 2894 #error "Unable to define raw_atomic64_sub_return" 2895 #endif 2896 } 2897 2898 /** 2899 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering 2900 * @i: s64 value to subtract 2901 * @v: pointer to atomic64_t 2902 * 2903 * Atomically updates @v to (@v - @i) with acquire ordering. 2904 * 2905 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere. 2906 * 2907 * Return: The updated value of @v. 2908 */ 2909 static __always_inline s64 2910 raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 2911 { 2912 #if defined(arch_atomic64_sub_return_acquire) 2913 return arch_atomic64_sub_return_acquire(i, v); 2914 #elif defined(arch_atomic64_sub_return_relaxed) 2915 s64 ret = arch_atomic64_sub_return_relaxed(i, v); 2916 __atomic_acquire_fence(); 2917 return ret; 2918 #elif defined(arch_atomic64_sub_return) 2919 return arch_atomic64_sub_return(i, v); 2920 #else 2921 #error "Unable to define raw_atomic64_sub_return_acquire" 2922 #endif 2923 } 2924 2925 /** 2926 * raw_atomic64_sub_return_release() - atomic subtract with release ordering 2927 * @i: s64 value to subtract 2928 * @v: pointer to atomic64_t 2929 * 2930 * Atomically updates @v to (@v - @i) with release ordering. 2931 * 2932 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere. 2933 * 2934 * Return: The updated value of @v. 2935 */ 2936 static __always_inline s64 2937 raw_atomic64_sub_return_release(s64 i, atomic64_t *v) 2938 { 2939 #if defined(arch_atomic64_sub_return_release) 2940 return arch_atomic64_sub_return_release(i, v); 2941 #elif defined(arch_atomic64_sub_return_relaxed) 2942 __atomic_release_fence(); 2943 return arch_atomic64_sub_return_relaxed(i, v); 2944 #elif defined(arch_atomic64_sub_return) 2945 return arch_atomic64_sub_return(i, v); 2946 #else 2947 #error "Unable to define raw_atomic64_sub_return_release" 2948 #endif 2949 } 2950 2951 /** 2952 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering 2953 * @i: s64 value to subtract 2954 * @v: pointer to atomic64_t 2955 * 2956 * Atomically updates @v to (@v - @i) with relaxed ordering. 2957 * 2958 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere. 2959 * 2960 * Return: The updated value of @v. 2961 */ 2962 static __always_inline s64 2963 raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 2964 { 2965 #if defined(arch_atomic64_sub_return_relaxed) 2966 return arch_atomic64_sub_return_relaxed(i, v); 2967 #elif defined(arch_atomic64_sub_return) 2968 return arch_atomic64_sub_return(i, v); 2969 #else 2970 #error "Unable to define raw_atomic64_sub_return_relaxed" 2971 #endif 2972 } 2973 2974 /** 2975 * raw_atomic64_fetch_sub() - atomic subtract with full ordering 2976 * @i: s64 value to subtract 2977 * @v: pointer to atomic64_t 2978 * 2979 * Atomically updates @v to (@v - @i) with full ordering. 2980 * 2981 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere. 2982 * 2983 * Return: The original value of @v. 2984 */ 2985 static __always_inline s64 2986 raw_atomic64_fetch_sub(s64 i, atomic64_t *v) 2987 { 2988 #if defined(arch_atomic64_fetch_sub) 2989 return arch_atomic64_fetch_sub(i, v); 2990 #elif defined(arch_atomic64_fetch_sub_relaxed) 2991 s64 ret; 2992 __atomic_pre_full_fence(); 2993 ret = arch_atomic64_fetch_sub_relaxed(i, v); 2994 __atomic_post_full_fence(); 2995 return ret; 2996 #else 2997 #error "Unable to define raw_atomic64_fetch_sub" 2998 #endif 2999 } 3000 3001 /** 3002 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering 3003 * @i: s64 value to subtract 3004 * @v: pointer to atomic64_t 3005 * 3006 * Atomically updates @v to (@v - @i) with acquire ordering. 3007 * 3008 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere. 3009 * 3010 * Return: The original value of @v. 3011 */ 3012 static __always_inline s64 3013 raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 3014 { 3015 #if defined(arch_atomic64_fetch_sub_acquire) 3016 return arch_atomic64_fetch_sub_acquire(i, v); 3017 #elif defined(arch_atomic64_fetch_sub_relaxed) 3018 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); 3019 __atomic_acquire_fence(); 3020 return ret; 3021 #elif defined(arch_atomic64_fetch_sub) 3022 return arch_atomic64_fetch_sub(i, v); 3023 #else 3024 #error "Unable to define raw_atomic64_fetch_sub_acquire" 3025 #endif 3026 } 3027 3028 /** 3029 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering 3030 * @i: s64 value to subtract 3031 * @v: pointer to atomic64_t 3032 * 3033 * Atomically updates @v to (@v - @i) with release ordering. 3034 * 3035 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere. 3036 * 3037 * Return: The original value of @v. 3038 */ 3039 static __always_inline s64 3040 raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 3041 { 3042 #if defined(arch_atomic64_fetch_sub_release) 3043 return arch_atomic64_fetch_sub_release(i, v); 3044 #elif defined(arch_atomic64_fetch_sub_relaxed) 3045 __atomic_release_fence(); 3046 return arch_atomic64_fetch_sub_relaxed(i, v); 3047 #elif defined(arch_atomic64_fetch_sub) 3048 return arch_atomic64_fetch_sub(i, v); 3049 #else 3050 #error "Unable to define raw_atomic64_fetch_sub_release" 3051 #endif 3052 } 3053 3054 /** 3055 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering 3056 * @i: s64 value to subtract 3057 * @v: pointer to atomic64_t 3058 * 3059 * Atomically updates @v to (@v - @i) with relaxed ordering. 3060 * 3061 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere. 3062 * 3063 * Return: The original value of @v. 3064 */ 3065 static __always_inline s64 3066 raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 3067 { 3068 #if defined(arch_atomic64_fetch_sub_relaxed) 3069 return arch_atomic64_fetch_sub_relaxed(i, v); 3070 #elif defined(arch_atomic64_fetch_sub) 3071 return arch_atomic64_fetch_sub(i, v); 3072 #else 3073 #error "Unable to define raw_atomic64_fetch_sub_relaxed" 3074 #endif 3075 } 3076 3077 /** 3078 * raw_atomic64_inc() - atomic increment with relaxed ordering 3079 * @v: pointer to atomic64_t 3080 * 3081 * Atomically updates @v to (@v + 1) with relaxed ordering. 3082 * 3083 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere. 3084 * 3085 * Return: Nothing. 3086 */ 3087 static __always_inline void 3088 raw_atomic64_inc(atomic64_t *v) 3089 { 3090 #if defined(arch_atomic64_inc) 3091 arch_atomic64_inc(v); 3092 #else 3093 raw_atomic64_add(1, v); 3094 #endif 3095 } 3096 3097 /** 3098 * raw_atomic64_inc_return() - atomic increment with full ordering 3099 * @v: pointer to atomic64_t 3100 * 3101 * Atomically updates @v to (@v + 1) with full ordering. 3102 * 3103 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere. 3104 * 3105 * Return: The updated value of @v. 3106 */ 3107 static __always_inline s64 3108 raw_atomic64_inc_return(atomic64_t *v) 3109 { 3110 #if defined(arch_atomic64_inc_return) 3111 return arch_atomic64_inc_return(v); 3112 #elif defined(arch_atomic64_inc_return_relaxed) 3113 s64 ret; 3114 __atomic_pre_full_fence(); 3115 ret = arch_atomic64_inc_return_relaxed(v); 3116 __atomic_post_full_fence(); 3117 return ret; 3118 #else 3119 return raw_atomic64_add_return(1, v); 3120 #endif 3121 } 3122 3123 /** 3124 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering 3125 * @v: pointer to atomic64_t 3126 * 3127 * Atomically updates @v to (@v + 1) with acquire ordering. 3128 * 3129 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere. 3130 * 3131 * Return: The updated value of @v. 3132 */ 3133 static __always_inline s64 3134 raw_atomic64_inc_return_acquire(atomic64_t *v) 3135 { 3136 #if defined(arch_atomic64_inc_return_acquire) 3137 return arch_atomic64_inc_return_acquire(v); 3138 #elif defined(arch_atomic64_inc_return_relaxed) 3139 s64 ret = arch_atomic64_inc_return_relaxed(v); 3140 __atomic_acquire_fence(); 3141 return ret; 3142 #elif defined(arch_atomic64_inc_return) 3143 return arch_atomic64_inc_return(v); 3144 #else 3145 return raw_atomic64_add_return_acquire(1, v); 3146 #endif 3147 } 3148 3149 /** 3150 * raw_atomic64_inc_return_release() - atomic increment with release ordering 3151 * @v: pointer to atomic64_t 3152 * 3153 * Atomically updates @v to (@v + 1) with release ordering. 3154 * 3155 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere. 3156 * 3157 * Return: The updated value of @v. 3158 */ 3159 static __always_inline s64 3160 raw_atomic64_inc_return_release(atomic64_t *v) 3161 { 3162 #if defined(arch_atomic64_inc_return_release) 3163 return arch_atomic64_inc_return_release(v); 3164 #elif defined(arch_atomic64_inc_return_relaxed) 3165 __atomic_release_fence(); 3166 return arch_atomic64_inc_return_relaxed(v); 3167 #elif defined(arch_atomic64_inc_return) 3168 return arch_atomic64_inc_return(v); 3169 #else 3170 return raw_atomic64_add_return_release(1, v); 3171 #endif 3172 } 3173 3174 /** 3175 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering 3176 * @v: pointer to atomic64_t 3177 * 3178 * Atomically updates @v to (@v + 1) with relaxed ordering. 3179 * 3180 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere. 3181 * 3182 * Return: The updated value of @v. 3183 */ 3184 static __always_inline s64 3185 raw_atomic64_inc_return_relaxed(atomic64_t *v) 3186 { 3187 #if defined(arch_atomic64_inc_return_relaxed) 3188 return arch_atomic64_inc_return_relaxed(v); 3189 #elif defined(arch_atomic64_inc_return) 3190 return arch_atomic64_inc_return(v); 3191 #else 3192 return raw_atomic64_add_return_relaxed(1, v); 3193 #endif 3194 } 3195 3196 /** 3197 * raw_atomic64_fetch_inc() - atomic increment with full ordering 3198 * @v: pointer to atomic64_t 3199 * 3200 * Atomically updates @v to (@v + 1) with full ordering. 3201 * 3202 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere. 3203 * 3204 * Return: The original value of @v. 3205 */ 3206 static __always_inline s64 3207 raw_atomic64_fetch_inc(atomic64_t *v) 3208 { 3209 #if defined(arch_atomic64_fetch_inc) 3210 return arch_atomic64_fetch_inc(v); 3211 #elif defined(arch_atomic64_fetch_inc_relaxed) 3212 s64 ret; 3213 __atomic_pre_full_fence(); 3214 ret = arch_atomic64_fetch_inc_relaxed(v); 3215 __atomic_post_full_fence(); 3216 return ret; 3217 #else 3218 return raw_atomic64_fetch_add(1, v); 3219 #endif 3220 } 3221 3222 /** 3223 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering 3224 * @v: pointer to atomic64_t 3225 * 3226 * Atomically updates @v to (@v + 1) with acquire ordering. 3227 * 3228 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere. 3229 * 3230 * Return: The original value of @v. 3231 */ 3232 static __always_inline s64 3233 raw_atomic64_fetch_inc_acquire(atomic64_t *v) 3234 { 3235 #if defined(arch_atomic64_fetch_inc_acquire) 3236 return arch_atomic64_fetch_inc_acquire(v); 3237 #elif defined(arch_atomic64_fetch_inc_relaxed) 3238 s64 ret = arch_atomic64_fetch_inc_relaxed(v); 3239 __atomic_acquire_fence(); 3240 return ret; 3241 #elif defined(arch_atomic64_fetch_inc) 3242 return arch_atomic64_fetch_inc(v); 3243 #else 3244 return raw_atomic64_fetch_add_acquire(1, v); 3245 #endif 3246 } 3247 3248 /** 3249 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering 3250 * @v: pointer to atomic64_t 3251 * 3252 * Atomically updates @v to (@v + 1) with release ordering. 3253 * 3254 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere. 3255 * 3256 * Return: The original value of @v. 3257 */ 3258 static __always_inline s64 3259 raw_atomic64_fetch_inc_release(atomic64_t *v) 3260 { 3261 #if defined(arch_atomic64_fetch_inc_release) 3262 return arch_atomic64_fetch_inc_release(v); 3263 #elif defined(arch_atomic64_fetch_inc_relaxed) 3264 __atomic_release_fence(); 3265 return arch_atomic64_fetch_inc_relaxed(v); 3266 #elif defined(arch_atomic64_fetch_inc) 3267 return arch_atomic64_fetch_inc(v); 3268 #else 3269 return raw_atomic64_fetch_add_release(1, v); 3270 #endif 3271 } 3272 3273 /** 3274 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering 3275 * @v: pointer to atomic64_t 3276 * 3277 * Atomically updates @v to (@v + 1) with relaxed ordering. 3278 * 3279 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere. 3280 * 3281 * Return: The original value of @v. 3282 */ 3283 static __always_inline s64 3284 raw_atomic64_fetch_inc_relaxed(atomic64_t *v) 3285 { 3286 #if defined(arch_atomic64_fetch_inc_relaxed) 3287 return arch_atomic64_fetch_inc_relaxed(v); 3288 #elif defined(arch_atomic64_fetch_inc) 3289 return arch_atomic64_fetch_inc(v); 3290 #else 3291 return raw_atomic64_fetch_add_relaxed(1, v); 3292 #endif 3293 } 3294 3295 /** 3296 * raw_atomic64_dec() - atomic decrement with relaxed ordering 3297 * @v: pointer to atomic64_t 3298 * 3299 * Atomically updates @v to (@v - 1) with relaxed ordering. 3300 * 3301 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere. 3302 * 3303 * Return: Nothing. 3304 */ 3305 static __always_inline void 3306 raw_atomic64_dec(atomic64_t *v) 3307 { 3308 #if defined(arch_atomic64_dec) 3309 arch_atomic64_dec(v); 3310 #else 3311 raw_atomic64_sub(1, v); 3312 #endif 3313 } 3314 3315 /** 3316 * raw_atomic64_dec_return() - atomic decrement with full ordering 3317 * @v: pointer to atomic64_t 3318 * 3319 * Atomically updates @v to (@v - 1) with full ordering. 3320 * 3321 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere. 3322 * 3323 * Return: The updated value of @v. 3324 */ 3325 static __always_inline s64 3326 raw_atomic64_dec_return(atomic64_t *v) 3327 { 3328 #if defined(arch_atomic64_dec_return) 3329 return arch_atomic64_dec_return(v); 3330 #elif defined(arch_atomic64_dec_return_relaxed) 3331 s64 ret; 3332 __atomic_pre_full_fence(); 3333 ret = arch_atomic64_dec_return_relaxed(v); 3334 __atomic_post_full_fence(); 3335 return ret; 3336 #else 3337 return raw_atomic64_sub_return(1, v); 3338 #endif 3339 } 3340 3341 /** 3342 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering 3343 * @v: pointer to atomic64_t 3344 * 3345 * Atomically updates @v to (@v - 1) with acquire ordering. 3346 * 3347 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere. 3348 * 3349 * Return: The updated value of @v. 3350 */ 3351 static __always_inline s64 3352 raw_atomic64_dec_return_acquire(atomic64_t *v) 3353 { 3354 #if defined(arch_atomic64_dec_return_acquire) 3355 return arch_atomic64_dec_return_acquire(v); 3356 #elif defined(arch_atomic64_dec_return_relaxed) 3357 s64 ret = arch_atomic64_dec_return_relaxed(v); 3358 __atomic_acquire_fence(); 3359 return ret; 3360 #elif defined(arch_atomic64_dec_return) 3361 return arch_atomic64_dec_return(v); 3362 #else 3363 return raw_atomic64_sub_return_acquire(1, v); 3364 #endif 3365 } 3366 3367 /** 3368 * raw_atomic64_dec_return_release() - atomic decrement with release ordering 3369 * @v: pointer to atomic64_t 3370 * 3371 * Atomically updates @v to (@v - 1) with release ordering. 3372 * 3373 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere. 3374 * 3375 * Return: The updated value of @v. 3376 */ 3377 static __always_inline s64 3378 raw_atomic64_dec_return_release(atomic64_t *v) 3379 { 3380 #if defined(arch_atomic64_dec_return_release) 3381 return arch_atomic64_dec_return_release(v); 3382 #elif defined(arch_atomic64_dec_return_relaxed) 3383 __atomic_release_fence(); 3384 return arch_atomic64_dec_return_relaxed(v); 3385 #elif defined(arch_atomic64_dec_return) 3386 return arch_atomic64_dec_return(v); 3387 #else 3388 return raw_atomic64_sub_return_release(1, v); 3389 #endif 3390 } 3391 3392 /** 3393 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering 3394 * @v: pointer to atomic64_t 3395 * 3396 * Atomically updates @v to (@v - 1) with relaxed ordering. 3397 * 3398 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere. 3399 * 3400 * Return: The updated value of @v. 3401 */ 3402 static __always_inline s64 3403 raw_atomic64_dec_return_relaxed(atomic64_t *v) 3404 { 3405 #if defined(arch_atomic64_dec_return_relaxed) 3406 return arch_atomic64_dec_return_relaxed(v); 3407 #elif defined(arch_atomic64_dec_return) 3408 return arch_atomic64_dec_return(v); 3409 #else 3410 return raw_atomic64_sub_return_relaxed(1, v); 3411 #endif 3412 } 3413 3414 /** 3415 * raw_atomic64_fetch_dec() - atomic decrement with full ordering 3416 * @v: pointer to atomic64_t 3417 * 3418 * Atomically updates @v to (@v - 1) with full ordering. 3419 * 3420 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere. 3421 * 3422 * Return: The original value of @v. 3423 */ 3424 static __always_inline s64 3425 raw_atomic64_fetch_dec(atomic64_t *v) 3426 { 3427 #if defined(arch_atomic64_fetch_dec) 3428 return arch_atomic64_fetch_dec(v); 3429 #elif defined(arch_atomic64_fetch_dec_relaxed) 3430 s64 ret; 3431 __atomic_pre_full_fence(); 3432 ret = arch_atomic64_fetch_dec_relaxed(v); 3433 __atomic_post_full_fence(); 3434 return ret; 3435 #else 3436 return raw_atomic64_fetch_sub(1, v); 3437 #endif 3438 } 3439 3440 /** 3441 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering 3442 * @v: pointer to atomic64_t 3443 * 3444 * Atomically updates @v to (@v - 1) with acquire ordering. 3445 * 3446 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere. 3447 * 3448 * Return: The original value of @v. 3449 */ 3450 static __always_inline s64 3451 raw_atomic64_fetch_dec_acquire(atomic64_t *v) 3452 { 3453 #if defined(arch_atomic64_fetch_dec_acquire) 3454 return arch_atomic64_fetch_dec_acquire(v); 3455 #elif defined(arch_atomic64_fetch_dec_relaxed) 3456 s64 ret = arch_atomic64_fetch_dec_relaxed(v); 3457 __atomic_acquire_fence(); 3458 return ret; 3459 #elif defined(arch_atomic64_fetch_dec) 3460 return arch_atomic64_fetch_dec(v); 3461 #else 3462 return raw_atomic64_fetch_sub_acquire(1, v); 3463 #endif 3464 } 3465 3466 /** 3467 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering 3468 * @v: pointer to atomic64_t 3469 * 3470 * Atomically updates @v to (@v - 1) with release ordering. 3471 * 3472 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere. 3473 * 3474 * Return: The original value of @v. 3475 */ 3476 static __always_inline s64 3477 raw_atomic64_fetch_dec_release(atomic64_t *v) 3478 { 3479 #if defined(arch_atomic64_fetch_dec_release) 3480 return arch_atomic64_fetch_dec_release(v); 3481 #elif defined(arch_atomic64_fetch_dec_relaxed) 3482 __atomic_release_fence(); 3483 return arch_atomic64_fetch_dec_relaxed(v); 3484 #elif defined(arch_atomic64_fetch_dec) 3485 return arch_atomic64_fetch_dec(v); 3486 #else 3487 return raw_atomic64_fetch_sub_release(1, v); 3488 #endif 3489 } 3490 3491 /** 3492 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering 3493 * @v: pointer to atomic64_t 3494 * 3495 * Atomically updates @v to (@v - 1) with relaxed ordering. 3496 * 3497 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere. 3498 * 3499 * Return: The original value of @v. 3500 */ 3501 static __always_inline s64 3502 raw_atomic64_fetch_dec_relaxed(atomic64_t *v) 3503 { 3504 #if defined(arch_atomic64_fetch_dec_relaxed) 3505 return arch_atomic64_fetch_dec_relaxed(v); 3506 #elif defined(arch_atomic64_fetch_dec) 3507 return arch_atomic64_fetch_dec(v); 3508 #else 3509 return raw_atomic64_fetch_sub_relaxed(1, v); 3510 #endif 3511 } 3512 3513 /** 3514 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering 3515 * @i: s64 value 3516 * @v: pointer to atomic64_t 3517 * 3518 * Atomically updates @v to (@v & @i) with relaxed ordering. 3519 * 3520 * Safe to use in noinstr code; prefer atomic64_and() elsewhere. 3521 * 3522 * Return: Nothing. 3523 */ 3524 static __always_inline void 3525 raw_atomic64_and(s64 i, atomic64_t *v) 3526 { 3527 arch_atomic64_and(i, v); 3528 } 3529 3530 /** 3531 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering 3532 * @i: s64 value 3533 * @v: pointer to atomic64_t 3534 * 3535 * Atomically updates @v to (@v & @i) with full ordering. 3536 * 3537 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere. 3538 * 3539 * Return: The original value of @v. 3540 */ 3541 static __always_inline s64 3542 raw_atomic64_fetch_and(s64 i, atomic64_t *v) 3543 { 3544 #if defined(arch_atomic64_fetch_and) 3545 return arch_atomic64_fetch_and(i, v); 3546 #elif defined(arch_atomic64_fetch_and_relaxed) 3547 s64 ret; 3548 __atomic_pre_full_fence(); 3549 ret = arch_atomic64_fetch_and_relaxed(i, v); 3550 __atomic_post_full_fence(); 3551 return ret; 3552 #else 3553 #error "Unable to define raw_atomic64_fetch_and" 3554 #endif 3555 } 3556 3557 /** 3558 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering 3559 * @i: s64 value 3560 * @v: pointer to atomic64_t 3561 * 3562 * Atomically updates @v to (@v & @i) with acquire ordering. 3563 * 3564 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere. 3565 * 3566 * Return: The original value of @v. 3567 */ 3568 static __always_inline s64 3569 raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 3570 { 3571 #if defined(arch_atomic64_fetch_and_acquire) 3572 return arch_atomic64_fetch_and_acquire(i, v); 3573 #elif defined(arch_atomic64_fetch_and_relaxed) 3574 s64 ret = arch_atomic64_fetch_and_relaxed(i, v); 3575 __atomic_acquire_fence(); 3576 return ret; 3577 #elif defined(arch_atomic64_fetch_and) 3578 return arch_atomic64_fetch_and(i, v); 3579 #else 3580 #error "Unable to define raw_atomic64_fetch_and_acquire" 3581 #endif 3582 } 3583 3584 /** 3585 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering 3586 * @i: s64 value 3587 * @v: pointer to atomic64_t 3588 * 3589 * Atomically updates @v to (@v & @i) with release ordering. 3590 * 3591 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere. 3592 * 3593 * Return: The original value of @v. 3594 */ 3595 static __always_inline s64 3596 raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) 3597 { 3598 #if defined(arch_atomic64_fetch_and_release) 3599 return arch_atomic64_fetch_and_release(i, v); 3600 #elif defined(arch_atomic64_fetch_and_relaxed) 3601 __atomic_release_fence(); 3602 return arch_atomic64_fetch_and_relaxed(i, v); 3603 #elif defined(arch_atomic64_fetch_and) 3604 return arch_atomic64_fetch_and(i, v); 3605 #else 3606 #error "Unable to define raw_atomic64_fetch_and_release" 3607 #endif 3608 } 3609 3610 /** 3611 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering 3612 * @i: s64 value 3613 * @v: pointer to atomic64_t 3614 * 3615 * Atomically updates @v to (@v & @i) with relaxed ordering. 3616 * 3617 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere. 3618 * 3619 * Return: The original value of @v. 3620 */ 3621 static __always_inline s64 3622 raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 3623 { 3624 #if defined(arch_atomic64_fetch_and_relaxed) 3625 return arch_atomic64_fetch_and_relaxed(i, v); 3626 #elif defined(arch_atomic64_fetch_and) 3627 return arch_atomic64_fetch_and(i, v); 3628 #else 3629 #error "Unable to define raw_atomic64_fetch_and_relaxed" 3630 #endif 3631 } 3632 3633 /** 3634 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering 3635 * @i: s64 value 3636 * @v: pointer to atomic64_t 3637 * 3638 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 3639 * 3640 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere. 3641 * 3642 * Return: Nothing. 3643 */ 3644 static __always_inline void 3645 raw_atomic64_andnot(s64 i, atomic64_t *v) 3646 { 3647 #if defined(arch_atomic64_andnot) 3648 arch_atomic64_andnot(i, v); 3649 #else 3650 raw_atomic64_and(~i, v); 3651 #endif 3652 } 3653 3654 /** 3655 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering 3656 * @i: s64 value 3657 * @v: pointer to atomic64_t 3658 * 3659 * Atomically updates @v to (@v & ~@i) with full ordering. 3660 * 3661 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere. 3662 * 3663 * Return: The original value of @v. 3664 */ 3665 static __always_inline s64 3666 raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) 3667 { 3668 #if defined(arch_atomic64_fetch_andnot) 3669 return arch_atomic64_fetch_andnot(i, v); 3670 #elif defined(arch_atomic64_fetch_andnot_relaxed) 3671 s64 ret; 3672 __atomic_pre_full_fence(); 3673 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 3674 __atomic_post_full_fence(); 3675 return ret; 3676 #else 3677 return raw_atomic64_fetch_and(~i, v); 3678 #endif 3679 } 3680 3681 /** 3682 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering 3683 * @i: s64 value 3684 * @v: pointer to atomic64_t 3685 * 3686 * Atomically updates @v to (@v & ~@i) with acquire ordering. 3687 * 3688 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere. 3689 * 3690 * Return: The original value of @v. 3691 */ 3692 static __always_inline s64 3693 raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 3694 { 3695 #if defined(arch_atomic64_fetch_andnot_acquire) 3696 return arch_atomic64_fetch_andnot_acquire(i, v); 3697 #elif defined(arch_atomic64_fetch_andnot_relaxed) 3698 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 3699 __atomic_acquire_fence(); 3700 return ret; 3701 #elif defined(arch_atomic64_fetch_andnot) 3702 return arch_atomic64_fetch_andnot(i, v); 3703 #else 3704 return raw_atomic64_fetch_and_acquire(~i, v); 3705 #endif 3706 } 3707 3708 /** 3709 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering 3710 * @i: s64 value 3711 * @v: pointer to atomic64_t 3712 * 3713 * Atomically updates @v to (@v & ~@i) with release ordering. 3714 * 3715 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere. 3716 * 3717 * Return: The original value of @v. 3718 */ 3719 static __always_inline s64 3720 raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 3721 { 3722 #if defined(arch_atomic64_fetch_andnot_release) 3723 return arch_atomic64_fetch_andnot_release(i, v); 3724 #elif defined(arch_atomic64_fetch_andnot_relaxed) 3725 __atomic_release_fence(); 3726 return arch_atomic64_fetch_andnot_relaxed(i, v); 3727 #elif defined(arch_atomic64_fetch_andnot) 3728 return arch_atomic64_fetch_andnot(i, v); 3729 #else 3730 return raw_atomic64_fetch_and_release(~i, v); 3731 #endif 3732 } 3733 3734 /** 3735 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering 3736 * @i: s64 value 3737 * @v: pointer to atomic64_t 3738 * 3739 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 3740 * 3741 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere. 3742 * 3743 * Return: The original value of @v. 3744 */ 3745 static __always_inline s64 3746 raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 3747 { 3748 #if defined(arch_atomic64_fetch_andnot_relaxed) 3749 return arch_atomic64_fetch_andnot_relaxed(i, v); 3750 #elif defined(arch_atomic64_fetch_andnot) 3751 return arch_atomic64_fetch_andnot(i, v); 3752 #else 3753 return raw_atomic64_fetch_and_relaxed(~i, v); 3754 #endif 3755 } 3756 3757 /** 3758 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering 3759 * @i: s64 value 3760 * @v: pointer to atomic64_t 3761 * 3762 * Atomically updates @v to (@v | @i) with relaxed ordering. 3763 * 3764 * Safe to use in noinstr code; prefer atomic64_or() elsewhere. 3765 * 3766 * Return: Nothing. 3767 */ 3768 static __always_inline void 3769 raw_atomic64_or(s64 i, atomic64_t *v) 3770 { 3771 arch_atomic64_or(i, v); 3772 } 3773 3774 /** 3775 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering 3776 * @i: s64 value 3777 * @v: pointer to atomic64_t 3778 * 3779 * Atomically updates @v to (@v | @i) with full ordering. 3780 * 3781 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere. 3782 * 3783 * Return: The original value of @v. 3784 */ 3785 static __always_inline s64 3786 raw_atomic64_fetch_or(s64 i, atomic64_t *v) 3787 { 3788 #if defined(arch_atomic64_fetch_or) 3789 return arch_atomic64_fetch_or(i, v); 3790 #elif defined(arch_atomic64_fetch_or_relaxed) 3791 s64 ret; 3792 __atomic_pre_full_fence(); 3793 ret = arch_atomic64_fetch_or_relaxed(i, v); 3794 __atomic_post_full_fence(); 3795 return ret; 3796 #else 3797 #error "Unable to define raw_atomic64_fetch_or" 3798 #endif 3799 } 3800 3801 /** 3802 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering 3803 * @i: s64 value 3804 * @v: pointer to atomic64_t 3805 * 3806 * Atomically updates @v to (@v | @i) with acquire ordering. 3807 * 3808 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere. 3809 * 3810 * Return: The original value of @v. 3811 */ 3812 static __always_inline s64 3813 raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 3814 { 3815 #if defined(arch_atomic64_fetch_or_acquire) 3816 return arch_atomic64_fetch_or_acquire(i, v); 3817 #elif defined(arch_atomic64_fetch_or_relaxed) 3818 s64 ret = arch_atomic64_fetch_or_relaxed(i, v); 3819 __atomic_acquire_fence(); 3820 return ret; 3821 #elif defined(arch_atomic64_fetch_or) 3822 return arch_atomic64_fetch_or(i, v); 3823 #else 3824 #error "Unable to define raw_atomic64_fetch_or_acquire" 3825 #endif 3826 } 3827 3828 /** 3829 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering 3830 * @i: s64 value 3831 * @v: pointer to atomic64_t 3832 * 3833 * Atomically updates @v to (@v | @i) with release ordering. 3834 * 3835 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere. 3836 * 3837 * Return: The original value of @v. 3838 */ 3839 static __always_inline s64 3840 raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) 3841 { 3842 #if defined(arch_atomic64_fetch_or_release) 3843 return arch_atomic64_fetch_or_release(i, v); 3844 #elif defined(arch_atomic64_fetch_or_relaxed) 3845 __atomic_release_fence(); 3846 return arch_atomic64_fetch_or_relaxed(i, v); 3847 #elif defined(arch_atomic64_fetch_or) 3848 return arch_atomic64_fetch_or(i, v); 3849 #else 3850 #error "Unable to define raw_atomic64_fetch_or_release" 3851 #endif 3852 } 3853 3854 /** 3855 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering 3856 * @i: s64 value 3857 * @v: pointer to atomic64_t 3858 * 3859 * Atomically updates @v to (@v | @i) with relaxed ordering. 3860 * 3861 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere. 3862 * 3863 * Return: The original value of @v. 3864 */ 3865 static __always_inline s64 3866 raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 3867 { 3868 #if defined(arch_atomic64_fetch_or_relaxed) 3869 return arch_atomic64_fetch_or_relaxed(i, v); 3870 #elif defined(arch_atomic64_fetch_or) 3871 return arch_atomic64_fetch_or(i, v); 3872 #else 3873 #error "Unable to define raw_atomic64_fetch_or_relaxed" 3874 #endif 3875 } 3876 3877 /** 3878 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering 3879 * @i: s64 value 3880 * @v: pointer to atomic64_t 3881 * 3882 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 3883 * 3884 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere. 3885 * 3886 * Return: Nothing. 3887 */ 3888 static __always_inline void 3889 raw_atomic64_xor(s64 i, atomic64_t *v) 3890 { 3891 arch_atomic64_xor(i, v); 3892 } 3893 3894 /** 3895 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering 3896 * @i: s64 value 3897 * @v: pointer to atomic64_t 3898 * 3899 * Atomically updates @v to (@v ^ @i) with full ordering. 3900 * 3901 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere. 3902 * 3903 * Return: The original value of @v. 3904 */ 3905 static __always_inline s64 3906 raw_atomic64_fetch_xor(s64 i, atomic64_t *v) 3907 { 3908 #if defined(arch_atomic64_fetch_xor) 3909 return arch_atomic64_fetch_xor(i, v); 3910 #elif defined(arch_atomic64_fetch_xor_relaxed) 3911 s64 ret; 3912 __atomic_pre_full_fence(); 3913 ret = arch_atomic64_fetch_xor_relaxed(i, v); 3914 __atomic_post_full_fence(); 3915 return ret; 3916 #else 3917 #error "Unable to define raw_atomic64_fetch_xor" 3918 #endif 3919 } 3920 3921 /** 3922 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering 3923 * @i: s64 value 3924 * @v: pointer to atomic64_t 3925 * 3926 * Atomically updates @v to (@v ^ @i) with acquire ordering. 3927 * 3928 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere. 3929 * 3930 * Return: The original value of @v. 3931 */ 3932 static __always_inline s64 3933 raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 3934 { 3935 #if defined(arch_atomic64_fetch_xor_acquire) 3936 return arch_atomic64_fetch_xor_acquire(i, v); 3937 #elif defined(arch_atomic64_fetch_xor_relaxed) 3938 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); 3939 __atomic_acquire_fence(); 3940 return ret; 3941 #elif defined(arch_atomic64_fetch_xor) 3942 return arch_atomic64_fetch_xor(i, v); 3943 #else 3944 #error "Unable to define raw_atomic64_fetch_xor_acquire" 3945 #endif 3946 } 3947 3948 /** 3949 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering 3950 * @i: s64 value 3951 * @v: pointer to atomic64_t 3952 * 3953 * Atomically updates @v to (@v ^ @i) with release ordering. 3954 * 3955 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere. 3956 * 3957 * Return: The original value of @v. 3958 */ 3959 static __always_inline s64 3960 raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 3961 { 3962 #if defined(arch_atomic64_fetch_xor_release) 3963 return arch_atomic64_fetch_xor_release(i, v); 3964 #elif defined(arch_atomic64_fetch_xor_relaxed) 3965 __atomic_release_fence(); 3966 return arch_atomic64_fetch_xor_relaxed(i, v); 3967 #elif defined(arch_atomic64_fetch_xor) 3968 return arch_atomic64_fetch_xor(i, v); 3969 #else 3970 #error "Unable to define raw_atomic64_fetch_xor_release" 3971 #endif 3972 } 3973 3974 /** 3975 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering 3976 * @i: s64 value 3977 * @v: pointer to atomic64_t 3978 * 3979 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 3980 * 3981 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere. 3982 * 3983 * Return: The original value of @v. 3984 */ 3985 static __always_inline s64 3986 raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 3987 { 3988 #if defined(arch_atomic64_fetch_xor_relaxed) 3989 return arch_atomic64_fetch_xor_relaxed(i, v); 3990 #elif defined(arch_atomic64_fetch_xor) 3991 return arch_atomic64_fetch_xor(i, v); 3992 #else 3993 #error "Unable to define raw_atomic64_fetch_xor_relaxed" 3994 #endif 3995 } 3996 3997 /** 3998 * raw_atomic64_xchg() - atomic exchange with full ordering 3999 * @v: pointer to atomic64_t 4000 * @new: s64 value to assign 4001 * 4002 * Atomically updates @v to @new with full ordering. 4003 * 4004 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere. 4005 * 4006 * Return: The original value of @v. 4007 */ 4008 static __always_inline s64 4009 raw_atomic64_xchg(atomic64_t *v, s64 new) 4010 { 4011 #if defined(arch_atomic64_xchg) 4012 return arch_atomic64_xchg(v, new); 4013 #elif defined(arch_atomic64_xchg_relaxed) 4014 s64 ret; 4015 __atomic_pre_full_fence(); 4016 ret = arch_atomic64_xchg_relaxed(v, new); 4017 __atomic_post_full_fence(); 4018 return ret; 4019 #else 4020 return raw_xchg(&v->counter, new); 4021 #endif 4022 } 4023 4024 /** 4025 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering 4026 * @v: pointer to atomic64_t 4027 * @new: s64 value to assign 4028 * 4029 * Atomically updates @v to @new with acquire ordering. 4030 * 4031 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere. 4032 * 4033 * Return: The original value of @v. 4034 */ 4035 static __always_inline s64 4036 raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) 4037 { 4038 #if defined(arch_atomic64_xchg_acquire) 4039 return arch_atomic64_xchg_acquire(v, new); 4040 #elif defined(arch_atomic64_xchg_relaxed) 4041 s64 ret = arch_atomic64_xchg_relaxed(v, new); 4042 __atomic_acquire_fence(); 4043 return ret; 4044 #elif defined(arch_atomic64_xchg) 4045 return arch_atomic64_xchg(v, new); 4046 #else 4047 return raw_xchg_acquire(&v->counter, new); 4048 #endif 4049 } 4050 4051 /** 4052 * raw_atomic64_xchg_release() - atomic exchange with release ordering 4053 * @v: pointer to atomic64_t 4054 * @new: s64 value to assign 4055 * 4056 * Atomically updates @v to @new with release ordering. 4057 * 4058 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere. 4059 * 4060 * Return: The original value of @v. 4061 */ 4062 static __always_inline s64 4063 raw_atomic64_xchg_release(atomic64_t *v, s64 new) 4064 { 4065 #if defined(arch_atomic64_xchg_release) 4066 return arch_atomic64_xchg_release(v, new); 4067 #elif defined(arch_atomic64_xchg_relaxed) 4068 __atomic_release_fence(); 4069 return arch_atomic64_xchg_relaxed(v, new); 4070 #elif defined(arch_atomic64_xchg) 4071 return arch_atomic64_xchg(v, new); 4072 #else 4073 return raw_xchg_release(&v->counter, new); 4074 #endif 4075 } 4076 4077 /** 4078 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering 4079 * @v: pointer to atomic64_t 4080 * @new: s64 value to assign 4081 * 4082 * Atomically updates @v to @new with relaxed ordering. 4083 * 4084 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere. 4085 * 4086 * Return: The original value of @v. 4087 */ 4088 static __always_inline s64 4089 raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) 4090 { 4091 #if defined(arch_atomic64_xchg_relaxed) 4092 return arch_atomic64_xchg_relaxed(v, new); 4093 #elif defined(arch_atomic64_xchg) 4094 return arch_atomic64_xchg(v, new); 4095 #else 4096 return raw_xchg_relaxed(&v->counter, new); 4097 #endif 4098 } 4099 4100 /** 4101 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering 4102 * @v: pointer to atomic64_t 4103 * @old: s64 value to compare with 4104 * @new: s64 value to assign 4105 * 4106 * If (@v == @old), atomically updates @v to @new with full ordering. 4107 * 4108 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere. 4109 * 4110 * Return: The original value of @v. 4111 */ 4112 static __always_inline s64 4113 raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 4114 { 4115 #if defined(arch_atomic64_cmpxchg) 4116 return arch_atomic64_cmpxchg(v, old, new); 4117 #elif defined(arch_atomic64_cmpxchg_relaxed) 4118 s64 ret; 4119 __atomic_pre_full_fence(); 4120 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 4121 __atomic_post_full_fence(); 4122 return ret; 4123 #else 4124 return raw_cmpxchg(&v->counter, old, new); 4125 #endif 4126 } 4127 4128 /** 4129 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 4130 * @v: pointer to atomic64_t 4131 * @old: s64 value to compare with 4132 * @new: s64 value to assign 4133 * 4134 * If (@v == @old), atomically updates @v to @new with acquire ordering. 4135 * 4136 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere. 4137 * 4138 * Return: The original value of @v. 4139 */ 4140 static __always_inline s64 4141 raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 4142 { 4143 #if defined(arch_atomic64_cmpxchg_acquire) 4144 return arch_atomic64_cmpxchg_acquire(v, old, new); 4145 #elif defined(arch_atomic64_cmpxchg_relaxed) 4146 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 4147 __atomic_acquire_fence(); 4148 return ret; 4149 #elif defined(arch_atomic64_cmpxchg) 4150 return arch_atomic64_cmpxchg(v, old, new); 4151 #else 4152 return raw_cmpxchg_acquire(&v->counter, old, new); 4153 #endif 4154 } 4155 4156 /** 4157 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering 4158 * @v: pointer to atomic64_t 4159 * @old: s64 value to compare with 4160 * @new: s64 value to assign 4161 * 4162 * If (@v == @old), atomically updates @v to @new with release ordering. 4163 * 4164 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere. 4165 * 4166 * Return: The original value of @v. 4167 */ 4168 static __always_inline s64 4169 raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 4170 { 4171 #if defined(arch_atomic64_cmpxchg_release) 4172 return arch_atomic64_cmpxchg_release(v, old, new); 4173 #elif defined(arch_atomic64_cmpxchg_relaxed) 4174 __atomic_release_fence(); 4175 return arch_atomic64_cmpxchg_relaxed(v, old, new); 4176 #elif defined(arch_atomic64_cmpxchg) 4177 return arch_atomic64_cmpxchg(v, old, new); 4178 #else 4179 return raw_cmpxchg_release(&v->counter, old, new); 4180 #endif 4181 } 4182 4183 /** 4184 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 4185 * @v: pointer to atomic64_t 4186 * @old: s64 value to compare with 4187 * @new: s64 value to assign 4188 * 4189 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 4190 * 4191 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere. 4192 * 4193 * Return: The original value of @v. 4194 */ 4195 static __always_inline s64 4196 raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 4197 { 4198 #if defined(arch_atomic64_cmpxchg_relaxed) 4199 return arch_atomic64_cmpxchg_relaxed(v, old, new); 4200 #elif defined(arch_atomic64_cmpxchg) 4201 return arch_atomic64_cmpxchg(v, old, new); 4202 #else 4203 return raw_cmpxchg_relaxed(&v->counter, old, new); 4204 #endif 4205 } 4206 4207 /** 4208 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering 4209 * @v: pointer to atomic64_t 4210 * @old: pointer to s64 value to compare with 4211 * @new: s64 value to assign 4212 * 4213 * If (@v == @old), atomically updates @v to @new with full ordering. 4214 * Otherwise, updates @old to the current value of @v. 4215 * 4216 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere. 4217 * 4218 * Return: @true if the exchange occured, @false otherwise. 4219 */ 4220 static __always_inline bool 4221 raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 4222 { 4223 #if defined(arch_atomic64_try_cmpxchg) 4224 return arch_atomic64_try_cmpxchg(v, old, new); 4225 #elif defined(arch_atomic64_try_cmpxchg_relaxed) 4226 bool ret; 4227 __atomic_pre_full_fence(); 4228 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4229 __atomic_post_full_fence(); 4230 return ret; 4231 #else 4232 s64 r, o = *old; 4233 r = raw_atomic64_cmpxchg(v, o, new); 4234 if (unlikely(r != o)) 4235 *old = r; 4236 return likely(r == o); 4237 #endif 4238 } 4239 4240 /** 4241 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 4242 * @v: pointer to atomic64_t 4243 * @old: pointer to s64 value to compare with 4244 * @new: s64 value to assign 4245 * 4246 * If (@v == @old), atomically updates @v to @new with acquire ordering. 4247 * Otherwise, updates @old to the current value of @v. 4248 * 4249 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere. 4250 * 4251 * Return: @true if the exchange occured, @false otherwise. 4252 */ 4253 static __always_inline bool 4254 raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 4255 { 4256 #if defined(arch_atomic64_try_cmpxchg_acquire) 4257 return arch_atomic64_try_cmpxchg_acquire(v, old, new); 4258 #elif defined(arch_atomic64_try_cmpxchg_relaxed) 4259 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4260 __atomic_acquire_fence(); 4261 return ret; 4262 #elif defined(arch_atomic64_try_cmpxchg) 4263 return arch_atomic64_try_cmpxchg(v, old, new); 4264 #else 4265 s64 r, o = *old; 4266 r = raw_atomic64_cmpxchg_acquire(v, o, new); 4267 if (unlikely(r != o)) 4268 *old = r; 4269 return likely(r == o); 4270 #endif 4271 } 4272 4273 /** 4274 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering 4275 * @v: pointer to atomic64_t 4276 * @old: pointer to s64 value to compare with 4277 * @new: s64 value to assign 4278 * 4279 * If (@v == @old), atomically updates @v to @new with release ordering. 4280 * Otherwise, updates @old to the current value of @v. 4281 * 4282 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere. 4283 * 4284 * Return: @true if the exchange occured, @false otherwise. 4285 */ 4286 static __always_inline bool 4287 raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 4288 { 4289 #if defined(arch_atomic64_try_cmpxchg_release) 4290 return arch_atomic64_try_cmpxchg_release(v, old, new); 4291 #elif defined(arch_atomic64_try_cmpxchg_relaxed) 4292 __atomic_release_fence(); 4293 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4294 #elif defined(arch_atomic64_try_cmpxchg) 4295 return arch_atomic64_try_cmpxchg(v, old, new); 4296 #else 4297 s64 r, o = *old; 4298 r = raw_atomic64_cmpxchg_release(v, o, new); 4299 if (unlikely(r != o)) 4300 *old = r; 4301 return likely(r == o); 4302 #endif 4303 } 4304 4305 /** 4306 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 4307 * @v: pointer to atomic64_t 4308 * @old: pointer to s64 value to compare with 4309 * @new: s64 value to assign 4310 * 4311 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 4312 * Otherwise, updates @old to the current value of @v. 4313 * 4314 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere. 4315 * 4316 * Return: @true if the exchange occured, @false otherwise. 4317 */ 4318 static __always_inline bool 4319 raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 4320 { 4321 #if defined(arch_atomic64_try_cmpxchg_relaxed) 4322 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4323 #elif defined(arch_atomic64_try_cmpxchg) 4324 return arch_atomic64_try_cmpxchg(v, old, new); 4325 #else 4326 s64 r, o = *old; 4327 r = raw_atomic64_cmpxchg_relaxed(v, o, new); 4328 if (unlikely(r != o)) 4329 *old = r; 4330 return likely(r == o); 4331 #endif 4332 } 4333 4334 /** 4335 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering 4336 * @i: s64 value to add 4337 * @v: pointer to atomic64_t 4338 * 4339 * Atomically updates @v to (@v - @i) with full ordering. 4340 * 4341 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere. 4342 * 4343 * Return: @true if the resulting value of @v is zero, @false otherwise. 4344 */ 4345 static __always_inline bool 4346 raw_atomic64_sub_and_test(s64 i, atomic64_t *v) 4347 { 4348 #if defined(arch_atomic64_sub_and_test) 4349 return arch_atomic64_sub_and_test(i, v); 4350 #else 4351 return raw_atomic64_sub_return(i, v) == 0; 4352 #endif 4353 } 4354 4355 /** 4356 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering 4357 * @v: pointer to atomic64_t 4358 * 4359 * Atomically updates @v to (@v - 1) with full ordering. 4360 * 4361 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere. 4362 * 4363 * Return: @true if the resulting value of @v is zero, @false otherwise. 4364 */ 4365 static __always_inline bool 4366 raw_atomic64_dec_and_test(atomic64_t *v) 4367 { 4368 #if defined(arch_atomic64_dec_and_test) 4369 return arch_atomic64_dec_and_test(v); 4370 #else 4371 return raw_atomic64_dec_return(v) == 0; 4372 #endif 4373 } 4374 4375 /** 4376 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering 4377 * @v: pointer to atomic64_t 4378 * 4379 * Atomically updates @v to (@v + 1) with full ordering. 4380 * 4381 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere. 4382 * 4383 * Return: @true if the resulting value of @v is zero, @false otherwise. 4384 */ 4385 static __always_inline bool 4386 raw_atomic64_inc_and_test(atomic64_t *v) 4387 { 4388 #if defined(arch_atomic64_inc_and_test) 4389 return arch_atomic64_inc_and_test(v); 4390 #else 4391 return raw_atomic64_inc_return(v) == 0; 4392 #endif 4393 } 4394 4395 /** 4396 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering 4397 * @i: s64 value to add 4398 * @v: pointer to atomic64_t 4399 * 4400 * Atomically updates @v to (@v + @i) with full ordering. 4401 * 4402 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere. 4403 * 4404 * Return: @true if the resulting value of @v is negative, @false otherwise. 4405 */ 4406 static __always_inline bool 4407 raw_atomic64_add_negative(s64 i, atomic64_t *v) 4408 { 4409 #if defined(arch_atomic64_add_negative) 4410 return arch_atomic64_add_negative(i, v); 4411 #elif defined(arch_atomic64_add_negative_relaxed) 4412 bool ret; 4413 __atomic_pre_full_fence(); 4414 ret = arch_atomic64_add_negative_relaxed(i, v); 4415 __atomic_post_full_fence(); 4416 return ret; 4417 #else 4418 return raw_atomic64_add_return(i, v) < 0; 4419 #endif 4420 } 4421 4422 /** 4423 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering 4424 * @i: s64 value to add 4425 * @v: pointer to atomic64_t 4426 * 4427 * Atomically updates @v to (@v + @i) with acquire ordering. 4428 * 4429 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere. 4430 * 4431 * Return: @true if the resulting value of @v is negative, @false otherwise. 4432 */ 4433 static __always_inline bool 4434 raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 4435 { 4436 #if defined(arch_atomic64_add_negative_acquire) 4437 return arch_atomic64_add_negative_acquire(i, v); 4438 #elif defined(arch_atomic64_add_negative_relaxed) 4439 bool ret = arch_atomic64_add_negative_relaxed(i, v); 4440 __atomic_acquire_fence(); 4441 return ret; 4442 #elif defined(arch_atomic64_add_negative) 4443 return arch_atomic64_add_negative(i, v); 4444 #else 4445 return raw_atomic64_add_return_acquire(i, v) < 0; 4446 #endif 4447 } 4448 4449 /** 4450 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering 4451 * @i: s64 value to add 4452 * @v: pointer to atomic64_t 4453 * 4454 * Atomically updates @v to (@v + @i) with release ordering. 4455 * 4456 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere. 4457 * 4458 * Return: @true if the resulting value of @v is negative, @false otherwise. 4459 */ 4460 static __always_inline bool 4461 raw_atomic64_add_negative_release(s64 i, atomic64_t *v) 4462 { 4463 #if defined(arch_atomic64_add_negative_release) 4464 return arch_atomic64_add_negative_release(i, v); 4465 #elif defined(arch_atomic64_add_negative_relaxed) 4466 __atomic_release_fence(); 4467 return arch_atomic64_add_negative_relaxed(i, v); 4468 #elif defined(arch_atomic64_add_negative) 4469 return arch_atomic64_add_negative(i, v); 4470 #else 4471 return raw_atomic64_add_return_release(i, v) < 0; 4472 #endif 4473 } 4474 4475 /** 4476 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering 4477 * @i: s64 value to add 4478 * @v: pointer to atomic64_t 4479 * 4480 * Atomically updates @v to (@v + @i) with relaxed ordering. 4481 * 4482 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere. 4483 * 4484 * Return: @true if the resulting value of @v is negative, @false otherwise. 4485 */ 4486 static __always_inline bool 4487 raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 4488 { 4489 #if defined(arch_atomic64_add_negative_relaxed) 4490 return arch_atomic64_add_negative_relaxed(i, v); 4491 #elif defined(arch_atomic64_add_negative) 4492 return arch_atomic64_add_negative(i, v); 4493 #else 4494 return raw_atomic64_add_return_relaxed(i, v) < 0; 4495 #endif 4496 } 4497 4498 /** 4499 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering 4500 * @v: pointer to atomic64_t 4501 * @a: s64 value to add 4502 * @u: s64 value to compare with 4503 * 4504 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 4505 * 4506 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere. 4507 * 4508 * Return: The original value of @v. 4509 */ 4510 static __always_inline s64 4511 raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 4512 { 4513 #if defined(arch_atomic64_fetch_add_unless) 4514 return arch_atomic64_fetch_add_unless(v, a, u); 4515 #else 4516 s64 c = raw_atomic64_read(v); 4517 4518 do { 4519 if (unlikely(c == u)) 4520 break; 4521 } while (!raw_atomic64_try_cmpxchg(v, &c, c + a)); 4522 4523 return c; 4524 #endif 4525 } 4526 4527 /** 4528 * raw_atomic64_add_unless() - atomic add unless value with full ordering 4529 * @v: pointer to atomic64_t 4530 * @a: s64 value to add 4531 * @u: s64 value to compare with 4532 * 4533 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 4534 * 4535 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere. 4536 * 4537 * Return: @true if @v was updated, @false otherwise. 4538 */ 4539 static __always_inline bool 4540 raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 4541 { 4542 #if defined(arch_atomic64_add_unless) 4543 return arch_atomic64_add_unless(v, a, u); 4544 #else 4545 return raw_atomic64_fetch_add_unless(v, a, u) != u; 4546 #endif 4547 } 4548 4549 /** 4550 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering 4551 * @v: pointer to atomic64_t 4552 * 4553 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. 4554 * 4555 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere. 4556 * 4557 * Return: @true if @v was updated, @false otherwise. 4558 */ 4559 static __always_inline bool 4560 raw_atomic64_inc_not_zero(atomic64_t *v) 4561 { 4562 #if defined(arch_atomic64_inc_not_zero) 4563 return arch_atomic64_inc_not_zero(v); 4564 #else 4565 return raw_atomic64_add_unless(v, 1, 0); 4566 #endif 4567 } 4568 4569 /** 4570 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering 4571 * @v: pointer to atomic64_t 4572 * 4573 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. 4574 * 4575 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere. 4576 * 4577 * Return: @true if @v was updated, @false otherwise. 4578 */ 4579 static __always_inline bool 4580 raw_atomic64_inc_unless_negative(atomic64_t *v) 4581 { 4582 #if defined(arch_atomic64_inc_unless_negative) 4583 return arch_atomic64_inc_unless_negative(v); 4584 #else 4585 s64 c = raw_atomic64_read(v); 4586 4587 do { 4588 if (unlikely(c < 0)) 4589 return false; 4590 } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1)); 4591 4592 return true; 4593 #endif 4594 } 4595 4596 /** 4597 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering 4598 * @v: pointer to atomic64_t 4599 * 4600 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. 4601 * 4602 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere. 4603 * 4604 * Return: @true if @v was updated, @false otherwise. 4605 */ 4606 static __always_inline bool 4607 raw_atomic64_dec_unless_positive(atomic64_t *v) 4608 { 4609 #if defined(arch_atomic64_dec_unless_positive) 4610 return arch_atomic64_dec_unless_positive(v); 4611 #else 4612 s64 c = raw_atomic64_read(v); 4613 4614 do { 4615 if (unlikely(c > 0)) 4616 return false; 4617 } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1)); 4618 4619 return true; 4620 #endif 4621 } 4622 4623 /** 4624 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering 4625 * @v: pointer to atomic64_t 4626 * 4627 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. 4628 * 4629 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere. 4630 * 4631 * Return: The old value of (@v - 1), regardless of whether @v was updated. 4632 */ 4633 static __always_inline s64 4634 raw_atomic64_dec_if_positive(atomic64_t *v) 4635 { 4636 #if defined(arch_atomic64_dec_if_positive) 4637 return arch_atomic64_dec_if_positive(v); 4638 #else 4639 s64 dec, c = raw_atomic64_read(v); 4640 4641 do { 4642 dec = c - 1; 4643 if (unlikely(dec < 0)) 4644 break; 4645 } while (!raw_atomic64_try_cmpxchg(v, &c, dec)); 4646 4647 return dec; 4648 #endif 4649 } 4650 4651 #endif /* _LINUX_ATOMIC_FALLBACK_H */ 4652 // 2fdd6702823fa842f9cea57a002e6e4476ae780c 4653