1 /* Atomic operations usable in machine independent code */ 2 #ifndef _LINUX_ATOMIC_H 3 #define _LINUX_ATOMIC_H 4 #include <asm/atomic.h> 5 #include <asm/barrier.h> 6 7 /* 8 * Relaxed variants of xchg, cmpxchg and some atomic operations. 9 * 10 * We support four variants: 11 * 12 * - Fully ordered: The default implementation, no suffix required. 13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix. 14 * - Release: Provides RELEASE semantics, _release suffix. 15 * - Relaxed: No ordering guarantees, _relaxed suffix. 16 * 17 * For compound atomics performing both a load and a store, ACQUIRE 18 * semantics apply only to the load and RELEASE semantics only to the 19 * store portion of the operation. Note that a failed cmpxchg_acquire 20 * does -not- imply any memory ordering constraints. 21 * 22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. 23 */ 24 25 #ifndef atomic_read_acquire 26 #define atomic_read_acquire(v) smp_load_acquire(&(v)->counter) 27 #endif 28 29 #ifndef atomic_set_release 30 #define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i)) 31 #endif 32 33 /* 34 * The idea here is to build acquire/release variants by adding explicit 35 * barriers on top of the relaxed variant. In the case where the relaxed 36 * variant is already fully ordered, no additional barriers are needed. 37 */ 38 #define __atomic_op_acquire(op, args...) \ 39 ({ \ 40 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \ 41 smp_mb__after_atomic(); \ 42 __ret; \ 43 }) 44 45 #define __atomic_op_release(op, args...) \ 46 ({ \ 47 smp_mb__before_atomic(); \ 48 op##_relaxed(args); \ 49 }) 50 51 #define __atomic_op_fence(op, args...) \ 52 ({ \ 53 typeof(op##_relaxed(args)) __ret; \ 54 smp_mb__before_atomic(); \ 55 __ret = op##_relaxed(args); \ 56 smp_mb__after_atomic(); \ 57 __ret; \ 58 }) 59 60 /* atomic_add_return_relaxed */ 61 #ifndef atomic_add_return_relaxed 62 #define atomic_add_return_relaxed atomic_add_return 63 #define atomic_add_return_acquire atomic_add_return 64 #define atomic_add_return_release atomic_add_return 65 66 #else /* atomic_add_return_relaxed */ 67 68 #ifndef atomic_add_return_acquire 69 #define atomic_add_return_acquire(...) \ 70 __atomic_op_acquire(atomic_add_return, __VA_ARGS__) 71 #endif 72 73 #ifndef atomic_add_return_release 74 #define atomic_add_return_release(...) \ 75 __atomic_op_release(atomic_add_return, __VA_ARGS__) 76 #endif 77 78 #ifndef atomic_add_return 79 #define atomic_add_return(...) \ 80 __atomic_op_fence(atomic_add_return, __VA_ARGS__) 81 #endif 82 #endif /* atomic_add_return_relaxed */ 83 84 /* atomic_sub_return_relaxed */ 85 #ifndef atomic_sub_return_relaxed 86 #define atomic_sub_return_relaxed atomic_sub_return 87 #define atomic_sub_return_acquire atomic_sub_return 88 #define atomic_sub_return_release atomic_sub_return 89 90 #else /* atomic_sub_return_relaxed */ 91 92 #ifndef atomic_sub_return_acquire 93 #define atomic_sub_return_acquire(...) \ 94 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__) 95 #endif 96 97 #ifndef atomic_sub_return_release 98 #define atomic_sub_return_release(...) \ 99 __atomic_op_release(atomic_sub_return, __VA_ARGS__) 100 #endif 101 102 #ifndef atomic_sub_return 103 #define atomic_sub_return(...) \ 104 __atomic_op_fence(atomic_sub_return, __VA_ARGS__) 105 #endif 106 #endif /* atomic_sub_return_relaxed */ 107 108 /* atomic_xchg_relaxed */ 109 #ifndef atomic_xchg_relaxed 110 #define atomic_xchg_relaxed atomic_xchg 111 #define atomic_xchg_acquire atomic_xchg 112 #define atomic_xchg_release atomic_xchg 113 114 #else /* atomic_xchg_relaxed */ 115 116 #ifndef atomic_xchg_acquire 117 #define atomic_xchg_acquire(...) \ 118 __atomic_op_acquire(atomic_xchg, __VA_ARGS__) 119 #endif 120 121 #ifndef atomic_xchg_release 122 #define atomic_xchg_release(...) \ 123 __atomic_op_release(atomic_xchg, __VA_ARGS__) 124 #endif 125 126 #ifndef atomic_xchg 127 #define atomic_xchg(...) \ 128 __atomic_op_fence(atomic_xchg, __VA_ARGS__) 129 #endif 130 #endif /* atomic_xchg_relaxed */ 131 132 /* atomic_cmpxchg_relaxed */ 133 #ifndef atomic_cmpxchg_relaxed 134 #define atomic_cmpxchg_relaxed atomic_cmpxchg 135 #define atomic_cmpxchg_acquire atomic_cmpxchg 136 #define atomic_cmpxchg_release atomic_cmpxchg 137 138 #else /* atomic_cmpxchg_relaxed */ 139 140 #ifndef atomic_cmpxchg_acquire 141 #define atomic_cmpxchg_acquire(...) \ 142 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__) 143 #endif 144 145 #ifndef atomic_cmpxchg_release 146 #define atomic_cmpxchg_release(...) \ 147 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__) 148 #endif 149 150 #ifndef atomic_cmpxchg 151 #define atomic_cmpxchg(...) \ 152 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__) 153 #endif 154 #endif /* atomic_cmpxchg_relaxed */ 155 156 #ifndef atomic64_read_acquire 157 #define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter) 158 #endif 159 160 #ifndef atomic64_set_release 161 #define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i)) 162 #endif 163 164 /* atomic64_add_return_relaxed */ 165 #ifndef atomic64_add_return_relaxed 166 #define atomic64_add_return_relaxed atomic64_add_return 167 #define atomic64_add_return_acquire atomic64_add_return 168 #define atomic64_add_return_release atomic64_add_return 169 170 #else /* atomic64_add_return_relaxed */ 171 172 #ifndef atomic64_add_return_acquire 173 #define atomic64_add_return_acquire(...) \ 174 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__) 175 #endif 176 177 #ifndef atomic64_add_return_release 178 #define atomic64_add_return_release(...) \ 179 __atomic_op_release(atomic64_add_return, __VA_ARGS__) 180 #endif 181 182 #ifndef atomic64_add_return 183 #define atomic64_add_return(...) \ 184 __atomic_op_fence(atomic64_add_return, __VA_ARGS__) 185 #endif 186 #endif /* atomic64_add_return_relaxed */ 187 188 /* atomic64_sub_return_relaxed */ 189 #ifndef atomic64_sub_return_relaxed 190 #define atomic64_sub_return_relaxed atomic64_sub_return 191 #define atomic64_sub_return_acquire atomic64_sub_return 192 #define atomic64_sub_return_release atomic64_sub_return 193 194 #else /* atomic64_sub_return_relaxed */ 195 196 #ifndef atomic64_sub_return_acquire 197 #define atomic64_sub_return_acquire(...) \ 198 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__) 199 #endif 200 201 #ifndef atomic64_sub_return_release 202 #define atomic64_sub_return_release(...) \ 203 __atomic_op_release(atomic64_sub_return, __VA_ARGS__) 204 #endif 205 206 #ifndef atomic64_sub_return 207 #define atomic64_sub_return(...) \ 208 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__) 209 #endif 210 #endif /* atomic64_sub_return_relaxed */ 211 212 /* atomic64_xchg_relaxed */ 213 #ifndef atomic64_xchg_relaxed 214 #define atomic64_xchg_relaxed atomic64_xchg 215 #define atomic64_xchg_acquire atomic64_xchg 216 #define atomic64_xchg_release atomic64_xchg 217 218 #else /* atomic64_xchg_relaxed */ 219 220 #ifndef atomic64_xchg_acquire 221 #define atomic64_xchg_acquire(...) \ 222 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__) 223 #endif 224 225 #ifndef atomic64_xchg_release 226 #define atomic64_xchg_release(...) \ 227 __atomic_op_release(atomic64_xchg, __VA_ARGS__) 228 #endif 229 230 #ifndef atomic64_xchg 231 #define atomic64_xchg(...) \ 232 __atomic_op_fence(atomic64_xchg, __VA_ARGS__) 233 #endif 234 #endif /* atomic64_xchg_relaxed */ 235 236 /* atomic64_cmpxchg_relaxed */ 237 #ifndef atomic64_cmpxchg_relaxed 238 #define atomic64_cmpxchg_relaxed atomic64_cmpxchg 239 #define atomic64_cmpxchg_acquire atomic64_cmpxchg 240 #define atomic64_cmpxchg_release atomic64_cmpxchg 241 242 #else /* atomic64_cmpxchg_relaxed */ 243 244 #ifndef atomic64_cmpxchg_acquire 245 #define atomic64_cmpxchg_acquire(...) \ 246 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__) 247 #endif 248 249 #ifndef atomic64_cmpxchg_release 250 #define atomic64_cmpxchg_release(...) \ 251 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__) 252 #endif 253 254 #ifndef atomic64_cmpxchg 255 #define atomic64_cmpxchg(...) \ 256 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__) 257 #endif 258 #endif /* atomic64_cmpxchg_relaxed */ 259 260 /* cmpxchg_relaxed */ 261 #ifndef cmpxchg_relaxed 262 #define cmpxchg_relaxed cmpxchg 263 #define cmpxchg_acquire cmpxchg 264 #define cmpxchg_release cmpxchg 265 266 #else /* cmpxchg_relaxed */ 267 268 #ifndef cmpxchg_acquire 269 #define cmpxchg_acquire(...) \ 270 __atomic_op_acquire(cmpxchg, __VA_ARGS__) 271 #endif 272 273 #ifndef cmpxchg_release 274 #define cmpxchg_release(...) \ 275 __atomic_op_release(cmpxchg, __VA_ARGS__) 276 #endif 277 278 #ifndef cmpxchg 279 #define cmpxchg(...) \ 280 __atomic_op_fence(cmpxchg, __VA_ARGS__) 281 #endif 282 #endif /* cmpxchg_relaxed */ 283 284 /* cmpxchg64_relaxed */ 285 #ifndef cmpxchg64_relaxed 286 #define cmpxchg64_relaxed cmpxchg64 287 #define cmpxchg64_acquire cmpxchg64 288 #define cmpxchg64_release cmpxchg64 289 290 #else /* cmpxchg64_relaxed */ 291 292 #ifndef cmpxchg64_acquire 293 #define cmpxchg64_acquire(...) \ 294 __atomic_op_acquire(cmpxchg64, __VA_ARGS__) 295 #endif 296 297 #ifndef cmpxchg64_release 298 #define cmpxchg64_release(...) \ 299 __atomic_op_release(cmpxchg64, __VA_ARGS__) 300 #endif 301 302 #ifndef cmpxchg64 303 #define cmpxchg64(...) \ 304 __atomic_op_fence(cmpxchg64, __VA_ARGS__) 305 #endif 306 #endif /* cmpxchg64_relaxed */ 307 308 /* xchg_relaxed */ 309 #ifndef xchg_relaxed 310 #define xchg_relaxed xchg 311 #define xchg_acquire xchg 312 #define xchg_release xchg 313 314 #else /* xchg_relaxed */ 315 316 #ifndef xchg_acquire 317 #define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__) 318 #endif 319 320 #ifndef xchg_release 321 #define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__) 322 #endif 323 324 #ifndef xchg 325 #define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__) 326 #endif 327 #endif /* xchg_relaxed */ 328 329 /** 330 * atomic_add_unless - add unless the number is already a given value 331 * @v: pointer of type atomic_t 332 * @a: the amount to add to v... 333 * @u: ...unless v is equal to u. 334 * 335 * Atomically adds @a to @v, so long as @v was not already @u. 336 * Returns non-zero if @v was not @u, and zero otherwise. 337 */ 338 static inline int atomic_add_unless(atomic_t *v, int a, int u) 339 { 340 return __atomic_add_unless(v, a, u) != u; 341 } 342 343 /** 344 * atomic_inc_not_zero - increment unless the number is zero 345 * @v: pointer of type atomic_t 346 * 347 * Atomically increments @v by 1, so long as @v is non-zero. 348 * Returns non-zero if @v was non-zero, and zero otherwise. 349 */ 350 #ifndef atomic_inc_not_zero 351 #define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0) 352 #endif 353 354 #ifndef atomic_andnot 355 static inline void atomic_andnot(int i, atomic_t *v) 356 { 357 atomic_and(~i, v); 358 } 359 #endif 360 361 static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v) 362 { 363 atomic_andnot(mask, v); 364 } 365 366 static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v) 367 { 368 atomic_or(mask, v); 369 } 370 371 /** 372 * atomic_inc_not_zero_hint - increment if not null 373 * @v: pointer of type atomic_t 374 * @hint: probable value of the atomic before the increment 375 * 376 * This version of atomic_inc_not_zero() gives a hint of probable 377 * value of the atomic. This helps processor to not read the memory 378 * before doing the atomic read/modify/write cycle, lowering 379 * number of bus transactions on some arches. 380 * 381 * Returns: 0 if increment was not done, 1 otherwise. 382 */ 383 #ifndef atomic_inc_not_zero_hint 384 static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint) 385 { 386 int val, c = hint; 387 388 /* sanity test, should be removed by compiler if hint is a constant */ 389 if (!hint) 390 return atomic_inc_not_zero(v); 391 392 do { 393 val = atomic_cmpxchg(v, c, c + 1); 394 if (val == c) 395 return 1; 396 c = val; 397 } while (c); 398 399 return 0; 400 } 401 #endif 402 403 #ifndef atomic_inc_unless_negative 404 static inline int atomic_inc_unless_negative(atomic_t *p) 405 { 406 int v, v1; 407 for (v = 0; v >= 0; v = v1) { 408 v1 = atomic_cmpxchg(p, v, v + 1); 409 if (likely(v1 == v)) 410 return 1; 411 } 412 return 0; 413 } 414 #endif 415 416 #ifndef atomic_dec_unless_positive 417 static inline int atomic_dec_unless_positive(atomic_t *p) 418 { 419 int v, v1; 420 for (v = 0; v <= 0; v = v1) { 421 v1 = atomic_cmpxchg(p, v, v - 1); 422 if (likely(v1 == v)) 423 return 1; 424 } 425 return 0; 426 } 427 #endif 428 429 /* 430 * atomic_dec_if_positive - decrement by 1 if old value positive 431 * @v: pointer of type atomic_t 432 * 433 * The function returns the old value of *v minus 1, even if 434 * the atomic variable, v, was not decremented. 435 */ 436 #ifndef atomic_dec_if_positive 437 static inline int atomic_dec_if_positive(atomic_t *v) 438 { 439 int c, old, dec; 440 c = atomic_read(v); 441 for (;;) { 442 dec = c - 1; 443 if (unlikely(dec < 0)) 444 break; 445 old = atomic_cmpxchg((v), c, dec); 446 if (likely(old == c)) 447 break; 448 c = old; 449 } 450 return dec; 451 } 452 #endif 453 454 #include <asm-generic/atomic-long.h> 455 #ifdef CONFIG_GENERIC_ATOMIC64 456 #include <asm-generic/atomic64.h> 457 #endif 458 459 #ifndef atomic64_andnot 460 static inline void atomic64_andnot(long long i, atomic64_t *v) 461 { 462 atomic64_and(~i, v); 463 } 464 #endif 465 466 #endif /* _LINUX_ATOMIC_H */ 467