1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1998 Doug Rabson. 5 * Copyright (c) 2001 Jake Burkholder. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * from: FreeBSD: src/sys/i386/include/atomic.h,v 1.20 2001/02/11 30 * $FreeBSD$ 31 */ 32 33 #ifndef _MACHINE_ATOMIC_H_ 34 #define _MACHINE_ATOMIC_H_ 35 36 #include <machine/cpufunc.h> 37 38 #define mb() __asm__ __volatile__ ("membar #MemIssue": : :"memory") 39 #define wmb() mb() 40 #define rmb() mb() 41 42 #include <sys/atomic_common.h> 43 44 /* Userland needs different ASI's. */ 45 #ifdef _KERNEL 46 #define __ASI_ATOMIC ASI_N 47 #else 48 #define __ASI_ATOMIC ASI_P 49 #endif 50 51 /* 52 * Various simple arithmetic on memory which is atomic in the presence 53 * of interrupts and multiple processors. See atomic(9) for details. 54 * Note that efficient hardware support exists only for the 32 and 64 55 * bit variants; the 8 and 16 bit versions are not provided and should 56 * not be used in MI code. 57 * 58 * This implementation takes advantage of the fact that the sparc64 59 * cas instruction is both a load and a store. The loop is often coded 60 * as follows: 61 * 62 * do { 63 * expect = *p; 64 * new = expect + 1; 65 * } while (cas(p, expect, new) != expect); 66 * 67 * which performs an unnnecessary load on each iteration that the cas 68 * operation fails. Modified as follows: 69 * 70 * expect = *p; 71 * for (;;) { 72 * new = expect + 1; 73 * result = cas(p, expect, new); 74 * if (result == expect) 75 * break; 76 * expect = result; 77 * } 78 * 79 * the return value of cas is used to avoid the extra reload. 80 * 81 * We only include a memory barrier in the rel variants as in total store 82 * order which we use for running the kernel and all of the userland atomic 83 * loads and stores behave as if the were followed by a membar with a mask 84 * of #LoadLoad | #LoadStore | #StoreStore. In order to be also sufficient 85 * for use of relaxed memory ordering, the atomic_cas() in the acq variants 86 * additionally would have to be followed by a membar #LoadLoad | #LoadStore. 87 * Due to the suggested assembly syntax of the membar operands containing a 88 * # character, they cannot be used in macros. The cmask and mmask bits thus 89 * are hard coded in machine/cpufunc.h and used here through macros. 90 * Hopefully the bit numbers won't change in the future. 91 */ 92 93 #define itype(sz) uint ## sz ## _t 94 95 #define atomic_cas_32(p, e, s) casa((p), (e), (s), __ASI_ATOMIC) 96 #define atomic_cas_64(p, e, s) casxa((p), (e), (s), __ASI_ATOMIC) 97 98 #define atomic_cas(p, e, s, sz) \ 99 atomic_cas_ ## sz((p), (e), (s)) 100 101 #define atomic_cas_acq(p, e, s, sz) ({ \ 102 itype(sz) v; \ 103 v = atomic_cas((p), (e), (s), sz); \ 104 __compiler_membar(); \ 105 v; \ 106 }) 107 108 #define atomic_cas_rel(p, e, s, sz) ({ \ 109 itype(sz) v; \ 110 membar(LoadStore | StoreStore); \ 111 v = atomic_cas((p), (e), (s), sz); \ 112 v; \ 113 }) 114 115 #define atomic_op(p, op, v, sz) ({ \ 116 itype(sz) e, r, s; \ 117 for (e = *(volatile itype(sz) *)(p);; e = r) { \ 118 s = e op (v); \ 119 r = atomic_cas_ ## sz((p), e, s); \ 120 if (r == e) \ 121 break; \ 122 } \ 123 e; \ 124 }) 125 126 #define atomic_op_acq(p, op, v, sz) ({ \ 127 itype(sz) t; \ 128 t = atomic_op((p), op, (v), sz); \ 129 __compiler_membar(); \ 130 t; \ 131 }) 132 133 #define atomic_op_rel(p, op, v, sz) ({ \ 134 itype(sz) t; \ 135 membar(LoadStore | StoreStore); \ 136 t = atomic_op((p), op, (v), sz); \ 137 t; \ 138 }) 139 140 #define atomic_ld_acq(p, sz) ({ \ 141 itype(sz) v; \ 142 v = atomic_cas((p), 0, 0, sz); \ 143 __compiler_membar(); \ 144 v; \ 145 }) 146 147 #define atomic_ld_clear(p, sz) ({ \ 148 itype(sz) e, r; \ 149 for (e = *(volatile itype(sz) *)(p);; e = r) { \ 150 r = atomic_cas((p), e, 0, sz); \ 151 if (r == e) \ 152 break; \ 153 } \ 154 e; \ 155 }) 156 157 #define atomic_st(p, v, sz) ({ \ 158 itype(sz) e, r; \ 159 for (e = *(volatile itype(sz) *)(p);; e = r) { \ 160 r = atomic_cas((p), e, (v), sz); \ 161 if (r == e) \ 162 break; \ 163 } \ 164 e; \ 165 }) 166 167 #define atomic_st_acq(p, v, sz) do { \ 168 atomic_st((p), (v), sz); \ 169 __compiler_membar(); \ 170 } while (0) 171 172 #define atomic_st_rel(p, v, sz) do { \ 173 membar(LoadStore | StoreStore); \ 174 atomic_st((p), (v), sz); \ 175 } while (0) 176 177 #define ATOMIC_GEN(name, ptype, vtype, atype, sz) \ 178 \ 179 static __inline vtype \ 180 atomic_add_ ## name(volatile ptype p, atype v) \ 181 { \ 182 return ((vtype)atomic_op((p), +, (v), sz)); \ 183 } \ 184 static __inline vtype \ 185 atomic_add_acq_ ## name(volatile ptype p, atype v) \ 186 { \ 187 return ((vtype)atomic_op_acq((p), +, (v), sz)); \ 188 } \ 189 static __inline vtype \ 190 atomic_add_rel_ ## name(volatile ptype p, atype v) \ 191 { \ 192 return ((vtype)atomic_op_rel((p), +, (v), sz)); \ 193 } \ 194 \ 195 static __inline vtype \ 196 atomic_clear_ ## name(volatile ptype p, atype v) \ 197 { \ 198 return ((vtype)atomic_op((p), &, ~(v), sz)); \ 199 } \ 200 static __inline vtype \ 201 atomic_clear_acq_ ## name(volatile ptype p, atype v) \ 202 { \ 203 return ((vtype)atomic_op_acq((p), &, ~(v), sz)); \ 204 } \ 205 static __inline vtype \ 206 atomic_clear_rel_ ## name(volatile ptype p, atype v) \ 207 { \ 208 return ((vtype)atomic_op_rel((p), &, ~(v), sz)); \ 209 } \ 210 \ 211 static __inline int \ 212 atomic_cmpset_ ## name(volatile ptype p, vtype e, vtype s) \ 213 { \ 214 return (((vtype)atomic_cas((p), (e), (s), sz)) == (e)); \ 215 } \ 216 static __inline int \ 217 atomic_cmpset_acq_ ## name(volatile ptype p, vtype e, vtype s) \ 218 { \ 219 return (((vtype)atomic_cas_acq((p), (e), (s), sz)) == (e)); \ 220 } \ 221 static __inline int \ 222 atomic_cmpset_rel_ ## name(volatile ptype p, vtype e, vtype s) \ 223 { \ 224 return (((vtype)atomic_cas_rel((p), (e), (s), sz)) == (e)); \ 225 } \ 226 \ 227 static __inline int \ 228 atomic_fcmpset_ ## name(volatile ptype p, vtype *ep, vtype s) \ 229 { \ 230 vtype t; \ 231 \ 232 t = (vtype)atomic_cas((p), (*ep), (s), sz); \ 233 if (t == (*ep)) \ 234 return (1); \ 235 *ep = t; \ 236 return (0); \ 237 } \ 238 static __inline int \ 239 atomic_fcmpset_acq_ ## name(volatile ptype p, vtype *ep, vtype s) \ 240 { \ 241 vtype t; \ 242 \ 243 t = (vtype)atomic_cas_acq((p), (*ep), (s), sz); \ 244 if (t == (*ep)) \ 245 return (1); \ 246 *ep = t; \ 247 return (0); \ 248 } \ 249 static __inline int \ 250 atomic_fcmpset_rel_ ## name(volatile ptype p, vtype *ep, vtype s) \ 251 { \ 252 vtype t; \ 253 \ 254 t = (vtype)atomic_cas_rel((p), (*ep), (s), sz); \ 255 if (t == (*ep)) \ 256 return (1); \ 257 *ep = t; \ 258 return (0); \ 259 } \ 260 \ 261 static __inline vtype \ 262 atomic_load_acq_ ## name(volatile ptype p) \ 263 { \ 264 return ((vtype)atomic_cas_acq((p), 0, 0, sz)); \ 265 } \ 266 \ 267 static __inline vtype \ 268 atomic_readandclear_ ## name(volatile ptype p) \ 269 { \ 270 return ((vtype)atomic_ld_clear((p), sz)); \ 271 } \ 272 \ 273 static __inline vtype \ 274 atomic_set_ ## name(volatile ptype p, atype v) \ 275 { \ 276 return ((vtype)atomic_op((p), |, (v), sz)); \ 277 } \ 278 static __inline vtype \ 279 atomic_set_acq_ ## name(volatile ptype p, atype v) \ 280 { \ 281 return ((vtype)atomic_op_acq((p), |, (v), sz)); \ 282 } \ 283 static __inline vtype \ 284 atomic_set_rel_ ## name(volatile ptype p, atype v) \ 285 { \ 286 return ((vtype)atomic_op_rel((p), |, (v), sz)); \ 287 } \ 288 \ 289 static __inline vtype \ 290 atomic_subtract_ ## name(volatile ptype p, atype v) \ 291 { \ 292 return ((vtype)atomic_op((p), -, (v), sz)); \ 293 } \ 294 static __inline vtype \ 295 atomic_subtract_acq_ ## name(volatile ptype p, atype v) \ 296 { \ 297 return ((vtype)atomic_op_acq((p), -, (v), sz)); \ 298 } \ 299 static __inline vtype \ 300 atomic_subtract_rel_ ## name(volatile ptype p, atype v) \ 301 { \ 302 return ((vtype)atomic_op_rel((p), -, (v), sz)); \ 303 } \ 304 \ 305 static __inline void \ 306 atomic_store_acq_ ## name(volatile ptype p, vtype v) \ 307 { \ 308 atomic_st_acq((p), (v), sz); \ 309 } \ 310 static __inline void \ 311 atomic_store_rel_ ## name(volatile ptype p, vtype v) \ 312 { \ 313 atomic_st_rel((p), (v), sz); \ 314 } \ 315 \ 316 static __inline vtype \ 317 atomic_swap_ ## name(volatile ptype p, vtype v) \ 318 { \ 319 return ((vtype)atomic_st((p), (v), sz)); \ 320 } 321 322 static __inline void 323 atomic_thread_fence_acq(void) 324 { 325 326 __compiler_membar(); 327 } 328 329 static __inline void 330 atomic_thread_fence_rel(void) 331 { 332 333 __compiler_membar(); 334 } 335 336 static __inline void 337 atomic_thread_fence_acq_rel(void) 338 { 339 340 __compiler_membar(); 341 } 342 343 static __inline void 344 atomic_thread_fence_seq_cst(void) 345 { 346 347 membar(LoadLoad | LoadStore | StoreStore | StoreLoad); 348 } 349 350 351 ATOMIC_GEN(int, u_int *, u_int, u_int, 32); 352 ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32); 353 354 ATOMIC_GEN(long, u_long *, u_long, u_long, 64); 355 ATOMIC_GEN(64, uint64_t *, uint64_t, uint64_t, 64); 356 357 ATOMIC_GEN(ptr, uintptr_t *, uintptr_t, uintptr_t, 64); 358 359 #define atomic_fetchadd_int atomic_add_int 360 #define atomic_fetchadd_32 atomic_add_32 361 #define atomic_fetchadd_long atomic_add_long 362 #define atomic_fetchadd_64 atomic_add_64 363 364 #undef ATOMIC_GEN 365 #undef atomic_cas 366 #undef atomic_cas_acq 367 #undef atomic_cas_rel 368 #undef atomic_op 369 #undef atomic_op_acq 370 #undef atomic_op_rel 371 #undef atomic_ld_acq 372 #undef atomic_ld_clear 373 #undef atomic_st 374 #undef atomic_st_acq 375 #undef atomic_st_rel 376 377 #endif /* !_MACHINE_ATOMIC_H_ */ 378