1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2008 Marcel Moolenaar
5 * Copyright (c) 2001 Benno Rice
6 * Copyright (c) 2001 David E. O'Brien
7 * Copyright (c) 1998 Doug Rabson
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * $FreeBSD$
32 */
33
34 #ifndef _MACHINE_ATOMIC_H_
35 #define _MACHINE_ATOMIC_H_
36
37 #ifndef _SYS_CDEFS_H_
38 #error this file needs sys/cdefs.h as a prerequisite
39 #endif
40
41 #include <sys/atomic_common.h>
42
43 /*
44 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
45 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
46 * of this file. See also Appendix B.2 of Book II of the architecture manual.
47 *
48 * Note that not all Book-E processors accept the light-weight sync variant.
49 * In particular, early models of E500 cores are known to wedge. Bank on all
50 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
51 * to use the heavier-weight sync.
52 */
53
54 #ifdef __powerpc64__
55 #define mb() __asm __volatile("sync" : : : "memory")
56 #define rmb() __asm __volatile("lwsync" : : : "memory")
57 #define wmb() __asm __volatile("lwsync" : : : "memory")
58 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory")
59 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
60 #else
61 #define mb() __asm __volatile("sync" : : : "memory")
62 #define rmb() __asm __volatile("sync" : : : "memory")
63 #define wmb() __asm __volatile("sync" : : : "memory")
64 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory")
65 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
66 #endif
67
68 static __inline void
powerpc_lwsync(void)69 powerpc_lwsync(void)
70 {
71
72 #ifdef __powerpc64__
73 __asm __volatile("lwsync" : : : "memory");
74 #else
75 __asm __volatile("sync" : : : "memory");
76 #endif
77 }
78
79 /*
80 * atomic_add(p, v)
81 * { *p += v; }
82 */
83
84 #define __atomic_add_int(p, v, t) \
85 __asm __volatile( \
86 "1: lwarx %0, 0, %2\n" \
87 " add %0, %3, %0\n" \
88 " stwcx. %0, 0, %2\n" \
89 " bne- 1b\n" \
90 : "=&r" (t), "=m" (*p) \
91 : "r" (p), "r" (v), "m" (*p) \
92 : "cr0", "memory") \
93 /* __atomic_add_int */
94
95 #ifdef __powerpc64__
96 #define __atomic_add_long(p, v, t) \
97 __asm __volatile( \
98 "1: ldarx %0, 0, %2\n" \
99 " add %0, %3, %0\n" \
100 " stdcx. %0, 0, %2\n" \
101 " bne- 1b\n" \
102 : "=&r" (t), "=m" (*p) \
103 : "r" (p), "r" (v), "m" (*p) \
104 : "cr0", "memory") \
105 /* __atomic_add_long */
106 #else
107 #define __atomic_add_long(p, v, t) \
108 __asm __volatile( \
109 "1: lwarx %0, 0, %2\n" \
110 " add %0, %3, %0\n" \
111 " stwcx. %0, 0, %2\n" \
112 " bne- 1b\n" \
113 : "=&r" (t), "=m" (*p) \
114 : "r" (p), "r" (v), "m" (*p) \
115 : "cr0", "memory") \
116 /* __atomic_add_long */
117 #endif
118
119 #define _ATOMIC_ADD(type) \
120 static __inline void \
121 atomic_add_##type(volatile u_##type *p, u_##type v) { \
122 u_##type t; \
123 __atomic_add_##type(p, v, t); \
124 } \
125 \
126 static __inline void \
127 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \
128 u_##type t; \
129 __atomic_add_##type(p, v, t); \
130 __ATOMIC_ACQ(); \
131 } \
132 \
133 static __inline void \
134 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \
135 u_##type t; \
136 __ATOMIC_REL(); \
137 __atomic_add_##type(p, v, t); \
138 } \
139 /* _ATOMIC_ADD */
140
141 _ATOMIC_ADD(int)
_ATOMIC_ADD(long)142 _ATOMIC_ADD(long)
143
144 #define atomic_add_32 atomic_add_int
145 #define atomic_add_acq_32 atomic_add_acq_int
146 #define atomic_add_rel_32 atomic_add_rel_int
147
148 #ifdef __powerpc64__
149 #define atomic_add_64 atomic_add_long
150 #define atomic_add_acq_64 atomic_add_acq_long
151 #define atomic_add_rel_64 atomic_add_rel_long
152
153 #define atomic_add_ptr atomic_add_long
154 #define atomic_add_acq_ptr atomic_add_acq_long
155 #define atomic_add_rel_ptr atomic_add_rel_long
156 #else
157 #define atomic_add_ptr atomic_add_int
158 #define atomic_add_acq_ptr atomic_add_acq_int
159 #define atomic_add_rel_ptr atomic_add_rel_int
160 #endif
161 #undef _ATOMIC_ADD
162 #undef __atomic_add_long
163 #undef __atomic_add_int
164
165 /*
166 * atomic_clear(p, v)
167 * { *p &= ~v; }
168 */
169
170 #define __atomic_clear_int(p, v, t) \
171 __asm __volatile( \
172 "1: lwarx %0, 0, %2\n" \
173 " andc %0, %0, %3\n" \
174 " stwcx. %0, 0, %2\n" \
175 " bne- 1b\n" \
176 : "=&r" (t), "=m" (*p) \
177 : "r" (p), "r" (v), "m" (*p) \
178 : "cr0", "memory") \
179 /* __atomic_clear_int */
180
181 #ifdef __powerpc64__
182 #define __atomic_clear_long(p, v, t) \
183 __asm __volatile( \
184 "1: ldarx %0, 0, %2\n" \
185 " andc %0, %0, %3\n" \
186 " stdcx. %0, 0, %2\n" \
187 " bne- 1b\n" \
188 : "=&r" (t), "=m" (*p) \
189 : "r" (p), "r" (v), "m" (*p) \
190 : "cr0", "memory") \
191 /* __atomic_clear_long */
192 #else
193 #define __atomic_clear_long(p, v, t) \
194 __asm __volatile( \
195 "1: lwarx %0, 0, %2\n" \
196 " andc %0, %0, %3\n" \
197 " stwcx. %0, 0, %2\n" \
198 " bne- 1b\n" \
199 : "=&r" (t), "=m" (*p) \
200 : "r" (p), "r" (v), "m" (*p) \
201 : "cr0", "memory") \
202 /* __atomic_clear_long */
203 #endif
204
205 #define _ATOMIC_CLEAR(type) \
206 static __inline void \
207 atomic_clear_##type(volatile u_##type *p, u_##type v) { \
208 u_##type t; \
209 __atomic_clear_##type(p, v, t); \
210 } \
211 \
212 static __inline void \
213 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
214 u_##type t; \
215 __atomic_clear_##type(p, v, t); \
216 __ATOMIC_ACQ(); \
217 } \
218 \
219 static __inline void \
220 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
221 u_##type t; \
222 __ATOMIC_REL(); \
223 __atomic_clear_##type(p, v, t); \
224 } \
225 /* _ATOMIC_CLEAR */
226
227
228 _ATOMIC_CLEAR(int)
229 _ATOMIC_CLEAR(long)
230
231 #define atomic_clear_32 atomic_clear_int
232 #define atomic_clear_acq_32 atomic_clear_acq_int
233 #define atomic_clear_rel_32 atomic_clear_rel_int
234
235 #ifdef __powerpc64__
236 #define atomic_clear_64 atomic_clear_long
237 #define atomic_clear_acq_64 atomic_clear_acq_long
238 #define atomic_clear_rel_64 atomic_clear_rel_long
239
240 #define atomic_clear_ptr atomic_clear_long
241 #define atomic_clear_acq_ptr atomic_clear_acq_long
242 #define atomic_clear_rel_ptr atomic_clear_rel_long
243 #else
244 #define atomic_clear_ptr atomic_clear_int
245 #define atomic_clear_acq_ptr atomic_clear_acq_int
246 #define atomic_clear_rel_ptr atomic_clear_rel_int
247 #endif
248 #undef _ATOMIC_CLEAR
249 #undef __atomic_clear_long
250 #undef __atomic_clear_int
251
252 /*
253 * atomic_cmpset(p, o, n)
254 */
255 /* TODO -- see below */
256
257 /*
258 * atomic_load_acq(p)
259 */
260 /* TODO -- see below */
261
262 /*
263 * atomic_readandclear(p)
264 */
265 /* TODO -- see below */
266
267 /*
268 * atomic_set(p, v)
269 * { *p |= v; }
270 */
271
272 #define __atomic_set_int(p, v, t) \
273 __asm __volatile( \
274 "1: lwarx %0, 0, %2\n" \
275 " or %0, %3, %0\n" \
276 " stwcx. %0, 0, %2\n" \
277 " bne- 1b\n" \
278 : "=&r" (t), "=m" (*p) \
279 : "r" (p), "r" (v), "m" (*p) \
280 : "cr0", "memory") \
281 /* __atomic_set_int */
282
283 #ifdef __powerpc64__
284 #define __atomic_set_long(p, v, t) \
285 __asm __volatile( \
286 "1: ldarx %0, 0, %2\n" \
287 " or %0, %3, %0\n" \
288 " stdcx. %0, 0, %2\n" \
289 " bne- 1b\n" \
290 : "=&r" (t), "=m" (*p) \
291 : "r" (p), "r" (v), "m" (*p) \
292 : "cr0", "memory") \
293 /* __atomic_set_long */
294 #else
295 #define __atomic_set_long(p, v, t) \
296 __asm __volatile( \
297 "1: lwarx %0, 0, %2\n" \
298 " or %0, %3, %0\n" \
299 " stwcx. %0, 0, %2\n" \
300 " bne- 1b\n" \
301 : "=&r" (t), "=m" (*p) \
302 : "r" (p), "r" (v), "m" (*p) \
303 : "cr0", "memory") \
304 /* __atomic_set_long */
305 #endif
306
307 #define _ATOMIC_SET(type) \
308 static __inline void \
309 atomic_set_##type(volatile u_##type *p, u_##type v) { \
310 u_##type t; \
311 __atomic_set_##type(p, v, t); \
312 } \
313 \
314 static __inline void \
315 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \
316 u_##type t; \
317 __atomic_set_##type(p, v, t); \
318 __ATOMIC_ACQ(); \
319 } \
320 \
321 static __inline void \
322 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \
323 u_##type t; \
324 __ATOMIC_REL(); \
325 __atomic_set_##type(p, v, t); \
326 } \
327 /* _ATOMIC_SET */
328
329 _ATOMIC_SET(int)
330 _ATOMIC_SET(long)
331
332 #define atomic_set_32 atomic_set_int
333 #define atomic_set_acq_32 atomic_set_acq_int
334 #define atomic_set_rel_32 atomic_set_rel_int
335
336 #ifdef __powerpc64__
337 #define atomic_set_64 atomic_set_long
338 #define atomic_set_acq_64 atomic_set_acq_long
339 #define atomic_set_rel_64 atomic_set_rel_long
340
341 #define atomic_set_ptr atomic_set_long
342 #define atomic_set_acq_ptr atomic_set_acq_long
343 #define atomic_set_rel_ptr atomic_set_rel_long
344 #else
345 #define atomic_set_ptr atomic_set_int
346 #define atomic_set_acq_ptr atomic_set_acq_int
347 #define atomic_set_rel_ptr atomic_set_rel_int
348 #endif
349 #undef _ATOMIC_SET
350 #undef __atomic_set_long
351 #undef __atomic_set_int
352
353 /*
354 * atomic_subtract(p, v)
355 * { *p -= v; }
356 */
357
358 #define __atomic_subtract_int(p, v, t) \
359 __asm __volatile( \
360 "1: lwarx %0, 0, %2\n" \
361 " subf %0, %3, %0\n" \
362 " stwcx. %0, 0, %2\n" \
363 " bne- 1b\n" \
364 : "=&r" (t), "=m" (*p) \
365 : "r" (p), "r" (v), "m" (*p) \
366 : "cr0", "memory") \
367 /* __atomic_subtract_int */
368
369 #ifdef __powerpc64__
370 #define __atomic_subtract_long(p, v, t) \
371 __asm __volatile( \
372 "1: ldarx %0, 0, %2\n" \
373 " subf %0, %3, %0\n" \
374 " stdcx. %0, 0, %2\n" \
375 " bne- 1b\n" \
376 : "=&r" (t), "=m" (*p) \
377 : "r" (p), "r" (v), "m" (*p) \
378 : "cr0", "memory") \
379 /* __atomic_subtract_long */
380 #else
381 #define __atomic_subtract_long(p, v, t) \
382 __asm __volatile( \
383 "1: lwarx %0, 0, %2\n" \
384 " subf %0, %3, %0\n" \
385 " stwcx. %0, 0, %2\n" \
386 " bne- 1b\n" \
387 : "=&r" (t), "=m" (*p) \
388 : "r" (p), "r" (v), "m" (*p) \
389 : "cr0", "memory") \
390 /* __atomic_subtract_long */
391 #endif
392
393 #define _ATOMIC_SUBTRACT(type) \
394 static __inline void \
395 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \
396 u_##type t; \
397 __atomic_subtract_##type(p, v, t); \
398 } \
399 \
400 static __inline void \
401 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \
402 u_##type t; \
403 __atomic_subtract_##type(p, v, t); \
404 __ATOMIC_ACQ(); \
405 } \
406 \
407 static __inline void \
408 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \
409 u_##type t; \
410 __ATOMIC_REL(); \
411 __atomic_subtract_##type(p, v, t); \
412 } \
413 /* _ATOMIC_SUBTRACT */
414
415 _ATOMIC_SUBTRACT(int)
416 _ATOMIC_SUBTRACT(long)
417
418 #define atomic_subtract_32 atomic_subtract_int
419 #define atomic_subtract_acq_32 atomic_subtract_acq_int
420 #define atomic_subtract_rel_32 atomic_subtract_rel_int
421
422 #ifdef __powerpc64__
423 #define atomic_subtract_64 atomic_subtract_long
424 #define atomic_subtract_acq_64 atomic_subract_acq_long
425 #define atomic_subtract_rel_64 atomic_subtract_rel_long
426
427 #define atomic_subtract_ptr atomic_subtract_long
428 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
429 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
430 #else
431 #define atomic_subtract_ptr atomic_subtract_int
432 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
433 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
434 #endif
435 #undef _ATOMIC_SUBTRACT
436 #undef __atomic_subtract_long
437 #undef __atomic_subtract_int
438
439 /*
440 * atomic_store_rel(p, v)
441 */
442 /* TODO -- see below */
443
444 /*
445 * Old/original implementations that still need revisiting.
446 */
447
448 static __inline u_int
449 atomic_readandclear_int(volatile u_int *addr)
450 {
451 u_int result,temp;
452
453 __asm __volatile (
454 "\tsync\n" /* drain writes */
455 "1:\tlwarx %0, 0, %3\n\t" /* load old value */
456 "li %1, 0\n\t" /* load new value */
457 "stwcx. %1, 0, %3\n\t" /* attempt to store */
458 "bne- 1b\n\t" /* spin if failed */
459 : "=&r"(result), "=&r"(temp), "=m" (*addr)
460 : "r" (addr), "m" (*addr)
461 : "cr0", "memory");
462
463 return (result);
464 }
465
466 #ifdef __powerpc64__
467 static __inline u_long
atomic_readandclear_long(volatile u_long * addr)468 atomic_readandclear_long(volatile u_long *addr)
469 {
470 u_long result,temp;
471
472 __asm __volatile (
473 "\tsync\n" /* drain writes */
474 "1:\tldarx %0, 0, %3\n\t" /* load old value */
475 "li %1, 0\n\t" /* load new value */
476 "stdcx. %1, 0, %3\n\t" /* attempt to store */
477 "bne- 1b\n\t" /* spin if failed */
478 : "=&r"(result), "=&r"(temp), "=m" (*addr)
479 : "r" (addr), "m" (*addr)
480 : "cr0", "memory");
481
482 return (result);
483 }
484 #endif
485
486 #define atomic_readandclear_32 atomic_readandclear_int
487
488 #ifdef __powerpc64__
489 #define atomic_readandclear_64 atomic_readandclear_long
490
491 #define atomic_readandclear_ptr atomic_readandclear_long
492 #else
493 static __inline u_long
atomic_readandclear_long(volatile u_long * addr)494 atomic_readandclear_long(volatile u_long *addr)
495 {
496
497 return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
498 }
499
500 #define atomic_readandclear_ptr atomic_readandclear_int
501 #endif
502
503 /*
504 * We assume that a = b will do atomic loads and stores.
505 */
506 #define ATOMIC_STORE_LOAD(TYPE) \
507 static __inline u_##TYPE \
508 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
509 { \
510 u_##TYPE v; \
511 \
512 v = *p; \
513 mb(); \
514 return (v); \
515 } \
516 \
517 static __inline void \
518 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
519 { \
520 \
521 powerpc_lwsync(); \
522 *p = v; \
523 }
524
525 ATOMIC_STORE_LOAD(int)
526
527 #define atomic_load_acq_32 atomic_load_acq_int
528 #define atomic_store_rel_32 atomic_store_rel_int
529
530 #ifdef __powerpc64__
ATOMIC_STORE_LOAD(long)531 ATOMIC_STORE_LOAD(long)
532
533 #define atomic_load_acq_64 atomic_load_acq_long
534 #define atomic_store_rel_64 atomic_store_rel_long
535
536 #define atomic_load_acq_ptr atomic_load_acq_long
537 #define atomic_store_rel_ptr atomic_store_rel_long
538 #else
539 static __inline u_long
540 atomic_load_acq_long(volatile u_long *addr)
541 {
542
543 return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
544 }
545
546 static __inline void
547 atomic_store_rel_long(volatile u_long *addr, u_long val)
548 {
549
550 atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
551 }
552
553 #define atomic_load_acq_ptr atomic_load_acq_int
554 #define atomic_store_rel_ptr atomic_store_rel_int
555 #endif
556 #undef ATOMIC_STORE_LOAD
557
558 /*
559 * Atomically compare the value stored at *p with cmpval and if the
560 * two values are equal, update the value of *p with newval. Returns
561 * zero if the compare failed, nonzero otherwise.
562 */
563 static __inline int
564 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
565 {
566 int ret;
567
568 __asm __volatile (
569 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
570 "cmplw %3, %0\n\t" /* compare */
571 "bne 2f\n\t" /* exit if not equal */
572 "stwcx. %4, 0, %2\n\t" /* attempt to store */
573 "bne- 1b\n\t" /* spin if failed */
574 "li %0, 1\n\t" /* success - retval = 1 */
575 "b 3f\n\t" /* we've succeeded */
576 "2:\n\t"
577 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
578 "li %0, 0\n\t" /* failure - retval = 0 */
579 "3:\n\t"
580 : "=&r" (ret), "=m" (*p)
581 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
582 : "cr0", "memory");
583
584 return (ret);
585 }
586 static __inline int
atomic_cmpset_long(volatile u_long * p,u_long cmpval,u_long newval)587 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
588 {
589 int ret;
590
591 __asm __volatile (
592 #ifdef __powerpc64__
593 "1:\tldarx %0, 0, %2\n\t" /* load old value */
594 "cmpld %3, %0\n\t" /* compare */
595 "bne 2f\n\t" /* exit if not equal */
596 "stdcx. %4, 0, %2\n\t" /* attempt to store */
597 #else
598 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
599 "cmplw %3, %0\n\t" /* compare */
600 "bne 2f\n\t" /* exit if not equal */
601 "stwcx. %4, 0, %2\n\t" /* attempt to store */
602 #endif
603 "bne- 1b\n\t" /* spin if failed */
604 "li %0, 1\n\t" /* success - retval = 1 */
605 "b 3f\n\t" /* we've succeeded */
606 "2:\n\t"
607 #ifdef __powerpc64__
608 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
609 #else
610 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
611 #endif
612 "li %0, 0\n\t" /* failure - retval = 0 */
613 "3:\n\t"
614 : "=&r" (ret), "=m" (*p)
615 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
616 : "cr0", "memory");
617
618 return (ret);
619 }
620
621 static __inline int
atomic_cmpset_acq_int(volatile u_int * p,u_int cmpval,u_int newval)622 atomic_cmpset_acq_int(volatile u_int *p, u_int cmpval, u_int newval)
623 {
624 int retval;
625
626 retval = atomic_cmpset_int(p, cmpval, newval);
627 __ATOMIC_ACQ();
628 return (retval);
629 }
630
631 static __inline int
atomic_cmpset_rel_int(volatile u_int * p,u_int cmpval,u_int newval)632 atomic_cmpset_rel_int(volatile u_int *p, u_int cmpval, u_int newval)
633 {
634 __ATOMIC_REL();
635 return (atomic_cmpset_int(p, cmpval, newval));
636 }
637
638 static __inline int
atomic_cmpset_acq_long(volatile u_long * p,u_long cmpval,u_long newval)639 atomic_cmpset_acq_long(volatile u_long *p, u_long cmpval, u_long newval)
640 {
641 u_long retval;
642
643 retval = atomic_cmpset_long(p, cmpval, newval);
644 __ATOMIC_ACQ();
645 return (retval);
646 }
647
648 static __inline int
atomic_cmpset_rel_long(volatile u_long * p,u_long cmpval,u_long newval)649 atomic_cmpset_rel_long(volatile u_long *p, u_long cmpval, u_long newval)
650 {
651 __ATOMIC_REL();
652 return (atomic_cmpset_long(p, cmpval, newval));
653 }
654
655 #define atomic_cmpset_32 atomic_cmpset_int
656 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int
657 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int
658
659 #ifdef __powerpc64__
660 #define atomic_cmpset_64 atomic_cmpset_long
661 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long
662 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long
663
664 #define atomic_cmpset_ptr atomic_cmpset_long
665 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
666 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
667 #else
668 #define atomic_cmpset_ptr atomic_cmpset_int
669 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int
670 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int
671 #endif
672
673 /*
674 * Atomically compare the value stored at *p with *cmpval and if the
675 * two values are equal, update the value of *p with newval. Returns
676 * zero if the compare failed and sets *cmpval to the read value from *p,
677 * nonzero otherwise.
678 */
679 static __inline int
atomic_fcmpset_int(volatile u_int * p,u_int * cmpval,u_int newval)680 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
681 {
682 int ret;
683
684 __asm __volatile (
685 "lwarx %0, 0, %3\n\t" /* load old value */
686 "cmplw %4, %0\n\t" /* compare */
687 "bne 1f\n\t" /* exit if not equal */
688 "stwcx. %5, 0, %3\n\t" /* attempt to store */
689 "bne- 1f\n\t" /* exit if failed */
690 "li %0, 1\n\t" /* success - retval = 1 */
691 "b 2f\n\t" /* we've succeeded */
692 "1:\n\t"
693 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
694 "stwx %0, 0, %7\n\t"
695 "li %0, 0\n\t" /* failure - retval = 0 */
696 "2:\n\t"
697 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
698 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
699 : "cr0", "memory");
700
701 return (ret);
702 }
703 static __inline int
atomic_fcmpset_long(volatile u_long * p,u_long * cmpval,u_long newval)704 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
705 {
706 int ret;
707
708 __asm __volatile (
709 #ifdef __powerpc64__
710 "ldarx %0, 0, %3\n\t" /* load old value */
711 "cmpld %4, %0\n\t" /* compare */
712 "bne 1f\n\t" /* exit if not equal */
713 "stdcx. %5, 0, %3\n\t" /* attempt to store */
714 #else
715 "lwarx %0, 0, %3\n\t" /* load old value */
716 "cmplw %4, %0\n\t" /* compare */
717 "bne 1f\n\t" /* exit if not equal */
718 "stwcx. %5, 0, %3\n\t" /* attempt to store */
719 #endif
720 "bne- 1f\n\t" /* exit if failed */
721 "li %0, 1\n\t" /* success - retval = 1 */
722 "b 2f\n\t" /* we've succeeded */
723 "1:\n\t"
724 #ifdef __powerpc64__
725 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
726 "stdx %0, 0, %7\n\t"
727 #else
728 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
729 "stwx %0, 0, %7\n\t"
730 #endif
731 "li %0, 0\n\t" /* failure - retval = 0 */
732 "2:\n\t"
733 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
734 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
735 : "cr0", "memory");
736
737 return (ret);
738 }
739
740 static __inline int
atomic_fcmpset_acq_int(volatile u_int * p,u_int * cmpval,u_int newval)741 atomic_fcmpset_acq_int(volatile u_int *p, u_int *cmpval, u_int newval)
742 {
743 int retval;
744
745 retval = atomic_fcmpset_int(p, cmpval, newval);
746 __ATOMIC_ACQ();
747 return (retval);
748 }
749
750 static __inline int
atomic_fcmpset_rel_int(volatile u_int * p,u_int * cmpval,u_int newval)751 atomic_fcmpset_rel_int(volatile u_int *p, u_int *cmpval, u_int newval)
752 {
753 __ATOMIC_REL();
754 return (atomic_fcmpset_int(p, cmpval, newval));
755 }
756
757 static __inline int
atomic_fcmpset_acq_long(volatile u_long * p,u_long * cmpval,u_long newval)758 atomic_fcmpset_acq_long(volatile u_long *p, u_long *cmpval, u_long newval)
759 {
760 u_long retval;
761
762 retval = atomic_fcmpset_long(p, cmpval, newval);
763 __ATOMIC_ACQ();
764 return (retval);
765 }
766
767 static __inline int
atomic_fcmpset_rel_long(volatile u_long * p,u_long * cmpval,u_long newval)768 atomic_fcmpset_rel_long(volatile u_long *p, u_long *cmpval, u_long newval)
769 {
770 __ATOMIC_REL();
771 return (atomic_fcmpset_long(p, cmpval, newval));
772 }
773
774 #define atomic_fcmpset_32 atomic_fcmpset_int
775 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
776 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
777
778 #ifdef __powerpc64__
779 #define atomic_fcmpset_64 atomic_fcmpset_long
780 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long
781 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long
782
783 #define atomic_fcmpset_ptr atomic_fcmpset_long
784 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long
785 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long
786 #else
787 #define atomic_fcmpset_ptr atomic_fcmpset_int
788 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int
789 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int
790 #endif
791
792 static __inline u_int
atomic_fetchadd_int(volatile u_int * p,u_int v)793 atomic_fetchadd_int(volatile u_int *p, u_int v)
794 {
795 u_int value;
796
797 do {
798 value = *p;
799 } while (!atomic_cmpset_int(p, value, value + v));
800 return (value);
801 }
802
803 static __inline u_long
atomic_fetchadd_long(volatile u_long * p,u_long v)804 atomic_fetchadd_long(volatile u_long *p, u_long v)
805 {
806 u_long value;
807
808 do {
809 value = *p;
810 } while (!atomic_cmpset_long(p, value, value + v));
811 return (value);
812 }
813
814 static __inline u_int
atomic_swap_32(volatile u_int * p,u_int v)815 atomic_swap_32(volatile u_int *p, u_int v)
816 {
817 u_int prev;
818
819 __asm __volatile(
820 "1: lwarx %0,0,%2\n"
821 " stwcx. %3,0,%2\n"
822 " bne- 1b\n"
823 : "=&r" (prev), "+m" (*(volatile u_int *)p)
824 : "r" (p), "r" (v)
825 : "cr0", "memory");
826
827 return (prev);
828 }
829
830 #ifdef __powerpc64__
831 static __inline u_long
atomic_swap_64(volatile u_long * p,u_long v)832 atomic_swap_64(volatile u_long *p, u_long v)
833 {
834 u_long prev;
835
836 __asm __volatile(
837 "1: ldarx %0,0,%2\n"
838 " stdcx. %3,0,%2\n"
839 " bne- 1b\n"
840 : "=&r" (prev), "+m" (*(volatile u_long *)p)
841 : "r" (p), "r" (v)
842 : "cr0", "memory");
843
844 return (prev);
845 }
846 #endif
847
848 #define atomic_fetchadd_32 atomic_fetchadd_int
849 #define atomic_swap_int atomic_swap_32
850
851 #ifdef __powerpc64__
852 #define atomic_fetchadd_64 atomic_fetchadd_long
853 #define atomic_swap_long atomic_swap_64
854 #define atomic_swap_ptr atomic_swap_64
855 #endif
856
857 #undef __ATOMIC_REL
858 #undef __ATOMIC_ACQ
859
860 static __inline void
atomic_thread_fence_acq(void)861 atomic_thread_fence_acq(void)
862 {
863
864 powerpc_lwsync();
865 }
866
867 static __inline void
atomic_thread_fence_rel(void)868 atomic_thread_fence_rel(void)
869 {
870
871 powerpc_lwsync();
872 }
873
874 static __inline void
atomic_thread_fence_acq_rel(void)875 atomic_thread_fence_acq_rel(void)
876 {
877
878 powerpc_lwsync();
879 }
880
881 static __inline void
atomic_thread_fence_seq_cst(void)882 atomic_thread_fence_seq_cst(void)
883 {
884
885 __asm __volatile("sync" : : : "memory");
886 }
887
888 #endif /* ! _MACHINE_ATOMIC_H_ */
889