1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2008 Marcel Moolenaar
5 * Copyright (c) 2001 Benno Rice
6 * Copyright (c) 2001 David E. O'Brien
7 * Copyright (c) 1998 Doug Rabson
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 *
31 * $FreeBSD$
32 */
33
34 #ifndef _MACHINE_ATOMIC_H_
35 #define _MACHINE_ATOMIC_H_
36
37 #ifndef _SYS_CDEFS_H_
38 #error this file needs sys/cdefs.h as a prerequisite
39 #endif
40
41 #include <sys/atomic_common.h>
42
43 #ifndef __powerpc64__
44 #include <sys/_atomic64e.h>
45 #endif
46
47 /*
48 * The __ATOMIC_REL/ACQ() macros provide memory barriers only in conjunction
49 * with the atomic lXarx/stXcx. sequences below. They are not exposed outside
50 * of this file. See also Appendix B.2 of Book II of the architecture manual.
51 *
52 * Note that not all Book-E processors accept the light-weight sync variant.
53 * In particular, early models of E500 cores are known to wedge. Bank on all
54 * 64-bit capable CPUs to accept lwsync properly and pressimize 32-bit CPUs
55 * to use the heavier-weight sync.
56 */
57
58 #ifdef __powerpc64__
59 #define mb() __asm __volatile("sync" : : : "memory")
60 #define rmb() __asm __volatile("lwsync" : : : "memory")
61 #define wmb() __asm __volatile("lwsync" : : : "memory")
62 #define __ATOMIC_REL() __asm __volatile("lwsync" : : : "memory")
63 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
64 #else
65 #define mb() __asm __volatile("sync" : : : "memory")
66 #define rmb() __asm __volatile("sync" : : : "memory")
67 #define wmb() __asm __volatile("sync" : : : "memory")
68 #define __ATOMIC_REL() __asm __volatile("sync" : : : "memory")
69 #define __ATOMIC_ACQ() __asm __volatile("isync" : : : "memory")
70 #endif
71
72 static __inline void
powerpc_lwsync(void)73 powerpc_lwsync(void)
74 {
75
76 #ifdef __powerpc64__
77 __asm __volatile("lwsync" : : : "memory");
78 #else
79 __asm __volatile("sync" : : : "memory");
80 #endif
81 }
82
83 /*
84 * atomic_add(p, v)
85 * { *p += v; }
86 */
87
88 #define __atomic_add_int(p, v, t) \
89 __asm __volatile( \
90 "1: lwarx %0, 0, %2\n" \
91 " add %0, %3, %0\n" \
92 " stwcx. %0, 0, %2\n" \
93 " bne- 1b\n" \
94 : "=&r" (t), "=m" (*p) \
95 : "r" (p), "r" (v), "m" (*p) \
96 : "cr0", "memory") \
97 /* __atomic_add_int */
98
99 #ifdef __powerpc64__
100 #define __atomic_add_long(p, v, t) \
101 __asm __volatile( \
102 "1: ldarx %0, 0, %2\n" \
103 " add %0, %3, %0\n" \
104 " stdcx. %0, 0, %2\n" \
105 " bne- 1b\n" \
106 : "=&r" (t), "=m" (*p) \
107 : "r" (p), "r" (v), "m" (*p) \
108 : "cr0", "memory") \
109 /* __atomic_add_long */
110 #else
111 #define __atomic_add_long(p, v, t) \
112 __asm __volatile( \
113 "1: lwarx %0, 0, %2\n" \
114 " add %0, %3, %0\n" \
115 " stwcx. %0, 0, %2\n" \
116 " bne- 1b\n" \
117 : "=&r" (t), "=m" (*p) \
118 : "r" (p), "r" (v), "m" (*p) \
119 : "cr0", "memory") \
120 /* __atomic_add_long */
121 #endif
122
123 #define _ATOMIC_ADD(type) \
124 static __inline void \
125 atomic_add_##type(volatile u_##type *p, u_##type v) { \
126 u_##type t; \
127 __atomic_add_##type(p, v, t); \
128 } \
129 \
130 static __inline void \
131 atomic_add_acq_##type(volatile u_##type *p, u_##type v) { \
132 u_##type t; \
133 __atomic_add_##type(p, v, t); \
134 __ATOMIC_ACQ(); \
135 } \
136 \
137 static __inline void \
138 atomic_add_rel_##type(volatile u_##type *p, u_##type v) { \
139 u_##type t; \
140 __ATOMIC_REL(); \
141 __atomic_add_##type(p, v, t); \
142 } \
143 /* _ATOMIC_ADD */
144
145 _ATOMIC_ADD(int)
_ATOMIC_ADD(long)146 _ATOMIC_ADD(long)
147
148 #define atomic_add_32 atomic_add_int
149 #define atomic_add_acq_32 atomic_add_acq_int
150 #define atomic_add_rel_32 atomic_add_rel_int
151
152 #ifdef __powerpc64__
153 #define atomic_add_64 atomic_add_long
154 #define atomic_add_acq_64 atomic_add_acq_long
155 #define atomic_add_rel_64 atomic_add_rel_long
156
157 #define atomic_add_ptr atomic_add_long
158 #define atomic_add_acq_ptr atomic_add_acq_long
159 #define atomic_add_rel_ptr atomic_add_rel_long
160 #else
161 #define atomic_add_ptr atomic_add_int
162 #define atomic_add_acq_ptr atomic_add_acq_int
163 #define atomic_add_rel_ptr atomic_add_rel_int
164 #endif
165 #undef _ATOMIC_ADD
166 #undef __atomic_add_long
167 #undef __atomic_add_int
168
169 /*
170 * atomic_clear(p, v)
171 * { *p &= ~v; }
172 */
173
174 #define __atomic_clear_int(p, v, t) \
175 __asm __volatile( \
176 "1: lwarx %0, 0, %2\n" \
177 " andc %0, %0, %3\n" \
178 " stwcx. %0, 0, %2\n" \
179 " bne- 1b\n" \
180 : "=&r" (t), "=m" (*p) \
181 : "r" (p), "r" (v), "m" (*p) \
182 : "cr0", "memory") \
183 /* __atomic_clear_int */
184
185 #ifdef __powerpc64__
186 #define __atomic_clear_long(p, v, t) \
187 __asm __volatile( \
188 "1: ldarx %0, 0, %2\n" \
189 " andc %0, %0, %3\n" \
190 " stdcx. %0, 0, %2\n" \
191 " bne- 1b\n" \
192 : "=&r" (t), "=m" (*p) \
193 : "r" (p), "r" (v), "m" (*p) \
194 : "cr0", "memory") \
195 /* __atomic_clear_long */
196 #else
197 #define __atomic_clear_long(p, v, t) \
198 __asm __volatile( \
199 "1: lwarx %0, 0, %2\n" \
200 " andc %0, %0, %3\n" \
201 " stwcx. %0, 0, %2\n" \
202 " bne- 1b\n" \
203 : "=&r" (t), "=m" (*p) \
204 : "r" (p), "r" (v), "m" (*p) \
205 : "cr0", "memory") \
206 /* __atomic_clear_long */
207 #endif
208
209 #define _ATOMIC_CLEAR(type) \
210 static __inline void \
211 atomic_clear_##type(volatile u_##type *p, u_##type v) { \
212 u_##type t; \
213 __atomic_clear_##type(p, v, t); \
214 } \
215 \
216 static __inline void \
217 atomic_clear_acq_##type(volatile u_##type *p, u_##type v) { \
218 u_##type t; \
219 __atomic_clear_##type(p, v, t); \
220 __ATOMIC_ACQ(); \
221 } \
222 \
223 static __inline void \
224 atomic_clear_rel_##type(volatile u_##type *p, u_##type v) { \
225 u_##type t; \
226 __ATOMIC_REL(); \
227 __atomic_clear_##type(p, v, t); \
228 } \
229 /* _ATOMIC_CLEAR */
230
231 _ATOMIC_CLEAR(int)
232 _ATOMIC_CLEAR(long)
233
234 #define atomic_clear_32 atomic_clear_int
235 #define atomic_clear_acq_32 atomic_clear_acq_int
236 #define atomic_clear_rel_32 atomic_clear_rel_int
237
238 #ifdef __powerpc64__
239 #define atomic_clear_64 atomic_clear_long
240 #define atomic_clear_acq_64 atomic_clear_acq_long
241 #define atomic_clear_rel_64 atomic_clear_rel_long
242
243 #define atomic_clear_ptr atomic_clear_long
244 #define atomic_clear_acq_ptr atomic_clear_acq_long
245 #define atomic_clear_rel_ptr atomic_clear_rel_long
246 #else
247 #define atomic_clear_ptr atomic_clear_int
248 #define atomic_clear_acq_ptr atomic_clear_acq_int
249 #define atomic_clear_rel_ptr atomic_clear_rel_int
250 #endif
251 #undef _ATOMIC_CLEAR
252 #undef __atomic_clear_long
253 #undef __atomic_clear_int
254
255 /*
256 * atomic_cmpset(p, o, n)
257 */
258 /* TODO -- see below */
259
260 /*
261 * atomic_load_acq(p)
262 */
263 /* TODO -- see below */
264
265 /*
266 * atomic_readandclear(p)
267 */
268 /* TODO -- see below */
269
270 /*
271 * atomic_set(p, v)
272 * { *p |= v; }
273 */
274
275 #define __atomic_set_int(p, v, t) \
276 __asm __volatile( \
277 "1: lwarx %0, 0, %2\n" \
278 " or %0, %3, %0\n" \
279 " stwcx. %0, 0, %2\n" \
280 " bne- 1b\n" \
281 : "=&r" (t), "=m" (*p) \
282 : "r" (p), "r" (v), "m" (*p) \
283 : "cr0", "memory") \
284 /* __atomic_set_int */
285
286 #ifdef __powerpc64__
287 #define __atomic_set_long(p, v, t) \
288 __asm __volatile( \
289 "1: ldarx %0, 0, %2\n" \
290 " or %0, %3, %0\n" \
291 " stdcx. %0, 0, %2\n" \
292 " bne- 1b\n" \
293 : "=&r" (t), "=m" (*p) \
294 : "r" (p), "r" (v), "m" (*p) \
295 : "cr0", "memory") \
296 /* __atomic_set_long */
297 #else
298 #define __atomic_set_long(p, v, t) \
299 __asm __volatile( \
300 "1: lwarx %0, 0, %2\n" \
301 " or %0, %3, %0\n" \
302 " stwcx. %0, 0, %2\n" \
303 " bne- 1b\n" \
304 : "=&r" (t), "=m" (*p) \
305 : "r" (p), "r" (v), "m" (*p) \
306 : "cr0", "memory") \
307 /* __atomic_set_long */
308 #endif
309
310 #define _ATOMIC_SET(type) \
311 static __inline void \
312 atomic_set_##type(volatile u_##type *p, u_##type v) { \
313 u_##type t; \
314 __atomic_set_##type(p, v, t); \
315 } \
316 \
317 static __inline void \
318 atomic_set_acq_##type(volatile u_##type *p, u_##type v) { \
319 u_##type t; \
320 __atomic_set_##type(p, v, t); \
321 __ATOMIC_ACQ(); \
322 } \
323 \
324 static __inline void \
325 atomic_set_rel_##type(volatile u_##type *p, u_##type v) { \
326 u_##type t; \
327 __ATOMIC_REL(); \
328 __atomic_set_##type(p, v, t); \
329 } \
330 /* _ATOMIC_SET */
331
332 _ATOMIC_SET(int)
333 _ATOMIC_SET(long)
334
335 #define atomic_set_32 atomic_set_int
336 #define atomic_set_acq_32 atomic_set_acq_int
337 #define atomic_set_rel_32 atomic_set_rel_int
338
339 #ifdef __powerpc64__
340 #define atomic_set_64 atomic_set_long
341 #define atomic_set_acq_64 atomic_set_acq_long
342 #define atomic_set_rel_64 atomic_set_rel_long
343
344 #define atomic_set_ptr atomic_set_long
345 #define atomic_set_acq_ptr atomic_set_acq_long
346 #define atomic_set_rel_ptr atomic_set_rel_long
347 #else
348 #define atomic_set_ptr atomic_set_int
349 #define atomic_set_acq_ptr atomic_set_acq_int
350 #define atomic_set_rel_ptr atomic_set_rel_int
351 #endif
352 #undef _ATOMIC_SET
353 #undef __atomic_set_long
354 #undef __atomic_set_int
355
356 /*
357 * atomic_subtract(p, v)
358 * { *p -= v; }
359 */
360
361 #define __atomic_subtract_int(p, v, t) \
362 __asm __volatile( \
363 "1: lwarx %0, 0, %2\n" \
364 " subf %0, %3, %0\n" \
365 " stwcx. %0, 0, %2\n" \
366 " bne- 1b\n" \
367 : "=&r" (t), "=m" (*p) \
368 : "r" (p), "r" (v), "m" (*p) \
369 : "cr0", "memory") \
370 /* __atomic_subtract_int */
371
372 #ifdef __powerpc64__
373 #define __atomic_subtract_long(p, v, t) \
374 __asm __volatile( \
375 "1: ldarx %0, 0, %2\n" \
376 " subf %0, %3, %0\n" \
377 " stdcx. %0, 0, %2\n" \
378 " bne- 1b\n" \
379 : "=&r" (t), "=m" (*p) \
380 : "r" (p), "r" (v), "m" (*p) \
381 : "cr0", "memory") \
382 /* __atomic_subtract_long */
383 #else
384 #define __atomic_subtract_long(p, v, t) \
385 __asm __volatile( \
386 "1: lwarx %0, 0, %2\n" \
387 " subf %0, %3, %0\n" \
388 " stwcx. %0, 0, %2\n" \
389 " bne- 1b\n" \
390 : "=&r" (t), "=m" (*p) \
391 : "r" (p), "r" (v), "m" (*p) \
392 : "cr0", "memory") \
393 /* __atomic_subtract_long */
394 #endif
395
396 #define _ATOMIC_SUBTRACT(type) \
397 static __inline void \
398 atomic_subtract_##type(volatile u_##type *p, u_##type v) { \
399 u_##type t; \
400 __atomic_subtract_##type(p, v, t); \
401 } \
402 \
403 static __inline void \
404 atomic_subtract_acq_##type(volatile u_##type *p, u_##type v) { \
405 u_##type t; \
406 __atomic_subtract_##type(p, v, t); \
407 __ATOMIC_ACQ(); \
408 } \
409 \
410 static __inline void \
411 atomic_subtract_rel_##type(volatile u_##type *p, u_##type v) { \
412 u_##type t; \
413 __ATOMIC_REL(); \
414 __atomic_subtract_##type(p, v, t); \
415 } \
416 /* _ATOMIC_SUBTRACT */
417
418 _ATOMIC_SUBTRACT(int)
419 _ATOMIC_SUBTRACT(long)
420
421 #define atomic_subtract_32 atomic_subtract_int
422 #define atomic_subtract_acq_32 atomic_subtract_acq_int
423 #define atomic_subtract_rel_32 atomic_subtract_rel_int
424
425 #ifdef __powerpc64__
426 #define atomic_subtract_64 atomic_subtract_long
427 #define atomic_subtract_acq_64 atomic_subract_acq_long
428 #define atomic_subtract_rel_64 atomic_subtract_rel_long
429
430 #define atomic_subtract_ptr atomic_subtract_long
431 #define atomic_subtract_acq_ptr atomic_subtract_acq_long
432 #define atomic_subtract_rel_ptr atomic_subtract_rel_long
433 #else
434 #define atomic_subtract_ptr atomic_subtract_int
435 #define atomic_subtract_acq_ptr atomic_subtract_acq_int
436 #define atomic_subtract_rel_ptr atomic_subtract_rel_int
437 #endif
438 #undef _ATOMIC_SUBTRACT
439 #undef __atomic_subtract_long
440 #undef __atomic_subtract_int
441
442 /*
443 * atomic_store_rel(p, v)
444 */
445 /* TODO -- see below */
446
447 /*
448 * Old/original implementations that still need revisiting.
449 */
450
451 static __inline u_int
452 atomic_readandclear_int(volatile u_int *addr)
453 {
454 u_int result,temp;
455
456 __asm __volatile (
457 "\tsync\n" /* drain writes */
458 "1:\tlwarx %0, 0, %3\n\t" /* load old value */
459 "li %1, 0\n\t" /* load new value */
460 "stwcx. %1, 0, %3\n\t" /* attempt to store */
461 "bne- 1b\n\t" /* spin if failed */
462 : "=&r"(result), "=&r"(temp), "=m" (*addr)
463 : "r" (addr), "m" (*addr)
464 : "cr0", "memory");
465
466 return (result);
467 }
468
469 #ifdef __powerpc64__
470 static __inline u_long
atomic_readandclear_long(volatile u_long * addr)471 atomic_readandclear_long(volatile u_long *addr)
472 {
473 u_long result,temp;
474
475 __asm __volatile (
476 "\tsync\n" /* drain writes */
477 "1:\tldarx %0, 0, %3\n\t" /* load old value */
478 "li %1, 0\n\t" /* load new value */
479 "stdcx. %1, 0, %3\n\t" /* attempt to store */
480 "bne- 1b\n\t" /* spin if failed */
481 : "=&r"(result), "=&r"(temp), "=m" (*addr)
482 : "r" (addr), "m" (*addr)
483 : "cr0", "memory");
484
485 return (result);
486 }
487 #endif
488
489 #define atomic_readandclear_32 atomic_readandclear_int
490
491 #ifdef __powerpc64__
492 #define atomic_readandclear_64 atomic_readandclear_long
493
494 #define atomic_readandclear_ptr atomic_readandclear_long
495 #else
496 static __inline u_long
atomic_readandclear_long(volatile u_long * addr)497 atomic_readandclear_long(volatile u_long *addr)
498 {
499
500 return ((u_long)atomic_readandclear_int((volatile u_int *)addr));
501 }
502
503 #define atomic_readandclear_ptr atomic_readandclear_int
504 #endif
505
506 /*
507 * We assume that a = b will do atomic loads and stores.
508 */
509 #define ATOMIC_STORE_LOAD(TYPE) \
510 static __inline u_##TYPE \
511 atomic_load_acq_##TYPE(volatile u_##TYPE *p) \
512 { \
513 u_##TYPE v; \
514 \
515 v = *p; \
516 powerpc_lwsync(); \
517 return (v); \
518 } \
519 \
520 static __inline void \
521 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
522 { \
523 \
524 powerpc_lwsync(); \
525 *p = v; \
526 }
527
528 ATOMIC_STORE_LOAD(int)
529
530 #define atomic_load_acq_32 atomic_load_acq_int
531 #define atomic_store_rel_32 atomic_store_rel_int
532
533 #ifdef __powerpc64__
ATOMIC_STORE_LOAD(long)534 ATOMIC_STORE_LOAD(long)
535
536 #define atomic_load_acq_64 atomic_load_acq_long
537 #define atomic_store_rel_64 atomic_store_rel_long
538
539 #define atomic_load_acq_ptr atomic_load_acq_long
540 #define atomic_store_rel_ptr atomic_store_rel_long
541 #else
542 static __inline u_long
543 atomic_load_acq_long(volatile u_long *addr)
544 {
545
546 return ((u_long)atomic_load_acq_int((volatile u_int *)addr));
547 }
548
549 static __inline void
550 atomic_store_rel_long(volatile u_long *addr, u_long val)
551 {
552
553 atomic_store_rel_int((volatile u_int *)addr, (u_int)val);
554 }
555
556 #define atomic_load_acq_ptr atomic_load_acq_int
557 #define atomic_store_rel_ptr atomic_store_rel_int
558 #endif
559 #undef ATOMIC_STORE_LOAD
560
561 /*
562 * Atomically compare the value stored at *p with cmpval and if the
563 * two values are equal, update the value of *p with newval. Returns
564 * zero if the compare failed, nonzero otherwise.
565 */
566 #ifdef ISA_206_ATOMICS
567 static __inline int
568 atomic_cmpset_char(volatile u_char *p, u_char cmpval, u_char newval)
569 {
570 int ret;
571
572 __asm __volatile (
573 "1:\tlbarx %0, 0, %2\n\t" /* load old value */
574 "cmplw %3, %0\n\t" /* compare */
575 "bne- 2f\n\t" /* exit if not equal */
576 "stbcx. %4, 0, %2\n\t" /* attempt to store */
577 "bne- 1b\n\t" /* spin if failed */
578 "li %0, 1\n\t" /* success - retval = 1 */
579 "b 3f\n\t" /* we've succeeded */
580 "2:\n\t"
581 "stbcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
582 "li %0, 0\n\t" /* failure - retval = 0 */
583 "3:\n\t"
584 : "=&r" (ret), "=m" (*p)
585 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
586 : "cr0", "memory");
587
588 return (ret);
589 }
590
591 static __inline int
atomic_cmpset_short(volatile u_short * p,u_short cmpval,u_short newval)592 atomic_cmpset_short(volatile u_short *p, u_short cmpval, u_short newval)
593 {
594 int ret;
595
596 __asm __volatile (
597 "1:\tlharx %0, 0, %2\n\t" /* load old value */
598 "cmplw %3, %0\n\t" /* compare */
599 "bne- 2f\n\t" /* exit if not equal */
600 "sthcx. %4, 0, %2\n\t" /* attempt to store */
601 "bne- 1b\n\t" /* spin if failed */
602 "li %0, 1\n\t" /* success - retval = 1 */
603 "b 3f\n\t" /* we've succeeded */
604 "2:\n\t"
605 "sthcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
606 "li %0, 0\n\t" /* failure - retval = 0 */
607 "3:\n\t"
608 : "=&r" (ret), "=m" (*p)
609 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
610 : "cr0", "memory");
611
612 return (ret);
613 }
614 #else
615 static __inline int
616 atomic_cmpset_masked(uint32_t *p, uint32_t cmpval, uint32_t newval,
617 uint32_t mask)
618 {
619 int ret;
620 uint32_t tmp;
621
622 __asm __volatile (
623 "1:\tlwarx %2, 0, %3\n\t" /* load old value */
624 "and %0, %2, %7\n\t"
625 "cmplw %4, %0\n\t" /* compare */
626 "bne- 2f\n\t" /* exit if not equal */
627 "andc %2, %2, %7\n\t"
628 "or %2, %2, %5\n\t"
629 "stwcx. %2, 0, %3\n\t" /* attempt to store */
630 "bne- 1b\n\t" /* spin if failed */
631 "li %0, 1\n\t" /* success - retval = 1 */
632 "b 3f\n\t" /* we've succeeded */
633 "2:\n\t"
634 "stwcx. %2, 0, %3\n\t" /* clear reservation (74xx) */
635 "li %0, 0\n\t" /* failure - retval = 0 */
636 "3:\n\t"
637 : "=&r" (ret), "=m" (*p), "+&r" (tmp)
638 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p),
639 "r" (mask)
640 : "cr0", "memory");
641
642 return (ret);
643 }
644
645 #define _atomic_cmpset_masked_word(a,o,v,m) atomic_cmpset_masked(a, o, v, m)
646 #endif
647
648 static __inline int
atomic_cmpset_int(volatile u_int * p,u_int cmpval,u_int newval)649 atomic_cmpset_int(volatile u_int* p, u_int cmpval, u_int newval)
650 {
651 int ret;
652
653 __asm __volatile (
654 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
655 "cmplw %3, %0\n\t" /* compare */
656 "bne- 2f\n\t" /* exit if not equal */
657 "stwcx. %4, 0, %2\n\t" /* attempt to store */
658 "bne- 1b\n\t" /* spin if failed */
659 "li %0, 1\n\t" /* success - retval = 1 */
660 "b 3f\n\t" /* we've succeeded */
661 "2:\n\t"
662 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
663 "li %0, 0\n\t" /* failure - retval = 0 */
664 "3:\n\t"
665 : "=&r" (ret), "=m" (*p)
666 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
667 : "cr0", "memory");
668
669 return (ret);
670 }
671 static __inline int
atomic_cmpset_long(volatile u_long * p,u_long cmpval,u_long newval)672 atomic_cmpset_long(volatile u_long* p, u_long cmpval, u_long newval)
673 {
674 int ret;
675
676 __asm __volatile (
677 #ifdef __powerpc64__
678 "1:\tldarx %0, 0, %2\n\t" /* load old value */
679 "cmpld %3, %0\n\t" /* compare */
680 "bne- 2f\n\t" /* exit if not equal */
681 "stdcx. %4, 0, %2\n\t" /* attempt to store */
682 #else
683 "1:\tlwarx %0, 0, %2\n\t" /* load old value */
684 "cmplw %3, %0\n\t" /* compare */
685 "bne- 2f\n\t" /* exit if not equal */
686 "stwcx. %4, 0, %2\n\t" /* attempt to store */
687 #endif
688 "bne- 1b\n\t" /* spin if failed */
689 "li %0, 1\n\t" /* success - retval = 1 */
690 "b 3f\n\t" /* we've succeeded */
691 "2:\n\t"
692 #ifdef __powerpc64__
693 "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
694 #else
695 "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
696 #endif
697 "li %0, 0\n\t" /* failure - retval = 0 */
698 "3:\n\t"
699 : "=&r" (ret), "=m" (*p)
700 : "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
701 : "cr0", "memory");
702
703 return (ret);
704 }
705
706 #define ATOMIC_CMPSET_ACQ_REL(type) \
707 static __inline int \
708 atomic_cmpset_acq_##type(volatile u_##type *p, \
709 u_##type cmpval, u_##type newval)\
710 {\
711 u_##type retval; \
712 retval = atomic_cmpset_##type(p, cmpval, newval);\
713 __ATOMIC_ACQ();\
714 return (retval);\
715 }\
716 static __inline int \
717 atomic_cmpset_rel_##type(volatile u_##type *p, \
718 u_##type cmpval, u_##type newval)\
719 {\
720 __ATOMIC_REL();\
721 return (atomic_cmpset_##type(p, cmpval, newval));\
722 }\
723 struct hack
724
725 ATOMIC_CMPSET_ACQ_REL(int);
726 ATOMIC_CMPSET_ACQ_REL(long);
727
728 #ifdef ISA_206_ATOMICS
729 #define atomic_cmpset_8 atomic_cmpset_char
730 #endif
731 #define atomic_cmpset_acq_8 atomic_cmpset_acq_char
732 #define atomic_cmpset_rel_8 atomic_cmpset_rel_char
733
734 #ifdef ISA_206_ATOMICS
735 #define atomic_cmpset_16 atomic_cmpset_short
736 #endif
737 #define atomic_cmpset_acq_16 atomic_cmpset_acq_short
738 #define atomic_cmpset_rel_16 atomic_cmpset_rel_short
739
740 #define atomic_cmpset_32 atomic_cmpset_int
741 #define atomic_cmpset_acq_32 atomic_cmpset_acq_int
742 #define atomic_cmpset_rel_32 atomic_cmpset_rel_int
743
744 #ifdef __powerpc64__
745 #define atomic_cmpset_64 atomic_cmpset_long
746 #define atomic_cmpset_acq_64 atomic_cmpset_acq_long
747 #define atomic_cmpset_rel_64 atomic_cmpset_rel_long
748
749 #define atomic_cmpset_ptr atomic_cmpset_long
750 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long
751 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long
752 #else
753 #define atomic_cmpset_ptr atomic_cmpset_int
754 #define atomic_cmpset_acq_ptr atomic_cmpset_acq_int
755 #define atomic_cmpset_rel_ptr atomic_cmpset_rel_int
756 #endif
757
758 /*
759 * Atomically compare the value stored at *p with *cmpval and if the
760 * two values are equal, update the value of *p with newval. Returns
761 * zero if the compare failed and sets *cmpval to the read value from *p,
762 * nonzero otherwise.
763 */
764 #ifdef ISA_206_ATOMICS
765 static __inline int
atomic_fcmpset_char(volatile u_char * p,u_char * cmpval,u_char newval)766 atomic_fcmpset_char(volatile u_char *p, u_char *cmpval, u_char newval)
767 {
768 int ret;
769
770 __asm __volatile (
771 "lbarx %0, 0, %3\n\t" /* load old value */
772 "cmplw %4, %0\n\t" /* compare */
773 "bne- 1f\n\t" /* exit if not equal */
774 "stbcx. %5, 0, %3\n\t" /* attempt to store */
775 "bne- 1f\n\t" /* exit if failed */
776 "li %0, 1\n\t" /* success - retval = 1 */
777 "b 2f\n\t" /* we've succeeded */
778 "1:\n\t"
779 "stbcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
780 "stwx %0, 0, %7\n\t"
781 "li %0, 0\n\t" /* failure - retval = 0 */
782 "2:\n\t"
783 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
784 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
785 : "cr0", "memory");
786
787 return (ret);
788 }
789
790 static __inline int
atomic_fcmpset_short(volatile u_short * p,u_short * cmpval,u_short newval)791 atomic_fcmpset_short(volatile u_short *p, u_short *cmpval, u_short newval)
792 {
793 int ret;
794
795 __asm __volatile (
796 "lharx %0, 0, %3\n\t" /* load old value */
797 "cmplw %4, %0\n\t" /* compare */
798 "bne- 1f\n\t" /* exit if not equal */
799 "sthcx. %5, 0, %3\n\t" /* attempt to store */
800 "bne- 1f\n\t" /* exit if failed */
801 "li %0, 1\n\t" /* success - retval = 1 */
802 "b 2f\n\t" /* we've succeeded */
803 "1:\n\t"
804 "sthcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
805 "stwx %0, 0, %7\n\t"
806 "li %0, 0\n\t" /* failure - retval = 0 */
807 "2:\n\t"
808 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
809 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
810 : "cr0", "memory");
811
812 return (ret);
813 }
814 #endif /* ISA_206_ATOMICS */
815
816 static __inline int
atomic_fcmpset_int(volatile u_int * p,u_int * cmpval,u_int newval)817 atomic_fcmpset_int(volatile u_int *p, u_int *cmpval, u_int newval)
818 {
819 int ret;
820
821 __asm __volatile (
822 "lwarx %0, 0, %3\n\t" /* load old value */
823 "cmplw %4, %0\n\t" /* compare */
824 "bne- 1f\n\t" /* exit if not equal */
825 "stwcx. %5, 0, %3\n\t" /* attempt to store */
826 "bne- 1f\n\t" /* exit if failed */
827 "li %0, 1\n\t" /* success - retval = 1 */
828 "b 2f\n\t" /* we've succeeded */
829 "1:\n\t"
830 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
831 "stwx %0, 0, %7\n\t"
832 "li %0, 0\n\t" /* failure - retval = 0 */
833 "2:\n\t"
834 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
835 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
836 : "cr0", "memory");
837
838 return (ret);
839 }
840 static __inline int
atomic_fcmpset_long(volatile u_long * p,u_long * cmpval,u_long newval)841 atomic_fcmpset_long(volatile u_long *p, u_long *cmpval, u_long newval)
842 {
843 int ret;
844
845 __asm __volatile (
846 #ifdef __powerpc64__
847 "ldarx %0, 0, %3\n\t" /* load old value */
848 "cmpld %4, %0\n\t" /* compare */
849 "bne- 1f\n\t" /* exit if not equal */
850 "stdcx. %5, 0, %3\n\t" /* attempt to store */
851 #else
852 "lwarx %0, 0, %3\n\t" /* load old value */
853 "cmplw %4, %0\n\t" /* compare */
854 "bne- 1f\n\t" /* exit if not equal */
855 "stwcx. %5, 0, %3\n\t" /* attempt to store */
856 #endif
857 "bne- 1f\n\t" /* exit if failed */
858 "li %0, 1\n\t" /* success - retval = 1 */
859 "b 2f\n\t" /* we've succeeded */
860 "1:\n\t"
861 #ifdef __powerpc64__
862 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
863 "stdx %0, 0, %7\n\t"
864 #else
865 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */
866 "stwx %0, 0, %7\n\t"
867 #endif
868 "li %0, 0\n\t" /* failure - retval = 0 */
869 "2:\n\t"
870 : "=&r" (ret), "=m" (*p), "=m" (*cmpval)
871 : "r" (p), "r" (*cmpval), "r" (newval), "m" (*p), "r"(cmpval)
872 : "cr0", "memory");
873
874 return (ret);
875 }
876
877 #define ATOMIC_FCMPSET_ACQ_REL(type) \
878 static __inline int \
879 atomic_fcmpset_acq_##type(volatile u_##type *p, \
880 u_##type *cmpval, u_##type newval)\
881 {\
882 u_##type retval; \
883 retval = atomic_fcmpset_##type(p, cmpval, newval);\
884 __ATOMIC_ACQ();\
885 return (retval);\
886 }\
887 static __inline int \
888 atomic_fcmpset_rel_##type(volatile u_##type *p, \
889 u_##type *cmpval, u_##type newval)\
890 {\
891 __ATOMIC_REL();\
892 return (atomic_fcmpset_##type(p, cmpval, newval));\
893 }\
894 struct hack
895
896 ATOMIC_FCMPSET_ACQ_REL(int);
897 ATOMIC_FCMPSET_ACQ_REL(long);
898
899 #ifdef ISA_206_ATOMICS
900 #define atomic_fcmpset_8 atomic_fcmpset_char
901 #endif
902 #define atomic_fcmpset_acq_8 atomic_fcmpset_acq_char
903 #define atomic_fcmpset_rel_8 atomic_fcmpset_rel_char
904
905 #ifdef ISA_206_ATOMICS
906 #define atomic_fcmpset_16 atomic_fcmpset_short
907 #endif
908 #define atomic_fcmpset_acq_16 atomic_fcmpset_acq_short
909 #define atomic_fcmpset_rel_16 atomic_fcmpset_rel_short
910
911 #define atomic_fcmpset_32 atomic_fcmpset_int
912 #define atomic_fcmpset_acq_32 atomic_fcmpset_acq_int
913 #define atomic_fcmpset_rel_32 atomic_fcmpset_rel_int
914
915 #ifdef __powerpc64__
916 #define atomic_fcmpset_64 atomic_fcmpset_long
917 #define atomic_fcmpset_acq_64 atomic_fcmpset_acq_long
918 #define atomic_fcmpset_rel_64 atomic_fcmpset_rel_long
919
920 #define atomic_fcmpset_ptr atomic_fcmpset_long
921 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_long
922 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_long
923 #else
924 #define atomic_fcmpset_ptr atomic_fcmpset_int
925 #define atomic_fcmpset_acq_ptr atomic_fcmpset_acq_int
926 #define atomic_fcmpset_rel_ptr atomic_fcmpset_rel_int
927 #endif
928
929 static __inline u_int
atomic_fetchadd_int(volatile u_int * p,u_int v)930 atomic_fetchadd_int(volatile u_int *p, u_int v)
931 {
932 u_int value;
933
934 do {
935 value = *p;
936 } while (!atomic_cmpset_int(p, value, value + v));
937 return (value);
938 }
939
940 static __inline u_long
atomic_fetchadd_long(volatile u_long * p,u_long v)941 atomic_fetchadd_long(volatile u_long *p, u_long v)
942 {
943 u_long value;
944
945 do {
946 value = *p;
947 } while (!atomic_cmpset_long(p, value, value + v));
948 return (value);
949 }
950
951 static __inline u_int
atomic_swap_32(volatile u_int * p,u_int v)952 atomic_swap_32(volatile u_int *p, u_int v)
953 {
954 u_int prev;
955
956 __asm __volatile(
957 "1: lwarx %0,0,%2\n"
958 " stwcx. %3,0,%2\n"
959 " bne- 1b\n"
960 : "=&r" (prev), "+m" (*(volatile u_int *)p)
961 : "r" (p), "r" (v)
962 : "cr0", "memory");
963
964 return (prev);
965 }
966
967 #ifdef __powerpc64__
968 static __inline u_long
atomic_swap_64(volatile u_long * p,u_long v)969 atomic_swap_64(volatile u_long *p, u_long v)
970 {
971 u_long prev;
972
973 __asm __volatile(
974 "1: ldarx %0,0,%2\n"
975 " stdcx. %3,0,%2\n"
976 " bne- 1b\n"
977 : "=&r" (prev), "+m" (*(volatile u_long *)p)
978 : "r" (p), "r" (v)
979 : "cr0", "memory");
980
981 return (prev);
982 }
983 #endif
984
985 #define atomic_fetchadd_32 atomic_fetchadd_int
986 #define atomic_swap_int atomic_swap_32
987
988 #ifdef __powerpc64__
989 #define atomic_fetchadd_64 atomic_fetchadd_long
990 #define atomic_swap_long atomic_swap_64
991 #define atomic_swap_ptr atomic_swap_64
992 #else
993 #define atomic_swap_long(p,v) atomic_swap_32((volatile u_int *)(p), v)
994 #define atomic_swap_ptr(p,v) atomic_swap_32((volatile u_int *)(p), v)
995 #endif
996
997 static __inline int
atomic_testandset_int(volatile u_int * p,u_int v)998 atomic_testandset_int(volatile u_int *p, u_int v)
999 {
1000 u_int m = (1u << (v & 0x1f));
1001 u_int res;
1002 u_int tmp;
1003
1004 __asm __volatile(
1005 "1: lwarx %0,0,%3\n"
1006 " and %1,%0,%4\n"
1007 " or %0,%0,%4\n"
1008 " stwcx. %0,0,%3\n"
1009 " bne- 1b\n"
1010 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1011 : "r"(p), "r"(m)
1012 : "cr0", "memory");
1013
1014 return (res != 0);
1015 }
1016
1017 static __inline int
atomic_testandclear_int(volatile u_int * p,u_int v)1018 atomic_testandclear_int(volatile u_int *p, u_int v)
1019 {
1020 u_int m = (1u << (v & 0x1f));
1021 u_int res;
1022 u_int tmp;
1023
1024 __asm __volatile(
1025 "1: lwarx %0,0,%3\n"
1026 " and %1,%0,%4\n"
1027 " andc %0,%0,%4\n"
1028 " stwcx. %0,0,%3\n"
1029 " bne- 1b\n"
1030 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1031 : "r"(p), "r"(m)
1032 : "cr0", "memory");
1033
1034 return (res != 0);
1035 }
1036
1037 #ifdef __powerpc64__
1038 static __inline int
atomic_testandset_long(volatile u_long * p,u_int v)1039 atomic_testandset_long(volatile u_long *p, u_int v)
1040 {
1041 u_long m = (1ul << (v & 0x3f));
1042 u_long res;
1043 u_long tmp;
1044
1045 __asm __volatile(
1046 "1: ldarx %0,0,%3\n"
1047 " and %1,%0,%4\n"
1048 " or %0,%0,%4\n"
1049 " stdcx. %0,0,%3\n"
1050 " bne- 1b\n"
1051 : "=&r"(tmp), "=&r"(res), "+m"(*(volatile u_long *)p)
1052 : "r"(p), "r"(m)
1053 : "cr0", "memory");
1054
1055 return (res != 0);
1056 }
1057
1058 static __inline int
atomic_testandclear_long(volatile u_long * p,u_int v)1059 atomic_testandclear_long(volatile u_long *p, u_int v)
1060 {
1061 u_long m = (1ul << (v & 0x3f));
1062 u_long res;
1063 u_long tmp;
1064
1065 __asm __volatile(
1066 "1: ldarx %0,0,%3\n"
1067 " and %1,%0,%4\n"
1068 " andc %0,%0,%4\n"
1069 " stdcx. %0,0,%3\n"
1070 " bne- 1b\n"
1071 : "=&r"(tmp), "=&r"(res), "+m"(*p)
1072 : "r"(p), "r"(m)
1073 : "cr0", "memory");
1074
1075 return (res != 0);
1076 }
1077 #else
1078 static __inline int
atomic_testandset_long(volatile u_long * p,u_int v)1079 atomic_testandset_long(volatile u_long *p, u_int v)
1080 {
1081 return (atomic_testandset_int((volatile u_int *)p, v));
1082 }
1083
1084 static __inline int
atomic_testandclear_long(volatile u_long * p,u_int v)1085 atomic_testandclear_long(volatile u_long *p, u_int v)
1086 {
1087 return (atomic_testandclear_int((volatile u_int *)p, v));
1088 }
1089 #endif
1090
1091 #define atomic_testandclear_32 atomic_testandclear_int
1092 #define atomic_testandset_32 atomic_testandset_int
1093
1094 static __inline int
atomic_testandset_acq_long(volatile u_long * p,u_int v)1095 atomic_testandset_acq_long(volatile u_long *p, u_int v)
1096 {
1097 u_int a = atomic_testandset_long(p, v);
1098 __ATOMIC_ACQ();
1099 return (a);
1100 }
1101
1102 #define atomic_testandclear_int atomic_testandclear_int
1103 #define atomic_testandset_int atomic_testandset_int
1104 #define atomic_testandclear_long atomic_testandclear_long
1105 #define atomic_testandset_long atomic_testandset_long
1106 #define atomic_testandset_acq_long atomic_testandset_acq_long
1107
1108 static __inline void
atomic_thread_fence_acq(void)1109 atomic_thread_fence_acq(void)
1110 {
1111
1112 powerpc_lwsync();
1113 }
1114
1115 static __inline void
atomic_thread_fence_rel(void)1116 atomic_thread_fence_rel(void)
1117 {
1118
1119 powerpc_lwsync();
1120 }
1121
1122 static __inline void
atomic_thread_fence_acq_rel(void)1123 atomic_thread_fence_acq_rel(void)
1124 {
1125
1126 powerpc_lwsync();
1127 }
1128
1129 static __inline void
atomic_thread_fence_seq_cst(void)1130 atomic_thread_fence_seq_cst(void)
1131 {
1132
1133 __asm __volatile("sync" : : : "memory");
1134 }
1135
1136 #ifndef ISA_206_ATOMICS
1137 #include <sys/_atomic_subword.h>
1138 #define atomic_cmpset_char atomic_cmpset_8
1139 #define atomic_cmpset_short atomic_cmpset_16
1140 #define atomic_fcmpset_char atomic_fcmpset_8
1141 #define atomic_fcmpset_short atomic_fcmpset_16
1142 #endif
1143
1144 /* These need sys/_atomic_subword.h on non-ISA-2.06-atomic platforms. */
1145 ATOMIC_CMPSET_ACQ_REL(char);
1146 ATOMIC_CMPSET_ACQ_REL(short);
1147
1148 ATOMIC_FCMPSET_ACQ_REL(char);
1149 ATOMIC_FCMPSET_ACQ_REL(short);
1150
1151 #undef __ATOMIC_REL
1152 #undef __ATOMIC_ACQ
1153
1154 #endif /* ! _MACHINE_ATOMIC_H_ */
1155