1 /*
2 * Copyright 2009-2015 Samy Al Bahra.
3 * Copyright 2011 David Joseph.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #ifndef CK_PR_H
29 #define CK_PR_H
30
31 #include <ck_cc.h>
32 #include <ck_limits.h>
33 #include <ck_md.h>
34 #include <ck_stdint.h>
35 #include <ck_stdbool.h>
36
37 #ifndef CK_USE_CC_BUILTINS
38 #if defined(__x86_64__)
39 #include "gcc/x86_64/ck_pr.h"
40 #elif defined(__x86__)
41 #include "gcc/x86/ck_pr.h"
42 #elif defined(__sparcv9__)
43 #include "gcc/sparcv9/ck_pr.h"
44 #elif defined(__ppc64__)
45 #include "gcc/ppc64/ck_pr.h"
46 #elif defined(__s390x__)
47 #include "gcc/s390x/ck_pr.h"
48 #elif defined(__ppc__)
49 #include "gcc/ppc/ck_pr.h"
50 #elif defined(__arm__)
51 #include "gcc/arm/ck_pr.h"
52 #elif defined(__aarch64__)
53 #include "gcc/aarch64/ck_pr.h"
54 #elif !defined(__GNUC__)
55 #error Your platform is unsupported
56 #endif
57 #endif /* !CK_USE_CC_BUILTINS */
58
59 #if defined(__GNUC__)
60 #include "gcc/ck_pr.h"
61 #endif
62
63 #define CK_PR_FENCE_EMIT(T) \
64 CK_CC_INLINE static void \
65 ck_pr_fence_##T(void) \
66 { \
67 ck_pr_fence_strict_##T(); \
68 return; \
69 }
70 #define CK_PR_FENCE_NOOP(T) \
71 CK_CC_INLINE static void \
72 ck_pr_fence_##T(void) \
73 { \
74 ck_pr_barrier(); \
75 return; \
76 }
77
78 /*
79 * None of the currently supported platforms allow for data-dependent
80 * load ordering.
81 */
82 CK_PR_FENCE_NOOP(load_depends)
83 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends
84
85 /*
86 * In memory models where atomic operations do not have serializing
87 * effects, atomic read-modify-write operations are modeled as stores.
88 */
89 #if defined(CK_MD_RMO)
90 /*
91 * Only stores to the same location have a global
92 * ordering.
93 */
CK_PR_FENCE_EMIT(atomic)94 CK_PR_FENCE_EMIT(atomic)
95 CK_PR_FENCE_EMIT(atomic_load)
96 CK_PR_FENCE_EMIT(atomic_store)
97 CK_PR_FENCE_EMIT(store_atomic)
98 CK_PR_FENCE_EMIT(load_atomic)
99 CK_PR_FENCE_EMIT(load_store)
100 CK_PR_FENCE_EMIT(store_load)
101 CK_PR_FENCE_EMIT(load)
102 CK_PR_FENCE_EMIT(store)
103 CK_PR_FENCE_EMIT(memory)
104 CK_PR_FENCE_EMIT(acquire)
105 CK_PR_FENCE_EMIT(release)
106 CK_PR_FENCE_EMIT(acqrel)
107 CK_PR_FENCE_EMIT(lock)
108 CK_PR_FENCE_EMIT(unlock)
109 #elif defined(CK_MD_PSO)
110 /*
111 * Anything can be re-ordered with respect to stores.
112 * Otherwise, loads are executed in-order.
113 */
114 CK_PR_FENCE_EMIT(atomic)
115 CK_PR_FENCE_NOOP(atomic_load)
116 CK_PR_FENCE_EMIT(atomic_store)
117 CK_PR_FENCE_EMIT(store_atomic)
118 CK_PR_FENCE_NOOP(load_atomic)
119 CK_PR_FENCE_EMIT(load_store)
120 CK_PR_FENCE_EMIT(store_load)
121 CK_PR_FENCE_NOOP(load)
122 CK_PR_FENCE_EMIT(store)
123 CK_PR_FENCE_EMIT(memory)
124 CK_PR_FENCE_EMIT(acquire)
125 CK_PR_FENCE_EMIT(release)
126 CK_PR_FENCE_EMIT(acqrel)
127 CK_PR_FENCE_EMIT(lock)
128 CK_PR_FENCE_EMIT(unlock)
129 #elif defined(CK_MD_TSO)
130 /*
131 * Only loads are re-ordered and only with respect to
132 * prior stores. Atomic operations are serializing.
133 */
134 CK_PR_FENCE_NOOP(atomic)
135 CK_PR_FENCE_NOOP(atomic_load)
136 CK_PR_FENCE_NOOP(atomic_store)
137 CK_PR_FENCE_NOOP(store_atomic)
138 CK_PR_FENCE_NOOP(load_atomic)
139 CK_PR_FENCE_NOOP(load_store)
140 CK_PR_FENCE_EMIT(store_load)
141 CK_PR_FENCE_NOOP(load)
142 CK_PR_FENCE_NOOP(store)
143 CK_PR_FENCE_EMIT(memory)
144 CK_PR_FENCE_NOOP(acquire)
145 CK_PR_FENCE_NOOP(release)
146 CK_PR_FENCE_NOOP(acqrel)
147 CK_PR_FENCE_NOOP(lock)
148 CK_PR_FENCE_NOOP(unlock)
149 #else
150 #error "No memory model has been defined."
151 #endif /* CK_MD_TSO */
152
153 #undef CK_PR_FENCE_EMIT
154 #undef CK_PR_FENCE_NOOP
155
156 #ifndef CK_F_PR_RFO
157 #define CK_F_PR_RFO
158 CK_CC_INLINE static void
159 ck_pr_rfo(const void *m)
160 {
161
162 (void)m;
163 return;
164 }
165 #endif /* CK_F_PR_RFO */
166
167 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \
168 ck_pr_md_store_##TYPE( \
169 ((void)sizeof(*(DST) = (VAL)), (DST)), \
170 (VAL))
171
172 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr)
173 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char)
174 #ifndef CK_PR_DISABLE_DOUBLE
175 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double)
176 #endif
177 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint)
178 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int)
179 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32)
180 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16)
181 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8)
182
183 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL))
184
185 #ifdef CK_F_PR_LOAD_64
186 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64)
187 #endif /* CK_F_PR_LOAD_64 */
188
189 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC))
190 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC))
191
192 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC))
193 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char)
194 #ifndef CK_PR_DISABLE_DOUBLE
195 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double)
196 #endif
197 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint)
198 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int)
199 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32)
200 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16)
201 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8)
202
203 #ifdef CK_F_PR_LOAD_64
204 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64)
205 #endif /* CK_F_PR_LOAD_64 */
206
207 #define CK_PR_BIN(K, S, M, T, P, C) \
208 CK_CC_INLINE static void \
209 ck_pr_##K##_##S(M *target, T value) \
210 { \
211 T previous; \
212 C punt; \
213 punt = ck_pr_md_load_##S(target); \
214 previous = (T)punt; \
215 while (ck_pr_cas_##S##_value(target, \
216 (C)previous, \
217 (C)(previous P value), \
218 &previous) == false) \
219 ck_pr_stall(); \
220 \
221 return; \
222 }
223
224 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T)
225
226 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
227
228 #ifndef CK_F_PR_ADD_CHAR
229 #define CK_F_PR_ADD_CHAR
230 CK_PR_BIN_S(add, char, char, +)
231 #endif /* CK_F_PR_ADD_CHAR */
232
233 #ifndef CK_F_PR_SUB_CHAR
234 #define CK_F_PR_SUB_CHAR
235 CK_PR_BIN_S(sub, char, char, -)
236 #endif /* CK_F_PR_SUB_CHAR */
237
238 #ifndef CK_F_PR_AND_CHAR
239 #define CK_F_PR_AND_CHAR
240 CK_PR_BIN_S(and, char, char, &)
241 #endif /* CK_F_PR_AND_CHAR */
242
243 #ifndef CK_F_PR_XOR_CHAR
244 #define CK_F_PR_XOR_CHAR
245 CK_PR_BIN_S(xor, char, char, ^)
246 #endif /* CK_F_PR_XOR_CHAR */
247
248 #ifndef CK_F_PR_OR_CHAR
249 #define CK_F_PR_OR_CHAR
250 CK_PR_BIN_S(or, char, char, |)
251 #endif /* CK_F_PR_OR_CHAR */
252
253 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
254
255 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
256
257 #ifndef CK_F_PR_ADD_INT
258 #define CK_F_PR_ADD_INT
259 CK_PR_BIN_S(add, int, int, +)
260 #endif /* CK_F_PR_ADD_INT */
261
262 #ifndef CK_F_PR_SUB_INT
263 #define CK_F_PR_SUB_INT
264 CK_PR_BIN_S(sub, int, int, -)
265 #endif /* CK_F_PR_SUB_INT */
266
267 #ifndef CK_F_PR_AND_INT
268 #define CK_F_PR_AND_INT
269 CK_PR_BIN_S(and, int, int, &)
270 #endif /* CK_F_PR_AND_INT */
271
272 #ifndef CK_F_PR_XOR_INT
273 #define CK_F_PR_XOR_INT
274 CK_PR_BIN_S(xor, int, int, ^)
275 #endif /* CK_F_PR_XOR_INT */
276
277 #ifndef CK_F_PR_OR_INT
278 #define CK_F_PR_OR_INT
279 CK_PR_BIN_S(or, int, int, |)
280 #endif /* CK_F_PR_OR_INT */
281
282 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
283
284 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
285 !defined(CK_PR_DISABLE_DOUBLE)
286
287 #ifndef CK_F_PR_ADD_DOUBLE
288 #define CK_F_PR_ADD_DOUBLE
289 CK_PR_BIN_S(add, double, double, +)
290 #endif /* CK_F_PR_ADD_DOUBLE */
291
292 #ifndef CK_F_PR_SUB_DOUBLE
293 #define CK_F_PR_SUB_DOUBLE
294 CK_PR_BIN_S(sub, double, double, -)
295 #endif /* CK_F_PR_SUB_DOUBLE */
296
297 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
298
299 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
300
301 #ifndef CK_F_PR_ADD_UINT
302 #define CK_F_PR_ADD_UINT
303 CK_PR_BIN_S(add, uint, unsigned int, +)
304 #endif /* CK_F_PR_ADD_UINT */
305
306 #ifndef CK_F_PR_SUB_UINT
307 #define CK_F_PR_SUB_UINT
308 CK_PR_BIN_S(sub, uint, unsigned int, -)
309 #endif /* CK_F_PR_SUB_UINT */
310
311 #ifndef CK_F_PR_AND_UINT
312 #define CK_F_PR_AND_UINT
313 CK_PR_BIN_S(and, uint, unsigned int, &)
314 #endif /* CK_F_PR_AND_UINT */
315
316 #ifndef CK_F_PR_XOR_UINT
317 #define CK_F_PR_XOR_UINT
318 CK_PR_BIN_S(xor, uint, unsigned int, ^)
319 #endif /* CK_F_PR_XOR_UINT */
320
321 #ifndef CK_F_PR_OR_UINT
322 #define CK_F_PR_OR_UINT
323 CK_PR_BIN_S(or, uint, unsigned int, |)
324 #endif /* CK_F_PR_OR_UINT */
325
326 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
327
328 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
329
330 #ifndef CK_F_PR_ADD_PTR
331 #define CK_F_PR_ADD_PTR
332 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *)
333 #endif /* CK_F_PR_ADD_PTR */
334
335 #ifndef CK_F_PR_SUB_PTR
336 #define CK_F_PR_SUB_PTR
337 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *)
338 #endif /* CK_F_PR_SUB_PTR */
339
340 #ifndef CK_F_PR_AND_PTR
341 #define CK_F_PR_AND_PTR
342 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *)
343 #endif /* CK_F_PR_AND_PTR */
344
345 #ifndef CK_F_PR_XOR_PTR
346 #define CK_F_PR_XOR_PTR
347 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *)
348 #endif /* CK_F_PR_XOR_PTR */
349
350 #ifndef CK_F_PR_OR_PTR
351 #define CK_F_PR_OR_PTR
352 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *)
353 #endif /* CK_F_PR_OR_PTR */
354
355 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
356
357 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
358
359 #ifndef CK_F_PR_ADD_64
360 #define CK_F_PR_ADD_64
361 CK_PR_BIN_S(add, 64, uint64_t, +)
362 #endif /* CK_F_PR_ADD_64 */
363
364 #ifndef CK_F_PR_SUB_64
365 #define CK_F_PR_SUB_64
366 CK_PR_BIN_S(sub, 64, uint64_t, -)
367 #endif /* CK_F_PR_SUB_64 */
368
369 #ifndef CK_F_PR_AND_64
370 #define CK_F_PR_AND_64
371 CK_PR_BIN_S(and, 64, uint64_t, &)
372 #endif /* CK_F_PR_AND_64 */
373
374 #ifndef CK_F_PR_XOR_64
375 #define CK_F_PR_XOR_64
376 CK_PR_BIN_S(xor, 64, uint64_t, ^)
377 #endif /* CK_F_PR_XOR_64 */
378
379 #ifndef CK_F_PR_OR_64
380 #define CK_F_PR_OR_64
381 CK_PR_BIN_S(or, 64, uint64_t, |)
382 #endif /* CK_F_PR_OR_64 */
383
384 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
385
386 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
387
388 #ifndef CK_F_PR_ADD_32
389 #define CK_F_PR_ADD_32
390 CK_PR_BIN_S(add, 32, uint32_t, +)
391 #endif /* CK_F_PR_ADD_32 */
392
393 #ifndef CK_F_PR_SUB_32
394 #define CK_F_PR_SUB_32
395 CK_PR_BIN_S(sub, 32, uint32_t, -)
396 #endif /* CK_F_PR_SUB_32 */
397
398 #ifndef CK_F_PR_AND_32
399 #define CK_F_PR_AND_32
400 CK_PR_BIN_S(and, 32, uint32_t, &)
401 #endif /* CK_F_PR_AND_32 */
402
403 #ifndef CK_F_PR_XOR_32
404 #define CK_F_PR_XOR_32
405 CK_PR_BIN_S(xor, 32, uint32_t, ^)
406 #endif /* CK_F_PR_XOR_32 */
407
408 #ifndef CK_F_PR_OR_32
409 #define CK_F_PR_OR_32
410 CK_PR_BIN_S(or, 32, uint32_t, |)
411 #endif /* CK_F_PR_OR_32 */
412
413 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
414
415 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
416
417 #ifndef CK_F_PR_ADD_16
418 #define CK_F_PR_ADD_16
419 CK_PR_BIN_S(add, 16, uint16_t, +)
420 #endif /* CK_F_PR_ADD_16 */
421
422 #ifndef CK_F_PR_SUB_16
423 #define CK_F_PR_SUB_16
424 CK_PR_BIN_S(sub, 16, uint16_t, -)
425 #endif /* CK_F_PR_SUB_16 */
426
427 #ifndef CK_F_PR_AND_16
428 #define CK_F_PR_AND_16
429 CK_PR_BIN_S(and, 16, uint16_t, &)
430 #endif /* CK_F_PR_AND_16 */
431
432 #ifndef CK_F_PR_XOR_16
433 #define CK_F_PR_XOR_16
434 CK_PR_BIN_S(xor, 16, uint16_t, ^)
435 #endif /* CK_F_PR_XOR_16 */
436
437 #ifndef CK_F_PR_OR_16
438 #define CK_F_PR_OR_16
439 CK_PR_BIN_S(or, 16, uint16_t, |)
440 #endif /* CK_F_PR_OR_16 */
441
442 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
443
444 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
445
446 #ifndef CK_F_PR_ADD_8
447 #define CK_F_PR_ADD_8
448 CK_PR_BIN_S(add, 8, uint8_t, +)
449 #endif /* CK_F_PR_ADD_8 */
450
451 #ifndef CK_F_PR_SUB_8
452 #define CK_F_PR_SUB_8
453 CK_PR_BIN_S(sub, 8, uint8_t, -)
454 #endif /* CK_F_PR_SUB_8 */
455
456 #ifndef CK_F_PR_AND_8
457 #define CK_F_PR_AND_8
458 CK_PR_BIN_S(and, 8, uint8_t, &)
459 #endif /* CK_F_PR_AND_8 */
460
461 #ifndef CK_F_PR_XOR_8
462 #define CK_F_PR_XOR_8
463 CK_PR_BIN_S(xor, 8, uint8_t, ^)
464 #endif /* CK_F_PR_XOR_8 */
465
466 #ifndef CK_F_PR_OR_8
467 #define CK_F_PR_OR_8
468 CK_PR_BIN_S(or, 8, uint8_t, |)
469 #endif /* CK_F_PR_OR_8 */
470
471 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
472
473 #undef CK_PR_BIN_S
474 #undef CK_PR_BIN
475
476 #define CK_PR_BTX(K, S, M, T, P, C, R) \
477 CK_CC_INLINE static bool \
478 ck_pr_##K##_##S(M *target, unsigned int offset) \
479 { \
480 T previous; \
481 C punt; \
482 punt = ck_pr_md_load_##S(target); \
483 previous = (T)punt; \
484 while (ck_pr_cas_##S##_value(target, (C)previous, \
485 (C)(previous P (R ((T)1 << offset))), &previous) == false) \
486 ck_pr_stall(); \
487 return ((previous >> offset) & 1); \
488 }
489
490 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R)
491
492 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
493
494 #ifndef CK_F_PR_BTC_INT
495 #define CK_F_PR_BTC_INT
496 CK_PR_BTX_S(btc, int, int, ^,)
497 #endif /* CK_F_PR_BTC_INT */
498
499 #ifndef CK_F_PR_BTR_INT
500 #define CK_F_PR_BTR_INT
501 CK_PR_BTX_S(btr, int, int, &, ~)
502 #endif /* CK_F_PR_BTR_INT */
503
504 #ifndef CK_F_PR_BTS_INT
505 #define CK_F_PR_BTS_INT
506 CK_PR_BTX_S(bts, int, int, |,)
507 #endif /* CK_F_PR_BTS_INT */
508
509 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
510
511 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
512
513 #ifndef CK_F_PR_BTC_UINT
514 #define CK_F_PR_BTC_UINT
515 CK_PR_BTX_S(btc, uint, unsigned int, ^,)
516 #endif /* CK_F_PR_BTC_UINT */
517
518 #ifndef CK_F_PR_BTR_UINT
519 #define CK_F_PR_BTR_UINT
520 CK_PR_BTX_S(btr, uint, unsigned int, &, ~)
521 #endif /* CK_F_PR_BTR_UINT */
522
523 #ifndef CK_F_PR_BTS_UINT
524 #define CK_F_PR_BTS_UINT
525 CK_PR_BTX_S(bts, uint, unsigned int, |,)
526 #endif /* CK_F_PR_BTS_UINT */
527
528 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
529
530 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
531
532 #ifndef CK_F_PR_BTC_PTR
533 #define CK_F_PR_BTC_PTR
534 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,)
535 #endif /* CK_F_PR_BTC_PTR */
536
537 #ifndef CK_F_PR_BTR_PTR
538 #define CK_F_PR_BTR_PTR
539 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~)
540 #endif /* CK_F_PR_BTR_PTR */
541
542 #ifndef CK_F_PR_BTS_PTR
543 #define CK_F_PR_BTS_PTR
544 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,)
545 #endif /* CK_F_PR_BTS_PTR */
546
547 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
548
549 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
550
551 #ifndef CK_F_PR_BTC_64
552 #define CK_F_PR_BTC_64
553 CK_PR_BTX_S(btc, 64, uint64_t, ^,)
554 #endif /* CK_F_PR_BTC_64 */
555
556 #ifndef CK_F_PR_BTR_64
557 #define CK_F_PR_BTR_64
558 CK_PR_BTX_S(btr, 64, uint64_t, &, ~)
559 #endif /* CK_F_PR_BTR_64 */
560
561 #ifndef CK_F_PR_BTS_64
562 #define CK_F_PR_BTS_64
563 CK_PR_BTX_S(bts, 64, uint64_t, |,)
564 #endif /* CK_F_PR_BTS_64 */
565
566 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
567
568 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
569
570 #ifndef CK_F_PR_BTC_32
571 #define CK_F_PR_BTC_32
572 CK_PR_BTX_S(btc, 32, uint32_t, ^,)
573 #endif /* CK_F_PR_BTC_32 */
574
575 #ifndef CK_F_PR_BTR_32
576 #define CK_F_PR_BTR_32
577 CK_PR_BTX_S(btr, 32, uint32_t, &, ~)
578 #endif /* CK_F_PR_BTR_32 */
579
580 #ifndef CK_F_PR_BTS_32
581 #define CK_F_PR_BTS_32
582 CK_PR_BTX_S(bts, 32, uint32_t, |,)
583 #endif /* CK_F_PR_BTS_32 */
584
585 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
586
587 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
588
589 #ifndef CK_F_PR_BTC_16
590 #define CK_F_PR_BTC_16
591 CK_PR_BTX_S(btc, 16, uint16_t, ^,)
592 #endif /* CK_F_PR_BTC_16 */
593
594 #ifndef CK_F_PR_BTR_16
595 #define CK_F_PR_BTR_16
596 CK_PR_BTX_S(btr, 16, uint16_t, &, ~)
597 #endif /* CK_F_PR_BTR_16 */
598
599 #ifndef CK_F_PR_BTS_16
600 #define CK_F_PR_BTS_16
601 CK_PR_BTX_S(bts, 16, uint16_t, |,)
602 #endif /* CK_F_PR_BTS_16 */
603
604 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
605
606 #undef CK_PR_BTX_S
607 #undef CK_PR_BTX
608
609 #define CK_PR_UNARY(K, X, S, M, T) \
610 CK_CC_INLINE static void \
611 ck_pr_##K##_##S(M *target) \
612 { \
613 ck_pr_##X##_##S(target, (T)1); \
614 return; \
615 }
616
617 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \
618 CK_CC_INLINE static bool \
619 ck_pr_##K##_##S##_is_zero(M *target) \
620 { \
621 T previous; \
622 C punt; \
623 punt = (C)ck_pr_md_load_##S(target); \
624 previous = (T)punt; \
625 while (ck_pr_cas_##S##_value(target, \
626 (C)previous, \
627 (C)(previous P 1), \
628 &previous) == false) \
629 ck_pr_stall(); \
630 return previous == (T)Z; \
631 }
632
633 #define CK_PR_UNARY_Z_STUB(K, S, M) \
634 CK_CC_INLINE static void \
635 ck_pr_##K##_##S##_zero(M *target, bool *zero) \
636 { \
637 *zero = ck_pr_##K##_##S##_is_zero(target); \
638 return; \
639 }
640
641 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M)
642 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) \
643 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \
644 CK_PR_UNARY_Z_STUB(K, S, M)
645
646 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
647
648 #ifndef CK_F_PR_INC_CHAR
649 #define CK_F_PR_INC_CHAR
650 CK_PR_UNARY_S(inc, add, char, char)
651 #endif /* CK_F_PR_INC_CHAR */
652
653 #ifndef CK_F_PR_INC_CHAR_ZERO
654 #define CK_F_PR_INC_CHAR_ZERO
655 CK_PR_UNARY_Z_S(inc, char, char, +, -1)
656 #else
657 CK_PR_UNARY_Z_STUB(inc, char, char)
658 #endif /* CK_F_PR_INC_CHAR_ZERO */
659
660 #ifndef CK_F_PR_DEC_CHAR
661 #define CK_F_PR_DEC_CHAR
662 CK_PR_UNARY_S(dec, sub, char, char)
663 #endif /* CK_F_PR_DEC_CHAR */
664
665 #ifndef CK_F_PR_DEC_CHAR_ZERO
666 #define CK_F_PR_DEC_CHAR_ZERO
667 CK_PR_UNARY_Z_S(dec, char, char, -, 1)
668 #else
669 CK_PR_UNARY_Z_STUB(dec, char, char)
670 #endif /* CK_F_PR_DEC_CHAR_ZERO */
671
672 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
673
674 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
675
676 #ifndef CK_F_PR_INC_INT
677 #define CK_F_PR_INC_INT
678 CK_PR_UNARY_S(inc, add, int, int)
679 #endif /* CK_F_PR_INC_INT */
680
681 #ifndef CK_F_PR_INC_INT_ZERO
682 #define CK_F_PR_INC_INT_ZERO
683 CK_PR_UNARY_Z_S(inc, int, int, +, -1)
684 #else
685 CK_PR_UNARY_Z_STUB(inc, int, int)
686 #endif /* CK_F_PR_INC_INT_ZERO */
687
688 #ifndef CK_F_PR_DEC_INT
689 #define CK_F_PR_DEC_INT
690 CK_PR_UNARY_S(dec, sub, int, int)
691 #endif /* CK_F_PR_DEC_INT */
692
693 #ifndef CK_F_PR_DEC_INT_ZERO
694 #define CK_F_PR_DEC_INT_ZERO
695 CK_PR_UNARY_Z_S(dec, int, int, -, 1)
696 #else
697 CK_PR_UNARY_Z_STUB(dec, int, int)
698 #endif /* CK_F_PR_DEC_INT_ZERO */
699
700 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
701
702 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
703 !defined(CK_PR_DISABLE_DOUBLE)
704
705 #ifndef CK_F_PR_INC_DOUBLE
706 #define CK_F_PR_INC_DOUBLE
707 CK_PR_UNARY_S(inc, add, double, double)
708 #endif /* CK_F_PR_INC_DOUBLE */
709
710 #ifndef CK_F_PR_DEC_DOUBLE
711 #define CK_F_PR_DEC_DOUBLE
712 CK_PR_UNARY_S(dec, sub, double, double)
713 #endif /* CK_F_PR_DEC_DOUBLE */
714
715 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
716
717 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
718
719 #ifndef CK_F_PR_INC_UINT
720 #define CK_F_PR_INC_UINT
721 CK_PR_UNARY_S(inc, add, uint, unsigned int)
722 #endif /* CK_F_PR_INC_UINT */
723
724 #ifndef CK_F_PR_INC_UINT_ZERO
725 #define CK_F_PR_INC_UINT_ZERO
726 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX)
727 #else
728 CK_PR_UNARY_Z_STUB(inc, uint, unsigned int)
729 #endif /* CK_F_PR_INC_UINT_ZERO */
730
731 #ifndef CK_F_PR_DEC_UINT
732 #define CK_F_PR_DEC_UINT
733 CK_PR_UNARY_S(dec, sub, uint, unsigned int)
734 #endif /* CK_F_PR_DEC_UINT */
735
736 #ifndef CK_F_PR_DEC_UINT_ZERO
737 #define CK_F_PR_DEC_UINT_ZERO
738 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1)
739 #else
740 CK_PR_UNARY_Z_STUB(dec, uint, unsigned int)
741 #endif /* CK_F_PR_DEC_UINT_ZERO */
742
743 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
744
745 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
746
747 #ifndef CK_F_PR_INC_PTR
748 #define CK_F_PR_INC_PTR
749 CK_PR_UNARY(inc, add, ptr, void, uintptr_t)
750 #endif /* CK_F_PR_INC_PTR */
751
752 #ifndef CK_F_PR_INC_PTR_ZERO
753 #define CK_F_PR_INC_PTR_ZERO
754 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX)
755 #else
756 CK_PR_UNARY_Z_STUB(inc, ptr, void)
757 #endif /* CK_F_PR_INC_PTR_ZERO */
758
759 #ifndef CK_F_PR_DEC_PTR
760 #define CK_F_PR_DEC_PTR
761 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t)
762 #endif /* CK_F_PR_DEC_PTR */
763
764 #ifndef CK_F_PR_DEC_PTR_ZERO
765 #define CK_F_PR_DEC_PTR_ZERO
766 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1)
767 #else
768 CK_PR_UNARY_Z_STUB(dec, ptr, void)
769 #endif /* CK_F_PR_DEC_PTR_ZERO */
770
771 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
772
773 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
774
775 #ifndef CK_F_PR_INC_64
776 #define CK_F_PR_INC_64
777 CK_PR_UNARY_S(inc, add, 64, uint64_t)
778 #endif /* CK_F_PR_INC_64 */
779
780 #ifndef CK_F_PR_INC_64_ZERO
781 #define CK_F_PR_INC_64_ZERO
782 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX)
783 #else
784 CK_PR_UNARY_Z_STUB(inc, 64, uint64_t)
785 #endif /* CK_F_PR_INC_64_ZERO */
786
787 #ifndef CK_F_PR_DEC_64
788 #define CK_F_PR_DEC_64
789 CK_PR_UNARY_S(dec, sub, 64, uint64_t)
790 #endif /* CK_F_PR_DEC_64 */
791
792 #ifndef CK_F_PR_DEC_64_ZERO
793 #define CK_F_PR_DEC_64_ZERO
794 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1)
795 #else
796 CK_PR_UNARY_Z_STUB(dec, 64, uint64_t)
797 #endif /* CK_F_PR_DEC_64_ZERO */
798
799 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
800
801 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
802
803 #ifndef CK_F_PR_INC_32
804 #define CK_F_PR_INC_32
805 CK_PR_UNARY_S(inc, add, 32, uint32_t)
806 #endif /* CK_F_PR_INC_32 */
807
808 #ifndef CK_F_PR_INC_32_ZERO
809 #define CK_F_PR_INC_32_ZERO
810 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX)
811 #else
812 CK_PR_UNARY_Z_STUB(inc, 32, uint32_t)
813 #endif /* CK_F_PR_INC_32_ZERO */
814
815 #ifndef CK_F_PR_DEC_32
816 #define CK_F_PR_DEC_32
817 CK_PR_UNARY_S(dec, sub, 32, uint32_t)
818 #endif /* CK_F_PR_DEC_32 */
819
820 #ifndef CK_F_PR_DEC_32_ZERO
821 #define CK_F_PR_DEC_32_ZERO
822 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1)
823 #else
824 CK_PR_UNARY_Z_STUB(dec, 32, uint32_t)
825 #endif /* CK_F_PR_DEC_32_ZERO */
826
827 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
828
829 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
830
831 #ifndef CK_F_PR_INC_16
832 #define CK_F_PR_INC_16
833 CK_PR_UNARY_S(inc, add, 16, uint16_t)
834 #endif /* CK_F_PR_INC_16 */
835
836 #ifndef CK_F_PR_INC_16_ZERO
837 #define CK_F_PR_INC_16_ZERO
838 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX)
839 #else
840 CK_PR_UNARY_Z_STUB(inc, 16, uint16_t)
841 #endif /* CK_F_PR_INC_16_ZERO */
842
843 #ifndef CK_F_PR_DEC_16
844 #define CK_F_PR_DEC_16
845 CK_PR_UNARY_S(dec, sub, 16, uint16_t)
846 #endif /* CK_F_PR_DEC_16 */
847
848 #ifndef CK_F_PR_DEC_16_ZERO
849 #define CK_F_PR_DEC_16_ZERO
850 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1)
851 #else
852 CK_PR_UNARY_Z_STUB(dec, 16, uint16_t)
853 #endif /* CK_F_PR_DEC_16_ZERO */
854
855 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
856
857 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
858
859 #ifndef CK_F_PR_INC_8
860 #define CK_F_PR_INC_8
861 CK_PR_UNARY_S(inc, add, 8, uint8_t)
862 #endif /* CK_F_PR_INC_8 */
863
864 #ifndef CK_F_PR_INC_8_ZERO
865 #define CK_F_PR_INC_8_ZERO
866 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX)
867 #else
868 CK_PR_UNARY_Z_STUB(inc, 8, uint8_t)
869 #endif /* CK_F_PR_INC_8_ZERO */
870
871 #ifndef CK_F_PR_DEC_8
872 #define CK_F_PR_DEC_8
873 CK_PR_UNARY_S(dec, sub, 8, uint8_t)
874 #endif /* CK_F_PR_DEC_8 */
875
876 #ifndef CK_F_PR_DEC_8_ZERO
877 #define CK_F_PR_DEC_8_ZERO
878 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1)
879 #else
880 CK_PR_UNARY_Z_STUB(dec, 8, uint8_t)
881 #endif /* CK_F_PR_DEC_8_ZERO */
882
883 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
884
885 #undef CK_PR_UNARY_Z_S
886 #undef CK_PR_UNARY_S
887 #undef CK_PR_UNARY_Z
888 #undef CK_PR_UNARY
889
890 #define CK_PR_N(K, S, M, T, P, C) \
891 CK_CC_INLINE static void \
892 ck_pr_##K##_##S(M *target) \
893 { \
894 T previous; \
895 C punt; \
896 punt = (C)ck_pr_md_load_##S(target); \
897 previous = (T)punt; \
898 while (ck_pr_cas_##S##_value(target, \
899 (C)previous, \
900 (C)(P previous), \
901 &previous) == false) \
902 ck_pr_stall(); \
903 \
904 return; \
905 }
906
907 #define CK_PR_N_Z(S, M, T, C) \
908 CK_CC_INLINE static void \
909 ck_pr_neg_##S##_zero(M *target, bool *zero) \
910 { \
911 T previous; \
912 C punt; \
913 punt = (C)ck_pr_md_load_##S(target); \
914 previous = (T)punt; \
915 while (ck_pr_cas_##S##_value(target, \
916 (C)previous, \
917 (C)(-previous), \
918 &previous) == false) \
919 ck_pr_stall(); \
920 \
921 *zero = previous == 0; \
922 return; \
923 }
924
925 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M)
926 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M)
927
928 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
929
930 #ifndef CK_F_PR_NOT_CHAR
931 #define CK_F_PR_NOT_CHAR
932 CK_PR_N_S(not, char, char, ~)
933 #endif /* CK_F_PR_NOT_CHAR */
934
935 #ifndef CK_F_PR_NEG_CHAR
936 #define CK_F_PR_NEG_CHAR
937 CK_PR_N_S(neg, char, char, -)
938 #endif /* CK_F_PR_NEG_CHAR */
939
940 #ifndef CK_F_PR_NEG_CHAR_ZERO
941 #define CK_F_PR_NEG_CHAR_ZERO
942 CK_PR_N_Z_S(char, char)
943 #endif /* CK_F_PR_NEG_CHAR_ZERO */
944
945 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
946
947 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
948
949 #ifndef CK_F_PR_NOT_INT
950 #define CK_F_PR_NOT_INT
951 CK_PR_N_S(not, int, int, ~)
952 #endif /* CK_F_PR_NOT_INT */
953
954 #ifndef CK_F_PR_NEG_INT
955 #define CK_F_PR_NEG_INT
956 CK_PR_N_S(neg, int, int, -)
957 #endif /* CK_F_PR_NEG_INT */
958
959 #ifndef CK_F_PR_NEG_INT_ZERO
960 #define CK_F_PR_NEG_INT_ZERO
961 CK_PR_N_Z_S(int, int)
962 #endif /* CK_F_PR_NEG_INT_ZERO */
963
964 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
965
966 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
967 !defined(CK_PR_DISABLE_DOUBLE)
968
969 #ifndef CK_F_PR_NEG_DOUBLE
970 #define CK_F_PR_NEG_DOUBLE
971 CK_PR_N_S(neg, double, double, -)
972 #endif /* CK_F_PR_NEG_DOUBLE */
973
974 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
975
976 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
977
978 #ifndef CK_F_PR_NOT_UINT
979 #define CK_F_PR_NOT_UINT
980 CK_PR_N_S(not, uint, unsigned int, ~)
981 #endif /* CK_F_PR_NOT_UINT */
982
983 #ifndef CK_F_PR_NEG_UINT
984 #define CK_F_PR_NEG_UINT
985 CK_PR_N_S(neg, uint, unsigned int, -)
986 #endif /* CK_F_PR_NEG_UINT */
987
988 #ifndef CK_F_PR_NEG_UINT_ZERO
989 #define CK_F_PR_NEG_UINT_ZERO
990 CK_PR_N_Z_S(uint, unsigned int)
991 #endif /* CK_F_PR_NEG_UINT_ZERO */
992
993 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
994
995 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
996
997 #ifndef CK_F_PR_NOT_PTR
998 #define CK_F_PR_NOT_PTR
999 CK_PR_N(not, ptr, void, uintptr_t, ~, void *)
1000 #endif /* CK_F_PR_NOT_PTR */
1001
1002 #ifndef CK_F_PR_NEG_PTR
1003 #define CK_F_PR_NEG_PTR
1004 CK_PR_N(neg, ptr, void, uintptr_t, -, void *)
1005 #endif /* CK_F_PR_NEG_PTR */
1006
1007 #ifndef CK_F_PR_NEG_PTR_ZERO
1008 #define CK_F_PR_NEG_PTR_ZERO
1009 CK_PR_N_Z(ptr, void, uintptr_t, void *)
1010 #endif /* CK_F_PR_NEG_PTR_ZERO */
1011
1012 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1013
1014 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1015
1016 #ifndef CK_F_PR_NOT_64
1017 #define CK_F_PR_NOT_64
1018 CK_PR_N_S(not, 64, uint64_t, ~)
1019 #endif /* CK_F_PR_NOT_64 */
1020
1021 #ifndef CK_F_PR_NEG_64
1022 #define CK_F_PR_NEG_64
1023 CK_PR_N_S(neg, 64, uint64_t, -)
1024 #endif /* CK_F_PR_NEG_64 */
1025
1026 #ifndef CK_F_PR_NEG_64_ZERO
1027 #define CK_F_PR_NEG_64_ZERO
1028 CK_PR_N_Z_S(64, uint64_t)
1029 #endif /* CK_F_PR_NEG_64_ZERO */
1030
1031 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1032
1033 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1034
1035 #ifndef CK_F_PR_NOT_32
1036 #define CK_F_PR_NOT_32
1037 CK_PR_N_S(not, 32, uint32_t, ~)
1038 #endif /* CK_F_PR_NOT_32 */
1039
1040 #ifndef CK_F_PR_NEG_32
1041 #define CK_F_PR_NEG_32
1042 CK_PR_N_S(neg, 32, uint32_t, -)
1043 #endif /* CK_F_PR_NEG_32 */
1044
1045 #ifndef CK_F_PR_NEG_32_ZERO
1046 #define CK_F_PR_NEG_32_ZERO
1047 CK_PR_N_Z_S(32, uint32_t)
1048 #endif /* CK_F_PR_NEG_32_ZERO */
1049
1050 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1051
1052 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1053
1054 #ifndef CK_F_PR_NOT_16
1055 #define CK_F_PR_NOT_16
1056 CK_PR_N_S(not, 16, uint16_t, ~)
1057 #endif /* CK_F_PR_NOT_16 */
1058
1059 #ifndef CK_F_PR_NEG_16
1060 #define CK_F_PR_NEG_16
1061 CK_PR_N_S(neg, 16, uint16_t, -)
1062 #endif /* CK_F_PR_NEG_16 */
1063
1064 #ifndef CK_F_PR_NEG_16_ZERO
1065 #define CK_F_PR_NEG_16_ZERO
1066 CK_PR_N_Z_S(16, uint16_t)
1067 #endif /* CK_F_PR_NEG_16_ZERO */
1068
1069 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1070
1071 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1072
1073 #ifndef CK_F_PR_NOT_8
1074 #define CK_F_PR_NOT_8
1075 CK_PR_N_S(not, 8, uint8_t, ~)
1076 #endif /* CK_F_PR_NOT_8 */
1077
1078 #ifndef CK_F_PR_NEG_8
1079 #define CK_F_PR_NEG_8
1080 CK_PR_N_S(neg, 8, uint8_t, -)
1081 #endif /* CK_F_PR_NEG_8 */
1082
1083 #ifndef CK_F_PR_NEG_8_ZERO
1084 #define CK_F_PR_NEG_8_ZERO
1085 CK_PR_N_Z_S(8, uint8_t)
1086 #endif /* CK_F_PR_NEG_8_ZERO */
1087
1088 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1089
1090 #undef CK_PR_N_Z_S
1091 #undef CK_PR_N_S
1092 #undef CK_PR_N_Z
1093 #undef CK_PR_N
1094
1095 #define CK_PR_FAA(S, M, T, C) \
1096 CK_CC_INLINE static C \
1097 ck_pr_faa_##S(M *target, T delta) \
1098 { \
1099 T previous; \
1100 C punt; \
1101 punt = (C)ck_pr_md_load_##S(target); \
1102 previous = (T)punt; \
1103 while (ck_pr_cas_##S##_value(target, \
1104 (C)previous, \
1105 (C)(previous + delta), \
1106 &previous) == false) \
1107 ck_pr_stall(); \
1108 \
1109 return ((C)previous); \
1110 }
1111
1112 #define CK_PR_FAS(S, M, C) \
1113 CK_CC_INLINE static C \
1114 ck_pr_fas_##S(M *target, C update) \
1115 { \
1116 C previous; \
1117 previous = ck_pr_md_load_##S(target); \
1118 while (ck_pr_cas_##S##_value(target, \
1119 previous, \
1120 update, \
1121 &previous) == false) \
1122 ck_pr_stall(); \
1123 \
1124 return (previous); \
1125 }
1126
1127 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M)
1128 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M)
1129
1130 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
1131
1132 #ifndef CK_F_PR_FAA_CHAR
1133 #define CK_F_PR_FAA_CHAR
1134 CK_PR_FAA_S(char, char)
1135 #endif /* CK_F_PR_FAA_CHAR */
1136
1137 #ifndef CK_F_PR_FAS_CHAR
1138 #define CK_F_PR_FAS_CHAR
1139 CK_PR_FAS_S(char, char)
1140 #endif /* CK_F_PR_FAS_CHAR */
1141
1142 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
1143
1144 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
1145
1146 #ifndef CK_F_PR_FAA_INT
1147 #define CK_F_PR_FAA_INT
1148 CK_PR_FAA_S(int, int)
1149 #endif /* CK_F_PR_FAA_INT */
1150
1151 #ifndef CK_F_PR_FAS_INT
1152 #define CK_F_PR_FAS_INT
1153 CK_PR_FAS_S(int, int)
1154 #endif /* CK_F_PR_FAS_INT */
1155
1156 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
1157
1158 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
1159 !defined(CK_PR_DISABLE_DOUBLE)
1160
1161 #ifndef CK_F_PR_FAA_DOUBLE
1162 #define CK_F_PR_FAA_DOUBLE
1163 CK_PR_FAA_S(double, double)
1164 #endif /* CK_F_PR_FAA_DOUBLE */
1165
1166 #ifndef CK_F_PR_FAS_DOUBLE
1167 #define CK_F_PR_FAS_DOUBLE
1168 CK_PR_FAS_S(double, double)
1169 #endif /* CK_F_PR_FAS_DOUBLE */
1170
1171 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
1172
1173 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
1174
1175 #ifndef CK_F_PR_FAA_UINT
1176 #define CK_F_PR_FAA_UINT
1177 CK_PR_FAA_S(uint, unsigned int)
1178 #endif /* CK_F_PR_FAA_UINT */
1179
1180 #ifndef CK_F_PR_FAS_UINT
1181 #define CK_F_PR_FAS_UINT
1182 CK_PR_FAS_S(uint, unsigned int)
1183 #endif /* CK_F_PR_FAS_UINT */
1184
1185 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1186
1187 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1188
1189 #ifndef CK_F_PR_FAA_PTR
1190 #define CK_F_PR_FAA_PTR
1191 CK_PR_FAA(ptr, void, uintptr_t, void *)
1192 #endif /* CK_F_PR_FAA_PTR */
1193
1194 #ifndef CK_F_PR_FAS_PTR
1195 #define CK_F_PR_FAS_PTR
1196 CK_PR_FAS(ptr, void, void *)
1197 #endif /* CK_F_PR_FAS_PTR */
1198
1199 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1200
1201 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1202
1203 #ifndef CK_F_PR_FAA_64
1204 #define CK_F_PR_FAA_64
1205 CK_PR_FAA_S(64, uint64_t)
1206 #endif /* CK_F_PR_FAA_64 */
1207
1208 #ifndef CK_F_PR_FAS_64
1209 #define CK_F_PR_FAS_64
1210 CK_PR_FAS_S(64, uint64_t)
1211 #endif /* CK_F_PR_FAS_64 */
1212
1213 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1214
1215 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1216
1217 #ifndef CK_F_PR_FAA_32
1218 #define CK_F_PR_FAA_32
1219 CK_PR_FAA_S(32, uint32_t)
1220 #endif /* CK_F_PR_FAA_32 */
1221
1222 #ifndef CK_F_PR_FAS_32
1223 #define CK_F_PR_FAS_32
1224 CK_PR_FAS_S(32, uint32_t)
1225 #endif /* CK_F_PR_FAS_32 */
1226
1227 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1228
1229 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1230
1231 #ifndef CK_F_PR_FAA_16
1232 #define CK_F_PR_FAA_16
1233 CK_PR_FAA_S(16, uint16_t)
1234 #endif /* CK_F_PR_FAA_16 */
1235
1236 #ifndef CK_F_PR_FAS_16
1237 #define CK_F_PR_FAS_16
1238 CK_PR_FAS_S(16, uint16_t)
1239 #endif /* CK_F_PR_FAS_16 */
1240
1241 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1242
1243 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1244
1245 #ifndef CK_F_PR_FAA_8
1246 #define CK_F_PR_FAA_8
1247 CK_PR_FAA_S(8, uint8_t)
1248 #endif /* CK_F_PR_FAA_8 */
1249
1250 #ifndef CK_F_PR_FAS_8
1251 #define CK_F_PR_FAS_8
1252 CK_PR_FAS_S(8, uint8_t)
1253 #endif /* CK_F_PR_FAS_8 */
1254
1255 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1256
1257 #undef CK_PR_FAA_S
1258 #undef CK_PR_FAS_S
1259 #undef CK_PR_FAA
1260 #undef CK_PR_FAS
1261
1262 #endif /* CK_PR_H */
1263