1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License"). You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (c) 2009 by Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 #include <atomic.h>
28 #include <assert.h>
29 #include <pthread.h>
30
31 /*
32 * All operations are implemented by serializing them through a global
33 * pthread mutex. This provides a correct generic implementation.
34 * However all supported architectures are encouraged to provide a
35 * native implementation is assembly for performance reasons.
36 */
37 pthread_mutex_t atomic_lock = PTHREAD_MUTEX_INITIALIZER;
38
39 /*
40 * These are the void returning variants
41 */
42 /* BEGIN CSTYLED */
43 #define ATOMIC_INC(name, type) \
44 void atomic_inc_##name(volatile type *target) \
45 { \
46 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
47 (*target)++; \
48 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
49 }
50
51 ATOMIC_INC(8, uint8_t)
ATOMIC_INC(uchar,uchar_t)52 ATOMIC_INC(uchar, uchar_t)
53 ATOMIC_INC(16, uint16_t)
54 ATOMIC_INC(ushort, ushort_t)
55 ATOMIC_INC(32, uint32_t)
56 ATOMIC_INC(uint, uint_t)
57 ATOMIC_INC(ulong, ulong_t)
58 ATOMIC_INC(64, uint64_t)
59
60
61 #define ATOMIC_DEC(name, type) \
62 void atomic_dec_##name(volatile type *target) \
63 { \
64 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
65 (*target)--; \
66 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
67 }
68
69 ATOMIC_DEC(8, uint8_t)
70 ATOMIC_DEC(uchar, uchar_t)
71 ATOMIC_DEC(16, uint16_t)
72 ATOMIC_DEC(ushort, ushort_t)
73 ATOMIC_DEC(32, uint32_t)
74 ATOMIC_DEC(uint, uint_t)
75 ATOMIC_DEC(ulong, ulong_t)
76 ATOMIC_DEC(64, uint64_t)
77
78
79 #define ATOMIC_ADD(name, type1, type2) \
80 void atomic_add_##name(volatile type1 *target, type2 bits) \
81 { \
82 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
83 *target += bits; \
84 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
85 }
86
87 ATOMIC_ADD(8, uint8_t, int8_t)
88 ATOMIC_ADD(char, uchar_t, signed char)
89 ATOMIC_ADD(16, uint16_t, int16_t)
90 ATOMIC_ADD(short, ushort_t, short)
91 ATOMIC_ADD(32, uint32_t, int32_t)
92 ATOMIC_ADD(int, uint_t, int)
93 ATOMIC_ADD(long, ulong_t, long)
94 ATOMIC_ADD(64, uint64_t, int64_t)
95
96 void
97 atomic_add_ptr(volatile void *target, ssize_t bits)
98 {
99 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
100 *(caddr_t *)target += bits;
101 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
102 }
103
104
105 #define ATOMIC_SUB(name, type1, type2) \
106 void atomic_sub_##name(volatile type1 *target, type2 bits) \
107 { \
108 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
109 *target -= bits; \
110 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
111 }
112
113 ATOMIC_SUB(8, uint8_t, int8_t)
ATOMIC_SUB(char,uchar_t,signed char)114 ATOMIC_SUB(char, uchar_t, signed char)
115 ATOMIC_SUB(16, uint16_t, int16_t)
116 ATOMIC_SUB(short, ushort_t, short)
117 ATOMIC_SUB(32, uint32_t, int32_t)
118 ATOMIC_SUB(int, uint_t, int)
119 ATOMIC_SUB(long, ulong_t, long)
120 ATOMIC_SUB(64, uint64_t, int64_t)
121
122 void
123 atomic_sub_ptr(volatile void *target, ssize_t bits)
124 {
125 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
126 *(caddr_t *)target -= bits;
127 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
128 }
129
130
131 #define ATOMIC_OR(name, type) \
132 void atomic_or_##name(volatile type *target, type bits) \
133 { \
134 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
135 *target |= bits; \
136 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
137 }
138
139 ATOMIC_OR(8, uint8_t)
ATOMIC_OR(uchar,uchar_t)140 ATOMIC_OR(uchar, uchar_t)
141 ATOMIC_OR(16, uint16_t)
142 ATOMIC_OR(ushort, ushort_t)
143 ATOMIC_OR(32, uint32_t)
144 ATOMIC_OR(uint, uint_t)
145 ATOMIC_OR(ulong, ulong_t)
146 ATOMIC_OR(64, uint64_t)
147
148
149 #define ATOMIC_AND(name, type) \
150 void atomic_and_##name(volatile type *target, type bits) \
151 { \
152 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
153 *target &= bits; \
154 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
155 }
156
157 ATOMIC_AND(8, uint8_t)
158 ATOMIC_AND(uchar, uchar_t)
159 ATOMIC_AND(16, uint16_t)
160 ATOMIC_AND(ushort, ushort_t)
161 ATOMIC_AND(32, uint32_t)
162 ATOMIC_AND(uint, uint_t)
163 ATOMIC_AND(ulong, ulong_t)
164 ATOMIC_AND(64, uint64_t)
165
166
167 /*
168 * New value returning variants
169 */
170
171 #define ATOMIC_INC_NV(name, type) \
172 type atomic_inc_##name##_nv(volatile type *target) \
173 { \
174 type rc; \
175 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
176 rc = (++(*target)); \
177 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
178 return (rc); \
179 }
180
181 ATOMIC_INC_NV(8, uint8_t)
182 ATOMIC_INC_NV(uchar, uchar_t)
183 ATOMIC_INC_NV(16, uint16_t)
184 ATOMIC_INC_NV(ushort, ushort_t)
185 ATOMIC_INC_NV(32, uint32_t)
186 ATOMIC_INC_NV(uint, uint_t)
187 ATOMIC_INC_NV(ulong, ulong_t)
188 ATOMIC_INC_NV(64, uint64_t)
189
190
191 #define ATOMIC_DEC_NV(name, type) \
192 type atomic_dec_##name##_nv(volatile type *target) \
193 { \
194 type rc; \
195 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
196 rc = (--(*target)); \
197 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
198 return (rc); \
199 }
200
201 ATOMIC_DEC_NV(8, uint8_t)
202 ATOMIC_DEC_NV(uchar, uchar_t)
203 ATOMIC_DEC_NV(16, uint16_t)
204 ATOMIC_DEC_NV(ushort, ushort_t)
205 ATOMIC_DEC_NV(32, uint32_t)
206 ATOMIC_DEC_NV(uint, uint_t)
207 ATOMIC_DEC_NV(ulong, ulong_t)
208 ATOMIC_DEC_NV(64, uint64_t)
209
210
211 #define ATOMIC_ADD_NV(name, type1, type2) \
212 type1 atomic_add_##name##_nv(volatile type1 *target, type2 bits)\
213 { \
214 type1 rc; \
215 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
216 rc = (*target += bits); \
217 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
218 return (rc); \
219 }
220
221 ATOMIC_ADD_NV(8, uint8_t, int8_t)
222 ATOMIC_ADD_NV(char, uchar_t, signed char)
223 ATOMIC_ADD_NV(16, uint16_t, int16_t)
224 ATOMIC_ADD_NV(short, ushort_t, short)
225 ATOMIC_ADD_NV(32, uint32_t, int32_t)
226 ATOMIC_ADD_NV(int, uint_t, int)
227 ATOMIC_ADD_NV(long, ulong_t, long)
228 ATOMIC_ADD_NV(64, uint64_t, int64_t)
229
230 void *
231 atomic_add_ptr_nv(volatile void *target, ssize_t bits)
232 {
233 void *ptr;
234
235 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
236 ptr = (*(caddr_t *)target += bits);
237 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
238
239 return (ptr);
240 }
241
242
243 #define ATOMIC_SUB_NV(name, type1, type2) \
244 type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits)\
245 { \
246 type1 rc; \
247 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
248 rc = (*target -= bits); \
249 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
250 return (rc); \
251 }
252
253 ATOMIC_SUB_NV(8, uint8_t, int8_t)
ATOMIC_SUB_NV(char,uchar_t,signed char)254 ATOMIC_SUB_NV(char, uchar_t, signed char)
255 ATOMIC_SUB_NV(16, uint16_t, int16_t)
256 ATOMIC_SUB_NV(short, ushort_t, short)
257 ATOMIC_SUB_NV(32, uint32_t, int32_t)
258 ATOMIC_SUB_NV(int, uint_t, int)
259 ATOMIC_SUB_NV(long, ulong_t, long)
260 ATOMIC_SUB_NV(64, uint64_t, int64_t)
261
262 void *
263 atomic_sub_ptr_nv(volatile void *target, ssize_t bits)
264 {
265 void *ptr;
266
267 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
268 ptr = (*(caddr_t *)target -= bits);
269 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
270
271 return (ptr);
272 }
273
274
275 #define ATOMIC_OR_NV(name, type) \
276 type atomic_or_##name##_nv(volatile type *target, type bits) \
277 { \
278 type rc; \
279 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
280 rc = (*target |= bits); \
281 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
282 return (rc); \
283 }
284
285 ATOMIC_OR_NV(8, uint8_t)
ATOMIC_OR_NV(uchar,uchar_t)286 ATOMIC_OR_NV(uchar, uchar_t)
287 ATOMIC_OR_NV(16, uint16_t)
288 ATOMIC_OR_NV(ushort, ushort_t)
289 ATOMIC_OR_NV(32, uint32_t)
290 ATOMIC_OR_NV(uint, uint_t)
291 ATOMIC_OR_NV(ulong, ulong_t)
292 ATOMIC_OR_NV(64, uint64_t)
293
294
295 #define ATOMIC_AND_NV(name, type) \
296 type atomic_and_##name##_nv(volatile type *target, type bits) \
297 { \
298 type rc; \
299 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
300 rc = (*target &= bits); \
301 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
302 return (rc); \
303 }
304
305 ATOMIC_AND_NV(8, uint8_t)
306 ATOMIC_AND_NV(uchar, uchar_t)
307 ATOMIC_AND_NV(16, uint16_t)
308 ATOMIC_AND_NV(ushort, ushort_t)
309 ATOMIC_AND_NV(32, uint32_t)
310 ATOMIC_AND_NV(uint, uint_t)
311 ATOMIC_AND_NV(ulong, ulong_t)
312 ATOMIC_AND_NV(64, uint64_t)
313
314
315 /*
316 * If *arg1 == arg2, set *arg1 = arg3; return old value
317 */
318
319 #define ATOMIC_CAS(name, type) \
320 type atomic_cas_##name(volatile type *target, type arg1, type arg2) \
321 { \
322 type old; \
323 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
324 old = *target; \
325 if (old == arg1) \
326 *target = arg2; \
327 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
328 return (old); \
329 }
330
331 ATOMIC_CAS(8, uint8_t)
332 ATOMIC_CAS(uchar, uchar_t)
333 ATOMIC_CAS(16, uint16_t)
334 ATOMIC_CAS(ushort, ushort_t)
335 ATOMIC_CAS(32, uint32_t)
336 ATOMIC_CAS(uint, uint_t)
337 ATOMIC_CAS(ulong, ulong_t)
338 ATOMIC_CAS(64, uint64_t)
339
340 void *
341 atomic_cas_ptr(volatile void *target, void *arg1, void *arg2)
342 {
343 void *old;
344
345 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
346 old = *(void **)target;
347 if (old == arg1)
348 *(void **)target = arg2;
349 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
350
351 return (old);
352 }
353
354
355 /*
356 * Swap target and return old value
357 */
358
359 #define ATOMIC_SWAP(name, type) \
360 type atomic_swap_##name(volatile type *target, type bits) \
361 { \
362 type old; \
363 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
364 old = *target; \
365 *target = bits; \
366 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
367 return (old); \
368 }
369
370 ATOMIC_SWAP(8, uint8_t)
ATOMIC_SWAP(uchar,uchar_t)371 ATOMIC_SWAP(uchar, uchar_t)
372 ATOMIC_SWAP(16, uint16_t)
373 ATOMIC_SWAP(ushort, ushort_t)
374 ATOMIC_SWAP(32, uint32_t)
375 ATOMIC_SWAP(uint, uint_t)
376 ATOMIC_SWAP(ulong, ulong_t)
377 ATOMIC_SWAP(64, uint64_t)
378 /* END CSTYLED */
379
380 void *
381 atomic_swap_ptr(volatile void *target, void *bits)
382 {
383 void *old;
384
385 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
386 old = *(void **)target;
387 *(void **)target = bits;
388 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
389
390 return (old);
391 }
392
393
394 int
atomic_set_long_excl(volatile ulong_t * target,uint_t value)395 atomic_set_long_excl(volatile ulong_t *target, uint_t value)
396 {
397 ulong_t bit;
398
399 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
400 bit = (1UL << value);
401 if ((*target & bit) != 0) {
402 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
403 return (-1);
404 }
405 *target |= bit;
406 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
407
408 return (0);
409 }
410
411 int
atomic_clear_long_excl(volatile ulong_t * target,uint_t value)412 atomic_clear_long_excl(volatile ulong_t *target, uint_t value)
413 {
414 ulong_t bit;
415
416 VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
417 bit = (1UL << value);
418 if ((*target & bit) == 0) {
419 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
420 return (-1);
421 }
422 *target &= ~bit;
423 VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
424
425 return (0);
426 }
427
428 void
membar_enter(void)429 membar_enter(void)
430 {
431 /* XXX - Implement me */
432 }
433
434 void
membar_exit(void)435 membar_exit(void)
436 {
437 /* XXX - Implement me */
438 }
439
440 void
membar_producer(void)441 membar_producer(void)
442 {
443 /* XXX - Implement me */
444 }
445
446 void
membar_consumer(void)447 membar_consumer(void)
448 {
449 /* XXX - Implement me */
450 }
451