1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 */
4
5 /*
6 * Inspired from FreeBSD src/sys/i386/include/atomic.h
7 * Copyright (c) 1998 Doug Rabson
8 * All rights reserved.
9 */
10
11 #ifndef _RTE_ATOMIC_X86_H_
12 #error do not include this file directly, use <rte_atomic.h> instead
13 #endif
14
15 #ifndef _RTE_ATOMIC_I686_H_
16 #define _RTE_ATOMIC_I686_H_
17
18 #include <stdint.h>
19 #include <rte_common.h>
20
21 /*------------------------- 64 bit atomic operations -------------------------*/
22
23 #ifndef RTE_FORCE_INTRINSICS
24 static inline int
rte_atomic64_cmpset(volatile uint64_t * dst,uint64_t exp,uint64_t src)25 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
26 {
27 uint8_t res;
28 RTE_STD_C11
29 union {
30 struct {
31 uint32_t l32;
32 uint32_t h32;
33 };
34 uint64_t u64;
35 } _exp, _src;
36
37 _exp.u64 = exp;
38 _src.u64 = src;
39
40 #ifndef __PIC__
41 asm volatile (
42 MPLOCKED
43 "cmpxchg8b (%[dst]);"
44 "setz %[res];"
45 : [res] "=a" (res) /* result in eax */
46 : [dst] "S" (dst), /* esi */
47 "b" (_src.l32), /* ebx */
48 "c" (_src.h32), /* ecx */
49 "a" (_exp.l32), /* eax */
50 "d" (_exp.h32) /* edx */
51 : "memory" ); /* no-clobber list */
52 #else
53 asm volatile (
54 "xchgl %%ebx, %%edi;\n"
55 MPLOCKED
56 "cmpxchg8b (%[dst]);"
57 "setz %[res];"
58 "xchgl %%ebx, %%edi;\n"
59 : [res] "=a" (res) /* result in eax */
60 : [dst] "S" (dst), /* esi */
61 "D" (_src.l32), /* ebx */
62 "c" (_src.h32), /* ecx */
63 "a" (_exp.l32), /* eax */
64 "d" (_exp.h32) /* edx */
65 : "memory" ); /* no-clobber list */
66 #endif
67
68 return res;
69 }
70
71 static inline uint64_t
rte_atomic64_exchange(volatile uint64_t * dest,uint64_t val)72 rte_atomic64_exchange(volatile uint64_t *dest, uint64_t val)
73 {
74 uint64_t old;
75
76 do {
77 old = *dest;
78 } while (rte_atomic64_cmpset(dest, old, val) == 0);
79
80 return old;
81 }
82
83 static inline void
rte_atomic64_init(rte_atomic64_t * v)84 rte_atomic64_init(rte_atomic64_t *v)
85 {
86 int success = 0;
87 uint64_t tmp;
88
89 while (success == 0) {
90 tmp = v->cnt;
91 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
92 tmp, 0);
93 }
94 }
95
96 static inline int64_t
rte_atomic64_read(rte_atomic64_t * v)97 rte_atomic64_read(rte_atomic64_t *v)
98 {
99 int success = 0;
100 uint64_t tmp;
101
102 while (success == 0) {
103 tmp = v->cnt;
104 /* replace the value by itself */
105 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
106 tmp, tmp);
107 }
108 return tmp;
109 }
110
111 static inline void
rte_atomic64_set(rte_atomic64_t * v,int64_t new_value)112 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
113 {
114 int success = 0;
115 uint64_t tmp;
116
117 while (success == 0) {
118 tmp = v->cnt;
119 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
120 tmp, new_value);
121 }
122 }
123
124 static inline void
rte_atomic64_add(rte_atomic64_t * v,int64_t inc)125 rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
126 {
127 int success = 0;
128 uint64_t tmp;
129
130 while (success == 0) {
131 tmp = v->cnt;
132 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
133 tmp, tmp + inc);
134 }
135 }
136
137 static inline void
rte_atomic64_sub(rte_atomic64_t * v,int64_t dec)138 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
139 {
140 int success = 0;
141 uint64_t tmp;
142
143 while (success == 0) {
144 tmp = v->cnt;
145 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
146 tmp, tmp - dec);
147 }
148 }
149
150 static inline void
rte_atomic64_inc(rte_atomic64_t * v)151 rte_atomic64_inc(rte_atomic64_t *v)
152 {
153 rte_atomic64_add(v, 1);
154 }
155
156 static inline void
rte_atomic64_dec(rte_atomic64_t * v)157 rte_atomic64_dec(rte_atomic64_t *v)
158 {
159 rte_atomic64_sub(v, 1);
160 }
161
162 static inline int64_t
rte_atomic64_add_return(rte_atomic64_t * v,int64_t inc)163 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
164 {
165 int success = 0;
166 uint64_t tmp;
167
168 while (success == 0) {
169 tmp = v->cnt;
170 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
171 tmp, tmp + inc);
172 }
173
174 return tmp + inc;
175 }
176
177 static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t * v,int64_t dec)178 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
179 {
180 int success = 0;
181 uint64_t tmp;
182
183 while (success == 0) {
184 tmp = v->cnt;
185 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
186 tmp, tmp - dec);
187 }
188
189 return tmp - dec;
190 }
191
rte_atomic64_inc_and_test(rte_atomic64_t * v)192 static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
193 {
194 return rte_atomic64_add_return(v, 1) == 0;
195 }
196
rte_atomic64_dec_and_test(rte_atomic64_t * v)197 static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
198 {
199 return rte_atomic64_sub_return(v, 1) == 0;
200 }
201
rte_atomic64_test_and_set(rte_atomic64_t * v)202 static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
203 {
204 return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
205 }
206
rte_atomic64_clear(rte_atomic64_t * v)207 static inline void rte_atomic64_clear(rte_atomic64_t *v)
208 {
209 rte_atomic64_set(v, 0);
210 }
211 #endif
212
213 #endif /* _RTE_ATOMIC_I686_H_ */
214