1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation.
3 */
4
5 /*
6 * Inspired from FreeBSD src/sys/i386/include/atomic.h
7 * Copyright (c) 1998 Doug Rabson
8 * All rights reserved.
9 */
10
11 #ifndef _RTE_ATOMIC_X86_H_
12 #error do not include this file directly, use <rte_atomic.h> instead
13 #endif
14
15 #ifndef _RTE_ATOMIC_I686_H_
16 #define _RTE_ATOMIC_I686_H_
17
18 #include <stdint.h>
19 #include <rte_common.h>
20 #include <rte_atomic.h>
21
22 /*------------------------- 64 bit atomic operations -------------------------*/
23
24 #ifndef RTE_FORCE_INTRINSICS
25 static inline int
rte_atomic64_cmpset(volatile uint64_t * dst,uint64_t exp,uint64_t src)26 rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
27 {
28 uint8_t res;
29 RTE_STD_C11
30 union {
31 struct {
32 uint32_t l32;
33 uint32_t h32;
34 };
35 uint64_t u64;
36 } _exp, _src;
37
38 _exp.u64 = exp;
39 _src.u64 = src;
40
41 #ifndef __PIC__
42 asm volatile (
43 MPLOCKED
44 "cmpxchg8b (%[dst]);"
45 "setz %[res];"
46 : [res] "=a" (res) /* result in eax */
47 : [dst] "S" (dst), /* esi */
48 "b" (_src.l32), /* ebx */
49 "c" (_src.h32), /* ecx */
50 "a" (_exp.l32), /* eax */
51 "d" (_exp.h32) /* edx */
52 : "memory" ); /* no-clobber list */
53 #else
54 asm volatile (
55 "xchgl %%ebx, %%edi;\n"
56 MPLOCKED
57 "cmpxchg8b (%[dst]);"
58 "setz %[res];"
59 "xchgl %%ebx, %%edi;\n"
60 : [res] "=a" (res) /* result in eax */
61 : [dst] "S" (dst), /* esi */
62 "D" (_src.l32), /* ebx */
63 "c" (_src.h32), /* ecx */
64 "a" (_exp.l32), /* eax */
65 "d" (_exp.h32) /* edx */
66 : "memory" ); /* no-clobber list */
67 #endif
68
69 return res;
70 }
71
72 static inline uint64_t
rte_atomic64_exchange(volatile uint64_t * dest,uint64_t val)73 rte_atomic64_exchange(volatile uint64_t *dest, uint64_t val)
74 {
75 uint64_t old;
76
77 do {
78 old = *dest;
79 } while (rte_atomic64_cmpset(dest, old, val) == 0);
80
81 return old;
82 }
83
84 static inline void
rte_atomic64_init(rte_atomic64_t * v)85 rte_atomic64_init(rte_atomic64_t *v)
86 {
87 int success = 0;
88 uint64_t tmp;
89
90 while (success == 0) {
91 tmp = v->cnt;
92 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
93 tmp, 0);
94 }
95 }
96
97 static inline int64_t
rte_atomic64_read(rte_atomic64_t * v)98 rte_atomic64_read(rte_atomic64_t *v)
99 {
100 int success = 0;
101 uint64_t tmp;
102
103 while (success == 0) {
104 tmp = v->cnt;
105 /* replace the value by itself */
106 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
107 tmp, tmp);
108 }
109 return tmp;
110 }
111
112 static inline void
rte_atomic64_set(rte_atomic64_t * v,int64_t new_value)113 rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
114 {
115 int success = 0;
116 uint64_t tmp;
117
118 while (success == 0) {
119 tmp = v->cnt;
120 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
121 tmp, new_value);
122 }
123 }
124
125 static inline void
rte_atomic64_add(rte_atomic64_t * v,int64_t inc)126 rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
127 {
128 int success = 0;
129 uint64_t tmp;
130
131 while (success == 0) {
132 tmp = v->cnt;
133 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
134 tmp, tmp + inc);
135 }
136 }
137
138 static inline void
rte_atomic64_sub(rte_atomic64_t * v,int64_t dec)139 rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
140 {
141 int success = 0;
142 uint64_t tmp;
143
144 while (success == 0) {
145 tmp = v->cnt;
146 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
147 tmp, tmp - dec);
148 }
149 }
150
151 static inline void
rte_atomic64_inc(rte_atomic64_t * v)152 rte_atomic64_inc(rte_atomic64_t *v)
153 {
154 rte_atomic64_add(v, 1);
155 }
156
157 static inline void
rte_atomic64_dec(rte_atomic64_t * v)158 rte_atomic64_dec(rte_atomic64_t *v)
159 {
160 rte_atomic64_sub(v, 1);
161 }
162
163 static inline int64_t
rte_atomic64_add_return(rte_atomic64_t * v,int64_t inc)164 rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
165 {
166 int success = 0;
167 uint64_t tmp;
168
169 while (success == 0) {
170 tmp = v->cnt;
171 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
172 tmp, tmp + inc);
173 }
174
175 return tmp + inc;
176 }
177
178 static inline int64_t
rte_atomic64_sub_return(rte_atomic64_t * v,int64_t dec)179 rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
180 {
181 int success = 0;
182 uint64_t tmp;
183
184 while (success == 0) {
185 tmp = v->cnt;
186 success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
187 tmp, tmp - dec);
188 }
189
190 return tmp - dec;
191 }
192
rte_atomic64_inc_and_test(rte_atomic64_t * v)193 static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
194 {
195 return rte_atomic64_add_return(v, 1) == 0;
196 }
197
rte_atomic64_dec_and_test(rte_atomic64_t * v)198 static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
199 {
200 return rte_atomic64_sub_return(v, 1) == 0;
201 }
202
rte_atomic64_test_and_set(rte_atomic64_t * v)203 static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
204 {
205 return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
206 }
207
rte_atomic64_clear(rte_atomic64_t * v)208 static inline void rte_atomic64_clear(rte_atomic64_t *v)
209 {
210 rte_atomic64_set(v, 0);
211 }
212 #endif
213
214 #endif /* _RTE_ATOMIC_I686_H_ */
215