1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright (c) 2010-2020 Intel Corporation
4 * Copyright (c) 2007-2009 Kip Macy [email protected]
5 * All rights reserved.
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
8 */
9
10 #ifndef _RTE_RING_HTS_H_
11 #define _RTE_RING_HTS_H_
12
13 /**
14 * @file rte_ring_hts.h
15 * @b EXPERIMENTAL: this API may change without prior notice
16 * It is not recommended to include this file directly.
17 * Please include <rte_ring.h> instead.
18 *
19 * Contains functions for serialized, aka Head-Tail Sync (HTS) ring mode.
20 * In that mode enqueue/dequeue operation is fully serialized:
21 * at any given moment only one enqueue/dequeue operation can proceed.
22 * This is achieved by allowing a thread to proceed with changing head.value
23 * only when head.value == tail.value.
24 * Both head and tail values are updated atomically (as one 64-bit value).
25 * To achieve that 64-bit CAS is used by head update routine.
26 */
27
28 #ifdef __cplusplus
29 extern "C" {
30 #endif
31
32 #include <rte_ring_hts_c11_mem.h>
33
34 /**
35 * @internal Enqueue several objects on the HTS ring.
36 *
37 * @param r
38 * A pointer to the ring structure.
39 * @param obj_table
40 * A pointer to a table of objects.
41 * @param esize
42 * The size of ring element, in bytes. It must be a multiple of 4.
43 * This must be the same value used while creating the ring. Otherwise
44 * the results are undefined.
45 * @param n
46 * The number of objects to add in the ring from the obj_table.
47 * @param behavior
48 * RTE_RING_QUEUE_FIXED: Enqueue a fixed number of items from a ring
49 * RTE_RING_QUEUE_VARIABLE: Enqueue as many items as possible from ring
50 * @param free_space
51 * returns the amount of space after the enqueue operation has finished
52 * @return
53 * Actual number of objects enqueued.
54 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
55 */
56 static __rte_always_inline unsigned int
__rte_ring_do_hts_enqueue_elem(struct rte_ring * r,const void * obj_table,uint32_t esize,uint32_t n,enum rte_ring_queue_behavior behavior,uint32_t * free_space)57 __rte_ring_do_hts_enqueue_elem(struct rte_ring *r, const void *obj_table,
58 uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
59 uint32_t *free_space)
60 {
61 uint32_t free, head;
62
63 n = __rte_ring_hts_move_prod_head(r, n, behavior, &head, &free);
64
65 if (n != 0) {
66 __rte_ring_enqueue_elems(r, head, obj_table, esize, n);
67 __rte_ring_hts_update_tail(&r->hts_prod, head, n, 1);
68 }
69
70 if (free_space != NULL)
71 *free_space = free - n;
72 return n;
73 }
74
75 /**
76 * @internal Dequeue several objects from the HTS ring.
77 *
78 * @param r
79 * A pointer to the ring structure.
80 * @param obj_table
81 * A pointer to a table of objects.
82 * @param esize
83 * The size of ring element, in bytes. It must be a multiple of 4.
84 * This must be the same value used while creating the ring. Otherwise
85 * the results are undefined.
86 * @param n
87 * The number of objects to pull from the ring.
88 * @param behavior
89 * RTE_RING_QUEUE_FIXED: Dequeue a fixed number of items from a ring
90 * RTE_RING_QUEUE_VARIABLE: Dequeue as many items as possible from ring
91 * @param available
92 * returns the number of remaining ring entries after the dequeue has finished
93 * @return
94 * - Actual number of objects dequeued.
95 * If behavior == RTE_RING_QUEUE_FIXED, this will be 0 or n only.
96 */
97 static __rte_always_inline unsigned int
__rte_ring_do_hts_dequeue_elem(struct rte_ring * r,void * obj_table,uint32_t esize,uint32_t n,enum rte_ring_queue_behavior behavior,uint32_t * available)98 __rte_ring_do_hts_dequeue_elem(struct rte_ring *r, void *obj_table,
99 uint32_t esize, uint32_t n, enum rte_ring_queue_behavior behavior,
100 uint32_t *available)
101 {
102 uint32_t entries, head;
103
104 n = __rte_ring_hts_move_cons_head(r, n, behavior, &head, &entries);
105
106 if (n != 0) {
107 __rte_ring_dequeue_elems(r, head, obj_table, esize, n);
108 __rte_ring_hts_update_tail(&r->hts_cons, head, n, 0);
109 }
110
111 if (available != NULL)
112 *available = entries - n;
113 return n;
114 }
115
116 /**
117 * Enqueue several objects on the HTS ring (multi-producers safe).
118 *
119 * @param r
120 * A pointer to the ring structure.
121 * @param obj_table
122 * A pointer to a table of objects.
123 * @param esize
124 * The size of ring element, in bytes. It must be a multiple of 4.
125 * This must be the same value used while creating the ring. Otherwise
126 * the results are undefined.
127 * @param n
128 * The number of objects to add in the ring from the obj_table.
129 * @param free_space
130 * if non-NULL, returns the amount of space in the ring after the
131 * enqueue operation has finished.
132 * @return
133 * The number of objects enqueued, either 0 or n
134 */
135 __rte_experimental
136 static __rte_always_inline unsigned int
rte_ring_mp_hts_enqueue_bulk_elem(struct rte_ring * r,const void * obj_table,unsigned int esize,unsigned int n,unsigned int * free_space)137 rte_ring_mp_hts_enqueue_bulk_elem(struct rte_ring *r, const void *obj_table,
138 unsigned int esize, unsigned int n, unsigned int *free_space)
139 {
140 return __rte_ring_do_hts_enqueue_elem(r, obj_table, esize, n,
141 RTE_RING_QUEUE_FIXED, free_space);
142 }
143
144 /**
145 * Dequeue several objects from an HTS ring (multi-consumers safe).
146 *
147 * @param r
148 * A pointer to the ring structure.
149 * @param obj_table
150 * A pointer to a table of objects that will be filled.
151 * @param esize
152 * The size of ring element, in bytes. It must be a multiple of 4.
153 * This must be the same value used while creating the ring. Otherwise
154 * the results are undefined.
155 * @param n
156 * The number of objects to dequeue from the ring to the obj_table.
157 * @param available
158 * If non-NULL, returns the number of remaining ring entries after the
159 * dequeue has finished.
160 * @return
161 * The number of objects dequeued, either 0 or n
162 */
163 __rte_experimental
164 static __rte_always_inline unsigned int
rte_ring_mc_hts_dequeue_bulk_elem(struct rte_ring * r,void * obj_table,unsigned int esize,unsigned int n,unsigned int * available)165 rte_ring_mc_hts_dequeue_bulk_elem(struct rte_ring *r, void *obj_table,
166 unsigned int esize, unsigned int n, unsigned int *available)
167 {
168 return __rte_ring_do_hts_dequeue_elem(r, obj_table, esize, n,
169 RTE_RING_QUEUE_FIXED, available);
170 }
171
172 /**
173 * Enqueue several objects on the HTS ring (multi-producers safe).
174 *
175 * @param r
176 * A pointer to the ring structure.
177 * @param obj_table
178 * A pointer to a table of objects.
179 * @param esize
180 * The size of ring element, in bytes. It must be a multiple of 4.
181 * This must be the same value used while creating the ring. Otherwise
182 * the results are undefined.
183 * @param n
184 * The number of objects to add in the ring from the obj_table.
185 * @param free_space
186 * if non-NULL, returns the amount of space in the ring after the
187 * enqueue operation has finished.
188 * @return
189 * - n: Actual number of objects enqueued.
190 */
191 __rte_experimental
192 static __rte_always_inline unsigned int
rte_ring_mp_hts_enqueue_burst_elem(struct rte_ring * r,const void * obj_table,unsigned int esize,unsigned int n,unsigned int * free_space)193 rte_ring_mp_hts_enqueue_burst_elem(struct rte_ring *r, const void *obj_table,
194 unsigned int esize, unsigned int n, unsigned int *free_space)
195 {
196 return __rte_ring_do_hts_enqueue_elem(r, obj_table, esize, n,
197 RTE_RING_QUEUE_VARIABLE, free_space);
198 }
199
200 /**
201 * Dequeue several objects from an HTS ring (multi-consumers safe).
202 * When the requested objects are more than the available objects,
203 * only dequeue the actual number of objects.
204 *
205 * @param r
206 * A pointer to the ring structure.
207 * @param obj_table
208 * A pointer to a table of objects that will be filled.
209 * @param esize
210 * The size of ring element, in bytes. It must be a multiple of 4.
211 * This must be the same value used while creating the ring. Otherwise
212 * the results are undefined.
213 * @param n
214 * The number of objects to dequeue from the ring to the obj_table.
215 * @param available
216 * If non-NULL, returns the number of remaining ring entries after the
217 * dequeue has finished.
218 * @return
219 * - n: Actual number of objects dequeued, 0 if ring is empty
220 */
221 __rte_experimental
222 static __rte_always_inline unsigned int
rte_ring_mc_hts_dequeue_burst_elem(struct rte_ring * r,void * obj_table,unsigned int esize,unsigned int n,unsigned int * available)223 rte_ring_mc_hts_dequeue_burst_elem(struct rte_ring *r, void *obj_table,
224 unsigned int esize, unsigned int n, unsigned int *available)
225 {
226 return __rte_ring_do_hts_dequeue_elem(r, obj_table, esize, n,
227 RTE_RING_QUEUE_VARIABLE, available);
228 }
229
230 /**
231 * Enqueue several objects on the HTS ring (multi-producers safe).
232 *
233 * @param r
234 * A pointer to the ring structure.
235 * @param obj_table
236 * A pointer to a table of void * pointers (objects).
237 * @param n
238 * The number of objects to add in the ring from the obj_table.
239 * @param free_space
240 * if non-NULL, returns the amount of space in the ring after the
241 * enqueue operation has finished.
242 * @return
243 * The number of objects enqueued, either 0 or n
244 */
245 __rte_experimental
246 static __rte_always_inline unsigned int
rte_ring_mp_hts_enqueue_bulk(struct rte_ring * r,void * const * obj_table,unsigned int n,unsigned int * free_space)247 rte_ring_mp_hts_enqueue_bulk(struct rte_ring *r, void * const *obj_table,
248 unsigned int n, unsigned int *free_space)
249 {
250 return rte_ring_mp_hts_enqueue_bulk_elem(r, obj_table,
251 sizeof(uintptr_t), n, free_space);
252 }
253
254 /**
255 * Dequeue several objects from an HTS ring (multi-consumers safe).
256 *
257 * @param r
258 * A pointer to the ring structure.
259 * @param obj_table
260 * A pointer to a table of void * pointers (objects) that will be filled.
261 * @param n
262 * The number of objects to dequeue from the ring to the obj_table.
263 * @param available
264 * If non-NULL, returns the number of remaining ring entries after the
265 * dequeue has finished.
266 * @return
267 * The number of objects dequeued, either 0 or n
268 */
269 __rte_experimental
270 static __rte_always_inline unsigned int
rte_ring_mc_hts_dequeue_bulk(struct rte_ring * r,void ** obj_table,unsigned int n,unsigned int * available)271 rte_ring_mc_hts_dequeue_bulk(struct rte_ring *r, void **obj_table,
272 unsigned int n, unsigned int *available)
273 {
274 return rte_ring_mc_hts_dequeue_bulk_elem(r, obj_table,
275 sizeof(uintptr_t), n, available);
276 }
277
278 /**
279 * Enqueue several objects on the HTS ring (multi-producers safe).
280 *
281 * @param r
282 * A pointer to the ring structure.
283 * @param obj_table
284 * A pointer to a table of void * pointers (objects).
285 * @param n
286 * The number of objects to add in the ring from the obj_table.
287 * @param free_space
288 * if non-NULL, returns the amount of space in the ring after the
289 * enqueue operation has finished.
290 * @return
291 * - n: Actual number of objects enqueued.
292 */
293 __rte_experimental
294 static __rte_always_inline unsigned int
rte_ring_mp_hts_enqueue_burst(struct rte_ring * r,void * const * obj_table,unsigned int n,unsigned int * free_space)295 rte_ring_mp_hts_enqueue_burst(struct rte_ring *r, void * const *obj_table,
296 unsigned int n, unsigned int *free_space)
297 {
298 return rte_ring_mp_hts_enqueue_burst_elem(r, obj_table,
299 sizeof(uintptr_t), n, free_space);
300 }
301
302 /**
303 * Dequeue several objects from an HTS ring (multi-consumers safe).
304 * When the requested objects are more than the available objects,
305 * only dequeue the actual number of objects.
306 *
307 * @param r
308 * A pointer to the ring structure.
309 * @param obj_table
310 * A pointer to a table of void * pointers (objects) that will be filled.
311 * @param n
312 * The number of objects to dequeue from the ring to the obj_table.
313 * @param available
314 * If non-NULL, returns the number of remaining ring entries after the
315 * dequeue has finished.
316 * @return
317 * - n: Actual number of objects dequeued, 0 if ring is empty
318 */
319 __rte_experimental
320 static __rte_always_inline unsigned int
rte_ring_mc_hts_dequeue_burst(struct rte_ring * r,void ** obj_table,unsigned int n,unsigned int * available)321 rte_ring_mc_hts_dequeue_burst(struct rte_ring *r, void **obj_table,
322 unsigned int n, unsigned int *available)
323 {
324 return rte_ring_mc_hts_dequeue_burst_elem(r, obj_table,
325 sizeof(uintptr_t), n, available);
326 }
327
328 #ifdef __cplusplus
329 }
330 #endif
331
332 #endif /* _RTE_RING_HTS_H_ */
333