1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright (c) 2010-2020 Intel Corporation
4 * Copyright (c) 2007-2009 Kip Macy [email protected]
5 * All rights reserved.
6 * Derived from FreeBSD's bufring.h
7 * Used as BSD-3 Licensed with permission from Kip Macy.
8 */
9
10 #ifndef _RTE_RING_PEEK_H_
11 #define _RTE_RING_PEEK_H_
12
13 /**
14 * @file
15 * It is not recommended to include this file directly.
16 * Please include <rte_ring_elem.h> instead.
17 *
18 * Ring Peek API
19 * Introduction of rte_ring with serialized producer/consumer (HTS sync mode)
20 * makes possible to split public enqueue/dequeue API into two phases:
21 * - enqueue/dequeue start
22 * - enqueue/dequeue finish
23 * That allows user to inspect objects in the ring without removing them
24 * from it (aka MT safe peek).
25 * Note that right now this new API is available only for two sync modes:
26 * 1) Single Producer/Single Consumer (RTE_RING_SYNC_ST)
27 * 2) Serialized Producer/Serialized Consumer (RTE_RING_SYNC_MT_HTS).
28 * It is a user responsibility to create/init ring with appropriate sync
29 * modes selected.
30 * As an example:
31 * // read 1 elem from the ring:
32 * n = rte_ring_dequeue_bulk_start(ring, &obj, 1, NULL);
33 * if (n != 0) {
34 * //examine object
35 * if (object_examine(obj) == KEEP)
36 * //decided to keep it in the ring.
37 * rte_ring_dequeue_finish(ring, 0);
38 * else
39 * //decided to remove it from the ring.
40 * rte_ring_dequeue_finish(ring, n);
41 * }
42 * Note that between _start_ and _finish_ none other thread can proceed
43 * with enqueue(/dequeue) operation till _finish_ completes.
44 */
45
46 #ifdef __cplusplus
47 extern "C" {
48 #endif
49
50 #include <rte_ring_peek_elem_pvt.h>
51
52 /**
53 * Start to enqueue several objects on the ring.
54 * Note that no actual objects are put in the queue by this function,
55 * it just reserves for user such ability.
56 * User has to call appropriate enqueue_elem_finish() to copy objects into the
57 * queue and complete given enqueue operation.
58 *
59 * @param r
60 * A pointer to the ring structure.
61 * @param n
62 * The number of objects to add in the ring from the obj_table.
63 * @param free_space
64 * if non-NULL, returns the amount of space in the ring after the
65 * enqueue operation has finished.
66 * @return
67 * The number of objects that can be enqueued, either 0 or n
68 */
69 static __rte_always_inline unsigned int
rte_ring_enqueue_bulk_elem_start(struct rte_ring * r,unsigned int n,unsigned int * free_space)70 rte_ring_enqueue_bulk_elem_start(struct rte_ring *r, unsigned int n,
71 unsigned int *free_space)
72 {
73 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_FIXED,
74 free_space);
75 }
76
77 /**
78 * Start to enqueue several objects on the ring.
79 * Note that no actual objects are put in the queue by this function,
80 * it just reserves for user such ability.
81 * User has to call appropriate enqueue_finish() to copy objects into the
82 * queue and complete given enqueue operation.
83 *
84 * @param r
85 * A pointer to the ring structure.
86 * @param n
87 * The number of objects to add in the ring from the obj_table.
88 * @param free_space
89 * if non-NULL, returns the amount of space in the ring after the
90 * enqueue operation has finished.
91 * @return
92 * The number of objects that can be enqueued, either 0 or n
93 */
94 static __rte_always_inline unsigned int
rte_ring_enqueue_bulk_start(struct rte_ring * r,unsigned int n,unsigned int * free_space)95 rte_ring_enqueue_bulk_start(struct rte_ring *r, unsigned int n,
96 unsigned int *free_space)
97 {
98 return rte_ring_enqueue_bulk_elem_start(r, n, free_space);
99 }
100
101 /**
102 * Start to enqueue several objects on the ring.
103 * Note that no actual objects are put in the queue by this function,
104 * it just reserves for user such ability.
105 * User has to call appropriate enqueue_elem_finish() to copy objects into the
106 * queue and complete given enqueue operation.
107 *
108 * @param r
109 * A pointer to the ring structure.
110 * @param n
111 * The number of objects to add in the ring from the obj_table.
112 * @param free_space
113 * if non-NULL, returns the amount of space in the ring after the
114 * enqueue operation has finished.
115 * @return
116 * Actual number of objects that can be enqueued.
117 */
118 static __rte_always_inline unsigned int
rte_ring_enqueue_burst_elem_start(struct rte_ring * r,unsigned int n,unsigned int * free_space)119 rte_ring_enqueue_burst_elem_start(struct rte_ring *r, unsigned int n,
120 unsigned int *free_space)
121 {
122 return __rte_ring_do_enqueue_start(r, n, RTE_RING_QUEUE_VARIABLE,
123 free_space);
124 }
125
126 /**
127 * Start to enqueue several objects on the ring.
128 * Note that no actual objects are put in the queue by this function,
129 * it just reserves for user such ability.
130 * User has to call appropriate enqueue_finish() to copy objects into the
131 * queue and complete given enqueue operation.
132 *
133 * @param r
134 * A pointer to the ring structure.
135 * @param n
136 * The number of objects to add in the ring from the obj_table.
137 * @param free_space
138 * if non-NULL, returns the amount of space in the ring after the
139 * enqueue operation has finished.
140 * @return
141 * Actual number of objects that can be enqueued.
142 */
143 static __rte_always_inline unsigned int
rte_ring_enqueue_burst_start(struct rte_ring * r,unsigned int n,unsigned int * free_space)144 rte_ring_enqueue_burst_start(struct rte_ring *r, unsigned int n,
145 unsigned int *free_space)
146 {
147 return rte_ring_enqueue_burst_elem_start(r, n, free_space);
148 }
149
150 /**
151 * Complete to enqueue several objects on the ring.
152 * Note that number of objects to enqueue should not exceed previous
153 * enqueue_start return value.
154 *
155 * @param r
156 * A pointer to the ring structure.
157 * @param obj_table
158 * A pointer to a table of objects.
159 * @param esize
160 * The size of ring element, in bytes. It must be a multiple of 4.
161 * This must be the same value used while creating the ring. Otherwise
162 * the results are undefined.
163 * @param n
164 * The number of objects to add to the ring from the obj_table.
165 */
166 static __rte_always_inline void
rte_ring_enqueue_elem_finish(struct rte_ring * r,const void * obj_table,unsigned int esize,unsigned int n)167 rte_ring_enqueue_elem_finish(struct rte_ring *r, const void *obj_table,
168 unsigned int esize, unsigned int n)
169 {
170 uint32_t tail;
171
172 switch (r->prod.sync_type) {
173 case RTE_RING_SYNC_ST:
174 n = __rte_ring_st_get_tail(&r->prod, &tail, n);
175 if (n != 0)
176 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n);
177 __rte_ring_st_set_head_tail(&r->prod, tail, n, 1);
178 break;
179 case RTE_RING_SYNC_MT_HTS:
180 n = __rte_ring_hts_get_tail(&r->hts_prod, &tail, n);
181 if (n != 0)
182 __rte_ring_enqueue_elems(r, tail, obj_table, esize, n);
183 __rte_ring_hts_set_head_tail(&r->hts_prod, tail, n, 1);
184 break;
185 case RTE_RING_SYNC_MT:
186 case RTE_RING_SYNC_MT_RTS:
187 default:
188 /* unsupported mode, shouldn't be here */
189 RTE_ASSERT(0);
190 }
191 }
192
193 /**
194 * Complete to enqueue several objects on the ring.
195 * Note that number of objects to enqueue should not exceed previous
196 * enqueue_start return value.
197 *
198 * @param r
199 * A pointer to the ring structure.
200 * @param obj_table
201 * A pointer to a table of objects.
202 * @param n
203 * The number of objects to add to the ring from the obj_table.
204 */
205 static __rte_always_inline void
rte_ring_enqueue_finish(struct rte_ring * r,void * const * obj_table,unsigned int n)206 rte_ring_enqueue_finish(struct rte_ring *r, void * const *obj_table,
207 unsigned int n)
208 {
209 rte_ring_enqueue_elem_finish(r, obj_table, sizeof(uintptr_t), n);
210 }
211
212 /**
213 * Start to dequeue several objects from the ring.
214 * Note that user has to call appropriate dequeue_finish()
215 * to complete given dequeue operation and actually remove objects the ring.
216 *
217 * @param r
218 * A pointer to the ring structure.
219 * @param obj_table
220 * A pointer to a table of objects that will be filled.
221 * @param esize
222 * The size of ring element, in bytes. It must be a multiple of 4.
223 * This must be the same value used while creating the ring. Otherwise
224 * the results are undefined.
225 * @param n
226 * The number of objects to dequeue from the ring to the obj_table.
227 * @param available
228 * If non-NULL, returns the number of remaining ring entries after the
229 * dequeue has finished.
230 * @return
231 * The number of objects dequeued, either 0 or n.
232 */
233 static __rte_always_inline unsigned int
rte_ring_dequeue_bulk_elem_start(struct rte_ring * r,void * obj_table,unsigned int esize,unsigned int n,unsigned int * available)234 rte_ring_dequeue_bulk_elem_start(struct rte_ring *r, void *obj_table,
235 unsigned int esize, unsigned int n, unsigned int *available)
236 {
237 return __rte_ring_do_dequeue_start(r, obj_table, esize, n,
238 RTE_RING_QUEUE_FIXED, available);
239 }
240
241 /**
242 * Start to dequeue several objects from the ring.
243 * Note that user has to call appropriate dequeue_finish()
244 * to complete given dequeue operation and actually remove objects the ring.
245 *
246 * @param r
247 * A pointer to the ring structure.
248 * @param obj_table
249 * A pointer to a table of void * pointers (objects) that will be filled.
250 * @param n
251 * The number of objects to dequeue from the ring to the obj_table.
252 * @param available
253 * If non-NULL, returns the number of remaining ring entries after the
254 * dequeue has finished.
255 * @return
256 * Actual number of objects dequeued.
257 */
258 static __rte_always_inline unsigned int
rte_ring_dequeue_bulk_start(struct rte_ring * r,void ** obj_table,unsigned int n,unsigned int * available)259 rte_ring_dequeue_bulk_start(struct rte_ring *r, void **obj_table,
260 unsigned int n, unsigned int *available)
261 {
262 return rte_ring_dequeue_bulk_elem_start(r, obj_table, sizeof(uintptr_t),
263 n, available);
264 }
265
266 /**
267 * Start to dequeue several objects from the ring.
268 * Note that user has to call appropriate dequeue_finish()
269 * to complete given dequeue operation and actually remove objects the ring.
270 *
271 * @param r
272 * A pointer to the ring structure.
273 * @param obj_table
274 * A pointer to a table of objects that will be filled.
275 * @param esize
276 * The size of ring element, in bytes. It must be a multiple of 4.
277 * This must be the same value used while creating the ring. Otherwise
278 * the results are undefined.
279 * @param n
280 * The number of objects to dequeue from the ring to the obj_table.
281 * @param available
282 * If non-NULL, returns the number of remaining ring entries after the
283 * dequeue has finished.
284 * @return
285 * The actual number of objects dequeued.
286 */
287 static __rte_always_inline unsigned int
rte_ring_dequeue_burst_elem_start(struct rte_ring * r,void * obj_table,unsigned int esize,unsigned int n,unsigned int * available)288 rte_ring_dequeue_burst_elem_start(struct rte_ring *r, void *obj_table,
289 unsigned int esize, unsigned int n, unsigned int *available)
290 {
291 return __rte_ring_do_dequeue_start(r, obj_table, esize, n,
292 RTE_RING_QUEUE_VARIABLE, available);
293 }
294
295 /**
296 * Start to dequeue several objects from the ring.
297 * Note that user has to call appropriate dequeue_finish()
298 * to complete given dequeue operation and actually remove objects the ring.
299 *
300 * @param r
301 * A pointer to the ring structure.
302 * @param obj_table
303 * A pointer to a table of void * pointers (objects) that will be filled.
304 * @param n
305 * The number of objects to dequeue from the ring to the obj_table.
306 * @param available
307 * If non-NULL, returns the number of remaining ring entries after the
308 * dequeue has finished.
309 * @return
310 * The actual number of objects dequeued.
311 */
312 static __rte_always_inline unsigned int
rte_ring_dequeue_burst_start(struct rte_ring * r,void ** obj_table,unsigned int n,unsigned int * available)313 rte_ring_dequeue_burst_start(struct rte_ring *r, void **obj_table,
314 unsigned int n, unsigned int *available)
315 {
316 return rte_ring_dequeue_burst_elem_start(r, obj_table,
317 sizeof(uintptr_t), n, available);
318 }
319
320 /**
321 * Complete to dequeue several objects from the ring.
322 * Note that number of objects to dequeue should not exceed previous
323 * dequeue_start return value.
324 *
325 * @param r
326 * A pointer to the ring structure.
327 * @param n
328 * The number of objects to remove from the ring.
329 */
330 static __rte_always_inline void
rte_ring_dequeue_elem_finish(struct rte_ring * r,unsigned int n)331 rte_ring_dequeue_elem_finish(struct rte_ring *r, unsigned int n)
332 {
333 uint32_t tail;
334
335 switch (r->cons.sync_type) {
336 case RTE_RING_SYNC_ST:
337 n = __rte_ring_st_get_tail(&r->cons, &tail, n);
338 __rte_ring_st_set_head_tail(&r->cons, tail, n, 0);
339 break;
340 case RTE_RING_SYNC_MT_HTS:
341 n = __rte_ring_hts_get_tail(&r->hts_cons, &tail, n);
342 __rte_ring_hts_set_head_tail(&r->hts_cons, tail, n, 0);
343 break;
344 case RTE_RING_SYNC_MT:
345 case RTE_RING_SYNC_MT_RTS:
346 default:
347 /* unsupported mode, shouldn't be here */
348 RTE_ASSERT(0);
349 }
350 }
351
352 /**
353 * Complete to dequeue several objects from the ring.
354 * Note that number of objects to dequeue should not exceed previous
355 * dequeue_start return value.
356 *
357 * @param r
358 * A pointer to the ring structure.
359 * @param n
360 * The number of objects to remove from the ring.
361 */
362 static __rte_always_inline void
rte_ring_dequeue_finish(struct rte_ring * r,unsigned int n)363 rte_ring_dequeue_finish(struct rte_ring *r, unsigned int n)
364 {
365 rte_ring_dequeue_elem_finish(r, n);
366 }
367
368 #ifdef __cplusplus
369 }
370 #endif
371
372 #endif /* _RTE_RING_PEEK_H_ */
373