1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2018-2020 Arm Limited
3 */
4
5 #ifndef _RTE_RCU_QSBR_H_
6 #define _RTE_RCU_QSBR_H_
7
8 /**
9 * @file
10 *
11 * RTE Quiescent State Based Reclamation (QSBR).
12 *
13 * @warning
14 * @b EXPERIMENTAL:
15 * All functions in this file may be changed or removed without prior notice.
16 *
17 * Quiescent State (QS) is any point in the thread execution
18 * where the thread does not hold a reference to a data structure
19 * in shared memory. While using lock-less data structures, the writer
20 * can safely free memory once all the reader threads have entered
21 * quiescent state.
22 *
23 * This library provides the ability for the readers to report quiescent
24 * state and for the writers to identify when all the readers have
25 * entered quiescent state.
26 */
27
28 #ifdef __cplusplus
29 extern "C" {
30 #endif
31
32 #include <stdbool.h>
33 #include <stdio.h>
34 #include <stdint.h>
35 #include <inttypes.h>
36 #include <errno.h>
37 #include <rte_common.h>
38 #include <rte_memory.h>
39 #include <rte_lcore.h>
40 #include <rte_debug.h>
41 #include <rte_atomic.h>
42 #include <rte_ring.h>
43
44 extern int rte_rcu_log_type;
45
46 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
47 #define __RTE_RCU_DP_LOG(level, fmt, args...) \
48 rte_log(RTE_LOG_ ## level, rte_rcu_log_type, \
49 "%s(): " fmt "\n", __func__, ## args)
50 #else
51 #define __RTE_RCU_DP_LOG(level, fmt, args...)
52 #endif
53
54 #if defined(RTE_LIBRTE_RCU_DEBUG)
55 #define __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, level, fmt, args...) do {\
56 if (v->qsbr_cnt[thread_id].lock_cnt) \
57 rte_log(RTE_LOG_ ## level, rte_rcu_log_type, \
58 "%s(): " fmt "\n", __func__, ## args); \
59 } while (0)
60 #else
61 #define __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, level, fmt, args...)
62 #endif
63
64 /* Registered thread IDs are stored as a bitmap of 64b element array.
65 * Given thread id needs to be converted to index into the array and
66 * the id within the array element.
67 */
68 #define __RTE_QSBR_THRID_ARRAY_ELM_SIZE (sizeof(uint64_t) * 8)
69 #define __RTE_QSBR_THRID_ARRAY_SIZE(max_threads) \
70 RTE_ALIGN(RTE_ALIGN_MUL_CEIL(max_threads, \
71 __RTE_QSBR_THRID_ARRAY_ELM_SIZE) >> 3, RTE_CACHE_LINE_SIZE)
72 #define __RTE_QSBR_THRID_ARRAY_ELM(v, i) ((uint64_t *) \
73 ((struct rte_rcu_qsbr_cnt *)(v + 1) + v->max_threads) + i)
74 #define __RTE_QSBR_THRID_INDEX_SHIFT 6
75 #define __RTE_QSBR_THRID_MASK 0x3f
76 #define RTE_QSBR_THRID_INVALID 0xffffffff
77
78 /* Worker thread counter */
79 struct rte_rcu_qsbr_cnt {
80 uint64_t cnt;
81 /**< Quiescent state counter. Value 0 indicates the thread is offline
82 * 64b counter is used to avoid adding more code to address
83 * counter overflow. Changing this to 32b would require additional
84 * changes to various APIs.
85 */
86 uint32_t lock_cnt;
87 /**< Lock counter. Used when RTE_LIBRTE_RCU_DEBUG is enabled */
88 } __rte_cache_aligned;
89
90 #define __RTE_QSBR_CNT_THR_OFFLINE 0
91 #define __RTE_QSBR_CNT_INIT 1
92 #define __RTE_QSBR_CNT_MAX ((uint64_t)~0)
93 #define __RTE_QSBR_TOKEN_SIZE sizeof(uint64_t)
94
95 /* RTE Quiescent State variable structure.
96 * This structure has two elements that vary in size based on the
97 * 'max_threads' parameter.
98 * 1) Quiescent state counter array
99 * 2) Register thread ID array
100 */
101 struct rte_rcu_qsbr {
102 uint64_t token __rte_cache_aligned;
103 /**< Counter to allow for multiple concurrent quiescent state queries */
104 uint64_t acked_token;
105 /**< Least token acked by all the threads in the last call to
106 * rte_rcu_qsbr_check API.
107 */
108
109 uint32_t num_elems __rte_cache_aligned;
110 /**< Number of elements in the thread ID array */
111 uint32_t num_threads;
112 /**< Number of threads currently using this QS variable */
113 uint32_t max_threads;
114 /**< Maximum number of threads using this QS variable */
115
116 struct rte_rcu_qsbr_cnt qsbr_cnt[0] __rte_cache_aligned;
117 /**< Quiescent state counter array of 'max_threads' elements */
118
119 /**< Registered thread IDs are stored in a bitmap array,
120 * after the quiescent state counter array.
121 */
122 } __rte_cache_aligned;
123
124 /**
125 * Call back function called to free the resources.
126 *
127 * @param p
128 * Pointer provided while creating the defer queue
129 * @param e
130 * Pointer to the resource data stored on the defer queue
131 * @param n
132 * Number of resources to free. Currently, this is set to 1.
133 *
134 * @return
135 * None
136 */
137 typedef void (*rte_rcu_qsbr_free_resource_t)(void *p, void *e, unsigned int n);
138
139 #define RTE_RCU_QSBR_DQ_NAMESIZE RTE_RING_NAMESIZE
140
141 /**
142 * Various flags supported.
143 */
144 /**< Enqueue and reclaim operations are multi-thread safe by default.
145 * The call back functions registered to free the resources are
146 * assumed to be multi-thread safe.
147 * Set this flag if multi-thread safety is not required.
148 */
149 #define RTE_RCU_QSBR_DQ_MT_UNSAFE 1
150
151 /**
152 * Parameters used when creating the defer queue.
153 */
154 struct rte_rcu_qsbr_dq_parameters {
155 const char *name;
156 /**< Name of the queue. */
157 uint32_t flags;
158 /**< Flags to control API behaviors */
159 uint32_t size;
160 /**< Number of entries in queue. Typically, this will be
161 * the same as the maximum number of entries supported in the
162 * lock free data structure.
163 * Data structures with unbounded number of entries is not
164 * supported currently.
165 */
166 uint32_t esize;
167 /**< Size (in bytes) of each element in the defer queue.
168 * This has to be multiple of 4B.
169 */
170 uint32_t trigger_reclaim_limit;
171 /**< Trigger automatic reclamation after the defer queue
172 * has at least these many resources waiting. This auto
173 * reclamation is triggered in rte_rcu_qsbr_dq_enqueue API
174 * call.
175 * If this is greater than 'size', auto reclamation is
176 * not triggered.
177 * If this is set to 0, auto reclamation is triggered
178 * in every call to rte_rcu_qsbr_dq_enqueue API.
179 */
180 uint32_t max_reclaim_size;
181 /**< When automatic reclamation is enabled, reclaim at the max
182 * these many resources. This should contain a valid value, if
183 * auto reclamation is on. Setting this to 'size' or greater will
184 * reclaim all possible resources currently on the defer queue.
185 */
186 rte_rcu_qsbr_free_resource_t free_fn;
187 /**< Function to call to free the resource. */
188 void *p;
189 /**< Pointer passed to the free function. Typically, this is the
190 * pointer to the data structure to which the resource to free
191 * belongs. This can be NULL.
192 */
193 struct rte_rcu_qsbr *v;
194 /**< RCU QSBR variable to use for this defer queue */
195 };
196
197 /* RTE defer queue structure.
198 * This structure holds the defer queue. The defer queue is used to
199 * hold the deleted entries from the data structure that are not
200 * yet freed.
201 */
202 struct rte_rcu_qsbr_dq;
203
204 /**
205 * Return the size of the memory occupied by a Quiescent State variable.
206 *
207 * @param max_threads
208 * Maximum number of threads reporting quiescent state on this variable.
209 * @return
210 * On success - size of memory in bytes required for this QS variable.
211 * On error - 1 with error code set in rte_errno.
212 * Possible rte_errno codes are:
213 * - EINVAL - max_threads is 0
214 */
215 size_t
216 rte_rcu_qsbr_get_memsize(uint32_t max_threads);
217
218 /**
219 * Initialize a Quiescent State (QS) variable.
220 *
221 * @param v
222 * QS variable
223 * @param max_threads
224 * Maximum number of threads reporting quiescent state on this variable.
225 * This should be the same value as passed to rte_rcu_qsbr_get_memsize.
226 * @return
227 * On success - 0
228 * On error - 1 with error code set in rte_errno.
229 * Possible rte_errno codes are:
230 * - EINVAL - max_threads is 0 or 'v' is NULL.
231 *
232 */
233 int
234 rte_rcu_qsbr_init(struct rte_rcu_qsbr *v, uint32_t max_threads);
235
236 /**
237 * Register a reader thread to report its quiescent state
238 * on a QS variable.
239 *
240 * This is implemented as a lock-free function. It is multi-thread
241 * safe.
242 * Any reader thread that wants to report its quiescent state must
243 * call this API. This can be called during initialization or as part
244 * of the packet processing loop.
245 *
246 * Note that rte_rcu_qsbr_thread_online must be called before the
247 * thread updates its quiescent state using rte_rcu_qsbr_quiescent.
248 *
249 * @param v
250 * QS variable
251 * @param thread_id
252 * Reader thread with this thread ID will report its quiescent state on
253 * the QS variable. thread_id is a value between 0 and (max_threads - 1).
254 * 'max_threads' is the parameter passed in 'rte_rcu_qsbr_init' API.
255 */
256 int
257 rte_rcu_qsbr_thread_register(struct rte_rcu_qsbr *v, unsigned int thread_id);
258
259 /**
260 * Remove a reader thread, from the list of threads reporting their
261 * quiescent state on a QS variable.
262 *
263 * This is implemented as a lock-free function. It is multi-thread safe.
264 * This API can be called from the reader threads during shutdown.
265 * Ongoing quiescent state queries will stop waiting for the status from this
266 * unregistered reader thread.
267 *
268 * @param v
269 * QS variable
270 * @param thread_id
271 * Reader thread with this thread ID will stop reporting its quiescent
272 * state on the QS variable.
273 */
274 int
275 rte_rcu_qsbr_thread_unregister(struct rte_rcu_qsbr *v, unsigned int thread_id);
276
277 /**
278 * Add a registered reader thread, to the list of threads reporting their
279 * quiescent state on a QS variable.
280 *
281 * This is implemented as a lock-free function. It is multi-thread
282 * safe.
283 *
284 * Any registered reader thread that wants to report its quiescent state must
285 * call this API before calling rte_rcu_qsbr_quiescent. This can be called
286 * during initialization or as part of the packet processing loop.
287 *
288 * The reader thread must call rte_rcu_qsbr_thread_offline API, before
289 * calling any functions that block, to ensure that rte_rcu_qsbr_check
290 * API does not wait indefinitely for the reader thread to update its QS.
291 *
292 * The reader thread must call rte_rcu_thread_online API, after the blocking
293 * function call returns, to ensure that rte_rcu_qsbr_check API
294 * waits for the reader thread to update its quiescent state.
295 *
296 * @param v
297 * QS variable
298 * @param thread_id
299 * Reader thread with this thread ID will report its quiescent state on
300 * the QS variable.
301 */
302 static __rte_always_inline void
rte_rcu_qsbr_thread_online(struct rte_rcu_qsbr * v,unsigned int thread_id)303 rte_rcu_qsbr_thread_online(struct rte_rcu_qsbr *v, unsigned int thread_id)
304 {
305 uint64_t t;
306
307 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
308
309 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n",
310 v->qsbr_cnt[thread_id].lock_cnt);
311
312 /* Copy the current value of token.
313 * The fence at the end of the function will ensure that
314 * the following will not move down after the load of any shared
315 * data structure.
316 */
317 t = __atomic_load_n(&v->token, __ATOMIC_RELAXED);
318
319 /* __atomic_store_n(cnt, __ATOMIC_RELAXED) is used to ensure
320 * 'cnt' (64b) is accessed atomically.
321 */
322 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
323 t, __ATOMIC_RELAXED);
324
325 /* The subsequent load of the data structure should not
326 * move above the store. Hence a store-load barrier
327 * is required.
328 * If the load of the data structure moves above the store,
329 * writer might not see that the reader is online, even though
330 * the reader is referencing the shared data structure.
331 */
332 #ifdef RTE_ARCH_X86_64
333 /* rte_smp_mb() for x86 is lighter */
334 rte_smp_mb();
335 #else
336 __atomic_thread_fence(__ATOMIC_SEQ_CST);
337 #endif
338 }
339
340 /**
341 * Remove a registered reader thread from the list of threads reporting their
342 * quiescent state on a QS variable.
343 *
344 * This is implemented as a lock-free function. It is multi-thread
345 * safe.
346 *
347 * This can be called during initialization or as part of the packet
348 * processing loop.
349 *
350 * The reader thread must call rte_rcu_qsbr_thread_offline API, before
351 * calling any functions that block, to ensure that rte_rcu_qsbr_check
352 * API does not wait indefinitely for the reader thread to update its QS.
353 *
354 * @param v
355 * QS variable
356 * @param thread_id
357 * rte_rcu_qsbr_check API will not wait for the reader thread with
358 * this thread ID to report its quiescent state on the QS variable.
359 */
360 static __rte_always_inline void
rte_rcu_qsbr_thread_offline(struct rte_rcu_qsbr * v,unsigned int thread_id)361 rte_rcu_qsbr_thread_offline(struct rte_rcu_qsbr *v, unsigned int thread_id)
362 {
363 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
364
365 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n",
366 v->qsbr_cnt[thread_id].lock_cnt);
367
368 /* The reader can go offline only after the load of the
369 * data structure is completed. i.e. any load of the
370 * data strcture can not move after this store.
371 */
372
373 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
374 __RTE_QSBR_CNT_THR_OFFLINE, __ATOMIC_RELEASE);
375 }
376
377 /**
378 * Acquire a lock for accessing a shared data structure.
379 *
380 * This is implemented as a lock-free function. It is multi-thread
381 * safe.
382 *
383 * This API is provided to aid debugging. This should be called before
384 * accessing a shared data structure.
385 *
386 * When RTE_LIBRTE_RCU_DEBUG is enabled a lock counter is incremented.
387 * Similarly rte_rcu_qsbr_unlock will decrement the counter. When the
388 * rte_rcu_qsbr_check API will verify that this counter is 0.
389 *
390 * When RTE_LIBRTE_RCU_DEBUG is disabled, this API will do nothing.
391 *
392 * @param v
393 * QS variable
394 * @param thread_id
395 * Reader thread id
396 */
397 static __rte_always_inline void
rte_rcu_qsbr_lock(__rte_unused struct rte_rcu_qsbr * v,__rte_unused unsigned int thread_id)398 rte_rcu_qsbr_lock(__rte_unused struct rte_rcu_qsbr *v,
399 __rte_unused unsigned int thread_id)
400 {
401 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
402
403 #if defined(RTE_LIBRTE_RCU_DEBUG)
404 /* Increment the lock counter */
405 __atomic_fetch_add(&v->qsbr_cnt[thread_id].lock_cnt,
406 1, __ATOMIC_ACQUIRE);
407 #endif
408 }
409
410 /**
411 * Release a lock after accessing a shared data structure.
412 *
413 * This is implemented as a lock-free function. It is multi-thread
414 * safe.
415 *
416 * This API is provided to aid debugging. This should be called after
417 * accessing a shared data structure.
418 *
419 * When RTE_LIBRTE_RCU_DEBUG is enabled, rte_rcu_qsbr_unlock will
420 * decrement a lock counter. rte_rcu_qsbr_check API will verify that this
421 * counter is 0.
422 *
423 * When RTE_LIBRTE_RCU_DEBUG is disabled, this API will do nothing.
424 *
425 * @param v
426 * QS variable
427 * @param thread_id
428 * Reader thread id
429 */
430 static __rte_always_inline void
rte_rcu_qsbr_unlock(__rte_unused struct rte_rcu_qsbr * v,__rte_unused unsigned int thread_id)431 rte_rcu_qsbr_unlock(__rte_unused struct rte_rcu_qsbr *v,
432 __rte_unused unsigned int thread_id)
433 {
434 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
435
436 #if defined(RTE_LIBRTE_RCU_DEBUG)
437 /* Decrement the lock counter */
438 __atomic_fetch_sub(&v->qsbr_cnt[thread_id].lock_cnt,
439 1, __ATOMIC_RELEASE);
440
441 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, WARNING,
442 "Lock counter %u. Nested locks?\n",
443 v->qsbr_cnt[thread_id].lock_cnt);
444 #endif
445 }
446
447 /**
448 * Ask the reader threads to report the quiescent state
449 * status.
450 *
451 * This is implemented as a lock-free function. It is multi-thread
452 * safe and can be called from worker threads.
453 *
454 * @param v
455 * QS variable
456 * @return
457 * - This is the token for this call of the API. This should be
458 * passed to rte_rcu_qsbr_check API.
459 */
460 static __rte_always_inline uint64_t
rte_rcu_qsbr_start(struct rte_rcu_qsbr * v)461 rte_rcu_qsbr_start(struct rte_rcu_qsbr *v)
462 {
463 uint64_t t;
464
465 RTE_ASSERT(v != NULL);
466
467 /* Release the changes to the shared data structure.
468 * This store release will ensure that changes to any data
469 * structure are visible to the workers before the token
470 * update is visible.
471 */
472 t = __atomic_add_fetch(&v->token, 1, __ATOMIC_RELEASE);
473
474 return t;
475 }
476
477 /**
478 * Update quiescent state for a reader thread.
479 *
480 * This is implemented as a lock-free function. It is multi-thread safe.
481 * All the reader threads registered to report their quiescent state
482 * on the QS variable must call this API.
483 *
484 * @param v
485 * QS variable
486 * @param thread_id
487 * Update the quiescent state for the reader with this thread ID.
488 */
489 static __rte_always_inline void
rte_rcu_qsbr_quiescent(struct rte_rcu_qsbr * v,unsigned int thread_id)490 rte_rcu_qsbr_quiescent(struct rte_rcu_qsbr *v, unsigned int thread_id)
491 {
492 uint64_t t;
493
494 RTE_ASSERT(v != NULL && thread_id < v->max_threads);
495
496 __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n",
497 v->qsbr_cnt[thread_id].lock_cnt);
498
499 /* Acquire the changes to the shared data structure released
500 * by rte_rcu_qsbr_start.
501 * Later loads of the shared data structure should not move
502 * above this load. Hence, use load-acquire.
503 */
504 t = __atomic_load_n(&v->token, __ATOMIC_ACQUIRE);
505
506 /* Check if there are updates available from the writer.
507 * Inform the writer that updates are visible to this reader.
508 * Prior loads of the shared data structure should not move
509 * beyond this store. Hence use store-release.
510 */
511 if (t != __atomic_load_n(&v->qsbr_cnt[thread_id].cnt, __ATOMIC_RELAXED))
512 __atomic_store_n(&v->qsbr_cnt[thread_id].cnt,
513 t, __ATOMIC_RELEASE);
514
515 __RTE_RCU_DP_LOG(DEBUG, "%s: update: token = %" PRIu64 ", Thread ID = %d",
516 __func__, t, thread_id);
517 }
518
519 /* Check the quiescent state counter for registered threads only, assuming
520 * that not all threads have registered.
521 */
522 static __rte_always_inline int
__rte_rcu_qsbr_check_selective(struct rte_rcu_qsbr * v,uint64_t t,bool wait)523 __rte_rcu_qsbr_check_selective(struct rte_rcu_qsbr *v, uint64_t t, bool wait)
524 {
525 uint32_t i, j, id;
526 uint64_t bmap;
527 uint64_t c;
528 uint64_t *reg_thread_id;
529 uint64_t acked_token = __RTE_QSBR_CNT_MAX;
530
531 for (i = 0, reg_thread_id = __RTE_QSBR_THRID_ARRAY_ELM(v, 0);
532 i < v->num_elems;
533 i++, reg_thread_id++) {
534 /* Load the current registered thread bit map before
535 * loading the reader thread quiescent state counters.
536 */
537 bmap = __atomic_load_n(reg_thread_id, __ATOMIC_ACQUIRE);
538 id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
539
540 while (bmap) {
541 j = __builtin_ctzl(bmap);
542 __RTE_RCU_DP_LOG(DEBUG,
543 "%s: check: token = %" PRIu64 ", wait = %d, Bit Map = 0x%" PRIx64 ", Thread ID = %d",
544 __func__, t, wait, bmap, id + j);
545 c = __atomic_load_n(
546 &v->qsbr_cnt[id + j].cnt,
547 __ATOMIC_ACQUIRE);
548 __RTE_RCU_DP_LOG(DEBUG,
549 "%s: status: token = %" PRIu64 ", wait = %d, Thread QS cnt = %" PRIu64 ", Thread ID = %d",
550 __func__, t, wait, c, id+j);
551
552 /* Counter is not checked for wrap-around condition
553 * as it is a 64b counter.
554 */
555 if (unlikely(c !=
556 __RTE_QSBR_CNT_THR_OFFLINE && c < t)) {
557 /* This thread is not in quiescent state */
558 if (!wait)
559 return 0;
560
561 rte_pause();
562 /* This thread might have unregistered.
563 * Re-read the bitmap.
564 */
565 bmap = __atomic_load_n(reg_thread_id,
566 __ATOMIC_ACQUIRE);
567
568 continue;
569 }
570
571 /* This thread is in quiescent state. Use the counter
572 * to find the least acknowledged token among all the
573 * readers.
574 */
575 if (c != __RTE_QSBR_CNT_THR_OFFLINE && acked_token > c)
576 acked_token = c;
577
578 bmap &= ~(1UL << j);
579 }
580 }
581
582 /* All readers are checked, update least acknowledged token.
583 * There might be multiple writers trying to update this. There is
584 * no need to update this very accurately using compare-and-swap.
585 */
586 if (acked_token != __RTE_QSBR_CNT_MAX)
587 __atomic_store_n(&v->acked_token, acked_token,
588 __ATOMIC_RELAXED);
589
590 return 1;
591 }
592
593 /* Check the quiescent state counter for all threads, assuming that
594 * all the threads have registered.
595 */
596 static __rte_always_inline int
__rte_rcu_qsbr_check_all(struct rte_rcu_qsbr * v,uint64_t t,bool wait)597 __rte_rcu_qsbr_check_all(struct rte_rcu_qsbr *v, uint64_t t, bool wait)
598 {
599 uint32_t i;
600 struct rte_rcu_qsbr_cnt *cnt;
601 uint64_t c;
602 uint64_t acked_token = __RTE_QSBR_CNT_MAX;
603
604 for (i = 0, cnt = v->qsbr_cnt; i < v->max_threads; i++, cnt++) {
605 __RTE_RCU_DP_LOG(DEBUG,
606 "%s: check: token = %" PRIu64 ", wait = %d, Thread ID = %d",
607 __func__, t, wait, i);
608 while (1) {
609 c = __atomic_load_n(&cnt->cnt, __ATOMIC_ACQUIRE);
610 __RTE_RCU_DP_LOG(DEBUG,
611 "%s: status: token = %" PRIu64 ", wait = %d, Thread QS cnt = %" PRIu64 ", Thread ID = %d",
612 __func__, t, wait, c, i);
613
614 /* Counter is not checked for wrap-around condition
615 * as it is a 64b counter.
616 */
617 if (likely(c == __RTE_QSBR_CNT_THR_OFFLINE || c >= t))
618 break;
619
620 /* This thread is not in quiescent state */
621 if (!wait)
622 return 0;
623
624 rte_pause();
625 }
626
627 /* This thread is in quiescent state. Use the counter to find
628 * the least acknowledged token among all the readers.
629 */
630 if (likely(c != __RTE_QSBR_CNT_THR_OFFLINE && acked_token > c))
631 acked_token = c;
632 }
633
634 /* All readers are checked, update least acknowledged token.
635 * There might be multiple writers trying to update this. There is
636 * no need to update this very accurately using compare-and-swap.
637 */
638 if (acked_token != __RTE_QSBR_CNT_MAX)
639 __atomic_store_n(&v->acked_token, acked_token,
640 __ATOMIC_RELAXED);
641
642 return 1;
643 }
644
645 /**
646 * Checks if all the reader threads have entered the quiescent state
647 * referenced by token.
648 *
649 * This is implemented as a lock-free function. It is multi-thread
650 * safe and can be called from the worker threads as well.
651 *
652 * If this API is called with 'wait' set to true, the following
653 * factors must be considered:
654 *
655 * 1) If the calling thread is also reporting the status on the
656 * same QS variable, it must update the quiescent state status, before
657 * calling this API.
658 *
659 * 2) In addition, while calling from multiple threads, only
660 * one of those threads can be reporting the quiescent state status
661 * on a given QS variable.
662 *
663 * @param v
664 * QS variable
665 * @param t
666 * Token returned by rte_rcu_qsbr_start API
667 * @param wait
668 * If true, block till all the reader threads have completed entering
669 * the quiescent state referenced by token 't'.
670 * @return
671 * - 0 if all reader threads have NOT passed through specified number
672 * of quiescent states.
673 * - 1 if all reader threads have passed through specified number
674 * of quiescent states.
675 */
676 static __rte_always_inline int
rte_rcu_qsbr_check(struct rte_rcu_qsbr * v,uint64_t t,bool wait)677 rte_rcu_qsbr_check(struct rte_rcu_qsbr *v, uint64_t t, bool wait)
678 {
679 RTE_ASSERT(v != NULL);
680
681 /* Check if all the readers have already acknowledged this token */
682 if (likely(t <= v->acked_token)) {
683 __RTE_RCU_DP_LOG(DEBUG,
684 "%s: check: token = %" PRIu64 ", wait = %d",
685 __func__, t, wait);
686 __RTE_RCU_DP_LOG(DEBUG,
687 "%s: status: least acked token = %" PRIu64,
688 __func__, v->acked_token);
689 return 1;
690 }
691
692 if (likely(v->num_threads == v->max_threads))
693 return __rte_rcu_qsbr_check_all(v, t, wait);
694 else
695 return __rte_rcu_qsbr_check_selective(v, t, wait);
696 }
697
698 /**
699 * Wait till the reader threads have entered quiescent state.
700 *
701 * This is implemented as a lock-free function. It is multi-thread safe.
702 * This API can be thought of as a wrapper around rte_rcu_qsbr_start and
703 * rte_rcu_qsbr_check APIs.
704 *
705 * If this API is called from multiple threads, only one of
706 * those threads can be reporting the quiescent state status on a
707 * given QS variable.
708 *
709 * @param v
710 * QS variable
711 * @param thread_id
712 * Thread ID of the caller if it is registered to report quiescent state
713 * on this QS variable (i.e. the calling thread is also part of the
714 * readside critical section). If not, pass RTE_QSBR_THRID_INVALID.
715 */
716 void
717 rte_rcu_qsbr_synchronize(struct rte_rcu_qsbr *v, unsigned int thread_id);
718
719 /**
720 * Dump the details of a single QS variables to a file.
721 *
722 * It is NOT multi-thread safe.
723 *
724 * @param f
725 * A pointer to a file for output
726 * @param v
727 * QS variable
728 * @return
729 * On success - 0
730 * On error - 1 with error code set in rte_errno.
731 * Possible rte_errno codes are:
732 * - EINVAL - NULL parameters are passed
733 */
734 int
735 rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v);
736
737 /**
738 * @warning
739 * @b EXPERIMENTAL: this API may change without prior notice
740 *
741 * Create a queue used to store the data structure elements that can
742 * be freed later. This queue is referred to as 'defer queue'.
743 *
744 * @param params
745 * Parameters to create a defer queue.
746 * @return
747 * On success - Valid pointer to defer queue
748 * On error - NULL
749 * Possible rte_errno codes are:
750 * - EINVAL - NULL parameters are passed
751 * - ENOMEM - Not enough memory
752 */
753 __rte_experimental
754 struct rte_rcu_qsbr_dq *
755 rte_rcu_qsbr_dq_create(const struct rte_rcu_qsbr_dq_parameters *params);
756
757 /**
758 * @warning
759 * @b EXPERIMENTAL: this API may change without prior notice
760 *
761 * Enqueue one resource to the defer queue and start the grace period.
762 * The resource will be freed later after at least one grace period
763 * is over.
764 *
765 * If the defer queue is full, it will attempt to reclaim resources.
766 * It will also reclaim resources at regular intervals to avoid
767 * the defer queue from growing too big.
768 *
769 * Multi-thread safety is provided as the defer queue configuration.
770 * When multi-thread safety is requested, it is possible that the
771 * resources are not stored in their order of deletion. This results
772 * in resources being held in the defer queue longer than they should.
773 *
774 * @param dq
775 * Defer queue to allocate an entry from.
776 * @param e
777 * Pointer to resource data to copy to the defer queue. The size of
778 * the data to copy is equal to the element size provided when the
779 * defer queue was created.
780 * @return
781 * On success - 0
782 * On error - 1 with rte_errno set to
783 * - EINVAL - NULL parameters are passed
784 * - ENOSPC - Defer queue is full. This condition can not happen
785 * if the defer queue size is equal (or larger) than the
786 * number of elements in the data structure.
787 */
788 __rte_experimental
789 int
790 rte_rcu_qsbr_dq_enqueue(struct rte_rcu_qsbr_dq *dq, void *e);
791
792 /**
793 * @warning
794 * @b EXPERIMENTAL: this API may change without prior notice
795 *
796 * Free resources from the defer queue.
797 *
798 * This API is multi-thread safe.
799 *
800 * @param dq
801 * Defer queue to free an entry from.
802 * @param n
803 * Maximum number of resources to free.
804 * @param freed
805 * Number of resources that were freed.
806 * @param pending
807 * Number of resources pending on the defer queue. This number might not
808 * be accurate if multi-thread safety is configured.
809 * @param available
810 * Number of resources that can be added to the defer queue.
811 * This number might not be accurate if multi-thread safety is configured.
812 * @return
813 * On successful reclamation of at least 1 resource - 0
814 * On error - 1 with rte_errno set to
815 * - EINVAL - NULL parameters are passed
816 */
817 __rte_experimental
818 int
819 rte_rcu_qsbr_dq_reclaim(struct rte_rcu_qsbr_dq *dq, unsigned int n,
820 unsigned int *freed, unsigned int *pending, unsigned int *available);
821
822 /**
823 * @warning
824 * @b EXPERIMENTAL: this API may change without prior notice
825 *
826 * Delete a defer queue.
827 *
828 * It tries to reclaim all the resources on the defer queue.
829 * If any of the resources have not completed the grace period
830 * the reclamation stops and returns immediately. The rest of
831 * the resources are not reclaimed and the defer queue is not
832 * freed.
833 *
834 * @param dq
835 * Defer queue to delete.
836 * @return
837 * On success - 0
838 * On error - 1
839 * Possible rte_errno codes are:
840 * - EAGAIN - Some of the resources have not completed at least 1 grace
841 * period, try again.
842 */
843 __rte_experimental
844 int
845 rte_rcu_qsbr_dq_delete(struct rte_rcu_qsbr_dq *dq);
846
847 #ifdef __cplusplus
848 }
849 #endif
850
851 #endif /* _RTE_RCU_QSBR_H_ */
852