1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 */
4
5
6
7 /**
8 * @internal when c11 memory model enabled use c11 atomic memory barrier.
9 * when under non c11 memory model use rte_smp_* memory barrier.
10 *
11 * @param src
12 * Pointer to the source data.
13 * @param dst
14 * Pointer to the destination data.
15 * @param value
16 * Data value.
17 */
18 #ifdef RTE_USE_C11_MEM_MODEL
19 #define __KNI_LOAD_ACQUIRE(src) ({ \
20 __atomic_load_n((src), __ATOMIC_ACQUIRE); \
21 })
22 #define __KNI_STORE_RELEASE(dst, value) do { \
23 __atomic_store_n((dst), value, __ATOMIC_RELEASE); \
24 } while(0)
25 #else
26 #define __KNI_LOAD_ACQUIRE(src) ({ \
27 typeof (*(src)) val = *(src); \
28 rte_smp_rmb(); \
29 val; \
30 })
31 #define __KNI_STORE_RELEASE(dst, value) do { \
32 *(dst) = value; \
33 rte_smp_wmb(); \
34 } while(0)
35 #endif
36
37 /**
38 * Initializes the kni fifo structure
39 */
40 static void
kni_fifo_init(struct rte_kni_fifo * fifo,unsigned size)41 kni_fifo_init(struct rte_kni_fifo *fifo, unsigned size)
42 {
43 /* Ensure size is power of 2 */
44 if (size & (size - 1))
45 rte_panic("KNI fifo size must be power of 2\n");
46
47 fifo->write = 0;
48 fifo->read = 0;
49 fifo->len = size;
50 fifo->elem_size = sizeof(void *);
51 }
52
53 /**
54 * Adds num elements into the fifo. Return the number actually written
55 */
56 static inline unsigned
kni_fifo_put(struct rte_kni_fifo * fifo,void ** data,unsigned num)57 kni_fifo_put(struct rte_kni_fifo *fifo, void **data, unsigned num)
58 {
59 unsigned i = 0;
60 unsigned fifo_write = fifo->write;
61 unsigned new_write = fifo_write;
62 unsigned fifo_read = __KNI_LOAD_ACQUIRE(&fifo->read);
63
64 for (i = 0; i < num; i++) {
65 new_write = (new_write + 1) & (fifo->len - 1);
66
67 if (new_write == fifo_read)
68 break;
69 fifo->buffer[fifo_write] = data[i];
70 fifo_write = new_write;
71 }
72 __KNI_STORE_RELEASE(&fifo->write, fifo_write);
73 return i;
74 }
75
76 /**
77 * Get up to num elements from the fifo. Return the number actually read
78 */
79 static inline unsigned
kni_fifo_get(struct rte_kni_fifo * fifo,void ** data,unsigned num)80 kni_fifo_get(struct rte_kni_fifo *fifo, void **data, unsigned num)
81 {
82 unsigned i = 0;
83 unsigned new_read = fifo->read;
84 unsigned fifo_write = __KNI_LOAD_ACQUIRE(&fifo->write);
85
86 for (i = 0; i < num; i++) {
87 if (new_read == fifo_write)
88 break;
89
90 data[i] = fifo->buffer[new_read];
91 new_read = (new_read + 1) & (fifo->len - 1);
92 }
93 __KNI_STORE_RELEASE(&fifo->read, new_read);
94 return i;
95 }
96
97 /**
98 * Get the num of elements in the fifo
99 */
100 static inline uint32_t
kni_fifo_count(struct rte_kni_fifo * fifo)101 kni_fifo_count(struct rte_kni_fifo *fifo)
102 {
103 unsigned fifo_write = __KNI_LOAD_ACQUIRE(&fifo->write);
104 unsigned fifo_read = __KNI_LOAD_ACQUIRE(&fifo->read);
105 return (fifo->len + fifo_write - fifo_read) & (fifo->len - 1);
106 }
107
108 /**
109 * Get the num of available elements in the fifo
110 */
111 static inline uint32_t
kni_fifo_free_count(struct rte_kni_fifo * fifo)112 kni_fifo_free_count(struct rte_kni_fifo *fifo)
113 {
114 uint32_t fifo_write = __KNI_LOAD_ACQUIRE(&fifo->write);
115 uint32_t fifo_read = __KNI_LOAD_ACQUIRE(&fifo->read);
116 return (fifo_read - fifo_write - 1) & (fifo->len - 1);
117 }
118