1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Intel Corporation
3 */
4
5 #ifndef _IQ_CHUNK_H_
6 #define _IQ_CHUNK_H_
7
8 #include <stdint.h>
9 #include <stdbool.h>
10 #include <rte_eventdev.h>
11
12 #define IQ_ROB_NAMESIZE 12
13
14 struct sw_queue_chunk {
15 struct rte_event events[SW_EVS_PER_Q_CHUNK];
16 struct sw_queue_chunk *next;
17 } __rte_cache_aligned;
18
19 static __rte_always_inline bool
iq_empty(struct sw_iq * iq)20 iq_empty(struct sw_iq *iq)
21 {
22 return (iq->count == 0);
23 }
24
25 static __rte_always_inline uint16_t
iq_count(const struct sw_iq * iq)26 iq_count(const struct sw_iq *iq)
27 {
28 return iq->count;
29 }
30
31 static __rte_always_inline struct sw_queue_chunk *
iq_alloc_chunk(struct sw_evdev * sw)32 iq_alloc_chunk(struct sw_evdev *sw)
33 {
34 struct sw_queue_chunk *chunk = sw->chunk_list_head;
35 sw->chunk_list_head = chunk->next;
36 chunk->next = NULL;
37 return chunk;
38 }
39
40 static __rte_always_inline void
iq_free_chunk(struct sw_evdev * sw,struct sw_queue_chunk * chunk)41 iq_free_chunk(struct sw_evdev *sw, struct sw_queue_chunk *chunk)
42 {
43 chunk->next = sw->chunk_list_head;
44 sw->chunk_list_head = chunk;
45 }
46
47 static __rte_always_inline void
iq_free_chunk_list(struct sw_evdev * sw,struct sw_queue_chunk * head)48 iq_free_chunk_list(struct sw_evdev *sw, struct sw_queue_chunk *head)
49 {
50 while (head) {
51 struct sw_queue_chunk *next;
52 next = head->next;
53 iq_free_chunk(sw, head);
54 head = next;
55 }
56 }
57
58 static __rte_always_inline void
iq_init(struct sw_evdev * sw,struct sw_iq * iq)59 iq_init(struct sw_evdev *sw, struct sw_iq *iq)
60 {
61 iq->head = iq_alloc_chunk(sw);
62 iq->tail = iq->head;
63 iq->head_idx = 0;
64 iq->tail_idx = 0;
65 iq->count = 0;
66 }
67
68 static __rte_always_inline void
iq_enqueue(struct sw_evdev * sw,struct sw_iq * iq,const struct rte_event * ev)69 iq_enqueue(struct sw_evdev *sw, struct sw_iq *iq, const struct rte_event *ev)
70 {
71 iq->tail->events[iq->tail_idx++] = *ev;
72 iq->count++;
73
74 if (unlikely(iq->tail_idx == SW_EVS_PER_Q_CHUNK)) {
75 /* The number of chunks is defined in relation to the total
76 * number of inflight events and number of IQS such that
77 * allocation will always succeed.
78 */
79 struct sw_queue_chunk *chunk = iq_alloc_chunk(sw);
80 iq->tail->next = chunk;
81 iq->tail = chunk;
82 iq->tail_idx = 0;
83 }
84 }
85
86 static __rte_always_inline void
iq_pop(struct sw_evdev * sw,struct sw_iq * iq)87 iq_pop(struct sw_evdev *sw, struct sw_iq *iq)
88 {
89 iq->head_idx++;
90 iq->count--;
91
92 if (unlikely(iq->head_idx == SW_EVS_PER_Q_CHUNK)) {
93 struct sw_queue_chunk *next = iq->head->next;
94 iq_free_chunk(sw, iq->head);
95 iq->head = next;
96 iq->head_idx = 0;
97 }
98 }
99
100 static __rte_always_inline const struct rte_event *
iq_peek(struct sw_iq * iq)101 iq_peek(struct sw_iq *iq)
102 {
103 return &iq->head->events[iq->head_idx];
104 }
105
106 /* Note: the caller must ensure that count <= iq_count() */
107 static __rte_always_inline uint16_t
iq_dequeue_burst(struct sw_evdev * sw,struct sw_iq * iq,struct rte_event * ev,uint16_t count)108 iq_dequeue_burst(struct sw_evdev *sw,
109 struct sw_iq *iq,
110 struct rte_event *ev,
111 uint16_t count)
112 {
113 struct sw_queue_chunk *current;
114 uint16_t total, index;
115
116 count = RTE_MIN(count, iq_count(iq));
117
118 current = iq->head;
119 index = iq->head_idx;
120 total = 0;
121
122 /* Loop over the chunks */
123 while (1) {
124 struct sw_queue_chunk *next;
125 for (; index < SW_EVS_PER_Q_CHUNK;) {
126 ev[total++] = current->events[index++];
127
128 if (unlikely(total == count))
129 goto done;
130 }
131
132 /* Move to the next chunk */
133 next = current->next;
134 iq_free_chunk(sw, current);
135 current = next;
136 index = 0;
137 }
138
139 done:
140 if (unlikely(index == SW_EVS_PER_Q_CHUNK)) {
141 struct sw_queue_chunk *next = current->next;
142 iq_free_chunk(sw, current);
143 iq->head = next;
144 iq->head_idx = 0;
145 } else {
146 iq->head = current;
147 iq->head_idx = index;
148 }
149
150 iq->count -= total;
151
152 return total;
153 }
154
155 static __rte_always_inline void
iq_put_back(struct sw_evdev * sw,struct sw_iq * iq,struct rte_event * ev,unsigned int count)156 iq_put_back(struct sw_evdev *sw,
157 struct sw_iq *iq,
158 struct rte_event *ev,
159 unsigned int count)
160 {
161 /* Put back events that fit in the current head chunk. If necessary,
162 * put back events in a new head chunk. The caller must ensure that
163 * count <= SW_EVS_PER_Q_CHUNK, to ensure that at most one new head is
164 * needed.
165 */
166 uint16_t avail_space = iq->head_idx;
167
168 if (avail_space >= count) {
169 const uint16_t idx = avail_space - count;
170 uint16_t i;
171
172 for (i = 0; i < count; i++)
173 iq->head->events[idx + i] = ev[i];
174
175 iq->head_idx = idx;
176 } else if (avail_space < count) {
177 const uint16_t remaining = count - avail_space;
178 struct sw_queue_chunk *new_head;
179 uint16_t i;
180
181 for (i = 0; i < avail_space; i++)
182 iq->head->events[i] = ev[remaining + i];
183
184 new_head = iq_alloc_chunk(sw);
185 new_head->next = iq->head;
186 iq->head = new_head;
187 iq->head_idx = SW_EVS_PER_Q_CHUNK - remaining;
188
189 for (i = 0; i < remaining; i++)
190 iq->head->events[iq->head_idx + i] = ev[i];
191 }
192
193 iq->count += count;
194 }
195
196 #endif /* _IQ_CHUNK_H_ */
197