1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2015 Intel Corporation
3 */
4
5 #ifndef _I40E_RXTX_VEC_COMMON_H_
6 #define _I40E_RXTX_VEC_COMMON_H_
7 #include <stdint.h>
8 #include <rte_ethdev_driver.h>
9 #include <rte_malloc.h>
10
11 #include "i40e_ethdev.h"
12 #include "i40e_rxtx.h"
13
14 static inline uint16_t
reassemble_packets(struct i40e_rx_queue * rxq,struct rte_mbuf ** rx_bufs,uint16_t nb_bufs,uint8_t * split_flags)15 reassemble_packets(struct i40e_rx_queue *rxq, struct rte_mbuf **rx_bufs,
16 uint16_t nb_bufs, uint8_t *split_flags)
17 {
18 struct rte_mbuf *pkts[RTE_I40E_VPMD_RX_BURST]; /*finished pkts*/
19 struct rte_mbuf *start = rxq->pkt_first_seg;
20 struct rte_mbuf *end = rxq->pkt_last_seg;
21 unsigned pkt_idx, buf_idx;
22
23 for (buf_idx = 0, pkt_idx = 0; buf_idx < nb_bufs; buf_idx++) {
24 if (end != NULL) {
25 /* processing a split packet */
26 end->next = rx_bufs[buf_idx];
27 rx_bufs[buf_idx]->data_len += rxq->crc_len;
28
29 start->nb_segs++;
30 start->pkt_len += rx_bufs[buf_idx]->data_len;
31 end = end->next;
32
33 if (!split_flags[buf_idx]) {
34 /* it's the last packet of the set */
35 start->hash = end->hash;
36 start->vlan_tci = end->vlan_tci;
37 start->ol_flags = end->ol_flags;
38 /* we need to strip crc for the whole packet */
39 start->pkt_len -= rxq->crc_len;
40 if (end->data_len > rxq->crc_len)
41 end->data_len -= rxq->crc_len;
42 else {
43 /* free up last mbuf */
44 struct rte_mbuf *secondlast = start;
45
46 start->nb_segs--;
47 while (secondlast->next != end)
48 secondlast = secondlast->next;
49 secondlast->data_len -= (rxq->crc_len -
50 end->data_len);
51 secondlast->next = NULL;
52 rte_pktmbuf_free_seg(end);
53 }
54 pkts[pkt_idx++] = start;
55 start = end = NULL;
56 }
57 } else {
58 /* not processing a split packet */
59 if (!split_flags[buf_idx]) {
60 /* not a split packet, save and skip */
61 pkts[pkt_idx++] = rx_bufs[buf_idx];
62 continue;
63 }
64 end = start = rx_bufs[buf_idx];
65 rx_bufs[buf_idx]->data_len += rxq->crc_len;
66 rx_bufs[buf_idx]->pkt_len += rxq->crc_len;
67 }
68 }
69
70 /* save the partial packet for next time */
71 rxq->pkt_first_seg = start;
72 rxq->pkt_last_seg = end;
73 memcpy(rx_bufs, pkts, pkt_idx * (sizeof(*pkts)));
74 return pkt_idx;
75 }
76
77 static __rte_always_inline int
i40e_tx_free_bufs(struct i40e_tx_queue * txq)78 i40e_tx_free_bufs(struct i40e_tx_queue *txq)
79 {
80 struct i40e_tx_entry *txep;
81 uint32_t n;
82 uint32_t i;
83 int nb_free = 0;
84 struct rte_mbuf *m, *free[RTE_I40E_TX_MAX_FREE_BUF_SZ];
85
86 /* check DD bits on threshold descriptor */
87 if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
88 rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
89 rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
90 return 0;
91
92 n = txq->tx_rs_thresh;
93
94 /* first buffer to free from S/W ring is at index
95 * tx_next_dd - (tx_rs_thresh-1)
96 */
97 txep = &txq->sw_ring[txq->tx_next_dd - (n - 1)];
98 m = rte_pktmbuf_prefree_seg(txep[0].mbuf);
99 if (likely(m != NULL)) {
100 free[0] = m;
101 nb_free = 1;
102 for (i = 1; i < n; i++) {
103 m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
104 if (likely(m != NULL)) {
105 if (likely(m->pool == free[0]->pool)) {
106 free[nb_free++] = m;
107 } else {
108 rte_mempool_put_bulk(free[0]->pool,
109 (void *)free,
110 nb_free);
111 free[0] = m;
112 nb_free = 1;
113 }
114 }
115 }
116 rte_mempool_put_bulk(free[0]->pool, (void **)free, nb_free);
117 } else {
118 for (i = 1; i < n; i++) {
119 m = rte_pktmbuf_prefree_seg(txep[i].mbuf);
120 if (m != NULL)
121 rte_mempool_put(m->pool, m);
122 }
123 }
124
125 /* buffers were freed, update counters */
126 txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
127 txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
128 if (txq->tx_next_dd >= txq->nb_tx_desc)
129 txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
130
131 return txq->tx_rs_thresh;
132 }
133
134 static __rte_always_inline void
tx_backlog_entry(struct i40e_tx_entry * txep,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)135 tx_backlog_entry(struct i40e_tx_entry *txep,
136 struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
137 {
138 int i;
139
140 for (i = 0; i < (int)nb_pkts; ++i)
141 txep[i].mbuf = tx_pkts[i];
142 }
143
144 static inline void
_i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue * rxq)145 _i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue *rxq)
146 {
147 const unsigned mask = rxq->nb_rx_desc - 1;
148 unsigned i;
149
150 if (rxq->sw_ring == NULL || rxq->rxrearm_nb >= rxq->nb_rx_desc)
151 return;
152
153 /* free all mbufs that are valid in the ring */
154 if (rxq->rxrearm_nb == 0) {
155 for (i = 0; i < rxq->nb_rx_desc; i++) {
156 if (rxq->sw_ring[i].mbuf != NULL)
157 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
158 }
159 } else {
160 for (i = rxq->rx_tail;
161 i != rxq->rxrearm_start;
162 i = (i + 1) & mask) {
163 if (rxq->sw_ring[i].mbuf != NULL)
164 rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
165 }
166 }
167
168 rxq->rxrearm_nb = rxq->nb_rx_desc;
169
170 /* set all entries to NULL */
171 memset(rxq->sw_ring, 0, sizeof(rxq->sw_ring[0]) * rxq->nb_rx_desc);
172 }
173
174 static inline int
i40e_rxq_vec_setup_default(struct i40e_rx_queue * rxq)175 i40e_rxq_vec_setup_default(struct i40e_rx_queue *rxq)
176 {
177 uintptr_t p;
178 struct rte_mbuf mb_def = { .buf_addr = 0 }; /* zeroed mbuf */
179
180 mb_def.nb_segs = 1;
181 mb_def.data_off = RTE_PKTMBUF_HEADROOM;
182 mb_def.port = rxq->port_id;
183 rte_mbuf_refcnt_set(&mb_def, 1);
184
185 /* prevent compiler reordering: rearm_data covers previous fields */
186 rte_compiler_barrier();
187 p = (uintptr_t)&mb_def.rearm_data;
188 rxq->mbuf_initializer = *(uint64_t *)p;
189 return 0;
190 }
191
192 static inline int
i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev * dev)193 i40e_rx_vec_dev_conf_condition_check_default(struct rte_eth_dev *dev)
194 {
195 #ifndef RTE_LIBRTE_IEEE1588
196 struct i40e_adapter *ad =
197 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
198 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
199 struct rte_fdir_conf *fconf = &dev->data->dev_conf.fdir_conf;
200 struct i40e_rx_queue *rxq;
201 uint16_t desc, i;
202 bool first_queue;
203
204 /* no fdir support */
205 if (fconf->mode != RTE_FDIR_MODE_NONE)
206 return -1;
207
208 /* no header split support */
209 if (rxmode->offloads & DEV_RX_OFFLOAD_HEADER_SPLIT)
210 return -1;
211
212 /* no QinQ support */
213 if (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
214 return -1;
215
216 /**
217 * Vector mode is allowed only when number of Rx queue
218 * descriptor is power of 2.
219 */
220 if (!dev->data->dev_started) {
221 first_queue = true;
222 for (i = 0; i < dev->data->nb_rx_queues; i++) {
223 rxq = dev->data->rx_queues[i];
224 if (!rxq)
225 continue;
226 desc = rxq->nb_rx_desc;
227 if (first_queue)
228 ad->rx_vec_allowed =
229 rte_is_power_of_2(desc);
230 else
231 ad->rx_vec_allowed =
232 ad->rx_vec_allowed ?
233 rte_is_power_of_2(desc) :
234 ad->rx_vec_allowed;
235 first_queue = false;
236 }
237 } else {
238 /* Only check the first queue's descriptor number */
239 for (i = 0; i < dev->data->nb_rx_queues; i++) {
240 rxq = dev->data->rx_queues[i];
241 if (!rxq)
242 continue;
243 desc = rxq->nb_rx_desc;
244 ad->rx_vec_allowed = rte_is_power_of_2(desc);
245 break;
246 }
247 }
248
249 return 0;
250 #else
251 RTE_SET_USED(dev);
252 return -1;
253 #endif
254 }
255 #endif
256