1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
4 */
5
6 #include <stdint.h>
7 #include <string.h>
8 #include <stdlib.h>
9
10 #include <rte_mbuf.h>
11 #include <rte_mempool.h>
12 #include <rte_prefetch.h>
13 #include <rte_vect.h>
14
15 #include <mlx5_glue.h>
16 #include <mlx5_prm.h>
17
18 #include "mlx5_defs.h"
19 #include "mlx5.h"
20 #include "mlx5_utils.h"
21 #include "mlx5_rxtx.h"
22 #include "mlx5_rx.h"
23 #include "mlx5_rxtx_vec.h"
24 #include "mlx5_autoconf.h"
25
26 #if defined RTE_ARCH_X86_64
27 #include "mlx5_rxtx_vec_sse.h"
28 #elif defined RTE_ARCH_ARM64
29 #include "mlx5_rxtx_vec_neon.h"
30 #elif defined RTE_ARCH_PPC_64
31 #include "mlx5_rxtx_vec_altivec.h"
32 #else
33 #error "This should not be compiled if SIMD instructions are not supported."
34 #endif
35
36 /**
37 * Skip error packets.
38 *
39 * @param rxq
40 * Pointer to RX queue structure.
41 * @param[out] pkts
42 * Array to store received packets.
43 * @param pkts_n
44 * Maximum number of packets in array.
45 *
46 * @return
47 * Number of packets successfully received (<= pkts_n).
48 */
49 static uint16_t
rxq_handle_pending_error(struct mlx5_rxq_data * rxq,struct rte_mbuf ** pkts,uint16_t pkts_n)50 rxq_handle_pending_error(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
51 uint16_t pkts_n)
52 {
53 uint16_t n = 0;
54 unsigned int i;
55 #ifdef MLX5_PMD_SOFT_COUNTERS
56 uint32_t err_bytes = 0;
57 #endif
58
59 for (i = 0; i < pkts_n; ++i) {
60 struct rte_mbuf *pkt = pkts[i];
61
62 if (pkt->packet_type == RTE_PTYPE_ALL_MASK || rxq->err_state) {
63 #ifdef MLX5_PMD_SOFT_COUNTERS
64 err_bytes += PKT_LEN(pkt);
65 #endif
66 rte_pktmbuf_free_seg(pkt);
67 } else {
68 pkts[n++] = pkt;
69 }
70 }
71 rxq->stats.idropped += (pkts_n - n);
72 #ifdef MLX5_PMD_SOFT_COUNTERS
73 /* Correct counters of errored completions. */
74 rxq->stats.ipackets -= (pkts_n - n);
75 rxq->stats.ibytes -= err_bytes;
76 #endif
77 mlx5_rx_err_handle(rxq, 1);
78 return n;
79 }
80
81 /**
82 * Replenish buffers for RX in bulk.
83 *
84 * @param rxq
85 * Pointer to RX queue structure.
86 */
87 static inline void
mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data * rxq)88 mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq)
89 {
90 const uint16_t q_n = 1 << rxq->elts_n;
91 const uint16_t q_mask = q_n - 1;
92 uint16_t n = q_n - (rxq->rq_ci - rxq->rq_pi);
93 uint16_t elts_idx = rxq->rq_ci & q_mask;
94 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
95 volatile struct mlx5_wqe_data_seg *wq =
96 &((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx];
97 unsigned int i;
98
99 if (n >= rxq->rq_repl_thresh) {
100 MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n));
101 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) >
102 MLX5_VPMD_DESCS_PER_LOOP);
103 /* Not to cross queue end. */
104 n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx);
105 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
106 rxq->stats.rx_nombuf += n;
107 return;
108 }
109 if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1)) {
110 for (i = 0; i < n; ++i) {
111 /*
112 * In order to support the mbufs with external attached
113 * data buffer we should use the buf_addr pointer
114 * instead of rte_mbuf_buf_addr(). It touches the mbuf
115 * itself and may impact the performance.
116 */
117 void *buf_addr = elts[i]->buf_addr;
118
119 wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
120 RTE_PKTMBUF_HEADROOM);
121 wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
122 }
123 } else {
124 for (i = 0; i < n; ++i) {
125 void *buf_addr = elts[i]->buf_addr;
126
127 wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
128 RTE_PKTMBUF_HEADROOM);
129 }
130 }
131 rxq->rq_ci += n;
132 /* Prevent overflowing into consumed mbufs. */
133 elts_idx = rxq->rq_ci & q_mask;
134 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
135 (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf;
136 rte_io_wmb();
137 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
138 }
139 }
140
141 /**
142 * Replenish buffers for MPRQ RX in bulk.
143 *
144 * @param rxq
145 * Pointer to RX queue structure.
146 */
147 static inline void
mlx5_rx_mprq_replenish_bulk_mbuf(struct mlx5_rxq_data * rxq)148 mlx5_rx_mprq_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq)
149 {
150 const uint16_t wqe_n = 1 << rxq->elts_n;
151 const uint32_t strd_n = RTE_BIT32(rxq->log_strd_num);
152 const uint32_t elts_n = wqe_n * strd_n;
153 const uint32_t wqe_mask = elts_n - 1;
154 uint32_t n = elts_n - (rxq->elts_ci - rxq->rq_pi);
155 uint32_t elts_idx = rxq->elts_ci & wqe_mask;
156 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
157 unsigned int i;
158
159 if (n >= rxq->rq_repl_thresh &&
160 rxq->elts_ci - rxq->rq_pi <=
161 rxq->rq_repl_thresh + MLX5_VPMD_RX_MAX_BURST) {
162 MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n));
163 MLX5_ASSERT(MLX5_VPMD_RXQ_RPLNSH_THRESH(elts_n) >
164 MLX5_VPMD_DESCS_PER_LOOP);
165 /* Not to cross queue end. */
166 n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, elts_n - elts_idx);
167 /* Limit replenish number to threshold value. */
168 n = RTE_MIN(n, rxq->rq_repl_thresh);
169 if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) {
170 rxq->stats.rx_nombuf += n;
171 return;
172 }
173 rxq->elts_ci += n;
174 /* Prevent overflowing into consumed mbufs. */
175 elts_idx = rxq->elts_ci & wqe_mask;
176 for (i = 0; i < MLX5_VPMD_DESCS_PER_LOOP; ++i)
177 (*rxq->elts)[elts_idx + i] = &rxq->fake_mbuf;
178 }
179 }
180
181 /**
182 * Copy or attach MPRQ buffers to RX SW ring.
183 *
184 * @param rxq
185 * Pointer to RX queue structure.
186 * @param pkts
187 * Pointer to array of packets to be stored.
188 * @param pkts_n
189 * Number of packets to be stored.
190 *
191 * @return
192 * Number of packets successfully copied/attached (<= pkts_n).
193 */
194 static inline uint16_t
rxq_copy_mprq_mbuf_v(struct mlx5_rxq_data * rxq,struct rte_mbuf ** pkts,uint16_t pkts_n)195 rxq_copy_mprq_mbuf_v(struct mlx5_rxq_data *rxq,
196 struct rte_mbuf **pkts, uint16_t pkts_n)
197 {
198 const uint16_t wqe_n = 1 << rxq->elts_n;
199 const uint16_t wqe_mask = wqe_n - 1;
200 const uint16_t strd_sz = RTE_BIT32(rxq->log_strd_sz);
201 const uint32_t strd_n = RTE_BIT32(rxq->log_strd_num);
202 const uint32_t elts_n = wqe_n * strd_n;
203 const uint32_t elts_mask = elts_n - 1;
204 uint32_t elts_idx = rxq->rq_pi & elts_mask;
205 struct rte_mbuf **elts = &(*rxq->elts)[elts_idx];
206 uint32_t rq_ci = rxq->rq_ci;
207 struct mlx5_mprq_buf *buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask];
208 uint16_t copied = 0;
209 uint16_t i = 0;
210
211 for (i = 0; i < pkts_n; ++i) {
212 uint16_t strd_cnt;
213 enum mlx5_rqx_code rxq_code;
214
215 if (rxq->consumed_strd == strd_n) {
216 /* Replace WQE if the buffer is still in use. */
217 mprq_buf_replace(rxq, rq_ci & wqe_mask);
218 /* Advance to the next WQE. */
219 rxq->consumed_strd = 0;
220 rq_ci++;
221 buf = (*rxq->mprq_bufs)[rq_ci & wqe_mask];
222 }
223
224 if (!elts[i]->pkt_len) {
225 rxq->consumed_strd = strd_n;
226 rte_pktmbuf_free_seg(elts[i]);
227 #ifdef MLX5_PMD_SOFT_COUNTERS
228 rxq->stats.ipackets -= 1;
229 #endif
230 continue;
231 }
232 strd_cnt = (elts[i]->pkt_len / strd_sz) +
233 ((elts[i]->pkt_len % strd_sz) ? 1 : 0);
234 rxq_code = mprq_buf_to_pkt(rxq, elts[i], elts[i]->pkt_len,
235 buf, rxq->consumed_strd, strd_cnt);
236 rxq->consumed_strd += strd_cnt;
237 if (unlikely(rxq_code != MLX5_RXQ_CODE_EXIT)) {
238 rte_pktmbuf_free_seg(elts[i]);
239 #ifdef MLX5_PMD_SOFT_COUNTERS
240 rxq->stats.ipackets -= 1;
241 rxq->stats.ibytes -= elts[i]->pkt_len;
242 #endif
243 if (rxq_code == MLX5_RXQ_CODE_NOMBUF) {
244 ++rxq->stats.rx_nombuf;
245 break;
246 }
247 if (rxq_code == MLX5_RXQ_CODE_DROPPED) {
248 ++rxq->stats.idropped;
249 continue;
250 }
251 }
252 pkts[copied++] = elts[i];
253 }
254 rxq->rq_pi += i;
255 rxq->cq_ci += i;
256 rte_io_wmb();
257 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
258 if (rq_ci != rxq->rq_ci) {
259 rxq->rq_ci = rq_ci;
260 rte_io_wmb();
261 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci);
262 }
263 return copied;
264 }
265
266 /**
267 * Receive burst of packets. An errored completion also consumes a mbuf, but the
268 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
269 * before returning to application.
270 *
271 * @param rxq
272 * Pointer to RX queue structure.
273 * @param[out] pkts
274 * Array to store received packets.
275 * @param pkts_n
276 * Maximum number of packets in array.
277 * @param[out] err
278 * Pointer to a flag. Set non-zero value if pkts array has at least one error
279 * packet to handle.
280 * @param[out] no_cq
281 * Pointer to a boolean. Set true if no new CQE seen.
282 *
283 * @return
284 * Number of packets received including errors (<= pkts_n).
285 */
286 static inline uint16_t
rxq_burst_v(struct mlx5_rxq_data * rxq,struct rte_mbuf ** pkts,uint16_t pkts_n,uint64_t * err,bool * no_cq)287 rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
288 uint16_t pkts_n, uint64_t *err, bool *no_cq)
289 {
290 const uint16_t q_n = 1 << rxq->cqe_n;
291 const uint16_t q_mask = q_n - 1;
292 const uint16_t e_n = 1 << rxq->elts_n;
293 const uint16_t e_mask = e_n - 1;
294 volatile struct mlx5_cqe *cq;
295 struct rte_mbuf **elts;
296 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
297 uint16_t nocmp_n = 0;
298 uint16_t rcvd_pkt = 0;
299 unsigned int cq_idx = rxq->cq_ci & q_mask;
300 unsigned int elts_idx;
301
302 MLX5_ASSERT(rxq->sges_n == 0);
303 MLX5_ASSERT(rxq->cqe_n == rxq->elts_n);
304 cq = &(*rxq->cqes)[cq_idx];
305 rte_prefetch0(cq);
306 rte_prefetch0(cq + 1);
307 rte_prefetch0(cq + 2);
308 rte_prefetch0(cq + 3);
309 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
310 mlx5_rx_replenish_bulk_mbuf(rxq);
311 /* See if there're unreturned mbufs from compressed CQE. */
312 rcvd_pkt = rxq->decompressed;
313 if (rcvd_pkt > 0) {
314 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
315 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask],
316 pkts, rcvd_pkt);
317 rxq->rq_pi += rcvd_pkt;
318 rxq->decompressed -= rcvd_pkt;
319 pkts += rcvd_pkt;
320 }
321 elts_idx = rxq->rq_pi & e_mask;
322 elts = &(*rxq->elts)[elts_idx];
323 /* Not to overflow pkts array. */
324 pkts_n = RTE_ALIGN_FLOOR(pkts_n - rcvd_pkt, MLX5_VPMD_DESCS_PER_LOOP);
325 /* Not to cross queue end. */
326 pkts_n = RTE_MIN(pkts_n, q_n - elts_idx);
327 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx);
328 if (!pkts_n) {
329 *no_cq = !rcvd_pkt;
330 return rcvd_pkt;
331 }
332 /* At this point, there shouldn't be any remaining packets. */
333 MLX5_ASSERT(rxq->decompressed == 0);
334 /* Process all the CQEs */
335 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx);
336 /* If no new CQE seen, return without updating cq_db. */
337 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) {
338 *no_cq = true;
339 return rcvd_pkt;
340 }
341 /* Update the consumer indexes for non-compressed CQEs. */
342 MLX5_ASSERT(nocmp_n <= pkts_n);
343 rxq->cq_ci += nocmp_n;
344 rxq->rq_pi += nocmp_n;
345 rcvd_pkt += nocmp_n;
346 /* Decompress the last CQE if compressed. */
347 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) {
348 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
349 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n],
350 &elts[nocmp_n]);
351 rxq->cq_ci += rxq->decompressed;
352 /* Return more packets if needed. */
353 if (nocmp_n < pkts_n) {
354 uint16_t n = rxq->decompressed;
355
356 n = RTE_MIN(n, pkts_n - nocmp_n);
357 rxq_copy_mbuf_v(&(*rxq->elts)[rxq->rq_pi & e_mask],
358 &pkts[nocmp_n], n);
359 rxq->rq_pi += n;
360 rcvd_pkt += n;
361 rxq->decompressed -= n;
362 }
363 }
364 rte_io_wmb();
365 *rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
366 *no_cq = !rcvd_pkt;
367 return rcvd_pkt;
368 }
369
370 /**
371 * DPDK callback for vectorized RX.
372 *
373 * @param dpdk_rxq
374 * Generic pointer to RX queue structure.
375 * @param[out] pkts
376 * Array to store received packets.
377 * @param pkts_n
378 * Maximum number of packets in array.
379 *
380 * @return
381 * Number of packets successfully received (<= pkts_n).
382 */
383 uint16_t
mlx5_rx_burst_vec(void * dpdk_rxq,struct rte_mbuf ** pkts,uint16_t pkts_n)384 mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
385 {
386 struct mlx5_rxq_data *rxq = dpdk_rxq;
387 uint16_t nb_rx = 0;
388 uint16_t tn = 0;
389 uint64_t err = 0;
390 bool no_cq = false;
391
392 do {
393 nb_rx = rxq_burst_v(rxq, pkts + tn, pkts_n - tn,
394 &err, &no_cq);
395 if (unlikely(err | rxq->err_state))
396 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx);
397 tn += nb_rx;
398 if (unlikely(no_cq))
399 break;
400 } while (tn != pkts_n);
401 return tn;
402 }
403
404 /**
405 * Receive burst of packets. An errored completion also consumes a mbuf, but the
406 * packet_type is set to be RTE_PTYPE_ALL_MASK. Marked mbufs should be freed
407 * before returning to application.
408 *
409 * @param rxq
410 * Pointer to RX queue structure.
411 * @param[out] pkts
412 * Array to store received packets.
413 * @param pkts_n
414 * Maximum number of packets in array.
415 * @param[out] err
416 * Pointer to a flag. Set non-zero value if pkts array has at least one error
417 * packet to handle.
418 * @param[out] no_cq
419 * Pointer to a boolean. Set true if no new CQE seen.
420 *
421 * @return
422 * Number of packets received including errors (<= pkts_n).
423 */
424 static inline uint16_t
rxq_burst_mprq_v(struct mlx5_rxq_data * rxq,struct rte_mbuf ** pkts,uint16_t pkts_n,uint64_t * err,bool * no_cq)425 rxq_burst_mprq_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts,
426 uint16_t pkts_n, uint64_t *err, bool *no_cq)
427 {
428 const uint16_t q_n = 1 << rxq->cqe_n;
429 const uint16_t q_mask = q_n - 1;
430 const uint16_t wqe_n = 1 << rxq->elts_n;
431 const uint32_t strd_n = RTE_BIT32(rxq->log_strd_num);
432 const uint32_t elts_n = wqe_n * strd_n;
433 const uint32_t elts_mask = elts_n - 1;
434 volatile struct mlx5_cqe *cq;
435 struct rte_mbuf **elts;
436 uint64_t comp_idx = MLX5_VPMD_DESCS_PER_LOOP;
437 uint16_t nocmp_n = 0;
438 uint16_t rcvd_pkt = 0;
439 uint16_t cp_pkt = 0;
440 unsigned int cq_idx = rxq->cq_ci & q_mask;
441 unsigned int elts_idx;
442
443 MLX5_ASSERT(rxq->sges_n == 0);
444 cq = &(*rxq->cqes)[cq_idx];
445 rte_prefetch0(cq);
446 rte_prefetch0(cq + 1);
447 rte_prefetch0(cq + 2);
448 rte_prefetch0(cq + 3);
449 pkts_n = RTE_MIN(pkts_n, MLX5_VPMD_RX_MAX_BURST);
450 mlx5_rx_mprq_replenish_bulk_mbuf(rxq);
451 /* Not to move past the allocated mbufs. */
452 pkts_n = RTE_MIN(pkts_n, rxq->elts_ci - rxq->rq_pi);
453 /* See if there're unreturned mbufs from compressed CQE. */
454 rcvd_pkt = rxq->decompressed;
455 if (rcvd_pkt > 0) {
456 rcvd_pkt = RTE_MIN(rcvd_pkt, pkts_n);
457 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, rcvd_pkt);
458 rxq->decompressed -= rcvd_pkt;
459 pkts += cp_pkt;
460 }
461 elts_idx = rxq->rq_pi & elts_mask;
462 elts = &(*rxq->elts)[elts_idx];
463 /* Not to overflow pkts array. */
464 pkts_n = RTE_ALIGN_FLOOR(pkts_n - cp_pkt, MLX5_VPMD_DESCS_PER_LOOP);
465 /* Not to cross queue end. */
466 pkts_n = RTE_MIN(pkts_n, elts_n - elts_idx);
467 pkts_n = RTE_MIN(pkts_n, q_n - cq_idx);
468 if (!pkts_n) {
469 *no_cq = !cp_pkt;
470 return cp_pkt;
471 }
472 /* At this point, there shouldn't be any remaining packets. */
473 MLX5_ASSERT(rxq->decompressed == 0);
474 /* Process all the CQEs */
475 nocmp_n = rxq_cq_process_v(rxq, cq, elts, pkts, pkts_n, err, &comp_idx);
476 /* If no new CQE seen, return without updating cq_db. */
477 if (unlikely(!nocmp_n && comp_idx == MLX5_VPMD_DESCS_PER_LOOP)) {
478 *no_cq = true;
479 return cp_pkt;
480 }
481 /* Update the consumer indexes for non-compressed CQEs. */
482 MLX5_ASSERT(nocmp_n <= pkts_n);
483 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, pkts, nocmp_n);
484 rcvd_pkt += cp_pkt;
485 /* Decompress the last CQE if compressed. */
486 if (comp_idx < MLX5_VPMD_DESCS_PER_LOOP) {
487 MLX5_ASSERT(comp_idx == (nocmp_n % MLX5_VPMD_DESCS_PER_LOOP));
488 rxq->decompressed = rxq_cq_decompress_v(rxq, &cq[nocmp_n],
489 &elts[nocmp_n]);
490 /* Return more packets if needed. */
491 if (nocmp_n < pkts_n) {
492 uint16_t n = rxq->decompressed;
493
494 n = RTE_MIN(n, pkts_n - nocmp_n);
495 cp_pkt = rxq_copy_mprq_mbuf_v(rxq, &pkts[cp_pkt], n);
496 rcvd_pkt += cp_pkt;
497 rxq->decompressed -= n;
498 }
499 }
500 *no_cq = !rcvd_pkt;
501 return rcvd_pkt;
502 }
503
504 /**
505 * DPDK callback for vectorized MPRQ RX.
506 *
507 * @param dpdk_rxq
508 * Generic pointer to RX queue structure.
509 * @param[out] pkts
510 * Array to store received packets.
511 * @param pkts_n
512 * Maximum number of packets in array.
513 *
514 * @return
515 * Number of packets successfully received (<= pkts_n).
516 */
517 uint16_t
mlx5_rx_burst_mprq_vec(void * dpdk_rxq,struct rte_mbuf ** pkts,uint16_t pkts_n)518 mlx5_rx_burst_mprq_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
519 {
520 struct mlx5_rxq_data *rxq = dpdk_rxq;
521 uint16_t nb_rx = 0;
522 uint16_t tn = 0;
523 uint64_t err = 0;
524 bool no_cq = false;
525
526 do {
527 nb_rx = rxq_burst_mprq_v(rxq, pkts + tn, pkts_n - tn,
528 &err, &no_cq);
529 if (unlikely(err | rxq->err_state))
530 nb_rx = rxq_handle_pending_error(rxq, pkts + tn, nb_rx);
531 tn += nb_rx;
532 if (unlikely(no_cq))
533 break;
534 } while (tn != pkts_n);
535 return tn;
536 }
537
538 /**
539 * Check a RX queue can support vectorized RX.
540 *
541 * @param rxq
542 * Pointer to RX queue.
543 *
544 * @return
545 * 1 if supported, negative errno value if not.
546 */
547 int __rte_cold
mlx5_rxq_check_vec_support(struct mlx5_rxq_data * rxq)548 mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
549 {
550 struct mlx5_rxq_ctrl *ctrl =
551 container_of(rxq, struct mlx5_rxq_ctrl, rxq);
552
553 if (!RXQ_PORT(ctrl)->config.rx_vec_en || rxq->sges_n != 0)
554 return -ENOTSUP;
555 if (rxq->lro)
556 return -ENOTSUP;
557 return 1;
558 }
559
560 /**
561 * Check a device can support vectorized RX.
562 *
563 * @param dev
564 * Pointer to Ethernet device.
565 *
566 * @return
567 * 1 if supported, negative errno value if not.
568 */
569 int __rte_cold
mlx5_check_vec_rx_support(struct rte_eth_dev * dev)570 mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
571 {
572 struct mlx5_priv *priv = dev->data->dev_private;
573 uint32_t i;
574
575 if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)
576 return -ENOTSUP;
577 if (!priv->config.rx_vec_en)
578 return -ENOTSUP;
579 /* All the configured queues should support. */
580 for (i = 0; i < priv->rxqs_n; ++i) {
581 struct mlx5_rxq_data *rxq_data = mlx5_rxq_data_get(dev, i);
582
583 if (!rxq_data)
584 continue;
585 if (mlx5_rxq_check_vec_support(rxq_data) < 0)
586 break;
587 }
588 if (i != priv->rxqs_n)
589 return -ENOTSUP;
590 return 1;
591 }
592