1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2020 Intel Corporation
3 */
4
5 #include <stdint.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <string.h>
9 #include <errno.h>
10
11 #include <rte_net.h>
12
13 #include "virtio_logs.h"
14 #include "virtio_ethdev.h"
15 #include "virtio_pci.h"
16 #include "virtqueue.h"
17
18 #define BYTE_SIZE 8
19 /* flag bits offset in packed ring desc higher 64bits */
20 #define FLAGS_BITS_OFFSET ((offsetof(struct vring_packed_desc, flags) - \
21 offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
22
23 #define PACKED_FLAGS_MASK ((0ULL | VRING_PACKED_DESC_F_AVAIL_USED) << \
24 FLAGS_BITS_OFFSET)
25
26 /* reference count offset in mbuf rearm data */
27 #define REFCNT_BITS_OFFSET ((offsetof(struct rte_mbuf, refcnt) - \
28 offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
29 /* segment number offset in mbuf rearm data */
30 #define SEG_NUM_BITS_OFFSET ((offsetof(struct rte_mbuf, nb_segs) - \
31 offsetof(struct rte_mbuf, rearm_data)) * BYTE_SIZE)
32
33 /* default rearm data */
34 #define DEFAULT_REARM_DATA (1ULL << SEG_NUM_BITS_OFFSET | \
35 1ULL << REFCNT_BITS_OFFSET)
36
37 /* id bits offset in packed ring desc higher 64bits */
38 #define ID_BITS_OFFSET ((offsetof(struct vring_packed_desc, id) - \
39 offsetof(struct vring_packed_desc, len)) * BYTE_SIZE)
40
41 /* net hdr short size mask */
42 #define NET_HDR_MASK 0x3F
43
44 #define PACKED_BATCH_SIZE (RTE_CACHE_LINE_SIZE / \
45 sizeof(struct vring_packed_desc))
46 #define PACKED_BATCH_MASK (PACKED_BATCH_SIZE - 1)
47
48 #ifdef VIRTIO_GCC_UNROLL_PRAGMA
49 #define virtio_for_each_try_unroll(iter, val, size) _Pragma("GCC unroll 4") \
50 for (iter = val; iter < size; iter++)
51 #endif
52
53 #ifdef VIRTIO_CLANG_UNROLL_PRAGMA
54 #define virtio_for_each_try_unroll(iter, val, size) _Pragma("unroll 4") \
55 for (iter = val; iter < size; iter++)
56 #endif
57
58 #ifdef VIRTIO_ICC_UNROLL_PRAGMA
59 #define virtio_for_each_try_unroll(iter, val, size) _Pragma("unroll (4)") \
60 for (iter = val; iter < size; iter++)
61 #endif
62
63 #ifndef virtio_for_each_try_unroll
64 #define virtio_for_each_try_unroll(iter, val, num) \
65 for (iter = val; iter < num; iter++)
66 #endif
67
68 static inline void
virtio_update_batch_stats(struct virtnet_stats * stats,uint16_t pkt_len1,uint16_t pkt_len2,uint16_t pkt_len3,uint16_t pkt_len4)69 virtio_update_batch_stats(struct virtnet_stats *stats,
70 uint16_t pkt_len1,
71 uint16_t pkt_len2,
72 uint16_t pkt_len3,
73 uint16_t pkt_len4)
74 {
75 stats->bytes += pkt_len1;
76 stats->bytes += pkt_len2;
77 stats->bytes += pkt_len3;
78 stats->bytes += pkt_len4;
79 }
80
81 static inline int
virtqueue_enqueue_batch_packed_vec(struct virtnet_tx * txvq,struct rte_mbuf ** tx_pkts)82 virtqueue_enqueue_batch_packed_vec(struct virtnet_tx *txvq,
83 struct rte_mbuf **tx_pkts)
84 {
85 struct virtqueue *vq = txvq->vq;
86 uint16_t head_size = vq->hw->vtnet_hdr_size;
87 uint16_t idx = vq->vq_avail_idx;
88 struct virtio_net_hdr *hdr;
89 struct vq_desc_extra *dxp;
90 uint16_t i, cmp;
91
92 if (vq->vq_avail_idx & PACKED_BATCH_MASK)
93 return -1;
94
95 if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
96 return -1;
97
98 /* Load four mbufs rearm data */
99 RTE_BUILD_BUG_ON(REFCNT_BITS_OFFSET >= 64);
100 RTE_BUILD_BUG_ON(SEG_NUM_BITS_OFFSET >= 64);
101 __m256i mbufs = _mm256_set_epi64x(*tx_pkts[3]->rearm_data,
102 *tx_pkts[2]->rearm_data,
103 *tx_pkts[1]->rearm_data,
104 *tx_pkts[0]->rearm_data);
105
106 /* refcnt=1 and nb_segs=1 */
107 __m256i mbuf_ref = _mm256_set1_epi64x(DEFAULT_REARM_DATA);
108 __m256i head_rooms = _mm256_set1_epi16(head_size);
109
110 /* Check refcnt and nb_segs */
111 const __mmask16 mask = 0x6 | 0x6 << 4 | 0x6 << 8 | 0x6 << 12;
112 cmp = _mm256_mask_cmpneq_epu16_mask(mask, mbufs, mbuf_ref);
113 if (unlikely(cmp))
114 return -1;
115
116 /* Check headroom is enough */
117 const __mmask16 data_mask = 0x1 | 0x1 << 4 | 0x1 << 8 | 0x1 << 12;
118 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_off) !=
119 offsetof(struct rte_mbuf, rearm_data));
120 cmp = _mm256_mask_cmplt_epu16_mask(data_mask, mbufs, head_rooms);
121 if (unlikely(cmp))
122 return -1;
123
124 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
125 dxp = &vq->vq_descx[idx + i];
126 dxp->ndescs = 1;
127 dxp->cookie = tx_pkts[i];
128 }
129
130 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
131 tx_pkts[i]->data_off -= head_size;
132 tx_pkts[i]->data_len += head_size;
133 }
134
135 __m512i descs_base = _mm512_set_epi64(tx_pkts[3]->data_len,
136 VIRTIO_MBUF_ADDR(tx_pkts[3], vq),
137 tx_pkts[2]->data_len,
138 VIRTIO_MBUF_ADDR(tx_pkts[2], vq),
139 tx_pkts[1]->data_len,
140 VIRTIO_MBUF_ADDR(tx_pkts[1], vq),
141 tx_pkts[0]->data_len,
142 VIRTIO_MBUF_ADDR(tx_pkts[0], vq));
143
144 /* id offset and data offset */
145 __m512i data_offsets = _mm512_set_epi64((uint64_t)3 << ID_BITS_OFFSET,
146 tx_pkts[3]->data_off,
147 (uint64_t)2 << ID_BITS_OFFSET,
148 tx_pkts[2]->data_off,
149 (uint64_t)1 << ID_BITS_OFFSET,
150 tx_pkts[1]->data_off,
151 0, tx_pkts[0]->data_off);
152
153 __m512i new_descs = _mm512_add_epi64(descs_base, data_offsets);
154
155 uint64_t flags_temp = (uint64_t)idx << ID_BITS_OFFSET |
156 (uint64_t)vq->vq_packed.cached_flags << FLAGS_BITS_OFFSET;
157
158 /* flags offset and guest virtual address offset */
159 __m128i flag_offset = _mm_set_epi64x(flags_temp, 0);
160 __m512i v_offset = _mm512_broadcast_i32x4(flag_offset);
161 __m512i v_desc = _mm512_add_epi64(new_descs, v_offset);
162
163 if (!vq->hw->has_tx_offload) {
164 __m128i all_mask = _mm_set1_epi16(0xFFFF);
165 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
166 hdr = rte_pktmbuf_mtod_offset(tx_pkts[i],
167 struct virtio_net_hdr *, -head_size);
168 __m128i v_hdr = _mm_loadu_si128((void *)hdr);
169 if (unlikely(_mm_mask_test_epi16_mask(NET_HDR_MASK,
170 v_hdr, all_mask))) {
171 __m128i all_zero = _mm_setzero_si128();
172 _mm_mask_storeu_epi16((void *)hdr,
173 NET_HDR_MASK, all_zero);
174 }
175 }
176 } else {
177 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
178 hdr = rte_pktmbuf_mtod_offset(tx_pkts[i],
179 struct virtio_net_hdr *, -head_size);
180 virtqueue_xmit_offload(hdr, tx_pkts[i], true);
181 }
182 }
183
184 /* Enqueue Packet buffers */
185 _mm512_storeu_si512((void *)&vq->vq_packed.ring.desc[idx], v_desc);
186
187 virtio_update_batch_stats(&txvq->stats, tx_pkts[0]->pkt_len,
188 tx_pkts[1]->pkt_len, tx_pkts[2]->pkt_len,
189 tx_pkts[3]->pkt_len);
190
191 vq->vq_avail_idx += PACKED_BATCH_SIZE;
192 vq->vq_free_cnt -= PACKED_BATCH_SIZE;
193
194 if (vq->vq_avail_idx >= vq->vq_nentries) {
195 vq->vq_avail_idx -= vq->vq_nentries;
196 vq->vq_packed.cached_flags ^=
197 VRING_PACKED_DESC_F_AVAIL_USED;
198 }
199
200 return 0;
201 }
202
203 static inline int
virtqueue_enqueue_single_packed_vec(struct virtnet_tx * txvq,struct rte_mbuf * txm)204 virtqueue_enqueue_single_packed_vec(struct virtnet_tx *txvq,
205 struct rte_mbuf *txm)
206 {
207 struct virtqueue *vq = txvq->vq;
208 struct virtio_hw *hw = vq->hw;
209 uint16_t hdr_size = hw->vtnet_hdr_size;
210 uint16_t slots, can_push = 0, use_indirect = 0;
211 int16_t need;
212
213 /* optimize ring usage */
214 if ((vtpci_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
215 vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) &&
216 rte_mbuf_refcnt_read(txm) == 1 &&
217 RTE_MBUF_DIRECT(txm) &&
218 txm->nb_segs == 1 &&
219 rte_pktmbuf_headroom(txm) >= hdr_size)
220 can_push = 1;
221 else if (vtpci_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
222 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
223 use_indirect = 1;
224 /* How many main ring entries are needed to this Tx?
225 * indirect => 1
226 * any_layout => number of segments
227 * default => number of segments + 1
228 */
229 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
230 need = slots - vq->vq_free_cnt;
231
232 /* Positive value indicates it need free vring descriptors */
233 if (unlikely(need > 0)) {
234 virtio_xmit_cleanup_inorder_packed(vq, need);
235 need = slots - vq->vq_free_cnt;
236 if (unlikely(need > 0)) {
237 PMD_TX_LOG(ERR,
238 "No free tx descriptors to transmit");
239 return -1;
240 }
241 }
242
243 /* Enqueue Packet buffers */
244 virtqueue_enqueue_xmit_packed(txvq, txm, slots, use_indirect,
245 can_push, 1);
246
247 txvq->stats.bytes += txm->pkt_len;
248 return 0;
249 }
250
251 uint16_t
virtio_xmit_pkts_packed_vec(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)252 virtio_xmit_pkts_packed_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
253 uint16_t nb_pkts)
254 {
255 struct virtnet_tx *txvq = tx_queue;
256 struct virtqueue *vq = txvq->vq;
257 struct virtio_hw *hw = vq->hw;
258 uint16_t nb_tx = 0;
259 uint16_t remained;
260
261 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
262 return nb_tx;
263
264 if (unlikely(nb_pkts < 1))
265 return nb_pkts;
266
267 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
268
269 if (vq->vq_free_cnt <= vq->vq_nentries - vq->vq_free_thresh)
270 virtio_xmit_cleanup_inorder_packed(vq, vq->vq_free_thresh);
271
272 remained = RTE_MIN(nb_pkts, vq->vq_free_cnt);
273
274 while (remained) {
275 if (remained >= PACKED_BATCH_SIZE) {
276 if (!virtqueue_enqueue_batch_packed_vec(txvq,
277 &tx_pkts[nb_tx])) {
278 nb_tx += PACKED_BATCH_SIZE;
279 remained -= PACKED_BATCH_SIZE;
280 continue;
281 }
282 }
283 if (!virtqueue_enqueue_single_packed_vec(txvq,
284 tx_pkts[nb_tx])) {
285 nb_tx++;
286 remained--;
287 continue;
288 }
289 break;
290 };
291
292 txvq->stats.packets += nb_tx;
293
294 if (likely(nb_tx)) {
295 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
296 virtqueue_notify(vq);
297 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
298 }
299 }
300
301 return nb_tx;
302 }
303
304 /* Optionally fill offload information in structure */
305 static inline int
virtio_vec_rx_offload(struct rte_mbuf * m,struct virtio_net_hdr * hdr)306 virtio_vec_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
307 {
308 struct rte_net_hdr_lens hdr_lens;
309 uint32_t hdrlen, ptype;
310 int l4_supported = 0;
311
312 /* nothing to do */
313 if (hdr->flags == 0)
314 return 0;
315
316 /* GSO not support in vec path, skip check */
317 m->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN;
318
319 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
320 m->packet_type = ptype;
321 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP ||
322 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP ||
323 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP)
324 l4_supported = 1;
325
326 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
327 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len;
328 if (hdr->csum_start <= hdrlen && l4_supported) {
329 m->ol_flags |= PKT_RX_L4_CKSUM_NONE;
330 } else {
331 /* Unknown proto or tunnel, do sw cksum. We can assume
332 * the cksum field is in the first segment since the
333 * buffers we provided to the host are large enough.
334 * In case of SCTP, this will be wrong since it's a CRC
335 * but there's nothing we can do.
336 */
337 uint16_t csum = 0, off;
338
339 if (rte_raw_cksum_mbuf(m, hdr->csum_start,
340 rte_pktmbuf_pkt_len(m) - hdr->csum_start,
341 &csum) < 0)
342 return -1;
343 if (likely(csum != 0xffff))
344 csum = ~csum;
345 off = hdr->csum_offset + hdr->csum_start;
346 if (rte_pktmbuf_data_len(m) >= off + 1)
347 *rte_pktmbuf_mtod_offset(m, uint16_t *,
348 off) = csum;
349 }
350 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) {
351 m->ol_flags |= PKT_RX_L4_CKSUM_GOOD;
352 }
353
354 return 0;
355 }
356
357 static inline uint16_t
virtqueue_dequeue_batch_packed_vec(struct virtnet_rx * rxvq,struct rte_mbuf ** rx_pkts)358 virtqueue_dequeue_batch_packed_vec(struct virtnet_rx *rxvq,
359 struct rte_mbuf **rx_pkts)
360 {
361 struct virtqueue *vq = rxvq->vq;
362 struct virtio_hw *hw = vq->hw;
363 uint16_t hdr_size = hw->vtnet_hdr_size;
364 uint64_t addrs[PACKED_BATCH_SIZE];
365 uint16_t id = vq->vq_used_cons_idx;
366 uint8_t desc_stats;
367 uint16_t i;
368 void *desc_addr;
369
370 if (id & PACKED_BATCH_MASK)
371 return -1;
372
373 if (unlikely((id + PACKED_BATCH_SIZE) > vq->vq_nentries))
374 return -1;
375
376 /* only care avail/used bits */
377 #if defined(RTE_ARCH_I686)
378 __m512i v_mask = _mm512_set4_epi64(PACKED_FLAGS_MASK, 0x0,
379 PACKED_FLAGS_MASK, 0x0);
380 #else
381 __m512i v_mask = _mm512_maskz_set1_epi64(0xaa, PACKED_FLAGS_MASK);
382 #endif
383 desc_addr = &vq->vq_packed.ring.desc[id];
384
385 __m512i v_desc = _mm512_loadu_si512(desc_addr);
386 __m512i v_flag = _mm512_and_epi64(v_desc, v_mask);
387
388 __m512i v_used_flag = _mm512_setzero_si512();
389 if (vq->vq_packed.used_wrap_counter)
390 #if defined(RTE_ARCH_I686)
391 v_used_flag = _mm512_set4_epi64(PACKED_FLAGS_MASK, 0x0,
392 PACKED_FLAGS_MASK, 0x0);
393 #else
394 v_used_flag = _mm512_maskz_set1_epi64(0xaa, PACKED_FLAGS_MASK);
395 #endif
396
397 /* Check all descs are used */
398 desc_stats = _mm512_cmpneq_epu64_mask(v_flag, v_used_flag);
399 if (desc_stats)
400 return -1;
401
402 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
403 rx_pkts[i] = (struct rte_mbuf *)vq->vq_descx[id + i].cookie;
404 rte_packet_prefetch(rte_pktmbuf_mtod(rx_pkts[i], void *));
405
406 addrs[i] = (uintptr_t)rx_pkts[i]->rx_descriptor_fields1;
407 }
408
409 /*
410 * load len from desc, store into mbuf pkt_len and data_len
411 * len limiated by l6bit buf_len, pkt_len[16:31] can be ignored
412 */
413 const __mmask16 mask = 0x6 | 0x6 << 4 | 0x6 << 8 | 0x6 << 12;
414 __m512i values = _mm512_maskz_shuffle_epi32(mask, v_desc, 0xAA);
415
416 /* reduce hdr_len from pkt_len and data_len */
417 __m512i mbuf_len_offset = _mm512_maskz_set1_epi32(mask,
418 (uint32_t)-hdr_size);
419
420 __m512i v_value = _mm512_add_epi32(values, mbuf_len_offset);
421
422 /* assert offset of data_len */
423 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
424 offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
425
426 __m512i v_index = _mm512_set_epi64(addrs[3] + 8, addrs[3],
427 addrs[2] + 8, addrs[2],
428 addrs[1] + 8, addrs[1],
429 addrs[0] + 8, addrs[0]);
430 /* batch store into mbufs */
431 _mm512_i64scatter_epi64(0, v_index, v_value, 1);
432
433 if (hw->has_rx_offload) {
434 virtio_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) {
435 char *addr = (char *)rx_pkts[i]->buf_addr +
436 RTE_PKTMBUF_HEADROOM - hdr_size;
437 virtio_vec_rx_offload(rx_pkts[i],
438 (struct virtio_net_hdr *)addr);
439 }
440 }
441
442 virtio_update_batch_stats(&rxvq->stats, rx_pkts[0]->pkt_len,
443 rx_pkts[1]->pkt_len, rx_pkts[2]->pkt_len,
444 rx_pkts[3]->pkt_len);
445
446 vq->vq_free_cnt += PACKED_BATCH_SIZE;
447
448 vq->vq_used_cons_idx += PACKED_BATCH_SIZE;
449 if (vq->vq_used_cons_idx >= vq->vq_nentries) {
450 vq->vq_used_cons_idx -= vq->vq_nentries;
451 vq->vq_packed.used_wrap_counter ^= 1;
452 }
453
454 return 0;
455 }
456
457 static uint16_t
virtqueue_dequeue_single_packed_vec(struct virtnet_rx * rxvq,struct rte_mbuf ** rx_pkts)458 virtqueue_dequeue_single_packed_vec(struct virtnet_rx *rxvq,
459 struct rte_mbuf **rx_pkts)
460 {
461 uint16_t used_idx, id;
462 uint32_t len;
463 struct virtqueue *vq = rxvq->vq;
464 struct virtio_hw *hw = vq->hw;
465 uint32_t hdr_size = hw->vtnet_hdr_size;
466 struct virtio_net_hdr *hdr;
467 struct vring_packed_desc *desc;
468 struct rte_mbuf *cookie;
469
470 desc = vq->vq_packed.ring.desc;
471 used_idx = vq->vq_used_cons_idx;
472 if (!desc_is_used(&desc[used_idx], vq))
473 return -1;
474
475 len = desc[used_idx].len;
476 id = desc[used_idx].id;
477 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
478 if (unlikely(cookie == NULL)) {
479 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
480 vq->vq_used_cons_idx);
481 return -1;
482 }
483 rte_prefetch0(cookie);
484 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
485
486 cookie->data_off = RTE_PKTMBUF_HEADROOM;
487 cookie->ol_flags = 0;
488 cookie->pkt_len = (uint32_t)(len - hdr_size);
489 cookie->data_len = (uint32_t)(len - hdr_size);
490
491 hdr = (struct virtio_net_hdr *)((char *)cookie->buf_addr +
492 RTE_PKTMBUF_HEADROOM - hdr_size);
493 if (hw->has_rx_offload)
494 virtio_vec_rx_offload(cookie, hdr);
495
496 *rx_pkts = cookie;
497
498 rxvq->stats.bytes += cookie->pkt_len;
499
500 vq->vq_free_cnt++;
501 vq->vq_used_cons_idx++;
502 if (vq->vq_used_cons_idx >= vq->vq_nentries) {
503 vq->vq_used_cons_idx -= vq->vq_nentries;
504 vq->vq_packed.used_wrap_counter ^= 1;
505 }
506
507 return 0;
508 }
509
510 static inline void
virtio_recv_refill_packed_vec(struct virtnet_rx * rxvq,struct rte_mbuf ** cookie,uint16_t num)511 virtio_recv_refill_packed_vec(struct virtnet_rx *rxvq,
512 struct rte_mbuf **cookie,
513 uint16_t num)
514 {
515 struct virtqueue *vq = rxvq->vq;
516 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
517 uint16_t flags = vq->vq_packed.cached_flags;
518 struct virtio_hw *hw = vq->hw;
519 struct vq_desc_extra *dxp;
520 uint16_t idx, i;
521 uint16_t batch_num, total_num = 0;
522 uint16_t head_idx = vq->vq_avail_idx;
523 uint16_t head_flag = vq->vq_packed.cached_flags;
524 uint64_t addr;
525
526 do {
527 idx = vq->vq_avail_idx;
528
529 batch_num = PACKED_BATCH_SIZE;
530 if (unlikely((idx + PACKED_BATCH_SIZE) > vq->vq_nentries))
531 batch_num = vq->vq_nentries - idx;
532 if (unlikely((total_num + batch_num) > num))
533 batch_num = num - total_num;
534
535 virtio_for_each_try_unroll(i, 0, batch_num) {
536 dxp = &vq->vq_descx[idx + i];
537 dxp->cookie = (void *)cookie[total_num + i];
538
539 addr = VIRTIO_MBUF_ADDR(cookie[total_num + i], vq) +
540 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
541 start_dp[idx + i].addr = addr;
542 start_dp[idx + i].len = cookie[total_num + i]->buf_len
543 - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
544 if (total_num || i) {
545 virtqueue_store_flags_packed(&start_dp[idx + i],
546 flags, hw->weak_barriers);
547 }
548 }
549
550 vq->vq_avail_idx += batch_num;
551 if (vq->vq_avail_idx >= vq->vq_nentries) {
552 vq->vq_avail_idx -= vq->vq_nentries;
553 vq->vq_packed.cached_flags ^=
554 VRING_PACKED_DESC_F_AVAIL_USED;
555 flags = vq->vq_packed.cached_flags;
556 }
557 total_num += batch_num;
558 } while (total_num < num);
559
560 virtqueue_store_flags_packed(&start_dp[head_idx], head_flag,
561 hw->weak_barriers);
562 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
563 }
564
565 uint16_t
virtio_recv_pkts_packed_vec(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)566 virtio_recv_pkts_packed_vec(void *rx_queue,
567 struct rte_mbuf **rx_pkts,
568 uint16_t nb_pkts)
569 {
570 struct virtnet_rx *rxvq = rx_queue;
571 struct virtqueue *vq = rxvq->vq;
572 struct virtio_hw *hw = vq->hw;
573 uint16_t num, nb_rx = 0;
574 uint32_t nb_enqueued = 0;
575 uint16_t free_cnt = vq->vq_free_thresh;
576
577 if (unlikely(hw->started == 0))
578 return nb_rx;
579
580 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts);
581 if (likely(num > PACKED_BATCH_SIZE))
582 num = num - ((vq->vq_used_cons_idx + num) % PACKED_BATCH_SIZE);
583
584 while (num) {
585 if (!virtqueue_dequeue_batch_packed_vec(rxvq,
586 &rx_pkts[nb_rx])) {
587 nb_rx += PACKED_BATCH_SIZE;
588 num -= PACKED_BATCH_SIZE;
589 continue;
590 }
591 if (!virtqueue_dequeue_single_packed_vec(rxvq,
592 &rx_pkts[nb_rx])) {
593 nb_rx++;
594 num--;
595 continue;
596 }
597 break;
598 };
599
600 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
601
602 rxvq->stats.packets += nb_rx;
603
604 if (likely(vq->vq_free_cnt >= free_cnt)) {
605 struct rte_mbuf *new_pkts[free_cnt];
606 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
607 free_cnt) == 0)) {
608 virtio_recv_refill_packed_vec(rxvq, new_pkts,
609 free_cnt);
610 nb_enqueued += free_cnt;
611 } else {
612 struct rte_eth_dev *dev =
613 &rte_eth_devices[rxvq->port_id];
614 dev->data->rx_mbuf_alloc_failed += free_cnt;
615 }
616 }
617
618 if (likely(nb_enqueued)) {
619 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
620 virtqueue_notify(vq);
621 PMD_RX_LOG(DEBUG, "Notified");
622 }
623 }
624
625 return nb_rx;
626 }
627