1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 */
4
5 #include <stdint.h>
6 #include <stdio.h>
7 #include <stdlib.h>
8 #include <string.h>
9 #include <errno.h>
10
11 #include <rte_cycles.h>
12 #include <rte_memory.h>
13 #include <rte_branch_prediction.h>
14 #include <rte_mempool.h>
15 #include <rte_malloc.h>
16 #include <rte_mbuf.h>
17 #include <rte_ether.h>
18 #include <ethdev_driver.h>
19 #include <rte_prefetch.h>
20 #include <rte_string_fns.h>
21 #include <rte_errno.h>
22 #include <rte_byteorder.h>
23 #include <rte_net.h>
24 #include <rte_ip.h>
25 #include <rte_udp.h>
26 #include <rte_tcp.h>
27
28 #include "virtio_logs.h"
29 #include "virtio_ethdev.h"
30 #include "virtio.h"
31 #include "virtqueue.h"
32 #include "virtio_rxtx.h"
33 #include "virtio_rxtx_simple.h"
34 #include "virtio_ring.h"
35
36 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
37 #define VIRTIO_DUMP_PACKET(m, len) rte_pktmbuf_dump(stdout, m, len)
38 #else
39 #define VIRTIO_DUMP_PACKET(m, len) do { } while (0)
40 #endif
41
42 void
vq_ring_free_inorder(struct virtqueue * vq,uint16_t desc_idx,uint16_t num)43 vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx, uint16_t num)
44 {
45 vq->vq_free_cnt += num;
46 vq->vq_desc_tail_idx = desc_idx & (vq->vq_nentries - 1);
47 }
48
49 void
vq_ring_free_chain(struct virtqueue * vq,uint16_t desc_idx)50 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
51 {
52 struct vring_desc *dp, *dp_tail;
53 struct vq_desc_extra *dxp;
54 uint16_t desc_idx_last = desc_idx;
55
56 dp = &vq->vq_split.ring.desc[desc_idx];
57 dxp = &vq->vq_descx[desc_idx];
58 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt + dxp->ndescs);
59 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
60 while (dp->flags & VRING_DESC_F_NEXT) {
61 desc_idx_last = dp->next;
62 dp = &vq->vq_split.ring.desc[dp->next];
63 }
64 }
65 dxp->ndescs = 0;
66
67 /*
68 * We must append the existing free chain, if any, to the end of
69 * newly freed chain. If the virtqueue was completely used, then
70 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
71 */
72 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END) {
73 vq->vq_desc_head_idx = desc_idx;
74 } else {
75 dp_tail = &vq->vq_split.ring.desc[vq->vq_desc_tail_idx];
76 dp_tail->next = desc_idx;
77 }
78
79 vq->vq_desc_tail_idx = desc_idx_last;
80 dp->next = VQ_RING_DESC_CHAIN_END;
81 }
82
83 void
virtio_update_packet_stats(struct virtnet_stats * stats,struct rte_mbuf * mbuf)84 virtio_update_packet_stats(struct virtnet_stats *stats, struct rte_mbuf *mbuf)
85 {
86 uint32_t s = mbuf->pkt_len;
87 struct rte_ether_addr *ea;
88
89 stats->bytes += s;
90
91 if (s == 64) {
92 stats->size_bins[1]++;
93 } else if (s > 64 && s < 1024) {
94 uint32_t bin;
95
96 /* count zeros, and offset into correct bin */
97 bin = (sizeof(s) * 8) - __builtin_clz(s) - 5;
98 stats->size_bins[bin]++;
99 } else {
100 if (s < 64)
101 stats->size_bins[0]++;
102 else if (s < 1519)
103 stats->size_bins[6]++;
104 else
105 stats->size_bins[7]++;
106 }
107
108 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *);
109 if (rte_is_multicast_ether_addr(ea)) {
110 if (rte_is_broadcast_ether_addr(ea))
111 stats->broadcast++;
112 else
113 stats->multicast++;
114 }
115 }
116
117 static inline void
virtio_rx_stats_updated(struct virtnet_rx * rxvq,struct rte_mbuf * m)118 virtio_rx_stats_updated(struct virtnet_rx *rxvq, struct rte_mbuf *m)
119 {
120 VIRTIO_DUMP_PACKET(m, m->data_len);
121
122 virtio_update_packet_stats(&rxvq->stats, m);
123 }
124
125 static uint16_t
virtqueue_dequeue_burst_rx_packed(struct virtqueue * vq,struct rte_mbuf ** rx_pkts,uint32_t * len,uint16_t num)126 virtqueue_dequeue_burst_rx_packed(struct virtqueue *vq,
127 struct rte_mbuf **rx_pkts,
128 uint32_t *len,
129 uint16_t num)
130 {
131 struct rte_mbuf *cookie;
132 uint16_t used_idx;
133 uint16_t id;
134 struct vring_packed_desc *desc;
135 uint16_t i;
136
137 desc = vq->vq_packed.ring.desc;
138
139 for (i = 0; i < num; i++) {
140 used_idx = vq->vq_used_cons_idx;
141 /* desc_is_used has a load-acquire or rte_io_rmb inside
142 * and wait for used desc in virtqueue.
143 */
144 if (!desc_is_used(&desc[used_idx], vq))
145 return i;
146 len[i] = desc[used_idx].len;
147 id = desc[used_idx].id;
148 cookie = (struct rte_mbuf *)vq->vq_descx[id].cookie;
149 if (unlikely(cookie == NULL)) {
150 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
151 vq->vq_used_cons_idx);
152 break;
153 }
154 rte_prefetch0(cookie);
155 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
156 rx_pkts[i] = cookie;
157
158 vq->vq_free_cnt++;
159 vq->vq_used_cons_idx++;
160 if (vq->vq_used_cons_idx >= vq->vq_nentries) {
161 vq->vq_used_cons_idx -= vq->vq_nentries;
162 vq->vq_packed.used_wrap_counter ^= 1;
163 }
164 }
165
166 return i;
167 }
168
169 static uint16_t
virtqueue_dequeue_burst_rx(struct virtqueue * vq,struct rte_mbuf ** rx_pkts,uint32_t * len,uint16_t num)170 virtqueue_dequeue_burst_rx(struct virtqueue *vq, struct rte_mbuf **rx_pkts,
171 uint32_t *len, uint16_t num)
172 {
173 struct vring_used_elem *uep;
174 struct rte_mbuf *cookie;
175 uint16_t used_idx, desc_idx;
176 uint16_t i;
177
178 /* Caller does the check */
179 for (i = 0; i < num ; i++) {
180 used_idx = (uint16_t)(vq->vq_used_cons_idx & (vq->vq_nentries - 1));
181 uep = &vq->vq_split.ring.used->ring[used_idx];
182 desc_idx = (uint16_t) uep->id;
183 len[i] = uep->len;
184 cookie = (struct rte_mbuf *)vq->vq_descx[desc_idx].cookie;
185
186 if (unlikely(cookie == NULL)) {
187 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
188 vq->vq_used_cons_idx);
189 break;
190 }
191
192 rte_prefetch0(cookie);
193 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
194 rx_pkts[i] = cookie;
195 vq->vq_used_cons_idx++;
196 vq_ring_free_chain(vq, desc_idx);
197 vq->vq_descx[desc_idx].cookie = NULL;
198 }
199
200 return i;
201 }
202
203 static uint16_t
virtqueue_dequeue_rx_inorder(struct virtqueue * vq,struct rte_mbuf ** rx_pkts,uint32_t * len,uint16_t num)204 virtqueue_dequeue_rx_inorder(struct virtqueue *vq,
205 struct rte_mbuf **rx_pkts,
206 uint32_t *len,
207 uint16_t num)
208 {
209 struct vring_used_elem *uep;
210 struct rte_mbuf *cookie;
211 uint16_t used_idx = 0;
212 uint16_t i;
213
214 if (unlikely(num == 0))
215 return 0;
216
217 for (i = 0; i < num; i++) {
218 used_idx = vq->vq_used_cons_idx & (vq->vq_nentries - 1);
219 /* Desc idx same as used idx */
220 uep = &vq->vq_split.ring.used->ring[used_idx];
221 len[i] = uep->len;
222 cookie = (struct rte_mbuf *)vq->vq_descx[used_idx].cookie;
223
224 if (unlikely(cookie == NULL)) {
225 PMD_DRV_LOG(ERR, "vring descriptor with no mbuf cookie at %u",
226 vq->vq_used_cons_idx);
227 break;
228 }
229
230 rte_prefetch0(cookie);
231 rte_packet_prefetch(rte_pktmbuf_mtod(cookie, void *));
232 rx_pkts[i] = cookie;
233 vq->vq_used_cons_idx++;
234 vq->vq_descx[used_idx].cookie = NULL;
235 }
236
237 vq_ring_free_inorder(vq, used_idx, i);
238 return i;
239 }
240
241 static inline int
virtqueue_enqueue_refill_inorder(struct virtqueue * vq,struct rte_mbuf ** cookies,uint16_t num)242 virtqueue_enqueue_refill_inorder(struct virtqueue *vq,
243 struct rte_mbuf **cookies,
244 uint16_t num)
245 {
246 struct vq_desc_extra *dxp;
247 struct virtio_hw *hw = vq->hw;
248 struct vring_desc *start_dp;
249 uint16_t head_idx, idx, i = 0;
250
251 if (unlikely(vq->vq_free_cnt == 0))
252 return -ENOSPC;
253 if (unlikely(vq->vq_free_cnt < num))
254 return -EMSGSIZE;
255
256 head_idx = vq->vq_desc_head_idx & (vq->vq_nentries - 1);
257 start_dp = vq->vq_split.ring.desc;
258
259 while (i < num) {
260 idx = head_idx & (vq->vq_nentries - 1);
261 dxp = &vq->vq_descx[idx];
262 dxp->cookie = (void *)cookies[i];
263 dxp->ndescs = 1;
264
265 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookies[i], vq) +
266 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
267 start_dp[idx].len = cookies[i]->buf_len -
268 RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
269 start_dp[idx].flags = VRING_DESC_F_WRITE;
270
271 vq_update_avail_ring(vq, idx);
272 head_idx++;
273 i++;
274 }
275
276 vq->vq_desc_head_idx += num;
277 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
278 return 0;
279 }
280
281 static inline int
virtqueue_enqueue_recv_refill(struct virtqueue * vq,struct rte_mbuf ** cookie,uint16_t num)282 virtqueue_enqueue_recv_refill(struct virtqueue *vq, struct rte_mbuf **cookie,
283 uint16_t num)
284 {
285 struct vq_desc_extra *dxp;
286 struct virtio_hw *hw = vq->hw;
287 struct vring_desc *start_dp = vq->vq_split.ring.desc;
288 uint16_t idx, i;
289
290 if (unlikely(vq->vq_free_cnt == 0))
291 return -ENOSPC;
292 if (unlikely(vq->vq_free_cnt < num))
293 return -EMSGSIZE;
294
295 if (unlikely(vq->vq_desc_head_idx >= vq->vq_nentries))
296 return -EFAULT;
297
298 for (i = 0; i < num; i++) {
299 idx = vq->vq_desc_head_idx;
300 dxp = &vq->vq_descx[idx];
301 dxp->cookie = (void *)cookie[i];
302 dxp->ndescs = 1;
303
304 start_dp[idx].addr = VIRTIO_MBUF_ADDR(cookie[i], vq) +
305 RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
306 start_dp[idx].len = cookie[i]->buf_len - RTE_PKTMBUF_HEADROOM +
307 hw->vtnet_hdr_size;
308 start_dp[idx].flags = VRING_DESC_F_WRITE;
309 vq->vq_desc_head_idx = start_dp[idx].next;
310 vq_update_avail_ring(vq, idx);
311 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END) {
312 vq->vq_desc_tail_idx = vq->vq_desc_head_idx;
313 break;
314 }
315 }
316
317 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
318
319 return 0;
320 }
321
322 static inline void
virtqueue_refill_single_packed(struct virtqueue * vq,struct vring_packed_desc * dp,struct rte_mbuf * cookie)323 virtqueue_refill_single_packed(struct virtqueue *vq,
324 struct vring_packed_desc *dp,
325 struct rte_mbuf *cookie)
326 {
327 uint16_t flags = vq->vq_packed.cached_flags;
328 struct virtio_hw *hw = vq->hw;
329
330 dp->addr = VIRTIO_MBUF_ADDR(cookie, vq) + RTE_PKTMBUF_HEADROOM - hw->vtnet_hdr_size;
331 dp->len = cookie->buf_len - RTE_PKTMBUF_HEADROOM + hw->vtnet_hdr_size;
332
333 virtqueue_store_flags_packed(dp, flags, hw->weak_barriers);
334
335 if (++vq->vq_avail_idx >= vq->vq_nentries) {
336 vq->vq_avail_idx -= vq->vq_nentries;
337 vq->vq_packed.cached_flags ^=
338 VRING_PACKED_DESC_F_AVAIL_USED;
339 flags = vq->vq_packed.cached_flags;
340 }
341 }
342
343 static inline int
virtqueue_enqueue_recv_refill_packed_init(struct virtqueue * vq,struct rte_mbuf ** cookie,uint16_t num)344 virtqueue_enqueue_recv_refill_packed_init(struct virtqueue *vq,
345 struct rte_mbuf **cookie, uint16_t num)
346 {
347 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
348 struct vq_desc_extra *dxp;
349 uint16_t idx;
350 int i;
351
352 if (unlikely(vq->vq_free_cnt == 0))
353 return -ENOSPC;
354 if (unlikely(vq->vq_free_cnt < num))
355 return -EMSGSIZE;
356
357 for (i = 0; i < num; i++) {
358 idx = vq->vq_avail_idx;
359 dxp = &vq->vq_descx[idx];
360 dxp->cookie = (void *)cookie[i];
361 dxp->ndescs = 1;
362
363 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]);
364 }
365 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
366 return 0;
367 }
368
369 static inline int
virtqueue_enqueue_recv_refill_packed(struct virtqueue * vq,struct rte_mbuf ** cookie,uint16_t num)370 virtqueue_enqueue_recv_refill_packed(struct virtqueue *vq,
371 struct rte_mbuf **cookie, uint16_t num)
372 {
373 struct vring_packed_desc *start_dp = vq->vq_packed.ring.desc;
374 struct vq_desc_extra *dxp;
375 uint16_t idx, did;
376 int i;
377
378 if (unlikely(vq->vq_free_cnt == 0))
379 return -ENOSPC;
380 if (unlikely(vq->vq_free_cnt < num))
381 return -EMSGSIZE;
382
383 for (i = 0; i < num; i++) {
384 idx = vq->vq_avail_idx;
385 did = start_dp[idx].id;
386 dxp = &vq->vq_descx[did];
387 dxp->cookie = (void *)cookie[i];
388 dxp->ndescs = 1;
389
390 virtqueue_refill_single_packed(vq, &start_dp[idx], cookie[i]);
391 }
392 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
393 return 0;
394 }
395
396 /* When doing TSO, the IP length is not included in the pseudo header
397 * checksum of the packet given to the PMD, but for virtio it is
398 * expected.
399 */
400 static void
virtio_tso_fix_cksum(struct rte_mbuf * m)401 virtio_tso_fix_cksum(struct rte_mbuf *m)
402 {
403 /* common case: header is not fragmented */
404 if (likely(rte_pktmbuf_data_len(m) >= m->l2_len + m->l3_len +
405 m->l4_len)) {
406 struct rte_ipv4_hdr *iph;
407 struct rte_ipv6_hdr *ip6h;
408 struct rte_tcp_hdr *th;
409 uint16_t prev_cksum, new_cksum, ip_len, ip_paylen;
410 uint32_t tmp;
411
412 iph = rte_pktmbuf_mtod_offset(m,
413 struct rte_ipv4_hdr *, m->l2_len);
414 th = RTE_PTR_ADD(iph, m->l3_len);
415 if ((iph->version_ihl >> 4) == 4) {
416 iph->hdr_checksum = 0;
417 iph->hdr_checksum = rte_ipv4_cksum(iph);
418 ip_len = iph->total_length;
419 ip_paylen = rte_cpu_to_be_16(rte_be_to_cpu_16(ip_len) -
420 m->l3_len);
421 } else {
422 ip6h = (struct rte_ipv6_hdr *)iph;
423 ip_paylen = ip6h->payload_len;
424 }
425
426 /* calculate the new phdr checksum not including ip_paylen */
427 prev_cksum = th->cksum;
428 tmp = prev_cksum;
429 tmp += ip_paylen;
430 tmp = (tmp & 0xffff) + (tmp >> 16);
431 new_cksum = tmp;
432
433 /* replace it in the packet */
434 th->cksum = new_cksum;
435 }
436 }
437
438
439
440
441 static inline void
virtqueue_enqueue_xmit_inorder(struct virtnet_tx * txvq,struct rte_mbuf ** cookies,uint16_t num)442 virtqueue_enqueue_xmit_inorder(struct virtnet_tx *txvq,
443 struct rte_mbuf **cookies,
444 uint16_t num)
445 {
446 struct vq_desc_extra *dxp;
447 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
448 struct vring_desc *start_dp;
449 struct virtio_net_hdr *hdr;
450 uint16_t idx;
451 int16_t head_size = vq->hw->vtnet_hdr_size;
452 uint16_t i = 0;
453
454 idx = vq->vq_desc_head_idx;
455 start_dp = vq->vq_split.ring.desc;
456
457 while (i < num) {
458 idx = idx & (vq->vq_nentries - 1);
459 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
460 dxp->cookie = (void *)cookies[i];
461 dxp->ndescs = 1;
462 virtio_update_packet_stats(&txvq->stats, cookies[i]);
463
464 hdr = rte_pktmbuf_mtod_offset(cookies[i],
465 struct virtio_net_hdr *, -head_size);
466
467 /* if offload disabled, hdr is not zeroed yet, do it now */
468 if (!vq->hw->has_tx_offload)
469 virtqueue_clear_net_hdr(hdr);
470 else
471 virtqueue_xmit_offload(hdr, cookies[i]);
472
473 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookies[i], vq) - head_size;
474 start_dp[idx].len = cookies[i]->data_len + head_size;
475 start_dp[idx].flags = 0;
476
477
478 vq_update_avail_ring(vq, idx);
479
480 idx++;
481 i++;
482 };
483
484 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - num);
485 vq->vq_desc_head_idx = idx & (vq->vq_nentries - 1);
486 }
487
488 static inline void
virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx * txvq,struct rte_mbuf * cookie,int in_order)489 virtqueue_enqueue_xmit_packed_fast(struct virtnet_tx *txvq,
490 struct rte_mbuf *cookie,
491 int in_order)
492 {
493 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
494 struct vring_packed_desc *dp;
495 struct vq_desc_extra *dxp;
496 uint16_t idx, id, flags;
497 int16_t head_size = vq->hw->vtnet_hdr_size;
498 struct virtio_net_hdr *hdr;
499
500 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
501 idx = vq->vq_avail_idx;
502 dp = &vq->vq_packed.ring.desc[idx];
503
504 dxp = &vq->vq_descx[id];
505 dxp->ndescs = 1;
506 dxp->cookie = cookie;
507
508 flags = vq->vq_packed.cached_flags;
509
510 /* prepend cannot fail, checked by caller */
511 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
512 -head_size);
513
514 /* if offload disabled, hdr is not zeroed yet, do it now */
515 if (!vq->hw->has_tx_offload)
516 virtqueue_clear_net_hdr(hdr);
517 else
518 virtqueue_xmit_offload(hdr, cookie);
519
520 dp->addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq) - head_size;
521 dp->len = cookie->data_len + head_size;
522 dp->id = id;
523
524 if (++vq->vq_avail_idx >= vq->vq_nentries) {
525 vq->vq_avail_idx -= vq->vq_nentries;
526 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED;
527 }
528
529 vq->vq_free_cnt--;
530
531 if (!in_order) {
532 vq->vq_desc_head_idx = dxp->next;
533 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
534 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
535 }
536
537 virtqueue_store_flags_packed(dp, flags, vq->hw->weak_barriers);
538 }
539
540 static inline void
virtqueue_enqueue_xmit(struct virtnet_tx * txvq,struct rte_mbuf * cookie,uint16_t needed,int use_indirect,int can_push,int in_order)541 virtqueue_enqueue_xmit(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
542 uint16_t needed, int use_indirect, int can_push,
543 int in_order)
544 {
545 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
546 struct vq_desc_extra *dxp;
547 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
548 struct vring_desc *start_dp;
549 uint16_t seg_num = cookie->nb_segs;
550 uint16_t head_idx, idx;
551 int16_t head_size = vq->hw->vtnet_hdr_size;
552 bool prepend_header = false;
553 struct virtio_net_hdr *hdr;
554
555 head_idx = vq->vq_desc_head_idx;
556 idx = head_idx;
557 if (in_order)
558 dxp = &vq->vq_descx[vq->vq_avail_idx & (vq->vq_nentries - 1)];
559 else
560 dxp = &vq->vq_descx[idx];
561 dxp->cookie = (void *)cookie;
562 dxp->ndescs = needed;
563
564 start_dp = vq->vq_split.ring.desc;
565
566 if (can_push) {
567 /* prepend cannot fail, checked by caller */
568 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
569 -head_size);
570 prepend_header = true;
571
572 /* if offload disabled, it is not zeroed below, do it now */
573 if (!vq->hw->has_tx_offload)
574 virtqueue_clear_net_hdr(hdr);
575 } else if (use_indirect) {
576 /* setup tx ring slot to point to indirect
577 * descriptor list stored in reserved region.
578 *
579 * the first slot in indirect ring is already preset
580 * to point to the header in reserved region
581 */
582 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
583 RTE_PTR_DIFF(&txr[idx].tx_indir, txr);
584 start_dp[idx].len = (seg_num + 1) * sizeof(struct vring_desc);
585 start_dp[idx].flags = VRING_DESC_F_INDIRECT;
586 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
587
588 /* loop below will fill in rest of the indirect elements */
589 start_dp = txr[idx].tx_indir;
590 idx = 1;
591 } else {
592 /* setup first tx ring slot to point to header
593 * stored in reserved region.
594 */
595 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
596 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
597 start_dp[idx].len = vq->hw->vtnet_hdr_size;
598 start_dp[idx].flags = VRING_DESC_F_NEXT;
599 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
600
601 idx = start_dp[idx].next;
602 }
603
604 if (vq->hw->has_tx_offload)
605 virtqueue_xmit_offload(hdr, cookie);
606
607 do {
608 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);
609 start_dp[idx].len = cookie->data_len;
610 if (prepend_header) {
611 start_dp[idx].addr -= head_size;
612 start_dp[idx].len += head_size;
613 prepend_header = false;
614 }
615 start_dp[idx].flags = cookie->next ? VRING_DESC_F_NEXT : 0;
616 idx = start_dp[idx].next;
617 } while ((cookie = cookie->next) != NULL);
618
619 if (use_indirect)
620 idx = vq->vq_split.ring.desc[head_idx].next;
621
622 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
623
624 vq->vq_desc_head_idx = idx;
625 vq_update_avail_ring(vq, head_idx);
626
627 if (!in_order) {
628 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
629 vq->vq_desc_tail_idx = idx;
630 }
631 }
632
633 void
virtio_dev_cq_start(struct rte_eth_dev * dev)634 virtio_dev_cq_start(struct rte_eth_dev *dev)
635 {
636 struct virtio_hw *hw = dev->data->dev_private;
637
638 if (hw->cvq) {
639 rte_spinlock_init(&hw->cvq->lock);
640 VIRTQUEUE_DUMP(virtnet_cq_to_vq(hw->cvq));
641 }
642 }
643
644 int
virtio_dev_rx_queue_setup(struct rte_eth_dev * dev,uint16_t queue_idx,uint16_t nb_desc,unsigned int socket_id __rte_unused,const struct rte_eth_rxconf * rx_conf,struct rte_mempool * mp)645 virtio_dev_rx_queue_setup(struct rte_eth_dev *dev,
646 uint16_t queue_idx,
647 uint16_t nb_desc,
648 unsigned int socket_id __rte_unused,
649 const struct rte_eth_rxconf *rx_conf,
650 struct rte_mempool *mp)
651 {
652 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
653 struct virtio_hw *hw = dev->data->dev_private;
654 struct virtqueue *vq = hw->vqs[vq_idx];
655 struct virtnet_rx *rxvq;
656 uint16_t rx_free_thresh;
657 uint16_t buf_size;
658 const char *error;
659
660 PMD_INIT_FUNC_TRACE();
661
662 if (rx_conf->rx_deferred_start) {
663 PMD_INIT_LOG(ERR, "Rx deferred start is not supported");
664 return -EINVAL;
665 }
666
667 buf_size = virtio_rx_mem_pool_buf_size(mp);
668 if (!virtio_rx_check_scatter(hw->max_rx_pkt_len, buf_size,
669 hw->rx_ol_scatter, &error)) {
670 PMD_INIT_LOG(ERR, "RxQ %u Rx scatter check failed: %s",
671 queue_idx, error);
672 return -EINVAL;
673 }
674
675 rx_free_thresh = rx_conf->rx_free_thresh;
676 if (rx_free_thresh == 0)
677 rx_free_thresh =
678 RTE_MIN(vq->vq_nentries / 4, DEFAULT_RX_FREE_THRESH);
679
680 if (rx_free_thresh & 0x3) {
681 PMD_INIT_LOG(ERR, "rx_free_thresh must be multiples of four."
682 " (rx_free_thresh=%u port=%u queue=%u)",
683 rx_free_thresh, dev->data->port_id, queue_idx);
684 return -EINVAL;
685 }
686
687 if (rx_free_thresh >= vq->vq_nentries) {
688 PMD_INIT_LOG(ERR, "rx_free_thresh must be less than the "
689 "number of RX entries (%u)."
690 " (rx_free_thresh=%u port=%u queue=%u)",
691 vq->vq_nentries,
692 rx_free_thresh, dev->data->port_id, queue_idx);
693 return -EINVAL;
694 }
695 vq->vq_free_thresh = rx_free_thresh;
696
697 /*
698 * For split ring vectorized path descriptors number must be
699 * equal to the ring size.
700 */
701 if (nb_desc > vq->vq_nentries ||
702 (!virtio_with_packed_queue(hw) && hw->use_vec_rx)) {
703 nb_desc = vq->vq_nentries;
704 }
705 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
706
707 rxvq = &vq->rxq;
708 rxvq->queue_id = queue_idx;
709 rxvq->mpool = mp;
710 dev->data->rx_queues[queue_idx] = rxvq;
711
712 return 0;
713 }
714
715 int
virtio_dev_rx_queue_setup_finish(struct rte_eth_dev * dev,uint16_t queue_idx)716 virtio_dev_rx_queue_setup_finish(struct rte_eth_dev *dev, uint16_t queue_idx)
717 {
718 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_RQ_QUEUE_IDX;
719 struct virtio_hw *hw = dev->data->dev_private;
720 struct virtqueue *vq = hw->vqs[vq_idx];
721 struct virtnet_rx *rxvq = &vq->rxq;
722 struct rte_mbuf *m;
723 uint16_t desc_idx;
724 int error, nbufs, i;
725 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER);
726
727 PMD_INIT_FUNC_TRACE();
728
729 /* Allocate blank mbufs for the each rx descriptor */
730 nbufs = 0;
731
732 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) {
733 for (desc_idx = 0; desc_idx < vq->vq_nentries;
734 desc_idx++) {
735 vq->vq_split.ring.avail->ring[desc_idx] = desc_idx;
736 vq->vq_split.ring.desc[desc_idx].flags =
737 VRING_DESC_F_WRITE;
738 }
739
740 virtio_rxq_vec_setup(rxvq);
741 }
742
743 memset(rxvq->fake_mbuf, 0, sizeof(*rxvq->fake_mbuf));
744 for (desc_idx = 0; desc_idx < RTE_PMD_VIRTIO_RX_MAX_BURST; desc_idx++)
745 vq->sw_ring[vq->vq_nentries + desc_idx] = rxvq->fake_mbuf;
746
747 if (hw->use_vec_rx && !virtio_with_packed_queue(hw)) {
748 while (vq->vq_free_cnt >= RTE_VIRTIO_VPMD_RX_REARM_THRESH) {
749 virtio_rxq_rearm_vec(rxvq);
750 nbufs += RTE_VIRTIO_VPMD_RX_REARM_THRESH;
751 }
752 } else if (!virtio_with_packed_queue(vq->hw) && in_order) {
753 if ((!virtqueue_full(vq))) {
754 uint16_t free_cnt = vq->vq_free_cnt;
755 struct rte_mbuf *pkts[free_cnt];
756
757 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, pkts,
758 free_cnt)) {
759 error = virtqueue_enqueue_refill_inorder(vq,
760 pkts,
761 free_cnt);
762 if (unlikely(error)) {
763 for (i = 0; i < free_cnt; i++)
764 rte_pktmbuf_free(pkts[i]);
765 } else {
766 nbufs += free_cnt;
767 }
768 }
769
770 vq_update_avail_idx(vq);
771 }
772 } else {
773 while (!virtqueue_full(vq)) {
774 m = rte_mbuf_raw_alloc(rxvq->mpool);
775 if (m == NULL)
776 break;
777
778 /* Enqueue allocated buffers */
779 if (virtio_with_packed_queue(vq->hw))
780 error = virtqueue_enqueue_recv_refill_packed_init(vq,
781 &m, 1);
782 else
783 error = virtqueue_enqueue_recv_refill(vq,
784 &m, 1);
785 if (error) {
786 rte_pktmbuf_free(m);
787 break;
788 }
789 nbufs++;
790 }
791
792 if (!virtio_with_packed_queue(vq->hw))
793 vq_update_avail_idx(vq);
794 }
795
796 PMD_INIT_LOG(DEBUG, "Allocated %d bufs", nbufs);
797
798 VIRTQUEUE_DUMP(vq);
799
800 return 0;
801 }
802
803 /*
804 * struct rte_eth_dev *dev: Used to update dev
805 * uint16_t nb_desc: Defaults to values read from config space
806 * unsigned int socket_id: Used to allocate memzone
807 * const struct rte_eth_txconf *tx_conf: Used to setup tx engine
808 * uint16_t queue_idx: Just used as an index in dev txq list
809 */
810 int
virtio_dev_tx_queue_setup(struct rte_eth_dev * dev,uint16_t queue_idx,uint16_t nb_desc,unsigned int socket_id __rte_unused,const struct rte_eth_txconf * tx_conf)811 virtio_dev_tx_queue_setup(struct rte_eth_dev *dev,
812 uint16_t queue_idx,
813 uint16_t nb_desc,
814 unsigned int socket_id __rte_unused,
815 const struct rte_eth_txconf *tx_conf)
816 {
817 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
818 struct virtio_hw *hw = dev->data->dev_private;
819 struct virtqueue *vq = hw->vqs[vq_idx];
820 struct virtnet_tx *txvq;
821 uint16_t tx_free_thresh;
822
823 PMD_INIT_FUNC_TRACE();
824
825 if (tx_conf->tx_deferred_start) {
826 PMD_INIT_LOG(ERR, "Tx deferred start is not supported");
827 return -EINVAL;
828 }
829
830 if (nb_desc == 0 || nb_desc > vq->vq_nentries)
831 nb_desc = vq->vq_nentries;
832 vq->vq_free_cnt = RTE_MIN(vq->vq_free_cnt, nb_desc);
833
834 txvq = &vq->txq;
835 txvq->queue_id = queue_idx;
836
837 tx_free_thresh = tx_conf->tx_free_thresh;
838 if (tx_free_thresh == 0)
839 tx_free_thresh =
840 RTE_MIN(vq->vq_nentries / 4, DEFAULT_TX_FREE_THRESH);
841
842 if (tx_free_thresh >= (vq->vq_nentries - 3)) {
843 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the "
844 "number of TX entries minus 3 (%u)."
845 " (tx_free_thresh=%u port=%u queue=%u)",
846 vq->vq_nentries - 3,
847 tx_free_thresh, dev->data->port_id, queue_idx);
848 return -EINVAL;
849 }
850
851 vq->vq_free_thresh = tx_free_thresh;
852
853 dev->data->tx_queues[queue_idx] = txvq;
854 return 0;
855 }
856
857 int
virtio_dev_tx_queue_setup_finish(struct rte_eth_dev * dev,uint16_t queue_idx)858 virtio_dev_tx_queue_setup_finish(struct rte_eth_dev *dev,
859 uint16_t queue_idx)
860 {
861 uint16_t vq_idx = 2 * queue_idx + VTNET_SQ_TQ_QUEUE_IDX;
862 struct virtio_hw *hw = dev->data->dev_private;
863 struct virtqueue *vq = hw->vqs[vq_idx];
864
865 PMD_INIT_FUNC_TRACE();
866
867 if (!virtio_with_packed_queue(hw)) {
868 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER))
869 vq->vq_split.ring.desc[vq->vq_nentries - 1].next = 0;
870 }
871
872 VIRTQUEUE_DUMP(vq);
873
874 return 0;
875 }
876
877 static inline void
virtio_discard_rxbuf(struct virtqueue * vq,struct rte_mbuf * m)878 virtio_discard_rxbuf(struct virtqueue *vq, struct rte_mbuf *m)
879 {
880 int error;
881 /*
882 * Requeue the discarded mbuf. This should always be
883 * successful since it was just dequeued.
884 */
885 if (virtio_with_packed_queue(vq->hw))
886 error = virtqueue_enqueue_recv_refill_packed(vq, &m, 1);
887 else
888 error = virtqueue_enqueue_recv_refill(vq, &m, 1);
889
890 if (unlikely(error)) {
891 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf");
892 rte_pktmbuf_free(m);
893 }
894 }
895
896 static inline void
virtio_discard_rxbuf_inorder(struct virtqueue * vq,struct rte_mbuf * m)897 virtio_discard_rxbuf_inorder(struct virtqueue *vq, struct rte_mbuf *m)
898 {
899 int error;
900
901 error = virtqueue_enqueue_refill_inorder(vq, &m, 1);
902 if (unlikely(error)) {
903 PMD_DRV_LOG(ERR, "cannot requeue discarded mbuf");
904 rte_pktmbuf_free(m);
905 }
906 }
907
908 /* Optionally fill offload information in structure */
909 static inline int
virtio_rx_offload(struct rte_mbuf * m,struct virtio_net_hdr * hdr)910 virtio_rx_offload(struct rte_mbuf *m, struct virtio_net_hdr *hdr)
911 {
912 struct rte_net_hdr_lens hdr_lens;
913 uint32_t hdrlen, ptype;
914 int l4_supported = 0;
915
916 /* nothing to do */
917 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE)
918 return 0;
919
920 m->ol_flags |= RTE_MBUF_F_RX_IP_CKSUM_UNKNOWN;
921
922 ptype = rte_net_get_ptype(m, &hdr_lens, RTE_PTYPE_ALL_MASK);
923 m->packet_type = ptype;
924 if ((ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_TCP ||
925 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_UDP ||
926 (ptype & RTE_PTYPE_L4_MASK) == RTE_PTYPE_L4_SCTP)
927 l4_supported = 1;
928
929 if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) {
930 hdrlen = hdr_lens.l2_len + hdr_lens.l3_len + hdr_lens.l4_len;
931 if (hdr->csum_start <= hdrlen && l4_supported) {
932 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_NONE;
933 } else {
934 /* Unknown proto or tunnel, do sw cksum. We can assume
935 * the cksum field is in the first segment since the
936 * buffers we provided to the host are large enough.
937 * In case of SCTP, this will be wrong since it's a CRC
938 * but there's nothing we can do.
939 */
940 uint16_t csum = 0, off;
941
942 if (rte_raw_cksum_mbuf(m, hdr->csum_start,
943 rte_pktmbuf_pkt_len(m) - hdr->csum_start,
944 &csum) < 0)
945 return -EINVAL;
946 if (likely(csum != 0xffff))
947 csum = ~csum;
948 off = hdr->csum_offset + hdr->csum_start;
949 if (rte_pktmbuf_data_len(m) >= off + 1)
950 *rte_pktmbuf_mtod_offset(m, uint16_t *,
951 off) = csum;
952 }
953 } else if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID && l4_supported) {
954 m->ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
955 }
956
957 /* GSO request, save required information in mbuf */
958 if (hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) {
959 /* Check unsupported modes */
960 if ((hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) ||
961 (hdr->gso_size == 0)) {
962 return -EINVAL;
963 }
964
965 /* Update mss lengths in mbuf */
966 m->tso_segsz = hdr->gso_size;
967 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
968 case VIRTIO_NET_HDR_GSO_TCPV4:
969 case VIRTIO_NET_HDR_GSO_TCPV6:
970 m->ol_flags |= RTE_MBUF_F_RX_LRO |
971 RTE_MBUF_F_RX_L4_CKSUM_NONE;
972 break;
973 default:
974 return -EINVAL;
975 }
976 }
977
978 return 0;
979 }
980
981 #define DESC_PER_CACHELINE (RTE_CACHE_LINE_SIZE / sizeof(struct vring_desc))
982 uint16_t
virtio_recv_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)983 virtio_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
984 {
985 struct virtnet_rx *rxvq = rx_queue;
986 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
987 struct virtio_hw *hw = vq->hw;
988 struct rte_mbuf *rxm;
989 uint16_t nb_used, num, nb_rx;
990 uint32_t len[VIRTIO_MBUF_BURST_SZ];
991 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
992 int error;
993 uint32_t i, nb_enqueued;
994 uint32_t hdr_size;
995 struct virtio_net_hdr *hdr;
996
997 nb_rx = 0;
998 if (unlikely(hw->started == 0))
999 return nb_rx;
1000
1001 nb_used = virtqueue_nused(vq);
1002
1003 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1004 if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1005 num = VIRTIO_MBUF_BURST_SZ;
1006 if (likely(num > DESC_PER_CACHELINE))
1007 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1008
1009 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1010 PMD_RX_LOG(DEBUG, "used:%d dequeue:%d", nb_used, num);
1011
1012 nb_enqueued = 0;
1013 hdr_size = hw->vtnet_hdr_size;
1014
1015 for (i = 0; i < num ; i++) {
1016 rxm = rcv_pkts[i];
1017
1018 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1019
1020 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1021 PMD_RX_LOG(ERR, "Packet drop");
1022 nb_enqueued++;
1023 virtio_discard_rxbuf(vq, rxm);
1024 rxvq->stats.errors++;
1025 continue;
1026 }
1027
1028 rxm->port = rxvq->port_id;
1029 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1030 rxm->ol_flags = 0;
1031 rxm->vlan_tci = 0;
1032
1033 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1034 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1035
1036 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1037 RTE_PKTMBUF_HEADROOM - hdr_size);
1038
1039 if (hw->vlan_strip)
1040 rte_vlan_strip(rxm);
1041
1042 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1043 virtio_discard_rxbuf(vq, rxm);
1044 rxvq->stats.errors++;
1045 continue;
1046 }
1047
1048 virtio_rx_stats_updated(rxvq, rxm);
1049
1050 rx_pkts[nb_rx++] = rxm;
1051 }
1052
1053 rxvq->stats.packets += nb_rx;
1054
1055 /* Allocate new mbuf for the used descriptor */
1056 if (likely(!virtqueue_full(vq))) {
1057 uint16_t free_cnt = vq->vq_free_cnt;
1058 struct rte_mbuf *new_pkts[free_cnt];
1059
1060 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1061 free_cnt) == 0)) {
1062 error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1063 free_cnt);
1064 if (unlikely(error)) {
1065 for (i = 0; i < free_cnt; i++)
1066 rte_pktmbuf_free(new_pkts[i]);
1067 }
1068 nb_enqueued += free_cnt;
1069 } else {
1070 struct rte_eth_dev *dev =
1071 &rte_eth_devices[rxvq->port_id];
1072 dev->data->rx_mbuf_alloc_failed += free_cnt;
1073 }
1074 }
1075
1076 if (likely(nb_enqueued)) {
1077 vq_update_avail_idx(vq);
1078
1079 if (unlikely(virtqueue_kick_prepare(vq))) {
1080 virtqueue_notify(vq);
1081 PMD_RX_LOG(DEBUG, "Notified");
1082 }
1083 }
1084
1085 return nb_rx;
1086 }
1087
1088 uint16_t
virtio_recv_pkts_packed(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)1089 virtio_recv_pkts_packed(void *rx_queue, struct rte_mbuf **rx_pkts,
1090 uint16_t nb_pkts)
1091 {
1092 struct virtnet_rx *rxvq = rx_queue;
1093 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1094 struct virtio_hw *hw = vq->hw;
1095 struct rte_mbuf *rxm;
1096 uint16_t num, nb_rx;
1097 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1098 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1099 int error;
1100 uint32_t i, nb_enqueued;
1101 uint32_t hdr_size;
1102 struct virtio_net_hdr *hdr;
1103
1104 nb_rx = 0;
1105 if (unlikely(hw->started == 0))
1106 return nb_rx;
1107
1108 num = RTE_MIN(VIRTIO_MBUF_BURST_SZ, nb_pkts);
1109 if (likely(num > DESC_PER_CACHELINE))
1110 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1111
1112 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1113 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1114
1115 nb_enqueued = 0;
1116 hdr_size = hw->vtnet_hdr_size;
1117
1118 for (i = 0; i < num; i++) {
1119 rxm = rcv_pkts[i];
1120
1121 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1122
1123 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1124 PMD_RX_LOG(ERR, "Packet drop");
1125 nb_enqueued++;
1126 virtio_discard_rxbuf(vq, rxm);
1127 rxvq->stats.errors++;
1128 continue;
1129 }
1130
1131 rxm->port = rxvq->port_id;
1132 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1133 rxm->ol_flags = 0;
1134 rxm->vlan_tci = 0;
1135
1136 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1137 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1138
1139 hdr = (struct virtio_net_hdr *)((char *)rxm->buf_addr +
1140 RTE_PKTMBUF_HEADROOM - hdr_size);
1141
1142 if (hw->vlan_strip)
1143 rte_vlan_strip(rxm);
1144
1145 if (hw->has_rx_offload && virtio_rx_offload(rxm, hdr) < 0) {
1146 virtio_discard_rxbuf(vq, rxm);
1147 rxvq->stats.errors++;
1148 continue;
1149 }
1150
1151 virtio_rx_stats_updated(rxvq, rxm);
1152
1153 rx_pkts[nb_rx++] = rxm;
1154 }
1155
1156 rxvq->stats.packets += nb_rx;
1157
1158 /* Allocate new mbuf for the used descriptor */
1159 if (likely(!virtqueue_full(vq))) {
1160 uint16_t free_cnt = vq->vq_free_cnt;
1161 struct rte_mbuf *new_pkts[free_cnt];
1162
1163 if (likely(rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts,
1164 free_cnt) == 0)) {
1165 error = virtqueue_enqueue_recv_refill_packed(vq,
1166 new_pkts, free_cnt);
1167 if (unlikely(error)) {
1168 for (i = 0; i < free_cnt; i++)
1169 rte_pktmbuf_free(new_pkts[i]);
1170 }
1171 nb_enqueued += free_cnt;
1172 } else {
1173 struct rte_eth_dev *dev =
1174 &rte_eth_devices[rxvq->port_id];
1175 dev->data->rx_mbuf_alloc_failed += free_cnt;
1176 }
1177 }
1178
1179 if (likely(nb_enqueued)) {
1180 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1181 virtqueue_notify(vq);
1182 PMD_RX_LOG(DEBUG, "Notified");
1183 }
1184 }
1185
1186 return nb_rx;
1187 }
1188
1189
1190 uint16_t
virtio_recv_pkts_inorder(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)1191 virtio_recv_pkts_inorder(void *rx_queue,
1192 struct rte_mbuf **rx_pkts,
1193 uint16_t nb_pkts)
1194 {
1195 struct virtnet_rx *rxvq = rx_queue;
1196 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1197 struct virtio_hw *hw = vq->hw;
1198 struct rte_mbuf *rxm;
1199 struct rte_mbuf *prev = NULL;
1200 uint16_t nb_used, num, nb_rx;
1201 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1202 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1203 int error;
1204 uint32_t nb_enqueued;
1205 uint32_t seg_num;
1206 uint32_t seg_res;
1207 uint32_t hdr_size;
1208 int32_t i;
1209
1210 nb_rx = 0;
1211 if (unlikely(hw->started == 0))
1212 return nb_rx;
1213
1214 nb_used = virtqueue_nused(vq);
1215 nb_used = RTE_MIN(nb_used, nb_pkts);
1216 nb_used = RTE_MIN(nb_used, VIRTIO_MBUF_BURST_SZ);
1217
1218 PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1219
1220 nb_enqueued = 0;
1221 seg_num = 1;
1222 seg_res = 0;
1223 hdr_size = hw->vtnet_hdr_size;
1224
1225 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len, nb_used);
1226
1227 for (i = 0; i < num; i++) {
1228 struct virtio_net_hdr_mrg_rxbuf *header;
1229
1230 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1231 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1232
1233 rxm = rcv_pkts[i];
1234
1235 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1236 PMD_RX_LOG(ERR, "Packet drop");
1237 nb_enqueued++;
1238 virtio_discard_rxbuf_inorder(vq, rxm);
1239 rxvq->stats.errors++;
1240 continue;
1241 }
1242
1243 header = (struct virtio_net_hdr_mrg_rxbuf *)
1244 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1245 - hdr_size);
1246
1247 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) {
1248 seg_num = header->num_buffers;
1249 if (seg_num == 0)
1250 seg_num = 1;
1251 } else {
1252 seg_num = 1;
1253 }
1254
1255 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1256 rxm->nb_segs = seg_num;
1257 rxm->ol_flags = 0;
1258 rxm->vlan_tci = 0;
1259 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1260 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1261
1262 rxm->port = rxvq->port_id;
1263
1264 rx_pkts[nb_rx] = rxm;
1265 prev = rxm;
1266
1267 if (vq->hw->has_rx_offload &&
1268 virtio_rx_offload(rxm, &header->hdr) < 0) {
1269 virtio_discard_rxbuf_inorder(vq, rxm);
1270 rxvq->stats.errors++;
1271 continue;
1272 }
1273
1274 if (hw->vlan_strip)
1275 rte_vlan_strip(rx_pkts[nb_rx]);
1276
1277 seg_res = seg_num - 1;
1278
1279 /* Merge remaining segments */
1280 while (seg_res != 0 && i < (num - 1)) {
1281 i++;
1282
1283 rxm = rcv_pkts[i];
1284 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1285 rxm->pkt_len = (uint32_t)(len[i]);
1286 rxm->data_len = (uint16_t)(len[i]);
1287
1288 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1289
1290 prev->next = rxm;
1291 prev = rxm;
1292 seg_res -= 1;
1293 }
1294
1295 if (!seg_res) {
1296 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1297 nb_rx++;
1298 }
1299 }
1300
1301 /* Last packet still need merge segments */
1302 while (seg_res != 0) {
1303 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1304 VIRTIO_MBUF_BURST_SZ);
1305
1306 if (likely(virtqueue_nused(vq) >= rcv_cnt)) {
1307 num = virtqueue_dequeue_rx_inorder(vq, rcv_pkts, len,
1308 rcv_cnt);
1309 uint16_t extra_idx = 0;
1310
1311 rcv_cnt = num;
1312 while (extra_idx < rcv_cnt) {
1313 rxm = rcv_pkts[extra_idx];
1314 rxm->data_off =
1315 RTE_PKTMBUF_HEADROOM - hdr_size;
1316 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1317 rxm->data_len = (uint16_t)(len[extra_idx]);
1318 prev->next = rxm;
1319 prev = rxm;
1320 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1321 extra_idx += 1;
1322 };
1323 seg_res -= rcv_cnt;
1324
1325 if (!seg_res) {
1326 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1327 nb_rx++;
1328 }
1329 } else {
1330 PMD_RX_LOG(ERR,
1331 "No enough segments for packet.");
1332 rte_pktmbuf_free(rx_pkts[nb_rx]);
1333 rxvq->stats.errors++;
1334 break;
1335 }
1336 }
1337
1338 rxvq->stats.packets += nb_rx;
1339
1340 /* Allocate new mbuf for the used descriptor */
1341
1342 if (likely(!virtqueue_full(vq))) {
1343 /* free_cnt may include mrg descs */
1344 uint16_t free_cnt = vq->vq_free_cnt;
1345 struct rte_mbuf *new_pkts[free_cnt];
1346
1347 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1348 error = virtqueue_enqueue_refill_inorder(vq, new_pkts,
1349 free_cnt);
1350 if (unlikely(error)) {
1351 for (i = 0; i < free_cnt; i++)
1352 rte_pktmbuf_free(new_pkts[i]);
1353 }
1354 nb_enqueued += free_cnt;
1355 } else {
1356 struct rte_eth_dev *dev =
1357 &rte_eth_devices[rxvq->port_id];
1358 dev->data->rx_mbuf_alloc_failed += free_cnt;
1359 }
1360 }
1361
1362 if (likely(nb_enqueued)) {
1363 vq_update_avail_idx(vq);
1364
1365 if (unlikely(virtqueue_kick_prepare(vq))) {
1366 virtqueue_notify(vq);
1367 PMD_RX_LOG(DEBUG, "Notified");
1368 }
1369 }
1370
1371 return nb_rx;
1372 }
1373
1374 uint16_t
virtio_recv_mergeable_pkts(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)1375 virtio_recv_mergeable_pkts(void *rx_queue,
1376 struct rte_mbuf **rx_pkts,
1377 uint16_t nb_pkts)
1378 {
1379 struct virtnet_rx *rxvq = rx_queue;
1380 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1381 struct virtio_hw *hw = vq->hw;
1382 struct rte_mbuf *rxm;
1383 struct rte_mbuf *prev = NULL;
1384 uint16_t nb_used, num, nb_rx = 0;
1385 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1386 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1387 int error;
1388 uint32_t nb_enqueued = 0;
1389 uint32_t seg_num = 0;
1390 uint32_t seg_res = 0;
1391 uint32_t hdr_size = hw->vtnet_hdr_size;
1392 int32_t i;
1393
1394 if (unlikely(hw->started == 0))
1395 return nb_rx;
1396
1397 nb_used = virtqueue_nused(vq);
1398
1399 PMD_RX_LOG(DEBUG, "used:%d", nb_used);
1400
1401 num = likely(nb_used <= nb_pkts) ? nb_used : nb_pkts;
1402 if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1403 num = VIRTIO_MBUF_BURST_SZ;
1404 if (likely(num > DESC_PER_CACHELINE))
1405 num = num - ((vq->vq_used_cons_idx + num) %
1406 DESC_PER_CACHELINE);
1407
1408
1409 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len, num);
1410
1411 for (i = 0; i < num; i++) {
1412 struct virtio_net_hdr_mrg_rxbuf *header;
1413
1414 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1415 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1416
1417 rxm = rcv_pkts[i];
1418
1419 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1420 PMD_RX_LOG(ERR, "Packet drop");
1421 nb_enqueued++;
1422 virtio_discard_rxbuf(vq, rxm);
1423 rxvq->stats.errors++;
1424 continue;
1425 }
1426
1427 header = (struct virtio_net_hdr_mrg_rxbuf *)
1428 ((char *)rxm->buf_addr + RTE_PKTMBUF_HEADROOM
1429 - hdr_size);
1430 seg_num = header->num_buffers;
1431 if (seg_num == 0)
1432 seg_num = 1;
1433
1434 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1435 rxm->nb_segs = seg_num;
1436 rxm->ol_flags = 0;
1437 rxm->vlan_tci = 0;
1438 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1439 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1440
1441 rxm->port = rxvq->port_id;
1442
1443 rx_pkts[nb_rx] = rxm;
1444 prev = rxm;
1445
1446 if (hw->has_rx_offload &&
1447 virtio_rx_offload(rxm, &header->hdr) < 0) {
1448 virtio_discard_rxbuf(vq, rxm);
1449 rxvq->stats.errors++;
1450 continue;
1451 }
1452
1453 if (hw->vlan_strip)
1454 rte_vlan_strip(rx_pkts[nb_rx]);
1455
1456 seg_res = seg_num - 1;
1457
1458 /* Merge remaining segments */
1459 while (seg_res != 0 && i < (num - 1)) {
1460 i++;
1461
1462 rxm = rcv_pkts[i];
1463 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1464 rxm->pkt_len = (uint32_t)(len[i]);
1465 rxm->data_len = (uint16_t)(len[i]);
1466
1467 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1468
1469 prev->next = rxm;
1470 prev = rxm;
1471 seg_res -= 1;
1472 }
1473
1474 if (!seg_res) {
1475 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1476 nb_rx++;
1477 }
1478 }
1479
1480 /* Last packet still need merge segments */
1481 while (seg_res != 0) {
1482 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1483 VIRTIO_MBUF_BURST_SZ);
1484
1485 if (likely(virtqueue_nused(vq) >= rcv_cnt)) {
1486 num = virtqueue_dequeue_burst_rx(vq, rcv_pkts, len,
1487 rcv_cnt);
1488 uint16_t extra_idx = 0;
1489
1490 rcv_cnt = num;
1491 while (extra_idx < rcv_cnt) {
1492 rxm = rcv_pkts[extra_idx];
1493 rxm->data_off =
1494 RTE_PKTMBUF_HEADROOM - hdr_size;
1495 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1496 rxm->data_len = (uint16_t)(len[extra_idx]);
1497 prev->next = rxm;
1498 prev = rxm;
1499 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1500 extra_idx += 1;
1501 };
1502 seg_res -= rcv_cnt;
1503
1504 if (!seg_res) {
1505 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1506 nb_rx++;
1507 }
1508 } else {
1509 PMD_RX_LOG(ERR,
1510 "No enough segments for packet.");
1511 rte_pktmbuf_free(rx_pkts[nb_rx]);
1512 rxvq->stats.errors++;
1513 break;
1514 }
1515 }
1516
1517 rxvq->stats.packets += nb_rx;
1518
1519 /* Allocate new mbuf for the used descriptor */
1520 if (likely(!virtqueue_full(vq))) {
1521 /* free_cnt may include mrg descs */
1522 uint16_t free_cnt = vq->vq_free_cnt;
1523 struct rte_mbuf *new_pkts[free_cnt];
1524
1525 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1526 error = virtqueue_enqueue_recv_refill(vq, new_pkts,
1527 free_cnt);
1528 if (unlikely(error)) {
1529 for (i = 0; i < free_cnt; i++)
1530 rte_pktmbuf_free(new_pkts[i]);
1531 }
1532 nb_enqueued += free_cnt;
1533 } else {
1534 struct rte_eth_dev *dev =
1535 &rte_eth_devices[rxvq->port_id];
1536 dev->data->rx_mbuf_alloc_failed += free_cnt;
1537 }
1538 }
1539
1540 if (likely(nb_enqueued)) {
1541 vq_update_avail_idx(vq);
1542
1543 if (unlikely(virtqueue_kick_prepare(vq))) {
1544 virtqueue_notify(vq);
1545 PMD_RX_LOG(DEBUG, "Notified");
1546 }
1547 }
1548
1549 return nb_rx;
1550 }
1551
1552 uint16_t
virtio_recv_mergeable_pkts_packed(void * rx_queue,struct rte_mbuf ** rx_pkts,uint16_t nb_pkts)1553 virtio_recv_mergeable_pkts_packed(void *rx_queue,
1554 struct rte_mbuf **rx_pkts,
1555 uint16_t nb_pkts)
1556 {
1557 struct virtnet_rx *rxvq = rx_queue;
1558 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq);
1559 struct virtio_hw *hw = vq->hw;
1560 struct rte_mbuf *rxm;
1561 struct rte_mbuf *prev = NULL;
1562 uint16_t num, nb_rx = 0;
1563 uint32_t len[VIRTIO_MBUF_BURST_SZ];
1564 struct rte_mbuf *rcv_pkts[VIRTIO_MBUF_BURST_SZ];
1565 uint32_t nb_enqueued = 0;
1566 uint32_t seg_num = 0;
1567 uint32_t seg_res = 0;
1568 uint32_t hdr_size = hw->vtnet_hdr_size;
1569 int32_t i;
1570 int error;
1571
1572 if (unlikely(hw->started == 0))
1573 return nb_rx;
1574
1575
1576 num = nb_pkts;
1577 if (unlikely(num > VIRTIO_MBUF_BURST_SZ))
1578 num = VIRTIO_MBUF_BURST_SZ;
1579 if (likely(num > DESC_PER_CACHELINE))
1580 num = num - ((vq->vq_used_cons_idx + num) % DESC_PER_CACHELINE);
1581
1582 num = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts, len, num);
1583
1584 for (i = 0; i < num; i++) {
1585 struct virtio_net_hdr_mrg_rxbuf *header;
1586
1587 PMD_RX_LOG(DEBUG, "dequeue:%d", num);
1588 PMD_RX_LOG(DEBUG, "packet len:%d", len[i]);
1589
1590 rxm = rcv_pkts[i];
1591
1592 if (unlikely(len[i] < hdr_size + RTE_ETHER_HDR_LEN)) {
1593 PMD_RX_LOG(ERR, "Packet drop");
1594 nb_enqueued++;
1595 virtio_discard_rxbuf(vq, rxm);
1596 rxvq->stats.errors++;
1597 continue;
1598 }
1599
1600 header = (struct virtio_net_hdr_mrg_rxbuf *)((char *)
1601 rxm->buf_addr + RTE_PKTMBUF_HEADROOM - hdr_size);
1602 seg_num = header->num_buffers;
1603
1604 if (seg_num == 0)
1605 seg_num = 1;
1606
1607 rxm->data_off = RTE_PKTMBUF_HEADROOM;
1608 rxm->nb_segs = seg_num;
1609 rxm->ol_flags = 0;
1610 rxm->vlan_tci = 0;
1611 rxm->pkt_len = (uint32_t)(len[i] - hdr_size);
1612 rxm->data_len = (uint16_t)(len[i] - hdr_size);
1613
1614 rxm->port = rxvq->port_id;
1615 rx_pkts[nb_rx] = rxm;
1616 prev = rxm;
1617
1618 if (hw->has_rx_offload &&
1619 virtio_rx_offload(rxm, &header->hdr) < 0) {
1620 virtio_discard_rxbuf(vq, rxm);
1621 rxvq->stats.errors++;
1622 continue;
1623 }
1624
1625 if (hw->vlan_strip)
1626 rte_vlan_strip(rx_pkts[nb_rx]);
1627
1628 seg_res = seg_num - 1;
1629
1630 /* Merge remaining segments */
1631 while (seg_res != 0 && i < (num - 1)) {
1632 i++;
1633
1634 rxm = rcv_pkts[i];
1635 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1636 rxm->pkt_len = (uint32_t)(len[i]);
1637 rxm->data_len = (uint16_t)(len[i]);
1638
1639 rx_pkts[nb_rx]->pkt_len += (uint32_t)(len[i]);
1640
1641 prev->next = rxm;
1642 prev = rxm;
1643 seg_res -= 1;
1644 }
1645
1646 if (!seg_res) {
1647 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1648 nb_rx++;
1649 }
1650 }
1651
1652 /* Last packet still need merge segments */
1653 while (seg_res != 0) {
1654 uint16_t rcv_cnt = RTE_MIN((uint16_t)seg_res,
1655 VIRTIO_MBUF_BURST_SZ);
1656 uint16_t extra_idx = 0;
1657
1658 rcv_cnt = virtqueue_dequeue_burst_rx_packed(vq, rcv_pkts,
1659 len, rcv_cnt);
1660 if (unlikely(rcv_cnt == 0)) {
1661 PMD_RX_LOG(ERR, "No enough segments for packet.");
1662 rte_pktmbuf_free(rx_pkts[nb_rx]);
1663 rxvq->stats.errors++;
1664 break;
1665 }
1666
1667 while (extra_idx < rcv_cnt) {
1668 rxm = rcv_pkts[extra_idx];
1669
1670 rxm->data_off = RTE_PKTMBUF_HEADROOM - hdr_size;
1671 rxm->pkt_len = (uint32_t)(len[extra_idx]);
1672 rxm->data_len = (uint16_t)(len[extra_idx]);
1673
1674 prev->next = rxm;
1675 prev = rxm;
1676 rx_pkts[nb_rx]->pkt_len += len[extra_idx];
1677 extra_idx += 1;
1678 }
1679 seg_res -= rcv_cnt;
1680 if (!seg_res) {
1681 virtio_rx_stats_updated(rxvq, rx_pkts[nb_rx]);
1682 nb_rx++;
1683 }
1684 }
1685
1686 rxvq->stats.packets += nb_rx;
1687
1688 /* Allocate new mbuf for the used descriptor */
1689 if (likely(!virtqueue_full(vq))) {
1690 /* free_cnt may include mrg descs */
1691 uint16_t free_cnt = vq->vq_free_cnt;
1692 struct rte_mbuf *new_pkts[free_cnt];
1693
1694 if (!rte_pktmbuf_alloc_bulk(rxvq->mpool, new_pkts, free_cnt)) {
1695 error = virtqueue_enqueue_recv_refill_packed(vq,
1696 new_pkts, free_cnt);
1697 if (unlikely(error)) {
1698 for (i = 0; i < free_cnt; i++)
1699 rte_pktmbuf_free(new_pkts[i]);
1700 }
1701 nb_enqueued += free_cnt;
1702 } else {
1703 struct rte_eth_dev *dev =
1704 &rte_eth_devices[rxvq->port_id];
1705 dev->data->rx_mbuf_alloc_failed += free_cnt;
1706 }
1707 }
1708
1709 if (likely(nb_enqueued)) {
1710 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1711 virtqueue_notify(vq);
1712 PMD_RX_LOG(DEBUG, "Notified");
1713 }
1714 }
1715
1716 return nb_rx;
1717 }
1718
1719 uint16_t
virtio_xmit_pkts_prepare(void * tx_queue __rte_unused,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)1720 virtio_xmit_pkts_prepare(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
1721 uint16_t nb_pkts)
1722 {
1723 uint16_t nb_tx;
1724 int error;
1725
1726 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1727 struct rte_mbuf *m = tx_pkts[nb_tx];
1728
1729 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
1730 error = rte_validate_tx_offload(m);
1731 if (unlikely(error)) {
1732 rte_errno = -error;
1733 break;
1734 }
1735 #endif
1736
1737 /* Do VLAN tag insertion */
1738 if (unlikely(m->ol_flags & RTE_MBUF_F_TX_VLAN)) {
1739 error = rte_vlan_insert(&m);
1740 /* rte_vlan_insert() may change pointer
1741 * even in the case of failure
1742 */
1743 tx_pkts[nb_tx] = m;
1744
1745 if (unlikely(error)) {
1746 rte_errno = -error;
1747 break;
1748 }
1749 }
1750
1751 error = rte_net_intel_cksum_prepare(m);
1752 if (unlikely(error)) {
1753 rte_errno = -error;
1754 break;
1755 }
1756
1757 if (m->ol_flags & RTE_MBUF_F_TX_TCP_SEG)
1758 virtio_tso_fix_cksum(m);
1759 }
1760
1761 return nb_tx;
1762 }
1763
1764 uint16_t
virtio_xmit_pkts_packed(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)1765 virtio_xmit_pkts_packed(void *tx_queue, struct rte_mbuf **tx_pkts,
1766 uint16_t nb_pkts)
1767 {
1768 struct virtnet_tx *txvq = tx_queue;
1769 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1770 struct virtio_hw *hw = vq->hw;
1771 uint16_t hdr_size = hw->vtnet_hdr_size;
1772 uint16_t nb_tx = 0;
1773 bool in_order = virtio_with_feature(hw, VIRTIO_F_IN_ORDER);
1774
1775 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1776 return nb_tx;
1777
1778 if (unlikely(nb_pkts < 1))
1779 return nb_pkts;
1780
1781 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1782
1783 if (nb_pkts > vq->vq_free_cnt)
1784 virtio_xmit_cleanup_packed(vq, nb_pkts - vq->vq_free_cnt,
1785 in_order);
1786
1787 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1788 struct rte_mbuf *txm = tx_pkts[nb_tx];
1789 int can_push = 0, use_indirect = 0, slots, need;
1790
1791 /* optimize ring usage */
1792 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1793 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1794 rte_mbuf_refcnt_read(txm) == 1 &&
1795 RTE_MBUF_DIRECT(txm) &&
1796 txm->nb_segs == 1 &&
1797 rte_pktmbuf_headroom(txm) >= hdr_size &&
1798 rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1799 __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
1800 can_push = 1;
1801 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
1802 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
1803 use_indirect = 1;
1804 /* How many main ring entries are needed to this Tx?
1805 * indirect => 1
1806 * any_layout => number of segments
1807 * default => number of segments + 1
1808 */
1809 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
1810 need = slots - vq->vq_free_cnt;
1811
1812 /* Positive value indicates it need free vring descriptors */
1813 if (unlikely(need > 0)) {
1814 virtio_xmit_cleanup_packed(vq, need, in_order);
1815 need = slots - vq->vq_free_cnt;
1816 if (unlikely(need > 0)) {
1817 PMD_TX_LOG(ERR,
1818 "No free tx descriptors to transmit");
1819 break;
1820 }
1821 }
1822
1823 /* Enqueue Packet buffers */
1824 if (can_push)
1825 virtqueue_enqueue_xmit_packed_fast(txvq, txm, in_order);
1826 else
1827 virtqueue_enqueue_xmit_packed(txvq, txm, slots,
1828 use_indirect, 0,
1829 in_order);
1830
1831 virtio_update_packet_stats(&txvq->stats, txm);
1832 }
1833
1834 txvq->stats.packets += nb_tx;
1835
1836 if (likely(nb_tx)) {
1837 if (unlikely(virtqueue_kick_prepare_packed(vq))) {
1838 virtqueue_notify(vq);
1839 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
1840 }
1841 }
1842
1843 return nb_tx;
1844 }
1845
1846 uint16_t
virtio_xmit_pkts(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)1847 virtio_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1848 {
1849 struct virtnet_tx *txvq = tx_queue;
1850 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1851 struct virtio_hw *hw = vq->hw;
1852 uint16_t hdr_size = hw->vtnet_hdr_size;
1853 uint16_t nb_used, nb_tx = 0;
1854
1855 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1856 return nb_tx;
1857
1858 if (unlikely(nb_pkts < 1))
1859 return nb_pkts;
1860
1861 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1862
1863 nb_used = virtqueue_nused(vq);
1864
1865 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
1866 virtio_xmit_cleanup(vq, nb_used);
1867
1868 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1869 struct rte_mbuf *txm = tx_pkts[nb_tx];
1870 int can_push = 0, use_indirect = 0, slots, need;
1871
1872 /* optimize ring usage */
1873 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1874 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1875 rte_mbuf_refcnt_read(txm) == 1 &&
1876 RTE_MBUF_DIRECT(txm) &&
1877 txm->nb_segs == 1 &&
1878 rte_pktmbuf_headroom(txm) >= hdr_size &&
1879 rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1880 __alignof__(struct virtio_net_hdr_mrg_rxbuf)))
1881 can_push = 1;
1882 else if (virtio_with_feature(hw, VIRTIO_RING_F_INDIRECT_DESC) &&
1883 txm->nb_segs < VIRTIO_MAX_TX_INDIRECT)
1884 use_indirect = 1;
1885
1886 /* How many main ring entries are needed to this Tx?
1887 * any_layout => number of segments
1888 * indirect => 1
1889 * default => number of segments + 1
1890 */
1891 slots = use_indirect ? 1 : (txm->nb_segs + !can_push);
1892 need = slots - vq->vq_free_cnt;
1893
1894 /* Positive value indicates it need free vring descriptors */
1895 if (unlikely(need > 0)) {
1896 nb_used = virtqueue_nused(vq);
1897
1898 need = RTE_MIN(need, (int)nb_used);
1899
1900 virtio_xmit_cleanup(vq, need);
1901 need = slots - vq->vq_free_cnt;
1902 if (unlikely(need > 0)) {
1903 PMD_TX_LOG(ERR,
1904 "No free tx descriptors to transmit");
1905 break;
1906 }
1907 }
1908
1909 /* Enqueue Packet buffers */
1910 virtqueue_enqueue_xmit(txvq, txm, slots, use_indirect,
1911 can_push, 0);
1912
1913 virtio_update_packet_stats(&txvq->stats, txm);
1914 }
1915
1916 txvq->stats.packets += nb_tx;
1917
1918 if (likely(nb_tx)) {
1919 vq_update_avail_idx(vq);
1920
1921 if (unlikely(virtqueue_kick_prepare(vq))) {
1922 virtqueue_notify(vq);
1923 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
1924 }
1925 }
1926
1927 return nb_tx;
1928 }
1929
1930 static __rte_always_inline int
virtio_xmit_try_cleanup_inorder(struct virtqueue * vq,uint16_t need)1931 virtio_xmit_try_cleanup_inorder(struct virtqueue *vq, uint16_t need)
1932 {
1933 uint16_t nb_used, nb_clean, nb_descs;
1934
1935 nb_descs = vq->vq_free_cnt + need;
1936 nb_used = virtqueue_nused(vq);
1937 nb_clean = RTE_MIN(need, (int)nb_used);
1938
1939 virtio_xmit_cleanup_inorder(vq, nb_clean);
1940
1941 return nb_descs - vq->vq_free_cnt;
1942 }
1943
1944 uint16_t
virtio_xmit_pkts_inorder(void * tx_queue,struct rte_mbuf ** tx_pkts,uint16_t nb_pkts)1945 virtio_xmit_pkts_inorder(void *tx_queue,
1946 struct rte_mbuf **tx_pkts,
1947 uint16_t nb_pkts)
1948 {
1949 struct virtnet_tx *txvq = tx_queue;
1950 struct virtqueue *vq = virtnet_txq_to_vq(txvq);
1951 struct virtio_hw *hw = vq->hw;
1952 uint16_t hdr_size = hw->vtnet_hdr_size;
1953 uint16_t nb_used, nb_tx = 0, nb_inorder_pkts = 0;
1954 struct rte_mbuf *inorder_pkts[nb_pkts];
1955 int need;
1956
1957 if (unlikely(hw->started == 0 && tx_pkts != hw->inject_pkts))
1958 return nb_tx;
1959
1960 if (unlikely(nb_pkts < 1))
1961 return nb_pkts;
1962
1963 VIRTQUEUE_DUMP(vq);
1964 PMD_TX_LOG(DEBUG, "%d packets to xmit", nb_pkts);
1965 nb_used = virtqueue_nused(vq);
1966
1967 if (likely(nb_used > vq->vq_nentries - vq->vq_free_thresh))
1968 virtio_xmit_cleanup_inorder(vq, nb_used);
1969
1970 for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1971 struct rte_mbuf *txm = tx_pkts[nb_tx];
1972 int slots;
1973
1974 /* optimize ring usage */
1975 if ((virtio_with_feature(hw, VIRTIO_F_ANY_LAYOUT) ||
1976 virtio_with_feature(hw, VIRTIO_F_VERSION_1)) &&
1977 rte_mbuf_refcnt_read(txm) == 1 &&
1978 RTE_MBUF_DIRECT(txm) &&
1979 txm->nb_segs == 1 &&
1980 rte_pktmbuf_headroom(txm) >= hdr_size &&
1981 rte_is_aligned(rte_pktmbuf_mtod(txm, char *),
1982 __alignof__(struct virtio_net_hdr_mrg_rxbuf))) {
1983 inorder_pkts[nb_inorder_pkts] = txm;
1984 nb_inorder_pkts++;
1985
1986 continue;
1987 }
1988
1989 if (nb_inorder_pkts) {
1990 need = nb_inorder_pkts - vq->vq_free_cnt;
1991 if (unlikely(need > 0)) {
1992 need = virtio_xmit_try_cleanup_inorder(vq,
1993 need);
1994 if (unlikely(need > 0)) {
1995 PMD_TX_LOG(ERR,
1996 "No free tx descriptors to "
1997 "transmit");
1998 break;
1999 }
2000 }
2001 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2002 nb_inorder_pkts);
2003 nb_inorder_pkts = 0;
2004 }
2005
2006 slots = txm->nb_segs + 1;
2007 need = slots - vq->vq_free_cnt;
2008 if (unlikely(need > 0)) {
2009 need = virtio_xmit_try_cleanup_inorder(vq, slots);
2010
2011 if (unlikely(need > 0)) {
2012 PMD_TX_LOG(ERR,
2013 "No free tx descriptors to transmit");
2014 break;
2015 }
2016 }
2017 /* Enqueue Packet buffers */
2018 virtqueue_enqueue_xmit(txvq, txm, slots, 0, 0, 1);
2019
2020 virtio_update_packet_stats(&txvq->stats, txm);
2021 }
2022
2023 /* Transmit all inorder packets */
2024 if (nb_inorder_pkts) {
2025 need = nb_inorder_pkts - vq->vq_free_cnt;
2026 if (unlikely(need > 0)) {
2027 need = virtio_xmit_try_cleanup_inorder(vq,
2028 need);
2029 if (unlikely(need > 0)) {
2030 PMD_TX_LOG(ERR,
2031 "No free tx descriptors to transmit");
2032 nb_inorder_pkts = vq->vq_free_cnt;
2033 nb_tx -= need;
2034 }
2035 }
2036
2037 virtqueue_enqueue_xmit_inorder(txvq, inorder_pkts,
2038 nb_inorder_pkts);
2039 }
2040
2041 txvq->stats.packets += nb_tx;
2042
2043 if (likely(nb_tx)) {
2044 vq_update_avail_idx(vq);
2045
2046 if (unlikely(virtqueue_kick_prepare(vq))) {
2047 virtqueue_notify(vq);
2048 PMD_TX_LOG(DEBUG, "Notified backend after xmit");
2049 }
2050 }
2051
2052 VIRTQUEUE_DUMP(vq);
2053
2054 return nb_tx;
2055 }
2056
2057 __rte_weak uint16_t
virtio_recv_pkts_packed_vec(void * rx_queue __rte_unused,struct rte_mbuf ** rx_pkts __rte_unused,uint16_t nb_pkts __rte_unused)2058 virtio_recv_pkts_packed_vec(void *rx_queue __rte_unused,
2059 struct rte_mbuf **rx_pkts __rte_unused,
2060 uint16_t nb_pkts __rte_unused)
2061 {
2062 return 0;
2063 }
2064
2065 __rte_weak uint16_t
virtio_xmit_pkts_packed_vec(void * tx_queue __rte_unused,struct rte_mbuf ** tx_pkts __rte_unused,uint16_t nb_pkts __rte_unused)2066 virtio_xmit_pkts_packed_vec(void *tx_queue __rte_unused,
2067 struct rte_mbuf **tx_pkts __rte_unused,
2068 uint16_t nb_pkts __rte_unused)
2069 {
2070 return 0;
2071 }
2072