1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2010-2014 Intel Corporation
3 */
4
5 #ifndef _VIRTQUEUE_H_
6 #define _VIRTQUEUE_H_
7
8 #include <stdint.h>
9
10 #include <rte_atomic.h>
11 #include <rte_memory.h>
12 #include <rte_mempool.h>
13 #include <rte_net.h>
14
15 #include "virtio_pci.h"
16 #include "virtio_ring.h"
17 #include "virtio_logs.h"
18 #include "virtio_rxtx.h"
19
20 struct rte_mbuf;
21
22 #define DEFAULT_TX_FREE_THRESH 32
23 #define DEFAULT_RX_FREE_THRESH 32
24
25 #define VIRTIO_MBUF_BURST_SZ 64
26 /*
27 * Per virtio_ring.h in Linux.
28 * For virtio_pci on SMP, we don't need to order with respect to MMIO
29 * accesses through relaxed memory I/O windows, so smp_mb() et al are
30 * sufficient.
31 *
32 * For using virtio to talk to real devices (eg. vDPA) we do need real
33 * barriers.
34 */
35 static inline void
virtio_mb(uint8_t weak_barriers)36 virtio_mb(uint8_t weak_barriers)
37 {
38 if (weak_barriers)
39 rte_smp_mb();
40 else
41 rte_mb();
42 }
43
44 static inline void
virtio_rmb(uint8_t weak_barriers)45 virtio_rmb(uint8_t weak_barriers)
46 {
47 if (weak_barriers)
48 rte_smp_rmb();
49 else
50 rte_io_rmb();
51 }
52
53 static inline void
virtio_wmb(uint8_t weak_barriers)54 virtio_wmb(uint8_t weak_barriers)
55 {
56 if (weak_barriers)
57 rte_smp_wmb();
58 else
59 rte_io_wmb();
60 }
61
62 static inline uint16_t
virtqueue_fetch_flags_packed(struct vring_packed_desc * dp,uint8_t weak_barriers)63 virtqueue_fetch_flags_packed(struct vring_packed_desc *dp,
64 uint8_t weak_barriers)
65 {
66 uint16_t flags;
67
68 if (weak_barriers) {
69 /* x86 prefers to using rte_smp_rmb over __atomic_load_n as it reports
70 * a better perf(~1.5%), which comes from the saved branch by the compiler.
71 * The if and else branch are identical with the smp and io barriers both
72 * defined as compiler barriers on x86.
73 */
74 #ifdef RTE_ARCH_X86_64
75 flags = dp->flags;
76 rte_smp_rmb();
77 #else
78 flags = __atomic_load_n(&dp->flags, __ATOMIC_ACQUIRE);
79 #endif
80 } else {
81 flags = dp->flags;
82 rte_io_rmb();
83 }
84
85 return flags;
86 }
87
88 static inline void
virtqueue_store_flags_packed(struct vring_packed_desc * dp,uint16_t flags,uint8_t weak_barriers)89 virtqueue_store_flags_packed(struct vring_packed_desc *dp,
90 uint16_t flags, uint8_t weak_barriers)
91 {
92 if (weak_barriers) {
93 /* x86 prefers to using rte_smp_wmb over __atomic_store_n as it reports
94 * a better perf(~1.5%), which comes from the saved branch by the compiler.
95 * The if and else branch are identical with the smp and io barriers both
96 * defined as compiler barriers on x86.
97 */
98 #ifdef RTE_ARCH_X86_64
99 rte_smp_wmb();
100 dp->flags = flags;
101 #else
102 __atomic_store_n(&dp->flags, flags, __ATOMIC_RELEASE);
103 #endif
104 } else {
105 rte_io_wmb();
106 dp->flags = flags;
107 }
108 }
109 #ifdef RTE_PMD_PACKET_PREFETCH
110 #define rte_packet_prefetch(p) rte_prefetch1(p)
111 #else
112 #define rte_packet_prefetch(p) do {} while(0)
113 #endif
114
115 #define VIRTQUEUE_MAX_NAME_SZ 32
116
117 #ifdef RTE_VIRTIO_USER
118 /**
119 * Return the physical address (or virtual address in case of
120 * virtio-user) of mbuf data buffer.
121 *
122 * The address is firstly casted to the word size (sizeof(uintptr_t))
123 * before casting it to uint64_t. This is to make it work with different
124 * combination of word size (64 bit and 32 bit) and virtio device
125 * (virtio-pci and virtio-user).
126 */
127 #define VIRTIO_MBUF_ADDR(mb, vq) \
128 ((uint64_t)(*(uintptr_t *)((uintptr_t)(mb) + (vq)->offset)))
129 #else
130 #define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_iova)
131 #endif
132
133 /**
134 * Return the physical address (or virtual address in case of
135 * virtio-user) of mbuf data buffer, taking care of mbuf data offset
136 */
137 #define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \
138 (VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off)
139
140 #define VTNET_SQ_RQ_QUEUE_IDX 0
141 #define VTNET_SQ_TQ_QUEUE_IDX 1
142 #define VTNET_SQ_CQ_QUEUE_IDX 2
143
144 enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 };
145 /**
146 * The maximum virtqueue size is 2^15. Use that value as the end of
147 * descriptor chain terminator since it will never be a valid index
148 * in the descriptor table. This is used to verify we are correctly
149 * handling vq_free_cnt.
150 */
151 #define VQ_RING_DESC_CHAIN_END 32768
152
153 /**
154 * Control the RX mode, ie. promiscuous, allmulti, etc...
155 * All commands require an "out" sg entry containing a 1 byte
156 * state value, zero = disable, non-zero = enable. Commands
157 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
158 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
159 */
160 #define VIRTIO_NET_CTRL_RX 0
161 #define VIRTIO_NET_CTRL_RX_PROMISC 0
162 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1
163 #define VIRTIO_NET_CTRL_RX_ALLUNI 2
164 #define VIRTIO_NET_CTRL_RX_NOMULTI 3
165 #define VIRTIO_NET_CTRL_RX_NOUNI 4
166 #define VIRTIO_NET_CTRL_RX_NOBCAST 5
167
168 /**
169 * Control the MAC
170 *
171 * The MAC filter table is managed by the hypervisor, the guest should
172 * assume the size is infinite. Filtering should be considered
173 * non-perfect, ie. based on hypervisor resources, the guest may
174 * received packets from sources not specified in the filter list.
175 *
176 * In addition to the class/cmd header, the TABLE_SET command requires
177 * two out scatterlists. Each contains a 4 byte count of entries followed
178 * by a concatenated byte stream of the ETH_ALEN MAC addresses. The
179 * first sg list contains unicast addresses, the second is for multicast.
180 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
181 * is available.
182 *
183 * The ADDR_SET command requests one out scatterlist, it contains a
184 * 6 bytes MAC address. This functionality is present if the
185 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
186 */
187 struct virtio_net_ctrl_mac {
188 uint32_t entries;
189 uint8_t macs[][RTE_ETHER_ADDR_LEN];
190 } __rte_packed;
191
192 #define VIRTIO_NET_CTRL_MAC 1
193 #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
194 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
195
196 /**
197 * Control VLAN filtering
198 *
199 * The VLAN filter table is controlled via a simple ADD/DEL interface.
200 * VLAN IDs not added may be filtered by the hypervisor. Del is the
201 * opposite of add. Both commands expect an out entry containing a 2
202 * byte VLAN ID. VLAN filtering is available with the
203 * VIRTIO_NET_F_CTRL_VLAN feature bit.
204 */
205 #define VIRTIO_NET_CTRL_VLAN 2
206 #define VIRTIO_NET_CTRL_VLAN_ADD 0
207 #define VIRTIO_NET_CTRL_VLAN_DEL 1
208
209 /*
210 * Control link announce acknowledgement
211 *
212 * The command VIRTIO_NET_CTRL_ANNOUNCE_ACK is used to indicate that
213 * driver has recevied the notification; device would clear the
214 * VIRTIO_NET_S_ANNOUNCE bit in the status field after it receives
215 * this command.
216 */
217 #define VIRTIO_NET_CTRL_ANNOUNCE 3
218 #define VIRTIO_NET_CTRL_ANNOUNCE_ACK 0
219
220 struct virtio_net_ctrl_hdr {
221 uint8_t class;
222 uint8_t cmd;
223 } __rte_packed;
224
225 typedef uint8_t virtio_net_ctrl_ack;
226
227 #define VIRTIO_NET_OK 0
228 #define VIRTIO_NET_ERR 1
229
230 #define VIRTIO_MAX_CTRL_DATA 2048
231
232 struct virtio_pmd_ctrl {
233 struct virtio_net_ctrl_hdr hdr;
234 virtio_net_ctrl_ack status;
235 uint8_t data[VIRTIO_MAX_CTRL_DATA];
236 };
237
238 struct vq_desc_extra {
239 void *cookie;
240 uint16_t ndescs;
241 uint16_t next;
242 };
243
244 struct virtqueue {
245 struct virtio_hw *hw; /**< virtio_hw structure pointer. */
246 union {
247 struct {
248 /**< vring keeping desc, used and avail */
249 struct vring ring;
250 } vq_split;
251
252 struct {
253 /**< vring keeping descs and events */
254 struct vring_packed ring;
255 bool used_wrap_counter;
256 uint16_t cached_flags; /**< cached flags for descs */
257 uint16_t event_flags_shadow;
258 } vq_packed;
259 };
260
261 uint16_t vq_used_cons_idx; /**< last consumed descriptor */
262 uint16_t vq_nentries; /**< vring desc numbers */
263 uint16_t vq_free_cnt; /**< num of desc available */
264 uint16_t vq_avail_idx; /**< sync until needed */
265 uint16_t vq_free_thresh; /**< free threshold */
266
267 void *vq_ring_virt_mem; /**< linear address of vring*/
268 unsigned int vq_ring_size;
269
270 union {
271 struct virtnet_rx rxq;
272 struct virtnet_tx txq;
273 struct virtnet_ctl cq;
274 };
275
276 rte_iova_t vq_ring_mem; /**< physical address of vring,
277 * or virtual address for virtio_user. */
278
279 /**
280 * Head of the free chain in the descriptor table. If
281 * there are no free descriptors, this will be set to
282 * VQ_RING_DESC_CHAIN_END.
283 */
284 uint16_t vq_desc_head_idx;
285 uint16_t vq_desc_tail_idx;
286 uint16_t vq_queue_index; /**< PCI queue index */
287 uint16_t offset; /**< relative offset to obtain addr in mbuf */
288 uint16_t *notify_addr;
289 struct rte_mbuf **sw_ring; /**< RX software ring. */
290 struct vq_desc_extra vq_descx[0];
291 };
292
293 /* If multiqueue is provided by host, then we suppport it. */
294 #define VIRTIO_NET_CTRL_MQ 4
295 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
296 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
297 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
298
299 /**
300 * This is the first element of the scatter-gather list. If you don't
301 * specify GSO or CSUM features, you can simply ignore the header.
302 */
303 struct virtio_net_hdr {
304 #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /**< Use csum_start,csum_offset*/
305 #define VIRTIO_NET_HDR_F_DATA_VALID 2 /**< Checksum is valid */
306 uint8_t flags;
307 #define VIRTIO_NET_HDR_GSO_NONE 0 /**< Not a GSO frame */
308 #define VIRTIO_NET_HDR_GSO_TCPV4 1 /**< GSO frame, IPv4 TCP (TSO) */
309 #define VIRTIO_NET_HDR_GSO_UDP 3 /**< GSO frame, IPv4 UDP (UFO) */
310 #define VIRTIO_NET_HDR_GSO_TCPV6 4 /**< GSO frame, IPv6 TCP */
311 #define VIRTIO_NET_HDR_GSO_ECN 0x80 /**< TCP has ECN set */
312 uint8_t gso_type;
313 uint16_t hdr_len; /**< Ethernet + IP + tcp/udp hdrs */
314 uint16_t gso_size; /**< Bytes to append to hdr_len per frame */
315 uint16_t csum_start; /**< Position to start checksumming from */
316 uint16_t csum_offset; /**< Offset after that to place checksum */
317 };
318
319 /**
320 * This is the version of the header to use when the MRG_RXBUF
321 * feature has been negotiated.
322 */
323 struct virtio_net_hdr_mrg_rxbuf {
324 struct virtio_net_hdr hdr;
325 uint16_t num_buffers; /**< Number of merged rx buffers */
326 };
327
328 /* Region reserved to allow for transmit header and indirect ring */
329 #define VIRTIO_MAX_TX_INDIRECT 8
330 struct virtio_tx_region {
331 struct virtio_net_hdr_mrg_rxbuf tx_hdr;
332 union {
333 struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT];
334 struct vring_packed_desc
335 tx_packed_indir[VIRTIO_MAX_TX_INDIRECT];
336 } __rte_aligned(16);
337 };
338
339 static inline int
desc_is_used(struct vring_packed_desc * desc,struct virtqueue * vq)340 desc_is_used(struct vring_packed_desc *desc, struct virtqueue *vq)
341 {
342 uint16_t used, avail, flags;
343
344 flags = virtqueue_fetch_flags_packed(desc, vq->hw->weak_barriers);
345 used = !!(flags & VRING_PACKED_DESC_F_USED);
346 avail = !!(flags & VRING_PACKED_DESC_F_AVAIL);
347
348 return avail == used && used == vq->vq_packed.used_wrap_counter;
349 }
350
351 static inline void
vring_desc_init_packed(struct virtqueue * vq,int n)352 vring_desc_init_packed(struct virtqueue *vq, int n)
353 {
354 int i;
355 for (i = 0; i < n - 1; i++) {
356 vq->vq_packed.ring.desc[i].id = i;
357 vq->vq_descx[i].next = i + 1;
358 }
359 vq->vq_packed.ring.desc[i].id = i;
360 vq->vq_descx[i].next = VQ_RING_DESC_CHAIN_END;
361 }
362
363 /* Chain all the descriptors in the ring with an END */
364 static inline void
vring_desc_init_split(struct vring_desc * dp,uint16_t n)365 vring_desc_init_split(struct vring_desc *dp, uint16_t n)
366 {
367 uint16_t i;
368
369 for (i = 0; i < n - 1; i++)
370 dp[i].next = (uint16_t)(i + 1);
371 dp[i].next = VQ_RING_DESC_CHAIN_END;
372 }
373
374 static inline void
vring_desc_init_indirect_packed(struct vring_packed_desc * dp,int n)375 vring_desc_init_indirect_packed(struct vring_packed_desc *dp, int n)
376 {
377 int i;
378 for (i = 0; i < n; i++) {
379 dp[i].id = (uint16_t)i;
380 dp[i].flags = VRING_DESC_F_WRITE;
381 }
382 }
383
384 /**
385 * Tell the backend not to interrupt us. Implementation for packed virtqueues.
386 */
387 static inline void
virtqueue_disable_intr_packed(struct virtqueue * vq)388 virtqueue_disable_intr_packed(struct virtqueue *vq)
389 {
390 if (vq->vq_packed.event_flags_shadow != RING_EVENT_FLAGS_DISABLE) {
391 vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_DISABLE;
392 vq->vq_packed.ring.driver->desc_event_flags =
393 vq->vq_packed.event_flags_shadow;
394 }
395 }
396
397 /**
398 * Tell the backend not to interrupt us. Implementation for split virtqueues.
399 */
400 static inline void
virtqueue_disable_intr_split(struct virtqueue * vq)401 virtqueue_disable_intr_split(struct virtqueue *vq)
402 {
403 vq->vq_split.ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
404 }
405
406 /**
407 * Tell the backend not to interrupt us.
408 */
409 static inline void
virtqueue_disable_intr(struct virtqueue * vq)410 virtqueue_disable_intr(struct virtqueue *vq)
411 {
412 if (vtpci_packed_queue(vq->hw))
413 virtqueue_disable_intr_packed(vq);
414 else
415 virtqueue_disable_intr_split(vq);
416 }
417
418 /**
419 * Tell the backend to interrupt. Implementation for packed virtqueues.
420 */
421 static inline void
virtqueue_enable_intr_packed(struct virtqueue * vq)422 virtqueue_enable_intr_packed(struct virtqueue *vq)
423 {
424 if (vq->vq_packed.event_flags_shadow == RING_EVENT_FLAGS_DISABLE) {
425 vq->vq_packed.event_flags_shadow = RING_EVENT_FLAGS_ENABLE;
426 vq->vq_packed.ring.driver->desc_event_flags =
427 vq->vq_packed.event_flags_shadow;
428 }
429 }
430
431 /**
432 * Tell the backend to interrupt. Implementation for split virtqueues.
433 */
434 static inline void
virtqueue_enable_intr_split(struct virtqueue * vq)435 virtqueue_enable_intr_split(struct virtqueue *vq)
436 {
437 vq->vq_split.ring.avail->flags &= (~VRING_AVAIL_F_NO_INTERRUPT);
438 }
439
440 /**
441 * Tell the backend to interrupt us.
442 */
443 static inline void
virtqueue_enable_intr(struct virtqueue * vq)444 virtqueue_enable_intr(struct virtqueue *vq)
445 {
446 if (vtpci_packed_queue(vq->hw))
447 virtqueue_enable_intr_packed(vq);
448 else
449 virtqueue_enable_intr_split(vq);
450 }
451
452 /**
453 * Dump virtqueue internal structures, for debug purpose only.
454 */
455 void virtqueue_dump(struct virtqueue *vq);
456 /**
457 * Get all mbufs to be freed.
458 */
459 struct rte_mbuf *virtqueue_detach_unused(struct virtqueue *vq);
460
461 /* Flush the elements in the used ring. */
462 void virtqueue_rxvq_flush(struct virtqueue *vq);
463
464 int virtqueue_rxvq_reset_packed(struct virtqueue *vq);
465
466 int virtqueue_txvq_reset_packed(struct virtqueue *vq);
467
468 static inline int
virtqueue_full(const struct virtqueue * vq)469 virtqueue_full(const struct virtqueue *vq)
470 {
471 return vq->vq_free_cnt == 0;
472 }
473
474 static inline int
virtio_get_queue_type(struct virtio_hw * hw,uint16_t vtpci_queue_idx)475 virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx)
476 {
477 if (vtpci_queue_idx == hw->max_queue_pairs * 2)
478 return VTNET_CQ;
479 else if (vtpci_queue_idx % 2 == 0)
480 return VTNET_RQ;
481 else
482 return VTNET_TQ;
483 }
484
485 /* virtqueue_nused has load-acquire or rte_io_rmb insed */
486 static inline uint16_t
virtqueue_nused(const struct virtqueue * vq)487 virtqueue_nused(const struct virtqueue *vq)
488 {
489 uint16_t idx;
490
491 if (vq->hw->weak_barriers) {
492 /**
493 * x86 prefers to using rte_smp_rmb over __atomic_load_n as it
494 * reports a slightly better perf, which comes from the saved
495 * branch by the compiler.
496 * The if and else branches are identical with the smp and io
497 * barriers both defined as compiler barriers on x86.
498 */
499 #ifdef RTE_ARCH_X86_64
500 idx = vq->vq_split.ring.used->idx;
501 rte_smp_rmb();
502 #else
503 idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx,
504 __ATOMIC_ACQUIRE);
505 #endif
506 } else {
507 idx = vq->vq_split.ring.used->idx;
508 rte_io_rmb();
509 }
510 return idx - vq->vq_used_cons_idx;
511 }
512
513 void vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx);
514 void vq_ring_free_chain_packed(struct virtqueue *vq, uint16_t used_idx);
515 void vq_ring_free_inorder(struct virtqueue *vq, uint16_t desc_idx,
516 uint16_t num);
517
518 static inline void
vq_update_avail_idx(struct virtqueue * vq)519 vq_update_avail_idx(struct virtqueue *vq)
520 {
521 if (vq->hw->weak_barriers) {
522 /* x86 prefers to using rte_smp_wmb over __atomic_store_n as
523 * it reports a slightly better perf, which comes from the
524 * saved branch by the compiler.
525 * The if and else branches are identical with the smp and
526 * io barriers both defined as compiler barriers on x86.
527 */
528 #ifdef RTE_ARCH_X86_64
529 rte_smp_wmb();
530 vq->vq_split.ring.avail->idx = vq->vq_avail_idx;
531 #else
532 __atomic_store_n(&vq->vq_split.ring.avail->idx,
533 vq->vq_avail_idx, __ATOMIC_RELEASE);
534 #endif
535 } else {
536 rte_io_wmb();
537 vq->vq_split.ring.avail->idx = vq->vq_avail_idx;
538 }
539 }
540
541 static inline void
vq_update_avail_ring(struct virtqueue * vq,uint16_t desc_idx)542 vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx)
543 {
544 uint16_t avail_idx;
545 /*
546 * Place the head of the descriptor chain into the next slot and make
547 * it usable to the host. The chain is made available now rather than
548 * deferring to virtqueue_notify() in the hopes that if the host is
549 * currently running on another CPU, we can keep it processing the new
550 * descriptor.
551 */
552 avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1));
553 if (unlikely(vq->vq_split.ring.avail->ring[avail_idx] != desc_idx))
554 vq->vq_split.ring.avail->ring[avail_idx] = desc_idx;
555 vq->vq_avail_idx++;
556 }
557
558 static inline int
virtqueue_kick_prepare(struct virtqueue * vq)559 virtqueue_kick_prepare(struct virtqueue *vq)
560 {
561 /*
562 * Ensure updated avail->idx is visible to vhost before reading
563 * the used->flags.
564 */
565 virtio_mb(vq->hw->weak_barriers);
566 return !(vq->vq_split.ring.used->flags & VRING_USED_F_NO_NOTIFY);
567 }
568
569 static inline int
virtqueue_kick_prepare_packed(struct virtqueue * vq)570 virtqueue_kick_prepare_packed(struct virtqueue *vq)
571 {
572 uint16_t flags;
573
574 /*
575 * Ensure updated data is visible to vhost before reading the flags.
576 */
577 virtio_mb(vq->hw->weak_barriers);
578 flags = vq->vq_packed.ring.device->desc_event_flags;
579
580 return flags != RING_EVENT_FLAGS_DISABLE;
581 }
582
583 /*
584 * virtqueue_kick_prepare*() or the virtio_wmb() should be called
585 * before this function to be sure that all the data is visible to vhost.
586 */
587 static inline void
virtqueue_notify(struct virtqueue * vq)588 virtqueue_notify(struct virtqueue *vq)
589 {
590 VTPCI_OPS(vq->hw)->notify_queue(vq->hw, vq);
591 }
592
593 #ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
594 #define VIRTQUEUE_DUMP(vq) do { \
595 uint16_t used_idx, nused; \
596 used_idx = __atomic_load_n(&(vq)->vq_split.ring.used->idx, \
597 __ATOMIC_RELAXED); \
598 nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \
599 if (vtpci_packed_queue((vq)->hw)) { \
600 PMD_INIT_LOG(DEBUG, \
601 "VQ: - size=%d; free=%d; used_cons_idx=%d; avail_idx=%d;" \
602 " cached_flags=0x%x; used_wrap_counter=%d", \
603 (vq)->vq_nentries, (vq)->vq_free_cnt, (vq)->vq_used_cons_idx, \
604 (vq)->vq_avail_idx, (vq)->vq_packed.cached_flags, \
605 (vq)->vq_packed.used_wrap_counter); \
606 break; \
607 } \
608 PMD_INIT_LOG(DEBUG, \
609 "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \
610 " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \
611 " avail.flags=0x%x; used.flags=0x%x", \
612 (vq)->vq_nentries, (vq)->vq_free_cnt, nused, (vq)->vq_desc_head_idx, \
613 (vq)->vq_split.ring.avail->idx, (vq)->vq_used_cons_idx, \
614 __atomic_load_n(&(vq)->vq_split.ring.used->idx, __ATOMIC_RELAXED), \
615 (vq)->vq_split.ring.avail->flags, (vq)->vq_split.ring.used->flags); \
616 } while (0)
617 #else
618 #define VIRTQUEUE_DUMP(vq) do { } while (0)
619 #endif
620
621 /* avoid write operation when necessary, to lessen cache issues */
622 #define ASSIGN_UNLESS_EQUAL(var, val) do { \
623 typeof(var) *const var_ = &(var); \
624 typeof(val) const val_ = (val); \
625 if (*var_ != val_) \
626 *var_ = val_; \
627 } while (0)
628
629 #define virtqueue_clear_net_hdr(hdr) do { \
630 typeof(hdr) hdr_ = (hdr); \
631 ASSIGN_UNLESS_EQUAL((hdr_)->csum_start, 0); \
632 ASSIGN_UNLESS_EQUAL((hdr_)->csum_offset, 0); \
633 ASSIGN_UNLESS_EQUAL((hdr_)->flags, 0); \
634 ASSIGN_UNLESS_EQUAL((hdr_)->gso_type, 0); \
635 ASSIGN_UNLESS_EQUAL((hdr_)->gso_size, 0); \
636 ASSIGN_UNLESS_EQUAL((hdr_)->hdr_len, 0); \
637 } while (0)
638
639 static inline void
virtqueue_xmit_offload(struct virtio_net_hdr * hdr,struct rte_mbuf * cookie,bool offload)640 virtqueue_xmit_offload(struct virtio_net_hdr *hdr,
641 struct rte_mbuf *cookie,
642 bool offload)
643 {
644 if (offload) {
645 if (cookie->ol_flags & PKT_TX_TCP_SEG)
646 cookie->ol_flags |= PKT_TX_TCP_CKSUM;
647
648 switch (cookie->ol_flags & PKT_TX_L4_MASK) {
649 case PKT_TX_UDP_CKSUM:
650 hdr->csum_start = cookie->l2_len + cookie->l3_len;
651 hdr->csum_offset = offsetof(struct rte_udp_hdr,
652 dgram_cksum);
653 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
654 break;
655
656 case PKT_TX_TCP_CKSUM:
657 hdr->csum_start = cookie->l2_len + cookie->l3_len;
658 hdr->csum_offset = offsetof(struct rte_tcp_hdr, cksum);
659 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
660 break;
661
662 default:
663 ASSIGN_UNLESS_EQUAL(hdr->csum_start, 0);
664 ASSIGN_UNLESS_EQUAL(hdr->csum_offset, 0);
665 ASSIGN_UNLESS_EQUAL(hdr->flags, 0);
666 break;
667 }
668
669 /* TCP Segmentation Offload */
670 if (cookie->ol_flags & PKT_TX_TCP_SEG) {
671 hdr->gso_type = (cookie->ol_flags & PKT_TX_IPV6) ?
672 VIRTIO_NET_HDR_GSO_TCPV6 :
673 VIRTIO_NET_HDR_GSO_TCPV4;
674 hdr->gso_size = cookie->tso_segsz;
675 hdr->hdr_len =
676 cookie->l2_len +
677 cookie->l3_len +
678 cookie->l4_len;
679 } else {
680 ASSIGN_UNLESS_EQUAL(hdr->gso_type, 0);
681 ASSIGN_UNLESS_EQUAL(hdr->gso_size, 0);
682 ASSIGN_UNLESS_EQUAL(hdr->hdr_len, 0);
683 }
684 }
685 }
686
687 static inline void
virtqueue_enqueue_xmit_packed(struct virtnet_tx * txvq,struct rte_mbuf * cookie,uint16_t needed,int use_indirect,int can_push,int in_order)688 virtqueue_enqueue_xmit_packed(struct virtnet_tx *txvq, struct rte_mbuf *cookie,
689 uint16_t needed, int use_indirect, int can_push,
690 int in_order)
691 {
692 struct virtio_tx_region *txr = txvq->virtio_net_hdr_mz->addr;
693 struct vq_desc_extra *dxp;
694 struct virtqueue *vq = txvq->vq;
695 struct vring_packed_desc *start_dp, *head_dp;
696 uint16_t idx, id, head_idx, head_flags;
697 int16_t head_size = vq->hw->vtnet_hdr_size;
698 struct virtio_net_hdr *hdr;
699 uint16_t prev;
700 bool prepend_header = false;
701 uint16_t seg_num = cookie->nb_segs;
702
703 id = in_order ? vq->vq_avail_idx : vq->vq_desc_head_idx;
704
705 dxp = &vq->vq_descx[id];
706 dxp->ndescs = needed;
707 dxp->cookie = cookie;
708
709 head_idx = vq->vq_avail_idx;
710 idx = head_idx;
711 prev = head_idx;
712 start_dp = vq->vq_packed.ring.desc;
713
714 head_dp = &vq->vq_packed.ring.desc[idx];
715 head_flags = cookie->next ? VRING_DESC_F_NEXT : 0;
716 head_flags |= vq->vq_packed.cached_flags;
717
718 if (can_push) {
719 /* prepend cannot fail, checked by caller */
720 hdr = rte_pktmbuf_mtod_offset(cookie, struct virtio_net_hdr *,
721 -head_size);
722 prepend_header = true;
723
724 /* if offload disabled, it is not zeroed below, do it now */
725 if (!vq->hw->has_tx_offload)
726 virtqueue_clear_net_hdr(hdr);
727 } else if (use_indirect) {
728 /* setup tx ring slot to point to indirect
729 * descriptor list stored in reserved region.
730 *
731 * the first slot in indirect ring is already preset
732 * to point to the header in reserved region
733 */
734 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
735 RTE_PTR_DIFF(&txr[idx].tx_packed_indir, txr);
736 start_dp[idx].len = (seg_num + 1) *
737 sizeof(struct vring_packed_desc);
738 /* reset flags for indirect desc */
739 head_flags = VRING_DESC_F_INDIRECT;
740 head_flags |= vq->vq_packed.cached_flags;
741 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
742
743 /* loop below will fill in rest of the indirect elements */
744 start_dp = txr[idx].tx_packed_indir;
745 idx = 1;
746 } else {
747 /* setup first tx ring slot to point to header
748 * stored in reserved region.
749 */
750 start_dp[idx].addr = txvq->virtio_net_hdr_mem +
751 RTE_PTR_DIFF(&txr[idx].tx_hdr, txr);
752 start_dp[idx].len = vq->hw->vtnet_hdr_size;
753 hdr = (struct virtio_net_hdr *)&txr[idx].tx_hdr;
754 idx++;
755 if (idx >= vq->vq_nentries) {
756 idx -= vq->vq_nentries;
757 vq->vq_packed.cached_flags ^=
758 VRING_PACKED_DESC_F_AVAIL_USED;
759 }
760 }
761
762 virtqueue_xmit_offload(hdr, cookie, vq->hw->has_tx_offload);
763
764 do {
765 uint16_t flags;
766
767 start_dp[idx].addr = VIRTIO_MBUF_DATA_DMA_ADDR(cookie, vq);
768 start_dp[idx].len = cookie->data_len;
769 if (prepend_header) {
770 start_dp[idx].addr -= head_size;
771 start_dp[idx].len += head_size;
772 prepend_header = false;
773 }
774
775 if (likely(idx != head_idx)) {
776 flags = cookie->next ? VRING_DESC_F_NEXT : 0;
777 flags |= vq->vq_packed.cached_flags;
778 start_dp[idx].flags = flags;
779 }
780 prev = idx;
781 idx++;
782 if (idx >= vq->vq_nentries) {
783 idx -= vq->vq_nentries;
784 vq->vq_packed.cached_flags ^=
785 VRING_PACKED_DESC_F_AVAIL_USED;
786 }
787 } while ((cookie = cookie->next) != NULL);
788
789 start_dp[prev].id = id;
790
791 if (use_indirect) {
792 idx = head_idx;
793 if (++idx >= vq->vq_nentries) {
794 idx -= vq->vq_nentries;
795 vq->vq_packed.cached_flags ^=
796 VRING_PACKED_DESC_F_AVAIL_USED;
797 }
798 }
799
800 vq->vq_free_cnt = (uint16_t)(vq->vq_free_cnt - needed);
801 vq->vq_avail_idx = idx;
802
803 if (!in_order) {
804 vq->vq_desc_head_idx = dxp->next;
805 if (vq->vq_desc_head_idx == VQ_RING_DESC_CHAIN_END)
806 vq->vq_desc_tail_idx = VQ_RING_DESC_CHAIN_END;
807 }
808
809 virtqueue_store_flags_packed(head_dp, head_flags,
810 vq->hw->weak_barriers);
811 }
812
813 static void
vq_ring_free_id_packed(struct virtqueue * vq,uint16_t id)814 vq_ring_free_id_packed(struct virtqueue *vq, uint16_t id)
815 {
816 struct vq_desc_extra *dxp;
817
818 dxp = &vq->vq_descx[id];
819 vq->vq_free_cnt += dxp->ndescs;
820
821 if (vq->vq_desc_tail_idx == VQ_RING_DESC_CHAIN_END)
822 vq->vq_desc_head_idx = id;
823 else
824 vq->vq_descx[vq->vq_desc_tail_idx].next = id;
825
826 vq->vq_desc_tail_idx = id;
827 dxp->next = VQ_RING_DESC_CHAIN_END;
828 }
829
830 static void
virtio_xmit_cleanup_inorder_packed(struct virtqueue * vq,int num)831 virtio_xmit_cleanup_inorder_packed(struct virtqueue *vq, int num)
832 {
833 uint16_t used_idx, id, curr_id, free_cnt = 0;
834 uint16_t size = vq->vq_nentries;
835 struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
836 struct vq_desc_extra *dxp;
837
838 used_idx = vq->vq_used_cons_idx;
839 /* desc_is_used has a load-acquire or rte_io_rmb inside
840 * and wait for used desc in virtqueue.
841 */
842 while (num > 0 && desc_is_used(&desc[used_idx], vq)) {
843 id = desc[used_idx].id;
844 do {
845 curr_id = used_idx;
846 dxp = &vq->vq_descx[used_idx];
847 used_idx += dxp->ndescs;
848 free_cnt += dxp->ndescs;
849 num -= dxp->ndescs;
850 if (used_idx >= size) {
851 used_idx -= size;
852 vq->vq_packed.used_wrap_counter ^= 1;
853 }
854 if (dxp->cookie != NULL) {
855 rte_pktmbuf_free(dxp->cookie);
856 dxp->cookie = NULL;
857 }
858 } while (curr_id != id);
859 }
860 vq->vq_used_cons_idx = used_idx;
861 vq->vq_free_cnt += free_cnt;
862 }
863
864 static void
virtio_xmit_cleanup_normal_packed(struct virtqueue * vq,int num)865 virtio_xmit_cleanup_normal_packed(struct virtqueue *vq, int num)
866 {
867 uint16_t used_idx, id;
868 uint16_t size = vq->vq_nentries;
869 struct vring_packed_desc *desc = vq->vq_packed.ring.desc;
870 struct vq_desc_extra *dxp;
871
872 used_idx = vq->vq_used_cons_idx;
873 /* desc_is_used has a load-acquire or rte_io_rmb inside
874 * and wait for used desc in virtqueue.
875 */
876 while (num-- && desc_is_used(&desc[used_idx], vq)) {
877 id = desc[used_idx].id;
878 dxp = &vq->vq_descx[id];
879 vq->vq_used_cons_idx += dxp->ndescs;
880 if (vq->vq_used_cons_idx >= size) {
881 vq->vq_used_cons_idx -= size;
882 vq->vq_packed.used_wrap_counter ^= 1;
883 }
884 vq_ring_free_id_packed(vq, id);
885 if (dxp->cookie != NULL) {
886 rte_pktmbuf_free(dxp->cookie);
887 dxp->cookie = NULL;
888 }
889 used_idx = vq->vq_used_cons_idx;
890 }
891 }
892
893 /* Cleanup from completed transmits. */
894 static inline void
virtio_xmit_cleanup_packed(struct virtqueue * vq,int num,int in_order)895 virtio_xmit_cleanup_packed(struct virtqueue *vq, int num, int in_order)
896 {
897 if (in_order)
898 virtio_xmit_cleanup_inorder_packed(vq, num);
899 else
900 virtio_xmit_cleanup_normal_packed(vq, num);
901 }
902
903 static inline void
virtio_xmit_cleanup(struct virtqueue * vq,uint16_t num)904 virtio_xmit_cleanup(struct virtqueue *vq, uint16_t num)
905 {
906 uint16_t i, used_idx, desc_idx;
907 for (i = 0; i < num; i++) {
908 struct vring_used_elem *uep;
909 struct vq_desc_extra *dxp;
910
911 used_idx = (uint16_t)(vq->vq_used_cons_idx &
912 (vq->vq_nentries - 1));
913 uep = &vq->vq_split.ring.used->ring[used_idx];
914
915 desc_idx = (uint16_t)uep->id;
916 dxp = &vq->vq_descx[desc_idx];
917 vq->vq_used_cons_idx++;
918 vq_ring_free_chain(vq, desc_idx);
919
920 if (dxp->cookie != NULL) {
921 rte_pktmbuf_free(dxp->cookie);
922 dxp->cookie = NULL;
923 }
924 }
925 }
926
927 /* Cleanup from completed inorder transmits. */
928 static __rte_always_inline void
virtio_xmit_cleanup_inorder(struct virtqueue * vq,uint16_t num)929 virtio_xmit_cleanup_inorder(struct virtqueue *vq, uint16_t num)
930 {
931 uint16_t i, idx = vq->vq_used_cons_idx;
932 int16_t free_cnt = 0;
933 struct vq_desc_extra *dxp = NULL;
934
935 if (unlikely(num == 0))
936 return;
937
938 for (i = 0; i < num; i++) {
939 dxp = &vq->vq_descx[idx++ & (vq->vq_nentries - 1)];
940 free_cnt += dxp->ndescs;
941 if (dxp->cookie != NULL) {
942 rte_pktmbuf_free(dxp->cookie);
943 dxp->cookie = NULL;
944 }
945 }
946
947 vq->vq_free_cnt += free_cnt;
948 vq->vq_used_cons_idx = idx;
949 }
950 #endif /* _VIRTQUEUE_H_ */
951