1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdbool.h> 7 #include <linux/virtio_net.h> 8 9 #include <rte_mbuf.h> 10 #include <rte_memcpy.h> 11 #include <rte_ether.h> 12 #include <rte_ip.h> 13 #include <rte_vhost.h> 14 #include <rte_tcp.h> 15 #include <rte_udp.h> 16 #include <rte_sctp.h> 17 #include <rte_arp.h> 18 #include <rte_spinlock.h> 19 #include <rte_malloc.h> 20 #include <rte_vhost_async.h> 21 22 #include "iotlb.h" 23 #include "vhost.h" 24 25 #define MAX_BATCH_LEN 256 26 27 #define VHOST_ASYNC_BATCH_THRESHOLD 32 28 29 static __rte_always_inline bool 30 rxvq_is_mergeable(struct virtio_net *dev) 31 { 32 return dev->features & (1ULL << VIRTIO_NET_F_MRG_RXBUF); 33 } 34 35 static __rte_always_inline bool 36 virtio_net_is_inorder(struct virtio_net *dev) 37 { 38 return dev->features & (1ULL << VIRTIO_F_IN_ORDER); 39 } 40 41 static bool 42 is_valid_virt_queue_idx(uint32_t idx, int is_tx, uint32_t nr_vring) 43 { 44 return (is_tx ^ (idx & 1)) == 0 && idx < nr_vring; 45 } 46 47 static inline void 48 do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq) 49 { 50 struct batch_copy_elem *elem = vq->batch_copy_elems; 51 uint16_t count = vq->batch_copy_nb_elems; 52 int i; 53 54 for (i = 0; i < count; i++) { 55 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len); 56 vhost_log_cache_write_iova(dev, vq, elem[i].log_addr, 57 elem[i].len); 58 PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0); 59 } 60 61 vq->batch_copy_nb_elems = 0; 62 } 63 64 static inline void 65 do_data_copy_dequeue(struct vhost_virtqueue *vq) 66 { 67 struct batch_copy_elem *elem = vq->batch_copy_elems; 68 uint16_t count = vq->batch_copy_nb_elems; 69 int i; 70 71 for (i = 0; i < count; i++) 72 rte_memcpy(elem[i].dst, elem[i].src, elem[i].len); 73 74 vq->batch_copy_nb_elems = 0; 75 } 76 77 static __rte_always_inline void 78 do_flush_shadow_used_ring_split(struct virtio_net *dev, 79 struct vhost_virtqueue *vq, 80 uint16_t to, uint16_t from, uint16_t size) 81 { 82 rte_memcpy(&vq->used->ring[to], 83 &vq->shadow_used_split[from], 84 size * sizeof(struct vring_used_elem)); 85 vhost_log_cache_used_vring(dev, vq, 86 offsetof(struct vring_used, ring[to]), 87 size * sizeof(struct vring_used_elem)); 88 } 89 90 static __rte_always_inline void 91 flush_shadow_used_ring_split(struct virtio_net *dev, struct vhost_virtqueue *vq) 92 { 93 uint16_t used_idx = vq->last_used_idx & (vq->size - 1); 94 95 if (used_idx + vq->shadow_used_idx <= vq->size) { 96 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, 97 vq->shadow_used_idx); 98 } else { 99 uint16_t size; 100 101 /* update used ring interval [used_idx, vq->size] */ 102 size = vq->size - used_idx; 103 do_flush_shadow_used_ring_split(dev, vq, used_idx, 0, size); 104 105 /* update the left half used ring interval [0, left_size] */ 106 do_flush_shadow_used_ring_split(dev, vq, 0, size, 107 vq->shadow_used_idx - size); 108 } 109 vq->last_used_idx += vq->shadow_used_idx; 110 111 vhost_log_cache_sync(dev, vq); 112 113 __atomic_add_fetch(&vq->used->idx, vq->shadow_used_idx, 114 __ATOMIC_RELEASE); 115 vq->shadow_used_idx = 0; 116 vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), 117 sizeof(vq->used->idx)); 118 } 119 120 static __rte_always_inline void 121 update_shadow_used_ring_split(struct vhost_virtqueue *vq, 122 uint16_t desc_idx, uint32_t len) 123 { 124 uint16_t i = vq->shadow_used_idx++; 125 126 vq->shadow_used_split[i].id = desc_idx; 127 vq->shadow_used_split[i].len = len; 128 } 129 130 static __rte_always_inline void 131 vhost_flush_enqueue_shadow_packed(struct virtio_net *dev, 132 struct vhost_virtqueue *vq) 133 { 134 int i; 135 uint16_t used_idx = vq->last_used_idx; 136 uint16_t head_idx = vq->last_used_idx; 137 uint16_t head_flags = 0; 138 139 /* Split loop in two to save memory barriers */ 140 for (i = 0; i < vq->shadow_used_idx; i++) { 141 vq->desc_packed[used_idx].id = vq->shadow_used_packed[i].id; 142 vq->desc_packed[used_idx].len = vq->shadow_used_packed[i].len; 143 144 used_idx += vq->shadow_used_packed[i].count; 145 if (used_idx >= vq->size) 146 used_idx -= vq->size; 147 } 148 149 /* The ordering for storing desc flags needs to be enforced. */ 150 rte_atomic_thread_fence(__ATOMIC_RELEASE); 151 152 for (i = 0; i < vq->shadow_used_idx; i++) { 153 uint16_t flags; 154 155 if (vq->shadow_used_packed[i].len) 156 flags = VRING_DESC_F_WRITE; 157 else 158 flags = 0; 159 160 if (vq->used_wrap_counter) { 161 flags |= VRING_DESC_F_USED; 162 flags |= VRING_DESC_F_AVAIL; 163 } else { 164 flags &= ~VRING_DESC_F_USED; 165 flags &= ~VRING_DESC_F_AVAIL; 166 } 167 168 if (i > 0) { 169 vq->desc_packed[vq->last_used_idx].flags = flags; 170 171 vhost_log_cache_used_vring(dev, vq, 172 vq->last_used_idx * 173 sizeof(struct vring_packed_desc), 174 sizeof(struct vring_packed_desc)); 175 } else { 176 head_idx = vq->last_used_idx; 177 head_flags = flags; 178 } 179 180 vq_inc_last_used_packed(vq, vq->shadow_used_packed[i].count); 181 } 182 183 vq->desc_packed[head_idx].flags = head_flags; 184 185 vhost_log_cache_used_vring(dev, vq, 186 head_idx * 187 sizeof(struct vring_packed_desc), 188 sizeof(struct vring_packed_desc)); 189 190 vq->shadow_used_idx = 0; 191 vhost_log_cache_sync(dev, vq); 192 } 193 194 static __rte_always_inline void 195 vhost_flush_dequeue_shadow_packed(struct virtio_net *dev, 196 struct vhost_virtqueue *vq) 197 { 198 struct vring_used_elem_packed *used_elem = &vq->shadow_used_packed[0]; 199 200 vq->desc_packed[vq->shadow_last_used_idx].id = used_elem->id; 201 /* desc flags is the synchronization point for virtio packed vring */ 202 __atomic_store_n(&vq->desc_packed[vq->shadow_last_used_idx].flags, 203 used_elem->flags, __ATOMIC_RELEASE); 204 205 vhost_log_cache_used_vring(dev, vq, vq->shadow_last_used_idx * 206 sizeof(struct vring_packed_desc), 207 sizeof(struct vring_packed_desc)); 208 vq->shadow_used_idx = 0; 209 vhost_log_cache_sync(dev, vq); 210 } 211 212 static __rte_always_inline void 213 vhost_flush_enqueue_batch_packed(struct virtio_net *dev, 214 struct vhost_virtqueue *vq, 215 uint64_t *lens, 216 uint16_t *ids) 217 { 218 uint16_t i; 219 uint16_t flags; 220 uint16_t last_used_idx = vq->last_used_idx; 221 struct vring_packed_desc *desc_base = &vq->desc_packed[last_used_idx]; 222 223 if (vq->shadow_used_idx) { 224 do_data_copy_enqueue(dev, vq); 225 vhost_flush_enqueue_shadow_packed(dev, vq); 226 } 227 228 flags = PACKED_DESC_ENQUEUE_USED_FLAG(vq->used_wrap_counter); 229 230 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 231 desc_base[i].id = ids[i]; 232 desc_base[i].len = lens[i]; 233 } 234 235 rte_atomic_thread_fence(__ATOMIC_RELEASE); 236 237 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 238 desc_base[i].flags = flags; 239 } 240 241 vhost_log_cache_used_vring(dev, vq, last_used_idx * 242 sizeof(struct vring_packed_desc), 243 sizeof(struct vring_packed_desc) * 244 PACKED_BATCH_SIZE); 245 vhost_log_cache_sync(dev, vq); 246 247 vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE); 248 } 249 250 static __rte_always_inline void 251 vhost_shadow_dequeue_batch_packed_inorder(struct vhost_virtqueue *vq, 252 uint16_t id) 253 { 254 vq->shadow_used_packed[0].id = id; 255 256 if (!vq->shadow_used_idx) { 257 vq->shadow_last_used_idx = vq->last_used_idx; 258 vq->shadow_used_packed[0].flags = 259 PACKED_DESC_DEQUEUE_USED_FLAG(vq->used_wrap_counter); 260 vq->shadow_used_packed[0].len = 0; 261 vq->shadow_used_packed[0].count = 1; 262 vq->shadow_used_idx++; 263 } 264 265 vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE); 266 } 267 268 static __rte_always_inline void 269 vhost_shadow_dequeue_batch_packed(struct virtio_net *dev, 270 struct vhost_virtqueue *vq, 271 uint16_t *ids) 272 { 273 uint16_t flags; 274 uint16_t i; 275 uint16_t begin; 276 277 flags = PACKED_DESC_DEQUEUE_USED_FLAG(vq->used_wrap_counter); 278 279 if (!vq->shadow_used_idx) { 280 vq->shadow_last_used_idx = vq->last_used_idx; 281 vq->shadow_used_packed[0].id = ids[0]; 282 vq->shadow_used_packed[0].len = 0; 283 vq->shadow_used_packed[0].count = 1; 284 vq->shadow_used_packed[0].flags = flags; 285 vq->shadow_used_idx++; 286 begin = 1; 287 } else 288 begin = 0; 289 290 vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE) { 291 vq->desc_packed[vq->last_used_idx + i].id = ids[i]; 292 vq->desc_packed[vq->last_used_idx + i].len = 0; 293 } 294 295 rte_atomic_thread_fence(__ATOMIC_RELEASE); 296 vhost_for_each_try_unroll(i, begin, PACKED_BATCH_SIZE) 297 vq->desc_packed[vq->last_used_idx + i].flags = flags; 298 299 vhost_log_cache_used_vring(dev, vq, vq->last_used_idx * 300 sizeof(struct vring_packed_desc), 301 sizeof(struct vring_packed_desc) * 302 PACKED_BATCH_SIZE); 303 vhost_log_cache_sync(dev, vq); 304 305 vq_inc_last_used_packed(vq, PACKED_BATCH_SIZE); 306 } 307 308 static __rte_always_inline void 309 vhost_shadow_dequeue_single_packed(struct vhost_virtqueue *vq, 310 uint16_t buf_id, 311 uint16_t count) 312 { 313 uint16_t flags; 314 315 flags = vq->desc_packed[vq->last_used_idx].flags; 316 if (vq->used_wrap_counter) { 317 flags |= VRING_DESC_F_USED; 318 flags |= VRING_DESC_F_AVAIL; 319 } else { 320 flags &= ~VRING_DESC_F_USED; 321 flags &= ~VRING_DESC_F_AVAIL; 322 } 323 324 if (!vq->shadow_used_idx) { 325 vq->shadow_last_used_idx = vq->last_used_idx; 326 327 vq->shadow_used_packed[0].id = buf_id; 328 vq->shadow_used_packed[0].len = 0; 329 vq->shadow_used_packed[0].flags = flags; 330 vq->shadow_used_idx++; 331 } else { 332 vq->desc_packed[vq->last_used_idx].id = buf_id; 333 vq->desc_packed[vq->last_used_idx].len = 0; 334 vq->desc_packed[vq->last_used_idx].flags = flags; 335 } 336 337 vq_inc_last_used_packed(vq, count); 338 } 339 340 static __rte_always_inline void 341 vhost_shadow_dequeue_single_packed_inorder(struct vhost_virtqueue *vq, 342 uint16_t buf_id, 343 uint16_t count) 344 { 345 uint16_t flags; 346 347 vq->shadow_used_packed[0].id = buf_id; 348 349 flags = vq->desc_packed[vq->last_used_idx].flags; 350 if (vq->used_wrap_counter) { 351 flags |= VRING_DESC_F_USED; 352 flags |= VRING_DESC_F_AVAIL; 353 } else { 354 flags &= ~VRING_DESC_F_USED; 355 flags &= ~VRING_DESC_F_AVAIL; 356 } 357 358 if (!vq->shadow_used_idx) { 359 vq->shadow_last_used_idx = vq->last_used_idx; 360 vq->shadow_used_packed[0].len = 0; 361 vq->shadow_used_packed[0].flags = flags; 362 vq->shadow_used_idx++; 363 } 364 365 vq_inc_last_used_packed(vq, count); 366 } 367 368 static __rte_always_inline void 369 vhost_shadow_enqueue_single_packed(struct virtio_net *dev, 370 struct vhost_virtqueue *vq, 371 uint32_t len[], 372 uint16_t id[], 373 uint16_t count[], 374 uint16_t num_buffers) 375 { 376 uint16_t i; 377 for (i = 0; i < num_buffers; i++) { 378 /* enqueue shadow flush action aligned with batch num */ 379 if (!vq->shadow_used_idx) 380 vq->shadow_aligned_idx = vq->last_used_idx & 381 PACKED_BATCH_MASK; 382 vq->shadow_used_packed[vq->shadow_used_idx].id = id[i]; 383 vq->shadow_used_packed[vq->shadow_used_idx].len = len[i]; 384 vq->shadow_used_packed[vq->shadow_used_idx].count = count[i]; 385 vq->shadow_aligned_idx += count[i]; 386 vq->shadow_used_idx++; 387 } 388 389 if (vq->shadow_aligned_idx >= PACKED_BATCH_SIZE) { 390 do_data_copy_enqueue(dev, vq); 391 vhost_flush_enqueue_shadow_packed(dev, vq); 392 } 393 } 394 395 /* avoid write operation when necessary, to lessen cache issues */ 396 #define ASSIGN_UNLESS_EQUAL(var, val) do { \ 397 if ((var) != (val)) \ 398 (var) = (val); \ 399 } while (0) 400 401 static __rte_always_inline void 402 virtio_enqueue_offload(struct rte_mbuf *m_buf, struct virtio_net_hdr *net_hdr) 403 { 404 uint64_t csum_l4 = m_buf->ol_flags & PKT_TX_L4_MASK; 405 406 if (m_buf->ol_flags & PKT_TX_TCP_SEG) 407 csum_l4 |= PKT_TX_TCP_CKSUM; 408 409 if (csum_l4) { 410 net_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 411 net_hdr->csum_start = m_buf->l2_len + m_buf->l3_len; 412 413 switch (csum_l4) { 414 case PKT_TX_TCP_CKSUM: 415 net_hdr->csum_offset = (offsetof(struct rte_tcp_hdr, 416 cksum)); 417 break; 418 case PKT_TX_UDP_CKSUM: 419 net_hdr->csum_offset = (offsetof(struct rte_udp_hdr, 420 dgram_cksum)); 421 break; 422 case PKT_TX_SCTP_CKSUM: 423 net_hdr->csum_offset = (offsetof(struct rte_sctp_hdr, 424 cksum)); 425 break; 426 } 427 } else { 428 ASSIGN_UNLESS_EQUAL(net_hdr->csum_start, 0); 429 ASSIGN_UNLESS_EQUAL(net_hdr->csum_offset, 0); 430 ASSIGN_UNLESS_EQUAL(net_hdr->flags, 0); 431 } 432 433 /* IP cksum verification cannot be bypassed, then calculate here */ 434 if (m_buf->ol_flags & PKT_TX_IP_CKSUM) { 435 struct rte_ipv4_hdr *ipv4_hdr; 436 437 ipv4_hdr = rte_pktmbuf_mtod_offset(m_buf, struct rte_ipv4_hdr *, 438 m_buf->l2_len); 439 ipv4_hdr->hdr_checksum = 0; 440 ipv4_hdr->hdr_checksum = rte_ipv4_cksum(ipv4_hdr); 441 } 442 443 if (m_buf->ol_flags & PKT_TX_TCP_SEG) { 444 if (m_buf->ol_flags & PKT_TX_IPV4) 445 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 446 else 447 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 448 net_hdr->gso_size = m_buf->tso_segsz; 449 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len 450 + m_buf->l4_len; 451 } else if (m_buf->ol_flags & PKT_TX_UDP_SEG) { 452 net_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP; 453 net_hdr->gso_size = m_buf->tso_segsz; 454 net_hdr->hdr_len = m_buf->l2_len + m_buf->l3_len + 455 m_buf->l4_len; 456 } else { 457 ASSIGN_UNLESS_EQUAL(net_hdr->gso_type, 0); 458 ASSIGN_UNLESS_EQUAL(net_hdr->gso_size, 0); 459 ASSIGN_UNLESS_EQUAL(net_hdr->hdr_len, 0); 460 } 461 } 462 463 static __rte_always_inline int 464 map_one_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, 465 struct buf_vector *buf_vec, uint16_t *vec_idx, 466 uint64_t desc_iova, uint64_t desc_len, uint8_t perm) 467 { 468 uint16_t vec_id = *vec_idx; 469 470 while (desc_len) { 471 uint64_t desc_addr; 472 uint64_t desc_chunck_len = desc_len; 473 474 if (unlikely(vec_id >= BUF_VECTOR_MAX)) 475 return -1; 476 477 desc_addr = vhost_iova_to_vva(dev, vq, 478 desc_iova, 479 &desc_chunck_len, 480 perm); 481 if (unlikely(!desc_addr)) 482 return -1; 483 484 rte_prefetch0((void *)(uintptr_t)desc_addr); 485 486 buf_vec[vec_id].buf_iova = desc_iova; 487 buf_vec[vec_id].buf_addr = desc_addr; 488 buf_vec[vec_id].buf_len = desc_chunck_len; 489 490 desc_len -= desc_chunck_len; 491 desc_iova += desc_chunck_len; 492 vec_id++; 493 } 494 *vec_idx = vec_id; 495 496 return 0; 497 } 498 499 static __rte_always_inline int 500 fill_vec_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq, 501 uint32_t avail_idx, uint16_t *vec_idx, 502 struct buf_vector *buf_vec, uint16_t *desc_chain_head, 503 uint32_t *desc_chain_len, uint8_t perm) 504 { 505 uint16_t idx = vq->avail->ring[avail_idx & (vq->size - 1)]; 506 uint16_t vec_id = *vec_idx; 507 uint32_t len = 0; 508 uint64_t dlen; 509 uint32_t nr_descs = vq->size; 510 uint32_t cnt = 0; 511 struct vring_desc *descs = vq->desc; 512 struct vring_desc *idesc = NULL; 513 514 if (unlikely(idx >= vq->size)) 515 return -1; 516 517 *desc_chain_head = idx; 518 519 if (vq->desc[idx].flags & VRING_DESC_F_INDIRECT) { 520 dlen = vq->desc[idx].len; 521 nr_descs = dlen / sizeof(struct vring_desc); 522 if (unlikely(nr_descs > vq->size)) 523 return -1; 524 525 descs = (struct vring_desc *)(uintptr_t) 526 vhost_iova_to_vva(dev, vq, vq->desc[idx].addr, 527 &dlen, 528 VHOST_ACCESS_RO); 529 if (unlikely(!descs)) 530 return -1; 531 532 if (unlikely(dlen < vq->desc[idx].len)) { 533 /* 534 * The indirect desc table is not contiguous 535 * in process VA space, we have to copy it. 536 */ 537 idesc = vhost_alloc_copy_ind_table(dev, vq, 538 vq->desc[idx].addr, vq->desc[idx].len); 539 if (unlikely(!idesc)) 540 return -1; 541 542 descs = idesc; 543 } 544 545 idx = 0; 546 } 547 548 while (1) { 549 if (unlikely(idx >= nr_descs || cnt++ >= nr_descs)) { 550 free_ind_table(idesc); 551 return -1; 552 } 553 554 dlen = descs[idx].len; 555 len += dlen; 556 557 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id, 558 descs[idx].addr, dlen, 559 perm))) { 560 free_ind_table(idesc); 561 return -1; 562 } 563 564 if ((descs[idx].flags & VRING_DESC_F_NEXT) == 0) 565 break; 566 567 idx = descs[idx].next; 568 } 569 570 *desc_chain_len = len; 571 *vec_idx = vec_id; 572 573 if (unlikely(!!idesc)) 574 free_ind_table(idesc); 575 576 return 0; 577 } 578 579 /* 580 * Returns -1 on fail, 0 on success 581 */ 582 static inline int 583 reserve_avail_buf_split(struct virtio_net *dev, struct vhost_virtqueue *vq, 584 uint32_t size, struct buf_vector *buf_vec, 585 uint16_t *num_buffers, uint16_t avail_head, 586 uint16_t *nr_vec) 587 { 588 uint16_t cur_idx; 589 uint16_t vec_idx = 0; 590 uint16_t max_tries, tries = 0; 591 592 uint16_t head_idx = 0; 593 uint32_t len = 0; 594 595 *num_buffers = 0; 596 cur_idx = vq->last_avail_idx; 597 598 if (rxvq_is_mergeable(dev)) 599 max_tries = vq->size - 1; 600 else 601 max_tries = 1; 602 603 while (size > 0) { 604 if (unlikely(cur_idx == avail_head)) 605 return -1; 606 /* 607 * if we tried all available ring items, and still 608 * can't get enough buf, it means something abnormal 609 * happened. 610 */ 611 if (unlikely(++tries > max_tries)) 612 return -1; 613 614 if (unlikely(fill_vec_buf_split(dev, vq, cur_idx, 615 &vec_idx, buf_vec, 616 &head_idx, &len, 617 VHOST_ACCESS_RW) < 0)) 618 return -1; 619 len = RTE_MIN(len, size); 620 update_shadow_used_ring_split(vq, head_idx, len); 621 size -= len; 622 623 cur_idx++; 624 *num_buffers += 1; 625 } 626 627 *nr_vec = vec_idx; 628 629 return 0; 630 } 631 632 static __rte_always_inline int 633 fill_vec_buf_packed_indirect(struct virtio_net *dev, 634 struct vhost_virtqueue *vq, 635 struct vring_packed_desc *desc, uint16_t *vec_idx, 636 struct buf_vector *buf_vec, uint32_t *len, uint8_t perm) 637 { 638 uint16_t i; 639 uint32_t nr_descs; 640 uint16_t vec_id = *vec_idx; 641 uint64_t dlen; 642 struct vring_packed_desc *descs, *idescs = NULL; 643 644 dlen = desc->len; 645 descs = (struct vring_packed_desc *)(uintptr_t) 646 vhost_iova_to_vva(dev, vq, desc->addr, &dlen, VHOST_ACCESS_RO); 647 if (unlikely(!descs)) 648 return -1; 649 650 if (unlikely(dlen < desc->len)) { 651 /* 652 * The indirect desc table is not contiguous 653 * in process VA space, we have to copy it. 654 */ 655 idescs = vhost_alloc_copy_ind_table(dev, 656 vq, desc->addr, desc->len); 657 if (unlikely(!idescs)) 658 return -1; 659 660 descs = idescs; 661 } 662 663 nr_descs = desc->len / sizeof(struct vring_packed_desc); 664 if (unlikely(nr_descs >= vq->size)) { 665 free_ind_table(idescs); 666 return -1; 667 } 668 669 for (i = 0; i < nr_descs; i++) { 670 if (unlikely(vec_id >= BUF_VECTOR_MAX)) { 671 free_ind_table(idescs); 672 return -1; 673 } 674 675 dlen = descs[i].len; 676 *len += dlen; 677 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id, 678 descs[i].addr, dlen, 679 perm))) 680 return -1; 681 } 682 *vec_idx = vec_id; 683 684 if (unlikely(!!idescs)) 685 free_ind_table(idescs); 686 687 return 0; 688 } 689 690 static __rte_always_inline int 691 fill_vec_buf_packed(struct virtio_net *dev, struct vhost_virtqueue *vq, 692 uint16_t avail_idx, uint16_t *desc_count, 693 struct buf_vector *buf_vec, uint16_t *vec_idx, 694 uint16_t *buf_id, uint32_t *len, uint8_t perm) 695 { 696 bool wrap_counter = vq->avail_wrap_counter; 697 struct vring_packed_desc *descs = vq->desc_packed; 698 uint16_t vec_id = *vec_idx; 699 uint64_t dlen; 700 701 if (avail_idx < vq->last_avail_idx) 702 wrap_counter ^= 1; 703 704 /* 705 * Perform a load-acquire barrier in desc_is_avail to 706 * enforce the ordering between desc flags and desc 707 * content. 708 */ 709 if (unlikely(!desc_is_avail(&descs[avail_idx], wrap_counter))) 710 return -1; 711 712 *desc_count = 0; 713 *len = 0; 714 715 while (1) { 716 if (unlikely(vec_id >= BUF_VECTOR_MAX)) 717 return -1; 718 719 if (unlikely(*desc_count >= vq->size)) 720 return -1; 721 722 *desc_count += 1; 723 *buf_id = descs[avail_idx].id; 724 725 if (descs[avail_idx].flags & VRING_DESC_F_INDIRECT) { 726 if (unlikely(fill_vec_buf_packed_indirect(dev, vq, 727 &descs[avail_idx], 728 &vec_id, buf_vec, 729 len, perm) < 0)) 730 return -1; 731 } else { 732 dlen = descs[avail_idx].len; 733 *len += dlen; 734 735 if (unlikely(map_one_desc(dev, vq, buf_vec, &vec_id, 736 descs[avail_idx].addr, 737 dlen, 738 perm))) 739 return -1; 740 } 741 742 if ((descs[avail_idx].flags & VRING_DESC_F_NEXT) == 0) 743 break; 744 745 if (++avail_idx >= vq->size) { 746 avail_idx -= vq->size; 747 wrap_counter ^= 1; 748 } 749 } 750 751 *vec_idx = vec_id; 752 753 return 0; 754 } 755 756 static __rte_noinline void 757 copy_vnet_hdr_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, 758 struct buf_vector *buf_vec, 759 struct virtio_net_hdr_mrg_rxbuf *hdr) 760 { 761 uint64_t len; 762 uint64_t remain = dev->vhost_hlen; 763 uint64_t src = (uint64_t)(uintptr_t)hdr, dst; 764 uint64_t iova = buf_vec->buf_iova; 765 766 while (remain) { 767 len = RTE_MIN(remain, 768 buf_vec->buf_len); 769 dst = buf_vec->buf_addr; 770 rte_memcpy((void *)(uintptr_t)dst, 771 (void *)(uintptr_t)src, 772 len); 773 774 PRINT_PACKET(dev, (uintptr_t)dst, 775 (uint32_t)len, 0); 776 vhost_log_cache_write_iova(dev, vq, 777 iova, len); 778 779 remain -= len; 780 iova += len; 781 src += len; 782 buf_vec++; 783 } 784 } 785 786 static __rte_always_inline int 787 copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, 788 struct rte_mbuf *m, struct buf_vector *buf_vec, 789 uint16_t nr_vec, uint16_t num_buffers) 790 { 791 uint32_t vec_idx = 0; 792 uint32_t mbuf_offset, mbuf_avail; 793 uint32_t buf_offset, buf_avail; 794 uint64_t buf_addr, buf_iova, buf_len; 795 uint32_t cpy_len; 796 uint64_t hdr_addr; 797 struct rte_mbuf *hdr_mbuf; 798 struct batch_copy_elem *batch_copy = vq->batch_copy_elems; 799 struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL; 800 int error = 0; 801 802 if (unlikely(m == NULL)) { 803 error = -1; 804 goto out; 805 } 806 807 buf_addr = buf_vec[vec_idx].buf_addr; 808 buf_iova = buf_vec[vec_idx].buf_iova; 809 buf_len = buf_vec[vec_idx].buf_len; 810 811 if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) { 812 error = -1; 813 goto out; 814 } 815 816 hdr_mbuf = m; 817 hdr_addr = buf_addr; 818 if (unlikely(buf_len < dev->vhost_hlen)) { 819 memset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf)); 820 hdr = &tmp_hdr; 821 } else 822 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr; 823 824 VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n", 825 dev->vid, num_buffers); 826 827 if (unlikely(buf_len < dev->vhost_hlen)) { 828 buf_offset = dev->vhost_hlen - buf_len; 829 vec_idx++; 830 buf_addr = buf_vec[vec_idx].buf_addr; 831 buf_iova = buf_vec[vec_idx].buf_iova; 832 buf_len = buf_vec[vec_idx].buf_len; 833 buf_avail = buf_len - buf_offset; 834 } else { 835 buf_offset = dev->vhost_hlen; 836 buf_avail = buf_len - dev->vhost_hlen; 837 } 838 839 mbuf_avail = rte_pktmbuf_data_len(m); 840 mbuf_offset = 0; 841 while (mbuf_avail != 0 || m->next != NULL) { 842 /* done with current buf, get the next one */ 843 if (buf_avail == 0) { 844 vec_idx++; 845 if (unlikely(vec_idx >= nr_vec)) { 846 error = -1; 847 goto out; 848 } 849 850 buf_addr = buf_vec[vec_idx].buf_addr; 851 buf_iova = buf_vec[vec_idx].buf_iova; 852 buf_len = buf_vec[vec_idx].buf_len; 853 854 buf_offset = 0; 855 buf_avail = buf_len; 856 } 857 858 /* done with current mbuf, get the next one */ 859 if (mbuf_avail == 0) { 860 m = m->next; 861 862 mbuf_offset = 0; 863 mbuf_avail = rte_pktmbuf_data_len(m); 864 } 865 866 if (hdr_addr) { 867 virtio_enqueue_offload(hdr_mbuf, &hdr->hdr); 868 if (rxvq_is_mergeable(dev)) 869 ASSIGN_UNLESS_EQUAL(hdr->num_buffers, 870 num_buffers); 871 872 if (unlikely(hdr == &tmp_hdr)) { 873 copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr); 874 } else { 875 PRINT_PACKET(dev, (uintptr_t)hdr_addr, 876 dev->vhost_hlen, 0); 877 vhost_log_cache_write_iova(dev, vq, 878 buf_vec[0].buf_iova, 879 dev->vhost_hlen); 880 } 881 882 hdr_addr = 0; 883 } 884 885 cpy_len = RTE_MIN(buf_avail, mbuf_avail); 886 887 if (likely(cpy_len > MAX_BATCH_LEN || 888 vq->batch_copy_nb_elems >= vq->size)) { 889 rte_memcpy((void *)((uintptr_t)(buf_addr + buf_offset)), 890 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), 891 cpy_len); 892 vhost_log_cache_write_iova(dev, vq, 893 buf_iova + buf_offset, 894 cpy_len); 895 PRINT_PACKET(dev, (uintptr_t)(buf_addr + buf_offset), 896 cpy_len, 0); 897 } else { 898 batch_copy[vq->batch_copy_nb_elems].dst = 899 (void *)((uintptr_t)(buf_addr + buf_offset)); 900 batch_copy[vq->batch_copy_nb_elems].src = 901 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); 902 batch_copy[vq->batch_copy_nb_elems].log_addr = 903 buf_iova + buf_offset; 904 batch_copy[vq->batch_copy_nb_elems].len = cpy_len; 905 vq->batch_copy_nb_elems++; 906 } 907 908 mbuf_avail -= cpy_len; 909 mbuf_offset += cpy_len; 910 buf_avail -= cpy_len; 911 buf_offset += cpy_len; 912 } 913 914 out: 915 916 return error; 917 } 918 919 static __rte_always_inline void 920 async_fill_vec(struct iovec *v, void *base, size_t len) 921 { 922 v->iov_base = base; 923 v->iov_len = len; 924 } 925 926 static __rte_always_inline void 927 async_fill_iter(struct rte_vhost_iov_iter *it, size_t count, 928 struct iovec *vec, unsigned long nr_seg) 929 { 930 it->offset = 0; 931 it->count = count; 932 933 if (count) { 934 it->iov = vec; 935 it->nr_segs = nr_seg; 936 } else { 937 it->iov = 0; 938 it->nr_segs = 0; 939 } 940 } 941 942 static __rte_always_inline void 943 async_fill_desc(struct rte_vhost_async_desc *desc, 944 struct rte_vhost_iov_iter *src, struct rte_vhost_iov_iter *dst) 945 { 946 desc->src = src; 947 desc->dst = dst; 948 } 949 950 static __rte_always_inline int 951 async_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, 952 struct rte_mbuf *m, struct buf_vector *buf_vec, 953 uint16_t nr_vec, uint16_t num_buffers, 954 struct iovec *src_iovec, struct iovec *dst_iovec, 955 struct rte_vhost_iov_iter *src_it, 956 struct rte_vhost_iov_iter *dst_it) 957 { 958 uint32_t vec_idx = 0; 959 uint32_t mbuf_offset, mbuf_avail; 960 uint32_t buf_offset, buf_avail; 961 uint64_t buf_addr, buf_iova, buf_len; 962 uint32_t cpy_len, cpy_threshold; 963 uint64_t hdr_addr; 964 struct rte_mbuf *hdr_mbuf; 965 struct batch_copy_elem *batch_copy = vq->batch_copy_elems; 966 struct virtio_net_hdr_mrg_rxbuf tmp_hdr, *hdr = NULL; 967 int error = 0; 968 uint64_t mapped_len; 969 970 uint32_t tlen = 0; 971 int tvec_idx = 0; 972 void *hpa; 973 974 if (unlikely(m == NULL)) { 975 error = -1; 976 goto out; 977 } 978 979 cpy_threshold = vq->async_threshold; 980 981 buf_addr = buf_vec[vec_idx].buf_addr; 982 buf_iova = buf_vec[vec_idx].buf_iova; 983 buf_len = buf_vec[vec_idx].buf_len; 984 985 if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) { 986 error = -1; 987 goto out; 988 } 989 990 hdr_mbuf = m; 991 hdr_addr = buf_addr; 992 if (unlikely(buf_len < dev->vhost_hlen)) { 993 memset(&tmp_hdr, 0, sizeof(struct virtio_net_hdr_mrg_rxbuf)); 994 hdr = &tmp_hdr; 995 } else 996 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(uintptr_t)hdr_addr; 997 998 VHOST_LOG_DATA(DEBUG, "(%d) RX: num merge buffers %d\n", 999 dev->vid, num_buffers); 1000 1001 if (unlikely(buf_len < dev->vhost_hlen)) { 1002 buf_offset = dev->vhost_hlen - buf_len; 1003 vec_idx++; 1004 buf_addr = buf_vec[vec_idx].buf_addr; 1005 buf_iova = buf_vec[vec_idx].buf_iova; 1006 buf_len = buf_vec[vec_idx].buf_len; 1007 buf_avail = buf_len - buf_offset; 1008 } else { 1009 buf_offset = dev->vhost_hlen; 1010 buf_avail = buf_len - dev->vhost_hlen; 1011 } 1012 1013 mbuf_avail = rte_pktmbuf_data_len(m); 1014 mbuf_offset = 0; 1015 1016 while (mbuf_avail != 0 || m->next != NULL) { 1017 /* done with current buf, get the next one */ 1018 if (buf_avail == 0) { 1019 vec_idx++; 1020 if (unlikely(vec_idx >= nr_vec)) { 1021 error = -1; 1022 goto out; 1023 } 1024 1025 buf_addr = buf_vec[vec_idx].buf_addr; 1026 buf_iova = buf_vec[vec_idx].buf_iova; 1027 buf_len = buf_vec[vec_idx].buf_len; 1028 1029 buf_offset = 0; 1030 buf_avail = buf_len; 1031 } 1032 1033 /* done with current mbuf, get the next one */ 1034 if (mbuf_avail == 0) { 1035 m = m->next; 1036 1037 mbuf_offset = 0; 1038 mbuf_avail = rte_pktmbuf_data_len(m); 1039 } 1040 1041 if (hdr_addr) { 1042 virtio_enqueue_offload(hdr_mbuf, &hdr->hdr); 1043 if (rxvq_is_mergeable(dev)) 1044 ASSIGN_UNLESS_EQUAL(hdr->num_buffers, 1045 num_buffers); 1046 1047 if (unlikely(hdr == &tmp_hdr)) { 1048 copy_vnet_hdr_to_desc(dev, vq, buf_vec, hdr); 1049 } else { 1050 PRINT_PACKET(dev, (uintptr_t)hdr_addr, 1051 dev->vhost_hlen, 0); 1052 vhost_log_cache_write_iova(dev, vq, 1053 buf_vec[0].buf_iova, 1054 dev->vhost_hlen); 1055 } 1056 1057 hdr_addr = 0; 1058 } 1059 1060 cpy_len = RTE_MIN(buf_avail, mbuf_avail); 1061 1062 while (unlikely(cpy_len && cpy_len >= cpy_threshold)) { 1063 hpa = (void *)(uintptr_t)gpa_to_first_hpa(dev, 1064 buf_iova + buf_offset, 1065 cpy_len, &mapped_len); 1066 1067 if (unlikely(!hpa || mapped_len < cpy_threshold)) 1068 break; 1069 1070 async_fill_vec(src_iovec + tvec_idx, 1071 (void *)(uintptr_t)rte_pktmbuf_iova_offset(m, 1072 mbuf_offset), (size_t)mapped_len); 1073 1074 async_fill_vec(dst_iovec + tvec_idx, 1075 hpa, (size_t)mapped_len); 1076 1077 tlen += (uint32_t)mapped_len; 1078 cpy_len -= (uint32_t)mapped_len; 1079 mbuf_avail -= (uint32_t)mapped_len; 1080 mbuf_offset += (uint32_t)mapped_len; 1081 buf_avail -= (uint32_t)mapped_len; 1082 buf_offset += (uint32_t)mapped_len; 1083 tvec_idx++; 1084 } 1085 1086 if (likely(cpy_len)) { 1087 if (unlikely(vq->batch_copy_nb_elems >= vq->size)) { 1088 rte_memcpy( 1089 (void *)((uintptr_t)(buf_addr + buf_offset)), 1090 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), 1091 cpy_len); 1092 1093 PRINT_PACKET(dev, 1094 (uintptr_t)(buf_addr + buf_offset), 1095 cpy_len, 0); 1096 } else { 1097 batch_copy[vq->batch_copy_nb_elems].dst = 1098 (void *)((uintptr_t)(buf_addr + buf_offset)); 1099 batch_copy[vq->batch_copy_nb_elems].src = 1100 rte_pktmbuf_mtod_offset(m, void *, mbuf_offset); 1101 batch_copy[vq->batch_copy_nb_elems].log_addr = 1102 buf_iova + buf_offset; 1103 batch_copy[vq->batch_copy_nb_elems].len = 1104 cpy_len; 1105 vq->batch_copy_nb_elems++; 1106 } 1107 1108 mbuf_avail -= cpy_len; 1109 mbuf_offset += cpy_len; 1110 buf_avail -= cpy_len; 1111 buf_offset += cpy_len; 1112 } 1113 1114 } 1115 1116 out: 1117 if (tlen) { 1118 async_fill_iter(src_it, tlen, src_iovec, tvec_idx); 1119 async_fill_iter(dst_it, tlen, dst_iovec, tvec_idx); 1120 } else { 1121 src_it->count = 0; 1122 } 1123 1124 return error; 1125 } 1126 1127 static __rte_always_inline int 1128 vhost_enqueue_single_packed(struct virtio_net *dev, 1129 struct vhost_virtqueue *vq, 1130 struct rte_mbuf *pkt, 1131 struct buf_vector *buf_vec, 1132 uint16_t *nr_descs) 1133 { 1134 uint16_t nr_vec = 0; 1135 uint16_t avail_idx = vq->last_avail_idx; 1136 uint16_t max_tries, tries = 0; 1137 uint16_t buf_id = 0; 1138 uint32_t len = 0; 1139 uint16_t desc_count; 1140 uint32_t size = pkt->pkt_len + sizeof(struct virtio_net_hdr_mrg_rxbuf); 1141 uint16_t num_buffers = 0; 1142 uint32_t buffer_len[vq->size]; 1143 uint16_t buffer_buf_id[vq->size]; 1144 uint16_t buffer_desc_count[vq->size]; 1145 1146 if (rxvq_is_mergeable(dev)) 1147 max_tries = vq->size - 1; 1148 else 1149 max_tries = 1; 1150 1151 while (size > 0) { 1152 /* 1153 * if we tried all available ring items, and still 1154 * can't get enough buf, it means something abnormal 1155 * happened. 1156 */ 1157 if (unlikely(++tries > max_tries)) 1158 return -1; 1159 1160 if (unlikely(fill_vec_buf_packed(dev, vq, 1161 avail_idx, &desc_count, 1162 buf_vec, &nr_vec, 1163 &buf_id, &len, 1164 VHOST_ACCESS_RW) < 0)) 1165 return -1; 1166 1167 len = RTE_MIN(len, size); 1168 size -= len; 1169 1170 buffer_len[num_buffers] = len; 1171 buffer_buf_id[num_buffers] = buf_id; 1172 buffer_desc_count[num_buffers] = desc_count; 1173 num_buffers += 1; 1174 1175 *nr_descs += desc_count; 1176 avail_idx += desc_count; 1177 if (avail_idx >= vq->size) 1178 avail_idx -= vq->size; 1179 } 1180 1181 if (copy_mbuf_to_desc(dev, vq, pkt, buf_vec, nr_vec, num_buffers) < 0) 1182 return -1; 1183 1184 vhost_shadow_enqueue_single_packed(dev, vq, buffer_len, buffer_buf_id, 1185 buffer_desc_count, num_buffers); 1186 1187 return 0; 1188 } 1189 1190 static __rte_noinline uint32_t 1191 virtio_dev_rx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, 1192 struct rte_mbuf **pkts, uint32_t count) 1193 { 1194 uint32_t pkt_idx = 0; 1195 uint16_t num_buffers; 1196 struct buf_vector buf_vec[BUF_VECTOR_MAX]; 1197 uint16_t avail_head; 1198 1199 /* 1200 * The ordering between avail index and 1201 * desc reads needs to be enforced. 1202 */ 1203 avail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE); 1204 1205 rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); 1206 1207 for (pkt_idx = 0; pkt_idx < count; pkt_idx++) { 1208 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; 1209 uint16_t nr_vec = 0; 1210 1211 if (unlikely(reserve_avail_buf_split(dev, vq, 1212 pkt_len, buf_vec, &num_buffers, 1213 avail_head, &nr_vec) < 0)) { 1214 VHOST_LOG_DATA(DEBUG, 1215 "(%d) failed to get enough desc from vring\n", 1216 dev->vid); 1217 vq->shadow_used_idx -= num_buffers; 1218 break; 1219 } 1220 1221 VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n", 1222 dev->vid, vq->last_avail_idx, 1223 vq->last_avail_idx + num_buffers); 1224 1225 if (copy_mbuf_to_desc(dev, vq, pkts[pkt_idx], 1226 buf_vec, nr_vec, 1227 num_buffers) < 0) { 1228 vq->shadow_used_idx -= num_buffers; 1229 break; 1230 } 1231 1232 vq->last_avail_idx += num_buffers; 1233 } 1234 1235 do_data_copy_enqueue(dev, vq); 1236 1237 if (likely(vq->shadow_used_idx)) { 1238 flush_shadow_used_ring_split(dev, vq); 1239 vhost_vring_call_split(dev, vq); 1240 } 1241 1242 return pkt_idx; 1243 } 1244 1245 static __rte_always_inline int 1246 virtio_dev_rx_batch_packed(struct virtio_net *dev, 1247 struct vhost_virtqueue *vq, 1248 struct rte_mbuf **pkts) 1249 { 1250 bool wrap_counter = vq->avail_wrap_counter; 1251 struct vring_packed_desc *descs = vq->desc_packed; 1252 uint16_t avail_idx = vq->last_avail_idx; 1253 uint64_t desc_addrs[PACKED_BATCH_SIZE]; 1254 struct virtio_net_hdr_mrg_rxbuf *hdrs[PACKED_BATCH_SIZE]; 1255 uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1256 uint64_t lens[PACKED_BATCH_SIZE]; 1257 uint16_t ids[PACKED_BATCH_SIZE]; 1258 uint16_t i; 1259 1260 if (unlikely(avail_idx & PACKED_BATCH_MASK)) 1261 return -1; 1262 1263 if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size)) 1264 return -1; 1265 1266 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 1267 if (unlikely(pkts[i]->next != NULL)) 1268 return -1; 1269 if (unlikely(!desc_is_avail(&descs[avail_idx + i], 1270 wrap_counter))) 1271 return -1; 1272 } 1273 1274 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 1275 lens[i] = descs[avail_idx + i].len; 1276 1277 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 1278 if (unlikely(pkts[i]->pkt_len > (lens[i] - buf_offset))) 1279 return -1; 1280 } 1281 1282 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 1283 desc_addrs[i] = vhost_iova_to_vva(dev, vq, 1284 descs[avail_idx + i].addr, 1285 &lens[i], 1286 VHOST_ACCESS_RW); 1287 1288 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 1289 if (unlikely(!desc_addrs[i])) 1290 return -1; 1291 if (unlikely(lens[i] != descs[avail_idx + i].len)) 1292 return -1; 1293 } 1294 1295 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 1296 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]); 1297 hdrs[i] = (struct virtio_net_hdr_mrg_rxbuf *) 1298 (uintptr_t)desc_addrs[i]; 1299 lens[i] = pkts[i]->pkt_len + 1300 sizeof(struct virtio_net_hdr_mrg_rxbuf); 1301 } 1302 1303 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 1304 virtio_enqueue_offload(pkts[i], &hdrs[i]->hdr); 1305 1306 vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE); 1307 1308 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 1309 rte_memcpy((void *)(uintptr_t)(desc_addrs[i] + buf_offset), 1310 rte_pktmbuf_mtod_offset(pkts[i], void *, 0), 1311 pkts[i]->pkt_len); 1312 } 1313 1314 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 1315 vhost_log_cache_write_iova(dev, vq, descs[avail_idx + i].addr, 1316 lens[i]); 1317 1318 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 1319 ids[i] = descs[avail_idx + i].id; 1320 1321 vhost_flush_enqueue_batch_packed(dev, vq, lens, ids); 1322 1323 return 0; 1324 } 1325 1326 static __rte_always_inline int16_t 1327 virtio_dev_rx_single_packed(struct virtio_net *dev, 1328 struct vhost_virtqueue *vq, 1329 struct rte_mbuf *pkt) 1330 { 1331 struct buf_vector buf_vec[BUF_VECTOR_MAX]; 1332 uint16_t nr_descs = 0; 1333 1334 if (unlikely(vhost_enqueue_single_packed(dev, vq, pkt, buf_vec, 1335 &nr_descs) < 0)) { 1336 VHOST_LOG_DATA(DEBUG, 1337 "(%d) failed to get enough desc from vring\n", 1338 dev->vid); 1339 return -1; 1340 } 1341 1342 VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n", 1343 dev->vid, vq->last_avail_idx, 1344 vq->last_avail_idx + nr_descs); 1345 1346 vq_inc_last_avail_packed(vq, nr_descs); 1347 1348 return 0; 1349 } 1350 1351 static __rte_noinline uint32_t 1352 virtio_dev_rx_packed(struct virtio_net *dev, 1353 struct vhost_virtqueue *__rte_restrict vq, 1354 struct rte_mbuf **__rte_restrict pkts, 1355 uint32_t count) 1356 { 1357 uint32_t pkt_idx = 0; 1358 1359 do { 1360 rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]); 1361 1362 if (count - pkt_idx >= PACKED_BATCH_SIZE) { 1363 if (!virtio_dev_rx_batch_packed(dev, vq, 1364 &pkts[pkt_idx])) { 1365 pkt_idx += PACKED_BATCH_SIZE; 1366 continue; 1367 } 1368 } 1369 1370 if (virtio_dev_rx_single_packed(dev, vq, pkts[pkt_idx])) 1371 break; 1372 pkt_idx++; 1373 1374 } while (pkt_idx < count); 1375 1376 if (vq->shadow_used_idx) { 1377 do_data_copy_enqueue(dev, vq); 1378 vhost_flush_enqueue_shadow_packed(dev, vq); 1379 } 1380 1381 if (pkt_idx) 1382 vhost_vring_call_packed(dev, vq); 1383 1384 return pkt_idx; 1385 } 1386 1387 static __rte_always_inline uint32_t 1388 virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, 1389 struct rte_mbuf **pkts, uint32_t count) 1390 { 1391 struct vhost_virtqueue *vq; 1392 uint32_t nb_tx = 0; 1393 1394 VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__); 1395 if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) { 1396 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n", 1397 dev->vid, __func__, queue_id); 1398 return 0; 1399 } 1400 1401 vq = dev->virtqueue[queue_id]; 1402 1403 rte_spinlock_lock(&vq->access_lock); 1404 1405 if (unlikely(!vq->enabled)) 1406 goto out_access_unlock; 1407 1408 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 1409 vhost_user_iotlb_rd_lock(vq); 1410 1411 if (unlikely(!vq->access_ok)) 1412 if (unlikely(vring_translate(dev, vq) < 0)) 1413 goto out; 1414 1415 count = RTE_MIN((uint32_t)MAX_PKT_BURST, count); 1416 if (count == 0) 1417 goto out; 1418 1419 if (vq_is_packed(dev)) 1420 nb_tx = virtio_dev_rx_packed(dev, vq, pkts, count); 1421 else 1422 nb_tx = virtio_dev_rx_split(dev, vq, pkts, count); 1423 1424 out: 1425 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 1426 vhost_user_iotlb_rd_unlock(vq); 1427 1428 out_access_unlock: 1429 rte_spinlock_unlock(&vq->access_lock); 1430 1431 return nb_tx; 1432 } 1433 1434 uint16_t 1435 rte_vhost_enqueue_burst(int vid, uint16_t queue_id, 1436 struct rte_mbuf **__rte_restrict pkts, uint16_t count) 1437 { 1438 struct virtio_net *dev = get_device(vid); 1439 1440 if (!dev) 1441 return 0; 1442 1443 if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { 1444 VHOST_LOG_DATA(ERR, 1445 "(%d) %s: built-in vhost net backend is disabled.\n", 1446 dev->vid, __func__); 1447 return 0; 1448 } 1449 1450 return virtio_dev_rx(dev, queue_id, pkts, count); 1451 } 1452 1453 static __rte_always_inline uint16_t 1454 virtio_dev_rx_async_get_info_idx(uint16_t pkts_idx, 1455 uint16_t vq_size, uint16_t n_inflight) 1456 { 1457 return pkts_idx > n_inflight ? (pkts_idx - n_inflight) : 1458 (vq_size - n_inflight + pkts_idx) & (vq_size - 1); 1459 } 1460 1461 static __rte_noinline uint32_t 1462 virtio_dev_rx_async_submit_split(struct virtio_net *dev, 1463 struct vhost_virtqueue *vq, uint16_t queue_id, 1464 struct rte_mbuf **pkts, uint32_t count, 1465 struct rte_mbuf **comp_pkts, uint32_t *comp_count) 1466 { 1467 uint32_t pkt_idx = 0, pkt_burst_idx = 0; 1468 uint16_t num_buffers; 1469 struct buf_vector buf_vec[BUF_VECTOR_MAX]; 1470 uint16_t avail_head; 1471 1472 struct rte_vhost_iov_iter *it_pool = vq->it_pool; 1473 struct iovec *vec_pool = vq->vec_pool; 1474 struct rte_vhost_async_desc tdes[MAX_PKT_BURST]; 1475 struct iovec *src_iovec = vec_pool; 1476 struct iovec *dst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >> 1); 1477 struct rte_vhost_iov_iter *src_it = it_pool; 1478 struct rte_vhost_iov_iter *dst_it = it_pool + 1; 1479 uint16_t slot_idx = 0; 1480 uint16_t segs_await = 0; 1481 struct async_inflight_info *pkts_info = vq->async_pkts_info; 1482 uint32_t n_pkts = 0, pkt_err = 0; 1483 uint32_t num_async_pkts = 0, num_done_pkts = 0; 1484 struct { 1485 uint16_t pkt_idx; 1486 uint16_t last_avail_idx; 1487 } async_pkts_log[MAX_PKT_BURST]; 1488 1489 /* 1490 * The ordering between avail index and desc reads need to be enforced. 1491 */ 1492 avail_head = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE); 1493 1494 rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); 1495 1496 for (pkt_idx = 0; pkt_idx < count; pkt_idx++) { 1497 uint32_t pkt_len = pkts[pkt_idx]->pkt_len + dev->vhost_hlen; 1498 uint16_t nr_vec = 0; 1499 1500 if (unlikely(reserve_avail_buf_split(dev, vq, 1501 pkt_len, buf_vec, &num_buffers, 1502 avail_head, &nr_vec) < 0)) { 1503 VHOST_LOG_DATA(DEBUG, 1504 "(%d) failed to get enough desc from vring\n", 1505 dev->vid); 1506 vq->shadow_used_idx -= num_buffers; 1507 break; 1508 } 1509 1510 VHOST_LOG_DATA(DEBUG, "(%d) current index %d | end index %d\n", 1511 dev->vid, vq->last_avail_idx, 1512 vq->last_avail_idx + num_buffers); 1513 1514 if (async_mbuf_to_desc(dev, vq, pkts[pkt_idx], 1515 buf_vec, nr_vec, num_buffers, 1516 src_iovec, dst_iovec, src_it, dst_it) < 0) { 1517 vq->shadow_used_idx -= num_buffers; 1518 break; 1519 } 1520 1521 slot_idx = (vq->async_pkts_idx + num_async_pkts) & 1522 (vq->size - 1); 1523 if (src_it->count) { 1524 uint16_t from, to; 1525 1526 async_fill_desc(&tdes[pkt_burst_idx++], src_it, dst_it); 1527 pkts_info[slot_idx].descs = num_buffers; 1528 pkts_info[slot_idx].mbuf = pkts[pkt_idx]; 1529 async_pkts_log[num_async_pkts].pkt_idx = pkt_idx; 1530 async_pkts_log[num_async_pkts++].last_avail_idx = 1531 vq->last_avail_idx; 1532 src_iovec += src_it->nr_segs; 1533 dst_iovec += dst_it->nr_segs; 1534 src_it += 2; 1535 dst_it += 2; 1536 segs_await += src_it->nr_segs; 1537 1538 /** 1539 * recover shadow used ring and keep DMA-occupied 1540 * descriptors. 1541 */ 1542 from = vq->shadow_used_idx - num_buffers; 1543 to = vq->async_desc_idx & (vq->size - 1); 1544 if (num_buffers + to <= vq->size) { 1545 rte_memcpy(&vq->async_descs_split[to], 1546 &vq->shadow_used_split[from], 1547 num_buffers * 1548 sizeof(struct vring_used_elem)); 1549 } else { 1550 int size = vq->size - to; 1551 1552 rte_memcpy(&vq->async_descs_split[to], 1553 &vq->shadow_used_split[from], 1554 size * 1555 sizeof(struct vring_used_elem)); 1556 rte_memcpy(vq->async_descs_split, 1557 &vq->shadow_used_split[from + 1558 size], (num_buffers - size) * 1559 sizeof(struct vring_used_elem)); 1560 } 1561 vq->async_desc_idx += num_buffers; 1562 vq->shadow_used_idx -= num_buffers; 1563 } else 1564 comp_pkts[num_done_pkts++] = pkts[pkt_idx]; 1565 1566 vq->last_avail_idx += num_buffers; 1567 1568 /* 1569 * conditions to trigger async device transfer: 1570 * - buffered packet number reaches transfer threshold 1571 * - unused async iov number is less than max vhost vector 1572 */ 1573 if (unlikely(pkt_burst_idx >= VHOST_ASYNC_BATCH_THRESHOLD || 1574 ((VHOST_MAX_ASYNC_VEC >> 1) - segs_await < 1575 BUF_VECTOR_MAX))) { 1576 n_pkts = vq->async_ops.transfer_data(dev->vid, 1577 queue_id, tdes, 0, pkt_burst_idx); 1578 src_iovec = vec_pool; 1579 dst_iovec = vec_pool + (VHOST_MAX_ASYNC_VEC >> 1); 1580 src_it = it_pool; 1581 dst_it = it_pool + 1; 1582 segs_await = 0; 1583 vq->async_pkts_inflight_n += n_pkts; 1584 1585 if (unlikely(n_pkts < pkt_burst_idx)) { 1586 /* 1587 * log error packets number here and do actual 1588 * error processing when applications poll 1589 * completion 1590 */ 1591 pkt_err = pkt_burst_idx - n_pkts; 1592 pkt_burst_idx = 0; 1593 break; 1594 } 1595 1596 pkt_burst_idx = 0; 1597 } 1598 } 1599 1600 if (pkt_burst_idx) { 1601 n_pkts = vq->async_ops.transfer_data(dev->vid, 1602 queue_id, tdes, 0, pkt_burst_idx); 1603 vq->async_pkts_inflight_n += n_pkts; 1604 1605 if (unlikely(n_pkts < pkt_burst_idx)) 1606 pkt_err = pkt_burst_idx - n_pkts; 1607 } 1608 1609 do_data_copy_enqueue(dev, vq); 1610 1611 if (unlikely(pkt_err)) { 1612 uint16_t num_descs = 0; 1613 1614 num_async_pkts -= pkt_err; 1615 /* calculate the sum of descriptors of DMA-error packets. */ 1616 while (pkt_err-- > 0) { 1617 num_descs += pkts_info[slot_idx & (vq->size - 1)].descs; 1618 slot_idx--; 1619 } 1620 vq->async_desc_idx -= num_descs; 1621 /* recover shadow used ring and available ring */ 1622 vq->shadow_used_idx -= (vq->last_avail_idx - 1623 async_pkts_log[num_async_pkts].last_avail_idx - 1624 num_descs); 1625 vq->last_avail_idx = 1626 async_pkts_log[num_async_pkts].last_avail_idx; 1627 pkt_idx = async_pkts_log[num_async_pkts].pkt_idx; 1628 num_done_pkts = pkt_idx - num_async_pkts; 1629 } 1630 1631 vq->async_pkts_idx += num_async_pkts; 1632 *comp_count = num_done_pkts; 1633 1634 if (likely(vq->shadow_used_idx)) { 1635 flush_shadow_used_ring_split(dev, vq); 1636 vhost_vring_call_split(dev, vq); 1637 } 1638 1639 return pkt_idx; 1640 } 1641 1642 uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id, 1643 struct rte_mbuf **pkts, uint16_t count) 1644 { 1645 struct virtio_net *dev = get_device(vid); 1646 struct vhost_virtqueue *vq; 1647 uint16_t n_pkts_cpl = 0, n_pkts_put = 0, n_descs = 0; 1648 uint16_t start_idx, pkts_idx, vq_size; 1649 struct async_inflight_info *pkts_info; 1650 uint16_t from, i; 1651 1652 if (!dev) 1653 return 0; 1654 1655 VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__); 1656 if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) { 1657 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n", 1658 dev->vid, __func__, queue_id); 1659 return 0; 1660 } 1661 1662 vq = dev->virtqueue[queue_id]; 1663 1664 if (unlikely(!vq->async_registered)) { 1665 VHOST_LOG_DATA(ERR, "(%d) %s: async not registered for queue id %d.\n", 1666 dev->vid, __func__, queue_id); 1667 return 0; 1668 } 1669 1670 rte_spinlock_lock(&vq->access_lock); 1671 1672 pkts_idx = vq->async_pkts_idx & (vq->size - 1); 1673 pkts_info = vq->async_pkts_info; 1674 vq_size = vq->size; 1675 start_idx = virtio_dev_rx_async_get_info_idx(pkts_idx, 1676 vq_size, vq->async_pkts_inflight_n); 1677 1678 if (count > vq->async_last_pkts_n) 1679 n_pkts_cpl = vq->async_ops.check_completed_copies(vid, 1680 queue_id, 0, count - vq->async_last_pkts_n); 1681 n_pkts_cpl += vq->async_last_pkts_n; 1682 1683 n_pkts_put = RTE_MIN(count, n_pkts_cpl); 1684 if (unlikely(n_pkts_put == 0)) { 1685 vq->async_last_pkts_n = n_pkts_cpl; 1686 goto done; 1687 } 1688 1689 for (i = 0; i < n_pkts_put; i++) { 1690 from = (start_idx + i) & (vq_size - 1); 1691 n_descs += pkts_info[from].descs; 1692 pkts[i] = pkts_info[from].mbuf; 1693 } 1694 vq->async_last_pkts_n = n_pkts_cpl - n_pkts_put; 1695 vq->async_pkts_inflight_n -= n_pkts_put; 1696 1697 if (likely(vq->enabled && vq->access_ok)) { 1698 uint16_t nr_left = n_descs; 1699 uint16_t nr_copy; 1700 uint16_t to; 1701 1702 /* write back completed descriptors to used ring */ 1703 do { 1704 from = vq->last_async_desc_idx & (vq->size - 1); 1705 nr_copy = nr_left + from <= vq->size ? nr_left : 1706 vq->size - from; 1707 to = vq->last_used_idx & (vq->size - 1); 1708 1709 if (to + nr_copy <= vq->size) { 1710 rte_memcpy(&vq->used->ring[to], 1711 &vq->async_descs_split[from], 1712 nr_copy * 1713 sizeof(struct vring_used_elem)); 1714 } else { 1715 uint16_t size = vq->size - to; 1716 1717 rte_memcpy(&vq->used->ring[to], 1718 &vq->async_descs_split[from], 1719 size * 1720 sizeof(struct vring_used_elem)); 1721 rte_memcpy(vq->used->ring, 1722 &vq->async_descs_split[from + 1723 size], (nr_copy - size) * 1724 sizeof(struct vring_used_elem)); 1725 } 1726 1727 vq->last_async_desc_idx += nr_copy; 1728 vq->last_used_idx += nr_copy; 1729 nr_left -= nr_copy; 1730 } while (nr_left > 0); 1731 1732 __atomic_add_fetch(&vq->used->idx, n_descs, __ATOMIC_RELEASE); 1733 vhost_vring_call_split(dev, vq); 1734 } else 1735 vq->last_async_desc_idx += n_descs; 1736 1737 done: 1738 rte_spinlock_unlock(&vq->access_lock); 1739 1740 return n_pkts_put; 1741 } 1742 1743 static __rte_always_inline uint32_t 1744 virtio_dev_rx_async_submit(struct virtio_net *dev, uint16_t queue_id, 1745 struct rte_mbuf **pkts, uint32_t count, 1746 struct rte_mbuf **comp_pkts, uint32_t *comp_count) 1747 { 1748 struct vhost_virtqueue *vq; 1749 uint32_t nb_tx = 0; 1750 1751 VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__); 1752 if (unlikely(!is_valid_virt_queue_idx(queue_id, 0, dev->nr_vring))) { 1753 VHOST_LOG_DATA(ERR, "(%d) %s: invalid virtqueue idx %d.\n", 1754 dev->vid, __func__, queue_id); 1755 return 0; 1756 } 1757 1758 vq = dev->virtqueue[queue_id]; 1759 1760 rte_spinlock_lock(&vq->access_lock); 1761 1762 if (unlikely(!vq->enabled || !vq->async_registered)) 1763 goto out_access_unlock; 1764 1765 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 1766 vhost_user_iotlb_rd_lock(vq); 1767 1768 if (unlikely(!vq->access_ok)) 1769 if (unlikely(vring_translate(dev, vq) < 0)) 1770 goto out; 1771 1772 count = RTE_MIN((uint32_t)MAX_PKT_BURST, count); 1773 if (count == 0) 1774 goto out; 1775 1776 /* TODO: packed queue not implemented */ 1777 if (vq_is_packed(dev)) 1778 nb_tx = 0; 1779 else 1780 nb_tx = virtio_dev_rx_async_submit_split(dev, 1781 vq, queue_id, pkts, count, comp_pkts, 1782 comp_count); 1783 1784 out: 1785 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 1786 vhost_user_iotlb_rd_unlock(vq); 1787 1788 out_access_unlock: 1789 rte_spinlock_unlock(&vq->access_lock); 1790 1791 return nb_tx; 1792 } 1793 1794 uint16_t 1795 rte_vhost_submit_enqueue_burst(int vid, uint16_t queue_id, 1796 struct rte_mbuf **pkts, uint16_t count, 1797 struct rte_mbuf **comp_pkts, uint32_t *comp_count) 1798 { 1799 struct virtio_net *dev = get_device(vid); 1800 1801 *comp_count = 0; 1802 if (!dev) 1803 return 0; 1804 1805 if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { 1806 VHOST_LOG_DATA(ERR, 1807 "(%d) %s: built-in vhost net backend is disabled.\n", 1808 dev->vid, __func__); 1809 return 0; 1810 } 1811 1812 return virtio_dev_rx_async_submit(dev, queue_id, pkts, count, comp_pkts, 1813 comp_count); 1814 } 1815 1816 static inline bool 1817 virtio_net_with_host_offload(struct virtio_net *dev) 1818 { 1819 if (dev->features & 1820 ((1ULL << VIRTIO_NET_F_CSUM) | 1821 (1ULL << VIRTIO_NET_F_HOST_ECN) | 1822 (1ULL << VIRTIO_NET_F_HOST_TSO4) | 1823 (1ULL << VIRTIO_NET_F_HOST_TSO6) | 1824 (1ULL << VIRTIO_NET_F_HOST_UFO))) 1825 return true; 1826 1827 return false; 1828 } 1829 1830 static void 1831 parse_ethernet(struct rte_mbuf *m, uint16_t *l4_proto, void **l4_hdr) 1832 { 1833 struct rte_ipv4_hdr *ipv4_hdr; 1834 struct rte_ipv6_hdr *ipv6_hdr; 1835 void *l3_hdr = NULL; 1836 struct rte_ether_hdr *eth_hdr; 1837 uint16_t ethertype; 1838 1839 eth_hdr = rte_pktmbuf_mtod(m, struct rte_ether_hdr *); 1840 1841 m->l2_len = sizeof(struct rte_ether_hdr); 1842 ethertype = rte_be_to_cpu_16(eth_hdr->ether_type); 1843 1844 if (ethertype == RTE_ETHER_TYPE_VLAN) { 1845 struct rte_vlan_hdr *vlan_hdr = 1846 (struct rte_vlan_hdr *)(eth_hdr + 1); 1847 1848 m->l2_len += sizeof(struct rte_vlan_hdr); 1849 ethertype = rte_be_to_cpu_16(vlan_hdr->eth_proto); 1850 } 1851 1852 l3_hdr = (char *)eth_hdr + m->l2_len; 1853 1854 switch (ethertype) { 1855 case RTE_ETHER_TYPE_IPV4: 1856 ipv4_hdr = l3_hdr; 1857 *l4_proto = ipv4_hdr->next_proto_id; 1858 m->l3_len = rte_ipv4_hdr_len(ipv4_hdr); 1859 *l4_hdr = (char *)l3_hdr + m->l3_len; 1860 m->ol_flags |= PKT_TX_IPV4; 1861 break; 1862 case RTE_ETHER_TYPE_IPV6: 1863 ipv6_hdr = l3_hdr; 1864 *l4_proto = ipv6_hdr->proto; 1865 m->l3_len = sizeof(struct rte_ipv6_hdr); 1866 *l4_hdr = (char *)l3_hdr + m->l3_len; 1867 m->ol_flags |= PKT_TX_IPV6; 1868 break; 1869 default: 1870 m->l3_len = 0; 1871 *l4_proto = 0; 1872 *l4_hdr = NULL; 1873 break; 1874 } 1875 } 1876 1877 static __rte_always_inline void 1878 vhost_dequeue_offload(struct virtio_net_hdr *hdr, struct rte_mbuf *m) 1879 { 1880 uint16_t l4_proto = 0; 1881 void *l4_hdr = NULL; 1882 struct rte_tcp_hdr *tcp_hdr = NULL; 1883 1884 if (hdr->flags == 0 && hdr->gso_type == VIRTIO_NET_HDR_GSO_NONE) 1885 return; 1886 1887 parse_ethernet(m, &l4_proto, &l4_hdr); 1888 if (hdr->flags == VIRTIO_NET_HDR_F_NEEDS_CSUM) { 1889 if (hdr->csum_start == (m->l2_len + m->l3_len)) { 1890 switch (hdr->csum_offset) { 1891 case (offsetof(struct rte_tcp_hdr, cksum)): 1892 if (l4_proto == IPPROTO_TCP) 1893 m->ol_flags |= PKT_TX_TCP_CKSUM; 1894 break; 1895 case (offsetof(struct rte_udp_hdr, dgram_cksum)): 1896 if (l4_proto == IPPROTO_UDP) 1897 m->ol_flags |= PKT_TX_UDP_CKSUM; 1898 break; 1899 case (offsetof(struct rte_sctp_hdr, cksum)): 1900 if (l4_proto == IPPROTO_SCTP) 1901 m->ol_flags |= PKT_TX_SCTP_CKSUM; 1902 break; 1903 default: 1904 break; 1905 } 1906 } 1907 } 1908 1909 if (l4_hdr && hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 1910 switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 1911 case VIRTIO_NET_HDR_GSO_TCPV4: 1912 case VIRTIO_NET_HDR_GSO_TCPV6: 1913 tcp_hdr = l4_hdr; 1914 m->ol_flags |= PKT_TX_TCP_SEG; 1915 m->tso_segsz = hdr->gso_size; 1916 m->l4_len = (tcp_hdr->data_off & 0xf0) >> 2; 1917 break; 1918 case VIRTIO_NET_HDR_GSO_UDP: 1919 m->ol_flags |= PKT_TX_UDP_SEG; 1920 m->tso_segsz = hdr->gso_size; 1921 m->l4_len = sizeof(struct rte_udp_hdr); 1922 break; 1923 default: 1924 VHOST_LOG_DATA(WARNING, 1925 "unsupported gso type %u.\n", hdr->gso_type); 1926 break; 1927 } 1928 } 1929 } 1930 1931 static __rte_noinline void 1932 copy_vnet_hdr_from_desc(struct virtio_net_hdr *hdr, 1933 struct buf_vector *buf_vec) 1934 { 1935 uint64_t len; 1936 uint64_t remain = sizeof(struct virtio_net_hdr); 1937 uint64_t src; 1938 uint64_t dst = (uint64_t)(uintptr_t)hdr; 1939 1940 while (remain) { 1941 len = RTE_MIN(remain, buf_vec->buf_len); 1942 src = buf_vec->buf_addr; 1943 rte_memcpy((void *)(uintptr_t)dst, 1944 (void *)(uintptr_t)src, len); 1945 1946 remain -= len; 1947 dst += len; 1948 buf_vec++; 1949 } 1950 } 1951 1952 static __rte_always_inline int 1953 copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, 1954 struct buf_vector *buf_vec, uint16_t nr_vec, 1955 struct rte_mbuf *m, struct rte_mempool *mbuf_pool) 1956 { 1957 uint32_t buf_avail, buf_offset; 1958 uint64_t buf_addr, buf_len; 1959 uint32_t mbuf_avail, mbuf_offset; 1960 uint32_t cpy_len; 1961 struct rte_mbuf *cur = m, *prev = m; 1962 struct virtio_net_hdr tmp_hdr; 1963 struct virtio_net_hdr *hdr = NULL; 1964 /* A counter to avoid desc dead loop chain */ 1965 uint16_t vec_idx = 0; 1966 struct batch_copy_elem *batch_copy = vq->batch_copy_elems; 1967 int error = 0; 1968 1969 buf_addr = buf_vec[vec_idx].buf_addr; 1970 buf_len = buf_vec[vec_idx].buf_len; 1971 1972 if (unlikely(buf_len < dev->vhost_hlen && nr_vec <= 1)) { 1973 error = -1; 1974 goto out; 1975 } 1976 1977 if (virtio_net_with_host_offload(dev)) { 1978 if (unlikely(buf_len < sizeof(struct virtio_net_hdr))) { 1979 /* 1980 * No luck, the virtio-net header doesn't fit 1981 * in a contiguous virtual area. 1982 */ 1983 copy_vnet_hdr_from_desc(&tmp_hdr, buf_vec); 1984 hdr = &tmp_hdr; 1985 } else { 1986 hdr = (struct virtio_net_hdr *)((uintptr_t)buf_addr); 1987 } 1988 } 1989 1990 /* 1991 * A virtio driver normally uses at least 2 desc buffers 1992 * for Tx: the first for storing the header, and others 1993 * for storing the data. 1994 */ 1995 if (unlikely(buf_len < dev->vhost_hlen)) { 1996 buf_offset = dev->vhost_hlen - buf_len; 1997 vec_idx++; 1998 buf_addr = buf_vec[vec_idx].buf_addr; 1999 buf_len = buf_vec[vec_idx].buf_len; 2000 buf_avail = buf_len - buf_offset; 2001 } else if (buf_len == dev->vhost_hlen) { 2002 if (unlikely(++vec_idx >= nr_vec)) 2003 goto out; 2004 buf_addr = buf_vec[vec_idx].buf_addr; 2005 buf_len = buf_vec[vec_idx].buf_len; 2006 2007 buf_offset = 0; 2008 buf_avail = buf_len; 2009 } else { 2010 buf_offset = dev->vhost_hlen; 2011 buf_avail = buf_vec[vec_idx].buf_len - dev->vhost_hlen; 2012 } 2013 2014 PRINT_PACKET(dev, 2015 (uintptr_t)(buf_addr + buf_offset), 2016 (uint32_t)buf_avail, 0); 2017 2018 mbuf_offset = 0; 2019 mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM; 2020 while (1) { 2021 cpy_len = RTE_MIN(buf_avail, mbuf_avail); 2022 2023 if (likely(cpy_len > MAX_BATCH_LEN || 2024 vq->batch_copy_nb_elems >= vq->size || 2025 (hdr && cur == m))) { 2026 rte_memcpy(rte_pktmbuf_mtod_offset(cur, void *, 2027 mbuf_offset), 2028 (void *)((uintptr_t)(buf_addr + 2029 buf_offset)), cpy_len); 2030 } else { 2031 batch_copy[vq->batch_copy_nb_elems].dst = 2032 rte_pktmbuf_mtod_offset(cur, void *, 2033 mbuf_offset); 2034 batch_copy[vq->batch_copy_nb_elems].src = 2035 (void *)((uintptr_t)(buf_addr + buf_offset)); 2036 batch_copy[vq->batch_copy_nb_elems].len = cpy_len; 2037 vq->batch_copy_nb_elems++; 2038 } 2039 2040 mbuf_avail -= cpy_len; 2041 mbuf_offset += cpy_len; 2042 buf_avail -= cpy_len; 2043 buf_offset += cpy_len; 2044 2045 /* This buf reaches to its end, get the next one */ 2046 if (buf_avail == 0) { 2047 if (++vec_idx >= nr_vec) 2048 break; 2049 2050 buf_addr = buf_vec[vec_idx].buf_addr; 2051 buf_len = buf_vec[vec_idx].buf_len; 2052 2053 buf_offset = 0; 2054 buf_avail = buf_len; 2055 2056 PRINT_PACKET(dev, (uintptr_t)buf_addr, 2057 (uint32_t)buf_avail, 0); 2058 } 2059 2060 /* 2061 * This mbuf reaches to its end, get a new one 2062 * to hold more data. 2063 */ 2064 if (mbuf_avail == 0) { 2065 cur = rte_pktmbuf_alloc(mbuf_pool); 2066 if (unlikely(cur == NULL)) { 2067 VHOST_LOG_DATA(ERR, "Failed to " 2068 "allocate memory for mbuf.\n"); 2069 error = -1; 2070 goto out; 2071 } 2072 2073 prev->next = cur; 2074 prev->data_len = mbuf_offset; 2075 m->nb_segs += 1; 2076 m->pkt_len += mbuf_offset; 2077 prev = cur; 2078 2079 mbuf_offset = 0; 2080 mbuf_avail = cur->buf_len - RTE_PKTMBUF_HEADROOM; 2081 } 2082 } 2083 2084 prev->data_len = mbuf_offset; 2085 m->pkt_len += mbuf_offset; 2086 2087 if (hdr) 2088 vhost_dequeue_offload(hdr, m); 2089 2090 out: 2091 2092 return error; 2093 } 2094 2095 static void 2096 virtio_dev_extbuf_free(void *addr __rte_unused, void *opaque) 2097 { 2098 rte_free(opaque); 2099 } 2100 2101 static int 2102 virtio_dev_extbuf_alloc(struct rte_mbuf *pkt, uint32_t size) 2103 { 2104 struct rte_mbuf_ext_shared_info *shinfo = NULL; 2105 uint32_t total_len = RTE_PKTMBUF_HEADROOM + size; 2106 uint16_t buf_len; 2107 rte_iova_t iova; 2108 void *buf; 2109 2110 total_len += sizeof(*shinfo) + sizeof(uintptr_t); 2111 total_len = RTE_ALIGN_CEIL(total_len, sizeof(uintptr_t)); 2112 2113 if (unlikely(total_len > UINT16_MAX)) 2114 return -ENOSPC; 2115 2116 buf_len = total_len; 2117 buf = rte_malloc(NULL, buf_len, RTE_CACHE_LINE_SIZE); 2118 if (unlikely(buf == NULL)) 2119 return -ENOMEM; 2120 2121 /* Initialize shinfo */ 2122 shinfo = rte_pktmbuf_ext_shinfo_init_helper(buf, &buf_len, 2123 virtio_dev_extbuf_free, buf); 2124 if (unlikely(shinfo == NULL)) { 2125 rte_free(buf); 2126 VHOST_LOG_DATA(ERR, "Failed to init shinfo\n"); 2127 return -1; 2128 } 2129 2130 iova = rte_malloc_virt2iova(buf); 2131 rte_pktmbuf_attach_extbuf(pkt, buf, iova, buf_len, shinfo); 2132 rte_pktmbuf_reset_headroom(pkt); 2133 2134 return 0; 2135 } 2136 2137 static __rte_always_inline int 2138 virtio_dev_pktmbuf_prep(struct virtio_net *dev, struct rte_mbuf *pkt, 2139 uint32_t data_len) 2140 { 2141 if (rte_pktmbuf_tailroom(pkt) >= data_len) 2142 return 0; 2143 2144 /* attach an external buffer if supported */ 2145 if (dev->extbuf && !virtio_dev_extbuf_alloc(pkt, data_len)) 2146 return 0; 2147 2148 /* check if chained buffers are allowed */ 2149 if (!dev->linearbuf) 2150 return 0; 2151 2152 return -1; 2153 } 2154 2155 /* 2156 * Allocate a host supported pktmbuf. 2157 */ 2158 static __rte_always_inline struct rte_mbuf * 2159 virtio_dev_pktmbuf_alloc(struct virtio_net *dev, struct rte_mempool *mp, 2160 uint32_t data_len) 2161 { 2162 struct rte_mbuf *pkt = rte_pktmbuf_alloc(mp); 2163 2164 if (unlikely(pkt == NULL)) { 2165 VHOST_LOG_DATA(ERR, 2166 "Failed to allocate memory for mbuf.\n"); 2167 return NULL; 2168 } 2169 2170 if (virtio_dev_pktmbuf_prep(dev, pkt, data_len)) { 2171 /* Data doesn't fit into the buffer and the host supports 2172 * only linear buffers 2173 */ 2174 rte_pktmbuf_free(pkt); 2175 return NULL; 2176 } 2177 2178 return pkt; 2179 } 2180 2181 static __rte_noinline uint16_t 2182 virtio_dev_tx_split(struct virtio_net *dev, struct vhost_virtqueue *vq, 2183 struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) 2184 { 2185 uint16_t i; 2186 uint16_t free_entries; 2187 uint16_t dropped = 0; 2188 static bool allocerr_warned; 2189 2190 /* 2191 * The ordering between avail index and 2192 * desc reads needs to be enforced. 2193 */ 2194 free_entries = __atomic_load_n(&vq->avail->idx, __ATOMIC_ACQUIRE) - 2195 vq->last_avail_idx; 2196 if (free_entries == 0) 2197 return 0; 2198 2199 rte_prefetch0(&vq->avail->ring[vq->last_avail_idx & (vq->size - 1)]); 2200 2201 VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__); 2202 2203 count = RTE_MIN(count, MAX_PKT_BURST); 2204 count = RTE_MIN(count, free_entries); 2205 VHOST_LOG_DATA(DEBUG, "(%d) about to dequeue %u buffers\n", 2206 dev->vid, count); 2207 2208 for (i = 0; i < count; i++) { 2209 struct buf_vector buf_vec[BUF_VECTOR_MAX]; 2210 uint16_t head_idx; 2211 uint32_t buf_len; 2212 uint16_t nr_vec = 0; 2213 int err; 2214 2215 if (unlikely(fill_vec_buf_split(dev, vq, 2216 vq->last_avail_idx + i, 2217 &nr_vec, buf_vec, 2218 &head_idx, &buf_len, 2219 VHOST_ACCESS_RO) < 0)) 2220 break; 2221 2222 update_shadow_used_ring_split(vq, head_idx, 0); 2223 2224 pkts[i] = virtio_dev_pktmbuf_alloc(dev, mbuf_pool, buf_len); 2225 if (unlikely(pkts[i] == NULL)) { 2226 /* 2227 * mbuf allocation fails for jumbo packets when external 2228 * buffer allocation is not allowed and linear buffer 2229 * is required. Drop this packet. 2230 */ 2231 if (!allocerr_warned) { 2232 VHOST_LOG_DATA(ERR, 2233 "Failed mbuf alloc of size %d from %s on %s.\n", 2234 buf_len, mbuf_pool->name, dev->ifname); 2235 allocerr_warned = true; 2236 } 2237 dropped += 1; 2238 i++; 2239 break; 2240 } 2241 2242 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts[i], 2243 mbuf_pool); 2244 if (unlikely(err)) { 2245 rte_pktmbuf_free(pkts[i]); 2246 if (!allocerr_warned) { 2247 VHOST_LOG_DATA(ERR, 2248 "Failed to copy desc to mbuf on %s.\n", 2249 dev->ifname); 2250 allocerr_warned = true; 2251 } 2252 dropped += 1; 2253 i++; 2254 break; 2255 } 2256 } 2257 2258 vq->last_avail_idx += i; 2259 2260 do_data_copy_dequeue(vq); 2261 if (unlikely(i < count)) 2262 vq->shadow_used_idx = i; 2263 if (likely(vq->shadow_used_idx)) { 2264 flush_shadow_used_ring_split(dev, vq); 2265 vhost_vring_call_split(dev, vq); 2266 } 2267 2268 return (i - dropped); 2269 } 2270 2271 static __rte_always_inline int 2272 vhost_reserve_avail_batch_packed(struct virtio_net *dev, 2273 struct vhost_virtqueue *vq, 2274 struct rte_mbuf **pkts, 2275 uint16_t avail_idx, 2276 uintptr_t *desc_addrs, 2277 uint16_t *ids) 2278 { 2279 bool wrap = vq->avail_wrap_counter; 2280 struct vring_packed_desc *descs = vq->desc_packed; 2281 uint64_t lens[PACKED_BATCH_SIZE]; 2282 uint64_t buf_lens[PACKED_BATCH_SIZE]; 2283 uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf); 2284 uint16_t flags, i; 2285 2286 if (unlikely(avail_idx & PACKED_BATCH_MASK)) 2287 return -1; 2288 if (unlikely((avail_idx + PACKED_BATCH_SIZE) > vq->size)) 2289 return -1; 2290 2291 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 2292 flags = descs[avail_idx + i].flags; 2293 if (unlikely((wrap != !!(flags & VRING_DESC_F_AVAIL)) || 2294 (wrap == !!(flags & VRING_DESC_F_USED)) || 2295 (flags & PACKED_DESC_SINGLE_DEQUEUE_FLAG))) 2296 return -1; 2297 } 2298 2299 rte_atomic_thread_fence(__ATOMIC_ACQUIRE); 2300 2301 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 2302 lens[i] = descs[avail_idx + i].len; 2303 2304 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 2305 desc_addrs[i] = vhost_iova_to_vva(dev, vq, 2306 descs[avail_idx + i].addr, 2307 &lens[i], VHOST_ACCESS_RW); 2308 } 2309 2310 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 2311 if (unlikely(!desc_addrs[i])) 2312 return -1; 2313 if (unlikely((lens[i] != descs[avail_idx + i].len))) 2314 return -1; 2315 } 2316 2317 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 2318 if (virtio_dev_pktmbuf_prep(dev, pkts[i], lens[i])) 2319 goto err; 2320 } 2321 2322 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 2323 buf_lens[i] = pkts[i]->buf_len - pkts[i]->data_off; 2324 2325 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 2326 if (unlikely(buf_lens[i] < (lens[i] - buf_offset))) 2327 goto err; 2328 } 2329 2330 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 2331 pkts[i]->pkt_len = lens[i] - buf_offset; 2332 pkts[i]->data_len = pkts[i]->pkt_len; 2333 ids[i] = descs[avail_idx + i].id; 2334 } 2335 2336 return 0; 2337 2338 err: 2339 return -1; 2340 } 2341 2342 static __rte_always_inline int 2343 virtio_dev_tx_batch_packed(struct virtio_net *dev, 2344 struct vhost_virtqueue *vq, 2345 struct rte_mbuf **pkts) 2346 { 2347 uint16_t avail_idx = vq->last_avail_idx; 2348 uint32_t buf_offset = sizeof(struct virtio_net_hdr_mrg_rxbuf); 2349 struct virtio_net_hdr *hdr; 2350 uintptr_t desc_addrs[PACKED_BATCH_SIZE]; 2351 uint16_t ids[PACKED_BATCH_SIZE]; 2352 uint16_t i; 2353 2354 if (vhost_reserve_avail_batch_packed(dev, vq, pkts, avail_idx, 2355 desc_addrs, ids)) 2356 return -1; 2357 2358 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 2359 rte_prefetch0((void *)(uintptr_t)desc_addrs[i]); 2360 2361 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) 2362 rte_memcpy(rte_pktmbuf_mtod_offset(pkts[i], void *, 0), 2363 (void *)(uintptr_t)(desc_addrs[i] + buf_offset), 2364 pkts[i]->pkt_len); 2365 2366 if (virtio_net_with_host_offload(dev)) { 2367 vhost_for_each_try_unroll(i, 0, PACKED_BATCH_SIZE) { 2368 hdr = (struct virtio_net_hdr *)(desc_addrs[i]); 2369 vhost_dequeue_offload(hdr, pkts[i]); 2370 } 2371 } 2372 2373 if (virtio_net_is_inorder(dev)) 2374 vhost_shadow_dequeue_batch_packed_inorder(vq, 2375 ids[PACKED_BATCH_SIZE - 1]); 2376 else 2377 vhost_shadow_dequeue_batch_packed(dev, vq, ids); 2378 2379 vq_inc_last_avail_packed(vq, PACKED_BATCH_SIZE); 2380 2381 return 0; 2382 } 2383 2384 static __rte_always_inline int 2385 vhost_dequeue_single_packed(struct virtio_net *dev, 2386 struct vhost_virtqueue *vq, 2387 struct rte_mempool *mbuf_pool, 2388 struct rte_mbuf *pkts, 2389 uint16_t *buf_id, 2390 uint16_t *desc_count) 2391 { 2392 struct buf_vector buf_vec[BUF_VECTOR_MAX]; 2393 uint32_t buf_len; 2394 uint16_t nr_vec = 0; 2395 int err; 2396 static bool allocerr_warned; 2397 2398 if (unlikely(fill_vec_buf_packed(dev, vq, 2399 vq->last_avail_idx, desc_count, 2400 buf_vec, &nr_vec, 2401 buf_id, &buf_len, 2402 VHOST_ACCESS_RO) < 0)) 2403 return -1; 2404 2405 if (unlikely(virtio_dev_pktmbuf_prep(dev, pkts, buf_len))) { 2406 if (!allocerr_warned) { 2407 VHOST_LOG_DATA(ERR, 2408 "Failed mbuf alloc of size %d from %s on %s.\n", 2409 buf_len, mbuf_pool->name, dev->ifname); 2410 allocerr_warned = true; 2411 } 2412 return -1; 2413 } 2414 2415 err = copy_desc_to_mbuf(dev, vq, buf_vec, nr_vec, pkts, 2416 mbuf_pool); 2417 if (unlikely(err)) { 2418 if (!allocerr_warned) { 2419 VHOST_LOG_DATA(ERR, 2420 "Failed to copy desc to mbuf on %s.\n", 2421 dev->ifname); 2422 allocerr_warned = true; 2423 } 2424 return -1; 2425 } 2426 2427 return 0; 2428 } 2429 2430 static __rte_always_inline int 2431 virtio_dev_tx_single_packed(struct virtio_net *dev, 2432 struct vhost_virtqueue *vq, 2433 struct rte_mempool *mbuf_pool, 2434 struct rte_mbuf *pkts) 2435 { 2436 2437 uint16_t buf_id, desc_count = 0; 2438 int ret; 2439 2440 ret = vhost_dequeue_single_packed(dev, vq, mbuf_pool, pkts, &buf_id, 2441 &desc_count); 2442 2443 if (likely(desc_count > 0)) { 2444 if (virtio_net_is_inorder(dev)) 2445 vhost_shadow_dequeue_single_packed_inorder(vq, buf_id, 2446 desc_count); 2447 else 2448 vhost_shadow_dequeue_single_packed(vq, buf_id, 2449 desc_count); 2450 2451 vq_inc_last_avail_packed(vq, desc_count); 2452 } 2453 2454 return ret; 2455 } 2456 2457 static __rte_noinline uint16_t 2458 virtio_dev_tx_packed(struct virtio_net *dev, 2459 struct vhost_virtqueue *__rte_restrict vq, 2460 struct rte_mempool *mbuf_pool, 2461 struct rte_mbuf **__rte_restrict pkts, 2462 uint32_t count) 2463 { 2464 uint32_t pkt_idx = 0; 2465 2466 if (rte_pktmbuf_alloc_bulk(mbuf_pool, pkts, count)) 2467 return 0; 2468 2469 do { 2470 rte_prefetch0(&vq->desc_packed[vq->last_avail_idx]); 2471 2472 if (count - pkt_idx >= PACKED_BATCH_SIZE) { 2473 if (!virtio_dev_tx_batch_packed(dev, vq, 2474 &pkts[pkt_idx])) { 2475 pkt_idx += PACKED_BATCH_SIZE; 2476 continue; 2477 } 2478 } 2479 2480 if (virtio_dev_tx_single_packed(dev, vq, mbuf_pool, 2481 pkts[pkt_idx])) 2482 break; 2483 pkt_idx++; 2484 } while (pkt_idx < count); 2485 2486 if (pkt_idx != count) 2487 rte_pktmbuf_free_bulk(&pkts[pkt_idx], count - pkt_idx); 2488 2489 if (vq->shadow_used_idx) { 2490 do_data_copy_dequeue(vq); 2491 2492 vhost_flush_dequeue_shadow_packed(dev, vq); 2493 vhost_vring_call_packed(dev, vq); 2494 } 2495 2496 return pkt_idx; 2497 } 2498 2499 uint16_t 2500 rte_vhost_dequeue_burst(int vid, uint16_t queue_id, 2501 struct rte_mempool *mbuf_pool, struct rte_mbuf **pkts, uint16_t count) 2502 { 2503 struct virtio_net *dev; 2504 struct rte_mbuf *rarp_mbuf = NULL; 2505 struct vhost_virtqueue *vq; 2506 int16_t success = 1; 2507 2508 dev = get_device(vid); 2509 if (!dev) 2510 return 0; 2511 2512 if (unlikely(!(dev->flags & VIRTIO_DEV_BUILTIN_VIRTIO_NET))) { 2513 VHOST_LOG_DATA(ERR, 2514 "(%d) %s: built-in vhost net backend is disabled.\n", 2515 dev->vid, __func__); 2516 return 0; 2517 } 2518 2519 if (unlikely(!is_valid_virt_queue_idx(queue_id, 1, dev->nr_vring))) { 2520 VHOST_LOG_DATA(ERR, 2521 "(%d) %s: invalid virtqueue idx %d.\n", 2522 dev->vid, __func__, queue_id); 2523 return 0; 2524 } 2525 2526 vq = dev->virtqueue[queue_id]; 2527 2528 if (unlikely(rte_spinlock_trylock(&vq->access_lock) == 0)) 2529 return 0; 2530 2531 if (unlikely(!vq->enabled)) { 2532 count = 0; 2533 goto out_access_unlock; 2534 } 2535 2536 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 2537 vhost_user_iotlb_rd_lock(vq); 2538 2539 if (unlikely(!vq->access_ok)) 2540 if (unlikely(vring_translate(dev, vq) < 0)) { 2541 count = 0; 2542 goto out; 2543 } 2544 2545 /* 2546 * Construct a RARP broadcast packet, and inject it to the "pkts" 2547 * array, to looks like that guest actually send such packet. 2548 * 2549 * Check user_send_rarp() for more information. 2550 * 2551 * broadcast_rarp shares a cacheline in the virtio_net structure 2552 * with some fields that are accessed during enqueue and 2553 * __atomic_compare_exchange_n causes a write if performed compare 2554 * and exchange. This could result in false sharing between enqueue 2555 * and dequeue. 2556 * 2557 * Prevent unnecessary false sharing by reading broadcast_rarp first 2558 * and only performing compare and exchange if the read indicates it 2559 * is likely to be set. 2560 */ 2561 if (unlikely(__atomic_load_n(&dev->broadcast_rarp, __ATOMIC_ACQUIRE) && 2562 __atomic_compare_exchange_n(&dev->broadcast_rarp, 2563 &success, 0, 0, __ATOMIC_RELEASE, __ATOMIC_RELAXED))) { 2564 2565 rarp_mbuf = rte_net_make_rarp_packet(mbuf_pool, &dev->mac); 2566 if (rarp_mbuf == NULL) { 2567 VHOST_LOG_DATA(ERR, "Failed to make RARP packet.\n"); 2568 count = 0; 2569 goto out; 2570 } 2571 count -= 1; 2572 } 2573 2574 if (vq_is_packed(dev)) 2575 count = virtio_dev_tx_packed(dev, vq, mbuf_pool, pkts, count); 2576 else 2577 count = virtio_dev_tx_split(dev, vq, mbuf_pool, pkts, count); 2578 2579 out: 2580 if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) 2581 vhost_user_iotlb_rd_unlock(vq); 2582 2583 out_access_unlock: 2584 rte_spinlock_unlock(&vq->access_lock); 2585 2586 if (unlikely(rarp_mbuf != NULL)) { 2587 /* 2588 * Inject it to the head of "pkts" array, so that switch's mac 2589 * learning table will get updated first. 2590 */ 2591 memmove(&pkts[1], pkts, count * sizeof(struct rte_mbuf *)); 2592 pkts[0] = rarp_mbuf; 2593 count += 1; 2594 } 2595 2596 return count; 2597 } 2598