1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2018 Intel Corporation 3 */ 4 5 #ifndef _VHOST_NET_CDEV_H_ 6 #define _VHOST_NET_CDEV_H_ 7 #include <stdint.h> 8 #include <stdio.h> 9 #include <stdbool.h> 10 #include <sys/types.h> 11 #include <sys/queue.h> 12 #include <unistd.h> 13 #include <linux/vhost.h> 14 #include <linux/virtio_net.h> 15 #include <sys/socket.h> 16 #include <linux/if.h> 17 18 #include <rte_log.h> 19 #include <rte_ether.h> 20 #include <rte_rwlock.h> 21 22 #include "rte_vhost.h" 23 #include "rte_vdpa.h" 24 25 /* Used to indicate that the device is running on a data core */ 26 #define VIRTIO_DEV_RUNNING 1 27 /* Used to indicate that the device is ready to operate */ 28 #define VIRTIO_DEV_READY 2 29 /* Used to indicate that the built-in vhost net device backend is enabled */ 30 #define VIRTIO_DEV_BUILTIN_VIRTIO_NET 4 31 /* Used to indicate that the device has its own data path and configured */ 32 #define VIRTIO_DEV_VDPA_CONFIGURED 8 33 34 /* Backend value set by guest. */ 35 #define VIRTIO_DEV_STOPPED -1 36 37 #define BUF_VECTOR_MAX 256 38 39 #define VHOST_LOG_CACHE_NR 32 40 41 /** 42 * Structure contains buffer address, length and descriptor index 43 * from vring to do scatter RX. 44 */ 45 struct buf_vector { 46 uint64_t buf_iova; 47 uint64_t buf_addr; 48 uint32_t buf_len; 49 uint32_t desc_idx; 50 }; 51 52 /* 53 * A structure to hold some fields needed in zero copy code path, 54 * mainly for associating an mbuf with the right desc_idx. 55 */ 56 struct zcopy_mbuf { 57 struct rte_mbuf *mbuf; 58 uint32_t desc_idx; 59 uint16_t desc_count; 60 uint16_t in_use; 61 62 TAILQ_ENTRY(zcopy_mbuf) next; 63 }; 64 TAILQ_HEAD(zcopy_mbuf_list, zcopy_mbuf); 65 66 /* 67 * Structure contains the info for each batched memory copy. 68 */ 69 struct batch_copy_elem { 70 void *dst; 71 void *src; 72 uint32_t len; 73 uint64_t log_addr; 74 }; 75 76 /* 77 * Structure that contains the info for batched dirty logging. 78 */ 79 struct log_cache_entry { 80 uint32_t offset; 81 unsigned long val; 82 }; 83 84 struct vring_used_elem_packed { 85 uint16_t id; 86 uint32_t len; 87 uint32_t count; 88 }; 89 90 /** 91 * Structure contains variables relevant to RX/TX virtqueues. 92 */ 93 struct vhost_virtqueue { 94 union { 95 struct vring_desc *desc; 96 struct vring_packed_desc *desc_packed; 97 }; 98 union { 99 struct vring_avail *avail; 100 struct vring_packed_desc_event *driver_event; 101 }; 102 union { 103 struct vring_used *used; 104 struct vring_packed_desc_event *device_event; 105 }; 106 uint32_t size; 107 108 uint16_t last_avail_idx; 109 uint16_t last_used_idx; 110 /* Last used index we notify to front end. */ 111 uint16_t signalled_used; 112 bool signalled_used_valid; 113 #define VIRTIO_INVALID_EVENTFD (-1) 114 #define VIRTIO_UNINITIALIZED_EVENTFD (-2) 115 116 /* Backend value to determine if device should started/stopped */ 117 int backend; 118 int enabled; 119 int access_ok; 120 rte_spinlock_t access_lock; 121 122 /* Used to notify the guest (trigger interrupt) */ 123 int callfd; 124 /* Currently unused as polling mode is enabled */ 125 int kickfd; 126 127 /* Physical address of used ring, for logging */ 128 uint64_t log_guest_addr; 129 130 uint16_t nr_zmbuf; 131 uint16_t zmbuf_size; 132 uint16_t last_zmbuf_idx; 133 struct zcopy_mbuf *zmbufs; 134 struct zcopy_mbuf_list zmbuf_list; 135 136 union { 137 struct vring_used_elem *shadow_used_split; 138 struct vring_used_elem_packed *shadow_used_packed; 139 }; 140 uint16_t shadow_used_idx; 141 struct vhost_vring_addr ring_addrs; 142 143 struct batch_copy_elem *batch_copy_elems; 144 uint16_t batch_copy_nb_elems; 145 bool used_wrap_counter; 146 bool avail_wrap_counter; 147 148 struct log_cache_entry log_cache[VHOST_LOG_CACHE_NR]; 149 uint16_t log_cache_nb_elem; 150 151 rte_rwlock_t iotlb_lock; 152 rte_rwlock_t iotlb_pending_lock; 153 struct rte_mempool *iotlb_pool; 154 TAILQ_HEAD(, vhost_iotlb_entry) iotlb_list; 155 int iotlb_cache_nr; 156 TAILQ_HEAD(, vhost_iotlb_entry) iotlb_pending_list; 157 } __rte_cache_aligned; 158 159 /* Old kernels have no such macros defined */ 160 #ifndef VIRTIO_NET_F_GUEST_ANNOUNCE 161 #define VIRTIO_NET_F_GUEST_ANNOUNCE 21 162 #endif 163 164 #ifndef VIRTIO_NET_F_MQ 165 #define VIRTIO_NET_F_MQ 22 166 #endif 167 168 #define VHOST_MAX_VRING 0x100 169 #define VHOST_MAX_QUEUE_PAIRS 0x80 170 171 #ifndef VIRTIO_NET_F_MTU 172 #define VIRTIO_NET_F_MTU 3 173 #endif 174 175 #ifndef VIRTIO_F_ANY_LAYOUT 176 #define VIRTIO_F_ANY_LAYOUT 27 177 #endif 178 179 /* Declare IOMMU related bits for older kernels */ 180 #ifndef VIRTIO_F_IOMMU_PLATFORM 181 182 #define VIRTIO_F_IOMMU_PLATFORM 33 183 184 struct vhost_iotlb_msg { 185 __u64 iova; 186 __u64 size; 187 __u64 uaddr; 188 #define VHOST_ACCESS_RO 0x1 189 #define VHOST_ACCESS_WO 0x2 190 #define VHOST_ACCESS_RW 0x3 191 __u8 perm; 192 #define VHOST_IOTLB_MISS 1 193 #define VHOST_IOTLB_UPDATE 2 194 #define VHOST_IOTLB_INVALIDATE 3 195 #define VHOST_IOTLB_ACCESS_FAIL 4 196 __u8 type; 197 }; 198 199 #define VHOST_IOTLB_MSG 0x1 200 201 struct vhost_msg { 202 int type; 203 union { 204 struct vhost_iotlb_msg iotlb; 205 __u8 padding[64]; 206 }; 207 }; 208 #endif 209 210 /* 211 * Define virtio 1.0 for older kernels 212 */ 213 #ifndef VIRTIO_F_VERSION_1 214 #define VIRTIO_F_VERSION_1 32 215 #endif 216 217 /* Declare packed ring related bits for older kernels */ 218 #ifndef VIRTIO_F_RING_PACKED 219 220 #define VIRTIO_F_RING_PACKED 34 221 222 struct vring_packed_desc { 223 uint64_t addr; 224 uint32_t len; 225 uint16_t id; 226 uint16_t flags; 227 }; 228 229 struct vring_packed_desc_event { 230 uint16_t off_wrap; 231 uint16_t flags; 232 }; 233 #endif 234 235 /* 236 * Declare below packed ring defines unconditionally 237 * as Kernel header might use different names. 238 */ 239 #define VRING_DESC_F_AVAIL (1ULL << 7) 240 #define VRING_DESC_F_USED (1ULL << 15) 241 242 #define VRING_EVENT_F_ENABLE 0x0 243 #define VRING_EVENT_F_DISABLE 0x1 244 #define VRING_EVENT_F_DESC 0x2 245 246 /* 247 * Available and used descs are in same order 248 */ 249 #ifndef VIRTIO_F_IN_ORDER 250 #define VIRTIO_F_IN_ORDER 35 251 #endif 252 253 /* Features supported by this builtin vhost-user net driver. */ 254 #define VIRTIO_NET_SUPPORTED_FEATURES ((1ULL << VIRTIO_NET_F_MRG_RXBUF) | \ 255 (1ULL << VIRTIO_F_ANY_LAYOUT) | \ 256 (1ULL << VIRTIO_NET_F_CTRL_VQ) | \ 257 (1ULL << VIRTIO_NET_F_CTRL_RX) | \ 258 (1ULL << VIRTIO_NET_F_GUEST_ANNOUNCE) | \ 259 (1ULL << VIRTIO_NET_F_MQ) | \ 260 (1ULL << VIRTIO_F_VERSION_1) | \ 261 (1ULL << VHOST_F_LOG_ALL) | \ 262 (1ULL << VHOST_USER_F_PROTOCOL_FEATURES) | \ 263 (1ULL << VIRTIO_NET_F_GSO) | \ 264 (1ULL << VIRTIO_NET_F_HOST_TSO4) | \ 265 (1ULL << VIRTIO_NET_F_HOST_TSO6) | \ 266 (1ULL << VIRTIO_NET_F_HOST_UFO) | \ 267 (1ULL << VIRTIO_NET_F_HOST_ECN) | \ 268 (1ULL << VIRTIO_NET_F_CSUM) | \ 269 (1ULL << VIRTIO_NET_F_GUEST_CSUM) | \ 270 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ 271 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ 272 (1ULL << VIRTIO_NET_F_GUEST_UFO) | \ 273 (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ 274 (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | \ 275 (1ULL << VIRTIO_RING_F_EVENT_IDX) | \ 276 (1ULL << VIRTIO_NET_F_MTU) | \ 277 (1ULL << VIRTIO_F_IN_ORDER) | \ 278 (1ULL << VIRTIO_F_IOMMU_PLATFORM) | \ 279 (1ULL << VIRTIO_F_RING_PACKED)) 280 281 282 struct guest_page { 283 uint64_t guest_phys_addr; 284 uint64_t host_phys_addr; 285 uint64_t size; 286 }; 287 288 /* The possible results of a message handling function */ 289 enum vh_result { 290 /* Message handling failed */ 291 VH_RESULT_ERR = -1, 292 /* Message handling successful */ 293 VH_RESULT_OK = 0, 294 /* Message handling successful and reply prepared */ 295 VH_RESULT_REPLY = 1, 296 }; 297 298 /** 299 * function prototype for the vhost backend to handler specific vhost user 300 * messages prior to the master message handling 301 * 302 * @param vid 303 * vhost device id 304 * @param msg 305 * Message pointer. 306 * @param skip_master 307 * If the handler requires skipping the master message handling, this variable 308 * shall be written 1, otherwise 0. 309 * @return 310 * VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply, 311 * VH_RESULT_ERR on failure 312 */ 313 typedef enum vh_result (*vhost_msg_pre_handle)(int vid, void *msg, 314 uint32_t *skip_master); 315 316 /** 317 * function prototype for the vhost backend to handler specific vhost user 318 * messages after the master message handling is done 319 * 320 * @param vid 321 * vhost device id 322 * @param msg 323 * Message pointer. 324 * @return 325 * VH_RESULT_OK on success, VH_RESULT_REPLY on success with reply, 326 * VH_RESULT_ERR on failure 327 */ 328 typedef enum vh_result (*vhost_msg_post_handle)(int vid, void *msg); 329 330 /** 331 * pre and post vhost user message handlers 332 */ 333 struct vhost_user_extern_ops { 334 vhost_msg_pre_handle pre_msg_handle; 335 vhost_msg_post_handle post_msg_handle; 336 }; 337 338 /** 339 * Device structure contains all configuration information relating 340 * to the device. 341 */ 342 struct virtio_net { 343 /* Frontend (QEMU) memory and memory region information */ 344 struct rte_vhost_memory *mem; 345 uint64_t features; 346 uint64_t protocol_features; 347 int vid; 348 uint32_t flags; 349 uint16_t vhost_hlen; 350 /* to tell if we need broadcast rarp packet */ 351 rte_atomic16_t broadcast_rarp; 352 uint32_t nr_vring; 353 int dequeue_zero_copy; 354 struct vhost_virtqueue *virtqueue[VHOST_MAX_QUEUE_PAIRS * 2]; 355 #define IF_NAME_SZ (PATH_MAX > IFNAMSIZ ? PATH_MAX : IFNAMSIZ) 356 char ifname[IF_NAME_SZ]; 357 uint64_t log_size; 358 uint64_t log_base; 359 uint64_t log_addr; 360 struct ether_addr mac; 361 uint16_t mtu; 362 363 struct vhost_device_ops const *notify_ops; 364 365 uint32_t nr_guest_pages; 366 uint32_t max_guest_pages; 367 struct guest_page *guest_pages; 368 369 int slave_req_fd; 370 rte_spinlock_t slave_req_lock; 371 372 int postcopy_ufd; 373 int postcopy_listening; 374 375 /* 376 * Device id to identify a specific backend device. 377 * It's set to -1 for the default software implementation. 378 */ 379 int vdpa_dev_id; 380 381 /* private data for virtio device */ 382 void *extern_data; 383 /* pre and post vhost user message handlers for the device */ 384 struct vhost_user_extern_ops extern_ops; 385 } __rte_cache_aligned; 386 387 static __rte_always_inline bool 388 vq_is_packed(struct virtio_net *dev) 389 { 390 return dev->features & (1ull << VIRTIO_F_RING_PACKED); 391 } 392 393 static inline bool 394 desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 395 { 396 uint16_t flags = *((volatile uint16_t *) &desc->flags); 397 398 return wrap_counter == !!(flags & VRING_DESC_F_AVAIL) && 399 wrap_counter != !!(flags & VRING_DESC_F_USED); 400 } 401 402 #define VHOST_LOG_PAGE 4096 403 404 /* 405 * Atomically set a bit in memory. 406 */ 407 static __rte_always_inline void 408 vhost_set_bit(unsigned int nr, volatile uint8_t *addr) 409 { 410 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 411 /* 412 * __sync_ built-ins are deprecated, but __atomic_ ones 413 * are sub-optimized in older GCC versions. 414 */ 415 __sync_fetch_and_or_1(addr, (1U << nr)); 416 #else 417 __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); 418 #endif 419 } 420 421 static __rte_always_inline void 422 vhost_log_page(uint8_t *log_base, uint64_t page) 423 { 424 vhost_set_bit(page % 8, &log_base[page / 8]); 425 } 426 427 static __rte_always_inline void 428 vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) 429 { 430 uint64_t page; 431 432 if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || 433 !dev->log_base || !len)) 434 return; 435 436 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 437 return; 438 439 /* To make sure guest memory updates are committed before logging */ 440 rte_smp_wmb(); 441 442 page = addr / VHOST_LOG_PAGE; 443 while (page * VHOST_LOG_PAGE < addr + len) { 444 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 445 page += 1; 446 } 447 } 448 449 static __rte_always_inline void 450 vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) 451 { 452 unsigned long *log_base; 453 int i; 454 455 if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || 456 !dev->log_base)) 457 return; 458 459 log_base = (unsigned long *)(uintptr_t)dev->log_base; 460 461 /* 462 * It is expected a write memory barrier has been issued 463 * before this function is called. 464 */ 465 466 for (i = 0; i < vq->log_cache_nb_elem; i++) { 467 struct log_cache_entry *elem = vq->log_cache + i; 468 469 #if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) 470 /* 471 * '__sync' builtins are deprecated, but '__atomic' ones 472 * are sub-optimized in older GCC versions. 473 */ 474 __sync_fetch_and_or(log_base + elem->offset, elem->val); 475 #else 476 __atomic_fetch_or(log_base + elem->offset, elem->val, 477 __ATOMIC_RELAXED); 478 #endif 479 } 480 481 rte_smp_wmb(); 482 483 vq->log_cache_nb_elem = 0; 484 } 485 486 static __rte_always_inline void 487 vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, 488 uint64_t page) 489 { 490 uint32_t bit_nr = page % (sizeof(unsigned long) << 3); 491 uint32_t offset = page / (sizeof(unsigned long) << 3); 492 int i; 493 494 for (i = 0; i < vq->log_cache_nb_elem; i++) { 495 struct log_cache_entry *elem = vq->log_cache + i; 496 497 if (elem->offset == offset) { 498 elem->val |= (1UL << bit_nr); 499 return; 500 } 501 } 502 503 if (unlikely(i >= VHOST_LOG_CACHE_NR)) { 504 /* 505 * No more room for a new log cache entry, 506 * so write the dirty log map directly. 507 */ 508 rte_smp_wmb(); 509 vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); 510 511 return; 512 } 513 514 vq->log_cache[i].offset = offset; 515 vq->log_cache[i].val = (1UL << bit_nr); 516 vq->log_cache_nb_elem++; 517 } 518 519 static __rte_always_inline void 520 vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, 521 uint64_t addr, uint64_t len) 522 { 523 uint64_t page; 524 525 if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || 526 !dev->log_base || !len)) 527 return; 528 529 if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) 530 return; 531 532 page = addr / VHOST_LOG_PAGE; 533 while (page * VHOST_LOG_PAGE < addr + len) { 534 vhost_log_cache_page(dev, vq, page); 535 page += 1; 536 } 537 } 538 539 static __rte_always_inline void 540 vhost_log_cache_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, 541 uint64_t offset, uint64_t len) 542 { 543 vhost_log_cache_write(dev, vq, vq->log_guest_addr + offset, len); 544 } 545 546 static __rte_always_inline void 547 vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, 548 uint64_t offset, uint64_t len) 549 { 550 vhost_log_write(dev, vq->log_guest_addr + offset, len); 551 } 552 553 /* Macros for printing using RTE_LOG */ 554 #define RTE_LOGTYPE_VHOST_CONFIG RTE_LOGTYPE_USER1 555 #define RTE_LOGTYPE_VHOST_DATA RTE_LOGTYPE_USER1 556 557 #ifdef RTE_LIBRTE_VHOST_DEBUG 558 #define VHOST_MAX_PRINT_BUFF 6072 559 #define VHOST_LOG_DEBUG(log_type, fmt, args...) \ 560 RTE_LOG(DEBUG, log_type, fmt, ##args) 561 #define PRINT_PACKET(device, addr, size, header) do { \ 562 char *pkt_addr = (char *)(addr); \ 563 unsigned int index; \ 564 char packet[VHOST_MAX_PRINT_BUFF]; \ 565 \ 566 if ((header)) \ 567 snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Header size %d: ", (device->vid), (size)); \ 568 else \ 569 snprintf(packet, VHOST_MAX_PRINT_BUFF, "(%d) Packet size %d: ", (device->vid), (size)); \ 570 for (index = 0; index < (size); index++) { \ 571 snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), \ 572 "%02hhx ", pkt_addr[index]); \ 573 } \ 574 snprintf(packet + strnlen(packet, VHOST_MAX_PRINT_BUFF), VHOST_MAX_PRINT_BUFF - strnlen(packet, VHOST_MAX_PRINT_BUFF), "\n"); \ 575 \ 576 VHOST_LOG_DEBUG(VHOST_DATA, "%s", packet); \ 577 } while (0) 578 #else 579 #define VHOST_LOG_DEBUG(log_type, fmt, args...) do {} while (0) 580 #define PRINT_PACKET(device, addr, size, header) do {} while (0) 581 #endif 582 583 extern uint64_t VHOST_FEATURES; 584 #define MAX_VHOST_DEVICE 1024 585 extern struct virtio_net *vhost_devices[MAX_VHOST_DEVICE]; 586 587 /* Convert guest physical address to host physical address */ 588 static __rte_always_inline rte_iova_t 589 gpa_to_hpa(struct virtio_net *dev, uint64_t gpa, uint64_t size) 590 { 591 uint32_t i; 592 struct guest_page *page; 593 594 for (i = 0; i < dev->nr_guest_pages; i++) { 595 page = &dev->guest_pages[i]; 596 597 if (gpa >= page->guest_phys_addr && 598 gpa + size < page->guest_phys_addr + page->size) { 599 return gpa - page->guest_phys_addr + 600 page->host_phys_addr; 601 } 602 } 603 604 return 0; 605 } 606 607 static __rte_always_inline struct virtio_net * 608 get_device(int vid) 609 { 610 struct virtio_net *dev = vhost_devices[vid]; 611 612 if (unlikely(!dev)) { 613 RTE_LOG(ERR, VHOST_CONFIG, 614 "(%d) device not found.\n", vid); 615 } 616 617 return dev; 618 } 619 620 int vhost_new_device(void); 621 void cleanup_device(struct virtio_net *dev, int destroy); 622 void reset_device(struct virtio_net *dev); 623 void vhost_destroy_device(int); 624 void vhost_destroy_device_notify(struct virtio_net *dev); 625 626 void cleanup_vq(struct vhost_virtqueue *vq, int destroy); 627 void free_vq(struct virtio_net *dev, struct vhost_virtqueue *vq); 628 629 int alloc_vring_queue(struct virtio_net *dev, uint32_t vring_idx); 630 631 void vhost_attach_vdpa_device(int vid, int did); 632 void vhost_detach_vdpa_device(int vid); 633 634 void vhost_set_ifname(int, const char *if_name, unsigned int if_len); 635 void vhost_enable_dequeue_zero_copy(int vid); 636 void vhost_set_builtin_virtio_net(int vid, bool enable); 637 638 struct vhost_device_ops const *vhost_driver_callback_get(const char *path); 639 640 /* 641 * Backend-specific cleanup. 642 * 643 * TODO: fix it; we have one backend now 644 */ 645 void vhost_backend_cleanup(struct virtio_net *dev); 646 647 uint64_t __vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, 648 uint64_t iova, uint64_t *len, uint8_t perm); 649 int vring_translate(struct virtio_net *dev, struct vhost_virtqueue *vq); 650 void vring_invalidate(struct virtio_net *dev, struct vhost_virtqueue *vq); 651 652 static __rte_always_inline uint64_t 653 vhost_iova_to_vva(struct virtio_net *dev, struct vhost_virtqueue *vq, 654 uint64_t iova, uint64_t *len, uint8_t perm) 655 { 656 if (!(dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) 657 return rte_vhost_va_from_guest_pa(dev->mem, iova, len); 658 659 return __vhost_iova_to_vva(dev, vq, iova, len, perm); 660 } 661 662 #define vhost_avail_event(vr) \ 663 (*(volatile uint16_t*)&(vr)->used->ring[(vr)->size]) 664 #define vhost_used_event(vr) \ 665 (*(volatile uint16_t*)&(vr)->avail->ring[(vr)->size]) 666 667 /* 668 * The following is used with VIRTIO_RING_F_EVENT_IDX. 669 * Assuming a given event_idx value from the other size, if we have 670 * just incremented index from old to new_idx, should we trigger an 671 * event? 672 */ 673 static __rte_always_inline int 674 vhost_need_event(uint16_t event_idx, uint16_t new_idx, uint16_t old) 675 { 676 return (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old); 677 } 678 679 static __rte_always_inline void 680 vhost_vring_call_split(struct virtio_net *dev, struct vhost_virtqueue *vq) 681 { 682 /* Flush used->idx update before we read avail->flags. */ 683 rte_smp_mb(); 684 685 /* Don't kick guest if we don't reach index specified by guest. */ 686 if (dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX)) { 687 uint16_t old = vq->signalled_used; 688 uint16_t new = vq->last_used_idx; 689 bool signalled_used_valid = vq->signalled_used_valid; 690 691 vq->signalled_used = new; 692 vq->signalled_used_valid = true; 693 694 VHOST_LOG_DEBUG(VHOST_DATA, "%s: used_event_idx=%d, old=%d, new=%d\n", 695 __func__, 696 vhost_used_event(vq), 697 old, new); 698 699 if ((vhost_need_event(vhost_used_event(vq), new, old) && 700 (vq->callfd >= 0)) || 701 unlikely(!signalled_used_valid)) 702 eventfd_write(vq->callfd, (eventfd_t) 1); 703 } else { 704 /* Kick the guest if necessary. */ 705 if (!(vq->avail->flags & VRING_AVAIL_F_NO_INTERRUPT) 706 && (vq->callfd >= 0)) 707 eventfd_write(vq->callfd, (eventfd_t)1); 708 } 709 } 710 711 static __rte_always_inline void 712 vhost_vring_call_packed(struct virtio_net *dev, struct vhost_virtqueue *vq) 713 { 714 uint16_t old, new, off, off_wrap; 715 bool signalled_used_valid, kick = false; 716 717 /* Flush used desc update. */ 718 rte_smp_mb(); 719 720 if (!(dev->features & (1ULL << VIRTIO_RING_F_EVENT_IDX))) { 721 if (vq->driver_event->flags != 722 VRING_EVENT_F_DISABLE) 723 kick = true; 724 goto kick; 725 } 726 727 old = vq->signalled_used; 728 new = vq->last_used_idx; 729 vq->signalled_used = new; 730 signalled_used_valid = vq->signalled_used_valid; 731 vq->signalled_used_valid = true; 732 733 if (vq->driver_event->flags != VRING_EVENT_F_DESC) { 734 if (vq->driver_event->flags != VRING_EVENT_F_DISABLE) 735 kick = true; 736 goto kick; 737 } 738 739 if (unlikely(!signalled_used_valid)) { 740 kick = true; 741 goto kick; 742 } 743 744 rte_smp_rmb(); 745 746 off_wrap = vq->driver_event->off_wrap; 747 off = off_wrap & ~(1 << 15); 748 749 if (new <= old) 750 old -= vq->size; 751 752 if (vq->used_wrap_counter != off_wrap >> 15) 753 off -= vq->size; 754 755 if (vhost_need_event(off, new, old)) 756 kick = true; 757 kick: 758 if (kick) 759 eventfd_write(vq->callfd, (eventfd_t)1); 760 } 761 762 static __rte_always_inline void 763 restore_mbuf(struct rte_mbuf *m) 764 { 765 uint32_t mbuf_size, priv_size; 766 767 while (m) { 768 priv_size = rte_pktmbuf_priv_size(m->pool); 769 mbuf_size = sizeof(struct rte_mbuf) + priv_size; 770 /* start of buffer is after mbuf structure and priv data */ 771 772 m->buf_addr = (char *)m + mbuf_size; 773 m->buf_iova = rte_mempool_virt2iova(m) + mbuf_size; 774 m = m->next; 775 } 776 } 777 778 static __rte_always_inline bool 779 mbuf_is_consumed(struct rte_mbuf *m) 780 { 781 while (m) { 782 if (rte_mbuf_refcnt_read(m) > 1) 783 return false; 784 m = m->next; 785 } 786 787 return true; 788 } 789 790 static __rte_always_inline void 791 put_zmbuf(struct zcopy_mbuf *zmbuf) 792 { 793 zmbuf->in_use = 0; 794 } 795 796 #endif /* _VHOST_NET_CDEV_H_ */ 797