1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016 IGEL Co., Ltd. 3 * Copyright(c) 2016-2018 Intel Corporation 4 */ 5 #include <unistd.h> 6 #include <pthread.h> 7 #include <stdbool.h> 8 9 #include <rte_mbuf.h> 10 #include <rte_ethdev_driver.h> 11 #include <rte_ethdev_vdev.h> 12 #include <rte_malloc.h> 13 #include <rte_memcpy.h> 14 #include <rte_bus_vdev.h> 15 #include <rte_kvargs.h> 16 #include <rte_vhost.h> 17 #include <rte_spinlock.h> 18 19 #include "rte_eth_vhost.h" 20 21 RTE_LOG_REGISTER(vhost_logtype, pmd.net.vhost, NOTICE); 22 23 #define VHOST_LOG(level, ...) \ 24 rte_log(RTE_LOG_ ## level, vhost_logtype, __VA_ARGS__) 25 26 enum {VIRTIO_RXQ, VIRTIO_TXQ, VIRTIO_QNUM}; 27 28 #define ETH_VHOST_IFACE_ARG "iface" 29 #define ETH_VHOST_QUEUES_ARG "queues" 30 #define ETH_VHOST_CLIENT_ARG "client" 31 #define ETH_VHOST_DEQUEUE_ZERO_COPY "dequeue-zero-copy" 32 #define ETH_VHOST_IOMMU_SUPPORT "iommu-support" 33 #define ETH_VHOST_POSTCOPY_SUPPORT "postcopy-support" 34 #define ETH_VHOST_VIRTIO_NET_F_HOST_TSO "tso" 35 #define ETH_VHOST_LINEAR_BUF "linear-buffer" 36 #define ETH_VHOST_EXT_BUF "ext-buffer" 37 #define VHOST_MAX_PKT_BURST 32 38 39 static const char *valid_arguments[] = { 40 ETH_VHOST_IFACE_ARG, 41 ETH_VHOST_QUEUES_ARG, 42 ETH_VHOST_CLIENT_ARG, 43 ETH_VHOST_DEQUEUE_ZERO_COPY, 44 ETH_VHOST_IOMMU_SUPPORT, 45 ETH_VHOST_POSTCOPY_SUPPORT, 46 ETH_VHOST_VIRTIO_NET_F_HOST_TSO, 47 ETH_VHOST_LINEAR_BUF, 48 ETH_VHOST_EXT_BUF, 49 NULL 50 }; 51 52 static struct rte_ether_addr base_eth_addr = { 53 .addr_bytes = { 54 0x56 /* V */, 55 0x48 /* H */, 56 0x4F /* O */, 57 0x53 /* S */, 58 0x54 /* T */, 59 0x00 60 } 61 }; 62 63 enum vhost_xstats_pkts { 64 VHOST_UNDERSIZE_PKT = 0, 65 VHOST_64_PKT, 66 VHOST_65_TO_127_PKT, 67 VHOST_128_TO_255_PKT, 68 VHOST_256_TO_511_PKT, 69 VHOST_512_TO_1023_PKT, 70 VHOST_1024_TO_1522_PKT, 71 VHOST_1523_TO_MAX_PKT, 72 VHOST_BROADCAST_PKT, 73 VHOST_MULTICAST_PKT, 74 VHOST_UNICAST_PKT, 75 VHOST_ERRORS_PKT, 76 VHOST_ERRORS_FRAGMENTED, 77 VHOST_ERRORS_JABBER, 78 VHOST_UNKNOWN_PROTOCOL, 79 VHOST_XSTATS_MAX, 80 }; 81 82 struct vhost_stats { 83 uint64_t pkts; 84 uint64_t bytes; 85 uint64_t missed_pkts; 86 uint64_t xstats[VHOST_XSTATS_MAX]; 87 }; 88 89 struct vhost_queue { 90 int vid; 91 rte_atomic32_t allow_queuing; 92 rte_atomic32_t while_queuing; 93 struct pmd_internal *internal; 94 struct rte_mempool *mb_pool; 95 uint16_t port; 96 uint16_t virtqueue_id; 97 struct vhost_stats stats; 98 }; 99 100 struct pmd_internal { 101 rte_atomic32_t dev_attached; 102 char *iface_name; 103 uint64_t flags; 104 uint64_t disable_flags; 105 uint16_t max_queues; 106 int vid; 107 rte_atomic32_t started; 108 uint8_t vlan_strip; 109 }; 110 111 struct internal_list { 112 TAILQ_ENTRY(internal_list) next; 113 struct rte_eth_dev *eth_dev; 114 }; 115 116 TAILQ_HEAD(internal_list_head, internal_list); 117 static struct internal_list_head internal_list = 118 TAILQ_HEAD_INITIALIZER(internal_list); 119 120 static pthread_mutex_t internal_list_lock = PTHREAD_MUTEX_INITIALIZER; 121 122 static struct rte_eth_link pmd_link = { 123 .link_speed = 10000, 124 .link_duplex = ETH_LINK_FULL_DUPLEX, 125 .link_status = ETH_LINK_DOWN 126 }; 127 128 struct rte_vhost_vring_state { 129 rte_spinlock_t lock; 130 131 bool cur[RTE_MAX_QUEUES_PER_PORT * 2]; 132 bool seen[RTE_MAX_QUEUES_PER_PORT * 2]; 133 unsigned int index; 134 unsigned int max_vring; 135 }; 136 137 static struct rte_vhost_vring_state *vring_states[RTE_MAX_ETHPORTS]; 138 139 #define VHOST_XSTATS_NAME_SIZE 64 140 141 struct vhost_xstats_name_off { 142 char name[VHOST_XSTATS_NAME_SIZE]; 143 uint64_t offset; 144 }; 145 146 /* [rx]_is prepended to the name string here */ 147 static const struct vhost_xstats_name_off vhost_rxport_stat_strings[] = { 148 {"good_packets", 149 offsetof(struct vhost_queue, stats.pkts)}, 150 {"total_bytes", 151 offsetof(struct vhost_queue, stats.bytes)}, 152 {"missed_pkts", 153 offsetof(struct vhost_queue, stats.missed_pkts)}, 154 {"broadcast_packets", 155 offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])}, 156 {"multicast_packets", 157 offsetof(struct vhost_queue, stats.xstats[VHOST_MULTICAST_PKT])}, 158 {"unicast_packets", 159 offsetof(struct vhost_queue, stats.xstats[VHOST_UNICAST_PKT])}, 160 {"undersize_packets", 161 offsetof(struct vhost_queue, stats.xstats[VHOST_UNDERSIZE_PKT])}, 162 {"size_64_packets", 163 offsetof(struct vhost_queue, stats.xstats[VHOST_64_PKT])}, 164 {"size_65_to_127_packets", 165 offsetof(struct vhost_queue, stats.xstats[VHOST_65_TO_127_PKT])}, 166 {"size_128_to_255_packets", 167 offsetof(struct vhost_queue, stats.xstats[VHOST_128_TO_255_PKT])}, 168 {"size_256_to_511_packets", 169 offsetof(struct vhost_queue, stats.xstats[VHOST_256_TO_511_PKT])}, 170 {"size_512_to_1023_packets", 171 offsetof(struct vhost_queue, stats.xstats[VHOST_512_TO_1023_PKT])}, 172 {"size_1024_to_1522_packets", 173 offsetof(struct vhost_queue, stats.xstats[VHOST_1024_TO_1522_PKT])}, 174 {"size_1523_to_max_packets", 175 offsetof(struct vhost_queue, stats.xstats[VHOST_1523_TO_MAX_PKT])}, 176 {"errors_with_bad_CRC", 177 offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_PKT])}, 178 {"fragmented_errors", 179 offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_FRAGMENTED])}, 180 {"jabber_errors", 181 offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_JABBER])}, 182 {"unknown_protos_packets", 183 offsetof(struct vhost_queue, stats.xstats[VHOST_UNKNOWN_PROTOCOL])}, 184 }; 185 186 /* [tx]_ is prepended to the name string here */ 187 static const struct vhost_xstats_name_off vhost_txport_stat_strings[] = { 188 {"good_packets", 189 offsetof(struct vhost_queue, stats.pkts)}, 190 {"total_bytes", 191 offsetof(struct vhost_queue, stats.bytes)}, 192 {"missed_pkts", 193 offsetof(struct vhost_queue, stats.missed_pkts)}, 194 {"broadcast_packets", 195 offsetof(struct vhost_queue, stats.xstats[VHOST_BROADCAST_PKT])}, 196 {"multicast_packets", 197 offsetof(struct vhost_queue, stats.xstats[VHOST_MULTICAST_PKT])}, 198 {"unicast_packets", 199 offsetof(struct vhost_queue, stats.xstats[VHOST_UNICAST_PKT])}, 200 {"undersize_packets", 201 offsetof(struct vhost_queue, stats.xstats[VHOST_UNDERSIZE_PKT])}, 202 {"size_64_packets", 203 offsetof(struct vhost_queue, stats.xstats[VHOST_64_PKT])}, 204 {"size_65_to_127_packets", 205 offsetof(struct vhost_queue, stats.xstats[VHOST_65_TO_127_PKT])}, 206 {"size_128_to_255_packets", 207 offsetof(struct vhost_queue, stats.xstats[VHOST_128_TO_255_PKT])}, 208 {"size_256_to_511_packets", 209 offsetof(struct vhost_queue, stats.xstats[VHOST_256_TO_511_PKT])}, 210 {"size_512_to_1023_packets", 211 offsetof(struct vhost_queue, stats.xstats[VHOST_512_TO_1023_PKT])}, 212 {"size_1024_to_1522_packets", 213 offsetof(struct vhost_queue, stats.xstats[VHOST_1024_TO_1522_PKT])}, 214 {"size_1523_to_max_packets", 215 offsetof(struct vhost_queue, stats.xstats[VHOST_1523_TO_MAX_PKT])}, 216 {"errors_with_bad_CRC", 217 offsetof(struct vhost_queue, stats.xstats[VHOST_ERRORS_PKT])}, 218 }; 219 220 #define VHOST_NB_XSTATS_RXPORT (sizeof(vhost_rxport_stat_strings) / \ 221 sizeof(vhost_rxport_stat_strings[0])) 222 223 #define VHOST_NB_XSTATS_TXPORT (sizeof(vhost_txport_stat_strings) / \ 224 sizeof(vhost_txport_stat_strings[0])) 225 226 static int 227 vhost_dev_xstats_reset(struct rte_eth_dev *dev) 228 { 229 struct vhost_queue *vq = NULL; 230 unsigned int i = 0; 231 232 for (i = 0; i < dev->data->nb_rx_queues; i++) { 233 vq = dev->data->rx_queues[i]; 234 if (!vq) 235 continue; 236 memset(&vq->stats, 0, sizeof(vq->stats)); 237 } 238 for (i = 0; i < dev->data->nb_tx_queues; i++) { 239 vq = dev->data->tx_queues[i]; 240 if (!vq) 241 continue; 242 memset(&vq->stats, 0, sizeof(vq->stats)); 243 } 244 245 return 0; 246 } 247 248 static int 249 vhost_dev_xstats_get_names(struct rte_eth_dev *dev __rte_unused, 250 struct rte_eth_xstat_name *xstats_names, 251 unsigned int limit __rte_unused) 252 { 253 unsigned int t = 0; 254 int count = 0; 255 int nstats = VHOST_NB_XSTATS_RXPORT + VHOST_NB_XSTATS_TXPORT; 256 257 if (!xstats_names) 258 return nstats; 259 for (t = 0; t < VHOST_NB_XSTATS_RXPORT; t++) { 260 snprintf(xstats_names[count].name, 261 sizeof(xstats_names[count].name), 262 "rx_%s", vhost_rxport_stat_strings[t].name); 263 count++; 264 } 265 for (t = 0; t < VHOST_NB_XSTATS_TXPORT; t++) { 266 snprintf(xstats_names[count].name, 267 sizeof(xstats_names[count].name), 268 "tx_%s", vhost_txport_stat_strings[t].name); 269 count++; 270 } 271 return count; 272 } 273 274 static int 275 vhost_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 276 unsigned int n) 277 { 278 unsigned int i; 279 unsigned int t; 280 unsigned int count = 0; 281 struct vhost_queue *vq = NULL; 282 unsigned int nxstats = VHOST_NB_XSTATS_RXPORT + VHOST_NB_XSTATS_TXPORT; 283 284 if (n < nxstats) 285 return nxstats; 286 287 for (i = 0; i < dev->data->nb_rx_queues; i++) { 288 vq = dev->data->rx_queues[i]; 289 if (!vq) 290 continue; 291 vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts 292 - (vq->stats.xstats[VHOST_BROADCAST_PKT] 293 + vq->stats.xstats[VHOST_MULTICAST_PKT]); 294 } 295 for (i = 0; i < dev->data->nb_tx_queues; i++) { 296 vq = dev->data->tx_queues[i]; 297 if (!vq) 298 continue; 299 vq->stats.xstats[VHOST_UNICAST_PKT] = vq->stats.pkts 300 + vq->stats.missed_pkts 301 - (vq->stats.xstats[VHOST_BROADCAST_PKT] 302 + vq->stats.xstats[VHOST_MULTICAST_PKT]); 303 } 304 for (t = 0; t < VHOST_NB_XSTATS_RXPORT; t++) { 305 xstats[count].value = 0; 306 for (i = 0; i < dev->data->nb_rx_queues; i++) { 307 vq = dev->data->rx_queues[i]; 308 if (!vq) 309 continue; 310 xstats[count].value += 311 *(uint64_t *)(((char *)vq) 312 + vhost_rxport_stat_strings[t].offset); 313 } 314 xstats[count].id = count; 315 count++; 316 } 317 for (t = 0; t < VHOST_NB_XSTATS_TXPORT; t++) { 318 xstats[count].value = 0; 319 for (i = 0; i < dev->data->nb_tx_queues; i++) { 320 vq = dev->data->tx_queues[i]; 321 if (!vq) 322 continue; 323 xstats[count].value += 324 *(uint64_t *)(((char *)vq) 325 + vhost_txport_stat_strings[t].offset); 326 } 327 xstats[count].id = count; 328 count++; 329 } 330 return count; 331 } 332 333 static inline void 334 vhost_count_multicast_broadcast(struct vhost_queue *vq, 335 struct rte_mbuf *mbuf) 336 { 337 struct rte_ether_addr *ea = NULL; 338 struct vhost_stats *pstats = &vq->stats; 339 340 ea = rte_pktmbuf_mtod(mbuf, struct rte_ether_addr *); 341 if (rte_is_multicast_ether_addr(ea)) { 342 if (rte_is_broadcast_ether_addr(ea)) 343 pstats->xstats[VHOST_BROADCAST_PKT]++; 344 else 345 pstats->xstats[VHOST_MULTICAST_PKT]++; 346 } 347 } 348 349 static void 350 vhost_update_packet_xstats(struct vhost_queue *vq, 351 struct rte_mbuf **bufs, 352 uint16_t count) 353 { 354 uint32_t pkt_len = 0; 355 uint64_t i = 0; 356 uint64_t index; 357 struct vhost_stats *pstats = &vq->stats; 358 359 for (i = 0; i < count ; i++) { 360 pkt_len = bufs[i]->pkt_len; 361 if (pkt_len == 64) { 362 pstats->xstats[VHOST_64_PKT]++; 363 } else if (pkt_len > 64 && pkt_len < 1024) { 364 index = (sizeof(pkt_len) * 8) 365 - __builtin_clz(pkt_len) - 5; 366 pstats->xstats[index]++; 367 } else { 368 if (pkt_len < 64) 369 pstats->xstats[VHOST_UNDERSIZE_PKT]++; 370 else if (pkt_len <= 1522) 371 pstats->xstats[VHOST_1024_TO_1522_PKT]++; 372 else if (pkt_len > 1522) 373 pstats->xstats[VHOST_1523_TO_MAX_PKT]++; 374 } 375 vhost_count_multicast_broadcast(vq, bufs[i]); 376 } 377 } 378 379 static uint16_t 380 eth_vhost_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 381 { 382 struct vhost_queue *r = q; 383 uint16_t i, nb_rx = 0; 384 uint16_t nb_receive = nb_bufs; 385 386 if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0)) 387 return 0; 388 389 rte_atomic32_set(&r->while_queuing, 1); 390 391 if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0)) 392 goto out; 393 394 /* Dequeue packets from guest TX queue */ 395 while (nb_receive) { 396 uint16_t nb_pkts; 397 uint16_t num = (uint16_t)RTE_MIN(nb_receive, 398 VHOST_MAX_PKT_BURST); 399 400 nb_pkts = rte_vhost_dequeue_burst(r->vid, r->virtqueue_id, 401 r->mb_pool, &bufs[nb_rx], 402 num); 403 404 nb_rx += nb_pkts; 405 nb_receive -= nb_pkts; 406 if (nb_pkts < num) 407 break; 408 } 409 410 r->stats.pkts += nb_rx; 411 412 for (i = 0; likely(i < nb_rx); i++) { 413 bufs[i]->port = r->port; 414 bufs[i]->vlan_tci = 0; 415 416 if (r->internal->vlan_strip) 417 rte_vlan_strip(bufs[i]); 418 419 r->stats.bytes += bufs[i]->pkt_len; 420 } 421 422 vhost_update_packet_xstats(r, bufs, nb_rx); 423 424 out: 425 rte_atomic32_set(&r->while_queuing, 0); 426 427 return nb_rx; 428 } 429 430 static uint16_t 431 eth_vhost_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs) 432 { 433 struct vhost_queue *r = q; 434 uint16_t i, nb_tx = 0; 435 uint16_t nb_send = 0; 436 437 if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0)) 438 return 0; 439 440 rte_atomic32_set(&r->while_queuing, 1); 441 442 if (unlikely(rte_atomic32_read(&r->allow_queuing) == 0)) 443 goto out; 444 445 for (i = 0; i < nb_bufs; i++) { 446 struct rte_mbuf *m = bufs[i]; 447 448 /* Do VLAN tag insertion */ 449 if (m->ol_flags & PKT_TX_VLAN_PKT) { 450 int error = rte_vlan_insert(&m); 451 if (unlikely(error)) { 452 rte_pktmbuf_free(m); 453 continue; 454 } 455 } 456 457 bufs[nb_send] = m; 458 ++nb_send; 459 } 460 461 /* Enqueue packets to guest RX queue */ 462 while (nb_send) { 463 uint16_t nb_pkts; 464 uint16_t num = (uint16_t)RTE_MIN(nb_send, 465 VHOST_MAX_PKT_BURST); 466 467 nb_pkts = rte_vhost_enqueue_burst(r->vid, r->virtqueue_id, 468 &bufs[nb_tx], num); 469 470 nb_tx += nb_pkts; 471 nb_send -= nb_pkts; 472 if (nb_pkts < num) 473 break; 474 } 475 476 r->stats.pkts += nb_tx; 477 r->stats.missed_pkts += nb_bufs - nb_tx; 478 479 for (i = 0; likely(i < nb_tx); i++) 480 r->stats.bytes += bufs[i]->pkt_len; 481 482 vhost_update_packet_xstats(r, bufs, nb_tx); 483 484 /* According to RFC2863 page42 section ifHCOutMulticastPkts and 485 * ifHCOutBroadcastPkts, the counters "multicast" and "broadcast" 486 * are increased when packets are not transmitted successfully. 487 */ 488 for (i = nb_tx; i < nb_bufs; i++) 489 vhost_count_multicast_broadcast(r, bufs[i]); 490 491 for (i = 0; likely(i < nb_tx); i++) 492 rte_pktmbuf_free(bufs[i]); 493 out: 494 rte_atomic32_set(&r->while_queuing, 0); 495 496 return nb_tx; 497 } 498 499 static inline struct internal_list * 500 find_internal_resource(char *ifname) 501 { 502 int found = 0; 503 struct internal_list *list; 504 struct pmd_internal *internal; 505 506 if (ifname == NULL) 507 return NULL; 508 509 pthread_mutex_lock(&internal_list_lock); 510 511 TAILQ_FOREACH(list, &internal_list, next) { 512 internal = list->eth_dev->data->dev_private; 513 if (!strcmp(internal->iface_name, ifname)) { 514 found = 1; 515 break; 516 } 517 } 518 519 pthread_mutex_unlock(&internal_list_lock); 520 521 if (!found) 522 return NULL; 523 524 return list; 525 } 526 527 static int 528 eth_rxq_intr_enable(struct rte_eth_dev *dev, uint16_t qid) 529 { 530 struct rte_vhost_vring vring; 531 struct vhost_queue *vq; 532 int ret = 0; 533 534 vq = dev->data->rx_queues[qid]; 535 if (!vq) { 536 VHOST_LOG(ERR, "rxq%d is not setup yet\n", qid); 537 return -1; 538 } 539 540 ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring); 541 if (ret < 0) { 542 VHOST_LOG(ERR, "Failed to get rxq%d's vring\n", qid); 543 return ret; 544 } 545 VHOST_LOG(INFO, "Enable interrupt for rxq%d\n", qid); 546 rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 1); 547 rte_wmb(); 548 549 return ret; 550 } 551 552 static int 553 eth_rxq_intr_disable(struct rte_eth_dev *dev, uint16_t qid) 554 { 555 struct rte_vhost_vring vring; 556 struct vhost_queue *vq; 557 int ret = 0; 558 559 vq = dev->data->rx_queues[qid]; 560 if (!vq) { 561 VHOST_LOG(ERR, "rxq%d is not setup yet\n", qid); 562 return -1; 563 } 564 565 ret = rte_vhost_get_vhost_vring(vq->vid, (qid << 1) + 1, &vring); 566 if (ret < 0) { 567 VHOST_LOG(ERR, "Failed to get rxq%d's vring", qid); 568 return ret; 569 } 570 VHOST_LOG(INFO, "Disable interrupt for rxq%d\n", qid); 571 rte_vhost_enable_guest_notification(vq->vid, (qid << 1) + 1, 0); 572 rte_wmb(); 573 574 return 0; 575 } 576 577 static void 578 eth_vhost_uninstall_intr(struct rte_eth_dev *dev) 579 { 580 struct rte_intr_handle *intr_handle = dev->intr_handle; 581 582 if (intr_handle) { 583 if (intr_handle->intr_vec) 584 free(intr_handle->intr_vec); 585 free(intr_handle); 586 } 587 588 dev->intr_handle = NULL; 589 } 590 591 static int 592 eth_vhost_install_intr(struct rte_eth_dev *dev) 593 { 594 struct rte_vhost_vring vring; 595 struct vhost_queue *vq; 596 int count = 0; 597 int nb_rxq = dev->data->nb_rx_queues; 598 int i; 599 int ret; 600 601 /* uninstall firstly if we are reconnecting */ 602 if (dev->intr_handle) 603 eth_vhost_uninstall_intr(dev); 604 605 dev->intr_handle = malloc(sizeof(*dev->intr_handle)); 606 if (!dev->intr_handle) { 607 VHOST_LOG(ERR, "Fail to allocate intr_handle\n"); 608 return -ENOMEM; 609 } 610 memset(dev->intr_handle, 0, sizeof(*dev->intr_handle)); 611 612 dev->intr_handle->efd_counter_size = sizeof(uint64_t); 613 614 dev->intr_handle->intr_vec = 615 malloc(nb_rxq * sizeof(dev->intr_handle->intr_vec[0])); 616 617 if (!dev->intr_handle->intr_vec) { 618 VHOST_LOG(ERR, 619 "Failed to allocate memory for interrupt vector\n"); 620 free(dev->intr_handle); 621 return -ENOMEM; 622 } 623 624 VHOST_LOG(INFO, "Prepare intr vec\n"); 625 for (i = 0; i < nb_rxq; i++) { 626 vq = dev->data->rx_queues[i]; 627 if (!vq) { 628 VHOST_LOG(INFO, "rxq-%d not setup yet, skip!\n", i); 629 continue; 630 } 631 632 ret = rte_vhost_get_vhost_vring(vq->vid, (i << 1) + 1, &vring); 633 if (ret < 0) { 634 VHOST_LOG(INFO, 635 "Failed to get rxq-%d's vring, skip!\n", i); 636 continue; 637 } 638 639 if (vring.kickfd < 0) { 640 VHOST_LOG(INFO, 641 "rxq-%d's kickfd is invalid, skip!\n", i); 642 continue; 643 } 644 dev->intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i; 645 dev->intr_handle->efds[i] = vring.kickfd; 646 count++; 647 VHOST_LOG(INFO, "Installed intr vec for rxq-%d\n", i); 648 } 649 650 dev->intr_handle->nb_efd = count; 651 dev->intr_handle->max_intr = count + 1; 652 dev->intr_handle->type = RTE_INTR_HANDLE_VDEV; 653 654 return 0; 655 } 656 657 static void 658 update_queuing_status(struct rte_eth_dev *dev) 659 { 660 struct pmd_internal *internal = dev->data->dev_private; 661 struct vhost_queue *vq; 662 unsigned int i; 663 int allow_queuing = 1; 664 665 if (!dev->data->rx_queues || !dev->data->tx_queues) 666 return; 667 668 if (rte_atomic32_read(&internal->started) == 0 || 669 rte_atomic32_read(&internal->dev_attached) == 0) 670 allow_queuing = 0; 671 672 /* Wait until rx/tx_pkt_burst stops accessing vhost device */ 673 for (i = 0; i < dev->data->nb_rx_queues; i++) { 674 vq = dev->data->rx_queues[i]; 675 if (vq == NULL) 676 continue; 677 rte_atomic32_set(&vq->allow_queuing, allow_queuing); 678 while (rte_atomic32_read(&vq->while_queuing)) 679 rte_pause(); 680 } 681 682 for (i = 0; i < dev->data->nb_tx_queues; i++) { 683 vq = dev->data->tx_queues[i]; 684 if (vq == NULL) 685 continue; 686 rte_atomic32_set(&vq->allow_queuing, allow_queuing); 687 while (rte_atomic32_read(&vq->while_queuing)) 688 rte_pause(); 689 } 690 } 691 692 static void 693 queue_setup(struct rte_eth_dev *eth_dev, struct pmd_internal *internal) 694 { 695 struct vhost_queue *vq; 696 int i; 697 698 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 699 vq = eth_dev->data->rx_queues[i]; 700 if (!vq) 701 continue; 702 vq->vid = internal->vid; 703 vq->internal = internal; 704 vq->port = eth_dev->data->port_id; 705 } 706 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 707 vq = eth_dev->data->tx_queues[i]; 708 if (!vq) 709 continue; 710 vq->vid = internal->vid; 711 vq->internal = internal; 712 vq->port = eth_dev->data->port_id; 713 } 714 } 715 716 static int 717 new_device(int vid) 718 { 719 struct rte_eth_dev *eth_dev; 720 struct internal_list *list; 721 struct pmd_internal *internal; 722 struct rte_eth_conf *dev_conf; 723 unsigned i; 724 char ifname[PATH_MAX]; 725 #ifdef RTE_LIBRTE_VHOST_NUMA 726 int newnode; 727 #endif 728 729 rte_vhost_get_ifname(vid, ifname, sizeof(ifname)); 730 list = find_internal_resource(ifname); 731 if (list == NULL) { 732 VHOST_LOG(INFO, "Invalid device name: %s\n", ifname); 733 return -1; 734 } 735 736 eth_dev = list->eth_dev; 737 internal = eth_dev->data->dev_private; 738 dev_conf = ð_dev->data->dev_conf; 739 740 #ifdef RTE_LIBRTE_VHOST_NUMA 741 newnode = rte_vhost_get_numa_node(vid); 742 if (newnode >= 0) 743 eth_dev->data->numa_node = newnode; 744 #endif 745 746 internal->vid = vid; 747 if (rte_atomic32_read(&internal->started) == 1) { 748 queue_setup(eth_dev, internal); 749 750 if (dev_conf->intr_conf.rxq) { 751 if (eth_vhost_install_intr(eth_dev) < 0) { 752 VHOST_LOG(INFO, 753 "Failed to install interrupt handler."); 754 return -1; 755 } 756 } 757 } else { 758 VHOST_LOG(INFO, "RX/TX queues not exist yet\n"); 759 } 760 761 for (i = 0; i < rte_vhost_get_vring_num(vid); i++) 762 rte_vhost_enable_guest_notification(vid, i, 0); 763 764 rte_vhost_get_mtu(vid, ð_dev->data->mtu); 765 766 eth_dev->data->dev_link.link_status = ETH_LINK_UP; 767 768 rte_atomic32_set(&internal->dev_attached, 1); 769 update_queuing_status(eth_dev); 770 771 VHOST_LOG(INFO, "Vhost device %d created\n", vid); 772 773 _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); 774 775 return 0; 776 } 777 778 static void 779 destroy_device(int vid) 780 { 781 struct rte_eth_dev *eth_dev; 782 struct pmd_internal *internal; 783 struct vhost_queue *vq; 784 struct internal_list *list; 785 char ifname[PATH_MAX]; 786 unsigned i; 787 struct rte_vhost_vring_state *state; 788 789 rte_vhost_get_ifname(vid, ifname, sizeof(ifname)); 790 list = find_internal_resource(ifname); 791 if (list == NULL) { 792 VHOST_LOG(ERR, "Invalid interface name: %s\n", ifname); 793 return; 794 } 795 eth_dev = list->eth_dev; 796 internal = eth_dev->data->dev_private; 797 798 rte_atomic32_set(&internal->dev_attached, 0); 799 update_queuing_status(eth_dev); 800 801 eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; 802 803 if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) { 804 for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { 805 vq = eth_dev->data->rx_queues[i]; 806 if (!vq) 807 continue; 808 vq->vid = -1; 809 } 810 for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { 811 vq = eth_dev->data->tx_queues[i]; 812 if (!vq) 813 continue; 814 vq->vid = -1; 815 } 816 } 817 818 state = vring_states[eth_dev->data->port_id]; 819 rte_spinlock_lock(&state->lock); 820 for (i = 0; i <= state->max_vring; i++) { 821 state->cur[i] = false; 822 state->seen[i] = false; 823 } 824 state->max_vring = 0; 825 rte_spinlock_unlock(&state->lock); 826 827 VHOST_LOG(INFO, "Vhost device %d destroyed\n", vid); 828 eth_vhost_uninstall_intr(eth_dev); 829 830 _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); 831 } 832 833 static int 834 vring_state_changed(int vid, uint16_t vring, int enable) 835 { 836 struct rte_vhost_vring_state *state; 837 struct rte_eth_dev *eth_dev; 838 struct internal_list *list; 839 char ifname[PATH_MAX]; 840 841 rte_vhost_get_ifname(vid, ifname, sizeof(ifname)); 842 list = find_internal_resource(ifname); 843 if (list == NULL) { 844 VHOST_LOG(ERR, "Invalid interface name: %s\n", ifname); 845 return -1; 846 } 847 848 eth_dev = list->eth_dev; 849 /* won't be NULL */ 850 state = vring_states[eth_dev->data->port_id]; 851 rte_spinlock_lock(&state->lock); 852 if (state->cur[vring] == enable) { 853 rte_spinlock_unlock(&state->lock); 854 return 0; 855 } 856 state->cur[vring] = enable; 857 state->max_vring = RTE_MAX(vring, state->max_vring); 858 rte_spinlock_unlock(&state->lock); 859 860 VHOST_LOG(INFO, "vring%u is %s\n", 861 vring, enable ? "enabled" : "disabled"); 862 863 _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_QUEUE_STATE, NULL); 864 865 return 0; 866 } 867 868 static struct vhost_device_ops vhost_ops = { 869 .new_device = new_device, 870 .destroy_device = destroy_device, 871 .vring_state_changed = vring_state_changed, 872 }; 873 874 static int 875 vhost_driver_setup(struct rte_eth_dev *eth_dev) 876 { 877 struct pmd_internal *internal = eth_dev->data->dev_private; 878 struct internal_list *list = NULL; 879 struct rte_vhost_vring_state *vring_state = NULL; 880 unsigned int numa_node = eth_dev->device->numa_node; 881 const char *name = eth_dev->device->name; 882 883 /* Don't try to setup again if it has already been done. */ 884 list = find_internal_resource(internal->iface_name); 885 if (list) 886 return 0; 887 888 list = rte_zmalloc_socket(name, sizeof(*list), 0, numa_node); 889 if (list == NULL) 890 return -1; 891 892 vring_state = rte_zmalloc_socket(name, sizeof(*vring_state), 893 0, numa_node); 894 if (vring_state == NULL) 895 goto free_list; 896 897 list->eth_dev = eth_dev; 898 pthread_mutex_lock(&internal_list_lock); 899 TAILQ_INSERT_TAIL(&internal_list, list, next); 900 pthread_mutex_unlock(&internal_list_lock); 901 902 rte_spinlock_init(&vring_state->lock); 903 vring_states[eth_dev->data->port_id] = vring_state; 904 905 if (rte_vhost_driver_register(internal->iface_name, internal->flags)) 906 goto list_remove; 907 908 if (internal->disable_flags) { 909 if (rte_vhost_driver_disable_features(internal->iface_name, 910 internal->disable_flags)) 911 goto drv_unreg; 912 } 913 914 if (rte_vhost_driver_callback_register(internal->iface_name, 915 &vhost_ops) < 0) { 916 VHOST_LOG(ERR, "Can't register callbacks\n"); 917 goto drv_unreg; 918 } 919 920 if (rte_vhost_driver_start(internal->iface_name) < 0) { 921 VHOST_LOG(ERR, "Failed to start driver for %s\n", 922 internal->iface_name); 923 goto drv_unreg; 924 } 925 926 return 0; 927 928 drv_unreg: 929 rte_vhost_driver_unregister(internal->iface_name); 930 list_remove: 931 vring_states[eth_dev->data->port_id] = NULL; 932 pthread_mutex_lock(&internal_list_lock); 933 TAILQ_REMOVE(&internal_list, list, next); 934 pthread_mutex_unlock(&internal_list_lock); 935 rte_free(vring_state); 936 free_list: 937 rte_free(list); 938 939 return -1; 940 } 941 942 int 943 rte_eth_vhost_get_queue_event(uint16_t port_id, 944 struct rte_eth_vhost_queue_event *event) 945 { 946 struct rte_vhost_vring_state *state; 947 unsigned int i; 948 int idx; 949 950 if (port_id >= RTE_MAX_ETHPORTS) { 951 VHOST_LOG(ERR, "Invalid port id\n"); 952 return -1; 953 } 954 955 state = vring_states[port_id]; 956 if (!state) { 957 VHOST_LOG(ERR, "Unused port\n"); 958 return -1; 959 } 960 961 rte_spinlock_lock(&state->lock); 962 for (i = 0; i <= state->max_vring; i++) { 963 idx = state->index++ % (state->max_vring + 1); 964 965 if (state->cur[idx] != state->seen[idx]) { 966 state->seen[idx] = state->cur[idx]; 967 event->queue_id = idx / 2; 968 event->rx = idx & 1; 969 event->enable = state->cur[idx]; 970 rte_spinlock_unlock(&state->lock); 971 return 0; 972 } 973 } 974 rte_spinlock_unlock(&state->lock); 975 976 return -1; 977 } 978 979 int 980 rte_eth_vhost_get_vid_from_port_id(uint16_t port_id) 981 { 982 struct internal_list *list; 983 struct rte_eth_dev *eth_dev; 984 struct vhost_queue *vq; 985 int vid = -1; 986 987 if (!rte_eth_dev_is_valid_port(port_id)) 988 return -1; 989 990 pthread_mutex_lock(&internal_list_lock); 991 992 TAILQ_FOREACH(list, &internal_list, next) { 993 eth_dev = list->eth_dev; 994 if (eth_dev->data->port_id == port_id) { 995 vq = eth_dev->data->rx_queues[0]; 996 if (vq) { 997 vid = vq->vid; 998 } 999 break; 1000 } 1001 } 1002 1003 pthread_mutex_unlock(&internal_list_lock); 1004 1005 return vid; 1006 } 1007 1008 static int 1009 eth_dev_configure(struct rte_eth_dev *dev) 1010 { 1011 struct pmd_internal *internal = dev->data->dev_private; 1012 const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 1013 1014 /* NOTE: the same process has to operate a vhost interface 1015 * from beginning to end (from eth_dev configure to eth_dev close). 1016 * It is user's responsibility at the moment. 1017 */ 1018 if (vhost_driver_setup(dev) < 0) 1019 return -1; 1020 1021 internal->vlan_strip = !!(rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 1022 1023 return 0; 1024 } 1025 1026 static int 1027 eth_dev_start(struct rte_eth_dev *eth_dev) 1028 { 1029 struct pmd_internal *internal = eth_dev->data->dev_private; 1030 struct rte_eth_conf *dev_conf = ð_dev->data->dev_conf; 1031 1032 queue_setup(eth_dev, internal); 1033 1034 if (rte_atomic32_read(&internal->dev_attached) == 1) { 1035 if (dev_conf->intr_conf.rxq) { 1036 if (eth_vhost_install_intr(eth_dev) < 0) { 1037 VHOST_LOG(INFO, 1038 "Failed to install interrupt handler."); 1039 return -1; 1040 } 1041 } 1042 } 1043 1044 rte_atomic32_set(&internal->started, 1); 1045 update_queuing_status(eth_dev); 1046 1047 return 0; 1048 } 1049 1050 static void 1051 eth_dev_stop(struct rte_eth_dev *dev) 1052 { 1053 struct pmd_internal *internal = dev->data->dev_private; 1054 1055 rte_atomic32_set(&internal->started, 0); 1056 update_queuing_status(dev); 1057 } 1058 1059 static void 1060 eth_dev_close(struct rte_eth_dev *dev) 1061 { 1062 struct pmd_internal *internal; 1063 struct internal_list *list; 1064 unsigned int i; 1065 1066 internal = dev->data->dev_private; 1067 if (!internal) 1068 return; 1069 1070 eth_dev_stop(dev); 1071 1072 list = find_internal_resource(internal->iface_name); 1073 if (list) { 1074 rte_vhost_driver_unregister(internal->iface_name); 1075 pthread_mutex_lock(&internal_list_lock); 1076 TAILQ_REMOVE(&internal_list, list, next); 1077 pthread_mutex_unlock(&internal_list_lock); 1078 rte_free(list); 1079 } 1080 1081 if (dev->data->rx_queues) 1082 for (i = 0; i < dev->data->nb_rx_queues; i++) 1083 rte_free(dev->data->rx_queues[i]); 1084 1085 if (dev->data->tx_queues) 1086 for (i = 0; i < dev->data->nb_tx_queues; i++) 1087 rte_free(dev->data->tx_queues[i]); 1088 1089 rte_free(internal->iface_name); 1090 rte_free(internal); 1091 1092 dev->data->dev_private = NULL; 1093 1094 rte_free(vring_states[dev->data->port_id]); 1095 vring_states[dev->data->port_id] = NULL; 1096 } 1097 1098 static int 1099 eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id, 1100 uint16_t nb_rx_desc __rte_unused, 1101 unsigned int socket_id, 1102 const struct rte_eth_rxconf *rx_conf __rte_unused, 1103 struct rte_mempool *mb_pool) 1104 { 1105 struct vhost_queue *vq; 1106 1107 vq = rte_zmalloc_socket(NULL, sizeof(struct vhost_queue), 1108 RTE_CACHE_LINE_SIZE, socket_id); 1109 if (vq == NULL) { 1110 VHOST_LOG(ERR, "Failed to allocate memory for rx queue\n"); 1111 return -ENOMEM; 1112 } 1113 1114 vq->mb_pool = mb_pool; 1115 vq->virtqueue_id = rx_queue_id * VIRTIO_QNUM + VIRTIO_TXQ; 1116 dev->data->rx_queues[rx_queue_id] = vq; 1117 1118 return 0; 1119 } 1120 1121 static int 1122 eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id, 1123 uint16_t nb_tx_desc __rte_unused, 1124 unsigned int socket_id, 1125 const struct rte_eth_txconf *tx_conf __rte_unused) 1126 { 1127 struct vhost_queue *vq; 1128 1129 vq = rte_zmalloc_socket(NULL, sizeof(struct vhost_queue), 1130 RTE_CACHE_LINE_SIZE, socket_id); 1131 if (vq == NULL) { 1132 VHOST_LOG(ERR, "Failed to allocate memory for tx queue\n"); 1133 return -ENOMEM; 1134 } 1135 1136 vq->virtqueue_id = tx_queue_id * VIRTIO_QNUM + VIRTIO_RXQ; 1137 dev->data->tx_queues[tx_queue_id] = vq; 1138 1139 return 0; 1140 } 1141 1142 static int 1143 eth_dev_info(struct rte_eth_dev *dev, 1144 struct rte_eth_dev_info *dev_info) 1145 { 1146 struct pmd_internal *internal; 1147 1148 internal = dev->data->dev_private; 1149 if (internal == NULL) { 1150 VHOST_LOG(ERR, "Invalid device specified\n"); 1151 return -ENODEV; 1152 } 1153 1154 dev_info->max_mac_addrs = 1; 1155 dev_info->max_rx_pktlen = (uint32_t)-1; 1156 dev_info->max_rx_queues = internal->max_queues; 1157 dev_info->max_tx_queues = internal->max_queues; 1158 dev_info->min_rx_bufsize = 0; 1159 1160 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 1161 DEV_TX_OFFLOAD_VLAN_INSERT; 1162 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 1163 1164 return 0; 1165 } 1166 1167 static int 1168 eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1169 { 1170 unsigned i; 1171 unsigned long rx_total = 0, tx_total = 0; 1172 unsigned long rx_total_bytes = 0, tx_total_bytes = 0; 1173 struct vhost_queue *vq; 1174 1175 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && 1176 i < dev->data->nb_rx_queues; i++) { 1177 if (dev->data->rx_queues[i] == NULL) 1178 continue; 1179 vq = dev->data->rx_queues[i]; 1180 stats->q_ipackets[i] = vq->stats.pkts; 1181 rx_total += stats->q_ipackets[i]; 1182 1183 stats->q_ibytes[i] = vq->stats.bytes; 1184 rx_total_bytes += stats->q_ibytes[i]; 1185 } 1186 1187 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && 1188 i < dev->data->nb_tx_queues; i++) { 1189 if (dev->data->tx_queues[i] == NULL) 1190 continue; 1191 vq = dev->data->tx_queues[i]; 1192 stats->q_opackets[i] = vq->stats.pkts; 1193 tx_total += stats->q_opackets[i]; 1194 1195 stats->q_obytes[i] = vq->stats.bytes; 1196 tx_total_bytes += stats->q_obytes[i]; 1197 } 1198 1199 stats->ipackets = rx_total; 1200 stats->opackets = tx_total; 1201 stats->ibytes = rx_total_bytes; 1202 stats->obytes = tx_total_bytes; 1203 1204 return 0; 1205 } 1206 1207 static int 1208 eth_stats_reset(struct rte_eth_dev *dev) 1209 { 1210 struct vhost_queue *vq; 1211 unsigned i; 1212 1213 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1214 if (dev->data->rx_queues[i] == NULL) 1215 continue; 1216 vq = dev->data->rx_queues[i]; 1217 vq->stats.pkts = 0; 1218 vq->stats.bytes = 0; 1219 } 1220 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1221 if (dev->data->tx_queues[i] == NULL) 1222 continue; 1223 vq = dev->data->tx_queues[i]; 1224 vq->stats.pkts = 0; 1225 vq->stats.bytes = 0; 1226 vq->stats.missed_pkts = 0; 1227 } 1228 1229 return 0; 1230 } 1231 1232 static void 1233 eth_queue_release(void *q) 1234 { 1235 rte_free(q); 1236 } 1237 1238 static int 1239 eth_tx_done_cleanup(void *txq __rte_unused, uint32_t free_cnt __rte_unused) 1240 { 1241 /* 1242 * vHost does not hang onto mbuf. eth_vhost_tx() copies packet data 1243 * and releases mbuf, so nothing to cleanup. 1244 */ 1245 return 0; 1246 } 1247 1248 static int 1249 eth_link_update(struct rte_eth_dev *dev __rte_unused, 1250 int wait_to_complete __rte_unused) 1251 { 1252 return 0; 1253 } 1254 1255 static uint32_t 1256 eth_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1257 { 1258 struct vhost_queue *vq; 1259 1260 vq = dev->data->rx_queues[rx_queue_id]; 1261 if (vq == NULL) 1262 return 0; 1263 1264 return rte_vhost_rx_queue_count(vq->vid, vq->virtqueue_id); 1265 } 1266 1267 static const struct eth_dev_ops ops = { 1268 .dev_start = eth_dev_start, 1269 .dev_stop = eth_dev_stop, 1270 .dev_close = eth_dev_close, 1271 .dev_configure = eth_dev_configure, 1272 .dev_infos_get = eth_dev_info, 1273 .rx_queue_setup = eth_rx_queue_setup, 1274 .tx_queue_setup = eth_tx_queue_setup, 1275 .rx_queue_release = eth_queue_release, 1276 .tx_queue_release = eth_queue_release, 1277 .tx_done_cleanup = eth_tx_done_cleanup, 1278 .rx_queue_count = eth_rx_queue_count, 1279 .link_update = eth_link_update, 1280 .stats_get = eth_stats_get, 1281 .stats_reset = eth_stats_reset, 1282 .xstats_reset = vhost_dev_xstats_reset, 1283 .xstats_get = vhost_dev_xstats_get, 1284 .xstats_get_names = vhost_dev_xstats_get_names, 1285 .rx_queue_intr_enable = eth_rxq_intr_enable, 1286 .rx_queue_intr_disable = eth_rxq_intr_disable, 1287 }; 1288 1289 static int 1290 eth_dev_vhost_create(struct rte_vdev_device *dev, char *iface_name, 1291 int16_t queues, const unsigned int numa_node, uint64_t flags, 1292 uint64_t disable_flags) 1293 { 1294 const char *name = rte_vdev_device_name(dev); 1295 struct rte_eth_dev_data *data; 1296 struct pmd_internal *internal = NULL; 1297 struct rte_eth_dev *eth_dev = NULL; 1298 struct rte_ether_addr *eth_addr = NULL; 1299 1300 VHOST_LOG(INFO, "Creating VHOST-USER backend on numa socket %u\n", 1301 numa_node); 1302 1303 /* reserve an ethdev entry */ 1304 eth_dev = rte_eth_vdev_allocate(dev, sizeof(*internal)); 1305 if (eth_dev == NULL) 1306 goto error; 1307 data = eth_dev->data; 1308 1309 eth_addr = rte_zmalloc_socket(name, sizeof(*eth_addr), 0, numa_node); 1310 if (eth_addr == NULL) 1311 goto error; 1312 data->mac_addrs = eth_addr; 1313 *eth_addr = base_eth_addr; 1314 eth_addr->addr_bytes[5] = eth_dev->data->port_id; 1315 1316 /* now put it all together 1317 * - store queue data in internal, 1318 * - point eth_dev_data to internals 1319 * - and point eth_dev structure to new eth_dev_data structure 1320 */ 1321 internal = eth_dev->data->dev_private; 1322 internal->iface_name = rte_malloc_socket(name, strlen(iface_name) + 1, 1323 0, numa_node); 1324 if (internal->iface_name == NULL) 1325 goto error; 1326 strcpy(internal->iface_name, iface_name); 1327 1328 data->nb_rx_queues = queues; 1329 data->nb_tx_queues = queues; 1330 internal->max_queues = queues; 1331 internal->vid = -1; 1332 internal->flags = flags; 1333 internal->disable_flags = disable_flags; 1334 data->dev_link = pmd_link; 1335 data->dev_flags = RTE_ETH_DEV_INTR_LSC | RTE_ETH_DEV_CLOSE_REMOVE; 1336 data->promiscuous = 1; 1337 data->all_multicast = 1; 1338 1339 eth_dev->dev_ops = &ops; 1340 1341 /* finally assign rx and tx ops */ 1342 eth_dev->rx_pkt_burst = eth_vhost_rx; 1343 eth_dev->tx_pkt_burst = eth_vhost_tx; 1344 1345 rte_eth_dev_probing_finish(eth_dev); 1346 return 0; 1347 1348 error: 1349 if (internal) 1350 rte_free(internal->iface_name); 1351 rte_eth_dev_release_port(eth_dev); 1352 1353 return -1; 1354 } 1355 1356 static inline int 1357 open_iface(const char *key __rte_unused, const char *value, void *extra_args) 1358 { 1359 const char **iface_name = extra_args; 1360 1361 if (value == NULL) 1362 return -1; 1363 1364 *iface_name = value; 1365 1366 return 0; 1367 } 1368 1369 static inline int 1370 open_int(const char *key __rte_unused, const char *value, void *extra_args) 1371 { 1372 uint16_t *n = extra_args; 1373 1374 if (value == NULL || extra_args == NULL) 1375 return -EINVAL; 1376 1377 *n = (uint16_t)strtoul(value, NULL, 0); 1378 if (*n == USHRT_MAX && errno == ERANGE) 1379 return -1; 1380 1381 return 0; 1382 } 1383 1384 static int 1385 rte_pmd_vhost_probe(struct rte_vdev_device *dev) 1386 { 1387 struct rte_kvargs *kvlist = NULL; 1388 int ret = 0; 1389 char *iface_name; 1390 uint16_t queues; 1391 uint64_t flags = 0; 1392 uint64_t disable_flags = 0; 1393 int client_mode = 0; 1394 int dequeue_zero_copy = 0; 1395 int iommu_support = 0; 1396 int postcopy_support = 0; 1397 int tso = 0; 1398 int linear_buf = 0; 1399 int ext_buf = 0; 1400 struct rte_eth_dev *eth_dev; 1401 const char *name = rte_vdev_device_name(dev); 1402 1403 VHOST_LOG(INFO, "Initializing pmd_vhost for %s\n", name); 1404 1405 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1406 eth_dev = rte_eth_dev_attach_secondary(name); 1407 if (!eth_dev) { 1408 VHOST_LOG(ERR, "Failed to probe %s\n", name); 1409 return -1; 1410 } 1411 eth_dev->rx_pkt_burst = eth_vhost_rx; 1412 eth_dev->tx_pkt_burst = eth_vhost_tx; 1413 eth_dev->dev_ops = &ops; 1414 if (dev->device.numa_node == SOCKET_ID_ANY) 1415 dev->device.numa_node = rte_socket_id(); 1416 eth_dev->device = &dev->device; 1417 rte_eth_dev_probing_finish(eth_dev); 1418 return 0; 1419 } 1420 1421 kvlist = rte_kvargs_parse(rte_vdev_device_args(dev), valid_arguments); 1422 if (kvlist == NULL) 1423 return -1; 1424 1425 if (rte_kvargs_count(kvlist, ETH_VHOST_IFACE_ARG) == 1) { 1426 ret = rte_kvargs_process(kvlist, ETH_VHOST_IFACE_ARG, 1427 &open_iface, &iface_name); 1428 if (ret < 0) 1429 goto out_free; 1430 } else { 1431 ret = -1; 1432 goto out_free; 1433 } 1434 1435 if (rte_kvargs_count(kvlist, ETH_VHOST_QUEUES_ARG) == 1) { 1436 ret = rte_kvargs_process(kvlist, ETH_VHOST_QUEUES_ARG, 1437 &open_int, &queues); 1438 if (ret < 0 || queues > RTE_MAX_QUEUES_PER_PORT) 1439 goto out_free; 1440 1441 } else 1442 queues = 1; 1443 1444 if (rte_kvargs_count(kvlist, ETH_VHOST_CLIENT_ARG) == 1) { 1445 ret = rte_kvargs_process(kvlist, ETH_VHOST_CLIENT_ARG, 1446 &open_int, &client_mode); 1447 if (ret < 0) 1448 goto out_free; 1449 1450 if (client_mode) 1451 flags |= RTE_VHOST_USER_CLIENT; 1452 } 1453 1454 if (rte_kvargs_count(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY) == 1) { 1455 ret = rte_kvargs_process(kvlist, ETH_VHOST_DEQUEUE_ZERO_COPY, 1456 &open_int, &dequeue_zero_copy); 1457 if (ret < 0) 1458 goto out_free; 1459 1460 if (dequeue_zero_copy) 1461 flags |= RTE_VHOST_USER_DEQUEUE_ZERO_COPY; 1462 } 1463 1464 if (rte_kvargs_count(kvlist, ETH_VHOST_IOMMU_SUPPORT) == 1) { 1465 ret = rte_kvargs_process(kvlist, ETH_VHOST_IOMMU_SUPPORT, 1466 &open_int, &iommu_support); 1467 if (ret < 0) 1468 goto out_free; 1469 1470 if (iommu_support) 1471 flags |= RTE_VHOST_USER_IOMMU_SUPPORT; 1472 } 1473 1474 if (rte_kvargs_count(kvlist, ETH_VHOST_POSTCOPY_SUPPORT) == 1) { 1475 ret = rte_kvargs_process(kvlist, ETH_VHOST_POSTCOPY_SUPPORT, 1476 &open_int, &postcopy_support); 1477 if (ret < 0) 1478 goto out_free; 1479 1480 if (postcopy_support) 1481 flags |= RTE_VHOST_USER_POSTCOPY_SUPPORT; 1482 } 1483 1484 if (rte_kvargs_count(kvlist, ETH_VHOST_VIRTIO_NET_F_HOST_TSO) == 1) { 1485 ret = rte_kvargs_process(kvlist, 1486 ETH_VHOST_VIRTIO_NET_F_HOST_TSO, 1487 &open_int, &tso); 1488 if (ret < 0) 1489 goto out_free; 1490 1491 if (tso == 0) { 1492 disable_flags |= (1ULL << VIRTIO_NET_F_HOST_TSO4); 1493 disable_flags |= (1ULL << VIRTIO_NET_F_HOST_TSO6); 1494 } 1495 } 1496 1497 if (rte_kvargs_count(kvlist, ETH_VHOST_LINEAR_BUF) == 1) { 1498 ret = rte_kvargs_process(kvlist, 1499 ETH_VHOST_LINEAR_BUF, 1500 &open_int, &linear_buf); 1501 if (ret < 0) 1502 goto out_free; 1503 1504 if (linear_buf == 1) 1505 flags |= RTE_VHOST_USER_LINEARBUF_SUPPORT; 1506 } 1507 1508 if (rte_kvargs_count(kvlist, ETH_VHOST_EXT_BUF) == 1) { 1509 ret = rte_kvargs_process(kvlist, 1510 ETH_VHOST_EXT_BUF, 1511 &open_int, &ext_buf); 1512 if (ret < 0) 1513 goto out_free; 1514 1515 if (ext_buf == 1) 1516 flags |= RTE_VHOST_USER_EXTBUF_SUPPORT; 1517 } 1518 1519 if (dev->device.numa_node == SOCKET_ID_ANY) 1520 dev->device.numa_node = rte_socket_id(); 1521 1522 ret = eth_dev_vhost_create(dev, iface_name, queues, 1523 dev->device.numa_node, flags, disable_flags); 1524 if (ret == -1) 1525 VHOST_LOG(ERR, "Failed to create %s\n", name); 1526 1527 out_free: 1528 rte_kvargs_free(kvlist); 1529 return ret; 1530 } 1531 1532 static int 1533 rte_pmd_vhost_remove(struct rte_vdev_device *dev) 1534 { 1535 const char *name; 1536 struct rte_eth_dev *eth_dev = NULL; 1537 1538 name = rte_vdev_device_name(dev); 1539 VHOST_LOG(INFO, "Un-Initializing pmd_vhost for %s\n", name); 1540 1541 /* find an ethdev entry */ 1542 eth_dev = rte_eth_dev_allocated(name); 1543 if (eth_dev == NULL) 1544 return 0; 1545 1546 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1547 return rte_eth_dev_release_port(eth_dev); 1548 1549 eth_dev_close(eth_dev); 1550 1551 rte_eth_dev_release_port(eth_dev); 1552 1553 return 0; 1554 } 1555 1556 static struct rte_vdev_driver pmd_vhost_drv = { 1557 .probe = rte_pmd_vhost_probe, 1558 .remove = rte_pmd_vhost_remove, 1559 }; 1560 1561 RTE_PMD_REGISTER_VDEV(net_vhost, pmd_vhost_drv); 1562 RTE_PMD_REGISTER_ALIAS(net_vhost, eth_vhost); 1563 RTE_PMD_REGISTER_PARAM_STRING(net_vhost, 1564 "iface=<ifc> " 1565 "queues=<int> " 1566 "client=<0|1> " 1567 "dequeue-zero-copy=<0|1> " 1568 "iommu-support=<0|1> " 1569 "postcopy-support=<0|1> " 1570 "tso=<0|1> " 1571 "linear-buffer=<0|1> " 1572 "ext-buffer=<0|1>"); 1573