1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2015 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdint.h> 35 #include <string.h> 36 #include <stdio.h> 37 #include <errno.h> 38 #include <unistd.h> 39 #ifdef RTE_EXEC_ENV_LINUXAPP 40 #include <dirent.h> 41 #include <fcntl.h> 42 #endif 43 44 #include <rte_ethdev.h> 45 #include <rte_memcpy.h> 46 #include <rte_string_fns.h> 47 #include <rte_memzone.h> 48 #include <rte_malloc.h> 49 #include <rte_atomic.h> 50 #include <rte_branch_prediction.h> 51 #include <rte_pci.h> 52 #include <rte_ether.h> 53 #include <rte_common.h> 54 #include <rte_errno.h> 55 56 #include <rte_memory.h> 57 #include <rte_eal.h> 58 #include <rte_dev.h> 59 60 #include "virtio_ethdev.h" 61 #include "virtio_pci.h" 62 #include "virtio_logs.h" 63 #include "virtqueue.h" 64 65 66 static int eth_virtio_dev_init(struct rte_eth_dev *eth_dev); 67 static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev); 68 static int virtio_dev_configure(struct rte_eth_dev *dev); 69 static int virtio_dev_start(struct rte_eth_dev *dev); 70 static void virtio_dev_stop(struct rte_eth_dev *dev); 71 static void virtio_dev_promiscuous_enable(struct rte_eth_dev *dev); 72 static void virtio_dev_promiscuous_disable(struct rte_eth_dev *dev); 73 static void virtio_dev_allmulticast_enable(struct rte_eth_dev *dev); 74 static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev); 75 static void virtio_dev_info_get(struct rte_eth_dev *dev, 76 struct rte_eth_dev_info *dev_info); 77 static int virtio_dev_link_update(struct rte_eth_dev *dev, 78 __rte_unused int wait_to_complete); 79 80 static void virtio_set_hwaddr(struct virtio_hw *hw); 81 static void virtio_get_hwaddr(struct virtio_hw *hw); 82 83 static void virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); 84 static void virtio_dev_stats_reset(struct rte_eth_dev *dev); 85 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev); 86 static int virtio_vlan_filter_set(struct rte_eth_dev *dev, 87 uint16_t vlan_id, int on); 88 static void virtio_mac_addr_add(struct rte_eth_dev *dev, 89 struct ether_addr *mac_addr, 90 uint32_t index, uint32_t vmdq __rte_unused); 91 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); 92 static void virtio_mac_addr_set(struct rte_eth_dev *dev, 93 struct ether_addr *mac_addr); 94 95 static int virtio_dev_queue_stats_mapping_set( 96 __rte_unused struct rte_eth_dev *eth_dev, 97 __rte_unused uint16_t queue_id, 98 __rte_unused uint8_t stat_idx, 99 __rte_unused uint8_t is_rx); 100 101 /* 102 * The set of PCI devices this driver supports 103 */ 104 static const struct rte_pci_id pci_id_virtio_map[] = { 105 106 #define RTE_PCI_DEV_ID_DECL_VIRTIO(vend, dev) {RTE_PCI_DEVICE(vend, dev)}, 107 #include "rte_pci_dev_ids.h" 108 109 { .vendor_id = 0, /* sentinel */ }, 110 }; 111 112 static int 113 virtio_send_command(struct virtqueue *vq, struct virtio_pmd_ctrl *ctrl, 114 int *dlen, int pkt_num) 115 { 116 uint32_t head, i; 117 int k, sum = 0; 118 virtio_net_ctrl_ack status = ~0; 119 struct virtio_pmd_ctrl result; 120 121 ctrl->status = status; 122 123 if (!(vq && vq->hw->cvq)) { 124 PMD_INIT_LOG(ERR, 125 "%s(): Control queue is not supported.", 126 __func__); 127 return -1; 128 } 129 head = vq->vq_desc_head_idx; 130 131 PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, " 132 "vq->hw->cvq = %p vq = %p", 133 vq->vq_desc_head_idx, status, vq->hw->cvq, vq); 134 135 if ((vq->vq_free_cnt < ((uint32_t)pkt_num + 2)) || (pkt_num < 1)) 136 return -1; 137 138 memcpy(vq->virtio_net_hdr_mz->addr, ctrl, 139 sizeof(struct virtio_pmd_ctrl)); 140 141 /* 142 * Format is enforced in qemu code: 143 * One TX packet for header; 144 * At least one TX packet per argument; 145 * One RX packet for ACK. 146 */ 147 vq->vq_ring.desc[head].flags = VRING_DESC_F_NEXT; 148 vq->vq_ring.desc[head].addr = vq->virtio_net_hdr_mz->phys_addr; 149 vq->vq_ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr); 150 vq->vq_free_cnt--; 151 i = vq->vq_ring.desc[head].next; 152 153 for (k = 0; k < pkt_num; k++) { 154 vq->vq_ring.desc[i].flags = VRING_DESC_F_NEXT; 155 vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr 156 + sizeof(struct virtio_net_ctrl_hdr) 157 + sizeof(ctrl->status) + sizeof(uint8_t)*sum; 158 vq->vq_ring.desc[i].len = dlen[k]; 159 sum += dlen[k]; 160 vq->vq_free_cnt--; 161 i = vq->vq_ring.desc[i].next; 162 } 163 164 vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE; 165 vq->vq_ring.desc[i].addr = vq->virtio_net_hdr_mz->phys_addr 166 + sizeof(struct virtio_net_ctrl_hdr); 167 vq->vq_ring.desc[i].len = sizeof(ctrl->status); 168 vq->vq_free_cnt--; 169 170 vq->vq_desc_head_idx = vq->vq_ring.desc[i].next; 171 172 vq_update_avail_ring(vq, head); 173 vq_update_avail_idx(vq); 174 175 PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index); 176 177 virtqueue_notify(vq); 178 179 rte_rmb(); 180 while (vq->vq_used_cons_idx == vq->vq_ring.used->idx) { 181 rte_rmb(); 182 usleep(100); 183 } 184 185 while (vq->vq_used_cons_idx != vq->vq_ring.used->idx) { 186 uint32_t idx, desc_idx, used_idx; 187 struct vring_used_elem *uep; 188 189 used_idx = (uint32_t)(vq->vq_used_cons_idx 190 & (vq->vq_nentries - 1)); 191 uep = &vq->vq_ring.used->ring[used_idx]; 192 idx = (uint32_t) uep->id; 193 desc_idx = idx; 194 195 while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) { 196 desc_idx = vq->vq_ring.desc[desc_idx].next; 197 vq->vq_free_cnt++; 198 } 199 200 vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx; 201 vq->vq_desc_head_idx = idx; 202 203 vq->vq_used_cons_idx++; 204 vq->vq_free_cnt++; 205 } 206 207 PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d", 208 vq->vq_free_cnt, vq->vq_desc_head_idx); 209 210 memcpy(&result, vq->virtio_net_hdr_mz->addr, 211 sizeof(struct virtio_pmd_ctrl)); 212 213 return result.status; 214 } 215 216 static int 217 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues) 218 { 219 struct virtio_hw *hw = dev->data->dev_private; 220 struct virtio_pmd_ctrl ctrl; 221 int dlen[1]; 222 int ret; 223 224 ctrl.hdr.class = VIRTIO_NET_CTRL_MQ; 225 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 226 memcpy(ctrl.data, &nb_queues, sizeof(uint16_t)); 227 228 dlen[0] = sizeof(uint16_t); 229 230 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 231 if (ret) { 232 PMD_INIT_LOG(ERR, "Multiqueue configured but send command " 233 "failed, this is too late now..."); 234 return -EINVAL; 235 } 236 237 return 0; 238 } 239 240 void 241 virtio_dev_queue_release(struct virtqueue *vq) { 242 struct virtio_hw *hw = vq->hw; 243 244 if (vq) { 245 /* Select and deactivate the queue */ 246 VIRTIO_WRITE_REG_2(hw, VIRTIO_PCI_QUEUE_SEL, vq->queue_id); 247 VIRTIO_WRITE_REG_4(hw, VIRTIO_PCI_QUEUE_PFN, 0); 248 249 rte_free(vq); 250 vq = NULL; 251 } 252 } 253 254 int virtio_dev_queue_setup(struct rte_eth_dev *dev, 255 int queue_type, 256 uint16_t queue_idx, 257 uint16_t vtpci_queue_idx, 258 uint16_t nb_desc, 259 unsigned int socket_id, 260 struct virtqueue **pvq) 261 { 262 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 263 const struct rte_memzone *mz; 264 uint16_t vq_size; 265 int size; 266 struct virtio_hw *hw = dev->data->dev_private; 267 struct virtqueue *vq = NULL; 268 269 /* Write the virtqueue index to the Queue Select Field */ 270 VIRTIO_WRITE_REG_2(hw, VIRTIO_PCI_QUEUE_SEL, vtpci_queue_idx); 271 PMD_INIT_LOG(DEBUG, "selecting queue: %d", vtpci_queue_idx); 272 273 /* 274 * Read the virtqueue size from the Queue Size field 275 * Always power of 2 and if 0 virtqueue does not exist 276 */ 277 vq_size = VIRTIO_READ_REG_2(hw, VIRTIO_PCI_QUEUE_NUM); 278 PMD_INIT_LOG(DEBUG, "vq_size: %d nb_desc:%d", vq_size, nb_desc); 279 if (vq_size == 0) { 280 PMD_INIT_LOG(ERR, "%s: virtqueue does not exist", __func__); 281 return -EINVAL; 282 } 283 284 if (!rte_is_power_of_2(vq_size)) { 285 PMD_INIT_LOG(ERR, "%s: virtqueue size is not powerof 2", __func__); 286 return -EINVAL; 287 } 288 289 if (queue_type == VTNET_RQ) { 290 snprintf(vq_name, sizeof(vq_name), "port%d_rvq%d", 291 dev->data->port_id, queue_idx); 292 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + 293 vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); 294 } else if (queue_type == VTNET_TQ) { 295 snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d", 296 dev->data->port_id, queue_idx); 297 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + 298 vq_size * sizeof(struct vq_desc_extra), RTE_CACHE_LINE_SIZE); 299 } else if (queue_type == VTNET_CQ) { 300 snprintf(vq_name, sizeof(vq_name), "port%d_cvq", 301 dev->data->port_id); 302 vq = rte_zmalloc(vq_name, sizeof(struct virtqueue) + 303 vq_size * sizeof(struct vq_desc_extra), 304 RTE_CACHE_LINE_SIZE); 305 } 306 if (vq == NULL) { 307 PMD_INIT_LOG(ERR, "%s: Can not allocate virtqueue", __func__); 308 return (-ENOMEM); 309 } 310 311 vq->hw = hw; 312 vq->port_id = dev->data->port_id; 313 vq->queue_id = queue_idx; 314 vq->vq_queue_index = vtpci_queue_idx; 315 vq->vq_nentries = vq_size; 316 317 if (nb_desc == 0 || nb_desc > vq_size) 318 nb_desc = vq_size; 319 vq->vq_free_cnt = nb_desc; 320 321 /* 322 * Reserve a memzone for vring elements 323 */ 324 size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN); 325 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN); 326 PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", size, vq->vq_ring_size); 327 328 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, 329 socket_id, 0, VIRTIO_PCI_VRING_ALIGN); 330 if (mz == NULL) { 331 if (rte_errno == EEXIST) 332 mz = rte_memzone_lookup(vq_name); 333 if (mz == NULL) { 334 rte_free(vq); 335 return -ENOMEM; 336 } 337 } 338 339 /* 340 * Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 341 * and only accepts 32 bit page frame number. 342 * Check if the allocated physical memory exceeds 16TB. 343 */ 344 if ((mz->phys_addr + vq->vq_ring_size - 1) >> (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 345 PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!"); 346 rte_free(vq); 347 return -ENOMEM; 348 } 349 350 memset(mz->addr, 0, sizeof(mz->len)); 351 vq->mz = mz; 352 vq->vq_ring_mem = mz->phys_addr; 353 vq->vq_ring_virt_mem = mz->addr; 354 PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%"PRIx64, (uint64_t)mz->phys_addr); 355 PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%"PRIx64, (uint64_t)(uintptr_t)mz->addr); 356 vq->virtio_net_hdr_mz = NULL; 357 vq->virtio_net_hdr_mem = 0; 358 359 if (queue_type == VTNET_TQ) { 360 /* 361 * For each xmit packet, allocate a virtio_net_hdr 362 */ 363 snprintf(vq_name, sizeof(vq_name), "port%d_tvq%d_hdrzone", 364 dev->data->port_id, queue_idx); 365 vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name, 366 vq_size * hw->vtnet_hdr_size, 367 socket_id, 0, RTE_CACHE_LINE_SIZE); 368 if (vq->virtio_net_hdr_mz == NULL) { 369 if (rte_errno == EEXIST) 370 vq->virtio_net_hdr_mz = 371 rte_memzone_lookup(vq_name); 372 if (vq->virtio_net_hdr_mz == NULL) { 373 rte_free(vq); 374 return -ENOMEM; 375 } 376 } 377 vq->virtio_net_hdr_mem = 378 vq->virtio_net_hdr_mz->phys_addr; 379 memset(vq->virtio_net_hdr_mz->addr, 0, 380 vq_size * hw->vtnet_hdr_size); 381 } else if (queue_type == VTNET_CQ) { 382 /* Allocate a page for control vq command, data and status */ 383 snprintf(vq_name, sizeof(vq_name), "port%d_cvq_hdrzone", 384 dev->data->port_id); 385 vq->virtio_net_hdr_mz = rte_memzone_reserve_aligned(vq_name, 386 PAGE_SIZE, socket_id, 0, RTE_CACHE_LINE_SIZE); 387 if (vq->virtio_net_hdr_mz == NULL) { 388 if (rte_errno == EEXIST) 389 vq->virtio_net_hdr_mz = 390 rte_memzone_lookup(vq_name); 391 if (vq->virtio_net_hdr_mz == NULL) { 392 rte_free(vq); 393 return -ENOMEM; 394 } 395 } 396 vq->virtio_net_hdr_mem = 397 vq->virtio_net_hdr_mz->phys_addr; 398 memset(vq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE); 399 } 400 401 /* 402 * Set guest physical address of the virtqueue 403 * in VIRTIO_PCI_QUEUE_PFN config register of device 404 */ 405 VIRTIO_WRITE_REG_4(hw, VIRTIO_PCI_QUEUE_PFN, 406 mz->phys_addr >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 407 *pvq = vq; 408 return 0; 409 } 410 411 static int 412 virtio_dev_cq_queue_setup(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx, 413 uint32_t socket_id) 414 { 415 struct virtqueue *vq; 416 int ret; 417 struct virtio_hw *hw = dev->data->dev_private; 418 419 PMD_INIT_FUNC_TRACE(); 420 ret = virtio_dev_queue_setup(dev, VTNET_CQ, VTNET_SQ_CQ_QUEUE_IDX, 421 vtpci_queue_idx, 0, socket_id, &vq); 422 if (ret < 0) { 423 PMD_INIT_LOG(ERR, "control vq initialization failed"); 424 return ret; 425 } 426 427 hw->cvq = vq; 428 return 0; 429 } 430 431 static void 432 virtio_free_queues(struct rte_eth_dev *dev) 433 { 434 unsigned int i; 435 436 for (i = 0; i < dev->data->nb_rx_queues; i++) 437 virtio_dev_rx_queue_release(dev->data->rx_queues[i]); 438 439 dev->data->nb_rx_queues = 0; 440 441 for (i = 0; i < dev->data->nb_tx_queues; i++) 442 virtio_dev_tx_queue_release(dev->data->tx_queues[i]); 443 444 dev->data->nb_tx_queues = 0; 445 } 446 447 static void 448 virtio_dev_close(struct rte_eth_dev *dev) 449 { 450 struct virtio_hw *hw = dev->data->dev_private; 451 struct rte_pci_device *pci_dev = dev->pci_dev; 452 453 PMD_INIT_LOG(DEBUG, "virtio_dev_close"); 454 455 /* reset the NIC */ 456 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 457 vtpci_irq_config(hw, VIRTIO_MSI_NO_VECTOR); 458 vtpci_reset(hw); 459 hw->started = 0; 460 virtio_dev_free_mbufs(dev); 461 virtio_free_queues(dev); 462 } 463 464 static void 465 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev) 466 { 467 struct virtio_hw *hw = dev->data->dev_private; 468 struct virtio_pmd_ctrl ctrl; 469 int dlen[1]; 470 int ret; 471 472 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 473 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 474 return; 475 } 476 477 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 478 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 479 ctrl.data[0] = 1; 480 dlen[0] = 1; 481 482 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 483 if (ret) 484 PMD_INIT_LOG(ERR, "Failed to enable promisc"); 485 } 486 487 static void 488 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev) 489 { 490 struct virtio_hw *hw = dev->data->dev_private; 491 struct virtio_pmd_ctrl ctrl; 492 int dlen[1]; 493 int ret; 494 495 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 496 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 497 return; 498 } 499 500 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 501 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 502 ctrl.data[0] = 0; 503 dlen[0] = 1; 504 505 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 506 if (ret) 507 PMD_INIT_LOG(ERR, "Failed to disable promisc"); 508 } 509 510 static void 511 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev) 512 { 513 struct virtio_hw *hw = dev->data->dev_private; 514 struct virtio_pmd_ctrl ctrl; 515 int dlen[1]; 516 int ret; 517 518 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 519 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 520 return; 521 } 522 523 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 524 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 525 ctrl.data[0] = 1; 526 dlen[0] = 1; 527 528 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 529 if (ret) 530 PMD_INIT_LOG(ERR, "Failed to enable allmulticast"); 531 } 532 533 static void 534 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) 535 { 536 struct virtio_hw *hw = dev->data->dev_private; 537 struct virtio_pmd_ctrl ctrl; 538 int dlen[1]; 539 int ret; 540 541 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 542 PMD_INIT_LOG(INFO, "host does not support rx control\n"); 543 return; 544 } 545 546 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 547 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 548 ctrl.data[0] = 0; 549 dlen[0] = 1; 550 551 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 552 if (ret) 553 PMD_INIT_LOG(ERR, "Failed to disable allmulticast"); 554 } 555 556 /* 557 * dev_ops for virtio, bare necessities for basic operation 558 */ 559 static const struct eth_dev_ops virtio_eth_dev_ops = { 560 .dev_configure = virtio_dev_configure, 561 .dev_start = virtio_dev_start, 562 .dev_stop = virtio_dev_stop, 563 .dev_close = virtio_dev_close, 564 .promiscuous_enable = virtio_dev_promiscuous_enable, 565 .promiscuous_disable = virtio_dev_promiscuous_disable, 566 .allmulticast_enable = virtio_dev_allmulticast_enable, 567 .allmulticast_disable = virtio_dev_allmulticast_disable, 568 569 .dev_infos_get = virtio_dev_info_get, 570 .stats_get = virtio_dev_stats_get, 571 .stats_reset = virtio_dev_stats_reset, 572 .link_update = virtio_dev_link_update, 573 .rx_queue_setup = virtio_dev_rx_queue_setup, 574 .rx_queue_release = virtio_dev_rx_queue_release, 575 .tx_queue_setup = virtio_dev_tx_queue_setup, 576 .tx_queue_release = virtio_dev_tx_queue_release, 577 /* collect stats per queue */ 578 .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, 579 .vlan_filter_set = virtio_vlan_filter_set, 580 .mac_addr_add = virtio_mac_addr_add, 581 .mac_addr_remove = virtio_mac_addr_remove, 582 .mac_addr_set = virtio_mac_addr_set, 583 }; 584 585 static inline int 586 virtio_dev_atomic_read_link_status(struct rte_eth_dev *dev, 587 struct rte_eth_link *link) 588 { 589 struct rte_eth_link *dst = link; 590 struct rte_eth_link *src = &(dev->data->dev_link); 591 592 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 593 *(uint64_t *)src) == 0) 594 return -1; 595 596 return 0; 597 } 598 599 /** 600 * Atomically writes the link status information into global 601 * structure rte_eth_dev. 602 * 603 * @param dev 604 * - Pointer to the structure rte_eth_dev to read from. 605 * - Pointer to the buffer to be saved with the link status. 606 * 607 * @return 608 * - On success, zero. 609 * - On failure, negative value. 610 */ 611 static inline int 612 virtio_dev_atomic_write_link_status(struct rte_eth_dev *dev, 613 struct rte_eth_link *link) 614 { 615 struct rte_eth_link *dst = &(dev->data->dev_link); 616 struct rte_eth_link *src = link; 617 618 if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, 619 *(uint64_t *)src) == 0) 620 return -1; 621 622 return 0; 623 } 624 625 static void 626 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 627 { 628 unsigned i; 629 630 for (i = 0; i < dev->data->nb_tx_queues; i++) { 631 const struct virtqueue *txvq = dev->data->tx_queues[i]; 632 if (txvq == NULL) 633 continue; 634 635 stats->opackets += txvq->packets; 636 stats->obytes += txvq->bytes; 637 stats->oerrors += txvq->errors; 638 639 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 640 stats->q_opackets[i] = txvq->packets; 641 stats->q_obytes[i] = txvq->bytes; 642 } 643 } 644 645 for (i = 0; i < dev->data->nb_rx_queues; i++) { 646 const struct virtqueue *rxvq = dev->data->rx_queues[i]; 647 if (rxvq == NULL) 648 continue; 649 650 stats->ipackets += rxvq->packets; 651 stats->ibytes += rxvq->bytes; 652 stats->ierrors += rxvq->errors; 653 654 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 655 stats->q_ipackets[i] = rxvq->packets; 656 stats->q_ibytes[i] = rxvq->bytes; 657 } 658 } 659 660 stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed; 661 } 662 663 static void 664 virtio_dev_stats_reset(struct rte_eth_dev *dev) 665 { 666 unsigned int i; 667 668 for (i = 0; i < dev->data->nb_tx_queues; i++) { 669 struct virtqueue *txvq = dev->data->tx_queues[i]; 670 if (txvq == NULL) 671 continue; 672 673 txvq->packets = 0; 674 txvq->bytes = 0; 675 txvq->errors = 0; 676 } 677 678 for (i = 0; i < dev->data->nb_rx_queues; i++) { 679 struct virtqueue *rxvq = dev->data->rx_queues[i]; 680 if (rxvq == NULL) 681 continue; 682 683 rxvq->packets = 0; 684 rxvq->bytes = 0; 685 rxvq->errors = 0; 686 } 687 688 dev->data->rx_mbuf_alloc_failed = 0; 689 } 690 691 static void 692 virtio_set_hwaddr(struct virtio_hw *hw) 693 { 694 vtpci_write_dev_config(hw, 695 offsetof(struct virtio_net_config, mac), 696 &hw->mac_addr, ETHER_ADDR_LEN); 697 } 698 699 static void 700 virtio_get_hwaddr(struct virtio_hw *hw) 701 { 702 if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) { 703 vtpci_read_dev_config(hw, 704 offsetof(struct virtio_net_config, mac), 705 &hw->mac_addr, ETHER_ADDR_LEN); 706 } else { 707 eth_random_addr(&hw->mac_addr[0]); 708 virtio_set_hwaddr(hw); 709 } 710 } 711 712 static void 713 virtio_mac_table_set(struct virtio_hw *hw, 714 const struct virtio_net_ctrl_mac *uc, 715 const struct virtio_net_ctrl_mac *mc) 716 { 717 struct virtio_pmd_ctrl ctrl; 718 int err, len[2]; 719 720 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 721 PMD_DRV_LOG(INFO, "host does not support mac table\n"); 722 return; 723 } 724 725 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 726 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 727 728 len[0] = uc->entries * ETHER_ADDR_LEN + sizeof(uc->entries); 729 memcpy(ctrl.data, uc, len[0]); 730 731 len[1] = mc->entries * ETHER_ADDR_LEN + sizeof(mc->entries); 732 memcpy(ctrl.data + len[0], mc, len[1]); 733 734 err = virtio_send_command(hw->cvq, &ctrl, len, 2); 735 if (err != 0) 736 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err); 737 } 738 739 static void 740 virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr, 741 uint32_t index, uint32_t vmdq __rte_unused) 742 { 743 struct virtio_hw *hw = dev->data->dev_private; 744 const struct ether_addr *addrs = dev->data->mac_addrs; 745 unsigned int i; 746 struct virtio_net_ctrl_mac *uc, *mc; 747 748 if (index >= VIRTIO_MAX_MAC_ADDRS) { 749 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 750 return; 751 } 752 753 uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries)); 754 uc->entries = 0; 755 mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries)); 756 mc->entries = 0; 757 758 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 759 const struct ether_addr *addr 760 = (i == index) ? mac_addr : addrs + i; 761 struct virtio_net_ctrl_mac *tbl 762 = is_multicast_ether_addr(addr) ? mc : uc; 763 764 memcpy(&tbl->macs[tbl->entries++], addr, ETHER_ADDR_LEN); 765 } 766 767 virtio_mac_table_set(hw, uc, mc); 768 } 769 770 static void 771 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 772 { 773 struct virtio_hw *hw = dev->data->dev_private; 774 struct ether_addr *addrs = dev->data->mac_addrs; 775 struct virtio_net_ctrl_mac *uc, *mc; 776 unsigned int i; 777 778 if (index >= VIRTIO_MAX_MAC_ADDRS) { 779 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 780 return; 781 } 782 783 uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries)); 784 uc->entries = 0; 785 mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries)); 786 mc->entries = 0; 787 788 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 789 struct virtio_net_ctrl_mac *tbl; 790 791 if (i == index || is_zero_ether_addr(addrs + i)) 792 continue; 793 794 tbl = is_multicast_ether_addr(addrs + i) ? mc : uc; 795 memcpy(&tbl->macs[tbl->entries++], addrs + i, ETHER_ADDR_LEN); 796 } 797 798 virtio_mac_table_set(hw, uc, mc); 799 } 800 801 static void 802 virtio_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) 803 { 804 struct virtio_hw *hw = dev->data->dev_private; 805 806 memcpy(hw->mac_addr, mac_addr, ETHER_ADDR_LEN); 807 808 /* Use atomic update if available */ 809 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 810 struct virtio_pmd_ctrl ctrl; 811 int len = ETHER_ADDR_LEN; 812 813 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 814 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 815 816 memcpy(ctrl.data, mac_addr, ETHER_ADDR_LEN); 817 virtio_send_command(hw->cvq, &ctrl, &len, 1); 818 } else if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) 819 virtio_set_hwaddr(hw); 820 } 821 822 static int 823 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 824 { 825 struct virtio_hw *hw = dev->data->dev_private; 826 struct virtio_pmd_ctrl ctrl; 827 int len; 828 829 if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) 830 return -ENOTSUP; 831 832 ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN; 833 ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 834 memcpy(ctrl.data, &vlan_id, sizeof(vlan_id)); 835 len = sizeof(vlan_id); 836 837 return virtio_send_command(hw->cvq, &ctrl, &len, 1); 838 } 839 840 static void 841 virtio_negotiate_features(struct virtio_hw *hw) 842 { 843 uint32_t host_features; 844 845 /* Prepare guest_features: feature that driver wants to support */ 846 hw->guest_features = VIRTIO_PMD_GUEST_FEATURES; 847 PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %x", 848 hw->guest_features); 849 850 /* Read device(host) feature bits */ 851 host_features = VIRTIO_READ_REG_4(hw, VIRTIO_PCI_HOST_FEATURES); 852 PMD_INIT_LOG(DEBUG, "host_features before negotiate = %x", 853 host_features); 854 855 /* 856 * Negotiate features: Subset of device feature bits are written back 857 * guest feature bits. 858 */ 859 hw->guest_features = vtpci_negotiate_features(hw, host_features); 860 PMD_INIT_LOG(DEBUG, "features after negotiate = %x", 861 hw->guest_features); 862 } 863 864 #ifdef RTE_EXEC_ENV_LINUXAPP 865 static int 866 parse_sysfs_value(const char *filename, unsigned long *val) 867 { 868 FILE *f; 869 char buf[BUFSIZ]; 870 char *end = NULL; 871 872 f = fopen(filename, "r"); 873 if (f == NULL) { 874 PMD_INIT_LOG(ERR, "%s(): cannot open sysfs value %s", 875 __func__, filename); 876 return -1; 877 } 878 879 if (fgets(buf, sizeof(buf), f) == NULL) { 880 PMD_INIT_LOG(ERR, "%s(): cannot read sysfs value %s", 881 __func__, filename); 882 fclose(f); 883 return -1; 884 } 885 *val = strtoul(buf, &end, 0); 886 if ((buf[0] == '\0') || (end == NULL) || (*end != '\n')) { 887 PMD_INIT_LOG(ERR, "%s(): cannot parse sysfs value %s", 888 __func__, filename); 889 fclose(f); 890 return -1; 891 } 892 fclose(f); 893 return 0; 894 } 895 896 static int get_uio_dev(struct rte_pci_addr *loc, char *buf, unsigned int buflen, 897 unsigned int *uio_num) 898 { 899 struct dirent *e; 900 DIR *dir; 901 char dirname[PATH_MAX]; 902 903 /* depending on kernel version, uio can be located in uio/uioX 904 * or uio:uioX */ 905 snprintf(dirname, sizeof(dirname), 906 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/uio", 907 loc->domain, loc->bus, loc->devid, loc->function); 908 dir = opendir(dirname); 909 if (dir == NULL) { 910 /* retry with the parent directory */ 911 snprintf(dirname, sizeof(dirname), 912 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT, 913 loc->domain, loc->bus, loc->devid, loc->function); 914 dir = opendir(dirname); 915 916 if (dir == NULL) { 917 PMD_INIT_LOG(ERR, "Cannot opendir %s", dirname); 918 return -1; 919 } 920 } 921 922 /* take the first file starting with "uio" */ 923 while ((e = readdir(dir)) != NULL) { 924 /* format could be uio%d ...*/ 925 int shortprefix_len = sizeof("uio") - 1; 926 /* ... or uio:uio%d */ 927 int longprefix_len = sizeof("uio:uio") - 1; 928 char *endptr; 929 930 if (strncmp(e->d_name, "uio", 3) != 0) 931 continue; 932 933 /* first try uio%d */ 934 errno = 0; 935 *uio_num = strtoull(e->d_name + shortprefix_len, &endptr, 10); 936 if (errno == 0 && endptr != (e->d_name + shortprefix_len)) { 937 snprintf(buf, buflen, "%s/uio%u", dirname, *uio_num); 938 break; 939 } 940 941 /* then try uio:uio%d */ 942 errno = 0; 943 *uio_num = strtoull(e->d_name + longprefix_len, &endptr, 10); 944 if (errno == 0 && endptr != (e->d_name + longprefix_len)) { 945 snprintf(buf, buflen, "%s/uio:uio%u", dirname, 946 *uio_num); 947 break; 948 } 949 } 950 closedir(dir); 951 952 /* No uio resource found */ 953 if (e == NULL) { 954 PMD_INIT_LOG(ERR, "Could not find uio resource"); 955 return -1; 956 } 957 958 return 0; 959 } 960 961 static int 962 virtio_has_msix(const struct rte_pci_addr *loc) 963 { 964 DIR *d; 965 char dirname[PATH_MAX]; 966 967 snprintf(dirname, sizeof(dirname), 968 SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/msi_irqs", 969 loc->domain, loc->bus, loc->devid, loc->function); 970 971 d = opendir(dirname); 972 if (d) 973 closedir(d); 974 975 return (d != NULL); 976 } 977 978 /* Extract I/O port numbers from sysfs */ 979 static int virtio_resource_init_by_uio(struct rte_pci_device *pci_dev) 980 { 981 char dirname[PATH_MAX]; 982 char filename[PATH_MAX]; 983 unsigned long start, size; 984 unsigned int uio_num; 985 986 if (get_uio_dev(&pci_dev->addr, dirname, sizeof(dirname), &uio_num) < 0) 987 return -1; 988 989 /* get portio size */ 990 snprintf(filename, sizeof(filename), 991 "%s/portio/port0/size", dirname); 992 if (parse_sysfs_value(filename, &size) < 0) { 993 PMD_INIT_LOG(ERR, "%s(): cannot parse size", 994 __func__); 995 return -1; 996 } 997 998 /* get portio start */ 999 snprintf(filename, sizeof(filename), 1000 "%s/portio/port0/start", dirname); 1001 if (parse_sysfs_value(filename, &start) < 0) { 1002 PMD_INIT_LOG(ERR, "%s(): cannot parse portio start", 1003 __func__); 1004 return -1; 1005 } 1006 pci_dev->mem_resource[0].addr = (void *)(uintptr_t)start; 1007 pci_dev->mem_resource[0].len = (uint64_t)size; 1008 PMD_INIT_LOG(DEBUG, 1009 "PCI Port IO found start=0x%lx with size=0x%lx", 1010 start, size); 1011 1012 /* save fd */ 1013 memset(dirname, 0, sizeof(dirname)); 1014 snprintf(dirname, sizeof(dirname), "/dev/uio%u", uio_num); 1015 pci_dev->intr_handle.fd = open(dirname, O_RDWR); 1016 if (pci_dev->intr_handle.fd < 0) { 1017 PMD_INIT_LOG(ERR, "Cannot open %s: %s\n", 1018 dirname, strerror(errno)); 1019 return -1; 1020 } 1021 1022 pci_dev->intr_handle.type = RTE_INTR_HANDLE_UIO; 1023 pci_dev->driver->drv_flags |= RTE_PCI_DRV_INTR_LSC; 1024 1025 return 0; 1026 } 1027 1028 /* Extract port I/O numbers from proc/ioports */ 1029 static int virtio_resource_init_by_ioports(struct rte_pci_device *pci_dev) 1030 { 1031 uint16_t start, end; 1032 int size; 1033 FILE *fp; 1034 char *line = NULL; 1035 char pci_id[16]; 1036 int found = 0; 1037 size_t linesz; 1038 1039 snprintf(pci_id, sizeof(pci_id), PCI_PRI_FMT, 1040 pci_dev->addr.domain, 1041 pci_dev->addr.bus, 1042 pci_dev->addr.devid, 1043 pci_dev->addr.function); 1044 1045 fp = fopen("/proc/ioports", "r"); 1046 if (fp == NULL) { 1047 PMD_INIT_LOG(ERR, "%s(): can't open ioports", __func__); 1048 return -1; 1049 } 1050 1051 while (getdelim(&line, &linesz, '\n', fp) > 0) { 1052 char *ptr = line; 1053 char *left; 1054 int n; 1055 1056 n = strcspn(ptr, ":"); 1057 ptr[n] = 0; 1058 left = &ptr[n+1]; 1059 1060 while (*left && isspace(*left)) 1061 left++; 1062 1063 if (!strncmp(left, pci_id, strlen(pci_id))) { 1064 found = 1; 1065 1066 while (*ptr && isspace(*ptr)) 1067 ptr++; 1068 1069 sscanf(ptr, "%04hx-%04hx", &start, &end); 1070 size = end - start + 1; 1071 1072 break; 1073 } 1074 } 1075 1076 free(line); 1077 fclose(fp); 1078 1079 if (!found) 1080 return -1; 1081 1082 pci_dev->mem_resource[0].addr = (void *)(uintptr_t)(uint32_t)start; 1083 pci_dev->mem_resource[0].len = (uint64_t)size; 1084 PMD_INIT_LOG(DEBUG, 1085 "PCI Port IO found start=0x%x with size=0x%x", 1086 start, size); 1087 1088 /* can't support lsc interrupt without uio */ 1089 pci_dev->driver->drv_flags &= ~RTE_PCI_DRV_INTR_LSC; 1090 1091 return 0; 1092 } 1093 1094 /* Extract I/O port numbers from sysfs */ 1095 static int virtio_resource_init(struct rte_pci_device *pci_dev) 1096 { 1097 if (virtio_resource_init_by_uio(pci_dev) == 0) 1098 return 0; 1099 else 1100 return virtio_resource_init_by_ioports(pci_dev); 1101 } 1102 1103 #else 1104 static int 1105 virtio_has_msix(const struct rte_pci_addr *loc __rte_unused) 1106 { 1107 /* nic_uio does not enable interrupts, return 0 (false). */ 1108 return 0; 1109 } 1110 1111 static int virtio_resource_init(struct rte_pci_device *pci_dev __rte_unused) 1112 { 1113 /* no setup required */ 1114 return 0; 1115 } 1116 #endif 1117 1118 /* 1119 * Process Virtio Config changed interrupt and call the callback 1120 * if link state changed. 1121 */ 1122 static void 1123 virtio_interrupt_handler(__rte_unused struct rte_intr_handle *handle, 1124 void *param) 1125 { 1126 struct rte_eth_dev *dev = param; 1127 struct virtio_hw *hw = dev->data->dev_private; 1128 uint8_t isr; 1129 1130 /* Read interrupt status which clears interrupt */ 1131 isr = vtpci_isr(hw); 1132 PMD_DRV_LOG(INFO, "interrupt status = %#x", isr); 1133 1134 if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0) 1135 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1136 1137 if (isr & VIRTIO_PCI_ISR_CONFIG) { 1138 if (virtio_dev_link_update(dev, 0) == 0) 1139 _rte_eth_dev_callback_process(dev, 1140 RTE_ETH_EVENT_INTR_LSC); 1141 } 1142 1143 } 1144 1145 static void 1146 rx_func_get(struct rte_eth_dev *eth_dev) 1147 { 1148 struct virtio_hw *hw = eth_dev->data->dev_private; 1149 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) 1150 eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts; 1151 else 1152 eth_dev->rx_pkt_burst = &virtio_recv_pkts; 1153 } 1154 1155 /* 1156 * This function is based on probe() function in virtio_pci.c 1157 * It returns 0 on success. 1158 */ 1159 static int 1160 eth_virtio_dev_init(struct rte_eth_dev *eth_dev) 1161 { 1162 struct virtio_hw *hw = eth_dev->data->dev_private; 1163 struct virtio_net_config *config; 1164 struct virtio_net_config local_config; 1165 uint32_t offset_conf = sizeof(config->mac); 1166 struct rte_pci_device *pci_dev; 1167 1168 RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr)); 1169 1170 eth_dev->dev_ops = &virtio_eth_dev_ops; 1171 eth_dev->tx_pkt_burst = &virtio_xmit_pkts; 1172 1173 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1174 rx_func_get(eth_dev); 1175 return 0; 1176 } 1177 1178 /* Allocate memory for storing MAC addresses */ 1179 eth_dev->data->mac_addrs = rte_zmalloc("virtio", ETHER_ADDR_LEN, 0); 1180 if (eth_dev->data->mac_addrs == NULL) { 1181 PMD_INIT_LOG(ERR, 1182 "Failed to allocate %d bytes needed to store MAC addresses", 1183 ETHER_ADDR_LEN); 1184 return -ENOMEM; 1185 } 1186 1187 pci_dev = eth_dev->pci_dev; 1188 if (virtio_resource_init(pci_dev) < 0) 1189 return -1; 1190 1191 hw->use_msix = virtio_has_msix(&pci_dev->addr); 1192 hw->io_base = (uint32_t)(uintptr_t)pci_dev->mem_resource[0].addr; 1193 1194 /* Reset the device although not necessary at startup */ 1195 vtpci_reset(hw); 1196 1197 /* Tell the host we've noticed this device. */ 1198 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1199 1200 /* Tell the host we've known how to drive the device. */ 1201 vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1202 virtio_negotiate_features(hw); 1203 1204 rx_func_get(eth_dev); 1205 1206 /* Setting up rx_header size for the device */ 1207 if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) 1208 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1209 else 1210 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 1211 1212 /* Copy the permanent MAC address to: virtio_hw */ 1213 virtio_get_hwaddr(hw); 1214 ether_addr_copy((struct ether_addr *) hw->mac_addr, 1215 ð_dev->data->mac_addrs[0]); 1216 PMD_INIT_LOG(DEBUG, 1217 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X", 1218 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 1219 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 1220 1221 if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { 1222 config = &local_config; 1223 1224 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1225 offset_conf += sizeof(config->status); 1226 } else { 1227 PMD_INIT_LOG(DEBUG, 1228 "VIRTIO_NET_F_STATUS is not supported"); 1229 config->status = 0; 1230 } 1231 1232 if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) { 1233 offset_conf += sizeof(config->max_virtqueue_pairs); 1234 } else { 1235 PMD_INIT_LOG(DEBUG, 1236 "VIRTIO_NET_F_MQ is not supported"); 1237 config->max_virtqueue_pairs = 1; 1238 } 1239 1240 vtpci_read_dev_config(hw, 0, (uint8_t *)config, offset_conf); 1241 1242 hw->max_rx_queues = 1243 (VIRTIO_MAX_RX_QUEUES < config->max_virtqueue_pairs) ? 1244 VIRTIO_MAX_RX_QUEUES : config->max_virtqueue_pairs; 1245 hw->max_tx_queues = 1246 (VIRTIO_MAX_TX_QUEUES < config->max_virtqueue_pairs) ? 1247 VIRTIO_MAX_TX_QUEUES : config->max_virtqueue_pairs; 1248 1249 virtio_dev_cq_queue_setup(eth_dev, 1250 config->max_virtqueue_pairs * 2, 1251 SOCKET_ID_ANY); 1252 1253 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d", 1254 config->max_virtqueue_pairs); 1255 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status); 1256 PMD_INIT_LOG(DEBUG, 1257 "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X", 1258 config->mac[0], config->mac[1], 1259 config->mac[2], config->mac[3], 1260 config->mac[4], config->mac[5]); 1261 } else { 1262 hw->max_rx_queues = 1; 1263 hw->max_tx_queues = 1; 1264 } 1265 1266 eth_dev->data->nb_rx_queues = hw->max_rx_queues; 1267 eth_dev->data->nb_tx_queues = hw->max_tx_queues; 1268 1269 PMD_INIT_LOG(DEBUG, "hw->max_rx_queues=%d hw->max_tx_queues=%d", 1270 hw->max_rx_queues, hw->max_tx_queues); 1271 PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x", 1272 eth_dev->data->port_id, pci_dev->id.vendor_id, 1273 pci_dev->id.device_id); 1274 1275 /* Setup interrupt callback */ 1276 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 1277 rte_intr_callback_register(&pci_dev->intr_handle, 1278 virtio_interrupt_handler, eth_dev); 1279 1280 virtio_dev_cq_start(eth_dev); 1281 1282 return 0; 1283 } 1284 1285 static int 1286 eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev) 1287 { 1288 struct rte_pci_device *pci_dev; 1289 struct virtio_hw *hw = eth_dev->data->dev_private; 1290 1291 PMD_INIT_FUNC_TRACE(); 1292 1293 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 1294 return -EPERM; 1295 1296 if (hw->started == 1) { 1297 virtio_dev_stop(eth_dev); 1298 virtio_dev_close(eth_dev); 1299 } 1300 pci_dev = eth_dev->pci_dev; 1301 1302 eth_dev->dev_ops = NULL; 1303 eth_dev->tx_pkt_burst = NULL; 1304 eth_dev->rx_pkt_burst = NULL; 1305 1306 virtio_dev_queue_release(hw->cvq); 1307 1308 rte_free(eth_dev->data->mac_addrs); 1309 eth_dev->data->mac_addrs = NULL; 1310 1311 /* reset interrupt callback */ 1312 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 1313 rte_intr_callback_unregister(&pci_dev->intr_handle, 1314 virtio_interrupt_handler, 1315 eth_dev); 1316 1317 PMD_INIT_LOG(DEBUG, "dev_uninit completed"); 1318 1319 return 0; 1320 } 1321 1322 static struct eth_driver rte_virtio_pmd = { 1323 .pci_drv = { 1324 .name = "rte_virtio_pmd", 1325 .id_table = pci_id_virtio_map, 1326 .drv_flags = RTE_PCI_DRV_DETACHABLE, 1327 }, 1328 .eth_dev_init = eth_virtio_dev_init, 1329 .eth_dev_uninit = eth_virtio_dev_uninit, 1330 .dev_private_size = sizeof(struct virtio_hw), 1331 }; 1332 1333 /* 1334 * Driver initialization routine. 1335 * Invoked once at EAL init time. 1336 * Register itself as the [Poll Mode] Driver of PCI virtio devices. 1337 * Returns 0 on success. 1338 */ 1339 static int 1340 rte_virtio_pmd_init(const char *name __rte_unused, 1341 const char *param __rte_unused) 1342 { 1343 if (rte_eal_iopl_init() != 0) { 1344 PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD"); 1345 return -1; 1346 } 1347 1348 rte_eth_driver_register(&rte_virtio_pmd); 1349 return 0; 1350 } 1351 1352 /* 1353 * Configure virtio device 1354 * It returns 0 on success. 1355 */ 1356 static int 1357 virtio_dev_configure(struct rte_eth_dev *dev) 1358 { 1359 const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 1360 struct virtio_hw *hw = dev->data->dev_private; 1361 struct rte_pci_device *pci_dev = dev->pci_dev; 1362 1363 PMD_INIT_LOG(DEBUG, "configure"); 1364 1365 if (rxmode->hw_ip_checksum) { 1366 PMD_DRV_LOG(ERR, "HW IP checksum not supported"); 1367 return (-EINVAL); 1368 } 1369 1370 hw->vlan_strip = rxmode->hw_vlan_strip; 1371 1372 if (rxmode->hw_vlan_filter 1373 && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { 1374 PMD_DRV_LOG(NOTICE, 1375 "vlan filtering not available on this host"); 1376 return -ENOTSUP; 1377 } 1378 1379 if (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC) 1380 if (vtpci_irq_config(hw, 0) == VIRTIO_MSI_NO_VECTOR) { 1381 PMD_DRV_LOG(ERR, "failed to set config vector"); 1382 return -EBUSY; 1383 } 1384 1385 return 0; 1386 } 1387 1388 1389 static int 1390 virtio_dev_start(struct rte_eth_dev *dev) 1391 { 1392 uint16_t nb_queues, i; 1393 struct virtio_hw *hw = dev->data->dev_private; 1394 struct rte_pci_device *pci_dev = dev->pci_dev; 1395 1396 /* check if lsc interrupt feature is enabled */ 1397 if ((dev->data->dev_conf.intr_conf.lsc) && 1398 (pci_dev->driver->drv_flags & RTE_PCI_DRV_INTR_LSC)) { 1399 if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1400 PMD_DRV_LOG(ERR, "link status not supported by host"); 1401 return -ENOTSUP; 1402 } 1403 1404 if (rte_intr_enable(&dev->pci_dev->intr_handle) < 0) { 1405 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1406 return -EIO; 1407 } 1408 } 1409 1410 /* Initialize Link state */ 1411 virtio_dev_link_update(dev, 0); 1412 1413 /* On restart after stop do not touch queues */ 1414 if (hw->started) 1415 return 0; 1416 1417 /* Do final configuration before rx/tx engine starts */ 1418 virtio_dev_rxtx_start(dev); 1419 vtpci_reinit_complete(hw); 1420 1421 hw->started = 1; 1422 1423 /*Notify the backend 1424 *Otherwise the tap backend might already stop its queue due to fullness. 1425 *vhost backend will have no chance to be waked up 1426 */ 1427 nb_queues = dev->data->nb_rx_queues; 1428 if (nb_queues > 1) { 1429 if (virtio_set_multiple_queues(dev, nb_queues) != 0) 1430 return -EINVAL; 1431 } 1432 1433 PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues); 1434 1435 for (i = 0; i < nb_queues; i++) 1436 virtqueue_notify(dev->data->rx_queues[i]); 1437 1438 PMD_INIT_LOG(DEBUG, "Notified backend at initialization"); 1439 1440 for (i = 0; i < dev->data->nb_rx_queues; i++) 1441 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); 1442 1443 for (i = 0; i < dev->data->nb_tx_queues; i++) 1444 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); 1445 1446 return 0; 1447 } 1448 1449 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) 1450 { 1451 struct rte_mbuf *buf; 1452 int i, mbuf_num = 0; 1453 1454 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1455 PMD_INIT_LOG(DEBUG, 1456 "Before freeing rxq[%d] used and unused buf", i); 1457 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); 1458 1459 PMD_INIT_LOG(DEBUG, "rx_queues[%d]=%p", 1460 i, dev->data->rx_queues[i]); 1461 while ((buf = (struct rte_mbuf *)virtqueue_detatch_unused( 1462 dev->data->rx_queues[i])) != NULL) { 1463 rte_pktmbuf_free(buf); 1464 mbuf_num++; 1465 } 1466 1467 PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num); 1468 PMD_INIT_LOG(DEBUG, 1469 "After freeing rxq[%d] used and unused buf", i); 1470 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->rx_queues[i]); 1471 } 1472 1473 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1474 PMD_INIT_LOG(DEBUG, 1475 "Before freeing txq[%d] used and unused bufs", 1476 i); 1477 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); 1478 1479 mbuf_num = 0; 1480 while ((buf = (struct rte_mbuf *)virtqueue_detatch_unused( 1481 dev->data->tx_queues[i])) != NULL) { 1482 rte_pktmbuf_free(buf); 1483 1484 mbuf_num++; 1485 } 1486 1487 PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num); 1488 PMD_INIT_LOG(DEBUG, 1489 "After freeing txq[%d] used and unused buf", i); 1490 VIRTQUEUE_DUMP((struct virtqueue *)dev->data->tx_queues[i]); 1491 } 1492 } 1493 1494 /* 1495 * Stop device: disable interrupt and mark link down 1496 */ 1497 static void 1498 virtio_dev_stop(struct rte_eth_dev *dev) 1499 { 1500 struct rte_eth_link link; 1501 1502 PMD_INIT_LOG(DEBUG, "stop"); 1503 1504 if (dev->data->dev_conf.intr_conf.lsc) 1505 rte_intr_disable(&dev->pci_dev->intr_handle); 1506 1507 memset(&link, 0, sizeof(link)); 1508 virtio_dev_atomic_write_link_status(dev, &link); 1509 } 1510 1511 static int 1512 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 1513 { 1514 struct rte_eth_link link, old; 1515 uint16_t status; 1516 struct virtio_hw *hw = dev->data->dev_private; 1517 memset(&link, 0, sizeof(link)); 1518 virtio_dev_atomic_read_link_status(dev, &link); 1519 old = link; 1520 link.link_duplex = FULL_DUPLEX; 1521 link.link_speed = SPEED_10G; 1522 1523 if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1524 PMD_INIT_LOG(DEBUG, "Get link status from hw"); 1525 vtpci_read_dev_config(hw, 1526 offsetof(struct virtio_net_config, status), 1527 &status, sizeof(status)); 1528 if ((status & VIRTIO_NET_S_LINK_UP) == 0) { 1529 link.link_status = 0; 1530 PMD_INIT_LOG(DEBUG, "Port %d is down", 1531 dev->data->port_id); 1532 } else { 1533 link.link_status = 1; 1534 PMD_INIT_LOG(DEBUG, "Port %d is up", 1535 dev->data->port_id); 1536 } 1537 } else { 1538 link.link_status = 1; /* Link up */ 1539 } 1540 virtio_dev_atomic_write_link_status(dev, &link); 1541 1542 return (old.link_status == link.link_status) ? -1 : 0; 1543 } 1544 1545 static void 1546 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 1547 { 1548 struct virtio_hw *hw = dev->data->dev_private; 1549 1550 dev_info->driver_name = dev->driver->pci_drv.name; 1551 dev_info->max_rx_queues = (uint16_t)hw->max_rx_queues; 1552 dev_info->max_tx_queues = (uint16_t)hw->max_tx_queues; 1553 dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE; 1554 dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN; 1555 dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS; 1556 dev_info->default_txconf = (struct rte_eth_txconf) { 1557 .txq_flags = ETH_TXQ_FLAGS_NOOFFLOADS 1558 }; 1559 } 1560 1561 /* 1562 * It enables testpmd to collect per queue stats. 1563 */ 1564 static int 1565 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev, 1566 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx, 1567 __rte_unused uint8_t is_rx) 1568 { 1569 return 0; 1570 } 1571 1572 static struct rte_driver rte_virtio_driver = { 1573 .type = PMD_PDEV, 1574 .init = rte_virtio_pmd_init, 1575 }; 1576 1577 PMD_REGISTER_DRIVER(rte_virtio_driver); 1578