1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <string.h> 7 #include <stdio.h> 8 #include <errno.h> 9 #include <unistd.h> 10 11 #include <ethdev_driver.h> 12 #include <rte_memcpy.h> 13 #include <rte_string_fns.h> 14 #include <rte_memzone.h> 15 #include <rte_malloc.h> 16 #include <rte_branch_prediction.h> 17 #include <rte_ether.h> 18 #include <rte_ip.h> 19 #include <rte_arp.h> 20 #include <rte_common.h> 21 #include <rte_errno.h> 22 #include <rte_cpuflags.h> 23 #include <rte_vect.h> 24 #include <rte_memory.h> 25 #include <rte_eal_paging.h> 26 #include <rte_eal.h> 27 #include <rte_dev.h> 28 #include <rte_cycles.h> 29 #include <rte_kvargs.h> 30 31 #include "virtio_ethdev.h" 32 #include "virtio.h" 33 #include "virtio_logs.h" 34 #include "virtqueue.h" 35 #include "virtio_rxtx.h" 36 #include "virtio_rxtx_simple.h" 37 #include "virtio_user/virtio_user_dev.h" 38 39 static int virtio_dev_configure(struct rte_eth_dev *dev); 40 static int virtio_dev_start(struct rte_eth_dev *dev); 41 static int virtio_dev_promiscuous_enable(struct rte_eth_dev *dev); 42 static int virtio_dev_promiscuous_disable(struct rte_eth_dev *dev); 43 static int virtio_dev_allmulticast_enable(struct rte_eth_dev *dev); 44 static int virtio_dev_allmulticast_disable(struct rte_eth_dev *dev); 45 static uint32_t virtio_dev_speed_capa_get(uint32_t speed); 46 static int virtio_dev_devargs_parse(struct rte_devargs *devargs, 47 uint32_t *speed, 48 int *vectorized); 49 static int virtio_dev_info_get(struct rte_eth_dev *dev, 50 struct rte_eth_dev_info *dev_info); 51 static int virtio_dev_link_update(struct rte_eth_dev *dev, 52 int wait_to_complete); 53 static int virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 54 55 static void virtio_set_hwaddr(struct virtio_hw *hw); 56 static void virtio_get_hwaddr(struct virtio_hw *hw); 57 58 static int virtio_dev_stats_get(struct rte_eth_dev *dev, 59 struct rte_eth_stats *stats); 60 static int virtio_dev_xstats_get(struct rte_eth_dev *dev, 61 struct rte_eth_xstat *xstats, unsigned n); 62 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, 63 struct rte_eth_xstat_name *xstats_names, 64 unsigned limit); 65 static int virtio_dev_stats_reset(struct rte_eth_dev *dev); 66 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev); 67 static int virtio_vlan_filter_set(struct rte_eth_dev *dev, 68 uint16_t vlan_id, int on); 69 static int virtio_mac_addr_add(struct rte_eth_dev *dev, 70 struct rte_ether_addr *mac_addr, 71 uint32_t index, uint32_t vmdq); 72 static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); 73 static int virtio_mac_addr_set(struct rte_eth_dev *dev, 74 struct rte_ether_addr *mac_addr); 75 76 static int virtio_intr_disable(struct rte_eth_dev *dev); 77 78 static int virtio_dev_queue_stats_mapping_set( 79 struct rte_eth_dev *eth_dev, 80 uint16_t queue_id, 81 uint8_t stat_idx, 82 uint8_t is_rx); 83 84 static void virtio_notify_peers(struct rte_eth_dev *dev); 85 static void virtio_ack_link_announce(struct rte_eth_dev *dev); 86 87 struct rte_virtio_xstats_name_off { 88 char name[RTE_ETH_XSTATS_NAME_SIZE]; 89 unsigned offset; 90 }; 91 92 /* [rt]x_qX_ is prepended to the name string here */ 93 static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = { 94 {"good_packets", offsetof(struct virtnet_rx, stats.packets)}, 95 {"good_bytes", offsetof(struct virtnet_rx, stats.bytes)}, 96 {"errors", offsetof(struct virtnet_rx, stats.errors)}, 97 {"multicast_packets", offsetof(struct virtnet_rx, stats.multicast)}, 98 {"broadcast_packets", offsetof(struct virtnet_rx, stats.broadcast)}, 99 {"undersize_packets", offsetof(struct virtnet_rx, stats.size_bins[0])}, 100 {"size_64_packets", offsetof(struct virtnet_rx, stats.size_bins[1])}, 101 {"size_65_127_packets", offsetof(struct virtnet_rx, stats.size_bins[2])}, 102 {"size_128_255_packets", offsetof(struct virtnet_rx, stats.size_bins[3])}, 103 {"size_256_511_packets", offsetof(struct virtnet_rx, stats.size_bins[4])}, 104 {"size_512_1023_packets", offsetof(struct virtnet_rx, stats.size_bins[5])}, 105 {"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])}, 106 {"size_1519_max_packets", offsetof(struct virtnet_rx, stats.size_bins[7])}, 107 }; 108 109 /* [rt]x_qX_ is prepended to the name string here */ 110 static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = { 111 {"good_packets", offsetof(struct virtnet_tx, stats.packets)}, 112 {"good_bytes", offsetof(struct virtnet_tx, stats.bytes)}, 113 {"multicast_packets", offsetof(struct virtnet_tx, stats.multicast)}, 114 {"broadcast_packets", offsetof(struct virtnet_tx, stats.broadcast)}, 115 {"undersize_packets", offsetof(struct virtnet_tx, stats.size_bins[0])}, 116 {"size_64_packets", offsetof(struct virtnet_tx, stats.size_bins[1])}, 117 {"size_65_127_packets", offsetof(struct virtnet_tx, stats.size_bins[2])}, 118 {"size_128_255_packets", offsetof(struct virtnet_tx, stats.size_bins[3])}, 119 {"size_256_511_packets", offsetof(struct virtnet_tx, stats.size_bins[4])}, 120 {"size_512_1023_packets", offsetof(struct virtnet_tx, stats.size_bins[5])}, 121 {"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])}, 122 {"size_1519_max_packets", offsetof(struct virtnet_tx, stats.size_bins[7])}, 123 }; 124 125 #define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \ 126 sizeof(rte_virtio_rxq_stat_strings[0])) 127 #define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \ 128 sizeof(rte_virtio_txq_stat_strings[0])) 129 130 struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS]; 131 132 static struct virtio_pmd_ctrl * 133 virtio_send_command_packed(struct virtnet_ctl *cvq, 134 struct virtio_pmd_ctrl *ctrl, 135 int *dlen, int pkt_num) 136 { 137 struct virtqueue *vq = virtnet_cq_to_vq(cvq); 138 int head; 139 struct vring_packed_desc *desc = vq->vq_packed.ring.desc; 140 struct virtio_pmd_ctrl *result; 141 uint16_t flags; 142 int sum = 0; 143 int nb_descs = 0; 144 int k; 145 146 /* 147 * Format is enforced in qemu code: 148 * One TX packet for header; 149 * At least one TX packet per argument; 150 * One RX packet for ACK. 151 */ 152 head = vq->vq_avail_idx; 153 flags = vq->vq_packed.cached_flags; 154 desc[head].addr = cvq->virtio_net_hdr_mem; 155 desc[head].len = sizeof(struct virtio_net_ctrl_hdr); 156 vq->vq_free_cnt--; 157 nb_descs++; 158 if (++vq->vq_avail_idx >= vq->vq_nentries) { 159 vq->vq_avail_idx -= vq->vq_nentries; 160 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 161 } 162 163 for (k = 0; k < pkt_num; k++) { 164 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem 165 + sizeof(struct virtio_net_ctrl_hdr) 166 + sizeof(ctrl->status) + sizeof(uint8_t) * sum; 167 desc[vq->vq_avail_idx].len = dlen[k]; 168 desc[vq->vq_avail_idx].flags = VRING_DESC_F_NEXT | 169 vq->vq_packed.cached_flags; 170 sum += dlen[k]; 171 vq->vq_free_cnt--; 172 nb_descs++; 173 if (++vq->vq_avail_idx >= vq->vq_nentries) { 174 vq->vq_avail_idx -= vq->vq_nentries; 175 vq->vq_packed.cached_flags ^= 176 VRING_PACKED_DESC_F_AVAIL_USED; 177 } 178 } 179 180 desc[vq->vq_avail_idx].addr = cvq->virtio_net_hdr_mem 181 + sizeof(struct virtio_net_ctrl_hdr); 182 desc[vq->vq_avail_idx].len = sizeof(ctrl->status); 183 desc[vq->vq_avail_idx].flags = VRING_DESC_F_WRITE | 184 vq->vq_packed.cached_flags; 185 vq->vq_free_cnt--; 186 nb_descs++; 187 if (++vq->vq_avail_idx >= vq->vq_nentries) { 188 vq->vq_avail_idx -= vq->vq_nentries; 189 vq->vq_packed.cached_flags ^= VRING_PACKED_DESC_F_AVAIL_USED; 190 } 191 192 virtqueue_store_flags_packed(&desc[head], VRING_DESC_F_NEXT | flags, 193 vq->hw->weak_barriers); 194 195 virtio_wmb(vq->hw->weak_barriers); 196 virtqueue_notify(vq); 197 198 /* wait for used desc in virtqueue 199 * desc_is_used has a load-acquire or rte_io_rmb inside 200 */ 201 while (!desc_is_used(&desc[head], vq)) 202 usleep(100); 203 204 /* now get used descriptors */ 205 vq->vq_free_cnt += nb_descs; 206 vq->vq_used_cons_idx += nb_descs; 207 if (vq->vq_used_cons_idx >= vq->vq_nentries) { 208 vq->vq_used_cons_idx -= vq->vq_nentries; 209 vq->vq_packed.used_wrap_counter ^= 1; 210 } 211 212 PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\n" 213 "vq->vq_avail_idx=%d\n" 214 "vq->vq_used_cons_idx=%d\n" 215 "vq->vq_packed.cached_flags=0x%x\n" 216 "vq->vq_packed.used_wrap_counter=%d", 217 vq->vq_free_cnt, 218 vq->vq_avail_idx, 219 vq->vq_used_cons_idx, 220 vq->vq_packed.cached_flags, 221 vq->vq_packed.used_wrap_counter); 222 223 result = cvq->virtio_net_hdr_mz->addr; 224 return result; 225 } 226 227 static struct virtio_pmd_ctrl * 228 virtio_send_command_split(struct virtnet_ctl *cvq, 229 struct virtio_pmd_ctrl *ctrl, 230 int *dlen, int pkt_num) 231 { 232 struct virtio_pmd_ctrl *result; 233 struct virtqueue *vq = virtnet_cq_to_vq(cvq); 234 uint32_t head, i; 235 int k, sum = 0; 236 237 head = vq->vq_desc_head_idx; 238 239 /* 240 * Format is enforced in qemu code: 241 * One TX packet for header; 242 * At least one TX packet per argument; 243 * One RX packet for ACK. 244 */ 245 vq->vq_split.ring.desc[head].flags = VRING_DESC_F_NEXT; 246 vq->vq_split.ring.desc[head].addr = cvq->virtio_net_hdr_mem; 247 vq->vq_split.ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr); 248 vq->vq_free_cnt--; 249 i = vq->vq_split.ring.desc[head].next; 250 251 for (k = 0; k < pkt_num; k++) { 252 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_NEXT; 253 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem 254 + sizeof(struct virtio_net_ctrl_hdr) 255 + sizeof(ctrl->status) + sizeof(uint8_t)*sum; 256 vq->vq_split.ring.desc[i].len = dlen[k]; 257 sum += dlen[k]; 258 vq->vq_free_cnt--; 259 i = vq->vq_split.ring.desc[i].next; 260 } 261 262 vq->vq_split.ring.desc[i].flags = VRING_DESC_F_WRITE; 263 vq->vq_split.ring.desc[i].addr = cvq->virtio_net_hdr_mem 264 + sizeof(struct virtio_net_ctrl_hdr); 265 vq->vq_split.ring.desc[i].len = sizeof(ctrl->status); 266 vq->vq_free_cnt--; 267 268 vq->vq_desc_head_idx = vq->vq_split.ring.desc[i].next; 269 270 vq_update_avail_ring(vq, head); 271 vq_update_avail_idx(vq); 272 273 PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index); 274 275 virtqueue_notify(vq); 276 277 while (virtqueue_nused(vq) == 0) 278 usleep(100); 279 280 while (virtqueue_nused(vq)) { 281 uint32_t idx, desc_idx, used_idx; 282 struct vring_used_elem *uep; 283 284 used_idx = (uint32_t)(vq->vq_used_cons_idx 285 & (vq->vq_nentries - 1)); 286 uep = &vq->vq_split.ring.used->ring[used_idx]; 287 idx = (uint32_t) uep->id; 288 desc_idx = idx; 289 290 while (vq->vq_split.ring.desc[desc_idx].flags & 291 VRING_DESC_F_NEXT) { 292 desc_idx = vq->vq_split.ring.desc[desc_idx].next; 293 vq->vq_free_cnt++; 294 } 295 296 vq->vq_split.ring.desc[desc_idx].next = vq->vq_desc_head_idx; 297 vq->vq_desc_head_idx = idx; 298 299 vq->vq_used_cons_idx++; 300 vq->vq_free_cnt++; 301 } 302 303 PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d", 304 vq->vq_free_cnt, vq->vq_desc_head_idx); 305 306 result = cvq->virtio_net_hdr_mz->addr; 307 return result; 308 } 309 310 static int 311 virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl, 312 int *dlen, int pkt_num) 313 { 314 virtio_net_ctrl_ack status = ~0; 315 struct virtio_pmd_ctrl *result; 316 struct virtqueue *vq; 317 318 ctrl->status = status; 319 320 if (!cvq) { 321 PMD_INIT_LOG(ERR, "Control queue is not supported."); 322 return -1; 323 } 324 325 rte_spinlock_lock(&cvq->lock); 326 vq = virtnet_cq_to_vq(cvq); 327 328 PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, " 329 "vq->hw->cvq = %p vq = %p", 330 vq->vq_desc_head_idx, status, vq->hw->cvq, vq); 331 332 if (vq->vq_free_cnt < pkt_num + 2 || pkt_num < 1) { 333 rte_spinlock_unlock(&cvq->lock); 334 return -1; 335 } 336 337 memcpy(cvq->virtio_net_hdr_mz->addr, ctrl, 338 sizeof(struct virtio_pmd_ctrl)); 339 340 if (virtio_with_packed_queue(vq->hw)) 341 result = virtio_send_command_packed(cvq, ctrl, dlen, pkt_num); 342 else 343 result = virtio_send_command_split(cvq, ctrl, dlen, pkt_num); 344 345 rte_spinlock_unlock(&cvq->lock); 346 return result->status; 347 } 348 349 static int 350 virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues) 351 { 352 struct virtio_hw *hw = dev->data->dev_private; 353 struct virtio_pmd_ctrl ctrl; 354 int dlen[1]; 355 int ret; 356 357 ctrl.hdr.class = VIRTIO_NET_CTRL_MQ; 358 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; 359 memcpy(ctrl.data, &nb_queues, sizeof(uint16_t)); 360 361 dlen[0] = sizeof(uint16_t); 362 363 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 364 if (ret) { 365 PMD_INIT_LOG(ERR, "Multiqueue configured but send command " 366 "failed, this is too late now..."); 367 return -EINVAL; 368 } 369 370 return 0; 371 } 372 373 static void 374 virtio_dev_queue_release(void *queue __rte_unused) 375 { 376 /* do nothing */ 377 } 378 379 static uint16_t 380 virtio_get_nr_vq(struct virtio_hw *hw) 381 { 382 uint16_t nr_vq = hw->max_queue_pairs * 2; 383 384 if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) 385 nr_vq += 1; 386 387 return nr_vq; 388 } 389 390 static void 391 virtio_init_vring(struct virtqueue *vq) 392 { 393 int size = vq->vq_nentries; 394 uint8_t *ring_mem = vq->vq_ring_virt_mem; 395 396 PMD_INIT_FUNC_TRACE(); 397 398 memset(ring_mem, 0, vq->vq_ring_size); 399 400 vq->vq_used_cons_idx = 0; 401 vq->vq_desc_head_idx = 0; 402 vq->vq_avail_idx = 0; 403 vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1); 404 vq->vq_free_cnt = vq->vq_nentries; 405 memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries); 406 if (virtio_with_packed_queue(vq->hw)) { 407 vring_init_packed(&vq->vq_packed.ring, ring_mem, 408 VIRTIO_VRING_ALIGN, size); 409 vring_desc_init_packed(vq, size); 410 } else { 411 struct vring *vr = &vq->vq_split.ring; 412 413 vring_init_split(vr, ring_mem, VIRTIO_VRING_ALIGN, size); 414 vring_desc_init_split(vr->desc, size); 415 } 416 /* 417 * Disable device(host) interrupting guest 418 */ 419 virtqueue_disable_intr(vq); 420 } 421 422 static int 423 virtio_init_queue(struct rte_eth_dev *dev, uint16_t queue_idx) 424 { 425 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 426 char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ]; 427 const struct rte_memzone *mz = NULL, *hdr_mz = NULL; 428 unsigned int vq_size, size; 429 struct virtio_hw *hw = dev->data->dev_private; 430 struct virtnet_rx *rxvq = NULL; 431 struct virtnet_tx *txvq = NULL; 432 struct virtnet_ctl *cvq = NULL; 433 struct virtqueue *vq; 434 size_t sz_hdr_mz = 0; 435 void *sw_ring = NULL; 436 int queue_type = virtio_get_queue_type(hw, queue_idx); 437 int ret; 438 int numa_node = dev->device->numa_node; 439 struct rte_mbuf *fake_mbuf = NULL; 440 441 PMD_INIT_LOG(INFO, "setting up queue: %u on NUMA node %d", 442 queue_idx, numa_node); 443 444 /* 445 * Read the virtqueue size from the Queue Size field 446 * Always power of 2 and if 0 virtqueue does not exist 447 */ 448 vq_size = VIRTIO_OPS(hw)->get_queue_num(hw, queue_idx); 449 PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size); 450 if (vq_size == 0) { 451 PMD_INIT_LOG(ERR, "virtqueue does not exist"); 452 return -EINVAL; 453 } 454 455 if (!virtio_with_packed_queue(hw) && !rte_is_power_of_2(vq_size)) { 456 PMD_INIT_LOG(ERR, "split virtqueue size is not power of 2"); 457 return -EINVAL; 458 } 459 460 snprintf(vq_name, sizeof(vq_name), "port%d_vq%d", 461 dev->data->port_id, queue_idx); 462 463 size = RTE_ALIGN_CEIL(sizeof(*vq) + 464 vq_size * sizeof(struct vq_desc_extra), 465 RTE_CACHE_LINE_SIZE); 466 if (queue_type == VTNET_TQ) { 467 /* 468 * For each xmit packet, allocate a virtio_net_hdr 469 * and indirect ring elements 470 */ 471 sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region); 472 } else if (queue_type == VTNET_CQ) { 473 /* Allocate a page for control vq command, data and status */ 474 sz_hdr_mz = rte_mem_page_size(); 475 } 476 477 vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE, 478 numa_node); 479 if (vq == NULL) { 480 PMD_INIT_LOG(ERR, "can not allocate vq"); 481 return -ENOMEM; 482 } 483 hw->vqs[queue_idx] = vq; 484 485 vq->hw = hw; 486 vq->vq_queue_index = queue_idx; 487 vq->vq_nentries = vq_size; 488 if (virtio_with_packed_queue(hw)) { 489 vq->vq_packed.used_wrap_counter = 1; 490 vq->vq_packed.cached_flags = VRING_PACKED_DESC_F_AVAIL; 491 vq->vq_packed.event_flags_shadow = 0; 492 if (queue_type == VTNET_RQ) 493 vq->vq_packed.cached_flags |= VRING_DESC_F_WRITE; 494 } 495 496 /* 497 * Reserve a memzone for vring elements 498 */ 499 size = vring_size(hw, vq_size, VIRTIO_VRING_ALIGN); 500 vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_VRING_ALIGN); 501 PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d", 502 size, vq->vq_ring_size); 503 504 mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size, 505 numa_node, RTE_MEMZONE_IOVA_CONTIG, 506 VIRTIO_VRING_ALIGN); 507 if (mz == NULL) { 508 if (rte_errno == EEXIST) 509 mz = rte_memzone_lookup(vq_name); 510 if (mz == NULL) { 511 ret = -ENOMEM; 512 goto free_vq; 513 } 514 } 515 516 memset(mz->addr, 0, mz->len); 517 518 if (hw->use_va) 519 vq->vq_ring_mem = (uintptr_t)mz->addr; 520 else 521 vq->vq_ring_mem = mz->iova; 522 523 vq->vq_ring_virt_mem = mz->addr; 524 PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem: 0x%" PRIx64, vq->vq_ring_mem); 525 PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: %p", vq->vq_ring_virt_mem); 526 527 virtio_init_vring(vq); 528 529 if (sz_hdr_mz) { 530 snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr", 531 dev->data->port_id, queue_idx); 532 hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz, 533 numa_node, RTE_MEMZONE_IOVA_CONTIG, 534 RTE_CACHE_LINE_SIZE); 535 if (hdr_mz == NULL) { 536 if (rte_errno == EEXIST) 537 hdr_mz = rte_memzone_lookup(vq_hdr_name); 538 if (hdr_mz == NULL) { 539 ret = -ENOMEM; 540 goto free_mz; 541 } 542 } 543 } 544 545 if (queue_type == VTNET_RQ) { 546 size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) * 547 sizeof(vq->sw_ring[0]); 548 549 sw_ring = rte_zmalloc_socket("sw_ring", sz_sw, 550 RTE_CACHE_LINE_SIZE, numa_node); 551 if (!sw_ring) { 552 PMD_INIT_LOG(ERR, "can not allocate RX soft ring"); 553 ret = -ENOMEM; 554 goto free_hdr_mz; 555 } 556 557 fake_mbuf = rte_zmalloc_socket("sw_ring", sizeof(*fake_mbuf), 558 RTE_CACHE_LINE_SIZE, numa_node); 559 if (!fake_mbuf) { 560 PMD_INIT_LOG(ERR, "can not allocate fake mbuf"); 561 ret = -ENOMEM; 562 goto free_sw_ring; 563 } 564 565 vq->sw_ring = sw_ring; 566 rxvq = &vq->rxq; 567 rxvq->port_id = dev->data->port_id; 568 rxvq->mz = mz; 569 rxvq->fake_mbuf = fake_mbuf; 570 } else if (queue_type == VTNET_TQ) { 571 txvq = &vq->txq; 572 txvq->port_id = dev->data->port_id; 573 txvq->mz = mz; 574 txvq->virtio_net_hdr_mz = hdr_mz; 575 if (hw->use_va) 576 txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr; 577 else 578 txvq->virtio_net_hdr_mem = hdr_mz->iova; 579 } else if (queue_type == VTNET_CQ) { 580 cvq = &vq->cq; 581 cvq->mz = mz; 582 cvq->virtio_net_hdr_mz = hdr_mz; 583 if (hw->use_va) 584 cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr; 585 else 586 cvq->virtio_net_hdr_mem = hdr_mz->iova; 587 memset(cvq->virtio_net_hdr_mz->addr, 0, rte_mem_page_size()); 588 589 hw->cvq = cvq; 590 } 591 592 if (hw->use_va) 593 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_addr); 594 else 595 vq->mbuf_addr_offset = offsetof(struct rte_mbuf, buf_iova); 596 597 if (queue_type == VTNET_TQ) { 598 struct virtio_tx_region *txr; 599 unsigned int i; 600 601 txr = hdr_mz->addr; 602 memset(txr, 0, vq_size * sizeof(*txr)); 603 for (i = 0; i < vq_size; i++) { 604 /* first indirect descriptor is always the tx header */ 605 if (!virtio_with_packed_queue(hw)) { 606 struct vring_desc *start_dp = txr[i].tx_indir; 607 vring_desc_init_split(start_dp, 608 RTE_DIM(txr[i].tx_indir)); 609 start_dp->addr = txvq->virtio_net_hdr_mem 610 + i * sizeof(*txr) 611 + offsetof(struct virtio_tx_region, 612 tx_hdr); 613 start_dp->len = hw->vtnet_hdr_size; 614 start_dp->flags = VRING_DESC_F_NEXT; 615 } else { 616 struct vring_packed_desc *start_dp = 617 txr[i].tx_packed_indir; 618 vring_desc_init_indirect_packed(start_dp, 619 RTE_DIM(txr[i].tx_packed_indir)); 620 start_dp->addr = txvq->virtio_net_hdr_mem 621 + i * sizeof(*txr) 622 + offsetof(struct virtio_tx_region, 623 tx_hdr); 624 start_dp->len = hw->vtnet_hdr_size; 625 } 626 } 627 } 628 629 if (VIRTIO_OPS(hw)->setup_queue(hw, vq) < 0) { 630 PMD_INIT_LOG(ERR, "setup_queue failed"); 631 ret = -EINVAL; 632 goto clean_vq; 633 } 634 635 return 0; 636 637 clean_vq: 638 hw->cvq = NULL; 639 rte_free(fake_mbuf); 640 free_sw_ring: 641 rte_free(sw_ring); 642 free_hdr_mz: 643 rte_memzone_free(hdr_mz); 644 free_mz: 645 rte_memzone_free(mz); 646 free_vq: 647 rte_free(vq); 648 hw->vqs[queue_idx] = NULL; 649 650 return ret; 651 } 652 653 static void 654 virtio_free_queues(struct virtio_hw *hw) 655 { 656 uint16_t nr_vq = virtio_get_nr_vq(hw); 657 struct virtqueue *vq; 658 int queue_type; 659 uint16_t i; 660 661 if (hw->vqs == NULL) 662 return; 663 664 for (i = 0; i < nr_vq; i++) { 665 vq = hw->vqs[i]; 666 if (!vq) 667 continue; 668 669 queue_type = virtio_get_queue_type(hw, i); 670 if (queue_type == VTNET_RQ) { 671 rte_free(vq->rxq.fake_mbuf); 672 rte_free(vq->sw_ring); 673 rte_memzone_free(vq->rxq.mz); 674 } else if (queue_type == VTNET_TQ) { 675 rte_memzone_free(vq->txq.mz); 676 rte_memzone_free(vq->txq.virtio_net_hdr_mz); 677 } else { 678 rte_memzone_free(vq->cq.mz); 679 rte_memzone_free(vq->cq.virtio_net_hdr_mz); 680 } 681 682 rte_free(vq); 683 hw->vqs[i] = NULL; 684 } 685 686 rte_free(hw->vqs); 687 hw->vqs = NULL; 688 } 689 690 static int 691 virtio_alloc_queues(struct rte_eth_dev *dev) 692 { 693 struct virtio_hw *hw = dev->data->dev_private; 694 uint16_t nr_vq = virtio_get_nr_vq(hw); 695 uint16_t i; 696 int ret; 697 698 hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0); 699 if (!hw->vqs) { 700 PMD_INIT_LOG(ERR, "failed to allocate vqs"); 701 return -ENOMEM; 702 } 703 704 for (i = 0; i < nr_vq; i++) { 705 ret = virtio_init_queue(dev, i); 706 if (ret < 0) { 707 virtio_free_queues(hw); 708 return ret; 709 } 710 } 711 712 return 0; 713 } 714 715 static void virtio_queues_unbind_intr(struct rte_eth_dev *dev); 716 717 int 718 virtio_dev_close(struct rte_eth_dev *dev) 719 { 720 struct virtio_hw *hw = dev->data->dev_private; 721 struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; 722 723 PMD_INIT_LOG(DEBUG, "virtio_dev_close"); 724 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 725 return 0; 726 727 if (!hw->opened) 728 return 0; 729 hw->opened = 0; 730 731 /* reset the NIC */ 732 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 733 VIRTIO_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR); 734 if (intr_conf->rxq) 735 virtio_queues_unbind_intr(dev); 736 737 if (intr_conf->lsc || intr_conf->rxq) { 738 virtio_intr_disable(dev); 739 rte_intr_efd_disable(dev->intr_handle); 740 rte_free(dev->intr_handle->intr_vec); 741 dev->intr_handle->intr_vec = NULL; 742 } 743 744 virtio_reset(hw); 745 virtio_dev_free_mbufs(dev); 746 virtio_free_queues(hw); 747 748 return VIRTIO_OPS(hw)->dev_close(hw); 749 } 750 751 static int 752 virtio_dev_promiscuous_enable(struct rte_eth_dev *dev) 753 { 754 struct virtio_hw *hw = dev->data->dev_private; 755 struct virtio_pmd_ctrl ctrl; 756 int dlen[1]; 757 int ret; 758 759 if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 760 PMD_INIT_LOG(INFO, "host does not support rx control"); 761 return -ENOTSUP; 762 } 763 764 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 765 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 766 ctrl.data[0] = 1; 767 dlen[0] = 1; 768 769 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 770 if (ret) { 771 PMD_INIT_LOG(ERR, "Failed to enable promisc"); 772 return -EAGAIN; 773 } 774 775 return 0; 776 } 777 778 static int 779 virtio_dev_promiscuous_disable(struct rte_eth_dev *dev) 780 { 781 struct virtio_hw *hw = dev->data->dev_private; 782 struct virtio_pmd_ctrl ctrl; 783 int dlen[1]; 784 int ret; 785 786 if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 787 PMD_INIT_LOG(INFO, "host does not support rx control"); 788 return -ENOTSUP; 789 } 790 791 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 792 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC; 793 ctrl.data[0] = 0; 794 dlen[0] = 1; 795 796 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 797 if (ret) { 798 PMD_INIT_LOG(ERR, "Failed to disable promisc"); 799 return -EAGAIN; 800 } 801 802 return 0; 803 } 804 805 static int 806 virtio_dev_allmulticast_enable(struct rte_eth_dev *dev) 807 { 808 struct virtio_hw *hw = dev->data->dev_private; 809 struct virtio_pmd_ctrl ctrl; 810 int dlen[1]; 811 int ret; 812 813 if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 814 PMD_INIT_LOG(INFO, "host does not support rx control"); 815 return -ENOTSUP; 816 } 817 818 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 819 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 820 ctrl.data[0] = 1; 821 dlen[0] = 1; 822 823 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 824 if (ret) { 825 PMD_INIT_LOG(ERR, "Failed to enable allmulticast"); 826 return -EAGAIN; 827 } 828 829 return 0; 830 } 831 832 static int 833 virtio_dev_allmulticast_disable(struct rte_eth_dev *dev) 834 { 835 struct virtio_hw *hw = dev->data->dev_private; 836 struct virtio_pmd_ctrl ctrl; 837 int dlen[1]; 838 int ret; 839 840 if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) { 841 PMD_INIT_LOG(INFO, "host does not support rx control"); 842 return -ENOTSUP; 843 } 844 845 ctrl.hdr.class = VIRTIO_NET_CTRL_RX; 846 ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI; 847 ctrl.data[0] = 0; 848 dlen[0] = 1; 849 850 ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1); 851 if (ret) { 852 PMD_INIT_LOG(ERR, "Failed to disable allmulticast"); 853 return -EAGAIN; 854 } 855 856 return 0; 857 } 858 859 uint16_t 860 virtio_rx_mem_pool_buf_size(struct rte_mempool *mp) 861 { 862 return rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM; 863 } 864 865 bool 866 virtio_rx_check_scatter(uint16_t max_rx_pkt_len, uint16_t rx_buf_size, 867 bool rx_scatter_enabled, const char **error) 868 { 869 if (!rx_scatter_enabled && max_rx_pkt_len > rx_buf_size) { 870 *error = "Rx scatter is disabled and RxQ mbuf pool object size is too small"; 871 return false; 872 } 873 874 return true; 875 } 876 877 static bool 878 virtio_check_scatter_on_all_rx_queues(struct rte_eth_dev *dev, 879 uint16_t frame_size) 880 { 881 struct virtio_hw *hw = dev->data->dev_private; 882 struct virtnet_rx *rxvq; 883 struct virtqueue *vq; 884 unsigned int qidx; 885 uint16_t buf_size; 886 const char *error; 887 888 if (hw->vqs == NULL) 889 return true; 890 891 for (qidx = 0; (vq = hw->vqs[2 * qidx + VTNET_SQ_RQ_QUEUE_IDX]) != NULL; 892 qidx++) { 893 rxvq = &vq->rxq; 894 if (rxvq->mpool == NULL) 895 continue; 896 buf_size = virtio_rx_mem_pool_buf_size(rxvq->mpool); 897 898 if (!virtio_rx_check_scatter(frame_size, buf_size, 899 hw->rx_ol_scatter, &error)) { 900 PMD_INIT_LOG(ERR, "MTU check for RxQ %u failed: %s", 901 qidx, error); 902 return false; 903 } 904 } 905 906 return true; 907 } 908 909 #define VLAN_TAG_LEN 4 /* 802.3ac tag (not DMA'd) */ 910 static int 911 virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 912 { 913 struct virtio_hw *hw = dev->data->dev_private; 914 uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN + 915 hw->vtnet_hdr_size; 916 uint32_t frame_size = mtu + ether_hdr_len; 917 uint32_t max_frame_size = hw->max_mtu + ether_hdr_len; 918 919 max_frame_size = RTE_MIN(max_frame_size, VIRTIO_MAX_RX_PKTLEN); 920 921 if (mtu < RTE_ETHER_MIN_MTU || frame_size > max_frame_size) { 922 PMD_INIT_LOG(ERR, "MTU should be between %d and %d", 923 RTE_ETHER_MIN_MTU, max_frame_size - ether_hdr_len); 924 return -EINVAL; 925 } 926 927 if (!virtio_check_scatter_on_all_rx_queues(dev, frame_size)) { 928 PMD_INIT_LOG(ERR, "MTU vs Rx scatter and Rx buffers check failed"); 929 return -EINVAL; 930 } 931 932 hw->max_rx_pkt_len = frame_size; 933 dev->data->dev_conf.rxmode.max_rx_pkt_len = hw->max_rx_pkt_len; 934 935 return 0; 936 } 937 938 static int 939 virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 940 { 941 struct virtio_hw *hw = dev->data->dev_private; 942 struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; 943 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 944 945 virtqueue_enable_intr(vq); 946 virtio_mb(hw->weak_barriers); 947 return 0; 948 } 949 950 static int 951 virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 952 { 953 struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id]; 954 struct virtqueue *vq = virtnet_rxq_to_vq(rxvq); 955 956 virtqueue_disable_intr(vq); 957 return 0; 958 } 959 960 /* 961 * dev_ops for virtio, bare necessities for basic operation 962 */ 963 static const struct eth_dev_ops virtio_eth_dev_ops = { 964 .dev_configure = virtio_dev_configure, 965 .dev_start = virtio_dev_start, 966 .dev_stop = virtio_dev_stop, 967 .dev_close = virtio_dev_close, 968 .promiscuous_enable = virtio_dev_promiscuous_enable, 969 .promiscuous_disable = virtio_dev_promiscuous_disable, 970 .allmulticast_enable = virtio_dev_allmulticast_enable, 971 .allmulticast_disable = virtio_dev_allmulticast_disable, 972 .mtu_set = virtio_mtu_set, 973 .dev_infos_get = virtio_dev_info_get, 974 .stats_get = virtio_dev_stats_get, 975 .xstats_get = virtio_dev_xstats_get, 976 .xstats_get_names = virtio_dev_xstats_get_names, 977 .stats_reset = virtio_dev_stats_reset, 978 .xstats_reset = virtio_dev_stats_reset, 979 .link_update = virtio_dev_link_update, 980 .vlan_offload_set = virtio_dev_vlan_offload_set, 981 .rx_queue_setup = virtio_dev_rx_queue_setup, 982 .rx_queue_intr_enable = virtio_dev_rx_queue_intr_enable, 983 .rx_queue_intr_disable = virtio_dev_rx_queue_intr_disable, 984 .rx_queue_release = virtio_dev_queue_release, 985 .tx_queue_setup = virtio_dev_tx_queue_setup, 986 .tx_queue_release = virtio_dev_queue_release, 987 /* collect stats per queue */ 988 .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, 989 .vlan_filter_set = virtio_vlan_filter_set, 990 .mac_addr_add = virtio_mac_addr_add, 991 .mac_addr_remove = virtio_mac_addr_remove, 992 .mac_addr_set = virtio_mac_addr_set, 993 }; 994 995 /* 996 * dev_ops for virtio-user in secondary processes, as we just have 997 * some limited supports currently. 998 */ 999 const struct eth_dev_ops virtio_user_secondary_eth_dev_ops = { 1000 .dev_infos_get = virtio_dev_info_get, 1001 .stats_get = virtio_dev_stats_get, 1002 .xstats_get = virtio_dev_xstats_get, 1003 .xstats_get_names = virtio_dev_xstats_get_names, 1004 .stats_reset = virtio_dev_stats_reset, 1005 .xstats_reset = virtio_dev_stats_reset, 1006 /* collect stats per queue */ 1007 .queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set, 1008 }; 1009 1010 static void 1011 virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1012 { 1013 unsigned i; 1014 1015 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1016 const struct virtnet_tx *txvq = dev->data->tx_queues[i]; 1017 if (txvq == NULL) 1018 continue; 1019 1020 stats->opackets += txvq->stats.packets; 1021 stats->obytes += txvq->stats.bytes; 1022 1023 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 1024 stats->q_opackets[i] = txvq->stats.packets; 1025 stats->q_obytes[i] = txvq->stats.bytes; 1026 } 1027 } 1028 1029 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1030 const struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 1031 if (rxvq == NULL) 1032 continue; 1033 1034 stats->ipackets += rxvq->stats.packets; 1035 stats->ibytes += rxvq->stats.bytes; 1036 stats->ierrors += rxvq->stats.errors; 1037 1038 if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { 1039 stats->q_ipackets[i] = rxvq->stats.packets; 1040 stats->q_ibytes[i] = rxvq->stats.bytes; 1041 } 1042 } 1043 1044 stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed; 1045 } 1046 1047 static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev, 1048 struct rte_eth_xstat_name *xstats_names, 1049 __rte_unused unsigned limit) 1050 { 1051 unsigned i; 1052 unsigned count = 0; 1053 unsigned t; 1054 1055 unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS + 1056 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS; 1057 1058 if (xstats_names != NULL) { 1059 /* Note: limit checked in rte_eth_xstats_names() */ 1060 1061 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1062 struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 1063 if (rxvq == NULL) 1064 continue; 1065 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) { 1066 snprintf(xstats_names[count].name, 1067 sizeof(xstats_names[count].name), 1068 "rx_q%u_%s", i, 1069 rte_virtio_rxq_stat_strings[t].name); 1070 count++; 1071 } 1072 } 1073 1074 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1075 struct virtnet_tx *txvq = dev->data->tx_queues[i]; 1076 if (txvq == NULL) 1077 continue; 1078 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) { 1079 snprintf(xstats_names[count].name, 1080 sizeof(xstats_names[count].name), 1081 "tx_q%u_%s", i, 1082 rte_virtio_txq_stat_strings[t].name); 1083 count++; 1084 } 1085 } 1086 return count; 1087 } 1088 return nstats; 1089 } 1090 1091 static int 1092 virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, 1093 unsigned n) 1094 { 1095 unsigned i; 1096 unsigned count = 0; 1097 1098 unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS + 1099 dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS; 1100 1101 if (n < nstats) 1102 return nstats; 1103 1104 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1105 struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 1106 1107 if (rxvq == NULL) 1108 continue; 1109 1110 unsigned t; 1111 1112 for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) { 1113 xstats[count].value = *(uint64_t *)(((char *)rxvq) + 1114 rte_virtio_rxq_stat_strings[t].offset); 1115 xstats[count].id = count; 1116 count++; 1117 } 1118 } 1119 1120 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1121 struct virtnet_tx *txvq = dev->data->tx_queues[i]; 1122 1123 if (txvq == NULL) 1124 continue; 1125 1126 unsigned t; 1127 1128 for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) { 1129 xstats[count].value = *(uint64_t *)(((char *)txvq) + 1130 rte_virtio_txq_stat_strings[t].offset); 1131 xstats[count].id = count; 1132 count++; 1133 } 1134 } 1135 1136 return count; 1137 } 1138 1139 static int 1140 virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1141 { 1142 virtio_update_stats(dev, stats); 1143 1144 return 0; 1145 } 1146 1147 static int 1148 virtio_dev_stats_reset(struct rte_eth_dev *dev) 1149 { 1150 unsigned int i; 1151 1152 for (i = 0; i < dev->data->nb_tx_queues; i++) { 1153 struct virtnet_tx *txvq = dev->data->tx_queues[i]; 1154 if (txvq == NULL) 1155 continue; 1156 1157 txvq->stats.packets = 0; 1158 txvq->stats.bytes = 0; 1159 txvq->stats.multicast = 0; 1160 txvq->stats.broadcast = 0; 1161 memset(txvq->stats.size_bins, 0, 1162 sizeof(txvq->stats.size_bins[0]) * 8); 1163 } 1164 1165 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1166 struct virtnet_rx *rxvq = dev->data->rx_queues[i]; 1167 if (rxvq == NULL) 1168 continue; 1169 1170 rxvq->stats.packets = 0; 1171 rxvq->stats.bytes = 0; 1172 rxvq->stats.errors = 0; 1173 rxvq->stats.multicast = 0; 1174 rxvq->stats.broadcast = 0; 1175 memset(rxvq->stats.size_bins, 0, 1176 sizeof(rxvq->stats.size_bins[0]) * 8); 1177 } 1178 1179 return 0; 1180 } 1181 1182 static void 1183 virtio_set_hwaddr(struct virtio_hw *hw) 1184 { 1185 virtio_write_dev_config(hw, 1186 offsetof(struct virtio_net_config, mac), 1187 &hw->mac_addr, RTE_ETHER_ADDR_LEN); 1188 } 1189 1190 static void 1191 virtio_get_hwaddr(struct virtio_hw *hw) 1192 { 1193 if (virtio_with_feature(hw, VIRTIO_NET_F_MAC)) { 1194 virtio_read_dev_config(hw, 1195 offsetof(struct virtio_net_config, mac), 1196 &hw->mac_addr, RTE_ETHER_ADDR_LEN); 1197 } else { 1198 rte_eth_random_addr(&hw->mac_addr[0]); 1199 virtio_set_hwaddr(hw); 1200 } 1201 } 1202 1203 static int 1204 virtio_mac_table_set(struct virtio_hw *hw, 1205 const struct virtio_net_ctrl_mac *uc, 1206 const struct virtio_net_ctrl_mac *mc) 1207 { 1208 struct virtio_pmd_ctrl ctrl; 1209 int err, len[2]; 1210 1211 if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 1212 PMD_DRV_LOG(INFO, "host does not support mac table"); 1213 return -1; 1214 } 1215 1216 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 1217 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET; 1218 1219 len[0] = uc->entries * RTE_ETHER_ADDR_LEN + sizeof(uc->entries); 1220 memcpy(ctrl.data, uc, len[0]); 1221 1222 len[1] = mc->entries * RTE_ETHER_ADDR_LEN + sizeof(mc->entries); 1223 memcpy(ctrl.data + len[0], mc, len[1]); 1224 1225 err = virtio_send_command(hw->cvq, &ctrl, len, 2); 1226 if (err != 0) 1227 PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err); 1228 return err; 1229 } 1230 1231 static int 1232 virtio_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, 1233 uint32_t index, uint32_t vmdq __rte_unused) 1234 { 1235 struct virtio_hw *hw = dev->data->dev_private; 1236 const struct rte_ether_addr *addrs = dev->data->mac_addrs; 1237 unsigned int i; 1238 struct virtio_net_ctrl_mac *uc, *mc; 1239 1240 if (index >= VIRTIO_MAX_MAC_ADDRS) { 1241 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 1242 return -EINVAL; 1243 } 1244 1245 uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1246 sizeof(uc->entries)); 1247 uc->entries = 0; 1248 mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1249 sizeof(mc->entries)); 1250 mc->entries = 0; 1251 1252 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 1253 const struct rte_ether_addr *addr 1254 = (i == index) ? mac_addr : addrs + i; 1255 struct virtio_net_ctrl_mac *tbl 1256 = rte_is_multicast_ether_addr(addr) ? mc : uc; 1257 1258 memcpy(&tbl->macs[tbl->entries++], addr, RTE_ETHER_ADDR_LEN); 1259 } 1260 1261 return virtio_mac_table_set(hw, uc, mc); 1262 } 1263 1264 static void 1265 virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) 1266 { 1267 struct virtio_hw *hw = dev->data->dev_private; 1268 struct rte_ether_addr *addrs = dev->data->mac_addrs; 1269 struct virtio_net_ctrl_mac *uc, *mc; 1270 unsigned int i; 1271 1272 if (index >= VIRTIO_MAX_MAC_ADDRS) { 1273 PMD_DRV_LOG(ERR, "mac address index %u out of range", index); 1274 return; 1275 } 1276 1277 uc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1278 sizeof(uc->entries)); 1279 uc->entries = 0; 1280 mc = alloca(VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN + 1281 sizeof(mc->entries)); 1282 mc->entries = 0; 1283 1284 for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) { 1285 struct virtio_net_ctrl_mac *tbl; 1286 1287 if (i == index || rte_is_zero_ether_addr(addrs + i)) 1288 continue; 1289 1290 tbl = rte_is_multicast_ether_addr(addrs + i) ? mc : uc; 1291 memcpy(&tbl->macs[tbl->entries++], addrs + i, 1292 RTE_ETHER_ADDR_LEN); 1293 } 1294 1295 virtio_mac_table_set(hw, uc, mc); 1296 } 1297 1298 static int 1299 virtio_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) 1300 { 1301 struct virtio_hw *hw = dev->data->dev_private; 1302 1303 memcpy(hw->mac_addr, mac_addr, RTE_ETHER_ADDR_LEN); 1304 1305 /* Use atomic update if available */ 1306 if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 1307 struct virtio_pmd_ctrl ctrl; 1308 int len = RTE_ETHER_ADDR_LEN; 1309 1310 ctrl.hdr.class = VIRTIO_NET_CTRL_MAC; 1311 ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET; 1312 1313 memcpy(ctrl.data, mac_addr, RTE_ETHER_ADDR_LEN); 1314 return virtio_send_command(hw->cvq, &ctrl, &len, 1); 1315 } 1316 1317 if (!virtio_with_feature(hw, VIRTIO_NET_F_MAC)) 1318 return -ENOTSUP; 1319 1320 virtio_set_hwaddr(hw); 1321 return 0; 1322 } 1323 1324 static int 1325 virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1326 { 1327 struct virtio_hw *hw = dev->data->dev_private; 1328 struct virtio_pmd_ctrl ctrl; 1329 int len; 1330 1331 if (!virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) 1332 return -ENOTSUP; 1333 1334 ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN; 1335 ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL; 1336 memcpy(ctrl.data, &vlan_id, sizeof(vlan_id)); 1337 len = sizeof(vlan_id); 1338 1339 return virtio_send_command(hw->cvq, &ctrl, &len, 1); 1340 } 1341 1342 static int 1343 virtio_intr_unmask(struct rte_eth_dev *dev) 1344 { 1345 struct virtio_hw *hw = dev->data->dev_private; 1346 1347 if (rte_intr_ack(dev->intr_handle) < 0) 1348 return -1; 1349 1350 if (VIRTIO_OPS(hw)->intr_detect) 1351 VIRTIO_OPS(hw)->intr_detect(hw); 1352 1353 return 0; 1354 } 1355 1356 static int 1357 virtio_intr_enable(struct rte_eth_dev *dev) 1358 { 1359 struct virtio_hw *hw = dev->data->dev_private; 1360 1361 if (rte_intr_enable(dev->intr_handle) < 0) 1362 return -1; 1363 1364 if (VIRTIO_OPS(hw)->intr_detect) 1365 VIRTIO_OPS(hw)->intr_detect(hw); 1366 1367 return 0; 1368 } 1369 1370 static int 1371 virtio_intr_disable(struct rte_eth_dev *dev) 1372 { 1373 struct virtio_hw *hw = dev->data->dev_private; 1374 1375 if (rte_intr_disable(dev->intr_handle) < 0) 1376 return -1; 1377 1378 if (VIRTIO_OPS(hw)->intr_detect) 1379 VIRTIO_OPS(hw)->intr_detect(hw); 1380 1381 return 0; 1382 } 1383 1384 static int 1385 virtio_ethdev_negotiate_features(struct virtio_hw *hw, uint64_t req_features) 1386 { 1387 uint64_t host_features; 1388 1389 /* Prepare guest_features: feature that driver wants to support */ 1390 PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64, 1391 req_features); 1392 1393 /* Read device(host) feature bits */ 1394 host_features = VIRTIO_OPS(hw)->get_features(hw); 1395 PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64, 1396 host_features); 1397 1398 /* If supported, ensure MTU value is valid before acknowledging it. */ 1399 if (host_features & req_features & (1ULL << VIRTIO_NET_F_MTU)) { 1400 struct virtio_net_config config; 1401 1402 virtio_read_dev_config(hw, 1403 offsetof(struct virtio_net_config, mtu), 1404 &config.mtu, sizeof(config.mtu)); 1405 1406 if (config.mtu < RTE_ETHER_MIN_MTU) 1407 req_features &= ~(1ULL << VIRTIO_NET_F_MTU); 1408 } 1409 1410 /* 1411 * Negotiate features: Subset of device feature bits are written back 1412 * guest feature bits. 1413 */ 1414 hw->guest_features = req_features; 1415 hw->guest_features = virtio_negotiate_features(hw, host_features); 1416 PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64, 1417 hw->guest_features); 1418 1419 if (VIRTIO_OPS(hw)->features_ok(hw) < 0) 1420 return -1; 1421 1422 if (virtio_with_feature(hw, VIRTIO_F_VERSION_1)) { 1423 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK); 1424 1425 if (!(virtio_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) { 1426 PMD_INIT_LOG(ERR, "Failed to set FEATURES_OK status!"); 1427 return -1; 1428 } 1429 } 1430 1431 hw->req_guest_features = req_features; 1432 1433 return 0; 1434 } 1435 1436 int 1437 virtio_dev_pause(struct rte_eth_dev *dev) 1438 { 1439 struct virtio_hw *hw = dev->data->dev_private; 1440 1441 rte_spinlock_lock(&hw->state_lock); 1442 1443 if (hw->started == 0) { 1444 /* Device is just stopped. */ 1445 rte_spinlock_unlock(&hw->state_lock); 1446 return -1; 1447 } 1448 hw->started = 0; 1449 /* 1450 * Prevent the worker threads from touching queues to avoid contention, 1451 * 1 ms should be enough for the ongoing Tx function to finish. 1452 */ 1453 rte_delay_ms(1); 1454 return 0; 1455 } 1456 1457 /* 1458 * Recover hw state to let the worker threads continue. 1459 */ 1460 void 1461 virtio_dev_resume(struct rte_eth_dev *dev) 1462 { 1463 struct virtio_hw *hw = dev->data->dev_private; 1464 1465 hw->started = 1; 1466 rte_spinlock_unlock(&hw->state_lock); 1467 } 1468 1469 /* 1470 * Should be called only after device is paused. 1471 */ 1472 int 1473 virtio_inject_pkts(struct rte_eth_dev *dev, struct rte_mbuf **tx_pkts, 1474 int nb_pkts) 1475 { 1476 struct virtio_hw *hw = dev->data->dev_private; 1477 struct virtnet_tx *txvq = dev->data->tx_queues[0]; 1478 int ret; 1479 1480 hw->inject_pkts = tx_pkts; 1481 ret = dev->tx_pkt_burst(txvq, tx_pkts, nb_pkts); 1482 hw->inject_pkts = NULL; 1483 1484 return ret; 1485 } 1486 1487 static void 1488 virtio_notify_peers(struct rte_eth_dev *dev) 1489 { 1490 struct virtio_hw *hw = dev->data->dev_private; 1491 struct virtnet_rx *rxvq; 1492 struct rte_mbuf *rarp_mbuf; 1493 1494 if (!dev->data->rx_queues) 1495 return; 1496 1497 rxvq = dev->data->rx_queues[0]; 1498 if (!rxvq) 1499 return; 1500 1501 rarp_mbuf = rte_net_make_rarp_packet(rxvq->mpool, 1502 (struct rte_ether_addr *)hw->mac_addr); 1503 if (rarp_mbuf == NULL) { 1504 PMD_DRV_LOG(ERR, "failed to make RARP packet."); 1505 return; 1506 } 1507 1508 /* If virtio port just stopped, no need to send RARP */ 1509 if (virtio_dev_pause(dev) < 0) { 1510 rte_pktmbuf_free(rarp_mbuf); 1511 return; 1512 } 1513 1514 virtio_inject_pkts(dev, &rarp_mbuf, 1); 1515 virtio_dev_resume(dev); 1516 } 1517 1518 static void 1519 virtio_ack_link_announce(struct rte_eth_dev *dev) 1520 { 1521 struct virtio_hw *hw = dev->data->dev_private; 1522 struct virtio_pmd_ctrl ctrl; 1523 1524 ctrl.hdr.class = VIRTIO_NET_CTRL_ANNOUNCE; 1525 ctrl.hdr.cmd = VIRTIO_NET_CTRL_ANNOUNCE_ACK; 1526 1527 virtio_send_command(hw->cvq, &ctrl, NULL, 0); 1528 } 1529 1530 /* 1531 * Process virtio config changed interrupt. Call the callback 1532 * if link state changed, generate gratuitous RARP packet if 1533 * the status indicates an ANNOUNCE. 1534 */ 1535 void 1536 virtio_interrupt_handler(void *param) 1537 { 1538 struct rte_eth_dev *dev = param; 1539 struct virtio_hw *hw = dev->data->dev_private; 1540 uint8_t isr; 1541 uint16_t status; 1542 1543 /* Read interrupt status which clears interrupt */ 1544 isr = virtio_get_isr(hw); 1545 PMD_DRV_LOG(INFO, "interrupt status = %#x", isr); 1546 1547 if (virtio_intr_unmask(dev) < 0) 1548 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1549 1550 if (isr & VIRTIO_ISR_CONFIG) { 1551 if (virtio_dev_link_update(dev, 0) == 0) 1552 rte_eth_dev_callback_process(dev, 1553 RTE_ETH_EVENT_INTR_LSC, 1554 NULL); 1555 1556 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1557 virtio_read_dev_config(hw, 1558 offsetof(struct virtio_net_config, status), 1559 &status, sizeof(status)); 1560 if (status & VIRTIO_NET_S_ANNOUNCE) { 1561 virtio_notify_peers(dev); 1562 if (hw->cvq) 1563 virtio_ack_link_announce(dev); 1564 } 1565 } 1566 } 1567 } 1568 1569 /* set rx and tx handlers according to what is supported */ 1570 static void 1571 set_rxtx_funcs(struct rte_eth_dev *eth_dev) 1572 { 1573 struct virtio_hw *hw = eth_dev->data->dev_private; 1574 1575 eth_dev->tx_pkt_prepare = virtio_xmit_pkts_prepare; 1576 if (virtio_with_packed_queue(hw)) { 1577 PMD_INIT_LOG(INFO, 1578 "virtio: using packed ring %s Tx path on port %u", 1579 hw->use_vec_tx ? "vectorized" : "standard", 1580 eth_dev->data->port_id); 1581 if (hw->use_vec_tx) 1582 eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed_vec; 1583 else 1584 eth_dev->tx_pkt_burst = virtio_xmit_pkts_packed; 1585 } else { 1586 if (hw->use_inorder_tx) { 1587 PMD_INIT_LOG(INFO, "virtio: using inorder Tx path on port %u", 1588 eth_dev->data->port_id); 1589 eth_dev->tx_pkt_burst = virtio_xmit_pkts_inorder; 1590 } else { 1591 PMD_INIT_LOG(INFO, "virtio: using standard Tx path on port %u", 1592 eth_dev->data->port_id); 1593 eth_dev->tx_pkt_burst = virtio_xmit_pkts; 1594 } 1595 } 1596 1597 if (virtio_with_packed_queue(hw)) { 1598 if (hw->use_vec_rx) { 1599 PMD_INIT_LOG(INFO, 1600 "virtio: using packed ring vectorized Rx path on port %u", 1601 eth_dev->data->port_id); 1602 eth_dev->rx_pkt_burst = 1603 &virtio_recv_pkts_packed_vec; 1604 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1605 PMD_INIT_LOG(INFO, 1606 "virtio: using packed ring mergeable buffer Rx path on port %u", 1607 eth_dev->data->port_id); 1608 eth_dev->rx_pkt_burst = 1609 &virtio_recv_mergeable_pkts_packed; 1610 } else { 1611 PMD_INIT_LOG(INFO, 1612 "virtio: using packed ring standard Rx path on port %u", 1613 eth_dev->data->port_id); 1614 eth_dev->rx_pkt_burst = &virtio_recv_pkts_packed; 1615 } 1616 } else { 1617 if (hw->use_vec_rx) { 1618 PMD_INIT_LOG(INFO, "virtio: using vectorized Rx path on port %u", 1619 eth_dev->data->port_id); 1620 eth_dev->rx_pkt_burst = virtio_recv_pkts_vec; 1621 } else if (hw->use_inorder_rx) { 1622 PMD_INIT_LOG(INFO, 1623 "virtio: using inorder Rx path on port %u", 1624 eth_dev->data->port_id); 1625 eth_dev->rx_pkt_burst = &virtio_recv_pkts_inorder; 1626 } else if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 1627 PMD_INIT_LOG(INFO, 1628 "virtio: using mergeable buffer Rx path on port %u", 1629 eth_dev->data->port_id); 1630 eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts; 1631 } else { 1632 PMD_INIT_LOG(INFO, "virtio: using standard Rx path on port %u", 1633 eth_dev->data->port_id); 1634 eth_dev->rx_pkt_burst = &virtio_recv_pkts; 1635 } 1636 } 1637 1638 } 1639 1640 /* Only support 1:1 queue/interrupt mapping so far. 1641 * TODO: support n:1 queue/interrupt mapping when there are limited number of 1642 * interrupt vectors (<N+1). 1643 */ 1644 static int 1645 virtio_queues_bind_intr(struct rte_eth_dev *dev) 1646 { 1647 uint32_t i; 1648 struct virtio_hw *hw = dev->data->dev_private; 1649 1650 PMD_INIT_LOG(INFO, "queue/interrupt binding"); 1651 for (i = 0; i < dev->data->nb_rx_queues; ++i) { 1652 dev->intr_handle->intr_vec[i] = i + 1; 1653 if (VIRTIO_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) == 1654 VIRTIO_MSI_NO_VECTOR) { 1655 PMD_DRV_LOG(ERR, "failed to set queue vector"); 1656 return -EBUSY; 1657 } 1658 } 1659 1660 return 0; 1661 } 1662 1663 static void 1664 virtio_queues_unbind_intr(struct rte_eth_dev *dev) 1665 { 1666 uint32_t i; 1667 struct virtio_hw *hw = dev->data->dev_private; 1668 1669 PMD_INIT_LOG(INFO, "queue/interrupt unbinding"); 1670 for (i = 0; i < dev->data->nb_rx_queues; ++i) 1671 VIRTIO_OPS(hw)->set_queue_irq(hw, 1672 hw->vqs[i * VTNET_CQ], 1673 VIRTIO_MSI_NO_VECTOR); 1674 } 1675 1676 static int 1677 virtio_configure_intr(struct rte_eth_dev *dev) 1678 { 1679 struct virtio_hw *hw = dev->data->dev_private; 1680 1681 if (!rte_intr_cap_multiple(dev->intr_handle)) { 1682 PMD_INIT_LOG(ERR, "Multiple intr vector not supported"); 1683 return -ENOTSUP; 1684 } 1685 1686 if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) { 1687 PMD_INIT_LOG(ERR, "Fail to create eventfd"); 1688 return -1; 1689 } 1690 1691 if (!dev->intr_handle->intr_vec) { 1692 dev->intr_handle->intr_vec = 1693 rte_zmalloc("intr_vec", 1694 hw->max_queue_pairs * sizeof(int), 0); 1695 if (!dev->intr_handle->intr_vec) { 1696 PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors", 1697 hw->max_queue_pairs); 1698 return -ENOMEM; 1699 } 1700 } 1701 1702 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 1703 /* Re-register callback to update max_intr */ 1704 rte_intr_callback_unregister(dev->intr_handle, 1705 virtio_interrupt_handler, 1706 dev); 1707 rte_intr_callback_register(dev->intr_handle, 1708 virtio_interrupt_handler, 1709 dev); 1710 } 1711 1712 /* DO NOT try to remove this! This function will enable msix, or QEMU 1713 * will encounter SIGSEGV when DRIVER_OK is sent. 1714 * And for legacy devices, this should be done before queue/vec binding 1715 * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR 1716 * (22) will be ignored. 1717 */ 1718 if (virtio_intr_enable(dev) < 0) { 1719 PMD_DRV_LOG(ERR, "interrupt enable failed"); 1720 return -1; 1721 } 1722 1723 if (virtio_queues_bind_intr(dev) < 0) { 1724 PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt"); 1725 return -1; 1726 } 1727 1728 return 0; 1729 } 1730 #define DUPLEX_UNKNOWN 0xff 1731 /* reset device and renegotiate features if needed */ 1732 static int 1733 virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features) 1734 { 1735 struct virtio_hw *hw = eth_dev->data->dev_private; 1736 struct virtio_net_config *config; 1737 struct virtio_net_config local_config; 1738 int ret; 1739 1740 /* Reset the device although not necessary at startup */ 1741 virtio_reset(hw); 1742 1743 if (hw->vqs) { 1744 virtio_dev_free_mbufs(eth_dev); 1745 virtio_free_queues(hw); 1746 } 1747 1748 /* Tell the host we've noticed this device. */ 1749 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK); 1750 1751 /* Tell the host we've known how to drive the device. */ 1752 virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER); 1753 if (virtio_ethdev_negotiate_features(hw, req_features) < 0) 1754 return -1; 1755 1756 hw->weak_barriers = !virtio_with_feature(hw, VIRTIO_F_ORDER_PLATFORM); 1757 1758 /* If host does not support both status and MSI-X then disable LSC */ 1759 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS) && hw->intr_lsc) 1760 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; 1761 else 1762 eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC; 1763 1764 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 1765 1766 /* Setting up rx_header size for the device */ 1767 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) || 1768 virtio_with_feature(hw, VIRTIO_F_VERSION_1) || 1769 virtio_with_packed_queue(hw)) 1770 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf); 1771 else 1772 hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr); 1773 1774 /* Copy the permanent MAC address to: virtio_hw */ 1775 virtio_get_hwaddr(hw); 1776 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac_addr, 1777 ð_dev->data->mac_addrs[0]); 1778 PMD_INIT_LOG(DEBUG, 1779 "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT, 1780 hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2], 1781 hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]); 1782 1783 if (hw->speed == ETH_SPEED_NUM_UNKNOWN) { 1784 if (virtio_with_feature(hw, VIRTIO_NET_F_SPEED_DUPLEX)) { 1785 config = &local_config; 1786 virtio_read_dev_config(hw, 1787 offsetof(struct virtio_net_config, speed), 1788 &config->speed, sizeof(config->speed)); 1789 virtio_read_dev_config(hw, 1790 offsetof(struct virtio_net_config, duplex), 1791 &config->duplex, sizeof(config->duplex)); 1792 hw->speed = config->speed; 1793 hw->duplex = config->duplex; 1794 } 1795 } 1796 if (hw->duplex == DUPLEX_UNKNOWN) 1797 hw->duplex = ETH_LINK_FULL_DUPLEX; 1798 PMD_INIT_LOG(DEBUG, "link speed = %d, duplex = %d", 1799 hw->speed, hw->duplex); 1800 if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) { 1801 config = &local_config; 1802 1803 virtio_read_dev_config(hw, 1804 offsetof(struct virtio_net_config, mac), 1805 &config->mac, sizeof(config->mac)); 1806 1807 if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) { 1808 virtio_read_dev_config(hw, 1809 offsetof(struct virtio_net_config, status), 1810 &config->status, sizeof(config->status)); 1811 } else { 1812 PMD_INIT_LOG(DEBUG, 1813 "VIRTIO_NET_F_STATUS is not supported"); 1814 config->status = 0; 1815 } 1816 1817 if (virtio_with_feature(hw, VIRTIO_NET_F_MQ)) { 1818 virtio_read_dev_config(hw, 1819 offsetof(struct virtio_net_config, max_virtqueue_pairs), 1820 &config->max_virtqueue_pairs, 1821 sizeof(config->max_virtqueue_pairs)); 1822 } else { 1823 PMD_INIT_LOG(DEBUG, 1824 "VIRTIO_NET_F_MQ is not supported"); 1825 config->max_virtqueue_pairs = 1; 1826 } 1827 1828 hw->max_queue_pairs = config->max_virtqueue_pairs; 1829 1830 if (virtio_with_feature(hw, VIRTIO_NET_F_MTU)) { 1831 virtio_read_dev_config(hw, 1832 offsetof(struct virtio_net_config, mtu), 1833 &config->mtu, 1834 sizeof(config->mtu)); 1835 1836 /* 1837 * MTU value has already been checked at negotiation 1838 * time, but check again in case it has changed since 1839 * then, which should not happen. 1840 */ 1841 if (config->mtu < RTE_ETHER_MIN_MTU) { 1842 PMD_INIT_LOG(ERR, "invalid max MTU value (%u)", 1843 config->mtu); 1844 return -1; 1845 } 1846 1847 hw->max_mtu = config->mtu; 1848 /* Set initial MTU to maximum one supported by vhost */ 1849 eth_dev->data->mtu = config->mtu; 1850 1851 } else { 1852 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN - 1853 VLAN_TAG_LEN - hw->vtnet_hdr_size; 1854 } 1855 1856 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d", 1857 config->max_virtqueue_pairs); 1858 PMD_INIT_LOG(DEBUG, "config->status=%d", config->status); 1859 PMD_INIT_LOG(DEBUG, 1860 "PORT MAC: " RTE_ETHER_ADDR_PRT_FMT, 1861 config->mac[0], config->mac[1], 1862 config->mac[2], config->mac[3], 1863 config->mac[4], config->mac[5]); 1864 } else { 1865 PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1"); 1866 hw->max_queue_pairs = 1; 1867 hw->max_mtu = VIRTIO_MAX_RX_PKTLEN - RTE_ETHER_HDR_LEN - 1868 VLAN_TAG_LEN - hw->vtnet_hdr_size; 1869 } 1870 1871 ret = virtio_alloc_queues(eth_dev); 1872 if (ret < 0) 1873 return ret; 1874 1875 if (eth_dev->data->dev_conf.intr_conf.rxq) { 1876 if (virtio_configure_intr(eth_dev) < 0) { 1877 PMD_INIT_LOG(ERR, "failed to configure interrupt"); 1878 virtio_free_queues(hw); 1879 return -1; 1880 } 1881 } 1882 1883 virtio_reinit_complete(hw); 1884 1885 return 0; 1886 } 1887 1888 /* 1889 * This function is based on probe() function in virtio_pci.c 1890 * It returns 0 on success. 1891 */ 1892 int 1893 eth_virtio_dev_init(struct rte_eth_dev *eth_dev) 1894 { 1895 struct virtio_hw *hw = eth_dev->data->dev_private; 1896 uint32_t speed = ETH_SPEED_NUM_UNKNOWN; 1897 int vectorized = 0; 1898 int ret; 1899 1900 if (sizeof(struct virtio_net_hdr_mrg_rxbuf) > RTE_PKTMBUF_HEADROOM) { 1901 PMD_INIT_LOG(ERR, 1902 "Not sufficient headroom required = %d, avail = %d", 1903 (int)sizeof(struct virtio_net_hdr_mrg_rxbuf), 1904 RTE_PKTMBUF_HEADROOM); 1905 1906 return -1; 1907 } 1908 1909 eth_dev->dev_ops = &virtio_eth_dev_ops; 1910 eth_dev->rx_descriptor_done = virtio_dev_rx_queue_done; 1911 1912 if (rte_eal_process_type() == RTE_PROC_SECONDARY) { 1913 set_rxtx_funcs(eth_dev); 1914 return 0; 1915 } 1916 1917 ret = virtio_dev_devargs_parse(eth_dev->device->devargs, &speed, &vectorized); 1918 if (ret < 0) 1919 return ret; 1920 hw->speed = speed; 1921 hw->duplex = DUPLEX_UNKNOWN; 1922 1923 /* Allocate memory for storing MAC addresses */ 1924 eth_dev->data->mac_addrs = rte_zmalloc("virtio", 1925 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN, 0); 1926 if (eth_dev->data->mac_addrs == NULL) { 1927 PMD_INIT_LOG(ERR, 1928 "Failed to allocate %d bytes needed to store MAC addresses", 1929 VIRTIO_MAX_MAC_ADDRS * RTE_ETHER_ADDR_LEN); 1930 return -ENOMEM; 1931 } 1932 1933 rte_spinlock_init(&hw->state_lock); 1934 1935 /* reset device and negotiate default features */ 1936 ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES); 1937 if (ret < 0) 1938 goto err_virtio_init; 1939 1940 if (vectorized) { 1941 if (!virtio_with_packed_queue(hw)) { 1942 hw->use_vec_rx = 1; 1943 } else { 1944 #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM) 1945 hw->use_vec_rx = 1; 1946 hw->use_vec_tx = 1; 1947 #else 1948 PMD_DRV_LOG(INFO, 1949 "building environment do not support packed ring vectorized"); 1950 #endif 1951 } 1952 } 1953 1954 hw->opened = 1; 1955 1956 return 0; 1957 1958 err_virtio_init: 1959 rte_free(eth_dev->data->mac_addrs); 1960 eth_dev->data->mac_addrs = NULL; 1961 return ret; 1962 } 1963 1964 static uint32_t 1965 virtio_dev_speed_capa_get(uint32_t speed) 1966 { 1967 switch (speed) { 1968 case ETH_SPEED_NUM_10G: 1969 return ETH_LINK_SPEED_10G; 1970 case ETH_SPEED_NUM_20G: 1971 return ETH_LINK_SPEED_20G; 1972 case ETH_SPEED_NUM_25G: 1973 return ETH_LINK_SPEED_25G; 1974 case ETH_SPEED_NUM_40G: 1975 return ETH_LINK_SPEED_40G; 1976 case ETH_SPEED_NUM_50G: 1977 return ETH_LINK_SPEED_50G; 1978 case ETH_SPEED_NUM_56G: 1979 return ETH_LINK_SPEED_56G; 1980 case ETH_SPEED_NUM_100G: 1981 return ETH_LINK_SPEED_100G; 1982 case ETH_SPEED_NUM_200G: 1983 return ETH_LINK_SPEED_200G; 1984 default: 1985 return 0; 1986 } 1987 } 1988 1989 static int vectorized_check_handler(__rte_unused const char *key, 1990 const char *value, void *ret_val) 1991 { 1992 if (strcmp(value, "1") == 0) 1993 *(int *)ret_val = 1; 1994 else 1995 *(int *)ret_val = 0; 1996 1997 return 0; 1998 } 1999 2000 #define VIRTIO_ARG_SPEED "speed" 2001 #define VIRTIO_ARG_VECTORIZED "vectorized" 2002 2003 static int 2004 link_speed_handler(const char *key __rte_unused, 2005 const char *value, void *ret_val) 2006 { 2007 uint32_t val; 2008 if (!value || !ret_val) 2009 return -EINVAL; 2010 val = strtoul(value, NULL, 0); 2011 /* validate input */ 2012 if (virtio_dev_speed_capa_get(val) == 0) 2013 return -EINVAL; 2014 *(uint32_t *)ret_val = val; 2015 2016 return 0; 2017 } 2018 2019 2020 static int 2021 virtio_dev_devargs_parse(struct rte_devargs *devargs, uint32_t *speed, int *vectorized) 2022 { 2023 struct rte_kvargs *kvlist; 2024 int ret = 0; 2025 2026 if (devargs == NULL) 2027 return 0; 2028 2029 kvlist = rte_kvargs_parse(devargs->args, NULL); 2030 if (kvlist == NULL) { 2031 PMD_INIT_LOG(ERR, "error when parsing param"); 2032 return 0; 2033 } 2034 2035 if (speed && rte_kvargs_count(kvlist, VIRTIO_ARG_SPEED) == 1) { 2036 ret = rte_kvargs_process(kvlist, 2037 VIRTIO_ARG_SPEED, 2038 link_speed_handler, speed); 2039 if (ret < 0) { 2040 PMD_INIT_LOG(ERR, "Failed to parse %s", 2041 VIRTIO_ARG_SPEED); 2042 goto exit; 2043 } 2044 } 2045 2046 if (vectorized && 2047 rte_kvargs_count(kvlist, VIRTIO_ARG_VECTORIZED) == 1) { 2048 ret = rte_kvargs_process(kvlist, 2049 VIRTIO_ARG_VECTORIZED, 2050 vectorized_check_handler, vectorized); 2051 if (ret < 0) { 2052 PMD_INIT_LOG(ERR, "Failed to parse %s", 2053 VIRTIO_ARG_VECTORIZED); 2054 goto exit; 2055 } 2056 } 2057 2058 exit: 2059 rte_kvargs_free(kvlist); 2060 return ret; 2061 } 2062 2063 static uint8_t 2064 rx_offload_enabled(struct virtio_hw *hw) 2065 { 2066 return virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM) || 2067 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || 2068 virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6); 2069 } 2070 2071 static uint8_t 2072 tx_offload_enabled(struct virtio_hw *hw) 2073 { 2074 return virtio_with_feature(hw, VIRTIO_NET_F_CSUM) || 2075 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO4) || 2076 virtio_with_feature(hw, VIRTIO_NET_F_HOST_TSO6); 2077 } 2078 2079 /* 2080 * Configure virtio device 2081 * It returns 0 on success. 2082 */ 2083 static int 2084 virtio_dev_configure(struct rte_eth_dev *dev) 2085 { 2086 const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 2087 const struct rte_eth_txmode *txmode = &dev->data->dev_conf.txmode; 2088 struct virtio_hw *hw = dev->data->dev_private; 2089 uint32_t ether_hdr_len = RTE_ETHER_HDR_LEN + VLAN_TAG_LEN + 2090 hw->vtnet_hdr_size; 2091 uint64_t rx_offloads = rxmode->offloads; 2092 uint64_t tx_offloads = txmode->offloads; 2093 uint64_t req_features; 2094 int ret; 2095 2096 PMD_INIT_LOG(DEBUG, "configure"); 2097 req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES; 2098 2099 if (rxmode->mq_mode != ETH_MQ_RX_NONE) { 2100 PMD_DRV_LOG(ERR, 2101 "Unsupported Rx multi queue mode %d", 2102 rxmode->mq_mode); 2103 return -EINVAL; 2104 } 2105 2106 if (txmode->mq_mode != ETH_MQ_TX_NONE) { 2107 PMD_DRV_LOG(ERR, 2108 "Unsupported Tx multi queue mode %d", 2109 txmode->mq_mode); 2110 return -EINVAL; 2111 } 2112 2113 if (dev->data->dev_conf.intr_conf.rxq) { 2114 ret = virtio_init_device(dev, hw->req_guest_features); 2115 if (ret < 0) 2116 return ret; 2117 } 2118 2119 if ((rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) && 2120 (rxmode->max_rx_pkt_len > hw->max_mtu + ether_hdr_len)) 2121 req_features &= ~(1ULL << VIRTIO_NET_F_MTU); 2122 2123 if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) 2124 hw->max_rx_pkt_len = rxmode->max_rx_pkt_len; 2125 else 2126 hw->max_rx_pkt_len = ether_hdr_len + dev->data->mtu; 2127 2128 if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | 2129 DEV_RX_OFFLOAD_TCP_CKSUM)) 2130 req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM); 2131 2132 if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) 2133 req_features |= 2134 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 2135 (1ULL << VIRTIO_NET_F_GUEST_TSO6); 2136 2137 if (tx_offloads & (DEV_TX_OFFLOAD_UDP_CKSUM | 2138 DEV_TX_OFFLOAD_TCP_CKSUM)) 2139 req_features |= (1ULL << VIRTIO_NET_F_CSUM); 2140 2141 if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) 2142 req_features |= 2143 (1ULL << VIRTIO_NET_F_HOST_TSO4) | 2144 (1ULL << VIRTIO_NET_F_HOST_TSO6); 2145 2146 /* if request features changed, reinit the device */ 2147 if (req_features != hw->req_guest_features) { 2148 ret = virtio_init_device(dev, req_features); 2149 if (ret < 0) 2150 return ret; 2151 } 2152 2153 if ((rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | 2154 DEV_RX_OFFLOAD_TCP_CKSUM)) && 2155 !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) { 2156 PMD_DRV_LOG(ERR, 2157 "rx checksum not available on this host"); 2158 return -ENOTSUP; 2159 } 2160 2161 if ((rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) && 2162 (!virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) || 2163 !virtio_with_feature(hw, VIRTIO_NET_F_GUEST_TSO6))) { 2164 PMD_DRV_LOG(ERR, 2165 "Large Receive Offload not available on this host"); 2166 return -ENOTSUP; 2167 } 2168 2169 /* start control queue */ 2170 if (virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) 2171 virtio_dev_cq_start(dev); 2172 2173 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 2174 hw->vlan_strip = 1; 2175 2176 hw->rx_ol_scatter = (rx_offloads & DEV_RX_OFFLOAD_SCATTER); 2177 2178 if ((rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER) && 2179 !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { 2180 PMD_DRV_LOG(ERR, 2181 "vlan filtering not available on this host"); 2182 return -ENOTSUP; 2183 } 2184 2185 hw->has_tx_offload = tx_offload_enabled(hw); 2186 hw->has_rx_offload = rx_offload_enabled(hw); 2187 2188 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 2189 /* Enable vector (0) for Link State Intrerrupt */ 2190 if (VIRTIO_OPS(hw)->set_config_irq(hw, 0) == 2191 VIRTIO_MSI_NO_VECTOR) { 2192 PMD_DRV_LOG(ERR, "failed to set config vector"); 2193 return -EBUSY; 2194 } 2195 2196 if (virtio_with_packed_queue(hw)) { 2197 #if defined(RTE_ARCH_X86_64) && defined(CC_AVX512_SUPPORT) 2198 if ((hw->use_vec_rx || hw->use_vec_tx) && 2199 (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F) || 2200 !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) || 2201 !virtio_with_feature(hw, VIRTIO_F_VERSION_1) || 2202 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_512)) { 2203 PMD_DRV_LOG(INFO, 2204 "disabled packed ring vectorized path for requirements not met"); 2205 hw->use_vec_rx = 0; 2206 hw->use_vec_tx = 0; 2207 } 2208 #elif defined(RTE_ARCH_ARM) 2209 if ((hw->use_vec_rx || hw->use_vec_tx) && 2210 (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) || 2211 !virtio_with_feature(hw, VIRTIO_F_IN_ORDER) || 2212 !virtio_with_feature(hw, VIRTIO_F_VERSION_1) || 2213 rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128)) { 2214 PMD_DRV_LOG(INFO, 2215 "disabled packed ring vectorized path for requirements not met"); 2216 hw->use_vec_rx = 0; 2217 hw->use_vec_tx = 0; 2218 } 2219 #else 2220 hw->use_vec_rx = 0; 2221 hw->use_vec_tx = 0; 2222 #endif 2223 2224 if (hw->use_vec_rx) { 2225 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 2226 PMD_DRV_LOG(INFO, 2227 "disabled packed ring vectorized rx for mrg_rxbuf enabled"); 2228 hw->use_vec_rx = 0; 2229 } 2230 2231 if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) { 2232 PMD_DRV_LOG(INFO, 2233 "disabled packed ring vectorized rx for TCP_LRO enabled"); 2234 hw->use_vec_rx = 0; 2235 } 2236 } 2237 } else { 2238 if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) { 2239 hw->use_inorder_tx = 1; 2240 hw->use_inorder_rx = 1; 2241 hw->use_vec_rx = 0; 2242 } 2243 2244 if (hw->use_vec_rx) { 2245 #if defined RTE_ARCH_ARM 2246 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { 2247 PMD_DRV_LOG(INFO, 2248 "disabled split ring vectorized path for requirement not met"); 2249 hw->use_vec_rx = 0; 2250 } 2251 #endif 2252 if (virtio_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF)) { 2253 PMD_DRV_LOG(INFO, 2254 "disabled split ring vectorized rx for mrg_rxbuf enabled"); 2255 hw->use_vec_rx = 0; 2256 } 2257 2258 if (rx_offloads & (DEV_RX_OFFLOAD_UDP_CKSUM | 2259 DEV_RX_OFFLOAD_TCP_CKSUM | 2260 DEV_RX_OFFLOAD_TCP_LRO | 2261 DEV_RX_OFFLOAD_VLAN_STRIP)) { 2262 PMD_DRV_LOG(INFO, 2263 "disabled split ring vectorized rx for offloading enabled"); 2264 hw->use_vec_rx = 0; 2265 } 2266 2267 if (rte_vect_get_max_simd_bitwidth() < RTE_VECT_SIMD_128) { 2268 PMD_DRV_LOG(INFO, 2269 "disabled split ring vectorized rx, max SIMD bitwidth too low"); 2270 hw->use_vec_rx = 0; 2271 } 2272 } 2273 } 2274 2275 return 0; 2276 } 2277 2278 2279 static int 2280 virtio_dev_start(struct rte_eth_dev *dev) 2281 { 2282 uint16_t nb_queues, i; 2283 struct virtqueue *vq; 2284 struct virtio_hw *hw = dev->data->dev_private; 2285 int ret; 2286 2287 /* Finish the initialization of the queues */ 2288 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2289 ret = virtio_dev_rx_queue_setup_finish(dev, i); 2290 if (ret < 0) 2291 return ret; 2292 } 2293 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2294 ret = virtio_dev_tx_queue_setup_finish(dev, i); 2295 if (ret < 0) 2296 return ret; 2297 } 2298 2299 /* check if lsc interrupt feature is enabled */ 2300 if (dev->data->dev_conf.intr_conf.lsc) { 2301 if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) { 2302 PMD_DRV_LOG(ERR, "link status not supported by host"); 2303 return -ENOTSUP; 2304 } 2305 } 2306 2307 /* Enable uio/vfio intr/eventfd mapping: althrough we already did that 2308 * in device configure, but it could be unmapped when device is 2309 * stopped. 2310 */ 2311 if (dev->data->dev_conf.intr_conf.lsc || 2312 dev->data->dev_conf.intr_conf.rxq) { 2313 virtio_intr_disable(dev); 2314 2315 /* Setup interrupt callback */ 2316 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) 2317 rte_intr_callback_register(dev->intr_handle, 2318 virtio_interrupt_handler, 2319 dev); 2320 2321 if (virtio_intr_enable(dev) < 0) { 2322 PMD_DRV_LOG(ERR, "interrupt enable failed"); 2323 return -EIO; 2324 } 2325 } 2326 2327 /*Notify the backend 2328 *Otherwise the tap backend might already stop its queue due to fullness. 2329 *vhost backend will have no chance to be waked up 2330 */ 2331 nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues); 2332 if (hw->max_queue_pairs > 1) { 2333 if (virtio_set_multiple_queues(dev, nb_queues) != 0) 2334 return -EINVAL; 2335 } 2336 2337 PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues); 2338 2339 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2340 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]); 2341 /* Flush the old packets */ 2342 virtqueue_rxvq_flush(vq); 2343 virtqueue_notify(vq); 2344 } 2345 2346 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2347 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]); 2348 virtqueue_notify(vq); 2349 } 2350 2351 PMD_INIT_LOG(DEBUG, "Notified backend at initialization"); 2352 2353 for (i = 0; i < dev->data->nb_rx_queues; i++) { 2354 vq = virtnet_rxq_to_vq(dev->data->rx_queues[i]); 2355 VIRTQUEUE_DUMP(vq); 2356 } 2357 2358 for (i = 0; i < dev->data->nb_tx_queues; i++) { 2359 vq = virtnet_txq_to_vq(dev->data->tx_queues[i]); 2360 VIRTQUEUE_DUMP(vq); 2361 } 2362 2363 set_rxtx_funcs(dev); 2364 hw->started = 1; 2365 2366 /* Initialize Link state */ 2367 virtio_dev_link_update(dev, 0); 2368 2369 return 0; 2370 } 2371 2372 static void virtio_dev_free_mbufs(struct rte_eth_dev *dev) 2373 { 2374 struct virtio_hw *hw = dev->data->dev_private; 2375 uint16_t nr_vq = virtio_get_nr_vq(hw); 2376 const char *type __rte_unused; 2377 unsigned int i, mbuf_num = 0; 2378 struct virtqueue *vq; 2379 struct rte_mbuf *buf; 2380 int queue_type; 2381 2382 if (hw->vqs == NULL) 2383 return; 2384 2385 for (i = 0; i < nr_vq; i++) { 2386 vq = hw->vqs[i]; 2387 if (!vq) 2388 continue; 2389 2390 queue_type = virtio_get_queue_type(hw, i); 2391 if (queue_type == VTNET_RQ) 2392 type = "rxq"; 2393 else if (queue_type == VTNET_TQ) 2394 type = "txq"; 2395 else 2396 continue; 2397 2398 PMD_INIT_LOG(DEBUG, 2399 "Before freeing %s[%d] used and unused buf", 2400 type, i); 2401 VIRTQUEUE_DUMP(vq); 2402 2403 while ((buf = virtqueue_detach_unused(vq)) != NULL) { 2404 rte_pktmbuf_free(buf); 2405 mbuf_num++; 2406 } 2407 2408 PMD_INIT_LOG(DEBUG, 2409 "After freeing %s[%d] used and unused buf", 2410 type, i); 2411 VIRTQUEUE_DUMP(vq); 2412 } 2413 2414 PMD_INIT_LOG(DEBUG, "%d mbufs freed", mbuf_num); 2415 } 2416 2417 static void 2418 virtio_tx_completed_cleanup(struct rte_eth_dev *dev) 2419 { 2420 struct virtio_hw *hw = dev->data->dev_private; 2421 struct virtqueue *vq; 2422 int qidx; 2423 void (*xmit_cleanup)(struct virtqueue *vq, uint16_t nb_used); 2424 2425 if (virtio_with_packed_queue(hw)) { 2426 if (hw->use_vec_tx) 2427 xmit_cleanup = &virtio_xmit_cleanup_inorder_packed; 2428 else if (virtio_with_feature(hw, VIRTIO_F_IN_ORDER)) 2429 xmit_cleanup = &virtio_xmit_cleanup_inorder_packed; 2430 else 2431 xmit_cleanup = &virtio_xmit_cleanup_normal_packed; 2432 } else { 2433 if (hw->use_inorder_tx) 2434 xmit_cleanup = &virtio_xmit_cleanup_inorder; 2435 else 2436 xmit_cleanup = &virtio_xmit_cleanup; 2437 } 2438 2439 for (qidx = 0; qidx < hw->max_queue_pairs; qidx++) { 2440 vq = hw->vqs[2 * qidx + VTNET_SQ_TQ_QUEUE_IDX]; 2441 if (vq != NULL) 2442 xmit_cleanup(vq, virtqueue_nused(vq)); 2443 } 2444 } 2445 2446 /* 2447 * Stop device: disable interrupt and mark link down 2448 */ 2449 int 2450 virtio_dev_stop(struct rte_eth_dev *dev) 2451 { 2452 struct virtio_hw *hw = dev->data->dev_private; 2453 struct rte_eth_link link; 2454 struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf; 2455 2456 PMD_INIT_LOG(DEBUG, "stop"); 2457 dev->data->dev_started = 0; 2458 2459 rte_spinlock_lock(&hw->state_lock); 2460 if (!hw->started) 2461 goto out_unlock; 2462 hw->started = 0; 2463 2464 virtio_tx_completed_cleanup(dev); 2465 2466 if (intr_conf->lsc || intr_conf->rxq) { 2467 virtio_intr_disable(dev); 2468 2469 /* Reset interrupt callback */ 2470 if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) { 2471 rte_intr_callback_unregister(dev->intr_handle, 2472 virtio_interrupt_handler, 2473 dev); 2474 } 2475 } 2476 2477 memset(&link, 0, sizeof(link)); 2478 rte_eth_linkstatus_set(dev, &link); 2479 out_unlock: 2480 rte_spinlock_unlock(&hw->state_lock); 2481 2482 return 0; 2483 } 2484 2485 static int 2486 virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) 2487 { 2488 struct rte_eth_link link; 2489 uint16_t status; 2490 struct virtio_hw *hw = dev->data->dev_private; 2491 2492 memset(&link, 0, sizeof(link)); 2493 link.link_duplex = hw->duplex; 2494 link.link_speed = hw->speed; 2495 link.link_autoneg = ETH_LINK_AUTONEG; 2496 2497 if (!hw->started) { 2498 link.link_status = ETH_LINK_DOWN; 2499 link.link_speed = ETH_SPEED_NUM_NONE; 2500 } else if (virtio_with_feature(hw, VIRTIO_NET_F_STATUS)) { 2501 PMD_INIT_LOG(DEBUG, "Get link status from hw"); 2502 virtio_read_dev_config(hw, 2503 offsetof(struct virtio_net_config, status), 2504 &status, sizeof(status)); 2505 if ((status & VIRTIO_NET_S_LINK_UP) == 0) { 2506 link.link_status = ETH_LINK_DOWN; 2507 link.link_speed = ETH_SPEED_NUM_NONE; 2508 PMD_INIT_LOG(DEBUG, "Port %d is down", 2509 dev->data->port_id); 2510 } else { 2511 link.link_status = ETH_LINK_UP; 2512 PMD_INIT_LOG(DEBUG, "Port %d is up", 2513 dev->data->port_id); 2514 } 2515 } else { 2516 link.link_status = ETH_LINK_UP; 2517 } 2518 2519 return rte_eth_linkstatus_set(dev, &link); 2520 } 2521 2522 static int 2523 virtio_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 2524 { 2525 const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 2526 struct virtio_hw *hw = dev->data->dev_private; 2527 uint64_t offloads = rxmode->offloads; 2528 2529 if (mask & ETH_VLAN_FILTER_MASK) { 2530 if ((offloads & DEV_RX_OFFLOAD_VLAN_FILTER) && 2531 !virtio_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) { 2532 2533 PMD_DRV_LOG(NOTICE, 2534 "vlan filtering not available on this host"); 2535 2536 return -ENOTSUP; 2537 } 2538 } 2539 2540 if (mask & ETH_VLAN_STRIP_MASK) 2541 hw->vlan_strip = !!(offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 2542 2543 return 0; 2544 } 2545 2546 static int 2547 virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2548 { 2549 uint64_t tso_mask, host_features; 2550 struct virtio_hw *hw = dev->data->dev_private; 2551 dev_info->speed_capa = virtio_dev_speed_capa_get(hw->speed); 2552 2553 dev_info->max_rx_queues = 2554 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES); 2555 dev_info->max_tx_queues = 2556 RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES); 2557 dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE; 2558 dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN; 2559 dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS; 2560 dev_info->max_mtu = hw->max_mtu; 2561 2562 host_features = VIRTIO_OPS(hw)->get_features(hw); 2563 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 2564 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_JUMBO_FRAME; 2565 if (host_features & (1ULL << VIRTIO_NET_F_MRG_RXBUF)) 2566 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_SCATTER; 2567 if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) { 2568 dev_info->rx_offload_capa |= 2569 DEV_RX_OFFLOAD_TCP_CKSUM | 2570 DEV_RX_OFFLOAD_UDP_CKSUM; 2571 } 2572 if (host_features & (1ULL << VIRTIO_NET_F_CTRL_VLAN)) 2573 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_VLAN_FILTER; 2574 tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 2575 (1ULL << VIRTIO_NET_F_GUEST_TSO6); 2576 if ((host_features & tso_mask) == tso_mask) 2577 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO; 2578 2579 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 2580 DEV_TX_OFFLOAD_VLAN_INSERT; 2581 if (host_features & (1ULL << VIRTIO_NET_F_CSUM)) { 2582 dev_info->tx_offload_capa |= 2583 DEV_TX_OFFLOAD_UDP_CKSUM | 2584 DEV_TX_OFFLOAD_TCP_CKSUM; 2585 } 2586 tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) | 2587 (1ULL << VIRTIO_NET_F_HOST_TSO6); 2588 if ((host_features & tso_mask) == tso_mask) 2589 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 2590 2591 if (host_features & (1ULL << VIRTIO_F_RING_PACKED)) { 2592 /* 2593 * According to 2.7 Packed Virtqueues, 2594 * 2.7.10.1 Structure Size and Alignment: 2595 * The Queue Size value does not have to be a power of 2. 2596 */ 2597 dev_info->rx_desc_lim.nb_max = UINT16_MAX; 2598 dev_info->tx_desc_lim.nb_max = UINT16_MAX; 2599 } else { 2600 /* 2601 * According to 2.6 Split Virtqueues: 2602 * Queue Size value is always a power of 2. The maximum Queue 2603 * Size value is 32768. 2604 */ 2605 dev_info->rx_desc_lim.nb_max = 32768; 2606 dev_info->tx_desc_lim.nb_max = 32768; 2607 } 2608 /* 2609 * Actual minimum is not the same for virtqueues of different kinds, 2610 * but to avoid tangling the code with separate branches, rely on 2611 * default thresholds since desc number must be at least of their size. 2612 */ 2613 dev_info->rx_desc_lim.nb_min = RTE_MAX(DEFAULT_RX_FREE_THRESH, 2614 RTE_VIRTIO_VPMD_RX_REARM_THRESH); 2615 dev_info->tx_desc_lim.nb_min = DEFAULT_TX_FREE_THRESH; 2616 dev_info->rx_desc_lim.nb_align = 1; 2617 dev_info->tx_desc_lim.nb_align = 1; 2618 2619 return 0; 2620 } 2621 2622 /* 2623 * It enables testpmd to collect per queue stats. 2624 */ 2625 static int 2626 virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev, 2627 __rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx, 2628 __rte_unused uint8_t is_rx) 2629 { 2630 return 0; 2631 } 2632 2633 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_init, init, NOTICE); 2634 RTE_LOG_REGISTER_SUFFIX(virtio_logtype_driver, driver, NOTICE); 2635