1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_interrupts.h> 17 #include <rte_debug.h> 18 #include <rte_pci.h> 19 #include <rte_atomic.h> 20 #include <rte_eal.h> 21 #include <rte_ether.h> 22 #include <rte_ethdev_driver.h> 23 #include <rte_ethdev_pci.h> 24 #include <rte_malloc.h> 25 #include <rte_memzone.h> 26 #include <rte_dev.h> 27 28 #include "iavf.h" 29 #include "iavf_rxtx.h" 30 #include "iavf_generic_flow.h" 31 32 static int iavf_dev_configure(struct rte_eth_dev *dev); 33 static int iavf_dev_start(struct rte_eth_dev *dev); 34 static void iavf_dev_stop(struct rte_eth_dev *dev); 35 static void iavf_dev_close(struct rte_eth_dev *dev); 36 static int iavf_dev_reset(struct rte_eth_dev *dev); 37 static int iavf_dev_info_get(struct rte_eth_dev *dev, 38 struct rte_eth_dev_info *dev_info); 39 static const uint32_t *iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev); 40 static int iavf_dev_stats_get(struct rte_eth_dev *dev, 41 struct rte_eth_stats *stats); 42 static int iavf_dev_stats_reset(struct rte_eth_dev *dev); 43 static int iavf_dev_promiscuous_enable(struct rte_eth_dev *dev); 44 static int iavf_dev_promiscuous_disable(struct rte_eth_dev *dev); 45 static int iavf_dev_allmulticast_enable(struct rte_eth_dev *dev); 46 static int iavf_dev_allmulticast_disable(struct rte_eth_dev *dev); 47 static int iavf_dev_add_mac_addr(struct rte_eth_dev *dev, 48 struct rte_ether_addr *addr, 49 uint32_t index, 50 uint32_t pool); 51 static void iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); 52 static int iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, 53 uint16_t vlan_id, int on); 54 static int iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 55 static int iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 56 struct rte_eth_rss_reta_entry64 *reta_conf, 57 uint16_t reta_size); 58 static int iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 59 struct rte_eth_rss_reta_entry64 *reta_conf, 60 uint16_t reta_size); 61 static int iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 62 struct rte_eth_rss_conf *rss_conf); 63 static int iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 64 struct rte_eth_rss_conf *rss_conf); 65 static int iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 66 static int iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 67 struct rte_ether_addr *mac_addr); 68 static int iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 69 uint16_t queue_id); 70 static int iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 71 uint16_t queue_id); 72 static int iavf_dev_filter_ctrl(struct rte_eth_dev *dev, 73 enum rte_filter_type filter_type, 74 enum rte_filter_op filter_op, 75 void *arg); 76 static int iavf_set_mc_addr_list(struct rte_eth_dev *dev, 77 struct rte_ether_addr *mc_addrs, 78 uint32_t mc_addrs_num); 79 80 static const struct rte_pci_id pci_id_iavf_map[] = { 81 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_ADAPTIVE_VF) }, 82 { .vendor_id = 0, /* sentinel */ }, 83 }; 84 85 static const struct eth_dev_ops iavf_eth_dev_ops = { 86 .dev_configure = iavf_dev_configure, 87 .dev_start = iavf_dev_start, 88 .dev_stop = iavf_dev_stop, 89 .dev_close = iavf_dev_close, 90 .dev_reset = iavf_dev_reset, 91 .dev_infos_get = iavf_dev_info_get, 92 .dev_supported_ptypes_get = iavf_dev_supported_ptypes_get, 93 .link_update = iavf_dev_link_update, 94 .stats_get = iavf_dev_stats_get, 95 .stats_reset = iavf_dev_stats_reset, 96 .promiscuous_enable = iavf_dev_promiscuous_enable, 97 .promiscuous_disable = iavf_dev_promiscuous_disable, 98 .allmulticast_enable = iavf_dev_allmulticast_enable, 99 .allmulticast_disable = iavf_dev_allmulticast_disable, 100 .mac_addr_add = iavf_dev_add_mac_addr, 101 .mac_addr_remove = iavf_dev_del_mac_addr, 102 .set_mc_addr_list = iavf_set_mc_addr_list, 103 .vlan_filter_set = iavf_dev_vlan_filter_set, 104 .vlan_offload_set = iavf_dev_vlan_offload_set, 105 .rx_queue_start = iavf_dev_rx_queue_start, 106 .rx_queue_stop = iavf_dev_rx_queue_stop, 107 .tx_queue_start = iavf_dev_tx_queue_start, 108 .tx_queue_stop = iavf_dev_tx_queue_stop, 109 .rx_queue_setup = iavf_dev_rx_queue_setup, 110 .rx_queue_release = iavf_dev_rx_queue_release, 111 .tx_queue_setup = iavf_dev_tx_queue_setup, 112 .tx_queue_release = iavf_dev_tx_queue_release, 113 .mac_addr_set = iavf_dev_set_default_mac_addr, 114 .reta_update = iavf_dev_rss_reta_update, 115 .reta_query = iavf_dev_rss_reta_query, 116 .rss_hash_update = iavf_dev_rss_hash_update, 117 .rss_hash_conf_get = iavf_dev_rss_hash_conf_get, 118 .rxq_info_get = iavf_dev_rxq_info_get, 119 .txq_info_get = iavf_dev_txq_info_get, 120 .mtu_set = iavf_dev_mtu_set, 121 .rx_queue_intr_enable = iavf_dev_rx_queue_intr_enable, 122 .rx_queue_intr_disable = iavf_dev_rx_queue_intr_disable, 123 .filter_ctrl = iavf_dev_filter_ctrl, 124 }; 125 126 static int 127 iavf_set_mc_addr_list(struct rte_eth_dev *dev, 128 struct rte_ether_addr *mc_addrs, 129 uint32_t mc_addrs_num) 130 { 131 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 132 struct iavf_adapter *adapter = 133 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 134 int err; 135 136 /* flush previous addresses */ 137 err = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 138 false); 139 if (err) 140 return err; 141 142 vf->mc_addrs_num = 0; 143 144 /* add new ones */ 145 err = iavf_add_del_mc_addr_list(adapter, mc_addrs, mc_addrs_num, true); 146 if (err) 147 return err; 148 149 vf->mc_addrs_num = mc_addrs_num; 150 memcpy(vf->mc_addrs, mc_addrs, mc_addrs_num * sizeof(*mc_addrs)); 151 152 return 0; 153 } 154 155 static int 156 iavf_init_rss(struct iavf_adapter *adapter) 157 { 158 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 159 struct rte_eth_rss_conf *rss_conf; 160 uint16_t i, j, nb_q; 161 int ret; 162 163 rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf; 164 nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues, 165 IAVF_MAX_NUM_QUEUES); 166 167 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 168 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 169 return -ENOTSUP; 170 } 171 if (adapter->eth_dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 172 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default"); 173 /* set all lut items to default queue */ 174 for (i = 0; i < vf->vf_res->rss_lut_size; i++) 175 vf->rss_lut[i] = 0; 176 ret = iavf_configure_rss_lut(adapter); 177 return ret; 178 } 179 180 /* In IAVF, RSS enablement is set by PF driver. It is not supported 181 * to set based on rss_conf->rss_hf. 182 */ 183 184 /* configure RSS key */ 185 if (!rss_conf->rss_key) { 186 /* Calculate the default hash key */ 187 for (i = 0; i <= vf->vf_res->rss_key_size; i++) 188 vf->rss_key[i] = (uint8_t)rte_rand(); 189 } else 190 rte_memcpy(vf->rss_key, rss_conf->rss_key, 191 RTE_MIN(rss_conf->rss_key_len, 192 vf->vf_res->rss_key_size)); 193 194 /* init RSS LUT table */ 195 for (i = 0, j = 0; i < vf->vf_res->rss_lut_size; i++, j++) { 196 if (j >= nb_q) 197 j = 0; 198 vf->rss_lut[i] = j; 199 } 200 /* send virtchnnl ops to configure rss*/ 201 ret = iavf_configure_rss_lut(adapter); 202 if (ret) 203 return ret; 204 ret = iavf_configure_rss_key(adapter); 205 if (ret) 206 return ret; 207 208 return 0; 209 } 210 211 static int 212 iavf_dev_configure(struct rte_eth_dev *dev) 213 { 214 struct iavf_adapter *ad = 215 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 216 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 217 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 218 219 ad->rx_bulk_alloc_allowed = true; 220 /* Initialize to TRUE. If any of Rx queues doesn't meet the 221 * vector Rx/Tx preconditions, it will be reset. 222 */ 223 ad->rx_vec_allowed = true; 224 ad->tx_vec_allowed = true; 225 226 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 227 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 228 229 /* Vlan stripping setting */ 230 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) { 231 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 232 iavf_enable_vlan_strip(ad); 233 else 234 iavf_disable_vlan_strip(ad); 235 } 236 237 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 238 if (iavf_init_rss(ad) != 0) { 239 PMD_DRV_LOG(ERR, "configure rss failed"); 240 return -1; 241 } 242 } 243 return 0; 244 } 245 246 static int 247 iavf_init_rxq(struct rte_eth_dev *dev, struct iavf_rx_queue *rxq) 248 { 249 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 250 struct rte_eth_dev_data *dev_data = dev->data; 251 uint16_t buf_size, max_pkt_len, len; 252 253 buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; 254 255 /* Calculate the maximum packet length allowed */ 256 len = rxq->rx_buf_len * IAVF_MAX_CHAINED_RX_BUFFERS; 257 max_pkt_len = RTE_MIN(len, dev->data->dev_conf.rxmode.max_rx_pkt_len); 258 259 /* Check if the jumbo frame and maximum packet length are set 260 * correctly. 261 */ 262 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 263 if (max_pkt_len <= RTE_ETHER_MAX_LEN || 264 max_pkt_len > IAVF_FRAME_SIZE_MAX) { 265 PMD_DRV_LOG(ERR, "maximum packet length must be " 266 "larger than %u and smaller than %u, " 267 "as jumbo frame is enabled", 268 (uint32_t)RTE_ETHER_MAX_LEN, 269 (uint32_t)IAVF_FRAME_SIZE_MAX); 270 return -EINVAL; 271 } 272 } else { 273 if (max_pkt_len < RTE_ETHER_MIN_LEN || 274 max_pkt_len > RTE_ETHER_MAX_LEN) { 275 PMD_DRV_LOG(ERR, "maximum packet length must be " 276 "larger than %u and smaller than %u, " 277 "as jumbo frame is disabled", 278 (uint32_t)RTE_ETHER_MIN_LEN, 279 (uint32_t)RTE_ETHER_MAX_LEN); 280 return -EINVAL; 281 } 282 } 283 284 rxq->max_pkt_len = max_pkt_len; 285 if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) || 286 rxq->max_pkt_len > buf_size) { 287 dev_data->scattered_rx = 1; 288 } 289 IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 290 IAVF_WRITE_FLUSH(hw); 291 292 return 0; 293 } 294 295 static int 296 iavf_init_queues(struct rte_eth_dev *dev) 297 { 298 struct iavf_rx_queue **rxq = 299 (struct iavf_rx_queue **)dev->data->rx_queues; 300 int i, ret = IAVF_SUCCESS; 301 302 for (i = 0; i < dev->data->nb_rx_queues; i++) { 303 if (!rxq[i] || !rxq[i]->q_set) 304 continue; 305 ret = iavf_init_rxq(dev, rxq[i]); 306 if (ret != IAVF_SUCCESS) 307 break; 308 } 309 /* set rx/tx function to vector/scatter/single-segment 310 * according to parameters 311 */ 312 iavf_set_rx_function(dev); 313 iavf_set_tx_function(dev); 314 315 return ret; 316 } 317 318 static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev, 319 struct rte_intr_handle *intr_handle) 320 { 321 struct iavf_adapter *adapter = 322 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 323 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 324 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 325 uint16_t interval, i; 326 int vec; 327 328 if (rte_intr_cap_multiple(intr_handle) && 329 dev->data->dev_conf.intr_conf.rxq) { 330 if (rte_intr_efd_enable(intr_handle, dev->data->nb_rx_queues)) 331 return -1; 332 } 333 334 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 335 intr_handle->intr_vec = 336 rte_zmalloc("intr_vec", 337 dev->data->nb_rx_queues * sizeof(int), 0); 338 if (!intr_handle->intr_vec) { 339 PMD_DRV_LOG(ERR, "Failed to allocate %d rx intr_vec", 340 dev->data->nb_rx_queues); 341 return -1; 342 } 343 } 344 345 if (!dev->data->dev_conf.intr_conf.rxq || 346 !rte_intr_dp_is_en(intr_handle)) { 347 /* Rx interrupt disabled, Map interrupt only for writeback */ 348 vf->nb_msix = 1; 349 if (vf->vf_res->vf_cap_flags & 350 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 351 /* If WB_ON_ITR supports, enable it */ 352 vf->msix_base = IAVF_RX_VEC_START; 353 IAVF_WRITE_REG(hw, 354 IAVF_VFINT_DYN_CTLN1(vf->msix_base - 1), 355 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK | 356 IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK); 357 } else { 358 /* If no WB_ON_ITR offload flags, need to set 359 * interrupt for descriptor write back. 360 */ 361 vf->msix_base = IAVF_MISC_VEC_ID; 362 363 /* set ITR to max */ 364 interval = iavf_calc_itr_interval( 365 IAVF_QUEUE_ITR_INTERVAL_MAX); 366 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 367 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 368 (IAVF_ITR_INDEX_DEFAULT << 369 IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT) | 370 (interval << 371 IAVF_VFINT_DYN_CTL01_INTERVAL_SHIFT)); 372 } 373 IAVF_WRITE_FLUSH(hw); 374 /* map all queues to the same interrupt */ 375 for (i = 0; i < dev->data->nb_rx_queues; i++) 376 vf->rxq_map[vf->msix_base] |= 1 << i; 377 } else { 378 if (!rte_intr_allow_others(intr_handle)) { 379 vf->nb_msix = 1; 380 vf->msix_base = IAVF_MISC_VEC_ID; 381 for (i = 0; i < dev->data->nb_rx_queues; i++) { 382 vf->rxq_map[vf->msix_base] |= 1 << i; 383 intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID; 384 } 385 PMD_DRV_LOG(DEBUG, 386 "vector %u are mapping to all Rx queues", 387 vf->msix_base); 388 } else { 389 /* If Rx interrupt is reuquired, and we can use 390 * multi interrupts, then the vec is from 1 391 */ 392 vf->nb_msix = RTE_MIN(vf->vf_res->max_vectors, 393 intr_handle->nb_efd); 394 vf->msix_base = IAVF_RX_VEC_START; 395 vec = IAVF_RX_VEC_START; 396 for (i = 0; i < dev->data->nb_rx_queues; i++) { 397 vf->rxq_map[vec] |= 1 << i; 398 intr_handle->intr_vec[i] = vec++; 399 if (vec >= vf->nb_msix) 400 vec = IAVF_RX_VEC_START; 401 } 402 PMD_DRV_LOG(DEBUG, 403 "%u vectors are mapping to %u Rx queues", 404 vf->nb_msix, dev->data->nb_rx_queues); 405 } 406 } 407 408 if (iavf_config_irq_map(adapter)) { 409 PMD_DRV_LOG(ERR, "config interrupt mapping failed"); 410 return -1; 411 } 412 return 0; 413 } 414 415 static int 416 iavf_start_queues(struct rte_eth_dev *dev) 417 { 418 struct iavf_rx_queue *rxq; 419 struct iavf_tx_queue *txq; 420 int i; 421 422 for (i = 0; i < dev->data->nb_tx_queues; i++) { 423 txq = dev->data->tx_queues[i]; 424 if (txq->tx_deferred_start) 425 continue; 426 if (iavf_dev_tx_queue_start(dev, i) != 0) { 427 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 428 return -1; 429 } 430 } 431 432 for (i = 0; i < dev->data->nb_rx_queues; i++) { 433 rxq = dev->data->rx_queues[i]; 434 if (rxq->rx_deferred_start) 435 continue; 436 if (iavf_dev_rx_queue_start(dev, i) != 0) { 437 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 438 return -1; 439 } 440 } 441 442 return 0; 443 } 444 445 static int 446 iavf_dev_start(struct rte_eth_dev *dev) 447 { 448 struct iavf_adapter *adapter = 449 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 450 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 451 struct rte_intr_handle *intr_handle = dev->intr_handle; 452 453 PMD_INIT_FUNC_TRACE(); 454 455 adapter->stopped = 0; 456 457 vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; 458 vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 459 dev->data->nb_tx_queues); 460 461 if (iavf_init_queues(dev) != 0) { 462 PMD_DRV_LOG(ERR, "failed to do Queue init"); 463 return -1; 464 } 465 466 if (iavf_configure_queues(adapter) != 0) { 467 PMD_DRV_LOG(ERR, "configure queues failed"); 468 goto err_queue; 469 } 470 471 if (iavf_config_rx_queues_irqs(dev, intr_handle) != 0) { 472 PMD_DRV_LOG(ERR, "configure irq failed"); 473 goto err_queue; 474 } 475 /* re-enable intr again, because efd assign may change */ 476 if (dev->data->dev_conf.intr_conf.rxq != 0) { 477 rte_intr_disable(intr_handle); 478 rte_intr_enable(intr_handle); 479 } 480 481 /* Set all mac addrs */ 482 iavf_add_del_all_mac_addr(adapter, true); 483 484 /* Set all multicast addresses */ 485 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 486 true); 487 488 if (iavf_start_queues(dev) != 0) { 489 PMD_DRV_LOG(ERR, "enable queues failed"); 490 goto err_mac; 491 } 492 493 return 0; 494 495 err_mac: 496 iavf_add_del_all_mac_addr(adapter, false); 497 err_queue: 498 return -1; 499 } 500 501 static void 502 iavf_dev_stop(struct rte_eth_dev *dev) 503 { 504 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 505 struct iavf_adapter *adapter = 506 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 507 struct rte_intr_handle *intr_handle = dev->intr_handle; 508 509 PMD_INIT_FUNC_TRACE(); 510 511 if (adapter->stopped == 1) 512 return; 513 514 iavf_stop_queues(dev); 515 516 /* Disable the interrupt for Rx */ 517 rte_intr_efd_disable(intr_handle); 518 /* Rx interrupt vector mapping free */ 519 if (intr_handle->intr_vec) { 520 rte_free(intr_handle->intr_vec); 521 intr_handle->intr_vec = NULL; 522 } 523 524 /* remove all mac addrs */ 525 iavf_add_del_all_mac_addr(adapter, false); 526 527 /* remove all multicast addresses */ 528 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 529 false); 530 531 adapter->stopped = 1; 532 } 533 534 static int 535 iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 536 { 537 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 538 539 dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs; 540 dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs; 541 dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN; 542 dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX; 543 dev_info->hash_key_size = vf->vf_res->rss_key_size; 544 dev_info->reta_size = vf->vf_res->rss_lut_size; 545 dev_info->flow_type_rss_offloads = IAVF_RSS_OFFLOAD_ALL; 546 dev_info->max_mac_addrs = IAVF_NUM_MACADDR_MAX; 547 dev_info->rx_offload_capa = 548 DEV_RX_OFFLOAD_VLAN_STRIP | 549 DEV_RX_OFFLOAD_QINQ_STRIP | 550 DEV_RX_OFFLOAD_IPV4_CKSUM | 551 DEV_RX_OFFLOAD_UDP_CKSUM | 552 DEV_RX_OFFLOAD_TCP_CKSUM | 553 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 554 DEV_RX_OFFLOAD_SCATTER | 555 DEV_RX_OFFLOAD_JUMBO_FRAME | 556 DEV_RX_OFFLOAD_VLAN_FILTER | 557 DEV_RX_OFFLOAD_RSS_HASH; 558 dev_info->tx_offload_capa = 559 DEV_TX_OFFLOAD_VLAN_INSERT | 560 DEV_TX_OFFLOAD_QINQ_INSERT | 561 DEV_TX_OFFLOAD_IPV4_CKSUM | 562 DEV_TX_OFFLOAD_UDP_CKSUM | 563 DEV_TX_OFFLOAD_TCP_CKSUM | 564 DEV_TX_OFFLOAD_SCTP_CKSUM | 565 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 566 DEV_TX_OFFLOAD_TCP_TSO | 567 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 568 DEV_TX_OFFLOAD_GRE_TNL_TSO | 569 DEV_TX_OFFLOAD_IPIP_TNL_TSO | 570 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | 571 DEV_TX_OFFLOAD_MULTI_SEGS; 572 573 dev_info->default_rxconf = (struct rte_eth_rxconf) { 574 .rx_free_thresh = IAVF_DEFAULT_RX_FREE_THRESH, 575 .rx_drop_en = 0, 576 .offloads = 0, 577 }; 578 579 dev_info->default_txconf = (struct rte_eth_txconf) { 580 .tx_free_thresh = IAVF_DEFAULT_TX_FREE_THRESH, 581 .tx_rs_thresh = IAVF_DEFAULT_TX_RS_THRESH, 582 .offloads = 0, 583 }; 584 585 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 586 .nb_max = IAVF_MAX_RING_DESC, 587 .nb_min = IAVF_MIN_RING_DESC, 588 .nb_align = IAVF_ALIGN_RING_DESC, 589 }; 590 591 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 592 .nb_max = IAVF_MAX_RING_DESC, 593 .nb_min = IAVF_MIN_RING_DESC, 594 .nb_align = IAVF_ALIGN_RING_DESC, 595 }; 596 597 return 0; 598 } 599 600 static const uint32_t * 601 iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) 602 { 603 static const uint32_t ptypes[] = { 604 RTE_PTYPE_L2_ETHER, 605 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 606 RTE_PTYPE_L4_FRAG, 607 RTE_PTYPE_L4_ICMP, 608 RTE_PTYPE_L4_NONFRAG, 609 RTE_PTYPE_L4_SCTP, 610 RTE_PTYPE_L4_TCP, 611 RTE_PTYPE_L4_UDP, 612 RTE_PTYPE_UNKNOWN 613 }; 614 return ptypes; 615 } 616 617 int 618 iavf_dev_link_update(struct rte_eth_dev *dev, 619 __rte_unused int wait_to_complete) 620 { 621 struct rte_eth_link new_link; 622 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 623 624 memset(&new_link, 0, sizeof(new_link)); 625 626 /* Only read status info stored in VF, and the info is updated 627 * when receive LINK_CHANGE evnet from PF by Virtchnnl. 628 */ 629 switch (vf->link_speed) { 630 case 10: 631 new_link.link_speed = ETH_SPEED_NUM_10M; 632 break; 633 case 100: 634 new_link.link_speed = ETH_SPEED_NUM_100M; 635 break; 636 case 1000: 637 new_link.link_speed = ETH_SPEED_NUM_1G; 638 break; 639 case 10000: 640 new_link.link_speed = ETH_SPEED_NUM_10G; 641 break; 642 case 20000: 643 new_link.link_speed = ETH_SPEED_NUM_20G; 644 break; 645 case 25000: 646 new_link.link_speed = ETH_SPEED_NUM_25G; 647 break; 648 case 40000: 649 new_link.link_speed = ETH_SPEED_NUM_40G; 650 break; 651 case 50000: 652 new_link.link_speed = ETH_SPEED_NUM_50G; 653 break; 654 case 100000: 655 new_link.link_speed = ETH_SPEED_NUM_100G; 656 break; 657 default: 658 new_link.link_speed = ETH_SPEED_NUM_NONE; 659 break; 660 } 661 662 new_link.link_duplex = ETH_LINK_FULL_DUPLEX; 663 new_link.link_status = vf->link_up ? ETH_LINK_UP : 664 ETH_LINK_DOWN; 665 new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 666 ETH_LINK_SPEED_FIXED); 667 668 if (rte_atomic64_cmpset((uint64_t *)&dev->data->dev_link, 669 *(uint64_t *)&dev->data->dev_link, 670 *(uint64_t *)&new_link) == 0) 671 return -1; 672 673 return 0; 674 } 675 676 static int 677 iavf_dev_promiscuous_enable(struct rte_eth_dev *dev) 678 { 679 struct iavf_adapter *adapter = 680 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 681 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 682 int ret; 683 684 if (vf->promisc_unicast_enabled) 685 return 0; 686 687 ret = iavf_config_promisc(adapter, true, vf->promisc_multicast_enabled); 688 if (!ret) 689 vf->promisc_unicast_enabled = true; 690 else if (ret == IAVF_NOT_SUPPORTED) 691 ret = -ENOTSUP; 692 else 693 ret = -EAGAIN; 694 695 return ret; 696 } 697 698 static int 699 iavf_dev_promiscuous_disable(struct rte_eth_dev *dev) 700 { 701 struct iavf_adapter *adapter = 702 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 703 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 704 int ret; 705 706 if (!vf->promisc_unicast_enabled) 707 return 0; 708 709 ret = iavf_config_promisc(adapter, false, 710 vf->promisc_multicast_enabled); 711 if (!ret) 712 vf->promisc_unicast_enabled = false; 713 else if (ret == IAVF_NOT_SUPPORTED) 714 ret = -ENOTSUP; 715 else 716 ret = -EAGAIN; 717 718 return ret; 719 } 720 721 static int 722 iavf_dev_allmulticast_enable(struct rte_eth_dev *dev) 723 { 724 struct iavf_adapter *adapter = 725 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 726 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 727 int ret; 728 729 if (vf->promisc_multicast_enabled) 730 return 0; 731 732 ret = iavf_config_promisc(adapter, vf->promisc_unicast_enabled, true); 733 if (!ret) 734 vf->promisc_multicast_enabled = true; 735 else if (ret == IAVF_NOT_SUPPORTED) 736 ret = -ENOTSUP; 737 else 738 ret = -EAGAIN; 739 740 return ret; 741 } 742 743 static int 744 iavf_dev_allmulticast_disable(struct rte_eth_dev *dev) 745 { 746 struct iavf_adapter *adapter = 747 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 748 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 749 int ret; 750 751 if (!vf->promisc_multicast_enabled) 752 return 0; 753 754 ret = iavf_config_promisc(adapter, vf->promisc_unicast_enabled, false); 755 if (!ret) 756 vf->promisc_multicast_enabled = false; 757 else if (ret == IAVF_NOT_SUPPORTED) 758 ret = -ENOTSUP; 759 else 760 ret = -EAGAIN; 761 762 return ret; 763 } 764 765 static int 766 iavf_dev_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr, 767 __rte_unused uint32_t index, 768 __rte_unused uint32_t pool) 769 { 770 struct iavf_adapter *adapter = 771 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 772 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 773 int err; 774 775 if (rte_is_zero_ether_addr(addr)) { 776 PMD_DRV_LOG(ERR, "Invalid Ethernet Address"); 777 return -EINVAL; 778 } 779 780 err = iavf_add_del_eth_addr(adapter, addr, true); 781 if (err) { 782 PMD_DRV_LOG(ERR, "fail to add MAC address"); 783 return -EIO; 784 } 785 786 vf->mac_num++; 787 788 return 0; 789 } 790 791 static void 792 iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) 793 { 794 struct iavf_adapter *adapter = 795 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 796 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 797 struct rte_ether_addr *addr; 798 int err; 799 800 addr = &dev->data->mac_addrs[index]; 801 802 err = iavf_add_del_eth_addr(adapter, addr, false); 803 if (err) 804 PMD_DRV_LOG(ERR, "fail to delete MAC address"); 805 806 vf->mac_num--; 807 } 808 809 static int 810 iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 811 { 812 struct iavf_adapter *adapter = 813 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 814 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 815 int err; 816 817 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 818 return -ENOTSUP; 819 820 err = iavf_add_del_vlan(adapter, vlan_id, on); 821 if (err) 822 return -EIO; 823 return 0; 824 } 825 826 static int 827 iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 828 { 829 struct iavf_adapter *adapter = 830 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 831 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 832 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 833 int err; 834 835 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 836 return -ENOTSUP; 837 838 /* Vlan stripping setting */ 839 if (mask & ETH_VLAN_STRIP_MASK) { 840 /* Enable or disable VLAN stripping */ 841 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 842 err = iavf_enable_vlan_strip(adapter); 843 else 844 err = iavf_disable_vlan_strip(adapter); 845 846 if (err) 847 return -EIO; 848 } 849 return 0; 850 } 851 852 static int 853 iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 854 struct rte_eth_rss_reta_entry64 *reta_conf, 855 uint16_t reta_size) 856 { 857 struct iavf_adapter *adapter = 858 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 859 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 860 uint8_t *lut; 861 uint16_t i, idx, shift; 862 int ret; 863 864 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 865 return -ENOTSUP; 866 867 if (reta_size != vf->vf_res->rss_lut_size) { 868 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 869 "(%d) doesn't match the number of hardware can " 870 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 871 return -EINVAL; 872 } 873 874 lut = rte_zmalloc("rss_lut", reta_size, 0); 875 if (!lut) { 876 PMD_DRV_LOG(ERR, "No memory can be allocated"); 877 return -ENOMEM; 878 } 879 /* store the old lut table temporarily */ 880 rte_memcpy(lut, vf->rss_lut, reta_size); 881 882 for (i = 0; i < reta_size; i++) { 883 idx = i / RTE_RETA_GROUP_SIZE; 884 shift = i % RTE_RETA_GROUP_SIZE; 885 if (reta_conf[idx].mask & (1ULL << shift)) 886 lut[i] = reta_conf[idx].reta[shift]; 887 } 888 889 rte_memcpy(vf->rss_lut, lut, reta_size); 890 /* send virtchnnl ops to configure rss*/ 891 ret = iavf_configure_rss_lut(adapter); 892 if (ret) /* revert back */ 893 rte_memcpy(vf->rss_lut, lut, reta_size); 894 rte_free(lut); 895 896 return ret; 897 } 898 899 static int 900 iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 901 struct rte_eth_rss_reta_entry64 *reta_conf, 902 uint16_t reta_size) 903 { 904 struct iavf_adapter *adapter = 905 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 906 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 907 uint16_t i, idx, shift; 908 909 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 910 return -ENOTSUP; 911 912 if (reta_size != vf->vf_res->rss_lut_size) { 913 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 914 "(%d) doesn't match the number of hardware can " 915 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 916 return -EINVAL; 917 } 918 919 for (i = 0; i < reta_size; i++) { 920 idx = i / RTE_RETA_GROUP_SIZE; 921 shift = i % RTE_RETA_GROUP_SIZE; 922 if (reta_conf[idx].mask & (1ULL << shift)) 923 reta_conf[idx].reta[shift] = vf->rss_lut[i]; 924 } 925 926 return 0; 927 } 928 929 static int 930 iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 931 struct rte_eth_rss_conf *rss_conf) 932 { 933 struct iavf_adapter *adapter = 934 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 935 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 936 937 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 938 return -ENOTSUP; 939 940 /* HENA setting, it is enabled by default, no change */ 941 if (!rss_conf->rss_key || rss_conf->rss_key_len == 0) { 942 PMD_DRV_LOG(DEBUG, "No key to be configured"); 943 return 0; 944 } else if (rss_conf->rss_key_len != vf->vf_res->rss_key_size) { 945 PMD_DRV_LOG(ERR, "The size of hash key configured " 946 "(%d) doesn't match the size of hardware can " 947 "support (%d)", rss_conf->rss_key_len, 948 vf->vf_res->rss_key_size); 949 return -EINVAL; 950 } 951 952 rte_memcpy(vf->rss_key, rss_conf->rss_key, rss_conf->rss_key_len); 953 954 return iavf_configure_rss_key(adapter); 955 } 956 957 static int 958 iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 959 struct rte_eth_rss_conf *rss_conf) 960 { 961 struct iavf_adapter *adapter = 962 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 963 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 964 965 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 966 return -ENOTSUP; 967 968 /* Just set it to default value now. */ 969 rss_conf->rss_hf = IAVF_RSS_OFFLOAD_ALL; 970 971 if (!rss_conf->rss_key) 972 return 0; 973 974 rss_conf->rss_key_len = vf->vf_res->rss_key_size; 975 rte_memcpy(rss_conf->rss_key, vf->rss_key, rss_conf->rss_key_len); 976 977 return 0; 978 } 979 980 static int 981 iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 982 { 983 uint32_t frame_size = mtu + IAVF_ETH_OVERHEAD; 984 int ret = 0; 985 986 if (mtu < RTE_ETHER_MIN_MTU || frame_size > IAVF_FRAME_SIZE_MAX) 987 return -EINVAL; 988 989 /* mtu setting is forbidden if port is start */ 990 if (dev->data->dev_started) { 991 PMD_DRV_LOG(ERR, "port must be stopped before configuration"); 992 return -EBUSY; 993 } 994 995 if (frame_size > RTE_ETHER_MAX_LEN) 996 dev->data->dev_conf.rxmode.offloads |= 997 DEV_RX_OFFLOAD_JUMBO_FRAME; 998 else 999 dev->data->dev_conf.rxmode.offloads &= 1000 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 1001 1002 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 1003 1004 return ret; 1005 } 1006 1007 static int 1008 iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 1009 struct rte_ether_addr *mac_addr) 1010 { 1011 struct iavf_adapter *adapter = 1012 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1013 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1014 struct rte_ether_addr *perm_addr, *old_addr; 1015 int ret; 1016 1017 old_addr = (struct rte_ether_addr *)hw->mac.addr; 1018 perm_addr = (struct rte_ether_addr *)hw->mac.perm_addr; 1019 1020 /* If the MAC address is configured by host, skip the setting */ 1021 if (rte_is_valid_assigned_ether_addr(perm_addr)) 1022 return -EPERM; 1023 1024 ret = iavf_add_del_eth_addr(adapter, old_addr, false); 1025 if (ret) 1026 PMD_DRV_LOG(ERR, "Fail to delete old MAC:" 1027 " %02X:%02X:%02X:%02X:%02X:%02X", 1028 old_addr->addr_bytes[0], 1029 old_addr->addr_bytes[1], 1030 old_addr->addr_bytes[2], 1031 old_addr->addr_bytes[3], 1032 old_addr->addr_bytes[4], 1033 old_addr->addr_bytes[5]); 1034 1035 ret = iavf_add_del_eth_addr(adapter, mac_addr, true); 1036 if (ret) 1037 PMD_DRV_LOG(ERR, "Fail to add new MAC:" 1038 " %02X:%02X:%02X:%02X:%02X:%02X", 1039 mac_addr->addr_bytes[0], 1040 mac_addr->addr_bytes[1], 1041 mac_addr->addr_bytes[2], 1042 mac_addr->addr_bytes[3], 1043 mac_addr->addr_bytes[4], 1044 mac_addr->addr_bytes[5]); 1045 1046 if (ret) 1047 return -EIO; 1048 1049 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)hw->mac.addr); 1050 return 0; 1051 } 1052 1053 static void 1054 iavf_stat_update_48(uint64_t *offset, uint64_t *stat) 1055 { 1056 if (*stat >= *offset) 1057 *stat = *stat - *offset; 1058 else 1059 *stat = (uint64_t)((*stat + 1060 ((uint64_t)1 << IAVF_48_BIT_WIDTH)) - *offset); 1061 1062 *stat &= IAVF_48_BIT_MASK; 1063 } 1064 1065 static void 1066 iavf_stat_update_32(uint64_t *offset, uint64_t *stat) 1067 { 1068 if (*stat >= *offset) 1069 *stat = (uint64_t)(*stat - *offset); 1070 else 1071 *stat = (uint64_t)((*stat + 1072 ((uint64_t)1 << IAVF_32_BIT_WIDTH)) - *offset); 1073 } 1074 1075 static void 1076 iavf_update_stats(struct iavf_vsi *vsi, struct virtchnl_eth_stats *nes) 1077 { 1078 struct virtchnl_eth_stats *oes = &vsi->eth_stats_offset; 1079 1080 iavf_stat_update_48(&oes->rx_bytes, &nes->rx_bytes); 1081 iavf_stat_update_48(&oes->rx_unicast, &nes->rx_unicast); 1082 iavf_stat_update_48(&oes->rx_multicast, &nes->rx_multicast); 1083 iavf_stat_update_48(&oes->rx_broadcast, &nes->rx_broadcast); 1084 iavf_stat_update_32(&oes->rx_discards, &nes->rx_discards); 1085 iavf_stat_update_48(&oes->tx_bytes, &nes->tx_bytes); 1086 iavf_stat_update_48(&oes->tx_unicast, &nes->tx_unicast); 1087 iavf_stat_update_48(&oes->tx_multicast, &nes->tx_multicast); 1088 iavf_stat_update_48(&oes->tx_broadcast, &nes->tx_broadcast); 1089 iavf_stat_update_32(&oes->tx_errors, &nes->tx_errors); 1090 iavf_stat_update_32(&oes->tx_discards, &nes->tx_discards); 1091 } 1092 1093 static int 1094 iavf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1095 { 1096 struct iavf_adapter *adapter = 1097 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1098 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1099 struct iavf_vsi *vsi = &vf->vsi; 1100 struct virtchnl_eth_stats *pstats = NULL; 1101 int ret; 1102 1103 ret = iavf_query_stats(adapter, &pstats); 1104 if (ret == 0) { 1105 iavf_update_stats(vsi, pstats); 1106 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + 1107 pstats->rx_broadcast - pstats->rx_discards; 1108 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + 1109 pstats->tx_unicast; 1110 stats->imissed = pstats->rx_discards; 1111 stats->oerrors = pstats->tx_errors + pstats->tx_discards; 1112 stats->ibytes = pstats->rx_bytes; 1113 stats->ibytes -= stats->ipackets * RTE_ETHER_CRC_LEN; 1114 stats->obytes = pstats->tx_bytes; 1115 } else { 1116 PMD_DRV_LOG(ERR, "Get statistics failed"); 1117 } 1118 return ret; 1119 } 1120 1121 static int 1122 iavf_dev_stats_reset(struct rte_eth_dev *dev) 1123 { 1124 int ret; 1125 struct iavf_adapter *adapter = 1126 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1127 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1128 struct iavf_vsi *vsi = &vf->vsi; 1129 struct virtchnl_eth_stats *pstats = NULL; 1130 1131 /* read stat values to clear hardware registers */ 1132 ret = iavf_query_stats(adapter, &pstats); 1133 if (ret != 0) 1134 return ret; 1135 1136 /* set stats offset base on current values */ 1137 vsi->eth_stats_offset = *pstats; 1138 1139 return 0; 1140 } 1141 1142 static int 1143 iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1144 { 1145 struct iavf_adapter *adapter = 1146 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1147 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1148 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1149 uint16_t msix_intr; 1150 1151 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1152 if (msix_intr == IAVF_MISC_VEC_ID) { 1153 PMD_DRV_LOG(INFO, "MISC is also enabled for control"); 1154 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1155 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 1156 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1157 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1158 } else { 1159 IAVF_WRITE_REG(hw, 1160 IAVF_VFINT_DYN_CTLN1 1161 (msix_intr - IAVF_RX_VEC_START), 1162 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 1163 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1164 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 1165 } 1166 1167 IAVF_WRITE_FLUSH(hw); 1168 1169 rte_intr_ack(&pci_dev->intr_handle); 1170 1171 return 0; 1172 } 1173 1174 static int 1175 iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1176 { 1177 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1178 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1179 uint16_t msix_intr; 1180 1181 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1182 if (msix_intr == IAVF_MISC_VEC_ID) { 1183 PMD_DRV_LOG(ERR, "MISC is used for control, cannot disable it"); 1184 return -EIO; 1185 } 1186 1187 IAVF_WRITE_REG(hw, 1188 IAVF_VFINT_DYN_CTLN1(msix_intr - IAVF_RX_VEC_START), 1189 0); 1190 1191 IAVF_WRITE_FLUSH(hw); 1192 return 0; 1193 } 1194 1195 static int 1196 iavf_check_vf_reset_done(struct iavf_hw *hw) 1197 { 1198 int i, reset; 1199 1200 for (i = 0; i < IAVF_RESET_WAIT_CNT; i++) { 1201 reset = IAVF_READ_REG(hw, IAVF_VFGEN_RSTAT) & 1202 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 1203 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 1204 if (reset == VIRTCHNL_VFR_VFACTIVE || 1205 reset == VIRTCHNL_VFR_COMPLETED) 1206 break; 1207 rte_delay_ms(20); 1208 } 1209 1210 if (i >= IAVF_RESET_WAIT_CNT) 1211 return -1; 1212 1213 return 0; 1214 } 1215 1216 static int 1217 iavf_init_vf(struct rte_eth_dev *dev) 1218 { 1219 int err, bufsz; 1220 struct iavf_adapter *adapter = 1221 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1222 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1223 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1224 1225 err = iavf_set_mac_type(hw); 1226 if (err) { 1227 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err); 1228 goto err; 1229 } 1230 1231 err = iavf_check_vf_reset_done(hw); 1232 if (err) { 1233 PMD_INIT_LOG(ERR, "VF is still resetting"); 1234 goto err; 1235 } 1236 1237 iavf_init_adminq_parameter(hw); 1238 err = iavf_init_adminq(hw); 1239 if (err) { 1240 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err); 1241 goto err; 1242 } 1243 1244 vf->aq_resp = rte_zmalloc("vf_aq_resp", IAVF_AQ_BUF_SZ, 0); 1245 if (!vf->aq_resp) { 1246 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); 1247 goto err_aq; 1248 } 1249 if (iavf_check_api_version(adapter) != 0) { 1250 PMD_INIT_LOG(ERR, "check_api version failed"); 1251 goto err_api; 1252 } 1253 1254 bufsz = sizeof(struct virtchnl_vf_resource) + 1255 (IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); 1256 vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); 1257 if (!vf->vf_res) { 1258 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 1259 goto err_api; 1260 } 1261 if (iavf_get_vf_resource(adapter) != 0) { 1262 PMD_INIT_LOG(ERR, "iavf_get_vf_config failed"); 1263 goto err_alloc; 1264 } 1265 /* Allocate memort for RSS info */ 1266 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 1267 vf->rss_key = rte_zmalloc("rss_key", 1268 vf->vf_res->rss_key_size, 0); 1269 if (!vf->rss_key) { 1270 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 1271 goto err_rss; 1272 } 1273 vf->rss_lut = rte_zmalloc("rss_lut", 1274 vf->vf_res->rss_lut_size, 0); 1275 if (!vf->rss_lut) { 1276 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 1277 goto err_rss; 1278 } 1279 } 1280 1281 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 1282 if (iavf_get_supported_rxdid(adapter) != 0) { 1283 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 1284 goto err_rss; 1285 } 1286 } 1287 1288 vf->vf_reset = false; 1289 1290 return 0; 1291 err_rss: 1292 rte_free(vf->rss_key); 1293 rte_free(vf->rss_lut); 1294 err_alloc: 1295 rte_free(vf->vf_res); 1296 vf->vsi_res = NULL; 1297 err_api: 1298 rte_free(vf->aq_resp); 1299 err_aq: 1300 iavf_shutdown_adminq(hw); 1301 err: 1302 return -1; 1303 } 1304 1305 /* Enable default admin queue interrupt setting */ 1306 static inline void 1307 iavf_enable_irq0(struct iavf_hw *hw) 1308 { 1309 /* Enable admin queue interrupt trigger */ 1310 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 1311 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 1312 1313 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1314 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 1315 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1316 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1317 1318 IAVF_WRITE_FLUSH(hw); 1319 } 1320 1321 static inline void 1322 iavf_disable_irq0(struct iavf_hw *hw) 1323 { 1324 /* Disable all interrupt types */ 1325 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 0); 1326 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1327 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1328 IAVF_WRITE_FLUSH(hw); 1329 } 1330 1331 static void 1332 iavf_dev_interrupt_handler(void *param) 1333 { 1334 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1335 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1336 1337 iavf_disable_irq0(hw); 1338 1339 iavf_handle_virtchnl_msg(dev); 1340 1341 iavf_enable_irq0(hw); 1342 } 1343 1344 static int 1345 iavf_dev_filter_ctrl(struct rte_eth_dev *dev, 1346 enum rte_filter_type filter_type, 1347 enum rte_filter_op filter_op, 1348 void *arg) 1349 { 1350 int ret = 0; 1351 1352 if (!dev) 1353 return -EINVAL; 1354 1355 switch (filter_type) { 1356 case RTE_ETH_FILTER_GENERIC: 1357 if (filter_op != RTE_ETH_FILTER_GET) 1358 return -EINVAL; 1359 *(const void **)arg = &iavf_flow_ops; 1360 break; 1361 default: 1362 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 1363 filter_type); 1364 ret = -EINVAL; 1365 break; 1366 } 1367 1368 return ret; 1369 } 1370 1371 1372 static int 1373 iavf_dev_init(struct rte_eth_dev *eth_dev) 1374 { 1375 struct iavf_adapter *adapter = 1376 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 1377 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1378 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1379 int ret = 0; 1380 1381 PMD_INIT_FUNC_TRACE(); 1382 1383 /* assign ops func pointer */ 1384 eth_dev->dev_ops = &iavf_eth_dev_ops; 1385 eth_dev->rx_queue_count = iavf_dev_rxq_count; 1386 eth_dev->rx_descriptor_status = iavf_dev_rx_desc_status; 1387 eth_dev->tx_descriptor_status = iavf_dev_tx_desc_status; 1388 eth_dev->rx_pkt_burst = &iavf_recv_pkts; 1389 eth_dev->tx_pkt_burst = &iavf_xmit_pkts; 1390 eth_dev->tx_pkt_prepare = &iavf_prep_pkts; 1391 1392 /* For secondary processes, we don't initialise any further as primary 1393 * has already done this work. Only check if we need a different RX 1394 * and TX function. 1395 */ 1396 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 1397 iavf_set_rx_function(eth_dev); 1398 iavf_set_tx_function(eth_dev); 1399 return 0; 1400 } 1401 rte_eth_copy_pci_info(eth_dev, pci_dev); 1402 1403 hw->vendor_id = pci_dev->id.vendor_id; 1404 hw->device_id = pci_dev->id.device_id; 1405 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 1406 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 1407 hw->bus.bus_id = pci_dev->addr.bus; 1408 hw->bus.device = pci_dev->addr.devid; 1409 hw->bus.func = pci_dev->addr.function; 1410 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1411 hw->back = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 1412 adapter->eth_dev = eth_dev; 1413 adapter->stopped = 1; 1414 1415 /* Pass the information to the rte_eth_dev_close() that it should also 1416 * release the private port resources. 1417 */ 1418 eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE; 1419 1420 if (iavf_init_vf(eth_dev) != 0) { 1421 PMD_INIT_LOG(ERR, "Init vf failed"); 1422 return -1; 1423 } 1424 1425 /* set default ptype table */ 1426 adapter->ptype_tbl = iavf_get_default_ptype_table(); 1427 1428 /* copy mac addr */ 1429 eth_dev->data->mac_addrs = rte_zmalloc( 1430 "iavf_mac", RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX, 0); 1431 if (!eth_dev->data->mac_addrs) { 1432 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to" 1433 " store MAC addresses", 1434 RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX); 1435 return -ENOMEM; 1436 } 1437 /* If the MAC address is not configured by host, 1438 * generate a random one. 1439 */ 1440 if (!rte_is_valid_assigned_ether_addr( 1441 (struct rte_ether_addr *)hw->mac.addr)) 1442 rte_eth_random_addr(hw->mac.addr); 1443 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 1444 ð_dev->data->mac_addrs[0]); 1445 1446 /* register callback func to eal lib */ 1447 rte_intr_callback_register(&pci_dev->intr_handle, 1448 iavf_dev_interrupt_handler, 1449 (void *)eth_dev); 1450 1451 /* enable uio intr after callback register */ 1452 rte_intr_enable(&pci_dev->intr_handle); 1453 1454 /* configure and enable device interrupt */ 1455 iavf_enable_irq0(hw); 1456 1457 ret = iavf_flow_init(adapter); 1458 if (ret) { 1459 PMD_INIT_LOG(ERR, "Failed to initialize flow"); 1460 return ret; 1461 } 1462 1463 return 0; 1464 } 1465 1466 static void 1467 iavf_dev_close(struct rte_eth_dev *dev) 1468 { 1469 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1470 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1471 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1472 struct iavf_adapter *adapter = 1473 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1474 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1475 1476 iavf_dev_stop(dev); 1477 iavf_flow_flush(dev, NULL); 1478 iavf_flow_uninit(adapter); 1479 iavf_shutdown_adminq(hw); 1480 /* disable uio intr before callback unregister */ 1481 rte_intr_disable(intr_handle); 1482 1483 /* unregister callback func from eal lib */ 1484 rte_intr_callback_unregister(intr_handle, 1485 iavf_dev_interrupt_handler, dev); 1486 iavf_disable_irq0(hw); 1487 1488 dev->dev_ops = NULL; 1489 dev->rx_pkt_burst = NULL; 1490 dev->tx_pkt_burst = NULL; 1491 1492 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 1493 if (vf->rss_lut) { 1494 rte_free(vf->rss_lut); 1495 vf->rss_lut = NULL; 1496 } 1497 if (vf->rss_key) { 1498 rte_free(vf->rss_key); 1499 vf->rss_key = NULL; 1500 } 1501 } 1502 1503 rte_free(vf->vf_res); 1504 vf->vsi_res = NULL; 1505 vf->vf_res = NULL; 1506 1507 rte_free(vf->aq_resp); 1508 vf->aq_resp = NULL; 1509 } 1510 1511 static int 1512 iavf_dev_uninit(struct rte_eth_dev *dev) 1513 { 1514 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1515 return -EPERM; 1516 1517 iavf_dev_close(dev); 1518 1519 return 0; 1520 } 1521 1522 /* 1523 * Reset VF device only to re-initialize resources in PMD layer 1524 */ 1525 static int 1526 iavf_dev_reset(struct rte_eth_dev *dev) 1527 { 1528 int ret; 1529 1530 ret = iavf_dev_uninit(dev); 1531 if (ret) 1532 return ret; 1533 1534 return iavf_dev_init(dev); 1535 } 1536 1537 static int 1538 iavf_dcf_cap_check_handler(__rte_unused const char *key, 1539 const char *value, __rte_unused void *opaque) 1540 { 1541 if (strcmp(value, "dcf")) 1542 return -1; 1543 1544 return 0; 1545 } 1546 1547 static int 1548 iavf_dcf_cap_selected(struct rte_devargs *devargs) 1549 { 1550 struct rte_kvargs *kvlist; 1551 const char *key = "cap"; 1552 int ret = 0; 1553 1554 if (devargs == NULL) 1555 return 0; 1556 1557 kvlist = rte_kvargs_parse(devargs->args, NULL); 1558 if (kvlist == NULL) 1559 return 0; 1560 1561 if (!rte_kvargs_count(kvlist, key)) 1562 goto exit; 1563 1564 /* dcf capability selected when there's a key-value pair: cap=dcf */ 1565 if (rte_kvargs_process(kvlist, key, 1566 iavf_dcf_cap_check_handler, NULL) < 0) 1567 goto exit; 1568 1569 ret = 1; 1570 1571 exit: 1572 rte_kvargs_free(kvlist); 1573 return ret; 1574 } 1575 1576 static int eth_iavf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1577 struct rte_pci_device *pci_dev) 1578 { 1579 if (iavf_dcf_cap_selected(pci_dev->device.devargs)) 1580 return 1; 1581 1582 return rte_eth_dev_pci_generic_probe(pci_dev, 1583 sizeof(struct iavf_adapter), iavf_dev_init); 1584 } 1585 1586 static int eth_iavf_pci_remove(struct rte_pci_device *pci_dev) 1587 { 1588 return rte_eth_dev_pci_generic_remove(pci_dev, iavf_dev_uninit); 1589 } 1590 1591 /* Adaptive virtual function driver struct */ 1592 static struct rte_pci_driver rte_iavf_pmd = { 1593 .id_table = pci_id_iavf_map, 1594 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 1595 .probe = eth_iavf_pci_probe, 1596 .remove = eth_iavf_pci_remove, 1597 }; 1598 1599 RTE_PMD_REGISTER_PCI(net_iavf, rte_iavf_pmd); 1600 RTE_PMD_REGISTER_PCI_TABLE(net_iavf, pci_id_iavf_map); 1601 RTE_PMD_REGISTER_KMOD_DEP(net_iavf, "* igb_uio | vfio-pci"); 1602 RTE_PMD_REGISTER_PARAM_STRING(net_iavf, "cap=dcf"); 1603 RTE_LOG_REGISTER(iavf_logtype_init, pmd.net.iavf.init, NOTICE); 1604 RTE_LOG_REGISTER(iavf_logtype_driver, pmd.net.iavf.driver, NOTICE); 1605 #ifdef RTE_LIBRTE_IAVF_DEBUG_RX 1606 RTE_LOG_REGISTER(iavf_logtype_rx, pmd.net.iavf.rx, DEBUG); 1607 #endif 1608 #ifdef RTE_LIBRTE_IAVF_DEBUG_TX 1609 RTE_LOG_REGISTER(iavf_logtype_tx, pmd.net.iavf.tx, DEBUG); 1610 #endif 1611 #ifdef RTE_LIBRTE_IAVF_DEBUG_TX_FREE 1612 RTE_LOG_REGISTER(iavf_logtype_tx_free, pmd.net.iavf.tx_free, DEBUG); 1613 #endif 1614