1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_interrupts.h> 17 #include <rte_debug.h> 18 #include <rte_pci.h> 19 #include <rte_atomic.h> 20 #include <rte_eal.h> 21 #include <rte_ether.h> 22 #include <rte_ethdev_driver.h> 23 #include <rte_ethdev_pci.h> 24 #include <rte_malloc.h> 25 #include <rte_memzone.h> 26 #include <rte_dev.h> 27 28 #include "iavf.h" 29 #include "iavf_rxtx.h" 30 #include "iavf_generic_flow.h" 31 #include "rte_pmd_iavf.h" 32 33 /* devargs */ 34 #define IAVF_PROTO_XTR_ARG "proto_xtr" 35 36 static const char * const iavf_valid_args[] = { 37 IAVF_PROTO_XTR_ARG, 38 NULL 39 }; 40 41 static const struct rte_mbuf_dynfield iavf_proto_xtr_metadata_param = { 42 .name = "intel_pmd_dynfield_proto_xtr_metadata", 43 .size = sizeof(uint32_t), 44 .align = __alignof__(uint32_t), 45 .flags = 0, 46 }; 47 48 struct iavf_proto_xtr_ol { 49 const struct rte_mbuf_dynflag param; 50 uint64_t *ol_flag; 51 bool required; 52 }; 53 54 static struct iavf_proto_xtr_ol iavf_proto_xtr_params[] = { 55 [IAVF_PROTO_XTR_VLAN] = { 56 .param = { .name = "intel_pmd_dynflag_proto_xtr_vlan" }, 57 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_vlan_mask }, 58 [IAVF_PROTO_XTR_IPV4] = { 59 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv4" }, 60 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask }, 61 [IAVF_PROTO_XTR_IPV6] = { 62 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6" }, 63 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask }, 64 [IAVF_PROTO_XTR_IPV6_FLOW] = { 65 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6_flow" }, 66 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask }, 67 [IAVF_PROTO_XTR_TCP] = { 68 .param = { .name = "intel_pmd_dynflag_proto_xtr_tcp" }, 69 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_tcp_mask }, 70 [IAVF_PROTO_XTR_IP_OFFSET] = { 71 .param = { .name = "intel_pmd_dynflag_proto_xtr_ip_offset" }, 72 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask }, 73 }; 74 75 static int iavf_dev_configure(struct rte_eth_dev *dev); 76 static int iavf_dev_start(struct rte_eth_dev *dev); 77 static int iavf_dev_stop(struct rte_eth_dev *dev); 78 static int iavf_dev_close(struct rte_eth_dev *dev); 79 static int iavf_dev_reset(struct rte_eth_dev *dev); 80 static int iavf_dev_info_get(struct rte_eth_dev *dev, 81 struct rte_eth_dev_info *dev_info); 82 static const uint32_t *iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev); 83 static int iavf_dev_stats_get(struct rte_eth_dev *dev, 84 struct rte_eth_stats *stats); 85 static int iavf_dev_stats_reset(struct rte_eth_dev *dev); 86 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 87 struct rte_eth_xstat *xstats, unsigned int n); 88 static int iavf_dev_xstats_get_names(struct rte_eth_dev *dev, 89 struct rte_eth_xstat_name *xstats_names, 90 unsigned int limit); 91 static int iavf_dev_promiscuous_enable(struct rte_eth_dev *dev); 92 static int iavf_dev_promiscuous_disable(struct rte_eth_dev *dev); 93 static int iavf_dev_allmulticast_enable(struct rte_eth_dev *dev); 94 static int iavf_dev_allmulticast_disable(struct rte_eth_dev *dev); 95 static int iavf_dev_add_mac_addr(struct rte_eth_dev *dev, 96 struct rte_ether_addr *addr, 97 uint32_t index, 98 uint32_t pool); 99 static void iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); 100 static int iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, 101 uint16_t vlan_id, int on); 102 static int iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 103 static int iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 104 struct rte_eth_rss_reta_entry64 *reta_conf, 105 uint16_t reta_size); 106 static int iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 107 struct rte_eth_rss_reta_entry64 *reta_conf, 108 uint16_t reta_size); 109 static int iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 110 struct rte_eth_rss_conf *rss_conf); 111 static int iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 112 struct rte_eth_rss_conf *rss_conf); 113 static int iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 114 static int iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 115 struct rte_ether_addr *mac_addr); 116 static int iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 117 uint16_t queue_id); 118 static int iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 119 uint16_t queue_id); 120 static int iavf_dev_filter_ctrl(struct rte_eth_dev *dev, 121 enum rte_filter_type filter_type, 122 enum rte_filter_op filter_op, 123 void *arg); 124 static int iavf_set_mc_addr_list(struct rte_eth_dev *dev, 125 struct rte_ether_addr *mc_addrs, 126 uint32_t mc_addrs_num); 127 128 static const struct rte_pci_id pci_id_iavf_map[] = { 129 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_ADAPTIVE_VF) }, 130 { .vendor_id = 0, /* sentinel */ }, 131 }; 132 133 struct rte_iavf_xstats_name_off { 134 char name[RTE_ETH_XSTATS_NAME_SIZE]; 135 unsigned int offset; 136 }; 137 138 static const struct rte_iavf_xstats_name_off rte_iavf_stats_strings[] = { 139 {"rx_bytes", offsetof(struct iavf_eth_stats, rx_bytes)}, 140 {"rx_unicast_packets", offsetof(struct iavf_eth_stats, rx_unicast)}, 141 {"rx_multicast_packets", offsetof(struct iavf_eth_stats, rx_multicast)}, 142 {"rx_broadcast_packets", offsetof(struct iavf_eth_stats, rx_broadcast)}, 143 {"rx_dropped_packets", offsetof(struct iavf_eth_stats, rx_discards)}, 144 {"rx_unknown_protocol_packets", offsetof(struct iavf_eth_stats, 145 rx_unknown_protocol)}, 146 {"tx_bytes", offsetof(struct iavf_eth_stats, tx_bytes)}, 147 {"tx_unicast_packets", offsetof(struct iavf_eth_stats, tx_unicast)}, 148 {"tx_multicast_packets", offsetof(struct iavf_eth_stats, tx_multicast)}, 149 {"tx_broadcast_packets", offsetof(struct iavf_eth_stats, tx_broadcast)}, 150 {"tx_dropped_packets", offsetof(struct iavf_eth_stats, tx_discards)}, 151 {"tx_error_packets", offsetof(struct iavf_eth_stats, tx_errors)}, 152 }; 153 154 #define IAVF_NB_XSTATS (sizeof(rte_iavf_stats_strings) / \ 155 sizeof(rte_iavf_stats_strings[0])) 156 157 static const struct eth_dev_ops iavf_eth_dev_ops = { 158 .dev_configure = iavf_dev_configure, 159 .dev_start = iavf_dev_start, 160 .dev_stop = iavf_dev_stop, 161 .dev_close = iavf_dev_close, 162 .dev_reset = iavf_dev_reset, 163 .dev_infos_get = iavf_dev_info_get, 164 .dev_supported_ptypes_get = iavf_dev_supported_ptypes_get, 165 .link_update = iavf_dev_link_update, 166 .stats_get = iavf_dev_stats_get, 167 .stats_reset = iavf_dev_stats_reset, 168 .xstats_get = iavf_dev_xstats_get, 169 .xstats_get_names = iavf_dev_xstats_get_names, 170 .xstats_reset = iavf_dev_stats_reset, 171 .promiscuous_enable = iavf_dev_promiscuous_enable, 172 .promiscuous_disable = iavf_dev_promiscuous_disable, 173 .allmulticast_enable = iavf_dev_allmulticast_enable, 174 .allmulticast_disable = iavf_dev_allmulticast_disable, 175 .mac_addr_add = iavf_dev_add_mac_addr, 176 .mac_addr_remove = iavf_dev_del_mac_addr, 177 .set_mc_addr_list = iavf_set_mc_addr_list, 178 .vlan_filter_set = iavf_dev_vlan_filter_set, 179 .vlan_offload_set = iavf_dev_vlan_offload_set, 180 .rx_queue_start = iavf_dev_rx_queue_start, 181 .rx_queue_stop = iavf_dev_rx_queue_stop, 182 .tx_queue_start = iavf_dev_tx_queue_start, 183 .tx_queue_stop = iavf_dev_tx_queue_stop, 184 .rx_queue_setup = iavf_dev_rx_queue_setup, 185 .rx_queue_release = iavf_dev_rx_queue_release, 186 .tx_queue_setup = iavf_dev_tx_queue_setup, 187 .tx_queue_release = iavf_dev_tx_queue_release, 188 .mac_addr_set = iavf_dev_set_default_mac_addr, 189 .reta_update = iavf_dev_rss_reta_update, 190 .reta_query = iavf_dev_rss_reta_query, 191 .rss_hash_update = iavf_dev_rss_hash_update, 192 .rss_hash_conf_get = iavf_dev_rss_hash_conf_get, 193 .rxq_info_get = iavf_dev_rxq_info_get, 194 .txq_info_get = iavf_dev_txq_info_get, 195 .mtu_set = iavf_dev_mtu_set, 196 .rx_queue_intr_enable = iavf_dev_rx_queue_intr_enable, 197 .rx_queue_intr_disable = iavf_dev_rx_queue_intr_disable, 198 .filter_ctrl = iavf_dev_filter_ctrl, 199 .tx_done_cleanup = iavf_dev_tx_done_cleanup, 200 }; 201 202 static int 203 iavf_set_mc_addr_list(struct rte_eth_dev *dev, 204 struct rte_ether_addr *mc_addrs, 205 uint32_t mc_addrs_num) 206 { 207 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 208 struct iavf_adapter *adapter = 209 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 210 int err, ret; 211 212 if (mc_addrs_num > IAVF_NUM_MACADDR_MAX) { 213 PMD_DRV_LOG(ERR, 214 "can't add more than a limited number (%u) of addresses.", 215 (uint32_t)IAVF_NUM_MACADDR_MAX); 216 return -EINVAL; 217 } 218 219 /* flush previous addresses */ 220 err = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 221 false); 222 if (err) 223 return err; 224 225 /* add new ones */ 226 err = iavf_add_del_mc_addr_list(adapter, mc_addrs, mc_addrs_num, true); 227 228 if (err) { 229 /* if adding mac address list fails, should add the previous 230 * addresses back. 231 */ 232 ret = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, 233 vf->mc_addrs_num, true); 234 if (ret) 235 return ret; 236 } else { 237 vf->mc_addrs_num = mc_addrs_num; 238 memcpy(vf->mc_addrs, 239 mc_addrs, mc_addrs_num * sizeof(*mc_addrs)); 240 } 241 242 return err; 243 } 244 245 static int 246 iavf_init_rss(struct iavf_adapter *adapter) 247 { 248 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 249 struct rte_eth_rss_conf *rss_conf; 250 uint16_t i, j, nb_q; 251 int ret; 252 253 rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf; 254 nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues, 255 vf->max_rss_qregion); 256 257 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 258 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 259 return -ENOTSUP; 260 } 261 if (adapter->eth_dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 262 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default"); 263 /* set all lut items to default queue */ 264 for (i = 0; i < vf->vf_res->rss_lut_size; i++) 265 vf->rss_lut[i] = 0; 266 ret = iavf_configure_rss_lut(adapter); 267 return ret; 268 } 269 270 /* configure RSS key */ 271 if (!rss_conf->rss_key) { 272 /* Calculate the default hash key */ 273 for (i = 0; i <= vf->vf_res->rss_key_size; i++) 274 vf->rss_key[i] = (uint8_t)rte_rand(); 275 } else 276 rte_memcpy(vf->rss_key, rss_conf->rss_key, 277 RTE_MIN(rss_conf->rss_key_len, 278 vf->vf_res->rss_key_size)); 279 280 /* init RSS LUT table */ 281 for (i = 0, j = 0; i < vf->vf_res->rss_lut_size; i++, j++) { 282 if (j >= nb_q) 283 j = 0; 284 vf->rss_lut[i] = j; 285 } 286 /* send virtchnnl ops to configure rss*/ 287 ret = iavf_configure_rss_lut(adapter); 288 if (ret) 289 return ret; 290 ret = iavf_configure_rss_key(adapter); 291 if (ret) 292 return ret; 293 294 /* Set RSS hash configuration based on rss_conf->rss_hf. */ 295 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 296 if (ret) { 297 PMD_DRV_LOG(ERR, "fail to set default RSS"); 298 return ret; 299 } 300 301 return 0; 302 } 303 304 static int 305 iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num) 306 { 307 struct iavf_adapter *ad = 308 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 309 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 310 int ret; 311 312 ret = iavf_request_queues(ad, num); 313 if (ret) { 314 PMD_DRV_LOG(ERR, "request queues from PF failed"); 315 return ret; 316 } 317 PMD_DRV_LOG(INFO, "change queue pairs from %u to %u", 318 vf->vsi_res->num_queue_pairs, num); 319 320 ret = iavf_dev_reset(dev); 321 if (ret) { 322 PMD_DRV_LOG(ERR, "vf reset failed"); 323 return ret; 324 } 325 326 return 0; 327 } 328 329 static int 330 iavf_dev_configure(struct rte_eth_dev *dev) 331 { 332 struct iavf_adapter *ad = 333 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 334 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 335 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 336 uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 337 dev->data->nb_tx_queues); 338 int ret; 339 340 ad->rx_bulk_alloc_allowed = true; 341 /* Initialize to TRUE. If any of Rx queues doesn't meet the 342 * vector Rx/Tx preconditions, it will be reset. 343 */ 344 ad->rx_vec_allowed = true; 345 ad->tx_vec_allowed = true; 346 347 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 348 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 349 350 /* Large VF setting */ 351 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) { 352 if (!(vf->vf_res->vf_cap_flags & 353 VIRTCHNL_VF_LARGE_NUM_QPAIRS)) { 354 PMD_DRV_LOG(ERR, "large VF is not supported"); 355 return -1; 356 } 357 358 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) { 359 PMD_DRV_LOG(ERR, "queue pairs number cannot be larger than %u", 360 IAVF_MAX_NUM_QUEUES_LV); 361 return -1; 362 } 363 364 ret = iavf_queues_req_reset(dev, num_queue_pairs); 365 if (ret) 366 return ret; 367 368 ret = iavf_get_max_rss_queue_region(ad); 369 if (ret) { 370 PMD_INIT_LOG(ERR, "get max rss queue region failed"); 371 return ret; 372 } 373 374 vf->lv_enabled = true; 375 } else { 376 /* Check if large VF is already enabled. If so, disable and 377 * release redundant queue resource. 378 * Or check if enough queue pairs. If not, request them from PF. 379 */ 380 if (vf->lv_enabled || 381 num_queue_pairs > vf->vsi_res->num_queue_pairs) { 382 ret = iavf_queues_req_reset(dev, num_queue_pairs); 383 if (ret) 384 return ret; 385 386 vf->lv_enabled = false; 387 } 388 /* if large VF is not required, use default rss queue region */ 389 vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT; 390 } 391 392 /* Vlan stripping setting */ 393 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) { 394 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 395 iavf_enable_vlan_strip(ad); 396 else 397 iavf_disable_vlan_strip(ad); 398 } 399 400 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 401 if (iavf_init_rss(ad) != 0) { 402 PMD_DRV_LOG(ERR, "configure rss failed"); 403 return -1; 404 } 405 } 406 return 0; 407 } 408 409 static int 410 iavf_init_rxq(struct rte_eth_dev *dev, struct iavf_rx_queue *rxq) 411 { 412 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 413 struct rte_eth_dev_data *dev_data = dev->data; 414 uint16_t buf_size, max_pkt_len, len; 415 416 buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; 417 418 /* Calculate the maximum packet length allowed */ 419 len = rxq->rx_buf_len * IAVF_MAX_CHAINED_RX_BUFFERS; 420 max_pkt_len = RTE_MIN(len, dev->data->dev_conf.rxmode.max_rx_pkt_len); 421 422 /* Check if the jumbo frame and maximum packet length are set 423 * correctly. 424 */ 425 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 426 if (max_pkt_len <= RTE_ETHER_MAX_LEN || 427 max_pkt_len > IAVF_FRAME_SIZE_MAX) { 428 PMD_DRV_LOG(ERR, "maximum packet length must be " 429 "larger than %u and smaller than %u, " 430 "as jumbo frame is enabled", 431 (uint32_t)RTE_ETHER_MAX_LEN, 432 (uint32_t)IAVF_FRAME_SIZE_MAX); 433 return -EINVAL; 434 } 435 } else { 436 if (max_pkt_len < RTE_ETHER_MIN_LEN || 437 max_pkt_len > RTE_ETHER_MAX_LEN) { 438 PMD_DRV_LOG(ERR, "maximum packet length must be " 439 "larger than %u and smaller than %u, " 440 "as jumbo frame is disabled", 441 (uint32_t)RTE_ETHER_MIN_LEN, 442 (uint32_t)RTE_ETHER_MAX_LEN); 443 return -EINVAL; 444 } 445 } 446 447 rxq->max_pkt_len = max_pkt_len; 448 if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) || 449 rxq->max_pkt_len > buf_size) { 450 dev_data->scattered_rx = 1; 451 } 452 IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 453 IAVF_WRITE_FLUSH(hw); 454 455 return 0; 456 } 457 458 static int 459 iavf_init_queues(struct rte_eth_dev *dev) 460 { 461 struct iavf_rx_queue **rxq = 462 (struct iavf_rx_queue **)dev->data->rx_queues; 463 int i, ret = IAVF_SUCCESS; 464 465 for (i = 0; i < dev->data->nb_rx_queues; i++) { 466 if (!rxq[i] || !rxq[i]->q_set) 467 continue; 468 ret = iavf_init_rxq(dev, rxq[i]); 469 if (ret != IAVF_SUCCESS) 470 break; 471 } 472 /* set rx/tx function to vector/scatter/single-segment 473 * according to parameters 474 */ 475 iavf_set_rx_function(dev); 476 iavf_set_tx_function(dev); 477 478 return ret; 479 } 480 481 static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev, 482 struct rte_intr_handle *intr_handle) 483 { 484 struct iavf_adapter *adapter = 485 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 486 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 487 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 488 struct iavf_qv_map *qv_map; 489 uint16_t interval, i; 490 int vec; 491 492 if (rte_intr_cap_multiple(intr_handle) && 493 dev->data->dev_conf.intr_conf.rxq) { 494 if (rte_intr_efd_enable(intr_handle, dev->data->nb_rx_queues)) 495 return -1; 496 } 497 498 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 499 intr_handle->intr_vec = 500 rte_zmalloc("intr_vec", 501 dev->data->nb_rx_queues * sizeof(int), 0); 502 if (!intr_handle->intr_vec) { 503 PMD_DRV_LOG(ERR, "Failed to allocate %d rx intr_vec", 504 dev->data->nb_rx_queues); 505 return -1; 506 } 507 } 508 509 qv_map = rte_zmalloc("qv_map", 510 dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0); 511 if (!qv_map) { 512 PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map", 513 dev->data->nb_rx_queues); 514 return -1; 515 } 516 517 if (!dev->data->dev_conf.intr_conf.rxq || 518 !rte_intr_dp_is_en(intr_handle)) { 519 /* Rx interrupt disabled, Map interrupt only for writeback */ 520 vf->nb_msix = 1; 521 if (vf->vf_res->vf_cap_flags & 522 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 523 /* If WB_ON_ITR supports, enable it */ 524 vf->msix_base = IAVF_RX_VEC_START; 525 /* Set the ITR for index zero, to 2us to make sure that 526 * we leave time for aggregation to occur, but don't 527 * increase latency dramatically. 528 */ 529 IAVF_WRITE_REG(hw, 530 IAVF_VFINT_DYN_CTLN1(vf->msix_base - 1), 531 (0 << IAVF_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) | 532 IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK | 533 (2UL << IAVF_VFINT_DYN_CTLN1_INTERVAL_SHIFT)); 534 /* debug - check for success! the return value 535 * should be 2, offset is 0x2800 536 */ 537 /* IAVF_READ_REG(hw, IAVF_VFINT_ITRN1(0, 0)); */ 538 } else { 539 /* If no WB_ON_ITR offload flags, need to set 540 * interrupt for descriptor write back. 541 */ 542 vf->msix_base = IAVF_MISC_VEC_ID; 543 544 /* set ITR to max */ 545 interval = iavf_calc_itr_interval( 546 IAVF_QUEUE_ITR_INTERVAL_MAX); 547 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 548 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 549 (IAVF_ITR_INDEX_DEFAULT << 550 IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT) | 551 (interval << 552 IAVF_VFINT_DYN_CTL01_INTERVAL_SHIFT)); 553 } 554 IAVF_WRITE_FLUSH(hw); 555 /* map all queues to the same interrupt */ 556 for (i = 0; i < dev->data->nb_rx_queues; i++) { 557 qv_map[i].queue_id = i; 558 qv_map[i].vector_id = vf->msix_base; 559 } 560 vf->qv_map = qv_map; 561 } else { 562 if (!rte_intr_allow_others(intr_handle)) { 563 vf->nb_msix = 1; 564 vf->msix_base = IAVF_MISC_VEC_ID; 565 for (i = 0; i < dev->data->nb_rx_queues; i++) { 566 qv_map[i].queue_id = i; 567 qv_map[i].vector_id = vf->msix_base; 568 intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID; 569 } 570 vf->qv_map = qv_map; 571 PMD_DRV_LOG(DEBUG, 572 "vector %u are mapping to all Rx queues", 573 vf->msix_base); 574 } else { 575 /* If Rx interrupt is reuquired, and we can use 576 * multi interrupts, then the vec is from 1 577 */ 578 vf->nb_msix = RTE_MIN(vf->vf_res->max_vectors, 579 intr_handle->nb_efd); 580 vf->msix_base = IAVF_RX_VEC_START; 581 vec = IAVF_RX_VEC_START; 582 for (i = 0; i < dev->data->nb_rx_queues; i++) { 583 qv_map[i].queue_id = i; 584 qv_map[i].vector_id = vec; 585 intr_handle->intr_vec[i] = vec++; 586 if (vec >= vf->nb_msix) 587 vec = IAVF_RX_VEC_START; 588 } 589 vf->qv_map = qv_map; 590 PMD_DRV_LOG(DEBUG, 591 "%u vectors are mapping to %u Rx queues", 592 vf->nb_msix, dev->data->nb_rx_queues); 593 } 594 } 595 596 if (!vf->lv_enabled) { 597 if (iavf_config_irq_map(adapter)) { 598 PMD_DRV_LOG(ERR, "config interrupt mapping failed"); 599 return -1; 600 } 601 } else { 602 uint16_t num_qv_maps = dev->data->nb_rx_queues; 603 uint16_t index = 0; 604 605 while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) { 606 if (iavf_config_irq_map_lv(adapter, 607 IAVF_IRQ_MAP_NUM_PER_BUF, index)) { 608 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 609 return -1; 610 } 611 num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF; 612 index += IAVF_IRQ_MAP_NUM_PER_BUF; 613 } 614 615 if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) { 616 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 617 return -1; 618 } 619 } 620 return 0; 621 } 622 623 static int 624 iavf_start_queues(struct rte_eth_dev *dev) 625 { 626 struct iavf_rx_queue *rxq; 627 struct iavf_tx_queue *txq; 628 int i; 629 630 for (i = 0; i < dev->data->nb_tx_queues; i++) { 631 txq = dev->data->tx_queues[i]; 632 if (txq->tx_deferred_start) 633 continue; 634 if (iavf_dev_tx_queue_start(dev, i) != 0) { 635 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 636 return -1; 637 } 638 } 639 640 for (i = 0; i < dev->data->nb_rx_queues; i++) { 641 rxq = dev->data->rx_queues[i]; 642 if (rxq->rx_deferred_start) 643 continue; 644 if (iavf_dev_rx_queue_start(dev, i) != 0) { 645 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 646 return -1; 647 } 648 } 649 650 return 0; 651 } 652 653 static int 654 iavf_dev_start(struct rte_eth_dev *dev) 655 { 656 struct iavf_adapter *adapter = 657 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 658 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 659 struct rte_intr_handle *intr_handle = dev->intr_handle; 660 uint16_t num_queue_pairs; 661 uint16_t index = 0; 662 663 PMD_INIT_FUNC_TRACE(); 664 665 adapter->stopped = 0; 666 667 vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; 668 vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 669 dev->data->nb_tx_queues); 670 num_queue_pairs = vf->num_queue_pairs; 671 672 if (iavf_init_queues(dev) != 0) { 673 PMD_DRV_LOG(ERR, "failed to do Queue init"); 674 return -1; 675 } 676 677 /* If needed, send configure queues msg multiple times to make the 678 * adminq buffer length smaller than the 4K limitation. 679 */ 680 while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) { 681 if (iavf_configure_queues(adapter, 682 IAVF_CFG_Q_NUM_PER_BUF, index) != 0) { 683 PMD_DRV_LOG(ERR, "configure queues failed"); 684 goto err_queue; 685 } 686 num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF; 687 index += IAVF_CFG_Q_NUM_PER_BUF; 688 } 689 690 if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) { 691 PMD_DRV_LOG(ERR, "configure queues failed"); 692 goto err_queue; 693 } 694 695 if (iavf_config_rx_queues_irqs(dev, intr_handle) != 0) { 696 PMD_DRV_LOG(ERR, "configure irq failed"); 697 goto err_queue; 698 } 699 /* re-enable intr again, because efd assign may change */ 700 if (dev->data->dev_conf.intr_conf.rxq != 0) { 701 rte_intr_disable(intr_handle); 702 rte_intr_enable(intr_handle); 703 } 704 705 /* Set all mac addrs */ 706 iavf_add_del_all_mac_addr(adapter, true); 707 708 /* Set all multicast addresses */ 709 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 710 true); 711 712 if (iavf_start_queues(dev) != 0) { 713 PMD_DRV_LOG(ERR, "enable queues failed"); 714 goto err_mac; 715 } 716 717 return 0; 718 719 err_mac: 720 iavf_add_del_all_mac_addr(adapter, false); 721 err_queue: 722 return -1; 723 } 724 725 static int 726 iavf_dev_stop(struct rte_eth_dev *dev) 727 { 728 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 729 struct iavf_adapter *adapter = 730 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 731 struct rte_intr_handle *intr_handle = dev->intr_handle; 732 733 PMD_INIT_FUNC_TRACE(); 734 735 if (adapter->stopped == 1) 736 return 0; 737 738 iavf_stop_queues(dev); 739 740 /* Disable the interrupt for Rx */ 741 rte_intr_efd_disable(intr_handle); 742 /* Rx interrupt vector mapping free */ 743 if (intr_handle->intr_vec) { 744 rte_free(intr_handle->intr_vec); 745 intr_handle->intr_vec = NULL; 746 } 747 748 /* remove all mac addrs */ 749 iavf_add_del_all_mac_addr(adapter, false); 750 751 /* remove all multicast addresses */ 752 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 753 false); 754 755 adapter->stopped = 1; 756 dev->data->dev_started = 0; 757 758 return 0; 759 } 760 761 static int 762 iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 763 { 764 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 765 766 dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV; 767 dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV; 768 dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN; 769 dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX; 770 dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD; 771 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 772 dev_info->hash_key_size = vf->vf_res->rss_key_size; 773 dev_info->reta_size = vf->vf_res->rss_lut_size; 774 dev_info->flow_type_rss_offloads = IAVF_RSS_OFFLOAD_ALL; 775 dev_info->max_mac_addrs = IAVF_NUM_MACADDR_MAX; 776 dev_info->rx_offload_capa = 777 DEV_RX_OFFLOAD_VLAN_STRIP | 778 DEV_RX_OFFLOAD_QINQ_STRIP | 779 DEV_RX_OFFLOAD_IPV4_CKSUM | 780 DEV_RX_OFFLOAD_UDP_CKSUM | 781 DEV_RX_OFFLOAD_TCP_CKSUM | 782 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 783 DEV_RX_OFFLOAD_SCATTER | 784 DEV_RX_OFFLOAD_JUMBO_FRAME | 785 DEV_RX_OFFLOAD_VLAN_FILTER | 786 DEV_RX_OFFLOAD_RSS_HASH; 787 dev_info->tx_offload_capa = 788 DEV_TX_OFFLOAD_VLAN_INSERT | 789 DEV_TX_OFFLOAD_QINQ_INSERT | 790 DEV_TX_OFFLOAD_IPV4_CKSUM | 791 DEV_TX_OFFLOAD_UDP_CKSUM | 792 DEV_TX_OFFLOAD_TCP_CKSUM | 793 DEV_TX_OFFLOAD_SCTP_CKSUM | 794 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 795 DEV_TX_OFFLOAD_TCP_TSO | 796 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 797 DEV_TX_OFFLOAD_GRE_TNL_TSO | 798 DEV_TX_OFFLOAD_IPIP_TNL_TSO | 799 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | 800 DEV_TX_OFFLOAD_MULTI_SEGS | 801 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 802 803 dev_info->default_rxconf = (struct rte_eth_rxconf) { 804 .rx_free_thresh = IAVF_DEFAULT_RX_FREE_THRESH, 805 .rx_drop_en = 0, 806 .offloads = 0, 807 }; 808 809 dev_info->default_txconf = (struct rte_eth_txconf) { 810 .tx_free_thresh = IAVF_DEFAULT_TX_FREE_THRESH, 811 .tx_rs_thresh = IAVF_DEFAULT_TX_RS_THRESH, 812 .offloads = 0, 813 }; 814 815 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 816 .nb_max = IAVF_MAX_RING_DESC, 817 .nb_min = IAVF_MIN_RING_DESC, 818 .nb_align = IAVF_ALIGN_RING_DESC, 819 }; 820 821 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 822 .nb_max = IAVF_MAX_RING_DESC, 823 .nb_min = IAVF_MIN_RING_DESC, 824 .nb_align = IAVF_ALIGN_RING_DESC, 825 }; 826 827 return 0; 828 } 829 830 static const uint32_t * 831 iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) 832 { 833 static const uint32_t ptypes[] = { 834 RTE_PTYPE_L2_ETHER, 835 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 836 RTE_PTYPE_L4_FRAG, 837 RTE_PTYPE_L4_ICMP, 838 RTE_PTYPE_L4_NONFRAG, 839 RTE_PTYPE_L4_SCTP, 840 RTE_PTYPE_L4_TCP, 841 RTE_PTYPE_L4_UDP, 842 RTE_PTYPE_UNKNOWN 843 }; 844 return ptypes; 845 } 846 847 int 848 iavf_dev_link_update(struct rte_eth_dev *dev, 849 __rte_unused int wait_to_complete) 850 { 851 struct rte_eth_link new_link; 852 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 853 854 memset(&new_link, 0, sizeof(new_link)); 855 856 /* Only read status info stored in VF, and the info is updated 857 * when receive LINK_CHANGE evnet from PF by Virtchnnl. 858 */ 859 switch (vf->link_speed) { 860 case 10: 861 new_link.link_speed = ETH_SPEED_NUM_10M; 862 break; 863 case 100: 864 new_link.link_speed = ETH_SPEED_NUM_100M; 865 break; 866 case 1000: 867 new_link.link_speed = ETH_SPEED_NUM_1G; 868 break; 869 case 10000: 870 new_link.link_speed = ETH_SPEED_NUM_10G; 871 break; 872 case 20000: 873 new_link.link_speed = ETH_SPEED_NUM_20G; 874 break; 875 case 25000: 876 new_link.link_speed = ETH_SPEED_NUM_25G; 877 break; 878 case 40000: 879 new_link.link_speed = ETH_SPEED_NUM_40G; 880 break; 881 case 50000: 882 new_link.link_speed = ETH_SPEED_NUM_50G; 883 break; 884 case 100000: 885 new_link.link_speed = ETH_SPEED_NUM_100G; 886 break; 887 default: 888 new_link.link_speed = ETH_SPEED_NUM_NONE; 889 break; 890 } 891 892 new_link.link_duplex = ETH_LINK_FULL_DUPLEX; 893 new_link.link_status = vf->link_up ? ETH_LINK_UP : 894 ETH_LINK_DOWN; 895 new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 896 ETH_LINK_SPEED_FIXED); 897 898 return rte_eth_linkstatus_set(dev, &new_link); 899 } 900 901 static int 902 iavf_dev_promiscuous_enable(struct rte_eth_dev *dev) 903 { 904 struct iavf_adapter *adapter = 905 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 906 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 907 908 return iavf_config_promisc(adapter, 909 true, vf->promisc_multicast_enabled); 910 } 911 912 static int 913 iavf_dev_promiscuous_disable(struct rte_eth_dev *dev) 914 { 915 struct iavf_adapter *adapter = 916 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 917 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 918 919 return iavf_config_promisc(adapter, 920 false, vf->promisc_multicast_enabled); 921 } 922 923 static int 924 iavf_dev_allmulticast_enable(struct rte_eth_dev *dev) 925 { 926 struct iavf_adapter *adapter = 927 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 928 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 929 930 return iavf_config_promisc(adapter, 931 vf->promisc_unicast_enabled, true); 932 } 933 934 static int 935 iavf_dev_allmulticast_disable(struct rte_eth_dev *dev) 936 { 937 struct iavf_adapter *adapter = 938 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 939 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 940 941 return iavf_config_promisc(adapter, 942 vf->promisc_unicast_enabled, false); 943 } 944 945 static int 946 iavf_dev_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr, 947 __rte_unused uint32_t index, 948 __rte_unused uint32_t pool) 949 { 950 struct iavf_adapter *adapter = 951 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 952 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 953 int err; 954 955 if (rte_is_zero_ether_addr(addr)) { 956 PMD_DRV_LOG(ERR, "Invalid Ethernet Address"); 957 return -EINVAL; 958 } 959 960 err = iavf_add_del_eth_addr(adapter, addr, true); 961 if (err) { 962 PMD_DRV_LOG(ERR, "fail to add MAC address"); 963 return -EIO; 964 } 965 966 vf->mac_num++; 967 968 return 0; 969 } 970 971 static void 972 iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) 973 { 974 struct iavf_adapter *adapter = 975 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 976 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 977 struct rte_ether_addr *addr; 978 int err; 979 980 addr = &dev->data->mac_addrs[index]; 981 982 err = iavf_add_del_eth_addr(adapter, addr, false); 983 if (err) 984 PMD_DRV_LOG(ERR, "fail to delete MAC address"); 985 986 vf->mac_num--; 987 } 988 989 static int 990 iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 991 { 992 struct iavf_adapter *adapter = 993 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 994 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 995 int err; 996 997 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 998 return -ENOTSUP; 999 1000 err = iavf_add_del_vlan(adapter, vlan_id, on); 1001 if (err) 1002 return -EIO; 1003 return 0; 1004 } 1005 1006 static int 1007 iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1008 { 1009 struct iavf_adapter *adapter = 1010 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1011 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1012 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 1013 int err; 1014 1015 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1016 return -ENOTSUP; 1017 1018 /* Vlan stripping setting */ 1019 if (mask & ETH_VLAN_STRIP_MASK) { 1020 /* Enable or disable VLAN stripping */ 1021 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1022 err = iavf_enable_vlan_strip(adapter); 1023 else 1024 err = iavf_disable_vlan_strip(adapter); 1025 1026 if (err) 1027 return -EIO; 1028 } 1029 return 0; 1030 } 1031 1032 static int 1033 iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 1034 struct rte_eth_rss_reta_entry64 *reta_conf, 1035 uint16_t reta_size) 1036 { 1037 struct iavf_adapter *adapter = 1038 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1039 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1040 uint8_t *lut; 1041 uint16_t i, idx, shift; 1042 int ret; 1043 1044 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1045 return -ENOTSUP; 1046 1047 if (reta_size != vf->vf_res->rss_lut_size) { 1048 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1049 "(%d) doesn't match the number of hardware can " 1050 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1051 return -EINVAL; 1052 } 1053 1054 lut = rte_zmalloc("rss_lut", reta_size, 0); 1055 if (!lut) { 1056 PMD_DRV_LOG(ERR, "No memory can be allocated"); 1057 return -ENOMEM; 1058 } 1059 /* store the old lut table temporarily */ 1060 rte_memcpy(lut, vf->rss_lut, reta_size); 1061 1062 for (i = 0; i < reta_size; i++) { 1063 idx = i / RTE_RETA_GROUP_SIZE; 1064 shift = i % RTE_RETA_GROUP_SIZE; 1065 if (reta_conf[idx].mask & (1ULL << shift)) 1066 lut[i] = reta_conf[idx].reta[shift]; 1067 } 1068 1069 rte_memcpy(vf->rss_lut, lut, reta_size); 1070 /* send virtchnnl ops to configure rss*/ 1071 ret = iavf_configure_rss_lut(adapter); 1072 if (ret) /* revert back */ 1073 rte_memcpy(vf->rss_lut, lut, reta_size); 1074 rte_free(lut); 1075 1076 return ret; 1077 } 1078 1079 static int 1080 iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 1081 struct rte_eth_rss_reta_entry64 *reta_conf, 1082 uint16_t reta_size) 1083 { 1084 struct iavf_adapter *adapter = 1085 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1086 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1087 uint16_t i, idx, shift; 1088 1089 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1090 return -ENOTSUP; 1091 1092 if (reta_size != vf->vf_res->rss_lut_size) { 1093 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1094 "(%d) doesn't match the number of hardware can " 1095 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1096 return -EINVAL; 1097 } 1098 1099 for (i = 0; i < reta_size; i++) { 1100 idx = i / RTE_RETA_GROUP_SIZE; 1101 shift = i % RTE_RETA_GROUP_SIZE; 1102 if (reta_conf[idx].mask & (1ULL << shift)) 1103 reta_conf[idx].reta[shift] = vf->rss_lut[i]; 1104 } 1105 1106 return 0; 1107 } 1108 1109 static int 1110 iavf_set_rss_key(struct iavf_adapter *adapter, uint8_t *key, uint8_t key_len) 1111 { 1112 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1113 1114 /* HENA setting, it is enabled by default, no change */ 1115 if (!key || key_len == 0) { 1116 PMD_DRV_LOG(DEBUG, "No key to be configured"); 1117 return 0; 1118 } else if (key_len != vf->vf_res->rss_key_size) { 1119 PMD_DRV_LOG(ERR, "The size of hash key configured " 1120 "(%d) doesn't match the size of hardware can " 1121 "support (%d)", key_len, 1122 vf->vf_res->rss_key_size); 1123 return -EINVAL; 1124 } 1125 1126 rte_memcpy(vf->rss_key, key, key_len); 1127 1128 return iavf_configure_rss_key(adapter); 1129 } 1130 1131 static int 1132 iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 1133 struct rte_eth_rss_conf *rss_conf) 1134 { 1135 struct iavf_adapter *adapter = 1136 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1137 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1138 int ret; 1139 1140 adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf = *rss_conf; 1141 1142 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1143 return -ENOTSUP; 1144 1145 /* Set hash key. */ 1146 ret = iavf_set_rss_key(adapter, rss_conf->rss_key, 1147 rss_conf->rss_key_len); 1148 if (ret) 1149 return ret; 1150 1151 if (rss_conf->rss_hf == 0) 1152 return 0; 1153 1154 /* Overwritten default RSS. */ 1155 ret = iavf_set_hena(adapter, 0); 1156 if (ret) 1157 PMD_DRV_LOG(ERR, "%s Remove rss vsi fail %d", 1158 __func__, ret); 1159 1160 /* Set new RSS configuration. */ 1161 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 1162 if (ret) { 1163 PMD_DRV_LOG(ERR, "fail to set new RSS"); 1164 return ret; 1165 } 1166 1167 return 0; 1168 } 1169 1170 static int 1171 iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 1172 struct rte_eth_rss_conf *rss_conf) 1173 { 1174 struct iavf_adapter *adapter = 1175 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1176 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1177 1178 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1179 return -ENOTSUP; 1180 1181 rss_conf->rss_hf = vf->rss_hf; 1182 1183 if (!rss_conf->rss_key) 1184 return 0; 1185 1186 rss_conf->rss_key_len = vf->vf_res->rss_key_size; 1187 rte_memcpy(rss_conf->rss_key, vf->rss_key, rss_conf->rss_key_len); 1188 1189 return 0; 1190 } 1191 1192 static int 1193 iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1194 { 1195 uint32_t frame_size = mtu + IAVF_ETH_OVERHEAD; 1196 int ret = 0; 1197 1198 if (mtu < RTE_ETHER_MIN_MTU || frame_size > IAVF_FRAME_SIZE_MAX) 1199 return -EINVAL; 1200 1201 /* mtu setting is forbidden if port is start */ 1202 if (dev->data->dev_started) { 1203 PMD_DRV_LOG(ERR, "port must be stopped before configuration"); 1204 return -EBUSY; 1205 } 1206 1207 if (frame_size > RTE_ETHER_MAX_LEN) 1208 dev->data->dev_conf.rxmode.offloads |= 1209 DEV_RX_OFFLOAD_JUMBO_FRAME; 1210 else 1211 dev->data->dev_conf.rxmode.offloads &= 1212 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 1213 1214 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 1215 1216 return ret; 1217 } 1218 1219 static int 1220 iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 1221 struct rte_ether_addr *mac_addr) 1222 { 1223 struct iavf_adapter *adapter = 1224 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1225 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1226 struct rte_ether_addr *perm_addr, *old_addr; 1227 int ret; 1228 1229 old_addr = (struct rte_ether_addr *)hw->mac.addr; 1230 perm_addr = (struct rte_ether_addr *)hw->mac.perm_addr; 1231 1232 /* If the MAC address is configured by host, skip the setting */ 1233 if (rte_is_valid_assigned_ether_addr(perm_addr)) 1234 return -EPERM; 1235 1236 ret = iavf_add_del_eth_addr(adapter, old_addr, false); 1237 if (ret) 1238 PMD_DRV_LOG(ERR, "Fail to delete old MAC:" 1239 " %02X:%02X:%02X:%02X:%02X:%02X", 1240 old_addr->addr_bytes[0], 1241 old_addr->addr_bytes[1], 1242 old_addr->addr_bytes[2], 1243 old_addr->addr_bytes[3], 1244 old_addr->addr_bytes[4], 1245 old_addr->addr_bytes[5]); 1246 1247 ret = iavf_add_del_eth_addr(adapter, mac_addr, true); 1248 if (ret) 1249 PMD_DRV_LOG(ERR, "Fail to add new MAC:" 1250 " %02X:%02X:%02X:%02X:%02X:%02X", 1251 mac_addr->addr_bytes[0], 1252 mac_addr->addr_bytes[1], 1253 mac_addr->addr_bytes[2], 1254 mac_addr->addr_bytes[3], 1255 mac_addr->addr_bytes[4], 1256 mac_addr->addr_bytes[5]); 1257 1258 if (ret) 1259 return -EIO; 1260 1261 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)hw->mac.addr); 1262 return 0; 1263 } 1264 1265 static void 1266 iavf_stat_update_48(uint64_t *offset, uint64_t *stat) 1267 { 1268 if (*stat >= *offset) 1269 *stat = *stat - *offset; 1270 else 1271 *stat = (uint64_t)((*stat + 1272 ((uint64_t)1 << IAVF_48_BIT_WIDTH)) - *offset); 1273 1274 *stat &= IAVF_48_BIT_MASK; 1275 } 1276 1277 static void 1278 iavf_stat_update_32(uint64_t *offset, uint64_t *stat) 1279 { 1280 if (*stat >= *offset) 1281 *stat = (uint64_t)(*stat - *offset); 1282 else 1283 *stat = (uint64_t)((*stat + 1284 ((uint64_t)1 << IAVF_32_BIT_WIDTH)) - *offset); 1285 } 1286 1287 static void 1288 iavf_update_stats(struct iavf_vsi *vsi, struct virtchnl_eth_stats *nes) 1289 { 1290 struct virtchnl_eth_stats *oes = &vsi->eth_stats_offset; 1291 1292 iavf_stat_update_48(&oes->rx_bytes, &nes->rx_bytes); 1293 iavf_stat_update_48(&oes->rx_unicast, &nes->rx_unicast); 1294 iavf_stat_update_48(&oes->rx_multicast, &nes->rx_multicast); 1295 iavf_stat_update_48(&oes->rx_broadcast, &nes->rx_broadcast); 1296 iavf_stat_update_32(&oes->rx_discards, &nes->rx_discards); 1297 iavf_stat_update_48(&oes->tx_bytes, &nes->tx_bytes); 1298 iavf_stat_update_48(&oes->tx_unicast, &nes->tx_unicast); 1299 iavf_stat_update_48(&oes->tx_multicast, &nes->tx_multicast); 1300 iavf_stat_update_48(&oes->tx_broadcast, &nes->tx_broadcast); 1301 iavf_stat_update_32(&oes->tx_errors, &nes->tx_errors); 1302 iavf_stat_update_32(&oes->tx_discards, &nes->tx_discards); 1303 } 1304 1305 static int 1306 iavf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1307 { 1308 struct iavf_adapter *adapter = 1309 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1310 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1311 struct iavf_vsi *vsi = &vf->vsi; 1312 struct virtchnl_eth_stats *pstats = NULL; 1313 int ret; 1314 1315 ret = iavf_query_stats(adapter, &pstats); 1316 if (ret == 0) { 1317 iavf_update_stats(vsi, pstats); 1318 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + 1319 pstats->rx_broadcast - pstats->rx_discards; 1320 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + 1321 pstats->tx_unicast; 1322 stats->imissed = pstats->rx_discards; 1323 stats->oerrors = pstats->tx_errors + pstats->tx_discards; 1324 stats->ibytes = pstats->rx_bytes; 1325 stats->ibytes -= stats->ipackets * RTE_ETHER_CRC_LEN; 1326 stats->obytes = pstats->tx_bytes; 1327 } else { 1328 PMD_DRV_LOG(ERR, "Get statistics failed"); 1329 } 1330 return ret; 1331 } 1332 1333 static int 1334 iavf_dev_stats_reset(struct rte_eth_dev *dev) 1335 { 1336 int ret; 1337 struct iavf_adapter *adapter = 1338 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1339 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1340 struct iavf_vsi *vsi = &vf->vsi; 1341 struct virtchnl_eth_stats *pstats = NULL; 1342 1343 /* read stat values to clear hardware registers */ 1344 ret = iavf_query_stats(adapter, &pstats); 1345 if (ret != 0) 1346 return ret; 1347 1348 /* set stats offset base on current values */ 1349 vsi->eth_stats_offset = *pstats; 1350 1351 return 0; 1352 } 1353 1354 static int iavf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1355 struct rte_eth_xstat_name *xstats_names, 1356 __rte_unused unsigned int limit) 1357 { 1358 unsigned int i; 1359 1360 if (xstats_names != NULL) 1361 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1362 snprintf(xstats_names[i].name, 1363 sizeof(xstats_names[i].name), 1364 "%s", rte_iavf_stats_strings[i].name); 1365 } 1366 return IAVF_NB_XSTATS; 1367 } 1368 1369 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 1370 struct rte_eth_xstat *xstats, unsigned int n) 1371 { 1372 int ret; 1373 unsigned int i; 1374 struct iavf_adapter *adapter = 1375 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1376 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1377 struct iavf_vsi *vsi = &vf->vsi; 1378 struct virtchnl_eth_stats *pstats = NULL; 1379 1380 if (n < IAVF_NB_XSTATS) 1381 return IAVF_NB_XSTATS; 1382 1383 ret = iavf_query_stats(adapter, &pstats); 1384 if (ret != 0) 1385 return 0; 1386 1387 if (!xstats) 1388 return 0; 1389 1390 iavf_update_stats(vsi, pstats); 1391 1392 /* loop over xstats array and values from pstats */ 1393 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1394 xstats[i].id = i; 1395 xstats[i].value = *(uint64_t *)(((char *)pstats) + 1396 rte_iavf_stats_strings[i].offset); 1397 } 1398 1399 return IAVF_NB_XSTATS; 1400 } 1401 1402 1403 static int 1404 iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1405 { 1406 struct iavf_adapter *adapter = 1407 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1408 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1409 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1410 uint16_t msix_intr; 1411 1412 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1413 if (msix_intr == IAVF_MISC_VEC_ID) { 1414 PMD_DRV_LOG(INFO, "MISC is also enabled for control"); 1415 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1416 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 1417 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1418 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1419 } else { 1420 IAVF_WRITE_REG(hw, 1421 IAVF_VFINT_DYN_CTLN1 1422 (msix_intr - IAVF_RX_VEC_START), 1423 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 1424 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1425 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 1426 } 1427 1428 IAVF_WRITE_FLUSH(hw); 1429 1430 rte_intr_ack(&pci_dev->intr_handle); 1431 1432 return 0; 1433 } 1434 1435 static int 1436 iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1437 { 1438 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1439 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1440 uint16_t msix_intr; 1441 1442 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1443 if (msix_intr == IAVF_MISC_VEC_ID) { 1444 PMD_DRV_LOG(ERR, "MISC is used for control, cannot disable it"); 1445 return -EIO; 1446 } 1447 1448 IAVF_WRITE_REG(hw, 1449 IAVF_VFINT_DYN_CTLN1(msix_intr - IAVF_RX_VEC_START), 1450 0); 1451 1452 IAVF_WRITE_FLUSH(hw); 1453 return 0; 1454 } 1455 1456 static int 1457 iavf_check_vf_reset_done(struct iavf_hw *hw) 1458 { 1459 int i, reset; 1460 1461 for (i = 0; i < IAVF_RESET_WAIT_CNT; i++) { 1462 reset = IAVF_READ_REG(hw, IAVF_VFGEN_RSTAT) & 1463 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 1464 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 1465 if (reset == VIRTCHNL_VFR_VFACTIVE || 1466 reset == VIRTCHNL_VFR_COMPLETED) 1467 break; 1468 rte_delay_ms(20); 1469 } 1470 1471 if (i >= IAVF_RESET_WAIT_CNT) 1472 return -1; 1473 1474 return 0; 1475 } 1476 1477 static int 1478 iavf_lookup_proto_xtr_type(const char *flex_name) 1479 { 1480 static struct { 1481 const char *name; 1482 enum iavf_proto_xtr_type type; 1483 } xtr_type_map[] = { 1484 { "vlan", IAVF_PROTO_XTR_VLAN }, 1485 { "ipv4", IAVF_PROTO_XTR_IPV4 }, 1486 { "ipv6", IAVF_PROTO_XTR_IPV6 }, 1487 { "ipv6_flow", IAVF_PROTO_XTR_IPV6_FLOW }, 1488 { "tcp", IAVF_PROTO_XTR_TCP }, 1489 { "ip_offset", IAVF_PROTO_XTR_IP_OFFSET }, 1490 }; 1491 uint32_t i; 1492 1493 for (i = 0; i < RTE_DIM(xtr_type_map); i++) { 1494 if (strcmp(flex_name, xtr_type_map[i].name) == 0) 1495 return xtr_type_map[i].type; 1496 } 1497 1498 PMD_DRV_LOG(ERR, "wrong proto_xtr type, " 1499 "it should be: vlan|ipv4|ipv6|ipv6_flow|tcp|ip_offset"); 1500 1501 return -1; 1502 } 1503 1504 /** 1505 * Parse elem, the elem could be single number/range or '(' ')' group 1506 * 1) A single number elem, it's just a simple digit. e.g. 9 1507 * 2) A single range elem, two digits with a '-' between. e.g. 2-6 1508 * 3) A group elem, combines multiple 1) or 2) with '( )'. e.g (0,2-4,6) 1509 * Within group elem, '-' used for a range separator; 1510 * ',' used for a single number. 1511 */ 1512 static int 1513 iavf_parse_queue_set(const char *input, int xtr_type, 1514 struct iavf_devargs *devargs) 1515 { 1516 const char *str = input; 1517 char *end = NULL; 1518 uint32_t min, max; 1519 uint32_t idx; 1520 1521 while (isblank(*str)) 1522 str++; 1523 1524 if (!isdigit(*str) && *str != '(') 1525 return -1; 1526 1527 /* process single number or single range of number */ 1528 if (*str != '(') { 1529 errno = 0; 1530 idx = strtoul(str, &end, 10); 1531 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1532 return -1; 1533 1534 while (isblank(*end)) 1535 end++; 1536 1537 min = idx; 1538 max = idx; 1539 1540 /* process single <number>-<number> */ 1541 if (*end == '-') { 1542 end++; 1543 while (isblank(*end)) 1544 end++; 1545 if (!isdigit(*end)) 1546 return -1; 1547 1548 errno = 0; 1549 idx = strtoul(end, &end, 10); 1550 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1551 return -1; 1552 1553 max = idx; 1554 while (isblank(*end)) 1555 end++; 1556 } 1557 1558 if (*end != ':') 1559 return -1; 1560 1561 for (idx = RTE_MIN(min, max); 1562 idx <= RTE_MAX(min, max); idx++) 1563 devargs->proto_xtr[idx] = xtr_type; 1564 1565 return 0; 1566 } 1567 1568 /* process set within bracket */ 1569 str++; 1570 while (isblank(*str)) 1571 str++; 1572 if (*str == '\0') 1573 return -1; 1574 1575 min = IAVF_MAX_QUEUE_NUM; 1576 do { 1577 /* go ahead to the first digit */ 1578 while (isblank(*str)) 1579 str++; 1580 if (!isdigit(*str)) 1581 return -1; 1582 1583 /* get the digit value */ 1584 errno = 0; 1585 idx = strtoul(str, &end, 10); 1586 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1587 return -1; 1588 1589 /* go ahead to separator '-',',' and ')' */ 1590 while (isblank(*end)) 1591 end++; 1592 if (*end == '-') { 1593 if (min == IAVF_MAX_QUEUE_NUM) 1594 min = idx; 1595 else /* avoid continuous '-' */ 1596 return -1; 1597 } else if (*end == ',' || *end == ')') { 1598 max = idx; 1599 if (min == IAVF_MAX_QUEUE_NUM) 1600 min = idx; 1601 1602 for (idx = RTE_MIN(min, max); 1603 idx <= RTE_MAX(min, max); idx++) 1604 devargs->proto_xtr[idx] = xtr_type; 1605 1606 min = IAVF_MAX_QUEUE_NUM; 1607 } else { 1608 return -1; 1609 } 1610 1611 str = end + 1; 1612 } while (*end != ')' && *end != '\0'); 1613 1614 return 0; 1615 } 1616 1617 static int 1618 iavf_parse_queue_proto_xtr(const char *queues, struct iavf_devargs *devargs) 1619 { 1620 const char *queue_start; 1621 uint32_t idx; 1622 int xtr_type; 1623 char flex_name[32]; 1624 1625 while (isblank(*queues)) 1626 queues++; 1627 1628 if (*queues != '[') { 1629 xtr_type = iavf_lookup_proto_xtr_type(queues); 1630 if (xtr_type < 0) 1631 return -1; 1632 1633 devargs->proto_xtr_dflt = xtr_type; 1634 1635 return 0; 1636 } 1637 1638 queues++; 1639 do { 1640 while (isblank(*queues)) 1641 queues++; 1642 if (*queues == '\0') 1643 return -1; 1644 1645 queue_start = queues; 1646 1647 /* go across a complete bracket */ 1648 if (*queue_start == '(') { 1649 queues += strcspn(queues, ")"); 1650 if (*queues != ')') 1651 return -1; 1652 } 1653 1654 /* scan the separator ':' */ 1655 queues += strcspn(queues, ":"); 1656 if (*queues++ != ':') 1657 return -1; 1658 while (isblank(*queues)) 1659 queues++; 1660 1661 for (idx = 0; ; idx++) { 1662 if (isblank(queues[idx]) || 1663 queues[idx] == ',' || 1664 queues[idx] == ']' || 1665 queues[idx] == '\0') 1666 break; 1667 1668 if (idx > sizeof(flex_name) - 2) 1669 return -1; 1670 1671 flex_name[idx] = queues[idx]; 1672 } 1673 flex_name[idx] = '\0'; 1674 xtr_type = iavf_lookup_proto_xtr_type(flex_name); 1675 if (xtr_type < 0) 1676 return -1; 1677 1678 queues += idx; 1679 1680 while (isblank(*queues) || *queues == ',' || *queues == ']') 1681 queues++; 1682 1683 if (iavf_parse_queue_set(queue_start, xtr_type, devargs) < 0) 1684 return -1; 1685 } while (*queues != '\0'); 1686 1687 return 0; 1688 } 1689 1690 static int 1691 iavf_handle_proto_xtr_arg(__rte_unused const char *key, const char *value, 1692 void *extra_args) 1693 { 1694 struct iavf_devargs *devargs = extra_args; 1695 1696 if (!value || !extra_args) 1697 return -EINVAL; 1698 1699 if (iavf_parse_queue_proto_xtr(value, devargs) < 0) { 1700 PMD_DRV_LOG(ERR, "the proto_xtr's parameter is wrong : '%s'", 1701 value); 1702 return -1; 1703 } 1704 1705 return 0; 1706 } 1707 1708 static int iavf_parse_devargs(struct rte_eth_dev *dev) 1709 { 1710 struct iavf_adapter *ad = 1711 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1712 struct rte_devargs *devargs = dev->device->devargs; 1713 struct rte_kvargs *kvlist; 1714 int ret; 1715 1716 if (!devargs) 1717 return 0; 1718 1719 kvlist = rte_kvargs_parse(devargs->args, iavf_valid_args); 1720 if (!kvlist) { 1721 PMD_INIT_LOG(ERR, "invalid kvargs key\n"); 1722 return -EINVAL; 1723 } 1724 1725 ad->devargs.proto_xtr_dflt = IAVF_PROTO_XTR_NONE; 1726 memset(ad->devargs.proto_xtr, IAVF_PROTO_XTR_NONE, 1727 sizeof(ad->devargs.proto_xtr)); 1728 1729 ret = rte_kvargs_process(kvlist, IAVF_PROTO_XTR_ARG, 1730 &iavf_handle_proto_xtr_arg, &ad->devargs); 1731 if (ret) 1732 goto bail; 1733 1734 bail: 1735 rte_kvargs_free(kvlist); 1736 return ret; 1737 } 1738 1739 static void 1740 iavf_init_proto_xtr(struct rte_eth_dev *dev) 1741 { 1742 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1743 struct iavf_adapter *ad = 1744 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1745 const struct iavf_proto_xtr_ol *xtr_ol; 1746 bool proto_xtr_enable = false; 1747 int offset; 1748 uint16_t i; 1749 1750 vf->proto_xtr = rte_zmalloc("vf proto xtr", 1751 vf->vsi_res->num_queue_pairs, 0); 1752 if (unlikely(!(vf->proto_xtr))) { 1753 PMD_DRV_LOG(ERR, "no memory for setting up proto_xtr's table"); 1754 return; 1755 } 1756 1757 for (i = 0; i < vf->vsi_res->num_queue_pairs; i++) { 1758 vf->proto_xtr[i] = ad->devargs.proto_xtr[i] != 1759 IAVF_PROTO_XTR_NONE ? 1760 ad->devargs.proto_xtr[i] : 1761 ad->devargs.proto_xtr_dflt; 1762 1763 if (vf->proto_xtr[i] != IAVF_PROTO_XTR_NONE) { 1764 uint8_t type = vf->proto_xtr[i]; 1765 1766 iavf_proto_xtr_params[type].required = true; 1767 proto_xtr_enable = true; 1768 } 1769 } 1770 1771 if (likely(!proto_xtr_enable)) 1772 return; 1773 1774 offset = rte_mbuf_dynfield_register(&iavf_proto_xtr_metadata_param); 1775 if (unlikely(offset == -1)) { 1776 PMD_DRV_LOG(ERR, 1777 "failed to extract protocol metadata, error %d", 1778 -rte_errno); 1779 return; 1780 } 1781 1782 PMD_DRV_LOG(DEBUG, 1783 "proto_xtr metadata offset in mbuf is : %d", 1784 offset); 1785 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = offset; 1786 1787 for (i = 0; i < RTE_DIM(iavf_proto_xtr_params); i++) { 1788 xtr_ol = &iavf_proto_xtr_params[i]; 1789 1790 uint8_t rxdid = iavf_proto_xtr_type_to_rxdid((uint8_t)i); 1791 1792 if (!xtr_ol->required) 1793 continue; 1794 1795 if (!(vf->supported_rxdid & BIT(rxdid))) { 1796 PMD_DRV_LOG(ERR, 1797 "rxdid[%u] is not supported in hardware", 1798 rxdid); 1799 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 1800 break; 1801 } 1802 1803 offset = rte_mbuf_dynflag_register(&xtr_ol->param); 1804 if (unlikely(offset == -1)) { 1805 PMD_DRV_LOG(ERR, 1806 "failed to register proto_xtr offload '%s', error %d", 1807 xtr_ol->param.name, -rte_errno); 1808 1809 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 1810 break; 1811 } 1812 1813 PMD_DRV_LOG(DEBUG, 1814 "proto_xtr offload '%s' offset in mbuf is : %d", 1815 xtr_ol->param.name, offset); 1816 *xtr_ol->ol_flag = 1ULL << offset; 1817 } 1818 } 1819 1820 static int 1821 iavf_init_vf(struct rte_eth_dev *dev) 1822 { 1823 int err, bufsz; 1824 struct iavf_adapter *adapter = 1825 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1826 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1827 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1828 1829 err = iavf_parse_devargs(dev); 1830 if (err) { 1831 PMD_INIT_LOG(ERR, "Failed to parse devargs"); 1832 goto err; 1833 } 1834 1835 err = iavf_set_mac_type(hw); 1836 if (err) { 1837 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err); 1838 goto err; 1839 } 1840 1841 err = iavf_check_vf_reset_done(hw); 1842 if (err) { 1843 PMD_INIT_LOG(ERR, "VF is still resetting"); 1844 goto err; 1845 } 1846 1847 iavf_init_adminq_parameter(hw); 1848 err = iavf_init_adminq(hw); 1849 if (err) { 1850 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err); 1851 goto err; 1852 } 1853 1854 vf->aq_resp = rte_zmalloc("vf_aq_resp", IAVF_AQ_BUF_SZ, 0); 1855 if (!vf->aq_resp) { 1856 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); 1857 goto err_aq; 1858 } 1859 if (iavf_check_api_version(adapter) != 0) { 1860 PMD_INIT_LOG(ERR, "check_api version failed"); 1861 goto err_api; 1862 } 1863 1864 bufsz = sizeof(struct virtchnl_vf_resource) + 1865 (IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); 1866 vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); 1867 if (!vf->vf_res) { 1868 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 1869 goto err_api; 1870 } 1871 if (iavf_get_vf_resource(adapter) != 0) { 1872 PMD_INIT_LOG(ERR, "iavf_get_vf_config failed"); 1873 goto err_alloc; 1874 } 1875 /* Allocate memort for RSS info */ 1876 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 1877 vf->rss_key = rte_zmalloc("rss_key", 1878 vf->vf_res->rss_key_size, 0); 1879 if (!vf->rss_key) { 1880 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 1881 goto err_rss; 1882 } 1883 vf->rss_lut = rte_zmalloc("rss_lut", 1884 vf->vf_res->rss_lut_size, 0); 1885 if (!vf->rss_lut) { 1886 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 1887 goto err_rss; 1888 } 1889 } 1890 1891 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 1892 if (iavf_get_supported_rxdid(adapter) != 0) { 1893 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 1894 goto err_rss; 1895 } 1896 } 1897 1898 iavf_init_proto_xtr(dev); 1899 1900 return 0; 1901 err_rss: 1902 rte_free(vf->rss_key); 1903 rte_free(vf->rss_lut); 1904 err_alloc: 1905 rte_free(vf->vf_res); 1906 vf->vsi_res = NULL; 1907 err_api: 1908 rte_free(vf->aq_resp); 1909 err_aq: 1910 iavf_shutdown_adminq(hw); 1911 err: 1912 return -1; 1913 } 1914 1915 /* Enable default admin queue interrupt setting */ 1916 static inline void 1917 iavf_enable_irq0(struct iavf_hw *hw) 1918 { 1919 /* Enable admin queue interrupt trigger */ 1920 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 1921 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 1922 1923 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1924 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 1925 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1926 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1927 1928 IAVF_WRITE_FLUSH(hw); 1929 } 1930 1931 static inline void 1932 iavf_disable_irq0(struct iavf_hw *hw) 1933 { 1934 /* Disable all interrupt types */ 1935 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 0); 1936 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1937 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1938 IAVF_WRITE_FLUSH(hw); 1939 } 1940 1941 static void 1942 iavf_dev_interrupt_handler(void *param) 1943 { 1944 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1945 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1946 1947 iavf_disable_irq0(hw); 1948 1949 iavf_handle_virtchnl_msg(dev); 1950 1951 iavf_enable_irq0(hw); 1952 } 1953 1954 static int 1955 iavf_dev_filter_ctrl(struct rte_eth_dev *dev, 1956 enum rte_filter_type filter_type, 1957 enum rte_filter_op filter_op, 1958 void *arg) 1959 { 1960 int ret = 0; 1961 1962 if (!dev) 1963 return -EINVAL; 1964 1965 switch (filter_type) { 1966 case RTE_ETH_FILTER_GENERIC: 1967 if (filter_op != RTE_ETH_FILTER_GET) 1968 return -EINVAL; 1969 *(const void **)arg = &iavf_flow_ops; 1970 break; 1971 default: 1972 PMD_DRV_LOG(WARNING, "Filter type (%d) not supported", 1973 filter_type); 1974 ret = -EINVAL; 1975 break; 1976 } 1977 1978 return ret; 1979 } 1980 1981 1982 static int 1983 iavf_dev_init(struct rte_eth_dev *eth_dev) 1984 { 1985 struct iavf_adapter *adapter = 1986 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 1987 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1988 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1989 int ret = 0; 1990 1991 PMD_INIT_FUNC_TRACE(); 1992 1993 /* assign ops func pointer */ 1994 eth_dev->dev_ops = &iavf_eth_dev_ops; 1995 eth_dev->rx_queue_count = iavf_dev_rxq_count; 1996 eth_dev->rx_descriptor_status = iavf_dev_rx_desc_status; 1997 eth_dev->tx_descriptor_status = iavf_dev_tx_desc_status; 1998 eth_dev->rx_pkt_burst = &iavf_recv_pkts; 1999 eth_dev->tx_pkt_burst = &iavf_xmit_pkts; 2000 eth_dev->tx_pkt_prepare = &iavf_prep_pkts; 2001 2002 /* For secondary processes, we don't initialise any further as primary 2003 * has already done this work. Only check if we need a different RX 2004 * and TX function. 2005 */ 2006 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2007 iavf_set_rx_function(eth_dev); 2008 iavf_set_tx_function(eth_dev); 2009 return 0; 2010 } 2011 rte_eth_copy_pci_info(eth_dev, pci_dev); 2012 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 2013 2014 hw->vendor_id = pci_dev->id.vendor_id; 2015 hw->device_id = pci_dev->id.device_id; 2016 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 2017 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 2018 hw->bus.bus_id = pci_dev->addr.bus; 2019 hw->bus.device = pci_dev->addr.devid; 2020 hw->bus.func = pci_dev->addr.function; 2021 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 2022 hw->back = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 2023 adapter->eth_dev = eth_dev; 2024 adapter->stopped = 1; 2025 2026 if (iavf_init_vf(eth_dev) != 0) { 2027 PMD_INIT_LOG(ERR, "Init vf failed"); 2028 return -1; 2029 } 2030 2031 /* set default ptype table */ 2032 adapter->ptype_tbl = iavf_get_default_ptype_table(); 2033 2034 /* copy mac addr */ 2035 eth_dev->data->mac_addrs = rte_zmalloc( 2036 "iavf_mac", RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX, 0); 2037 if (!eth_dev->data->mac_addrs) { 2038 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to" 2039 " store MAC addresses", 2040 RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX); 2041 return -ENOMEM; 2042 } 2043 /* If the MAC address is not configured by host, 2044 * generate a random one. 2045 */ 2046 if (!rte_is_valid_assigned_ether_addr( 2047 (struct rte_ether_addr *)hw->mac.addr)) 2048 rte_eth_random_addr(hw->mac.addr); 2049 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 2050 ð_dev->data->mac_addrs[0]); 2051 2052 /* register callback func to eal lib */ 2053 rte_intr_callback_register(&pci_dev->intr_handle, 2054 iavf_dev_interrupt_handler, 2055 (void *)eth_dev); 2056 2057 /* enable uio intr after callback register */ 2058 rte_intr_enable(&pci_dev->intr_handle); 2059 2060 /* configure and enable device interrupt */ 2061 iavf_enable_irq0(hw); 2062 2063 ret = iavf_flow_init(adapter); 2064 if (ret) { 2065 PMD_INIT_LOG(ERR, "Failed to initialize flow"); 2066 return ret; 2067 } 2068 2069 /* Set hena = 0 to ask PF to cleanup all existing RSS. */ 2070 ret = iavf_set_hena(adapter, 0); 2071 if (ret) { 2072 PMD_DRV_LOG(ERR, "fail to disable default PF RSS"); 2073 return ret; 2074 } 2075 2076 return 0; 2077 } 2078 2079 static int 2080 iavf_dev_close(struct rte_eth_dev *dev) 2081 { 2082 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2083 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2084 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2085 struct iavf_adapter *adapter = 2086 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2087 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2088 int ret; 2089 2090 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2091 return 0; 2092 2093 ret = iavf_dev_stop(dev); 2094 2095 iavf_flow_flush(dev, NULL); 2096 iavf_flow_uninit(adapter); 2097 2098 /* 2099 * disable promiscuous mode before reset vf 2100 * it is a workaround solution when work with kernel driver 2101 * and it is not the normal way 2102 */ 2103 if (vf->promisc_unicast_enabled || vf->promisc_multicast_enabled) 2104 iavf_config_promisc(adapter, false, false); 2105 2106 iavf_shutdown_adminq(hw); 2107 /* disable uio intr before callback unregister */ 2108 rte_intr_disable(intr_handle); 2109 2110 /* unregister callback func from eal lib */ 2111 rte_intr_callback_unregister(intr_handle, 2112 iavf_dev_interrupt_handler, dev); 2113 iavf_disable_irq0(hw); 2114 2115 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2116 if (vf->rss_lut) { 2117 rte_free(vf->rss_lut); 2118 vf->rss_lut = NULL; 2119 } 2120 if (vf->rss_key) { 2121 rte_free(vf->rss_key); 2122 vf->rss_key = NULL; 2123 } 2124 } 2125 2126 rte_free(vf->vf_res); 2127 vf->vsi_res = NULL; 2128 vf->vf_res = NULL; 2129 2130 rte_free(vf->aq_resp); 2131 vf->aq_resp = NULL; 2132 2133 vf->vf_reset = false; 2134 2135 return ret; 2136 } 2137 2138 static int 2139 iavf_dev_uninit(struct rte_eth_dev *dev) 2140 { 2141 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2142 return -EPERM; 2143 2144 iavf_dev_close(dev); 2145 2146 return 0; 2147 } 2148 2149 /* 2150 * Reset VF device only to re-initialize resources in PMD layer 2151 */ 2152 static int 2153 iavf_dev_reset(struct rte_eth_dev *dev) 2154 { 2155 int ret; 2156 2157 ret = iavf_dev_uninit(dev); 2158 if (ret) 2159 return ret; 2160 2161 return iavf_dev_init(dev); 2162 } 2163 2164 static int 2165 iavf_dcf_cap_check_handler(__rte_unused const char *key, 2166 const char *value, __rte_unused void *opaque) 2167 { 2168 if (strcmp(value, "dcf")) 2169 return -1; 2170 2171 return 0; 2172 } 2173 2174 static int 2175 iavf_dcf_cap_selected(struct rte_devargs *devargs) 2176 { 2177 struct rte_kvargs *kvlist; 2178 const char *key = "cap"; 2179 int ret = 0; 2180 2181 if (devargs == NULL) 2182 return 0; 2183 2184 kvlist = rte_kvargs_parse(devargs->args, NULL); 2185 if (kvlist == NULL) 2186 return 0; 2187 2188 if (!rte_kvargs_count(kvlist, key)) 2189 goto exit; 2190 2191 /* dcf capability selected when there's a key-value pair: cap=dcf */ 2192 if (rte_kvargs_process(kvlist, key, 2193 iavf_dcf_cap_check_handler, NULL) < 0) 2194 goto exit; 2195 2196 ret = 1; 2197 2198 exit: 2199 rte_kvargs_free(kvlist); 2200 return ret; 2201 } 2202 2203 static int eth_iavf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2204 struct rte_pci_device *pci_dev) 2205 { 2206 if (iavf_dcf_cap_selected(pci_dev->device.devargs)) 2207 return 1; 2208 2209 return rte_eth_dev_pci_generic_probe(pci_dev, 2210 sizeof(struct iavf_adapter), iavf_dev_init); 2211 } 2212 2213 static int eth_iavf_pci_remove(struct rte_pci_device *pci_dev) 2214 { 2215 return rte_eth_dev_pci_generic_remove(pci_dev, iavf_dev_uninit); 2216 } 2217 2218 /* Adaptive virtual function driver struct */ 2219 static struct rte_pci_driver rte_iavf_pmd = { 2220 .id_table = pci_id_iavf_map, 2221 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 2222 .probe = eth_iavf_pci_probe, 2223 .remove = eth_iavf_pci_remove, 2224 }; 2225 2226 RTE_PMD_REGISTER_PCI(net_iavf, rte_iavf_pmd); 2227 RTE_PMD_REGISTER_PCI_TABLE(net_iavf, pci_id_iavf_map); 2228 RTE_PMD_REGISTER_KMOD_DEP(net_iavf, "* igb_uio | vfio-pci"); 2229 RTE_PMD_REGISTER_PARAM_STRING(net_iavf, "cap=dcf"); 2230 RTE_LOG_REGISTER(iavf_logtype_init, pmd.net.iavf.init, NOTICE); 2231 RTE_LOG_REGISTER(iavf_logtype_driver, pmd.net.iavf.driver, NOTICE); 2232 #ifdef RTE_LIBRTE_IAVF_DEBUG_RX 2233 RTE_LOG_REGISTER(iavf_logtype_rx, pmd.net.iavf.rx, DEBUG); 2234 #endif 2235 #ifdef RTE_LIBRTE_IAVF_DEBUG_TX 2236 RTE_LOG_REGISTER(iavf_logtype_tx, pmd.net.iavf.tx, DEBUG); 2237 #endif 2238 #ifdef RTE_LIBRTE_IAVF_DEBUG_TX_FREE 2239 RTE_LOG_REGISTER(iavf_logtype_tx_free, pmd.net.iavf.tx_free, DEBUG); 2240 #endif 2241