1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_interrupts.h> 17 #include <rte_debug.h> 18 #include <rte_pci.h> 19 #include <rte_atomic.h> 20 #include <rte_eal.h> 21 #include <rte_ether.h> 22 #include <ethdev_driver.h> 23 #include <ethdev_pci.h> 24 #include <rte_malloc.h> 25 #include <rte_memzone.h> 26 #include <rte_dev.h> 27 28 #include "iavf.h" 29 #include "iavf_rxtx.h" 30 #include "iavf_generic_flow.h" 31 #include "rte_pmd_iavf.h" 32 33 /* devargs */ 34 #define IAVF_PROTO_XTR_ARG "proto_xtr" 35 36 static const char * const iavf_valid_args[] = { 37 IAVF_PROTO_XTR_ARG, 38 NULL 39 }; 40 41 static const struct rte_mbuf_dynfield iavf_proto_xtr_metadata_param = { 42 .name = "intel_pmd_dynfield_proto_xtr_metadata", 43 .size = sizeof(uint32_t), 44 .align = __alignof__(uint32_t), 45 .flags = 0, 46 }; 47 48 struct iavf_proto_xtr_ol { 49 const struct rte_mbuf_dynflag param; 50 uint64_t *ol_flag; 51 bool required; 52 }; 53 54 static struct iavf_proto_xtr_ol iavf_proto_xtr_params[] = { 55 [IAVF_PROTO_XTR_VLAN] = { 56 .param = { .name = "intel_pmd_dynflag_proto_xtr_vlan" }, 57 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_vlan_mask }, 58 [IAVF_PROTO_XTR_IPV4] = { 59 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv4" }, 60 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask }, 61 [IAVF_PROTO_XTR_IPV6] = { 62 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6" }, 63 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask }, 64 [IAVF_PROTO_XTR_IPV6_FLOW] = { 65 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6_flow" }, 66 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask }, 67 [IAVF_PROTO_XTR_TCP] = { 68 .param = { .name = "intel_pmd_dynflag_proto_xtr_tcp" }, 69 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_tcp_mask }, 70 [IAVF_PROTO_XTR_IP_OFFSET] = { 71 .param = { .name = "intel_pmd_dynflag_proto_xtr_ip_offset" }, 72 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask }, 73 }; 74 75 static int iavf_dev_configure(struct rte_eth_dev *dev); 76 static int iavf_dev_start(struct rte_eth_dev *dev); 77 static int iavf_dev_stop(struct rte_eth_dev *dev); 78 static int iavf_dev_close(struct rte_eth_dev *dev); 79 static int iavf_dev_reset(struct rte_eth_dev *dev); 80 static int iavf_dev_info_get(struct rte_eth_dev *dev, 81 struct rte_eth_dev_info *dev_info); 82 static const uint32_t *iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev); 83 static int iavf_dev_stats_get(struct rte_eth_dev *dev, 84 struct rte_eth_stats *stats); 85 static int iavf_dev_stats_reset(struct rte_eth_dev *dev); 86 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 87 struct rte_eth_xstat *xstats, unsigned int n); 88 static int iavf_dev_xstats_get_names(struct rte_eth_dev *dev, 89 struct rte_eth_xstat_name *xstats_names, 90 unsigned int limit); 91 static int iavf_dev_promiscuous_enable(struct rte_eth_dev *dev); 92 static int iavf_dev_promiscuous_disable(struct rte_eth_dev *dev); 93 static int iavf_dev_allmulticast_enable(struct rte_eth_dev *dev); 94 static int iavf_dev_allmulticast_disable(struct rte_eth_dev *dev); 95 static int iavf_dev_add_mac_addr(struct rte_eth_dev *dev, 96 struct rte_ether_addr *addr, 97 uint32_t index, 98 uint32_t pool); 99 static void iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); 100 static int iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, 101 uint16_t vlan_id, int on); 102 static int iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 103 static int iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 104 struct rte_eth_rss_reta_entry64 *reta_conf, 105 uint16_t reta_size); 106 static int iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 107 struct rte_eth_rss_reta_entry64 *reta_conf, 108 uint16_t reta_size); 109 static int iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 110 struct rte_eth_rss_conf *rss_conf); 111 static int iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 112 struct rte_eth_rss_conf *rss_conf); 113 static int iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 114 static int iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 115 struct rte_ether_addr *mac_addr); 116 static int iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 117 uint16_t queue_id); 118 static int iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 119 uint16_t queue_id); 120 static int iavf_dev_flow_ops_get(struct rte_eth_dev *dev, 121 const struct rte_flow_ops **ops); 122 static int iavf_set_mc_addr_list(struct rte_eth_dev *dev, 123 struct rte_ether_addr *mc_addrs, 124 uint32_t mc_addrs_num); 125 static int iavf_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg); 126 127 static const struct rte_pci_id pci_id_iavf_map[] = { 128 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_ADAPTIVE_VF) }, 129 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_VF) }, 130 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_VF_HV) }, 131 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_X722_VF) }, 132 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_X722_A0_VF) }, 133 { .vendor_id = 0, /* sentinel */ }, 134 }; 135 136 struct rte_iavf_xstats_name_off { 137 char name[RTE_ETH_XSTATS_NAME_SIZE]; 138 unsigned int offset; 139 }; 140 141 static const struct rte_iavf_xstats_name_off rte_iavf_stats_strings[] = { 142 {"rx_bytes", offsetof(struct iavf_eth_stats, rx_bytes)}, 143 {"rx_unicast_packets", offsetof(struct iavf_eth_stats, rx_unicast)}, 144 {"rx_multicast_packets", offsetof(struct iavf_eth_stats, rx_multicast)}, 145 {"rx_broadcast_packets", offsetof(struct iavf_eth_stats, rx_broadcast)}, 146 {"rx_dropped_packets", offsetof(struct iavf_eth_stats, rx_discards)}, 147 {"rx_unknown_protocol_packets", offsetof(struct iavf_eth_stats, 148 rx_unknown_protocol)}, 149 {"tx_bytes", offsetof(struct iavf_eth_stats, tx_bytes)}, 150 {"tx_unicast_packets", offsetof(struct iavf_eth_stats, tx_unicast)}, 151 {"tx_multicast_packets", offsetof(struct iavf_eth_stats, tx_multicast)}, 152 {"tx_broadcast_packets", offsetof(struct iavf_eth_stats, tx_broadcast)}, 153 {"tx_dropped_packets", offsetof(struct iavf_eth_stats, tx_discards)}, 154 {"tx_error_packets", offsetof(struct iavf_eth_stats, tx_errors)}, 155 }; 156 157 #define IAVF_NB_XSTATS (sizeof(rte_iavf_stats_strings) / \ 158 sizeof(rte_iavf_stats_strings[0])) 159 160 static const struct eth_dev_ops iavf_eth_dev_ops = { 161 .dev_configure = iavf_dev_configure, 162 .dev_start = iavf_dev_start, 163 .dev_stop = iavf_dev_stop, 164 .dev_close = iavf_dev_close, 165 .dev_reset = iavf_dev_reset, 166 .dev_infos_get = iavf_dev_info_get, 167 .dev_supported_ptypes_get = iavf_dev_supported_ptypes_get, 168 .link_update = iavf_dev_link_update, 169 .stats_get = iavf_dev_stats_get, 170 .stats_reset = iavf_dev_stats_reset, 171 .xstats_get = iavf_dev_xstats_get, 172 .xstats_get_names = iavf_dev_xstats_get_names, 173 .xstats_reset = iavf_dev_stats_reset, 174 .promiscuous_enable = iavf_dev_promiscuous_enable, 175 .promiscuous_disable = iavf_dev_promiscuous_disable, 176 .allmulticast_enable = iavf_dev_allmulticast_enable, 177 .allmulticast_disable = iavf_dev_allmulticast_disable, 178 .mac_addr_add = iavf_dev_add_mac_addr, 179 .mac_addr_remove = iavf_dev_del_mac_addr, 180 .set_mc_addr_list = iavf_set_mc_addr_list, 181 .vlan_filter_set = iavf_dev_vlan_filter_set, 182 .vlan_offload_set = iavf_dev_vlan_offload_set, 183 .rx_queue_start = iavf_dev_rx_queue_start, 184 .rx_queue_stop = iavf_dev_rx_queue_stop, 185 .tx_queue_start = iavf_dev_tx_queue_start, 186 .tx_queue_stop = iavf_dev_tx_queue_stop, 187 .rx_queue_setup = iavf_dev_rx_queue_setup, 188 .rx_queue_release = iavf_dev_rx_queue_release, 189 .tx_queue_setup = iavf_dev_tx_queue_setup, 190 .tx_queue_release = iavf_dev_tx_queue_release, 191 .mac_addr_set = iavf_dev_set_default_mac_addr, 192 .reta_update = iavf_dev_rss_reta_update, 193 .reta_query = iavf_dev_rss_reta_query, 194 .rss_hash_update = iavf_dev_rss_hash_update, 195 .rss_hash_conf_get = iavf_dev_rss_hash_conf_get, 196 .rxq_info_get = iavf_dev_rxq_info_get, 197 .txq_info_get = iavf_dev_txq_info_get, 198 .mtu_set = iavf_dev_mtu_set, 199 .rx_queue_intr_enable = iavf_dev_rx_queue_intr_enable, 200 .rx_queue_intr_disable = iavf_dev_rx_queue_intr_disable, 201 .flow_ops_get = iavf_dev_flow_ops_get, 202 .tx_done_cleanup = iavf_dev_tx_done_cleanup, 203 .get_monitor_addr = iavf_get_monitor_addr, 204 .tm_ops_get = iavf_tm_ops_get, 205 }; 206 207 static int 208 iavf_tm_ops_get(struct rte_eth_dev *dev __rte_unused, 209 void *arg) 210 { 211 if (!arg) 212 return -EINVAL; 213 214 *(const void **)arg = &iavf_tm_ops; 215 216 return 0; 217 } 218 219 static int 220 iavf_set_mc_addr_list(struct rte_eth_dev *dev, 221 struct rte_ether_addr *mc_addrs, 222 uint32_t mc_addrs_num) 223 { 224 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 225 struct iavf_adapter *adapter = 226 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 227 int err, ret; 228 229 if (mc_addrs_num > IAVF_NUM_MACADDR_MAX) { 230 PMD_DRV_LOG(ERR, 231 "can't add more than a limited number (%u) of addresses.", 232 (uint32_t)IAVF_NUM_MACADDR_MAX); 233 return -EINVAL; 234 } 235 236 /* flush previous addresses */ 237 err = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 238 false); 239 if (err) 240 return err; 241 242 /* add new ones */ 243 err = iavf_add_del_mc_addr_list(adapter, mc_addrs, mc_addrs_num, true); 244 245 if (err) { 246 /* if adding mac address list fails, should add the previous 247 * addresses back. 248 */ 249 ret = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, 250 vf->mc_addrs_num, true); 251 if (ret) 252 return ret; 253 } else { 254 vf->mc_addrs_num = mc_addrs_num; 255 memcpy(vf->mc_addrs, 256 mc_addrs, mc_addrs_num * sizeof(*mc_addrs)); 257 } 258 259 return err; 260 } 261 262 static void 263 iavf_config_rss_hf(struct iavf_adapter *adapter, uint64_t rss_hf) 264 { 265 static const uint64_t map_hena_rss[] = { 266 /* IPv4 */ 267 [IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] = 268 ETH_RSS_NONFRAG_IPV4_UDP, 269 [IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] = 270 ETH_RSS_NONFRAG_IPV4_UDP, 271 [IAVF_FILTER_PCTYPE_NONF_IPV4_UDP] = 272 ETH_RSS_NONFRAG_IPV4_UDP, 273 [IAVF_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] = 274 ETH_RSS_NONFRAG_IPV4_TCP, 275 [IAVF_FILTER_PCTYPE_NONF_IPV4_TCP] = 276 ETH_RSS_NONFRAG_IPV4_TCP, 277 [IAVF_FILTER_PCTYPE_NONF_IPV4_SCTP] = 278 ETH_RSS_NONFRAG_IPV4_SCTP, 279 [IAVF_FILTER_PCTYPE_NONF_IPV4_OTHER] = 280 ETH_RSS_NONFRAG_IPV4_OTHER, 281 [IAVF_FILTER_PCTYPE_FRAG_IPV4] = ETH_RSS_FRAG_IPV4, 282 283 /* IPv6 */ 284 [IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] = 285 ETH_RSS_NONFRAG_IPV6_UDP, 286 [IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] = 287 ETH_RSS_NONFRAG_IPV6_UDP, 288 [IAVF_FILTER_PCTYPE_NONF_IPV6_UDP] = 289 ETH_RSS_NONFRAG_IPV6_UDP, 290 [IAVF_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] = 291 ETH_RSS_NONFRAG_IPV6_TCP, 292 [IAVF_FILTER_PCTYPE_NONF_IPV6_TCP] = 293 ETH_RSS_NONFRAG_IPV6_TCP, 294 [IAVF_FILTER_PCTYPE_NONF_IPV6_SCTP] = 295 ETH_RSS_NONFRAG_IPV6_SCTP, 296 [IAVF_FILTER_PCTYPE_NONF_IPV6_OTHER] = 297 ETH_RSS_NONFRAG_IPV6_OTHER, 298 [IAVF_FILTER_PCTYPE_FRAG_IPV6] = ETH_RSS_FRAG_IPV6, 299 300 /* L2 Payload */ 301 [IAVF_FILTER_PCTYPE_L2_PAYLOAD] = ETH_RSS_L2_PAYLOAD 302 }; 303 304 const uint64_t ipv4_rss = ETH_RSS_NONFRAG_IPV4_UDP | 305 ETH_RSS_NONFRAG_IPV4_TCP | 306 ETH_RSS_NONFRAG_IPV4_SCTP | 307 ETH_RSS_NONFRAG_IPV4_OTHER | 308 ETH_RSS_FRAG_IPV4; 309 310 const uint64_t ipv6_rss = ETH_RSS_NONFRAG_IPV6_UDP | 311 ETH_RSS_NONFRAG_IPV6_TCP | 312 ETH_RSS_NONFRAG_IPV6_SCTP | 313 ETH_RSS_NONFRAG_IPV6_OTHER | 314 ETH_RSS_FRAG_IPV6; 315 316 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 317 uint64_t caps = 0, hena = 0, valid_rss_hf = 0; 318 uint32_t i; 319 int ret; 320 321 ret = iavf_get_hena_caps(adapter, &caps); 322 if (ret) { 323 /** 324 * RSS offload type configuration is not a necessary feature 325 * for VF, so here just print a warning and return. 326 */ 327 PMD_DRV_LOG(WARNING, 328 "fail to get RSS offload type caps, ret: %d", ret); 329 return; 330 } 331 332 /** 333 * ETH_RSS_IPV4 and ETH_RSS_IPV6 can be considered as 2 334 * generalizations of all other IPv4 and IPv6 RSS types. 335 */ 336 if (rss_hf & ETH_RSS_IPV4) 337 rss_hf |= ipv4_rss; 338 339 if (rss_hf & ETH_RSS_IPV6) 340 rss_hf |= ipv6_rss; 341 342 RTE_BUILD_BUG_ON(RTE_DIM(map_hena_rss) > sizeof(uint64_t) * CHAR_BIT); 343 344 for (i = 0; i < RTE_DIM(map_hena_rss); i++) { 345 uint64_t bit = BIT_ULL(i); 346 347 if ((caps & bit) && (map_hena_rss[i] & rss_hf)) { 348 valid_rss_hf |= map_hena_rss[i]; 349 hena |= bit; 350 } 351 } 352 353 ret = iavf_set_hena(adapter, hena); 354 if (ret) { 355 /** 356 * RSS offload type configuration is not a necessary feature 357 * for VF, so here just print a warning and return. 358 */ 359 PMD_DRV_LOG(WARNING, 360 "fail to set RSS offload types, ret: %d", ret); 361 return; 362 } 363 364 if (valid_rss_hf & ipv4_rss) 365 valid_rss_hf |= rss_hf & ETH_RSS_IPV4; 366 367 if (valid_rss_hf & ipv6_rss) 368 valid_rss_hf |= rss_hf & ETH_RSS_IPV6; 369 370 if (rss_hf & ~valid_rss_hf) 371 PMD_DRV_LOG(WARNING, "Unsupported rss_hf 0x%" PRIx64, 372 rss_hf & ~valid_rss_hf); 373 374 vf->rss_hf = valid_rss_hf; 375 } 376 377 static int 378 iavf_init_rss(struct iavf_adapter *adapter) 379 { 380 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 381 struct rte_eth_rss_conf *rss_conf; 382 uint16_t i, j, nb_q; 383 int ret; 384 385 rss_conf = &adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf; 386 nb_q = RTE_MIN(adapter->eth_dev->data->nb_rx_queues, 387 vf->max_rss_qregion); 388 389 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 390 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 391 return -ENOTSUP; 392 } 393 394 /* configure RSS key */ 395 if (!rss_conf->rss_key) { 396 /* Calculate the default hash key */ 397 for (i = 0; i < vf->vf_res->rss_key_size; i++) 398 vf->rss_key[i] = (uint8_t)rte_rand(); 399 } else 400 rte_memcpy(vf->rss_key, rss_conf->rss_key, 401 RTE_MIN(rss_conf->rss_key_len, 402 vf->vf_res->rss_key_size)); 403 404 /* init RSS LUT table */ 405 for (i = 0, j = 0; i < vf->vf_res->rss_lut_size; i++, j++) { 406 if (j >= nb_q) 407 j = 0; 408 vf->rss_lut[i] = j; 409 } 410 /* send virtchnnl ops to configure rss*/ 411 ret = iavf_configure_rss_lut(adapter); 412 if (ret) 413 return ret; 414 ret = iavf_configure_rss_key(adapter); 415 if (ret) 416 return ret; 417 418 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF) { 419 /* Set RSS hash configuration based on rss_conf->rss_hf. */ 420 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 421 if (ret) { 422 PMD_DRV_LOG(ERR, "fail to set default RSS"); 423 return ret; 424 } 425 } else { 426 iavf_config_rss_hf(adapter, rss_conf->rss_hf); 427 } 428 429 return 0; 430 } 431 432 static int 433 iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num) 434 { 435 struct iavf_adapter *ad = 436 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 437 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 438 int ret; 439 440 ret = iavf_request_queues(ad, num); 441 if (ret) { 442 PMD_DRV_LOG(ERR, "request queues from PF failed"); 443 return ret; 444 } 445 PMD_DRV_LOG(INFO, "change queue pairs from %u to %u", 446 vf->vsi_res->num_queue_pairs, num); 447 448 ret = iavf_dev_reset(dev); 449 if (ret) { 450 PMD_DRV_LOG(ERR, "vf reset failed"); 451 return ret; 452 } 453 454 return 0; 455 } 456 457 static int 458 iavf_dev_vlan_insert_set(struct rte_eth_dev *dev) 459 { 460 struct iavf_adapter *adapter = 461 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 462 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 463 bool enable; 464 465 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2)) 466 return 0; 467 468 enable = !!(dev->data->dev_conf.txmode.offloads & 469 DEV_TX_OFFLOAD_VLAN_INSERT); 470 iavf_config_vlan_insert_v2(adapter, enable); 471 472 return 0; 473 } 474 475 static int 476 iavf_dev_init_vlan(struct rte_eth_dev *dev) 477 { 478 int err; 479 480 err = iavf_dev_vlan_offload_set(dev, 481 ETH_VLAN_STRIP_MASK | 482 ETH_QINQ_STRIP_MASK | 483 ETH_VLAN_FILTER_MASK | 484 ETH_VLAN_EXTEND_MASK); 485 if (err) { 486 PMD_DRV_LOG(ERR, "Failed to update vlan offload"); 487 return err; 488 } 489 490 err = iavf_dev_vlan_insert_set(dev); 491 if (err) 492 PMD_DRV_LOG(ERR, "Failed to update vlan insertion"); 493 494 return err; 495 } 496 497 static int 498 iavf_dev_configure(struct rte_eth_dev *dev) 499 { 500 struct iavf_adapter *ad = 501 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 502 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 503 uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 504 dev->data->nb_tx_queues); 505 int ret; 506 507 ad->rx_bulk_alloc_allowed = true; 508 /* Initialize to TRUE. If any of Rx queues doesn't meet the 509 * vector Rx/Tx preconditions, it will be reset. 510 */ 511 ad->rx_vec_allowed = true; 512 ad->tx_vec_allowed = true; 513 514 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 515 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 516 517 /* Large VF setting */ 518 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) { 519 if (!(vf->vf_res->vf_cap_flags & 520 VIRTCHNL_VF_LARGE_NUM_QPAIRS)) { 521 PMD_DRV_LOG(ERR, "large VF is not supported"); 522 return -1; 523 } 524 525 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) { 526 PMD_DRV_LOG(ERR, "queue pairs number cannot be larger than %u", 527 IAVF_MAX_NUM_QUEUES_LV); 528 return -1; 529 } 530 531 ret = iavf_queues_req_reset(dev, num_queue_pairs); 532 if (ret) 533 return ret; 534 535 ret = iavf_get_max_rss_queue_region(ad); 536 if (ret) { 537 PMD_INIT_LOG(ERR, "get max rss queue region failed"); 538 return ret; 539 } 540 541 vf->lv_enabled = true; 542 } else { 543 /* Check if large VF is already enabled. If so, disable and 544 * release redundant queue resource. 545 * Or check if enough queue pairs. If not, request them from PF. 546 */ 547 if (vf->lv_enabled || 548 num_queue_pairs > vf->vsi_res->num_queue_pairs) { 549 ret = iavf_queues_req_reset(dev, num_queue_pairs); 550 if (ret) 551 return ret; 552 553 vf->lv_enabled = false; 554 } 555 /* if large VF is not required, use default rss queue region */ 556 vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT; 557 } 558 559 ret = iavf_dev_init_vlan(dev); 560 if (ret) 561 PMD_DRV_LOG(ERR, "configure VLAN failed: %d", ret); 562 563 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 564 if (iavf_init_rss(ad) != 0) { 565 PMD_DRV_LOG(ERR, "configure rss failed"); 566 return -1; 567 } 568 } 569 return 0; 570 } 571 572 static int 573 iavf_init_rxq(struct rte_eth_dev *dev, struct iavf_rx_queue *rxq) 574 { 575 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 576 struct rte_eth_dev_data *dev_data = dev->data; 577 uint16_t buf_size, max_pkt_len; 578 579 buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; 580 581 /* Calculate the maximum packet length allowed */ 582 max_pkt_len = RTE_MIN((uint32_t) 583 rxq->rx_buf_len * IAVF_MAX_CHAINED_RX_BUFFERS, 584 dev->data->dev_conf.rxmode.max_rx_pkt_len); 585 586 /* Check if the jumbo frame and maximum packet length are set 587 * correctly. 588 */ 589 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 590 if (max_pkt_len <= IAVF_ETH_MAX_LEN || 591 max_pkt_len > IAVF_FRAME_SIZE_MAX) { 592 PMD_DRV_LOG(ERR, "maximum packet length must be " 593 "larger than %u and smaller than %u, " 594 "as jumbo frame is enabled", 595 (uint32_t)IAVF_ETH_MAX_LEN, 596 (uint32_t)IAVF_FRAME_SIZE_MAX); 597 return -EINVAL; 598 } 599 } else { 600 if (max_pkt_len < RTE_ETHER_MIN_LEN || 601 max_pkt_len > IAVF_ETH_MAX_LEN) { 602 PMD_DRV_LOG(ERR, "maximum packet length must be " 603 "larger than %u and smaller than %u, " 604 "as jumbo frame is disabled", 605 (uint32_t)RTE_ETHER_MIN_LEN, 606 (uint32_t)IAVF_ETH_MAX_LEN); 607 return -EINVAL; 608 } 609 } 610 611 rxq->max_pkt_len = max_pkt_len; 612 if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) || 613 rxq->max_pkt_len > buf_size) { 614 dev_data->scattered_rx = 1; 615 } 616 IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 617 IAVF_WRITE_FLUSH(hw); 618 619 return 0; 620 } 621 622 static int 623 iavf_init_queues(struct rte_eth_dev *dev) 624 { 625 struct iavf_rx_queue **rxq = 626 (struct iavf_rx_queue **)dev->data->rx_queues; 627 int i, ret = IAVF_SUCCESS; 628 629 for (i = 0; i < dev->data->nb_rx_queues; i++) { 630 if (!rxq[i] || !rxq[i]->q_set) 631 continue; 632 ret = iavf_init_rxq(dev, rxq[i]); 633 if (ret != IAVF_SUCCESS) 634 break; 635 } 636 /* set rx/tx function to vector/scatter/single-segment 637 * according to parameters 638 */ 639 iavf_set_rx_function(dev); 640 iavf_set_tx_function(dev); 641 642 return ret; 643 } 644 645 static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev, 646 struct rte_intr_handle *intr_handle) 647 { 648 struct iavf_adapter *adapter = 649 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 650 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 651 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 652 struct iavf_qv_map *qv_map; 653 uint16_t interval, i; 654 int vec; 655 656 if (rte_intr_cap_multiple(intr_handle) && 657 dev->data->dev_conf.intr_conf.rxq) { 658 if (rte_intr_efd_enable(intr_handle, dev->data->nb_rx_queues)) 659 return -1; 660 } 661 662 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 663 intr_handle->intr_vec = 664 rte_zmalloc("intr_vec", 665 dev->data->nb_rx_queues * sizeof(int), 0); 666 if (!intr_handle->intr_vec) { 667 PMD_DRV_LOG(ERR, "Failed to allocate %d rx intr_vec", 668 dev->data->nb_rx_queues); 669 return -1; 670 } 671 } 672 673 qv_map = rte_zmalloc("qv_map", 674 dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0); 675 if (!qv_map) { 676 PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map", 677 dev->data->nb_rx_queues); 678 return -1; 679 } 680 681 if (!dev->data->dev_conf.intr_conf.rxq || 682 !rte_intr_dp_is_en(intr_handle)) { 683 /* Rx interrupt disabled, Map interrupt only for writeback */ 684 vf->nb_msix = 1; 685 if (vf->vf_res->vf_cap_flags & 686 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 687 /* If WB_ON_ITR supports, enable it */ 688 vf->msix_base = IAVF_RX_VEC_START; 689 /* Set the ITR for index zero, to 2us to make sure that 690 * we leave time for aggregation to occur, but don't 691 * increase latency dramatically. 692 */ 693 IAVF_WRITE_REG(hw, 694 IAVF_VFINT_DYN_CTLN1(vf->msix_base - 1), 695 (0 << IAVF_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) | 696 IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK | 697 (2UL << IAVF_VFINT_DYN_CTLN1_INTERVAL_SHIFT)); 698 /* debug - check for success! the return value 699 * should be 2, offset is 0x2800 700 */ 701 /* IAVF_READ_REG(hw, IAVF_VFINT_ITRN1(0, 0)); */ 702 } else { 703 /* If no WB_ON_ITR offload flags, need to set 704 * interrupt for descriptor write back. 705 */ 706 vf->msix_base = IAVF_MISC_VEC_ID; 707 708 /* set ITR to max */ 709 interval = iavf_calc_itr_interval( 710 IAVF_QUEUE_ITR_INTERVAL_MAX); 711 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 712 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 713 (IAVF_ITR_INDEX_DEFAULT << 714 IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT) | 715 (interval << 716 IAVF_VFINT_DYN_CTL01_INTERVAL_SHIFT)); 717 } 718 IAVF_WRITE_FLUSH(hw); 719 /* map all queues to the same interrupt */ 720 for (i = 0; i < dev->data->nb_rx_queues; i++) { 721 qv_map[i].queue_id = i; 722 qv_map[i].vector_id = vf->msix_base; 723 } 724 vf->qv_map = qv_map; 725 } else { 726 if (!rte_intr_allow_others(intr_handle)) { 727 vf->nb_msix = 1; 728 vf->msix_base = IAVF_MISC_VEC_ID; 729 for (i = 0; i < dev->data->nb_rx_queues; i++) { 730 qv_map[i].queue_id = i; 731 qv_map[i].vector_id = vf->msix_base; 732 intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID; 733 } 734 vf->qv_map = qv_map; 735 PMD_DRV_LOG(DEBUG, 736 "vector %u are mapping to all Rx queues", 737 vf->msix_base); 738 } else { 739 /* If Rx interrupt is reuquired, and we can use 740 * multi interrupts, then the vec is from 1 741 */ 742 vf->nb_msix = RTE_MIN(intr_handle->nb_efd, 743 (uint16_t)(vf->vf_res->max_vectors - 1)); 744 vf->msix_base = IAVF_RX_VEC_START; 745 vec = IAVF_RX_VEC_START; 746 for (i = 0; i < dev->data->nb_rx_queues; i++) { 747 qv_map[i].queue_id = i; 748 qv_map[i].vector_id = vec; 749 intr_handle->intr_vec[i] = vec++; 750 if (vec >= vf->nb_msix + IAVF_RX_VEC_START) 751 vec = IAVF_RX_VEC_START; 752 } 753 vf->qv_map = qv_map; 754 PMD_DRV_LOG(DEBUG, 755 "%u vectors are mapping to %u Rx queues", 756 vf->nb_msix, dev->data->nb_rx_queues); 757 } 758 } 759 760 if (!vf->lv_enabled) { 761 if (iavf_config_irq_map(adapter)) { 762 PMD_DRV_LOG(ERR, "config interrupt mapping failed"); 763 return -1; 764 } 765 } else { 766 uint16_t num_qv_maps = dev->data->nb_rx_queues; 767 uint16_t index = 0; 768 769 while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) { 770 if (iavf_config_irq_map_lv(adapter, 771 IAVF_IRQ_MAP_NUM_PER_BUF, index)) { 772 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 773 return -1; 774 } 775 num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF; 776 index += IAVF_IRQ_MAP_NUM_PER_BUF; 777 } 778 779 if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) { 780 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 781 return -1; 782 } 783 } 784 return 0; 785 } 786 787 static int 788 iavf_start_queues(struct rte_eth_dev *dev) 789 { 790 struct iavf_rx_queue *rxq; 791 struct iavf_tx_queue *txq; 792 int i; 793 794 for (i = 0; i < dev->data->nb_tx_queues; i++) { 795 txq = dev->data->tx_queues[i]; 796 if (txq->tx_deferred_start) 797 continue; 798 if (iavf_dev_tx_queue_start(dev, i) != 0) { 799 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 800 return -1; 801 } 802 } 803 804 for (i = 0; i < dev->data->nb_rx_queues; i++) { 805 rxq = dev->data->rx_queues[i]; 806 if (rxq->rx_deferred_start) 807 continue; 808 if (iavf_dev_rx_queue_start(dev, i) != 0) { 809 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 810 return -1; 811 } 812 } 813 814 return 0; 815 } 816 817 static int 818 iavf_dev_start(struct rte_eth_dev *dev) 819 { 820 struct iavf_adapter *adapter = 821 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 822 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 823 struct rte_intr_handle *intr_handle = dev->intr_handle; 824 uint16_t num_queue_pairs; 825 uint16_t index = 0; 826 827 PMD_INIT_FUNC_TRACE(); 828 829 adapter->stopped = 0; 830 831 vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; 832 vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 833 dev->data->nb_tx_queues); 834 num_queue_pairs = vf->num_queue_pairs; 835 836 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 837 if (iavf_get_qos_cap(adapter)) { 838 PMD_INIT_LOG(ERR, "Failed to get qos capability"); 839 return -1; 840 } 841 842 if (iavf_init_queues(dev) != 0) { 843 PMD_DRV_LOG(ERR, "failed to do Queue init"); 844 return -1; 845 } 846 847 /* If needed, send configure queues msg multiple times to make the 848 * adminq buffer length smaller than the 4K limitation. 849 */ 850 while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) { 851 if (iavf_configure_queues(adapter, 852 IAVF_CFG_Q_NUM_PER_BUF, index) != 0) { 853 PMD_DRV_LOG(ERR, "configure queues failed"); 854 goto err_queue; 855 } 856 num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF; 857 index += IAVF_CFG_Q_NUM_PER_BUF; 858 } 859 860 if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) { 861 PMD_DRV_LOG(ERR, "configure queues failed"); 862 goto err_queue; 863 } 864 865 if (iavf_config_rx_queues_irqs(dev, intr_handle) != 0) { 866 PMD_DRV_LOG(ERR, "configure irq failed"); 867 goto err_queue; 868 } 869 /* re-enable intr again, because efd assign may change */ 870 if (dev->data->dev_conf.intr_conf.rxq != 0) { 871 rte_intr_disable(intr_handle); 872 rte_intr_enable(intr_handle); 873 } 874 875 /* Set all mac addrs */ 876 iavf_add_del_all_mac_addr(adapter, true); 877 878 /* Set all multicast addresses */ 879 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 880 true); 881 882 if (iavf_start_queues(dev) != 0) { 883 PMD_DRV_LOG(ERR, "enable queues failed"); 884 goto err_mac; 885 } 886 887 return 0; 888 889 err_mac: 890 iavf_add_del_all_mac_addr(adapter, false); 891 err_queue: 892 return -1; 893 } 894 895 static int 896 iavf_dev_stop(struct rte_eth_dev *dev) 897 { 898 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 899 struct iavf_adapter *adapter = 900 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 901 struct rte_intr_handle *intr_handle = dev->intr_handle; 902 903 PMD_INIT_FUNC_TRACE(); 904 905 if (adapter->stopped == 1) 906 return 0; 907 908 iavf_stop_queues(dev); 909 910 /* Disable the interrupt for Rx */ 911 rte_intr_efd_disable(intr_handle); 912 /* Rx interrupt vector mapping free */ 913 if (intr_handle->intr_vec) { 914 rte_free(intr_handle->intr_vec); 915 intr_handle->intr_vec = NULL; 916 } 917 918 /* remove all mac addrs */ 919 iavf_add_del_all_mac_addr(adapter, false); 920 921 /* remove all multicast addresses */ 922 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 923 false); 924 925 adapter->stopped = 1; 926 dev->data->dev_started = 0; 927 928 return 0; 929 } 930 931 static int 932 iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 933 { 934 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 935 936 dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV; 937 dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV; 938 dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN; 939 dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX; 940 dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD; 941 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 942 dev_info->hash_key_size = vf->vf_res->rss_key_size; 943 dev_info->reta_size = vf->vf_res->rss_lut_size; 944 dev_info->flow_type_rss_offloads = IAVF_RSS_OFFLOAD_ALL; 945 dev_info->max_mac_addrs = IAVF_NUM_MACADDR_MAX; 946 dev_info->rx_offload_capa = 947 DEV_RX_OFFLOAD_VLAN_STRIP | 948 DEV_RX_OFFLOAD_QINQ_STRIP | 949 DEV_RX_OFFLOAD_IPV4_CKSUM | 950 DEV_RX_OFFLOAD_UDP_CKSUM | 951 DEV_RX_OFFLOAD_TCP_CKSUM | 952 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 953 DEV_RX_OFFLOAD_SCATTER | 954 DEV_RX_OFFLOAD_JUMBO_FRAME | 955 DEV_RX_OFFLOAD_VLAN_FILTER | 956 DEV_RX_OFFLOAD_RSS_HASH; 957 958 dev_info->tx_offload_capa = 959 DEV_TX_OFFLOAD_VLAN_INSERT | 960 DEV_TX_OFFLOAD_QINQ_INSERT | 961 DEV_TX_OFFLOAD_IPV4_CKSUM | 962 DEV_TX_OFFLOAD_UDP_CKSUM | 963 DEV_TX_OFFLOAD_TCP_CKSUM | 964 DEV_TX_OFFLOAD_SCTP_CKSUM | 965 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 966 DEV_TX_OFFLOAD_TCP_TSO | 967 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 968 DEV_TX_OFFLOAD_GRE_TNL_TSO | 969 DEV_TX_OFFLOAD_IPIP_TNL_TSO | 970 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | 971 DEV_TX_OFFLOAD_MULTI_SEGS | 972 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 973 974 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_CRC) 975 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_KEEP_CRC; 976 977 dev_info->default_rxconf = (struct rte_eth_rxconf) { 978 .rx_free_thresh = IAVF_DEFAULT_RX_FREE_THRESH, 979 .rx_drop_en = 0, 980 .offloads = 0, 981 }; 982 983 dev_info->default_txconf = (struct rte_eth_txconf) { 984 .tx_free_thresh = IAVF_DEFAULT_TX_FREE_THRESH, 985 .tx_rs_thresh = IAVF_DEFAULT_TX_RS_THRESH, 986 .offloads = 0, 987 }; 988 989 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 990 .nb_max = IAVF_MAX_RING_DESC, 991 .nb_min = IAVF_MIN_RING_DESC, 992 .nb_align = IAVF_ALIGN_RING_DESC, 993 }; 994 995 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 996 .nb_max = IAVF_MAX_RING_DESC, 997 .nb_min = IAVF_MIN_RING_DESC, 998 .nb_align = IAVF_ALIGN_RING_DESC, 999 }; 1000 1001 return 0; 1002 } 1003 1004 static const uint32_t * 1005 iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) 1006 { 1007 static const uint32_t ptypes[] = { 1008 RTE_PTYPE_L2_ETHER, 1009 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 1010 RTE_PTYPE_L4_FRAG, 1011 RTE_PTYPE_L4_ICMP, 1012 RTE_PTYPE_L4_NONFRAG, 1013 RTE_PTYPE_L4_SCTP, 1014 RTE_PTYPE_L4_TCP, 1015 RTE_PTYPE_L4_UDP, 1016 RTE_PTYPE_UNKNOWN 1017 }; 1018 return ptypes; 1019 } 1020 1021 int 1022 iavf_dev_link_update(struct rte_eth_dev *dev, 1023 __rte_unused int wait_to_complete) 1024 { 1025 struct rte_eth_link new_link; 1026 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1027 1028 memset(&new_link, 0, sizeof(new_link)); 1029 1030 /* Only read status info stored in VF, and the info is updated 1031 * when receive LINK_CHANGE evnet from PF by Virtchnnl. 1032 */ 1033 switch (vf->link_speed) { 1034 case 10: 1035 new_link.link_speed = ETH_SPEED_NUM_10M; 1036 break; 1037 case 100: 1038 new_link.link_speed = ETH_SPEED_NUM_100M; 1039 break; 1040 case 1000: 1041 new_link.link_speed = ETH_SPEED_NUM_1G; 1042 break; 1043 case 10000: 1044 new_link.link_speed = ETH_SPEED_NUM_10G; 1045 break; 1046 case 20000: 1047 new_link.link_speed = ETH_SPEED_NUM_20G; 1048 break; 1049 case 25000: 1050 new_link.link_speed = ETH_SPEED_NUM_25G; 1051 break; 1052 case 40000: 1053 new_link.link_speed = ETH_SPEED_NUM_40G; 1054 break; 1055 case 50000: 1056 new_link.link_speed = ETH_SPEED_NUM_50G; 1057 break; 1058 case 100000: 1059 new_link.link_speed = ETH_SPEED_NUM_100G; 1060 break; 1061 default: 1062 new_link.link_speed = ETH_SPEED_NUM_NONE; 1063 break; 1064 } 1065 1066 new_link.link_duplex = ETH_LINK_FULL_DUPLEX; 1067 new_link.link_status = vf->link_up ? ETH_LINK_UP : 1068 ETH_LINK_DOWN; 1069 new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 1070 ETH_LINK_SPEED_FIXED); 1071 1072 return rte_eth_linkstatus_set(dev, &new_link); 1073 } 1074 1075 static int 1076 iavf_dev_promiscuous_enable(struct rte_eth_dev *dev) 1077 { 1078 struct iavf_adapter *adapter = 1079 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1080 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1081 1082 return iavf_config_promisc(adapter, 1083 true, vf->promisc_multicast_enabled); 1084 } 1085 1086 static int 1087 iavf_dev_promiscuous_disable(struct rte_eth_dev *dev) 1088 { 1089 struct iavf_adapter *adapter = 1090 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1091 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1092 1093 return iavf_config_promisc(adapter, 1094 false, vf->promisc_multicast_enabled); 1095 } 1096 1097 static int 1098 iavf_dev_allmulticast_enable(struct rte_eth_dev *dev) 1099 { 1100 struct iavf_adapter *adapter = 1101 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1102 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1103 1104 return iavf_config_promisc(adapter, 1105 vf->promisc_unicast_enabled, true); 1106 } 1107 1108 static int 1109 iavf_dev_allmulticast_disable(struct rte_eth_dev *dev) 1110 { 1111 struct iavf_adapter *adapter = 1112 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1113 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1114 1115 return iavf_config_promisc(adapter, 1116 vf->promisc_unicast_enabled, false); 1117 } 1118 1119 static int 1120 iavf_dev_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr, 1121 __rte_unused uint32_t index, 1122 __rte_unused uint32_t pool) 1123 { 1124 struct iavf_adapter *adapter = 1125 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1126 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1127 int err; 1128 1129 if (rte_is_zero_ether_addr(addr)) { 1130 PMD_DRV_LOG(ERR, "Invalid Ethernet Address"); 1131 return -EINVAL; 1132 } 1133 1134 err = iavf_add_del_eth_addr(adapter, addr, true, VIRTCHNL_ETHER_ADDR_EXTRA); 1135 if (err) { 1136 PMD_DRV_LOG(ERR, "fail to add MAC address"); 1137 return -EIO; 1138 } 1139 1140 vf->mac_num++; 1141 1142 return 0; 1143 } 1144 1145 static void 1146 iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) 1147 { 1148 struct iavf_adapter *adapter = 1149 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1150 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1151 struct rte_ether_addr *addr; 1152 int err; 1153 1154 addr = &dev->data->mac_addrs[index]; 1155 1156 err = iavf_add_del_eth_addr(adapter, addr, false, VIRTCHNL_ETHER_ADDR_EXTRA); 1157 if (err) 1158 PMD_DRV_LOG(ERR, "fail to delete MAC address"); 1159 1160 vf->mac_num--; 1161 } 1162 1163 static int 1164 iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1165 { 1166 struct iavf_adapter *adapter = 1167 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1168 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1169 int err; 1170 1171 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 1172 err = iavf_add_del_vlan_v2(adapter, vlan_id, on); 1173 if (err) 1174 return -EIO; 1175 return 0; 1176 } 1177 1178 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1179 return -ENOTSUP; 1180 1181 err = iavf_add_del_vlan(adapter, vlan_id, on); 1182 if (err) 1183 return -EIO; 1184 return 0; 1185 } 1186 1187 static void 1188 iavf_iterate_vlan_filters_v2(struct rte_eth_dev *dev, bool enable) 1189 { 1190 struct rte_vlan_filter_conf *vfc = &dev->data->vlan_filter_conf; 1191 struct iavf_adapter *adapter = 1192 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1193 uint32_t i, j; 1194 uint64_t ids; 1195 1196 for (i = 0; i < RTE_DIM(vfc->ids); i++) { 1197 if (vfc->ids[i] == 0) 1198 continue; 1199 1200 ids = vfc->ids[i]; 1201 for (j = 0; ids != 0 && j < 64; j++, ids >>= 1) { 1202 if (ids & 1) 1203 iavf_add_del_vlan_v2(adapter, 1204 64 * i + j, enable); 1205 } 1206 } 1207 } 1208 1209 static int 1210 iavf_dev_vlan_offload_set_v2(struct rte_eth_dev *dev, int mask) 1211 { 1212 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 1213 struct iavf_adapter *adapter = 1214 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1215 bool enable; 1216 int err; 1217 1218 if (mask & ETH_VLAN_FILTER_MASK) { 1219 enable = !!(rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER); 1220 1221 iavf_iterate_vlan_filters_v2(dev, enable); 1222 } 1223 1224 if (mask & ETH_VLAN_STRIP_MASK) { 1225 enable = !!(rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 1226 1227 err = iavf_config_vlan_strip_v2(adapter, enable); 1228 /* If not support, the stripping is already disabled by PF */ 1229 if (err == -ENOTSUP && !enable) 1230 err = 0; 1231 if (err) 1232 return -EIO; 1233 } 1234 1235 return 0; 1236 } 1237 1238 static int 1239 iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1240 { 1241 struct iavf_adapter *adapter = 1242 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1243 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1244 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 1245 int err; 1246 1247 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) 1248 return iavf_dev_vlan_offload_set_v2(dev, mask); 1249 1250 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1251 return -ENOTSUP; 1252 1253 /* Vlan stripping setting */ 1254 if (mask & ETH_VLAN_STRIP_MASK) { 1255 /* Enable or disable VLAN stripping */ 1256 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1257 err = iavf_enable_vlan_strip(adapter); 1258 else 1259 err = iavf_disable_vlan_strip(adapter); 1260 1261 if (err) 1262 return -EIO; 1263 } 1264 return 0; 1265 } 1266 1267 static int 1268 iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 1269 struct rte_eth_rss_reta_entry64 *reta_conf, 1270 uint16_t reta_size) 1271 { 1272 struct iavf_adapter *adapter = 1273 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1274 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1275 uint8_t *lut; 1276 uint16_t i, idx, shift; 1277 int ret; 1278 1279 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1280 return -ENOTSUP; 1281 1282 if (reta_size != vf->vf_res->rss_lut_size) { 1283 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1284 "(%d) doesn't match the number of hardware can " 1285 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1286 return -EINVAL; 1287 } 1288 1289 lut = rte_zmalloc("rss_lut", reta_size, 0); 1290 if (!lut) { 1291 PMD_DRV_LOG(ERR, "No memory can be allocated"); 1292 return -ENOMEM; 1293 } 1294 /* store the old lut table temporarily */ 1295 rte_memcpy(lut, vf->rss_lut, reta_size); 1296 1297 for (i = 0; i < reta_size; i++) { 1298 idx = i / RTE_RETA_GROUP_SIZE; 1299 shift = i % RTE_RETA_GROUP_SIZE; 1300 if (reta_conf[idx].mask & (1ULL << shift)) 1301 lut[i] = reta_conf[idx].reta[shift]; 1302 } 1303 1304 rte_memcpy(vf->rss_lut, lut, reta_size); 1305 /* send virtchnnl ops to configure rss*/ 1306 ret = iavf_configure_rss_lut(adapter); 1307 if (ret) /* revert back */ 1308 rte_memcpy(vf->rss_lut, lut, reta_size); 1309 rte_free(lut); 1310 1311 return ret; 1312 } 1313 1314 static int 1315 iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 1316 struct rte_eth_rss_reta_entry64 *reta_conf, 1317 uint16_t reta_size) 1318 { 1319 struct iavf_adapter *adapter = 1320 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1321 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1322 uint16_t i, idx, shift; 1323 1324 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1325 return -ENOTSUP; 1326 1327 if (reta_size != vf->vf_res->rss_lut_size) { 1328 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1329 "(%d) doesn't match the number of hardware can " 1330 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1331 return -EINVAL; 1332 } 1333 1334 for (i = 0; i < reta_size; i++) { 1335 idx = i / RTE_RETA_GROUP_SIZE; 1336 shift = i % RTE_RETA_GROUP_SIZE; 1337 if (reta_conf[idx].mask & (1ULL << shift)) 1338 reta_conf[idx].reta[shift] = vf->rss_lut[i]; 1339 } 1340 1341 return 0; 1342 } 1343 1344 static int 1345 iavf_set_rss_key(struct iavf_adapter *adapter, uint8_t *key, uint8_t key_len) 1346 { 1347 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1348 1349 /* HENA setting, it is enabled by default, no change */ 1350 if (!key || key_len == 0) { 1351 PMD_DRV_LOG(DEBUG, "No key to be configured"); 1352 return 0; 1353 } else if (key_len != vf->vf_res->rss_key_size) { 1354 PMD_DRV_LOG(ERR, "The size of hash key configured " 1355 "(%d) doesn't match the size of hardware can " 1356 "support (%d)", key_len, 1357 vf->vf_res->rss_key_size); 1358 return -EINVAL; 1359 } 1360 1361 rte_memcpy(vf->rss_key, key, key_len); 1362 1363 return iavf_configure_rss_key(adapter); 1364 } 1365 1366 static int 1367 iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 1368 struct rte_eth_rss_conf *rss_conf) 1369 { 1370 struct iavf_adapter *adapter = 1371 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1372 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1373 int ret; 1374 1375 adapter->eth_dev->data->dev_conf.rx_adv_conf.rss_conf = *rss_conf; 1376 1377 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1378 return -ENOTSUP; 1379 1380 /* Set hash key. */ 1381 ret = iavf_set_rss_key(adapter, rss_conf->rss_key, 1382 rss_conf->rss_key_len); 1383 if (ret) 1384 return ret; 1385 1386 if (rss_conf->rss_hf == 0) { 1387 vf->rss_hf = 0; 1388 ret = iavf_set_hena(adapter, 0); 1389 1390 /* It is a workaround, temporarily allow error to be returned 1391 * due to possible lack of PF handling for hena = 0. 1392 */ 1393 if (ret) 1394 PMD_DRV_LOG(WARNING, "fail to clean existing RSS, lack PF support"); 1395 return 0; 1396 } 1397 1398 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF) { 1399 /* Clear existing RSS. */ 1400 ret = iavf_set_hena(adapter, 0); 1401 1402 /* It is a workaround, temporarily allow error to be returned 1403 * due to possible lack of PF handling for hena = 0. 1404 */ 1405 if (ret) 1406 PMD_DRV_LOG(WARNING, "fail to clean existing RSS," 1407 "lack PF support"); 1408 1409 /* Set new RSS configuration. */ 1410 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 1411 if (ret) { 1412 PMD_DRV_LOG(ERR, "fail to set new RSS"); 1413 return ret; 1414 } 1415 } else { 1416 iavf_config_rss_hf(adapter, rss_conf->rss_hf); 1417 } 1418 1419 return 0; 1420 } 1421 1422 static int 1423 iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 1424 struct rte_eth_rss_conf *rss_conf) 1425 { 1426 struct iavf_adapter *adapter = 1427 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1428 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1429 1430 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1431 return -ENOTSUP; 1432 1433 rss_conf->rss_hf = vf->rss_hf; 1434 1435 if (!rss_conf->rss_key) 1436 return 0; 1437 1438 rss_conf->rss_key_len = vf->vf_res->rss_key_size; 1439 rte_memcpy(rss_conf->rss_key, vf->rss_key, rss_conf->rss_key_len); 1440 1441 return 0; 1442 } 1443 1444 static int 1445 iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1446 { 1447 uint32_t frame_size = mtu + IAVF_ETH_OVERHEAD; 1448 int ret = 0; 1449 1450 if (mtu < RTE_ETHER_MIN_MTU || frame_size > IAVF_FRAME_SIZE_MAX) 1451 return -EINVAL; 1452 1453 /* mtu setting is forbidden if port is start */ 1454 if (dev->data->dev_started) { 1455 PMD_DRV_LOG(ERR, "port must be stopped before configuration"); 1456 return -EBUSY; 1457 } 1458 1459 if (frame_size > IAVF_ETH_MAX_LEN) 1460 dev->data->dev_conf.rxmode.offloads |= 1461 DEV_RX_OFFLOAD_JUMBO_FRAME; 1462 else 1463 dev->data->dev_conf.rxmode.offloads &= 1464 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 1465 1466 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 1467 1468 return ret; 1469 } 1470 1471 static int 1472 iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 1473 struct rte_ether_addr *mac_addr) 1474 { 1475 struct iavf_adapter *adapter = 1476 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1477 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1478 struct rte_ether_addr *old_addr; 1479 int ret; 1480 1481 old_addr = (struct rte_ether_addr *)hw->mac.addr; 1482 1483 if (rte_is_same_ether_addr(old_addr, mac_addr)) 1484 return 0; 1485 1486 ret = iavf_add_del_eth_addr(adapter, old_addr, false, VIRTCHNL_ETHER_ADDR_PRIMARY); 1487 if (ret) 1488 PMD_DRV_LOG(ERR, "Fail to delete old MAC:" 1489 RTE_ETHER_ADDR_PRT_FMT, 1490 RTE_ETHER_ADDR_BYTES(old_addr)); 1491 1492 ret = iavf_add_del_eth_addr(adapter, mac_addr, true, VIRTCHNL_ETHER_ADDR_PRIMARY); 1493 if (ret) 1494 PMD_DRV_LOG(ERR, "Fail to add new MAC:" 1495 RTE_ETHER_ADDR_PRT_FMT, 1496 RTE_ETHER_ADDR_BYTES(mac_addr)); 1497 1498 if (ret) 1499 return -EIO; 1500 1501 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)hw->mac.addr); 1502 return 0; 1503 } 1504 1505 static void 1506 iavf_stat_update_48(uint64_t *offset, uint64_t *stat) 1507 { 1508 if (*stat >= *offset) 1509 *stat = *stat - *offset; 1510 else 1511 *stat = (uint64_t)((*stat + 1512 ((uint64_t)1 << IAVF_48_BIT_WIDTH)) - *offset); 1513 1514 *stat &= IAVF_48_BIT_MASK; 1515 } 1516 1517 static void 1518 iavf_stat_update_32(uint64_t *offset, uint64_t *stat) 1519 { 1520 if (*stat >= *offset) 1521 *stat = (uint64_t)(*stat - *offset); 1522 else 1523 *stat = (uint64_t)((*stat + 1524 ((uint64_t)1 << IAVF_32_BIT_WIDTH)) - *offset); 1525 } 1526 1527 static void 1528 iavf_update_stats(struct iavf_vsi *vsi, struct virtchnl_eth_stats *nes) 1529 { 1530 struct virtchnl_eth_stats *oes = &vsi->eth_stats_offset; 1531 1532 iavf_stat_update_48(&oes->rx_bytes, &nes->rx_bytes); 1533 iavf_stat_update_48(&oes->rx_unicast, &nes->rx_unicast); 1534 iavf_stat_update_48(&oes->rx_multicast, &nes->rx_multicast); 1535 iavf_stat_update_48(&oes->rx_broadcast, &nes->rx_broadcast); 1536 iavf_stat_update_32(&oes->rx_discards, &nes->rx_discards); 1537 iavf_stat_update_48(&oes->tx_bytes, &nes->tx_bytes); 1538 iavf_stat_update_48(&oes->tx_unicast, &nes->tx_unicast); 1539 iavf_stat_update_48(&oes->tx_multicast, &nes->tx_multicast); 1540 iavf_stat_update_48(&oes->tx_broadcast, &nes->tx_broadcast); 1541 iavf_stat_update_32(&oes->tx_errors, &nes->tx_errors); 1542 iavf_stat_update_32(&oes->tx_discards, &nes->tx_discards); 1543 } 1544 1545 static int 1546 iavf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1547 { 1548 struct iavf_adapter *adapter = 1549 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1550 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1551 struct iavf_vsi *vsi = &vf->vsi; 1552 struct virtchnl_eth_stats *pstats = NULL; 1553 int ret; 1554 1555 ret = iavf_query_stats(adapter, &pstats); 1556 if (ret == 0) { 1557 uint8_t crc_stats_len = (dev->data->dev_conf.rxmode.offloads & 1558 DEV_RX_OFFLOAD_KEEP_CRC) ? 0 : 1559 RTE_ETHER_CRC_LEN; 1560 iavf_update_stats(vsi, pstats); 1561 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + 1562 pstats->rx_broadcast - pstats->rx_discards; 1563 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + 1564 pstats->tx_unicast; 1565 stats->imissed = pstats->rx_discards; 1566 stats->oerrors = pstats->tx_errors + pstats->tx_discards; 1567 stats->ibytes = pstats->rx_bytes; 1568 stats->ibytes -= stats->ipackets * crc_stats_len; 1569 stats->obytes = pstats->tx_bytes; 1570 } else { 1571 PMD_DRV_LOG(ERR, "Get statistics failed"); 1572 } 1573 return ret; 1574 } 1575 1576 static int 1577 iavf_dev_stats_reset(struct rte_eth_dev *dev) 1578 { 1579 int ret; 1580 struct iavf_adapter *adapter = 1581 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1582 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1583 struct iavf_vsi *vsi = &vf->vsi; 1584 struct virtchnl_eth_stats *pstats = NULL; 1585 1586 /* read stat values to clear hardware registers */ 1587 ret = iavf_query_stats(adapter, &pstats); 1588 if (ret != 0) 1589 return ret; 1590 1591 /* set stats offset base on current values */ 1592 vsi->eth_stats_offset = *pstats; 1593 1594 return 0; 1595 } 1596 1597 static int iavf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1598 struct rte_eth_xstat_name *xstats_names, 1599 __rte_unused unsigned int limit) 1600 { 1601 unsigned int i; 1602 1603 if (xstats_names != NULL) 1604 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1605 snprintf(xstats_names[i].name, 1606 sizeof(xstats_names[i].name), 1607 "%s", rte_iavf_stats_strings[i].name); 1608 } 1609 return IAVF_NB_XSTATS; 1610 } 1611 1612 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 1613 struct rte_eth_xstat *xstats, unsigned int n) 1614 { 1615 int ret; 1616 unsigned int i; 1617 struct iavf_adapter *adapter = 1618 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1619 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1620 struct iavf_vsi *vsi = &vf->vsi; 1621 struct virtchnl_eth_stats *pstats = NULL; 1622 1623 if (n < IAVF_NB_XSTATS) 1624 return IAVF_NB_XSTATS; 1625 1626 ret = iavf_query_stats(adapter, &pstats); 1627 if (ret != 0) 1628 return 0; 1629 1630 if (!xstats) 1631 return 0; 1632 1633 iavf_update_stats(vsi, pstats); 1634 1635 /* loop over xstats array and values from pstats */ 1636 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1637 xstats[i].id = i; 1638 xstats[i].value = *(uint64_t *)(((char *)pstats) + 1639 rte_iavf_stats_strings[i].offset); 1640 } 1641 1642 return IAVF_NB_XSTATS; 1643 } 1644 1645 1646 static int 1647 iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1648 { 1649 struct iavf_adapter *adapter = 1650 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1651 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1652 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1653 uint16_t msix_intr; 1654 1655 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1656 if (msix_intr == IAVF_MISC_VEC_ID) { 1657 PMD_DRV_LOG(INFO, "MISC is also enabled for control"); 1658 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1659 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 1660 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1661 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1662 } else { 1663 IAVF_WRITE_REG(hw, 1664 IAVF_VFINT_DYN_CTLN1 1665 (msix_intr - IAVF_RX_VEC_START), 1666 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 1667 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1668 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 1669 } 1670 1671 IAVF_WRITE_FLUSH(hw); 1672 1673 rte_intr_ack(&pci_dev->intr_handle); 1674 1675 return 0; 1676 } 1677 1678 static int 1679 iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1680 { 1681 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1682 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1683 uint16_t msix_intr; 1684 1685 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1686 if (msix_intr == IAVF_MISC_VEC_ID) { 1687 PMD_DRV_LOG(ERR, "MISC is used for control, cannot disable it"); 1688 return -EIO; 1689 } 1690 1691 IAVF_WRITE_REG(hw, 1692 IAVF_VFINT_DYN_CTLN1(msix_intr - IAVF_RX_VEC_START), 1693 0); 1694 1695 IAVF_WRITE_FLUSH(hw); 1696 return 0; 1697 } 1698 1699 static int 1700 iavf_check_vf_reset_done(struct iavf_hw *hw) 1701 { 1702 int i, reset; 1703 1704 for (i = 0; i < IAVF_RESET_WAIT_CNT; i++) { 1705 reset = IAVF_READ_REG(hw, IAVF_VFGEN_RSTAT) & 1706 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 1707 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 1708 if (reset == VIRTCHNL_VFR_VFACTIVE || 1709 reset == VIRTCHNL_VFR_COMPLETED) 1710 break; 1711 rte_delay_ms(20); 1712 } 1713 1714 if (i >= IAVF_RESET_WAIT_CNT) 1715 return -1; 1716 1717 return 0; 1718 } 1719 1720 static int 1721 iavf_lookup_proto_xtr_type(const char *flex_name) 1722 { 1723 static struct { 1724 const char *name; 1725 enum iavf_proto_xtr_type type; 1726 } xtr_type_map[] = { 1727 { "vlan", IAVF_PROTO_XTR_VLAN }, 1728 { "ipv4", IAVF_PROTO_XTR_IPV4 }, 1729 { "ipv6", IAVF_PROTO_XTR_IPV6 }, 1730 { "ipv6_flow", IAVF_PROTO_XTR_IPV6_FLOW }, 1731 { "tcp", IAVF_PROTO_XTR_TCP }, 1732 { "ip_offset", IAVF_PROTO_XTR_IP_OFFSET }, 1733 }; 1734 uint32_t i; 1735 1736 for (i = 0; i < RTE_DIM(xtr_type_map); i++) { 1737 if (strcmp(flex_name, xtr_type_map[i].name) == 0) 1738 return xtr_type_map[i].type; 1739 } 1740 1741 PMD_DRV_LOG(ERR, "wrong proto_xtr type, " 1742 "it should be: vlan|ipv4|ipv6|ipv6_flow|tcp|ip_offset"); 1743 1744 return -1; 1745 } 1746 1747 /** 1748 * Parse elem, the elem could be single number/range or '(' ')' group 1749 * 1) A single number elem, it's just a simple digit. e.g. 9 1750 * 2) A single range elem, two digits with a '-' between. e.g. 2-6 1751 * 3) A group elem, combines multiple 1) or 2) with '( )'. e.g (0,2-4,6) 1752 * Within group elem, '-' used for a range separator; 1753 * ',' used for a single number. 1754 */ 1755 static int 1756 iavf_parse_queue_set(const char *input, int xtr_type, 1757 struct iavf_devargs *devargs) 1758 { 1759 const char *str = input; 1760 char *end = NULL; 1761 uint32_t min, max; 1762 uint32_t idx; 1763 1764 while (isblank(*str)) 1765 str++; 1766 1767 if (!isdigit(*str) && *str != '(') 1768 return -1; 1769 1770 /* process single number or single range of number */ 1771 if (*str != '(') { 1772 errno = 0; 1773 idx = strtoul(str, &end, 10); 1774 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1775 return -1; 1776 1777 while (isblank(*end)) 1778 end++; 1779 1780 min = idx; 1781 max = idx; 1782 1783 /* process single <number>-<number> */ 1784 if (*end == '-') { 1785 end++; 1786 while (isblank(*end)) 1787 end++; 1788 if (!isdigit(*end)) 1789 return -1; 1790 1791 errno = 0; 1792 idx = strtoul(end, &end, 10); 1793 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1794 return -1; 1795 1796 max = idx; 1797 while (isblank(*end)) 1798 end++; 1799 } 1800 1801 if (*end != ':') 1802 return -1; 1803 1804 for (idx = RTE_MIN(min, max); 1805 idx <= RTE_MAX(min, max); idx++) 1806 devargs->proto_xtr[idx] = xtr_type; 1807 1808 return 0; 1809 } 1810 1811 /* process set within bracket */ 1812 str++; 1813 while (isblank(*str)) 1814 str++; 1815 if (*str == '\0') 1816 return -1; 1817 1818 min = IAVF_MAX_QUEUE_NUM; 1819 do { 1820 /* go ahead to the first digit */ 1821 while (isblank(*str)) 1822 str++; 1823 if (!isdigit(*str)) 1824 return -1; 1825 1826 /* get the digit value */ 1827 errno = 0; 1828 idx = strtoul(str, &end, 10); 1829 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1830 return -1; 1831 1832 /* go ahead to separator '-',',' and ')' */ 1833 while (isblank(*end)) 1834 end++; 1835 if (*end == '-') { 1836 if (min == IAVF_MAX_QUEUE_NUM) 1837 min = idx; 1838 else /* avoid continuous '-' */ 1839 return -1; 1840 } else if (*end == ',' || *end == ')') { 1841 max = idx; 1842 if (min == IAVF_MAX_QUEUE_NUM) 1843 min = idx; 1844 1845 for (idx = RTE_MIN(min, max); 1846 idx <= RTE_MAX(min, max); idx++) 1847 devargs->proto_xtr[idx] = xtr_type; 1848 1849 min = IAVF_MAX_QUEUE_NUM; 1850 } else { 1851 return -1; 1852 } 1853 1854 str = end + 1; 1855 } while (*end != ')' && *end != '\0'); 1856 1857 return 0; 1858 } 1859 1860 static int 1861 iavf_parse_queue_proto_xtr(const char *queues, struct iavf_devargs *devargs) 1862 { 1863 const char *queue_start; 1864 uint32_t idx; 1865 int xtr_type; 1866 char flex_name[32]; 1867 1868 while (isblank(*queues)) 1869 queues++; 1870 1871 if (*queues != '[') { 1872 xtr_type = iavf_lookup_proto_xtr_type(queues); 1873 if (xtr_type < 0) 1874 return -1; 1875 1876 devargs->proto_xtr_dflt = xtr_type; 1877 1878 return 0; 1879 } 1880 1881 queues++; 1882 do { 1883 while (isblank(*queues)) 1884 queues++; 1885 if (*queues == '\0') 1886 return -1; 1887 1888 queue_start = queues; 1889 1890 /* go across a complete bracket */ 1891 if (*queue_start == '(') { 1892 queues += strcspn(queues, ")"); 1893 if (*queues != ')') 1894 return -1; 1895 } 1896 1897 /* scan the separator ':' */ 1898 queues += strcspn(queues, ":"); 1899 if (*queues++ != ':') 1900 return -1; 1901 while (isblank(*queues)) 1902 queues++; 1903 1904 for (idx = 0; ; idx++) { 1905 if (isblank(queues[idx]) || 1906 queues[idx] == ',' || 1907 queues[idx] == ']' || 1908 queues[idx] == '\0') 1909 break; 1910 1911 if (idx > sizeof(flex_name) - 2) 1912 return -1; 1913 1914 flex_name[idx] = queues[idx]; 1915 } 1916 flex_name[idx] = '\0'; 1917 xtr_type = iavf_lookup_proto_xtr_type(flex_name); 1918 if (xtr_type < 0) 1919 return -1; 1920 1921 queues += idx; 1922 1923 while (isblank(*queues) || *queues == ',' || *queues == ']') 1924 queues++; 1925 1926 if (iavf_parse_queue_set(queue_start, xtr_type, devargs) < 0) 1927 return -1; 1928 } while (*queues != '\0'); 1929 1930 return 0; 1931 } 1932 1933 static int 1934 iavf_handle_proto_xtr_arg(__rte_unused const char *key, const char *value, 1935 void *extra_args) 1936 { 1937 struct iavf_devargs *devargs = extra_args; 1938 1939 if (!value || !extra_args) 1940 return -EINVAL; 1941 1942 if (iavf_parse_queue_proto_xtr(value, devargs) < 0) { 1943 PMD_DRV_LOG(ERR, "the proto_xtr's parameter is wrong : '%s'", 1944 value); 1945 return -1; 1946 } 1947 1948 return 0; 1949 } 1950 1951 static int iavf_parse_devargs(struct rte_eth_dev *dev) 1952 { 1953 struct iavf_adapter *ad = 1954 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1955 struct rte_devargs *devargs = dev->device->devargs; 1956 struct rte_kvargs *kvlist; 1957 int ret; 1958 1959 if (!devargs) 1960 return 0; 1961 1962 kvlist = rte_kvargs_parse(devargs->args, iavf_valid_args); 1963 if (!kvlist) { 1964 PMD_INIT_LOG(ERR, "invalid kvargs key\n"); 1965 return -EINVAL; 1966 } 1967 1968 ad->devargs.proto_xtr_dflt = IAVF_PROTO_XTR_NONE; 1969 memset(ad->devargs.proto_xtr, IAVF_PROTO_XTR_NONE, 1970 sizeof(ad->devargs.proto_xtr)); 1971 1972 ret = rte_kvargs_process(kvlist, IAVF_PROTO_XTR_ARG, 1973 &iavf_handle_proto_xtr_arg, &ad->devargs); 1974 if (ret) 1975 goto bail; 1976 1977 bail: 1978 rte_kvargs_free(kvlist); 1979 return ret; 1980 } 1981 1982 static void 1983 iavf_init_proto_xtr(struct rte_eth_dev *dev) 1984 { 1985 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1986 struct iavf_adapter *ad = 1987 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1988 const struct iavf_proto_xtr_ol *xtr_ol; 1989 bool proto_xtr_enable = false; 1990 int offset; 1991 uint16_t i; 1992 1993 vf->proto_xtr = rte_zmalloc("vf proto xtr", 1994 vf->vsi_res->num_queue_pairs, 0); 1995 if (unlikely(!(vf->proto_xtr))) { 1996 PMD_DRV_LOG(ERR, "no memory for setting up proto_xtr's table"); 1997 return; 1998 } 1999 2000 for (i = 0; i < vf->vsi_res->num_queue_pairs; i++) { 2001 vf->proto_xtr[i] = ad->devargs.proto_xtr[i] != 2002 IAVF_PROTO_XTR_NONE ? 2003 ad->devargs.proto_xtr[i] : 2004 ad->devargs.proto_xtr_dflt; 2005 2006 if (vf->proto_xtr[i] != IAVF_PROTO_XTR_NONE) { 2007 uint8_t type = vf->proto_xtr[i]; 2008 2009 iavf_proto_xtr_params[type].required = true; 2010 proto_xtr_enable = true; 2011 } 2012 } 2013 2014 if (likely(!proto_xtr_enable)) 2015 return; 2016 2017 offset = rte_mbuf_dynfield_register(&iavf_proto_xtr_metadata_param); 2018 if (unlikely(offset == -1)) { 2019 PMD_DRV_LOG(ERR, 2020 "failed to extract protocol metadata, error %d", 2021 -rte_errno); 2022 return; 2023 } 2024 2025 PMD_DRV_LOG(DEBUG, 2026 "proto_xtr metadata offset in mbuf is : %d", 2027 offset); 2028 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = offset; 2029 2030 for (i = 0; i < RTE_DIM(iavf_proto_xtr_params); i++) { 2031 xtr_ol = &iavf_proto_xtr_params[i]; 2032 2033 uint8_t rxdid = iavf_proto_xtr_type_to_rxdid((uint8_t)i); 2034 2035 if (!xtr_ol->required) 2036 continue; 2037 2038 if (!(vf->supported_rxdid & BIT(rxdid))) { 2039 PMD_DRV_LOG(ERR, 2040 "rxdid[%u] is not supported in hardware", 2041 rxdid); 2042 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 2043 break; 2044 } 2045 2046 offset = rte_mbuf_dynflag_register(&xtr_ol->param); 2047 if (unlikely(offset == -1)) { 2048 PMD_DRV_LOG(ERR, 2049 "failed to register proto_xtr offload '%s', error %d", 2050 xtr_ol->param.name, -rte_errno); 2051 2052 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 2053 break; 2054 } 2055 2056 PMD_DRV_LOG(DEBUG, 2057 "proto_xtr offload '%s' offset in mbuf is : %d", 2058 xtr_ol->param.name, offset); 2059 *xtr_ol->ol_flag = 1ULL << offset; 2060 } 2061 } 2062 2063 static int 2064 iavf_init_vf(struct rte_eth_dev *dev) 2065 { 2066 int err, bufsz; 2067 struct iavf_adapter *adapter = 2068 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2069 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2070 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2071 2072 err = iavf_parse_devargs(dev); 2073 if (err) { 2074 PMD_INIT_LOG(ERR, "Failed to parse devargs"); 2075 goto err; 2076 } 2077 2078 err = iavf_set_mac_type(hw); 2079 if (err) { 2080 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err); 2081 goto err; 2082 } 2083 2084 err = iavf_check_vf_reset_done(hw); 2085 if (err) { 2086 PMD_INIT_LOG(ERR, "VF is still resetting"); 2087 goto err; 2088 } 2089 2090 iavf_init_adminq_parameter(hw); 2091 err = iavf_init_adminq(hw); 2092 if (err) { 2093 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err); 2094 goto err; 2095 } 2096 2097 vf->aq_resp = rte_zmalloc("vf_aq_resp", IAVF_AQ_BUF_SZ, 0); 2098 if (!vf->aq_resp) { 2099 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); 2100 goto err_aq; 2101 } 2102 if (iavf_check_api_version(adapter) != 0) { 2103 PMD_INIT_LOG(ERR, "check_api version failed"); 2104 goto err_api; 2105 } 2106 2107 bufsz = sizeof(struct virtchnl_vf_resource) + 2108 (IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); 2109 vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); 2110 if (!vf->vf_res) { 2111 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 2112 goto err_api; 2113 } 2114 2115 if (iavf_get_vf_resource(adapter) != 0) { 2116 PMD_INIT_LOG(ERR, "iavf_get_vf_config failed"); 2117 goto err_alloc; 2118 } 2119 /* Allocate memort for RSS info */ 2120 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2121 vf->rss_key = rte_zmalloc("rss_key", 2122 vf->vf_res->rss_key_size, 0); 2123 if (!vf->rss_key) { 2124 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 2125 goto err_rss; 2126 } 2127 vf->rss_lut = rte_zmalloc("rss_lut", 2128 vf->vf_res->rss_lut_size, 0); 2129 if (!vf->rss_lut) { 2130 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 2131 goto err_rss; 2132 } 2133 } 2134 2135 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 2136 if (iavf_get_supported_rxdid(adapter) != 0) { 2137 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 2138 goto err_rss; 2139 } 2140 } 2141 2142 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 2143 if (iavf_get_vlan_offload_caps_v2(adapter) != 0) { 2144 PMD_INIT_LOG(ERR, "failed to do get VLAN offload v2 capabilities"); 2145 goto err_rss; 2146 } 2147 } 2148 2149 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) { 2150 bufsz = sizeof(struct virtchnl_qos_cap_list) + 2151 IAVF_MAX_TRAFFIC_CLASS * 2152 sizeof(struct virtchnl_qos_cap_elem); 2153 vf->qos_cap = rte_zmalloc("qos_cap", bufsz, 0); 2154 if (!vf->qos_cap) { 2155 PMD_INIT_LOG(ERR, "unable to allocate qos_cap memory"); 2156 goto err_rss; 2157 } 2158 iavf_tm_conf_init(dev); 2159 } 2160 2161 iavf_init_proto_xtr(dev); 2162 2163 return 0; 2164 err_rss: 2165 rte_free(vf->rss_key); 2166 rte_free(vf->rss_lut); 2167 err_alloc: 2168 rte_free(vf->qos_cap); 2169 rte_free(vf->vf_res); 2170 vf->vsi_res = NULL; 2171 err_api: 2172 rte_free(vf->aq_resp); 2173 err_aq: 2174 iavf_shutdown_adminq(hw); 2175 err: 2176 return -1; 2177 } 2178 2179 /* Enable default admin queue interrupt setting */ 2180 static inline void 2181 iavf_enable_irq0(struct iavf_hw *hw) 2182 { 2183 /* Enable admin queue interrupt trigger */ 2184 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 2185 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 2186 2187 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 2188 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 2189 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 2190 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 2191 2192 IAVF_WRITE_FLUSH(hw); 2193 } 2194 2195 static inline void 2196 iavf_disable_irq0(struct iavf_hw *hw) 2197 { 2198 /* Disable all interrupt types */ 2199 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 0); 2200 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 2201 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 2202 IAVF_WRITE_FLUSH(hw); 2203 } 2204 2205 static void 2206 iavf_dev_interrupt_handler(void *param) 2207 { 2208 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2209 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2210 2211 iavf_disable_irq0(hw); 2212 2213 iavf_handle_virtchnl_msg(dev); 2214 2215 iavf_enable_irq0(hw); 2216 } 2217 2218 static int 2219 iavf_dev_flow_ops_get(struct rte_eth_dev *dev, 2220 const struct rte_flow_ops **ops) 2221 { 2222 if (!dev) 2223 return -EINVAL; 2224 2225 *ops = &iavf_flow_ops; 2226 return 0; 2227 } 2228 2229 static void 2230 iavf_default_rss_disable(struct iavf_adapter *adapter) 2231 { 2232 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 2233 int ret = 0; 2234 2235 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2236 /* Set hena = 0 to ask PF to cleanup all existing RSS. */ 2237 ret = iavf_set_hena(adapter, 0); 2238 if (ret) 2239 /* It is a workaround, temporarily allow error to be 2240 * returned due to possible lack of PF handling for 2241 * hena = 0. 2242 */ 2243 PMD_INIT_LOG(WARNING, "fail to disable default RSS," 2244 "lack PF support"); 2245 } 2246 } 2247 2248 static int 2249 iavf_dev_init(struct rte_eth_dev *eth_dev) 2250 { 2251 struct iavf_adapter *adapter = 2252 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 2253 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 2254 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 2255 int ret = 0; 2256 2257 PMD_INIT_FUNC_TRACE(); 2258 2259 /* assign ops func pointer */ 2260 eth_dev->dev_ops = &iavf_eth_dev_ops; 2261 eth_dev->rx_queue_count = iavf_dev_rxq_count; 2262 eth_dev->rx_descriptor_status = iavf_dev_rx_desc_status; 2263 eth_dev->tx_descriptor_status = iavf_dev_tx_desc_status; 2264 eth_dev->rx_pkt_burst = &iavf_recv_pkts; 2265 eth_dev->tx_pkt_burst = &iavf_xmit_pkts; 2266 eth_dev->tx_pkt_prepare = &iavf_prep_pkts; 2267 2268 /* For secondary processes, we don't initialise any further as primary 2269 * has already done this work. Only check if we need a different RX 2270 * and TX function. 2271 */ 2272 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2273 iavf_set_rx_function(eth_dev); 2274 iavf_set_tx_function(eth_dev); 2275 return 0; 2276 } 2277 rte_eth_copy_pci_info(eth_dev, pci_dev); 2278 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 2279 2280 hw->vendor_id = pci_dev->id.vendor_id; 2281 hw->device_id = pci_dev->id.device_id; 2282 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 2283 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 2284 hw->bus.bus_id = pci_dev->addr.bus; 2285 hw->bus.device = pci_dev->addr.devid; 2286 hw->bus.func = pci_dev->addr.function; 2287 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 2288 hw->back = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 2289 adapter->eth_dev = eth_dev; 2290 adapter->stopped = 1; 2291 2292 if (iavf_init_vf(eth_dev) != 0) { 2293 PMD_INIT_LOG(ERR, "Init vf failed"); 2294 return -1; 2295 } 2296 2297 /* set default ptype table */ 2298 adapter->ptype_tbl = iavf_get_default_ptype_table(); 2299 2300 /* copy mac addr */ 2301 eth_dev->data->mac_addrs = rte_zmalloc( 2302 "iavf_mac", RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX, 0); 2303 if (!eth_dev->data->mac_addrs) { 2304 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to" 2305 " store MAC addresses", 2306 RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX); 2307 return -ENOMEM; 2308 } 2309 /* If the MAC address is not configured by host, 2310 * generate a random one. 2311 */ 2312 if (!rte_is_valid_assigned_ether_addr( 2313 (struct rte_ether_addr *)hw->mac.addr)) 2314 rte_eth_random_addr(hw->mac.addr); 2315 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 2316 ð_dev->data->mac_addrs[0]); 2317 2318 /* register callback func to eal lib */ 2319 rte_intr_callback_register(&pci_dev->intr_handle, 2320 iavf_dev_interrupt_handler, 2321 (void *)eth_dev); 2322 2323 /* enable uio intr after callback register */ 2324 rte_intr_enable(&pci_dev->intr_handle); 2325 2326 /* configure and enable device interrupt */ 2327 iavf_enable_irq0(hw); 2328 2329 ret = iavf_flow_init(adapter); 2330 if (ret) { 2331 PMD_INIT_LOG(ERR, "Failed to initialize flow"); 2332 return ret; 2333 } 2334 2335 iavf_default_rss_disable(adapter); 2336 2337 return 0; 2338 } 2339 2340 static int 2341 iavf_dev_close(struct rte_eth_dev *dev) 2342 { 2343 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2344 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2345 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2346 struct iavf_adapter *adapter = 2347 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2348 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2349 int ret; 2350 2351 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2352 return 0; 2353 2354 ret = iavf_dev_stop(dev); 2355 2356 iavf_flow_flush(dev, NULL); 2357 iavf_flow_uninit(adapter); 2358 2359 /* 2360 * disable promiscuous mode before reset vf 2361 * it is a workaround solution when work with kernel driver 2362 * and it is not the normal way 2363 */ 2364 if (vf->promisc_unicast_enabled || vf->promisc_multicast_enabled) 2365 iavf_config_promisc(adapter, false, false); 2366 2367 iavf_shutdown_adminq(hw); 2368 /* disable uio intr before callback unregister */ 2369 rte_intr_disable(intr_handle); 2370 2371 /* unregister callback func from eal lib */ 2372 rte_intr_callback_unregister(intr_handle, 2373 iavf_dev_interrupt_handler, dev); 2374 iavf_disable_irq0(hw); 2375 2376 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 2377 iavf_tm_conf_uninit(dev); 2378 2379 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2380 if (vf->rss_lut) { 2381 rte_free(vf->rss_lut); 2382 vf->rss_lut = NULL; 2383 } 2384 if (vf->rss_key) { 2385 rte_free(vf->rss_key); 2386 vf->rss_key = NULL; 2387 } 2388 } 2389 2390 rte_free(vf->vf_res); 2391 vf->vsi_res = NULL; 2392 vf->vf_res = NULL; 2393 2394 rte_free(vf->aq_resp); 2395 vf->aq_resp = NULL; 2396 2397 /* 2398 * If the VF is reset via VFLR, the device will be knocked out of bus 2399 * master mode, and the driver will fail to recover from the reset. Fix 2400 * this by enabling bus mastering after every reset. In a non-VFLR case, 2401 * the bus master bit will not be disabled, and this call will have no 2402 * effect. 2403 */ 2404 if (vf->vf_reset && !rte_pci_set_bus_master(pci_dev, true)) 2405 vf->vf_reset = false; 2406 2407 return ret; 2408 } 2409 2410 static int 2411 iavf_dev_uninit(struct rte_eth_dev *dev) 2412 { 2413 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2414 return -EPERM; 2415 2416 iavf_dev_close(dev); 2417 2418 return 0; 2419 } 2420 2421 /* 2422 * Reset VF device only to re-initialize resources in PMD layer 2423 */ 2424 static int 2425 iavf_dev_reset(struct rte_eth_dev *dev) 2426 { 2427 int ret; 2428 2429 ret = iavf_dev_uninit(dev); 2430 if (ret) 2431 return ret; 2432 2433 return iavf_dev_init(dev); 2434 } 2435 2436 static int 2437 iavf_dcf_cap_check_handler(__rte_unused const char *key, 2438 const char *value, __rte_unused void *opaque) 2439 { 2440 if (strcmp(value, "dcf")) 2441 return -1; 2442 2443 return 0; 2444 } 2445 2446 static int 2447 iavf_dcf_cap_selected(struct rte_devargs *devargs) 2448 { 2449 struct rte_kvargs *kvlist; 2450 const char *key = "cap"; 2451 int ret = 0; 2452 2453 if (devargs == NULL) 2454 return 0; 2455 2456 kvlist = rte_kvargs_parse(devargs->args, NULL); 2457 if (kvlist == NULL) 2458 return 0; 2459 2460 if (!rte_kvargs_count(kvlist, key)) 2461 goto exit; 2462 2463 /* dcf capability selected when there's a key-value pair: cap=dcf */ 2464 if (rte_kvargs_process(kvlist, key, 2465 iavf_dcf_cap_check_handler, NULL) < 0) 2466 goto exit; 2467 2468 ret = 1; 2469 2470 exit: 2471 rte_kvargs_free(kvlist); 2472 return ret; 2473 } 2474 2475 static int 2476 iavf_drv_i40evf_check_handler(__rte_unused const char *key, 2477 const char *value, __rte_unused void *opaque) 2478 { 2479 if (strcmp(value, "i40evf")) 2480 return -1; 2481 2482 return 0; 2483 } 2484 2485 static int 2486 iavf_drv_i40evf_selected(struct rte_devargs *devargs, uint16_t device_id) 2487 { 2488 struct rte_kvargs *kvlist; 2489 int ret = 0; 2490 2491 if (device_id != IAVF_DEV_ID_VF && 2492 device_id != IAVF_DEV_ID_VF_HV && 2493 device_id != IAVF_DEV_ID_X722_VF && 2494 device_id != IAVF_DEV_ID_X722_A0_VF) 2495 return 0; 2496 2497 if (devargs == NULL) 2498 return 0; 2499 2500 kvlist = rte_kvargs_parse(devargs->args, NULL); 2501 if (kvlist == NULL) 2502 return 0; 2503 2504 if (!rte_kvargs_count(kvlist, RTE_DEVARGS_KEY_DRIVER)) 2505 goto exit; 2506 2507 /* i40evf driver selected when there's a key-value pair: 2508 * driver=i40evf 2509 */ 2510 if (rte_kvargs_process(kvlist, RTE_DEVARGS_KEY_DRIVER, 2511 iavf_drv_i40evf_check_handler, NULL) < 0) 2512 goto exit; 2513 2514 ret = 1; 2515 2516 exit: 2517 rte_kvargs_free(kvlist); 2518 return ret; 2519 } 2520 2521 static int eth_iavf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2522 struct rte_pci_device *pci_dev) 2523 { 2524 if (iavf_dcf_cap_selected(pci_dev->device.devargs) || 2525 iavf_drv_i40evf_selected(pci_dev->device.devargs, 2526 pci_dev->id.device_id)) 2527 return 1; 2528 2529 return rte_eth_dev_pci_generic_probe(pci_dev, 2530 sizeof(struct iavf_adapter), iavf_dev_init); 2531 } 2532 2533 static int eth_iavf_pci_remove(struct rte_pci_device *pci_dev) 2534 { 2535 return rte_eth_dev_pci_generic_remove(pci_dev, iavf_dev_uninit); 2536 } 2537 2538 /* Adaptive virtual function driver struct */ 2539 static struct rte_pci_driver rte_iavf_pmd = { 2540 .id_table = pci_id_iavf_map, 2541 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 2542 .probe = eth_iavf_pci_probe, 2543 .remove = eth_iavf_pci_remove, 2544 }; 2545 2546 RTE_PMD_REGISTER_PCI(net_iavf, rte_iavf_pmd); 2547 RTE_PMD_REGISTER_PCI_TABLE(net_iavf, pci_id_iavf_map); 2548 RTE_PMD_REGISTER_KMOD_DEP(net_iavf, "* igb_uio | vfio-pci"); 2549 RTE_PMD_REGISTER_PARAM_STRING(net_iavf, "cap=dcf driver=i40evf"); 2550 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_init, init, NOTICE); 2551 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_driver, driver, NOTICE); 2552 #ifdef RTE_ETHDEV_DEBUG_RX 2553 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_rx, rx, DEBUG); 2554 #endif 2555 #ifdef RTE_ETHDEV_DEBUG_TX 2556 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_tx, tx, DEBUG); 2557 #endif 2558