1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_interrupts.h> 17 #include <rte_debug.h> 18 #include <rte_pci.h> 19 #include <rte_alarm.h> 20 #include <rte_atomic.h> 21 #include <rte_eal.h> 22 #include <rte_ether.h> 23 #include <ethdev_driver.h> 24 #include <ethdev_pci.h> 25 #include <rte_malloc.h> 26 #include <rte_memzone.h> 27 #include <rte_dev.h> 28 29 #include "iavf.h" 30 #include "iavf_rxtx.h" 31 #include "iavf_generic_flow.h" 32 #include "rte_pmd_iavf.h" 33 34 /* devargs */ 35 #define IAVF_PROTO_XTR_ARG "proto_xtr" 36 37 static const char * const iavf_valid_args[] = { 38 IAVF_PROTO_XTR_ARG, 39 NULL 40 }; 41 42 static const struct rte_mbuf_dynfield iavf_proto_xtr_metadata_param = { 43 .name = "intel_pmd_dynfield_proto_xtr_metadata", 44 .size = sizeof(uint32_t), 45 .align = __alignof__(uint32_t), 46 .flags = 0, 47 }; 48 49 struct iavf_proto_xtr_ol { 50 const struct rte_mbuf_dynflag param; 51 uint64_t *ol_flag; 52 bool required; 53 }; 54 55 static struct iavf_proto_xtr_ol iavf_proto_xtr_params[] = { 56 [IAVF_PROTO_XTR_VLAN] = { 57 .param = { .name = "intel_pmd_dynflag_proto_xtr_vlan" }, 58 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_vlan_mask }, 59 [IAVF_PROTO_XTR_IPV4] = { 60 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv4" }, 61 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv4_mask }, 62 [IAVF_PROTO_XTR_IPV6] = { 63 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6" }, 64 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_mask }, 65 [IAVF_PROTO_XTR_IPV6_FLOW] = { 66 .param = { .name = "intel_pmd_dynflag_proto_xtr_ipv6_flow" }, 67 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ipv6_flow_mask }, 68 [IAVF_PROTO_XTR_TCP] = { 69 .param = { .name = "intel_pmd_dynflag_proto_xtr_tcp" }, 70 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_tcp_mask }, 71 [IAVF_PROTO_XTR_IP_OFFSET] = { 72 .param = { .name = "intel_pmd_dynflag_proto_xtr_ip_offset" }, 73 .ol_flag = &rte_pmd_ifd_dynflag_proto_xtr_ip_offset_mask }, 74 }; 75 76 static int iavf_dev_configure(struct rte_eth_dev *dev); 77 static int iavf_dev_start(struct rte_eth_dev *dev); 78 static int iavf_dev_stop(struct rte_eth_dev *dev); 79 static int iavf_dev_close(struct rte_eth_dev *dev); 80 static int iavf_dev_reset(struct rte_eth_dev *dev); 81 static int iavf_dev_info_get(struct rte_eth_dev *dev, 82 struct rte_eth_dev_info *dev_info); 83 static const uint32_t *iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev); 84 static int iavf_dev_stats_get(struct rte_eth_dev *dev, 85 struct rte_eth_stats *stats); 86 static int iavf_dev_stats_reset(struct rte_eth_dev *dev); 87 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 88 struct rte_eth_xstat *xstats, unsigned int n); 89 static int iavf_dev_xstats_get_names(struct rte_eth_dev *dev, 90 struct rte_eth_xstat_name *xstats_names, 91 unsigned int limit); 92 static int iavf_dev_promiscuous_enable(struct rte_eth_dev *dev); 93 static int iavf_dev_promiscuous_disable(struct rte_eth_dev *dev); 94 static int iavf_dev_allmulticast_enable(struct rte_eth_dev *dev); 95 static int iavf_dev_allmulticast_disable(struct rte_eth_dev *dev); 96 static int iavf_dev_add_mac_addr(struct rte_eth_dev *dev, 97 struct rte_ether_addr *addr, 98 uint32_t index, 99 uint32_t pool); 100 static void iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); 101 static int iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, 102 uint16_t vlan_id, int on); 103 static int iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask); 104 static int iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 105 struct rte_eth_rss_reta_entry64 *reta_conf, 106 uint16_t reta_size); 107 static int iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 108 struct rte_eth_rss_reta_entry64 *reta_conf, 109 uint16_t reta_size); 110 static int iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 111 struct rte_eth_rss_conf *rss_conf); 112 static int iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 113 struct rte_eth_rss_conf *rss_conf); 114 static int iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 115 static int iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 116 struct rte_ether_addr *mac_addr); 117 static int iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, 118 uint16_t queue_id); 119 static int iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, 120 uint16_t queue_id); 121 static int iavf_dev_flow_ops_get(struct rte_eth_dev *dev, 122 const struct rte_flow_ops **ops); 123 static int iavf_set_mc_addr_list(struct rte_eth_dev *dev, 124 struct rte_ether_addr *mc_addrs, 125 uint32_t mc_addrs_num); 126 static int iavf_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *arg); 127 128 static const struct rte_pci_id pci_id_iavf_map[] = { 129 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_ADAPTIVE_VF) }, 130 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_VF) }, 131 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_VF_HV) }, 132 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_X722_VF) }, 133 { RTE_PCI_DEVICE(IAVF_INTEL_VENDOR_ID, IAVF_DEV_ID_X722_A0_VF) }, 134 { .vendor_id = 0, /* sentinel */ }, 135 }; 136 137 struct rte_iavf_xstats_name_off { 138 char name[RTE_ETH_XSTATS_NAME_SIZE]; 139 unsigned int offset; 140 }; 141 142 static const struct rte_iavf_xstats_name_off rte_iavf_stats_strings[] = { 143 {"rx_bytes", offsetof(struct iavf_eth_stats, rx_bytes)}, 144 {"rx_unicast_packets", offsetof(struct iavf_eth_stats, rx_unicast)}, 145 {"rx_multicast_packets", offsetof(struct iavf_eth_stats, rx_multicast)}, 146 {"rx_broadcast_packets", offsetof(struct iavf_eth_stats, rx_broadcast)}, 147 {"rx_dropped_packets", offsetof(struct iavf_eth_stats, rx_discards)}, 148 {"rx_unknown_protocol_packets", offsetof(struct iavf_eth_stats, 149 rx_unknown_protocol)}, 150 {"tx_bytes", offsetof(struct iavf_eth_stats, tx_bytes)}, 151 {"tx_unicast_packets", offsetof(struct iavf_eth_stats, tx_unicast)}, 152 {"tx_multicast_packets", offsetof(struct iavf_eth_stats, tx_multicast)}, 153 {"tx_broadcast_packets", offsetof(struct iavf_eth_stats, tx_broadcast)}, 154 {"tx_dropped_packets", offsetof(struct iavf_eth_stats, tx_discards)}, 155 {"tx_error_packets", offsetof(struct iavf_eth_stats, tx_errors)}, 156 }; 157 158 #define IAVF_NB_XSTATS (sizeof(rte_iavf_stats_strings) / \ 159 sizeof(rte_iavf_stats_strings[0])) 160 161 static const struct eth_dev_ops iavf_eth_dev_ops = { 162 .dev_configure = iavf_dev_configure, 163 .dev_start = iavf_dev_start, 164 .dev_stop = iavf_dev_stop, 165 .dev_close = iavf_dev_close, 166 .dev_reset = iavf_dev_reset, 167 .dev_infos_get = iavf_dev_info_get, 168 .dev_supported_ptypes_get = iavf_dev_supported_ptypes_get, 169 .link_update = iavf_dev_link_update, 170 .stats_get = iavf_dev_stats_get, 171 .stats_reset = iavf_dev_stats_reset, 172 .xstats_get = iavf_dev_xstats_get, 173 .xstats_get_names = iavf_dev_xstats_get_names, 174 .xstats_reset = iavf_dev_stats_reset, 175 .promiscuous_enable = iavf_dev_promiscuous_enable, 176 .promiscuous_disable = iavf_dev_promiscuous_disable, 177 .allmulticast_enable = iavf_dev_allmulticast_enable, 178 .allmulticast_disable = iavf_dev_allmulticast_disable, 179 .mac_addr_add = iavf_dev_add_mac_addr, 180 .mac_addr_remove = iavf_dev_del_mac_addr, 181 .set_mc_addr_list = iavf_set_mc_addr_list, 182 .vlan_filter_set = iavf_dev_vlan_filter_set, 183 .vlan_offload_set = iavf_dev_vlan_offload_set, 184 .rx_queue_start = iavf_dev_rx_queue_start, 185 .rx_queue_stop = iavf_dev_rx_queue_stop, 186 .tx_queue_start = iavf_dev_tx_queue_start, 187 .tx_queue_stop = iavf_dev_tx_queue_stop, 188 .rx_queue_setup = iavf_dev_rx_queue_setup, 189 .rx_queue_release = iavf_dev_rx_queue_release, 190 .tx_queue_setup = iavf_dev_tx_queue_setup, 191 .tx_queue_release = iavf_dev_tx_queue_release, 192 .mac_addr_set = iavf_dev_set_default_mac_addr, 193 .reta_update = iavf_dev_rss_reta_update, 194 .reta_query = iavf_dev_rss_reta_query, 195 .rss_hash_update = iavf_dev_rss_hash_update, 196 .rss_hash_conf_get = iavf_dev_rss_hash_conf_get, 197 .rxq_info_get = iavf_dev_rxq_info_get, 198 .txq_info_get = iavf_dev_txq_info_get, 199 .mtu_set = iavf_dev_mtu_set, 200 .rx_queue_intr_enable = iavf_dev_rx_queue_intr_enable, 201 .rx_queue_intr_disable = iavf_dev_rx_queue_intr_disable, 202 .flow_ops_get = iavf_dev_flow_ops_get, 203 .tx_done_cleanup = iavf_dev_tx_done_cleanup, 204 .get_monitor_addr = iavf_get_monitor_addr, 205 .tm_ops_get = iavf_tm_ops_get, 206 }; 207 208 static int 209 iavf_tm_ops_get(struct rte_eth_dev *dev __rte_unused, 210 void *arg) 211 { 212 if (!arg) 213 return -EINVAL; 214 215 *(const void **)arg = &iavf_tm_ops; 216 217 return 0; 218 } 219 220 static int 221 iavf_set_mc_addr_list(struct rte_eth_dev *dev, 222 struct rte_ether_addr *mc_addrs, 223 uint32_t mc_addrs_num) 224 { 225 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 226 struct iavf_adapter *adapter = 227 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 228 int err, ret; 229 230 if (mc_addrs_num > IAVF_NUM_MACADDR_MAX) { 231 PMD_DRV_LOG(ERR, 232 "can't add more than a limited number (%u) of addresses.", 233 (uint32_t)IAVF_NUM_MACADDR_MAX); 234 return -EINVAL; 235 } 236 237 /* flush previous addresses */ 238 err = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 239 false); 240 if (err) 241 return err; 242 243 /* add new ones */ 244 err = iavf_add_del_mc_addr_list(adapter, mc_addrs, mc_addrs_num, true); 245 246 if (err) { 247 /* if adding mac address list fails, should add the previous 248 * addresses back. 249 */ 250 ret = iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, 251 vf->mc_addrs_num, true); 252 if (ret) 253 return ret; 254 } else { 255 vf->mc_addrs_num = mc_addrs_num; 256 memcpy(vf->mc_addrs, 257 mc_addrs, mc_addrs_num * sizeof(*mc_addrs)); 258 } 259 260 return err; 261 } 262 263 static void 264 iavf_config_rss_hf(struct iavf_adapter *adapter, uint64_t rss_hf) 265 { 266 static const uint64_t map_hena_rss[] = { 267 /* IPv4 */ 268 [IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV4_UDP] = 269 ETH_RSS_NONFRAG_IPV4_UDP, 270 [IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV4_UDP] = 271 ETH_RSS_NONFRAG_IPV4_UDP, 272 [IAVF_FILTER_PCTYPE_NONF_IPV4_UDP] = 273 ETH_RSS_NONFRAG_IPV4_UDP, 274 [IAVF_FILTER_PCTYPE_NONF_IPV4_TCP_SYN_NO_ACK] = 275 ETH_RSS_NONFRAG_IPV4_TCP, 276 [IAVF_FILTER_PCTYPE_NONF_IPV4_TCP] = 277 ETH_RSS_NONFRAG_IPV4_TCP, 278 [IAVF_FILTER_PCTYPE_NONF_IPV4_SCTP] = 279 ETH_RSS_NONFRAG_IPV4_SCTP, 280 [IAVF_FILTER_PCTYPE_NONF_IPV4_OTHER] = 281 ETH_RSS_NONFRAG_IPV4_OTHER, 282 [IAVF_FILTER_PCTYPE_FRAG_IPV4] = ETH_RSS_FRAG_IPV4, 283 284 /* IPv6 */ 285 [IAVF_FILTER_PCTYPE_NONF_UNICAST_IPV6_UDP] = 286 ETH_RSS_NONFRAG_IPV6_UDP, 287 [IAVF_FILTER_PCTYPE_NONF_MULTICAST_IPV6_UDP] = 288 ETH_RSS_NONFRAG_IPV6_UDP, 289 [IAVF_FILTER_PCTYPE_NONF_IPV6_UDP] = 290 ETH_RSS_NONFRAG_IPV6_UDP, 291 [IAVF_FILTER_PCTYPE_NONF_IPV6_TCP_SYN_NO_ACK] = 292 ETH_RSS_NONFRAG_IPV6_TCP, 293 [IAVF_FILTER_PCTYPE_NONF_IPV6_TCP] = 294 ETH_RSS_NONFRAG_IPV6_TCP, 295 [IAVF_FILTER_PCTYPE_NONF_IPV6_SCTP] = 296 ETH_RSS_NONFRAG_IPV6_SCTP, 297 [IAVF_FILTER_PCTYPE_NONF_IPV6_OTHER] = 298 ETH_RSS_NONFRAG_IPV6_OTHER, 299 [IAVF_FILTER_PCTYPE_FRAG_IPV6] = ETH_RSS_FRAG_IPV6, 300 301 /* L2 Payload */ 302 [IAVF_FILTER_PCTYPE_L2_PAYLOAD] = ETH_RSS_L2_PAYLOAD 303 }; 304 305 const uint64_t ipv4_rss = ETH_RSS_NONFRAG_IPV4_UDP | 306 ETH_RSS_NONFRAG_IPV4_TCP | 307 ETH_RSS_NONFRAG_IPV4_SCTP | 308 ETH_RSS_NONFRAG_IPV4_OTHER | 309 ETH_RSS_FRAG_IPV4; 310 311 const uint64_t ipv6_rss = ETH_RSS_NONFRAG_IPV6_UDP | 312 ETH_RSS_NONFRAG_IPV6_TCP | 313 ETH_RSS_NONFRAG_IPV6_SCTP | 314 ETH_RSS_NONFRAG_IPV6_OTHER | 315 ETH_RSS_FRAG_IPV6; 316 317 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 318 uint64_t caps = 0, hena = 0, valid_rss_hf = 0; 319 uint32_t i; 320 int ret; 321 322 ret = iavf_get_hena_caps(adapter, &caps); 323 if (ret) { 324 /** 325 * RSS offload type configuration is not a necessary feature 326 * for VF, so here just print a warning and return. 327 */ 328 PMD_DRV_LOG(WARNING, 329 "fail to get RSS offload type caps, ret: %d", ret); 330 return; 331 } 332 333 /** 334 * ETH_RSS_IPV4 and ETH_RSS_IPV6 can be considered as 2 335 * generalizations of all other IPv4 and IPv6 RSS types. 336 */ 337 if (rss_hf & ETH_RSS_IPV4) 338 rss_hf |= ipv4_rss; 339 340 if (rss_hf & ETH_RSS_IPV6) 341 rss_hf |= ipv6_rss; 342 343 RTE_BUILD_BUG_ON(RTE_DIM(map_hena_rss) > sizeof(uint64_t) * CHAR_BIT); 344 345 for (i = 0; i < RTE_DIM(map_hena_rss); i++) { 346 uint64_t bit = BIT_ULL(i); 347 348 if ((caps & bit) && (map_hena_rss[i] & rss_hf)) { 349 valid_rss_hf |= map_hena_rss[i]; 350 hena |= bit; 351 } 352 } 353 354 ret = iavf_set_hena(adapter, hena); 355 if (ret) { 356 /** 357 * RSS offload type configuration is not a necessary feature 358 * for VF, so here just print a warning and return. 359 */ 360 PMD_DRV_LOG(WARNING, 361 "fail to set RSS offload types, ret: %d", ret); 362 return; 363 } 364 365 if (valid_rss_hf & ipv4_rss) 366 valid_rss_hf |= rss_hf & ETH_RSS_IPV4; 367 368 if (valid_rss_hf & ipv6_rss) 369 valid_rss_hf |= rss_hf & ETH_RSS_IPV6; 370 371 if (rss_hf & ~valid_rss_hf) 372 PMD_DRV_LOG(WARNING, "Unsupported rss_hf 0x%" PRIx64, 373 rss_hf & ~valid_rss_hf); 374 375 vf->rss_hf = valid_rss_hf; 376 } 377 378 static int 379 iavf_init_rss(struct iavf_adapter *adapter) 380 { 381 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 382 struct rte_eth_rss_conf *rss_conf; 383 uint16_t i, j, nb_q; 384 int ret; 385 386 rss_conf = &adapter->dev_data->dev_conf.rx_adv_conf.rss_conf; 387 nb_q = RTE_MIN(adapter->dev_data->nb_rx_queues, 388 vf->max_rss_qregion); 389 390 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 391 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 392 return -ENOTSUP; 393 } 394 395 /* configure RSS key */ 396 if (!rss_conf->rss_key) { 397 /* Calculate the default hash key */ 398 for (i = 0; i < vf->vf_res->rss_key_size; i++) 399 vf->rss_key[i] = (uint8_t)rte_rand(); 400 } else 401 rte_memcpy(vf->rss_key, rss_conf->rss_key, 402 RTE_MIN(rss_conf->rss_key_len, 403 vf->vf_res->rss_key_size)); 404 405 /* init RSS LUT table */ 406 for (i = 0, j = 0; i < vf->vf_res->rss_lut_size; i++, j++) { 407 if (j >= nb_q) 408 j = 0; 409 vf->rss_lut[i] = j; 410 } 411 /* send virtchnnl ops to configure rss*/ 412 ret = iavf_configure_rss_lut(adapter); 413 if (ret) 414 return ret; 415 ret = iavf_configure_rss_key(adapter); 416 if (ret) 417 return ret; 418 419 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF) { 420 /* Set RSS hash configuration based on rss_conf->rss_hf. */ 421 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 422 if (ret) { 423 PMD_DRV_LOG(ERR, "fail to set default RSS"); 424 return ret; 425 } 426 } else { 427 iavf_config_rss_hf(adapter, rss_conf->rss_hf); 428 } 429 430 return 0; 431 } 432 433 static int 434 iavf_queues_req_reset(struct rte_eth_dev *dev, uint16_t num) 435 { 436 struct iavf_adapter *ad = 437 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 438 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 439 int ret; 440 441 ret = iavf_request_queues(dev, num); 442 if (ret) { 443 PMD_DRV_LOG(ERR, "request queues from PF failed"); 444 return ret; 445 } 446 PMD_DRV_LOG(INFO, "change queue pairs from %u to %u", 447 vf->vsi_res->num_queue_pairs, num); 448 449 ret = iavf_dev_reset(dev); 450 if (ret) { 451 PMD_DRV_LOG(ERR, "vf reset failed"); 452 return ret; 453 } 454 455 return 0; 456 } 457 458 static int 459 iavf_dev_vlan_insert_set(struct rte_eth_dev *dev) 460 { 461 struct iavf_adapter *adapter = 462 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 463 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 464 bool enable; 465 466 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2)) 467 return 0; 468 469 enable = !!(dev->data->dev_conf.txmode.offloads & 470 DEV_TX_OFFLOAD_VLAN_INSERT); 471 iavf_config_vlan_insert_v2(adapter, enable); 472 473 return 0; 474 } 475 476 static int 477 iavf_dev_init_vlan(struct rte_eth_dev *dev) 478 { 479 int err; 480 481 err = iavf_dev_vlan_offload_set(dev, 482 ETH_VLAN_STRIP_MASK | 483 ETH_QINQ_STRIP_MASK | 484 ETH_VLAN_FILTER_MASK | 485 ETH_VLAN_EXTEND_MASK); 486 if (err) { 487 PMD_DRV_LOG(ERR, "Failed to update vlan offload"); 488 return err; 489 } 490 491 err = iavf_dev_vlan_insert_set(dev); 492 if (err) 493 PMD_DRV_LOG(ERR, "Failed to update vlan insertion"); 494 495 return err; 496 } 497 498 static int 499 iavf_dev_configure(struct rte_eth_dev *dev) 500 { 501 struct iavf_adapter *ad = 502 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 503 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(ad); 504 uint16_t num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 505 dev->data->nb_tx_queues); 506 int ret; 507 508 ad->rx_bulk_alloc_allowed = true; 509 /* Initialize to TRUE. If any of Rx queues doesn't meet the 510 * vector Rx/Tx preconditions, it will be reset. 511 */ 512 ad->rx_vec_allowed = true; 513 ad->tx_vec_allowed = true; 514 515 if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) 516 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH; 517 518 /* Large VF setting */ 519 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_DFLT) { 520 if (!(vf->vf_res->vf_cap_flags & 521 VIRTCHNL_VF_LARGE_NUM_QPAIRS)) { 522 PMD_DRV_LOG(ERR, "large VF is not supported"); 523 return -1; 524 } 525 526 if (num_queue_pairs > IAVF_MAX_NUM_QUEUES_LV) { 527 PMD_DRV_LOG(ERR, "queue pairs number cannot be larger than %u", 528 IAVF_MAX_NUM_QUEUES_LV); 529 return -1; 530 } 531 532 ret = iavf_queues_req_reset(dev, num_queue_pairs); 533 if (ret) 534 return ret; 535 536 ret = iavf_get_max_rss_queue_region(ad); 537 if (ret) { 538 PMD_INIT_LOG(ERR, "get max rss queue region failed"); 539 return ret; 540 } 541 542 vf->lv_enabled = true; 543 } else { 544 /* Check if large VF is already enabled. If so, disable and 545 * release redundant queue resource. 546 * Or check if enough queue pairs. If not, request them from PF. 547 */ 548 if (vf->lv_enabled || 549 num_queue_pairs > vf->vsi_res->num_queue_pairs) { 550 ret = iavf_queues_req_reset(dev, num_queue_pairs); 551 if (ret) 552 return ret; 553 554 vf->lv_enabled = false; 555 } 556 /* if large VF is not required, use default rss queue region */ 557 vf->max_rss_qregion = IAVF_MAX_NUM_QUEUES_DFLT; 558 } 559 560 ret = iavf_dev_init_vlan(dev); 561 if (ret) 562 PMD_DRV_LOG(ERR, "configure VLAN failed: %d", ret); 563 564 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 565 if (iavf_init_rss(ad) != 0) { 566 PMD_DRV_LOG(ERR, "configure rss failed"); 567 return -1; 568 } 569 } 570 return 0; 571 } 572 573 static int 574 iavf_init_rxq(struct rte_eth_dev *dev, struct iavf_rx_queue *rxq) 575 { 576 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 577 struct rte_eth_dev_data *dev_data = dev->data; 578 uint16_t buf_size, max_pkt_len; 579 580 buf_size = rte_pktmbuf_data_room_size(rxq->mp) - RTE_PKTMBUF_HEADROOM; 581 582 /* Calculate the maximum packet length allowed */ 583 max_pkt_len = RTE_MIN((uint32_t) 584 rxq->rx_buf_len * IAVF_MAX_CHAINED_RX_BUFFERS, 585 dev->data->dev_conf.rxmode.max_rx_pkt_len); 586 587 /* Check if the jumbo frame and maximum packet length are set 588 * correctly. 589 */ 590 if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 591 if (max_pkt_len <= IAVF_ETH_MAX_LEN || 592 max_pkt_len > IAVF_FRAME_SIZE_MAX) { 593 PMD_DRV_LOG(ERR, "maximum packet length must be " 594 "larger than %u and smaller than %u, " 595 "as jumbo frame is enabled", 596 (uint32_t)IAVF_ETH_MAX_LEN, 597 (uint32_t)IAVF_FRAME_SIZE_MAX); 598 return -EINVAL; 599 } 600 } else { 601 if (max_pkt_len < RTE_ETHER_MIN_LEN || 602 max_pkt_len > IAVF_ETH_MAX_LEN) { 603 PMD_DRV_LOG(ERR, "maximum packet length must be " 604 "larger than %u and smaller than %u, " 605 "as jumbo frame is disabled", 606 (uint32_t)RTE_ETHER_MIN_LEN, 607 (uint32_t)IAVF_ETH_MAX_LEN); 608 return -EINVAL; 609 } 610 } 611 612 rxq->max_pkt_len = max_pkt_len; 613 if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) || 614 rxq->max_pkt_len > buf_size) { 615 dev_data->scattered_rx = 1; 616 } 617 IAVF_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 618 IAVF_WRITE_FLUSH(hw); 619 620 return 0; 621 } 622 623 static int 624 iavf_init_queues(struct rte_eth_dev *dev) 625 { 626 struct iavf_rx_queue **rxq = 627 (struct iavf_rx_queue **)dev->data->rx_queues; 628 int i, ret = IAVF_SUCCESS; 629 630 for (i = 0; i < dev->data->nb_rx_queues; i++) { 631 if (!rxq[i] || !rxq[i]->q_set) 632 continue; 633 ret = iavf_init_rxq(dev, rxq[i]); 634 if (ret != IAVF_SUCCESS) 635 break; 636 } 637 /* set rx/tx function to vector/scatter/single-segment 638 * according to parameters 639 */ 640 iavf_set_rx_function(dev); 641 iavf_set_tx_function(dev); 642 643 return ret; 644 } 645 646 static int iavf_config_rx_queues_irqs(struct rte_eth_dev *dev, 647 struct rte_intr_handle *intr_handle) 648 { 649 struct iavf_adapter *adapter = 650 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 651 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 652 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 653 struct iavf_qv_map *qv_map; 654 uint16_t interval, i; 655 int vec; 656 657 if (rte_intr_cap_multiple(intr_handle) && 658 dev->data->dev_conf.intr_conf.rxq) { 659 if (rte_intr_efd_enable(intr_handle, dev->data->nb_rx_queues)) 660 return -1; 661 } 662 663 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 664 intr_handle->intr_vec = 665 rte_zmalloc("intr_vec", 666 dev->data->nb_rx_queues * sizeof(int), 0); 667 if (!intr_handle->intr_vec) { 668 PMD_DRV_LOG(ERR, "Failed to allocate %d rx intr_vec", 669 dev->data->nb_rx_queues); 670 return -1; 671 } 672 } 673 674 qv_map = rte_zmalloc("qv_map", 675 dev->data->nb_rx_queues * sizeof(struct iavf_qv_map), 0); 676 if (!qv_map) { 677 PMD_DRV_LOG(ERR, "Failed to allocate %d queue-vector map", 678 dev->data->nb_rx_queues); 679 goto qv_map_alloc_err; 680 } 681 682 if (!dev->data->dev_conf.intr_conf.rxq || 683 !rte_intr_dp_is_en(intr_handle)) { 684 /* Rx interrupt disabled, Map interrupt only for writeback */ 685 vf->nb_msix = 1; 686 if (vf->vf_res->vf_cap_flags & 687 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 688 /* If WB_ON_ITR supports, enable it */ 689 vf->msix_base = IAVF_RX_VEC_START; 690 /* Set the ITR for index zero, to 2us to make sure that 691 * we leave time for aggregation to occur, but don't 692 * increase latency dramatically. 693 */ 694 IAVF_WRITE_REG(hw, 695 IAVF_VFINT_DYN_CTLN1(vf->msix_base - 1), 696 (0 << IAVF_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) | 697 IAVF_VFINT_DYN_CTLN1_WB_ON_ITR_MASK | 698 (2UL << IAVF_VFINT_DYN_CTLN1_INTERVAL_SHIFT)); 699 /* debug - check for success! the return value 700 * should be 2, offset is 0x2800 701 */ 702 /* IAVF_READ_REG(hw, IAVF_VFINT_ITRN1(0, 0)); */ 703 } else { 704 /* If no WB_ON_ITR offload flags, need to set 705 * interrupt for descriptor write back. 706 */ 707 vf->msix_base = IAVF_MISC_VEC_ID; 708 709 /* set ITR to default */ 710 interval = iavf_calc_itr_interval( 711 IAVF_QUEUE_ITR_INTERVAL_DEFAULT); 712 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 713 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 714 (IAVF_ITR_INDEX_DEFAULT << 715 IAVF_VFINT_DYN_CTL01_ITR_INDX_SHIFT) | 716 (interval << 717 IAVF_VFINT_DYN_CTL01_INTERVAL_SHIFT)); 718 } 719 IAVF_WRITE_FLUSH(hw); 720 /* map all queues to the same interrupt */ 721 for (i = 0; i < dev->data->nb_rx_queues; i++) { 722 qv_map[i].queue_id = i; 723 qv_map[i].vector_id = vf->msix_base; 724 } 725 vf->qv_map = qv_map; 726 } else { 727 if (!rte_intr_allow_others(intr_handle)) { 728 vf->nb_msix = 1; 729 vf->msix_base = IAVF_MISC_VEC_ID; 730 for (i = 0; i < dev->data->nb_rx_queues; i++) { 731 qv_map[i].queue_id = i; 732 qv_map[i].vector_id = vf->msix_base; 733 intr_handle->intr_vec[i] = IAVF_MISC_VEC_ID; 734 } 735 vf->qv_map = qv_map; 736 PMD_DRV_LOG(DEBUG, 737 "vector %u are mapping to all Rx queues", 738 vf->msix_base); 739 } else { 740 /* If Rx interrupt is reuquired, and we can use 741 * multi interrupts, then the vec is from 1 742 */ 743 vf->nb_msix = RTE_MIN(intr_handle->nb_efd, 744 (uint16_t)(vf->vf_res->max_vectors - 1)); 745 vf->msix_base = IAVF_RX_VEC_START; 746 vec = IAVF_RX_VEC_START; 747 for (i = 0; i < dev->data->nb_rx_queues; i++) { 748 qv_map[i].queue_id = i; 749 qv_map[i].vector_id = vec; 750 intr_handle->intr_vec[i] = vec++; 751 if (vec >= vf->nb_msix + IAVF_RX_VEC_START) 752 vec = IAVF_RX_VEC_START; 753 } 754 vf->qv_map = qv_map; 755 PMD_DRV_LOG(DEBUG, 756 "%u vectors are mapping to %u Rx queues", 757 vf->nb_msix, dev->data->nb_rx_queues); 758 } 759 } 760 761 if (!vf->lv_enabled) { 762 if (iavf_config_irq_map(adapter)) { 763 PMD_DRV_LOG(ERR, "config interrupt mapping failed"); 764 goto config_irq_map_err; 765 } 766 } else { 767 uint16_t num_qv_maps = dev->data->nb_rx_queues; 768 uint16_t index = 0; 769 770 while (num_qv_maps > IAVF_IRQ_MAP_NUM_PER_BUF) { 771 if (iavf_config_irq_map_lv(adapter, 772 IAVF_IRQ_MAP_NUM_PER_BUF, index)) { 773 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 774 goto config_irq_map_err; 775 } 776 num_qv_maps -= IAVF_IRQ_MAP_NUM_PER_BUF; 777 index += IAVF_IRQ_MAP_NUM_PER_BUF; 778 } 779 780 if (iavf_config_irq_map_lv(adapter, num_qv_maps, index)) { 781 PMD_DRV_LOG(ERR, "config interrupt mapping for large VF failed"); 782 goto config_irq_map_err; 783 } 784 } 785 return 0; 786 787 config_irq_map_err: 788 rte_free(vf->qv_map); 789 vf->qv_map = NULL; 790 791 qv_map_alloc_err: 792 rte_free(intr_handle->intr_vec); 793 intr_handle->intr_vec = NULL; 794 795 return -1; 796 } 797 798 static int 799 iavf_start_queues(struct rte_eth_dev *dev) 800 { 801 struct iavf_rx_queue *rxq; 802 struct iavf_tx_queue *txq; 803 int i; 804 805 for (i = 0; i < dev->data->nb_tx_queues; i++) { 806 txq = dev->data->tx_queues[i]; 807 if (txq->tx_deferred_start) 808 continue; 809 if (iavf_dev_tx_queue_start(dev, i) != 0) { 810 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 811 return -1; 812 } 813 } 814 815 for (i = 0; i < dev->data->nb_rx_queues; i++) { 816 rxq = dev->data->rx_queues[i]; 817 if (rxq->rx_deferred_start) 818 continue; 819 if (iavf_dev_rx_queue_start(dev, i) != 0) { 820 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 821 return -1; 822 } 823 } 824 825 return 0; 826 } 827 828 static int 829 iavf_dev_start(struct rte_eth_dev *dev) 830 { 831 struct iavf_adapter *adapter = 832 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 833 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 834 struct rte_intr_handle *intr_handle = dev->intr_handle; 835 uint16_t num_queue_pairs; 836 uint16_t index = 0; 837 838 PMD_INIT_FUNC_TRACE(); 839 840 adapter->stopped = 0; 841 842 vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; 843 vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 844 dev->data->nb_tx_queues); 845 num_queue_pairs = vf->num_queue_pairs; 846 847 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 848 if (iavf_get_qos_cap(adapter)) { 849 PMD_INIT_LOG(ERR, "Failed to get qos capability"); 850 return -1; 851 } 852 853 if (iavf_init_queues(dev) != 0) { 854 PMD_DRV_LOG(ERR, "failed to do Queue init"); 855 return -1; 856 } 857 858 /* If needed, send configure queues msg multiple times to make the 859 * adminq buffer length smaller than the 4K limitation. 860 */ 861 while (num_queue_pairs > IAVF_CFG_Q_NUM_PER_BUF) { 862 if (iavf_configure_queues(adapter, 863 IAVF_CFG_Q_NUM_PER_BUF, index) != 0) { 864 PMD_DRV_LOG(ERR, "configure queues failed"); 865 goto err_queue; 866 } 867 num_queue_pairs -= IAVF_CFG_Q_NUM_PER_BUF; 868 index += IAVF_CFG_Q_NUM_PER_BUF; 869 } 870 871 if (iavf_configure_queues(adapter, num_queue_pairs, index) != 0) { 872 PMD_DRV_LOG(ERR, "configure queues failed"); 873 goto err_queue; 874 } 875 876 if (iavf_config_rx_queues_irqs(dev, intr_handle) != 0) { 877 PMD_DRV_LOG(ERR, "configure irq failed"); 878 goto err_queue; 879 } 880 /* re-enable intr again, because efd assign may change */ 881 if (dev->data->dev_conf.intr_conf.rxq != 0) { 882 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 883 rte_intr_disable(intr_handle); 884 rte_intr_enable(intr_handle); 885 } 886 887 /* Set all mac addrs */ 888 iavf_add_del_all_mac_addr(adapter, true); 889 890 /* Set all multicast addresses */ 891 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 892 true); 893 894 if (iavf_start_queues(dev) != 0) { 895 PMD_DRV_LOG(ERR, "enable queues failed"); 896 goto err_mac; 897 } 898 899 return 0; 900 901 err_mac: 902 iavf_add_del_all_mac_addr(adapter, false); 903 err_queue: 904 return -1; 905 } 906 907 static int 908 iavf_dev_stop(struct rte_eth_dev *dev) 909 { 910 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 911 struct iavf_adapter *adapter = 912 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 913 struct rte_intr_handle *intr_handle = dev->intr_handle; 914 915 PMD_INIT_FUNC_TRACE(); 916 917 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) && 918 dev->data->dev_conf.intr_conf.rxq != 0) 919 rte_intr_disable(intr_handle); 920 921 if (adapter->stopped == 1) 922 return 0; 923 924 iavf_stop_queues(dev); 925 926 /* Disable the interrupt for Rx */ 927 rte_intr_efd_disable(intr_handle); 928 /* Rx interrupt vector mapping free */ 929 if (intr_handle->intr_vec) { 930 rte_free(intr_handle->intr_vec); 931 intr_handle->intr_vec = NULL; 932 } 933 934 /* remove all mac addrs */ 935 iavf_add_del_all_mac_addr(adapter, false); 936 937 /* remove all multicast addresses */ 938 iavf_add_del_mc_addr_list(adapter, vf->mc_addrs, vf->mc_addrs_num, 939 false); 940 941 adapter->stopped = 1; 942 dev->data->dev_started = 0; 943 944 return 0; 945 } 946 947 static int 948 iavf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 949 { 950 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 951 952 dev_info->max_rx_queues = IAVF_MAX_NUM_QUEUES_LV; 953 dev_info->max_tx_queues = IAVF_MAX_NUM_QUEUES_LV; 954 dev_info->min_rx_bufsize = IAVF_BUF_SIZE_MIN; 955 dev_info->max_rx_pktlen = IAVF_FRAME_SIZE_MAX; 956 dev_info->max_mtu = dev_info->max_rx_pktlen - IAVF_ETH_OVERHEAD; 957 dev_info->min_mtu = RTE_ETHER_MIN_MTU; 958 dev_info->hash_key_size = vf->vf_res->rss_key_size; 959 dev_info->reta_size = vf->vf_res->rss_lut_size; 960 dev_info->flow_type_rss_offloads = IAVF_RSS_OFFLOAD_ALL; 961 dev_info->max_mac_addrs = IAVF_NUM_MACADDR_MAX; 962 dev_info->rx_offload_capa = 963 DEV_RX_OFFLOAD_VLAN_STRIP | 964 DEV_RX_OFFLOAD_QINQ_STRIP | 965 DEV_RX_OFFLOAD_IPV4_CKSUM | 966 DEV_RX_OFFLOAD_UDP_CKSUM | 967 DEV_RX_OFFLOAD_TCP_CKSUM | 968 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 969 DEV_RX_OFFLOAD_SCATTER | 970 DEV_RX_OFFLOAD_JUMBO_FRAME | 971 DEV_RX_OFFLOAD_VLAN_FILTER | 972 DEV_RX_OFFLOAD_RSS_HASH; 973 974 dev_info->tx_offload_capa = 975 DEV_TX_OFFLOAD_VLAN_INSERT | 976 DEV_TX_OFFLOAD_QINQ_INSERT | 977 DEV_TX_OFFLOAD_IPV4_CKSUM | 978 DEV_TX_OFFLOAD_UDP_CKSUM | 979 DEV_TX_OFFLOAD_TCP_CKSUM | 980 DEV_TX_OFFLOAD_SCTP_CKSUM | 981 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 982 DEV_TX_OFFLOAD_TCP_TSO | 983 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 984 DEV_TX_OFFLOAD_GRE_TNL_TSO | 985 DEV_TX_OFFLOAD_IPIP_TNL_TSO | 986 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | 987 DEV_TX_OFFLOAD_MULTI_SEGS | 988 DEV_TX_OFFLOAD_MBUF_FAST_FREE; 989 990 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_CRC) 991 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_KEEP_CRC; 992 993 dev_info->default_rxconf = (struct rte_eth_rxconf) { 994 .rx_free_thresh = IAVF_DEFAULT_RX_FREE_THRESH, 995 .rx_drop_en = 0, 996 .offloads = 0, 997 }; 998 999 dev_info->default_txconf = (struct rte_eth_txconf) { 1000 .tx_free_thresh = IAVF_DEFAULT_TX_FREE_THRESH, 1001 .tx_rs_thresh = IAVF_DEFAULT_TX_RS_THRESH, 1002 .offloads = 0, 1003 }; 1004 1005 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 1006 .nb_max = IAVF_MAX_RING_DESC, 1007 .nb_min = IAVF_MIN_RING_DESC, 1008 .nb_align = IAVF_ALIGN_RING_DESC, 1009 }; 1010 1011 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 1012 .nb_max = IAVF_MAX_RING_DESC, 1013 .nb_min = IAVF_MIN_RING_DESC, 1014 .nb_align = IAVF_ALIGN_RING_DESC, 1015 }; 1016 1017 return 0; 1018 } 1019 1020 static const uint32_t * 1021 iavf_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused) 1022 { 1023 static const uint32_t ptypes[] = { 1024 RTE_PTYPE_L2_ETHER, 1025 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN, 1026 RTE_PTYPE_L4_FRAG, 1027 RTE_PTYPE_L4_ICMP, 1028 RTE_PTYPE_L4_NONFRAG, 1029 RTE_PTYPE_L4_SCTP, 1030 RTE_PTYPE_L4_TCP, 1031 RTE_PTYPE_L4_UDP, 1032 RTE_PTYPE_UNKNOWN 1033 }; 1034 return ptypes; 1035 } 1036 1037 int 1038 iavf_dev_link_update(struct rte_eth_dev *dev, 1039 __rte_unused int wait_to_complete) 1040 { 1041 struct rte_eth_link new_link; 1042 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1043 1044 memset(&new_link, 0, sizeof(new_link)); 1045 1046 /* Only read status info stored in VF, and the info is updated 1047 * when receive LINK_CHANGE evnet from PF by Virtchnnl. 1048 */ 1049 switch (vf->link_speed) { 1050 case 10: 1051 new_link.link_speed = ETH_SPEED_NUM_10M; 1052 break; 1053 case 100: 1054 new_link.link_speed = ETH_SPEED_NUM_100M; 1055 break; 1056 case 1000: 1057 new_link.link_speed = ETH_SPEED_NUM_1G; 1058 break; 1059 case 10000: 1060 new_link.link_speed = ETH_SPEED_NUM_10G; 1061 break; 1062 case 20000: 1063 new_link.link_speed = ETH_SPEED_NUM_20G; 1064 break; 1065 case 25000: 1066 new_link.link_speed = ETH_SPEED_NUM_25G; 1067 break; 1068 case 40000: 1069 new_link.link_speed = ETH_SPEED_NUM_40G; 1070 break; 1071 case 50000: 1072 new_link.link_speed = ETH_SPEED_NUM_50G; 1073 break; 1074 case 100000: 1075 new_link.link_speed = ETH_SPEED_NUM_100G; 1076 break; 1077 default: 1078 new_link.link_speed = ETH_SPEED_NUM_NONE; 1079 break; 1080 } 1081 1082 new_link.link_duplex = ETH_LINK_FULL_DUPLEX; 1083 new_link.link_status = vf->link_up ? ETH_LINK_UP : 1084 ETH_LINK_DOWN; 1085 new_link.link_autoneg = !(dev->data->dev_conf.link_speeds & 1086 ETH_LINK_SPEED_FIXED); 1087 1088 return rte_eth_linkstatus_set(dev, &new_link); 1089 } 1090 1091 static int 1092 iavf_dev_promiscuous_enable(struct rte_eth_dev *dev) 1093 { 1094 struct iavf_adapter *adapter = 1095 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1096 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1097 1098 return iavf_config_promisc(adapter, 1099 true, vf->promisc_multicast_enabled); 1100 } 1101 1102 static int 1103 iavf_dev_promiscuous_disable(struct rte_eth_dev *dev) 1104 { 1105 struct iavf_adapter *adapter = 1106 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1107 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1108 1109 return iavf_config_promisc(adapter, 1110 false, vf->promisc_multicast_enabled); 1111 } 1112 1113 static int 1114 iavf_dev_allmulticast_enable(struct rte_eth_dev *dev) 1115 { 1116 struct iavf_adapter *adapter = 1117 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1118 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1119 1120 return iavf_config_promisc(adapter, 1121 vf->promisc_unicast_enabled, true); 1122 } 1123 1124 static int 1125 iavf_dev_allmulticast_disable(struct rte_eth_dev *dev) 1126 { 1127 struct iavf_adapter *adapter = 1128 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1129 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1130 1131 return iavf_config_promisc(adapter, 1132 vf->promisc_unicast_enabled, false); 1133 } 1134 1135 static int 1136 iavf_dev_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *addr, 1137 __rte_unused uint32_t index, 1138 __rte_unused uint32_t pool) 1139 { 1140 struct iavf_adapter *adapter = 1141 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1142 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1143 int err; 1144 1145 if (rte_is_zero_ether_addr(addr)) { 1146 PMD_DRV_LOG(ERR, "Invalid Ethernet Address"); 1147 return -EINVAL; 1148 } 1149 1150 err = iavf_add_del_eth_addr(adapter, addr, true, VIRTCHNL_ETHER_ADDR_EXTRA); 1151 if (err) { 1152 PMD_DRV_LOG(ERR, "fail to add MAC address"); 1153 return -EIO; 1154 } 1155 1156 vf->mac_num++; 1157 1158 return 0; 1159 } 1160 1161 static void 1162 iavf_dev_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) 1163 { 1164 struct iavf_adapter *adapter = 1165 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1166 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1167 struct rte_ether_addr *addr; 1168 int err; 1169 1170 addr = &dev->data->mac_addrs[index]; 1171 1172 err = iavf_add_del_eth_addr(adapter, addr, false, VIRTCHNL_ETHER_ADDR_EXTRA); 1173 if (err) 1174 PMD_DRV_LOG(ERR, "fail to delete MAC address"); 1175 1176 vf->mac_num--; 1177 } 1178 1179 static int 1180 iavf_dev_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1181 { 1182 struct iavf_adapter *adapter = 1183 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1184 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1185 int err; 1186 1187 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 1188 err = iavf_add_del_vlan_v2(adapter, vlan_id, on); 1189 if (err) 1190 return -EIO; 1191 return 0; 1192 } 1193 1194 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1195 return -ENOTSUP; 1196 1197 err = iavf_add_del_vlan(adapter, vlan_id, on); 1198 if (err) 1199 return -EIO; 1200 return 0; 1201 } 1202 1203 static void 1204 iavf_iterate_vlan_filters_v2(struct rte_eth_dev *dev, bool enable) 1205 { 1206 struct rte_vlan_filter_conf *vfc = &dev->data->vlan_filter_conf; 1207 struct iavf_adapter *adapter = 1208 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1209 uint32_t i, j; 1210 uint64_t ids; 1211 1212 for (i = 0; i < RTE_DIM(vfc->ids); i++) { 1213 if (vfc->ids[i] == 0) 1214 continue; 1215 1216 ids = vfc->ids[i]; 1217 for (j = 0; ids != 0 && j < 64; j++, ids >>= 1) { 1218 if (ids & 1) 1219 iavf_add_del_vlan_v2(adapter, 1220 64 * i + j, enable); 1221 } 1222 } 1223 } 1224 1225 static int 1226 iavf_dev_vlan_offload_set_v2(struct rte_eth_dev *dev, int mask) 1227 { 1228 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode; 1229 struct iavf_adapter *adapter = 1230 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1231 bool enable; 1232 int err; 1233 1234 if (mask & ETH_VLAN_FILTER_MASK) { 1235 enable = !!(rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER); 1236 1237 iavf_iterate_vlan_filters_v2(dev, enable); 1238 } 1239 1240 if (mask & ETH_VLAN_STRIP_MASK) { 1241 enable = !!(rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP); 1242 1243 err = iavf_config_vlan_strip_v2(adapter, enable); 1244 /* If not support, the stripping is already disabled by PF */ 1245 if (err == -ENOTSUP && !enable) 1246 err = 0; 1247 if (err) 1248 return -EIO; 1249 } 1250 1251 return 0; 1252 } 1253 1254 static int 1255 iavf_dev_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1256 { 1257 struct iavf_adapter *adapter = 1258 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1259 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1260 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 1261 int err; 1262 1263 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) 1264 return iavf_dev_vlan_offload_set_v2(dev, mask); 1265 1266 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1267 return -ENOTSUP; 1268 1269 /* Vlan stripping setting */ 1270 if (mask & ETH_VLAN_STRIP_MASK) { 1271 /* Enable or disable VLAN stripping */ 1272 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1273 err = iavf_enable_vlan_strip(adapter); 1274 else 1275 err = iavf_disable_vlan_strip(adapter); 1276 1277 if (err) 1278 return -EIO; 1279 } 1280 return 0; 1281 } 1282 1283 static int 1284 iavf_dev_rss_reta_update(struct rte_eth_dev *dev, 1285 struct rte_eth_rss_reta_entry64 *reta_conf, 1286 uint16_t reta_size) 1287 { 1288 struct iavf_adapter *adapter = 1289 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1290 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1291 uint8_t *lut; 1292 uint16_t i, idx, shift; 1293 int ret; 1294 1295 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1296 return -ENOTSUP; 1297 1298 if (reta_size != vf->vf_res->rss_lut_size) { 1299 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1300 "(%d) doesn't match the number of hardware can " 1301 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1302 return -EINVAL; 1303 } 1304 1305 lut = rte_zmalloc("rss_lut", reta_size, 0); 1306 if (!lut) { 1307 PMD_DRV_LOG(ERR, "No memory can be allocated"); 1308 return -ENOMEM; 1309 } 1310 /* store the old lut table temporarily */ 1311 rte_memcpy(lut, vf->rss_lut, reta_size); 1312 1313 for (i = 0; i < reta_size; i++) { 1314 idx = i / RTE_RETA_GROUP_SIZE; 1315 shift = i % RTE_RETA_GROUP_SIZE; 1316 if (reta_conf[idx].mask & (1ULL << shift)) 1317 lut[i] = reta_conf[idx].reta[shift]; 1318 } 1319 1320 rte_memcpy(vf->rss_lut, lut, reta_size); 1321 /* send virtchnnl ops to configure rss*/ 1322 ret = iavf_configure_rss_lut(adapter); 1323 if (ret) /* revert back */ 1324 rte_memcpy(vf->rss_lut, lut, reta_size); 1325 rte_free(lut); 1326 1327 return ret; 1328 } 1329 1330 static int 1331 iavf_dev_rss_reta_query(struct rte_eth_dev *dev, 1332 struct rte_eth_rss_reta_entry64 *reta_conf, 1333 uint16_t reta_size) 1334 { 1335 struct iavf_adapter *adapter = 1336 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1337 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1338 uint16_t i, idx, shift; 1339 1340 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1341 return -ENOTSUP; 1342 1343 if (reta_size != vf->vf_res->rss_lut_size) { 1344 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 1345 "(%d) doesn't match the number of hardware can " 1346 "support (%d)", reta_size, vf->vf_res->rss_lut_size); 1347 return -EINVAL; 1348 } 1349 1350 for (i = 0; i < reta_size; i++) { 1351 idx = i / RTE_RETA_GROUP_SIZE; 1352 shift = i % RTE_RETA_GROUP_SIZE; 1353 if (reta_conf[idx].mask & (1ULL << shift)) 1354 reta_conf[idx].reta[shift] = vf->rss_lut[i]; 1355 } 1356 1357 return 0; 1358 } 1359 1360 static int 1361 iavf_set_rss_key(struct iavf_adapter *adapter, uint8_t *key, uint8_t key_len) 1362 { 1363 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1364 1365 /* HENA setting, it is enabled by default, no change */ 1366 if (!key || key_len == 0) { 1367 PMD_DRV_LOG(DEBUG, "No key to be configured"); 1368 return 0; 1369 } else if (key_len != vf->vf_res->rss_key_size) { 1370 PMD_DRV_LOG(ERR, "The size of hash key configured " 1371 "(%d) doesn't match the size of hardware can " 1372 "support (%d)", key_len, 1373 vf->vf_res->rss_key_size); 1374 return -EINVAL; 1375 } 1376 1377 rte_memcpy(vf->rss_key, key, key_len); 1378 1379 return iavf_configure_rss_key(adapter); 1380 } 1381 1382 static int 1383 iavf_dev_rss_hash_update(struct rte_eth_dev *dev, 1384 struct rte_eth_rss_conf *rss_conf) 1385 { 1386 struct iavf_adapter *adapter = 1387 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1388 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1389 int ret; 1390 1391 adapter->dev_data->dev_conf.rx_adv_conf.rss_conf = *rss_conf; 1392 1393 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1394 return -ENOTSUP; 1395 1396 /* Set hash key. */ 1397 ret = iavf_set_rss_key(adapter, rss_conf->rss_key, 1398 rss_conf->rss_key_len); 1399 if (ret) 1400 return ret; 1401 1402 if (rss_conf->rss_hf == 0) { 1403 vf->rss_hf = 0; 1404 ret = iavf_set_hena(adapter, 0); 1405 1406 /* It is a workaround, temporarily allow error to be returned 1407 * due to possible lack of PF handling for hena = 0. 1408 */ 1409 if (ret) 1410 PMD_DRV_LOG(WARNING, "fail to clean existing RSS, lack PF support"); 1411 return 0; 1412 } 1413 1414 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF) { 1415 /* Clear existing RSS. */ 1416 ret = iavf_set_hena(adapter, 0); 1417 1418 /* It is a workaround, temporarily allow error to be returned 1419 * due to possible lack of PF handling for hena = 0. 1420 */ 1421 if (ret) 1422 PMD_DRV_LOG(WARNING, "fail to clean existing RSS," 1423 "lack PF support"); 1424 1425 /* Set new RSS configuration. */ 1426 ret = iavf_rss_hash_set(adapter, rss_conf->rss_hf, true); 1427 if (ret) { 1428 PMD_DRV_LOG(ERR, "fail to set new RSS"); 1429 return ret; 1430 } 1431 } else { 1432 iavf_config_rss_hf(adapter, rss_conf->rss_hf); 1433 } 1434 1435 return 0; 1436 } 1437 1438 static int 1439 iavf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 1440 struct rte_eth_rss_conf *rss_conf) 1441 { 1442 struct iavf_adapter *adapter = 1443 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1444 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1445 1446 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) 1447 return -ENOTSUP; 1448 1449 rss_conf->rss_hf = vf->rss_hf; 1450 1451 if (!rss_conf->rss_key) 1452 return 0; 1453 1454 rss_conf->rss_key_len = vf->vf_res->rss_key_size; 1455 rte_memcpy(rss_conf->rss_key, vf->rss_key, rss_conf->rss_key_len); 1456 1457 return 0; 1458 } 1459 1460 static int 1461 iavf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 1462 { 1463 uint32_t frame_size = mtu + IAVF_ETH_OVERHEAD; 1464 int ret = 0; 1465 1466 if (mtu < RTE_ETHER_MIN_MTU || frame_size > IAVF_FRAME_SIZE_MAX) 1467 return -EINVAL; 1468 1469 /* mtu setting is forbidden if port is start */ 1470 if (dev->data->dev_started) { 1471 PMD_DRV_LOG(ERR, "port must be stopped before configuration"); 1472 return -EBUSY; 1473 } 1474 1475 if (frame_size > IAVF_ETH_MAX_LEN) 1476 dev->data->dev_conf.rxmode.offloads |= 1477 DEV_RX_OFFLOAD_JUMBO_FRAME; 1478 else 1479 dev->data->dev_conf.rxmode.offloads &= 1480 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 1481 1482 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 1483 1484 return ret; 1485 } 1486 1487 static int 1488 iavf_dev_set_default_mac_addr(struct rte_eth_dev *dev, 1489 struct rte_ether_addr *mac_addr) 1490 { 1491 struct iavf_adapter *adapter = 1492 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1493 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1494 struct rte_ether_addr *old_addr; 1495 int ret; 1496 1497 old_addr = (struct rte_ether_addr *)hw->mac.addr; 1498 1499 if (rte_is_same_ether_addr(old_addr, mac_addr)) 1500 return 0; 1501 1502 ret = iavf_add_del_eth_addr(adapter, old_addr, false, VIRTCHNL_ETHER_ADDR_PRIMARY); 1503 if (ret) 1504 PMD_DRV_LOG(ERR, "Fail to delete old MAC:" 1505 RTE_ETHER_ADDR_PRT_FMT, 1506 RTE_ETHER_ADDR_BYTES(old_addr)); 1507 1508 ret = iavf_add_del_eth_addr(adapter, mac_addr, true, VIRTCHNL_ETHER_ADDR_PRIMARY); 1509 if (ret) 1510 PMD_DRV_LOG(ERR, "Fail to add new MAC:" 1511 RTE_ETHER_ADDR_PRT_FMT, 1512 RTE_ETHER_ADDR_BYTES(mac_addr)); 1513 1514 if (ret) 1515 return -EIO; 1516 1517 rte_ether_addr_copy(mac_addr, (struct rte_ether_addr *)hw->mac.addr); 1518 return 0; 1519 } 1520 1521 static void 1522 iavf_stat_update_48(uint64_t *offset, uint64_t *stat) 1523 { 1524 if (*stat >= *offset) 1525 *stat = *stat - *offset; 1526 else 1527 *stat = (uint64_t)((*stat + 1528 ((uint64_t)1 << IAVF_48_BIT_WIDTH)) - *offset); 1529 1530 *stat &= IAVF_48_BIT_MASK; 1531 } 1532 1533 static void 1534 iavf_stat_update_32(uint64_t *offset, uint64_t *stat) 1535 { 1536 if (*stat >= *offset) 1537 *stat = (uint64_t)(*stat - *offset); 1538 else 1539 *stat = (uint64_t)((*stat + 1540 ((uint64_t)1 << IAVF_32_BIT_WIDTH)) - *offset); 1541 } 1542 1543 static void 1544 iavf_update_stats(struct iavf_vsi *vsi, struct virtchnl_eth_stats *nes) 1545 { 1546 struct virtchnl_eth_stats *oes = &vsi->eth_stats_offset; 1547 1548 iavf_stat_update_48(&oes->rx_bytes, &nes->rx_bytes); 1549 iavf_stat_update_48(&oes->rx_unicast, &nes->rx_unicast); 1550 iavf_stat_update_48(&oes->rx_multicast, &nes->rx_multicast); 1551 iavf_stat_update_48(&oes->rx_broadcast, &nes->rx_broadcast); 1552 iavf_stat_update_32(&oes->rx_discards, &nes->rx_discards); 1553 iavf_stat_update_48(&oes->tx_bytes, &nes->tx_bytes); 1554 iavf_stat_update_48(&oes->tx_unicast, &nes->tx_unicast); 1555 iavf_stat_update_48(&oes->tx_multicast, &nes->tx_multicast); 1556 iavf_stat_update_48(&oes->tx_broadcast, &nes->tx_broadcast); 1557 iavf_stat_update_32(&oes->tx_errors, &nes->tx_errors); 1558 iavf_stat_update_32(&oes->tx_discards, &nes->tx_discards); 1559 } 1560 1561 static int 1562 iavf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 1563 { 1564 struct iavf_adapter *adapter = 1565 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1566 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1567 struct iavf_vsi *vsi = &vf->vsi; 1568 struct virtchnl_eth_stats *pstats = NULL; 1569 int ret; 1570 1571 ret = iavf_query_stats(adapter, &pstats); 1572 if (ret == 0) { 1573 uint8_t crc_stats_len = (dev->data->dev_conf.rxmode.offloads & 1574 DEV_RX_OFFLOAD_KEEP_CRC) ? 0 : 1575 RTE_ETHER_CRC_LEN; 1576 iavf_update_stats(vsi, pstats); 1577 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + 1578 pstats->rx_broadcast - pstats->rx_discards; 1579 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + 1580 pstats->tx_unicast; 1581 stats->imissed = pstats->rx_discards; 1582 stats->oerrors = pstats->tx_errors + pstats->tx_discards; 1583 stats->ibytes = pstats->rx_bytes; 1584 stats->ibytes -= stats->ipackets * crc_stats_len; 1585 stats->obytes = pstats->tx_bytes; 1586 } else { 1587 PMD_DRV_LOG(ERR, "Get statistics failed"); 1588 } 1589 return ret; 1590 } 1591 1592 static int 1593 iavf_dev_stats_reset(struct rte_eth_dev *dev) 1594 { 1595 int ret; 1596 struct iavf_adapter *adapter = 1597 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1598 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1599 struct iavf_vsi *vsi = &vf->vsi; 1600 struct virtchnl_eth_stats *pstats = NULL; 1601 1602 /* read stat values to clear hardware registers */ 1603 ret = iavf_query_stats(adapter, &pstats); 1604 if (ret != 0) 1605 return ret; 1606 1607 /* set stats offset base on current values */ 1608 vsi->eth_stats_offset = *pstats; 1609 1610 return 0; 1611 } 1612 1613 static int iavf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 1614 struct rte_eth_xstat_name *xstats_names, 1615 __rte_unused unsigned int limit) 1616 { 1617 unsigned int i; 1618 1619 if (xstats_names != NULL) 1620 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1621 snprintf(xstats_names[i].name, 1622 sizeof(xstats_names[i].name), 1623 "%s", rte_iavf_stats_strings[i].name); 1624 } 1625 return IAVF_NB_XSTATS; 1626 } 1627 1628 static int iavf_dev_xstats_get(struct rte_eth_dev *dev, 1629 struct rte_eth_xstat *xstats, unsigned int n) 1630 { 1631 int ret; 1632 unsigned int i; 1633 struct iavf_adapter *adapter = 1634 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1635 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1636 struct iavf_vsi *vsi = &vf->vsi; 1637 struct virtchnl_eth_stats *pstats = NULL; 1638 1639 if (n < IAVF_NB_XSTATS) 1640 return IAVF_NB_XSTATS; 1641 1642 ret = iavf_query_stats(adapter, &pstats); 1643 if (ret != 0) 1644 return 0; 1645 1646 if (!xstats) 1647 return 0; 1648 1649 iavf_update_stats(vsi, pstats); 1650 1651 /* loop over xstats array and values from pstats */ 1652 for (i = 0; i < IAVF_NB_XSTATS; i++) { 1653 xstats[i].id = i; 1654 xstats[i].value = *(uint64_t *)(((char *)pstats) + 1655 rte_iavf_stats_strings[i].offset); 1656 } 1657 1658 return IAVF_NB_XSTATS; 1659 } 1660 1661 1662 static int 1663 iavf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1664 { 1665 struct iavf_adapter *adapter = 1666 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1667 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1668 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 1669 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 1670 uint16_t msix_intr; 1671 1672 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1673 if (msix_intr == IAVF_MISC_VEC_ID) { 1674 PMD_DRV_LOG(INFO, "MISC is also enabled for control"); 1675 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 1676 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 1677 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1678 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 1679 } else { 1680 IAVF_WRITE_REG(hw, 1681 IAVF_VFINT_DYN_CTLN1 1682 (msix_intr - IAVF_RX_VEC_START), 1683 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 1684 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 1685 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 1686 } 1687 1688 IAVF_WRITE_FLUSH(hw); 1689 1690 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 1691 rte_intr_ack(&pci_dev->intr_handle); 1692 1693 return 0; 1694 } 1695 1696 static int 1697 iavf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1698 { 1699 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1700 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1701 uint16_t msix_intr; 1702 1703 msix_intr = pci_dev->intr_handle.intr_vec[queue_id]; 1704 if (msix_intr == IAVF_MISC_VEC_ID) { 1705 PMD_DRV_LOG(ERR, "MISC is used for control, cannot disable it"); 1706 return -EIO; 1707 } 1708 1709 IAVF_WRITE_REG(hw, 1710 IAVF_VFINT_DYN_CTLN1(msix_intr - IAVF_RX_VEC_START), 1711 0); 1712 1713 IAVF_WRITE_FLUSH(hw); 1714 return 0; 1715 } 1716 1717 static int 1718 iavf_check_vf_reset_done(struct iavf_hw *hw) 1719 { 1720 int i, reset; 1721 1722 for (i = 0; i < IAVF_RESET_WAIT_CNT; i++) { 1723 reset = IAVF_READ_REG(hw, IAVF_VFGEN_RSTAT) & 1724 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 1725 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 1726 if (reset == VIRTCHNL_VFR_VFACTIVE || 1727 reset == VIRTCHNL_VFR_COMPLETED) 1728 break; 1729 rte_delay_ms(20); 1730 } 1731 1732 if (i >= IAVF_RESET_WAIT_CNT) 1733 return -1; 1734 1735 return 0; 1736 } 1737 1738 static int 1739 iavf_lookup_proto_xtr_type(const char *flex_name) 1740 { 1741 static struct { 1742 const char *name; 1743 enum iavf_proto_xtr_type type; 1744 } xtr_type_map[] = { 1745 { "vlan", IAVF_PROTO_XTR_VLAN }, 1746 { "ipv4", IAVF_PROTO_XTR_IPV4 }, 1747 { "ipv6", IAVF_PROTO_XTR_IPV6 }, 1748 { "ipv6_flow", IAVF_PROTO_XTR_IPV6_FLOW }, 1749 { "tcp", IAVF_PROTO_XTR_TCP }, 1750 { "ip_offset", IAVF_PROTO_XTR_IP_OFFSET }, 1751 }; 1752 uint32_t i; 1753 1754 for (i = 0; i < RTE_DIM(xtr_type_map); i++) { 1755 if (strcmp(flex_name, xtr_type_map[i].name) == 0) 1756 return xtr_type_map[i].type; 1757 } 1758 1759 PMD_DRV_LOG(ERR, "wrong proto_xtr type, " 1760 "it should be: vlan|ipv4|ipv6|ipv6_flow|tcp|ip_offset"); 1761 1762 return -1; 1763 } 1764 1765 /** 1766 * Parse elem, the elem could be single number/range or '(' ')' group 1767 * 1) A single number elem, it's just a simple digit. e.g. 9 1768 * 2) A single range elem, two digits with a '-' between. e.g. 2-6 1769 * 3) A group elem, combines multiple 1) or 2) with '( )'. e.g (0,2-4,6) 1770 * Within group elem, '-' used for a range separator; 1771 * ',' used for a single number. 1772 */ 1773 static int 1774 iavf_parse_queue_set(const char *input, int xtr_type, 1775 struct iavf_devargs *devargs) 1776 { 1777 const char *str = input; 1778 char *end = NULL; 1779 uint32_t min, max; 1780 uint32_t idx; 1781 1782 while (isblank(*str)) 1783 str++; 1784 1785 if (!isdigit(*str) && *str != '(') 1786 return -1; 1787 1788 /* process single number or single range of number */ 1789 if (*str != '(') { 1790 errno = 0; 1791 idx = strtoul(str, &end, 10); 1792 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1793 return -1; 1794 1795 while (isblank(*end)) 1796 end++; 1797 1798 min = idx; 1799 max = idx; 1800 1801 /* process single <number>-<number> */ 1802 if (*end == '-') { 1803 end++; 1804 while (isblank(*end)) 1805 end++; 1806 if (!isdigit(*end)) 1807 return -1; 1808 1809 errno = 0; 1810 idx = strtoul(end, &end, 10); 1811 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1812 return -1; 1813 1814 max = idx; 1815 while (isblank(*end)) 1816 end++; 1817 } 1818 1819 if (*end != ':') 1820 return -1; 1821 1822 for (idx = RTE_MIN(min, max); 1823 idx <= RTE_MAX(min, max); idx++) 1824 devargs->proto_xtr[idx] = xtr_type; 1825 1826 return 0; 1827 } 1828 1829 /* process set within bracket */ 1830 str++; 1831 while (isblank(*str)) 1832 str++; 1833 if (*str == '\0') 1834 return -1; 1835 1836 min = IAVF_MAX_QUEUE_NUM; 1837 do { 1838 /* go ahead to the first digit */ 1839 while (isblank(*str)) 1840 str++; 1841 if (!isdigit(*str)) 1842 return -1; 1843 1844 /* get the digit value */ 1845 errno = 0; 1846 idx = strtoul(str, &end, 10); 1847 if (errno || !end || idx >= IAVF_MAX_QUEUE_NUM) 1848 return -1; 1849 1850 /* go ahead to separator '-',',' and ')' */ 1851 while (isblank(*end)) 1852 end++; 1853 if (*end == '-') { 1854 if (min == IAVF_MAX_QUEUE_NUM) 1855 min = idx; 1856 else /* avoid continuous '-' */ 1857 return -1; 1858 } else if (*end == ',' || *end == ')') { 1859 max = idx; 1860 if (min == IAVF_MAX_QUEUE_NUM) 1861 min = idx; 1862 1863 for (idx = RTE_MIN(min, max); 1864 idx <= RTE_MAX(min, max); idx++) 1865 devargs->proto_xtr[idx] = xtr_type; 1866 1867 min = IAVF_MAX_QUEUE_NUM; 1868 } else { 1869 return -1; 1870 } 1871 1872 str = end + 1; 1873 } while (*end != ')' && *end != '\0'); 1874 1875 return 0; 1876 } 1877 1878 static int 1879 iavf_parse_queue_proto_xtr(const char *queues, struct iavf_devargs *devargs) 1880 { 1881 const char *queue_start; 1882 uint32_t idx; 1883 int xtr_type; 1884 char flex_name[32]; 1885 1886 while (isblank(*queues)) 1887 queues++; 1888 1889 if (*queues != '[') { 1890 xtr_type = iavf_lookup_proto_xtr_type(queues); 1891 if (xtr_type < 0) 1892 return -1; 1893 1894 devargs->proto_xtr_dflt = xtr_type; 1895 1896 return 0; 1897 } 1898 1899 queues++; 1900 do { 1901 while (isblank(*queues)) 1902 queues++; 1903 if (*queues == '\0') 1904 return -1; 1905 1906 queue_start = queues; 1907 1908 /* go across a complete bracket */ 1909 if (*queue_start == '(') { 1910 queues += strcspn(queues, ")"); 1911 if (*queues != ')') 1912 return -1; 1913 } 1914 1915 /* scan the separator ':' */ 1916 queues += strcspn(queues, ":"); 1917 if (*queues++ != ':') 1918 return -1; 1919 while (isblank(*queues)) 1920 queues++; 1921 1922 for (idx = 0; ; idx++) { 1923 if (isblank(queues[idx]) || 1924 queues[idx] == ',' || 1925 queues[idx] == ']' || 1926 queues[idx] == '\0') 1927 break; 1928 1929 if (idx > sizeof(flex_name) - 2) 1930 return -1; 1931 1932 flex_name[idx] = queues[idx]; 1933 } 1934 flex_name[idx] = '\0'; 1935 xtr_type = iavf_lookup_proto_xtr_type(flex_name); 1936 if (xtr_type < 0) 1937 return -1; 1938 1939 queues += idx; 1940 1941 while (isblank(*queues) || *queues == ',' || *queues == ']') 1942 queues++; 1943 1944 if (iavf_parse_queue_set(queue_start, xtr_type, devargs) < 0) 1945 return -1; 1946 } while (*queues != '\0'); 1947 1948 return 0; 1949 } 1950 1951 static int 1952 iavf_handle_proto_xtr_arg(__rte_unused const char *key, const char *value, 1953 void *extra_args) 1954 { 1955 struct iavf_devargs *devargs = extra_args; 1956 1957 if (!value || !extra_args) 1958 return -EINVAL; 1959 1960 if (iavf_parse_queue_proto_xtr(value, devargs) < 0) { 1961 PMD_DRV_LOG(ERR, "the proto_xtr's parameter is wrong : '%s'", 1962 value); 1963 return -1; 1964 } 1965 1966 return 0; 1967 } 1968 1969 static int iavf_parse_devargs(struct rte_eth_dev *dev) 1970 { 1971 struct iavf_adapter *ad = 1972 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1973 struct rte_devargs *devargs = dev->device->devargs; 1974 struct rte_kvargs *kvlist; 1975 int ret; 1976 1977 if (!devargs) 1978 return 0; 1979 1980 kvlist = rte_kvargs_parse(devargs->args, iavf_valid_args); 1981 if (!kvlist) { 1982 PMD_INIT_LOG(ERR, "invalid kvargs key\n"); 1983 return -EINVAL; 1984 } 1985 1986 ad->devargs.proto_xtr_dflt = IAVF_PROTO_XTR_NONE; 1987 memset(ad->devargs.proto_xtr, IAVF_PROTO_XTR_NONE, 1988 sizeof(ad->devargs.proto_xtr)); 1989 1990 ret = rte_kvargs_process(kvlist, IAVF_PROTO_XTR_ARG, 1991 &iavf_handle_proto_xtr_arg, &ad->devargs); 1992 if (ret) 1993 goto bail; 1994 1995 bail: 1996 rte_kvargs_free(kvlist); 1997 return ret; 1998 } 1999 2000 static void 2001 iavf_init_proto_xtr(struct rte_eth_dev *dev) 2002 { 2003 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2004 struct iavf_adapter *ad = 2005 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2006 const struct iavf_proto_xtr_ol *xtr_ol; 2007 bool proto_xtr_enable = false; 2008 int offset; 2009 uint16_t i; 2010 2011 vf->proto_xtr = rte_zmalloc("vf proto xtr", 2012 vf->vsi_res->num_queue_pairs, 0); 2013 if (unlikely(!(vf->proto_xtr))) { 2014 PMD_DRV_LOG(ERR, "no memory for setting up proto_xtr's table"); 2015 return; 2016 } 2017 2018 for (i = 0; i < vf->vsi_res->num_queue_pairs; i++) { 2019 vf->proto_xtr[i] = ad->devargs.proto_xtr[i] != 2020 IAVF_PROTO_XTR_NONE ? 2021 ad->devargs.proto_xtr[i] : 2022 ad->devargs.proto_xtr_dflt; 2023 2024 if (vf->proto_xtr[i] != IAVF_PROTO_XTR_NONE) { 2025 uint8_t type = vf->proto_xtr[i]; 2026 2027 iavf_proto_xtr_params[type].required = true; 2028 proto_xtr_enable = true; 2029 } 2030 } 2031 2032 if (likely(!proto_xtr_enable)) 2033 return; 2034 2035 offset = rte_mbuf_dynfield_register(&iavf_proto_xtr_metadata_param); 2036 if (unlikely(offset == -1)) { 2037 PMD_DRV_LOG(ERR, 2038 "failed to extract protocol metadata, error %d", 2039 -rte_errno); 2040 return; 2041 } 2042 2043 PMD_DRV_LOG(DEBUG, 2044 "proto_xtr metadata offset in mbuf is : %d", 2045 offset); 2046 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = offset; 2047 2048 for (i = 0; i < RTE_DIM(iavf_proto_xtr_params); i++) { 2049 xtr_ol = &iavf_proto_xtr_params[i]; 2050 2051 uint8_t rxdid = iavf_proto_xtr_type_to_rxdid((uint8_t)i); 2052 2053 if (!xtr_ol->required) 2054 continue; 2055 2056 if (!(vf->supported_rxdid & BIT(rxdid))) { 2057 PMD_DRV_LOG(ERR, 2058 "rxdid[%u] is not supported in hardware", 2059 rxdid); 2060 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 2061 break; 2062 } 2063 2064 offset = rte_mbuf_dynflag_register(&xtr_ol->param); 2065 if (unlikely(offset == -1)) { 2066 PMD_DRV_LOG(ERR, 2067 "failed to register proto_xtr offload '%s', error %d", 2068 xtr_ol->param.name, -rte_errno); 2069 2070 rte_pmd_ifd_dynfield_proto_xtr_metadata_offs = -1; 2071 break; 2072 } 2073 2074 PMD_DRV_LOG(DEBUG, 2075 "proto_xtr offload '%s' offset in mbuf is : %d", 2076 xtr_ol->param.name, offset); 2077 *xtr_ol->ol_flag = 1ULL << offset; 2078 } 2079 } 2080 2081 static int 2082 iavf_init_vf(struct rte_eth_dev *dev) 2083 { 2084 int err, bufsz; 2085 struct iavf_adapter *adapter = 2086 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2087 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2088 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2089 2090 vf->eth_dev = dev; 2091 2092 err = iavf_parse_devargs(dev); 2093 if (err) { 2094 PMD_INIT_LOG(ERR, "Failed to parse devargs"); 2095 goto err; 2096 } 2097 2098 err = iavf_set_mac_type(hw); 2099 if (err) { 2100 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err); 2101 goto err; 2102 } 2103 2104 err = iavf_check_vf_reset_done(hw); 2105 if (err) { 2106 PMD_INIT_LOG(ERR, "VF is still resetting"); 2107 goto err; 2108 } 2109 2110 iavf_init_adminq_parameter(hw); 2111 err = iavf_init_adminq(hw); 2112 if (err) { 2113 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err); 2114 goto err; 2115 } 2116 2117 vf->aq_resp = rte_zmalloc("vf_aq_resp", IAVF_AQ_BUF_SZ, 0); 2118 if (!vf->aq_resp) { 2119 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); 2120 goto err_aq; 2121 } 2122 if (iavf_check_api_version(adapter) != 0) { 2123 PMD_INIT_LOG(ERR, "check_api version failed"); 2124 goto err_api; 2125 } 2126 2127 bufsz = sizeof(struct virtchnl_vf_resource) + 2128 (IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); 2129 vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); 2130 if (!vf->vf_res) { 2131 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 2132 goto err_api; 2133 } 2134 2135 if (iavf_get_vf_resource(adapter) != 0) { 2136 PMD_INIT_LOG(ERR, "iavf_get_vf_config failed"); 2137 goto err_alloc; 2138 } 2139 /* Allocate memort for RSS info */ 2140 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2141 vf->rss_key = rte_zmalloc("rss_key", 2142 vf->vf_res->rss_key_size, 0); 2143 if (!vf->rss_key) { 2144 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 2145 goto err_rss; 2146 } 2147 vf->rss_lut = rte_zmalloc("rss_lut", 2148 vf->vf_res->rss_lut_size, 0); 2149 if (!vf->rss_lut) { 2150 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 2151 goto err_rss; 2152 } 2153 } 2154 2155 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 2156 if (iavf_get_supported_rxdid(adapter) != 0) { 2157 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 2158 goto err_rss; 2159 } 2160 } 2161 2162 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN_V2) { 2163 if (iavf_get_vlan_offload_caps_v2(adapter) != 0) { 2164 PMD_INIT_LOG(ERR, "failed to do get VLAN offload v2 capabilities"); 2165 goto err_rss; 2166 } 2167 } 2168 2169 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) { 2170 bufsz = sizeof(struct virtchnl_qos_cap_list) + 2171 IAVF_MAX_TRAFFIC_CLASS * 2172 sizeof(struct virtchnl_qos_cap_elem); 2173 vf->qos_cap = rte_zmalloc("qos_cap", bufsz, 0); 2174 if (!vf->qos_cap) { 2175 PMD_INIT_LOG(ERR, "unable to allocate qos_cap memory"); 2176 goto err_rss; 2177 } 2178 iavf_tm_conf_init(dev); 2179 } 2180 2181 iavf_init_proto_xtr(dev); 2182 2183 return 0; 2184 err_rss: 2185 rte_free(vf->rss_key); 2186 rte_free(vf->rss_lut); 2187 err_alloc: 2188 rte_free(vf->qos_cap); 2189 rte_free(vf->vf_res); 2190 vf->vsi_res = NULL; 2191 err_api: 2192 rte_free(vf->aq_resp); 2193 err_aq: 2194 iavf_shutdown_adminq(hw); 2195 err: 2196 return -1; 2197 } 2198 2199 static void 2200 iavf_uninit_vf(struct rte_eth_dev *dev) 2201 { 2202 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2203 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2204 2205 iavf_shutdown_adminq(hw); 2206 2207 rte_free(vf->vf_res); 2208 vf->vsi_res = NULL; 2209 vf->vf_res = NULL; 2210 2211 rte_free(vf->aq_resp); 2212 vf->aq_resp = NULL; 2213 2214 rte_free(vf->qos_cap); 2215 vf->qos_cap = NULL; 2216 2217 rte_free(vf->rss_lut); 2218 vf->rss_lut = NULL; 2219 rte_free(vf->rss_key); 2220 vf->rss_key = NULL; 2221 } 2222 2223 /* Enable default admin queue interrupt setting */ 2224 static inline void 2225 iavf_enable_irq0(struct iavf_hw *hw) 2226 { 2227 /* Enable admin queue interrupt trigger */ 2228 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 2229 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 2230 2231 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 2232 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 2233 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 2234 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 2235 2236 IAVF_WRITE_FLUSH(hw); 2237 } 2238 2239 static inline void 2240 iavf_disable_irq0(struct iavf_hw *hw) 2241 { 2242 /* Disable all interrupt types */ 2243 IAVF_WRITE_REG(hw, IAVF_VFINT_ICR0_ENA1, 0); 2244 IAVF_WRITE_REG(hw, IAVF_VFINT_DYN_CTL01, 2245 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 2246 IAVF_WRITE_FLUSH(hw); 2247 } 2248 2249 static void 2250 iavf_dev_interrupt_handler(void *param) 2251 { 2252 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2253 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2254 2255 iavf_disable_irq0(hw); 2256 2257 iavf_handle_virtchnl_msg(dev); 2258 2259 iavf_enable_irq0(hw); 2260 } 2261 2262 void 2263 iavf_dev_alarm_handler(void *param) 2264 { 2265 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 2266 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2267 uint32_t icr0; 2268 2269 iavf_disable_irq0(hw); 2270 2271 /* read out interrupt causes */ 2272 icr0 = IAVF_READ_REG(hw, IAVF_VFINT_ICR01); 2273 2274 if (icr0 & IAVF_VFINT_ICR01_ADMINQ_MASK) { 2275 PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported"); 2276 iavf_handle_virtchnl_msg(dev); 2277 } 2278 2279 iavf_enable_irq0(hw); 2280 2281 rte_eal_alarm_set(IAVF_ALARM_INTERVAL, 2282 iavf_dev_alarm_handler, dev); 2283 } 2284 2285 static int 2286 iavf_dev_flow_ops_get(struct rte_eth_dev *dev, 2287 const struct rte_flow_ops **ops) 2288 { 2289 if (!dev) 2290 return -EINVAL; 2291 2292 *ops = &iavf_flow_ops; 2293 return 0; 2294 } 2295 2296 static void 2297 iavf_default_rss_disable(struct iavf_adapter *adapter) 2298 { 2299 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 2300 int ret = 0; 2301 2302 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2303 /* Set hena = 0 to ask PF to cleanup all existing RSS. */ 2304 ret = iavf_set_hena(adapter, 0); 2305 if (ret) 2306 /* It is a workaround, temporarily allow error to be 2307 * returned due to possible lack of PF handling for 2308 * hena = 0. 2309 */ 2310 PMD_INIT_LOG(WARNING, "fail to disable default RSS," 2311 "lack PF support"); 2312 } 2313 } 2314 2315 static int 2316 iavf_dev_init(struct rte_eth_dev *eth_dev) 2317 { 2318 struct iavf_adapter *adapter = 2319 IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 2320 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(adapter); 2321 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(adapter); 2322 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 2323 int ret = 0; 2324 2325 PMD_INIT_FUNC_TRACE(); 2326 2327 /* assign ops func pointer */ 2328 eth_dev->dev_ops = &iavf_eth_dev_ops; 2329 eth_dev->rx_queue_count = iavf_dev_rxq_count; 2330 eth_dev->rx_descriptor_status = iavf_dev_rx_desc_status; 2331 eth_dev->tx_descriptor_status = iavf_dev_tx_desc_status; 2332 eth_dev->rx_pkt_burst = &iavf_recv_pkts; 2333 eth_dev->tx_pkt_burst = &iavf_xmit_pkts; 2334 eth_dev->tx_pkt_prepare = &iavf_prep_pkts; 2335 2336 /* For secondary processes, we don't initialise any further as primary 2337 * has already done this work. Only check if we need a different RX 2338 * and TX function. 2339 */ 2340 if (rte_eal_process_type() != RTE_PROC_PRIMARY) { 2341 iavf_set_rx_function(eth_dev); 2342 iavf_set_tx_function(eth_dev); 2343 return 0; 2344 } 2345 rte_eth_copy_pci_info(eth_dev, pci_dev); 2346 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS; 2347 2348 hw->vendor_id = pci_dev->id.vendor_id; 2349 hw->device_id = pci_dev->id.device_id; 2350 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 2351 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 2352 hw->bus.bus_id = pci_dev->addr.bus; 2353 hw->bus.device = pci_dev->addr.devid; 2354 hw->bus.func = pci_dev->addr.function; 2355 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 2356 hw->back = IAVF_DEV_PRIVATE_TO_ADAPTER(eth_dev->data->dev_private); 2357 adapter->dev_data = eth_dev->data; 2358 adapter->stopped = 1; 2359 2360 if (iavf_init_vf(eth_dev) != 0) { 2361 PMD_INIT_LOG(ERR, "Init vf failed"); 2362 return -1; 2363 } 2364 2365 /* set default ptype table */ 2366 adapter->ptype_tbl = iavf_get_default_ptype_table(); 2367 2368 /* copy mac addr */ 2369 eth_dev->data->mac_addrs = rte_zmalloc( 2370 "iavf_mac", RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX, 0); 2371 if (!eth_dev->data->mac_addrs) { 2372 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to" 2373 " store MAC addresses", 2374 RTE_ETHER_ADDR_LEN * IAVF_NUM_MACADDR_MAX); 2375 ret = -ENOMEM; 2376 goto init_vf_err; 2377 } 2378 /* If the MAC address is not configured by host, 2379 * generate a random one. 2380 */ 2381 if (!rte_is_valid_assigned_ether_addr( 2382 (struct rte_ether_addr *)hw->mac.addr)) 2383 rte_eth_random_addr(hw->mac.addr); 2384 rte_ether_addr_copy((struct rte_ether_addr *)hw->mac.addr, 2385 ð_dev->data->mac_addrs[0]); 2386 2387 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 2388 /* register callback func to eal lib */ 2389 rte_intr_callback_register(&pci_dev->intr_handle, 2390 iavf_dev_interrupt_handler, 2391 (void *)eth_dev); 2392 2393 /* enable uio intr after callback register */ 2394 rte_intr_enable(&pci_dev->intr_handle); 2395 } else { 2396 rte_eal_alarm_set(IAVF_ALARM_INTERVAL, 2397 iavf_dev_alarm_handler, eth_dev); 2398 } 2399 2400 /* configure and enable device interrupt */ 2401 iavf_enable_irq0(hw); 2402 2403 ret = iavf_flow_init(adapter); 2404 if (ret) { 2405 PMD_INIT_LOG(ERR, "Failed to initialize flow"); 2406 goto flow_init_err; 2407 } 2408 2409 iavf_default_rss_disable(adapter); 2410 2411 return 0; 2412 2413 flow_init_err: 2414 rte_free(eth_dev->data->mac_addrs); 2415 eth_dev->data->mac_addrs = NULL; 2416 2417 init_vf_err: 2418 iavf_uninit_vf(eth_dev); 2419 2420 return ret; 2421 } 2422 2423 static int 2424 iavf_dev_close(struct rte_eth_dev *dev) 2425 { 2426 struct iavf_hw *hw = IAVF_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2427 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2428 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2429 struct iavf_adapter *adapter = 2430 IAVF_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 2431 struct iavf_info *vf = IAVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2432 int ret; 2433 2434 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2435 return 0; 2436 2437 ret = iavf_dev_stop(dev); 2438 2439 iavf_flow_flush(dev, NULL); 2440 iavf_flow_uninit(adapter); 2441 2442 /* 2443 * disable promiscuous mode before reset vf 2444 * it is a workaround solution when work with kernel driver 2445 * and it is not the normal way 2446 */ 2447 if (vf->promisc_unicast_enabled || vf->promisc_multicast_enabled) 2448 iavf_config_promisc(adapter, false, false); 2449 2450 iavf_shutdown_adminq(hw); 2451 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) { 2452 /* disable uio intr before callback unregister */ 2453 rte_intr_disable(intr_handle); 2454 2455 /* unregister callback func from eal lib */ 2456 rte_intr_callback_unregister(intr_handle, 2457 iavf_dev_interrupt_handler, dev); 2458 } else { 2459 rte_eal_alarm_cancel(iavf_dev_alarm_handler, dev); 2460 } 2461 iavf_disable_irq0(hw); 2462 2463 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 2464 iavf_tm_conf_uninit(dev); 2465 2466 if (vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2467 if (vf->rss_lut) { 2468 rte_free(vf->rss_lut); 2469 vf->rss_lut = NULL; 2470 } 2471 if (vf->rss_key) { 2472 rte_free(vf->rss_key); 2473 vf->rss_key = NULL; 2474 } 2475 } 2476 2477 rte_free(vf->vf_res); 2478 vf->vsi_res = NULL; 2479 vf->vf_res = NULL; 2480 2481 rte_free(vf->aq_resp); 2482 vf->aq_resp = NULL; 2483 2484 /* 2485 * If the VF is reset via VFLR, the device will be knocked out of bus 2486 * master mode, and the driver will fail to recover from the reset. Fix 2487 * this by enabling bus mastering after every reset. In a non-VFLR case, 2488 * the bus master bit will not be disabled, and this call will have no 2489 * effect. 2490 */ 2491 if (vf->vf_reset && !rte_pci_set_bus_master(pci_dev, true)) 2492 vf->vf_reset = false; 2493 2494 return ret; 2495 } 2496 2497 static int 2498 iavf_dev_uninit(struct rte_eth_dev *dev) 2499 { 2500 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 2501 return -EPERM; 2502 2503 iavf_dev_close(dev); 2504 2505 return 0; 2506 } 2507 2508 /* 2509 * Reset VF device only to re-initialize resources in PMD layer 2510 */ 2511 static int 2512 iavf_dev_reset(struct rte_eth_dev *dev) 2513 { 2514 int ret; 2515 2516 ret = iavf_dev_uninit(dev); 2517 if (ret) 2518 return ret; 2519 2520 return iavf_dev_init(dev); 2521 } 2522 2523 static int 2524 iavf_dcf_cap_check_handler(__rte_unused const char *key, 2525 const char *value, __rte_unused void *opaque) 2526 { 2527 if (strcmp(value, "dcf")) 2528 return -1; 2529 2530 return 0; 2531 } 2532 2533 static int 2534 iavf_dcf_cap_selected(struct rte_devargs *devargs) 2535 { 2536 struct rte_kvargs *kvlist; 2537 const char *key = "cap"; 2538 int ret = 0; 2539 2540 if (devargs == NULL) 2541 return 0; 2542 2543 kvlist = rte_kvargs_parse(devargs->args, NULL); 2544 if (kvlist == NULL) 2545 return 0; 2546 2547 if (!rte_kvargs_count(kvlist, key)) 2548 goto exit; 2549 2550 /* dcf capability selected when there's a key-value pair: cap=dcf */ 2551 if (rte_kvargs_process(kvlist, key, 2552 iavf_dcf_cap_check_handler, NULL) < 0) 2553 goto exit; 2554 2555 ret = 1; 2556 2557 exit: 2558 rte_kvargs_free(kvlist); 2559 return ret; 2560 } 2561 2562 static int eth_iavf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 2563 struct rte_pci_device *pci_dev) 2564 { 2565 if (iavf_dcf_cap_selected(pci_dev->device.devargs)) 2566 return 1; 2567 2568 return rte_eth_dev_pci_generic_probe(pci_dev, 2569 sizeof(struct iavf_adapter), iavf_dev_init); 2570 } 2571 2572 static int eth_iavf_pci_remove(struct rte_pci_device *pci_dev) 2573 { 2574 return rte_eth_dev_pci_generic_remove(pci_dev, iavf_dev_uninit); 2575 } 2576 2577 /* Adaptive virtual function driver struct */ 2578 static struct rte_pci_driver rte_iavf_pmd = { 2579 .id_table = pci_id_iavf_map, 2580 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, 2581 .probe = eth_iavf_pci_probe, 2582 .remove = eth_iavf_pci_remove, 2583 }; 2584 2585 RTE_PMD_REGISTER_PCI(net_iavf, rte_iavf_pmd); 2586 RTE_PMD_REGISTER_PCI_TABLE(net_iavf, pci_id_iavf_map); 2587 RTE_PMD_REGISTER_KMOD_DEP(net_iavf, "* igb_uio | vfio-pci"); 2588 RTE_PMD_REGISTER_PARAM_STRING(net_iavf, "cap=dcf"); 2589 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_init, init, NOTICE); 2590 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_driver, driver, NOTICE); 2591 #ifdef RTE_ETHDEV_DEBUG_RX 2592 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_rx, rx, DEBUG); 2593 #endif 2594 #ifdef RTE_ETHDEV_DEBUG_TX 2595 RTE_LOG_REGISTER_SUFFIX(iavf_logtype_tx, tx, DEBUG); 2596 #endif 2597