1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2016 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 #include <rte_cycles.h> 16 17 #include <rte_interrupts.h> 18 #include <rte_log.h> 19 #include <rte_debug.h> 20 #include <rte_pci.h> 21 #include <rte_bus_pci.h> 22 #include <rte_atomic.h> 23 #include <rte_branch_prediction.h> 24 #include <rte_memory.h> 25 #include <rte_eal.h> 26 #include <rte_alarm.h> 27 #include <rte_ether.h> 28 #include <rte_ethdev_driver.h> 29 #include <rte_ethdev_pci.h> 30 #include <rte_malloc.h> 31 #include <rte_dev.h> 32 33 #include "i40e_logs.h" 34 #include "base/i40e_prototype.h" 35 #include "base/i40e_adminq_cmd.h" 36 #include "base/i40e_type.h" 37 38 #include "i40e_rxtx.h" 39 #include "i40e_ethdev.h" 40 #include "i40e_pf.h" 41 42 /* busy wait delay in msec */ 43 #define I40EVF_BUSY_WAIT_DELAY 10 44 #define I40EVF_BUSY_WAIT_COUNT 50 45 #define MAX_RESET_WAIT_CNT 20 46 47 #define I40EVF_ALARM_INTERVAL 50000 /* us */ 48 49 struct i40evf_arq_msg_info { 50 enum virtchnl_ops ops; 51 enum i40e_status_code result; 52 uint16_t buf_len; 53 uint16_t msg_len; 54 uint8_t *msg; 55 }; 56 57 struct vf_cmd_info { 58 enum virtchnl_ops ops; 59 uint8_t *in_args; 60 uint32_t in_args_size; 61 uint8_t *out_buffer; 62 /* Input & output type. pass in buffer size and pass out 63 * actual return result 64 */ 65 uint32_t out_size; 66 }; 67 68 enum i40evf_aq_result { 69 I40EVF_MSG_ERR = -1, /* Meet error when accessing admin queue */ 70 I40EVF_MSG_NON, /* Read nothing from admin queue */ 71 I40EVF_MSG_SYS, /* Read system msg from admin queue */ 72 I40EVF_MSG_CMD, /* Read async command result */ 73 }; 74 75 static int i40evf_dev_configure(struct rte_eth_dev *dev); 76 static int i40evf_dev_start(struct rte_eth_dev *dev); 77 static void i40evf_dev_stop(struct rte_eth_dev *dev); 78 static void i40evf_dev_info_get(struct rte_eth_dev *dev, 79 struct rte_eth_dev_info *dev_info); 80 static int i40evf_dev_link_update(struct rte_eth_dev *dev, 81 int wait_to_complete); 82 static int i40evf_dev_stats_get(struct rte_eth_dev *dev, 83 struct rte_eth_stats *stats); 84 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev, 85 struct rte_eth_xstat *xstats, unsigned n); 86 static int i40evf_dev_xstats_get_names(struct rte_eth_dev *dev, 87 struct rte_eth_xstat_name *xstats_names, 88 unsigned limit); 89 static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev); 90 static int i40evf_vlan_filter_set(struct rte_eth_dev *dev, 91 uint16_t vlan_id, int on); 92 static int i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask); 93 static void i40evf_dev_close(struct rte_eth_dev *dev); 94 static int i40evf_dev_reset(struct rte_eth_dev *dev); 95 static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev); 96 static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev); 97 static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev); 98 static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev); 99 static int i40evf_init_vlan(struct rte_eth_dev *dev); 100 static int i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, 101 uint16_t rx_queue_id); 102 static int i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, 103 uint16_t rx_queue_id); 104 static int i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, 105 uint16_t tx_queue_id); 106 static int i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, 107 uint16_t tx_queue_id); 108 static int i40evf_add_mac_addr(struct rte_eth_dev *dev, 109 struct ether_addr *addr, 110 uint32_t index, 111 uint32_t pool); 112 static void i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index); 113 static int i40evf_dev_rss_reta_update(struct rte_eth_dev *dev, 114 struct rte_eth_rss_reta_entry64 *reta_conf, 115 uint16_t reta_size); 116 static int i40evf_dev_rss_reta_query(struct rte_eth_dev *dev, 117 struct rte_eth_rss_reta_entry64 *reta_conf, 118 uint16_t reta_size); 119 static int i40evf_config_rss(struct i40e_vf *vf); 120 static int i40evf_dev_rss_hash_update(struct rte_eth_dev *dev, 121 struct rte_eth_rss_conf *rss_conf); 122 static int i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 123 struct rte_eth_rss_conf *rss_conf); 124 static int i40evf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); 125 static int i40evf_set_default_mac_addr(struct rte_eth_dev *dev, 126 struct ether_addr *mac_addr); 127 static int 128 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id); 129 static int 130 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id); 131 static void i40evf_handle_pf_event(struct rte_eth_dev *dev, 132 uint8_t *msg, 133 uint16_t msglen); 134 135 static int 136 i40evf_add_del_mc_addr_list(struct rte_eth_dev *dev, 137 struct ether_addr *mc_addr_set, 138 uint32_t nb_mc_addr, bool add); 139 static int 140 i40evf_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set, 141 uint32_t nb_mc_addr); 142 143 /* Default hash key buffer for RSS */ 144 static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1]; 145 146 struct rte_i40evf_xstats_name_off { 147 char name[RTE_ETH_XSTATS_NAME_SIZE]; 148 unsigned offset; 149 }; 150 151 static const struct rte_i40evf_xstats_name_off rte_i40evf_stats_strings[] = { 152 {"rx_bytes", offsetof(struct i40e_eth_stats, rx_bytes)}, 153 {"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)}, 154 {"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)}, 155 {"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)}, 156 {"rx_dropped_packets", offsetof(struct i40e_eth_stats, rx_discards)}, 157 {"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats, 158 rx_unknown_protocol)}, 159 {"tx_bytes", offsetof(struct i40e_eth_stats, tx_bytes)}, 160 {"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_unicast)}, 161 {"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_multicast)}, 162 {"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_broadcast)}, 163 {"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_discards)}, 164 {"tx_error_packets", offsetof(struct i40e_eth_stats, tx_errors)}, 165 }; 166 167 #define I40EVF_NB_XSTATS (sizeof(rte_i40evf_stats_strings) / \ 168 sizeof(rte_i40evf_stats_strings[0])) 169 170 static const struct eth_dev_ops i40evf_eth_dev_ops = { 171 .dev_configure = i40evf_dev_configure, 172 .dev_start = i40evf_dev_start, 173 .dev_stop = i40evf_dev_stop, 174 .promiscuous_enable = i40evf_dev_promiscuous_enable, 175 .promiscuous_disable = i40evf_dev_promiscuous_disable, 176 .allmulticast_enable = i40evf_dev_allmulticast_enable, 177 .allmulticast_disable = i40evf_dev_allmulticast_disable, 178 .link_update = i40evf_dev_link_update, 179 .stats_get = i40evf_dev_stats_get, 180 .stats_reset = i40evf_dev_xstats_reset, 181 .xstats_get = i40evf_dev_xstats_get, 182 .xstats_get_names = i40evf_dev_xstats_get_names, 183 .xstats_reset = i40evf_dev_xstats_reset, 184 .dev_close = i40evf_dev_close, 185 .dev_reset = i40evf_dev_reset, 186 .dev_infos_get = i40evf_dev_info_get, 187 .dev_supported_ptypes_get = i40e_dev_supported_ptypes_get, 188 .vlan_filter_set = i40evf_vlan_filter_set, 189 .vlan_offload_set = i40evf_vlan_offload_set, 190 .rx_queue_start = i40evf_dev_rx_queue_start, 191 .rx_queue_stop = i40evf_dev_rx_queue_stop, 192 .tx_queue_start = i40evf_dev_tx_queue_start, 193 .tx_queue_stop = i40evf_dev_tx_queue_stop, 194 .rx_queue_setup = i40e_dev_rx_queue_setup, 195 .rx_queue_release = i40e_dev_rx_queue_release, 196 .rx_queue_intr_enable = i40evf_dev_rx_queue_intr_enable, 197 .rx_queue_intr_disable = i40evf_dev_rx_queue_intr_disable, 198 .rx_descriptor_done = i40e_dev_rx_descriptor_done, 199 .rx_descriptor_status = i40e_dev_rx_descriptor_status, 200 .tx_descriptor_status = i40e_dev_tx_descriptor_status, 201 .tx_queue_setup = i40e_dev_tx_queue_setup, 202 .tx_queue_release = i40e_dev_tx_queue_release, 203 .rx_queue_count = i40e_dev_rx_queue_count, 204 .rxq_info_get = i40e_rxq_info_get, 205 .txq_info_get = i40e_txq_info_get, 206 .mac_addr_add = i40evf_add_mac_addr, 207 .mac_addr_remove = i40evf_del_mac_addr, 208 .set_mc_addr_list = i40evf_set_mc_addr_list, 209 .reta_update = i40evf_dev_rss_reta_update, 210 .reta_query = i40evf_dev_rss_reta_query, 211 .rss_hash_update = i40evf_dev_rss_hash_update, 212 .rss_hash_conf_get = i40evf_dev_rss_hash_conf_get, 213 .mtu_set = i40evf_dev_mtu_set, 214 .mac_addr_set = i40evf_set_default_mac_addr, 215 }; 216 217 /* 218 * Read data in admin queue to get msg from pf driver 219 */ 220 static enum i40evf_aq_result 221 i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data) 222 { 223 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 224 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 225 struct i40e_arq_event_info event; 226 enum virtchnl_ops opcode; 227 enum i40e_status_code retval; 228 int ret; 229 enum i40evf_aq_result result = I40EVF_MSG_NON; 230 231 event.buf_len = data->buf_len; 232 event.msg_buf = data->msg; 233 ret = i40e_clean_arq_element(hw, &event, NULL); 234 /* Can't read any msg from adminQ */ 235 if (ret) { 236 if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK) 237 result = I40EVF_MSG_ERR; 238 return result; 239 } 240 241 opcode = (enum virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high); 242 retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low); 243 /* pf sys event */ 244 if (opcode == VIRTCHNL_OP_EVENT) { 245 struct virtchnl_pf_event *vpe = 246 (struct virtchnl_pf_event *)event.msg_buf; 247 248 result = I40EVF_MSG_SYS; 249 switch (vpe->event) { 250 case VIRTCHNL_EVENT_LINK_CHANGE: 251 vf->link_up = 252 vpe->event_data.link_event.link_status; 253 vf->link_speed = 254 vpe->event_data.link_event.link_speed; 255 vf->pend_msg |= PFMSG_LINK_CHANGE; 256 PMD_DRV_LOG(INFO, "Link status update:%s", 257 vf->link_up ? "up" : "down"); 258 break; 259 case VIRTCHNL_EVENT_RESET_IMPENDING: 260 vf->vf_reset = true; 261 vf->pend_msg |= PFMSG_RESET_IMPENDING; 262 PMD_DRV_LOG(INFO, "vf is reseting"); 263 break; 264 case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: 265 vf->dev_closed = true; 266 vf->pend_msg |= PFMSG_DRIVER_CLOSE; 267 PMD_DRV_LOG(INFO, "PF driver closed"); 268 break; 269 default: 270 PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf", 271 __func__, vpe->event); 272 } 273 } else { 274 /* async reply msg on command issued by vf previously */ 275 result = I40EVF_MSG_CMD; 276 /* Actual data length read from PF */ 277 data->msg_len = event.msg_len; 278 } 279 280 data->result = retval; 281 data->ops = opcode; 282 283 return result; 284 } 285 286 /** 287 * clear current command. Only call in case execute 288 * _atomic_set_cmd successfully. 289 */ 290 static inline void 291 _clear_cmd(struct i40e_vf *vf) 292 { 293 rte_wmb(); 294 vf->pend_cmd = VIRTCHNL_OP_UNKNOWN; 295 } 296 297 /* 298 * Check there is pending cmd in execution. If none, set new command. 299 */ 300 static inline int 301 _atomic_set_cmd(struct i40e_vf *vf, enum virtchnl_ops ops) 302 { 303 int ret = rte_atomic32_cmpset(&vf->pend_cmd, 304 VIRTCHNL_OP_UNKNOWN, ops); 305 306 if (!ret) 307 PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd); 308 309 return !ret; 310 } 311 312 #define MAX_TRY_TIMES 200 313 #define ASQ_DELAY_MS 10 314 315 static int 316 i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args) 317 { 318 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 319 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 320 struct i40evf_arq_msg_info info; 321 enum i40evf_aq_result ret; 322 int err, i = 0; 323 324 if (_atomic_set_cmd(vf, args->ops)) 325 return -1; 326 327 info.msg = args->out_buffer; 328 info.buf_len = args->out_size; 329 info.ops = VIRTCHNL_OP_UNKNOWN; 330 info.result = I40E_SUCCESS; 331 332 err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS, 333 args->in_args, args->in_args_size, NULL); 334 if (err) { 335 PMD_DRV_LOG(ERR, "fail to send cmd %d", args->ops); 336 _clear_cmd(vf); 337 return err; 338 } 339 340 switch (args->ops) { 341 case VIRTCHNL_OP_RESET_VF: 342 /*no need to process in this function */ 343 err = 0; 344 break; 345 case VIRTCHNL_OP_VERSION: 346 case VIRTCHNL_OP_GET_VF_RESOURCES: 347 /* for init adminq commands, need to poll the response */ 348 err = -1; 349 do { 350 ret = i40evf_read_pfmsg(dev, &info); 351 vf->cmd_retval = info.result; 352 if (ret == I40EVF_MSG_CMD) { 353 err = 0; 354 break; 355 } else if (ret == I40EVF_MSG_ERR) 356 break; 357 rte_delay_ms(ASQ_DELAY_MS); 358 /* If don't read msg or read sys event, continue */ 359 } while (i++ < MAX_TRY_TIMES); 360 _clear_cmd(vf); 361 break; 362 363 default: 364 /* for other adminq in running time, waiting the cmd done flag */ 365 err = -1; 366 do { 367 if (vf->pend_cmd == VIRTCHNL_OP_UNKNOWN) { 368 err = 0; 369 break; 370 } 371 rte_delay_ms(ASQ_DELAY_MS); 372 /* If don't read msg or read sys event, continue */ 373 } while (i++ < MAX_TRY_TIMES); 374 /* If there's no response is received, clear command */ 375 if (i >= MAX_TRY_TIMES) { 376 PMD_DRV_LOG(WARNING, "No response for %d", args->ops); 377 _clear_cmd(vf); 378 } 379 break; 380 } 381 382 return err | vf->cmd_retval; 383 } 384 385 /* 386 * Check API version with sync wait until version read or fail from admin queue 387 */ 388 static int 389 i40evf_check_api_version(struct rte_eth_dev *dev) 390 { 391 struct virtchnl_version_info version, *pver; 392 int err; 393 struct vf_cmd_info args; 394 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 395 396 version.major = VIRTCHNL_VERSION_MAJOR; 397 version.minor = VIRTCHNL_VERSION_MINOR; 398 399 args.ops = VIRTCHNL_OP_VERSION; 400 args.in_args = (uint8_t *)&version; 401 args.in_args_size = sizeof(version); 402 args.out_buffer = vf->aq_resp; 403 args.out_size = I40E_AQ_BUF_SZ; 404 405 err = i40evf_execute_vf_cmd(dev, &args); 406 if (err) { 407 PMD_INIT_LOG(ERR, "fail to execute command OP_VERSION"); 408 return err; 409 } 410 411 pver = (struct virtchnl_version_info *)args.out_buffer; 412 vf->version_major = pver->major; 413 vf->version_minor = pver->minor; 414 if ((vf->version_major == VIRTCHNL_VERSION_MAJOR) && 415 (vf->version_minor <= VIRTCHNL_VERSION_MINOR)) 416 PMD_DRV_LOG(INFO, "Peer is Linux PF host"); 417 else { 418 PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)", 419 vf->version_major, vf->version_minor, 420 VIRTCHNL_VERSION_MAJOR, 421 VIRTCHNL_VERSION_MINOR); 422 return -1; 423 } 424 425 return 0; 426 } 427 428 static int 429 i40evf_get_vf_resource(struct rte_eth_dev *dev) 430 { 431 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 432 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 433 int err; 434 struct vf_cmd_info args; 435 uint32_t caps, len; 436 437 args.ops = VIRTCHNL_OP_GET_VF_RESOURCES; 438 args.out_buffer = vf->aq_resp; 439 args.out_size = I40E_AQ_BUF_SZ; 440 if (PF_IS_V11(vf)) { 441 caps = VIRTCHNL_VF_OFFLOAD_L2 | 442 VIRTCHNL_VF_OFFLOAD_RSS_AQ | 443 VIRTCHNL_VF_OFFLOAD_RSS_REG | 444 VIRTCHNL_VF_OFFLOAD_VLAN | 445 VIRTCHNL_VF_OFFLOAD_RX_POLLING; 446 args.in_args = (uint8_t *)∩︀ 447 args.in_args_size = sizeof(caps); 448 } else { 449 args.in_args = NULL; 450 args.in_args_size = 0; 451 } 452 err = i40evf_execute_vf_cmd(dev, &args); 453 454 if (err) { 455 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_VF_RESOURCE"); 456 return err; 457 } 458 459 len = sizeof(struct virtchnl_vf_resource) + 460 I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource); 461 462 rte_memcpy(vf->vf_res, args.out_buffer, 463 RTE_MIN(args.out_size, len)); 464 i40e_vf_parse_hw_config(hw, vf->vf_res); 465 466 return 0; 467 } 468 469 static int 470 i40evf_config_promisc(struct rte_eth_dev *dev, 471 bool enable_unicast, 472 bool enable_multicast) 473 { 474 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 475 int err; 476 struct vf_cmd_info args; 477 struct virtchnl_promisc_info promisc; 478 479 promisc.flags = 0; 480 promisc.vsi_id = vf->vsi_res->vsi_id; 481 482 if (enable_unicast) 483 promisc.flags |= FLAG_VF_UNICAST_PROMISC; 484 485 if (enable_multicast) 486 promisc.flags |= FLAG_VF_MULTICAST_PROMISC; 487 488 args.ops = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; 489 args.in_args = (uint8_t *)&promisc; 490 args.in_args_size = sizeof(promisc); 491 args.out_buffer = vf->aq_resp; 492 args.out_size = I40E_AQ_BUF_SZ; 493 494 err = i40evf_execute_vf_cmd(dev, &args); 495 496 if (err) 497 PMD_DRV_LOG(ERR, "fail to execute command " 498 "CONFIG_PROMISCUOUS_MODE"); 499 return err; 500 } 501 502 static int 503 i40evf_enable_vlan_strip(struct rte_eth_dev *dev) 504 { 505 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 506 struct vf_cmd_info args; 507 int ret; 508 509 memset(&args, 0, sizeof(args)); 510 args.ops = VIRTCHNL_OP_ENABLE_VLAN_STRIPPING; 511 args.in_args = NULL; 512 args.in_args_size = 0; 513 args.out_buffer = vf->aq_resp; 514 args.out_size = I40E_AQ_BUF_SZ; 515 ret = i40evf_execute_vf_cmd(dev, &args); 516 if (ret) 517 PMD_DRV_LOG(ERR, "Failed to execute command of " 518 "VIRTCHNL_OP_ENABLE_VLAN_STRIPPING"); 519 520 return ret; 521 } 522 523 static int 524 i40evf_disable_vlan_strip(struct rte_eth_dev *dev) 525 { 526 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 527 struct vf_cmd_info args; 528 int ret; 529 530 memset(&args, 0, sizeof(args)); 531 args.ops = VIRTCHNL_OP_DISABLE_VLAN_STRIPPING; 532 args.in_args = NULL; 533 args.in_args_size = 0; 534 args.out_buffer = vf->aq_resp; 535 args.out_size = I40E_AQ_BUF_SZ; 536 ret = i40evf_execute_vf_cmd(dev, &args); 537 if (ret) 538 PMD_DRV_LOG(ERR, "Failed to execute command of " 539 "VIRTCHNL_OP_DISABLE_VLAN_STRIPPING"); 540 541 return ret; 542 } 543 544 static void 545 i40evf_fill_virtchnl_vsi_txq_info(struct virtchnl_txq_info *txq_info, 546 uint16_t vsi_id, 547 uint16_t queue_id, 548 uint16_t nb_txq, 549 struct i40e_tx_queue *txq) 550 { 551 txq_info->vsi_id = vsi_id; 552 txq_info->queue_id = queue_id; 553 if (queue_id < nb_txq && txq) { 554 txq_info->ring_len = txq->nb_tx_desc; 555 txq_info->dma_ring_addr = txq->tx_ring_phys_addr; 556 } 557 } 558 559 static void 560 i40evf_fill_virtchnl_vsi_rxq_info(struct virtchnl_rxq_info *rxq_info, 561 uint16_t vsi_id, 562 uint16_t queue_id, 563 uint16_t nb_rxq, 564 uint32_t max_pkt_size, 565 struct i40e_rx_queue *rxq) 566 { 567 rxq_info->vsi_id = vsi_id; 568 rxq_info->queue_id = queue_id; 569 rxq_info->max_pkt_size = max_pkt_size; 570 if (queue_id < nb_rxq && rxq) { 571 rxq_info->ring_len = rxq->nb_rx_desc; 572 rxq_info->dma_ring_addr = rxq->rx_ring_phys_addr; 573 rxq_info->databuffer_size = 574 (rte_pktmbuf_data_room_size(rxq->mp) - 575 RTE_PKTMBUF_HEADROOM); 576 } 577 } 578 579 static int 580 i40evf_configure_vsi_queues(struct rte_eth_dev *dev) 581 { 582 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 583 struct i40e_rx_queue **rxq = 584 (struct i40e_rx_queue **)dev->data->rx_queues; 585 struct i40e_tx_queue **txq = 586 (struct i40e_tx_queue **)dev->data->tx_queues; 587 struct virtchnl_vsi_queue_config_info *vc_vqci; 588 struct virtchnl_queue_pair_info *vc_qpi; 589 struct vf_cmd_info args; 590 uint16_t i, nb_qp = vf->num_queue_pairs; 591 const uint32_t size = 592 I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqci, nb_qp); 593 uint8_t buff[size]; 594 int ret; 595 596 memset(buff, 0, sizeof(buff)); 597 vc_vqci = (struct virtchnl_vsi_queue_config_info *)buff; 598 vc_vqci->vsi_id = vf->vsi_res->vsi_id; 599 vc_vqci->num_queue_pairs = nb_qp; 600 601 for (i = 0, vc_qpi = vc_vqci->qpair; i < nb_qp; i++, vc_qpi++) { 602 i40evf_fill_virtchnl_vsi_txq_info(&vc_qpi->txq, 603 vc_vqci->vsi_id, i, dev->data->nb_tx_queues, 604 txq ? txq[i] : NULL); 605 i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpi->rxq, 606 vc_vqci->vsi_id, i, dev->data->nb_rx_queues, 607 vf->max_pkt_len, rxq ? rxq[i] : NULL); 608 } 609 memset(&args, 0, sizeof(args)); 610 args.ops = VIRTCHNL_OP_CONFIG_VSI_QUEUES; 611 args.in_args = (uint8_t *)vc_vqci; 612 args.in_args_size = size; 613 args.out_buffer = vf->aq_resp; 614 args.out_size = I40E_AQ_BUF_SZ; 615 ret = i40evf_execute_vf_cmd(dev, &args); 616 if (ret) 617 PMD_DRV_LOG(ERR, "Failed to execute command of " 618 "VIRTCHNL_OP_CONFIG_VSI_QUEUES"); 619 620 return ret; 621 } 622 623 static int 624 i40evf_config_irq_map(struct rte_eth_dev *dev) 625 { 626 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 627 struct vf_cmd_info args; 628 uint8_t cmd_buffer[sizeof(struct virtchnl_irq_map_info) + \ 629 sizeof(struct virtchnl_vector_map)]; 630 struct virtchnl_irq_map_info *map_info; 631 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 632 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 633 uint32_t vector_id; 634 int i, err; 635 636 if (dev->data->dev_conf.intr_conf.rxq != 0 && 637 rte_intr_allow_others(intr_handle)) 638 vector_id = I40E_RX_VEC_START; 639 else 640 vector_id = I40E_MISC_VEC_ID; 641 642 map_info = (struct virtchnl_irq_map_info *)cmd_buffer; 643 map_info->num_vectors = 1; 644 map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT; 645 map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id; 646 /* Alway use default dynamic MSIX interrupt */ 647 map_info->vecmap[0].vector_id = vector_id; 648 /* Don't map any tx queue */ 649 map_info->vecmap[0].txq_map = 0; 650 map_info->vecmap[0].rxq_map = 0; 651 for (i = 0; i < dev->data->nb_rx_queues; i++) { 652 map_info->vecmap[0].rxq_map |= 1 << i; 653 if (rte_intr_dp_is_en(intr_handle)) 654 intr_handle->intr_vec[i] = vector_id; 655 } 656 657 args.ops = VIRTCHNL_OP_CONFIG_IRQ_MAP; 658 args.in_args = (u8 *)cmd_buffer; 659 args.in_args_size = sizeof(cmd_buffer); 660 args.out_buffer = vf->aq_resp; 661 args.out_size = I40E_AQ_BUF_SZ; 662 err = i40evf_execute_vf_cmd(dev, &args); 663 if (err) 664 PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES"); 665 666 return err; 667 } 668 669 static int 670 i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid, 671 bool on) 672 { 673 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 674 struct virtchnl_queue_select queue_select; 675 int err; 676 struct vf_cmd_info args; 677 memset(&queue_select, 0, sizeof(queue_select)); 678 queue_select.vsi_id = vf->vsi_res->vsi_id; 679 680 if (isrx) 681 queue_select.rx_queues |= 1 << qid; 682 else 683 queue_select.tx_queues |= 1 << qid; 684 685 if (on) 686 args.ops = VIRTCHNL_OP_ENABLE_QUEUES; 687 else 688 args.ops = VIRTCHNL_OP_DISABLE_QUEUES; 689 args.in_args = (u8 *)&queue_select; 690 args.in_args_size = sizeof(queue_select); 691 args.out_buffer = vf->aq_resp; 692 args.out_size = I40E_AQ_BUF_SZ; 693 err = i40evf_execute_vf_cmd(dev, &args); 694 if (err) 695 PMD_DRV_LOG(ERR, "fail to switch %s %u %s", 696 isrx ? "RX" : "TX", qid, on ? "on" : "off"); 697 698 return err; 699 } 700 701 static int 702 i40evf_start_queues(struct rte_eth_dev *dev) 703 { 704 struct rte_eth_dev_data *dev_data = dev->data; 705 int i; 706 struct i40e_rx_queue *rxq; 707 struct i40e_tx_queue *txq; 708 709 for (i = 0; i < dev->data->nb_rx_queues; i++) { 710 rxq = dev_data->rx_queues[i]; 711 if (rxq->rx_deferred_start) 712 continue; 713 if (i40evf_dev_rx_queue_start(dev, i) != 0) { 714 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 715 return -1; 716 } 717 } 718 719 for (i = 0; i < dev->data->nb_tx_queues; i++) { 720 txq = dev_data->tx_queues[i]; 721 if (txq->tx_deferred_start) 722 continue; 723 if (i40evf_dev_tx_queue_start(dev, i) != 0) { 724 PMD_DRV_LOG(ERR, "Fail to start queue %u", i); 725 return -1; 726 } 727 } 728 729 return 0; 730 } 731 732 static int 733 i40evf_stop_queues(struct rte_eth_dev *dev) 734 { 735 int i; 736 737 /* Stop TX queues first */ 738 for (i = 0; i < dev->data->nb_tx_queues; i++) { 739 if (i40evf_dev_tx_queue_stop(dev, i) != 0) { 740 PMD_DRV_LOG(ERR, "Fail to stop queue %u", i); 741 return -1; 742 } 743 } 744 745 /* Then stop RX queues */ 746 for (i = 0; i < dev->data->nb_rx_queues; i++) { 747 if (i40evf_dev_rx_queue_stop(dev, i) != 0) { 748 PMD_DRV_LOG(ERR, "Fail to stop queue %u", i); 749 return -1; 750 } 751 } 752 753 return 0; 754 } 755 756 static int 757 i40evf_add_mac_addr(struct rte_eth_dev *dev, 758 struct ether_addr *addr, 759 __rte_unused uint32_t index, 760 __rte_unused uint32_t pool) 761 { 762 struct virtchnl_ether_addr_list *list; 763 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 764 uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \ 765 sizeof(struct virtchnl_ether_addr)]; 766 int err; 767 struct vf_cmd_info args; 768 769 if (is_zero_ether_addr(addr)) { 770 PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x", 771 addr->addr_bytes[0], addr->addr_bytes[1], 772 addr->addr_bytes[2], addr->addr_bytes[3], 773 addr->addr_bytes[4], addr->addr_bytes[5]); 774 return I40E_ERR_INVALID_MAC_ADDR; 775 } 776 777 list = (struct virtchnl_ether_addr_list *)cmd_buffer; 778 list->vsi_id = vf->vsi_res->vsi_id; 779 list->num_elements = 1; 780 rte_memcpy(list->list[0].addr, addr->addr_bytes, 781 sizeof(addr->addr_bytes)); 782 783 args.ops = VIRTCHNL_OP_ADD_ETH_ADDR; 784 args.in_args = cmd_buffer; 785 args.in_args_size = sizeof(cmd_buffer); 786 args.out_buffer = vf->aq_resp; 787 args.out_size = I40E_AQ_BUF_SZ; 788 err = i40evf_execute_vf_cmd(dev, &args); 789 if (err) 790 PMD_DRV_LOG(ERR, "fail to execute command " 791 "OP_ADD_ETHER_ADDRESS"); 792 else 793 vf->vsi.mac_num++; 794 795 return err; 796 } 797 798 static void 799 i40evf_del_mac_addr_by_addr(struct rte_eth_dev *dev, 800 struct ether_addr *addr) 801 { 802 struct virtchnl_ether_addr_list *list; 803 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 804 uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + \ 805 sizeof(struct virtchnl_ether_addr)]; 806 int err; 807 struct vf_cmd_info args; 808 809 if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) { 810 PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x", 811 addr->addr_bytes[0], addr->addr_bytes[1], 812 addr->addr_bytes[2], addr->addr_bytes[3], 813 addr->addr_bytes[4], addr->addr_bytes[5]); 814 return; 815 } 816 817 list = (struct virtchnl_ether_addr_list *)cmd_buffer; 818 list->vsi_id = vf->vsi_res->vsi_id; 819 list->num_elements = 1; 820 rte_memcpy(list->list[0].addr, addr->addr_bytes, 821 sizeof(addr->addr_bytes)); 822 823 args.ops = VIRTCHNL_OP_DEL_ETH_ADDR; 824 args.in_args = cmd_buffer; 825 args.in_args_size = sizeof(cmd_buffer); 826 args.out_buffer = vf->aq_resp; 827 args.out_size = I40E_AQ_BUF_SZ; 828 err = i40evf_execute_vf_cmd(dev, &args); 829 if (err) 830 PMD_DRV_LOG(ERR, "fail to execute command " 831 "OP_DEL_ETHER_ADDRESS"); 832 else 833 vf->vsi.mac_num--; 834 return; 835 } 836 837 static void 838 i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index) 839 { 840 struct rte_eth_dev_data *data = dev->data; 841 struct ether_addr *addr; 842 843 addr = &data->mac_addrs[index]; 844 845 i40evf_del_mac_addr_by_addr(dev, addr); 846 } 847 848 static int 849 i40evf_query_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats) 850 { 851 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 852 struct virtchnl_queue_select q_stats; 853 int err; 854 struct vf_cmd_info args; 855 856 memset(&q_stats, 0, sizeof(q_stats)); 857 q_stats.vsi_id = vf->vsi_res->vsi_id; 858 args.ops = VIRTCHNL_OP_GET_STATS; 859 args.in_args = (u8 *)&q_stats; 860 args.in_args_size = sizeof(q_stats); 861 args.out_buffer = vf->aq_resp; 862 args.out_size = I40E_AQ_BUF_SZ; 863 864 err = i40evf_execute_vf_cmd(dev, &args); 865 if (err) { 866 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS"); 867 *pstats = NULL; 868 return err; 869 } 870 *pstats = (struct i40e_eth_stats *)args.out_buffer; 871 return 0; 872 } 873 874 static void 875 i40evf_stat_update_48(uint64_t *offset, 876 uint64_t *stat) 877 { 878 if (*stat >= *offset) 879 *stat = *stat - *offset; 880 else 881 *stat = (uint64_t)((*stat + 882 ((uint64_t)1 << I40E_48_BIT_WIDTH)) - *offset); 883 884 *stat &= I40E_48_BIT_MASK; 885 } 886 887 static void 888 i40evf_stat_update_32(uint64_t *offset, 889 uint64_t *stat) 890 { 891 if (*stat >= *offset) 892 *stat = (uint64_t)(*stat - *offset); 893 else 894 *stat = (uint64_t)((*stat + 895 ((uint64_t)1 << I40E_32_BIT_WIDTH)) - *offset); 896 } 897 898 static void 899 i40evf_update_stats(struct i40e_vsi *vsi, 900 struct i40e_eth_stats *nes) 901 { 902 struct i40e_eth_stats *oes = &vsi->eth_stats_offset; 903 904 i40evf_stat_update_48(&oes->rx_bytes, 905 &nes->rx_bytes); 906 i40evf_stat_update_48(&oes->rx_unicast, 907 &nes->rx_unicast); 908 i40evf_stat_update_48(&oes->rx_multicast, 909 &nes->rx_multicast); 910 i40evf_stat_update_48(&oes->rx_broadcast, 911 &nes->rx_broadcast); 912 i40evf_stat_update_32(&oes->rx_discards, 913 &nes->rx_discards); 914 i40evf_stat_update_32(&oes->rx_unknown_protocol, 915 &nes->rx_unknown_protocol); 916 i40evf_stat_update_48(&oes->tx_bytes, 917 &nes->tx_bytes); 918 i40evf_stat_update_48(&oes->tx_unicast, 919 &nes->tx_unicast); 920 i40evf_stat_update_48(&oes->tx_multicast, 921 &nes->tx_multicast); 922 i40evf_stat_update_48(&oes->tx_broadcast, 923 &nes->tx_broadcast); 924 i40evf_stat_update_32(&oes->tx_errors, &nes->tx_errors); 925 i40evf_stat_update_32(&oes->tx_discards, &nes->tx_discards); 926 } 927 928 static void 929 i40evf_dev_xstats_reset(struct rte_eth_dev *dev) 930 { 931 int ret; 932 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 933 struct i40e_eth_stats *pstats = NULL; 934 935 /* read stat values to clear hardware registers */ 936 ret = i40evf_query_stats(dev, &pstats); 937 938 /* set stats offset base on current values */ 939 if (ret == 0) 940 vf->vsi.eth_stats_offset = *pstats; 941 } 942 943 static int i40evf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev, 944 struct rte_eth_xstat_name *xstats_names, 945 __rte_unused unsigned limit) 946 { 947 unsigned i; 948 949 if (xstats_names != NULL) 950 for (i = 0; i < I40EVF_NB_XSTATS; i++) { 951 snprintf(xstats_names[i].name, 952 sizeof(xstats_names[i].name), 953 "%s", rte_i40evf_stats_strings[i].name); 954 } 955 return I40EVF_NB_XSTATS; 956 } 957 958 static int i40evf_dev_xstats_get(struct rte_eth_dev *dev, 959 struct rte_eth_xstat *xstats, unsigned n) 960 { 961 int ret; 962 unsigned i; 963 struct i40e_eth_stats *pstats = NULL; 964 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 965 struct i40e_vsi *vsi = &vf->vsi; 966 967 if (n < I40EVF_NB_XSTATS) 968 return I40EVF_NB_XSTATS; 969 970 ret = i40evf_query_stats(dev, &pstats); 971 if (ret != 0) 972 return 0; 973 974 if (!xstats) 975 return 0; 976 977 i40evf_update_stats(vsi, pstats); 978 979 /* loop over xstats array and values from pstats */ 980 for (i = 0; i < I40EVF_NB_XSTATS; i++) { 981 xstats[i].id = i; 982 xstats[i].value = *(uint64_t *)(((char *)pstats) + 983 rte_i40evf_stats_strings[i].offset); 984 } 985 986 return I40EVF_NB_XSTATS; 987 } 988 989 static int 990 i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid) 991 { 992 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 993 struct virtchnl_vlan_filter_list *vlan_list; 994 uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) + 995 sizeof(uint16_t)]; 996 int err; 997 struct vf_cmd_info args; 998 999 vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer; 1000 vlan_list->vsi_id = vf->vsi_res->vsi_id; 1001 vlan_list->num_elements = 1; 1002 vlan_list->vlan_id[0] = vlanid; 1003 1004 args.ops = VIRTCHNL_OP_ADD_VLAN; 1005 args.in_args = (u8 *)&cmd_buffer; 1006 args.in_args_size = sizeof(cmd_buffer); 1007 args.out_buffer = vf->aq_resp; 1008 args.out_size = I40E_AQ_BUF_SZ; 1009 err = i40evf_execute_vf_cmd(dev, &args); 1010 if (err) 1011 PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_VLAN"); 1012 1013 return err; 1014 } 1015 1016 static int 1017 i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid) 1018 { 1019 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1020 struct virtchnl_vlan_filter_list *vlan_list; 1021 uint8_t cmd_buffer[sizeof(struct virtchnl_vlan_filter_list) + 1022 sizeof(uint16_t)]; 1023 int err; 1024 struct vf_cmd_info args; 1025 1026 vlan_list = (struct virtchnl_vlan_filter_list *)cmd_buffer; 1027 vlan_list->vsi_id = vf->vsi_res->vsi_id; 1028 vlan_list->num_elements = 1; 1029 vlan_list->vlan_id[0] = vlanid; 1030 1031 args.ops = VIRTCHNL_OP_DEL_VLAN; 1032 args.in_args = (u8 *)&cmd_buffer; 1033 args.in_args_size = sizeof(cmd_buffer); 1034 args.out_buffer = vf->aq_resp; 1035 args.out_size = I40E_AQ_BUF_SZ; 1036 err = i40evf_execute_vf_cmd(dev, &args); 1037 if (err) 1038 PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_VLAN"); 1039 1040 return err; 1041 } 1042 1043 static const struct rte_pci_id pci_id_i40evf_map[] = { 1044 { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF) }, 1045 { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV) }, 1046 { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF) }, 1047 { RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF) }, 1048 { .vendor_id = 0, /* sentinel */ }, 1049 }; 1050 1051 /* Disable IRQ0 */ 1052 static inline void 1053 i40evf_disable_irq0(struct i40e_hw *hw) 1054 { 1055 /* Disable all interrupt types */ 1056 I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0); 1057 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 1058 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK); 1059 I40EVF_WRITE_FLUSH(hw); 1060 } 1061 1062 /* Enable IRQ0 */ 1063 static inline void 1064 i40evf_enable_irq0(struct i40e_hw *hw) 1065 { 1066 /* Enable admin queue interrupt trigger */ 1067 uint32_t val; 1068 1069 i40evf_disable_irq0(hw); 1070 val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1); 1071 val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK | 1072 I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK; 1073 I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val); 1074 1075 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 1076 I40E_VFINT_DYN_CTL01_INTENA_MASK | 1077 I40E_VFINT_DYN_CTL01_CLEARPBA_MASK | 1078 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK); 1079 1080 I40EVF_WRITE_FLUSH(hw); 1081 } 1082 1083 static int 1084 i40evf_check_vf_reset_done(struct rte_eth_dev *dev) 1085 { 1086 int i, reset; 1087 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1088 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1089 1090 for (i = 0; i < MAX_RESET_WAIT_CNT; i++) { 1091 reset = I40E_READ_REG(hw, I40E_VFGEN_RSTAT) & 1092 I40E_VFGEN_RSTAT_VFR_STATE_MASK; 1093 reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT; 1094 if (reset == VIRTCHNL_VFR_VFACTIVE || 1095 reset == VIRTCHNL_VFR_COMPLETED) 1096 break; 1097 rte_delay_ms(50); 1098 } 1099 1100 if (i >= MAX_RESET_WAIT_CNT) 1101 return -1; 1102 1103 vf->vf_reset = false; 1104 vf->pend_msg &= ~PFMSG_RESET_IMPENDING; 1105 1106 return 0; 1107 } 1108 static int 1109 i40evf_reset_vf(struct rte_eth_dev *dev) 1110 { 1111 int ret; 1112 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1113 1114 if (i40e_vf_reset(hw) != I40E_SUCCESS) { 1115 PMD_INIT_LOG(ERR, "Reset VF NIC failed"); 1116 return -1; 1117 } 1118 /** 1119 * After issuing vf reset command to pf, pf won't necessarily 1120 * reset vf, it depends on what state it exactly is. If it's not 1121 * initialized yet, it won't have vf reset since it's in a certain 1122 * state. If not, it will try to reset. Even vf is reset, pf will 1123 * set I40E_VFGEN_RSTAT to COMPLETE first, then wait 10ms and set 1124 * it to ACTIVE. In this duration, vf may not catch the moment that 1125 * COMPLETE is set. So, for vf, we'll try to wait a long time. 1126 */ 1127 rte_delay_ms(200); 1128 1129 ret = i40evf_check_vf_reset_done(dev); 1130 if (ret) { 1131 PMD_INIT_LOG(ERR, "VF is still resetting"); 1132 return ret; 1133 } 1134 1135 return 0; 1136 } 1137 1138 static int 1139 i40evf_init_vf(struct rte_eth_dev *dev) 1140 { 1141 int i, err, bufsz; 1142 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1143 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1144 uint16_t interval = 1145 i40e_calc_itr_interval(0, 0); 1146 1147 vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1148 vf->dev_data = dev->data; 1149 err = i40e_set_mac_type(hw); 1150 if (err) { 1151 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err); 1152 goto err; 1153 } 1154 1155 err = i40evf_check_vf_reset_done(dev); 1156 if (err) 1157 goto err; 1158 1159 i40e_init_adminq_parameter(hw); 1160 err = i40e_init_adminq(hw); 1161 if (err) { 1162 PMD_INIT_LOG(ERR, "init_adminq failed: %d", err); 1163 goto err; 1164 } 1165 1166 /* Reset VF and wait until it's complete */ 1167 if (i40evf_reset_vf(dev)) { 1168 PMD_INIT_LOG(ERR, "reset NIC failed"); 1169 goto err_aq; 1170 } 1171 1172 /* VF reset, shutdown admin queue and initialize again */ 1173 if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) { 1174 PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed"); 1175 goto err; 1176 } 1177 1178 i40e_init_adminq_parameter(hw); 1179 if (i40e_init_adminq(hw) != I40E_SUCCESS) { 1180 PMD_INIT_LOG(ERR, "init_adminq failed"); 1181 goto err; 1182 } 1183 1184 vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0); 1185 if (!vf->aq_resp) { 1186 PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory"); 1187 goto err_aq; 1188 } 1189 if (i40evf_check_api_version(dev) != 0) { 1190 PMD_INIT_LOG(ERR, "check_api version failed"); 1191 goto err_api; 1192 } 1193 bufsz = sizeof(struct virtchnl_vf_resource) + 1194 (I40E_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)); 1195 vf->vf_res = rte_zmalloc("vf_res", bufsz, 0); 1196 if (!vf->vf_res) { 1197 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 1198 goto err_api; 1199 } 1200 1201 if (i40evf_get_vf_resource(dev) != 0) { 1202 PMD_INIT_LOG(ERR, "i40evf_get_vf_config failed"); 1203 goto err_alloc; 1204 } 1205 1206 /* got VF config message back from PF, now we can parse it */ 1207 for (i = 0; i < vf->vf_res->num_vsis; i++) { 1208 if (vf->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 1209 vf->vsi_res = &vf->vf_res->vsi_res[i]; 1210 } 1211 1212 if (!vf->vsi_res) { 1213 PMD_INIT_LOG(ERR, "no LAN VSI found"); 1214 goto err_alloc; 1215 } 1216 1217 if (hw->mac.type == I40E_MAC_X722_VF) 1218 vf->flags = I40E_FLAG_RSS_AQ_CAPABLE; 1219 vf->vsi.vsi_id = vf->vsi_res->vsi_id; 1220 1221 switch (vf->vsi_res->vsi_type) { 1222 case VIRTCHNL_VSI_SRIOV: 1223 vf->vsi.type = I40E_VSI_SRIOV; 1224 break; 1225 default: 1226 vf->vsi.type = I40E_VSI_TYPE_UNKNOWN; 1227 break; 1228 } 1229 vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs; 1230 vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1231 1232 /* Store the MAC address configured by host, or generate random one */ 1233 if (is_valid_assigned_ether_addr((struct ether_addr *)hw->mac.addr)) 1234 vf->flags |= I40E_FLAG_VF_MAC_BY_PF; 1235 else 1236 eth_random_addr(hw->mac.addr); /* Generate a random one */ 1237 1238 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 1239 (I40E_ITR_INDEX_DEFAULT << 1240 I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) | 1241 (interval << 1242 I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT)); 1243 I40EVF_WRITE_FLUSH(hw); 1244 1245 return 0; 1246 1247 err_alloc: 1248 rte_free(vf->vf_res); 1249 vf->vsi_res = NULL; 1250 err_api: 1251 rte_free(vf->aq_resp); 1252 err_aq: 1253 i40e_shutdown_adminq(hw); /* ignore error */ 1254 err: 1255 return -1; 1256 } 1257 1258 static int 1259 i40evf_uninit_vf(struct rte_eth_dev *dev) 1260 { 1261 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1262 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1263 1264 PMD_INIT_FUNC_TRACE(); 1265 1266 if (hw->adapter_closed == 0) 1267 i40evf_dev_close(dev); 1268 rte_free(vf->vf_res); 1269 vf->vf_res = NULL; 1270 rte_free(vf->aq_resp); 1271 vf->aq_resp = NULL; 1272 1273 return 0; 1274 } 1275 1276 static void 1277 i40evf_handle_pf_event(struct rte_eth_dev *dev, uint8_t *msg, 1278 __rte_unused uint16_t msglen) 1279 { 1280 struct virtchnl_pf_event *pf_msg = 1281 (struct virtchnl_pf_event *)msg; 1282 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1283 1284 switch (pf_msg->event) { 1285 case VIRTCHNL_EVENT_RESET_IMPENDING: 1286 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event"); 1287 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, 1288 NULL); 1289 break; 1290 case VIRTCHNL_EVENT_LINK_CHANGE: 1291 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event"); 1292 vf->link_up = pf_msg->event_data.link_event.link_status; 1293 vf->link_speed = pf_msg->event_data.link_event.link_speed; 1294 break; 1295 case VIRTCHNL_EVENT_PF_DRIVER_CLOSE: 1296 PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event"); 1297 break; 1298 default: 1299 PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event); 1300 break; 1301 } 1302 } 1303 1304 static void 1305 i40evf_handle_aq_msg(struct rte_eth_dev *dev) 1306 { 1307 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1308 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1309 struct i40e_arq_event_info info; 1310 uint16_t pending, aq_opc; 1311 enum virtchnl_ops msg_opc; 1312 enum i40e_status_code msg_ret; 1313 int ret; 1314 1315 info.buf_len = I40E_AQ_BUF_SZ; 1316 if (!vf->aq_resp) { 1317 PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL"); 1318 return; 1319 } 1320 info.msg_buf = vf->aq_resp; 1321 1322 pending = 1; 1323 while (pending) { 1324 ret = i40e_clean_arq_element(hw, &info, &pending); 1325 1326 if (ret != I40E_SUCCESS) { 1327 PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ," 1328 "ret: %d", ret); 1329 break; 1330 } 1331 aq_opc = rte_le_to_cpu_16(info.desc.opcode); 1332 /* For the message sent from pf to vf, opcode is stored in 1333 * cookie_high of struct i40e_aq_desc, while return error code 1334 * are stored in cookie_low, Which is done by 1335 * i40e_aq_send_msg_to_vf in PF driver.*/ 1336 msg_opc = (enum virtchnl_ops)rte_le_to_cpu_32( 1337 info.desc.cookie_high); 1338 msg_ret = (enum i40e_status_code)rte_le_to_cpu_32( 1339 info.desc.cookie_low); 1340 switch (aq_opc) { 1341 case i40e_aqc_opc_send_msg_to_vf: 1342 if (msg_opc == VIRTCHNL_OP_EVENT) 1343 /* process event*/ 1344 i40evf_handle_pf_event(dev, info.msg_buf, 1345 info.msg_len); 1346 else { 1347 /* read message and it's expected one */ 1348 if (msg_opc == vf->pend_cmd) { 1349 vf->cmd_retval = msg_ret; 1350 /* prevent compiler reordering */ 1351 rte_compiler_barrier(); 1352 _clear_cmd(vf); 1353 } else 1354 PMD_DRV_LOG(ERR, "command mismatch," 1355 "expect %u, get %u", 1356 vf->pend_cmd, msg_opc); 1357 PMD_DRV_LOG(DEBUG, "adminq response is received," 1358 " opcode = %d", msg_opc); 1359 } 1360 break; 1361 default: 1362 PMD_DRV_LOG(ERR, "Request %u is not supported yet", 1363 aq_opc); 1364 break; 1365 } 1366 } 1367 } 1368 1369 /** 1370 * Interrupt handler triggered by NIC for handling 1371 * specific interrupt. Only adminq interrupt is processed in VF. 1372 * 1373 * @param handle 1374 * Pointer to interrupt handle. 1375 * @param param 1376 * The address of parameter (struct rte_eth_dev *) regsitered before. 1377 * 1378 * @return 1379 * void 1380 */ 1381 static void 1382 i40evf_dev_alarm_handler(void *param) 1383 { 1384 struct rte_eth_dev *dev = (struct rte_eth_dev *)param; 1385 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1386 uint32_t icr0; 1387 1388 i40evf_disable_irq0(hw); 1389 1390 /* read out interrupt causes */ 1391 icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01); 1392 1393 /* No interrupt event indicated */ 1394 if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) 1395 goto done; 1396 1397 if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) { 1398 PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported"); 1399 i40evf_handle_aq_msg(dev); 1400 } 1401 1402 /* Link Status Change interrupt */ 1403 if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK) 1404 PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported," 1405 " do nothing"); 1406 1407 done: 1408 i40evf_enable_irq0(hw); 1409 rte_eal_alarm_set(I40EVF_ALARM_INTERVAL, 1410 i40evf_dev_alarm_handler, dev); 1411 } 1412 1413 static int 1414 i40evf_dev_init(struct rte_eth_dev *eth_dev) 1415 { 1416 struct i40e_hw *hw 1417 = I40E_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private); 1418 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 1419 1420 PMD_INIT_FUNC_TRACE(); 1421 1422 /* assign ops func pointer */ 1423 eth_dev->dev_ops = &i40evf_eth_dev_ops; 1424 eth_dev->rx_pkt_burst = &i40e_recv_pkts; 1425 eth_dev->tx_pkt_burst = &i40e_xmit_pkts; 1426 1427 /* 1428 * For secondary processes, we don't initialise any further as primary 1429 * has already done this work. 1430 */ 1431 if (rte_eal_process_type() != RTE_PROC_PRIMARY){ 1432 i40e_set_rx_function(eth_dev); 1433 i40e_set_tx_function(eth_dev); 1434 return 0; 1435 } 1436 i40e_set_default_ptype_table(eth_dev); 1437 rte_eth_copy_pci_info(eth_dev, pci_dev); 1438 1439 hw->vendor_id = pci_dev->id.vendor_id; 1440 hw->device_id = pci_dev->id.device_id; 1441 hw->subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 1442 hw->subsystem_device_id = pci_dev->id.subsystem_device_id; 1443 hw->bus.device = pci_dev->addr.devid; 1444 hw->bus.func = pci_dev->addr.function; 1445 hw->hw_addr = (void *)pci_dev->mem_resource[0].addr; 1446 hw->adapter_stopped = 0; 1447 hw->adapter_closed = 0; 1448 1449 if(i40evf_init_vf(eth_dev) != 0) { 1450 PMD_INIT_LOG(ERR, "Init vf failed"); 1451 return -1; 1452 } 1453 1454 i40e_set_default_pctype_table(eth_dev); 1455 rte_eal_alarm_set(I40EVF_ALARM_INTERVAL, 1456 i40evf_dev_alarm_handler, eth_dev); 1457 1458 /* configure and enable device interrupt */ 1459 i40evf_enable_irq0(hw); 1460 1461 /* copy mac addr */ 1462 eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac", 1463 ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX, 1464 0); 1465 if (eth_dev->data->mac_addrs == NULL) { 1466 PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to" 1467 " store MAC addresses", 1468 ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX); 1469 return -ENOMEM; 1470 } 1471 ether_addr_copy((struct ether_addr *)hw->mac.addr, 1472 ð_dev->data->mac_addrs[0]); 1473 1474 return 0; 1475 } 1476 1477 static int 1478 i40evf_dev_uninit(struct rte_eth_dev *eth_dev) 1479 { 1480 PMD_INIT_FUNC_TRACE(); 1481 1482 if (rte_eal_process_type() != RTE_PROC_PRIMARY) 1483 return -EPERM; 1484 1485 eth_dev->dev_ops = NULL; 1486 eth_dev->rx_pkt_burst = NULL; 1487 eth_dev->tx_pkt_burst = NULL; 1488 1489 if (i40evf_uninit_vf(eth_dev) != 0) { 1490 PMD_INIT_LOG(ERR, "i40evf_uninit_vf failed"); 1491 return -1; 1492 } 1493 1494 return 0; 1495 } 1496 1497 static int eth_i40evf_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, 1498 struct rte_pci_device *pci_dev) 1499 { 1500 return rte_eth_dev_pci_generic_probe(pci_dev, 1501 sizeof(struct i40e_adapter), i40evf_dev_init); 1502 } 1503 1504 static int eth_i40evf_pci_remove(struct rte_pci_device *pci_dev) 1505 { 1506 return rte_eth_dev_pci_generic_remove(pci_dev, i40evf_dev_uninit); 1507 } 1508 1509 /* 1510 * virtual function driver struct 1511 */ 1512 static struct rte_pci_driver rte_i40evf_pmd = { 1513 .id_table = pci_id_i40evf_map, 1514 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA, 1515 .probe = eth_i40evf_pci_probe, 1516 .remove = eth_i40evf_pci_remove, 1517 }; 1518 1519 RTE_PMD_REGISTER_PCI(net_i40e_vf, rte_i40evf_pmd); 1520 RTE_PMD_REGISTER_PCI_TABLE(net_i40e_vf, pci_id_i40evf_map); 1521 RTE_PMD_REGISTER_KMOD_DEP(net_i40e_vf, "* igb_uio | vfio-pci"); 1522 1523 static int 1524 i40evf_dev_configure(struct rte_eth_dev *dev) 1525 { 1526 struct i40e_adapter *ad = 1527 I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private); 1528 1529 /* Initialize to TRUE. If any of Rx queues doesn't meet the bulk 1530 * allocation or vector Rx preconditions we will reset it. 1531 */ 1532 ad->rx_bulk_alloc_allowed = true; 1533 ad->rx_vec_allowed = true; 1534 ad->tx_simple_allowed = true; 1535 ad->tx_vec_allowed = true; 1536 1537 return i40evf_init_vlan(dev); 1538 } 1539 1540 static int 1541 i40evf_init_vlan(struct rte_eth_dev *dev) 1542 { 1543 /* Apply vlan offload setting */ 1544 i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK); 1545 1546 return 0; 1547 } 1548 1549 static int 1550 i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask) 1551 { 1552 struct rte_eth_conf *dev_conf = &dev->data->dev_conf; 1553 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1554 1555 if (!(vf->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)) 1556 return -ENOTSUP; 1557 1558 /* Vlan stripping setting */ 1559 if (mask & ETH_VLAN_STRIP_MASK) { 1560 /* Enable or disable VLAN stripping */ 1561 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_VLAN_STRIP) 1562 i40evf_enable_vlan_strip(dev); 1563 else 1564 i40evf_disable_vlan_strip(dev); 1565 } 1566 1567 return 0; 1568 } 1569 1570 static int 1571 i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1572 { 1573 struct i40e_rx_queue *rxq; 1574 int err; 1575 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1576 1577 PMD_INIT_FUNC_TRACE(); 1578 1579 rxq = dev->data->rx_queues[rx_queue_id]; 1580 1581 err = i40e_alloc_rx_queue_mbufs(rxq); 1582 if (err) { 1583 PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf"); 1584 return err; 1585 } 1586 1587 rte_wmb(); 1588 1589 /* Init the RX tail register. */ 1590 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 1591 I40EVF_WRITE_FLUSH(hw); 1592 1593 /* Ready to switch the queue on */ 1594 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE); 1595 if (err) { 1596 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on", 1597 rx_queue_id); 1598 return err; 1599 } 1600 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1601 1602 return 0; 1603 } 1604 1605 static int 1606 i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) 1607 { 1608 struct i40e_rx_queue *rxq; 1609 int err; 1610 1611 rxq = dev->data->rx_queues[rx_queue_id]; 1612 1613 err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE); 1614 if (err) { 1615 PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off", 1616 rx_queue_id); 1617 return err; 1618 } 1619 1620 i40e_rx_queue_release_mbufs(rxq); 1621 i40e_reset_rx_queue(rxq); 1622 dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1623 1624 return 0; 1625 } 1626 1627 static int 1628 i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1629 { 1630 int err; 1631 1632 PMD_INIT_FUNC_TRACE(); 1633 1634 /* Ready to switch the queue on */ 1635 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE); 1636 if (err) { 1637 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on", 1638 tx_queue_id); 1639 return err; 1640 } 1641 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1642 1643 return 0; 1644 } 1645 1646 static int 1647 i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) 1648 { 1649 struct i40e_tx_queue *txq; 1650 int err; 1651 1652 txq = dev->data->tx_queues[tx_queue_id]; 1653 1654 err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE); 1655 if (err) { 1656 PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off", 1657 tx_queue_id); 1658 return err; 1659 } 1660 1661 i40e_tx_queue_release_mbufs(txq); 1662 i40e_reset_tx_queue(txq); 1663 dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; 1664 1665 return 0; 1666 } 1667 1668 static int 1669 i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) 1670 { 1671 int ret; 1672 1673 if (on) 1674 ret = i40evf_add_vlan(dev, vlan_id); 1675 else 1676 ret = i40evf_del_vlan(dev,vlan_id); 1677 1678 return ret; 1679 } 1680 1681 static int 1682 i40evf_rxq_init(struct rte_eth_dev *dev, struct i40e_rx_queue *rxq) 1683 { 1684 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1685 struct rte_eth_dev_data *dev_data = dev->data; 1686 struct rte_pktmbuf_pool_private *mbp_priv; 1687 uint16_t buf_size, len; 1688 1689 rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL1(rxq->queue_id); 1690 I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1); 1691 I40EVF_WRITE_FLUSH(hw); 1692 1693 /* Calculate the maximum packet length allowed */ 1694 mbp_priv = rte_mempool_get_priv(rxq->mp); 1695 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size - 1696 RTE_PKTMBUF_HEADROOM); 1697 rxq->hs_mode = i40e_header_split_none; 1698 rxq->rx_hdr_len = 0; 1699 rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); 1700 len = rxq->rx_buf_len * I40E_MAX_CHAINED_RX_BUFFERS; 1701 rxq->max_pkt_len = RTE_MIN(len, 1702 dev_data->dev_conf.rxmode.max_rx_pkt_len); 1703 1704 /** 1705 * Check if the jumbo frame and maximum packet length are set correctly 1706 */ 1707 if (dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) { 1708 if (rxq->max_pkt_len <= ETHER_MAX_LEN || 1709 rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) { 1710 PMD_DRV_LOG(ERR, "maximum packet length must be " 1711 "larger than %u and smaller than %u, as jumbo " 1712 "frame is enabled", (uint32_t)ETHER_MAX_LEN, 1713 (uint32_t)I40E_FRAME_SIZE_MAX); 1714 return I40E_ERR_CONFIG; 1715 } 1716 } else { 1717 if (rxq->max_pkt_len < ETHER_MIN_LEN || 1718 rxq->max_pkt_len > ETHER_MAX_LEN) { 1719 PMD_DRV_LOG(ERR, "maximum packet length must be " 1720 "larger than %u and smaller than %u, as jumbo " 1721 "frame is disabled", (uint32_t)ETHER_MIN_LEN, 1722 (uint32_t)ETHER_MAX_LEN); 1723 return I40E_ERR_CONFIG; 1724 } 1725 } 1726 1727 if ((dev_data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER) || 1728 rxq->max_pkt_len > buf_size) 1729 dev_data->scattered_rx = 1; 1730 1731 return 0; 1732 } 1733 1734 static int 1735 i40evf_rx_init(struct rte_eth_dev *dev) 1736 { 1737 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1738 uint16_t i; 1739 int ret = I40E_SUCCESS; 1740 struct i40e_rx_queue **rxq = 1741 (struct i40e_rx_queue **)dev->data->rx_queues; 1742 1743 i40evf_config_rss(vf); 1744 for (i = 0; i < dev->data->nb_rx_queues; i++) { 1745 if (!rxq[i] || !rxq[i]->q_set) 1746 continue; 1747 ret = i40evf_rxq_init(dev, rxq[i]); 1748 if (ret != I40E_SUCCESS) 1749 break; 1750 } 1751 if (ret == I40E_SUCCESS) 1752 i40e_set_rx_function(dev); 1753 1754 return ret; 1755 } 1756 1757 static void 1758 i40evf_tx_init(struct rte_eth_dev *dev) 1759 { 1760 uint16_t i; 1761 struct i40e_tx_queue **txq = 1762 (struct i40e_tx_queue **)dev->data->tx_queues; 1763 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1764 1765 for (i = 0; i < dev->data->nb_tx_queues; i++) 1766 txq[i]->qtx_tail = hw->hw_addr + I40E_QTX_TAIL1(i); 1767 1768 i40e_set_tx_function(dev); 1769 } 1770 1771 static inline void 1772 i40evf_enable_queues_intr(struct rte_eth_dev *dev) 1773 { 1774 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1775 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1776 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1777 1778 if (!rte_intr_allow_others(intr_handle)) { 1779 I40E_WRITE_REG(hw, 1780 I40E_VFINT_DYN_CTL01, 1781 I40E_VFINT_DYN_CTL01_INTENA_MASK | 1782 I40E_VFINT_DYN_CTL01_CLEARPBA_MASK | 1783 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK); 1784 I40EVF_WRITE_FLUSH(hw); 1785 return; 1786 } 1787 1788 I40EVF_WRITE_FLUSH(hw); 1789 } 1790 1791 static inline void 1792 i40evf_disable_queues_intr(struct rte_eth_dev *dev) 1793 { 1794 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1795 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1796 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1797 1798 if (!rte_intr_allow_others(intr_handle)) { 1799 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 1800 I40E_VFINT_DYN_CTL01_ITR_INDX_MASK); 1801 I40EVF_WRITE_FLUSH(hw); 1802 return; 1803 } 1804 1805 I40EVF_WRITE_FLUSH(hw); 1806 } 1807 1808 static int 1809 i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id) 1810 { 1811 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1812 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1813 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1814 uint16_t interval = 1815 i40e_calc_itr_interval(0, 0); 1816 uint16_t msix_intr; 1817 1818 msix_intr = intr_handle->intr_vec[queue_id]; 1819 if (msix_intr == I40E_MISC_VEC_ID) 1820 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 1821 I40E_VFINT_DYN_CTL01_INTENA_MASK | 1822 I40E_VFINT_DYN_CTL01_CLEARPBA_MASK | 1823 (0 << I40E_VFINT_DYN_CTL01_ITR_INDX_SHIFT) | 1824 (interval << 1825 I40E_VFINT_DYN_CTL01_INTERVAL_SHIFT)); 1826 else 1827 I40E_WRITE_REG(hw, 1828 I40E_VFINT_DYN_CTLN1(msix_intr - 1829 I40E_RX_VEC_START), 1830 I40E_VFINT_DYN_CTLN1_INTENA_MASK | 1831 I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK | 1832 (0 << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) | 1833 (interval << 1834 I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT)); 1835 1836 I40EVF_WRITE_FLUSH(hw); 1837 1838 return 0; 1839 } 1840 1841 static int 1842 i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id) 1843 { 1844 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1845 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1846 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1847 uint16_t msix_intr; 1848 1849 msix_intr = intr_handle->intr_vec[queue_id]; 1850 if (msix_intr == I40E_MISC_VEC_ID) 1851 I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0); 1852 else 1853 I40E_WRITE_REG(hw, 1854 I40E_VFINT_DYN_CTLN1(msix_intr - 1855 I40E_RX_VEC_START), 1856 0); 1857 1858 I40EVF_WRITE_FLUSH(hw); 1859 1860 return 0; 1861 } 1862 1863 static void 1864 i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add) 1865 { 1866 struct virtchnl_ether_addr_list *list; 1867 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1868 int err, i, j; 1869 int next_begin = 0; 1870 int begin = 0; 1871 uint32_t len; 1872 struct ether_addr *addr; 1873 struct vf_cmd_info args; 1874 1875 do { 1876 j = 0; 1877 len = sizeof(struct virtchnl_ether_addr_list); 1878 for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) { 1879 if (is_zero_ether_addr(&dev->data->mac_addrs[i])) 1880 continue; 1881 len += sizeof(struct virtchnl_ether_addr); 1882 if (len >= I40E_AQ_BUF_SZ) { 1883 next_begin = i + 1; 1884 break; 1885 } 1886 } 1887 1888 list = rte_zmalloc("i40evf_del_mac_buffer", len, 0); 1889 if (!list) { 1890 PMD_DRV_LOG(ERR, "fail to allocate memory"); 1891 return; 1892 } 1893 1894 for (i = begin; i < next_begin; i++) { 1895 addr = &dev->data->mac_addrs[i]; 1896 if (is_zero_ether_addr(addr)) 1897 continue; 1898 rte_memcpy(list->list[j].addr, addr->addr_bytes, 1899 sizeof(addr->addr_bytes)); 1900 PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x", 1901 addr->addr_bytes[0], addr->addr_bytes[1], 1902 addr->addr_bytes[2], addr->addr_bytes[3], 1903 addr->addr_bytes[4], addr->addr_bytes[5]); 1904 j++; 1905 } 1906 list->vsi_id = vf->vsi_res->vsi_id; 1907 list->num_elements = j; 1908 args.ops = add ? VIRTCHNL_OP_ADD_ETH_ADDR : 1909 VIRTCHNL_OP_DEL_ETH_ADDR; 1910 args.in_args = (uint8_t *)list; 1911 args.in_args_size = len; 1912 args.out_buffer = vf->aq_resp; 1913 args.out_size = I40E_AQ_BUF_SZ; 1914 err = i40evf_execute_vf_cmd(dev, &args); 1915 if (err) { 1916 PMD_DRV_LOG(ERR, "fail to execute command %s", 1917 add ? "OP_ADD_ETHER_ADDRESS" : 1918 "OP_DEL_ETHER_ADDRESS"); 1919 } else { 1920 if (add) 1921 vf->vsi.mac_num++; 1922 else 1923 vf->vsi.mac_num--; 1924 } 1925 rte_free(list); 1926 begin = next_begin; 1927 } while (begin < I40E_NUM_MACADDR_MAX); 1928 } 1929 1930 static int 1931 i40evf_dev_start(struct rte_eth_dev *dev) 1932 { 1933 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 1934 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 1935 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 1936 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 1937 uint32_t intr_vector = 0; 1938 1939 PMD_INIT_FUNC_TRACE(); 1940 1941 hw->adapter_stopped = 0; 1942 1943 vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len; 1944 vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues, 1945 dev->data->nb_tx_queues); 1946 1947 /* check and configure queue intr-vector mapping */ 1948 if (rte_intr_cap_multiple(intr_handle) && 1949 dev->data->dev_conf.intr_conf.rxq) { 1950 intr_vector = dev->data->nb_rx_queues; 1951 if (rte_intr_efd_enable(intr_handle, intr_vector)) 1952 return -1; 1953 } 1954 1955 if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) { 1956 intr_handle->intr_vec = 1957 rte_zmalloc("intr_vec", 1958 dev->data->nb_rx_queues * sizeof(int), 0); 1959 if (!intr_handle->intr_vec) { 1960 PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues" 1961 " intr_vec", dev->data->nb_rx_queues); 1962 return -ENOMEM; 1963 } 1964 } 1965 1966 if (i40evf_rx_init(dev) != 0){ 1967 PMD_DRV_LOG(ERR, "failed to do RX init"); 1968 return -1; 1969 } 1970 1971 i40evf_tx_init(dev); 1972 1973 if (i40evf_configure_vsi_queues(dev) != 0) { 1974 PMD_DRV_LOG(ERR, "configure queues failed"); 1975 goto err_queue; 1976 } 1977 if (i40evf_config_irq_map(dev)) { 1978 PMD_DRV_LOG(ERR, "config_irq_map failed"); 1979 goto err_queue; 1980 } 1981 1982 /* Set all mac addrs */ 1983 i40evf_add_del_all_mac_addr(dev, TRUE); 1984 /* Set all multicast addresses */ 1985 i40evf_add_del_mc_addr_list(dev, vf->mc_addrs, vf->mc_addrs_num, 1986 TRUE); 1987 1988 if (i40evf_start_queues(dev) != 0) { 1989 PMD_DRV_LOG(ERR, "enable queues failed"); 1990 goto err_mac; 1991 } 1992 1993 /* only enable interrupt in rx interrupt mode */ 1994 if (dev->data->dev_conf.intr_conf.rxq != 0) 1995 rte_intr_enable(intr_handle); 1996 1997 i40evf_enable_queues_intr(dev); 1998 1999 return 0; 2000 2001 err_mac: 2002 i40evf_add_del_all_mac_addr(dev, FALSE); 2003 i40evf_add_del_mc_addr_list(dev, vf->mc_addrs, vf->mc_addrs_num, 2004 FALSE); 2005 err_queue: 2006 return -1; 2007 } 2008 2009 static void 2010 i40evf_dev_stop(struct rte_eth_dev *dev) 2011 { 2012 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev); 2013 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 2014 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2015 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2016 2017 PMD_INIT_FUNC_TRACE(); 2018 2019 if (dev->data->dev_conf.intr_conf.rxq != 0) 2020 rte_intr_disable(intr_handle); 2021 2022 if (hw->adapter_stopped == 1) 2023 return; 2024 i40evf_stop_queues(dev); 2025 i40evf_disable_queues_intr(dev); 2026 i40e_dev_clear_queues(dev); 2027 2028 /* Clean datapath event and queue/vec mapping */ 2029 rte_intr_efd_disable(intr_handle); 2030 if (intr_handle->intr_vec) { 2031 rte_free(intr_handle->intr_vec); 2032 intr_handle->intr_vec = NULL; 2033 } 2034 /* remove all mac addrs */ 2035 i40evf_add_del_all_mac_addr(dev, FALSE); 2036 /* remove all multicast addresses */ 2037 i40evf_add_del_mc_addr_list(dev, vf->mc_addrs, vf->mc_addrs_num, 2038 FALSE); 2039 hw->adapter_stopped = 1; 2040 2041 } 2042 2043 static int 2044 i40evf_dev_link_update(struct rte_eth_dev *dev, 2045 __rte_unused int wait_to_complete) 2046 { 2047 struct rte_eth_link new_link; 2048 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2049 /* 2050 * DPDK pf host provide interfacet to acquire link status 2051 * while Linux driver does not 2052 */ 2053 2054 memset(&new_link, 0, sizeof(new_link)); 2055 /* Linux driver PF host */ 2056 switch (vf->link_speed) { 2057 case I40E_LINK_SPEED_100MB: 2058 new_link.link_speed = ETH_SPEED_NUM_100M; 2059 break; 2060 case I40E_LINK_SPEED_1GB: 2061 new_link.link_speed = ETH_SPEED_NUM_1G; 2062 break; 2063 case I40E_LINK_SPEED_10GB: 2064 new_link.link_speed = ETH_SPEED_NUM_10G; 2065 break; 2066 case I40E_LINK_SPEED_20GB: 2067 new_link.link_speed = ETH_SPEED_NUM_20G; 2068 break; 2069 case I40E_LINK_SPEED_25GB: 2070 new_link.link_speed = ETH_SPEED_NUM_25G; 2071 break; 2072 case I40E_LINK_SPEED_40GB: 2073 new_link.link_speed = ETH_SPEED_NUM_40G; 2074 break; 2075 default: 2076 new_link.link_speed = ETH_SPEED_NUM_100M; 2077 break; 2078 } 2079 /* full duplex only */ 2080 new_link.link_duplex = ETH_LINK_FULL_DUPLEX; 2081 new_link.link_status = vf->link_up ? ETH_LINK_UP : 2082 ETH_LINK_DOWN; 2083 new_link.link_autoneg = 2084 !(dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED); 2085 2086 return rte_eth_linkstatus_set(dev, &new_link); 2087 } 2088 2089 static void 2090 i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev) 2091 { 2092 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2093 int ret; 2094 2095 /* If enabled, just return */ 2096 if (vf->promisc_unicast_enabled) 2097 return; 2098 2099 ret = i40evf_config_promisc(dev, 1, vf->promisc_multicast_enabled); 2100 if (ret == 0) 2101 vf->promisc_unicast_enabled = TRUE; 2102 } 2103 2104 static void 2105 i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev) 2106 { 2107 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2108 int ret; 2109 2110 /* If disabled, just return */ 2111 if (!vf->promisc_unicast_enabled) 2112 return; 2113 2114 ret = i40evf_config_promisc(dev, 0, vf->promisc_multicast_enabled); 2115 if (ret == 0) 2116 vf->promisc_unicast_enabled = FALSE; 2117 } 2118 2119 static void 2120 i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev) 2121 { 2122 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2123 int ret; 2124 2125 /* If enabled, just return */ 2126 if (vf->promisc_multicast_enabled) 2127 return; 2128 2129 ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 1); 2130 if (ret == 0) 2131 vf->promisc_multicast_enabled = TRUE; 2132 } 2133 2134 static void 2135 i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev) 2136 { 2137 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2138 int ret; 2139 2140 /* If enabled, just return */ 2141 if (!vf->promisc_multicast_enabled) 2142 return; 2143 2144 ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 0); 2145 if (ret == 0) 2146 vf->promisc_multicast_enabled = FALSE; 2147 } 2148 2149 static void 2150 i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) 2151 { 2152 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2153 2154 dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs; 2155 dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs; 2156 dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN; 2157 dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX; 2158 dev_info->hash_key_size = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t); 2159 dev_info->reta_size = ETH_RSS_RETA_SIZE_64; 2160 dev_info->flow_type_rss_offloads = vf->adapter->flow_types_mask; 2161 dev_info->max_mac_addrs = I40E_NUM_MACADDR_MAX; 2162 dev_info->rx_queue_offload_capa = 0; 2163 dev_info->rx_offload_capa = 2164 DEV_RX_OFFLOAD_VLAN_STRIP | 2165 DEV_RX_OFFLOAD_QINQ_STRIP | 2166 DEV_RX_OFFLOAD_IPV4_CKSUM | 2167 DEV_RX_OFFLOAD_UDP_CKSUM | 2168 DEV_RX_OFFLOAD_TCP_CKSUM | 2169 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | 2170 DEV_RX_OFFLOAD_SCATTER | 2171 DEV_RX_OFFLOAD_JUMBO_FRAME | 2172 DEV_RX_OFFLOAD_VLAN_FILTER; 2173 2174 dev_info->tx_queue_offload_capa = 0; 2175 dev_info->tx_offload_capa = 2176 DEV_TX_OFFLOAD_VLAN_INSERT | 2177 DEV_TX_OFFLOAD_QINQ_INSERT | 2178 DEV_TX_OFFLOAD_IPV4_CKSUM | 2179 DEV_TX_OFFLOAD_UDP_CKSUM | 2180 DEV_TX_OFFLOAD_TCP_CKSUM | 2181 DEV_TX_OFFLOAD_SCTP_CKSUM | 2182 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | 2183 DEV_TX_OFFLOAD_TCP_TSO | 2184 DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 2185 DEV_TX_OFFLOAD_GRE_TNL_TSO | 2186 DEV_TX_OFFLOAD_IPIP_TNL_TSO | 2187 DEV_TX_OFFLOAD_GENEVE_TNL_TSO | 2188 DEV_TX_OFFLOAD_MULTI_SEGS; 2189 2190 dev_info->default_rxconf = (struct rte_eth_rxconf) { 2191 .rx_thresh = { 2192 .pthresh = I40E_DEFAULT_RX_PTHRESH, 2193 .hthresh = I40E_DEFAULT_RX_HTHRESH, 2194 .wthresh = I40E_DEFAULT_RX_WTHRESH, 2195 }, 2196 .rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH, 2197 .rx_drop_en = 0, 2198 .offloads = 0, 2199 }; 2200 2201 dev_info->default_txconf = (struct rte_eth_txconf) { 2202 .tx_thresh = { 2203 .pthresh = I40E_DEFAULT_TX_PTHRESH, 2204 .hthresh = I40E_DEFAULT_TX_HTHRESH, 2205 .wthresh = I40E_DEFAULT_TX_WTHRESH, 2206 }, 2207 .tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH, 2208 .tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH, 2209 .offloads = 0, 2210 }; 2211 2212 dev_info->rx_desc_lim = (struct rte_eth_desc_lim) { 2213 .nb_max = I40E_MAX_RING_DESC, 2214 .nb_min = I40E_MIN_RING_DESC, 2215 .nb_align = I40E_ALIGN_RING_DESC, 2216 }; 2217 2218 dev_info->tx_desc_lim = (struct rte_eth_desc_lim) { 2219 .nb_max = I40E_MAX_RING_DESC, 2220 .nb_min = I40E_MIN_RING_DESC, 2221 .nb_align = I40E_ALIGN_RING_DESC, 2222 }; 2223 } 2224 2225 static int 2226 i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) 2227 { 2228 int ret; 2229 struct i40e_eth_stats *pstats = NULL; 2230 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2231 struct i40e_vsi *vsi = &vf->vsi; 2232 2233 ret = i40evf_query_stats(dev, &pstats); 2234 if (ret == 0) { 2235 i40evf_update_stats(vsi, pstats); 2236 2237 stats->ipackets = pstats->rx_unicast + pstats->rx_multicast + 2238 pstats->rx_broadcast; 2239 stats->opackets = pstats->tx_broadcast + pstats->tx_multicast + 2240 pstats->tx_unicast; 2241 stats->imissed = pstats->rx_discards; 2242 stats->oerrors = pstats->tx_errors + pstats->tx_discards; 2243 stats->ibytes = pstats->rx_bytes; 2244 stats->obytes = pstats->tx_bytes; 2245 } else { 2246 PMD_DRV_LOG(ERR, "Get statistics failed"); 2247 } 2248 return ret; 2249 } 2250 2251 static void 2252 i40evf_dev_close(struct rte_eth_dev *dev) 2253 { 2254 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2255 2256 i40evf_dev_stop(dev); 2257 i40e_dev_free_queues(dev); 2258 /* 2259 * disable promiscuous mode before reset vf 2260 * it is a workaround solution when work with kernel driver 2261 * and it is not the normal way 2262 */ 2263 i40evf_dev_promiscuous_disable(dev); 2264 i40evf_dev_allmulticast_disable(dev); 2265 rte_eal_alarm_cancel(i40evf_dev_alarm_handler, dev); 2266 2267 i40evf_reset_vf(dev); 2268 i40e_shutdown_adminq(hw); 2269 i40evf_disable_irq0(hw); 2270 hw->adapter_closed = 1; 2271 } 2272 2273 /* 2274 * Reset VF device only to re-initialize resources in PMD layer 2275 */ 2276 static int 2277 i40evf_dev_reset(struct rte_eth_dev *dev) 2278 { 2279 int ret; 2280 2281 ret = i40evf_dev_uninit(dev); 2282 if (ret) 2283 return ret; 2284 2285 ret = i40evf_dev_init(dev); 2286 2287 return ret; 2288 } 2289 2290 static int 2291 i40evf_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size) 2292 { 2293 struct i40e_vf *vf = I40E_VSI_TO_VF(vsi); 2294 struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); 2295 int ret; 2296 2297 if (!lut) 2298 return -EINVAL; 2299 2300 if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { 2301 ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, FALSE, 2302 lut, lut_size); 2303 if (ret) { 2304 PMD_DRV_LOG(ERR, "Failed to get RSS lookup table"); 2305 return ret; 2306 } 2307 } else { 2308 uint32_t *lut_dw = (uint32_t *)lut; 2309 uint16_t i, lut_size_dw = lut_size / 4; 2310 2311 for (i = 0; i < lut_size_dw; i++) 2312 lut_dw[i] = I40E_READ_REG(hw, I40E_VFQF_HLUT(i)); 2313 } 2314 2315 return 0; 2316 } 2317 2318 static int 2319 i40evf_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size) 2320 { 2321 struct i40e_vf *vf; 2322 struct i40e_hw *hw; 2323 int ret; 2324 2325 if (!vsi || !lut) 2326 return -EINVAL; 2327 2328 vf = I40E_VSI_TO_VF(vsi); 2329 hw = I40E_VSI_TO_HW(vsi); 2330 2331 if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { 2332 ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, FALSE, 2333 lut, lut_size); 2334 if (ret) { 2335 PMD_DRV_LOG(ERR, "Failed to set RSS lookup table"); 2336 return ret; 2337 } 2338 } else { 2339 uint32_t *lut_dw = (uint32_t *)lut; 2340 uint16_t i, lut_size_dw = lut_size / 4; 2341 2342 for (i = 0; i < lut_size_dw; i++) 2343 I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i), lut_dw[i]); 2344 I40EVF_WRITE_FLUSH(hw); 2345 } 2346 2347 return 0; 2348 } 2349 2350 static int 2351 i40evf_dev_rss_reta_update(struct rte_eth_dev *dev, 2352 struct rte_eth_rss_reta_entry64 *reta_conf, 2353 uint16_t reta_size) 2354 { 2355 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2356 uint8_t *lut; 2357 uint16_t i, idx, shift; 2358 int ret; 2359 2360 if (reta_size != ETH_RSS_RETA_SIZE_64) { 2361 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 2362 "(%d) doesn't match the number of hardware can " 2363 "support (%d)", reta_size, ETH_RSS_RETA_SIZE_64); 2364 return -EINVAL; 2365 } 2366 2367 lut = rte_zmalloc("i40e_rss_lut", reta_size, 0); 2368 if (!lut) { 2369 PMD_DRV_LOG(ERR, "No memory can be allocated"); 2370 return -ENOMEM; 2371 } 2372 ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size); 2373 if (ret) 2374 goto out; 2375 for (i = 0; i < reta_size; i++) { 2376 idx = i / RTE_RETA_GROUP_SIZE; 2377 shift = i % RTE_RETA_GROUP_SIZE; 2378 if (reta_conf[idx].mask & (1ULL << shift)) 2379 lut[i] = reta_conf[idx].reta[shift]; 2380 } 2381 ret = i40evf_set_rss_lut(&vf->vsi, lut, reta_size); 2382 2383 out: 2384 rte_free(lut); 2385 2386 return ret; 2387 } 2388 2389 static int 2390 i40evf_dev_rss_reta_query(struct rte_eth_dev *dev, 2391 struct rte_eth_rss_reta_entry64 *reta_conf, 2392 uint16_t reta_size) 2393 { 2394 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2395 uint16_t i, idx, shift; 2396 uint8_t *lut; 2397 int ret; 2398 2399 if (reta_size != ETH_RSS_RETA_SIZE_64) { 2400 PMD_DRV_LOG(ERR, "The size of hash lookup table configured " 2401 "(%d) doesn't match the number of hardware can " 2402 "support (%d)", reta_size, ETH_RSS_RETA_SIZE_64); 2403 return -EINVAL; 2404 } 2405 2406 lut = rte_zmalloc("i40e_rss_lut", reta_size, 0); 2407 if (!lut) { 2408 PMD_DRV_LOG(ERR, "No memory can be allocated"); 2409 return -ENOMEM; 2410 } 2411 2412 ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size); 2413 if (ret) 2414 goto out; 2415 for (i = 0; i < reta_size; i++) { 2416 idx = i / RTE_RETA_GROUP_SIZE; 2417 shift = i % RTE_RETA_GROUP_SIZE; 2418 if (reta_conf[idx].mask & (1ULL << shift)) 2419 reta_conf[idx].reta[shift] = lut[i]; 2420 } 2421 2422 out: 2423 rte_free(lut); 2424 2425 return ret; 2426 } 2427 2428 static int 2429 i40evf_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len) 2430 { 2431 struct i40e_vf *vf = I40E_VSI_TO_VF(vsi); 2432 struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); 2433 int ret = 0; 2434 2435 if (!key || key_len == 0) { 2436 PMD_DRV_LOG(DEBUG, "No key to be configured"); 2437 return 0; 2438 } else if (key_len != (I40E_VFQF_HKEY_MAX_INDEX + 1) * 2439 sizeof(uint32_t)) { 2440 PMD_DRV_LOG(ERR, "Invalid key length %u", key_len); 2441 return -EINVAL; 2442 } 2443 2444 if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { 2445 struct i40e_aqc_get_set_rss_key_data *key_dw = 2446 (struct i40e_aqc_get_set_rss_key_data *)key; 2447 2448 ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw); 2449 if (ret) 2450 PMD_INIT_LOG(ERR, "Failed to configure RSS key " 2451 "via AQ"); 2452 } else { 2453 uint32_t *hash_key = (uint32_t *)key; 2454 uint16_t i; 2455 2456 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++) 2457 i40e_write_rx_ctl(hw, I40E_VFQF_HKEY(i), hash_key[i]); 2458 I40EVF_WRITE_FLUSH(hw); 2459 } 2460 2461 return ret; 2462 } 2463 2464 static int 2465 i40evf_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len) 2466 { 2467 struct i40e_vf *vf = I40E_VSI_TO_VF(vsi); 2468 struct i40e_hw *hw = I40E_VSI_TO_HW(vsi); 2469 int ret; 2470 2471 if (!key || !key_len) 2472 return -EINVAL; 2473 2474 if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) { 2475 ret = i40e_aq_get_rss_key(hw, vsi->vsi_id, 2476 (struct i40e_aqc_get_set_rss_key_data *)key); 2477 if (ret) { 2478 PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ"); 2479 return ret; 2480 } 2481 } else { 2482 uint32_t *key_dw = (uint32_t *)key; 2483 uint16_t i; 2484 2485 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++) 2486 key_dw[i] = i40e_read_rx_ctl(hw, I40E_VFQF_HKEY(i)); 2487 } 2488 *key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t); 2489 2490 return 0; 2491 } 2492 2493 static int 2494 i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf) 2495 { 2496 struct i40e_hw *hw = I40E_VF_TO_HW(vf); 2497 uint64_t hena; 2498 int ret; 2499 2500 ret = i40evf_set_rss_key(&vf->vsi, rss_conf->rss_key, 2501 rss_conf->rss_key_len); 2502 if (ret) 2503 return ret; 2504 2505 hena = i40e_config_hena(vf->adapter, rss_conf->rss_hf); 2506 i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena); 2507 i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32)); 2508 I40EVF_WRITE_FLUSH(hw); 2509 2510 return 0; 2511 } 2512 2513 static void 2514 i40evf_disable_rss(struct i40e_vf *vf) 2515 { 2516 struct i40e_hw *hw = I40E_VF_TO_HW(vf); 2517 2518 i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), 0); 2519 i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), 0); 2520 I40EVF_WRITE_FLUSH(hw); 2521 } 2522 2523 static int 2524 i40evf_config_rss(struct i40e_vf *vf) 2525 { 2526 struct i40e_hw *hw = I40E_VF_TO_HW(vf); 2527 struct rte_eth_rss_conf rss_conf; 2528 uint32_t i, j, lut = 0, nb_q = (I40E_VFQF_HLUT_MAX_INDEX + 1) * 4; 2529 uint16_t num; 2530 2531 if (vf->dev_data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 2532 i40evf_disable_rss(vf); 2533 PMD_DRV_LOG(DEBUG, "RSS not configured"); 2534 return 0; 2535 } 2536 2537 num = RTE_MIN(vf->dev_data->nb_rx_queues, I40E_MAX_QP_NUM_PER_VF); 2538 /* Fill out the look up table */ 2539 for (i = 0, j = 0; i < nb_q; i++, j++) { 2540 if (j >= num) 2541 j = 0; 2542 lut = (lut << 8) | j; 2543 if ((i & 3) == 3) 2544 I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i >> 2), lut); 2545 } 2546 2547 rss_conf = vf->dev_data->dev_conf.rx_adv_conf.rss_conf; 2548 if ((rss_conf.rss_hf & vf->adapter->flow_types_mask) == 0) { 2549 i40evf_disable_rss(vf); 2550 PMD_DRV_LOG(DEBUG, "No hash flag is set"); 2551 return 0; 2552 } 2553 2554 if (rss_conf.rss_key == NULL || rss_conf.rss_key_len < 2555 (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) { 2556 /* Calculate the default hash key */ 2557 for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++) 2558 rss_key_default[i] = (uint32_t)rte_rand(); 2559 rss_conf.rss_key = (uint8_t *)rss_key_default; 2560 rss_conf.rss_key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) * 2561 sizeof(uint32_t); 2562 } 2563 2564 return i40evf_hw_rss_hash_set(vf, &rss_conf); 2565 } 2566 2567 static int 2568 i40evf_dev_rss_hash_update(struct rte_eth_dev *dev, 2569 struct rte_eth_rss_conf *rss_conf) 2570 { 2571 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2572 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2573 uint64_t rss_hf = rss_conf->rss_hf & vf->adapter->flow_types_mask; 2574 uint64_t hena; 2575 2576 hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0)); 2577 hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32; 2578 2579 if (!(hena & vf->adapter->pctypes_mask)) { /* RSS disabled */ 2580 if (rss_hf != 0) /* Enable RSS */ 2581 return -EINVAL; 2582 return 0; 2583 } 2584 2585 /* RSS enabled */ 2586 if (rss_hf == 0) /* Disable RSS */ 2587 return -EINVAL; 2588 2589 return i40evf_hw_rss_hash_set(vf, rss_conf); 2590 } 2591 2592 static int 2593 i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev, 2594 struct rte_eth_rss_conf *rss_conf) 2595 { 2596 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2597 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2598 uint64_t hena; 2599 2600 i40evf_get_rss_key(&vf->vsi, rss_conf->rss_key, 2601 &rss_conf->rss_key_len); 2602 2603 hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0)); 2604 hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32; 2605 rss_conf->rss_hf = i40e_parse_hena(vf->adapter, hena); 2606 2607 return 0; 2608 } 2609 2610 static int 2611 i40evf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) 2612 { 2613 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2614 struct rte_eth_dev_data *dev_data = vf->dev_data; 2615 uint32_t frame_size = mtu + I40E_ETH_OVERHEAD; 2616 int ret = 0; 2617 2618 /* check if mtu is within the allowed range */ 2619 if ((mtu < ETHER_MIN_MTU) || (frame_size > I40E_FRAME_SIZE_MAX)) 2620 return -EINVAL; 2621 2622 /* mtu setting is forbidden if port is start */ 2623 if (dev_data->dev_started) { 2624 PMD_DRV_LOG(ERR, "port %d must be stopped before configuration", 2625 dev_data->port_id); 2626 return -EBUSY; 2627 } 2628 2629 if (frame_size > ETHER_MAX_LEN) 2630 dev_data->dev_conf.rxmode.offloads |= 2631 DEV_RX_OFFLOAD_JUMBO_FRAME; 2632 else 2633 dev_data->dev_conf.rxmode.offloads &= 2634 ~DEV_RX_OFFLOAD_JUMBO_FRAME; 2635 dev_data->dev_conf.rxmode.max_rx_pkt_len = frame_size; 2636 2637 return ret; 2638 } 2639 2640 static int 2641 i40evf_set_default_mac_addr(struct rte_eth_dev *dev, 2642 struct ether_addr *mac_addr) 2643 { 2644 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2645 struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); 2646 2647 if (!is_valid_assigned_ether_addr(mac_addr)) { 2648 PMD_DRV_LOG(ERR, "Tried to set invalid MAC address."); 2649 return -EINVAL; 2650 } 2651 2652 if (vf->flags & I40E_FLAG_VF_MAC_BY_PF) 2653 return -EPERM; 2654 2655 i40evf_del_mac_addr_by_addr(dev, (struct ether_addr *)hw->mac.addr); 2656 2657 if (i40evf_add_mac_addr(dev, mac_addr, 0, 0) != 0) 2658 return -EIO; 2659 2660 ether_addr_copy(mac_addr, (struct ether_addr *)hw->mac.addr); 2661 return 0; 2662 } 2663 2664 static int 2665 i40evf_add_del_mc_addr_list(struct rte_eth_dev *dev, 2666 struct ether_addr *mc_addrs, 2667 uint32_t mc_addrs_num, bool add) 2668 { 2669 struct virtchnl_ether_addr_list *list; 2670 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2671 uint8_t cmd_buffer[sizeof(struct virtchnl_ether_addr_list) + 2672 (I40E_NUM_MACADDR_MAX * sizeof(struct virtchnl_ether_addr))]; 2673 uint32_t i; 2674 int err; 2675 struct vf_cmd_info args; 2676 2677 if (mc_addrs == NULL || mc_addrs_num == 0) 2678 return 0; 2679 2680 if (mc_addrs_num > I40E_NUM_MACADDR_MAX) 2681 return -EINVAL; 2682 2683 list = (struct virtchnl_ether_addr_list *)cmd_buffer; 2684 list->vsi_id = vf->vsi_res->vsi_id; 2685 list->num_elements = mc_addrs_num; 2686 2687 for (i = 0; i < mc_addrs_num; i++) { 2688 if (!I40E_IS_MULTICAST(mc_addrs[i].addr_bytes)) { 2689 PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x", 2690 mc_addrs[i].addr_bytes[0], 2691 mc_addrs[i].addr_bytes[1], 2692 mc_addrs[i].addr_bytes[2], 2693 mc_addrs[i].addr_bytes[3], 2694 mc_addrs[i].addr_bytes[4], 2695 mc_addrs[i].addr_bytes[5]); 2696 return -EINVAL; 2697 } 2698 2699 memcpy(list->list[i].addr, mc_addrs[i].addr_bytes, 2700 sizeof(list->list[i].addr)); 2701 } 2702 2703 args.ops = add ? VIRTCHNL_OP_ADD_ETH_ADDR : VIRTCHNL_OP_DEL_ETH_ADDR; 2704 args.in_args = cmd_buffer; 2705 args.in_args_size = sizeof(struct virtchnl_ether_addr_list) + 2706 i * sizeof(struct virtchnl_ether_addr); 2707 args.out_buffer = vf->aq_resp; 2708 args.out_size = I40E_AQ_BUF_SZ; 2709 err = i40evf_execute_vf_cmd(dev, &args); 2710 if (err) { 2711 PMD_DRV_LOG(ERR, "fail to execute command %s", 2712 add ? "OP_ADD_ETH_ADDR" : "OP_DEL_ETH_ADDR"); 2713 return err; 2714 } 2715 2716 return 0; 2717 } 2718 2719 static int 2720 i40evf_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addrs, 2721 uint32_t mc_addrs_num) 2722 { 2723 struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private); 2724 int err; 2725 2726 /* flush previous addresses */ 2727 err = i40evf_add_del_mc_addr_list(dev, vf->mc_addrs, vf->mc_addrs_num, 2728 FALSE); 2729 if (err) 2730 return err; 2731 2732 vf->mc_addrs_num = 0; 2733 2734 /* add new ones */ 2735 err = i40evf_add_del_mc_addr_list(dev, mc_addrs, mc_addrs_num, 2736 TRUE); 2737 if (err) 2738 return err; 2739 2740 vf->mc_addrs_num = mc_addrs_num; 2741 memcpy(vf->mc_addrs, mc_addrs, mc_addrs_num * sizeof(*mc_addrs)); 2742 2743 return 0; 2744 } 2745