1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_pci.h> 17 #include <rte_atomic.h> 18 #include <rte_eal.h> 19 #include <rte_ether.h> 20 #include <rte_ethdev_driver.h> 21 #include <rte_ethdev_pci.h> 22 #include <rte_malloc.h> 23 #include <rte_memzone.h> 24 #include <rte_dev.h> 25 26 #include "ice_dcf.h" 27 #include "ice_rxtx.h" 28 29 #define ICE_DCF_AQ_LEN 32 30 #define ICE_DCF_AQ_BUF_SZ 4096 31 32 #define ICE_DCF_ARQ_MAX_RETRIES 200 33 #define ICE_DCF_ARQ_CHECK_TIME 2 /* msecs */ 34 35 #define ICE_DCF_VF_RES_BUF_SZ \ 36 (sizeof(struct virtchnl_vf_resource) + \ 37 IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)) 38 39 static __rte_always_inline int 40 ice_dcf_send_cmd_req_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 41 uint8_t *req_msg, uint16_t req_msglen) 42 { 43 return iavf_aq_send_msg_to_pf(&hw->avf, op, IAVF_SUCCESS, 44 req_msg, req_msglen, NULL); 45 } 46 47 static int 48 ice_dcf_recv_cmd_rsp_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 49 uint8_t *rsp_msgbuf, uint16_t rsp_buflen, 50 uint16_t *rsp_msglen) 51 { 52 struct iavf_arq_event_info event; 53 enum virtchnl_ops v_op; 54 int i = 0; 55 int err; 56 57 event.buf_len = rsp_buflen; 58 event.msg_buf = rsp_msgbuf; 59 60 do { 61 err = iavf_clean_arq_element(&hw->avf, &event, NULL); 62 if (err != IAVF_SUCCESS) 63 goto again; 64 65 v_op = rte_le_to_cpu_32(event.desc.cookie_high); 66 if (v_op != op) 67 goto again; 68 69 if (rsp_msglen != NULL) 70 *rsp_msglen = event.msg_len; 71 return rte_le_to_cpu_32(event.desc.cookie_low); 72 73 again: 74 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 75 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 76 77 return -EIO; 78 } 79 80 static __rte_always_inline void 81 ice_dcf_aq_cmd_clear(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 82 { 83 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 84 85 TAILQ_REMOVE(&hw->vc_cmd_queue, cmd, next); 86 87 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 88 } 89 90 static __rte_always_inline void 91 ice_dcf_vc_cmd_set(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 92 { 93 cmd->v_ret = IAVF_ERR_NOT_READY; 94 cmd->rsp_msglen = 0; 95 cmd->pending = 1; 96 97 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 98 99 TAILQ_INSERT_TAIL(&hw->vc_cmd_queue, cmd, next); 100 101 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 102 } 103 104 static __rte_always_inline int 105 ice_dcf_vc_cmd_send(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 106 { 107 return iavf_aq_send_msg_to_pf(&hw->avf, 108 cmd->v_op, IAVF_SUCCESS, 109 cmd->req_msg, cmd->req_msglen, NULL); 110 } 111 112 static __rte_always_inline void 113 ice_dcf_aq_cmd_handle(struct ice_dcf_hw *hw, struct iavf_arq_event_info *info) 114 { 115 struct dcf_virtchnl_cmd *cmd; 116 enum virtchnl_ops v_op; 117 enum iavf_status v_ret; 118 uint16_t aq_op; 119 120 aq_op = rte_le_to_cpu_16(info->desc.opcode); 121 if (unlikely(aq_op != iavf_aqc_opc_send_msg_to_vf)) { 122 PMD_DRV_LOG(ERR, 123 "Request %u is not supported yet", aq_op); 124 return; 125 } 126 127 v_op = rte_le_to_cpu_32(info->desc.cookie_high); 128 if (v_op == VIRTCHNL_OP_EVENT) { 129 if (hw->vc_event_msg_cb != NULL) 130 hw->vc_event_msg_cb(hw, 131 info->msg_buf, 132 info->msg_len); 133 return; 134 } 135 136 v_ret = rte_le_to_cpu_32(info->desc.cookie_low); 137 138 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 139 140 TAILQ_FOREACH(cmd, &hw->vc_cmd_queue, next) { 141 if (cmd->v_op == v_op && cmd->pending) { 142 cmd->v_ret = v_ret; 143 cmd->rsp_msglen = RTE_MIN(info->msg_len, 144 cmd->rsp_buflen); 145 if (likely(cmd->rsp_msglen != 0)) 146 rte_memcpy(cmd->rsp_msgbuf, info->msg_buf, 147 cmd->rsp_msglen); 148 149 /* prevent compiler reordering */ 150 rte_compiler_barrier(); 151 cmd->pending = 0; 152 break; 153 } 154 } 155 156 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 157 } 158 159 static void 160 ice_dcf_handle_virtchnl_msg(struct ice_dcf_hw *hw) 161 { 162 struct iavf_arq_event_info info; 163 uint16_t pending = 1; 164 int ret; 165 166 info.buf_len = ICE_DCF_AQ_BUF_SZ; 167 info.msg_buf = hw->arq_buf; 168 169 while (pending) { 170 ret = iavf_clean_arq_element(&hw->avf, &info, &pending); 171 if (ret != IAVF_SUCCESS) 172 break; 173 174 ice_dcf_aq_cmd_handle(hw, &info); 175 } 176 } 177 178 static int 179 ice_dcf_init_check_api_version(struct ice_dcf_hw *hw) 180 { 181 #define ICE_CPF_VIRTCHNL_VERSION_MAJOR_START 1 182 #define ICE_CPF_VIRTCHNL_VERSION_MINOR_START 1 183 struct virtchnl_version_info version, *pver; 184 int err; 185 186 version.major = VIRTCHNL_VERSION_MAJOR; 187 version.minor = VIRTCHNL_VERSION_MINOR; 188 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_VERSION, 189 (uint8_t *)&version, sizeof(version)); 190 if (err) { 191 PMD_INIT_LOG(ERR, "Failed to send OP_VERSION"); 192 return err; 193 } 194 195 pver = &hw->virtchnl_version; 196 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_VERSION, 197 (uint8_t *)pver, sizeof(*pver), NULL); 198 if (err) { 199 PMD_INIT_LOG(ERR, "Failed to get response of OP_VERSION"); 200 return -1; 201 } 202 203 PMD_INIT_LOG(DEBUG, 204 "Peer PF API version: %u.%u", pver->major, pver->minor); 205 206 if (pver->major < ICE_CPF_VIRTCHNL_VERSION_MAJOR_START || 207 (pver->major == ICE_CPF_VIRTCHNL_VERSION_MAJOR_START && 208 pver->minor < ICE_CPF_VIRTCHNL_VERSION_MINOR_START)) { 209 PMD_INIT_LOG(ERR, 210 "VIRTCHNL API version should not be lower than (%u.%u)", 211 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START, 212 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START); 213 return -1; 214 } else if (pver->major > VIRTCHNL_VERSION_MAJOR || 215 (pver->major == VIRTCHNL_VERSION_MAJOR && 216 pver->minor > VIRTCHNL_VERSION_MINOR)) { 217 PMD_INIT_LOG(ERR, 218 "PF/VF API version mismatch:(%u.%u)-(%u.%u)", 219 pver->major, pver->minor, 220 VIRTCHNL_VERSION_MAJOR, VIRTCHNL_VERSION_MINOR); 221 return -1; 222 } 223 224 PMD_INIT_LOG(DEBUG, "Peer is supported PF host"); 225 226 return 0; 227 } 228 229 static int 230 ice_dcf_get_vf_resource(struct ice_dcf_hw *hw) 231 { 232 uint32_t caps; 233 int err, i; 234 235 caps = VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RX_POLLING | 236 VIRTCHNL_VF_CAP_ADV_LINK_SPEED | VIRTCHNL_VF_CAP_DCF | 237 VF_BASE_MODE_OFFLOADS | VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC; 238 239 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 240 (uint8_t *)&caps, sizeof(caps)); 241 if (err) { 242 PMD_DRV_LOG(ERR, "Failed to send msg OP_GET_VF_RESOURCE"); 243 return err; 244 } 245 246 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 247 (uint8_t *)hw->vf_res, 248 ICE_DCF_VF_RES_BUF_SZ, NULL); 249 if (err) { 250 PMD_DRV_LOG(ERR, "Failed to get response of OP_GET_VF_RESOURCE"); 251 return -1; 252 } 253 254 iavf_vf_parse_hw_config(&hw->avf, hw->vf_res); 255 256 hw->vsi_res = NULL; 257 for (i = 0; i < hw->vf_res->num_vsis; i++) { 258 if (hw->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 259 hw->vsi_res = &hw->vf_res->vsi_res[i]; 260 } 261 262 if (!hw->vsi_res) { 263 PMD_DRV_LOG(ERR, "no LAN VSI found"); 264 return -1; 265 } 266 267 hw->vsi_id = hw->vsi_res->vsi_id; 268 PMD_DRV_LOG(DEBUG, "VSI ID is %u", hw->vsi_id); 269 270 return 0; 271 } 272 273 static int 274 ice_dcf_get_vf_vsi_map(struct ice_dcf_hw *hw) 275 { 276 struct virtchnl_dcf_vsi_map *vsi_map; 277 uint32_t valid_msg_len; 278 uint16_t len; 279 int err; 280 281 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 282 NULL, 0); 283 if (err) { 284 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_GET_VSI_MAP"); 285 return err; 286 } 287 288 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 289 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, 290 &len); 291 if (err) { 292 PMD_DRV_LOG(ERR, "Failed to get response of OP_DCF_GET_VSI_MAP"); 293 return err; 294 } 295 296 vsi_map = (struct virtchnl_dcf_vsi_map *)hw->arq_buf; 297 valid_msg_len = (vsi_map->num_vfs - 1) * sizeof(vsi_map->vf_vsi[0]) + 298 sizeof(*vsi_map); 299 if (len != valid_msg_len) { 300 PMD_DRV_LOG(ERR, "invalid vf vsi map response with length %u", 301 len); 302 return -EINVAL; 303 } 304 305 if (hw->num_vfs != 0 && hw->num_vfs != vsi_map->num_vfs) { 306 PMD_DRV_LOG(ERR, "The number VSI map (%u) doesn't match the number of VFs (%u)", 307 vsi_map->num_vfs, hw->num_vfs); 308 return -EINVAL; 309 } 310 311 len = vsi_map->num_vfs * sizeof(vsi_map->vf_vsi[0]); 312 313 if (!hw->vf_vsi_map) { 314 hw->vf_vsi_map = rte_zmalloc("vf_vsi_ctx", len, 0); 315 if (!hw->vf_vsi_map) { 316 PMD_DRV_LOG(ERR, "Failed to alloc memory for VSI context"); 317 return -ENOMEM; 318 } 319 320 hw->num_vfs = vsi_map->num_vfs; 321 } 322 323 if (!memcmp(hw->vf_vsi_map, vsi_map->vf_vsi, len)) { 324 PMD_DRV_LOG(DEBUG, "VF VSI map doesn't change"); 325 return 1; 326 } 327 328 rte_memcpy(hw->vf_vsi_map, vsi_map->vf_vsi, len); 329 return 0; 330 } 331 332 static int 333 ice_dcf_mode_disable(struct ice_dcf_hw *hw) 334 { 335 int err; 336 337 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 338 NULL, 0); 339 if (err) { 340 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_DISABLE"); 341 return err; 342 } 343 344 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 345 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, NULL); 346 if (err) { 347 PMD_DRV_LOG(ERR, 348 "Failed to get response of OP_DCF_DISABLE %d", 349 err); 350 return -1; 351 } 352 353 return 0; 354 } 355 356 static int 357 ice_dcf_check_reset_done(struct ice_dcf_hw *hw) 358 { 359 #define ICE_DCF_RESET_WAIT_CNT 50 360 struct iavf_hw *avf = &hw->avf; 361 int i, reset; 362 363 for (i = 0; i < ICE_DCF_RESET_WAIT_CNT; i++) { 364 reset = IAVF_READ_REG(avf, IAVF_VFGEN_RSTAT) & 365 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 366 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 367 368 if (reset == VIRTCHNL_VFR_VFACTIVE || 369 reset == VIRTCHNL_VFR_COMPLETED) 370 break; 371 372 rte_delay_ms(20); 373 } 374 375 if (i >= ICE_DCF_RESET_WAIT_CNT) 376 return -1; 377 378 return 0; 379 } 380 381 static inline void 382 ice_dcf_enable_irq0(struct ice_dcf_hw *hw) 383 { 384 struct iavf_hw *avf = &hw->avf; 385 386 /* Enable admin queue interrupt trigger */ 387 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 388 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 389 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 390 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 391 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 392 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 393 394 IAVF_WRITE_FLUSH(avf); 395 } 396 397 static inline void 398 ice_dcf_disable_irq0(struct ice_dcf_hw *hw) 399 { 400 struct iavf_hw *avf = &hw->avf; 401 402 /* Disable all interrupt types */ 403 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 0); 404 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 405 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 406 407 IAVF_WRITE_FLUSH(avf); 408 } 409 410 static void 411 ice_dcf_dev_interrupt_handler(void *param) 412 { 413 struct ice_dcf_hw *hw = param; 414 415 ice_dcf_disable_irq0(hw); 416 417 ice_dcf_handle_virtchnl_msg(hw); 418 419 ice_dcf_enable_irq0(hw); 420 } 421 422 int 423 ice_dcf_execute_virtchnl_cmd(struct ice_dcf_hw *hw, 424 struct dcf_virtchnl_cmd *cmd) 425 { 426 int i = 0; 427 int err; 428 429 if ((cmd->req_msg && !cmd->req_msglen) || 430 (!cmd->req_msg && cmd->req_msglen) || 431 (cmd->rsp_msgbuf && !cmd->rsp_buflen) || 432 (!cmd->rsp_msgbuf && cmd->rsp_buflen)) 433 return -EINVAL; 434 435 rte_spinlock_lock(&hw->vc_cmd_send_lock); 436 ice_dcf_vc_cmd_set(hw, cmd); 437 438 err = ice_dcf_vc_cmd_send(hw, cmd); 439 if (err) { 440 PMD_DRV_LOG(ERR, "fail to send cmd %d", cmd->v_op); 441 goto ret; 442 } 443 444 do { 445 if (!cmd->pending) 446 break; 447 448 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 449 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 450 451 if (cmd->v_ret != IAVF_SUCCESS) { 452 err = -1; 453 PMD_DRV_LOG(ERR, 454 "No response (%d times) or return failure (%d) for cmd %d", 455 i, cmd->v_ret, cmd->v_op); 456 } 457 458 ret: 459 ice_dcf_aq_cmd_clear(hw, cmd); 460 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 461 return err; 462 } 463 464 int 465 ice_dcf_send_aq_cmd(void *dcf_hw, struct ice_aq_desc *desc, 466 void *buf, uint16_t buf_size) 467 { 468 struct dcf_virtchnl_cmd desc_cmd, buff_cmd; 469 struct ice_dcf_hw *hw = dcf_hw; 470 int err = 0; 471 int i = 0; 472 473 if ((buf && !buf_size) || (!buf && buf_size) || 474 buf_size > ICE_DCF_AQ_BUF_SZ) 475 return -EINVAL; 476 477 desc_cmd.v_op = VIRTCHNL_OP_DCF_CMD_DESC; 478 desc_cmd.req_msglen = sizeof(*desc); 479 desc_cmd.req_msg = (uint8_t *)desc; 480 desc_cmd.rsp_buflen = sizeof(*desc); 481 desc_cmd.rsp_msgbuf = (uint8_t *)desc; 482 483 if (buf == NULL) 484 return ice_dcf_execute_virtchnl_cmd(hw, &desc_cmd); 485 486 desc->flags |= rte_cpu_to_le_16(ICE_AQ_FLAG_BUF); 487 488 buff_cmd.v_op = VIRTCHNL_OP_DCF_CMD_BUFF; 489 buff_cmd.req_msglen = buf_size; 490 buff_cmd.req_msg = buf; 491 buff_cmd.rsp_buflen = buf_size; 492 buff_cmd.rsp_msgbuf = buf; 493 494 rte_spinlock_lock(&hw->vc_cmd_send_lock); 495 ice_dcf_vc_cmd_set(hw, &desc_cmd); 496 ice_dcf_vc_cmd_set(hw, &buff_cmd); 497 498 if (ice_dcf_vc_cmd_send(hw, &desc_cmd) || 499 ice_dcf_vc_cmd_send(hw, &buff_cmd)) { 500 err = -1; 501 PMD_DRV_LOG(ERR, "fail to send OP_DCF_CMD_DESC/BUFF"); 502 goto ret; 503 } 504 505 do { 506 if ((!desc_cmd.pending && !buff_cmd.pending) || 507 (!desc_cmd.pending && desc_cmd.v_ret != IAVF_SUCCESS) || 508 (!buff_cmd.pending && buff_cmd.v_ret != IAVF_SUCCESS)) 509 break; 510 511 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 512 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 513 514 if (desc_cmd.v_ret != IAVF_SUCCESS || buff_cmd.v_ret != IAVF_SUCCESS) { 515 err = -1; 516 PMD_DRV_LOG(ERR, 517 "No response (%d times) or return failure (desc: %d / buff: %d)", 518 i, desc_cmd.v_ret, buff_cmd.v_ret); 519 } 520 521 ret: 522 ice_dcf_aq_cmd_clear(hw, &desc_cmd); 523 ice_dcf_aq_cmd_clear(hw, &buff_cmd); 524 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 525 526 return err; 527 } 528 529 int 530 ice_dcf_handle_vsi_update_event(struct ice_dcf_hw *hw) 531 { 532 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(hw->eth_dev); 533 int err = 0; 534 535 rte_spinlock_lock(&hw->vc_cmd_send_lock); 536 537 rte_intr_disable(&pci_dev->intr_handle); 538 ice_dcf_disable_irq0(hw); 539 540 if (ice_dcf_get_vf_resource(hw) || ice_dcf_get_vf_vsi_map(hw) < 0) 541 err = -1; 542 543 rte_intr_enable(&pci_dev->intr_handle); 544 ice_dcf_enable_irq0(hw); 545 546 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 547 548 return err; 549 } 550 551 static int 552 ice_dcf_get_supported_rxdid(struct ice_dcf_hw *hw) 553 { 554 int err; 555 556 err = ice_dcf_send_cmd_req_no_irq(hw, 557 VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 558 NULL, 0); 559 if (err) { 560 PMD_INIT_LOG(ERR, "Failed to send OP_GET_SUPPORTED_RXDIDS"); 561 return -1; 562 } 563 564 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 565 (uint8_t *)&hw->supported_rxdid, 566 sizeof(uint64_t), NULL); 567 if (err) { 568 PMD_INIT_LOG(ERR, "Failed to get response of OP_GET_SUPPORTED_RXDIDS"); 569 return -1; 570 } 571 572 return 0; 573 } 574 575 int 576 ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 577 { 578 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 579 int ret; 580 581 hw->avf.hw_addr = pci_dev->mem_resource[0].addr; 582 hw->avf.back = hw; 583 584 hw->avf.bus.bus_id = pci_dev->addr.bus; 585 hw->avf.bus.device = pci_dev->addr.devid; 586 hw->avf.bus.func = pci_dev->addr.function; 587 588 hw->avf.device_id = pci_dev->id.device_id; 589 hw->avf.vendor_id = pci_dev->id.vendor_id; 590 hw->avf.subsystem_device_id = pci_dev->id.subsystem_device_id; 591 hw->avf.subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 592 593 hw->avf.aq.num_arq_entries = ICE_DCF_AQ_LEN; 594 hw->avf.aq.num_asq_entries = ICE_DCF_AQ_LEN; 595 hw->avf.aq.arq_buf_size = ICE_DCF_AQ_BUF_SZ; 596 hw->avf.aq.asq_buf_size = ICE_DCF_AQ_BUF_SZ; 597 598 rte_spinlock_init(&hw->vc_cmd_send_lock); 599 rte_spinlock_init(&hw->vc_cmd_queue_lock); 600 TAILQ_INIT(&hw->vc_cmd_queue); 601 602 hw->arq_buf = rte_zmalloc("arq_buf", ICE_DCF_AQ_BUF_SZ, 0); 603 if (hw->arq_buf == NULL) { 604 PMD_INIT_LOG(ERR, "unable to allocate AdminQ buffer memory"); 605 goto err; 606 } 607 608 ret = iavf_set_mac_type(&hw->avf); 609 if (ret) { 610 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", ret); 611 goto err; 612 } 613 614 ret = ice_dcf_check_reset_done(hw); 615 if (ret) { 616 PMD_INIT_LOG(ERR, "VF is still resetting"); 617 goto err; 618 } 619 620 ret = iavf_init_adminq(&hw->avf); 621 if (ret) { 622 PMD_INIT_LOG(ERR, "init_adminq failed: %d", ret); 623 goto err; 624 } 625 626 if (ice_dcf_init_check_api_version(hw)) { 627 PMD_INIT_LOG(ERR, "check_api version failed"); 628 goto err_api; 629 } 630 631 hw->vf_res = rte_zmalloc("vf_res", ICE_DCF_VF_RES_BUF_SZ, 0); 632 if (hw->vf_res == NULL) { 633 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 634 goto err_api; 635 } 636 637 if (ice_dcf_get_vf_resource(hw)) { 638 PMD_INIT_LOG(ERR, "Failed to get VF resource"); 639 goto err_alloc; 640 } 641 642 if (ice_dcf_get_vf_vsi_map(hw) < 0) { 643 PMD_INIT_LOG(ERR, "Failed to get VF VSI map"); 644 ice_dcf_mode_disable(hw); 645 goto err_alloc; 646 } 647 648 /* Allocate memory for RSS info */ 649 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 650 hw->rss_key = rte_zmalloc(NULL, 651 hw->vf_res->rss_key_size, 0); 652 if (!hw->rss_key) { 653 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 654 goto err_alloc; 655 } 656 hw->rss_lut = rte_zmalloc("rss_lut", 657 hw->vf_res->rss_lut_size, 0); 658 if (!hw->rss_lut) { 659 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 660 goto err_rss; 661 } 662 } 663 664 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 665 if (ice_dcf_get_supported_rxdid(hw) != 0) { 666 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 667 goto err_rss; 668 } 669 } 670 671 hw->eth_dev = eth_dev; 672 rte_intr_callback_register(&pci_dev->intr_handle, 673 ice_dcf_dev_interrupt_handler, hw); 674 rte_intr_enable(&pci_dev->intr_handle); 675 ice_dcf_enable_irq0(hw); 676 677 return 0; 678 679 err_rss: 680 rte_free(hw->rss_key); 681 rte_free(hw->rss_lut); 682 err_alloc: 683 rte_free(hw->vf_res); 684 err_api: 685 iavf_shutdown_adminq(&hw->avf); 686 err: 687 rte_free(hw->arq_buf); 688 689 return -1; 690 } 691 692 void 693 ice_dcf_uninit_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 694 { 695 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 696 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 697 698 ice_dcf_disable_irq0(hw); 699 rte_intr_disable(intr_handle); 700 rte_intr_callback_unregister(intr_handle, 701 ice_dcf_dev_interrupt_handler, hw); 702 703 ice_dcf_mode_disable(hw); 704 iavf_shutdown_adminq(&hw->avf); 705 706 rte_free(hw->arq_buf); 707 rte_free(hw->vf_vsi_map); 708 rte_free(hw->vf_res); 709 rte_free(hw->rss_lut); 710 rte_free(hw->rss_key); 711 } 712 713 static int 714 ice_dcf_configure_rss_key(struct ice_dcf_hw *hw) 715 { 716 struct virtchnl_rss_key *rss_key; 717 struct dcf_virtchnl_cmd args; 718 int len, err; 719 720 len = sizeof(*rss_key) + hw->vf_res->rss_key_size - 1; 721 rss_key = rte_zmalloc("rss_key", len, 0); 722 if (!rss_key) 723 return -ENOMEM; 724 725 rss_key->vsi_id = hw->vsi_res->vsi_id; 726 rss_key->key_len = hw->vf_res->rss_key_size; 727 rte_memcpy(rss_key->key, hw->rss_key, hw->vf_res->rss_key_size); 728 729 args.v_op = VIRTCHNL_OP_CONFIG_RSS_KEY; 730 args.req_msglen = len; 731 args.req_msg = (uint8_t *)rss_key; 732 args.rsp_msglen = 0; 733 args.rsp_buflen = 0; 734 args.rsp_msgbuf = NULL; 735 args.pending = 0; 736 737 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 738 if (err) 739 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_KEY"); 740 741 rte_free(rss_key); 742 return err; 743 } 744 745 static int 746 ice_dcf_configure_rss_lut(struct ice_dcf_hw *hw) 747 { 748 struct virtchnl_rss_lut *rss_lut; 749 struct dcf_virtchnl_cmd args; 750 int len, err; 751 752 len = sizeof(*rss_lut) + hw->vf_res->rss_lut_size - 1; 753 rss_lut = rte_zmalloc("rss_lut", len, 0); 754 if (!rss_lut) 755 return -ENOMEM; 756 757 rss_lut->vsi_id = hw->vsi_res->vsi_id; 758 rss_lut->lut_entries = hw->vf_res->rss_lut_size; 759 rte_memcpy(rss_lut->lut, hw->rss_lut, hw->vf_res->rss_lut_size); 760 761 args.v_op = VIRTCHNL_OP_CONFIG_RSS_LUT; 762 args.req_msglen = len; 763 args.req_msg = (uint8_t *)rss_lut; 764 args.rsp_msglen = 0; 765 args.rsp_buflen = 0; 766 args.rsp_msgbuf = NULL; 767 args.pending = 0; 768 769 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 770 if (err) 771 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_LUT"); 772 773 rte_free(rss_lut); 774 return err; 775 } 776 777 int 778 ice_dcf_init_rss(struct ice_dcf_hw *hw) 779 { 780 struct rte_eth_dev *dev = hw->eth_dev; 781 struct rte_eth_rss_conf *rss_conf; 782 uint8_t i, j, nb_q; 783 int ret; 784 785 rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf; 786 nb_q = dev->data->nb_rx_queues; 787 788 if (!(hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 789 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 790 return -ENOTSUP; 791 } 792 if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 793 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default"); 794 /* set all lut items to default queue */ 795 memset(hw->rss_lut, 0, hw->vf_res->rss_lut_size); 796 return ice_dcf_configure_rss_lut(hw); 797 } 798 799 /* In IAVF, RSS enablement is set by PF driver. It is not supported 800 * to set based on rss_conf->rss_hf. 801 */ 802 803 /* configure RSS key */ 804 if (!rss_conf->rss_key) 805 /* Calculate the default hash key */ 806 for (i = 0; i < hw->vf_res->rss_key_size; i++) 807 hw->rss_key[i] = (uint8_t)rte_rand(); 808 else 809 rte_memcpy(hw->rss_key, rss_conf->rss_key, 810 RTE_MIN(rss_conf->rss_key_len, 811 hw->vf_res->rss_key_size)); 812 813 /* init RSS LUT table */ 814 for (i = 0, j = 0; i < hw->vf_res->rss_lut_size; i++, j++) { 815 if (j >= nb_q) 816 j = 0; 817 hw->rss_lut[i] = j; 818 } 819 /* send virtchnnl ops to configure rss*/ 820 ret = ice_dcf_configure_rss_lut(hw); 821 if (ret) 822 return ret; 823 ret = ice_dcf_configure_rss_key(hw); 824 if (ret) 825 return ret; 826 827 return 0; 828 } 829 830 #define IAVF_RXDID_LEGACY_1 1 831 #define IAVF_RXDID_COMMS_GENERIC 16 832 833 int 834 ice_dcf_configure_queues(struct ice_dcf_hw *hw) 835 { 836 struct ice_rx_queue **rxq = 837 (struct ice_rx_queue **)hw->eth_dev->data->rx_queues; 838 struct ice_tx_queue **txq = 839 (struct ice_tx_queue **)hw->eth_dev->data->tx_queues; 840 struct virtchnl_vsi_queue_config_info *vc_config; 841 struct virtchnl_queue_pair_info *vc_qp; 842 struct dcf_virtchnl_cmd args; 843 uint16_t i, size; 844 int err; 845 846 size = sizeof(*vc_config) + 847 sizeof(vc_config->qpair[0]) * hw->num_queue_pairs; 848 vc_config = rte_zmalloc("cfg_queue", size, 0); 849 if (!vc_config) 850 return -ENOMEM; 851 852 vc_config->vsi_id = hw->vsi_res->vsi_id; 853 vc_config->num_queue_pairs = hw->num_queue_pairs; 854 855 for (i = 0, vc_qp = vc_config->qpair; 856 i < hw->num_queue_pairs; 857 i++, vc_qp++) { 858 vc_qp->txq.vsi_id = hw->vsi_res->vsi_id; 859 vc_qp->txq.queue_id = i; 860 if (i < hw->eth_dev->data->nb_tx_queues) { 861 vc_qp->txq.ring_len = txq[i]->nb_tx_desc; 862 vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_dma; 863 } 864 vc_qp->rxq.vsi_id = hw->vsi_res->vsi_id; 865 vc_qp->rxq.queue_id = i; 866 vc_qp->rxq.max_pkt_size = rxq[i]->max_pkt_len; 867 868 if (i >= hw->eth_dev->data->nb_rx_queues) 869 continue; 870 871 vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc; 872 vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_dma; 873 vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len; 874 875 if (hw->vf_res->vf_cap_flags & 876 VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && 877 hw->supported_rxdid & 878 BIT(IAVF_RXDID_COMMS_GENERIC)) { 879 vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_GENERIC; 880 PMD_DRV_LOG(NOTICE, "request RXDID == %d in " 881 "Queue[%d]", vc_qp->rxq.rxdid, i); 882 } else { 883 PMD_DRV_LOG(ERR, "RXDID 16 is not supported"); 884 return -EINVAL; 885 } 886 } 887 888 memset(&args, 0, sizeof(args)); 889 args.v_op = VIRTCHNL_OP_CONFIG_VSI_QUEUES; 890 args.req_msg = (uint8_t *)vc_config; 891 args.req_msglen = size; 892 893 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 894 if (err) 895 PMD_DRV_LOG(ERR, "Failed to execute command of" 896 " VIRTCHNL_OP_CONFIG_VSI_QUEUES"); 897 898 rte_free(vc_config); 899 return err; 900 } 901 902 int 903 ice_dcf_config_irq_map(struct ice_dcf_hw *hw) 904 { 905 struct virtchnl_irq_map_info *map_info; 906 struct virtchnl_vector_map *vecmap; 907 struct dcf_virtchnl_cmd args; 908 int len, i, err; 909 910 len = sizeof(struct virtchnl_irq_map_info) + 911 sizeof(struct virtchnl_vector_map) * hw->nb_msix; 912 913 map_info = rte_zmalloc("map_info", len, 0); 914 if (!map_info) 915 return -ENOMEM; 916 917 map_info->num_vectors = hw->nb_msix; 918 for (i = 0; i < hw->nb_msix; i++) { 919 vecmap = &map_info->vecmap[i]; 920 vecmap->vsi_id = hw->vsi_res->vsi_id; 921 vecmap->rxitr_idx = 0; 922 vecmap->vector_id = hw->msix_base + i; 923 vecmap->txq_map = 0; 924 vecmap->rxq_map = hw->rxq_map[hw->msix_base + i]; 925 } 926 927 memset(&args, 0, sizeof(args)); 928 args.v_op = VIRTCHNL_OP_CONFIG_IRQ_MAP; 929 args.req_msg = (u8 *)map_info; 930 args.req_msglen = len; 931 932 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 933 if (err) 934 PMD_DRV_LOG(ERR, "fail to execute command OP_CONFIG_IRQ_MAP"); 935 936 rte_free(map_info); 937 return err; 938 } 939 940 int 941 ice_dcf_switch_queue(struct ice_dcf_hw *hw, uint16_t qid, bool rx, bool on) 942 { 943 struct virtchnl_queue_select queue_select; 944 struct dcf_virtchnl_cmd args; 945 int err; 946 947 memset(&queue_select, 0, sizeof(queue_select)); 948 queue_select.vsi_id = hw->vsi_res->vsi_id; 949 if (rx) 950 queue_select.rx_queues |= 1 << qid; 951 else 952 queue_select.tx_queues |= 1 << qid; 953 954 memset(&args, 0, sizeof(args)); 955 if (on) 956 args.v_op = VIRTCHNL_OP_ENABLE_QUEUES; 957 else 958 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 959 960 args.req_msg = (u8 *)&queue_select; 961 args.req_msglen = sizeof(queue_select); 962 963 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 964 if (err) 965 PMD_DRV_LOG(ERR, "Failed to execute command of %s", 966 on ? "OP_ENABLE_QUEUES" : "OP_DISABLE_QUEUES"); 967 968 return err; 969 } 970 971 int 972 ice_dcf_disable_queues(struct ice_dcf_hw *hw) 973 { 974 struct virtchnl_queue_select queue_select; 975 struct dcf_virtchnl_cmd args; 976 int err; 977 978 memset(&queue_select, 0, sizeof(queue_select)); 979 queue_select.vsi_id = hw->vsi_res->vsi_id; 980 981 queue_select.rx_queues = BIT(hw->eth_dev->data->nb_rx_queues) - 1; 982 queue_select.tx_queues = BIT(hw->eth_dev->data->nb_tx_queues) - 1; 983 984 memset(&args, 0, sizeof(args)); 985 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 986 args.req_msg = (u8 *)&queue_select; 987 args.req_msglen = sizeof(queue_select); 988 989 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 990 if (err) 991 PMD_DRV_LOG(ERR, 992 "Failed to execute command of OP_DISABLE_QUEUES"); 993 994 return err; 995 } 996 997 int 998 ice_dcf_query_stats(struct ice_dcf_hw *hw, 999 struct virtchnl_eth_stats *pstats) 1000 { 1001 struct virtchnl_queue_select q_stats; 1002 struct dcf_virtchnl_cmd args; 1003 int err; 1004 1005 memset(&q_stats, 0, sizeof(q_stats)); 1006 q_stats.vsi_id = hw->vsi_res->vsi_id; 1007 1008 args.v_op = VIRTCHNL_OP_GET_STATS; 1009 args.req_msg = (uint8_t *)&q_stats; 1010 args.req_msglen = sizeof(q_stats); 1011 args.rsp_msglen = sizeof(*pstats); 1012 args.rsp_msgbuf = (uint8_t *)pstats; 1013 args.rsp_buflen = sizeof(*pstats); 1014 1015 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1016 if (err) { 1017 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS"); 1018 return err; 1019 } 1020 1021 return 0; 1022 } 1023 1024 int 1025 ice_dcf_add_del_all_mac_addr(struct ice_dcf_hw *hw, bool add) 1026 { 1027 struct virtchnl_ether_addr_list *list; 1028 struct rte_ether_addr *addr; 1029 struct dcf_virtchnl_cmd args; 1030 int len, err = 0; 1031 1032 len = sizeof(struct virtchnl_ether_addr_list); 1033 addr = hw->eth_dev->data->mac_addrs; 1034 len += sizeof(struct virtchnl_ether_addr); 1035 1036 list = rte_zmalloc(NULL, len, 0); 1037 if (!list) { 1038 PMD_DRV_LOG(ERR, "fail to allocate memory"); 1039 return -ENOMEM; 1040 } 1041 1042 rte_memcpy(list->list[0].addr, addr->addr_bytes, 1043 sizeof(addr->addr_bytes)); 1044 PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x", 1045 addr->addr_bytes[0], addr->addr_bytes[1], 1046 addr->addr_bytes[2], addr->addr_bytes[3], 1047 addr->addr_bytes[4], addr->addr_bytes[5]); 1048 1049 list->vsi_id = hw->vsi_res->vsi_id; 1050 list->num_elements = 1; 1051 1052 memset(&args, 0, sizeof(args)); 1053 args.v_op = add ? VIRTCHNL_OP_ADD_ETH_ADDR : 1054 VIRTCHNL_OP_DEL_ETH_ADDR; 1055 args.req_msg = (uint8_t *)list; 1056 args.req_msglen = len; 1057 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1058 if (err) 1059 PMD_DRV_LOG(ERR, "fail to execute command %s", 1060 add ? "OP_ADD_ETHER_ADDRESS" : 1061 "OP_DEL_ETHER_ADDRESS"); 1062 rte_free(list); 1063 return err; 1064 } 1065