1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_pci.h> 17 #include <rte_atomic.h> 18 #include <rte_eal.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <ethdev_pci.h> 22 #include <rte_malloc.h> 23 #include <rte_memzone.h> 24 #include <rte_dev.h> 25 26 #include "ice_dcf.h" 27 #include "ice_rxtx.h" 28 29 #define ICE_DCF_AQ_LEN 32 30 #define ICE_DCF_AQ_BUF_SZ 4096 31 32 #define ICE_DCF_ARQ_MAX_RETRIES 200 33 #define ICE_DCF_ARQ_CHECK_TIME 2 /* msecs */ 34 35 #define ICE_DCF_VF_RES_BUF_SZ \ 36 (sizeof(struct virtchnl_vf_resource) + \ 37 IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)) 38 39 static __rte_always_inline int 40 ice_dcf_send_cmd_req_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 41 uint8_t *req_msg, uint16_t req_msglen) 42 { 43 return iavf_aq_send_msg_to_pf(&hw->avf, op, IAVF_SUCCESS, 44 req_msg, req_msglen, NULL); 45 } 46 47 static int 48 ice_dcf_recv_cmd_rsp_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 49 uint8_t *rsp_msgbuf, uint16_t rsp_buflen, 50 uint16_t *rsp_msglen) 51 { 52 struct iavf_arq_event_info event; 53 enum virtchnl_ops v_op; 54 int i = 0; 55 int err; 56 57 event.buf_len = rsp_buflen; 58 event.msg_buf = rsp_msgbuf; 59 60 do { 61 err = iavf_clean_arq_element(&hw->avf, &event, NULL); 62 if (err != IAVF_SUCCESS) 63 goto again; 64 65 v_op = rte_le_to_cpu_32(event.desc.cookie_high); 66 if (v_op != op) 67 goto again; 68 69 if (rsp_msglen != NULL) 70 *rsp_msglen = event.msg_len; 71 return rte_le_to_cpu_32(event.desc.cookie_low); 72 73 again: 74 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 75 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 76 77 return -EIO; 78 } 79 80 static __rte_always_inline void 81 ice_dcf_aq_cmd_clear(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 82 { 83 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 84 85 TAILQ_REMOVE(&hw->vc_cmd_queue, cmd, next); 86 87 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 88 } 89 90 static __rte_always_inline void 91 ice_dcf_vc_cmd_set(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 92 { 93 cmd->v_ret = IAVF_ERR_NOT_READY; 94 cmd->rsp_msglen = 0; 95 cmd->pending = 1; 96 97 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 98 99 TAILQ_INSERT_TAIL(&hw->vc_cmd_queue, cmd, next); 100 101 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 102 } 103 104 static __rte_always_inline int 105 ice_dcf_vc_cmd_send(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 106 { 107 return iavf_aq_send_msg_to_pf(&hw->avf, 108 cmd->v_op, IAVF_SUCCESS, 109 cmd->req_msg, cmd->req_msglen, NULL); 110 } 111 112 static __rte_always_inline void 113 ice_dcf_aq_cmd_handle(struct ice_dcf_hw *hw, struct iavf_arq_event_info *info) 114 { 115 struct dcf_virtchnl_cmd *cmd; 116 enum virtchnl_ops v_op; 117 enum iavf_status v_ret; 118 uint16_t aq_op; 119 120 aq_op = rte_le_to_cpu_16(info->desc.opcode); 121 if (unlikely(aq_op != iavf_aqc_opc_send_msg_to_vf)) { 122 PMD_DRV_LOG(ERR, 123 "Request %u is not supported yet", aq_op); 124 return; 125 } 126 127 v_op = rte_le_to_cpu_32(info->desc.cookie_high); 128 if (v_op == VIRTCHNL_OP_EVENT) { 129 if (hw->vc_event_msg_cb != NULL) 130 hw->vc_event_msg_cb(hw, 131 info->msg_buf, 132 info->msg_len); 133 return; 134 } 135 136 v_ret = rte_le_to_cpu_32(info->desc.cookie_low); 137 138 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 139 140 TAILQ_FOREACH(cmd, &hw->vc_cmd_queue, next) { 141 if (cmd->v_op == v_op && cmd->pending) { 142 cmd->v_ret = v_ret; 143 cmd->rsp_msglen = RTE_MIN(info->msg_len, 144 cmd->rsp_buflen); 145 if (likely(cmd->rsp_msglen != 0)) 146 rte_memcpy(cmd->rsp_msgbuf, info->msg_buf, 147 cmd->rsp_msglen); 148 149 /* prevent compiler reordering */ 150 rte_compiler_barrier(); 151 cmd->pending = 0; 152 break; 153 } 154 } 155 156 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 157 } 158 159 static void 160 ice_dcf_handle_virtchnl_msg(struct ice_dcf_hw *hw) 161 { 162 struct iavf_arq_event_info info; 163 uint16_t pending = 1; 164 int ret; 165 166 info.buf_len = ICE_DCF_AQ_BUF_SZ; 167 info.msg_buf = hw->arq_buf; 168 169 while (pending) { 170 ret = iavf_clean_arq_element(&hw->avf, &info, &pending); 171 if (ret != IAVF_SUCCESS) 172 break; 173 174 ice_dcf_aq_cmd_handle(hw, &info); 175 } 176 } 177 178 static int 179 ice_dcf_init_check_api_version(struct ice_dcf_hw *hw) 180 { 181 #define ICE_CPF_VIRTCHNL_VERSION_MAJOR_START 1 182 #define ICE_CPF_VIRTCHNL_VERSION_MINOR_START 1 183 struct virtchnl_version_info version, *pver; 184 int err; 185 186 version.major = VIRTCHNL_VERSION_MAJOR; 187 version.minor = VIRTCHNL_VERSION_MINOR; 188 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_VERSION, 189 (uint8_t *)&version, sizeof(version)); 190 if (err) { 191 PMD_INIT_LOG(ERR, "Failed to send OP_VERSION"); 192 return err; 193 } 194 195 pver = &hw->virtchnl_version; 196 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_VERSION, 197 (uint8_t *)pver, sizeof(*pver), NULL); 198 if (err) { 199 PMD_INIT_LOG(ERR, "Failed to get response of OP_VERSION"); 200 return -1; 201 } 202 203 PMD_INIT_LOG(DEBUG, 204 "Peer PF API version: %u.%u", pver->major, pver->minor); 205 206 if (pver->major < ICE_CPF_VIRTCHNL_VERSION_MAJOR_START || 207 (pver->major == ICE_CPF_VIRTCHNL_VERSION_MAJOR_START && 208 pver->minor < ICE_CPF_VIRTCHNL_VERSION_MINOR_START)) { 209 PMD_INIT_LOG(ERR, 210 "VIRTCHNL API version should not be lower than (%u.%u)", 211 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START, 212 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START); 213 return -1; 214 } else if (pver->major > VIRTCHNL_VERSION_MAJOR || 215 (pver->major == VIRTCHNL_VERSION_MAJOR && 216 pver->minor > VIRTCHNL_VERSION_MINOR)) { 217 PMD_INIT_LOG(ERR, 218 "PF/VF API version mismatch:(%u.%u)-(%u.%u)", 219 pver->major, pver->minor, 220 VIRTCHNL_VERSION_MAJOR, VIRTCHNL_VERSION_MINOR); 221 return -1; 222 } 223 224 PMD_INIT_LOG(DEBUG, "Peer is supported PF host"); 225 226 return 0; 227 } 228 229 static int 230 ice_dcf_get_vf_resource(struct ice_dcf_hw *hw) 231 { 232 uint32_t caps; 233 int err, i; 234 235 caps = VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RX_POLLING | 236 VIRTCHNL_VF_CAP_ADV_LINK_SPEED | VIRTCHNL_VF_CAP_DCF | 237 VIRTCHNL_VF_OFFLOAD_VLAN_V2 | 238 VF_BASE_MODE_OFFLOADS | VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC; 239 240 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 241 (uint8_t *)&caps, sizeof(caps)); 242 if (err) { 243 PMD_DRV_LOG(ERR, "Failed to send msg OP_GET_VF_RESOURCE"); 244 return err; 245 } 246 247 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 248 (uint8_t *)hw->vf_res, 249 ICE_DCF_VF_RES_BUF_SZ, NULL); 250 if (err) { 251 PMD_DRV_LOG(ERR, "Failed to get response of OP_GET_VF_RESOURCE"); 252 return -1; 253 } 254 255 iavf_vf_parse_hw_config(&hw->avf, hw->vf_res); 256 257 hw->vsi_res = NULL; 258 for (i = 0; i < hw->vf_res->num_vsis; i++) { 259 if (hw->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 260 hw->vsi_res = &hw->vf_res->vsi_res[i]; 261 } 262 263 if (!hw->vsi_res) { 264 PMD_DRV_LOG(ERR, "no LAN VSI found"); 265 return -1; 266 } 267 268 hw->vsi_id = hw->vsi_res->vsi_id; 269 PMD_DRV_LOG(DEBUG, "VSI ID is %u", hw->vsi_id); 270 271 return 0; 272 } 273 274 static int 275 ice_dcf_get_vf_vsi_map(struct ice_dcf_hw *hw) 276 { 277 struct virtchnl_dcf_vsi_map *vsi_map; 278 uint32_t valid_msg_len; 279 uint16_t len; 280 int err; 281 282 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 283 NULL, 0); 284 if (err) { 285 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_GET_VSI_MAP"); 286 return err; 287 } 288 289 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 290 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, 291 &len); 292 if (err) { 293 PMD_DRV_LOG(ERR, "Failed to get response of OP_DCF_GET_VSI_MAP"); 294 return err; 295 } 296 297 vsi_map = (struct virtchnl_dcf_vsi_map *)hw->arq_buf; 298 valid_msg_len = (vsi_map->num_vfs - 1) * sizeof(vsi_map->vf_vsi[0]) + 299 sizeof(*vsi_map); 300 if (len != valid_msg_len) { 301 PMD_DRV_LOG(ERR, "invalid vf vsi map response with length %u", 302 len); 303 return -EINVAL; 304 } 305 306 if (hw->num_vfs != 0 && hw->num_vfs != vsi_map->num_vfs) { 307 PMD_DRV_LOG(ERR, "The number VSI map (%u) doesn't match the number of VFs (%u)", 308 vsi_map->num_vfs, hw->num_vfs); 309 return -EINVAL; 310 } 311 312 len = vsi_map->num_vfs * sizeof(vsi_map->vf_vsi[0]); 313 314 if (!hw->vf_vsi_map) { 315 hw->vf_vsi_map = rte_zmalloc("vf_vsi_ctx", len, 0); 316 if (!hw->vf_vsi_map) { 317 PMD_DRV_LOG(ERR, "Failed to alloc memory for VSI context"); 318 return -ENOMEM; 319 } 320 321 hw->num_vfs = vsi_map->num_vfs; 322 hw->pf_vsi_id = vsi_map->pf_vsi; 323 } 324 325 if (!memcmp(hw->vf_vsi_map, vsi_map->vf_vsi, len)) { 326 PMD_DRV_LOG(DEBUG, "VF VSI map doesn't change"); 327 return 1; 328 } 329 330 rte_memcpy(hw->vf_vsi_map, vsi_map->vf_vsi, len); 331 return 0; 332 } 333 334 static int 335 ice_dcf_mode_disable(struct ice_dcf_hw *hw) 336 { 337 int err; 338 339 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 340 NULL, 0); 341 if (err) { 342 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_DISABLE"); 343 return err; 344 } 345 346 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 347 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, NULL); 348 if (err) { 349 PMD_DRV_LOG(ERR, 350 "Failed to get response of OP_DCF_DISABLE %d", 351 err); 352 return -1; 353 } 354 355 return 0; 356 } 357 358 static int 359 ice_dcf_check_reset_done(struct ice_dcf_hw *hw) 360 { 361 #define ICE_DCF_RESET_WAIT_CNT 50 362 struct iavf_hw *avf = &hw->avf; 363 int i, reset; 364 365 for (i = 0; i < ICE_DCF_RESET_WAIT_CNT; i++) { 366 reset = IAVF_READ_REG(avf, IAVF_VFGEN_RSTAT) & 367 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 368 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 369 370 if (reset == VIRTCHNL_VFR_VFACTIVE || 371 reset == VIRTCHNL_VFR_COMPLETED) 372 break; 373 374 rte_delay_ms(20); 375 } 376 377 if (i >= ICE_DCF_RESET_WAIT_CNT) 378 return -1; 379 380 return 0; 381 } 382 383 static inline void 384 ice_dcf_enable_irq0(struct ice_dcf_hw *hw) 385 { 386 struct iavf_hw *avf = &hw->avf; 387 388 /* Enable admin queue interrupt trigger */ 389 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 390 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 391 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 392 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 393 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 394 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 395 396 IAVF_WRITE_FLUSH(avf); 397 } 398 399 static inline void 400 ice_dcf_disable_irq0(struct ice_dcf_hw *hw) 401 { 402 struct iavf_hw *avf = &hw->avf; 403 404 /* Disable all interrupt types */ 405 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 0); 406 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 407 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 408 409 IAVF_WRITE_FLUSH(avf); 410 } 411 412 static void 413 ice_dcf_dev_interrupt_handler(void *param) 414 { 415 struct ice_dcf_hw *hw = param; 416 417 ice_dcf_disable_irq0(hw); 418 419 ice_dcf_handle_virtchnl_msg(hw); 420 421 ice_dcf_enable_irq0(hw); 422 } 423 424 int 425 ice_dcf_execute_virtchnl_cmd(struct ice_dcf_hw *hw, 426 struct dcf_virtchnl_cmd *cmd) 427 { 428 int i = 0; 429 int err; 430 431 if ((cmd->req_msg && !cmd->req_msglen) || 432 (!cmd->req_msg && cmd->req_msglen) || 433 (cmd->rsp_msgbuf && !cmd->rsp_buflen) || 434 (!cmd->rsp_msgbuf && cmd->rsp_buflen)) 435 return -EINVAL; 436 437 rte_spinlock_lock(&hw->vc_cmd_send_lock); 438 ice_dcf_vc_cmd_set(hw, cmd); 439 440 err = ice_dcf_vc_cmd_send(hw, cmd); 441 if (err) { 442 PMD_DRV_LOG(ERR, "fail to send cmd %d", cmd->v_op); 443 goto ret; 444 } 445 446 do { 447 if (!cmd->pending) 448 break; 449 450 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 451 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 452 453 if (cmd->v_ret != IAVF_SUCCESS) { 454 err = -1; 455 PMD_DRV_LOG(ERR, 456 "No response (%d times) or return failure (%d) for cmd %d", 457 i, cmd->v_ret, cmd->v_op); 458 } 459 460 ret: 461 ice_dcf_aq_cmd_clear(hw, cmd); 462 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 463 return err; 464 } 465 466 int 467 ice_dcf_send_aq_cmd(void *dcf_hw, struct ice_aq_desc *desc, 468 void *buf, uint16_t buf_size) 469 { 470 struct dcf_virtchnl_cmd desc_cmd, buff_cmd; 471 struct ice_dcf_hw *hw = dcf_hw; 472 int err = 0; 473 int i = 0; 474 475 if ((buf && !buf_size) || (!buf && buf_size) || 476 buf_size > ICE_DCF_AQ_BUF_SZ) 477 return -EINVAL; 478 479 desc_cmd.v_op = VIRTCHNL_OP_DCF_CMD_DESC; 480 desc_cmd.req_msglen = sizeof(*desc); 481 desc_cmd.req_msg = (uint8_t *)desc; 482 desc_cmd.rsp_buflen = sizeof(*desc); 483 desc_cmd.rsp_msgbuf = (uint8_t *)desc; 484 485 if (buf == NULL) 486 return ice_dcf_execute_virtchnl_cmd(hw, &desc_cmd); 487 488 desc->flags |= rte_cpu_to_le_16(ICE_AQ_FLAG_BUF); 489 490 buff_cmd.v_op = VIRTCHNL_OP_DCF_CMD_BUFF; 491 buff_cmd.req_msglen = buf_size; 492 buff_cmd.req_msg = buf; 493 buff_cmd.rsp_buflen = buf_size; 494 buff_cmd.rsp_msgbuf = buf; 495 496 rte_spinlock_lock(&hw->vc_cmd_send_lock); 497 ice_dcf_vc_cmd_set(hw, &desc_cmd); 498 ice_dcf_vc_cmd_set(hw, &buff_cmd); 499 500 if (ice_dcf_vc_cmd_send(hw, &desc_cmd) || 501 ice_dcf_vc_cmd_send(hw, &buff_cmd)) { 502 err = -1; 503 PMD_DRV_LOG(ERR, "fail to send OP_DCF_CMD_DESC/BUFF"); 504 goto ret; 505 } 506 507 do { 508 if (!desc_cmd.pending && !buff_cmd.pending) 509 break; 510 511 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 512 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 513 514 if (desc_cmd.v_ret != IAVF_SUCCESS || buff_cmd.v_ret != IAVF_SUCCESS) { 515 err = -1; 516 PMD_DRV_LOG(ERR, 517 "No response (%d times) or return failure (desc: %d / buff: %d)", 518 i, desc_cmd.v_ret, buff_cmd.v_ret); 519 } 520 521 ret: 522 ice_dcf_aq_cmd_clear(hw, &desc_cmd); 523 ice_dcf_aq_cmd_clear(hw, &buff_cmd); 524 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 525 526 return err; 527 } 528 529 int 530 ice_dcf_handle_vsi_update_event(struct ice_dcf_hw *hw) 531 { 532 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(hw->eth_dev); 533 int err = 0; 534 535 rte_spinlock_lock(&hw->vc_cmd_send_lock); 536 537 rte_intr_disable(&pci_dev->intr_handle); 538 ice_dcf_disable_irq0(hw); 539 540 if (ice_dcf_get_vf_resource(hw) || ice_dcf_get_vf_vsi_map(hw) < 0) 541 err = -1; 542 543 rte_intr_enable(&pci_dev->intr_handle); 544 ice_dcf_enable_irq0(hw); 545 546 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 547 548 return err; 549 } 550 551 static int 552 ice_dcf_get_supported_rxdid(struct ice_dcf_hw *hw) 553 { 554 int err; 555 556 err = ice_dcf_send_cmd_req_no_irq(hw, 557 VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 558 NULL, 0); 559 if (err) { 560 PMD_INIT_LOG(ERR, "Failed to send OP_GET_SUPPORTED_RXDIDS"); 561 return -1; 562 } 563 564 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 565 (uint8_t *)&hw->supported_rxdid, 566 sizeof(uint64_t), NULL); 567 if (err) { 568 PMD_INIT_LOG(ERR, "Failed to get response of OP_GET_SUPPORTED_RXDIDS"); 569 return -1; 570 } 571 572 return 0; 573 } 574 575 int 576 ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 577 { 578 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 579 int ret; 580 581 hw->avf.hw_addr = pci_dev->mem_resource[0].addr; 582 hw->avf.back = hw; 583 584 hw->avf.bus.bus_id = pci_dev->addr.bus; 585 hw->avf.bus.device = pci_dev->addr.devid; 586 hw->avf.bus.func = pci_dev->addr.function; 587 588 hw->avf.device_id = pci_dev->id.device_id; 589 hw->avf.vendor_id = pci_dev->id.vendor_id; 590 hw->avf.subsystem_device_id = pci_dev->id.subsystem_device_id; 591 hw->avf.subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 592 593 hw->avf.aq.num_arq_entries = ICE_DCF_AQ_LEN; 594 hw->avf.aq.num_asq_entries = ICE_DCF_AQ_LEN; 595 hw->avf.aq.arq_buf_size = ICE_DCF_AQ_BUF_SZ; 596 hw->avf.aq.asq_buf_size = ICE_DCF_AQ_BUF_SZ; 597 598 rte_spinlock_init(&hw->vc_cmd_send_lock); 599 rte_spinlock_init(&hw->vc_cmd_queue_lock); 600 TAILQ_INIT(&hw->vc_cmd_queue); 601 602 hw->arq_buf = rte_zmalloc("arq_buf", ICE_DCF_AQ_BUF_SZ, 0); 603 if (hw->arq_buf == NULL) { 604 PMD_INIT_LOG(ERR, "unable to allocate AdminQ buffer memory"); 605 goto err; 606 } 607 608 ret = iavf_set_mac_type(&hw->avf); 609 if (ret) { 610 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", ret); 611 goto err; 612 } 613 614 ret = ice_dcf_check_reset_done(hw); 615 if (ret) { 616 PMD_INIT_LOG(ERR, "VF is still resetting"); 617 goto err; 618 } 619 620 ret = iavf_init_adminq(&hw->avf); 621 if (ret) { 622 PMD_INIT_LOG(ERR, "init_adminq failed: %d", ret); 623 goto err; 624 } 625 626 if (ice_dcf_init_check_api_version(hw)) { 627 PMD_INIT_LOG(ERR, "check_api version failed"); 628 goto err_api; 629 } 630 631 hw->vf_res = rte_zmalloc("vf_res", ICE_DCF_VF_RES_BUF_SZ, 0); 632 if (hw->vf_res == NULL) { 633 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 634 goto err_api; 635 } 636 637 if (ice_dcf_get_vf_resource(hw)) { 638 PMD_INIT_LOG(ERR, "Failed to get VF resource"); 639 goto err_alloc; 640 } 641 642 if (ice_dcf_get_vf_vsi_map(hw) < 0) { 643 PMD_INIT_LOG(ERR, "Failed to get VF VSI map"); 644 ice_dcf_mode_disable(hw); 645 goto err_alloc; 646 } 647 648 /* Allocate memory for RSS info */ 649 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 650 hw->rss_key = rte_zmalloc(NULL, 651 hw->vf_res->rss_key_size, 0); 652 if (!hw->rss_key) { 653 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 654 goto err_alloc; 655 } 656 hw->rss_lut = rte_zmalloc("rss_lut", 657 hw->vf_res->rss_lut_size, 0); 658 if (!hw->rss_lut) { 659 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 660 goto err_rss; 661 } 662 } 663 664 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 665 if (ice_dcf_get_supported_rxdid(hw) != 0) { 666 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 667 goto err_rss; 668 } 669 } 670 671 hw->eth_dev = eth_dev; 672 rte_intr_callback_register(&pci_dev->intr_handle, 673 ice_dcf_dev_interrupt_handler, hw); 674 rte_intr_enable(&pci_dev->intr_handle); 675 ice_dcf_enable_irq0(hw); 676 677 return 0; 678 679 err_rss: 680 rte_free(hw->rss_key); 681 rte_free(hw->rss_lut); 682 err_alloc: 683 rte_free(hw->vf_res); 684 err_api: 685 iavf_shutdown_adminq(&hw->avf); 686 err: 687 rte_free(hw->arq_buf); 688 689 return -1; 690 } 691 692 void 693 ice_dcf_uninit_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 694 { 695 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 696 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 697 698 ice_dcf_disable_irq0(hw); 699 rte_intr_disable(intr_handle); 700 rte_intr_callback_unregister(intr_handle, 701 ice_dcf_dev_interrupt_handler, hw); 702 703 ice_dcf_mode_disable(hw); 704 iavf_shutdown_adminq(&hw->avf); 705 706 rte_free(hw->arq_buf); 707 rte_free(hw->vf_vsi_map); 708 rte_free(hw->vf_res); 709 rte_free(hw->rss_lut); 710 rte_free(hw->rss_key); 711 } 712 713 static int 714 ice_dcf_configure_rss_key(struct ice_dcf_hw *hw) 715 { 716 struct virtchnl_rss_key *rss_key; 717 struct dcf_virtchnl_cmd args; 718 int len, err; 719 720 len = sizeof(*rss_key) + hw->vf_res->rss_key_size - 1; 721 rss_key = rte_zmalloc("rss_key", len, 0); 722 if (!rss_key) 723 return -ENOMEM; 724 725 rss_key->vsi_id = hw->vsi_res->vsi_id; 726 rss_key->key_len = hw->vf_res->rss_key_size; 727 rte_memcpy(rss_key->key, hw->rss_key, hw->vf_res->rss_key_size); 728 729 args.v_op = VIRTCHNL_OP_CONFIG_RSS_KEY; 730 args.req_msglen = len; 731 args.req_msg = (uint8_t *)rss_key; 732 args.rsp_msglen = 0; 733 args.rsp_buflen = 0; 734 args.rsp_msgbuf = NULL; 735 args.pending = 0; 736 737 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 738 if (err) 739 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_KEY"); 740 741 rte_free(rss_key); 742 return err; 743 } 744 745 static int 746 ice_dcf_configure_rss_lut(struct ice_dcf_hw *hw) 747 { 748 struct virtchnl_rss_lut *rss_lut; 749 struct dcf_virtchnl_cmd args; 750 int len, err; 751 752 len = sizeof(*rss_lut) + hw->vf_res->rss_lut_size - 1; 753 rss_lut = rte_zmalloc("rss_lut", len, 0); 754 if (!rss_lut) 755 return -ENOMEM; 756 757 rss_lut->vsi_id = hw->vsi_res->vsi_id; 758 rss_lut->lut_entries = hw->vf_res->rss_lut_size; 759 rte_memcpy(rss_lut->lut, hw->rss_lut, hw->vf_res->rss_lut_size); 760 761 args.v_op = VIRTCHNL_OP_CONFIG_RSS_LUT; 762 args.req_msglen = len; 763 args.req_msg = (uint8_t *)rss_lut; 764 args.rsp_msglen = 0; 765 args.rsp_buflen = 0; 766 args.rsp_msgbuf = NULL; 767 args.pending = 0; 768 769 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 770 if (err) 771 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_LUT"); 772 773 rte_free(rss_lut); 774 return err; 775 } 776 777 int 778 ice_dcf_init_rss(struct ice_dcf_hw *hw) 779 { 780 struct rte_eth_dev *dev = hw->eth_dev; 781 struct rte_eth_rss_conf *rss_conf; 782 uint8_t i, j, nb_q; 783 int ret; 784 785 rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf; 786 nb_q = dev->data->nb_rx_queues; 787 788 if (!(hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 789 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 790 return -ENOTSUP; 791 } 792 if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 793 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default"); 794 /* set all lut items to default queue */ 795 memset(hw->rss_lut, 0, hw->vf_res->rss_lut_size); 796 return ice_dcf_configure_rss_lut(hw); 797 } 798 799 /* In IAVF, RSS enablement is set by PF driver. It is not supported 800 * to set based on rss_conf->rss_hf. 801 */ 802 803 /* configure RSS key */ 804 if (!rss_conf->rss_key) 805 /* Calculate the default hash key */ 806 for (i = 0; i < hw->vf_res->rss_key_size; i++) 807 hw->rss_key[i] = (uint8_t)rte_rand(); 808 else 809 rte_memcpy(hw->rss_key, rss_conf->rss_key, 810 RTE_MIN(rss_conf->rss_key_len, 811 hw->vf_res->rss_key_size)); 812 813 /* init RSS LUT table */ 814 for (i = 0, j = 0; i < hw->vf_res->rss_lut_size; i++, j++) { 815 if (j >= nb_q) 816 j = 0; 817 hw->rss_lut[i] = j; 818 } 819 /* send virtchnnl ops to configure rss*/ 820 ret = ice_dcf_configure_rss_lut(hw); 821 if (ret) 822 return ret; 823 ret = ice_dcf_configure_rss_key(hw); 824 if (ret) 825 return ret; 826 827 return 0; 828 } 829 830 #define IAVF_RXDID_LEGACY_0 0 831 #define IAVF_RXDID_LEGACY_1 1 832 #define IAVF_RXDID_COMMS_GENERIC 16 833 834 int 835 ice_dcf_configure_queues(struct ice_dcf_hw *hw) 836 { 837 struct ice_rx_queue **rxq = 838 (struct ice_rx_queue **)hw->eth_dev->data->rx_queues; 839 struct ice_tx_queue **txq = 840 (struct ice_tx_queue **)hw->eth_dev->data->tx_queues; 841 struct virtchnl_vsi_queue_config_info *vc_config; 842 struct virtchnl_queue_pair_info *vc_qp; 843 struct dcf_virtchnl_cmd args; 844 uint16_t i, size; 845 int err; 846 847 size = sizeof(*vc_config) + 848 sizeof(vc_config->qpair[0]) * hw->num_queue_pairs; 849 vc_config = rte_zmalloc("cfg_queue", size, 0); 850 if (!vc_config) 851 return -ENOMEM; 852 853 vc_config->vsi_id = hw->vsi_res->vsi_id; 854 vc_config->num_queue_pairs = hw->num_queue_pairs; 855 856 for (i = 0, vc_qp = vc_config->qpair; 857 i < hw->num_queue_pairs; 858 i++, vc_qp++) { 859 vc_qp->txq.vsi_id = hw->vsi_res->vsi_id; 860 vc_qp->txq.queue_id = i; 861 if (i < hw->eth_dev->data->nb_tx_queues) { 862 vc_qp->txq.ring_len = txq[i]->nb_tx_desc; 863 vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_dma; 864 } 865 vc_qp->rxq.vsi_id = hw->vsi_res->vsi_id; 866 vc_qp->rxq.queue_id = i; 867 vc_qp->rxq.max_pkt_size = rxq[i]->max_pkt_len; 868 869 if (i >= hw->eth_dev->data->nb_rx_queues) 870 continue; 871 872 vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc; 873 vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_dma; 874 vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len; 875 876 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 877 if (hw->vf_res->vf_cap_flags & 878 VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && 879 hw->supported_rxdid & 880 BIT(IAVF_RXDID_COMMS_GENERIC)) { 881 vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_GENERIC; 882 PMD_DRV_LOG(NOTICE, "request RXDID == %d in " 883 "Queue[%d]", vc_qp->rxq.rxdid, i); 884 } else { 885 PMD_DRV_LOG(ERR, "RXDID 16 is not supported"); 886 return -EINVAL; 887 } 888 #else 889 if (hw->vf_res->vf_cap_flags & 890 VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && 891 hw->supported_rxdid & 892 BIT(IAVF_RXDID_LEGACY_0)) { 893 vc_qp->rxq.rxdid = IAVF_RXDID_LEGACY_0; 894 PMD_DRV_LOG(NOTICE, "request RXDID == %d in " 895 "Queue[%d]", vc_qp->rxq.rxdid, i); 896 } else { 897 PMD_DRV_LOG(ERR, "RXDID == 0 is not supported"); 898 return -EINVAL; 899 } 900 #endif 901 ice_select_rxd_to_pkt_fields_handler(rxq[i], vc_qp->rxq.rxdid); 902 } 903 904 memset(&args, 0, sizeof(args)); 905 args.v_op = VIRTCHNL_OP_CONFIG_VSI_QUEUES; 906 args.req_msg = (uint8_t *)vc_config; 907 args.req_msglen = size; 908 909 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 910 if (err) 911 PMD_DRV_LOG(ERR, "Failed to execute command of" 912 " VIRTCHNL_OP_CONFIG_VSI_QUEUES"); 913 914 rte_free(vc_config); 915 return err; 916 } 917 918 int 919 ice_dcf_config_irq_map(struct ice_dcf_hw *hw) 920 { 921 struct virtchnl_irq_map_info *map_info; 922 struct virtchnl_vector_map *vecmap; 923 struct dcf_virtchnl_cmd args; 924 int len, i, err; 925 926 len = sizeof(struct virtchnl_irq_map_info) + 927 sizeof(struct virtchnl_vector_map) * hw->nb_msix; 928 929 map_info = rte_zmalloc("map_info", len, 0); 930 if (!map_info) 931 return -ENOMEM; 932 933 map_info->num_vectors = hw->nb_msix; 934 for (i = 0; i < hw->nb_msix; i++) { 935 vecmap = &map_info->vecmap[i]; 936 vecmap->vsi_id = hw->vsi_res->vsi_id; 937 vecmap->rxitr_idx = 0; 938 vecmap->vector_id = hw->msix_base + i; 939 vecmap->txq_map = 0; 940 vecmap->rxq_map = hw->rxq_map[hw->msix_base + i]; 941 } 942 943 memset(&args, 0, sizeof(args)); 944 args.v_op = VIRTCHNL_OP_CONFIG_IRQ_MAP; 945 args.req_msg = (u8 *)map_info; 946 args.req_msglen = len; 947 948 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 949 if (err) 950 PMD_DRV_LOG(ERR, "fail to execute command OP_CONFIG_IRQ_MAP"); 951 952 rte_free(map_info); 953 return err; 954 } 955 956 int 957 ice_dcf_switch_queue(struct ice_dcf_hw *hw, uint16_t qid, bool rx, bool on) 958 { 959 struct virtchnl_queue_select queue_select; 960 struct dcf_virtchnl_cmd args; 961 int err; 962 963 memset(&queue_select, 0, sizeof(queue_select)); 964 queue_select.vsi_id = hw->vsi_res->vsi_id; 965 if (rx) 966 queue_select.rx_queues |= 1 << qid; 967 else 968 queue_select.tx_queues |= 1 << qid; 969 970 memset(&args, 0, sizeof(args)); 971 if (on) 972 args.v_op = VIRTCHNL_OP_ENABLE_QUEUES; 973 else 974 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 975 976 args.req_msg = (u8 *)&queue_select; 977 args.req_msglen = sizeof(queue_select); 978 979 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 980 if (err) 981 PMD_DRV_LOG(ERR, "Failed to execute command of %s", 982 on ? "OP_ENABLE_QUEUES" : "OP_DISABLE_QUEUES"); 983 984 return err; 985 } 986 987 int 988 ice_dcf_disable_queues(struct ice_dcf_hw *hw) 989 { 990 struct virtchnl_queue_select queue_select; 991 struct dcf_virtchnl_cmd args; 992 int err; 993 994 memset(&queue_select, 0, sizeof(queue_select)); 995 queue_select.vsi_id = hw->vsi_res->vsi_id; 996 997 queue_select.rx_queues = BIT(hw->eth_dev->data->nb_rx_queues) - 1; 998 queue_select.tx_queues = BIT(hw->eth_dev->data->nb_tx_queues) - 1; 999 1000 memset(&args, 0, sizeof(args)); 1001 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 1002 args.req_msg = (u8 *)&queue_select; 1003 args.req_msglen = sizeof(queue_select); 1004 1005 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1006 if (err) 1007 PMD_DRV_LOG(ERR, 1008 "Failed to execute command of OP_DISABLE_QUEUES"); 1009 1010 return err; 1011 } 1012 1013 int 1014 ice_dcf_query_stats(struct ice_dcf_hw *hw, 1015 struct virtchnl_eth_stats *pstats) 1016 { 1017 struct virtchnl_queue_select q_stats; 1018 struct dcf_virtchnl_cmd args; 1019 int err; 1020 1021 memset(&q_stats, 0, sizeof(q_stats)); 1022 q_stats.vsi_id = hw->vsi_res->vsi_id; 1023 1024 args.v_op = VIRTCHNL_OP_GET_STATS; 1025 args.req_msg = (uint8_t *)&q_stats; 1026 args.req_msglen = sizeof(q_stats); 1027 args.rsp_msglen = sizeof(*pstats); 1028 args.rsp_msgbuf = (uint8_t *)pstats; 1029 args.rsp_buflen = sizeof(*pstats); 1030 1031 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1032 if (err) { 1033 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS"); 1034 return err; 1035 } 1036 1037 return 0; 1038 } 1039 1040 int 1041 ice_dcf_add_del_all_mac_addr(struct ice_dcf_hw *hw, bool add) 1042 { 1043 struct virtchnl_ether_addr_list *list; 1044 struct rte_ether_addr *addr; 1045 struct dcf_virtchnl_cmd args; 1046 int len, err = 0; 1047 1048 len = sizeof(struct virtchnl_ether_addr_list); 1049 addr = hw->eth_dev->data->mac_addrs; 1050 len += sizeof(struct virtchnl_ether_addr); 1051 1052 list = rte_zmalloc(NULL, len, 0); 1053 if (!list) { 1054 PMD_DRV_LOG(ERR, "fail to allocate memory"); 1055 return -ENOMEM; 1056 } 1057 1058 rte_memcpy(list->list[0].addr, addr->addr_bytes, 1059 sizeof(addr->addr_bytes)); 1060 PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x", 1061 addr->addr_bytes[0], addr->addr_bytes[1], 1062 addr->addr_bytes[2], addr->addr_bytes[3], 1063 addr->addr_bytes[4], addr->addr_bytes[5]); 1064 1065 list->vsi_id = hw->vsi_res->vsi_id; 1066 list->num_elements = 1; 1067 1068 memset(&args, 0, sizeof(args)); 1069 args.v_op = add ? VIRTCHNL_OP_ADD_ETH_ADDR : 1070 VIRTCHNL_OP_DEL_ETH_ADDR; 1071 args.req_msg = (uint8_t *)list; 1072 args.req_msglen = len; 1073 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1074 if (err) 1075 PMD_DRV_LOG(ERR, "fail to execute command %s", 1076 add ? "OP_ADD_ETHER_ADDRESS" : 1077 "OP_DEL_ETHER_ADDRESS"); 1078 rte_free(list); 1079 return err; 1080 } 1081