1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_pci.h> 17 #include <rte_atomic.h> 18 #include <rte_eal.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <ethdev_pci.h> 22 #include <rte_malloc.h> 23 #include <rte_memzone.h> 24 #include <rte_dev.h> 25 26 #include "ice_dcf.h" 27 #include "ice_rxtx.h" 28 29 #define ICE_DCF_AQ_LEN 32 30 #define ICE_DCF_AQ_BUF_SZ 4096 31 32 #define ICE_DCF_ARQ_MAX_RETRIES 200 33 #define ICE_DCF_ARQ_CHECK_TIME 2 /* msecs */ 34 35 #define ICE_DCF_VF_RES_BUF_SZ \ 36 (sizeof(struct virtchnl_vf_resource) + \ 37 IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)) 38 39 static __rte_always_inline int 40 ice_dcf_send_cmd_req_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 41 uint8_t *req_msg, uint16_t req_msglen) 42 { 43 return iavf_aq_send_msg_to_pf(&hw->avf, op, IAVF_SUCCESS, 44 req_msg, req_msglen, NULL); 45 } 46 47 static int 48 ice_dcf_recv_cmd_rsp_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 49 uint8_t *rsp_msgbuf, uint16_t rsp_buflen, 50 uint16_t *rsp_msglen) 51 { 52 struct iavf_arq_event_info event; 53 enum virtchnl_ops v_op; 54 int i = 0; 55 int err; 56 57 event.buf_len = rsp_buflen; 58 event.msg_buf = rsp_msgbuf; 59 60 do { 61 err = iavf_clean_arq_element(&hw->avf, &event, NULL); 62 if (err != IAVF_SUCCESS) 63 goto again; 64 65 v_op = rte_le_to_cpu_32(event.desc.cookie_high); 66 if (v_op != op) 67 goto again; 68 69 if (rsp_msglen != NULL) 70 *rsp_msglen = event.msg_len; 71 return rte_le_to_cpu_32(event.desc.cookie_low); 72 73 again: 74 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 75 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 76 77 return -EIO; 78 } 79 80 static __rte_always_inline void 81 ice_dcf_aq_cmd_clear(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 82 { 83 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 84 85 TAILQ_REMOVE(&hw->vc_cmd_queue, cmd, next); 86 87 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 88 } 89 90 static __rte_always_inline void 91 ice_dcf_vc_cmd_set(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 92 { 93 cmd->v_ret = IAVF_ERR_NOT_READY; 94 cmd->rsp_msglen = 0; 95 cmd->pending = 1; 96 97 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 98 99 TAILQ_INSERT_TAIL(&hw->vc_cmd_queue, cmd, next); 100 101 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 102 } 103 104 static __rte_always_inline int 105 ice_dcf_vc_cmd_send(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 106 { 107 return iavf_aq_send_msg_to_pf(&hw->avf, 108 cmd->v_op, IAVF_SUCCESS, 109 cmd->req_msg, cmd->req_msglen, NULL); 110 } 111 112 static __rte_always_inline void 113 ice_dcf_aq_cmd_handle(struct ice_dcf_hw *hw, struct iavf_arq_event_info *info) 114 { 115 struct dcf_virtchnl_cmd *cmd; 116 enum virtchnl_ops v_op; 117 enum iavf_status v_ret; 118 uint16_t aq_op; 119 120 aq_op = rte_le_to_cpu_16(info->desc.opcode); 121 if (unlikely(aq_op != iavf_aqc_opc_send_msg_to_vf)) { 122 PMD_DRV_LOG(ERR, 123 "Request %u is not supported yet", aq_op); 124 return; 125 } 126 127 v_op = rte_le_to_cpu_32(info->desc.cookie_high); 128 if (v_op == VIRTCHNL_OP_EVENT) { 129 if (hw->vc_event_msg_cb != NULL) 130 hw->vc_event_msg_cb(hw, 131 info->msg_buf, 132 info->msg_len); 133 return; 134 } 135 136 v_ret = rte_le_to_cpu_32(info->desc.cookie_low); 137 138 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 139 140 TAILQ_FOREACH(cmd, &hw->vc_cmd_queue, next) { 141 if (cmd->v_op == v_op && cmd->pending) { 142 cmd->v_ret = v_ret; 143 cmd->rsp_msglen = RTE_MIN(info->msg_len, 144 cmd->rsp_buflen); 145 if (likely(cmd->rsp_msglen != 0)) 146 rte_memcpy(cmd->rsp_msgbuf, info->msg_buf, 147 cmd->rsp_msglen); 148 149 /* prevent compiler reordering */ 150 rte_compiler_barrier(); 151 cmd->pending = 0; 152 break; 153 } 154 } 155 156 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 157 } 158 159 static void 160 ice_dcf_handle_virtchnl_msg(struct ice_dcf_hw *hw) 161 { 162 struct iavf_arq_event_info info; 163 uint16_t pending = 1; 164 int ret; 165 166 info.buf_len = ICE_DCF_AQ_BUF_SZ; 167 info.msg_buf = hw->arq_buf; 168 169 while (pending) { 170 ret = iavf_clean_arq_element(&hw->avf, &info, &pending); 171 if (ret != IAVF_SUCCESS) 172 break; 173 174 ice_dcf_aq_cmd_handle(hw, &info); 175 } 176 } 177 178 static int 179 ice_dcf_init_check_api_version(struct ice_dcf_hw *hw) 180 { 181 #define ICE_CPF_VIRTCHNL_VERSION_MAJOR_START 1 182 #define ICE_CPF_VIRTCHNL_VERSION_MINOR_START 1 183 struct virtchnl_version_info version, *pver; 184 int err; 185 186 version.major = VIRTCHNL_VERSION_MAJOR; 187 version.minor = VIRTCHNL_VERSION_MINOR; 188 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_VERSION, 189 (uint8_t *)&version, sizeof(version)); 190 if (err) { 191 PMD_INIT_LOG(ERR, "Failed to send OP_VERSION"); 192 return err; 193 } 194 195 pver = &hw->virtchnl_version; 196 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_VERSION, 197 (uint8_t *)pver, sizeof(*pver), NULL); 198 if (err) { 199 PMD_INIT_LOG(ERR, "Failed to get response of OP_VERSION"); 200 return -1; 201 } 202 203 PMD_INIT_LOG(DEBUG, 204 "Peer PF API version: %u.%u", pver->major, pver->minor); 205 206 if (pver->major < ICE_CPF_VIRTCHNL_VERSION_MAJOR_START || 207 (pver->major == ICE_CPF_VIRTCHNL_VERSION_MAJOR_START && 208 pver->minor < ICE_CPF_VIRTCHNL_VERSION_MINOR_START)) { 209 PMD_INIT_LOG(ERR, 210 "VIRTCHNL API version should not be lower than (%u.%u)", 211 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START, 212 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START); 213 return -1; 214 } else if (pver->major > VIRTCHNL_VERSION_MAJOR || 215 (pver->major == VIRTCHNL_VERSION_MAJOR && 216 pver->minor > VIRTCHNL_VERSION_MINOR)) { 217 PMD_INIT_LOG(ERR, 218 "PF/VF API version mismatch:(%u.%u)-(%u.%u)", 219 pver->major, pver->minor, 220 VIRTCHNL_VERSION_MAJOR, VIRTCHNL_VERSION_MINOR); 221 return -1; 222 } 223 224 PMD_INIT_LOG(DEBUG, "Peer is supported PF host"); 225 226 return 0; 227 } 228 229 static int 230 ice_dcf_get_vf_resource(struct ice_dcf_hw *hw) 231 { 232 uint32_t caps; 233 int err, i; 234 235 caps = VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RX_POLLING | 236 VIRTCHNL_VF_CAP_ADV_LINK_SPEED | VIRTCHNL_VF_CAP_DCF | 237 VIRTCHNL_VF_OFFLOAD_VLAN_V2 | 238 VF_BASE_MODE_OFFLOADS | VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC | 239 VIRTCHNL_VF_OFFLOAD_QOS; 240 241 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 242 (uint8_t *)&caps, sizeof(caps)); 243 if (err) { 244 PMD_DRV_LOG(ERR, "Failed to send msg OP_GET_VF_RESOURCE"); 245 return err; 246 } 247 248 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 249 (uint8_t *)hw->vf_res, 250 ICE_DCF_VF_RES_BUF_SZ, NULL); 251 if (err) { 252 PMD_DRV_LOG(ERR, "Failed to get response of OP_GET_VF_RESOURCE"); 253 return -1; 254 } 255 256 iavf_vf_parse_hw_config(&hw->avf, hw->vf_res); 257 258 hw->vsi_res = NULL; 259 for (i = 0; i < hw->vf_res->num_vsis; i++) { 260 if (hw->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 261 hw->vsi_res = &hw->vf_res->vsi_res[i]; 262 } 263 264 if (!hw->vsi_res) { 265 PMD_DRV_LOG(ERR, "no LAN VSI found"); 266 return -1; 267 } 268 269 hw->vsi_id = hw->vsi_res->vsi_id; 270 PMD_DRV_LOG(DEBUG, "VSI ID is %u", hw->vsi_id); 271 272 return 0; 273 } 274 275 static int 276 ice_dcf_get_vf_vsi_map(struct ice_dcf_hw *hw) 277 { 278 struct virtchnl_dcf_vsi_map *vsi_map; 279 uint32_t valid_msg_len; 280 uint16_t len; 281 int err; 282 283 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 284 NULL, 0); 285 if (err) { 286 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_GET_VSI_MAP"); 287 return err; 288 } 289 290 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 291 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, 292 &len); 293 if (err) { 294 PMD_DRV_LOG(ERR, "Failed to get response of OP_DCF_GET_VSI_MAP"); 295 return err; 296 } 297 298 vsi_map = (struct virtchnl_dcf_vsi_map *)hw->arq_buf; 299 valid_msg_len = (vsi_map->num_vfs - 1) * sizeof(vsi_map->vf_vsi[0]) + 300 sizeof(*vsi_map); 301 if (len != valid_msg_len) { 302 PMD_DRV_LOG(ERR, "invalid vf vsi map response with length %u", 303 len); 304 return -EINVAL; 305 } 306 307 if (hw->num_vfs != 0 && hw->num_vfs != vsi_map->num_vfs) { 308 PMD_DRV_LOG(ERR, "The number VSI map (%u) doesn't match the number of VFs (%u)", 309 vsi_map->num_vfs, hw->num_vfs); 310 return -EINVAL; 311 } 312 313 len = vsi_map->num_vfs * sizeof(vsi_map->vf_vsi[0]); 314 315 if (!hw->vf_vsi_map) { 316 hw->vf_vsi_map = rte_zmalloc("vf_vsi_ctx", len, 0); 317 if (!hw->vf_vsi_map) { 318 PMD_DRV_LOG(ERR, "Failed to alloc memory for VSI context"); 319 return -ENOMEM; 320 } 321 322 hw->num_vfs = vsi_map->num_vfs; 323 hw->pf_vsi_id = vsi_map->pf_vsi; 324 } 325 326 if (!memcmp(hw->vf_vsi_map, vsi_map->vf_vsi, len)) { 327 PMD_DRV_LOG(DEBUG, "VF VSI map doesn't change"); 328 return 1; 329 } 330 331 rte_memcpy(hw->vf_vsi_map, vsi_map->vf_vsi, len); 332 return 0; 333 } 334 335 static int 336 ice_dcf_mode_disable(struct ice_dcf_hw *hw) 337 { 338 int err; 339 340 if (hw->resetting) 341 return 0; 342 343 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 344 NULL, 0); 345 if (err) { 346 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_DISABLE"); 347 return err; 348 } 349 350 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 351 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, NULL); 352 if (err) { 353 PMD_DRV_LOG(ERR, 354 "Failed to get response of OP_DCF_DISABLE %d", 355 err); 356 return -1; 357 } 358 359 return 0; 360 } 361 362 static int 363 ice_dcf_check_reset_done(struct ice_dcf_hw *hw) 364 { 365 #define ICE_DCF_RESET_WAIT_CNT 50 366 struct iavf_hw *avf = &hw->avf; 367 int i, reset; 368 369 for (i = 0; i < ICE_DCF_RESET_WAIT_CNT; i++) { 370 reset = IAVF_READ_REG(avf, IAVF_VFGEN_RSTAT) & 371 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 372 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 373 374 if (reset == VIRTCHNL_VFR_VFACTIVE || 375 reset == VIRTCHNL_VFR_COMPLETED) 376 break; 377 378 rte_delay_ms(20); 379 } 380 381 if (i >= ICE_DCF_RESET_WAIT_CNT) 382 return -1; 383 384 return 0; 385 } 386 387 static inline void 388 ice_dcf_enable_irq0(struct ice_dcf_hw *hw) 389 { 390 struct iavf_hw *avf = &hw->avf; 391 392 /* Enable admin queue interrupt trigger */ 393 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 394 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 395 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 396 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 397 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 398 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 399 400 IAVF_WRITE_FLUSH(avf); 401 } 402 403 static inline void 404 ice_dcf_disable_irq0(struct ice_dcf_hw *hw) 405 { 406 struct iavf_hw *avf = &hw->avf; 407 408 /* Disable all interrupt types */ 409 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 0); 410 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 411 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 412 413 IAVF_WRITE_FLUSH(avf); 414 } 415 416 static void 417 ice_dcf_dev_interrupt_handler(void *param) 418 { 419 struct ice_dcf_hw *hw = param; 420 421 ice_dcf_disable_irq0(hw); 422 423 ice_dcf_handle_virtchnl_msg(hw); 424 425 ice_dcf_enable_irq0(hw); 426 } 427 428 int 429 ice_dcf_execute_virtchnl_cmd(struct ice_dcf_hw *hw, 430 struct dcf_virtchnl_cmd *cmd) 431 { 432 int i = 0; 433 int err; 434 435 if ((cmd->req_msg && !cmd->req_msglen) || 436 (!cmd->req_msg && cmd->req_msglen) || 437 (cmd->rsp_msgbuf && !cmd->rsp_buflen) || 438 (!cmd->rsp_msgbuf && cmd->rsp_buflen)) 439 return -EINVAL; 440 441 rte_spinlock_lock(&hw->vc_cmd_send_lock); 442 ice_dcf_vc_cmd_set(hw, cmd); 443 444 err = ice_dcf_vc_cmd_send(hw, cmd); 445 if (err) { 446 PMD_DRV_LOG(ERR, "fail to send cmd %d", cmd->v_op); 447 goto ret; 448 } 449 450 do { 451 if (!cmd->pending) 452 break; 453 454 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 455 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 456 457 if (cmd->v_ret != IAVF_SUCCESS) { 458 err = -1; 459 PMD_DRV_LOG(ERR, 460 "No response (%d times) or return failure (%d) for cmd %d", 461 i, cmd->v_ret, cmd->v_op); 462 } 463 464 ret: 465 ice_dcf_aq_cmd_clear(hw, cmd); 466 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 467 return err; 468 } 469 470 int 471 ice_dcf_send_aq_cmd(void *dcf_hw, struct ice_aq_desc *desc, 472 void *buf, uint16_t buf_size) 473 { 474 struct dcf_virtchnl_cmd desc_cmd, buff_cmd; 475 struct ice_dcf_hw *hw = dcf_hw; 476 int err = 0; 477 int i = 0; 478 479 if ((buf && !buf_size) || (!buf && buf_size) || 480 buf_size > ICE_DCF_AQ_BUF_SZ) 481 return -EINVAL; 482 483 desc_cmd.v_op = VIRTCHNL_OP_DCF_CMD_DESC; 484 desc_cmd.req_msglen = sizeof(*desc); 485 desc_cmd.req_msg = (uint8_t *)desc; 486 desc_cmd.rsp_buflen = sizeof(*desc); 487 desc_cmd.rsp_msgbuf = (uint8_t *)desc; 488 489 if (buf == NULL) 490 return ice_dcf_execute_virtchnl_cmd(hw, &desc_cmd); 491 492 desc->flags |= rte_cpu_to_le_16(ICE_AQ_FLAG_BUF); 493 494 buff_cmd.v_op = VIRTCHNL_OP_DCF_CMD_BUFF; 495 buff_cmd.req_msglen = buf_size; 496 buff_cmd.req_msg = buf; 497 buff_cmd.rsp_buflen = buf_size; 498 buff_cmd.rsp_msgbuf = buf; 499 500 rte_spinlock_lock(&hw->vc_cmd_send_lock); 501 ice_dcf_vc_cmd_set(hw, &desc_cmd); 502 ice_dcf_vc_cmd_set(hw, &buff_cmd); 503 504 if (ice_dcf_vc_cmd_send(hw, &desc_cmd) || 505 ice_dcf_vc_cmd_send(hw, &buff_cmd)) { 506 err = -1; 507 PMD_DRV_LOG(ERR, "fail to send OP_DCF_CMD_DESC/BUFF"); 508 goto ret; 509 } 510 511 do { 512 if (!desc_cmd.pending && !buff_cmd.pending) 513 break; 514 515 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 516 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 517 518 if (desc_cmd.v_ret != IAVF_SUCCESS || buff_cmd.v_ret != IAVF_SUCCESS) { 519 err = -1; 520 PMD_DRV_LOG(ERR, 521 "No response (%d times) or return failure (desc: %d / buff: %d)", 522 i, desc_cmd.v_ret, buff_cmd.v_ret); 523 } 524 525 ret: 526 ice_dcf_aq_cmd_clear(hw, &desc_cmd); 527 ice_dcf_aq_cmd_clear(hw, &buff_cmd); 528 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 529 530 return err; 531 } 532 533 int 534 ice_dcf_handle_vsi_update_event(struct ice_dcf_hw *hw) 535 { 536 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(hw->eth_dev); 537 int err = 0; 538 539 rte_spinlock_lock(&hw->vc_cmd_send_lock); 540 541 rte_intr_disable(&pci_dev->intr_handle); 542 ice_dcf_disable_irq0(hw); 543 544 if (ice_dcf_get_vf_resource(hw) || ice_dcf_get_vf_vsi_map(hw) < 0) 545 err = -1; 546 547 rte_intr_enable(&pci_dev->intr_handle); 548 ice_dcf_enable_irq0(hw); 549 550 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 551 552 return err; 553 } 554 555 static int 556 ice_dcf_get_supported_rxdid(struct ice_dcf_hw *hw) 557 { 558 int err; 559 560 err = ice_dcf_send_cmd_req_no_irq(hw, 561 VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 562 NULL, 0); 563 if (err) { 564 PMD_INIT_LOG(ERR, "Failed to send OP_GET_SUPPORTED_RXDIDS"); 565 return -1; 566 } 567 568 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 569 (uint8_t *)&hw->supported_rxdid, 570 sizeof(uint64_t), NULL); 571 if (err) { 572 PMD_INIT_LOG(ERR, "Failed to get response of OP_GET_SUPPORTED_RXDIDS"); 573 return -1; 574 } 575 576 return 0; 577 } 578 579 int 580 ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 581 { 582 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 583 int ret, size; 584 585 hw->avf.hw_addr = pci_dev->mem_resource[0].addr; 586 hw->avf.back = hw; 587 588 hw->avf.bus.bus_id = pci_dev->addr.bus; 589 hw->avf.bus.device = pci_dev->addr.devid; 590 hw->avf.bus.func = pci_dev->addr.function; 591 592 hw->avf.device_id = pci_dev->id.device_id; 593 hw->avf.vendor_id = pci_dev->id.vendor_id; 594 hw->avf.subsystem_device_id = pci_dev->id.subsystem_device_id; 595 hw->avf.subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 596 597 hw->avf.aq.num_arq_entries = ICE_DCF_AQ_LEN; 598 hw->avf.aq.num_asq_entries = ICE_DCF_AQ_LEN; 599 hw->avf.aq.arq_buf_size = ICE_DCF_AQ_BUF_SZ; 600 hw->avf.aq.asq_buf_size = ICE_DCF_AQ_BUF_SZ; 601 602 rte_spinlock_init(&hw->vc_cmd_send_lock); 603 rte_spinlock_init(&hw->vc_cmd_queue_lock); 604 TAILQ_INIT(&hw->vc_cmd_queue); 605 606 hw->arq_buf = rte_zmalloc("arq_buf", ICE_DCF_AQ_BUF_SZ, 0); 607 if (hw->arq_buf == NULL) { 608 PMD_INIT_LOG(ERR, "unable to allocate AdminQ buffer memory"); 609 goto err; 610 } 611 612 ret = iavf_set_mac_type(&hw->avf); 613 if (ret) { 614 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", ret); 615 goto err; 616 } 617 618 ret = ice_dcf_check_reset_done(hw); 619 if (ret) { 620 PMD_INIT_LOG(ERR, "VF is still resetting"); 621 goto err; 622 } 623 624 ret = iavf_init_adminq(&hw->avf); 625 if (ret) { 626 PMD_INIT_LOG(ERR, "init_adminq failed: %d", ret); 627 goto err; 628 } 629 630 if (ice_dcf_init_check_api_version(hw)) { 631 PMD_INIT_LOG(ERR, "check_api version failed"); 632 goto err_api; 633 } 634 635 hw->vf_res = rte_zmalloc("vf_res", ICE_DCF_VF_RES_BUF_SZ, 0); 636 if (hw->vf_res == NULL) { 637 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 638 goto err_api; 639 } 640 641 if (ice_dcf_get_vf_resource(hw)) { 642 PMD_INIT_LOG(ERR, "Failed to get VF resource"); 643 goto err_alloc; 644 } 645 646 if (ice_dcf_get_vf_vsi_map(hw) < 0) { 647 PMD_INIT_LOG(ERR, "Failed to get VF VSI map"); 648 ice_dcf_mode_disable(hw); 649 goto err_alloc; 650 } 651 652 /* Allocate memory for RSS info */ 653 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 654 hw->rss_key = rte_zmalloc(NULL, 655 hw->vf_res->rss_key_size, 0); 656 if (!hw->rss_key) { 657 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 658 goto err_alloc; 659 } 660 hw->rss_lut = rte_zmalloc("rss_lut", 661 hw->vf_res->rss_lut_size, 0); 662 if (!hw->rss_lut) { 663 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 664 goto err_rss; 665 } 666 } 667 668 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 669 if (ice_dcf_get_supported_rxdid(hw) != 0) { 670 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 671 goto err_rss; 672 } 673 } 674 675 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) { 676 ice_dcf_tm_conf_init(eth_dev); 677 size = sizeof(struct virtchnl_dcf_bw_cfg_list *) * hw->num_vfs; 678 hw->qos_bw_cfg = rte_zmalloc("qos_bw_cfg", size, 0); 679 if (!hw->qos_bw_cfg) { 680 PMD_INIT_LOG(ERR, "no memory for qos_bw_cfg"); 681 goto err_rss; 682 } 683 } 684 685 hw->eth_dev = eth_dev; 686 rte_intr_callback_register(&pci_dev->intr_handle, 687 ice_dcf_dev_interrupt_handler, hw); 688 rte_intr_enable(&pci_dev->intr_handle); 689 ice_dcf_enable_irq0(hw); 690 691 return 0; 692 693 err_rss: 694 rte_free(hw->rss_key); 695 rte_free(hw->rss_lut); 696 err_alloc: 697 rte_free(hw->vf_res); 698 err_api: 699 iavf_shutdown_adminq(&hw->avf); 700 err: 701 rte_free(hw->arq_buf); 702 703 return -1; 704 } 705 706 void 707 ice_dcf_uninit_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 708 { 709 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 710 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 711 712 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 713 if (hw->tm_conf.committed) { 714 ice_dcf_clear_bw(hw); 715 ice_dcf_tm_conf_uninit(eth_dev); 716 } 717 718 ice_dcf_disable_irq0(hw); 719 rte_intr_disable(intr_handle); 720 rte_intr_callback_unregister(intr_handle, 721 ice_dcf_dev_interrupt_handler, hw); 722 723 ice_dcf_mode_disable(hw); 724 iavf_shutdown_adminq(&hw->avf); 725 726 rte_free(hw->arq_buf); 727 hw->arq_buf = NULL; 728 729 rte_free(hw->vf_vsi_map); 730 hw->vf_vsi_map = NULL; 731 732 rte_free(hw->vf_res); 733 hw->vf_res = NULL; 734 735 rte_free(hw->rss_lut); 736 hw->rss_lut = NULL; 737 738 rte_free(hw->rss_key); 739 hw->rss_key = NULL; 740 741 rte_free(hw->qos_bw_cfg); 742 hw->qos_bw_cfg = NULL; 743 744 rte_free(hw->ets_config); 745 hw->ets_config = NULL; 746 } 747 748 static int 749 ice_dcf_configure_rss_key(struct ice_dcf_hw *hw) 750 { 751 struct virtchnl_rss_key *rss_key; 752 struct dcf_virtchnl_cmd args; 753 int len, err; 754 755 len = sizeof(*rss_key) + hw->vf_res->rss_key_size - 1; 756 rss_key = rte_zmalloc("rss_key", len, 0); 757 if (!rss_key) 758 return -ENOMEM; 759 760 rss_key->vsi_id = hw->vsi_res->vsi_id; 761 rss_key->key_len = hw->vf_res->rss_key_size; 762 rte_memcpy(rss_key->key, hw->rss_key, hw->vf_res->rss_key_size); 763 764 args.v_op = VIRTCHNL_OP_CONFIG_RSS_KEY; 765 args.req_msglen = len; 766 args.req_msg = (uint8_t *)rss_key; 767 args.rsp_msglen = 0; 768 args.rsp_buflen = 0; 769 args.rsp_msgbuf = NULL; 770 args.pending = 0; 771 772 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 773 if (err) 774 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_KEY"); 775 776 rte_free(rss_key); 777 return err; 778 } 779 780 static int 781 ice_dcf_configure_rss_lut(struct ice_dcf_hw *hw) 782 { 783 struct virtchnl_rss_lut *rss_lut; 784 struct dcf_virtchnl_cmd args; 785 int len, err; 786 787 len = sizeof(*rss_lut) + hw->vf_res->rss_lut_size - 1; 788 rss_lut = rte_zmalloc("rss_lut", len, 0); 789 if (!rss_lut) 790 return -ENOMEM; 791 792 rss_lut->vsi_id = hw->vsi_res->vsi_id; 793 rss_lut->lut_entries = hw->vf_res->rss_lut_size; 794 rte_memcpy(rss_lut->lut, hw->rss_lut, hw->vf_res->rss_lut_size); 795 796 args.v_op = VIRTCHNL_OP_CONFIG_RSS_LUT; 797 args.req_msglen = len; 798 args.req_msg = (uint8_t *)rss_lut; 799 args.rsp_msglen = 0; 800 args.rsp_buflen = 0; 801 args.rsp_msgbuf = NULL; 802 args.pending = 0; 803 804 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 805 if (err) 806 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_LUT"); 807 808 rte_free(rss_lut); 809 return err; 810 } 811 812 int 813 ice_dcf_init_rss(struct ice_dcf_hw *hw) 814 { 815 struct rte_eth_dev *dev = hw->eth_dev; 816 struct rte_eth_rss_conf *rss_conf; 817 uint8_t i, j, nb_q; 818 int ret; 819 820 rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf; 821 nb_q = dev->data->nb_rx_queues; 822 823 if (!(hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 824 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 825 return -ENOTSUP; 826 } 827 if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 828 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default"); 829 /* set all lut items to default queue */ 830 memset(hw->rss_lut, 0, hw->vf_res->rss_lut_size); 831 return ice_dcf_configure_rss_lut(hw); 832 } 833 834 /* In IAVF, RSS enablement is set by PF driver. It is not supported 835 * to set based on rss_conf->rss_hf. 836 */ 837 838 /* configure RSS key */ 839 if (!rss_conf->rss_key) 840 /* Calculate the default hash key */ 841 for (i = 0; i < hw->vf_res->rss_key_size; i++) 842 hw->rss_key[i] = (uint8_t)rte_rand(); 843 else 844 rte_memcpy(hw->rss_key, rss_conf->rss_key, 845 RTE_MIN(rss_conf->rss_key_len, 846 hw->vf_res->rss_key_size)); 847 848 /* init RSS LUT table */ 849 for (i = 0, j = 0; i < hw->vf_res->rss_lut_size; i++, j++) { 850 if (j >= nb_q) 851 j = 0; 852 hw->rss_lut[i] = j; 853 } 854 /* send virtchnnl ops to configure rss*/ 855 ret = ice_dcf_configure_rss_lut(hw); 856 if (ret) 857 return ret; 858 ret = ice_dcf_configure_rss_key(hw); 859 if (ret) 860 return ret; 861 862 return 0; 863 } 864 865 #define IAVF_RXDID_LEGACY_0 0 866 #define IAVF_RXDID_LEGACY_1 1 867 #define IAVF_RXDID_COMMS_OVS_1 22 868 869 int 870 ice_dcf_configure_queues(struct ice_dcf_hw *hw) 871 { 872 struct ice_rx_queue **rxq = 873 (struct ice_rx_queue **)hw->eth_dev->data->rx_queues; 874 struct ice_tx_queue **txq = 875 (struct ice_tx_queue **)hw->eth_dev->data->tx_queues; 876 struct virtchnl_vsi_queue_config_info *vc_config; 877 struct virtchnl_queue_pair_info *vc_qp; 878 struct dcf_virtchnl_cmd args; 879 uint16_t i, size; 880 int err; 881 882 size = sizeof(*vc_config) + 883 sizeof(vc_config->qpair[0]) * hw->num_queue_pairs; 884 vc_config = rte_zmalloc("cfg_queue", size, 0); 885 if (!vc_config) 886 return -ENOMEM; 887 888 vc_config->vsi_id = hw->vsi_res->vsi_id; 889 vc_config->num_queue_pairs = hw->num_queue_pairs; 890 891 for (i = 0, vc_qp = vc_config->qpair; 892 i < hw->num_queue_pairs; 893 i++, vc_qp++) { 894 vc_qp->txq.vsi_id = hw->vsi_res->vsi_id; 895 vc_qp->txq.queue_id = i; 896 if (i < hw->eth_dev->data->nb_tx_queues) { 897 vc_qp->txq.ring_len = txq[i]->nb_tx_desc; 898 vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_dma; 899 } 900 vc_qp->rxq.vsi_id = hw->vsi_res->vsi_id; 901 vc_qp->rxq.queue_id = i; 902 903 if (i >= hw->eth_dev->data->nb_rx_queues) 904 continue; 905 906 vc_qp->rxq.max_pkt_size = rxq[i]->max_pkt_len; 907 vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc; 908 vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_dma; 909 vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len; 910 911 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 912 if (hw->vf_res->vf_cap_flags & 913 VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && 914 hw->supported_rxdid & 915 BIT(IAVF_RXDID_COMMS_OVS_1)) { 916 vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_OVS_1; 917 PMD_DRV_LOG(NOTICE, "request RXDID == %d in " 918 "Queue[%d]", vc_qp->rxq.rxdid, i); 919 } else { 920 PMD_DRV_LOG(ERR, "RXDID 16 is not supported"); 921 return -EINVAL; 922 } 923 #else 924 if (hw->vf_res->vf_cap_flags & 925 VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && 926 hw->supported_rxdid & 927 BIT(IAVF_RXDID_LEGACY_0)) { 928 vc_qp->rxq.rxdid = IAVF_RXDID_LEGACY_0; 929 PMD_DRV_LOG(NOTICE, "request RXDID == %d in " 930 "Queue[%d]", vc_qp->rxq.rxdid, i); 931 } else { 932 PMD_DRV_LOG(ERR, "RXDID == 0 is not supported"); 933 return -EINVAL; 934 } 935 #endif 936 ice_select_rxd_to_pkt_fields_handler(rxq[i], vc_qp->rxq.rxdid); 937 } 938 939 memset(&args, 0, sizeof(args)); 940 args.v_op = VIRTCHNL_OP_CONFIG_VSI_QUEUES; 941 args.req_msg = (uint8_t *)vc_config; 942 args.req_msglen = size; 943 944 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 945 if (err) 946 PMD_DRV_LOG(ERR, "Failed to execute command of" 947 " VIRTCHNL_OP_CONFIG_VSI_QUEUES"); 948 949 rte_free(vc_config); 950 return err; 951 } 952 953 int 954 ice_dcf_config_irq_map(struct ice_dcf_hw *hw) 955 { 956 struct virtchnl_irq_map_info *map_info; 957 struct virtchnl_vector_map *vecmap; 958 struct dcf_virtchnl_cmd args; 959 int len, i, err; 960 961 len = sizeof(struct virtchnl_irq_map_info) + 962 sizeof(struct virtchnl_vector_map) * hw->nb_msix; 963 964 map_info = rte_zmalloc("map_info", len, 0); 965 if (!map_info) 966 return -ENOMEM; 967 968 map_info->num_vectors = hw->nb_msix; 969 for (i = 0; i < hw->nb_msix; i++) { 970 vecmap = &map_info->vecmap[i]; 971 vecmap->vsi_id = hw->vsi_res->vsi_id; 972 vecmap->rxitr_idx = 0; 973 vecmap->vector_id = hw->msix_base + i; 974 vecmap->txq_map = 0; 975 vecmap->rxq_map = hw->rxq_map[hw->msix_base + i]; 976 } 977 978 memset(&args, 0, sizeof(args)); 979 args.v_op = VIRTCHNL_OP_CONFIG_IRQ_MAP; 980 args.req_msg = (u8 *)map_info; 981 args.req_msglen = len; 982 983 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 984 if (err) 985 PMD_DRV_LOG(ERR, "fail to execute command OP_CONFIG_IRQ_MAP"); 986 987 rte_free(map_info); 988 return err; 989 } 990 991 int 992 ice_dcf_switch_queue(struct ice_dcf_hw *hw, uint16_t qid, bool rx, bool on) 993 { 994 struct virtchnl_queue_select queue_select; 995 struct dcf_virtchnl_cmd args; 996 int err; 997 998 memset(&queue_select, 0, sizeof(queue_select)); 999 queue_select.vsi_id = hw->vsi_res->vsi_id; 1000 if (rx) 1001 queue_select.rx_queues |= 1 << qid; 1002 else 1003 queue_select.tx_queues |= 1 << qid; 1004 1005 memset(&args, 0, sizeof(args)); 1006 if (on) 1007 args.v_op = VIRTCHNL_OP_ENABLE_QUEUES; 1008 else 1009 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 1010 1011 args.req_msg = (u8 *)&queue_select; 1012 args.req_msglen = sizeof(queue_select); 1013 1014 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1015 if (err) 1016 PMD_DRV_LOG(ERR, "Failed to execute command of %s", 1017 on ? "OP_ENABLE_QUEUES" : "OP_DISABLE_QUEUES"); 1018 1019 return err; 1020 } 1021 1022 int 1023 ice_dcf_disable_queues(struct ice_dcf_hw *hw) 1024 { 1025 struct virtchnl_queue_select queue_select; 1026 struct dcf_virtchnl_cmd args; 1027 int err; 1028 1029 if (hw->resetting) 1030 return 0; 1031 1032 memset(&queue_select, 0, sizeof(queue_select)); 1033 queue_select.vsi_id = hw->vsi_res->vsi_id; 1034 1035 queue_select.rx_queues = BIT(hw->eth_dev->data->nb_rx_queues) - 1; 1036 queue_select.tx_queues = BIT(hw->eth_dev->data->nb_tx_queues) - 1; 1037 1038 memset(&args, 0, sizeof(args)); 1039 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 1040 args.req_msg = (u8 *)&queue_select; 1041 args.req_msglen = sizeof(queue_select); 1042 1043 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1044 if (err) 1045 PMD_DRV_LOG(ERR, 1046 "Failed to execute command of OP_DISABLE_QUEUES"); 1047 1048 return err; 1049 } 1050 1051 int 1052 ice_dcf_query_stats(struct ice_dcf_hw *hw, 1053 struct virtchnl_eth_stats *pstats) 1054 { 1055 struct virtchnl_queue_select q_stats; 1056 struct dcf_virtchnl_cmd args; 1057 int err; 1058 1059 memset(&q_stats, 0, sizeof(q_stats)); 1060 q_stats.vsi_id = hw->vsi_res->vsi_id; 1061 1062 args.v_op = VIRTCHNL_OP_GET_STATS; 1063 args.req_msg = (uint8_t *)&q_stats; 1064 args.req_msglen = sizeof(q_stats); 1065 args.rsp_msglen = sizeof(*pstats); 1066 args.rsp_msgbuf = (uint8_t *)pstats; 1067 args.rsp_buflen = sizeof(*pstats); 1068 1069 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1070 if (err) { 1071 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS"); 1072 return err; 1073 } 1074 1075 return 0; 1076 } 1077 1078 int 1079 ice_dcf_add_del_all_mac_addr(struct ice_dcf_hw *hw, bool add) 1080 { 1081 struct virtchnl_ether_addr_list *list; 1082 struct rte_ether_addr *addr; 1083 struct dcf_virtchnl_cmd args; 1084 int len, err = 0; 1085 1086 if (hw->resetting) { 1087 if (!add) 1088 return 0; 1089 1090 PMD_DRV_LOG(ERR, "fail to add all MACs for VF resetting"); 1091 return -EIO; 1092 } 1093 1094 len = sizeof(struct virtchnl_ether_addr_list); 1095 addr = hw->eth_dev->data->mac_addrs; 1096 len += sizeof(struct virtchnl_ether_addr); 1097 1098 list = rte_zmalloc(NULL, len, 0); 1099 if (!list) { 1100 PMD_DRV_LOG(ERR, "fail to allocate memory"); 1101 return -ENOMEM; 1102 } 1103 1104 rte_memcpy(list->list[0].addr, addr->addr_bytes, 1105 sizeof(addr->addr_bytes)); 1106 PMD_DRV_LOG(DEBUG, "add/rm mac:" RTE_ETHER_ADDR_PRT_FMT, 1107 RTE_ETHER_ADDR_BYTES(addr)); 1108 1109 list->vsi_id = hw->vsi_res->vsi_id; 1110 list->num_elements = 1; 1111 1112 memset(&args, 0, sizeof(args)); 1113 args.v_op = add ? VIRTCHNL_OP_ADD_ETH_ADDR : 1114 VIRTCHNL_OP_DEL_ETH_ADDR; 1115 args.req_msg = (uint8_t *)list; 1116 args.req_msglen = len; 1117 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1118 if (err) 1119 PMD_DRV_LOG(ERR, "fail to execute command %s", 1120 add ? "OP_ADD_ETHER_ADDRESS" : 1121 "OP_DEL_ETHER_ADDRESS"); 1122 rte_free(list); 1123 return err; 1124 } 1125