1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <errno.h> 8 #include <stdint.h> 9 #include <string.h> 10 #include <unistd.h> 11 #include <stdarg.h> 12 #include <inttypes.h> 13 #include <rte_byteorder.h> 14 #include <rte_common.h> 15 16 #include <rte_pci.h> 17 #include <rte_atomic.h> 18 #include <rte_eal.h> 19 #include <rte_ether.h> 20 #include <ethdev_driver.h> 21 #include <ethdev_pci.h> 22 #include <rte_malloc.h> 23 #include <rte_memzone.h> 24 #include <rte_dev.h> 25 26 #include "ice_dcf.h" 27 #include "ice_rxtx.h" 28 29 #define ICE_DCF_AQ_LEN 32 30 #define ICE_DCF_AQ_BUF_SZ 4096 31 32 #define ICE_DCF_ARQ_MAX_RETRIES 200 33 #define ICE_DCF_ARQ_CHECK_TIME 2 /* msecs */ 34 35 #define ICE_DCF_VF_RES_BUF_SZ \ 36 (sizeof(struct virtchnl_vf_resource) + \ 37 IAVF_MAX_VF_VSI * sizeof(struct virtchnl_vsi_resource)) 38 39 static __rte_always_inline int 40 ice_dcf_send_cmd_req_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 41 uint8_t *req_msg, uint16_t req_msglen) 42 { 43 return iavf_aq_send_msg_to_pf(&hw->avf, op, IAVF_SUCCESS, 44 req_msg, req_msglen, NULL); 45 } 46 47 static int 48 ice_dcf_recv_cmd_rsp_no_irq(struct ice_dcf_hw *hw, enum virtchnl_ops op, 49 uint8_t *rsp_msgbuf, uint16_t rsp_buflen, 50 uint16_t *rsp_msglen) 51 { 52 struct iavf_arq_event_info event; 53 enum virtchnl_ops v_op; 54 int i = 0; 55 int err; 56 57 event.buf_len = rsp_buflen; 58 event.msg_buf = rsp_msgbuf; 59 60 do { 61 err = iavf_clean_arq_element(&hw->avf, &event, NULL); 62 if (err != IAVF_SUCCESS) 63 goto again; 64 65 v_op = rte_le_to_cpu_32(event.desc.cookie_high); 66 if (v_op != op) 67 goto again; 68 69 if (rsp_msglen != NULL) 70 *rsp_msglen = event.msg_len; 71 return rte_le_to_cpu_32(event.desc.cookie_low); 72 73 again: 74 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 75 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 76 77 return -EIO; 78 } 79 80 static __rte_always_inline void 81 ice_dcf_aq_cmd_clear(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 82 { 83 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 84 85 TAILQ_REMOVE(&hw->vc_cmd_queue, cmd, next); 86 87 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 88 } 89 90 static __rte_always_inline void 91 ice_dcf_vc_cmd_set(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 92 { 93 cmd->v_ret = IAVF_ERR_NOT_READY; 94 cmd->rsp_msglen = 0; 95 cmd->pending = 1; 96 97 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 98 99 TAILQ_INSERT_TAIL(&hw->vc_cmd_queue, cmd, next); 100 101 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 102 } 103 104 static __rte_always_inline int 105 ice_dcf_vc_cmd_send(struct ice_dcf_hw *hw, struct dcf_virtchnl_cmd *cmd) 106 { 107 return iavf_aq_send_msg_to_pf(&hw->avf, 108 cmd->v_op, IAVF_SUCCESS, 109 cmd->req_msg, cmd->req_msglen, NULL); 110 } 111 112 static __rte_always_inline void 113 ice_dcf_aq_cmd_handle(struct ice_dcf_hw *hw, struct iavf_arq_event_info *info) 114 { 115 struct dcf_virtchnl_cmd *cmd; 116 enum virtchnl_ops v_op; 117 enum iavf_status v_ret; 118 uint16_t aq_op; 119 120 aq_op = rte_le_to_cpu_16(info->desc.opcode); 121 if (unlikely(aq_op != iavf_aqc_opc_send_msg_to_vf)) { 122 PMD_DRV_LOG(ERR, 123 "Request %u is not supported yet", aq_op); 124 return; 125 } 126 127 v_op = rte_le_to_cpu_32(info->desc.cookie_high); 128 if (v_op == VIRTCHNL_OP_EVENT) { 129 if (hw->vc_event_msg_cb != NULL) 130 hw->vc_event_msg_cb(hw, 131 info->msg_buf, 132 info->msg_len); 133 return; 134 } 135 136 v_ret = rte_le_to_cpu_32(info->desc.cookie_low); 137 138 rte_spinlock_lock(&hw->vc_cmd_queue_lock); 139 140 TAILQ_FOREACH(cmd, &hw->vc_cmd_queue, next) { 141 if (cmd->v_op == v_op && cmd->pending) { 142 cmd->v_ret = v_ret; 143 cmd->rsp_msglen = RTE_MIN(info->msg_len, 144 cmd->rsp_buflen); 145 if (likely(cmd->rsp_msglen != 0)) 146 rte_memcpy(cmd->rsp_msgbuf, info->msg_buf, 147 cmd->rsp_msglen); 148 149 /* prevent compiler reordering */ 150 rte_compiler_barrier(); 151 cmd->pending = 0; 152 break; 153 } 154 } 155 156 rte_spinlock_unlock(&hw->vc_cmd_queue_lock); 157 } 158 159 static void 160 ice_dcf_handle_virtchnl_msg(struct ice_dcf_hw *hw) 161 { 162 struct iavf_arq_event_info info; 163 uint16_t pending = 1; 164 int ret; 165 166 info.buf_len = ICE_DCF_AQ_BUF_SZ; 167 info.msg_buf = hw->arq_buf; 168 169 while (pending) { 170 ret = iavf_clean_arq_element(&hw->avf, &info, &pending); 171 if (ret != IAVF_SUCCESS) 172 break; 173 174 ice_dcf_aq_cmd_handle(hw, &info); 175 } 176 } 177 178 static int 179 ice_dcf_init_check_api_version(struct ice_dcf_hw *hw) 180 { 181 #define ICE_CPF_VIRTCHNL_VERSION_MAJOR_START 1 182 #define ICE_CPF_VIRTCHNL_VERSION_MINOR_START 1 183 struct virtchnl_version_info version, *pver; 184 int err; 185 186 version.major = VIRTCHNL_VERSION_MAJOR; 187 version.minor = VIRTCHNL_VERSION_MINOR; 188 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_VERSION, 189 (uint8_t *)&version, sizeof(version)); 190 if (err) { 191 PMD_INIT_LOG(ERR, "Failed to send OP_VERSION"); 192 return err; 193 } 194 195 pver = &hw->virtchnl_version; 196 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_VERSION, 197 (uint8_t *)pver, sizeof(*pver), NULL); 198 if (err) { 199 PMD_INIT_LOG(ERR, "Failed to get response of OP_VERSION"); 200 return -1; 201 } 202 203 PMD_INIT_LOG(DEBUG, 204 "Peer PF API version: %u.%u", pver->major, pver->minor); 205 206 if (pver->major < ICE_CPF_VIRTCHNL_VERSION_MAJOR_START || 207 (pver->major == ICE_CPF_VIRTCHNL_VERSION_MAJOR_START && 208 pver->minor < ICE_CPF_VIRTCHNL_VERSION_MINOR_START)) { 209 PMD_INIT_LOG(ERR, 210 "VIRTCHNL API version should not be lower than (%u.%u)", 211 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START, 212 ICE_CPF_VIRTCHNL_VERSION_MAJOR_START); 213 return -1; 214 } else if (pver->major > VIRTCHNL_VERSION_MAJOR || 215 (pver->major == VIRTCHNL_VERSION_MAJOR && 216 pver->minor > VIRTCHNL_VERSION_MINOR)) { 217 PMD_INIT_LOG(ERR, 218 "PF/VF API version mismatch:(%u.%u)-(%u.%u)", 219 pver->major, pver->minor, 220 VIRTCHNL_VERSION_MAJOR, VIRTCHNL_VERSION_MINOR); 221 return -1; 222 } 223 224 PMD_INIT_LOG(DEBUG, "Peer is supported PF host"); 225 226 return 0; 227 } 228 229 static int 230 ice_dcf_get_vf_resource(struct ice_dcf_hw *hw) 231 { 232 uint32_t caps; 233 int err, i; 234 235 caps = VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RX_POLLING | 236 VIRTCHNL_VF_CAP_ADV_LINK_SPEED | VIRTCHNL_VF_CAP_DCF | 237 VIRTCHNL_VF_OFFLOAD_VLAN_V2 | 238 VF_BASE_MODE_OFFLOADS | VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC | 239 VIRTCHNL_VF_OFFLOAD_QOS; 240 241 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 242 (uint8_t *)&caps, sizeof(caps)); 243 if (err) { 244 PMD_DRV_LOG(ERR, "Failed to send msg OP_GET_VF_RESOURCE"); 245 return err; 246 } 247 248 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_VF_RESOURCES, 249 (uint8_t *)hw->vf_res, 250 ICE_DCF_VF_RES_BUF_SZ, NULL); 251 if (err) { 252 PMD_DRV_LOG(ERR, "Failed to get response of OP_GET_VF_RESOURCE"); 253 return -1; 254 } 255 256 iavf_vf_parse_hw_config(&hw->avf, hw->vf_res); 257 258 hw->vsi_res = NULL; 259 for (i = 0; i < hw->vf_res->num_vsis; i++) { 260 if (hw->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 261 hw->vsi_res = &hw->vf_res->vsi_res[i]; 262 } 263 264 if (!hw->vsi_res) { 265 PMD_DRV_LOG(ERR, "no LAN VSI found"); 266 return -1; 267 } 268 269 hw->vsi_id = hw->vsi_res->vsi_id; 270 PMD_DRV_LOG(DEBUG, "VSI ID is %u", hw->vsi_id); 271 272 return 0; 273 } 274 275 static int 276 ice_dcf_get_vf_vsi_map(struct ice_dcf_hw *hw) 277 { 278 struct virtchnl_dcf_vsi_map *vsi_map; 279 uint32_t valid_msg_len; 280 uint16_t len; 281 int err; 282 283 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 284 NULL, 0); 285 if (err) { 286 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_GET_VSI_MAP"); 287 return err; 288 } 289 290 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_GET_VSI_MAP, 291 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, 292 &len); 293 if (err) { 294 PMD_DRV_LOG(ERR, "Failed to get response of OP_DCF_GET_VSI_MAP"); 295 return err; 296 } 297 298 vsi_map = (struct virtchnl_dcf_vsi_map *)hw->arq_buf; 299 valid_msg_len = (vsi_map->num_vfs - 1) * sizeof(vsi_map->vf_vsi[0]) + 300 sizeof(*vsi_map); 301 if (len != valid_msg_len) { 302 PMD_DRV_LOG(ERR, "invalid vf vsi map response with length %u", 303 len); 304 return -EINVAL; 305 } 306 307 if (hw->num_vfs != 0 && hw->num_vfs != vsi_map->num_vfs) { 308 PMD_DRV_LOG(ERR, "The number VSI map (%u) doesn't match the number of VFs (%u)", 309 vsi_map->num_vfs, hw->num_vfs); 310 return -EINVAL; 311 } 312 313 len = vsi_map->num_vfs * sizeof(vsi_map->vf_vsi[0]); 314 315 if (!hw->vf_vsi_map) { 316 hw->vf_vsi_map = rte_zmalloc("vf_vsi_ctx", len, 0); 317 if (!hw->vf_vsi_map) { 318 PMD_DRV_LOG(ERR, "Failed to alloc memory for VSI context"); 319 return -ENOMEM; 320 } 321 322 hw->num_vfs = vsi_map->num_vfs; 323 hw->pf_vsi_id = vsi_map->pf_vsi; 324 } 325 326 if (!memcmp(hw->vf_vsi_map, vsi_map->vf_vsi, len)) { 327 PMD_DRV_LOG(DEBUG, "VF VSI map doesn't change"); 328 return 1; 329 } 330 331 rte_memcpy(hw->vf_vsi_map, vsi_map->vf_vsi, len); 332 return 0; 333 } 334 335 static int 336 ice_dcf_mode_disable(struct ice_dcf_hw *hw) 337 { 338 int err; 339 340 err = ice_dcf_send_cmd_req_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 341 NULL, 0); 342 if (err) { 343 PMD_DRV_LOG(ERR, "Failed to send msg OP_DCF_DISABLE"); 344 return err; 345 } 346 347 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_DCF_DISABLE, 348 hw->arq_buf, ICE_DCF_AQ_BUF_SZ, NULL); 349 if (err) { 350 PMD_DRV_LOG(ERR, 351 "Failed to get response of OP_DCF_DISABLE %d", 352 err); 353 return -1; 354 } 355 356 return 0; 357 } 358 359 static int 360 ice_dcf_check_reset_done(struct ice_dcf_hw *hw) 361 { 362 #define ICE_DCF_RESET_WAIT_CNT 50 363 struct iavf_hw *avf = &hw->avf; 364 int i, reset; 365 366 for (i = 0; i < ICE_DCF_RESET_WAIT_CNT; i++) { 367 reset = IAVF_READ_REG(avf, IAVF_VFGEN_RSTAT) & 368 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 369 reset = reset >> IAVF_VFGEN_RSTAT_VFR_STATE_SHIFT; 370 371 if (reset == VIRTCHNL_VFR_VFACTIVE || 372 reset == VIRTCHNL_VFR_COMPLETED) 373 break; 374 375 rte_delay_ms(20); 376 } 377 378 if (i >= ICE_DCF_RESET_WAIT_CNT) 379 return -1; 380 381 return 0; 382 } 383 384 static inline void 385 ice_dcf_enable_irq0(struct ice_dcf_hw *hw) 386 { 387 struct iavf_hw *avf = &hw->avf; 388 389 /* Enable admin queue interrupt trigger */ 390 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 391 IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 392 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 393 IAVF_VFINT_DYN_CTL01_INTENA_MASK | 394 IAVF_VFINT_DYN_CTL01_CLEARPBA_MASK | 395 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 396 397 IAVF_WRITE_FLUSH(avf); 398 } 399 400 static inline void 401 ice_dcf_disable_irq0(struct ice_dcf_hw *hw) 402 { 403 struct iavf_hw *avf = &hw->avf; 404 405 /* Disable all interrupt types */ 406 IAVF_WRITE_REG(avf, IAVF_VFINT_ICR0_ENA1, 0); 407 IAVF_WRITE_REG(avf, IAVF_VFINT_DYN_CTL01, 408 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 409 410 IAVF_WRITE_FLUSH(avf); 411 } 412 413 static void 414 ice_dcf_dev_interrupt_handler(void *param) 415 { 416 struct ice_dcf_hw *hw = param; 417 418 ice_dcf_disable_irq0(hw); 419 420 ice_dcf_handle_virtchnl_msg(hw); 421 422 ice_dcf_enable_irq0(hw); 423 } 424 425 int 426 ice_dcf_execute_virtchnl_cmd(struct ice_dcf_hw *hw, 427 struct dcf_virtchnl_cmd *cmd) 428 { 429 int i = 0; 430 int err; 431 432 if ((cmd->req_msg && !cmd->req_msglen) || 433 (!cmd->req_msg && cmd->req_msglen) || 434 (cmd->rsp_msgbuf && !cmd->rsp_buflen) || 435 (!cmd->rsp_msgbuf && cmd->rsp_buflen)) 436 return -EINVAL; 437 438 rte_spinlock_lock(&hw->vc_cmd_send_lock); 439 ice_dcf_vc_cmd_set(hw, cmd); 440 441 err = ice_dcf_vc_cmd_send(hw, cmd); 442 if (err) { 443 PMD_DRV_LOG(ERR, "fail to send cmd %d", cmd->v_op); 444 goto ret; 445 } 446 447 do { 448 if (!cmd->pending) 449 break; 450 451 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 452 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 453 454 if (cmd->v_ret != IAVF_SUCCESS) { 455 err = -1; 456 PMD_DRV_LOG(ERR, 457 "No response (%d times) or return failure (%d) for cmd %d", 458 i, cmd->v_ret, cmd->v_op); 459 } 460 461 ret: 462 ice_dcf_aq_cmd_clear(hw, cmd); 463 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 464 return err; 465 } 466 467 int 468 ice_dcf_send_aq_cmd(void *dcf_hw, struct ice_aq_desc *desc, 469 void *buf, uint16_t buf_size) 470 { 471 struct dcf_virtchnl_cmd desc_cmd, buff_cmd; 472 struct ice_dcf_hw *hw = dcf_hw; 473 int err = 0; 474 int i = 0; 475 476 if ((buf && !buf_size) || (!buf && buf_size) || 477 buf_size > ICE_DCF_AQ_BUF_SZ) 478 return -EINVAL; 479 480 desc_cmd.v_op = VIRTCHNL_OP_DCF_CMD_DESC; 481 desc_cmd.req_msglen = sizeof(*desc); 482 desc_cmd.req_msg = (uint8_t *)desc; 483 desc_cmd.rsp_buflen = sizeof(*desc); 484 desc_cmd.rsp_msgbuf = (uint8_t *)desc; 485 486 if (buf == NULL) 487 return ice_dcf_execute_virtchnl_cmd(hw, &desc_cmd); 488 489 desc->flags |= rte_cpu_to_le_16(ICE_AQ_FLAG_BUF); 490 491 buff_cmd.v_op = VIRTCHNL_OP_DCF_CMD_BUFF; 492 buff_cmd.req_msglen = buf_size; 493 buff_cmd.req_msg = buf; 494 buff_cmd.rsp_buflen = buf_size; 495 buff_cmd.rsp_msgbuf = buf; 496 497 rte_spinlock_lock(&hw->vc_cmd_send_lock); 498 ice_dcf_vc_cmd_set(hw, &desc_cmd); 499 ice_dcf_vc_cmd_set(hw, &buff_cmd); 500 501 if (ice_dcf_vc_cmd_send(hw, &desc_cmd) || 502 ice_dcf_vc_cmd_send(hw, &buff_cmd)) { 503 err = -1; 504 PMD_DRV_LOG(ERR, "fail to send OP_DCF_CMD_DESC/BUFF"); 505 goto ret; 506 } 507 508 do { 509 if (!desc_cmd.pending && !buff_cmd.pending) 510 break; 511 512 rte_delay_ms(ICE_DCF_ARQ_CHECK_TIME); 513 } while (i++ < ICE_DCF_ARQ_MAX_RETRIES); 514 515 if (desc_cmd.v_ret != IAVF_SUCCESS || buff_cmd.v_ret != IAVF_SUCCESS) { 516 err = -1; 517 PMD_DRV_LOG(ERR, 518 "No response (%d times) or return failure (desc: %d / buff: %d)", 519 i, desc_cmd.v_ret, buff_cmd.v_ret); 520 } 521 522 ret: 523 ice_dcf_aq_cmd_clear(hw, &desc_cmd); 524 ice_dcf_aq_cmd_clear(hw, &buff_cmd); 525 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 526 527 return err; 528 } 529 530 int 531 ice_dcf_handle_vsi_update_event(struct ice_dcf_hw *hw) 532 { 533 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(hw->eth_dev); 534 int err = 0; 535 536 rte_spinlock_lock(&hw->vc_cmd_send_lock); 537 538 rte_intr_disable(&pci_dev->intr_handle); 539 ice_dcf_disable_irq0(hw); 540 541 if (ice_dcf_get_vf_resource(hw) || ice_dcf_get_vf_vsi_map(hw) < 0) 542 err = -1; 543 544 rte_intr_enable(&pci_dev->intr_handle); 545 ice_dcf_enable_irq0(hw); 546 547 rte_spinlock_unlock(&hw->vc_cmd_send_lock); 548 549 return err; 550 } 551 552 static int 553 ice_dcf_get_supported_rxdid(struct ice_dcf_hw *hw) 554 { 555 int err; 556 557 err = ice_dcf_send_cmd_req_no_irq(hw, 558 VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 559 NULL, 0); 560 if (err) { 561 PMD_INIT_LOG(ERR, "Failed to send OP_GET_SUPPORTED_RXDIDS"); 562 return -1; 563 } 564 565 err = ice_dcf_recv_cmd_rsp_no_irq(hw, VIRTCHNL_OP_GET_SUPPORTED_RXDIDS, 566 (uint8_t *)&hw->supported_rxdid, 567 sizeof(uint64_t), NULL); 568 if (err) { 569 PMD_INIT_LOG(ERR, "Failed to get response of OP_GET_SUPPORTED_RXDIDS"); 570 return -1; 571 } 572 573 return 0; 574 } 575 576 int 577 ice_dcf_init_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 578 { 579 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 580 int ret; 581 582 hw->avf.hw_addr = pci_dev->mem_resource[0].addr; 583 hw->avf.back = hw; 584 585 hw->avf.bus.bus_id = pci_dev->addr.bus; 586 hw->avf.bus.device = pci_dev->addr.devid; 587 hw->avf.bus.func = pci_dev->addr.function; 588 589 hw->avf.device_id = pci_dev->id.device_id; 590 hw->avf.vendor_id = pci_dev->id.vendor_id; 591 hw->avf.subsystem_device_id = pci_dev->id.subsystem_device_id; 592 hw->avf.subsystem_vendor_id = pci_dev->id.subsystem_vendor_id; 593 594 hw->avf.aq.num_arq_entries = ICE_DCF_AQ_LEN; 595 hw->avf.aq.num_asq_entries = ICE_DCF_AQ_LEN; 596 hw->avf.aq.arq_buf_size = ICE_DCF_AQ_BUF_SZ; 597 hw->avf.aq.asq_buf_size = ICE_DCF_AQ_BUF_SZ; 598 599 rte_spinlock_init(&hw->vc_cmd_send_lock); 600 rte_spinlock_init(&hw->vc_cmd_queue_lock); 601 TAILQ_INIT(&hw->vc_cmd_queue); 602 603 hw->arq_buf = rte_zmalloc("arq_buf", ICE_DCF_AQ_BUF_SZ, 0); 604 if (hw->arq_buf == NULL) { 605 PMD_INIT_LOG(ERR, "unable to allocate AdminQ buffer memory"); 606 goto err; 607 } 608 609 ret = iavf_set_mac_type(&hw->avf); 610 if (ret) { 611 PMD_INIT_LOG(ERR, "set_mac_type failed: %d", ret); 612 goto err; 613 } 614 615 ret = ice_dcf_check_reset_done(hw); 616 if (ret) { 617 PMD_INIT_LOG(ERR, "VF is still resetting"); 618 goto err; 619 } 620 621 ret = iavf_init_adminq(&hw->avf); 622 if (ret) { 623 PMD_INIT_LOG(ERR, "init_adminq failed: %d", ret); 624 goto err; 625 } 626 627 if (ice_dcf_init_check_api_version(hw)) { 628 PMD_INIT_LOG(ERR, "check_api version failed"); 629 goto err_api; 630 } 631 632 hw->vf_res = rte_zmalloc("vf_res", ICE_DCF_VF_RES_BUF_SZ, 0); 633 if (hw->vf_res == NULL) { 634 PMD_INIT_LOG(ERR, "unable to allocate vf_res memory"); 635 goto err_api; 636 } 637 638 if (ice_dcf_get_vf_resource(hw)) { 639 PMD_INIT_LOG(ERR, "Failed to get VF resource"); 640 goto err_alloc; 641 } 642 643 if (ice_dcf_get_vf_vsi_map(hw) < 0) { 644 PMD_INIT_LOG(ERR, "Failed to get VF VSI map"); 645 ice_dcf_mode_disable(hw); 646 goto err_alloc; 647 } 648 649 /* Allocate memory for RSS info */ 650 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 651 hw->rss_key = rte_zmalloc(NULL, 652 hw->vf_res->rss_key_size, 0); 653 if (!hw->rss_key) { 654 PMD_INIT_LOG(ERR, "unable to allocate rss_key memory"); 655 goto err_alloc; 656 } 657 hw->rss_lut = rte_zmalloc("rss_lut", 658 hw->vf_res->rss_lut_size, 0); 659 if (!hw->rss_lut) { 660 PMD_INIT_LOG(ERR, "unable to allocate rss_lut memory"); 661 goto err_rss; 662 } 663 } 664 665 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { 666 if (ice_dcf_get_supported_rxdid(hw) != 0) { 667 PMD_INIT_LOG(ERR, "failed to do get supported rxdid"); 668 goto err_rss; 669 } 670 } 671 672 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 673 ice_dcf_tm_conf_init(eth_dev); 674 675 hw->eth_dev = eth_dev; 676 rte_intr_callback_register(&pci_dev->intr_handle, 677 ice_dcf_dev_interrupt_handler, hw); 678 rte_intr_enable(&pci_dev->intr_handle); 679 ice_dcf_enable_irq0(hw); 680 681 return 0; 682 683 err_rss: 684 rte_free(hw->rss_key); 685 rte_free(hw->rss_lut); 686 err_alloc: 687 rte_free(hw->vf_res); 688 err_api: 689 iavf_shutdown_adminq(&hw->avf); 690 err: 691 rte_free(hw->arq_buf); 692 693 return -1; 694 } 695 696 void 697 ice_dcf_uninit_hw(struct rte_eth_dev *eth_dev, struct ice_dcf_hw *hw) 698 { 699 struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); 700 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; 701 702 ice_dcf_disable_irq0(hw); 703 rte_intr_disable(intr_handle); 704 rte_intr_callback_unregister(intr_handle, 705 ice_dcf_dev_interrupt_handler, hw); 706 707 ice_dcf_mode_disable(hw); 708 iavf_shutdown_adminq(&hw->avf); 709 710 if (hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_QOS) 711 ice_dcf_tm_conf_uninit(eth_dev); 712 713 rte_free(hw->arq_buf); 714 rte_free(hw->vf_vsi_map); 715 rte_free(hw->vf_res); 716 rte_free(hw->rss_lut); 717 rte_free(hw->rss_key); 718 } 719 720 static int 721 ice_dcf_configure_rss_key(struct ice_dcf_hw *hw) 722 { 723 struct virtchnl_rss_key *rss_key; 724 struct dcf_virtchnl_cmd args; 725 int len, err; 726 727 len = sizeof(*rss_key) + hw->vf_res->rss_key_size - 1; 728 rss_key = rte_zmalloc("rss_key", len, 0); 729 if (!rss_key) 730 return -ENOMEM; 731 732 rss_key->vsi_id = hw->vsi_res->vsi_id; 733 rss_key->key_len = hw->vf_res->rss_key_size; 734 rte_memcpy(rss_key->key, hw->rss_key, hw->vf_res->rss_key_size); 735 736 args.v_op = VIRTCHNL_OP_CONFIG_RSS_KEY; 737 args.req_msglen = len; 738 args.req_msg = (uint8_t *)rss_key; 739 args.rsp_msglen = 0; 740 args.rsp_buflen = 0; 741 args.rsp_msgbuf = NULL; 742 args.pending = 0; 743 744 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 745 if (err) 746 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_KEY"); 747 748 rte_free(rss_key); 749 return err; 750 } 751 752 static int 753 ice_dcf_configure_rss_lut(struct ice_dcf_hw *hw) 754 { 755 struct virtchnl_rss_lut *rss_lut; 756 struct dcf_virtchnl_cmd args; 757 int len, err; 758 759 len = sizeof(*rss_lut) + hw->vf_res->rss_lut_size - 1; 760 rss_lut = rte_zmalloc("rss_lut", len, 0); 761 if (!rss_lut) 762 return -ENOMEM; 763 764 rss_lut->vsi_id = hw->vsi_res->vsi_id; 765 rss_lut->lut_entries = hw->vf_res->rss_lut_size; 766 rte_memcpy(rss_lut->lut, hw->rss_lut, hw->vf_res->rss_lut_size); 767 768 args.v_op = VIRTCHNL_OP_CONFIG_RSS_LUT; 769 args.req_msglen = len; 770 args.req_msg = (uint8_t *)rss_lut; 771 args.rsp_msglen = 0; 772 args.rsp_buflen = 0; 773 args.rsp_msgbuf = NULL; 774 args.pending = 0; 775 776 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 777 if (err) 778 PMD_INIT_LOG(ERR, "Failed to execute OP_CONFIG_RSS_LUT"); 779 780 rte_free(rss_lut); 781 return err; 782 } 783 784 int 785 ice_dcf_init_rss(struct ice_dcf_hw *hw) 786 { 787 struct rte_eth_dev *dev = hw->eth_dev; 788 struct rte_eth_rss_conf *rss_conf; 789 uint8_t i, j, nb_q; 790 int ret; 791 792 rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf; 793 nb_q = dev->data->nb_rx_queues; 794 795 if (!(hw->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF)) { 796 PMD_DRV_LOG(DEBUG, "RSS is not supported"); 797 return -ENOTSUP; 798 } 799 if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) { 800 PMD_DRV_LOG(WARNING, "RSS is enabled by PF by default"); 801 /* set all lut items to default queue */ 802 memset(hw->rss_lut, 0, hw->vf_res->rss_lut_size); 803 return ice_dcf_configure_rss_lut(hw); 804 } 805 806 /* In IAVF, RSS enablement is set by PF driver. It is not supported 807 * to set based on rss_conf->rss_hf. 808 */ 809 810 /* configure RSS key */ 811 if (!rss_conf->rss_key) 812 /* Calculate the default hash key */ 813 for (i = 0; i < hw->vf_res->rss_key_size; i++) 814 hw->rss_key[i] = (uint8_t)rte_rand(); 815 else 816 rte_memcpy(hw->rss_key, rss_conf->rss_key, 817 RTE_MIN(rss_conf->rss_key_len, 818 hw->vf_res->rss_key_size)); 819 820 /* init RSS LUT table */ 821 for (i = 0, j = 0; i < hw->vf_res->rss_lut_size; i++, j++) { 822 if (j >= nb_q) 823 j = 0; 824 hw->rss_lut[i] = j; 825 } 826 /* send virtchnnl ops to configure rss*/ 827 ret = ice_dcf_configure_rss_lut(hw); 828 if (ret) 829 return ret; 830 ret = ice_dcf_configure_rss_key(hw); 831 if (ret) 832 return ret; 833 834 return 0; 835 } 836 837 #define IAVF_RXDID_LEGACY_0 0 838 #define IAVF_RXDID_LEGACY_1 1 839 #define IAVF_RXDID_COMMS_GENERIC 16 840 841 int 842 ice_dcf_configure_queues(struct ice_dcf_hw *hw) 843 { 844 struct ice_rx_queue **rxq = 845 (struct ice_rx_queue **)hw->eth_dev->data->rx_queues; 846 struct ice_tx_queue **txq = 847 (struct ice_tx_queue **)hw->eth_dev->data->tx_queues; 848 struct virtchnl_vsi_queue_config_info *vc_config; 849 struct virtchnl_queue_pair_info *vc_qp; 850 struct dcf_virtchnl_cmd args; 851 uint16_t i, size; 852 int err; 853 854 size = sizeof(*vc_config) + 855 sizeof(vc_config->qpair[0]) * hw->num_queue_pairs; 856 vc_config = rte_zmalloc("cfg_queue", size, 0); 857 if (!vc_config) 858 return -ENOMEM; 859 860 vc_config->vsi_id = hw->vsi_res->vsi_id; 861 vc_config->num_queue_pairs = hw->num_queue_pairs; 862 863 for (i = 0, vc_qp = vc_config->qpair; 864 i < hw->num_queue_pairs; 865 i++, vc_qp++) { 866 vc_qp->txq.vsi_id = hw->vsi_res->vsi_id; 867 vc_qp->txq.queue_id = i; 868 if (i < hw->eth_dev->data->nb_tx_queues) { 869 vc_qp->txq.ring_len = txq[i]->nb_tx_desc; 870 vc_qp->txq.dma_ring_addr = txq[i]->tx_ring_dma; 871 } 872 vc_qp->rxq.vsi_id = hw->vsi_res->vsi_id; 873 vc_qp->rxq.queue_id = i; 874 vc_qp->rxq.max_pkt_size = rxq[i]->max_pkt_len; 875 876 if (i >= hw->eth_dev->data->nb_rx_queues) 877 continue; 878 879 vc_qp->rxq.ring_len = rxq[i]->nb_rx_desc; 880 vc_qp->rxq.dma_ring_addr = rxq[i]->rx_ring_dma; 881 vc_qp->rxq.databuffer_size = rxq[i]->rx_buf_len; 882 883 #ifndef RTE_LIBRTE_ICE_16BYTE_RX_DESC 884 if (hw->vf_res->vf_cap_flags & 885 VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && 886 hw->supported_rxdid & 887 BIT(IAVF_RXDID_COMMS_GENERIC)) { 888 vc_qp->rxq.rxdid = IAVF_RXDID_COMMS_GENERIC; 889 PMD_DRV_LOG(NOTICE, "request RXDID == %d in " 890 "Queue[%d]", vc_qp->rxq.rxdid, i); 891 } else { 892 PMD_DRV_LOG(ERR, "RXDID 16 is not supported"); 893 return -EINVAL; 894 } 895 #else 896 if (hw->vf_res->vf_cap_flags & 897 VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC && 898 hw->supported_rxdid & 899 BIT(IAVF_RXDID_LEGACY_0)) { 900 vc_qp->rxq.rxdid = IAVF_RXDID_LEGACY_0; 901 PMD_DRV_LOG(NOTICE, "request RXDID == %d in " 902 "Queue[%d]", vc_qp->rxq.rxdid, i); 903 } else { 904 PMD_DRV_LOG(ERR, "RXDID == 0 is not supported"); 905 return -EINVAL; 906 } 907 #endif 908 ice_select_rxd_to_pkt_fields_handler(rxq[i], vc_qp->rxq.rxdid); 909 } 910 911 memset(&args, 0, sizeof(args)); 912 args.v_op = VIRTCHNL_OP_CONFIG_VSI_QUEUES; 913 args.req_msg = (uint8_t *)vc_config; 914 args.req_msglen = size; 915 916 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 917 if (err) 918 PMD_DRV_LOG(ERR, "Failed to execute command of" 919 " VIRTCHNL_OP_CONFIG_VSI_QUEUES"); 920 921 rte_free(vc_config); 922 return err; 923 } 924 925 int 926 ice_dcf_config_irq_map(struct ice_dcf_hw *hw) 927 { 928 struct virtchnl_irq_map_info *map_info; 929 struct virtchnl_vector_map *vecmap; 930 struct dcf_virtchnl_cmd args; 931 int len, i, err; 932 933 len = sizeof(struct virtchnl_irq_map_info) + 934 sizeof(struct virtchnl_vector_map) * hw->nb_msix; 935 936 map_info = rte_zmalloc("map_info", len, 0); 937 if (!map_info) 938 return -ENOMEM; 939 940 map_info->num_vectors = hw->nb_msix; 941 for (i = 0; i < hw->nb_msix; i++) { 942 vecmap = &map_info->vecmap[i]; 943 vecmap->vsi_id = hw->vsi_res->vsi_id; 944 vecmap->rxitr_idx = 0; 945 vecmap->vector_id = hw->msix_base + i; 946 vecmap->txq_map = 0; 947 vecmap->rxq_map = hw->rxq_map[hw->msix_base + i]; 948 } 949 950 memset(&args, 0, sizeof(args)); 951 args.v_op = VIRTCHNL_OP_CONFIG_IRQ_MAP; 952 args.req_msg = (u8 *)map_info; 953 args.req_msglen = len; 954 955 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 956 if (err) 957 PMD_DRV_LOG(ERR, "fail to execute command OP_CONFIG_IRQ_MAP"); 958 959 rte_free(map_info); 960 return err; 961 } 962 963 int 964 ice_dcf_switch_queue(struct ice_dcf_hw *hw, uint16_t qid, bool rx, bool on) 965 { 966 struct virtchnl_queue_select queue_select; 967 struct dcf_virtchnl_cmd args; 968 int err; 969 970 memset(&queue_select, 0, sizeof(queue_select)); 971 queue_select.vsi_id = hw->vsi_res->vsi_id; 972 if (rx) 973 queue_select.rx_queues |= 1 << qid; 974 else 975 queue_select.tx_queues |= 1 << qid; 976 977 memset(&args, 0, sizeof(args)); 978 if (on) 979 args.v_op = VIRTCHNL_OP_ENABLE_QUEUES; 980 else 981 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 982 983 args.req_msg = (u8 *)&queue_select; 984 args.req_msglen = sizeof(queue_select); 985 986 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 987 if (err) 988 PMD_DRV_LOG(ERR, "Failed to execute command of %s", 989 on ? "OP_ENABLE_QUEUES" : "OP_DISABLE_QUEUES"); 990 991 return err; 992 } 993 994 int 995 ice_dcf_disable_queues(struct ice_dcf_hw *hw) 996 { 997 struct virtchnl_queue_select queue_select; 998 struct dcf_virtchnl_cmd args; 999 int err; 1000 1001 memset(&queue_select, 0, sizeof(queue_select)); 1002 queue_select.vsi_id = hw->vsi_res->vsi_id; 1003 1004 queue_select.rx_queues = BIT(hw->eth_dev->data->nb_rx_queues) - 1; 1005 queue_select.tx_queues = BIT(hw->eth_dev->data->nb_tx_queues) - 1; 1006 1007 memset(&args, 0, sizeof(args)); 1008 args.v_op = VIRTCHNL_OP_DISABLE_QUEUES; 1009 args.req_msg = (u8 *)&queue_select; 1010 args.req_msglen = sizeof(queue_select); 1011 1012 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1013 if (err) 1014 PMD_DRV_LOG(ERR, 1015 "Failed to execute command of OP_DISABLE_QUEUES"); 1016 1017 return err; 1018 } 1019 1020 int 1021 ice_dcf_query_stats(struct ice_dcf_hw *hw, 1022 struct virtchnl_eth_stats *pstats) 1023 { 1024 struct virtchnl_queue_select q_stats; 1025 struct dcf_virtchnl_cmd args; 1026 int err; 1027 1028 memset(&q_stats, 0, sizeof(q_stats)); 1029 q_stats.vsi_id = hw->vsi_res->vsi_id; 1030 1031 args.v_op = VIRTCHNL_OP_GET_STATS; 1032 args.req_msg = (uint8_t *)&q_stats; 1033 args.req_msglen = sizeof(q_stats); 1034 args.rsp_msglen = sizeof(*pstats); 1035 args.rsp_msgbuf = (uint8_t *)pstats; 1036 args.rsp_buflen = sizeof(*pstats); 1037 1038 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1039 if (err) { 1040 PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS"); 1041 return err; 1042 } 1043 1044 return 0; 1045 } 1046 1047 int 1048 ice_dcf_add_del_all_mac_addr(struct ice_dcf_hw *hw, bool add) 1049 { 1050 struct virtchnl_ether_addr_list *list; 1051 struct rte_ether_addr *addr; 1052 struct dcf_virtchnl_cmd args; 1053 int len, err = 0; 1054 1055 len = sizeof(struct virtchnl_ether_addr_list); 1056 addr = hw->eth_dev->data->mac_addrs; 1057 len += sizeof(struct virtchnl_ether_addr); 1058 1059 list = rte_zmalloc(NULL, len, 0); 1060 if (!list) { 1061 PMD_DRV_LOG(ERR, "fail to allocate memory"); 1062 return -ENOMEM; 1063 } 1064 1065 rte_memcpy(list->list[0].addr, addr->addr_bytes, 1066 sizeof(addr->addr_bytes)); 1067 PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x", 1068 addr->addr_bytes[0], addr->addr_bytes[1], 1069 addr->addr_bytes[2], addr->addr_bytes[3], 1070 addr->addr_bytes[4], addr->addr_bytes[5]); 1071 1072 list->vsi_id = hw->vsi_res->vsi_id; 1073 list->num_elements = 1; 1074 1075 memset(&args, 0, sizeof(args)); 1076 args.v_op = add ? VIRTCHNL_OP_ADD_ETH_ADDR : 1077 VIRTCHNL_OP_DEL_ETH_ADDR; 1078 args.req_msg = (uint8_t *)list; 1079 args.req_msglen = len; 1080 err = ice_dcf_execute_virtchnl_cmd(hw, &args); 1081 if (err) 1082 PMD_DRV_LOG(ERR, "fail to execute command %s", 1083 add ? "OP_ADD_ETHER_ADDRESS" : 1084 "OP_DEL_ETHER_ADDRESS"); 1085 rte_free(list); 1086 return err; 1087 } 1088