1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include "roc_api.h" 6 #include "roc_priv.h" 7 8 #define CPT_IQ_FC_LEN 128 9 #define CPT_IQ_GRP_LEN 16 10 11 #define CPT_IQ_NB_DESC_MULTIPLIER 40 12 13 /* The effective queue size to software is (CPT_LF_Q_SIZE[SIZE_DIV40] - 1 - 8). 14 * 15 * CPT requires 320 free entries (+8). And 40 entries are required for 16 * allowing CPT to discard packet when the queues are full (+1). 17 */ 18 #define CPT_IQ_NB_DESC_SIZE_DIV40(nb_desc) \ 19 (PLT_DIV_CEIL(nb_desc, CPT_IQ_NB_DESC_MULTIPLIER) + 1 + 8) 20 21 #define CPT_IQ_GRP_SIZE(nb_desc) \ 22 (CPT_IQ_NB_DESC_SIZE_DIV40(nb_desc) * CPT_IQ_GRP_LEN) 23 24 #define CPT_LF_MAX_NB_DESC 128000 25 #define CPT_LF_DEFAULT_NB_DESC 1024 26 27 static void 28 cpt_lf_misc_intr_enb_dis(struct roc_cpt_lf *lf, bool enb) 29 { 30 /* Enable all cpt lf error irqs except RQ_DISABLED and CQ_DISABLED */ 31 if (enb) 32 plt_write64((BIT_ULL(6) | BIT_ULL(5) | BIT_ULL(3) | BIT_ULL(2) | 33 BIT_ULL(1)), 34 lf->rbase + CPT_LF_MISC_INT_ENA_W1S); 35 else 36 plt_write64((BIT_ULL(6) | BIT_ULL(5) | BIT_ULL(3) | BIT_ULL(2) | 37 BIT_ULL(1)), 38 lf->rbase + CPT_LF_MISC_INT_ENA_W1C); 39 } 40 41 static void 42 cpt_lf_misc_irq(void *param) 43 { 44 struct roc_cpt_lf *lf = (struct roc_cpt_lf *)param; 45 struct dev *dev = lf->dev; 46 uint64_t intr; 47 48 intr = plt_read64(lf->rbase + CPT_LF_MISC_INT); 49 if (intr == 0) 50 return; 51 52 plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf); 53 54 /* Dump lf registers */ 55 cpt_lf_print(lf); 56 57 /* Clear interrupt */ 58 plt_write64(intr, lf->rbase + CPT_LF_MISC_INT); 59 } 60 61 static int 62 cpt_lf_register_misc_irq(struct roc_cpt_lf *lf) 63 { 64 struct plt_pci_device *pci_dev = lf->pci_dev; 65 struct plt_intr_handle *handle; 66 int rc, vec; 67 68 handle = pci_dev->intr_handle; 69 70 vec = lf->msixoff + CPT_LF_INT_VEC_MISC; 71 /* Clear err interrupt */ 72 cpt_lf_misc_intr_enb_dis(lf, false); 73 /* Set used interrupt vectors */ 74 rc = dev_irq_register(handle, cpt_lf_misc_irq, lf, vec); 75 /* Enable all dev interrupt except for RQ_DISABLED */ 76 cpt_lf_misc_intr_enb_dis(lf, true); 77 78 return rc; 79 } 80 81 static void 82 cpt_lf_unregister_misc_irq(struct roc_cpt_lf *lf) 83 { 84 struct plt_pci_device *pci_dev = lf->pci_dev; 85 struct plt_intr_handle *handle; 86 int vec; 87 88 handle = pci_dev->intr_handle; 89 90 vec = lf->msixoff + CPT_LF_INT_VEC_MISC; 91 /* Clear err interrupt */ 92 cpt_lf_misc_intr_enb_dis(lf, false); 93 dev_irq_unregister(handle, cpt_lf_misc_irq, lf, vec); 94 } 95 96 static void 97 cpt_lf_done_intr_enb_dis(struct roc_cpt_lf *lf, bool enb) 98 { 99 if (enb) 100 plt_write64(0x1, lf->rbase + CPT_LF_DONE_INT_ENA_W1S); 101 else 102 plt_write64(0x1, lf->rbase + CPT_LF_DONE_INT_ENA_W1C); 103 } 104 105 static void 106 cpt_lf_done_irq(void *param) 107 { 108 struct roc_cpt_lf *lf = param; 109 uint64_t done_wait; 110 uint64_t intr; 111 112 /* Read the number of completed requests */ 113 intr = plt_read64(lf->rbase + CPT_LF_DONE); 114 if (intr == 0) 115 return; 116 117 done_wait = plt_read64(lf->rbase + CPT_LF_DONE_WAIT); 118 119 /* Acknowledge the number of completed requests */ 120 plt_write64(intr, lf->rbase + CPT_LF_DONE_ACK); 121 122 plt_write64(done_wait, lf->rbase + CPT_LF_DONE_WAIT); 123 } 124 125 static int 126 cpt_lf_register_done_irq(struct roc_cpt_lf *lf) 127 { 128 struct plt_pci_device *pci_dev = lf->pci_dev; 129 struct plt_intr_handle *handle; 130 int rc, vec; 131 132 handle = pci_dev->intr_handle; 133 134 vec = lf->msixoff + CPT_LF_INT_VEC_DONE; 135 136 /* Clear done interrupt */ 137 cpt_lf_done_intr_enb_dis(lf, false); 138 139 /* Set used interrupt vectors */ 140 rc = dev_irq_register(handle, cpt_lf_done_irq, lf, vec); 141 142 /* Enable done interrupt */ 143 cpt_lf_done_intr_enb_dis(lf, true); 144 145 return rc; 146 } 147 148 static void 149 cpt_lf_unregister_done_irq(struct roc_cpt_lf *lf) 150 { 151 struct plt_pci_device *pci_dev = lf->pci_dev; 152 struct plt_intr_handle *handle; 153 int vec; 154 155 handle = pci_dev->intr_handle; 156 157 vec = lf->msixoff + CPT_LF_INT_VEC_DONE; 158 159 /* Clear done interrupt */ 160 cpt_lf_done_intr_enb_dis(lf, false); 161 dev_irq_unregister(handle, cpt_lf_done_irq, lf, vec); 162 } 163 164 static int 165 cpt_lf_register_irqs(struct roc_cpt_lf *lf) 166 { 167 int rc; 168 169 if (lf->msixoff == MSIX_VECTOR_INVALID) { 170 plt_err("Invalid CPTLF MSIX vector offset vector: 0x%x", 171 lf->msixoff); 172 return -EINVAL; 173 } 174 175 /* Register lf err interrupt */ 176 rc = cpt_lf_register_misc_irq(lf); 177 if (rc) 178 plt_err("Error registering IRQs"); 179 180 rc = cpt_lf_register_done_irq(lf); 181 if (rc) 182 plt_err("Error registering IRQs"); 183 184 return rc; 185 } 186 187 static void 188 cpt_lf_unregister_irqs(struct roc_cpt_lf *lf) 189 { 190 cpt_lf_unregister_misc_irq(lf); 191 cpt_lf_unregister_done_irq(lf); 192 } 193 194 static void 195 cpt_lf_dump(struct roc_cpt_lf *lf) 196 { 197 plt_cpt_dbg("CPT LF"); 198 plt_cpt_dbg("RBASE: 0x%016" PRIx64, lf->rbase); 199 plt_cpt_dbg("LMT_BASE: 0x%016" PRIx64, lf->lmt_base); 200 plt_cpt_dbg("MSIXOFF: 0x%x", lf->msixoff); 201 plt_cpt_dbg("LF_ID: 0x%x", lf->lf_id); 202 plt_cpt_dbg("NB DESC: %d", lf->nb_desc); 203 plt_cpt_dbg("FC_ADDR: 0x%016" PRIx64, (uintptr_t)lf->fc_addr); 204 plt_cpt_dbg("CQ.VADDR: 0x%016" PRIx64, (uintptr_t)lf->iq_vaddr); 205 206 plt_cpt_dbg("CPT LF REG:"); 207 plt_cpt_dbg("LF_CTL[0x%016llx]: 0x%016" PRIx64, CPT_LF_CTL, 208 plt_read64(lf->rbase + CPT_LF_CTL)); 209 plt_cpt_dbg("LF_INPROG[0x%016llx]: 0x%016" PRIx64, CPT_LF_INPROG, 210 plt_read64(lf->rbase + CPT_LF_INPROG)); 211 212 plt_cpt_dbg("Q_BASE[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_BASE, 213 plt_read64(lf->rbase + CPT_LF_Q_BASE)); 214 plt_cpt_dbg("Q_SIZE[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_SIZE, 215 plt_read64(lf->rbase + CPT_LF_Q_SIZE)); 216 plt_cpt_dbg("Q_INST_PTR[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_INST_PTR, 217 plt_read64(lf->rbase + CPT_LF_Q_INST_PTR)); 218 plt_cpt_dbg("Q_GRP_PTR[0x%016llx]: 0x%016" PRIx64, CPT_LF_Q_GRP_PTR, 219 plt_read64(lf->rbase + CPT_LF_Q_GRP_PTR)); 220 } 221 222 int 223 cpt_lf_outb_cfg(struct dev *dev, uint16_t sso_pf_func, uint16_t nix_pf_func, 224 uint8_t lf_id, bool ena) 225 { 226 struct cpt_inline_ipsec_cfg_msg *req; 227 struct mbox *mbox = dev->mbox; 228 229 req = mbox_alloc_msg_cpt_inline_ipsec_cfg(mbox); 230 if (req == NULL) 231 return -ENOSPC; 232 233 req->dir = CPT_INLINE_OUTBOUND; 234 req->slot = lf_id; 235 if (ena) { 236 req->enable = 1; 237 req->sso_pf_func = sso_pf_func; 238 req->nix_pf_func = nix_pf_func; 239 } else { 240 req->enable = 0; 241 } 242 243 return mbox_process(mbox); 244 } 245 246 int 247 roc_cpt_inline_ipsec_cfg(struct dev *cpt_dev, uint8_t lf_id, 248 struct roc_nix *roc_nix) 249 { 250 bool ena = roc_nix ? true : false; 251 uint16_t nix_pf_func = 0; 252 uint16_t sso_pf_func = 0; 253 254 if (ena) { 255 nix_pf_func = roc_nix_get_pf_func(roc_nix); 256 sso_pf_func = idev_sso_pffunc_get(); 257 } 258 259 return cpt_lf_outb_cfg(cpt_dev, sso_pf_func, nix_pf_func, lf_id, ena); 260 } 261 262 int 263 roc_cpt_inline_ipsec_inb_cfg(struct roc_cpt *roc_cpt, uint16_t param1, 264 uint16_t param2) 265 { 266 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt); 267 struct cpt_rx_inline_lf_cfg_msg *req; 268 struct mbox *mbox; 269 270 mbox = cpt->dev.mbox; 271 272 req = mbox_alloc_msg_cpt_rx_inline_lf_cfg(mbox); 273 if (req == NULL) 274 return -ENOSPC; 275 276 req->sso_pf_func = idev_sso_pffunc_get(); 277 req->param1 = param1; 278 req->param2 = param2; 279 280 return mbox_process(mbox); 281 } 282 283 int 284 roc_cpt_rxc_time_cfg(struct roc_cpt *roc_cpt, struct roc_cpt_rxc_time_cfg *cfg) 285 { 286 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt); 287 struct cpt_rxc_time_cfg_req *req; 288 struct dev *dev = &cpt->dev; 289 290 req = mbox_alloc_msg_cpt_rxc_time_cfg(dev->mbox); 291 if (req == NULL) 292 return -ENOSPC; 293 294 req->blkaddr = 0; 295 296 /* The step value is in microseconds. */ 297 req->step = cfg->step; 298 299 /* The timeout will be: limit * step microseconds */ 300 req->zombie_limit = cfg->zombie_limit; 301 req->zombie_thres = cfg->zombie_thres; 302 303 /* The timeout will be: limit * step microseconds */ 304 req->active_limit = cfg->active_limit; 305 req->active_thres = cfg->active_thres; 306 307 return mbox_process(dev->mbox); 308 } 309 310 int 311 cpt_get_msix_offset(struct dev *dev, struct msix_offset_rsp **msix_rsp) 312 { 313 struct mbox *mbox = dev->mbox; 314 int rc; 315 316 /* Get MSIX vector offsets */ 317 mbox_alloc_msg_msix_offset(mbox); 318 rc = mbox_process_msg(mbox, (void *)msix_rsp); 319 320 return rc; 321 } 322 323 int 324 cpt_lfs_attach(struct dev *dev, uint8_t blkaddr, bool modify, uint16_t nb_lf) 325 { 326 struct mbox *mbox = dev->mbox; 327 struct rsrc_attach_req *req; 328 329 if (blkaddr != RVU_BLOCK_ADDR_CPT0 && blkaddr != RVU_BLOCK_ADDR_CPT1) 330 return -EINVAL; 331 332 /* Attach CPT(lf) */ 333 req = mbox_alloc_msg_attach_resources(mbox); 334 if (req == NULL) 335 return -ENOSPC; 336 337 req->cptlfs = nb_lf; 338 req->modify = modify; 339 req->cpt_blkaddr = blkaddr; 340 341 return mbox_process(mbox); 342 } 343 344 int 345 cpt_lfs_detach(struct dev *dev) 346 { 347 struct mbox *mbox = dev->mbox; 348 struct rsrc_detach_req *req; 349 350 req = mbox_alloc_msg_detach_resources(mbox); 351 if (req == NULL) 352 return -ENOSPC; 353 354 req->cptlfs = 1; 355 req->partial = 1; 356 357 return mbox_process(mbox); 358 } 359 360 static int 361 cpt_available_lfs_get(struct dev *dev, uint16_t *nb_lf) 362 { 363 struct mbox *mbox = dev->mbox; 364 struct free_rsrcs_rsp *rsp; 365 int rc; 366 367 mbox_alloc_msg_free_rsrc_cnt(mbox); 368 369 rc = mbox_process_msg(mbox, (void *)&rsp); 370 if (rc) 371 return -EIO; 372 373 *nb_lf = PLT_MAX((uint16_t)rsp->cpt, (uint16_t)rsp->cpt1); 374 return 0; 375 } 376 377 int 378 cpt_lfs_alloc(struct dev *dev, uint8_t eng_grpmsk, uint8_t blkaddr, 379 bool inl_dev_sso) 380 { 381 struct cpt_lf_alloc_req_msg *req; 382 struct mbox *mbox = dev->mbox; 383 384 if (blkaddr != RVU_BLOCK_ADDR_CPT0 && blkaddr != RVU_BLOCK_ADDR_CPT1) 385 return -EINVAL; 386 387 req = mbox_alloc_msg_cpt_lf_alloc(mbox); 388 if (!req) 389 return -ENOSPC; 390 391 req->nix_pf_func = 0; 392 if (inl_dev_sso && nix_inl_dev_pffunc_get()) 393 req->sso_pf_func = nix_inl_dev_pffunc_get(); 394 else 395 req->sso_pf_func = idev_sso_pffunc_get(); 396 req->eng_grpmsk = eng_grpmsk; 397 req->blkaddr = blkaddr; 398 399 return mbox_process(mbox); 400 } 401 402 int 403 cpt_lfs_free(struct dev *dev) 404 { 405 mbox_alloc_msg_cpt_lf_free(dev->mbox); 406 407 return mbox_process(dev->mbox); 408 } 409 410 static int 411 cpt_hardware_caps_get(struct dev *dev, struct roc_cpt *roc_cpt) 412 { 413 struct cpt_caps_rsp_msg *rsp; 414 int ret; 415 416 mbox_alloc_msg_cpt_caps_get(dev->mbox); 417 418 ret = mbox_process_msg(dev->mbox, (void *)&rsp); 419 if (ret) 420 return -EIO; 421 422 roc_cpt->cpt_revision = rsp->cpt_revision; 423 mbox_memcpy(roc_cpt->hw_caps, rsp->eng_caps, 424 sizeof(union cpt_eng_caps) * CPT_MAX_ENG_TYPES); 425 426 return 0; 427 } 428 429 static uint32_t 430 cpt_lf_iq_mem_calc(uint32_t nb_desc) 431 { 432 uint32_t len; 433 434 /* Space for instruction group memory */ 435 len = CPT_IQ_GRP_SIZE(nb_desc); 436 437 /* Align to 128B */ 438 len = PLT_ALIGN(len, ROC_ALIGN); 439 440 /* Space for FC */ 441 len += CPT_IQ_FC_LEN; 442 443 /* For instruction queues */ 444 len += PLT_ALIGN(CPT_IQ_NB_DESC_SIZE_DIV40(nb_desc) * 445 CPT_IQ_NB_DESC_MULTIPLIER * 446 sizeof(struct cpt_inst_s), 447 ROC_ALIGN); 448 449 return len; 450 } 451 452 static inline void 453 cpt_iq_init(struct roc_cpt_lf *lf) 454 { 455 union cpt_lf_q_size lf_q_size = {.u = 0x0}; 456 union cpt_lf_q_base lf_q_base = {.u = 0x0}; 457 uintptr_t addr; 458 459 lf->io_addr = lf->rbase + CPT_LF_NQX(0); 460 461 /* Disable command queue */ 462 roc_cpt_iq_disable(lf); 463 464 /* Set command queue base address */ 465 addr = (uintptr_t)lf->iq_vaddr + 466 PLT_ALIGN(CPT_IQ_GRP_SIZE(lf->nb_desc), ROC_ALIGN); 467 468 lf_q_base.u = addr; 469 470 plt_write64(lf_q_base.u, lf->rbase + CPT_LF_Q_BASE); 471 472 /* Set command queue size */ 473 lf_q_size.s.size_div40 = CPT_IQ_NB_DESC_SIZE_DIV40(lf->nb_desc); 474 plt_write64(lf_q_size.u, lf->rbase + CPT_LF_Q_SIZE); 475 476 lf->fc_addr = (uint64_t *)addr; 477 lf->fc_hyst_bits = plt_log2_u32(lf->nb_desc) / 2; 478 lf->fc_thresh = lf->nb_desc - (lf->nb_desc % (1 << lf->fc_hyst_bits)); 479 } 480 481 int 482 roc_cpt_dev_configure(struct roc_cpt *roc_cpt, int nb_lf) 483 { 484 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt); 485 uint8_t blkaddr[ROC_CPT_MAX_BLKS]; 486 struct msix_offset_rsp *rsp; 487 uint8_t eng_grpmsk; 488 int blknum = 0; 489 int rc, i; 490 491 blkaddr[0] = RVU_BLOCK_ADDR_CPT0; 492 blkaddr[1] = RVU_BLOCK_ADDR_CPT1; 493 494 if ((roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_98XX) && 495 (cpt->dev.pf_func & 0x1)) 496 blknum = (blknum + 1) % ROC_CPT_MAX_BLKS; 497 498 /* Request LF resources */ 499 rc = cpt_lfs_attach(&cpt->dev, blkaddr[blknum], true, nb_lf); 500 501 /* Request LFs from another block if current block has less LFs */ 502 if (roc_cpt->cpt_revision == ROC_CPT_REVISION_ID_98XX && rc == ENOSPC) { 503 blknum = (blknum + 1) % ROC_CPT_MAX_BLKS; 504 rc = cpt_lfs_attach(&cpt->dev, blkaddr[blknum], true, nb_lf); 505 } 506 if (rc) { 507 plt_err("Could not attach LFs"); 508 return rc; 509 } 510 511 for (i = 0; i < nb_lf; i++) 512 cpt->lf_blkaddr[i] = blkaddr[blknum]; 513 514 eng_grpmsk = (1 << roc_cpt->eng_grp[CPT_ENG_TYPE_AE]) | 515 (1 << roc_cpt->eng_grp[CPT_ENG_TYPE_SE]) | 516 (1 << roc_cpt->eng_grp[CPT_ENG_TYPE_IE]); 517 518 rc = cpt_lfs_alloc(&cpt->dev, eng_grpmsk, blkaddr[blknum], false); 519 if (rc) 520 goto lfs_detach; 521 522 rc = cpt_get_msix_offset(&cpt->dev, &rsp); 523 if (rc) 524 goto lfs_free; 525 526 for (i = 0; i < nb_lf; i++) 527 cpt->lf_msix_off[i] = 528 (cpt->lf_blkaddr[i] == RVU_BLOCK_ADDR_CPT1) ? 529 rsp->cpt1_lf_msixoff[i] : 530 rsp->cptlf_msixoff[i]; 531 532 roc_cpt->nb_lf = nb_lf; 533 534 return 0; 535 536 lfs_free: 537 cpt_lfs_free(&cpt->dev); 538 lfs_detach: 539 cpt_lfs_detach(&cpt->dev); 540 return rc; 541 } 542 543 uint64_t 544 cpt_get_blkaddr(struct dev *dev) 545 { 546 uint64_t reg; 547 uint64_t off; 548 549 /* Reading the discovery register to know which CPT is the LF 550 * attached to. Assume CPT LF's of only one block are attached 551 * to a pffunc. 552 */ 553 if (dev_is_vf(dev)) 554 off = RVU_VF_BLOCK_ADDRX_DISC(RVU_BLOCK_ADDR_CPT1); 555 else 556 off = RVU_PF_BLOCK_ADDRX_DISC(RVU_BLOCK_ADDR_CPT1); 557 558 reg = plt_read64(dev->bar2 + off); 559 560 return reg & 0x1FFULL ? RVU_BLOCK_ADDR_CPT1 : RVU_BLOCK_ADDR_CPT0; 561 } 562 563 int 564 cpt_lf_init(struct roc_cpt_lf *lf) 565 { 566 struct dev *dev = lf->dev; 567 uint64_t blkaddr; 568 void *iq_mem; 569 int rc; 570 571 if (lf->nb_desc == 0 || lf->nb_desc > CPT_LF_MAX_NB_DESC) 572 lf->nb_desc = CPT_LF_DEFAULT_NB_DESC; 573 574 /* Allocate memory for instruction queue for CPT LF. */ 575 iq_mem = plt_zmalloc(cpt_lf_iq_mem_calc(lf->nb_desc), ROC_ALIGN); 576 if (iq_mem == NULL) 577 return -ENOMEM; 578 plt_atomic_thread_fence(__ATOMIC_ACQ_REL); 579 580 blkaddr = cpt_get_blkaddr(dev); 581 lf->rbase = dev->bar2 + ((blkaddr << 20) | (lf->lf_id << 12)); 582 lf->iq_vaddr = iq_mem; 583 lf->lmt_base = dev->lmt_base; 584 lf->pf_func = dev->pf_func; 585 586 /* Initialize instruction queue */ 587 cpt_iq_init(lf); 588 589 rc = cpt_lf_register_irqs(lf); 590 if (rc) 591 goto disable_iq; 592 593 return 0; 594 595 disable_iq: 596 roc_cpt_iq_disable(lf); 597 plt_free(iq_mem); 598 return rc; 599 } 600 601 int 602 roc_cpt_lf_init(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf) 603 { 604 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt); 605 int rc; 606 607 lf->dev = &cpt->dev; 608 lf->roc_cpt = roc_cpt; 609 lf->msixoff = cpt->lf_msix_off[lf->lf_id]; 610 lf->pci_dev = cpt->pci_dev; 611 612 rc = cpt_lf_init(lf); 613 if (rc) 614 return rc; 615 616 /* LF init successful */ 617 roc_cpt->lf[lf->lf_id] = lf; 618 return rc; 619 } 620 621 int 622 roc_cpt_dev_init(struct roc_cpt *roc_cpt) 623 { 624 struct plt_pci_device *pci_dev; 625 uint16_t nb_lf_avail; 626 struct dev *dev; 627 struct cpt *cpt; 628 int rc; 629 630 if (roc_cpt == NULL || roc_cpt->pci_dev == NULL) 631 return -EINVAL; 632 633 PLT_STATIC_ASSERT(sizeof(struct cpt) <= ROC_CPT_MEM_SZ); 634 635 cpt = roc_cpt_to_cpt_priv(roc_cpt); 636 memset(cpt, 0, sizeof(*cpt)); 637 pci_dev = roc_cpt->pci_dev; 638 dev = &cpt->dev; 639 640 /* Initialize device */ 641 rc = dev_init(dev, pci_dev); 642 if (rc) { 643 plt_err("Failed to init roc device"); 644 goto fail; 645 } 646 647 cpt->pci_dev = pci_dev; 648 roc_cpt->lmt_base = dev->lmt_base; 649 650 rc = cpt_hardware_caps_get(dev, roc_cpt); 651 if (rc) { 652 plt_err("Could not determine hardware capabilities"); 653 goto fail; 654 } 655 656 rc = cpt_available_lfs_get(&cpt->dev, &nb_lf_avail); 657 if (rc) { 658 plt_err("Could not get available lfs"); 659 goto fail; 660 } 661 662 /* Reserve 1 CPT LF for inline inbound */ 663 nb_lf_avail = PLT_MIN(nb_lf_avail, (uint16_t)(ROC_CPT_MAX_LFS - 1)); 664 665 roc_cpt->nb_lf_avail = nb_lf_avail; 666 667 dev->roc_cpt = roc_cpt; 668 669 /* Set it to idev if not already present */ 670 if (!roc_idev_cpt_get()) 671 roc_idev_cpt_set(roc_cpt); 672 673 return 0; 674 675 fail: 676 return rc; 677 } 678 679 int 680 roc_cpt_lf_ctx_flush(struct roc_cpt_lf *lf, void *cptr, bool inval) 681 { 682 union cpt_lf_ctx_flush reg; 683 684 if (lf == NULL) { 685 plt_err("Could not trigger CTX flush"); 686 return -ENOTSUP; 687 } 688 689 reg.u = 0; 690 reg.s.inval = inval; 691 reg.s.cptr = (uintptr_t)cptr >> 7; 692 693 plt_write64(reg.u, lf->rbase + CPT_LF_CTX_FLUSH); 694 695 return 0; 696 } 697 698 int 699 roc_cpt_lf_ctx_reload(struct roc_cpt_lf *lf, void *cptr) 700 { 701 union cpt_lf_ctx_reload reg; 702 703 if (lf == NULL) { 704 plt_err("Could not trigger CTX reload"); 705 return -ENOTSUP; 706 } 707 708 reg.u = 0; 709 reg.s.cptr = (uintptr_t)cptr >> 7; 710 711 plt_write64(reg.u, lf->rbase + CPT_LF_CTX_RELOAD); 712 713 return 0; 714 } 715 716 void 717 cpt_lf_fini(struct roc_cpt_lf *lf) 718 { 719 /* Unregister IRQ's */ 720 cpt_lf_unregister_irqs(lf); 721 722 /* Disable IQ */ 723 roc_cpt_iq_disable(lf); 724 725 /* Free memory */ 726 plt_free(lf->iq_vaddr); 727 lf->iq_vaddr = NULL; 728 } 729 730 void 731 roc_cpt_lf_fini(struct roc_cpt_lf *lf) 732 { 733 if (lf == NULL) 734 return; 735 lf->roc_cpt->lf[lf->lf_id] = NULL; 736 cpt_lf_fini(lf); 737 } 738 739 int 740 roc_cpt_dev_fini(struct roc_cpt *roc_cpt) 741 { 742 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt); 743 744 if (cpt == NULL) 745 return -EINVAL; 746 747 /* Remove idev references */ 748 if (roc_idev_cpt_get() == roc_cpt) 749 roc_idev_cpt_set(NULL); 750 751 roc_cpt->nb_lf_avail = 0; 752 753 roc_cpt->lmt_base = 0; 754 755 return dev_fini(&cpt->dev, cpt->pci_dev); 756 } 757 758 void 759 roc_cpt_dev_clear(struct roc_cpt *roc_cpt) 760 { 761 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt); 762 int i; 763 764 if (cpt == NULL) 765 return; 766 767 for (i = 0; i < roc_cpt->nb_lf; i++) 768 cpt->lf_msix_off[i] = 0; 769 770 roc_cpt->nb_lf = 0; 771 772 cpt_lfs_free(&cpt->dev); 773 774 cpt_lfs_detach(&cpt->dev); 775 } 776 777 int 778 roc_cpt_eng_grp_add(struct roc_cpt *roc_cpt, enum cpt_eng_type eng_type) 779 { 780 struct cpt *cpt = roc_cpt_to_cpt_priv(roc_cpt); 781 struct dev *dev = &cpt->dev; 782 struct cpt_eng_grp_req *req; 783 struct cpt_eng_grp_rsp *rsp; 784 int ret; 785 786 req = mbox_alloc_msg_cpt_eng_grp_get(dev->mbox); 787 if (req == NULL) 788 return -EIO; 789 790 switch (eng_type) { 791 case CPT_ENG_TYPE_AE: 792 case CPT_ENG_TYPE_SE: 793 case CPT_ENG_TYPE_IE: 794 break; 795 default: 796 return -EINVAL; 797 } 798 799 req->eng_type = eng_type; 800 ret = mbox_process_msg(dev->mbox, (void *)&rsp); 801 if (ret) 802 return -EIO; 803 804 if (rsp->eng_grp_num > 8) { 805 plt_err("Invalid CPT engine group"); 806 return -ENOTSUP; 807 } 808 809 roc_cpt->eng_grp[eng_type] = rsp->eng_grp_num; 810 811 return rsp->eng_grp_num; 812 } 813 814 void 815 roc_cpt_iq_disable(struct roc_cpt_lf *lf) 816 { 817 volatile union cpt_lf_q_grp_ptr grp_ptr = {.u = 0x0}; 818 volatile union cpt_lf_inprog lf_inprog = {.u = 0x0}; 819 union cpt_lf_ctl lf_ctl = {.u = 0x0}; 820 int timeout = 20; 821 int cnt; 822 823 /* Disable instructions enqueuing */ 824 plt_write64(lf_ctl.u, lf->rbase + CPT_LF_CTL); 825 826 /* Wait for instruction queue to become empty */ 827 do { 828 lf_inprog.u = plt_read64(lf->rbase + CPT_LF_INPROG); 829 if (!lf_inprog.s.inflight) 830 break; 831 832 plt_delay_ms(20); 833 if (timeout-- < 0) { 834 plt_err("CPT LF %d is still busy", lf->lf_id); 835 break; 836 } 837 838 } while (1); 839 840 /* Disable executions in the LF's queue. 841 * The queue should be empty at this point 842 */ 843 lf_inprog.s.eena = 0x0; 844 plt_write64(lf_inprog.u, lf->rbase + CPT_LF_INPROG); 845 846 /* Wait for instruction queue to become empty */ 847 cnt = 0; 848 do { 849 lf_inprog.u = plt_read64(lf->rbase + CPT_LF_INPROG); 850 if (lf_inprog.s.grb_partial) 851 cnt = 0; 852 else 853 cnt++; 854 grp_ptr.u = plt_read64(lf->rbase + CPT_LF_Q_GRP_PTR); 855 } while ((cnt < 10) && (grp_ptr.s.nq_ptr != grp_ptr.s.dq_ptr)); 856 857 cnt = 0; 858 do { 859 lf_inprog.u = plt_read64(lf->rbase + CPT_LF_INPROG); 860 if ((lf_inprog.s.inflight == 0) && (lf_inprog.s.gwb_cnt < 40) && 861 ((lf_inprog.s.grb_cnt == 0) || (lf_inprog.s.grb_cnt == 40))) 862 cnt++; 863 else 864 cnt = 0; 865 } while (cnt < 10); 866 } 867 868 void 869 roc_cpt_iq_enable(struct roc_cpt_lf *lf) 870 { 871 union cpt_lf_inprog lf_inprog; 872 union cpt_lf_ctl lf_ctl; 873 874 /* Disable command queue */ 875 roc_cpt_iq_disable(lf); 876 877 /* Enable instruction queue enqueuing */ 878 lf_ctl.u = plt_read64(lf->rbase + CPT_LF_CTL); 879 lf_ctl.s.ena = 1; 880 lf_ctl.s.fc_ena = 1; 881 lf_ctl.s.fc_up_crossing = 0; 882 lf_ctl.s.fc_hyst_bits = lf->fc_hyst_bits; 883 plt_write64(lf_ctl.u, lf->rbase + CPT_LF_CTL); 884 885 /* Enable command queue execution */ 886 lf_inprog.u = plt_read64(lf->rbase + CPT_LF_INPROG); 887 lf_inprog.s.eena = 1; 888 plt_write64(lf_inprog.u, lf->rbase + CPT_LF_INPROG); 889 890 cpt_lf_dump(lf); 891 } 892 893 int 894 roc_cpt_lmtline_init(struct roc_cpt *roc_cpt, struct roc_cpt_lmtline *lmtline, 895 int lf_id) 896 { 897 struct roc_cpt_lf *lf; 898 899 lf = roc_cpt->lf[lf_id]; 900 if (lf == NULL) 901 return -ENOTSUP; 902 903 lmtline->io_addr = lf->io_addr; 904 if (roc_model_is_cn10k()) 905 lmtline->io_addr |= ROC_CN10K_CPT_INST_DW_M1 << 4; 906 907 lmtline->fc_addr = lf->fc_addr; 908 lmtline->lmt_base = lf->lmt_base; 909 910 return 0; 911 } 912 913 int 914 roc_cpt_ctx_write(struct roc_cpt_lf *lf, void *sa_dptr, void *sa_cptr, 915 uint16_t sa_len) 916 { 917 uintptr_t lmt_base = lf->lmt_base; 918 uint64_t lmt_arg, io_addr; 919 struct cpt_inst_s *inst; 920 union cpt_res_s *res; 921 uint16_t lmt_id; 922 uint64_t *dptr; 923 int i; 924 925 ROC_LMT_CPT_BASE_ID_GET(lmt_base, lmt_id); 926 inst = (struct cpt_inst_s *)lmt_base; 927 928 memset(inst, 0, sizeof(struct cpt_inst_s)); 929 930 res = plt_zmalloc(sizeof(*res), ROC_CPT_RES_ALIGN); 931 if (res == NULL) { 932 plt_err("Couldn't allocate memory for result address"); 933 return -ENOMEM; 934 } 935 936 dptr = plt_zmalloc(sa_len, 8); 937 if (dptr == NULL) { 938 plt_err("Couldn't allocate memory for SA dptr"); 939 plt_free(res); 940 return -ENOMEM; 941 } 942 943 for (i = 0; i < (sa_len / 8); i++) 944 dptr[i] = plt_cpu_to_be_64(((uint64_t *)sa_dptr)[i]); 945 946 /* Fill CPT_INST_S for WRITE_SA microcode op */ 947 res->cn10k.compcode = CPT_COMP_NOT_DONE; 948 inst->res_addr = (uint64_t)res; 949 inst->dptr = (uint64_t)dptr; 950 inst->w4.s.param2 = sa_len >> 3; 951 inst->w4.s.dlen = sa_len; 952 inst->w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_WRITE_SA; 953 inst->w4.s.opcode_minor = ROC_IE_OT_MINOR_OP_WRITE_SA; 954 inst->w7.s.cptr = (uint64_t)sa_cptr; 955 inst->w7.s.ctx_val = 1; 956 inst->w7.s.egrp = ROC_CPT_DFLT_ENG_GRP_SE_IE; 957 958 lmt_arg = ROC_CN10K_CPT_LMT_ARG | (uint64_t)lmt_id; 959 io_addr = lf->io_addr | ROC_CN10K_CPT_INST_DW_M1 << 4; 960 961 roc_lmt_submit_steorl(lmt_arg, io_addr); 962 plt_wmb(); 963 964 /* Wait until CPT instruction completes */ 965 while (res->cn10k.compcode == CPT_COMP_NOT_DONE) 966 plt_delay_ms(1); 967 968 plt_free(res); 969 plt_free(dptr); 970 971 return 0; 972 } 973