1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 5 #include <fcntl.h> 6 #include <inttypes.h> 7 #include <string.h> 8 #include <sys/mman.h> 9 #include <unistd.h> 10 11 #include "roc_api.h" 12 #include "roc_priv.h" 13 14 /* PCI Extended capability ID */ 15 #define ROC_PCI_EXT_CAP_ID_SRIOV 0x10 /* SRIOV cap */ 16 17 /* Single Root I/O Virtualization */ 18 #define ROC_PCI_SRIOV_TOTAL_VF 0x0e /* Total VFs */ 19 20 static void * 21 mbox_mem_map(off_t off, size_t size) 22 { 23 void *va = MAP_FAILED; 24 int mem_fd; 25 26 if (size <= 0 || !off) { 27 plt_err("Invalid mbox area off 0x%lx size %lu", off, size); 28 goto error; 29 } 30 31 mem_fd = open("/dev/mem", O_RDWR); 32 if (mem_fd < 0) 33 goto error; 34 35 va = plt_mmap(NULL, size, PLT_PROT_READ | PLT_PROT_WRITE, 36 PLT_MAP_SHARED, mem_fd, off); 37 close(mem_fd); 38 39 if (va == MAP_FAILED) 40 plt_err("Failed to mmap sz=0x%zx, fd=%d, off=%jd", size, mem_fd, 41 (intmax_t)off); 42 error: 43 return va; 44 } 45 46 static void 47 mbox_mem_unmap(void *va, size_t size) 48 { 49 if (va) 50 munmap(va, size); 51 } 52 53 static int 54 pf_af_sync_msg(struct dev *dev, struct mbox_msghdr **rsp) 55 { 56 uint32_t timeout = 0, sleep = 1; 57 struct mbox *mbox = dev->mbox; 58 struct mbox_dev *mdev = &mbox->dev[0]; 59 60 volatile uint64_t int_status; 61 struct mbox_msghdr *msghdr; 62 uint64_t off; 63 int rc = 0; 64 65 /* We need to disable PF interrupts. We are in timer interrupt */ 66 plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C); 67 68 /* Send message */ 69 mbox_msg_send(mbox, 0); 70 71 do { 72 plt_delay_ms(sleep); 73 timeout += sleep; 74 if (timeout >= mbox->rsp_tmo) { 75 plt_err("Message timeout: %dms", mbox->rsp_tmo); 76 rc = -EIO; 77 break; 78 } 79 int_status = plt_read64(dev->bar2 + RVU_PF_INT); 80 } while ((int_status & 0x1) != 0x1); 81 82 /* Clear */ 83 plt_write64(int_status, dev->bar2 + RVU_PF_INT); 84 85 /* Enable interrupts */ 86 plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S); 87 88 if (rc == 0) { 89 /* Get message */ 90 off = mbox->rx_start + 91 PLT_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); 92 msghdr = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + off); 93 if (rsp) 94 *rsp = msghdr; 95 rc = msghdr->rc; 96 } 97 98 return rc; 99 } 100 101 static int 102 af_pf_wait_msg(struct dev *dev, uint16_t vf, int num_msg) 103 { 104 uint32_t timeout = 0, sleep = 1; 105 struct mbox *mbox = dev->mbox; 106 struct mbox_dev *mdev = &mbox->dev[0]; 107 volatile uint64_t int_status; 108 struct mbox_hdr *req_hdr; 109 struct mbox_msghdr *msg; 110 struct mbox_msghdr *rsp; 111 uint64_t offset; 112 size_t size; 113 int i; 114 115 /* We need to disable PF interrupts. We are in timer interrupt */ 116 plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C); 117 118 /* Send message */ 119 mbox_msg_send(mbox, 0); 120 121 do { 122 plt_delay_ms(sleep); 123 timeout++; 124 if (timeout >= mbox->rsp_tmo) { 125 plt_err("Routed messages %d timeout: %dms", num_msg, 126 mbox->rsp_tmo); 127 break; 128 } 129 int_status = plt_read64(dev->bar2 + RVU_PF_INT); 130 } while ((int_status & 0x1) != 0x1); 131 132 /* Clear */ 133 plt_write64(~0ull, dev->bar2 + RVU_PF_INT); 134 135 /* Enable interrupts */ 136 plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S); 137 138 plt_spinlock_lock(&mdev->mbox_lock); 139 140 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start); 141 if (req_hdr->num_msgs != num_msg) 142 plt_err("Routed messages: %d received: %d", num_msg, 143 req_hdr->num_msgs); 144 145 /* Get messages from mbox */ 146 offset = mbox->rx_start + 147 PLT_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); 148 for (i = 0; i < req_hdr->num_msgs; i++) { 149 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset); 150 size = mbox->rx_start + msg->next_msgoff - offset; 151 152 /* Reserve PF/VF mbox message */ 153 size = PLT_ALIGN(size, MBOX_MSG_ALIGN); 154 rsp = mbox_alloc_msg(&dev->mbox_vfpf, vf, size); 155 mbox_rsp_init(msg->id, rsp); 156 157 /* Copy message from AF<->PF mbox to PF<->VF mbox */ 158 mbox_memcpy((uint8_t *)rsp + sizeof(struct mbox_msghdr), 159 (uint8_t *)msg + sizeof(struct mbox_msghdr), 160 size - sizeof(struct mbox_msghdr)); 161 162 /* Set status and sender pf_func data */ 163 rsp->rc = msg->rc; 164 rsp->pcifunc = msg->pcifunc; 165 166 offset = mbox->rx_start + msg->next_msgoff; 167 } 168 plt_spinlock_unlock(&mdev->mbox_lock); 169 170 return req_hdr->num_msgs; 171 } 172 173 static int 174 vf_pf_process_msgs(struct dev *dev, uint16_t vf) 175 { 176 struct mbox *mbox = &dev->mbox_vfpf; 177 struct mbox_dev *mdev = &mbox->dev[vf]; 178 struct mbox_hdr *req_hdr; 179 struct mbox_msghdr *msg; 180 int offset, routed = 0; 181 size_t size; 182 uint16_t i; 183 184 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start); 185 if (!req_hdr->num_msgs) 186 return 0; 187 188 offset = mbox->rx_start + PLT_ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); 189 190 for (i = 0; i < req_hdr->num_msgs; i++) { 191 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset); 192 size = mbox->rx_start + msg->next_msgoff - offset; 193 194 /* RVU_PF_FUNC_S */ 195 msg->pcifunc = dev_pf_func(dev->pf, vf); 196 197 if (msg->id == MBOX_MSG_READY) { 198 struct ready_msg_rsp *rsp; 199 uint16_t max_bits = sizeof(dev->active_vfs[0]) * 8; 200 201 /* Handle READY message in PF */ 202 dev->active_vfs[vf / max_bits] |= 203 BIT_ULL(vf % max_bits); 204 rsp = (struct ready_msg_rsp *)mbox_alloc_msg( 205 mbox, vf, sizeof(*rsp)); 206 mbox_rsp_init(msg->id, rsp); 207 208 /* PF/VF function ID */ 209 rsp->hdr.pcifunc = msg->pcifunc; 210 rsp->hdr.rc = 0; 211 } else { 212 struct mbox_msghdr *af_req; 213 /* Reserve AF/PF mbox message */ 214 size = PLT_ALIGN(size, MBOX_MSG_ALIGN); 215 af_req = mbox_alloc_msg(dev->mbox, 0, size); 216 if (af_req == NULL) 217 return -ENOSPC; 218 mbox_req_init(msg->id, af_req); 219 220 /* Copy message from VF<->PF mbox to PF<->AF mbox */ 221 mbox_memcpy((uint8_t *)af_req + 222 sizeof(struct mbox_msghdr), 223 (uint8_t *)msg + sizeof(struct mbox_msghdr), 224 size - sizeof(struct mbox_msghdr)); 225 af_req->pcifunc = msg->pcifunc; 226 routed++; 227 } 228 offset = mbox->rx_start + msg->next_msgoff; 229 } 230 231 if (routed > 0) { 232 plt_base_dbg("pf:%d routed %d messages from vf:%d to AF", 233 dev->pf, routed, vf); 234 af_pf_wait_msg(dev, vf, routed); 235 mbox_reset(dev->mbox, 0); 236 } 237 238 /* Send mbox responses to VF */ 239 if (mdev->num_msgs) { 240 plt_base_dbg("pf:%d reply %d messages to vf:%d", dev->pf, 241 mdev->num_msgs, vf); 242 mbox_msg_send(mbox, vf); 243 } 244 245 return i; 246 } 247 248 static int 249 vf_pf_process_up_msgs(struct dev *dev, uint16_t vf) 250 { 251 struct mbox *mbox = &dev->mbox_vfpf_up; 252 struct mbox_dev *mdev = &mbox->dev[vf]; 253 struct mbox_hdr *req_hdr; 254 struct mbox_msghdr *msg; 255 int msgs_acked = 0; 256 int offset; 257 uint16_t i; 258 259 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start); 260 if (req_hdr->num_msgs == 0) 261 return 0; 262 263 offset = mbox->rx_start + PLT_ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); 264 265 for (i = 0; i < req_hdr->num_msgs; i++) { 266 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset); 267 268 msgs_acked++; 269 /* RVU_PF_FUNC_S */ 270 msg->pcifunc = dev_pf_func(dev->pf, vf); 271 272 switch (msg->id) { 273 case MBOX_MSG_CGX_LINK_EVENT: 274 plt_base_dbg("PF: Msg 0x%x (%s) fn:0x%x (pf:%d,vf:%d)", 275 msg->id, mbox_id2name(msg->id), 276 msg->pcifunc, dev_get_pf(msg->pcifunc), 277 dev_get_vf(msg->pcifunc)); 278 break; 279 case MBOX_MSG_CGX_PTP_RX_INFO: 280 plt_base_dbg("PF: Msg 0x%x (%s) fn:0x%x (pf:%d,vf:%d)", 281 msg->id, mbox_id2name(msg->id), 282 msg->pcifunc, dev_get_pf(msg->pcifunc), 283 dev_get_vf(msg->pcifunc)); 284 break; 285 default: 286 plt_err("Not handled UP msg 0x%x (%s) func:0x%x", 287 msg->id, mbox_id2name(msg->id), msg->pcifunc); 288 } 289 offset = mbox->rx_start + msg->next_msgoff; 290 } 291 mbox_reset(mbox, vf); 292 mdev->msgs_acked = msgs_acked; 293 plt_wmb(); 294 295 return i; 296 } 297 298 static void 299 roc_vf_pf_mbox_handle_msg(void *param) 300 { 301 uint16_t vf, max_vf, max_bits; 302 struct dev *dev = param; 303 304 max_bits = sizeof(dev->intr.bits[0]) * sizeof(uint64_t); 305 max_vf = max_bits * MAX_VFPF_DWORD_BITS; 306 307 for (vf = 0; vf < max_vf; vf++) { 308 if (dev->intr.bits[vf / max_bits] & BIT_ULL(vf % max_bits)) { 309 plt_base_dbg("Process vf:%d request (pf:%d, vf:%d)", vf, 310 dev->pf, dev->vf); 311 vf_pf_process_msgs(dev, vf); 312 /* UP messages */ 313 vf_pf_process_up_msgs(dev, vf); 314 dev->intr.bits[vf / max_bits] &= 315 ~(BIT_ULL(vf % max_bits)); 316 } 317 } 318 dev->timer_set = 0; 319 } 320 321 static void 322 roc_vf_pf_mbox_irq(void *param) 323 { 324 struct dev *dev = param; 325 bool alarm_set = false; 326 uint64_t intr; 327 int vfpf; 328 329 for (vfpf = 0; vfpf < MAX_VFPF_DWORD_BITS; ++vfpf) { 330 intr = plt_read64(dev->bar2 + RVU_PF_VFPF_MBOX_INTX(vfpf)); 331 if (!intr) 332 continue; 333 334 plt_base_dbg("vfpf: %d intr: 0x%" PRIx64 " (pf:%d, vf:%d)", 335 vfpf, intr, dev->pf, dev->vf); 336 337 /* Save and clear intr bits */ 338 dev->intr.bits[vfpf] |= intr; 339 plt_write64(intr, dev->bar2 + RVU_PF_VFPF_MBOX_INTX(vfpf)); 340 alarm_set = true; 341 } 342 343 if (!dev->timer_set && alarm_set) { 344 dev->timer_set = 1; 345 /* Start timer to handle messages */ 346 plt_alarm_set(VF_PF_MBOX_TIMER_MS, roc_vf_pf_mbox_handle_msg, 347 dev); 348 } 349 } 350 351 static void 352 process_msgs(struct dev *dev, struct mbox *mbox) 353 { 354 struct mbox_dev *mdev = &mbox->dev[0]; 355 struct mbox_hdr *req_hdr; 356 struct mbox_msghdr *msg; 357 int msgs_acked = 0; 358 int offset; 359 uint16_t i; 360 361 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start); 362 if (req_hdr->num_msgs == 0) 363 return; 364 365 offset = mbox->rx_start + PLT_ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); 366 for (i = 0; i < req_hdr->num_msgs; i++) { 367 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset); 368 369 msgs_acked++; 370 plt_base_dbg("Message 0x%x (%s) pf:%d/vf:%d", msg->id, 371 mbox_id2name(msg->id), dev_get_pf(msg->pcifunc), 372 dev_get_vf(msg->pcifunc)); 373 374 switch (msg->id) { 375 /* Add message id's that are handled here */ 376 case MBOX_MSG_READY: 377 /* Get our identity */ 378 dev->pf_func = msg->pcifunc; 379 break; 380 381 default: 382 if (msg->rc) 383 plt_err("Message (%s) response has err=%d", 384 mbox_id2name(msg->id), msg->rc); 385 break; 386 } 387 offset = mbox->rx_start + msg->next_msgoff; 388 } 389 390 mbox_reset(mbox, 0); 391 /* Update acked if someone is waiting a message */ 392 mdev->msgs_acked = msgs_acked; 393 plt_wmb(); 394 } 395 396 /* Copies the message received from AF and sends it to VF */ 397 static void 398 pf_vf_mbox_send_up_msg(struct dev *dev, void *rec_msg) 399 { 400 uint16_t max_bits = sizeof(dev->active_vfs[0]) * sizeof(uint64_t); 401 struct mbox *vf_mbox = &dev->mbox_vfpf_up; 402 struct msg_req *msg = rec_msg; 403 struct mbox_msghdr *vf_msg; 404 uint16_t vf; 405 size_t size; 406 407 size = PLT_ALIGN(mbox_id2size(msg->hdr.id), MBOX_MSG_ALIGN); 408 /* Send UP message to all VF's */ 409 for (vf = 0; vf < vf_mbox->ndevs; vf++) { 410 /* VF active */ 411 if (!(dev->active_vfs[vf / max_bits] & (BIT_ULL(vf)))) 412 continue; 413 414 plt_base_dbg("(%s) size: %zx to VF: %d", 415 mbox_id2name(msg->hdr.id), size, vf); 416 417 /* Reserve PF/VF mbox message */ 418 vf_msg = mbox_alloc_msg(vf_mbox, vf, size); 419 if (!vf_msg) { 420 plt_err("Failed to alloc VF%d UP message", vf); 421 continue; 422 } 423 mbox_req_init(msg->hdr.id, vf_msg); 424 425 /* 426 * Copy message from AF<->PF UP mbox 427 * to PF<->VF UP mbox 428 */ 429 mbox_memcpy((uint8_t *)vf_msg + sizeof(struct mbox_msghdr), 430 (uint8_t *)msg + sizeof(struct mbox_msghdr), 431 size - sizeof(struct mbox_msghdr)); 432 433 vf_msg->rc = msg->hdr.rc; 434 /* Set PF to be a sender */ 435 vf_msg->pcifunc = dev->pf_func; 436 437 /* Send to VF */ 438 mbox_msg_send(vf_mbox, vf); 439 } 440 } 441 442 static int 443 mbox_up_handler_cgx_link_event(struct dev *dev, struct cgx_link_info_msg *msg, 444 struct msg_rsp *rsp) 445 { 446 struct cgx_link_user_info *linfo = &msg->link_info; 447 void *roc_nix = dev->roc_nix; 448 449 plt_base_dbg("pf:%d/vf:%d NIC Link %s --> 0x%x (%s) from: pf:%d/vf:%d", 450 dev_get_pf(dev->pf_func), dev_get_vf(dev->pf_func), 451 linfo->link_up ? "UP" : "DOWN", msg->hdr.id, 452 mbox_id2name(msg->hdr.id), dev_get_pf(msg->hdr.pcifunc), 453 dev_get_vf(msg->hdr.pcifunc)); 454 455 /* PF gets link notification from AF */ 456 if (dev_get_pf(msg->hdr.pcifunc) == 0) { 457 if (dev->ops && dev->ops->link_status_update) 458 dev->ops->link_status_update(roc_nix, linfo); 459 460 /* Forward the same message as received from AF to VF */ 461 pf_vf_mbox_send_up_msg(dev, msg); 462 } else { 463 /* VF gets link up notification */ 464 if (dev->ops && dev->ops->link_status_update) 465 dev->ops->link_status_update(roc_nix, linfo); 466 } 467 468 rsp->hdr.rc = 0; 469 return 0; 470 } 471 472 static int 473 mbox_up_handler_cgx_ptp_rx_info(struct dev *dev, 474 struct cgx_ptp_rx_info_msg *msg, 475 struct msg_rsp *rsp) 476 { 477 void *roc_nix = dev->roc_nix; 478 479 plt_base_dbg("pf:%d/vf:%d PTP mode %s --> 0x%x (%s) from: pf:%d/vf:%d", 480 dev_get_pf(dev->pf_func), dev_get_vf(dev->pf_func), 481 msg->ptp_en ? "ENABLED" : "DISABLED", msg->hdr.id, 482 mbox_id2name(msg->hdr.id), dev_get_pf(msg->hdr.pcifunc), 483 dev_get_vf(msg->hdr.pcifunc)); 484 485 /* PF gets PTP notification from AF */ 486 if (dev_get_pf(msg->hdr.pcifunc) == 0) { 487 if (dev->ops && dev->ops->ptp_info_update) 488 dev->ops->ptp_info_update(roc_nix, msg->ptp_en); 489 490 /* Forward the same message as received from AF to VF */ 491 pf_vf_mbox_send_up_msg(dev, msg); 492 } else { 493 /* VF gets PTP notification */ 494 if (dev->ops && dev->ops->ptp_info_update) 495 dev->ops->ptp_info_update(roc_nix, msg->ptp_en); 496 } 497 498 rsp->hdr.rc = 0; 499 return 0; 500 } 501 502 static int 503 mbox_process_msgs_up(struct dev *dev, struct mbox_msghdr *req) 504 { 505 /* Check if valid, if not reply with a invalid msg */ 506 if (req->sig != MBOX_REQ_SIG) 507 return -EIO; 508 509 switch (req->id) { 510 default: 511 reply_invalid_msg(&dev->mbox_up, 0, 0, req->id); 512 break; 513 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \ 514 case _id: { \ 515 struct _rsp_type *rsp; \ 516 int err; \ 517 rsp = (struct _rsp_type *)mbox_alloc_msg( \ 518 &dev->mbox_up, 0, sizeof(struct _rsp_type)); \ 519 if (!rsp) \ 520 return -ENOMEM; \ 521 rsp->hdr.id = _id; \ 522 rsp->hdr.sig = MBOX_RSP_SIG; \ 523 rsp->hdr.pcifunc = dev->pf_func; \ 524 rsp->hdr.rc = 0; \ 525 err = mbox_up_handler_##_fn_name(dev, (struct _req_type *)req, \ 526 rsp); \ 527 return err; \ 528 } 529 MBOX_UP_CGX_MESSAGES 530 #undef M 531 } 532 533 return -ENODEV; 534 } 535 536 static void 537 process_msgs_up(struct dev *dev, struct mbox *mbox) 538 { 539 struct mbox_dev *mdev = &mbox->dev[0]; 540 struct mbox_hdr *req_hdr; 541 struct mbox_msghdr *msg; 542 int i, err, offset; 543 544 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start); 545 if (req_hdr->num_msgs == 0) 546 return; 547 548 offset = mbox->rx_start + PLT_ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); 549 for (i = 0; i < req_hdr->num_msgs; i++) { 550 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset); 551 552 plt_base_dbg("Message 0x%x (%s) pf:%d/vf:%d", msg->id, 553 mbox_id2name(msg->id), dev_get_pf(msg->pcifunc), 554 dev_get_vf(msg->pcifunc)); 555 err = mbox_process_msgs_up(dev, msg); 556 if (err) 557 plt_err("Error %d handling 0x%x (%s)", err, msg->id, 558 mbox_id2name(msg->id)); 559 offset = mbox->rx_start + msg->next_msgoff; 560 } 561 /* Send mbox responses */ 562 if (mdev->num_msgs) { 563 plt_base_dbg("Reply num_msgs:%d", mdev->num_msgs); 564 mbox_msg_send(mbox, 0); 565 } 566 } 567 568 static void 569 roc_pf_vf_mbox_irq(void *param) 570 { 571 struct dev *dev = param; 572 uint64_t intr; 573 574 intr = plt_read64(dev->bar2 + RVU_VF_INT); 575 if (intr == 0) 576 plt_base_dbg("Proceeding to check mbox UP messages if any"); 577 578 plt_write64(intr, dev->bar2 + RVU_VF_INT); 579 plt_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf); 580 581 /* First process all configuration messages */ 582 process_msgs(dev, dev->mbox); 583 584 /* Process Uplink messages */ 585 process_msgs_up(dev, &dev->mbox_up); 586 } 587 588 static void 589 roc_af_pf_mbox_irq(void *param) 590 { 591 struct dev *dev = param; 592 uint64_t intr; 593 594 intr = plt_read64(dev->bar2 + RVU_PF_INT); 595 if (intr == 0) 596 plt_base_dbg("Proceeding to check mbox UP messages if any"); 597 598 plt_write64(intr, dev->bar2 + RVU_PF_INT); 599 plt_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf); 600 601 /* First process all configuration messages */ 602 process_msgs(dev, dev->mbox); 603 604 /* Process Uplink messages */ 605 process_msgs_up(dev, &dev->mbox_up); 606 } 607 608 static int 609 mbox_register_pf_irq(struct plt_pci_device *pci_dev, struct dev *dev) 610 { 611 struct plt_intr_handle *intr_handle = &pci_dev->intr_handle; 612 int i, rc; 613 614 /* HW clear irq */ 615 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i) 616 plt_write64(~0ull, 617 dev->bar2 + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(i)); 618 619 plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C); 620 621 dev->timer_set = 0; 622 623 /* MBOX interrupt for VF(0...63) <-> PF */ 624 rc = dev_irq_register(intr_handle, roc_vf_pf_mbox_irq, dev, 625 RVU_PF_INT_VEC_VFPF_MBOX0); 626 627 if (rc) { 628 plt_err("Fail to register PF(VF0-63) mbox irq"); 629 return rc; 630 } 631 /* MBOX interrupt for VF(64...128) <-> PF */ 632 rc = dev_irq_register(intr_handle, roc_vf_pf_mbox_irq, dev, 633 RVU_PF_INT_VEC_VFPF_MBOX1); 634 635 if (rc) { 636 plt_err("Fail to register PF(VF64-128) mbox irq"); 637 return rc; 638 } 639 /* MBOX interrupt AF <-> PF */ 640 rc = dev_irq_register(intr_handle, roc_af_pf_mbox_irq, dev, 641 RVU_PF_INT_VEC_AFPF_MBOX); 642 if (rc) { 643 plt_err("Fail to register AF<->PF mbox irq"); 644 return rc; 645 } 646 647 /* HW enable intr */ 648 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i) 649 plt_write64(~0ull, 650 dev->bar2 + RVU_PF_VFPF_MBOX_INT_ENA_W1SX(i)); 651 652 plt_write64(~0ull, dev->bar2 + RVU_PF_INT); 653 plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S); 654 655 return rc; 656 } 657 658 static int 659 mbox_register_vf_irq(struct plt_pci_device *pci_dev, struct dev *dev) 660 { 661 struct plt_intr_handle *intr_handle = &pci_dev->intr_handle; 662 int rc; 663 664 /* Clear irq */ 665 plt_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1C); 666 667 /* MBOX interrupt PF <-> VF */ 668 rc = dev_irq_register(intr_handle, roc_pf_vf_mbox_irq, dev, 669 RVU_VF_INT_VEC_MBOX); 670 if (rc) { 671 plt_err("Fail to register PF<->VF mbox irq"); 672 return rc; 673 } 674 675 /* HW enable intr */ 676 plt_write64(~0ull, dev->bar2 + RVU_VF_INT); 677 plt_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1S); 678 679 return rc; 680 } 681 682 static int 683 mbox_register_irq(struct plt_pci_device *pci_dev, struct dev *dev) 684 { 685 if (dev_is_vf(dev)) 686 return mbox_register_vf_irq(pci_dev, dev); 687 else 688 return mbox_register_pf_irq(pci_dev, dev); 689 } 690 691 static void 692 mbox_unregister_pf_irq(struct plt_pci_device *pci_dev, struct dev *dev) 693 { 694 struct plt_intr_handle *intr_handle = &pci_dev->intr_handle; 695 int i; 696 697 /* HW clear irq */ 698 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i) 699 plt_write64(~0ull, 700 dev->bar2 + RVU_PF_VFPF_MBOX_INT_ENA_W1CX(i)); 701 702 plt_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C); 703 704 dev->timer_set = 0; 705 706 plt_alarm_cancel(roc_vf_pf_mbox_handle_msg, dev); 707 708 /* Unregister the interrupt handler for each vectors */ 709 /* MBOX interrupt for VF(0...63) <-> PF */ 710 dev_irq_unregister(intr_handle, roc_vf_pf_mbox_irq, dev, 711 RVU_PF_INT_VEC_VFPF_MBOX0); 712 713 /* MBOX interrupt for VF(64...128) <-> PF */ 714 dev_irq_unregister(intr_handle, roc_vf_pf_mbox_irq, dev, 715 RVU_PF_INT_VEC_VFPF_MBOX1); 716 717 /* MBOX interrupt AF <-> PF */ 718 dev_irq_unregister(intr_handle, roc_af_pf_mbox_irq, dev, 719 RVU_PF_INT_VEC_AFPF_MBOX); 720 } 721 722 static void 723 mbox_unregister_vf_irq(struct plt_pci_device *pci_dev, struct dev *dev) 724 { 725 struct plt_intr_handle *intr_handle = &pci_dev->intr_handle; 726 727 /* Clear irq */ 728 plt_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1C); 729 730 /* Unregister the interrupt handler */ 731 dev_irq_unregister(intr_handle, roc_pf_vf_mbox_irq, dev, 732 RVU_VF_INT_VEC_MBOX); 733 } 734 735 static void 736 mbox_unregister_irq(struct plt_pci_device *pci_dev, struct dev *dev) 737 { 738 if (dev_is_vf(dev)) 739 mbox_unregister_vf_irq(pci_dev, dev); 740 else 741 mbox_unregister_pf_irq(pci_dev, dev); 742 } 743 744 static int 745 vf_flr_send_msg(struct dev *dev, uint16_t vf) 746 { 747 struct mbox *mbox = dev->mbox; 748 struct msg_req *req; 749 int rc; 750 751 req = mbox_alloc_msg_vf_flr(mbox); 752 if (req == NULL) 753 return -ENOSPC; 754 /* Overwrite pcifunc to indicate VF */ 755 req->hdr.pcifunc = dev_pf_func(dev->pf, vf); 756 757 /* Sync message in interrupt context */ 758 rc = pf_af_sync_msg(dev, NULL); 759 if (rc) 760 plt_err("Failed to send VF FLR mbox msg, rc=%d", rc); 761 762 return rc; 763 } 764 765 static void 766 roc_pf_vf_flr_irq(void *param) 767 { 768 struct dev *dev = (struct dev *)param; 769 uint16_t max_vf = 64, vf; 770 uintptr_t bar2; 771 uint64_t intr; 772 int i; 773 774 max_vf = (dev->maxvf > 0) ? dev->maxvf : 64; 775 bar2 = dev->bar2; 776 777 plt_base_dbg("FLR VF interrupt: max_vf: %d", max_vf); 778 779 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i) { 780 intr = plt_read64(bar2 + RVU_PF_VFFLR_INTX(i)); 781 if (!intr) 782 continue; 783 784 for (vf = 0; vf < max_vf; vf++) { 785 if (!(intr & (1ULL << vf))) 786 continue; 787 788 plt_base_dbg("FLR: i :%d intr: 0x%" PRIx64 ", vf-%d", i, 789 intr, (64 * i + vf)); 790 /* Clear interrupt */ 791 plt_write64(BIT_ULL(vf), bar2 + RVU_PF_VFFLR_INTX(i)); 792 /* Disable the interrupt */ 793 plt_write64(BIT_ULL(vf), 794 bar2 + RVU_PF_VFFLR_INT_ENA_W1CX(i)); 795 /* Inform AF about VF reset */ 796 vf_flr_send_msg(dev, vf); 797 798 /* Signal FLR finish */ 799 plt_write64(BIT_ULL(vf), bar2 + RVU_PF_VFTRPENDX(i)); 800 /* Enable interrupt */ 801 plt_write64(~0ull, bar2 + RVU_PF_VFFLR_INT_ENA_W1SX(i)); 802 } 803 } 804 } 805 806 static int 807 vf_flr_unregister_irqs(struct plt_pci_device *pci_dev, struct dev *dev) 808 { 809 struct plt_intr_handle *intr_handle = &pci_dev->intr_handle; 810 int i; 811 812 plt_base_dbg("Unregister VF FLR interrupts for %s", pci_dev->name); 813 814 /* HW clear irq */ 815 for (i = 0; i < MAX_VFPF_DWORD_BITS; i++) 816 plt_write64(~0ull, dev->bar2 + RVU_PF_VFFLR_INT_ENA_W1CX(i)); 817 818 dev_irq_unregister(intr_handle, roc_pf_vf_flr_irq, dev, 819 RVU_PF_INT_VEC_VFFLR0); 820 821 dev_irq_unregister(intr_handle, roc_pf_vf_flr_irq, dev, 822 RVU_PF_INT_VEC_VFFLR1); 823 824 return 0; 825 } 826 827 static int 828 vf_flr_register_irqs(struct plt_pci_device *pci_dev, struct dev *dev) 829 { 830 struct plt_intr_handle *handle = &pci_dev->intr_handle; 831 int i, rc; 832 833 plt_base_dbg("Register VF FLR interrupts for %s", pci_dev->name); 834 835 rc = dev_irq_register(handle, roc_pf_vf_flr_irq, dev, 836 RVU_PF_INT_VEC_VFFLR0); 837 if (rc) 838 plt_err("Failed to init RVU_PF_INT_VEC_VFFLR0 rc=%d", rc); 839 840 rc = dev_irq_register(handle, roc_pf_vf_flr_irq, dev, 841 RVU_PF_INT_VEC_VFFLR1); 842 if (rc) 843 plt_err("Failed to init RVU_PF_INT_VEC_VFFLR1 rc=%d", rc); 844 845 /* Enable HW interrupt */ 846 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i) { 847 plt_write64(~0ull, dev->bar2 + RVU_PF_VFFLR_INTX(i)); 848 plt_write64(~0ull, dev->bar2 + RVU_PF_VFTRPENDX(i)); 849 plt_write64(~0ull, dev->bar2 + RVU_PF_VFFLR_INT_ENA_W1SX(i)); 850 } 851 return 0; 852 } 853 854 int 855 dev_active_vfs(struct dev *dev) 856 { 857 int i, count = 0; 858 859 for (i = 0; i < MAX_VFPF_DWORD_BITS; i++) 860 count += __builtin_popcount(dev->active_vfs[i]); 861 862 return count; 863 } 864 865 static void 866 dev_vf_hwcap_update(struct plt_pci_device *pci_dev, struct dev *dev) 867 { 868 switch (pci_dev->id.device_id) { 869 case PCI_DEVID_CNXK_RVU_PF: 870 break; 871 case PCI_DEVID_CNXK_RVU_SSO_TIM_VF: 872 case PCI_DEVID_CNXK_RVU_NPA_VF: 873 case PCI_DEVID_CNXK_RVU_AF_VF: 874 case PCI_DEVID_CNXK_RVU_VF: 875 case PCI_DEVID_CNXK_RVU_SDP_VF: 876 dev->hwcap |= DEV_HWCAP_F_VF; 877 break; 878 } 879 } 880 881 static uintptr_t 882 dev_vf_mbase_get(struct plt_pci_device *pci_dev, struct dev *dev) 883 { 884 void *vf_mbase = NULL; 885 uintptr_t pa; 886 887 if (dev_is_vf(dev)) 888 return 0; 889 890 /* For CN10K onwards, it is just after PF MBOX */ 891 if (!roc_model_is_cn9k()) 892 return dev->bar4 + MBOX_SIZE; 893 894 pa = plt_read64(dev->bar2 + RVU_PF_VF_BAR4_ADDR); 895 if (!pa) { 896 plt_err("Invalid VF mbox base pa"); 897 return pa; 898 } 899 900 vf_mbase = mbox_mem_map(pa, MBOX_SIZE * pci_dev->max_vfs); 901 if (vf_mbase == MAP_FAILED) { 902 plt_err("Failed to mmap vf mbase at pa 0x%lx, rc=%d", pa, 903 errno); 904 return 0; 905 } 906 return (uintptr_t)vf_mbase; 907 } 908 909 static void 910 dev_vf_mbase_put(struct plt_pci_device *pci_dev, uintptr_t vf_mbase) 911 { 912 if (!vf_mbase || !pci_dev->max_vfs || !roc_model_is_cn9k()) 913 return; 914 915 mbox_mem_unmap((void *)vf_mbase, MBOX_SIZE * pci_dev->max_vfs); 916 } 917 918 static uint16_t 919 dev_pf_total_vfs(struct plt_pci_device *pci_dev) 920 { 921 uint16_t total_vfs = 0; 922 int sriov_pos, rc; 923 924 sriov_pos = 925 plt_pci_find_ext_capability(pci_dev, ROC_PCI_EXT_CAP_ID_SRIOV); 926 if (sriov_pos <= 0) { 927 plt_warn("Unable to find SRIOV cap, rc=%d", sriov_pos); 928 return 0; 929 } 930 931 rc = plt_pci_read_config(pci_dev, &total_vfs, 2, 932 sriov_pos + ROC_PCI_SRIOV_TOTAL_VF); 933 if (rc < 0) { 934 plt_warn("Unable to read SRIOV cap, rc=%d", rc); 935 return 0; 936 } 937 938 return total_vfs; 939 } 940 941 static int 942 dev_setup_shared_lmt_region(struct mbox *mbox) 943 { 944 struct lmtst_tbl_setup_req *req; 945 946 req = mbox_alloc_msg_lmtst_tbl_setup(mbox); 947 req->pcifunc = idev_lmt_pffunc_get(); 948 949 return mbox_process(mbox); 950 } 951 952 static int 953 dev_lmt_setup(struct plt_pci_device *pci_dev, struct dev *dev) 954 { 955 struct idev_cfg *idev; 956 int rc; 957 958 if (roc_model_is_cn9k()) { 959 dev->lmt_base = dev->bar2 + (RVU_BLOCK_ADDR_LMT << 20); 960 return 0; 961 } 962 963 /* [CN10K, .) */ 964 965 /* Set common lmt region from second pf_func onwards. */ 966 if (!dev->disable_shared_lmt && idev_lmt_pffunc_get() && 967 dev->pf_func != idev_lmt_pffunc_get()) { 968 rc = dev_setup_shared_lmt_region(dev->mbox); 969 if (!rc) { 970 dev->lmt_base = roc_idev_lmt_base_addr_get(); 971 return rc; 972 } 973 plt_err("Failed to setup shared lmt region, pf_func %d err %d " 974 "Using respective LMT region per pf func", 975 dev->pf_func, rc); 976 } 977 978 if (dev_is_vf(dev)) { 979 /* VF BAR4 should always be sufficient enough to 980 * hold LMT lines. 981 */ 982 if (pci_dev->mem_resource[4].len < 983 (RVU_LMT_LINE_MAX * RVU_LMT_SZ)) { 984 plt_err("Not enough bar4 space for lmt lines"); 985 return -EFAULT; 986 } 987 988 dev->lmt_base = dev->bar4; 989 } else { 990 uint64_t bar4_mbox_sz = MBOX_SIZE; 991 992 /* PF BAR4 should always be sufficient enough to 993 * hold PF-AF MBOX + PF-VF MBOX + LMT lines. 994 */ 995 if (pci_dev->mem_resource[4].len < 996 (bar4_mbox_sz + (RVU_LMT_LINE_MAX * RVU_LMT_SZ))) { 997 plt_err("Not enough bar4 space for lmt lines and mbox"); 998 return -EFAULT; 999 } 1000 1001 /* LMT base is just after total VF MBOX area */ 1002 bar4_mbox_sz += (MBOX_SIZE * dev_pf_total_vfs(pci_dev)); 1003 dev->lmt_base = dev->bar4 + bar4_mbox_sz; 1004 } 1005 1006 /* Base LMT address should be chosen from only those pci funcs which 1007 * participate in LMT shared mode. 1008 */ 1009 if (!dev->disable_shared_lmt) { 1010 idev = idev_get_cfg(); 1011 if (!__atomic_load_n(&idev->lmt_pf_func, __ATOMIC_ACQUIRE)) { 1012 idev->lmt_base_addr = dev->lmt_base; 1013 idev->lmt_pf_func = dev->pf_func; 1014 idev->num_lmtlines = RVU_LMT_LINE_MAX; 1015 } 1016 } 1017 1018 return 0; 1019 } 1020 1021 int 1022 dev_init(struct dev *dev, struct plt_pci_device *pci_dev) 1023 { 1024 int direction, up_direction, rc; 1025 uintptr_t bar2, bar4, mbox; 1026 uintptr_t vf_mbase = 0; 1027 uint64_t intr_offset; 1028 1029 bar2 = (uintptr_t)pci_dev->mem_resource[2].addr; 1030 bar4 = (uintptr_t)pci_dev->mem_resource[4].addr; 1031 if (bar2 == 0 || bar4 == 0) { 1032 plt_err("Failed to get PCI bars"); 1033 rc = -ENODEV; 1034 goto error; 1035 } 1036 1037 /* Trigger fault on bar2 and bar4 regions 1038 * to avoid BUG_ON in remap_pfn_range() 1039 * in latest kernel. 1040 */ 1041 *(volatile uint64_t *)bar2; 1042 *(volatile uint64_t *)bar4; 1043 1044 /* Check ROC model supported */ 1045 if (roc_model->flag == 0) { 1046 rc = UTIL_ERR_INVALID_MODEL; 1047 goto error; 1048 } 1049 1050 dev->maxvf = pci_dev->max_vfs; 1051 dev->bar2 = bar2; 1052 dev->bar4 = bar4; 1053 dev_vf_hwcap_update(pci_dev, dev); 1054 1055 if (dev_is_vf(dev)) { 1056 mbox = (roc_model_is_cn9k() ? 1057 bar4 : (bar2 + RVU_VF_MBOX_REGION)); 1058 direction = MBOX_DIR_VFPF; 1059 up_direction = MBOX_DIR_VFPF_UP; 1060 intr_offset = RVU_VF_INT; 1061 } else { 1062 mbox = bar4; 1063 direction = MBOX_DIR_PFAF; 1064 up_direction = MBOX_DIR_PFAF_UP; 1065 intr_offset = RVU_PF_INT; 1066 } 1067 1068 /* Initialize the local mbox */ 1069 rc = mbox_init(&dev->mbox_local, mbox, bar2, direction, 1, intr_offset); 1070 if (rc) 1071 goto error; 1072 dev->mbox = &dev->mbox_local; 1073 1074 rc = mbox_init(&dev->mbox_up, mbox, bar2, up_direction, 1, intr_offset); 1075 if (rc) 1076 goto mbox_fini; 1077 1078 /* Register mbox interrupts */ 1079 rc = mbox_register_irq(pci_dev, dev); 1080 if (rc) 1081 goto mbox_fini; 1082 1083 /* Check the readiness of PF/VF */ 1084 rc = send_ready_msg(dev->mbox, &dev->pf_func); 1085 if (rc) 1086 goto mbox_unregister; 1087 1088 dev->pf = dev_get_pf(dev->pf_func); 1089 dev->vf = dev_get_vf(dev->pf_func); 1090 memset(&dev->active_vfs, 0, sizeof(dev->active_vfs)); 1091 1092 /* Allocate memory for device ops */ 1093 dev->ops = plt_zmalloc(sizeof(struct dev_ops), 0); 1094 if (dev->ops == NULL) { 1095 rc = -ENOMEM; 1096 goto mbox_unregister; 1097 } 1098 1099 /* Found VF devices in a PF device */ 1100 if (pci_dev->max_vfs > 0) { 1101 /* Remap mbox area for all vf's */ 1102 vf_mbase = dev_vf_mbase_get(pci_dev, dev); 1103 if (!vf_mbase) { 1104 rc = -ENODEV; 1105 goto mbox_unregister; 1106 } 1107 /* Init mbox object */ 1108 rc = mbox_init(&dev->mbox_vfpf, vf_mbase, bar2, MBOX_DIR_PFVF, 1109 pci_dev->max_vfs, intr_offset); 1110 if (rc) 1111 goto iounmap; 1112 1113 /* PF -> VF UP messages */ 1114 rc = mbox_init(&dev->mbox_vfpf_up, vf_mbase, bar2, 1115 MBOX_DIR_PFVF_UP, pci_dev->max_vfs, intr_offset); 1116 if (rc) 1117 goto iounmap; 1118 } 1119 1120 /* Register VF-FLR irq handlers */ 1121 if (!dev_is_vf(dev)) { 1122 rc = vf_flr_register_irqs(pci_dev, dev); 1123 if (rc) 1124 goto iounmap; 1125 } 1126 dev->mbox_active = 1; 1127 1128 rc = npa_lf_init(dev, pci_dev); 1129 if (rc) 1130 goto iounmap; 1131 1132 /* Setup LMT line base */ 1133 rc = dev_lmt_setup(pci_dev, dev); 1134 if (rc) 1135 goto iounmap; 1136 1137 return rc; 1138 iounmap: 1139 dev_vf_mbase_put(pci_dev, vf_mbase); 1140 mbox_unregister: 1141 mbox_unregister_irq(pci_dev, dev); 1142 if (dev->ops) 1143 plt_free(dev->ops); 1144 mbox_fini: 1145 mbox_fini(dev->mbox); 1146 mbox_fini(&dev->mbox_up); 1147 error: 1148 return rc; 1149 } 1150 1151 int 1152 dev_fini(struct dev *dev, struct plt_pci_device *pci_dev) 1153 { 1154 struct plt_intr_handle *intr_handle = &pci_dev->intr_handle; 1155 struct mbox *mbox; 1156 1157 /* Check if this dev hosts npalf and has 1+ refs */ 1158 if (idev_npa_lf_active(dev) > 1) 1159 return -EAGAIN; 1160 1161 /* Clear references to this pci dev */ 1162 npa_lf_fini(); 1163 1164 mbox_unregister_irq(pci_dev, dev); 1165 1166 if (!dev_is_vf(dev)) 1167 vf_flr_unregister_irqs(pci_dev, dev); 1168 /* Release PF - VF */ 1169 mbox = &dev->mbox_vfpf; 1170 if (mbox->hwbase && mbox->dev) 1171 dev_vf_mbase_put(pci_dev, mbox->hwbase); 1172 1173 if (dev->ops) 1174 plt_free(dev->ops); 1175 1176 mbox_fini(mbox); 1177 mbox = &dev->mbox_vfpf_up; 1178 mbox_fini(mbox); 1179 1180 /* Release PF - AF */ 1181 mbox = dev->mbox; 1182 mbox_fini(mbox); 1183 mbox = &dev->mbox_up; 1184 mbox_fini(mbox); 1185 dev->mbox_active = 0; 1186 1187 /* Disable MSIX vectors */ 1188 dev_irqs_disable(intr_handle); 1189 return 0; 1190 } 1191