1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/errno.h> 7 #include <linux/io.h> 8 #include <linux/slab.h> 9 #include <linux/etherdevice.h> 10 #include "ionic.h" 11 #include "ionic_dev.h" 12 #include "ionic_lif.h" 13 14 static void ionic_watchdog_cb(struct timer_list *t) 15 { 16 struct ionic *ionic = from_timer(ionic, t, watchdog_timer); 17 struct ionic_lif *lif = ionic->lif; 18 struct ionic_deferred_work *work; 19 int hb; 20 21 mod_timer(&ionic->watchdog_timer, 22 round_jiffies(jiffies + ionic->watchdog_period)); 23 24 if (!lif) 25 return; 26 27 hb = ionic_heartbeat_check(ionic); 28 dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n", 29 __func__, hb, netif_running(lif->netdev), 30 test_bit(IONIC_LIF_F_UP, lif->state)); 31 32 if (hb >= 0 && 33 !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 34 ionic_link_status_check_request(lif, CAN_NOT_SLEEP); 35 36 if (test_bit(IONIC_LIF_F_FILTER_SYNC_NEEDED, lif->state) && 37 !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) { 38 work = kzalloc(sizeof(*work), GFP_ATOMIC); 39 if (!work) { 40 netdev_err(lif->netdev, "rxmode change dropped\n"); 41 return; 42 } 43 44 work->type = IONIC_DW_TYPE_RX_MODE; 45 netdev_dbg(lif->netdev, "deferred: rx_mode\n"); 46 ionic_lif_deferred_enqueue(&lif->deferred, work); 47 } 48 } 49 50 static void ionic_watchdog_init(struct ionic *ionic) 51 { 52 struct ionic_dev *idev = &ionic->idev; 53 54 timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0); 55 ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ; 56 57 /* set times to ensure the first check will proceed */ 58 atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ); 59 idev->last_hb_time = jiffies - 2 * ionic->watchdog_period; 60 /* init as ready, so no transition if the first check succeeds */ 61 idev->last_fw_hb = 0; 62 idev->fw_hb_ready = true; 63 idev->fw_status_ready = true; 64 idev->fw_generation = IONIC_FW_STS_F_GENERATION & 65 ioread8(&idev->dev_info_regs->fw_status); 66 } 67 68 void ionic_init_devinfo(struct ionic *ionic) 69 { 70 struct ionic_dev *idev = &ionic->idev; 71 72 idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type); 73 idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev); 74 75 memcpy_fromio(idev->dev_info.fw_version, 76 idev->dev_info_regs->fw_version, 77 IONIC_DEVINFO_FWVERS_BUFLEN); 78 79 memcpy_fromio(idev->dev_info.serial_num, 80 idev->dev_info_regs->serial_num, 81 IONIC_DEVINFO_SERIAL_BUFLEN); 82 83 idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0; 84 idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0; 85 86 dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version); 87 } 88 89 int ionic_dev_setup(struct ionic *ionic) 90 { 91 struct ionic_dev_bar *bar = ionic->bars; 92 unsigned int num_bars = ionic->num_bars; 93 struct ionic_dev *idev = &ionic->idev; 94 struct device *dev = ionic->dev; 95 u32 sig; 96 97 /* BAR0: dev_cmd and interrupts */ 98 if (num_bars < 1) { 99 dev_err(dev, "No bars found, aborting\n"); 100 return -EFAULT; 101 } 102 103 if (bar->len < IONIC_BAR0_SIZE) { 104 dev_err(dev, "Resource bar size %lu too small, aborting\n", 105 bar->len); 106 return -EFAULT; 107 } 108 109 idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET; 110 idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET; 111 idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET; 112 idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET; 113 114 idev->hwstamp_regs = &idev->dev_info_regs->hwstamp; 115 116 sig = ioread32(&idev->dev_info_regs->signature); 117 if (sig != IONIC_DEV_INFO_SIGNATURE) { 118 dev_err(dev, "Incompatible firmware signature %x", sig); 119 return -EFAULT; 120 } 121 122 ionic_init_devinfo(ionic); 123 124 /* BAR1: doorbells */ 125 bar++; 126 if (num_bars < 2) { 127 dev_err(dev, "Doorbell bar missing, aborting\n"); 128 return -EFAULT; 129 } 130 131 ionic_watchdog_init(ionic); 132 133 idev->db_pages = bar->vaddr; 134 idev->phy_db_pages = bar->bus_addr; 135 136 return 0; 137 } 138 139 /* Devcmd Interface */ 140 bool ionic_is_fw_running(struct ionic_dev *idev) 141 { 142 u8 fw_status = ioread8(&idev->dev_info_regs->fw_status); 143 144 /* firmware is useful only if the running bit is set and 145 * fw_status != 0xff (bad PCI read) 146 */ 147 return (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING); 148 } 149 150 int ionic_heartbeat_check(struct ionic *ionic) 151 { 152 unsigned long check_time, last_check_time; 153 struct ionic_dev *idev = &ionic->idev; 154 struct ionic_lif *lif = ionic->lif; 155 bool fw_status_ready = true; 156 bool fw_hb_ready; 157 u8 fw_generation; 158 u8 fw_status; 159 u32 fw_hb; 160 161 /* wait a least one second before testing again */ 162 check_time = jiffies; 163 last_check_time = atomic_long_read(&idev->last_check_time); 164 do_check_time: 165 if (time_before(check_time, last_check_time + HZ)) 166 return 0; 167 if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time, 168 &last_check_time, check_time)) { 169 /* if called concurrently, only the first should proceed. */ 170 dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__); 171 goto do_check_time; 172 } 173 174 fw_status = ioread8(&idev->dev_info_regs->fw_status); 175 176 /* If fw_status is not ready don't bother with the generation */ 177 if (!ionic_is_fw_running(idev)) { 178 fw_status_ready = false; 179 } else { 180 fw_generation = fw_status & IONIC_FW_STS_F_GENERATION; 181 if (idev->fw_generation != fw_generation) { 182 dev_info(ionic->dev, "FW generation 0x%02x -> 0x%02x\n", 183 idev->fw_generation, fw_generation); 184 185 idev->fw_generation = fw_generation; 186 187 /* If the generation changed, the fw status is not 188 * ready so we need to trigger a fw-down cycle. After 189 * the down, the next watchdog will see the fw is up 190 * and the generation value stable, so will trigger 191 * the fw-up activity. 192 * 193 * If we had already moved to FW_RESET from a RESET event, 194 * it is possible that we never saw the fw_status go to 0, 195 * so we fake the current idev->fw_status_ready here to 196 * force the transition and get FW up again. 197 */ 198 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 199 idev->fw_status_ready = false; /* go to running */ 200 else 201 fw_status_ready = false; /* go to down */ 202 } 203 } 204 205 /* is this a transition? */ 206 if (fw_status_ready != idev->fw_status_ready) { 207 bool trigger = false; 208 209 if (!fw_status_ready && lif && 210 !test_bit(IONIC_LIF_F_FW_RESET, lif->state) && 211 !test_and_set_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) { 212 dev_info(ionic->dev, "FW stopped 0x%02x\n", fw_status); 213 trigger = true; 214 215 } else if (fw_status_ready && lif && 216 test_bit(IONIC_LIF_F_FW_RESET, lif->state) && 217 !test_bit(IONIC_LIF_F_FW_STOPPING, lif->state)) { 218 dev_info(ionic->dev, "FW running 0x%02x\n", fw_status); 219 trigger = true; 220 } 221 222 if (trigger) { 223 struct ionic_deferred_work *work; 224 225 idev->fw_status_ready = fw_status_ready; 226 227 work = kzalloc(sizeof(*work), GFP_ATOMIC); 228 if (work) { 229 work->type = IONIC_DW_TYPE_LIF_RESET; 230 work->fw_status = fw_status_ready; 231 ionic_lif_deferred_enqueue(&lif->deferred, work); 232 } 233 } 234 } 235 236 if (!idev->fw_status_ready) 237 return -ENXIO; 238 239 /* Because of some variability in the actual FW heartbeat, we 240 * wait longer than the DEVCMD_TIMEOUT before checking again. 241 */ 242 last_check_time = idev->last_hb_time; 243 if (time_before(check_time, last_check_time + DEVCMD_TIMEOUT * 2 * HZ)) 244 return 0; 245 246 fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat); 247 fw_hb_ready = fw_hb != idev->last_fw_hb; 248 249 /* early FW version had no heartbeat, so fake it */ 250 if (!fw_hb_ready && !fw_hb) 251 fw_hb_ready = true; 252 253 dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n", 254 __func__, fw_hb, idev->last_fw_hb, fw_hb_ready); 255 256 idev->last_fw_hb = fw_hb; 257 258 /* log a transition */ 259 if (fw_hb_ready != idev->fw_hb_ready) { 260 idev->fw_hb_ready = fw_hb_ready; 261 if (!fw_hb_ready) 262 dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb); 263 else 264 dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb); 265 } 266 267 if (!fw_hb_ready) 268 return -ENXIO; 269 270 idev->last_hb_time = check_time; 271 272 return 0; 273 } 274 275 u8 ionic_dev_cmd_status(struct ionic_dev *idev) 276 { 277 return ioread8(&idev->dev_cmd_regs->comp.comp.status); 278 } 279 280 bool ionic_dev_cmd_done(struct ionic_dev *idev) 281 { 282 return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE; 283 } 284 285 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp) 286 { 287 memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp)); 288 } 289 290 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd) 291 { 292 memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd)); 293 iowrite32(0, &idev->dev_cmd_regs->done); 294 iowrite32(1, &idev->dev_cmd_regs->doorbell); 295 } 296 297 /* Device commands */ 298 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver) 299 { 300 union ionic_dev_cmd cmd = { 301 .identify.opcode = IONIC_CMD_IDENTIFY, 302 .identify.ver = ver, 303 }; 304 305 ionic_dev_cmd_go(idev, &cmd); 306 } 307 308 void ionic_dev_cmd_init(struct ionic_dev *idev) 309 { 310 union ionic_dev_cmd cmd = { 311 .init.opcode = IONIC_CMD_INIT, 312 .init.type = 0, 313 }; 314 315 ionic_dev_cmd_go(idev, &cmd); 316 } 317 318 void ionic_dev_cmd_reset(struct ionic_dev *idev) 319 { 320 union ionic_dev_cmd cmd = { 321 .reset.opcode = IONIC_CMD_RESET, 322 }; 323 324 ionic_dev_cmd_go(idev, &cmd); 325 } 326 327 /* Port commands */ 328 void ionic_dev_cmd_port_identify(struct ionic_dev *idev) 329 { 330 union ionic_dev_cmd cmd = { 331 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY, 332 .port_init.index = 0, 333 }; 334 335 ionic_dev_cmd_go(idev, &cmd); 336 } 337 338 void ionic_dev_cmd_port_init(struct ionic_dev *idev) 339 { 340 union ionic_dev_cmd cmd = { 341 .port_init.opcode = IONIC_CMD_PORT_INIT, 342 .port_init.index = 0, 343 .port_init.info_pa = cpu_to_le64(idev->port_info_pa), 344 }; 345 346 ionic_dev_cmd_go(idev, &cmd); 347 } 348 349 void ionic_dev_cmd_port_reset(struct ionic_dev *idev) 350 { 351 union ionic_dev_cmd cmd = { 352 .port_reset.opcode = IONIC_CMD_PORT_RESET, 353 .port_reset.index = 0, 354 }; 355 356 ionic_dev_cmd_go(idev, &cmd); 357 } 358 359 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state) 360 { 361 union ionic_dev_cmd cmd = { 362 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 363 .port_setattr.index = 0, 364 .port_setattr.attr = IONIC_PORT_ATTR_STATE, 365 .port_setattr.state = state, 366 }; 367 368 ionic_dev_cmd_go(idev, &cmd); 369 } 370 371 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed) 372 { 373 union ionic_dev_cmd cmd = { 374 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 375 .port_setattr.index = 0, 376 .port_setattr.attr = IONIC_PORT_ATTR_SPEED, 377 .port_setattr.speed = cpu_to_le32(speed), 378 }; 379 380 ionic_dev_cmd_go(idev, &cmd); 381 } 382 383 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable) 384 { 385 union ionic_dev_cmd cmd = { 386 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 387 .port_setattr.index = 0, 388 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG, 389 .port_setattr.an_enable = an_enable, 390 }; 391 392 ionic_dev_cmd_go(idev, &cmd); 393 } 394 395 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type) 396 { 397 union ionic_dev_cmd cmd = { 398 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 399 .port_setattr.index = 0, 400 .port_setattr.attr = IONIC_PORT_ATTR_FEC, 401 .port_setattr.fec_type = fec_type, 402 }; 403 404 ionic_dev_cmd_go(idev, &cmd); 405 } 406 407 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type) 408 { 409 union ionic_dev_cmd cmd = { 410 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 411 .port_setattr.index = 0, 412 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE, 413 .port_setattr.pause_type = pause_type, 414 }; 415 416 ionic_dev_cmd_go(idev, &cmd); 417 } 418 419 /* VF commands */ 420 int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data) 421 { 422 union ionic_dev_cmd cmd = { 423 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR, 424 .vf_setattr.attr = attr, 425 .vf_setattr.vf_index = cpu_to_le16(vf), 426 }; 427 int err; 428 429 switch (attr) { 430 case IONIC_VF_ATTR_SPOOFCHK: 431 cmd.vf_setattr.spoofchk = *data; 432 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n", 433 __func__, vf, *data); 434 break; 435 case IONIC_VF_ATTR_TRUST: 436 cmd.vf_setattr.trust = *data; 437 dev_dbg(ionic->dev, "%s: vf %d trust %d\n", 438 __func__, vf, *data); 439 break; 440 case IONIC_VF_ATTR_LINKSTATE: 441 cmd.vf_setattr.linkstate = *data; 442 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n", 443 __func__, vf, *data); 444 break; 445 case IONIC_VF_ATTR_MAC: 446 ether_addr_copy(cmd.vf_setattr.macaddr, data); 447 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n", 448 __func__, vf, data); 449 break; 450 case IONIC_VF_ATTR_VLAN: 451 cmd.vf_setattr.vlanid = cpu_to_le16(*(u16 *)data); 452 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n", 453 __func__, vf, *(u16 *)data); 454 break; 455 case IONIC_VF_ATTR_RATE: 456 cmd.vf_setattr.maxrate = cpu_to_le32(*(u32 *)data); 457 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n", 458 __func__, vf, *(u32 *)data); 459 break; 460 case IONIC_VF_ATTR_STATSADDR: 461 cmd.vf_setattr.stats_pa = cpu_to_le64(*(u64 *)data); 462 dev_dbg(ionic->dev, "%s: vf %d stats_pa 0x%08llx\n", 463 __func__, vf, *(u64 *)data); 464 break; 465 default: 466 return -EINVAL; 467 } 468 469 mutex_lock(&ionic->dev_cmd_lock); 470 ionic_dev_cmd_go(&ionic->idev, &cmd); 471 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); 472 mutex_unlock(&ionic->dev_cmd_lock); 473 474 return err; 475 } 476 477 int ionic_dev_cmd_vf_getattr(struct ionic *ionic, int vf, u8 attr, 478 struct ionic_vf_getattr_comp *comp) 479 { 480 union ionic_dev_cmd cmd = { 481 .vf_getattr.opcode = IONIC_CMD_VF_GETATTR, 482 .vf_getattr.attr = attr, 483 .vf_getattr.vf_index = cpu_to_le16(vf), 484 }; 485 int err; 486 487 if (vf >= ionic->num_vfs) 488 return -EINVAL; 489 490 switch (attr) { 491 case IONIC_VF_ATTR_SPOOFCHK: 492 case IONIC_VF_ATTR_TRUST: 493 case IONIC_VF_ATTR_LINKSTATE: 494 case IONIC_VF_ATTR_MAC: 495 case IONIC_VF_ATTR_VLAN: 496 case IONIC_VF_ATTR_RATE: 497 break; 498 case IONIC_VF_ATTR_STATSADDR: 499 default: 500 return -EINVAL; 501 } 502 503 mutex_lock(&ionic->dev_cmd_lock); 504 ionic_dev_cmd_go(&ionic->idev, &cmd); 505 err = ionic_dev_cmd_wait_nomsg(ionic, DEVCMD_TIMEOUT); 506 memcpy_fromio(comp, &ionic->idev.dev_cmd_regs->comp.vf_getattr, 507 sizeof(*comp)); 508 mutex_unlock(&ionic->dev_cmd_lock); 509 510 if (err && comp->status != IONIC_RC_ENOSUPP) 511 ionic_dev_cmd_dev_err_print(ionic, cmd.vf_getattr.opcode, 512 comp->status, err); 513 514 return err; 515 } 516 517 /* LIF commands */ 518 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev, 519 u16 lif_type, u8 qtype, u8 qver) 520 { 521 union ionic_dev_cmd cmd = { 522 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY, 523 .q_identify.lif_type = cpu_to_le16(lif_type), 524 .q_identify.type = qtype, 525 .q_identify.ver = qver, 526 }; 527 528 ionic_dev_cmd_go(idev, &cmd); 529 } 530 531 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver) 532 { 533 union ionic_dev_cmd cmd = { 534 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY, 535 .lif_identify.type = type, 536 .lif_identify.ver = ver, 537 }; 538 539 ionic_dev_cmd_go(idev, &cmd); 540 } 541 542 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index, 543 dma_addr_t info_pa) 544 { 545 union ionic_dev_cmd cmd = { 546 .lif_init.opcode = IONIC_CMD_LIF_INIT, 547 .lif_init.index = cpu_to_le16(lif_index), 548 .lif_init.info_pa = cpu_to_le64(info_pa), 549 }; 550 551 ionic_dev_cmd_go(idev, &cmd); 552 } 553 554 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index) 555 { 556 union ionic_dev_cmd cmd = { 557 .lif_init.opcode = IONIC_CMD_LIF_RESET, 558 .lif_init.index = cpu_to_le16(lif_index), 559 }; 560 561 ionic_dev_cmd_go(idev, &cmd); 562 } 563 564 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq, 565 u16 lif_index, u16 intr_index) 566 { 567 struct ionic_queue *q = &qcq->q; 568 struct ionic_cq *cq = &qcq->cq; 569 570 union ionic_dev_cmd cmd = { 571 .q_init.opcode = IONIC_CMD_Q_INIT, 572 .q_init.lif_index = cpu_to_le16(lif_index), 573 .q_init.type = q->type, 574 .q_init.ver = qcq->q.lif->qtype_info[q->type].version, 575 .q_init.index = cpu_to_le32(q->index), 576 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ | 577 IONIC_QINIT_F_ENA), 578 .q_init.pid = cpu_to_le16(q->pid), 579 .q_init.intr_index = cpu_to_le16(intr_index), 580 .q_init.ring_size = ilog2(q->num_descs), 581 .q_init.ring_base = cpu_to_le64(q->base_pa), 582 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa), 583 }; 584 585 ionic_dev_cmd_go(idev, &cmd); 586 } 587 588 int ionic_db_page_num(struct ionic_lif *lif, int pid) 589 { 590 return (lif->hw_index * lif->dbid_count) + pid; 591 } 592 593 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, 594 struct ionic_intr_info *intr, 595 unsigned int num_descs, size_t desc_size) 596 { 597 unsigned int ring_size; 598 599 if (desc_size == 0 || !is_power_of_2(num_descs)) 600 return -EINVAL; 601 602 ring_size = ilog2(num_descs); 603 if (ring_size < 2 || ring_size > 16) 604 return -EINVAL; 605 606 cq->lif = lif; 607 cq->bound_intr = intr; 608 cq->num_descs = num_descs; 609 cq->desc_size = desc_size; 610 cq->tail_idx = 0; 611 cq->done_color = 1; 612 613 return 0; 614 } 615 616 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa) 617 { 618 struct ionic_cq_info *cur; 619 unsigned int i; 620 621 cq->base = base; 622 cq->base_pa = base_pa; 623 624 for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++) 625 cur->cq_desc = base + (i * cq->desc_size); 626 } 627 628 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q) 629 { 630 cq->bound_q = q; 631 } 632 633 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do, 634 ionic_cq_cb cb, ionic_cq_done_cb done_cb, 635 void *done_arg) 636 { 637 struct ionic_cq_info *cq_info; 638 unsigned int work_done = 0; 639 640 if (work_to_do == 0) 641 return 0; 642 643 cq_info = &cq->info[cq->tail_idx]; 644 while (cb(cq, cq_info)) { 645 if (cq->tail_idx == cq->num_descs - 1) 646 cq->done_color = !cq->done_color; 647 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 648 cq_info = &cq->info[cq->tail_idx]; 649 650 if (++work_done >= work_to_do) 651 break; 652 } 653 654 if (work_done && done_cb) 655 done_cb(done_arg); 656 657 return work_done; 658 } 659 660 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, 661 struct ionic_queue *q, unsigned int index, const char *name, 662 unsigned int num_descs, size_t desc_size, 663 size_t sg_desc_size, unsigned int pid) 664 { 665 unsigned int ring_size; 666 667 if (desc_size == 0 || !is_power_of_2(num_descs)) 668 return -EINVAL; 669 670 ring_size = ilog2(num_descs); 671 if (ring_size < 2 || ring_size > 16) 672 return -EINVAL; 673 674 q->lif = lif; 675 q->idev = idev; 676 q->index = index; 677 q->num_descs = num_descs; 678 q->desc_size = desc_size; 679 q->sg_desc_size = sg_desc_size; 680 q->tail_idx = 0; 681 q->head_idx = 0; 682 q->pid = pid; 683 684 snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index); 685 686 return 0; 687 } 688 689 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa) 690 { 691 struct ionic_desc_info *cur; 692 unsigned int i; 693 694 q->base = base; 695 q->base_pa = base_pa; 696 697 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) 698 cur->desc = base + (i * q->desc_size); 699 } 700 701 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa) 702 { 703 struct ionic_desc_info *cur; 704 unsigned int i; 705 706 q->sg_base = base; 707 q->sg_base_pa = base_pa; 708 709 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) 710 cur->sg_desc = base + (i * q->sg_desc_size); 711 } 712 713 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb, 714 void *cb_arg) 715 { 716 struct ionic_desc_info *desc_info; 717 struct ionic_lif *lif = q->lif; 718 struct device *dev = q->dev; 719 720 desc_info = &q->info[q->head_idx]; 721 desc_info->cb = cb; 722 desc_info->cb_arg = cb_arg; 723 724 q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); 725 726 dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n", 727 q->lif->index, q->name, q->hw_type, q->hw_index, 728 q->head_idx, ring_doorbell); 729 730 if (ring_doorbell) 731 ionic_dbell_ring(lif->kern_dbpage, q->hw_type, 732 q->dbval | q->head_idx); 733 } 734 735 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos) 736 { 737 unsigned int mask, tail, head; 738 739 mask = q->num_descs - 1; 740 tail = q->tail_idx; 741 head = q->head_idx; 742 743 return ((pos - tail) & mask) < ((head - tail) & mask); 744 } 745 746 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info, 747 unsigned int stop_index) 748 { 749 struct ionic_desc_info *desc_info; 750 ionic_desc_cb cb; 751 void *cb_arg; 752 u16 index; 753 754 /* check for empty queue */ 755 if (q->tail_idx == q->head_idx) 756 return; 757 758 /* stop index must be for a descriptor that is not yet completed */ 759 if (unlikely(!ionic_q_is_posted(q, stop_index))) 760 dev_err(q->dev, 761 "ionic stop is not posted %s stop %u tail %u head %u\n", 762 q->name, stop_index, q->tail_idx, q->head_idx); 763 764 do { 765 desc_info = &q->info[q->tail_idx]; 766 index = q->tail_idx; 767 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 768 769 cb = desc_info->cb; 770 cb_arg = desc_info->cb_arg; 771 772 desc_info->cb = NULL; 773 desc_info->cb_arg = NULL; 774 775 if (cb) 776 cb(q, desc_info, cq_info, cb_arg); 777 } while (index != stop_index); 778 } 779