1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2007-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 #if EFSYS_OPT_MON_MCDI 10 #include "mcdi_mon.h" 11 #endif 12 13 #define EFX_EV_PRESENT(_qword) \ 14 (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff && \ 15 EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff) 16 17 18 19 #if EFSYS_OPT_SIENA 20 21 static __checkReturn efx_rc_t 22 siena_ev_init( 23 __in efx_nic_t *enp); 24 25 static void 26 siena_ev_fini( 27 __in efx_nic_t *enp); 28 29 static __checkReturn efx_rc_t 30 siena_ev_qcreate( 31 __in efx_nic_t *enp, 32 __in unsigned int index, 33 __in efsys_mem_t *esmp, 34 __in size_t ndescs, 35 __in uint32_t id, 36 __in uint32_t us, 37 __in uint32_t flags, 38 __in efx_evq_t *eep); 39 40 static void 41 siena_ev_qdestroy( 42 __in efx_evq_t *eep); 43 44 static __checkReturn efx_rc_t 45 siena_ev_qprime( 46 __in efx_evq_t *eep, 47 __in unsigned int count); 48 49 static void 50 siena_ev_qpost( 51 __in efx_evq_t *eep, 52 __in uint16_t data); 53 54 static __checkReturn efx_rc_t 55 siena_ev_qmoderate( 56 __in efx_evq_t *eep, 57 __in unsigned int us); 58 59 #if EFSYS_OPT_QSTATS 60 static void 61 siena_ev_qstats_update( 62 __in efx_evq_t *eep, 63 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat); 64 65 #endif 66 67 #endif /* EFSYS_OPT_SIENA */ 68 69 #if EFSYS_OPT_SIENA 70 static const efx_ev_ops_t __efx_ev_siena_ops = { 71 siena_ev_init, /* eevo_init */ 72 siena_ev_fini, /* eevo_fini */ 73 siena_ev_qcreate, /* eevo_qcreate */ 74 siena_ev_qdestroy, /* eevo_qdestroy */ 75 siena_ev_qprime, /* eevo_qprime */ 76 siena_ev_qpost, /* eevo_qpost */ 77 siena_ev_qmoderate, /* eevo_qmoderate */ 78 #if EFSYS_OPT_QSTATS 79 siena_ev_qstats_update, /* eevo_qstats_update */ 80 #endif 81 }; 82 #endif /* EFSYS_OPT_SIENA */ 83 84 #if EFX_OPTS_EF10() 85 static const efx_ev_ops_t __efx_ev_ef10_ops = { 86 ef10_ev_init, /* eevo_init */ 87 ef10_ev_fini, /* eevo_fini */ 88 ef10_ev_qcreate, /* eevo_qcreate */ 89 ef10_ev_qdestroy, /* eevo_qdestroy */ 90 ef10_ev_qprime, /* eevo_qprime */ 91 ef10_ev_qpost, /* eevo_qpost */ 92 ef10_ev_qmoderate, /* eevo_qmoderate */ 93 #if EFSYS_OPT_QSTATS 94 ef10_ev_qstats_update, /* eevo_qstats_update */ 95 #endif 96 }; 97 #endif /* EFX_OPTS_EF10() */ 98 99 100 __checkReturn efx_rc_t 101 efx_ev_init( 102 __in efx_nic_t *enp) 103 { 104 const efx_ev_ops_t *eevop; 105 efx_rc_t rc; 106 107 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 108 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); 109 110 if (enp->en_mod_flags & EFX_MOD_EV) { 111 rc = EINVAL; 112 goto fail1; 113 } 114 115 switch (enp->en_family) { 116 #if EFSYS_OPT_SIENA 117 case EFX_FAMILY_SIENA: 118 eevop = &__efx_ev_siena_ops; 119 break; 120 #endif /* EFSYS_OPT_SIENA */ 121 122 #if EFSYS_OPT_HUNTINGTON 123 case EFX_FAMILY_HUNTINGTON: 124 eevop = &__efx_ev_ef10_ops; 125 break; 126 #endif /* EFSYS_OPT_HUNTINGTON */ 127 128 #if EFSYS_OPT_MEDFORD 129 case EFX_FAMILY_MEDFORD: 130 eevop = &__efx_ev_ef10_ops; 131 break; 132 #endif /* EFSYS_OPT_MEDFORD */ 133 134 #if EFSYS_OPT_MEDFORD2 135 case EFX_FAMILY_MEDFORD2: 136 eevop = &__efx_ev_ef10_ops; 137 break; 138 #endif /* EFSYS_OPT_MEDFORD2 */ 139 140 default: 141 EFSYS_ASSERT(0); 142 rc = ENOTSUP; 143 goto fail1; 144 } 145 146 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); 147 148 if ((rc = eevop->eevo_init(enp)) != 0) 149 goto fail2; 150 151 enp->en_eevop = eevop; 152 enp->en_mod_flags |= EFX_MOD_EV; 153 return (0); 154 155 fail2: 156 EFSYS_PROBE(fail2); 157 158 fail1: 159 EFSYS_PROBE1(fail1, efx_rc_t, rc); 160 161 enp->en_eevop = NULL; 162 enp->en_mod_flags &= ~EFX_MOD_EV; 163 return (rc); 164 } 165 166 __checkReturn size_t 167 efx_evq_size( 168 __in const efx_nic_t *enp, 169 __in unsigned int ndescs) 170 { 171 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 172 173 return (ndescs * encp->enc_ev_desc_size); 174 } 175 176 __checkReturn unsigned int 177 efx_evq_nbufs( 178 __in const efx_nic_t *enp, 179 __in unsigned int ndescs) 180 { 181 return (EFX_DIV_ROUND_UP(efx_evq_size(enp, ndescs), EFX_BUF_SIZE)); 182 } 183 184 void 185 efx_ev_fini( 186 __in efx_nic_t *enp) 187 { 188 const efx_ev_ops_t *eevop = enp->en_eevop; 189 190 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 191 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); 192 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); 193 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); 194 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); 195 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); 196 197 eevop->eevo_fini(enp); 198 199 enp->en_eevop = NULL; 200 enp->en_mod_flags &= ~EFX_MOD_EV; 201 } 202 203 204 __checkReturn efx_rc_t 205 efx_ev_qcreate( 206 __in efx_nic_t *enp, 207 __in unsigned int index, 208 __in efsys_mem_t *esmp, 209 __in size_t ndescs, 210 __in uint32_t id, 211 __in uint32_t us, 212 __in uint32_t flags, 213 __deref_out efx_evq_t **eepp) 214 { 215 const efx_ev_ops_t *eevop = enp->en_eevop; 216 efx_evq_t *eep; 217 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 218 efx_rc_t rc; 219 220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); 222 223 EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, 224 enp->en_nic_cfg.enc_evq_limit); 225 226 switch (flags & EFX_EVQ_FLAGS_NOTIFY_MASK) { 227 case EFX_EVQ_FLAGS_NOTIFY_INTERRUPT: 228 break; 229 case EFX_EVQ_FLAGS_NOTIFY_DISABLED: 230 if (us != 0) { 231 rc = EINVAL; 232 goto fail1; 233 } 234 break; 235 default: 236 rc = EINVAL; 237 goto fail2; 238 } 239 240 EFSYS_ASSERT(ISP2(encp->enc_evq_max_nevs)); 241 EFSYS_ASSERT(ISP2(encp->enc_evq_min_nevs)); 242 243 if (!ISP2(ndescs) || 244 ndescs < encp->enc_evq_min_nevs || 245 ndescs > encp->enc_evq_max_nevs) { 246 rc = EINVAL; 247 goto fail3; 248 } 249 250 /* Allocate an EVQ object */ 251 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep); 252 if (eep == NULL) { 253 rc = ENOMEM; 254 goto fail4; 255 } 256 257 eep->ee_magic = EFX_EVQ_MAGIC; 258 eep->ee_enp = enp; 259 eep->ee_index = index; 260 eep->ee_mask = ndescs - 1; 261 eep->ee_flags = flags; 262 eep->ee_esmp = esmp; 263 264 /* 265 * Set outputs before the queue is created because interrupts may be 266 * raised for events immediately after the queue is created, before the 267 * function call below returns. See bug58606. 268 * 269 * The eepp pointer passed in by the client must therefore point to data 270 * shared with the client's event processing context. 271 */ 272 enp->en_ev_qcount++; 273 *eepp = eep; 274 275 if ((rc = eevop->eevo_qcreate(enp, index, esmp, ndescs, id, us, flags, 276 eep)) != 0) 277 goto fail5; 278 279 return (0); 280 281 fail5: 282 EFSYS_PROBE(fail5); 283 284 *eepp = NULL; 285 enp->en_ev_qcount--; 286 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); 287 fail4: 288 EFSYS_PROBE(fail4); 289 fail3: 290 EFSYS_PROBE(fail3); 291 fail2: 292 EFSYS_PROBE(fail2); 293 fail1: 294 EFSYS_PROBE1(fail1, efx_rc_t, rc); 295 return (rc); 296 } 297 298 void 299 efx_ev_qdestroy( 300 __in efx_evq_t *eep) 301 { 302 efx_nic_t *enp = eep->ee_enp; 303 const efx_ev_ops_t *eevop = enp->en_eevop; 304 305 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 306 307 EFSYS_ASSERT(enp->en_ev_qcount != 0); 308 --enp->en_ev_qcount; 309 310 eevop->eevo_qdestroy(eep); 311 312 /* Free the EVQ object */ 313 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); 314 } 315 316 __checkReturn efx_rc_t 317 efx_ev_qprime( 318 __in efx_evq_t *eep, 319 __in unsigned int count) 320 { 321 efx_nic_t *enp = eep->ee_enp; 322 const efx_ev_ops_t *eevop = enp->en_eevop; 323 efx_rc_t rc; 324 325 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 326 327 if (!(enp->en_mod_flags & EFX_MOD_INTR)) { 328 rc = EINVAL; 329 goto fail1; 330 } 331 332 if ((rc = eevop->eevo_qprime(eep, count)) != 0) 333 goto fail2; 334 335 return (0); 336 337 fail2: 338 EFSYS_PROBE(fail2); 339 fail1: 340 EFSYS_PROBE1(fail1, efx_rc_t, rc); 341 return (rc); 342 } 343 344 __checkReturn boolean_t 345 efx_ev_qpending( 346 __in efx_evq_t *eep, 347 __in unsigned int count) 348 { 349 size_t offset; 350 efx_qword_t qword; 351 352 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 353 354 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 355 EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword); 356 357 return (EFX_EV_PRESENT(qword)); 358 } 359 360 #if EFSYS_OPT_EV_PREFETCH 361 362 void 363 efx_ev_qprefetch( 364 __in efx_evq_t *eep, 365 __in unsigned int count) 366 { 367 unsigned int offset; 368 369 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 370 371 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 372 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 373 } 374 375 #endif /* EFSYS_OPT_EV_PREFETCH */ 376 377 #define EFX_EV_BATCH 8 378 379 void 380 efx_ev_qpoll( 381 __in efx_evq_t *eep, 382 __inout unsigned int *countp, 383 __in const efx_ev_callbacks_t *eecp, 384 __in_opt void *arg) 385 { 386 efx_qword_t ev[EFX_EV_BATCH]; 387 unsigned int batch; 388 unsigned int total; 389 unsigned int count; 390 unsigned int index; 391 size_t offset; 392 393 /* Ensure events codes match for EF10 (Huntington/Medford) and Siena */ 394 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN); 395 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH); 396 397 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV); 398 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV); 399 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV); 400 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV == 401 FSE_AZ_EV_CODE_DRV_GEN_EV); 402 #if EFSYS_OPT_MCDI 403 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV == 404 FSE_AZ_EV_CODE_MCDI_EVRESPONSE); 405 #endif 406 407 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 408 EFSYS_ASSERT(countp != NULL); 409 EFSYS_ASSERT(eecp != NULL); 410 411 count = *countp; 412 do { 413 /* Read up until the end of the batch period */ 414 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1)); 415 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 416 for (total = 0; total < batch; ++total) { 417 EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total])); 418 419 if (!EFX_EV_PRESENT(ev[total])) 420 break; 421 422 EFSYS_PROBE3(event, unsigned int, eep->ee_index, 423 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1), 424 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0)); 425 426 offset += sizeof (efx_qword_t); 427 } 428 429 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1) 430 /* 431 * Prefetch the next batch when we get within PREFETCH_PERIOD 432 * of a completed batch. If the batch is smaller, then prefetch 433 * immediately. 434 */ 435 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD) 436 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 437 #endif /* EFSYS_OPT_EV_PREFETCH */ 438 439 /* Process the batch of events */ 440 for (index = 0; index < total; ++index) { 441 boolean_t should_abort; 442 uint32_t code; 443 444 #if EFSYS_OPT_EV_PREFETCH 445 /* Prefetch if we've now reached the batch period */ 446 if (total == batch && 447 index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) { 448 offset = (count + batch) & eep->ee_mask; 449 offset *= sizeof (efx_qword_t); 450 451 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 452 } 453 #endif /* EFSYS_OPT_EV_PREFETCH */ 454 455 EFX_EV_QSTAT_INCR(eep, EV_ALL); 456 457 code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE); 458 switch (code) { 459 case FSE_AZ_EV_CODE_RX_EV: 460 should_abort = eep->ee_rx(eep, 461 &(ev[index]), eecp, arg); 462 break; 463 case FSE_AZ_EV_CODE_TX_EV: 464 should_abort = eep->ee_tx(eep, 465 &(ev[index]), eecp, arg); 466 break; 467 case FSE_AZ_EV_CODE_DRIVER_EV: 468 should_abort = eep->ee_driver(eep, 469 &(ev[index]), eecp, arg); 470 break; 471 case FSE_AZ_EV_CODE_DRV_GEN_EV: 472 should_abort = eep->ee_drv_gen(eep, 473 &(ev[index]), eecp, arg); 474 break; 475 #if EFSYS_OPT_MCDI 476 case FSE_AZ_EV_CODE_MCDI_EVRESPONSE: 477 should_abort = eep->ee_mcdi(eep, 478 &(ev[index]), eecp, arg); 479 break; 480 #endif 481 case FSE_AZ_EV_CODE_GLOBAL_EV: 482 if (eep->ee_global) { 483 should_abort = eep->ee_global(eep, 484 &(ev[index]), eecp, arg); 485 break; 486 } 487 /* else fallthrough */ 488 default: 489 EFSYS_PROBE3(bad_event, 490 unsigned int, eep->ee_index, 491 uint32_t, 492 EFX_QWORD_FIELD(ev[index], EFX_DWORD_1), 493 uint32_t, 494 EFX_QWORD_FIELD(ev[index], EFX_DWORD_0)); 495 496 EFSYS_ASSERT(eecp->eec_exception != NULL); 497 (void) eecp->eec_exception(arg, 498 EFX_EXCEPTION_EV_ERROR, code); 499 should_abort = B_TRUE; 500 } 501 if (should_abort) { 502 /* Ignore subsequent events */ 503 total = index + 1; 504 505 /* 506 * Poison batch to ensure the outer 507 * loop is broken out of. 508 */ 509 EFSYS_ASSERT(batch <= EFX_EV_BATCH); 510 batch += (EFX_EV_BATCH << 1); 511 EFSYS_ASSERT(total != batch); 512 break; 513 } 514 } 515 516 /* 517 * Now that the hardware has most likely moved onto dma'ing 518 * into the next cache line, clear the processed events. Take 519 * care to only clear out events that we've processed 520 */ 521 EFX_SET_QWORD(ev[0]); 522 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 523 for (index = 0; index < total; ++index) { 524 EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0])); 525 offset += sizeof (efx_qword_t); 526 } 527 528 count += total; 529 530 } while (total == batch); 531 532 *countp = count; 533 } 534 535 void 536 efx_ev_qpost( 537 __in efx_evq_t *eep, 538 __in uint16_t data) 539 { 540 efx_nic_t *enp = eep->ee_enp; 541 const efx_ev_ops_t *eevop = enp->en_eevop; 542 543 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 544 545 EFSYS_ASSERT(eevop != NULL && 546 eevop->eevo_qpost != NULL); 547 548 eevop->eevo_qpost(eep, data); 549 } 550 551 __checkReturn efx_rc_t 552 efx_ev_usecs_to_ticks( 553 __in efx_nic_t *enp, 554 __in unsigned int us, 555 __out unsigned int *ticksp) 556 { 557 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 558 unsigned int ticks; 559 efx_rc_t rc; 560 561 if (encp->enc_evq_timer_quantum_ns == 0) { 562 rc = ENOTSUP; 563 goto fail1; 564 } 565 566 /* Convert microseconds to a timer tick count */ 567 if (us == 0) 568 ticks = 0; 569 else if (us * 1000 < encp->enc_evq_timer_quantum_ns) 570 ticks = 1; /* Never round down to zero */ 571 else 572 ticks = us * 1000 / encp->enc_evq_timer_quantum_ns; 573 574 *ticksp = ticks; 575 return (0); 576 577 fail1: 578 EFSYS_PROBE1(fail1, efx_rc_t, rc); 579 return (rc); 580 } 581 582 __checkReturn efx_rc_t 583 efx_ev_qmoderate( 584 __in efx_evq_t *eep, 585 __in unsigned int us) 586 { 587 efx_nic_t *enp = eep->ee_enp; 588 const efx_ev_ops_t *eevop = enp->en_eevop; 589 efx_rc_t rc; 590 591 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 592 593 if ((eep->ee_flags & EFX_EVQ_FLAGS_NOTIFY_MASK) == 594 EFX_EVQ_FLAGS_NOTIFY_DISABLED) { 595 rc = EINVAL; 596 goto fail1; 597 } 598 599 if ((rc = eevop->eevo_qmoderate(eep, us)) != 0) 600 goto fail2; 601 602 return (0); 603 604 fail2: 605 EFSYS_PROBE(fail2); 606 fail1: 607 EFSYS_PROBE1(fail1, efx_rc_t, rc); 608 return (rc); 609 } 610 611 #if EFSYS_OPT_QSTATS 612 void 613 efx_ev_qstats_update( 614 __in efx_evq_t *eep, 615 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) 616 617 { efx_nic_t *enp = eep->ee_enp; 618 const efx_ev_ops_t *eevop = enp->en_eevop; 619 620 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 621 622 eevop->eevo_qstats_update(eep, stat); 623 } 624 625 #endif /* EFSYS_OPT_QSTATS */ 626 627 #if EFSYS_OPT_SIENA 628 629 static __checkReturn efx_rc_t 630 siena_ev_init( 631 __in efx_nic_t *enp) 632 { 633 efx_oword_t oword; 634 635 /* 636 * Program the event queue for receive and transmit queue 637 * flush events. 638 */ 639 EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword); 640 EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0); 641 EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword); 642 643 return (0); 644 645 } 646 647 static __checkReturn boolean_t 648 siena_ev_rx_not_ok( 649 __in efx_evq_t *eep, 650 __in efx_qword_t *eqp, 651 __in uint32_t label, 652 __in uint32_t id, 653 __inout uint16_t *flagsp) 654 { 655 boolean_t ignore = B_FALSE; 656 657 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) { 658 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC); 659 EFSYS_PROBE(tobe_disc); 660 /* 661 * Assume this is a unicast address mismatch, unless below 662 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or 663 * EV_RX_PAUSE_FRM_ERR is set. 664 */ 665 (*flagsp) |= EFX_ADDR_MISMATCH; 666 } 667 668 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) { 669 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id); 670 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); 671 (*flagsp) |= EFX_DISCARD; 672 673 #if EFSYS_OPT_RX_SCATTER 674 /* 675 * Lookout for payload queue ran dry errors and ignore them. 676 * 677 * Sadly for the header/data split cases, the descriptor 678 * pointer in this event refers to the header queue and 679 * therefore cannot be easily detected as duplicate. 680 * So we drop these and rely on the receive processing seeing 681 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard 682 * the partially received packet. 683 */ 684 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) && 685 (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) && 686 (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0)) 687 ignore = B_TRUE; 688 #endif /* EFSYS_OPT_RX_SCATTER */ 689 } 690 691 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) { 692 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); 693 EFSYS_PROBE(crc_err); 694 (*flagsp) &= ~EFX_ADDR_MISMATCH; 695 (*flagsp) |= EFX_DISCARD; 696 } 697 698 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) { 699 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR); 700 EFSYS_PROBE(pause_frm_err); 701 (*flagsp) &= ~EFX_ADDR_MISMATCH; 702 (*flagsp) |= EFX_DISCARD; 703 } 704 705 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) { 706 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR); 707 EFSYS_PROBE(owner_id_err); 708 (*flagsp) |= EFX_DISCARD; 709 } 710 711 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) { 712 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); 713 EFSYS_PROBE(ipv4_err); 714 (*flagsp) &= ~EFX_CKSUM_IPV4; 715 } 716 717 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) { 718 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); 719 EFSYS_PROBE(udp_chk_err); 720 (*flagsp) &= ~EFX_CKSUM_TCPUDP; 721 } 722 723 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) { 724 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR); 725 726 /* 727 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This 728 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error 729 * condition. 730 */ 731 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP); 732 } 733 734 return (ignore); 735 } 736 737 static __checkReturn boolean_t 738 siena_ev_rx( 739 __in efx_evq_t *eep, 740 __in efx_qword_t *eqp, 741 __in const efx_ev_callbacks_t *eecp, 742 __in_opt void *arg) 743 { 744 uint32_t id; 745 uint32_t size; 746 uint32_t label; 747 boolean_t ok; 748 #if EFSYS_OPT_RX_SCATTER 749 boolean_t sop; 750 boolean_t jumbo_cont; 751 #endif /* EFSYS_OPT_RX_SCATTER */ 752 uint32_t hdr_type; 753 boolean_t is_v6; 754 uint16_t flags; 755 boolean_t ignore; 756 boolean_t should_abort; 757 758 EFX_EV_QSTAT_INCR(eep, EV_RX); 759 760 /* Basic packet information */ 761 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR); 762 size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT); 763 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL); 764 ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0); 765 766 #if EFSYS_OPT_RX_SCATTER 767 sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0); 768 jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0); 769 #endif /* EFSYS_OPT_RX_SCATTER */ 770 771 hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE); 772 773 is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0); 774 775 /* 776 * If packet is marked as OK and packet type is TCP/IP or 777 * UDP/IP or other IP, then we can rely on the hardware checksums. 778 */ 779 switch (hdr_type) { 780 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP: 781 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP; 782 if (is_v6) { 783 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6); 784 flags |= EFX_PKT_IPV6; 785 } else { 786 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4); 787 flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 788 } 789 break; 790 791 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP: 792 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP; 793 if (is_v6) { 794 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6); 795 flags |= EFX_PKT_IPV6; 796 } else { 797 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4); 798 flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 799 } 800 break; 801 802 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER: 803 if (is_v6) { 804 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6); 805 flags = EFX_PKT_IPV6; 806 } else { 807 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4); 808 flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 809 } 810 break; 811 812 case FSE_AZ_RX_EV_HDR_TYPE_OTHER: 813 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP); 814 flags = 0; 815 break; 816 817 default: 818 EFSYS_ASSERT(B_FALSE); 819 flags = 0; 820 break; 821 } 822 823 #if EFSYS_OPT_RX_SCATTER 824 /* Report scatter and header/lookahead split buffer flags */ 825 if (sop) 826 flags |= EFX_PKT_START; 827 if (jumbo_cont) 828 flags |= EFX_PKT_CONT; 829 #endif /* EFSYS_OPT_RX_SCATTER */ 830 831 /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */ 832 if (!ok) { 833 ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags); 834 if (ignore) { 835 EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, 836 uint32_t, size, uint16_t, flags); 837 838 return (B_FALSE); 839 } 840 } 841 842 /* If we're not discarding the packet then it is ok */ 843 if (~flags & EFX_DISCARD) 844 EFX_EV_QSTAT_INCR(eep, EV_RX_OK); 845 846 /* Detect multicast packets that didn't match the filter */ 847 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) { 848 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT); 849 850 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) { 851 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH); 852 } else { 853 EFSYS_PROBE(mcast_mismatch); 854 flags |= EFX_ADDR_MISMATCH; 855 } 856 } else { 857 flags |= EFX_PKT_UNICAST; 858 } 859 860 /* 861 * The packet parser in Siena can abort parsing packets under 862 * certain error conditions, setting the PKT_NOT_PARSED bit 863 * (which clears PKT_OK). If this is set, then don't trust 864 * the PKT_TYPE field. 865 */ 866 if (!ok) { 867 uint32_t parse_err; 868 869 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED); 870 if (parse_err != 0) 871 flags |= EFX_CHECK_VLAN; 872 } 873 874 if (~flags & EFX_CHECK_VLAN) { 875 uint32_t pkt_type; 876 877 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE); 878 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN) 879 flags |= EFX_PKT_VLAN_TAGGED; 880 } 881 882 EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, 883 uint32_t, size, uint16_t, flags); 884 885 EFSYS_ASSERT(eecp->eec_rx != NULL); 886 should_abort = eecp->eec_rx(arg, label, id, size, flags); 887 888 return (should_abort); 889 } 890 891 static __checkReturn boolean_t 892 siena_ev_tx( 893 __in efx_evq_t *eep, 894 __in efx_qword_t *eqp, 895 __in const efx_ev_callbacks_t *eecp, 896 __in_opt void *arg) 897 { 898 uint32_t id; 899 uint32_t label; 900 boolean_t should_abort; 901 902 EFX_EV_QSTAT_INCR(eep, EV_TX); 903 904 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 && 905 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 && 906 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 && 907 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) { 908 909 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR); 910 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL); 911 912 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id); 913 914 EFSYS_ASSERT(eecp->eec_tx != NULL); 915 should_abort = eecp->eec_tx(arg, label, id); 916 917 return (should_abort); 918 } 919 920 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0) 921 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 922 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 923 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 924 925 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0) 926 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR); 927 928 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0) 929 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG); 930 931 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0) 932 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL); 933 934 EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED); 935 return (B_FALSE); 936 } 937 938 static __checkReturn boolean_t 939 siena_ev_global( 940 __in efx_evq_t *eep, 941 __in efx_qword_t *eqp, 942 __in const efx_ev_callbacks_t *eecp, 943 __in_opt void *arg) 944 { 945 _NOTE(ARGUNUSED(eqp, eecp, arg)) 946 947 EFX_EV_QSTAT_INCR(eep, EV_GLOBAL); 948 949 return (B_FALSE); 950 } 951 952 static __checkReturn boolean_t 953 siena_ev_driver( 954 __in efx_evq_t *eep, 955 __in efx_qword_t *eqp, 956 __in const efx_ev_callbacks_t *eecp, 957 __in_opt void *arg) 958 { 959 boolean_t should_abort; 960 961 EFX_EV_QSTAT_INCR(eep, EV_DRIVER); 962 should_abort = B_FALSE; 963 964 switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) { 965 case FSE_AZ_TX_DESCQ_FLS_DONE_EV: { 966 uint32_t txq_index; 967 968 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE); 969 970 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 971 972 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index); 973 974 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL); 975 should_abort = eecp->eec_txq_flush_done(arg, txq_index); 976 977 break; 978 } 979 case FSE_AZ_RX_DESCQ_FLS_DONE_EV: { 980 uint32_t rxq_index; 981 uint32_t failed; 982 983 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); 984 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); 985 986 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL); 987 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL); 988 989 if (failed) { 990 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED); 991 992 EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index); 993 994 should_abort = eecp->eec_rxq_flush_failed(arg, 995 rxq_index); 996 } else { 997 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE); 998 999 EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index); 1000 1001 should_abort = eecp->eec_rxq_flush_done(arg, rxq_index); 1002 } 1003 1004 break; 1005 } 1006 case FSE_AZ_EVQ_INIT_DONE_EV: 1007 EFSYS_ASSERT(eecp->eec_initialized != NULL); 1008 should_abort = eecp->eec_initialized(arg); 1009 1010 break; 1011 1012 case FSE_AZ_EVQ_NOT_EN_EV: 1013 EFSYS_PROBE(evq_not_en); 1014 break; 1015 1016 case FSE_AZ_SRM_UPD_DONE_EV: { 1017 uint32_t code; 1018 1019 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE); 1020 1021 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 1022 1023 EFSYS_ASSERT(eecp->eec_sram != NULL); 1024 should_abort = eecp->eec_sram(arg, code); 1025 1026 break; 1027 } 1028 case FSE_AZ_WAKE_UP_EV: { 1029 uint32_t id; 1030 1031 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 1032 1033 EFSYS_ASSERT(eecp->eec_wake_up != NULL); 1034 should_abort = eecp->eec_wake_up(arg, id); 1035 1036 break; 1037 } 1038 case FSE_AZ_TX_PKT_NON_TCP_UDP: 1039 EFSYS_PROBE(tx_pkt_non_tcp_udp); 1040 break; 1041 1042 case FSE_AZ_TIMER_EV: { 1043 uint32_t id; 1044 1045 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 1046 1047 EFSYS_ASSERT(eecp->eec_timer != NULL); 1048 should_abort = eecp->eec_timer(arg, id); 1049 1050 break; 1051 } 1052 case FSE_AZ_RX_DSC_ERROR_EV: 1053 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR); 1054 1055 EFSYS_PROBE(rx_dsc_error); 1056 1057 EFSYS_ASSERT(eecp->eec_exception != NULL); 1058 should_abort = eecp->eec_exception(arg, 1059 EFX_EXCEPTION_RX_DSC_ERROR, 0); 1060 1061 break; 1062 1063 case FSE_AZ_TX_DSC_ERROR_EV: 1064 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR); 1065 1066 EFSYS_PROBE(tx_dsc_error); 1067 1068 EFSYS_ASSERT(eecp->eec_exception != NULL); 1069 should_abort = eecp->eec_exception(arg, 1070 EFX_EXCEPTION_TX_DSC_ERROR, 0); 1071 1072 break; 1073 1074 default: 1075 break; 1076 } 1077 1078 return (should_abort); 1079 } 1080 1081 static __checkReturn boolean_t 1082 siena_ev_drv_gen( 1083 __in efx_evq_t *eep, 1084 __in efx_qword_t *eqp, 1085 __in const efx_ev_callbacks_t *eecp, 1086 __in_opt void *arg) 1087 { 1088 uint32_t data; 1089 boolean_t should_abort; 1090 1091 EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN); 1092 1093 data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0); 1094 if (data >= ((uint32_t)1 << 16)) { 1095 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 1096 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 1097 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 1098 return (B_TRUE); 1099 } 1100 1101 EFSYS_ASSERT(eecp->eec_software != NULL); 1102 should_abort = eecp->eec_software(arg, (uint16_t)data); 1103 1104 return (should_abort); 1105 } 1106 1107 #if EFSYS_OPT_MCDI 1108 1109 static __checkReturn boolean_t 1110 siena_ev_mcdi( 1111 __in efx_evq_t *eep, 1112 __in efx_qword_t *eqp, 1113 __in const efx_ev_callbacks_t *eecp, 1114 __in_opt void *arg) 1115 { 1116 efx_nic_t *enp = eep->ee_enp; 1117 unsigned int code; 1118 boolean_t should_abort = B_FALSE; 1119 1120 EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); 1121 1122 if (enp->en_family != EFX_FAMILY_SIENA) 1123 goto out; 1124 1125 EFSYS_ASSERT(eecp->eec_link_change != NULL); 1126 EFSYS_ASSERT(eecp->eec_exception != NULL); 1127 #if EFSYS_OPT_MON_STATS 1128 EFSYS_ASSERT(eecp->eec_monitor != NULL); 1129 #endif 1130 1131 EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE); 1132 1133 code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE); 1134 switch (code) { 1135 case MCDI_EVENT_CODE_BADSSERT: 1136 efx_mcdi_ev_death(enp, EINTR); 1137 break; 1138 1139 case MCDI_EVENT_CODE_CMDDONE: 1140 efx_mcdi_ev_cpl(enp, 1141 MCDI_EV_FIELD(eqp, CMDDONE_SEQ), 1142 MCDI_EV_FIELD(eqp, CMDDONE_DATALEN), 1143 MCDI_EV_FIELD(eqp, CMDDONE_ERRNO)); 1144 break; 1145 1146 case MCDI_EVENT_CODE_LINKCHANGE: { 1147 efx_link_mode_t link_mode; 1148 1149 siena_phy_link_ev(enp, eqp, &link_mode); 1150 should_abort = eecp->eec_link_change(arg, link_mode); 1151 break; 1152 } 1153 case MCDI_EVENT_CODE_SENSOREVT: { 1154 #if EFSYS_OPT_MON_STATS 1155 efx_mon_stat_t id; 1156 efx_mon_stat_value_t value; 1157 efx_rc_t rc; 1158 1159 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0) 1160 should_abort = eecp->eec_monitor(arg, id, value); 1161 else if (rc == ENOTSUP) { 1162 should_abort = eecp->eec_exception(arg, 1163 EFX_EXCEPTION_UNKNOWN_SENSOREVT, 1164 MCDI_EV_FIELD(eqp, DATA)); 1165 } else 1166 EFSYS_ASSERT(rc == ENODEV); /* Wrong port */ 1167 #else 1168 should_abort = B_FALSE; 1169 #endif 1170 break; 1171 } 1172 case MCDI_EVENT_CODE_SCHEDERR: 1173 /* Informational only */ 1174 break; 1175 1176 case MCDI_EVENT_CODE_REBOOT: 1177 efx_mcdi_ev_death(enp, EIO); 1178 break; 1179 1180 case MCDI_EVENT_CODE_MAC_STATS_DMA: 1181 #if EFSYS_OPT_MAC_STATS 1182 if (eecp->eec_mac_stats != NULL) { 1183 eecp->eec_mac_stats(arg, 1184 MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION)); 1185 } 1186 #endif 1187 break; 1188 1189 case MCDI_EVENT_CODE_FWALERT: { 1190 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON); 1191 1192 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS) 1193 should_abort = eecp->eec_exception(arg, 1194 EFX_EXCEPTION_FWALERT_SRAM, 1195 MCDI_EV_FIELD(eqp, FWALERT_DATA)); 1196 else 1197 should_abort = eecp->eec_exception(arg, 1198 EFX_EXCEPTION_UNKNOWN_FWALERT, 1199 MCDI_EV_FIELD(eqp, DATA)); 1200 break; 1201 } 1202 1203 default: 1204 EFSYS_PROBE1(mc_pcol_error, int, code); 1205 break; 1206 } 1207 1208 out: 1209 return (should_abort); 1210 } 1211 1212 #endif /* EFSYS_OPT_MCDI */ 1213 1214 static __checkReturn efx_rc_t 1215 siena_ev_qprime( 1216 __in efx_evq_t *eep, 1217 __in unsigned int count) 1218 { 1219 efx_nic_t *enp = eep->ee_enp; 1220 uint32_t rptr; 1221 efx_dword_t dword; 1222 1223 rptr = count & eep->ee_mask; 1224 1225 EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr); 1226 1227 EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index, 1228 &dword, B_FALSE); 1229 1230 return (0); 1231 } 1232 1233 static void 1234 siena_ev_qpost( 1235 __in efx_evq_t *eep, 1236 __in uint16_t data) 1237 { 1238 efx_nic_t *enp = eep->ee_enp; 1239 efx_qword_t ev; 1240 efx_oword_t oword; 1241 1242 EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV, 1243 FSF_AZ_EV_DATA_DW0, (uint32_t)data); 1244 1245 EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index, 1246 EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0), 1247 EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1)); 1248 1249 EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword); 1250 } 1251 1252 static __checkReturn efx_rc_t 1253 siena_ev_qmoderate( 1254 __in efx_evq_t *eep, 1255 __in unsigned int us) 1256 { 1257 efx_nic_t *enp = eep->ee_enp; 1258 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1259 unsigned int locked; 1260 efx_dword_t dword; 1261 efx_rc_t rc; 1262 1263 if (us > encp->enc_evq_timer_max_us) { 1264 rc = EINVAL; 1265 goto fail1; 1266 } 1267 1268 /* If the value is zero then disable the timer */ 1269 if (us == 0) { 1270 EFX_POPULATE_DWORD_2(dword, 1271 FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS, 1272 FRF_CZ_TC_TIMER_VAL, 0); 1273 } else { 1274 unsigned int ticks; 1275 1276 if ((rc = efx_ev_usecs_to_ticks(enp, us, &ticks)) != 0) 1277 goto fail2; 1278 1279 EFSYS_ASSERT(ticks > 0); 1280 EFX_POPULATE_DWORD_2(dword, 1281 FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF, 1282 FRF_CZ_TC_TIMER_VAL, ticks - 1); 1283 } 1284 1285 locked = (eep->ee_index == 0) ? 1 : 0; 1286 1287 EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0, 1288 eep->ee_index, &dword, locked); 1289 1290 return (0); 1291 1292 fail2: 1293 EFSYS_PROBE(fail2); 1294 fail1: 1295 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1296 1297 return (rc); 1298 } 1299 1300 static __checkReturn efx_rc_t 1301 siena_ev_qcreate( 1302 __in efx_nic_t *enp, 1303 __in unsigned int index, 1304 __in efsys_mem_t *esmp, 1305 __in size_t ndescs, 1306 __in uint32_t id, 1307 __in uint32_t us, 1308 __in uint32_t flags, 1309 __in efx_evq_t *eep) 1310 { 1311 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1312 uint32_t size; 1313 efx_oword_t oword; 1314 efx_rc_t rc; 1315 boolean_t notify_mode; 1316 1317 _NOTE(ARGUNUSED(esmp)) 1318 1319 if (index >= encp->enc_evq_limit) { 1320 rc = EINVAL; 1321 goto fail1; 1322 } 1323 #if EFSYS_OPT_RX_SCALE 1324 if (enp->en_intr.ei_type == EFX_INTR_LINE && 1325 index >= EFX_MAXRSS_LEGACY) { 1326 rc = EINVAL; 1327 goto fail2; 1328 } 1329 #endif 1330 for (size = 0; 1331 (1U << size) <= encp->enc_evq_max_nevs / encp->enc_evq_min_nevs; 1332 size++) 1333 if ((1U << size) == (uint32_t)ndescs / encp->enc_evq_min_nevs) 1334 break; 1335 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1336 rc = EINVAL; 1337 goto fail3; 1338 } 1339 1340 /* Set up the handler table */ 1341 eep->ee_rx = siena_ev_rx; 1342 eep->ee_tx = siena_ev_tx; 1343 eep->ee_driver = siena_ev_driver; 1344 eep->ee_global = siena_ev_global; 1345 eep->ee_drv_gen = siena_ev_drv_gen; 1346 #if EFSYS_OPT_MCDI 1347 eep->ee_mcdi = siena_ev_mcdi; 1348 #endif /* EFSYS_OPT_MCDI */ 1349 1350 notify_mode = ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) != 1351 EFX_EVQ_FLAGS_NOTIFY_INTERRUPT); 1352 1353 /* Set up the new event queue */ 1354 EFX_POPULATE_OWORD_3(oword, FRF_CZ_TIMER_Q_EN, 1, 1355 FRF_CZ_HOST_NOTIFY_MODE, notify_mode, 1356 FRF_CZ_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS); 1357 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE); 1358 1359 EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size, 1360 FRF_AZ_EVQ_BUF_BASE_ID, id); 1361 1362 EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE); 1363 1364 /* Set initial interrupt moderation */ 1365 siena_ev_qmoderate(eep, us); 1366 1367 return (0); 1368 1369 fail3: 1370 EFSYS_PROBE(fail3); 1371 #if EFSYS_OPT_RX_SCALE 1372 fail2: 1373 EFSYS_PROBE(fail2); 1374 #endif 1375 fail1: 1376 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1377 1378 return (rc); 1379 } 1380 1381 #endif /* EFSYS_OPT_SIENA */ 1382 1383 #if EFSYS_OPT_QSTATS 1384 #if EFSYS_OPT_NAMES 1385 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock ac223f7134058b4f */ 1386 static const char * const __efx_ev_qstat_name[] = { 1387 "all", 1388 "rx", 1389 "rx_ok", 1390 "rx_frm_trunc", 1391 "rx_tobe_disc", 1392 "rx_pause_frm_err", 1393 "rx_buf_owner_id_err", 1394 "rx_ipv4_hdr_chksum_err", 1395 "rx_tcp_udp_chksum_err", 1396 "rx_eth_crc_err", 1397 "rx_ip_frag_err", 1398 "rx_mcast_pkt", 1399 "rx_mcast_hash_match", 1400 "rx_tcp_ipv4", 1401 "rx_tcp_ipv6", 1402 "rx_udp_ipv4", 1403 "rx_udp_ipv6", 1404 "rx_other_ipv4", 1405 "rx_other_ipv6", 1406 "rx_non_ip", 1407 "rx_batch", 1408 "tx", 1409 "tx_wq_ff_full", 1410 "tx_pkt_err", 1411 "tx_pkt_too_big", 1412 "tx_unexpected", 1413 "global", 1414 "global_mnt", 1415 "driver", 1416 "driver_srm_upd_done", 1417 "driver_tx_descq_fls_done", 1418 "driver_rx_descq_fls_done", 1419 "driver_rx_descq_fls_failed", 1420 "driver_rx_dsc_error", 1421 "driver_tx_dsc_error", 1422 "drv_gen", 1423 "mcdi_response", 1424 "rx_parse_incomplete", 1425 }; 1426 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */ 1427 1428 const char * 1429 efx_ev_qstat_name( 1430 __in efx_nic_t *enp, 1431 __in unsigned int id) 1432 { 1433 _NOTE(ARGUNUSED(enp)) 1434 1435 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1436 EFSYS_ASSERT3U(id, <, EV_NQSTATS); 1437 1438 return (__efx_ev_qstat_name[id]); 1439 } 1440 #endif /* EFSYS_OPT_NAMES */ 1441 #endif /* EFSYS_OPT_QSTATS */ 1442 1443 #if EFSYS_OPT_SIENA 1444 1445 #if EFSYS_OPT_QSTATS 1446 static void 1447 siena_ev_qstats_update( 1448 __in efx_evq_t *eep, 1449 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) 1450 { 1451 unsigned int id; 1452 1453 for (id = 0; id < EV_NQSTATS; id++) { 1454 efsys_stat_t *essp = &stat[id]; 1455 1456 EFSYS_STAT_INCR(essp, eep->ee_stat[id]); 1457 eep->ee_stat[id] = 0; 1458 } 1459 } 1460 #endif /* EFSYS_OPT_QSTATS */ 1461 1462 static void 1463 siena_ev_qdestroy( 1464 __in efx_evq_t *eep) 1465 { 1466 efx_nic_t *enp = eep->ee_enp; 1467 efx_oword_t oword; 1468 1469 /* Purge event queue */ 1470 EFX_ZERO_OWORD(oword); 1471 1472 EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, 1473 eep->ee_index, &oword, B_TRUE); 1474 1475 EFX_ZERO_OWORD(oword); 1476 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE); 1477 } 1478 1479 static void 1480 siena_ev_fini( 1481 __in efx_nic_t *enp) 1482 { 1483 _NOTE(ARGUNUSED(enp)) 1484 } 1485 1486 #endif /* EFSYS_OPT_SIENA */ 1487