1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2007-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 #if EFSYS_OPT_QSTATS 11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 12 do { \ 13 (_etp)->et_stat[_stat]++; \ 14 _NOTE(CONSTANTCONDITION) \ 15 } while (B_FALSE) 16 #else 17 #define EFX_TX_QSTAT_INCR(_etp, _stat) 18 #endif 19 20 #if EFSYS_OPT_SIENA 21 22 static __checkReturn efx_rc_t 23 siena_tx_init( 24 __in efx_nic_t *enp); 25 26 static void 27 siena_tx_fini( 28 __in efx_nic_t *enp); 29 30 static __checkReturn efx_rc_t 31 siena_tx_qcreate( 32 __in efx_nic_t *enp, 33 __in unsigned int index, 34 __in unsigned int label, 35 __in efsys_mem_t *esmp, 36 __in size_t ndescs, 37 __in uint32_t id, 38 __in uint16_t flags, 39 __in efx_evq_t *eep, 40 __in efx_txq_t *etp, 41 __out unsigned int *addedp); 42 43 static void 44 siena_tx_qdestroy( 45 __in efx_txq_t *etp); 46 47 static __checkReturn efx_rc_t 48 siena_tx_qpost( 49 __in efx_txq_t *etp, 50 __in_ecount(ndescs) efx_buffer_t *eb, 51 __in unsigned int ndescs, 52 __in unsigned int completed, 53 __inout unsigned int *addedp); 54 55 static void 56 siena_tx_qpush( 57 __in efx_txq_t *etp, 58 __in unsigned int added, 59 __in unsigned int pushed); 60 61 static __checkReturn efx_rc_t 62 siena_tx_qpace( 63 __in efx_txq_t *etp, 64 __in unsigned int ns); 65 66 static __checkReturn efx_rc_t 67 siena_tx_qflush( 68 __in efx_txq_t *etp); 69 70 static void 71 siena_tx_qenable( 72 __in efx_txq_t *etp); 73 74 __checkReturn efx_rc_t 75 siena_tx_qdesc_post( 76 __in efx_txq_t *etp, 77 __in_ecount(ndescs) efx_desc_t *ed, 78 __in unsigned int ndescs, 79 __in unsigned int completed, 80 __inout unsigned int *addedp); 81 82 void 83 siena_tx_qdesc_dma_create( 84 __in efx_txq_t *etp, 85 __in efsys_dma_addr_t addr, 86 __in size_t size, 87 __in boolean_t eop, 88 __out efx_desc_t *edp); 89 90 #if EFSYS_OPT_QSTATS 91 static void 92 siena_tx_qstats_update( 93 __in efx_txq_t *etp, 94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 95 #endif 96 97 #endif /* EFSYS_OPT_SIENA */ 98 99 100 #if EFSYS_OPT_SIENA 101 static const efx_tx_ops_t __efx_tx_siena_ops = { 102 siena_tx_init, /* etxo_init */ 103 siena_tx_fini, /* etxo_fini */ 104 siena_tx_qcreate, /* etxo_qcreate */ 105 siena_tx_qdestroy, /* etxo_qdestroy */ 106 siena_tx_qpost, /* etxo_qpost */ 107 siena_tx_qpush, /* etxo_qpush */ 108 siena_tx_qpace, /* etxo_qpace */ 109 siena_tx_qflush, /* etxo_qflush */ 110 siena_tx_qenable, /* etxo_qenable */ 111 NULL, /* etxo_qpio_enable */ 112 NULL, /* etxo_qpio_disable */ 113 NULL, /* etxo_qpio_write */ 114 NULL, /* etxo_qpio_post */ 115 siena_tx_qdesc_post, /* etxo_qdesc_post */ 116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 117 NULL, /* etxo_qdesc_tso_create */ 118 NULL, /* etxo_qdesc_tso2_create */ 119 NULL, /* etxo_qdesc_vlantci_create */ 120 NULL, /* etxo_qdesc_checksum_create */ 121 #if EFSYS_OPT_QSTATS 122 siena_tx_qstats_update, /* etxo_qstats_update */ 123 #endif 124 }; 125 #endif /* EFSYS_OPT_SIENA */ 126 127 #if EFSYS_OPT_HUNTINGTON 128 static const efx_tx_ops_t __efx_tx_hunt_ops = { 129 ef10_tx_init, /* etxo_init */ 130 ef10_tx_fini, /* etxo_fini */ 131 ef10_tx_qcreate, /* etxo_qcreate */ 132 ef10_tx_qdestroy, /* etxo_qdestroy */ 133 ef10_tx_qpost, /* etxo_qpost */ 134 ef10_tx_qpush, /* etxo_qpush */ 135 ef10_tx_qpace, /* etxo_qpace */ 136 ef10_tx_qflush, /* etxo_qflush */ 137 ef10_tx_qenable, /* etxo_qenable */ 138 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 139 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 140 ef10_tx_qpio_write, /* etxo_qpio_write */ 141 ef10_tx_qpio_post, /* etxo_qpio_post */ 142 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 148 #if EFSYS_OPT_QSTATS 149 ef10_tx_qstats_update, /* etxo_qstats_update */ 150 #endif 151 }; 152 #endif /* EFSYS_OPT_HUNTINGTON */ 153 154 #if EFSYS_OPT_MEDFORD 155 static const efx_tx_ops_t __efx_tx_medford_ops = { 156 ef10_tx_init, /* etxo_init */ 157 ef10_tx_fini, /* etxo_fini */ 158 ef10_tx_qcreate, /* etxo_qcreate */ 159 ef10_tx_qdestroy, /* etxo_qdestroy */ 160 ef10_tx_qpost, /* etxo_qpost */ 161 ef10_tx_qpush, /* etxo_qpush */ 162 ef10_tx_qpace, /* etxo_qpace */ 163 ef10_tx_qflush, /* etxo_qflush */ 164 ef10_tx_qenable, /* etxo_qenable */ 165 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 166 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 167 ef10_tx_qpio_write, /* etxo_qpio_write */ 168 ef10_tx_qpio_post, /* etxo_qpio_post */ 169 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 171 NULL, /* etxo_qdesc_tso_create */ 172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 175 #if EFSYS_OPT_QSTATS 176 ef10_tx_qstats_update, /* etxo_qstats_update */ 177 #endif 178 }; 179 #endif /* EFSYS_OPT_MEDFORD */ 180 181 #if EFSYS_OPT_MEDFORD2 182 static const efx_tx_ops_t __efx_tx_medford2_ops = { 183 ef10_tx_init, /* etxo_init */ 184 ef10_tx_fini, /* etxo_fini */ 185 ef10_tx_qcreate, /* etxo_qcreate */ 186 ef10_tx_qdestroy, /* etxo_qdestroy */ 187 ef10_tx_qpost, /* etxo_qpost */ 188 ef10_tx_qpush, /* etxo_qpush */ 189 ef10_tx_qpace, /* etxo_qpace */ 190 ef10_tx_qflush, /* etxo_qflush */ 191 ef10_tx_qenable, /* etxo_qenable */ 192 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 193 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 194 ef10_tx_qpio_write, /* etxo_qpio_write */ 195 ef10_tx_qpio_post, /* etxo_qpio_post */ 196 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 198 NULL, /* etxo_qdesc_tso_create */ 199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 202 #if EFSYS_OPT_QSTATS 203 ef10_tx_qstats_update, /* etxo_qstats_update */ 204 #endif 205 }; 206 #endif /* EFSYS_OPT_MEDFORD2 */ 207 208 209 __checkReturn efx_rc_t 210 efx_tx_init( 211 __in efx_nic_t *enp) 212 { 213 const efx_tx_ops_t *etxop; 214 efx_rc_t rc; 215 216 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 217 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 218 219 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 220 rc = EINVAL; 221 goto fail1; 222 } 223 224 if (enp->en_mod_flags & EFX_MOD_TX) { 225 rc = EINVAL; 226 goto fail2; 227 } 228 229 switch (enp->en_family) { 230 #if EFSYS_OPT_SIENA 231 case EFX_FAMILY_SIENA: 232 etxop = &__efx_tx_siena_ops; 233 break; 234 #endif /* EFSYS_OPT_SIENA */ 235 236 #if EFSYS_OPT_HUNTINGTON 237 case EFX_FAMILY_HUNTINGTON: 238 etxop = &__efx_tx_hunt_ops; 239 break; 240 #endif /* EFSYS_OPT_HUNTINGTON */ 241 242 #if EFSYS_OPT_MEDFORD 243 case EFX_FAMILY_MEDFORD: 244 etxop = &__efx_tx_medford_ops; 245 break; 246 #endif /* EFSYS_OPT_MEDFORD */ 247 248 #if EFSYS_OPT_MEDFORD2 249 case EFX_FAMILY_MEDFORD2: 250 etxop = &__efx_tx_medford2_ops; 251 break; 252 #endif /* EFSYS_OPT_MEDFORD2 */ 253 254 default: 255 EFSYS_ASSERT(0); 256 rc = ENOTSUP; 257 goto fail3; 258 } 259 260 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 261 262 if ((rc = etxop->etxo_init(enp)) != 0) 263 goto fail4; 264 265 enp->en_etxop = etxop; 266 enp->en_mod_flags |= EFX_MOD_TX; 267 return (0); 268 269 fail4: 270 EFSYS_PROBE(fail4); 271 fail3: 272 EFSYS_PROBE(fail3); 273 fail2: 274 EFSYS_PROBE(fail2); 275 fail1: 276 EFSYS_PROBE1(fail1, efx_rc_t, rc); 277 278 enp->en_etxop = NULL; 279 enp->en_mod_flags &= ~EFX_MOD_TX; 280 return (rc); 281 } 282 283 void 284 efx_tx_fini( 285 __in efx_nic_t *enp) 286 { 287 const efx_tx_ops_t *etxop = enp->en_etxop; 288 289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 291 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 292 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 293 294 etxop->etxo_fini(enp); 295 296 enp->en_etxop = NULL; 297 enp->en_mod_flags &= ~EFX_MOD_TX; 298 } 299 300 __checkReturn size_t 301 efx_txq_size( 302 __in const efx_nic_t *enp, 303 __in unsigned int ndescs) 304 { 305 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 306 307 return (ndescs * encp->enc_tx_desc_size); 308 } 309 310 __checkReturn unsigned int 311 efx_txq_nbufs( 312 __in const efx_nic_t *enp, 313 __in unsigned int ndescs) 314 { 315 return (EFX_DIV_ROUND_UP(efx_txq_size(enp, ndescs), EFX_BUF_SIZE)); 316 } 317 318 __checkReturn efx_rc_t 319 efx_tx_qcreate( 320 __in efx_nic_t *enp, 321 __in unsigned int index, 322 __in unsigned int label, 323 __in efsys_mem_t *esmp, 324 __in size_t ndescs, 325 __in uint32_t id, 326 __in uint16_t flags, 327 __in efx_evq_t *eep, 328 __deref_out efx_txq_t **etpp, 329 __out unsigned int *addedp) 330 { 331 const efx_tx_ops_t *etxop = enp->en_etxop; 332 efx_txq_t *etp; 333 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 334 efx_rc_t rc; 335 336 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 337 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 338 339 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, 340 enp->en_nic_cfg.enc_txq_limit); 341 342 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); 343 EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs)); 344 345 if (!ISP2(ndescs) || 346 ndescs < encp->enc_txq_min_ndescs || 347 ndescs > encp->enc_txq_max_ndescs) { 348 rc = EINVAL; 349 goto fail1; 350 } 351 352 /* Allocate an TXQ object */ 353 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 354 355 if (etp == NULL) { 356 rc = ENOMEM; 357 goto fail2; 358 } 359 360 etp->et_magic = EFX_TXQ_MAGIC; 361 etp->et_enp = enp; 362 etp->et_index = index; 363 etp->et_mask = ndescs - 1; 364 etp->et_esmp = esmp; 365 366 /* Initial descriptor index may be modified by etxo_qcreate */ 367 *addedp = 0; 368 369 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 370 ndescs, id, flags, eep, etp, addedp)) != 0) 371 goto fail3; 372 373 enp->en_tx_qcount++; 374 *etpp = etp; 375 376 return (0); 377 378 fail3: 379 EFSYS_PROBE(fail3); 380 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 381 fail2: 382 EFSYS_PROBE(fail2); 383 fail1: 384 EFSYS_PROBE1(fail1, efx_rc_t, rc); 385 return (rc); 386 } 387 388 void 389 efx_tx_qdestroy( 390 __in efx_txq_t *etp) 391 { 392 efx_nic_t *enp = etp->et_enp; 393 const efx_tx_ops_t *etxop = enp->en_etxop; 394 395 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 396 397 EFSYS_ASSERT(enp->en_tx_qcount != 0); 398 --enp->en_tx_qcount; 399 400 etxop->etxo_qdestroy(etp); 401 402 /* Free the TXQ object */ 403 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 404 } 405 406 __checkReturn efx_rc_t 407 efx_tx_qpost( 408 __in efx_txq_t *etp, 409 __in_ecount(ndescs) efx_buffer_t *eb, 410 __in unsigned int ndescs, 411 __in unsigned int completed, 412 __inout unsigned int *addedp) 413 { 414 efx_nic_t *enp = etp->et_enp; 415 const efx_tx_ops_t *etxop = enp->en_etxop; 416 efx_rc_t rc; 417 418 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 419 420 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0) 421 goto fail1; 422 423 return (0); 424 425 fail1: 426 EFSYS_PROBE1(fail1, efx_rc_t, rc); 427 return (rc); 428 } 429 430 void 431 efx_tx_qpush( 432 __in efx_txq_t *etp, 433 __in unsigned int added, 434 __in unsigned int pushed) 435 { 436 efx_nic_t *enp = etp->et_enp; 437 const efx_tx_ops_t *etxop = enp->en_etxop; 438 439 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 440 441 etxop->etxo_qpush(etp, added, pushed); 442 } 443 444 __checkReturn efx_rc_t 445 efx_tx_qpace( 446 __in efx_txq_t *etp, 447 __in unsigned int ns) 448 { 449 efx_nic_t *enp = etp->et_enp; 450 const efx_tx_ops_t *etxop = enp->en_etxop; 451 efx_rc_t rc; 452 453 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 454 455 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 456 goto fail1; 457 458 return (0); 459 460 fail1: 461 EFSYS_PROBE1(fail1, efx_rc_t, rc); 462 return (rc); 463 } 464 465 __checkReturn efx_rc_t 466 efx_tx_qflush( 467 __in efx_txq_t *etp) 468 { 469 efx_nic_t *enp = etp->et_enp; 470 const efx_tx_ops_t *etxop = enp->en_etxop; 471 efx_rc_t rc; 472 473 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 474 475 if ((rc = etxop->etxo_qflush(etp)) != 0) 476 goto fail1; 477 478 return (0); 479 480 fail1: 481 EFSYS_PROBE1(fail1, efx_rc_t, rc); 482 return (rc); 483 } 484 485 void 486 efx_tx_qenable( 487 __in efx_txq_t *etp) 488 { 489 efx_nic_t *enp = etp->et_enp; 490 const efx_tx_ops_t *etxop = enp->en_etxop; 491 492 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 493 494 etxop->etxo_qenable(etp); 495 } 496 497 __checkReturn efx_rc_t 498 efx_tx_qpio_enable( 499 __in efx_txq_t *etp) 500 { 501 efx_nic_t *enp = etp->et_enp; 502 const efx_tx_ops_t *etxop = enp->en_etxop; 503 efx_rc_t rc; 504 505 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 506 507 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 508 rc = ENOTSUP; 509 goto fail1; 510 } 511 if (etxop->etxo_qpio_enable == NULL) { 512 rc = ENOTSUP; 513 goto fail2; 514 } 515 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 516 goto fail3; 517 518 return (0); 519 520 fail3: 521 EFSYS_PROBE(fail3); 522 fail2: 523 EFSYS_PROBE(fail2); 524 fail1: 525 EFSYS_PROBE1(fail1, efx_rc_t, rc); 526 return (rc); 527 } 528 529 void 530 efx_tx_qpio_disable( 531 __in efx_txq_t *etp) 532 { 533 efx_nic_t *enp = etp->et_enp; 534 const efx_tx_ops_t *etxop = enp->en_etxop; 535 536 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 537 538 if (etxop->etxo_qpio_disable != NULL) 539 etxop->etxo_qpio_disable(etp); 540 } 541 542 __checkReturn efx_rc_t 543 efx_tx_qpio_write( 544 __in efx_txq_t *etp, 545 __in_ecount(buf_length) uint8_t *buffer, 546 __in size_t buf_length, 547 __in size_t pio_buf_offset) 548 { 549 efx_nic_t *enp = etp->et_enp; 550 const efx_tx_ops_t *etxop = enp->en_etxop; 551 efx_rc_t rc; 552 553 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 554 555 if (etxop->etxo_qpio_write != NULL) { 556 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 557 pio_buf_offset)) != 0) 558 goto fail1; 559 return (0); 560 } 561 562 return (ENOTSUP); 563 564 fail1: 565 EFSYS_PROBE1(fail1, efx_rc_t, rc); 566 return (rc); 567 } 568 569 __checkReturn efx_rc_t 570 efx_tx_qpio_post( 571 __in efx_txq_t *etp, 572 __in size_t pkt_length, 573 __in unsigned int completed, 574 __inout unsigned int *addedp) 575 { 576 efx_nic_t *enp = etp->et_enp; 577 const efx_tx_ops_t *etxop = enp->en_etxop; 578 efx_rc_t rc; 579 580 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 581 582 if (etxop->etxo_qpio_post != NULL) { 583 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 584 addedp)) != 0) 585 goto fail1; 586 return (0); 587 } 588 589 return (ENOTSUP); 590 591 fail1: 592 EFSYS_PROBE1(fail1, efx_rc_t, rc); 593 return (rc); 594 } 595 596 __checkReturn efx_rc_t 597 efx_tx_qdesc_post( 598 __in efx_txq_t *etp, 599 __in_ecount(ndescs) efx_desc_t *ed, 600 __in unsigned int ndescs, 601 __in unsigned int completed, 602 __inout unsigned int *addedp) 603 { 604 efx_nic_t *enp = etp->et_enp; 605 const efx_tx_ops_t *etxop = enp->en_etxop; 606 607 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 608 609 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp)); 610 } 611 612 void 613 efx_tx_qdesc_dma_create( 614 __in efx_txq_t *etp, 615 __in efsys_dma_addr_t addr, 616 __in size_t size, 617 __in boolean_t eop, 618 __out efx_desc_t *edp) 619 { 620 efx_nic_t *enp = etp->et_enp; 621 const efx_tx_ops_t *etxop = enp->en_etxop; 622 623 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 624 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 625 626 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 627 } 628 629 void 630 efx_tx_qdesc_tso_create( 631 __in efx_txq_t *etp, 632 __in uint16_t ipv4_id, 633 __in uint32_t tcp_seq, 634 __in uint8_t tcp_flags, 635 __out efx_desc_t *edp) 636 { 637 efx_nic_t *enp = etp->et_enp; 638 const efx_tx_ops_t *etxop = enp->en_etxop; 639 640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 641 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 642 643 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 644 } 645 646 void 647 efx_tx_qdesc_tso2_create( 648 __in efx_txq_t *etp, 649 __in uint16_t ipv4_id, 650 __in uint16_t outer_ipv4_id, 651 __in uint32_t tcp_seq, 652 __in uint16_t mss, 653 __out_ecount(count) efx_desc_t *edp, 654 __in int count) 655 { 656 efx_nic_t *enp = etp->et_enp; 657 const efx_tx_ops_t *etxop = enp->en_etxop; 658 659 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 660 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 661 662 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id, 663 tcp_seq, mss, edp, count); 664 } 665 666 void 667 efx_tx_qdesc_vlantci_create( 668 __in efx_txq_t *etp, 669 __in uint16_t tci, 670 __out efx_desc_t *edp) 671 { 672 efx_nic_t *enp = etp->et_enp; 673 const efx_tx_ops_t *etxop = enp->en_etxop; 674 675 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 676 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 677 678 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 679 } 680 681 void 682 efx_tx_qdesc_checksum_create( 683 __in efx_txq_t *etp, 684 __in uint16_t flags, 685 __out efx_desc_t *edp) 686 { 687 efx_nic_t *enp = etp->et_enp; 688 const efx_tx_ops_t *etxop = enp->en_etxop; 689 690 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 691 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL); 692 693 etxop->etxo_qdesc_checksum_create(etp, flags, edp); 694 } 695 696 697 #if EFSYS_OPT_QSTATS 698 void 699 efx_tx_qstats_update( 700 __in efx_txq_t *etp, 701 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 702 { 703 efx_nic_t *enp = etp->et_enp; 704 const efx_tx_ops_t *etxop = enp->en_etxop; 705 706 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 707 708 etxop->etxo_qstats_update(etp, stat); 709 } 710 #endif 711 712 713 #if EFSYS_OPT_SIENA 714 715 static __checkReturn efx_rc_t 716 siena_tx_init( 717 __in efx_nic_t *enp) 718 { 719 efx_oword_t oword; 720 721 /* 722 * Disable the timer-based TX DMA backoff and allow TX DMA to be 723 * controlled by the RX FIFO fill level (although always allow a 724 * minimal trickle). 725 */ 726 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 727 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 728 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 729 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 730 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 732 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 733 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 734 735 /* 736 * Filter all packets less than 14 bytes to avoid parsing 737 * errors. 738 */ 739 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 740 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 741 742 /* 743 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 744 * descriptors (which is bad). 745 */ 746 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 747 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 748 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 749 750 return (0); 751 } 752 753 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 754 do { \ 755 unsigned int id; \ 756 size_t offset; \ 757 efx_qword_t qword; \ 758 \ 759 id = (_added)++ & (_etp)->et_mask; \ 760 offset = id * sizeof (efx_qword_t); \ 761 \ 762 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 763 unsigned int, id, efsys_dma_addr_t, (_addr), \ 764 size_t, (_size), boolean_t, (_eop)); \ 765 \ 766 EFX_POPULATE_QWORD_4(qword, \ 767 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 768 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 769 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 770 (uint32_t)((_addr) & 0xffffffff), \ 771 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 772 (uint32_t)((_addr) >> 32)); \ 773 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 774 \ 775 _NOTE(CONSTANTCONDITION) \ 776 } while (B_FALSE) 777 778 static __checkReturn efx_rc_t 779 siena_tx_qpost( 780 __in efx_txq_t *etp, 781 __in_ecount(ndescs) efx_buffer_t *eb, 782 __in unsigned int ndescs, 783 __in unsigned int completed, 784 __inout unsigned int *addedp) 785 { 786 unsigned int added = *addedp; 787 unsigned int i; 788 789 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) 790 return (ENOSPC); 791 792 for (i = 0; i < ndescs; i++) { 793 efx_buffer_t *ebp = &eb[i]; 794 efsys_dma_addr_t start = ebp->eb_addr; 795 size_t size = ebp->eb_size; 796 efsys_dma_addr_t end = start + size; 797 798 /* 799 * Fragments must not span 4k boundaries. 800 * Here it is a stricter requirement than the maximum length. 801 */ 802 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1, 803 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 804 805 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 806 } 807 808 EFX_TX_QSTAT_INCR(etp, TX_POST); 809 810 *addedp = added; 811 return (0); 812 } 813 814 static void 815 siena_tx_qpush( 816 __in efx_txq_t *etp, 817 __in unsigned int added, 818 __in unsigned int pushed) 819 { 820 efx_nic_t *enp = etp->et_enp; 821 uint32_t wptr; 822 efx_dword_t dword; 823 efx_oword_t oword; 824 825 /* Push the populated descriptors out */ 826 wptr = added & etp->et_mask; 827 828 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 829 830 /* Only write the third DWORD */ 831 EFX_POPULATE_DWORD_1(dword, 832 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 833 834 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 835 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 836 wptr, pushed & etp->et_mask); 837 EFSYS_PIO_WRITE_BARRIER(); 838 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 839 etp->et_index, &dword, B_FALSE); 840 } 841 842 #define EFX_MAX_PACE_VALUE 20 843 844 static __checkReturn efx_rc_t 845 siena_tx_qpace( 846 __in efx_txq_t *etp, 847 __in unsigned int ns) 848 { 849 efx_nic_t *enp = etp->et_enp; 850 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 851 efx_oword_t oword; 852 unsigned int pace_val; 853 unsigned int timer_period; 854 efx_rc_t rc; 855 856 if (ns == 0) { 857 pace_val = 0; 858 } else { 859 /* 860 * The pace_val to write into the table is s.t 861 * ns <= timer_period * (2 ^ pace_val) 862 */ 863 timer_period = 104 / encp->enc_clk_mult; 864 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 865 if ((timer_period << pace_val) >= ns) 866 break; 867 } 868 } 869 if (pace_val > EFX_MAX_PACE_VALUE) { 870 rc = EINVAL; 871 goto fail1; 872 } 873 874 /* Update the pacing table */ 875 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 876 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 877 &oword, B_TRUE); 878 879 return (0); 880 881 fail1: 882 EFSYS_PROBE1(fail1, efx_rc_t, rc); 883 884 return (rc); 885 } 886 887 static __checkReturn efx_rc_t 888 siena_tx_qflush( 889 __in efx_txq_t *etp) 890 { 891 efx_nic_t *enp = etp->et_enp; 892 efx_oword_t oword; 893 uint32_t label; 894 895 efx_tx_qpace(etp, 0); 896 897 label = etp->et_index; 898 899 /* Flush the queue */ 900 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 901 FRF_AZ_TX_FLUSH_DESCQ, label); 902 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 903 904 return (0); 905 } 906 907 static void 908 siena_tx_qenable( 909 __in efx_txq_t *etp) 910 { 911 efx_nic_t *enp = etp->et_enp; 912 efx_oword_t oword; 913 914 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 915 etp->et_index, &oword, B_TRUE); 916 917 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 918 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 919 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 920 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 921 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 922 923 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 924 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 925 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 926 927 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 928 etp->et_index, &oword, B_TRUE); 929 } 930 931 static __checkReturn efx_rc_t 932 siena_tx_qcreate( 933 __in efx_nic_t *enp, 934 __in unsigned int index, 935 __in unsigned int label, 936 __in efsys_mem_t *esmp, 937 __in size_t ndescs, 938 __in uint32_t id, 939 __in uint16_t flags, 940 __in efx_evq_t *eep, 941 __in efx_txq_t *etp, 942 __out unsigned int *addedp) 943 { 944 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 945 efx_oword_t oword; 946 uint32_t size; 947 uint16_t inner_csum; 948 efx_rc_t rc; 949 950 _NOTE(ARGUNUSED(esmp)) 951 952 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 953 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 954 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 955 956 if (index >= encp->enc_txq_limit) { 957 rc = EINVAL; 958 goto fail1; 959 } 960 for (size = 0; 961 (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs; 962 size++) 963 if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs) 964 break; 965 if (id + (1 << size) >= encp->enc_buftbl_limit) { 966 rc = EINVAL; 967 goto fail2; 968 } 969 970 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; 971 if ((flags & inner_csum) != 0) { 972 rc = EINVAL; 973 goto fail3; 974 } 975 976 /* Set up the new descriptor queue */ 977 *addedp = 0; 978 979 EFX_POPULATE_OWORD_6(oword, 980 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 981 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 982 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 983 FRF_AZ_TX_DESCQ_LABEL, label, 984 FRF_AZ_TX_DESCQ_SIZE, size, 985 FRF_AZ_TX_DESCQ_TYPE, 0); 986 987 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 988 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 989 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 990 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 991 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 992 993 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 994 etp->et_index, &oword, B_TRUE); 995 996 return (0); 997 998 fail3: 999 EFSYS_PROBE(fail3); 1000 fail2: 1001 EFSYS_PROBE(fail2); 1002 fail1: 1003 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1004 1005 return (rc); 1006 } 1007 1008 __checkReturn efx_rc_t 1009 siena_tx_qdesc_post( 1010 __in efx_txq_t *etp, 1011 __in_ecount(ndescs) efx_desc_t *ed, 1012 __in unsigned int ndescs, 1013 __in unsigned int completed, 1014 __inout unsigned int *addedp) 1015 { 1016 unsigned int added = *addedp; 1017 unsigned int i; 1018 efx_rc_t rc; 1019 1020 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 1021 rc = ENOSPC; 1022 goto fail1; 1023 } 1024 1025 for (i = 0; i < ndescs; i++) { 1026 efx_desc_t *edp = &ed[i]; 1027 unsigned int id; 1028 size_t offset; 1029 1030 id = added++ & etp->et_mask; 1031 offset = id * sizeof (efx_desc_t); 1032 1033 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 1034 } 1035 1036 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1037 unsigned int, added, unsigned int, ndescs); 1038 1039 EFX_TX_QSTAT_INCR(etp, TX_POST); 1040 1041 *addedp = added; 1042 return (0); 1043 1044 fail1: 1045 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1046 return (rc); 1047 } 1048 1049 void 1050 siena_tx_qdesc_dma_create( 1051 __in efx_txq_t *etp, 1052 __in efsys_dma_addr_t addr, 1053 __in size_t size, 1054 __in boolean_t eop, 1055 __out efx_desc_t *edp) 1056 { 1057 /* 1058 * Fragments must not span 4k boundaries. 1059 * Here it is a stricter requirement than the maximum length. 1060 */ 1061 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1, 1062 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1063 1064 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1065 efsys_dma_addr_t, addr, 1066 size_t, size, boolean_t, eop); 1067 1068 EFX_POPULATE_QWORD_4(edp->ed_eq, 1069 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1070 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1071 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1072 (uint32_t)(addr & 0xffffffff), 1073 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1074 (uint32_t)(addr >> 32)); 1075 } 1076 1077 #endif /* EFSYS_OPT_SIENA */ 1078 1079 #if EFSYS_OPT_QSTATS 1080 #if EFSYS_OPT_NAMES 1081 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1082 static const char * const __efx_tx_qstat_name[] = { 1083 "post", 1084 "post_pio", 1085 }; 1086 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1087 1088 const char * 1089 efx_tx_qstat_name( 1090 __in efx_nic_t *enp, 1091 __in unsigned int id) 1092 { 1093 _NOTE(ARGUNUSED(enp)) 1094 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1095 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1096 1097 return (__efx_tx_qstat_name[id]); 1098 } 1099 #endif /* EFSYS_OPT_NAMES */ 1100 #endif /* EFSYS_OPT_QSTATS */ 1101 1102 #if EFSYS_OPT_SIENA 1103 1104 #if EFSYS_OPT_QSTATS 1105 static void 1106 siena_tx_qstats_update( 1107 __in efx_txq_t *etp, 1108 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1109 { 1110 unsigned int id; 1111 1112 for (id = 0; id < TX_NQSTATS; id++) { 1113 efsys_stat_t *essp = &stat[id]; 1114 1115 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1116 etp->et_stat[id] = 0; 1117 } 1118 } 1119 #endif /* EFSYS_OPT_QSTATS */ 1120 1121 static void 1122 siena_tx_qdestroy( 1123 __in efx_txq_t *etp) 1124 { 1125 efx_nic_t *enp = etp->et_enp; 1126 efx_oword_t oword; 1127 1128 /* Purge descriptor queue */ 1129 EFX_ZERO_OWORD(oword); 1130 1131 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1132 etp->et_index, &oword, B_TRUE); 1133 } 1134 1135 static void 1136 siena_tx_fini( 1137 __in efx_nic_t *enp) 1138 { 1139 _NOTE(ARGUNUSED(enp)) 1140 } 1141 1142 #endif /* EFSYS_OPT_SIENA */ 1143