1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2007-2016 Solarflare Communications Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
23 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
25 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
26 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 * The views and conclusions contained in the software and documentation are
29 * those of the authors and should not be interpreted as representing official
30 * policies, either expressed or implied, of the FreeBSD Project.
31 */
32
33 #include <sys/cdefs.h>
34 __FBSDID("$FreeBSD$");
35
36 #include "efx.h"
37 #include "efx_impl.h"
38
39 #if EFSYS_OPT_QSTATS
40 #define EFX_TX_QSTAT_INCR(_etp, _stat) \
41 do { \
42 (_etp)->et_stat[_stat]++; \
43 _NOTE(CONSTANTCONDITION) \
44 } while (B_FALSE)
45 #else
46 #define EFX_TX_QSTAT_INCR(_etp, _stat)
47 #endif
48
49 #if EFSYS_OPT_SIENA
50
51 static __checkReturn efx_rc_t
52 siena_tx_init(
53 __in efx_nic_t *enp);
54
55 static void
56 siena_tx_fini(
57 __in efx_nic_t *enp);
58
59 static __checkReturn efx_rc_t
60 siena_tx_qcreate(
61 __in efx_nic_t *enp,
62 __in unsigned int index,
63 __in unsigned int label,
64 __in efsys_mem_t *esmp,
65 __in size_t n,
66 __in uint32_t id,
67 __in uint16_t flags,
68 __in efx_evq_t *eep,
69 __in efx_txq_t *etp,
70 __out unsigned int *addedp);
71
72 static void
73 siena_tx_qdestroy(
74 __in efx_txq_t *etp);
75
76 static __checkReturn efx_rc_t
77 siena_tx_qpost(
78 __in efx_txq_t *etp,
79 __in_ecount(n) efx_buffer_t *eb,
80 __in unsigned int n,
81 __in unsigned int completed,
82 __inout unsigned int *addedp);
83
84 static void
85 siena_tx_qpush(
86 __in efx_txq_t *etp,
87 __in unsigned int added,
88 __in unsigned int pushed);
89
90 static __checkReturn efx_rc_t
91 siena_tx_qpace(
92 __in efx_txq_t *etp,
93 __in unsigned int ns);
94
95 static __checkReturn efx_rc_t
96 siena_tx_qflush(
97 __in efx_txq_t *etp);
98
99 static void
100 siena_tx_qenable(
101 __in efx_txq_t *etp);
102
103 __checkReturn efx_rc_t
104 siena_tx_qdesc_post(
105 __in efx_txq_t *etp,
106 __in_ecount(n) efx_desc_t *ed,
107 __in unsigned int n,
108 __in unsigned int completed,
109 __inout unsigned int *addedp);
110
111 void
112 siena_tx_qdesc_dma_create(
113 __in efx_txq_t *etp,
114 __in efsys_dma_addr_t addr,
115 __in size_t size,
116 __in boolean_t eop,
117 __out efx_desc_t *edp);
118
119 #if EFSYS_OPT_QSTATS
120 static void
121 siena_tx_qstats_update(
122 __in efx_txq_t *etp,
123 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat);
124 #endif
125
126 #endif /* EFSYS_OPT_SIENA */
127
128
129 #if EFSYS_OPT_SIENA
130 static const efx_tx_ops_t __efx_tx_siena_ops = {
131 siena_tx_init, /* etxo_init */
132 siena_tx_fini, /* etxo_fini */
133 siena_tx_qcreate, /* etxo_qcreate */
134 siena_tx_qdestroy, /* etxo_qdestroy */
135 siena_tx_qpost, /* etxo_qpost */
136 siena_tx_qpush, /* etxo_qpush */
137 siena_tx_qpace, /* etxo_qpace */
138 siena_tx_qflush, /* etxo_qflush */
139 siena_tx_qenable, /* etxo_qenable */
140 NULL, /* etxo_qpio_enable */
141 NULL, /* etxo_qpio_disable */
142 NULL, /* etxo_qpio_write */
143 NULL, /* etxo_qpio_post */
144 siena_tx_qdesc_post, /* etxo_qdesc_post */
145 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
146 NULL, /* etxo_qdesc_tso_create */
147 NULL, /* etxo_qdesc_tso2_create */
148 NULL, /* etxo_qdesc_vlantci_create */
149 NULL, /* etxo_qdesc_checksum_create */
150 #if EFSYS_OPT_QSTATS
151 siena_tx_qstats_update, /* etxo_qstats_update */
152 #endif
153 };
154 #endif /* EFSYS_OPT_SIENA */
155
156 #if EFSYS_OPT_HUNTINGTON
157 static const efx_tx_ops_t __efx_tx_hunt_ops = {
158 ef10_tx_init, /* etxo_init */
159 ef10_tx_fini, /* etxo_fini */
160 ef10_tx_qcreate, /* etxo_qcreate */
161 ef10_tx_qdestroy, /* etxo_qdestroy */
162 ef10_tx_qpost, /* etxo_qpost */
163 ef10_tx_qpush, /* etxo_qpush */
164 ef10_tx_qpace, /* etxo_qpace */
165 ef10_tx_qflush, /* etxo_qflush */
166 ef10_tx_qenable, /* etxo_qenable */
167 ef10_tx_qpio_enable, /* etxo_qpio_enable */
168 ef10_tx_qpio_disable, /* etxo_qpio_disable */
169 ef10_tx_qpio_write, /* etxo_qpio_write */
170 ef10_tx_qpio_post, /* etxo_qpio_post */
171 ef10_tx_qdesc_post, /* etxo_qdesc_post */
172 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
173 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */
174 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
175 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
176 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
177 #if EFSYS_OPT_QSTATS
178 ef10_tx_qstats_update, /* etxo_qstats_update */
179 #endif
180 };
181 #endif /* EFSYS_OPT_HUNTINGTON */
182
183 #if EFSYS_OPT_MEDFORD
184 static const efx_tx_ops_t __efx_tx_medford_ops = {
185 ef10_tx_init, /* etxo_init */
186 ef10_tx_fini, /* etxo_fini */
187 ef10_tx_qcreate, /* etxo_qcreate */
188 ef10_tx_qdestroy, /* etxo_qdestroy */
189 ef10_tx_qpost, /* etxo_qpost */
190 ef10_tx_qpush, /* etxo_qpush */
191 ef10_tx_qpace, /* etxo_qpace */
192 ef10_tx_qflush, /* etxo_qflush */
193 ef10_tx_qenable, /* etxo_qenable */
194 ef10_tx_qpio_enable, /* etxo_qpio_enable */
195 ef10_tx_qpio_disable, /* etxo_qpio_disable */
196 ef10_tx_qpio_write, /* etxo_qpio_write */
197 ef10_tx_qpio_post, /* etxo_qpio_post */
198 ef10_tx_qdesc_post, /* etxo_qdesc_post */
199 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */
200 NULL, /* etxo_qdesc_tso_create */
201 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */
202 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */
203 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */
204 #if EFSYS_OPT_QSTATS
205 ef10_tx_qstats_update, /* etxo_qstats_update */
206 #endif
207 };
208 #endif /* EFSYS_OPT_MEDFORD */
209
210 __checkReturn efx_rc_t
efx_tx_init(__in efx_nic_t * enp)211 efx_tx_init(
212 __in efx_nic_t *enp)
213 {
214 const efx_tx_ops_t *etxop;
215 efx_rc_t rc;
216
217 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
218 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
219
220 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
221 rc = EINVAL;
222 goto fail1;
223 }
224
225 if (enp->en_mod_flags & EFX_MOD_TX) {
226 rc = EINVAL;
227 goto fail2;
228 }
229
230 switch (enp->en_family) {
231 #if EFSYS_OPT_SIENA
232 case EFX_FAMILY_SIENA:
233 etxop = &__efx_tx_siena_ops;
234 break;
235 #endif /* EFSYS_OPT_SIENA */
236
237 #if EFSYS_OPT_HUNTINGTON
238 case EFX_FAMILY_HUNTINGTON:
239 etxop = &__efx_tx_hunt_ops;
240 break;
241 #endif /* EFSYS_OPT_HUNTINGTON */
242
243 #if EFSYS_OPT_MEDFORD
244 case EFX_FAMILY_MEDFORD:
245 etxop = &__efx_tx_medford_ops;
246 break;
247 #endif /* EFSYS_OPT_MEDFORD */
248
249 default:
250 EFSYS_ASSERT(0);
251 rc = ENOTSUP;
252 goto fail3;
253 }
254
255 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
256
257 if ((rc = etxop->etxo_init(enp)) != 0)
258 goto fail4;
259
260 enp->en_etxop = etxop;
261 enp->en_mod_flags |= EFX_MOD_TX;
262 return (0);
263
264 fail4:
265 EFSYS_PROBE(fail4);
266 fail3:
267 EFSYS_PROBE(fail3);
268 fail2:
269 EFSYS_PROBE(fail2);
270 fail1:
271 EFSYS_PROBE1(fail1, efx_rc_t, rc);
272
273 enp->en_etxop = NULL;
274 enp->en_mod_flags &= ~EFX_MOD_TX;
275 return (rc);
276 }
277
278 void
efx_tx_fini(__in efx_nic_t * enp)279 efx_tx_fini(
280 __in efx_nic_t *enp)
281 {
282 const efx_tx_ops_t *etxop = enp->en_etxop;
283
284 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
285 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
286 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
287 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0);
288
289 etxop->etxo_fini(enp);
290
291 enp->en_etxop = NULL;
292 enp->en_mod_flags &= ~EFX_MOD_TX;
293 }
294
295 __checkReturn efx_rc_t
efx_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__deref_out efx_txq_t ** etpp,__out unsigned int * addedp)296 efx_tx_qcreate(
297 __in efx_nic_t *enp,
298 __in unsigned int index,
299 __in unsigned int label,
300 __in efsys_mem_t *esmp,
301 __in size_t n,
302 __in uint32_t id,
303 __in uint16_t flags,
304 __in efx_evq_t *eep,
305 __deref_out efx_txq_t **etpp,
306 __out unsigned int *addedp)
307 {
308 const efx_tx_ops_t *etxop = enp->en_etxop;
309 efx_txq_t *etp;
310 efx_rc_t rc;
311
312 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
313 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX);
314
315 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <,
316 enp->en_nic_cfg.enc_txq_limit);
317
318 /* Allocate an TXQ object */
319 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp);
320
321 if (etp == NULL) {
322 rc = ENOMEM;
323 goto fail1;
324 }
325
326 etp->et_magic = EFX_TXQ_MAGIC;
327 etp->et_enp = enp;
328 etp->et_index = index;
329 etp->et_mask = n - 1;
330 etp->et_esmp = esmp;
331
332 /* Initial descriptor index may be modified by etxo_qcreate */
333 *addedp = 0;
334
335 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp,
336 n, id, flags, eep, etp, addedp)) != 0)
337 goto fail2;
338
339 enp->en_tx_qcount++;
340 *etpp = etp;
341
342 return (0);
343
344 fail2:
345 EFSYS_PROBE(fail2);
346 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
347 fail1:
348 EFSYS_PROBE1(fail1, efx_rc_t, rc);
349 return (rc);
350 }
351
352 void
efx_tx_qdestroy(__in efx_txq_t * etp)353 efx_tx_qdestroy(
354 __in efx_txq_t *etp)
355 {
356 efx_nic_t *enp = etp->et_enp;
357 const efx_tx_ops_t *etxop = enp->en_etxop;
358
359 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
360
361 EFSYS_ASSERT(enp->en_tx_qcount != 0);
362 --enp->en_tx_qcount;
363
364 etxop->etxo_qdestroy(etp);
365
366 /* Free the TXQ object */
367 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp);
368 }
369
370 __checkReturn efx_rc_t
efx_tx_qpost(__in efx_txq_t * etp,__in_ecount (n)efx_buffer_t * eb,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)371 efx_tx_qpost(
372 __in efx_txq_t *etp,
373 __in_ecount(n) efx_buffer_t *eb,
374 __in unsigned int n,
375 __in unsigned int completed,
376 __inout unsigned int *addedp)
377 {
378 efx_nic_t *enp = etp->et_enp;
379 const efx_tx_ops_t *etxop = enp->en_etxop;
380 efx_rc_t rc;
381
382 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
383
384 if ((rc = etxop->etxo_qpost(etp, eb,
385 n, completed, addedp)) != 0)
386 goto fail1;
387
388 return (0);
389
390 fail1:
391 EFSYS_PROBE1(fail1, efx_rc_t, rc);
392 return (rc);
393 }
394
395 void
efx_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)396 efx_tx_qpush(
397 __in efx_txq_t *etp,
398 __in unsigned int added,
399 __in unsigned int pushed)
400 {
401 efx_nic_t *enp = etp->et_enp;
402 const efx_tx_ops_t *etxop = enp->en_etxop;
403
404 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
405
406 etxop->etxo_qpush(etp, added, pushed);
407 }
408
409 __checkReturn efx_rc_t
efx_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)410 efx_tx_qpace(
411 __in efx_txq_t *etp,
412 __in unsigned int ns)
413 {
414 efx_nic_t *enp = etp->et_enp;
415 const efx_tx_ops_t *etxop = enp->en_etxop;
416 efx_rc_t rc;
417
418 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
419
420 if ((rc = etxop->etxo_qpace(etp, ns)) != 0)
421 goto fail1;
422
423 return (0);
424
425 fail1:
426 EFSYS_PROBE1(fail1, efx_rc_t, rc);
427 return (rc);
428 }
429
430 __checkReturn efx_rc_t
efx_tx_qflush(__in efx_txq_t * etp)431 efx_tx_qflush(
432 __in efx_txq_t *etp)
433 {
434 efx_nic_t *enp = etp->et_enp;
435 const efx_tx_ops_t *etxop = enp->en_etxop;
436 efx_rc_t rc;
437
438 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
439
440 if ((rc = etxop->etxo_qflush(etp)) != 0)
441 goto fail1;
442
443 return (0);
444
445 fail1:
446 EFSYS_PROBE1(fail1, efx_rc_t, rc);
447 return (rc);
448 }
449
450 void
efx_tx_qenable(__in efx_txq_t * etp)451 efx_tx_qenable(
452 __in efx_txq_t *etp)
453 {
454 efx_nic_t *enp = etp->et_enp;
455 const efx_tx_ops_t *etxop = enp->en_etxop;
456
457 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
458
459 etxop->etxo_qenable(etp);
460 }
461
462 __checkReturn efx_rc_t
efx_tx_qpio_enable(__in efx_txq_t * etp)463 efx_tx_qpio_enable(
464 __in efx_txq_t *etp)
465 {
466 efx_nic_t *enp = etp->et_enp;
467 const efx_tx_ops_t *etxop = enp->en_etxop;
468 efx_rc_t rc;
469
470 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
471
472 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) {
473 rc = ENOTSUP;
474 goto fail1;
475 }
476 if (etxop->etxo_qpio_enable == NULL) {
477 rc = ENOTSUP;
478 goto fail2;
479 }
480 if ((rc = etxop->etxo_qpio_enable(etp)) != 0)
481 goto fail3;
482
483 return (0);
484
485 fail3:
486 EFSYS_PROBE(fail3);
487 fail2:
488 EFSYS_PROBE(fail2);
489 fail1:
490 EFSYS_PROBE1(fail1, efx_rc_t, rc);
491 return (rc);
492 }
493
494 void
efx_tx_qpio_disable(__in efx_txq_t * etp)495 efx_tx_qpio_disable(
496 __in efx_txq_t *etp)
497 {
498 efx_nic_t *enp = etp->et_enp;
499 const efx_tx_ops_t *etxop = enp->en_etxop;
500
501 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
502
503 if (etxop->etxo_qpio_disable != NULL)
504 etxop->etxo_qpio_disable(etp);
505 }
506
507 __checkReturn efx_rc_t
efx_tx_qpio_write(__in efx_txq_t * etp,__in_ecount (buf_length)uint8_t * buffer,__in size_t buf_length,__in size_t pio_buf_offset)508 efx_tx_qpio_write(
509 __in efx_txq_t *etp,
510 __in_ecount(buf_length) uint8_t *buffer,
511 __in size_t buf_length,
512 __in size_t pio_buf_offset)
513 {
514 efx_nic_t *enp = etp->et_enp;
515 const efx_tx_ops_t *etxop = enp->en_etxop;
516 efx_rc_t rc;
517
518 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
519
520 if (etxop->etxo_qpio_write != NULL) {
521 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length,
522 pio_buf_offset)) != 0)
523 goto fail1;
524 return (0);
525 }
526
527 return (ENOTSUP);
528
529 fail1:
530 EFSYS_PROBE1(fail1, efx_rc_t, rc);
531 return (rc);
532 }
533
534 __checkReturn efx_rc_t
efx_tx_qpio_post(__in efx_txq_t * etp,__in size_t pkt_length,__in unsigned int completed,__inout unsigned int * addedp)535 efx_tx_qpio_post(
536 __in efx_txq_t *etp,
537 __in size_t pkt_length,
538 __in unsigned int completed,
539 __inout unsigned int *addedp)
540 {
541 efx_nic_t *enp = etp->et_enp;
542 const efx_tx_ops_t *etxop = enp->en_etxop;
543 efx_rc_t rc;
544
545 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
546
547 if (etxop->etxo_qpio_post != NULL) {
548 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed,
549 addedp)) != 0)
550 goto fail1;
551 return (0);
552 }
553
554 return (ENOTSUP);
555
556 fail1:
557 EFSYS_PROBE1(fail1, efx_rc_t, rc);
558 return (rc);
559 }
560
561 __checkReturn efx_rc_t
efx_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (n)efx_desc_t * ed,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)562 efx_tx_qdesc_post(
563 __in efx_txq_t *etp,
564 __in_ecount(n) efx_desc_t *ed,
565 __in unsigned int n,
566 __in unsigned int completed,
567 __inout unsigned int *addedp)
568 {
569 efx_nic_t *enp = etp->et_enp;
570 const efx_tx_ops_t *etxop = enp->en_etxop;
571 efx_rc_t rc;
572
573 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
574
575 if ((rc = etxop->etxo_qdesc_post(etp, ed,
576 n, completed, addedp)) != 0)
577 goto fail1;
578
579 return (0);
580
581 fail1:
582 EFSYS_PROBE1(fail1, efx_rc_t, rc);
583 return (rc);
584 }
585
586 void
efx_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)587 efx_tx_qdesc_dma_create(
588 __in efx_txq_t *etp,
589 __in efsys_dma_addr_t addr,
590 __in size_t size,
591 __in boolean_t eop,
592 __out efx_desc_t *edp)
593 {
594 efx_nic_t *enp = etp->et_enp;
595 const efx_tx_ops_t *etxop = enp->en_etxop;
596
597 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
598 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL);
599
600 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp);
601 }
602
603 void
efx_tx_qdesc_tso_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint32_t tcp_seq,__in uint8_t tcp_flags,__out efx_desc_t * edp)604 efx_tx_qdesc_tso_create(
605 __in efx_txq_t *etp,
606 __in uint16_t ipv4_id,
607 __in uint32_t tcp_seq,
608 __in uint8_t tcp_flags,
609 __out efx_desc_t *edp)
610 {
611 efx_nic_t *enp = etp->et_enp;
612 const efx_tx_ops_t *etxop = enp->en_etxop;
613
614 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
615 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL);
616
617 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp);
618 }
619
620 void
efx_tx_qdesc_tso2_create(__in efx_txq_t * etp,__in uint16_t ipv4_id,__in uint32_t tcp_seq,__in uint16_t mss,__out_ecount (count)efx_desc_t * edp,__in int count)621 efx_tx_qdesc_tso2_create(
622 __in efx_txq_t *etp,
623 __in uint16_t ipv4_id,
624 __in uint32_t tcp_seq,
625 __in uint16_t mss,
626 __out_ecount(count) efx_desc_t *edp,
627 __in int count)
628 {
629 efx_nic_t *enp = etp->et_enp;
630 const efx_tx_ops_t *etxop = enp->en_etxop;
631
632 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
633 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL);
634
635 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, tcp_seq, mss, edp, count);
636 }
637
638 void
efx_tx_qdesc_vlantci_create(__in efx_txq_t * etp,__in uint16_t tci,__out efx_desc_t * edp)639 efx_tx_qdesc_vlantci_create(
640 __in efx_txq_t *etp,
641 __in uint16_t tci,
642 __out efx_desc_t *edp)
643 {
644 efx_nic_t *enp = etp->et_enp;
645 const efx_tx_ops_t *etxop = enp->en_etxop;
646
647 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
648 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL);
649
650 etxop->etxo_qdesc_vlantci_create(etp, tci, edp);
651 }
652
653 void
efx_tx_qdesc_checksum_create(__in efx_txq_t * etp,__in uint16_t flags,__out efx_desc_t * edp)654 efx_tx_qdesc_checksum_create(
655 __in efx_txq_t *etp,
656 __in uint16_t flags,
657 __out efx_desc_t *edp)
658 {
659 efx_nic_t *enp = etp->et_enp;
660 const efx_tx_ops_t *etxop = enp->en_etxop;
661
662 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
663 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL);
664
665 etxop->etxo_qdesc_checksum_create(etp, flags, edp);
666 }
667
668
669 #if EFSYS_OPT_QSTATS
670 void
efx_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)671 efx_tx_qstats_update(
672 __in efx_txq_t *etp,
673 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
674 {
675 efx_nic_t *enp = etp->et_enp;
676 const efx_tx_ops_t *etxop = enp->en_etxop;
677
678 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC);
679
680 etxop->etxo_qstats_update(etp, stat);
681 }
682 #endif
683
684
685 #if EFSYS_OPT_SIENA
686
687 static __checkReturn efx_rc_t
siena_tx_init(__in efx_nic_t * enp)688 siena_tx_init(
689 __in efx_nic_t *enp)
690 {
691 efx_oword_t oword;
692
693 /*
694 * Disable the timer-based TX DMA backoff and allow TX DMA to be
695 * controlled by the RX FIFO fill level (although always allow a
696 * minimal trickle).
697 */
698 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword);
699 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe);
700 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1);
701 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1);
702 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0);
703 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1);
704 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2);
705 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff);
706
707 /*
708 * Filter all packets less than 14 bytes to avoid parsing
709 * errors.
710 */
711 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1);
712 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword);
713
714 /*
715 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16
716 * descriptors (which is bad).
717 */
718 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword);
719 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0);
720 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword);
721
722 return (0);
723 }
724
725 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \
726 do { \
727 unsigned int id; \
728 size_t offset; \
729 efx_qword_t qword; \
730 \
731 id = (_added)++ & (_etp)->et_mask; \
732 offset = id * sizeof (efx_qword_t); \
733 \
734 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \
735 unsigned int, id, efsys_dma_addr_t, (_addr), \
736 size_t, (_size), boolean_t, (_eop)); \
737 \
738 EFX_POPULATE_QWORD_4(qword, \
739 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \
740 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \
741 FSF_AZ_TX_KER_BUF_ADDR_DW0, \
742 (uint32_t)((_addr) & 0xffffffff), \
743 FSF_AZ_TX_KER_BUF_ADDR_DW1, \
744 (uint32_t)((_addr) >> 32)); \
745 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \
746 \
747 _NOTE(CONSTANTCONDITION) \
748 } while (B_FALSE)
749
750 static __checkReturn efx_rc_t
siena_tx_qpost(__in efx_txq_t * etp,__in_ecount (n)efx_buffer_t * eb,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)751 siena_tx_qpost(
752 __in efx_txq_t *etp,
753 __in_ecount(n) efx_buffer_t *eb,
754 __in unsigned int n,
755 __in unsigned int completed,
756 __inout unsigned int *addedp)
757 {
758 unsigned int added = *addedp;
759 unsigned int i;
760 int rc = ENOSPC;
761
762 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1))
763 goto fail1;
764
765 for (i = 0; i < n; i++) {
766 efx_buffer_t *ebp = &eb[i];
767 efsys_dma_addr_t start = ebp->eb_addr;
768 size_t size = ebp->eb_size;
769 efsys_dma_addr_t end = start + size;
770
771 /*
772 * Fragments must not span 4k boundaries.
773 * Here it is a stricter requirement than the maximum length.
774 */
775 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1,
776 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end);
777
778 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added);
779 }
780
781 EFX_TX_QSTAT_INCR(etp, TX_POST);
782
783 *addedp = added;
784 return (0);
785
786 fail1:
787 EFSYS_PROBE1(fail1, efx_rc_t, rc);
788
789 return (rc);
790 }
791
792 static void
siena_tx_qpush(__in efx_txq_t * etp,__in unsigned int added,__in unsigned int pushed)793 siena_tx_qpush(
794 __in efx_txq_t *etp,
795 __in unsigned int added,
796 __in unsigned int pushed)
797 {
798 efx_nic_t *enp = etp->et_enp;
799 uint32_t wptr;
800 efx_dword_t dword;
801 efx_oword_t oword;
802
803 /* Push the populated descriptors out */
804 wptr = added & etp->et_mask;
805
806 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr);
807
808 /* Only write the third DWORD */
809 EFX_POPULATE_DWORD_1(dword,
810 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
811
812 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
813 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1,
814 wptr, pushed & etp->et_mask);
815 EFSYS_PIO_WRITE_BARRIER();
816 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0,
817 etp->et_index, &dword, B_FALSE);
818 }
819
820 #define EFX_MAX_PACE_VALUE 20
821 #define EFX_TX_PACE_CLOCK_BASE 104
822
823 static __checkReturn efx_rc_t
siena_tx_qpace(__in efx_txq_t * etp,__in unsigned int ns)824 siena_tx_qpace(
825 __in efx_txq_t *etp,
826 __in unsigned int ns)
827 {
828 efx_nic_t *enp = etp->et_enp;
829 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
830 efx_oword_t oword;
831 unsigned int pace_val;
832 unsigned int timer_period;
833 efx_rc_t rc;
834
835 if (ns == 0) {
836 pace_val = 0;
837 } else {
838 /*
839 * The pace_val to write into the table is s.t
840 * ns <= timer_period * (2 ^ pace_val)
841 */
842 timer_period = EFX_TX_PACE_CLOCK_BASE / encp->enc_clk_mult;
843 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) {
844 if ((timer_period << pace_val) >= ns)
845 break;
846 }
847 }
848 if (pace_val > EFX_MAX_PACE_VALUE) {
849 rc = EINVAL;
850 goto fail1;
851 }
852
853 /* Update the pacing table */
854 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val);
855 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index,
856 &oword, B_TRUE);
857
858 return (0);
859
860 fail1:
861 EFSYS_PROBE1(fail1, efx_rc_t, rc);
862
863 return (rc);
864 }
865
866 static __checkReturn efx_rc_t
siena_tx_qflush(__in efx_txq_t * etp)867 siena_tx_qflush(
868 __in efx_txq_t *etp)
869 {
870 efx_nic_t *enp = etp->et_enp;
871 efx_oword_t oword;
872 uint32_t label;
873
874 efx_tx_qpace(etp, 0);
875
876 label = etp->et_index;
877
878 /* Flush the queue */
879 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1,
880 FRF_AZ_TX_FLUSH_DESCQ, label);
881 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword);
882
883 return (0);
884 }
885
886 static void
siena_tx_qenable(__in efx_txq_t * etp)887 siena_tx_qenable(
888 __in efx_txq_t *etp)
889 {
890 efx_nic_t *enp = etp->et_enp;
891 efx_oword_t oword;
892
893 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL,
894 etp->et_index, &oword, B_TRUE);
895
896 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index,
897 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3),
898 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2),
899 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1),
900 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0));
901
902 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0);
903 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0);
904 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1);
905
906 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
907 etp->et_index, &oword, B_TRUE);
908 }
909
910 static __checkReturn efx_rc_t
siena_tx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efsys_mem_t * esmp,__in size_t n,__in uint32_t id,__in uint16_t flags,__in efx_evq_t * eep,__in efx_txq_t * etp,__out unsigned int * addedp)911 siena_tx_qcreate(
912 __in efx_nic_t *enp,
913 __in unsigned int index,
914 __in unsigned int label,
915 __in efsys_mem_t *esmp,
916 __in size_t n,
917 __in uint32_t id,
918 __in uint16_t flags,
919 __in efx_evq_t *eep,
920 __in efx_txq_t *etp,
921 __out unsigned int *addedp)
922 {
923 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
924 efx_oword_t oword;
925 uint32_t size;
926 uint16_t inner_csum;
927 efx_rc_t rc;
928
929 _NOTE(ARGUNUSED(esmp))
930
931 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS ==
932 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH));
933 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS);
934
935 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs));
936 EFX_STATIC_ASSERT(ISP2(EFX_TXQ_MINNDESCS));
937
938 if (!ISP2(n) || (n < EFX_TXQ_MINNDESCS) || (n > EFX_EVQ_MAXNEVS)) {
939 rc = EINVAL;
940 goto fail1;
941 }
942 if (index >= encp->enc_txq_limit) {
943 rc = EINVAL;
944 goto fail2;
945 }
946 for (size = 0;
947 (1 << size) <= (int)(encp->enc_txq_max_ndescs / EFX_TXQ_MINNDESCS);
948 size++)
949 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS))
950 break;
951 if (id + (1 << size) >= encp->enc_buftbl_limit) {
952 rc = EINVAL;
953 goto fail3;
954 }
955
956 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP;
957 if ((flags & inner_csum) != 0) {
958 rc = EINVAL;
959 goto fail4;
960 }
961
962 /* Set up the new descriptor queue */
963 *addedp = 0;
964
965 EFX_POPULATE_OWORD_6(oword,
966 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id,
967 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index,
968 FRF_AZ_TX_DESCQ_OWNER_ID, 0,
969 FRF_AZ_TX_DESCQ_LABEL, label,
970 FRF_AZ_TX_DESCQ_SIZE, size,
971 FRF_AZ_TX_DESCQ_TYPE, 0);
972
973 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1);
974 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS,
975 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1);
976 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS,
977 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1);
978
979 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
980 etp->et_index, &oword, B_TRUE);
981
982 return (0);
983
984 fail4:
985 EFSYS_PROBE(fail4);
986 fail3:
987 EFSYS_PROBE(fail3);
988 fail2:
989 EFSYS_PROBE(fail2);
990 fail1:
991 EFSYS_PROBE1(fail1, efx_rc_t, rc);
992
993 return (rc);
994 }
995
996 __checkReturn efx_rc_t
siena_tx_qdesc_post(__in efx_txq_t * etp,__in_ecount (n)efx_desc_t * ed,__in unsigned int n,__in unsigned int completed,__inout unsigned int * addedp)997 siena_tx_qdesc_post(
998 __in efx_txq_t *etp,
999 __in_ecount(n) efx_desc_t *ed,
1000 __in unsigned int n,
1001 __in unsigned int completed,
1002 __inout unsigned int *addedp)
1003 {
1004 unsigned int added = *addedp;
1005 unsigned int i;
1006 efx_rc_t rc;
1007
1008 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) {
1009 rc = ENOSPC;
1010 goto fail1;
1011 }
1012
1013 for (i = 0; i < n; i++) {
1014 efx_desc_t *edp = &ed[i];
1015 unsigned int id;
1016 size_t offset;
1017
1018 id = added++ & etp->et_mask;
1019 offset = id * sizeof (efx_desc_t);
1020
1021 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq);
1022 }
1023
1024 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index,
1025 unsigned int, added, unsigned int, n);
1026
1027 EFX_TX_QSTAT_INCR(etp, TX_POST);
1028
1029 *addedp = added;
1030 return (0);
1031
1032 fail1:
1033 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1034 return (rc);
1035 }
1036
1037 void
siena_tx_qdesc_dma_create(__in efx_txq_t * etp,__in efsys_dma_addr_t addr,__in size_t size,__in boolean_t eop,__out efx_desc_t * edp)1038 siena_tx_qdesc_dma_create(
1039 __in efx_txq_t *etp,
1040 __in efsys_dma_addr_t addr,
1041 __in size_t size,
1042 __in boolean_t eop,
1043 __out efx_desc_t *edp)
1044 {
1045 /*
1046 * Fragments must not span 4k boundaries.
1047 * Here it is a stricter requirement than the maximum length.
1048 */
1049 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1,
1050 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size);
1051
1052 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index,
1053 efsys_dma_addr_t, addr,
1054 size_t, size, boolean_t, eop);
1055
1056 EFX_POPULATE_QWORD_4(edp->ed_eq,
1057 FSF_AZ_TX_KER_CONT, eop ? 0 : 1,
1058 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size,
1059 FSF_AZ_TX_KER_BUF_ADDR_DW0,
1060 (uint32_t)(addr & 0xffffffff),
1061 FSF_AZ_TX_KER_BUF_ADDR_DW1,
1062 (uint32_t)(addr >> 32));
1063 }
1064
1065 #endif /* EFSYS_OPT_SIENA */
1066
1067 #if EFSYS_OPT_QSTATS
1068 #if EFSYS_OPT_NAMES
1069 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */
1070 static const char * const __efx_tx_qstat_name[] = {
1071 "post",
1072 "post_pio",
1073 };
1074 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */
1075
1076 const char *
efx_tx_qstat_name(__in efx_nic_t * enp,__in unsigned int id)1077 efx_tx_qstat_name(
1078 __in efx_nic_t *enp,
1079 __in unsigned int id)
1080 {
1081 _NOTE(ARGUNUSED(enp))
1082 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
1083 EFSYS_ASSERT3U(id, <, TX_NQSTATS);
1084
1085 return (__efx_tx_qstat_name[id]);
1086 }
1087 #endif /* EFSYS_OPT_NAMES */
1088 #endif /* EFSYS_OPT_QSTATS */
1089
1090 #if EFSYS_OPT_SIENA
1091
1092 #if EFSYS_OPT_QSTATS
1093 static void
siena_tx_qstats_update(__in efx_txq_t * etp,__inout_ecount (TX_NQSTATS)efsys_stat_t * stat)1094 siena_tx_qstats_update(
1095 __in efx_txq_t *etp,
1096 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat)
1097 {
1098 unsigned int id;
1099
1100 for (id = 0; id < TX_NQSTATS; id++) {
1101 efsys_stat_t *essp = &stat[id];
1102
1103 EFSYS_STAT_INCR(essp, etp->et_stat[id]);
1104 etp->et_stat[id] = 0;
1105 }
1106 }
1107 #endif /* EFSYS_OPT_QSTATS */
1108
1109 static void
siena_tx_qdestroy(__in efx_txq_t * etp)1110 siena_tx_qdestroy(
1111 __in efx_txq_t *etp)
1112 {
1113 efx_nic_t *enp = etp->et_enp;
1114 efx_oword_t oword;
1115
1116 /* Purge descriptor queue */
1117 EFX_ZERO_OWORD(oword);
1118
1119 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL,
1120 etp->et_index, &oword, B_TRUE);
1121 }
1122
1123 static void
siena_tx_fini(__in efx_nic_t * enp)1124 siena_tx_fini(
1125 __in efx_nic_t *enp)
1126 {
1127 _NOTE(ARGUNUSED(enp))
1128 }
1129
1130 #endif /* EFSYS_OPT_SIENA */
1131