1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2021 Xilinx, Inc.
4 * Copyright(c) 2007-2019 Solarflare Communications Inc.
5 */
6
7 #include "efx.h"
8 #include "efx_impl.h"
9
10
11 #if EFSYS_OPT_SIENA
12
13 static __checkReturn efx_rc_t
14 siena_rx_init(
15 __in efx_nic_t *enp);
16
17 static void
18 siena_rx_fini(
19 __in efx_nic_t *enp);
20
21 #if EFSYS_OPT_RX_SCATTER
22 static __checkReturn efx_rc_t
23 siena_rx_scatter_enable(
24 __in efx_nic_t *enp,
25 __in unsigned int buf_size);
26 #endif /* EFSYS_OPT_RX_SCATTER */
27
28 #if EFSYS_OPT_RX_SCALE
29 static __checkReturn efx_rc_t
30 siena_rx_scale_mode_set(
31 __in efx_nic_t *enp,
32 __in uint32_t rss_context,
33 __in efx_rx_hash_alg_t alg,
34 __in efx_rx_hash_type_t type,
35 __in boolean_t insert);
36
37 static __checkReturn efx_rc_t
38 siena_rx_scale_key_set(
39 __in efx_nic_t *enp,
40 __in uint32_t rss_context,
41 __in_ecount(n) uint8_t *key,
42 __in size_t n);
43
44 static __checkReturn efx_rc_t
45 siena_rx_scale_tbl_set(
46 __in efx_nic_t *enp,
47 __in uint32_t rss_context,
48 __in_ecount(nentries) unsigned int *table,
49 __in size_t nentries);
50
51 static __checkReturn uint32_t
52 siena_rx_prefix_hash(
53 __in efx_nic_t *enp,
54 __in efx_rx_hash_alg_t func,
55 __in uint8_t *buffer);
56
57 #endif /* EFSYS_OPT_RX_SCALE */
58
59 static __checkReturn efx_rc_t
60 siena_rx_prefix_pktlen(
61 __in efx_nic_t *enp,
62 __in uint8_t *buffer,
63 __out uint16_t *lengthp);
64
65 static void
66 siena_rx_qpost(
67 __in efx_rxq_t *erp,
68 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
69 __in size_t size,
70 __in unsigned int ndescs,
71 __in unsigned int completed,
72 __in unsigned int added);
73
74 static void
75 siena_rx_qpush(
76 __in efx_rxq_t *erp,
77 __in unsigned int added,
78 __inout unsigned int *pushedp);
79
80 #if EFSYS_OPT_RX_PACKED_STREAM
81 static void
82 siena_rx_qpush_ps_credits(
83 __in efx_rxq_t *erp);
84
85 static __checkReturn uint8_t *
86 siena_rx_qps_packet_info(
87 __in efx_rxq_t *erp,
88 __in uint8_t *buffer,
89 __in uint32_t buffer_length,
90 __in uint32_t current_offset,
91 __out uint16_t *lengthp,
92 __out uint32_t *next_offsetp,
93 __out uint32_t *timestamp);
94 #endif
95
96 static __checkReturn efx_rc_t
97 siena_rx_qflush(
98 __in efx_rxq_t *erp);
99
100 static void
101 siena_rx_qenable(
102 __in efx_rxq_t *erp);
103
104 static __checkReturn efx_rc_t
105 siena_rx_qcreate(
106 __in efx_nic_t *enp,
107 __in unsigned int index,
108 __in unsigned int label,
109 __in efx_rxq_type_t type,
110 __in_opt const efx_rxq_type_data_t *type_data,
111 __in efsys_mem_t *esmp,
112 __in size_t ndescs,
113 __in uint32_t id,
114 __in unsigned int flags,
115 __in efx_evq_t *eep,
116 __in efx_rxq_t *erp);
117
118 static void
119 siena_rx_qdestroy(
120 __in efx_rxq_t *erp);
121
122 #endif /* EFSYS_OPT_SIENA */
123
124
125 #if EFSYS_OPT_SIENA
126 static const efx_rx_ops_t __efx_rx_siena_ops = {
127 siena_rx_init, /* erxo_init */
128 siena_rx_fini, /* erxo_fini */
129 #if EFSYS_OPT_RX_SCATTER
130 siena_rx_scatter_enable, /* erxo_scatter_enable */
131 #endif
132 #if EFSYS_OPT_RX_SCALE
133 NULL, /* erxo_scale_context_alloc */
134 NULL, /* erxo_scale_context_free */
135 siena_rx_scale_mode_set, /* erxo_scale_mode_set */
136 siena_rx_scale_key_set, /* erxo_scale_key_set */
137 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */
138 siena_rx_prefix_hash, /* erxo_prefix_hash */
139 #endif
140 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */
141 siena_rx_qpost, /* erxo_qpost */
142 siena_rx_qpush, /* erxo_qpush */
143 #if EFSYS_OPT_RX_PACKED_STREAM
144 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
145 siena_rx_qps_packet_info, /* erxo_qps_packet_info */
146 #endif
147 siena_rx_qflush, /* erxo_qflush */
148 siena_rx_qenable, /* erxo_qenable */
149 siena_rx_qcreate, /* erxo_qcreate */
150 siena_rx_qdestroy, /* erxo_qdestroy */
151 };
152 #endif /* EFSYS_OPT_SIENA */
153
154 #if EFX_OPTS_EF10()
155 static const efx_rx_ops_t __efx_rx_ef10_ops = {
156 ef10_rx_init, /* erxo_init */
157 ef10_rx_fini, /* erxo_fini */
158 #if EFSYS_OPT_RX_SCATTER
159 ef10_rx_scatter_enable, /* erxo_scatter_enable */
160 #endif
161 #if EFSYS_OPT_RX_SCALE
162 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */
163 ef10_rx_scale_context_free, /* erxo_scale_context_free */
164 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */
165 ef10_rx_scale_key_set, /* erxo_scale_key_set */
166 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */
167 ef10_rx_prefix_hash, /* erxo_prefix_hash */
168 #endif
169 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */
170 ef10_rx_qpost, /* erxo_qpost */
171 ef10_rx_qpush, /* erxo_qpush */
172 #if EFSYS_OPT_RX_PACKED_STREAM
173 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
174 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */
175 #endif
176 ef10_rx_qflush, /* erxo_qflush */
177 ef10_rx_qenable, /* erxo_qenable */
178 ef10_rx_qcreate, /* erxo_qcreate */
179 ef10_rx_qdestroy, /* erxo_qdestroy */
180 };
181 #endif /* EFX_OPTS_EF10() */
182
183 #if EFSYS_OPT_RIVERHEAD
184 static const efx_rx_ops_t __efx_rx_rhead_ops = {
185 rhead_rx_init, /* erxo_init */
186 rhead_rx_fini, /* erxo_fini */
187 #if EFSYS_OPT_RX_SCATTER
188 rhead_rx_scatter_enable, /* erxo_scatter_enable */
189 #endif
190 #if EFSYS_OPT_RX_SCALE
191 rhead_rx_scale_context_alloc, /* erxo_scale_context_alloc */
192 rhead_rx_scale_context_free, /* erxo_scale_context_free */
193 rhead_rx_scale_mode_set, /* erxo_scale_mode_set */
194 rhead_rx_scale_key_set, /* erxo_scale_key_set */
195 rhead_rx_scale_tbl_set, /* erxo_scale_tbl_set */
196 rhead_rx_prefix_hash, /* erxo_prefix_hash */
197 #endif
198 rhead_rx_prefix_pktlen, /* erxo_prefix_pktlen */
199 rhead_rx_qpost, /* erxo_qpost */
200 rhead_rx_qpush, /* erxo_qpush */
201 #if EFSYS_OPT_RX_PACKED_STREAM
202 NULL, /* erxo_qpush_ps_credits */
203 NULL, /* erxo_qps_packet_info */
204 #endif
205 rhead_rx_qflush, /* erxo_qflush */
206 rhead_rx_qenable, /* erxo_qenable */
207 rhead_rx_qcreate, /* erxo_qcreate */
208 rhead_rx_qdestroy, /* erxo_qdestroy */
209 };
210 #endif /* EFSYS_OPT_RIVERHEAD */
211
212
213 __checkReturn efx_rc_t
efx_rx_init(__inout efx_nic_t * enp)214 efx_rx_init(
215 __inout efx_nic_t *enp)
216 {
217 const efx_rx_ops_t *erxop;
218 efx_rc_t rc;
219
220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
222
223 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 rc = EINVAL;
225 goto fail1;
226 }
227
228 if (enp->en_mod_flags & EFX_MOD_RX) {
229 rc = EINVAL;
230 goto fail2;
231 }
232
233 switch (enp->en_family) {
234 #if EFSYS_OPT_SIENA
235 case EFX_FAMILY_SIENA:
236 erxop = &__efx_rx_siena_ops;
237 break;
238 #endif /* EFSYS_OPT_SIENA */
239
240 #if EFSYS_OPT_HUNTINGTON
241 case EFX_FAMILY_HUNTINGTON:
242 erxop = &__efx_rx_ef10_ops;
243 break;
244 #endif /* EFSYS_OPT_HUNTINGTON */
245
246 #if EFSYS_OPT_MEDFORD
247 case EFX_FAMILY_MEDFORD:
248 erxop = &__efx_rx_ef10_ops;
249 break;
250 #endif /* EFSYS_OPT_MEDFORD */
251
252 #if EFSYS_OPT_MEDFORD2
253 case EFX_FAMILY_MEDFORD2:
254 erxop = &__efx_rx_ef10_ops;
255 break;
256 #endif /* EFSYS_OPT_MEDFORD2 */
257
258 #if EFSYS_OPT_RIVERHEAD
259 case EFX_FAMILY_RIVERHEAD:
260 erxop = &__efx_rx_rhead_ops;
261 break;
262 #endif /* EFSYS_OPT_RIVERHEAD */
263
264 default:
265 EFSYS_ASSERT(0);
266 rc = ENOTSUP;
267 goto fail3;
268 }
269
270 if ((rc = erxop->erxo_init(enp)) != 0)
271 goto fail4;
272
273 enp->en_erxop = erxop;
274 enp->en_mod_flags |= EFX_MOD_RX;
275 return (0);
276
277 fail4:
278 EFSYS_PROBE(fail4);
279 fail3:
280 EFSYS_PROBE(fail3);
281 fail2:
282 EFSYS_PROBE(fail2);
283 fail1:
284 EFSYS_PROBE1(fail1, efx_rc_t, rc);
285
286 enp->en_erxop = NULL;
287 enp->en_mod_flags &= ~EFX_MOD_RX;
288 return (rc);
289 }
290
291 void
efx_rx_fini(__in efx_nic_t * enp)292 efx_rx_fini(
293 __in efx_nic_t *enp)
294 {
295 const efx_rx_ops_t *erxop = enp->en_erxop;
296
297 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
298 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
299 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
300 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
301
302 erxop->erxo_fini(enp);
303
304 enp->en_erxop = NULL;
305 enp->en_mod_flags &= ~EFX_MOD_RX;
306 }
307
308 #if EFSYS_OPT_RX_SCATTER
309 __checkReturn efx_rc_t
efx_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)310 efx_rx_scatter_enable(
311 __in efx_nic_t *enp,
312 __in unsigned int buf_size)
313 {
314 const efx_rx_ops_t *erxop = enp->en_erxop;
315 efx_rc_t rc;
316
317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
319
320 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
321 goto fail1;
322
323 return (0);
324
325 fail1:
326 EFSYS_PROBE1(fail1, efx_rc_t, rc);
327 return (rc);
328 }
329 #endif /* EFSYS_OPT_RX_SCATTER */
330
331 #if EFSYS_OPT_RX_SCALE
332 __checkReturn efx_rc_t
efx_rx_scale_hash_flags_get(__in efx_nic_t * enp,__in efx_rx_hash_alg_t hash_alg,__out_ecount_part (max_nflags,* nflagsp)unsigned int * flagsp,__in unsigned int max_nflags,__out unsigned int * nflagsp)333 efx_rx_scale_hash_flags_get(
334 __in efx_nic_t *enp,
335 __in efx_rx_hash_alg_t hash_alg,
336 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp,
337 __in unsigned int max_nflags,
338 __out unsigned int *nflagsp)
339 {
340 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
341 unsigned int nflags = 0;
342 efx_rc_t rc;
343
344 if (flagsp == NULL || nflagsp == NULL) {
345 rc = EINVAL;
346 goto fail1;
347 }
348
349 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) {
350 nflags = 0;
351 goto done;
352 }
353
354 /* Helper to add flags word to flags array without buffer overflow */
355 #define INSERT_FLAGS(_flags) \
356 do { \
357 if (nflags >= max_nflags) { \
358 rc = E2BIG; \
359 goto fail2; \
360 } \
361 *(flagsp + nflags) = (_flags); \
362 nflags++; \
363 \
364 _NOTE(CONSTANTCONDITION) \
365 } while (B_FALSE)
366
367 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) {
368 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE));
369 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE));
370 }
371
372 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) &&
373 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) {
374 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST));
375 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC));
376
377 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST));
378 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC));
379
380 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE));
381 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST));
382 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC));
383
384 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE));
385 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST));
386 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC));
387 }
388
389 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE));
390 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE));
391
392 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE));
393 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE));
394
395 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) {
396 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST));
397 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC));
398
399 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST));
400 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC));
401
402 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE));
403 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST));
404 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC));
405
406 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE));
407 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST));
408 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC));
409
410 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST));
411 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC));
412
413 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST));
414 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC));
415 }
416
417 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE));
418 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE));
419
420 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE));
421 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE));
422
423 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE));
424 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE));
425
426 #undef INSERT_FLAGS
427
428 done:
429 *nflagsp = nflags;
430 return (0);
431
432 fail2:
433 EFSYS_PROBE(fail2);
434 fail1:
435 EFSYS_PROBE1(fail1, efx_rc_t, rc);
436
437 return (rc);
438 }
439
440 __checkReturn efx_rc_t
efx_rx_hash_default_support_get(__in efx_nic_t * enp,__out efx_rx_hash_support_t * supportp)441 efx_rx_hash_default_support_get(
442 __in efx_nic_t *enp,
443 __out efx_rx_hash_support_t *supportp)
444 {
445 efx_rc_t rc;
446
447 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
448 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
449
450 if (supportp == NULL) {
451 rc = EINVAL;
452 goto fail1;
453 }
454
455 /*
456 * Report the hashing support the client gets by default if it
457 * does not allocate an RSS context itself.
458 */
459 *supportp = enp->en_hash_support;
460
461 return (0);
462
463 fail1:
464 EFSYS_PROBE1(fail1, efx_rc_t, rc);
465
466 return (rc);
467 }
468
469 __checkReturn efx_rc_t
efx_rx_scale_default_support_get(__in efx_nic_t * enp,__out efx_rx_scale_context_type_t * typep)470 efx_rx_scale_default_support_get(
471 __in efx_nic_t *enp,
472 __out efx_rx_scale_context_type_t *typep)
473 {
474 efx_rc_t rc;
475
476 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
477 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
478
479 if (typep == NULL) {
480 rc = EINVAL;
481 goto fail1;
482 }
483
484 /*
485 * Report the RSS support the client gets by default if it
486 * does not allocate an RSS context itself.
487 */
488 *typep = enp->en_rss_context_type;
489
490 return (0);
491
492 fail1:
493 EFSYS_PROBE1(fail1, efx_rc_t, rc);
494
495 return (rc);
496 }
497 #endif /* EFSYS_OPT_RX_SCALE */
498
499 #if EFSYS_OPT_RX_SCALE
500 __checkReturn efx_rc_t
efx_rx_scale_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__out uint32_t * rss_contextp)501 efx_rx_scale_context_alloc(
502 __in efx_nic_t *enp,
503 __in efx_rx_scale_context_type_t type,
504 __in uint32_t num_queues,
505 __out uint32_t *rss_contextp)
506 {
507 uint32_t table_nentries = EFX_RSS_TBL_SIZE;
508 const efx_rx_ops_t *erxop = enp->en_erxop;
509 efx_rc_t rc;
510
511 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
512 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
513
514 if (erxop->erxo_scale_context_alloc == NULL) {
515 rc = ENOTSUP;
516 goto fail1;
517 }
518
519 if (type == EFX_RX_SCALE_EVEN_SPREAD)
520 table_nentries = 0;
521
522 if ((rc = erxop->erxo_scale_context_alloc(enp, type, num_queues,
523 table_nentries, rss_contextp)) != 0) {
524 goto fail2;
525 }
526
527 return (0);
528
529 fail2:
530 EFSYS_PROBE(fail2);
531 fail1:
532 EFSYS_PROBE1(fail1, efx_rc_t, rc);
533 return (rc);
534 }
535 #endif /* EFSYS_OPT_RX_SCALE */
536
537 #if EFSYS_OPT_RX_SCALE
538 __checkReturn efx_rc_t
efx_rx_scale_context_alloc_v2(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__in uint32_t table_nentries,__out uint32_t * rss_contextp)539 efx_rx_scale_context_alloc_v2(
540 __in efx_nic_t *enp,
541 __in efx_rx_scale_context_type_t type,
542 __in uint32_t num_queues,
543 __in uint32_t table_nentries,
544 __out uint32_t *rss_contextp)
545 {
546 const efx_rx_ops_t *erxop = enp->en_erxop;
547 efx_rc_t rc;
548
549 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
550 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
551
552 if (erxop->erxo_scale_context_alloc == NULL) {
553 rc = ENOTSUP;
554 goto fail1;
555 }
556
557 if ((rc = erxop->erxo_scale_context_alloc(enp, type,
558 num_queues, table_nentries, rss_contextp)) != 0) {
559 goto fail2;
560 }
561
562 return (0);
563
564 fail2:
565 EFSYS_PROBE(fail2);
566 fail1:
567 EFSYS_PROBE1(fail1, efx_rc_t, rc);
568 return (rc);
569 }
570 #endif /* EFSYS_OPT_RX_SCALE */
571
572 #if EFSYS_OPT_RX_SCALE
573 __checkReturn efx_rc_t
efx_rx_scale_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)574 efx_rx_scale_context_free(
575 __in efx_nic_t *enp,
576 __in uint32_t rss_context)
577 {
578 const efx_rx_ops_t *erxop = enp->en_erxop;
579 efx_rc_t rc;
580
581 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
582 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
583
584 if (erxop->erxo_scale_context_free == NULL) {
585 rc = ENOTSUP;
586 goto fail1;
587 }
588 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
589 goto fail2;
590
591 return (0);
592
593 fail2:
594 EFSYS_PROBE(fail2);
595 fail1:
596 EFSYS_PROBE1(fail1, efx_rc_t, rc);
597 return (rc);
598 }
599 #endif /* EFSYS_OPT_RX_SCALE */
600
601 #if EFSYS_OPT_RX_SCALE
602 __checkReturn efx_rc_t
efx_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)603 efx_rx_scale_mode_set(
604 __in efx_nic_t *enp,
605 __in uint32_t rss_context,
606 __in efx_rx_hash_alg_t alg,
607 __in efx_rx_hash_type_t type,
608 __in boolean_t insert)
609 {
610 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
611 const efx_rx_ops_t *erxop = enp->en_erxop;
612 efx_rx_hash_type_t type_check;
613 unsigned int i;
614 efx_rc_t rc;
615
616 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
617 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
618
619 /*
620 * Legacy flags and modern bits cannot be
621 * used at the same time in the hash type.
622 */
623 if ((type & EFX_RX_HASH_LEGACY_MASK) &&
624 (type & ~EFX_RX_HASH_LEGACY_MASK)) {
625 rc = EINVAL;
626 goto fail1;
627 }
628
629 /*
630 * If RSS hash type is represented by additional bits
631 * in the value, the latter need to be verified since
632 * not all bit combinations are valid RSS modes. Also,
633 * depending on the firmware, some valid combinations
634 * may be unsupported. Discern additional bits in the
635 * type value and try to recognise valid combinations.
636 * If some bits remain unrecognised, report the error.
637 */
638 type_check = type & ~EFX_RX_HASH_LEGACY_MASK;
639 if (type_check != 0) {
640 unsigned int type_flags[EFX_RX_HASH_NFLAGS];
641 unsigned int type_nflags;
642
643 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags,
644 EFX_ARRAY_SIZE(type_flags), &type_nflags);
645 if (rc != 0)
646 goto fail2;
647
648 for (i = 0; i < type_nflags; ++i) {
649 if ((type_check & type_flags[i]) == type_flags[i])
650 type_check &= ~(type_flags[i]);
651 }
652
653 if (type_check != 0) {
654 rc = EINVAL;
655 goto fail3;
656 }
657 }
658
659 /*
660 * Translate EFX_RX_HASH() flags to their legacy counterparts
661 * provided that the FW claims no support for additional modes.
662 */
663 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) {
664 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) |
665 EFX_RX_HASH(IPV4_TCP, 2TUPLE);
666 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) |
667 EFX_RX_HASH(IPV6_TCP, 2TUPLE);
668 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
669 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
670
671 if ((type & t_ipv4) == t_ipv4)
672 type |= EFX_RX_HASH_IPV4;
673 if ((type & t_ipv6) == t_ipv6)
674 type |= EFX_RX_HASH_IPV6;
675
676 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) {
677 if ((type & t_ipv4_tcp) == t_ipv4_tcp)
678 type |= EFX_RX_HASH_TCPIPV4;
679 if ((type & t_ipv6_tcp) == t_ipv6_tcp)
680 type |= EFX_RX_HASH_TCPIPV6;
681 }
682
683 type &= EFX_RX_HASH_LEGACY_MASK;
684 }
685
686 if (erxop->erxo_scale_mode_set != NULL) {
687 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg,
688 type, insert)) != 0)
689 goto fail4;
690 }
691
692 return (0);
693
694 fail4:
695 EFSYS_PROBE(fail4);
696 fail3:
697 EFSYS_PROBE(fail3);
698 fail2:
699 EFSYS_PROBE(fail2);
700 fail1:
701 EFSYS_PROBE1(fail1, efx_rc_t, rc);
702 return (rc);
703 }
704 #endif /* EFSYS_OPT_RX_SCALE */
705
706 #if EFSYS_OPT_RX_SCALE
707 __checkReturn efx_rc_t
efx_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)708 efx_rx_scale_key_set(
709 __in efx_nic_t *enp,
710 __in uint32_t rss_context,
711 __in_ecount(n) uint8_t *key,
712 __in size_t n)
713 {
714 const efx_rx_ops_t *erxop = enp->en_erxop;
715 efx_rc_t rc;
716
717 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
718 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
719
720 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0)
721 goto fail1;
722
723 return (0);
724
725 fail1:
726 EFSYS_PROBE1(fail1, efx_rc_t, rc);
727
728 return (rc);
729 }
730 #endif /* EFSYS_OPT_RX_SCALE */
731
732 #if EFSYS_OPT_RX_SCALE
733 __checkReturn efx_rc_t
efx_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (nentries)unsigned int * table,__in size_t nentries)734 efx_rx_scale_tbl_set(
735 __in efx_nic_t *enp,
736 __in uint32_t rss_context,
737 __in_ecount(nentries) unsigned int *table,
738 __in size_t nentries)
739 {
740 const efx_rx_ops_t *erxop = enp->en_erxop;
741 efx_rc_t rc;
742
743 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
744 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
745
746 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table,
747 nentries)) != 0)
748 goto fail1;
749
750 return (0);
751
752 fail1:
753 EFSYS_PROBE1(fail1, efx_rc_t, rc);
754
755 return (rc);
756 }
757 #endif /* EFSYS_OPT_RX_SCALE */
758
759 void
efx_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)760 efx_rx_qpost(
761 __in efx_rxq_t *erp,
762 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
763 __in size_t size,
764 __in unsigned int ndescs,
765 __in unsigned int completed,
766 __in unsigned int added)
767 {
768 efx_nic_t *enp = erp->er_enp;
769 const efx_rx_ops_t *erxop = enp->en_erxop;
770
771 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
772 EFSYS_ASSERT(erp->er_buf_size == 0 || size == erp->er_buf_size);
773
774 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added);
775 }
776
777 #if EFSYS_OPT_RX_PACKED_STREAM
778
779 void
efx_rx_qpush_ps_credits(__in efx_rxq_t * erp)780 efx_rx_qpush_ps_credits(
781 __in efx_rxq_t *erp)
782 {
783 efx_nic_t *enp = erp->er_enp;
784 const efx_rx_ops_t *erxop = enp->en_erxop;
785
786 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
787
788 erxop->erxo_qpush_ps_credits(erp);
789 }
790
791 __checkReturn uint8_t *
efx_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)792 efx_rx_qps_packet_info(
793 __in efx_rxq_t *erp,
794 __in uint8_t *buffer,
795 __in uint32_t buffer_length,
796 __in uint32_t current_offset,
797 __out uint16_t *lengthp,
798 __out uint32_t *next_offsetp,
799 __out uint32_t *timestamp)
800 {
801 efx_nic_t *enp = erp->er_enp;
802 const efx_rx_ops_t *erxop = enp->en_erxop;
803
804 return (erxop->erxo_qps_packet_info(erp, buffer,
805 buffer_length, current_offset, lengthp,
806 next_offsetp, timestamp));
807 }
808
809 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
810
811 void
efx_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)812 efx_rx_qpush(
813 __in efx_rxq_t *erp,
814 __in unsigned int added,
815 __inout unsigned int *pushedp)
816 {
817 efx_nic_t *enp = erp->er_enp;
818 const efx_rx_ops_t *erxop = enp->en_erxop;
819
820 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
821
822 erxop->erxo_qpush(erp, added, pushedp);
823 }
824
825 __checkReturn efx_rc_t
efx_rx_qflush(__in efx_rxq_t * erp)826 efx_rx_qflush(
827 __in efx_rxq_t *erp)
828 {
829 efx_nic_t *enp = erp->er_enp;
830 const efx_rx_ops_t *erxop = enp->en_erxop;
831 efx_rc_t rc;
832
833 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
834
835 if ((rc = erxop->erxo_qflush(erp)) != 0)
836 goto fail1;
837
838 return (0);
839
840 fail1:
841 EFSYS_PROBE1(fail1, efx_rc_t, rc);
842
843 return (rc);
844 }
845
846 __checkReturn size_t
efx_rxq_size(__in const efx_nic_t * enp,__in unsigned int ndescs)847 efx_rxq_size(
848 __in const efx_nic_t *enp,
849 __in unsigned int ndescs)
850 {
851 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
852
853 return (ndescs * encp->enc_rx_desc_size);
854 }
855
856 __checkReturn unsigned int
efx_rxq_nbufs(__in const efx_nic_t * enp,__in unsigned int ndescs)857 efx_rxq_nbufs(
858 __in const efx_nic_t *enp,
859 __in unsigned int ndescs)
860 {
861 return (EFX_DIV_ROUND_UP(efx_rxq_size(enp, ndescs), EFX_BUF_SIZE));
862 }
863
864 void
efx_rx_qenable(__in efx_rxq_t * erp)865 efx_rx_qenable(
866 __in efx_rxq_t *erp)
867 {
868 efx_nic_t *enp = erp->er_enp;
869 const efx_rx_ops_t *erxop = enp->en_erxop;
870
871 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
872
873 erxop->erxo_qenable(erp);
874 }
875
876 static __checkReturn efx_rc_t
efx_rx_qcreate_internal(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)877 efx_rx_qcreate_internal(
878 __in efx_nic_t *enp,
879 __in unsigned int index,
880 __in unsigned int label,
881 __in efx_rxq_type_t type,
882 __in_opt const efx_rxq_type_data_t *type_data,
883 __in efsys_mem_t *esmp,
884 __in size_t ndescs,
885 __in uint32_t id,
886 __in unsigned int flags,
887 __in efx_evq_t *eep,
888 __deref_out efx_rxq_t **erpp)
889 {
890 const efx_rx_ops_t *erxop = enp->en_erxop;
891 efx_rxq_t *erp;
892 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
893 efx_rc_t rc;
894
895 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
896 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
897
898 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
899
900 EFSYS_ASSERT(ISP2(encp->enc_rxq_max_ndescs));
901 EFSYS_ASSERT(ISP2(encp->enc_rxq_min_ndescs));
902
903 if (index >= encp->enc_rxq_limit) {
904 rc = EINVAL;
905 goto fail1;
906 }
907
908 if (!ISP2(ndescs) ||
909 ndescs < encp->enc_rxq_min_ndescs ||
910 ndescs > encp->enc_rxq_max_ndescs) {
911 rc = EINVAL;
912 goto fail2;
913 }
914
915 /* Allocate an RXQ object */
916 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
917
918 if (erp == NULL) {
919 rc = ENOMEM;
920 goto fail3;
921 }
922
923 erp->er_magic = EFX_RXQ_MAGIC;
924 erp->er_enp = enp;
925 erp->er_index = index;
926 erp->er_mask = ndescs - 1;
927 erp->er_esmp = esmp;
928
929 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp,
930 ndescs, id, flags, eep, erp)) != 0)
931 goto fail4;
932
933 /* Sanity check queue creation result */
934 if (flags & EFX_RXQ_FLAG_RSS_HASH) {
935 const efx_rx_prefix_layout_t *erplp = &erp->er_prefix_layout;
936 const efx_rx_prefix_field_info_t *rss_hash_field;
937
938 rss_hash_field =
939 &erplp->erpl_fields[EFX_RX_PREFIX_FIELD_RSS_HASH];
940 if (rss_hash_field->erpfi_width_bits == 0)
941 goto fail5;
942 }
943
944 enp->en_rx_qcount++;
945 *erpp = erp;
946
947 return (0);
948
949 fail5:
950 EFSYS_PROBE(fail5);
951
952 erxop->erxo_qdestroy(erp);
953 fail4:
954 EFSYS_PROBE(fail4);
955
956 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
957 fail3:
958 EFSYS_PROBE(fail3);
959 fail2:
960 EFSYS_PROBE(fail2);
961 fail1:
962 EFSYS_PROBE1(fail1, efx_rc_t, rc);
963
964 return (rc);
965 }
966
967 __checkReturn efx_rc_t
efx_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in size_t buf_size,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)968 efx_rx_qcreate(
969 __in efx_nic_t *enp,
970 __in unsigned int index,
971 __in unsigned int label,
972 __in efx_rxq_type_t type,
973 __in size_t buf_size,
974 __in efsys_mem_t *esmp,
975 __in size_t ndescs,
976 __in uint32_t id,
977 __in unsigned int flags,
978 __in efx_evq_t *eep,
979 __deref_out efx_rxq_t **erpp)
980 {
981 efx_rxq_type_data_t type_data;
982
983 memset(&type_data, 0, sizeof (type_data));
984
985 type_data.ertd_default.ed_buf_size = buf_size;
986
987 return efx_rx_qcreate_internal(enp, index, label, type, &type_data,
988 esmp, ndescs, id, flags, eep, erpp);
989 }
990
991 #if EFSYS_OPT_RX_PACKED_STREAM
992
993 __checkReturn efx_rc_t
efx_rx_qcreate_packed_stream(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in uint32_t ps_buf_size,__in efsys_mem_t * esmp,__in size_t ndescs,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)994 efx_rx_qcreate_packed_stream(
995 __in efx_nic_t *enp,
996 __in unsigned int index,
997 __in unsigned int label,
998 __in uint32_t ps_buf_size,
999 __in efsys_mem_t *esmp,
1000 __in size_t ndescs,
1001 __in efx_evq_t *eep,
1002 __deref_out efx_rxq_t **erpp)
1003 {
1004 efx_rxq_type_data_t type_data;
1005
1006 memset(&type_data, 0, sizeof (type_data));
1007
1008 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size;
1009
1010 return efx_rx_qcreate_internal(enp, index, label,
1011 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs,
1012 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp);
1013 }
1014
1015 #endif
1016
1017 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
1018
1019 __checkReturn efx_rc_t
efx_rx_qcreate_es_super_buffer(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in uint32_t n_bufs_per_desc,__in uint32_t max_dma_len,__in uint32_t buf_stride,__in uint32_t hol_block_timeout,__in efsys_mem_t * esmp,__in size_t ndescs,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)1020 efx_rx_qcreate_es_super_buffer(
1021 __in efx_nic_t *enp,
1022 __in unsigned int index,
1023 __in unsigned int label,
1024 __in uint32_t n_bufs_per_desc,
1025 __in uint32_t max_dma_len,
1026 __in uint32_t buf_stride,
1027 __in uint32_t hol_block_timeout,
1028 __in efsys_mem_t *esmp,
1029 __in size_t ndescs,
1030 __in unsigned int flags,
1031 __in efx_evq_t *eep,
1032 __deref_out efx_rxq_t **erpp)
1033 {
1034 efx_rc_t rc;
1035 efx_rxq_type_data_t type_data;
1036
1037 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) {
1038 rc = EINVAL;
1039 goto fail1;
1040 }
1041
1042 memset(&type_data, 0, sizeof (type_data));
1043
1044 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc;
1045 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len;
1046 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride;
1047 type_data.ertd_es_super_buffer.eessb_hol_block_timeout =
1048 hol_block_timeout;
1049
1050 rc = efx_rx_qcreate_internal(enp, index, label,
1051 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs,
1052 0 /* id unused on EF10 */, flags, eep, erpp);
1053 if (rc != 0)
1054 goto fail2;
1055
1056 return (0);
1057
1058 fail2:
1059 EFSYS_PROBE(fail2);
1060 fail1:
1061 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1062
1063 return (rc);
1064 }
1065
1066 #endif
1067
1068
1069 void
efx_rx_qdestroy(__in efx_rxq_t * erp)1070 efx_rx_qdestroy(
1071 __in efx_rxq_t *erp)
1072 {
1073 efx_nic_t *enp = erp->er_enp;
1074 const efx_rx_ops_t *erxop = enp->en_erxop;
1075
1076 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1077
1078 EFSYS_ASSERT(enp->en_rx_qcount != 0);
1079 --enp->en_rx_qcount;
1080
1081 erxop->erxo_qdestroy(erp);
1082
1083 /* Free the RXQ object */
1084 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1085 }
1086
1087 __checkReturn efx_rc_t
efx_pseudo_hdr_pkt_length_get(__in efx_rxq_t * erp,__in uint8_t * buffer,__out uint16_t * lengthp)1088 efx_pseudo_hdr_pkt_length_get(
1089 __in efx_rxq_t *erp,
1090 __in uint8_t *buffer,
1091 __out uint16_t *lengthp)
1092 {
1093 efx_nic_t *enp = erp->er_enp;
1094 const efx_rx_ops_t *erxop = enp->en_erxop;
1095
1096 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1097
1098 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
1099 }
1100
1101 #if EFSYS_OPT_RX_SCALE
1102 __checkReturn uint32_t
efx_pseudo_hdr_hash_get(__in efx_rxq_t * erp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)1103 efx_pseudo_hdr_hash_get(
1104 __in efx_rxq_t *erp,
1105 __in efx_rx_hash_alg_t func,
1106 __in uint8_t *buffer)
1107 {
1108 efx_nic_t *enp = erp->er_enp;
1109 const efx_rx_ops_t *erxop = enp->en_erxop;
1110
1111 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1112
1113 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
1114 return (erxop->erxo_prefix_hash(enp, func, buffer));
1115 }
1116 #endif /* EFSYS_OPT_RX_SCALE */
1117
1118 __checkReturn efx_rc_t
efx_rx_prefix_get_layout(__in const efx_rxq_t * erp,__out efx_rx_prefix_layout_t * erplp)1119 efx_rx_prefix_get_layout(
1120 __in const efx_rxq_t *erp,
1121 __out efx_rx_prefix_layout_t *erplp)
1122 {
1123 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1124
1125 *erplp = erp->er_prefix_layout;
1126
1127 return (0);
1128 }
1129
1130 #if EFSYS_OPT_SIENA
1131
1132 static __checkReturn efx_rc_t
siena_rx_init(__in efx_nic_t * enp)1133 siena_rx_init(
1134 __in efx_nic_t *enp)
1135 {
1136 efx_oword_t oword;
1137 unsigned int index;
1138
1139 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
1140
1141 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
1142 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
1143 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
1144 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
1145 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
1146 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
1147 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
1148
1149 /* Zero the RSS table */
1150 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
1151 index++) {
1152 EFX_ZERO_OWORD(oword);
1153 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1154 index, &oword, B_TRUE);
1155 }
1156
1157 #if EFSYS_OPT_RX_SCALE
1158 /* The RSS key and indirection table are writable. */
1159 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
1160
1161 /* Hardware can insert RX hash with/without RSS */
1162 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
1163 #endif /* EFSYS_OPT_RX_SCALE */
1164
1165 return (0);
1166 }
1167
1168 #if EFSYS_OPT_RX_SCATTER
1169 static __checkReturn efx_rc_t
siena_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)1170 siena_rx_scatter_enable(
1171 __in efx_nic_t *enp,
1172 __in unsigned int buf_size)
1173 {
1174 unsigned int nbuf32;
1175 efx_oword_t oword;
1176 efx_rc_t rc;
1177
1178 nbuf32 = buf_size / 32;
1179 if ((nbuf32 == 0) ||
1180 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
1181 ((buf_size % 32) != 0)) {
1182 rc = EINVAL;
1183 goto fail1;
1184 }
1185
1186 if (enp->en_rx_qcount > 0) {
1187 rc = EBUSY;
1188 goto fail2;
1189 }
1190
1191 /* Set scatter buffer size */
1192 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
1193 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
1194 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
1195
1196 /* Enable scatter for packets not matching a filter */
1197 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
1198 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
1199 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
1200
1201 return (0);
1202
1203 fail2:
1204 EFSYS_PROBE(fail2);
1205 fail1:
1206 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1207
1208 return (rc);
1209 }
1210 #endif /* EFSYS_OPT_RX_SCATTER */
1211
1212
1213 #define EFX_RX_LFSR_HASH(_enp, _insert) \
1214 do { \
1215 efx_oword_t oword; \
1216 \
1217 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1218 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \
1219 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \
1220 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \
1221 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
1222 (_insert) ? 1 : 0); \
1223 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1224 \
1225 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \
1226 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
1227 &oword); \
1228 EFX_SET_OWORD_FIELD(oword, \
1229 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \
1230 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
1231 &oword); \
1232 } \
1233 \
1234 _NOTE(CONSTANTCONDITION) \
1235 } while (B_FALSE)
1236
1237 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \
1238 do { \
1239 efx_oword_t oword; \
1240 \
1241 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1242 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \
1243 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \
1244 (_ip) ? 1 : 0); \
1245 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \
1246 (_tcp) ? 0 : 1); \
1247 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
1248 (_insert) ? 1 : 0); \
1249 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1250 \
1251 _NOTE(CONSTANTCONDITION) \
1252 } while (B_FALSE)
1253
1254 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \
1255 do { \
1256 efx_oword_t oword; \
1257 \
1258 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
1259 EFX_SET_OWORD_FIELD(oword, \
1260 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \
1261 EFX_SET_OWORD_FIELD(oword, \
1262 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
1263 EFX_SET_OWORD_FIELD(oword, \
1264 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \
1265 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
1266 \
1267 (_rc) = 0; \
1268 \
1269 _NOTE(CONSTANTCONDITION) \
1270 } while (B_FALSE)
1271
1272
1273 #if EFSYS_OPT_RX_SCALE
1274
1275 static __checkReturn efx_rc_t
siena_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)1276 siena_rx_scale_mode_set(
1277 __in efx_nic_t *enp,
1278 __in uint32_t rss_context,
1279 __in efx_rx_hash_alg_t alg,
1280 __in efx_rx_hash_type_t type,
1281 __in boolean_t insert)
1282 {
1283 efx_rc_t rc;
1284
1285 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1286 rc = EINVAL;
1287 goto fail1;
1288 }
1289
1290 switch (alg) {
1291 case EFX_RX_HASHALG_LFSR:
1292 EFX_RX_LFSR_HASH(enp, insert);
1293 break;
1294
1295 case EFX_RX_HASHALG_TOEPLITZ:
1296 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
1297 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE,
1298 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE);
1299
1300 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
1301 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE,
1302 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE,
1303 rc);
1304 if (rc != 0)
1305 goto fail2;
1306
1307 break;
1308
1309 default:
1310 rc = EINVAL;
1311 goto fail3;
1312 }
1313
1314 return (0);
1315
1316 fail3:
1317 EFSYS_PROBE(fail3);
1318 fail2:
1319 EFSYS_PROBE(fail2);
1320 fail1:
1321 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1322
1323 EFX_RX_LFSR_HASH(enp, B_FALSE);
1324
1325 return (rc);
1326 }
1327 #endif
1328
1329 #if EFSYS_OPT_RX_SCALE
1330 static __checkReturn efx_rc_t
siena_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)1331 siena_rx_scale_key_set(
1332 __in efx_nic_t *enp,
1333 __in uint32_t rss_context,
1334 __in_ecount(n) uint8_t *key,
1335 __in size_t n)
1336 {
1337 efx_oword_t oword;
1338 unsigned int byte;
1339 unsigned int offset;
1340 efx_rc_t rc;
1341
1342 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1343 rc = EINVAL;
1344 goto fail1;
1345 }
1346
1347 byte = 0;
1348
1349 /* Write Toeplitz IPv4 hash key */
1350 EFX_ZERO_OWORD(oword);
1351 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1352 offset > 0 && byte < n;
1353 --offset)
1354 oword.eo_u8[offset - 1] = key[byte++];
1355
1356 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1357
1358 byte = 0;
1359
1360 /* Verify Toeplitz IPv4 hash key */
1361 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1362 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1363 offset > 0 && byte < n;
1364 --offset) {
1365 if (oword.eo_u8[offset - 1] != key[byte++]) {
1366 rc = EFAULT;
1367 goto fail2;
1368 }
1369 }
1370
1371 if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
1372 goto done;
1373
1374 byte = 0;
1375
1376 /* Write Toeplitz IPv6 hash key 3 */
1377 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1378 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1379 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1380 offset > 0 && byte < n;
1381 --offset)
1382 oword.eo_u8[offset - 1] = key[byte++];
1383
1384 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1385
1386 /* Write Toeplitz IPv6 hash key 2 */
1387 EFX_ZERO_OWORD(oword);
1388 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1389 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1390 offset > 0 && byte < n;
1391 --offset)
1392 oword.eo_u8[offset - 1] = key[byte++];
1393
1394 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1395
1396 /* Write Toeplitz IPv6 hash key 1 */
1397 EFX_ZERO_OWORD(oword);
1398 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1399 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1400 offset > 0 && byte < n;
1401 --offset)
1402 oword.eo_u8[offset - 1] = key[byte++];
1403
1404 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1405
1406 byte = 0;
1407
1408 /* Verify Toeplitz IPv6 hash key 3 */
1409 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1410 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1411 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1412 offset > 0 && byte < n;
1413 --offset) {
1414 if (oword.eo_u8[offset - 1] != key[byte++]) {
1415 rc = EFAULT;
1416 goto fail3;
1417 }
1418 }
1419
1420 /* Verify Toeplitz IPv6 hash key 2 */
1421 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1422 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1423 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1424 offset > 0 && byte < n;
1425 --offset) {
1426 if (oword.eo_u8[offset - 1] != key[byte++]) {
1427 rc = EFAULT;
1428 goto fail4;
1429 }
1430 }
1431
1432 /* Verify Toeplitz IPv6 hash key 1 */
1433 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1434 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1435 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1436 offset > 0 && byte < n;
1437 --offset) {
1438 if (oword.eo_u8[offset - 1] != key[byte++]) {
1439 rc = EFAULT;
1440 goto fail5;
1441 }
1442 }
1443
1444 done:
1445 return (0);
1446
1447 fail5:
1448 EFSYS_PROBE(fail5);
1449 fail4:
1450 EFSYS_PROBE(fail4);
1451 fail3:
1452 EFSYS_PROBE(fail3);
1453 fail2:
1454 EFSYS_PROBE(fail2);
1455 fail1:
1456 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1457
1458 return (rc);
1459 }
1460 #endif
1461
1462 #if EFSYS_OPT_RX_SCALE
1463 static __checkReturn efx_rc_t
siena_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (nentries)unsigned int * table,__in size_t nentries)1464 siena_rx_scale_tbl_set(
1465 __in efx_nic_t *enp,
1466 __in uint32_t rss_context,
1467 __in_ecount(nentries) unsigned int *table,
1468 __in size_t nentries)
1469 {
1470 efx_oword_t oword;
1471 int index;
1472 efx_rc_t rc;
1473
1474 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1475 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1476
1477 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1478 rc = EINVAL;
1479 goto fail1;
1480 }
1481
1482 if (nentries > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1483 rc = EINVAL;
1484 goto fail2;
1485 }
1486
1487 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1488 uint32_t byte;
1489
1490 /* Calculate the entry to place in the table */
1491 byte = (nentries > 0) ? (uint32_t)table[index % nentries] : 0;
1492
1493 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1494
1495 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1496
1497 /* Write the table */
1498 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1499 index, &oword, B_TRUE);
1500 }
1501
1502 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1503 uint32_t byte;
1504
1505 /* Determine if we're starting a new batch */
1506 byte = (nentries > 0) ? (uint32_t)table[index % nentries] : 0;
1507
1508 /* Read the table */
1509 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1510 index, &oword, B_TRUE);
1511
1512 /* Verify the entry */
1513 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1514 rc = EFAULT;
1515 goto fail3;
1516 }
1517 }
1518
1519 return (0);
1520
1521 fail3:
1522 EFSYS_PROBE(fail3);
1523 fail2:
1524 EFSYS_PROBE(fail2);
1525 fail1:
1526 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1527
1528 return (rc);
1529 }
1530 #endif
1531
1532 /*
1533 * Falcon/Siena pseudo-header
1534 * --------------------------
1535 *
1536 * Receive packets are prefixed by an optional 16 byte pseudo-header.
1537 * The pseudo-header is a byte array of one of the forms:
1538 *
1539 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1540 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1541 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1542 *
1543 * where:
1544 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian)
1545 * LL.LL LFSR hash (16-bit big-endian)
1546 */
1547
1548 /*
1549 * Provide Rx prefix layout with Toeplitz hash only since LSFR is
1550 * used by no supported drivers.
1551 *
1552 * Siena does not support Rx prefix choice via MC_CMD_GET_RX_PREFIX_ID
1553 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID.
1554 */
1555 static const efx_rx_prefix_layout_t siena_toeplitz_rx_prefix_layout = {
1556 .erpl_id = 0,
1557 .erpl_length = 16,
1558 .erpl_fields = {
1559 [EFX_RX_PREFIX_FIELD_RSS_HASH] = { 12 * 8, 32, B_TRUE },
1560 }
1561 };
1562
1563 #if EFSYS_OPT_RX_SCALE
1564 static __checkReturn uint32_t
siena_rx_prefix_hash(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)1565 siena_rx_prefix_hash(
1566 __in efx_nic_t *enp,
1567 __in efx_rx_hash_alg_t func,
1568 __in uint8_t *buffer)
1569 {
1570 _NOTE(ARGUNUSED(enp))
1571
1572 switch (func) {
1573 case EFX_RX_HASHALG_TOEPLITZ:
1574 return ((buffer[12] << 24) |
1575 (buffer[13] << 16) |
1576 (buffer[14] << 8) |
1577 buffer[15]);
1578
1579 case EFX_RX_HASHALG_LFSR:
1580 return ((buffer[14] << 8) | buffer[15]);
1581
1582 default:
1583 EFSYS_ASSERT(0);
1584 return (0);
1585 }
1586 }
1587 #endif /* EFSYS_OPT_RX_SCALE */
1588
1589 static __checkReturn efx_rc_t
siena_rx_prefix_pktlen(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)1590 siena_rx_prefix_pktlen(
1591 __in efx_nic_t *enp,
1592 __in uint8_t *buffer,
1593 __out uint16_t *lengthp)
1594 {
1595 _NOTE(ARGUNUSED(enp, buffer, lengthp))
1596
1597 /* Not supported by Falcon/Siena hardware */
1598 EFSYS_ASSERT(0);
1599 return (ENOTSUP);
1600 }
1601
1602
1603 static void
siena_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)1604 siena_rx_qpost(
1605 __in efx_rxq_t *erp,
1606 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
1607 __in size_t size,
1608 __in unsigned int ndescs,
1609 __in unsigned int completed,
1610 __in unsigned int added)
1611 {
1612 efx_qword_t qword;
1613 unsigned int i;
1614 unsigned int offset;
1615 unsigned int id;
1616
1617 /* The client driver must not overfill the queue */
1618 EFSYS_ASSERT3U(added - completed + ndescs, <=,
1619 EFX_RXQ_LIMIT(erp->er_mask + 1));
1620
1621 id = added & (erp->er_mask);
1622 for (i = 0; i < ndescs; i++) {
1623 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1624 unsigned int, id, efsys_dma_addr_t, addrp[i],
1625 size_t, size);
1626
1627 EFX_POPULATE_QWORD_3(qword,
1628 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1629 FSF_AZ_RX_KER_BUF_ADDR_DW0,
1630 (uint32_t)(addrp[i] & 0xffffffff),
1631 FSF_AZ_RX_KER_BUF_ADDR_DW1,
1632 (uint32_t)(addrp[i] >> 32));
1633
1634 offset = id * sizeof (efx_qword_t);
1635 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1636
1637 id = (id + 1) & (erp->er_mask);
1638 }
1639 }
1640
1641 static void
siena_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)1642 siena_rx_qpush(
1643 __in efx_rxq_t *erp,
1644 __in unsigned int added,
1645 __inout unsigned int *pushedp)
1646 {
1647 efx_nic_t *enp = erp->er_enp;
1648 unsigned int pushed = *pushedp;
1649 uint32_t wptr;
1650 efx_oword_t oword;
1651 efx_dword_t dword;
1652
1653 /* All descriptors are pushed */
1654 *pushedp = added;
1655
1656 /* Push the populated descriptors out */
1657 wptr = added & erp->er_mask;
1658
1659 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1660
1661 /* Only write the third DWORD */
1662 EFX_POPULATE_DWORD_1(dword,
1663 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1664
1665 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1666 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1667 SIENA_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask);
1668 EFSYS_PIO_WRITE_BARRIER();
1669 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1670 erp->er_index, &dword, B_FALSE);
1671 }
1672
1673 #if EFSYS_OPT_RX_PACKED_STREAM
1674 static void
siena_rx_qpush_ps_credits(__in efx_rxq_t * erp)1675 siena_rx_qpush_ps_credits(
1676 __in efx_rxq_t *erp)
1677 {
1678 /* Not supported by Siena hardware */
1679 EFSYS_ASSERT(0);
1680 }
1681
1682 static uint8_t *
siena_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)1683 siena_rx_qps_packet_info(
1684 __in efx_rxq_t *erp,
1685 __in uint8_t *buffer,
1686 __in uint32_t buffer_length,
1687 __in uint32_t current_offset,
1688 __out uint16_t *lengthp,
1689 __out uint32_t *next_offsetp,
1690 __out uint32_t *timestamp)
1691 {
1692 /* Not supported by Siena hardware */
1693 EFSYS_ASSERT(0);
1694
1695 return (NULL);
1696 }
1697 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1698
1699 static __checkReturn efx_rc_t
siena_rx_qflush(__in efx_rxq_t * erp)1700 siena_rx_qflush(
1701 __in efx_rxq_t *erp)
1702 {
1703 efx_nic_t *enp = erp->er_enp;
1704 efx_oword_t oword;
1705 uint32_t label;
1706
1707 label = erp->er_index;
1708
1709 /* Flush the queue */
1710 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1711 FRF_AZ_RX_FLUSH_DESCQ, label);
1712 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1713
1714 return (0);
1715 }
1716
1717 static void
siena_rx_qenable(__in efx_rxq_t * erp)1718 siena_rx_qenable(
1719 __in efx_rxq_t *erp)
1720 {
1721 efx_nic_t *enp = erp->er_enp;
1722 efx_oword_t oword;
1723
1724 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1725
1726 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1727 erp->er_index, &oword, B_TRUE);
1728
1729 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1730 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1731 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1732
1733 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1734 erp->er_index, &oword, B_TRUE);
1735 }
1736
1737 static __checkReturn efx_rc_t
siena_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__in efx_rxq_t * erp)1738 siena_rx_qcreate(
1739 __in efx_nic_t *enp,
1740 __in unsigned int index,
1741 __in unsigned int label,
1742 __in efx_rxq_type_t type,
1743 __in_opt const efx_rxq_type_data_t *type_data,
1744 __in efsys_mem_t *esmp,
1745 __in size_t ndescs,
1746 __in uint32_t id,
1747 __in unsigned int flags,
1748 __in efx_evq_t *eep,
1749 __in efx_rxq_t *erp)
1750 {
1751 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1752 efx_oword_t oword;
1753 uint32_t size;
1754 boolean_t jumbo = B_FALSE;
1755 efx_rc_t rc;
1756
1757 _NOTE(ARGUNUSED(esmp))
1758
1759 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1760 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1761 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1762
1763 for (size = 0;
1764 (1U << size) <= encp->enc_rxq_max_ndescs / encp->enc_rxq_min_ndescs;
1765 size++)
1766 if ((1U << size) == (uint32_t)ndescs / encp->enc_rxq_min_ndescs)
1767 break;
1768 if (id + (1 << size) >= encp->enc_buftbl_limit) {
1769 rc = EINVAL;
1770 goto fail1;
1771 }
1772
1773 switch (type) {
1774 case EFX_RXQ_TYPE_DEFAULT:
1775 erp->er_buf_size = type_data->ertd_default.ed_buf_size;
1776 /*
1777 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated
1778 * it is always delivered from HW in the pseudo-header.
1779 */
1780 break;
1781
1782 default:
1783 rc = EINVAL;
1784 goto fail2;
1785 }
1786
1787 #if EFSYS_OPT_RX_SCATTER
1788 #define SUPPORTED_RXQ_FLAGS EFX_RXQ_FLAG_SCATTER
1789 #else
1790 #define SUPPORTED_RXQ_FLAGS EFX_RXQ_FLAG_NONE
1791 #endif
1792 /* Reject flags for unsupported queue features */
1793 if ((flags & ~SUPPORTED_RXQ_FLAGS) != 0) {
1794 rc = EINVAL;
1795 goto fail3;
1796 }
1797 #undef SUPPORTED_RXQ_FLAGS
1798
1799 if (flags & EFX_RXQ_FLAG_SCATTER)
1800 jumbo = B_TRUE;
1801
1802 /* Set up the new descriptor queue */
1803 EFX_POPULATE_OWORD_7(oword,
1804 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1805 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1806 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1807 FRF_AZ_RX_DESCQ_LABEL, label,
1808 FRF_AZ_RX_DESCQ_SIZE, size,
1809 FRF_AZ_RX_DESCQ_TYPE, 0,
1810 FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1811
1812 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1813 erp->er_index, &oword, B_TRUE);
1814
1815 erp->er_prefix_layout = siena_toeplitz_rx_prefix_layout;
1816
1817 return (0);
1818
1819 fail3:
1820 EFSYS_PROBE(fail3);
1821 fail2:
1822 EFSYS_PROBE(fail2);
1823 fail1:
1824 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1825
1826 return (rc);
1827 }
1828
1829 static void
siena_rx_qdestroy(__in efx_rxq_t * erp)1830 siena_rx_qdestroy(
1831 __in efx_rxq_t *erp)
1832 {
1833 efx_nic_t *enp = erp->er_enp;
1834 efx_oword_t oword;
1835
1836 /* Purge descriptor queue */
1837 EFX_ZERO_OWORD(oword);
1838
1839 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1840 erp->er_index, &oword, B_TRUE);
1841 }
1842
1843 static void
siena_rx_fini(__in efx_nic_t * enp)1844 siena_rx_fini(
1845 __in efx_nic_t *enp)
1846 {
1847 _NOTE(ARGUNUSED(enp))
1848 }
1849
1850 #endif /* EFSYS_OPT_SIENA */
1851
1852 static __checkReturn boolean_t
efx_rx_prefix_layout_fields_match(__in const efx_rx_prefix_field_info_t * erpfip1,__in const efx_rx_prefix_field_info_t * erpfip2)1853 efx_rx_prefix_layout_fields_match(
1854 __in const efx_rx_prefix_field_info_t *erpfip1,
1855 __in const efx_rx_prefix_field_info_t *erpfip2)
1856 {
1857 if (erpfip1->erpfi_offset_bits != erpfip2->erpfi_offset_bits)
1858 return (B_FALSE);
1859
1860 if (erpfip1->erpfi_width_bits != erpfip2->erpfi_width_bits)
1861 return (B_FALSE);
1862
1863 if (erpfip1->erpfi_big_endian != erpfip2->erpfi_big_endian)
1864 return (B_FALSE);
1865
1866 return (B_TRUE);
1867 }
1868
1869 __checkReturn uint32_t
efx_rx_prefix_layout_check(__in const efx_rx_prefix_layout_t * available,__in const efx_rx_prefix_layout_t * wanted)1870 efx_rx_prefix_layout_check(
1871 __in const efx_rx_prefix_layout_t *available,
1872 __in const efx_rx_prefix_layout_t *wanted)
1873 {
1874 uint32_t result = 0;
1875 unsigned int i;
1876
1877 EFX_STATIC_ASSERT(EFX_RX_PREFIX_NFIELDS < sizeof (result) * 8);
1878 for (i = 0; i < EFX_RX_PREFIX_NFIELDS; ++i) {
1879 /* Skip the field if driver does not want to use it */
1880 if (wanted->erpl_fields[i].erpfi_width_bits == 0)
1881 continue;
1882
1883 if (efx_rx_prefix_layout_fields_match(
1884 &available->erpl_fields[i],
1885 &wanted->erpl_fields[i]) == B_FALSE)
1886 result |= (1U << i);
1887 }
1888
1889 return (result);
1890 }
1891