1 /* SPDX-License-Identifier: BSD-3-Clause
2 *
3 * Copyright(c) 2019-2020 Xilinx, Inc.
4 * Copyright(c) 2007-2019 Solarflare Communications Inc.
5 */
6
7 #include "efx.h"
8 #include "efx_impl.h"
9
10
11 #if EFSYS_OPT_SIENA
12
13 static __checkReturn efx_rc_t
14 siena_rx_init(
15 __in efx_nic_t *enp);
16
17 static void
18 siena_rx_fini(
19 __in efx_nic_t *enp);
20
21 #if EFSYS_OPT_RX_SCATTER
22 static __checkReturn efx_rc_t
23 siena_rx_scatter_enable(
24 __in efx_nic_t *enp,
25 __in unsigned int buf_size);
26 #endif /* EFSYS_OPT_RX_SCATTER */
27
28 #if EFSYS_OPT_RX_SCALE
29 static __checkReturn efx_rc_t
30 siena_rx_scale_mode_set(
31 __in efx_nic_t *enp,
32 __in uint32_t rss_context,
33 __in efx_rx_hash_alg_t alg,
34 __in efx_rx_hash_type_t type,
35 __in boolean_t insert);
36
37 static __checkReturn efx_rc_t
38 siena_rx_scale_key_set(
39 __in efx_nic_t *enp,
40 __in uint32_t rss_context,
41 __in_ecount(n) uint8_t *key,
42 __in size_t n);
43
44 static __checkReturn efx_rc_t
45 siena_rx_scale_tbl_set(
46 __in efx_nic_t *enp,
47 __in uint32_t rss_context,
48 __in_ecount(n) unsigned int *table,
49 __in size_t n);
50
51 static __checkReturn uint32_t
52 siena_rx_prefix_hash(
53 __in efx_nic_t *enp,
54 __in efx_rx_hash_alg_t func,
55 __in uint8_t *buffer);
56
57 #endif /* EFSYS_OPT_RX_SCALE */
58
59 static __checkReturn efx_rc_t
60 siena_rx_prefix_pktlen(
61 __in efx_nic_t *enp,
62 __in uint8_t *buffer,
63 __out uint16_t *lengthp);
64
65 static void
66 siena_rx_qpost(
67 __in efx_rxq_t *erp,
68 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
69 __in size_t size,
70 __in unsigned int ndescs,
71 __in unsigned int completed,
72 __in unsigned int added);
73
74 static void
75 siena_rx_qpush(
76 __in efx_rxq_t *erp,
77 __in unsigned int added,
78 __inout unsigned int *pushedp);
79
80 #if EFSYS_OPT_RX_PACKED_STREAM
81 static void
82 siena_rx_qpush_ps_credits(
83 __in efx_rxq_t *erp);
84
85 static __checkReturn uint8_t *
86 siena_rx_qps_packet_info(
87 __in efx_rxq_t *erp,
88 __in uint8_t *buffer,
89 __in uint32_t buffer_length,
90 __in uint32_t current_offset,
91 __out uint16_t *lengthp,
92 __out uint32_t *next_offsetp,
93 __out uint32_t *timestamp);
94 #endif
95
96 static __checkReturn efx_rc_t
97 siena_rx_qflush(
98 __in efx_rxq_t *erp);
99
100 static void
101 siena_rx_qenable(
102 __in efx_rxq_t *erp);
103
104 static __checkReturn efx_rc_t
105 siena_rx_qcreate(
106 __in efx_nic_t *enp,
107 __in unsigned int index,
108 __in unsigned int label,
109 __in efx_rxq_type_t type,
110 __in_opt const efx_rxq_type_data_t *type_data,
111 __in efsys_mem_t *esmp,
112 __in size_t ndescs,
113 __in uint32_t id,
114 __in unsigned int flags,
115 __in efx_evq_t *eep,
116 __in efx_rxq_t *erp);
117
118 static void
119 siena_rx_qdestroy(
120 __in efx_rxq_t *erp);
121
122 #endif /* EFSYS_OPT_SIENA */
123
124
125 #if EFSYS_OPT_SIENA
126 static const efx_rx_ops_t __efx_rx_siena_ops = {
127 siena_rx_init, /* erxo_init */
128 siena_rx_fini, /* erxo_fini */
129 #if EFSYS_OPT_RX_SCATTER
130 siena_rx_scatter_enable, /* erxo_scatter_enable */
131 #endif
132 #if EFSYS_OPT_RX_SCALE
133 NULL, /* erxo_scale_context_alloc */
134 NULL, /* erxo_scale_context_free */
135 siena_rx_scale_mode_set, /* erxo_scale_mode_set */
136 siena_rx_scale_key_set, /* erxo_scale_key_set */
137 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */
138 siena_rx_prefix_hash, /* erxo_prefix_hash */
139 #endif
140 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */
141 siena_rx_qpost, /* erxo_qpost */
142 siena_rx_qpush, /* erxo_qpush */
143 #if EFSYS_OPT_RX_PACKED_STREAM
144 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
145 siena_rx_qps_packet_info, /* erxo_qps_packet_info */
146 #endif
147 siena_rx_qflush, /* erxo_qflush */
148 siena_rx_qenable, /* erxo_qenable */
149 siena_rx_qcreate, /* erxo_qcreate */
150 siena_rx_qdestroy, /* erxo_qdestroy */
151 };
152 #endif /* EFSYS_OPT_SIENA */
153
154 #if EFX_OPTS_EF10()
155 static const efx_rx_ops_t __efx_rx_ef10_ops = {
156 ef10_rx_init, /* erxo_init */
157 ef10_rx_fini, /* erxo_fini */
158 #if EFSYS_OPT_RX_SCATTER
159 ef10_rx_scatter_enable, /* erxo_scatter_enable */
160 #endif
161 #if EFSYS_OPT_RX_SCALE
162 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */
163 ef10_rx_scale_context_free, /* erxo_scale_context_free */
164 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */
165 ef10_rx_scale_key_set, /* erxo_scale_key_set */
166 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */
167 ef10_rx_prefix_hash, /* erxo_prefix_hash */
168 #endif
169 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */
170 ef10_rx_qpost, /* erxo_qpost */
171 ef10_rx_qpush, /* erxo_qpush */
172 #if EFSYS_OPT_RX_PACKED_STREAM
173 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
174 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */
175 #endif
176 ef10_rx_qflush, /* erxo_qflush */
177 ef10_rx_qenable, /* erxo_qenable */
178 ef10_rx_qcreate, /* erxo_qcreate */
179 ef10_rx_qdestroy, /* erxo_qdestroy */
180 };
181 #endif /* EFX_OPTS_EF10() */
182
183 #if EFSYS_OPT_RIVERHEAD
184 static const efx_rx_ops_t __efx_rx_rhead_ops = {
185 rhead_rx_init, /* erxo_init */
186 rhead_rx_fini, /* erxo_fini */
187 #if EFSYS_OPT_RX_SCATTER
188 rhead_rx_scatter_enable, /* erxo_scatter_enable */
189 #endif
190 #if EFSYS_OPT_RX_SCALE
191 rhead_rx_scale_context_alloc, /* erxo_scale_context_alloc */
192 rhead_rx_scale_context_free, /* erxo_scale_context_free */
193 rhead_rx_scale_mode_set, /* erxo_scale_mode_set */
194 rhead_rx_scale_key_set, /* erxo_scale_key_set */
195 rhead_rx_scale_tbl_set, /* erxo_scale_tbl_set */
196 rhead_rx_prefix_hash, /* erxo_prefix_hash */
197 #endif
198 rhead_rx_prefix_pktlen, /* erxo_prefix_pktlen */
199 rhead_rx_qpost, /* erxo_qpost */
200 rhead_rx_qpush, /* erxo_qpush */
201 #if EFSYS_OPT_RX_PACKED_STREAM
202 NULL, /* erxo_qpush_ps_credits */
203 NULL, /* erxo_qps_packet_info */
204 #endif
205 rhead_rx_qflush, /* erxo_qflush */
206 rhead_rx_qenable, /* erxo_qenable */
207 rhead_rx_qcreate, /* erxo_qcreate */
208 rhead_rx_qdestroy, /* erxo_qdestroy */
209 };
210 #endif /* EFSYS_OPT_RIVERHEAD */
211
212
213 __checkReturn efx_rc_t
efx_rx_init(__inout efx_nic_t * enp)214 efx_rx_init(
215 __inout efx_nic_t *enp)
216 {
217 const efx_rx_ops_t *erxop;
218 efx_rc_t rc;
219
220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
222
223 if (!(enp->en_mod_flags & EFX_MOD_EV)) {
224 rc = EINVAL;
225 goto fail1;
226 }
227
228 if (enp->en_mod_flags & EFX_MOD_RX) {
229 rc = EINVAL;
230 goto fail2;
231 }
232
233 switch (enp->en_family) {
234 #if EFSYS_OPT_SIENA
235 case EFX_FAMILY_SIENA:
236 erxop = &__efx_rx_siena_ops;
237 break;
238 #endif /* EFSYS_OPT_SIENA */
239
240 #if EFSYS_OPT_HUNTINGTON
241 case EFX_FAMILY_HUNTINGTON:
242 erxop = &__efx_rx_ef10_ops;
243 break;
244 #endif /* EFSYS_OPT_HUNTINGTON */
245
246 #if EFSYS_OPT_MEDFORD
247 case EFX_FAMILY_MEDFORD:
248 erxop = &__efx_rx_ef10_ops;
249 break;
250 #endif /* EFSYS_OPT_MEDFORD */
251
252 #if EFSYS_OPT_MEDFORD2
253 case EFX_FAMILY_MEDFORD2:
254 erxop = &__efx_rx_ef10_ops;
255 break;
256 #endif /* EFSYS_OPT_MEDFORD2 */
257
258 #if EFSYS_OPT_RIVERHEAD
259 case EFX_FAMILY_RIVERHEAD:
260 erxop = &__efx_rx_rhead_ops;
261 break;
262 #endif /* EFSYS_OPT_RIVERHEAD */
263
264 default:
265 EFSYS_ASSERT(0);
266 rc = ENOTSUP;
267 goto fail3;
268 }
269
270 if ((rc = erxop->erxo_init(enp)) != 0)
271 goto fail4;
272
273 enp->en_erxop = erxop;
274 enp->en_mod_flags |= EFX_MOD_RX;
275 return (0);
276
277 fail4:
278 EFSYS_PROBE(fail4);
279 fail3:
280 EFSYS_PROBE(fail3);
281 fail2:
282 EFSYS_PROBE(fail2);
283 fail1:
284 EFSYS_PROBE1(fail1, efx_rc_t, rc);
285
286 enp->en_erxop = NULL;
287 enp->en_mod_flags &= ~EFX_MOD_RX;
288 return (rc);
289 }
290
291 void
efx_rx_fini(__in efx_nic_t * enp)292 efx_rx_fini(
293 __in efx_nic_t *enp)
294 {
295 const efx_rx_ops_t *erxop = enp->en_erxop;
296
297 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
298 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC);
299 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
300 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0);
301
302 erxop->erxo_fini(enp);
303
304 enp->en_erxop = NULL;
305 enp->en_mod_flags &= ~EFX_MOD_RX;
306 }
307
308 #if EFSYS_OPT_RX_SCATTER
309 __checkReturn efx_rc_t
efx_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)310 efx_rx_scatter_enable(
311 __in efx_nic_t *enp,
312 __in unsigned int buf_size)
313 {
314 const efx_rx_ops_t *erxop = enp->en_erxop;
315 efx_rc_t rc;
316
317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
319
320 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0)
321 goto fail1;
322
323 return (0);
324
325 fail1:
326 EFSYS_PROBE1(fail1, efx_rc_t, rc);
327 return (rc);
328 }
329 #endif /* EFSYS_OPT_RX_SCATTER */
330
331 #if EFSYS_OPT_RX_SCALE
332 __checkReturn efx_rc_t
efx_rx_scale_hash_flags_get(__in efx_nic_t * enp,__in efx_rx_hash_alg_t hash_alg,__out_ecount_part (max_nflags,* nflagsp)unsigned int * flagsp,__in unsigned int max_nflags,__out unsigned int * nflagsp)333 efx_rx_scale_hash_flags_get(
334 __in efx_nic_t *enp,
335 __in efx_rx_hash_alg_t hash_alg,
336 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp,
337 __in unsigned int max_nflags,
338 __out unsigned int *nflagsp)
339 {
340 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
341 unsigned int nflags = 0;
342 efx_rc_t rc;
343
344 if (flagsp == NULL || nflagsp == NULL) {
345 rc = EINVAL;
346 goto fail1;
347 }
348
349 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) {
350 nflags = 0;
351 goto done;
352 }
353
354 /* Helper to add flags word to flags array without buffer overflow */
355 #define INSERT_FLAGS(_flags) \
356 do { \
357 if (nflags >= max_nflags) { \
358 rc = E2BIG; \
359 goto fail2; \
360 } \
361 *(flagsp + nflags) = (_flags); \
362 nflags++; \
363 \
364 _NOTE(CONSTANTCONDITION) \
365 } while (B_FALSE)
366
367 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) {
368 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE));
369 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE));
370 }
371
372 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) &&
373 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) {
374 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST));
375 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC));
376
377 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST));
378 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC));
379
380 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE));
381 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST));
382 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC));
383
384 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE));
385 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST));
386 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC));
387 }
388
389 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE));
390 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE));
391
392 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE));
393 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE));
394
395 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) {
396 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST));
397 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC));
398
399 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST));
400 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC));
401
402 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE));
403 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST));
404 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC));
405
406 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE));
407 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST));
408 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC));
409
410 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST));
411 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC));
412
413 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST));
414 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC));
415 }
416
417 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE));
418 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE));
419
420 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE));
421 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE));
422
423 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE));
424 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE));
425
426 #undef INSERT_FLAGS
427
428 done:
429 *nflagsp = nflags;
430 return (0);
431
432 fail2:
433 EFSYS_PROBE(fail2);
434 fail1:
435 EFSYS_PROBE1(fail1, efx_rc_t, rc);
436
437 return (rc);
438 }
439
440 __checkReturn efx_rc_t
efx_rx_hash_default_support_get(__in efx_nic_t * enp,__out efx_rx_hash_support_t * supportp)441 efx_rx_hash_default_support_get(
442 __in efx_nic_t *enp,
443 __out efx_rx_hash_support_t *supportp)
444 {
445 efx_rc_t rc;
446
447 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
448 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
449
450 if (supportp == NULL) {
451 rc = EINVAL;
452 goto fail1;
453 }
454
455 /*
456 * Report the hashing support the client gets by default if it
457 * does not allocate an RSS context itself.
458 */
459 *supportp = enp->en_hash_support;
460
461 return (0);
462
463 fail1:
464 EFSYS_PROBE1(fail1, efx_rc_t, rc);
465
466 return (rc);
467 }
468
469 __checkReturn efx_rc_t
efx_rx_scale_default_support_get(__in efx_nic_t * enp,__out efx_rx_scale_context_type_t * typep)470 efx_rx_scale_default_support_get(
471 __in efx_nic_t *enp,
472 __out efx_rx_scale_context_type_t *typep)
473 {
474 efx_rc_t rc;
475
476 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
477 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
478
479 if (typep == NULL) {
480 rc = EINVAL;
481 goto fail1;
482 }
483
484 /*
485 * Report the RSS support the client gets by default if it
486 * does not allocate an RSS context itself.
487 */
488 *typep = enp->en_rss_context_type;
489
490 return (0);
491
492 fail1:
493 EFSYS_PROBE1(fail1, efx_rc_t, rc);
494
495 return (rc);
496 }
497 #endif /* EFSYS_OPT_RX_SCALE */
498
499 #if EFSYS_OPT_RX_SCALE
500 __checkReturn efx_rc_t
efx_rx_scale_context_alloc(__in efx_nic_t * enp,__in efx_rx_scale_context_type_t type,__in uint32_t num_queues,__out uint32_t * rss_contextp)501 efx_rx_scale_context_alloc(
502 __in efx_nic_t *enp,
503 __in efx_rx_scale_context_type_t type,
504 __in uint32_t num_queues,
505 __out uint32_t *rss_contextp)
506 {
507 const efx_rx_ops_t *erxop = enp->en_erxop;
508 efx_rc_t rc;
509
510 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
511 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
512
513 if (erxop->erxo_scale_context_alloc == NULL) {
514 rc = ENOTSUP;
515 goto fail1;
516 }
517 if ((rc = erxop->erxo_scale_context_alloc(enp, type,
518 num_queues, rss_contextp)) != 0) {
519 goto fail2;
520 }
521
522 return (0);
523
524 fail2:
525 EFSYS_PROBE(fail2);
526 fail1:
527 EFSYS_PROBE1(fail1, efx_rc_t, rc);
528 return (rc);
529 }
530 #endif /* EFSYS_OPT_RX_SCALE */
531
532 #if EFSYS_OPT_RX_SCALE
533 __checkReturn efx_rc_t
efx_rx_scale_context_free(__in efx_nic_t * enp,__in uint32_t rss_context)534 efx_rx_scale_context_free(
535 __in efx_nic_t *enp,
536 __in uint32_t rss_context)
537 {
538 const efx_rx_ops_t *erxop = enp->en_erxop;
539 efx_rc_t rc;
540
541 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
542 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
543
544 if (erxop->erxo_scale_context_free == NULL) {
545 rc = ENOTSUP;
546 goto fail1;
547 }
548 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0)
549 goto fail2;
550
551 return (0);
552
553 fail2:
554 EFSYS_PROBE(fail2);
555 fail1:
556 EFSYS_PROBE1(fail1, efx_rc_t, rc);
557 return (rc);
558 }
559 #endif /* EFSYS_OPT_RX_SCALE */
560
561 #if EFSYS_OPT_RX_SCALE
562 __checkReturn efx_rc_t
efx_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)563 efx_rx_scale_mode_set(
564 __in efx_nic_t *enp,
565 __in uint32_t rss_context,
566 __in efx_rx_hash_alg_t alg,
567 __in efx_rx_hash_type_t type,
568 __in boolean_t insert)
569 {
570 efx_nic_cfg_t *encp = &enp->en_nic_cfg;
571 const efx_rx_ops_t *erxop = enp->en_erxop;
572 efx_rx_hash_type_t type_check;
573 unsigned int i;
574 efx_rc_t rc;
575
576 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
577 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
578
579 /*
580 * Legacy flags and modern bits cannot be
581 * used at the same time in the hash type.
582 */
583 if ((type & EFX_RX_HASH_LEGACY_MASK) &&
584 (type & ~EFX_RX_HASH_LEGACY_MASK)) {
585 rc = EINVAL;
586 goto fail1;
587 }
588
589 /*
590 * If RSS hash type is represented by additional bits
591 * in the value, the latter need to be verified since
592 * not all bit combinations are valid RSS modes. Also,
593 * depending on the firmware, some valid combinations
594 * may be unsupported. Discern additional bits in the
595 * type value and try to recognise valid combinations.
596 * If some bits remain unrecognised, report the error.
597 */
598 type_check = type & ~EFX_RX_HASH_LEGACY_MASK;
599 if (type_check != 0) {
600 unsigned int type_flags[EFX_RX_HASH_NFLAGS];
601 unsigned int type_nflags;
602
603 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags,
604 EFX_ARRAY_SIZE(type_flags), &type_nflags);
605 if (rc != 0)
606 goto fail2;
607
608 for (i = 0; i < type_nflags; ++i) {
609 if ((type_check & type_flags[i]) == type_flags[i])
610 type_check &= ~(type_flags[i]);
611 }
612
613 if (type_check != 0) {
614 rc = EINVAL;
615 goto fail3;
616 }
617 }
618
619 /*
620 * Translate EFX_RX_HASH() flags to their legacy counterparts
621 * provided that the FW claims no support for additional modes.
622 */
623 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) {
624 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) |
625 EFX_RX_HASH(IPV4_TCP, 2TUPLE);
626 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) |
627 EFX_RX_HASH(IPV6_TCP, 2TUPLE);
628 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE);
629 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE);
630
631 if ((type & t_ipv4) == t_ipv4)
632 type |= EFX_RX_HASH_IPV4;
633 if ((type & t_ipv6) == t_ipv6)
634 type |= EFX_RX_HASH_IPV6;
635
636 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) {
637 if ((type & t_ipv4_tcp) == t_ipv4_tcp)
638 type |= EFX_RX_HASH_TCPIPV4;
639 if ((type & t_ipv6_tcp) == t_ipv6_tcp)
640 type |= EFX_RX_HASH_TCPIPV6;
641 }
642
643 type &= EFX_RX_HASH_LEGACY_MASK;
644 }
645
646 if (erxop->erxo_scale_mode_set != NULL) {
647 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg,
648 type, insert)) != 0)
649 goto fail4;
650 }
651
652 return (0);
653
654 fail4:
655 EFSYS_PROBE(fail4);
656 fail3:
657 EFSYS_PROBE(fail3);
658 fail2:
659 EFSYS_PROBE(fail2);
660 fail1:
661 EFSYS_PROBE1(fail1, efx_rc_t, rc);
662 return (rc);
663 }
664 #endif /* EFSYS_OPT_RX_SCALE */
665
666 #if EFSYS_OPT_RX_SCALE
667 __checkReturn efx_rc_t
efx_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)668 efx_rx_scale_key_set(
669 __in efx_nic_t *enp,
670 __in uint32_t rss_context,
671 __in_ecount(n) uint8_t *key,
672 __in size_t n)
673 {
674 const efx_rx_ops_t *erxop = enp->en_erxop;
675 efx_rc_t rc;
676
677 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
678 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
679
680 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0)
681 goto fail1;
682
683 return (0);
684
685 fail1:
686 EFSYS_PROBE1(fail1, efx_rc_t, rc);
687
688 return (rc);
689 }
690 #endif /* EFSYS_OPT_RX_SCALE */
691
692 #if EFSYS_OPT_RX_SCALE
693 __checkReturn efx_rc_t
efx_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)unsigned int * table,__in size_t n)694 efx_rx_scale_tbl_set(
695 __in efx_nic_t *enp,
696 __in uint32_t rss_context,
697 __in_ecount(n) unsigned int *table,
698 __in size_t n)
699 {
700 const efx_rx_ops_t *erxop = enp->en_erxop;
701 efx_rc_t rc;
702
703 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
704 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
705
706 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0)
707 goto fail1;
708
709 return (0);
710
711 fail1:
712 EFSYS_PROBE1(fail1, efx_rc_t, rc);
713
714 return (rc);
715 }
716 #endif /* EFSYS_OPT_RX_SCALE */
717
718 void
efx_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)719 efx_rx_qpost(
720 __in efx_rxq_t *erp,
721 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
722 __in size_t size,
723 __in unsigned int ndescs,
724 __in unsigned int completed,
725 __in unsigned int added)
726 {
727 efx_nic_t *enp = erp->er_enp;
728 const efx_rx_ops_t *erxop = enp->en_erxop;
729
730 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
731 EFSYS_ASSERT(erp->er_buf_size == 0 || size == erp->er_buf_size);
732
733 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added);
734 }
735
736 #if EFSYS_OPT_RX_PACKED_STREAM
737
738 void
efx_rx_qpush_ps_credits(__in efx_rxq_t * erp)739 efx_rx_qpush_ps_credits(
740 __in efx_rxq_t *erp)
741 {
742 efx_nic_t *enp = erp->er_enp;
743 const efx_rx_ops_t *erxop = enp->en_erxop;
744
745 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
746
747 erxop->erxo_qpush_ps_credits(erp);
748 }
749
750 __checkReturn uint8_t *
efx_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)751 efx_rx_qps_packet_info(
752 __in efx_rxq_t *erp,
753 __in uint8_t *buffer,
754 __in uint32_t buffer_length,
755 __in uint32_t current_offset,
756 __out uint16_t *lengthp,
757 __out uint32_t *next_offsetp,
758 __out uint32_t *timestamp)
759 {
760 efx_nic_t *enp = erp->er_enp;
761 const efx_rx_ops_t *erxop = enp->en_erxop;
762
763 return (erxop->erxo_qps_packet_info(erp, buffer,
764 buffer_length, current_offset, lengthp,
765 next_offsetp, timestamp));
766 }
767
768 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
769
770 void
efx_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)771 efx_rx_qpush(
772 __in efx_rxq_t *erp,
773 __in unsigned int added,
774 __inout unsigned int *pushedp)
775 {
776 efx_nic_t *enp = erp->er_enp;
777 const efx_rx_ops_t *erxop = enp->en_erxop;
778
779 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
780
781 erxop->erxo_qpush(erp, added, pushedp);
782 }
783
784 __checkReturn efx_rc_t
efx_rx_qflush(__in efx_rxq_t * erp)785 efx_rx_qflush(
786 __in efx_rxq_t *erp)
787 {
788 efx_nic_t *enp = erp->er_enp;
789 const efx_rx_ops_t *erxop = enp->en_erxop;
790 efx_rc_t rc;
791
792 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
793
794 if ((rc = erxop->erxo_qflush(erp)) != 0)
795 goto fail1;
796
797 return (0);
798
799 fail1:
800 EFSYS_PROBE1(fail1, efx_rc_t, rc);
801
802 return (rc);
803 }
804
805 __checkReturn size_t
efx_rxq_size(__in const efx_nic_t * enp,__in unsigned int ndescs)806 efx_rxq_size(
807 __in const efx_nic_t *enp,
808 __in unsigned int ndescs)
809 {
810 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
811
812 return (ndescs * encp->enc_rx_desc_size);
813 }
814
815 __checkReturn unsigned int
efx_rxq_nbufs(__in const efx_nic_t * enp,__in unsigned int ndescs)816 efx_rxq_nbufs(
817 __in const efx_nic_t *enp,
818 __in unsigned int ndescs)
819 {
820 return (EFX_DIV_ROUND_UP(efx_rxq_size(enp, ndescs), EFX_BUF_SIZE));
821 }
822
823 void
efx_rx_qenable(__in efx_rxq_t * erp)824 efx_rx_qenable(
825 __in efx_rxq_t *erp)
826 {
827 efx_nic_t *enp = erp->er_enp;
828 const efx_rx_ops_t *erxop = enp->en_erxop;
829
830 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
831
832 erxop->erxo_qenable(erp);
833 }
834
835 static __checkReturn efx_rc_t
efx_rx_qcreate_internal(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)836 efx_rx_qcreate_internal(
837 __in efx_nic_t *enp,
838 __in unsigned int index,
839 __in unsigned int label,
840 __in efx_rxq_type_t type,
841 __in_opt const efx_rxq_type_data_t *type_data,
842 __in efsys_mem_t *esmp,
843 __in size_t ndescs,
844 __in uint32_t id,
845 __in unsigned int flags,
846 __in efx_evq_t *eep,
847 __deref_out efx_rxq_t **erpp)
848 {
849 const efx_rx_ops_t *erxop = enp->en_erxop;
850 efx_rxq_t *erp;
851 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
852 efx_rc_t rc;
853
854 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC);
855 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX);
856
857 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit);
858
859 EFSYS_ASSERT(ISP2(encp->enc_rxq_max_ndescs));
860 EFSYS_ASSERT(ISP2(encp->enc_rxq_min_ndescs));
861
862 if (index >= encp->enc_rxq_limit) {
863 rc = EINVAL;
864 goto fail1;
865 }
866
867 if (!ISP2(ndescs) ||
868 ndescs < encp->enc_rxq_min_ndescs ||
869 ndescs > encp->enc_rxq_max_ndescs) {
870 rc = EINVAL;
871 goto fail2;
872 }
873
874 /* Allocate an RXQ object */
875 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp);
876
877 if (erp == NULL) {
878 rc = ENOMEM;
879 goto fail3;
880 }
881
882 erp->er_magic = EFX_RXQ_MAGIC;
883 erp->er_enp = enp;
884 erp->er_index = index;
885 erp->er_mask = ndescs - 1;
886 erp->er_esmp = esmp;
887
888 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp,
889 ndescs, id, flags, eep, erp)) != 0)
890 goto fail4;
891
892 /* Sanity check queue creation result */
893 if (flags & EFX_RXQ_FLAG_RSS_HASH) {
894 const efx_rx_prefix_layout_t *erplp = &erp->er_prefix_layout;
895 const efx_rx_prefix_field_info_t *rss_hash_field;
896
897 rss_hash_field =
898 &erplp->erpl_fields[EFX_RX_PREFIX_FIELD_RSS_HASH];
899 if (rss_hash_field->erpfi_width_bits == 0)
900 goto fail5;
901 }
902
903 enp->en_rx_qcount++;
904 *erpp = erp;
905
906 return (0);
907
908 fail5:
909 EFSYS_PROBE(fail5);
910
911 erxop->erxo_qdestroy(erp);
912 fail4:
913 EFSYS_PROBE(fail4);
914
915 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
916 fail3:
917 EFSYS_PROBE(fail3);
918 fail2:
919 EFSYS_PROBE(fail2);
920 fail1:
921 EFSYS_PROBE1(fail1, efx_rc_t, rc);
922
923 return (rc);
924 }
925
926 __checkReturn efx_rc_t
efx_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in size_t buf_size,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)927 efx_rx_qcreate(
928 __in efx_nic_t *enp,
929 __in unsigned int index,
930 __in unsigned int label,
931 __in efx_rxq_type_t type,
932 __in size_t buf_size,
933 __in efsys_mem_t *esmp,
934 __in size_t ndescs,
935 __in uint32_t id,
936 __in unsigned int flags,
937 __in efx_evq_t *eep,
938 __deref_out efx_rxq_t **erpp)
939 {
940 efx_rxq_type_data_t type_data;
941
942 memset(&type_data, 0, sizeof (type_data));
943
944 type_data.ertd_default.ed_buf_size = buf_size;
945
946 return efx_rx_qcreate_internal(enp, index, label, type, &type_data,
947 esmp, ndescs, id, flags, eep, erpp);
948 }
949
950 #if EFSYS_OPT_RX_PACKED_STREAM
951
952 __checkReturn efx_rc_t
efx_rx_qcreate_packed_stream(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in uint32_t ps_buf_size,__in efsys_mem_t * esmp,__in size_t ndescs,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)953 efx_rx_qcreate_packed_stream(
954 __in efx_nic_t *enp,
955 __in unsigned int index,
956 __in unsigned int label,
957 __in uint32_t ps_buf_size,
958 __in efsys_mem_t *esmp,
959 __in size_t ndescs,
960 __in efx_evq_t *eep,
961 __deref_out efx_rxq_t **erpp)
962 {
963 efx_rxq_type_data_t type_data;
964
965 memset(&type_data, 0, sizeof (type_data));
966
967 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size;
968
969 return efx_rx_qcreate_internal(enp, index, label,
970 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs,
971 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp);
972 }
973
974 #endif
975
976 #if EFSYS_OPT_RX_ES_SUPER_BUFFER
977
978 __checkReturn efx_rc_t
efx_rx_qcreate_es_super_buffer(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in uint32_t n_bufs_per_desc,__in uint32_t max_dma_len,__in uint32_t buf_stride,__in uint32_t hol_block_timeout,__in efsys_mem_t * esmp,__in size_t ndescs,__in unsigned int flags,__in efx_evq_t * eep,__deref_out efx_rxq_t ** erpp)979 efx_rx_qcreate_es_super_buffer(
980 __in efx_nic_t *enp,
981 __in unsigned int index,
982 __in unsigned int label,
983 __in uint32_t n_bufs_per_desc,
984 __in uint32_t max_dma_len,
985 __in uint32_t buf_stride,
986 __in uint32_t hol_block_timeout,
987 __in efsys_mem_t *esmp,
988 __in size_t ndescs,
989 __in unsigned int flags,
990 __in efx_evq_t *eep,
991 __deref_out efx_rxq_t **erpp)
992 {
993 efx_rc_t rc;
994 efx_rxq_type_data_t type_data;
995
996 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) {
997 rc = EINVAL;
998 goto fail1;
999 }
1000
1001 memset(&type_data, 0, sizeof (type_data));
1002
1003 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc;
1004 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len;
1005 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride;
1006 type_data.ertd_es_super_buffer.eessb_hol_block_timeout =
1007 hol_block_timeout;
1008
1009 rc = efx_rx_qcreate_internal(enp, index, label,
1010 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs,
1011 0 /* id unused on EF10 */, flags, eep, erpp);
1012 if (rc != 0)
1013 goto fail2;
1014
1015 return (0);
1016
1017 fail2:
1018 EFSYS_PROBE(fail2);
1019 fail1:
1020 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1021
1022 return (rc);
1023 }
1024
1025 #endif
1026
1027
1028 void
efx_rx_qdestroy(__in efx_rxq_t * erp)1029 efx_rx_qdestroy(
1030 __in efx_rxq_t *erp)
1031 {
1032 efx_nic_t *enp = erp->er_enp;
1033 const efx_rx_ops_t *erxop = enp->en_erxop;
1034
1035 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1036
1037 EFSYS_ASSERT(enp->en_rx_qcount != 0);
1038 --enp->en_rx_qcount;
1039
1040 erxop->erxo_qdestroy(erp);
1041
1042 /* Free the RXQ object */
1043 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp);
1044 }
1045
1046 __checkReturn efx_rc_t
efx_pseudo_hdr_pkt_length_get(__in efx_rxq_t * erp,__in uint8_t * buffer,__out uint16_t * lengthp)1047 efx_pseudo_hdr_pkt_length_get(
1048 __in efx_rxq_t *erp,
1049 __in uint8_t *buffer,
1050 __out uint16_t *lengthp)
1051 {
1052 efx_nic_t *enp = erp->er_enp;
1053 const efx_rx_ops_t *erxop = enp->en_erxop;
1054
1055 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1056
1057 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp));
1058 }
1059
1060 #if EFSYS_OPT_RX_SCALE
1061 __checkReturn uint32_t
efx_pseudo_hdr_hash_get(__in efx_rxq_t * erp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)1062 efx_pseudo_hdr_hash_get(
1063 __in efx_rxq_t *erp,
1064 __in efx_rx_hash_alg_t func,
1065 __in uint8_t *buffer)
1066 {
1067 efx_nic_t *enp = erp->er_enp;
1068 const efx_rx_ops_t *erxop = enp->en_erxop;
1069
1070 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1071
1072 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE);
1073 return (erxop->erxo_prefix_hash(enp, func, buffer));
1074 }
1075 #endif /* EFSYS_OPT_RX_SCALE */
1076
1077 __checkReturn efx_rc_t
efx_rx_prefix_get_layout(__in const efx_rxq_t * erp,__out efx_rx_prefix_layout_t * erplp)1078 efx_rx_prefix_get_layout(
1079 __in const efx_rxq_t *erp,
1080 __out efx_rx_prefix_layout_t *erplp)
1081 {
1082 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1083
1084 *erplp = erp->er_prefix_layout;
1085
1086 return (0);
1087 }
1088
1089 #if EFSYS_OPT_SIENA
1090
1091 static __checkReturn efx_rc_t
siena_rx_init(__in efx_nic_t * enp)1092 siena_rx_init(
1093 __in efx_nic_t *enp)
1094 {
1095 efx_oword_t oword;
1096 unsigned int index;
1097
1098 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
1099
1100 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0);
1101 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0);
1102 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0);
1103 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0);
1104 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0);
1105 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32);
1106 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
1107
1108 /* Zero the RSS table */
1109 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS;
1110 index++) {
1111 EFX_ZERO_OWORD(oword);
1112 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1113 index, &oword, B_TRUE);
1114 }
1115
1116 #if EFSYS_OPT_RX_SCALE
1117 /* The RSS key and indirection table are writable. */
1118 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE;
1119
1120 /* Hardware can insert RX hash with/without RSS */
1121 enp->en_hash_support = EFX_RX_HASH_AVAILABLE;
1122 #endif /* EFSYS_OPT_RX_SCALE */
1123
1124 return (0);
1125 }
1126
1127 #if EFSYS_OPT_RX_SCATTER
1128 static __checkReturn efx_rc_t
siena_rx_scatter_enable(__in efx_nic_t * enp,__in unsigned int buf_size)1129 siena_rx_scatter_enable(
1130 __in efx_nic_t *enp,
1131 __in unsigned int buf_size)
1132 {
1133 unsigned int nbuf32;
1134 efx_oword_t oword;
1135 efx_rc_t rc;
1136
1137 nbuf32 = buf_size / 32;
1138 if ((nbuf32 == 0) ||
1139 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) ||
1140 ((buf_size % 32) != 0)) {
1141 rc = EINVAL;
1142 goto fail1;
1143 }
1144
1145 if (enp->en_rx_qcount > 0) {
1146 rc = EBUSY;
1147 goto fail2;
1148 }
1149
1150 /* Set scatter buffer size */
1151 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword);
1152 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32);
1153 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword);
1154
1155 /* Enable scatter for packets not matching a filter */
1156 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
1157 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1);
1158 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword);
1159
1160 return (0);
1161
1162 fail2:
1163 EFSYS_PROBE(fail2);
1164 fail1:
1165 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1166
1167 return (rc);
1168 }
1169 #endif /* EFSYS_OPT_RX_SCATTER */
1170
1171
1172 #define EFX_RX_LFSR_HASH(_enp, _insert) \
1173 do { \
1174 efx_oword_t oword; \
1175 \
1176 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1177 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \
1178 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \
1179 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \
1180 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
1181 (_insert) ? 1 : 0); \
1182 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1183 \
1184 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \
1185 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
1186 &oword); \
1187 EFX_SET_OWORD_FIELD(oword, \
1188 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \
1189 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \
1190 &oword); \
1191 } \
1192 \
1193 _NOTE(CONSTANTCONDITION) \
1194 } while (B_FALSE)
1195
1196 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \
1197 do { \
1198 efx_oword_t oword; \
1199 \
1200 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1201 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \
1202 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \
1203 (_ip) ? 1 : 0); \
1204 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \
1205 (_tcp) ? 0 : 1); \
1206 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \
1207 (_insert) ? 1 : 0); \
1208 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \
1209 \
1210 _NOTE(CONSTANTCONDITION) \
1211 } while (B_FALSE)
1212
1213 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \
1214 do { \
1215 efx_oword_t oword; \
1216 \
1217 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
1218 EFX_SET_OWORD_FIELD(oword, \
1219 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \
1220 EFX_SET_OWORD_FIELD(oword, \
1221 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \
1222 EFX_SET_OWORD_FIELD(oword, \
1223 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \
1224 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \
1225 \
1226 (_rc) = 0; \
1227 \
1228 _NOTE(CONSTANTCONDITION) \
1229 } while (B_FALSE)
1230
1231
1232 #if EFSYS_OPT_RX_SCALE
1233
1234 static __checkReturn efx_rc_t
siena_rx_scale_mode_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in efx_rx_hash_alg_t alg,__in efx_rx_hash_type_t type,__in boolean_t insert)1235 siena_rx_scale_mode_set(
1236 __in efx_nic_t *enp,
1237 __in uint32_t rss_context,
1238 __in efx_rx_hash_alg_t alg,
1239 __in efx_rx_hash_type_t type,
1240 __in boolean_t insert)
1241 {
1242 efx_rc_t rc;
1243
1244 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1245 rc = EINVAL;
1246 goto fail1;
1247 }
1248
1249 switch (alg) {
1250 case EFX_RX_HASHALG_LFSR:
1251 EFX_RX_LFSR_HASH(enp, insert);
1252 break;
1253
1254 case EFX_RX_HASHALG_TOEPLITZ:
1255 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert,
1256 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE,
1257 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE);
1258
1259 EFX_RX_TOEPLITZ_IPV6_HASH(enp,
1260 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE,
1261 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE,
1262 rc);
1263 if (rc != 0)
1264 goto fail2;
1265
1266 break;
1267
1268 default:
1269 rc = EINVAL;
1270 goto fail3;
1271 }
1272
1273 return (0);
1274
1275 fail3:
1276 EFSYS_PROBE(fail3);
1277 fail2:
1278 EFSYS_PROBE(fail2);
1279 fail1:
1280 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1281
1282 EFX_RX_LFSR_HASH(enp, B_FALSE);
1283
1284 return (rc);
1285 }
1286 #endif
1287
1288 #if EFSYS_OPT_RX_SCALE
1289 static __checkReturn efx_rc_t
siena_rx_scale_key_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)uint8_t * key,__in size_t n)1290 siena_rx_scale_key_set(
1291 __in efx_nic_t *enp,
1292 __in uint32_t rss_context,
1293 __in_ecount(n) uint8_t *key,
1294 __in size_t n)
1295 {
1296 efx_oword_t oword;
1297 unsigned int byte;
1298 unsigned int offset;
1299 efx_rc_t rc;
1300
1301 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1302 rc = EINVAL;
1303 goto fail1;
1304 }
1305
1306 byte = 0;
1307
1308 /* Write Toeplitz IPv4 hash key */
1309 EFX_ZERO_OWORD(oword);
1310 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1311 offset > 0 && byte < n;
1312 --offset)
1313 oword.eo_u8[offset - 1] = key[byte++];
1314
1315 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1316
1317 byte = 0;
1318
1319 /* Verify Toeplitz IPv4 hash key */
1320 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword);
1321 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8;
1322 offset > 0 && byte < n;
1323 --offset) {
1324 if (oword.eo_u8[offset - 1] != key[byte++]) {
1325 rc = EFAULT;
1326 goto fail2;
1327 }
1328 }
1329
1330 if ((enp->en_features & EFX_FEATURE_IPV6) == 0)
1331 goto done;
1332
1333 byte = 0;
1334
1335 /* Write Toeplitz IPv6 hash key 3 */
1336 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1337 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1338 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1339 offset > 0 && byte < n;
1340 --offset)
1341 oword.eo_u8[offset - 1] = key[byte++];
1342
1343 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1344
1345 /* Write Toeplitz IPv6 hash key 2 */
1346 EFX_ZERO_OWORD(oword);
1347 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1348 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1349 offset > 0 && byte < n;
1350 --offset)
1351 oword.eo_u8[offset - 1] = key[byte++];
1352
1353 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1354
1355 /* Write Toeplitz IPv6 hash key 1 */
1356 EFX_ZERO_OWORD(oword);
1357 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1358 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1359 offset > 0 && byte < n;
1360 --offset)
1361 oword.eo_u8[offset - 1] = key[byte++];
1362
1363 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1364
1365 byte = 0;
1366
1367 /* Verify Toeplitz IPv6 hash key 3 */
1368 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword);
1369 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN +
1370 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8;
1371 offset > 0 && byte < n;
1372 --offset) {
1373 if (oword.eo_u8[offset - 1] != key[byte++]) {
1374 rc = EFAULT;
1375 goto fail3;
1376 }
1377 }
1378
1379 /* Verify Toeplitz IPv6 hash key 2 */
1380 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword);
1381 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN +
1382 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8;
1383 offset > 0 && byte < n;
1384 --offset) {
1385 if (oword.eo_u8[offset - 1] != key[byte++]) {
1386 rc = EFAULT;
1387 goto fail4;
1388 }
1389 }
1390
1391 /* Verify Toeplitz IPv6 hash key 1 */
1392 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword);
1393 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN +
1394 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8;
1395 offset > 0 && byte < n;
1396 --offset) {
1397 if (oword.eo_u8[offset - 1] != key[byte++]) {
1398 rc = EFAULT;
1399 goto fail5;
1400 }
1401 }
1402
1403 done:
1404 return (0);
1405
1406 fail5:
1407 EFSYS_PROBE(fail5);
1408 fail4:
1409 EFSYS_PROBE(fail4);
1410 fail3:
1411 EFSYS_PROBE(fail3);
1412 fail2:
1413 EFSYS_PROBE(fail2);
1414 fail1:
1415 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1416
1417 return (rc);
1418 }
1419 #endif
1420
1421 #if EFSYS_OPT_RX_SCALE
1422 static __checkReturn efx_rc_t
siena_rx_scale_tbl_set(__in efx_nic_t * enp,__in uint32_t rss_context,__in_ecount (n)unsigned int * table,__in size_t n)1423 siena_rx_scale_tbl_set(
1424 __in efx_nic_t *enp,
1425 __in uint32_t rss_context,
1426 __in_ecount(n) unsigned int *table,
1427 __in size_t n)
1428 {
1429 efx_oword_t oword;
1430 int index;
1431 efx_rc_t rc;
1432
1433 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS);
1434 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH));
1435
1436 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) {
1437 rc = EINVAL;
1438 goto fail1;
1439 }
1440
1441 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) {
1442 rc = EINVAL;
1443 goto fail2;
1444 }
1445
1446 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) {
1447 uint32_t byte;
1448
1449 /* Calculate the entry to place in the table */
1450 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1451
1452 EFSYS_PROBE2(table, int, index, uint32_t, byte);
1453
1454 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte);
1455
1456 /* Write the table */
1457 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL,
1458 index, &oword, B_TRUE);
1459 }
1460
1461 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) {
1462 uint32_t byte;
1463
1464 /* Determine if we're starting a new batch */
1465 byte = (n > 0) ? (uint32_t)table[index % n] : 0;
1466
1467 /* Read the table */
1468 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL,
1469 index, &oword, B_TRUE);
1470
1471 /* Verify the entry */
1472 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) {
1473 rc = EFAULT;
1474 goto fail3;
1475 }
1476 }
1477
1478 return (0);
1479
1480 fail3:
1481 EFSYS_PROBE(fail3);
1482 fail2:
1483 EFSYS_PROBE(fail2);
1484 fail1:
1485 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1486
1487 return (rc);
1488 }
1489 #endif
1490
1491 /*
1492 * Falcon/Siena pseudo-header
1493 * --------------------------
1494 *
1495 * Receive packets are prefixed by an optional 16 byte pseudo-header.
1496 * The pseudo-header is a byte array of one of the forms:
1497 *
1498 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
1499 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT
1500 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL
1501 *
1502 * where:
1503 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian)
1504 * LL.LL LFSR hash (16-bit big-endian)
1505 */
1506
1507 /*
1508 * Provide Rx prefix layout with Toeplitz hash only since LSFR is
1509 * used by no supported drivers.
1510 *
1511 * Siena does not support Rx prefix choice via MC_CMD_GET_RX_PREFIX_ID
1512 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID.
1513 */
1514 static const efx_rx_prefix_layout_t siena_toeplitz_rx_prefix_layout = {
1515 .erpl_id = 0,
1516 .erpl_length = 16,
1517 .erpl_fields = {
1518 [EFX_RX_PREFIX_FIELD_RSS_HASH] = { 12 * 8, 32, B_TRUE },
1519 }
1520 };
1521
1522 #if EFSYS_OPT_RX_SCALE
1523 static __checkReturn uint32_t
siena_rx_prefix_hash(__in efx_nic_t * enp,__in efx_rx_hash_alg_t func,__in uint8_t * buffer)1524 siena_rx_prefix_hash(
1525 __in efx_nic_t *enp,
1526 __in efx_rx_hash_alg_t func,
1527 __in uint8_t *buffer)
1528 {
1529 _NOTE(ARGUNUSED(enp))
1530
1531 switch (func) {
1532 case EFX_RX_HASHALG_TOEPLITZ:
1533 return ((buffer[12] << 24) |
1534 (buffer[13] << 16) |
1535 (buffer[14] << 8) |
1536 buffer[15]);
1537
1538 case EFX_RX_HASHALG_LFSR:
1539 return ((buffer[14] << 8) | buffer[15]);
1540
1541 default:
1542 EFSYS_ASSERT(0);
1543 return (0);
1544 }
1545 }
1546 #endif /* EFSYS_OPT_RX_SCALE */
1547
1548 static __checkReturn efx_rc_t
siena_rx_prefix_pktlen(__in efx_nic_t * enp,__in uint8_t * buffer,__out uint16_t * lengthp)1549 siena_rx_prefix_pktlen(
1550 __in efx_nic_t *enp,
1551 __in uint8_t *buffer,
1552 __out uint16_t *lengthp)
1553 {
1554 _NOTE(ARGUNUSED(enp, buffer, lengthp))
1555
1556 /* Not supported by Falcon/Siena hardware */
1557 EFSYS_ASSERT(0);
1558 return (ENOTSUP);
1559 }
1560
1561
1562 static void
siena_rx_qpost(__in efx_rxq_t * erp,__in_ecount (ndescs)efsys_dma_addr_t * addrp,__in size_t size,__in unsigned int ndescs,__in unsigned int completed,__in unsigned int added)1563 siena_rx_qpost(
1564 __in efx_rxq_t *erp,
1565 __in_ecount(ndescs) efsys_dma_addr_t *addrp,
1566 __in size_t size,
1567 __in unsigned int ndescs,
1568 __in unsigned int completed,
1569 __in unsigned int added)
1570 {
1571 efx_qword_t qword;
1572 unsigned int i;
1573 unsigned int offset;
1574 unsigned int id;
1575
1576 /* The client driver must not overfill the queue */
1577 EFSYS_ASSERT3U(added - completed + ndescs, <=,
1578 EFX_RXQ_LIMIT(erp->er_mask + 1));
1579
1580 id = added & (erp->er_mask);
1581 for (i = 0; i < ndescs; i++) {
1582 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index,
1583 unsigned int, id, efsys_dma_addr_t, addrp[i],
1584 size_t, size);
1585
1586 EFX_POPULATE_QWORD_3(qword,
1587 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size),
1588 FSF_AZ_RX_KER_BUF_ADDR_DW0,
1589 (uint32_t)(addrp[i] & 0xffffffff),
1590 FSF_AZ_RX_KER_BUF_ADDR_DW1,
1591 (uint32_t)(addrp[i] >> 32));
1592
1593 offset = id * sizeof (efx_qword_t);
1594 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword);
1595
1596 id = (id + 1) & (erp->er_mask);
1597 }
1598 }
1599
1600 static void
siena_rx_qpush(__in efx_rxq_t * erp,__in unsigned int added,__inout unsigned int * pushedp)1601 siena_rx_qpush(
1602 __in efx_rxq_t *erp,
1603 __in unsigned int added,
1604 __inout unsigned int *pushedp)
1605 {
1606 efx_nic_t *enp = erp->er_enp;
1607 unsigned int pushed = *pushedp;
1608 uint32_t wptr;
1609 efx_oword_t oword;
1610 efx_dword_t dword;
1611
1612 /* All descriptors are pushed */
1613 *pushedp = added;
1614
1615 /* Push the populated descriptors out */
1616 wptr = added & erp->er_mask;
1617
1618 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr);
1619
1620 /* Only write the third DWORD */
1621 EFX_POPULATE_DWORD_1(dword,
1622 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3));
1623
1624 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */
1625 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1,
1626 SIENA_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask);
1627 EFSYS_PIO_WRITE_BARRIER();
1628 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
1629 erp->er_index, &dword, B_FALSE);
1630 }
1631
1632 #if EFSYS_OPT_RX_PACKED_STREAM
1633 static void
siena_rx_qpush_ps_credits(__in efx_rxq_t * erp)1634 siena_rx_qpush_ps_credits(
1635 __in efx_rxq_t *erp)
1636 {
1637 /* Not supported by Siena hardware */
1638 EFSYS_ASSERT(0);
1639 }
1640
1641 static uint8_t *
siena_rx_qps_packet_info(__in efx_rxq_t * erp,__in uint8_t * buffer,__in uint32_t buffer_length,__in uint32_t current_offset,__out uint16_t * lengthp,__out uint32_t * next_offsetp,__out uint32_t * timestamp)1642 siena_rx_qps_packet_info(
1643 __in efx_rxq_t *erp,
1644 __in uint8_t *buffer,
1645 __in uint32_t buffer_length,
1646 __in uint32_t current_offset,
1647 __out uint16_t *lengthp,
1648 __out uint32_t *next_offsetp,
1649 __out uint32_t *timestamp)
1650 {
1651 /* Not supported by Siena hardware */
1652 EFSYS_ASSERT(0);
1653
1654 return (NULL);
1655 }
1656 #endif /* EFSYS_OPT_RX_PACKED_STREAM */
1657
1658 static __checkReturn efx_rc_t
siena_rx_qflush(__in efx_rxq_t * erp)1659 siena_rx_qflush(
1660 __in efx_rxq_t *erp)
1661 {
1662 efx_nic_t *enp = erp->er_enp;
1663 efx_oword_t oword;
1664 uint32_t label;
1665
1666 label = erp->er_index;
1667
1668 /* Flush the queue */
1669 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1,
1670 FRF_AZ_RX_FLUSH_DESCQ, label);
1671 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword);
1672
1673 return (0);
1674 }
1675
1676 static void
siena_rx_qenable(__in efx_rxq_t * erp)1677 siena_rx_qenable(
1678 __in efx_rxq_t *erp)
1679 {
1680 efx_nic_t *enp = erp->er_enp;
1681 efx_oword_t oword;
1682
1683 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
1684
1685 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL,
1686 erp->er_index, &oword, B_TRUE);
1687
1688 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0);
1689 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0);
1690 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1);
1691
1692 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1693 erp->er_index, &oword, B_TRUE);
1694 }
1695
1696 static __checkReturn efx_rc_t
siena_rx_qcreate(__in efx_nic_t * enp,__in unsigned int index,__in unsigned int label,__in efx_rxq_type_t type,__in_opt const efx_rxq_type_data_t * type_data,__in efsys_mem_t * esmp,__in size_t ndescs,__in uint32_t id,__in unsigned int flags,__in efx_evq_t * eep,__in efx_rxq_t * erp)1697 siena_rx_qcreate(
1698 __in efx_nic_t *enp,
1699 __in unsigned int index,
1700 __in unsigned int label,
1701 __in efx_rxq_type_t type,
1702 __in_opt const efx_rxq_type_data_t *type_data,
1703 __in efsys_mem_t *esmp,
1704 __in size_t ndescs,
1705 __in uint32_t id,
1706 __in unsigned int flags,
1707 __in efx_evq_t *eep,
1708 __in efx_rxq_t *erp)
1709 {
1710 efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
1711 efx_oword_t oword;
1712 uint32_t size;
1713 boolean_t jumbo = B_FALSE;
1714 efx_rc_t rc;
1715
1716 _NOTE(ARGUNUSED(esmp))
1717
1718 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS ==
1719 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH));
1720 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS);
1721
1722 for (size = 0;
1723 (1U << size) <= encp->enc_rxq_max_ndescs / encp->enc_rxq_min_ndescs;
1724 size++)
1725 if ((1U << size) == (uint32_t)ndescs / encp->enc_rxq_min_ndescs)
1726 break;
1727 if (id + (1 << size) >= encp->enc_buftbl_limit) {
1728 rc = EINVAL;
1729 goto fail1;
1730 }
1731
1732 switch (type) {
1733 case EFX_RXQ_TYPE_DEFAULT:
1734 erp->er_buf_size = type_data->ertd_default.ed_buf_size;
1735 /*
1736 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated
1737 * it is always delivered from HW in the pseudo-header.
1738 */
1739 break;
1740
1741 default:
1742 rc = EINVAL;
1743 goto fail2;
1744 }
1745
1746 if (flags & EFX_RXQ_FLAG_SCATTER) {
1747 #if EFSYS_OPT_RX_SCATTER
1748 jumbo = B_TRUE;
1749 #else
1750 rc = EINVAL;
1751 goto fail3;
1752 #endif /* EFSYS_OPT_RX_SCATTER */
1753 }
1754
1755 /* Set up the new descriptor queue */
1756 EFX_POPULATE_OWORD_7(oword,
1757 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id,
1758 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index,
1759 FRF_AZ_RX_DESCQ_OWNER_ID, 0,
1760 FRF_AZ_RX_DESCQ_LABEL, label,
1761 FRF_AZ_RX_DESCQ_SIZE, size,
1762 FRF_AZ_RX_DESCQ_TYPE, 0,
1763 FRF_AZ_RX_DESCQ_JUMBO, jumbo);
1764
1765 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1766 erp->er_index, &oword, B_TRUE);
1767
1768 erp->er_prefix_layout = siena_toeplitz_rx_prefix_layout;
1769
1770 return (0);
1771
1772 #if !EFSYS_OPT_RX_SCATTER
1773 fail3:
1774 EFSYS_PROBE(fail3);
1775 #endif
1776 fail2:
1777 EFSYS_PROBE(fail2);
1778 fail1:
1779 EFSYS_PROBE1(fail1, efx_rc_t, rc);
1780
1781 return (rc);
1782 }
1783
1784 static void
siena_rx_qdestroy(__in efx_rxq_t * erp)1785 siena_rx_qdestroy(
1786 __in efx_rxq_t *erp)
1787 {
1788 efx_nic_t *enp = erp->er_enp;
1789 efx_oword_t oword;
1790
1791 /* Purge descriptor queue */
1792 EFX_ZERO_OWORD(oword);
1793
1794 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL,
1795 erp->er_index, &oword, B_TRUE);
1796 }
1797
1798 static void
siena_rx_fini(__in efx_nic_t * enp)1799 siena_rx_fini(
1800 __in efx_nic_t *enp)
1801 {
1802 _NOTE(ARGUNUSED(enp))
1803 }
1804
1805 #endif /* EFSYS_OPT_SIENA */
1806
1807 static __checkReturn boolean_t
efx_rx_prefix_layout_fields_match(__in const efx_rx_prefix_field_info_t * erpfip1,__in const efx_rx_prefix_field_info_t * erpfip2)1808 efx_rx_prefix_layout_fields_match(
1809 __in const efx_rx_prefix_field_info_t *erpfip1,
1810 __in const efx_rx_prefix_field_info_t *erpfip2)
1811 {
1812 if (erpfip1->erpfi_offset_bits != erpfip2->erpfi_offset_bits)
1813 return (B_FALSE);
1814
1815 if (erpfip1->erpfi_width_bits != erpfip2->erpfi_width_bits)
1816 return (B_FALSE);
1817
1818 if (erpfip1->erpfi_big_endian != erpfip2->erpfi_big_endian)
1819 return (B_FALSE);
1820
1821 return (B_TRUE);
1822 }
1823
1824 __checkReturn uint32_t
efx_rx_prefix_layout_check(__in const efx_rx_prefix_layout_t * available,__in const efx_rx_prefix_layout_t * wanted)1825 efx_rx_prefix_layout_check(
1826 __in const efx_rx_prefix_layout_t *available,
1827 __in const efx_rx_prefix_layout_t *wanted)
1828 {
1829 uint32_t result = 0;
1830 unsigned int i;
1831
1832 EFX_STATIC_ASSERT(EFX_RX_PREFIX_NFIELDS < sizeof (result) * 8);
1833 for (i = 0; i < EFX_RX_PREFIX_NFIELDS; ++i) {
1834 /* Skip the field if driver does not want to use it */
1835 if (wanted->erpl_fields[i].erpfi_width_bits == 0)
1836 continue;
1837
1838 if (efx_rx_prefix_layout_fields_match(
1839 &available->erpl_fields[i],
1840 &wanted->erpl_fields[i]) == B_FALSE)
1841 result |= (1U << i);
1842 }
1843
1844 return (result);
1845 }
1846