1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2012-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFX_OPTS_EF10() 12 13 14 static __checkReturn efx_rc_t 15 efx_mcdi_init_rxq( 16 __in efx_nic_t *enp, 17 __in uint32_t ndescs, 18 __in efx_evq_t *eep, 19 __in uint32_t label, 20 __in uint32_t instance, 21 __in efsys_mem_t *esmp, 22 __in boolean_t disable_scatter, 23 __in boolean_t want_inner_classes, 24 __in uint32_t buf_size, 25 __in uint32_t ps_bufsize, 26 __in uint32_t es_bufs_per_desc, 27 __in uint32_t es_max_dma_len, 28 __in uint32_t es_buf_stride, 29 __in uint32_t hol_block_timeout) 30 { 31 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 32 efx_mcdi_req_t req; 33 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_INIT_RXQ_V4_IN_LEN, 34 MC_CMD_INIT_RXQ_V4_OUT_LEN); 35 int npages = efx_rxq_nbufs(enp, ndescs); 36 int i; 37 efx_qword_t *dma_addr; 38 uint64_t addr; 39 efx_rc_t rc; 40 uint32_t dma_mode; 41 boolean_t want_outer_classes; 42 boolean_t no_cont_ev; 43 44 EFSYS_ASSERT3U(ndescs, <=, encp->enc_rxq_max_ndescs); 45 46 if ((esmp == NULL) || 47 (EFSYS_MEM_SIZE(esmp) < efx_rxq_size(enp, ndescs))) { 48 rc = EINVAL; 49 goto fail1; 50 } 51 52 no_cont_ev = (eep->ee_flags & EFX_EVQ_FLAGS_NO_CONT_EV); 53 if ((no_cont_ev == B_TRUE) && (disable_scatter == B_FALSE)) { 54 /* TODO: Support scatter in NO_CONT_EV mode */ 55 rc = EINVAL; 56 goto fail2; 57 } 58 59 if (ps_bufsize > 0) 60 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM; 61 else if (es_bufs_per_desc > 0) 62 dma_mode = MC_CMD_INIT_RXQ_V3_IN_EQUAL_STRIDE_SUPER_BUFFER; 63 else 64 dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET; 65 66 if (encp->enc_tunnel_encapsulations_supported != 0 && 67 !want_inner_classes) { 68 /* 69 * WANT_OUTER_CLASSES can only be specified on hardware which 70 * supports tunnel encapsulation offloads, even though it is 71 * effectively the behaviour the hardware gives. 72 * 73 * Also, on hardware which does support such offloads, older 74 * firmware rejects the flag if the offloads are not supported 75 * by the current firmware variant, which means this may fail if 76 * the capabilities are not updated when the firmware variant 77 * changes. This is not an issue on newer firmware, as it was 78 * changed in bug 69842 (v6.4.2.1007) to permit this flag to be 79 * specified on all firmware variants. 80 */ 81 want_outer_classes = B_TRUE; 82 } else { 83 want_outer_classes = B_FALSE; 84 } 85 86 req.emr_cmd = MC_CMD_INIT_RXQ; 87 req.emr_in_buf = payload; 88 req.emr_in_length = MC_CMD_INIT_RXQ_V4_IN_LEN; 89 req.emr_out_buf = payload; 90 req.emr_out_length = MC_CMD_INIT_RXQ_V4_OUT_LEN; 91 92 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, ndescs); 93 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, eep->ee_index); 94 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label); 95 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance); 96 MCDI_IN_POPULATE_DWORD_10(req, INIT_RXQ_EXT_IN_FLAGS, 97 INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0, 98 INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0, 99 INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0, 100 INIT_RXQ_EXT_IN_CRC_MODE, 0, 101 INIT_RXQ_EXT_IN_FLAG_PREFIX, 1, 102 INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter, 103 INIT_RXQ_EXT_IN_DMA_MODE, 104 dma_mode, 105 INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize, 106 INIT_RXQ_EXT_IN_FLAG_WANT_OUTER_CLASSES, want_outer_classes, 107 INIT_RXQ_EXT_IN_FLAG_NO_CONT_EV, no_cont_ev); 108 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0); 109 MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, enp->en_vport_id); 110 111 if (es_bufs_per_desc > 0) { 112 MCDI_IN_SET_DWORD(req, 113 INIT_RXQ_V3_IN_ES_PACKET_BUFFERS_PER_BUCKET, 114 es_bufs_per_desc); 115 MCDI_IN_SET_DWORD(req, 116 INIT_RXQ_V3_IN_ES_MAX_DMA_LEN, es_max_dma_len); 117 MCDI_IN_SET_DWORD(req, 118 INIT_RXQ_V3_IN_ES_PACKET_STRIDE, es_buf_stride); 119 MCDI_IN_SET_DWORD(req, 120 INIT_RXQ_V3_IN_ES_HEAD_OF_LINE_BLOCK_TIMEOUT, 121 hol_block_timeout); 122 } 123 124 if (encp->enc_init_rxq_with_buffer_size) 125 MCDI_IN_SET_DWORD(req, INIT_RXQ_V4_IN_BUFFER_SIZE_BYTES, 126 buf_size); 127 128 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR); 129 addr = EFSYS_MEM_ADDR(esmp); 130 131 for (i = 0; i < npages; i++) { 132 EFX_POPULATE_QWORD_2(*dma_addr, 133 EFX_DWORD_1, (uint32_t)(addr >> 32), 134 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 135 136 dma_addr++; 137 addr += EFX_BUF_SIZE; 138 } 139 140 efx_mcdi_execute(enp, &req); 141 142 if (req.emr_rc != 0) { 143 rc = req.emr_rc; 144 goto fail3; 145 } 146 147 return (0); 148 149 fail3: 150 EFSYS_PROBE(fail3); 151 fail2: 152 EFSYS_PROBE(fail2); 153 fail1: 154 EFSYS_PROBE1(fail1, efx_rc_t, rc); 155 156 return (rc); 157 } 158 159 static __checkReturn efx_rc_t 160 efx_mcdi_fini_rxq( 161 __in efx_nic_t *enp, 162 __in uint32_t instance) 163 { 164 efx_mcdi_req_t req; 165 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_FINI_RXQ_IN_LEN, 166 MC_CMD_FINI_RXQ_OUT_LEN); 167 efx_rc_t rc; 168 169 req.emr_cmd = MC_CMD_FINI_RXQ; 170 req.emr_in_buf = payload; 171 req.emr_in_length = MC_CMD_FINI_RXQ_IN_LEN; 172 req.emr_out_buf = payload; 173 req.emr_out_length = MC_CMD_FINI_RXQ_OUT_LEN; 174 175 MCDI_IN_SET_DWORD(req, FINI_RXQ_IN_INSTANCE, instance); 176 177 efx_mcdi_execute_quiet(enp, &req); 178 179 if (req.emr_rc != 0) { 180 rc = req.emr_rc; 181 goto fail1; 182 } 183 184 return (0); 185 186 fail1: 187 /* 188 * EALREADY is not an error, but indicates that the MC has rebooted and 189 * that the RXQ has already been destroyed. 190 */ 191 if (rc != EALREADY) 192 EFSYS_PROBE1(fail1, efx_rc_t, rc); 193 194 return (rc); 195 } 196 197 #if EFSYS_OPT_RX_SCALE 198 static __checkReturn efx_rc_t 199 efx_mcdi_rss_context_alloc( 200 __in efx_nic_t *enp, 201 __in efx_rx_scale_context_type_t type, 202 __in uint32_t num_queues, 203 __out uint32_t *rss_contextp) 204 { 205 efx_mcdi_req_t req; 206 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 207 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 208 uint32_t rss_context; 209 uint32_t context_type; 210 efx_rc_t rc; 211 212 if (num_queues > EFX_MAXRSS) { 213 rc = EINVAL; 214 goto fail1; 215 } 216 217 switch (type) { 218 case EFX_RX_SCALE_EXCLUSIVE: 219 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 220 break; 221 case EFX_RX_SCALE_SHARED: 222 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 223 break; 224 default: 225 rc = EINVAL; 226 goto fail2; 227 } 228 229 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 230 req.emr_in_buf = payload; 231 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 232 req.emr_out_buf = payload; 233 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 234 235 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 236 enp->en_vport_id); 237 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 238 239 /* 240 * For exclusive contexts, NUM_QUEUES is only used to validate 241 * indirection table offsets. 242 * For shared contexts, the provided context will spread traffic over 243 * NUM_QUEUES many queues. 244 */ 245 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 246 247 efx_mcdi_execute(enp, &req); 248 249 if (req.emr_rc != 0) { 250 rc = req.emr_rc; 251 goto fail3; 252 } 253 254 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 255 rc = EMSGSIZE; 256 goto fail4; 257 } 258 259 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 260 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 261 rc = ENOENT; 262 goto fail5; 263 } 264 265 *rss_contextp = rss_context; 266 267 return (0); 268 269 fail5: 270 EFSYS_PROBE(fail5); 271 fail4: 272 EFSYS_PROBE(fail4); 273 fail3: 274 EFSYS_PROBE(fail3); 275 fail2: 276 EFSYS_PROBE(fail2); 277 fail1: 278 EFSYS_PROBE1(fail1, efx_rc_t, rc); 279 280 return (rc); 281 } 282 #endif /* EFSYS_OPT_RX_SCALE */ 283 284 #if EFSYS_OPT_RX_SCALE 285 static efx_rc_t 286 efx_mcdi_rss_context_free( 287 __in efx_nic_t *enp, 288 __in uint32_t rss_context) 289 { 290 efx_mcdi_req_t req; 291 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 292 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN); 293 efx_rc_t rc; 294 295 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 296 rc = EINVAL; 297 goto fail1; 298 } 299 300 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 301 req.emr_in_buf = payload; 302 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 303 req.emr_out_buf = payload; 304 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 305 306 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 307 308 efx_mcdi_execute_quiet(enp, &req); 309 310 if (req.emr_rc != 0) { 311 rc = req.emr_rc; 312 goto fail2; 313 } 314 315 return (0); 316 317 fail2: 318 EFSYS_PROBE(fail2); 319 fail1: 320 EFSYS_PROBE1(fail1, efx_rc_t, rc); 321 322 return (rc); 323 } 324 #endif /* EFSYS_OPT_RX_SCALE */ 325 326 #if EFSYS_OPT_RX_SCALE 327 static efx_rc_t 328 efx_mcdi_rss_context_set_flags( 329 __in efx_nic_t *enp, 330 __in uint32_t rss_context, 331 __in efx_rx_hash_type_t type) 332 { 333 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 334 efx_mcdi_req_t req; 335 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 336 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN); 337 efx_rc_t rc; 338 339 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN == 340 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN); 341 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH == 342 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH); 343 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN == 344 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN); 345 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH == 346 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH); 347 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN == 348 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN); 349 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH == 350 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH); 351 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN == 352 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN); 353 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH == 354 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH); 355 356 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 357 rc = EINVAL; 358 goto fail1; 359 } 360 361 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 362 req.emr_in_buf = payload; 363 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 364 req.emr_out_buf = payload; 365 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 366 367 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 368 rss_context); 369 370 /* 371 * If the firmware lacks support for additional modes, RSS_MODE 372 * fields must contain zeros, otherwise the operation will fail. 373 */ 374 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) 375 type &= EFX_RX_HASH_LEGACY_MASK; 376 377 MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 378 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 379 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 380 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 381 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 382 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 383 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 384 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 385 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0, 386 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE, 387 (type >> EFX_RX_CLASS_IPV4_TCP_LBN) & 388 EFX_MASK32(EFX_RX_CLASS_IPV4_TCP), 389 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE, 390 (type >> EFX_RX_CLASS_IPV4_UDP_LBN) & 391 EFX_MASK32(EFX_RX_CLASS_IPV4_UDP), 392 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE, 393 (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4), 394 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE, 395 (type >> EFX_RX_CLASS_IPV6_TCP_LBN) & 396 EFX_MASK32(EFX_RX_CLASS_IPV6_TCP), 397 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE, 398 (type >> EFX_RX_CLASS_IPV6_UDP_LBN) & 399 EFX_MASK32(EFX_RX_CLASS_IPV6_UDP), 400 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE, 401 (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6)); 402 403 efx_mcdi_execute(enp, &req); 404 405 if (req.emr_rc != 0) { 406 rc = req.emr_rc; 407 goto fail2; 408 } 409 410 return (0); 411 412 fail2: 413 EFSYS_PROBE(fail2); 414 fail1: 415 EFSYS_PROBE1(fail1, efx_rc_t, rc); 416 417 return (rc); 418 } 419 #endif /* EFSYS_OPT_RX_SCALE */ 420 421 #if EFSYS_OPT_RX_SCALE 422 static efx_rc_t 423 efx_mcdi_rss_context_set_key( 424 __in efx_nic_t *enp, 425 __in uint32_t rss_context, 426 __in_ecount(n) uint8_t *key, 427 __in size_t n) 428 { 429 efx_mcdi_req_t req; 430 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 431 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN); 432 efx_rc_t rc; 433 434 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 435 rc = EINVAL; 436 goto fail1; 437 } 438 439 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 440 req.emr_in_buf = payload; 441 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 442 req.emr_out_buf = payload; 443 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 444 445 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 446 rss_context); 447 448 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 449 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 450 rc = EINVAL; 451 goto fail2; 452 } 453 454 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 455 key, n); 456 457 efx_mcdi_execute(enp, &req); 458 459 if (req.emr_rc != 0) { 460 rc = req.emr_rc; 461 goto fail3; 462 } 463 464 return (0); 465 466 fail3: 467 EFSYS_PROBE(fail3); 468 fail2: 469 EFSYS_PROBE(fail2); 470 fail1: 471 EFSYS_PROBE1(fail1, efx_rc_t, rc); 472 473 return (rc); 474 } 475 #endif /* EFSYS_OPT_RX_SCALE */ 476 477 #if EFSYS_OPT_RX_SCALE 478 static efx_rc_t 479 efx_mcdi_rss_context_set_table( 480 __in efx_nic_t *enp, 481 __in uint32_t rss_context, 482 __in_ecount(n) unsigned int *table, 483 __in size_t n) 484 { 485 efx_mcdi_req_t req; 486 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 487 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN); 488 uint8_t *req_table; 489 int i, rc; 490 491 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 492 rc = EINVAL; 493 goto fail1; 494 } 495 496 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 497 req.emr_in_buf = payload; 498 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 499 req.emr_out_buf = payload; 500 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 501 502 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 503 rss_context); 504 505 req_table = 506 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 507 508 for (i = 0; 509 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 510 i++) { 511 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 512 } 513 514 efx_mcdi_execute(enp, &req); 515 516 if (req.emr_rc != 0) { 517 rc = req.emr_rc; 518 goto fail2; 519 } 520 521 return (0); 522 523 fail2: 524 EFSYS_PROBE(fail2); 525 fail1: 526 EFSYS_PROBE1(fail1, efx_rc_t, rc); 527 528 return (rc); 529 } 530 #endif /* EFSYS_OPT_RX_SCALE */ 531 532 533 __checkReturn efx_rc_t 534 ef10_rx_init( 535 __in efx_nic_t *enp) 536 { 537 #if EFSYS_OPT_RX_SCALE 538 539 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 540 &enp->en_rss_context) == 0) { 541 /* 542 * Allocated an exclusive RSS context, which allows both the 543 * indirection table and key to be modified. 544 */ 545 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 546 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 547 } else { 548 /* 549 * Failed to allocate an exclusive RSS context. Continue 550 * operation without support for RSS. The pseudo-header in 551 * received packets will not contain a Toeplitz hash value. 552 */ 553 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 554 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 555 } 556 557 #endif /* EFSYS_OPT_RX_SCALE */ 558 559 return (0); 560 } 561 562 #if EFSYS_OPT_RX_SCATTER 563 __checkReturn efx_rc_t 564 ef10_rx_scatter_enable( 565 __in efx_nic_t *enp, 566 __in unsigned int buf_size) 567 { 568 _NOTE(ARGUNUSED(enp, buf_size)) 569 return (0); 570 } 571 #endif /* EFSYS_OPT_RX_SCATTER */ 572 573 #if EFSYS_OPT_RX_SCALE 574 __checkReturn efx_rc_t 575 ef10_rx_scale_context_alloc( 576 __in efx_nic_t *enp, 577 __in efx_rx_scale_context_type_t type, 578 __in uint32_t num_queues, 579 __out uint32_t *rss_contextp) 580 { 581 efx_rc_t rc; 582 583 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); 584 if (rc != 0) 585 goto fail1; 586 587 return (0); 588 589 fail1: 590 EFSYS_PROBE1(fail1, efx_rc_t, rc); 591 return (rc); 592 } 593 #endif /* EFSYS_OPT_RX_SCALE */ 594 595 #if EFSYS_OPT_RX_SCALE 596 __checkReturn efx_rc_t 597 ef10_rx_scale_context_free( 598 __in efx_nic_t *enp, 599 __in uint32_t rss_context) 600 { 601 efx_rc_t rc; 602 603 rc = efx_mcdi_rss_context_free(enp, rss_context); 604 if (rc != 0) 605 goto fail1; 606 607 return (0); 608 609 fail1: 610 EFSYS_PROBE1(fail1, efx_rc_t, rc); 611 return (rc); 612 } 613 #endif /* EFSYS_OPT_RX_SCALE */ 614 615 #if EFSYS_OPT_RX_SCALE 616 __checkReturn efx_rc_t 617 ef10_rx_scale_mode_set( 618 __in efx_nic_t *enp, 619 __in uint32_t rss_context, 620 __in efx_rx_hash_alg_t alg, 621 __in efx_rx_hash_type_t type, 622 __in boolean_t insert) 623 { 624 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 625 efx_rc_t rc; 626 627 EFSYS_ASSERT3U(insert, ==, B_TRUE); 628 629 if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 || 630 insert == B_FALSE) { 631 rc = EINVAL; 632 goto fail1; 633 } 634 635 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 636 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 637 rc = ENOTSUP; 638 goto fail2; 639 } 640 rss_context = enp->en_rss_context; 641 } 642 643 if ((rc = efx_mcdi_rss_context_set_flags(enp, 644 rss_context, type)) != 0) 645 goto fail3; 646 647 return (0); 648 649 fail3: 650 EFSYS_PROBE(fail3); 651 fail2: 652 EFSYS_PROBE(fail2); 653 fail1: 654 EFSYS_PROBE1(fail1, efx_rc_t, rc); 655 656 return (rc); 657 } 658 #endif /* EFSYS_OPT_RX_SCALE */ 659 660 #if EFSYS_OPT_RX_SCALE 661 __checkReturn efx_rc_t 662 ef10_rx_scale_key_set( 663 __in efx_nic_t *enp, 664 __in uint32_t rss_context, 665 __in_ecount(n) uint8_t *key, 666 __in size_t n) 667 { 668 efx_rc_t rc; 669 670 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == 671 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 672 673 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 674 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 675 rc = ENOTSUP; 676 goto fail1; 677 } 678 rss_context = enp->en_rss_context; 679 } 680 681 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) 682 goto fail2; 683 684 return (0); 685 686 fail2: 687 EFSYS_PROBE(fail2); 688 fail1: 689 EFSYS_PROBE1(fail1, efx_rc_t, rc); 690 691 return (rc); 692 } 693 #endif /* EFSYS_OPT_RX_SCALE */ 694 695 #if EFSYS_OPT_RX_SCALE 696 __checkReturn efx_rc_t 697 ef10_rx_scale_tbl_set( 698 __in efx_nic_t *enp, 699 __in uint32_t rss_context, 700 __in_ecount(n) unsigned int *table, 701 __in size_t n) 702 { 703 efx_rc_t rc; 704 705 706 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 707 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 708 rc = ENOTSUP; 709 goto fail1; 710 } 711 rss_context = enp->en_rss_context; 712 } 713 714 if ((rc = efx_mcdi_rss_context_set_table(enp, 715 rss_context, table, n)) != 0) 716 goto fail2; 717 718 return (0); 719 720 fail2: 721 EFSYS_PROBE(fail2); 722 fail1: 723 EFSYS_PROBE1(fail1, efx_rc_t, rc); 724 725 return (rc); 726 } 727 #endif /* EFSYS_OPT_RX_SCALE */ 728 729 730 /* 731 * EF10 RX pseudo-header 732 * --------------------- 733 * 734 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 735 * 736 * +00: Toeplitz hash value. 737 * (32bit little-endian) 738 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 739 * (16bit big-endian) 740 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 741 * (16bit big-endian) 742 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 743 * (16bit little-endian) 744 * +10: MAC timestamp. Zero if timestamping is not enabled. 745 * (32bit little-endian) 746 * 747 * See "The RX Pseudo-header" in SF-109306-TC. 748 */ 749 750 __checkReturn efx_rc_t 751 ef10_rx_prefix_pktlen( 752 __in efx_nic_t *enp, 753 __in uint8_t *buffer, 754 __out uint16_t *lengthp) 755 { 756 _NOTE(ARGUNUSED(enp)) 757 758 /* 759 * The RX pseudo-header contains the packet length, excluding the 760 * pseudo-header. If the hardware receive datapath was operating in 761 * cut-through mode then the length in the RX pseudo-header will be 762 * zero, and the packet length must be obtained from the DMA length 763 * reported in the RX event. 764 */ 765 *lengthp = buffer[8] | (buffer[9] << 8); 766 return (0); 767 } 768 769 #if EFSYS_OPT_RX_SCALE 770 __checkReturn uint32_t 771 ef10_rx_prefix_hash( 772 __in efx_nic_t *enp, 773 __in efx_rx_hash_alg_t func, 774 __in uint8_t *buffer) 775 { 776 _NOTE(ARGUNUSED(enp)) 777 778 switch (func) { 779 case EFX_RX_HASHALG_PACKED_STREAM: 780 case EFX_RX_HASHALG_TOEPLITZ: 781 return (buffer[0] | 782 (buffer[1] << 8) | 783 (buffer[2] << 16) | 784 (buffer[3] << 24)); 785 786 default: 787 EFSYS_ASSERT(0); 788 return (0); 789 } 790 } 791 #endif /* EFSYS_OPT_RX_SCALE */ 792 793 #if EFSYS_OPT_RX_PACKED_STREAM 794 /* 795 * Fake length for RXQ descriptors in packed stream mode 796 * to make hardware happy 797 */ 798 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 799 #endif 800 801 void 802 ef10_rx_qpost( 803 __in efx_rxq_t *erp, 804 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 805 __in size_t size, 806 __in unsigned int ndescs, 807 __in unsigned int completed, 808 __in unsigned int added) 809 { 810 efx_qword_t qword; 811 unsigned int i; 812 unsigned int offset; 813 unsigned int id; 814 815 _NOTE(ARGUNUSED(completed)) 816 817 #if EFSYS_OPT_RX_PACKED_STREAM 818 /* 819 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT 820 * and equal to 0 after applying mask. Hardware does not like it. 821 */ 822 if (erp->er_ev_qstate->eers_rx_packed_stream) 823 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; 824 #endif 825 826 /* The client driver must not overfill the queue */ 827 EFSYS_ASSERT3U(added - completed + ndescs, <=, 828 EFX_RXQ_LIMIT(erp->er_mask + 1)); 829 830 id = added & (erp->er_mask); 831 for (i = 0; i < ndescs; i++) { 832 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 833 unsigned int, id, efsys_dma_addr_t, addrp[i], 834 size_t, size); 835 836 EFX_POPULATE_QWORD_3(qword, 837 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 838 ESF_DZ_RX_KER_BUF_ADDR_DW0, 839 (uint32_t)(addrp[i] & 0xffffffff), 840 ESF_DZ_RX_KER_BUF_ADDR_DW1, 841 (uint32_t)(addrp[i] >> 32)); 842 843 offset = id * sizeof (efx_qword_t); 844 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 845 846 id = (id + 1) & (erp->er_mask); 847 } 848 } 849 850 void 851 ef10_rx_qpush( 852 __in efx_rxq_t *erp, 853 __in unsigned int added, 854 __inout unsigned int *pushedp) 855 { 856 efx_nic_t *enp = erp->er_enp; 857 unsigned int pushed = *pushedp; 858 uint32_t wptr; 859 efx_dword_t dword; 860 861 /* Hardware has alignment restriction for WPTR */ 862 wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN); 863 if (pushed == wptr) 864 return; 865 866 *pushedp = wptr; 867 868 /* Push the populated descriptors out */ 869 wptr &= erp->er_mask; 870 871 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 872 873 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 874 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 875 wptr, pushed & erp->er_mask); 876 EFSYS_PIO_WRITE_BARRIER(); 877 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 878 erp->er_index, &dword, B_FALSE); 879 } 880 881 #if EFSYS_OPT_RX_PACKED_STREAM 882 883 void 884 ef10_rx_qpush_ps_credits( 885 __in efx_rxq_t *erp) 886 { 887 efx_nic_t *enp = erp->er_enp; 888 efx_dword_t dword; 889 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 890 uint32_t credits; 891 892 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 893 894 if (rxq_state->eers_rx_packed_stream_credits == 0) 895 return; 896 897 /* 898 * It is a bug if we think that FW has utilized more 899 * credits than it is allowed to have (maximum). However, 900 * make sure that we do not credit more than maximum anyway. 901 */ 902 credits = MIN(rxq_state->eers_rx_packed_stream_credits, 903 EFX_RX_PACKED_STREAM_MAX_CREDITS); 904 EFX_POPULATE_DWORD_3(dword, 905 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, 906 ERF_DZ_RX_DESC_MAGIC_CMD, 907 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, 908 ERF_DZ_RX_DESC_MAGIC_DATA, credits); 909 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 910 erp->er_index, &dword, B_FALSE); 911 912 rxq_state->eers_rx_packed_stream_credits = 0; 913 } 914 915 /* 916 * In accordance with SF-112241-TC the received data has the following layout: 917 * - 8 byte pseudo-header which consist of: 918 * - 4 byte little-endian timestamp 919 * - 2 byte little-endian captured length in bytes 920 * - 2 byte little-endian original packet length in bytes 921 * - captured packet bytes 922 * - optional padding to align to 64 bytes boundary 923 * - 64 bytes scratch space for the host software 924 */ 925 __checkReturn uint8_t * 926 ef10_rx_qps_packet_info( 927 __in efx_rxq_t *erp, 928 __in uint8_t *buffer, 929 __in uint32_t buffer_length, 930 __in uint32_t current_offset, 931 __out uint16_t *lengthp, 932 __out uint32_t *next_offsetp, 933 __out uint32_t *timestamp) 934 { 935 uint16_t buf_len; 936 uint8_t *pkt_start; 937 efx_qword_t *qwordp; 938 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 939 940 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 941 942 buffer += current_offset; 943 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; 944 945 qwordp = (efx_qword_t *)buffer; 946 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); 947 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); 948 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); 949 950 buf_len = EFX_P2ROUNDUP(uint16_t, 951 buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, 952 EFX_RX_PACKED_STREAM_ALIGNMENT); 953 *next_offsetp = 954 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; 955 956 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); 957 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); 958 959 if ((*next_offsetp ^ current_offset) & 960 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) 961 rxq_state->eers_rx_packed_stream_credits++; 962 963 return (pkt_start); 964 } 965 966 967 #endif 968 969 __checkReturn efx_rc_t 970 ef10_rx_qflush( 971 __in efx_rxq_t *erp) 972 { 973 efx_nic_t *enp = erp->er_enp; 974 efx_rc_t rc; 975 976 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 977 goto fail1; 978 979 return (0); 980 981 fail1: 982 /* 983 * EALREADY is not an error, but indicates that the MC has rebooted and 984 * that the RXQ has already been destroyed. Callers need to know that 985 * the RXQ flush has completed to avoid waiting until timeout for a 986 * flush done event that will not be delivered. 987 */ 988 if (rc != EALREADY) 989 EFSYS_PROBE1(fail1, efx_rc_t, rc); 990 991 return (rc); 992 } 993 994 void 995 ef10_rx_qenable( 996 __in efx_rxq_t *erp) 997 { 998 /* FIXME */ 999 _NOTE(ARGUNUSED(erp)) 1000 /* FIXME */ 1001 } 1002 1003 __checkReturn efx_rc_t 1004 ef10_rx_qcreate( 1005 __in efx_nic_t *enp, 1006 __in unsigned int index, 1007 __in unsigned int label, 1008 __in efx_rxq_type_t type, 1009 __in_opt const efx_rxq_type_data_t *type_data, 1010 __in efsys_mem_t *esmp, 1011 __in size_t ndescs, 1012 __in uint32_t id, 1013 __in unsigned int flags, 1014 __in efx_evq_t *eep, 1015 __in efx_rxq_t *erp) 1016 { 1017 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1018 efx_rc_t rc; 1019 boolean_t disable_scatter; 1020 boolean_t want_inner_classes; 1021 unsigned int ps_buf_size; 1022 uint32_t es_bufs_per_desc = 0; 1023 uint32_t es_max_dma_len = 0; 1024 uint32_t es_buf_stride = 0; 1025 uint32_t hol_block_timeout = 0; 1026 1027 _NOTE(ARGUNUSED(id, erp)) 1028 1029 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 1030 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1031 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 1032 1033 if (index >= encp->enc_rxq_limit) { 1034 rc = EINVAL; 1035 goto fail1; 1036 } 1037 1038 switch (type) { 1039 case EFX_RXQ_TYPE_DEFAULT: 1040 if (type_data == NULL) { 1041 rc = EINVAL; 1042 goto fail2; 1043 } 1044 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 1045 ps_buf_size = 0; 1046 break; 1047 #if EFSYS_OPT_RX_PACKED_STREAM 1048 case EFX_RXQ_TYPE_PACKED_STREAM: 1049 if (type_data == NULL) { 1050 rc = EINVAL; 1051 goto fail3; 1052 } 1053 switch (type_data->ertd_packed_stream.eps_buf_size) { 1054 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M: 1055 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; 1056 break; 1057 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K: 1058 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; 1059 break; 1060 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K: 1061 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; 1062 break; 1063 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K: 1064 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; 1065 break; 1066 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K: 1067 ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; 1068 break; 1069 default: 1070 rc = ENOTSUP; 1071 goto fail4; 1072 } 1073 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size; 1074 break; 1075 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1076 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1077 case EFX_RXQ_TYPE_ES_SUPER_BUFFER: 1078 if (type_data == NULL) { 1079 rc = EINVAL; 1080 goto fail5; 1081 } 1082 ps_buf_size = 0; 1083 es_bufs_per_desc = 1084 type_data->ertd_es_super_buffer.eessb_bufs_per_desc; 1085 es_max_dma_len = 1086 type_data->ertd_es_super_buffer.eessb_max_dma_len; 1087 es_buf_stride = 1088 type_data->ertd_es_super_buffer.eessb_buf_stride; 1089 hol_block_timeout = 1090 type_data->ertd_es_super_buffer.eessb_hol_block_timeout; 1091 break; 1092 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1093 default: 1094 rc = ENOTSUP; 1095 goto fail6; 1096 } 1097 1098 #if EFSYS_OPT_RX_PACKED_STREAM 1099 if (ps_buf_size != 0) { 1100 /* Check if datapath firmware supports packed stream mode */ 1101 if (encp->enc_rx_packed_stream_supported == B_FALSE) { 1102 rc = ENOTSUP; 1103 goto fail7; 1104 } 1105 /* Check if packed stream allows configurable buffer sizes */ 1106 if ((ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) && 1107 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { 1108 rc = ENOTSUP; 1109 goto fail8; 1110 } 1111 } 1112 #else /* EFSYS_OPT_RX_PACKED_STREAM */ 1113 EFSYS_ASSERT(ps_buf_size == 0); 1114 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1115 1116 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1117 if (es_bufs_per_desc > 0) { 1118 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) { 1119 rc = ENOTSUP; 1120 goto fail9; 1121 } 1122 if (!EFX_IS_P2ALIGNED(uint32_t, es_max_dma_len, 1123 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1124 rc = EINVAL; 1125 goto fail10; 1126 } 1127 if (!EFX_IS_P2ALIGNED(uint32_t, es_buf_stride, 1128 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1129 rc = EINVAL; 1130 goto fail11; 1131 } 1132 } 1133 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1134 EFSYS_ASSERT(es_bufs_per_desc == 0); 1135 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1136 1137 /* Scatter can only be disabled if the firmware supports doing so */ 1138 if (flags & EFX_RXQ_FLAG_SCATTER) 1139 disable_scatter = B_FALSE; 1140 else 1141 disable_scatter = encp->enc_rx_disable_scatter_supported; 1142 1143 if (flags & EFX_RXQ_FLAG_INNER_CLASSES) 1144 want_inner_classes = B_TRUE; 1145 else 1146 want_inner_classes = B_FALSE; 1147 1148 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index, 1149 esmp, disable_scatter, want_inner_classes, erp->er_buf_size, 1150 ps_buf_size, es_bufs_per_desc, es_max_dma_len, 1151 es_buf_stride, hol_block_timeout)) != 0) 1152 goto fail12; 1153 1154 erp->er_eep = eep; 1155 erp->er_label = label; 1156 1157 ef10_ev_rxlabel_init(eep, erp, label, type); 1158 1159 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; 1160 1161 return (0); 1162 1163 fail12: 1164 EFSYS_PROBE(fail12); 1165 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1166 fail11: 1167 EFSYS_PROBE(fail11); 1168 fail10: 1169 EFSYS_PROBE(fail10); 1170 fail9: 1171 EFSYS_PROBE(fail9); 1172 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1173 #if EFSYS_OPT_RX_PACKED_STREAM 1174 fail8: 1175 EFSYS_PROBE(fail8); 1176 fail7: 1177 EFSYS_PROBE(fail7); 1178 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1179 fail6: 1180 EFSYS_PROBE(fail6); 1181 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1182 fail5: 1183 EFSYS_PROBE(fail5); 1184 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1185 #if EFSYS_OPT_RX_PACKED_STREAM 1186 fail4: 1187 EFSYS_PROBE(fail4); 1188 fail3: 1189 EFSYS_PROBE(fail3); 1190 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1191 fail2: 1192 EFSYS_PROBE(fail2); 1193 fail1: 1194 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1195 1196 return (rc); 1197 } 1198 1199 void 1200 ef10_rx_qdestroy( 1201 __in efx_rxq_t *erp) 1202 { 1203 efx_nic_t *enp = erp->er_enp; 1204 efx_evq_t *eep = erp->er_eep; 1205 unsigned int label = erp->er_label; 1206 1207 ef10_ev_rxlabel_fini(eep, label); 1208 1209 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1210 --enp->en_rx_qcount; 1211 1212 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1213 } 1214 1215 void 1216 ef10_rx_fini( 1217 __in efx_nic_t *enp) 1218 { 1219 #if EFSYS_OPT_RX_SCALE 1220 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) 1221 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 1222 enp->en_rss_context = 0; 1223 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 1224 #else 1225 _NOTE(ARGUNUSED(enp)) 1226 #endif /* EFSYS_OPT_RX_SCALE */ 1227 } 1228 1229 #endif /* EFX_OPTS_EF10() */ 1230