1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2017 6WIND S.A. 3 * Copyright 2017 Mellanox Technologies, Ltd 4 */ 5 6 /** 7 * @file 8 * Rx queues configuration for mlx4 driver. 9 */ 10 11 #include <assert.h> 12 #include <errno.h> 13 #include <stddef.h> 14 #include <stdint.h> 15 #include <string.h> 16 17 /* Verbs headers do not support -pedantic. */ 18 #ifdef PEDANTIC 19 #pragma GCC diagnostic ignored "-Wpedantic" 20 #endif 21 #include <infiniband/mlx4dv.h> 22 #include <infiniband/verbs.h> 23 #ifdef PEDANTIC 24 #pragma GCC diagnostic error "-Wpedantic" 25 #endif 26 27 #include <rte_byteorder.h> 28 #include <rte_common.h> 29 #include <rte_errno.h> 30 #include <rte_ethdev_driver.h> 31 #include <rte_flow.h> 32 #include <rte_malloc.h> 33 #include <rte_mbuf.h> 34 #include <rte_mempool.h> 35 36 #include "mlx4.h" 37 #include "mlx4_glue.h" 38 #include "mlx4_flow.h" 39 #include "mlx4_rxtx.h" 40 #include "mlx4_utils.h" 41 42 /** 43 * Historical RSS hash key. 44 * 45 * This used to be the default for mlx4 in Linux before v3.19 switched to 46 * generating random hash keys through netdev_rss_key_fill(). 47 * 48 * It is used in this PMD for consistency with past DPDK releases but can 49 * now be overridden through user configuration. 50 * 51 * Note: this is not const to work around API quirks. 52 */ 53 uint8_t 54 mlx4_rss_hash_key_default[MLX4_RSS_HASH_KEY_SIZE] = { 55 0x2c, 0xc6, 0x81, 0xd1, 56 0x5b, 0xdb, 0xf4, 0xf7, 57 0xfc, 0xa2, 0x83, 0x19, 58 0xdb, 0x1a, 0x3e, 0x94, 59 0x6b, 0x9e, 0x38, 0xd9, 60 0x2c, 0x9c, 0x03, 0xd1, 61 0xad, 0x99, 0x44, 0xa7, 62 0xd9, 0x56, 0x3d, 0x59, 63 0x06, 0x3c, 0x25, 0xf3, 64 0xfc, 0x1f, 0xdc, 0x2a, 65 }; 66 67 /** 68 * Obtain a RSS context with specified properties. 69 * 70 * Used when creating a flow rule targeting one or several Rx queues. 71 * 72 * If a matching RSS context already exists, it is returned with its 73 * reference count incremented. 74 * 75 * @param priv 76 * Pointer to private structure. 77 * @param fields 78 * Fields for RSS processing (Verbs format). 79 * @param[in] key 80 * Hash key to use (whose size is exactly MLX4_RSS_HASH_KEY_SIZE). 81 * @param queues 82 * Number of target queues. 83 * @param[in] queue_id 84 * Target queues. 85 * 86 * @return 87 * Pointer to RSS context on success, NULL otherwise and rte_errno is set. 88 */ 89 struct mlx4_rss * 90 mlx4_rss_get(struct priv *priv, uint64_t fields, 91 const uint8_t key[MLX4_RSS_HASH_KEY_SIZE], 92 uint16_t queues, const uint16_t queue_id[]) 93 { 94 struct mlx4_rss *rss; 95 size_t queue_id_size = sizeof(queue_id[0]) * queues; 96 97 LIST_FOREACH(rss, &priv->rss, next) 98 if (fields == rss->fields && 99 queues == rss->queues && 100 !memcmp(key, rss->key, MLX4_RSS_HASH_KEY_SIZE) && 101 !memcmp(queue_id, rss->queue_id, queue_id_size)) { 102 ++rss->refcnt; 103 return rss; 104 } 105 rss = rte_malloc(__func__, offsetof(struct mlx4_rss, queue_id) + 106 queue_id_size, 0); 107 if (!rss) 108 goto error; 109 *rss = (struct mlx4_rss){ 110 .priv = priv, 111 .refcnt = 1, 112 .usecnt = 0, 113 .qp = NULL, 114 .ind = NULL, 115 .fields = fields, 116 .queues = queues, 117 }; 118 memcpy(rss->key, key, MLX4_RSS_HASH_KEY_SIZE); 119 memcpy(rss->queue_id, queue_id, queue_id_size); 120 LIST_INSERT_HEAD(&priv->rss, rss, next); 121 return rss; 122 error: 123 rte_errno = ENOMEM; 124 return NULL; 125 } 126 127 /** 128 * Release a RSS context instance. 129 * 130 * Used when destroying a flow rule targeting one or several Rx queues. 131 * 132 * This function decrements the reference count of the context and destroys 133 * it after reaching 0. The context must have no users at this point; all 134 * prior calls to mlx4_rss_attach() must have been followed by matching 135 * calls to mlx4_rss_detach(). 136 * 137 * @param rss 138 * RSS context to release. 139 */ 140 void 141 mlx4_rss_put(struct mlx4_rss *rss) 142 { 143 assert(rss->refcnt); 144 if (--rss->refcnt) 145 return; 146 assert(!rss->usecnt); 147 assert(!rss->qp); 148 assert(!rss->ind); 149 LIST_REMOVE(rss, next); 150 rte_free(rss); 151 } 152 153 /** 154 * Attach a user to a RSS context instance. 155 * 156 * Used when the RSS QP and indirection table objects must be instantiated, 157 * that is, when a flow rule must be enabled. 158 * 159 * This function increments the usage count of the context. 160 * 161 * @param rss 162 * RSS context to attach to. 163 * 164 * @return 165 * 0 on success, a negative errno value otherwise and rte_errno is set. 166 */ 167 int 168 mlx4_rss_attach(struct mlx4_rss *rss) 169 { 170 assert(rss->refcnt); 171 if (rss->usecnt++) { 172 assert(rss->qp); 173 assert(rss->ind); 174 return 0; 175 } 176 177 struct ibv_wq *ind_tbl[rss->queues]; 178 struct priv *priv = rss->priv; 179 const char *msg; 180 unsigned int i = 0; 181 int ret; 182 183 if (!rte_is_power_of_2(RTE_DIM(ind_tbl))) { 184 ret = EINVAL; 185 msg = "number of RSS queues must be a power of two"; 186 goto error; 187 } 188 for (i = 0; i != RTE_DIM(ind_tbl); ++i) { 189 uint16_t id = rss->queue_id[i]; 190 struct rxq *rxq = NULL; 191 192 if (id < priv->dev->data->nb_rx_queues) 193 rxq = priv->dev->data->rx_queues[id]; 194 if (!rxq) { 195 ret = EINVAL; 196 msg = "RSS target queue is not configured"; 197 goto error; 198 } 199 ret = mlx4_rxq_attach(rxq); 200 if (ret) { 201 ret = -ret; 202 msg = "unable to attach RSS target queue"; 203 goto error; 204 } 205 ind_tbl[i] = rxq->wq; 206 } 207 rss->ind = mlx4_glue->create_rwq_ind_table 208 (priv->ctx, 209 &(struct ibv_rwq_ind_table_init_attr){ 210 .log_ind_tbl_size = rte_log2_u32(RTE_DIM(ind_tbl)), 211 .ind_tbl = ind_tbl, 212 .comp_mask = 0, 213 }); 214 if (!rss->ind) { 215 ret = errno ? errno : EINVAL; 216 msg = "RSS indirection table creation failure"; 217 goto error; 218 } 219 rss->qp = mlx4_glue->create_qp_ex 220 (priv->ctx, 221 &(struct ibv_qp_init_attr_ex){ 222 .comp_mask = (IBV_QP_INIT_ATTR_PD | 223 IBV_QP_INIT_ATTR_RX_HASH | 224 IBV_QP_INIT_ATTR_IND_TABLE), 225 .qp_type = IBV_QPT_RAW_PACKET, 226 .pd = priv->pd, 227 .rwq_ind_tbl = rss->ind, 228 .rx_hash_conf = { 229 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, 230 .rx_hash_key_len = MLX4_RSS_HASH_KEY_SIZE, 231 .rx_hash_key = rss->key, 232 .rx_hash_fields_mask = rss->fields, 233 }, 234 }); 235 if (!rss->qp) { 236 ret = errno ? errno : EINVAL; 237 msg = "RSS hash QP creation failure"; 238 goto error; 239 } 240 ret = mlx4_glue->modify_qp 241 (rss->qp, 242 &(struct ibv_qp_attr){ 243 .qp_state = IBV_QPS_INIT, 244 .port_num = priv->port, 245 }, 246 IBV_QP_STATE | IBV_QP_PORT); 247 if (ret) { 248 msg = "failed to switch RSS hash QP to INIT state"; 249 goto error; 250 } 251 ret = mlx4_glue->modify_qp 252 (rss->qp, 253 &(struct ibv_qp_attr){ 254 .qp_state = IBV_QPS_RTR, 255 }, 256 IBV_QP_STATE); 257 if (ret) { 258 msg = "failed to switch RSS hash QP to RTR state"; 259 goto error; 260 } 261 return 0; 262 error: 263 if (rss->qp) { 264 claim_zero(mlx4_glue->destroy_qp(rss->qp)); 265 rss->qp = NULL; 266 } 267 if (rss->ind) { 268 claim_zero(mlx4_glue->destroy_rwq_ind_table(rss->ind)); 269 rss->ind = NULL; 270 } 271 while (i--) 272 mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); 273 ERROR("mlx4: %s", msg); 274 --rss->usecnt; 275 rte_errno = ret; 276 return -ret; 277 } 278 279 /** 280 * Detach a user from a RSS context instance. 281 * 282 * Used when disabling (not destroying) a flow rule. 283 * 284 * This function decrements the usage count of the context and destroys 285 * usage resources after reaching 0. 286 * 287 * @param rss 288 * RSS context to detach from. 289 */ 290 void 291 mlx4_rss_detach(struct mlx4_rss *rss) 292 { 293 struct priv *priv = rss->priv; 294 unsigned int i; 295 296 assert(rss->refcnt); 297 assert(rss->qp); 298 assert(rss->ind); 299 if (--rss->usecnt) 300 return; 301 claim_zero(mlx4_glue->destroy_qp(rss->qp)); 302 rss->qp = NULL; 303 claim_zero(mlx4_glue->destroy_rwq_ind_table(rss->ind)); 304 rss->ind = NULL; 305 for (i = 0; i != rss->queues; ++i) 306 mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); 307 } 308 309 /** 310 * Initialize common RSS context resources. 311 * 312 * Because ConnectX-3 hardware limitations require a fixed order in the 313 * indirection table, WQs must be allocated sequentially to be part of a 314 * common RSS context. 315 * 316 * Since a newly created WQ cannot be moved to a different context, this 317 * function allocates them all at once, one for each configured Rx queue, 318 * as well as all related resources (CQs and mbufs). 319 * 320 * This must therefore be done before creating any Rx flow rules relying on 321 * indirection tables. 322 * 323 * @param priv 324 * Pointer to private structure. 325 * 326 * @return 327 * 0 on success, a negative errno value otherwise and rte_errno is set. 328 */ 329 int 330 mlx4_rss_init(struct priv *priv) 331 { 332 struct rte_eth_dev *dev = priv->dev; 333 uint8_t log2_range = rte_log2_u32(dev->data->nb_rx_queues); 334 uint32_t wq_num_prev = 0; 335 const char *msg; 336 unsigned int i; 337 int ret; 338 339 if (priv->rss_init) 340 return 0; 341 if (priv->dev->data->nb_rx_queues > priv->hw_rss_max_qps) { 342 ERROR("RSS does not support more than %d queues", 343 priv->hw_rss_max_qps); 344 rte_errno = EINVAL; 345 return -rte_errno; 346 } 347 /* Prepare range for RSS contexts before creating the first WQ. */ 348 ret = mlx4_glue->dv_set_context_attr 349 (priv->ctx, 350 MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, 351 &log2_range); 352 if (ret) { 353 ERROR("cannot set up range size for RSS context to %u" 354 " (for %u Rx queues), error: %s", 355 1 << log2_range, dev->data->nb_rx_queues, strerror(ret)); 356 rte_errno = ret; 357 return -ret; 358 } 359 for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { 360 struct rxq *rxq = priv->dev->data->rx_queues[i]; 361 struct ibv_cq *cq; 362 struct ibv_wq *wq; 363 uint32_t wq_num; 364 365 /* Attach the configured Rx queues. */ 366 if (rxq) { 367 assert(!rxq->usecnt); 368 ret = mlx4_rxq_attach(rxq); 369 if (!ret) { 370 wq_num = rxq->wq->wq_num; 371 goto wq_num_check; 372 } 373 ret = -ret; 374 msg = "unable to create Rx queue resources"; 375 goto error; 376 } 377 /* 378 * WQs are temporarily allocated for unconfigured Rx queues 379 * to maintain proper index alignment in indirection table 380 * by skipping unused WQ numbers. 381 * 382 * The reason this works at all even though these WQs are 383 * immediately destroyed is that WQNs are allocated 384 * sequentially and are guaranteed to never be reused in the 385 * same context by the underlying implementation. 386 */ 387 cq = mlx4_glue->create_cq(priv->ctx, 1, NULL, NULL, 0); 388 if (!cq) { 389 ret = ENOMEM; 390 msg = "placeholder CQ creation failure"; 391 goto error; 392 } 393 wq = mlx4_glue->create_wq 394 (priv->ctx, 395 &(struct ibv_wq_init_attr){ 396 .wq_type = IBV_WQT_RQ, 397 .max_wr = 1, 398 .max_sge = 1, 399 .pd = priv->pd, 400 .cq = cq, 401 }); 402 if (wq) { 403 wq_num = wq->wq_num; 404 claim_zero(mlx4_glue->destroy_wq(wq)); 405 } else { 406 wq_num = 0; /* Shut up GCC 4.8 warnings. */ 407 } 408 claim_zero(mlx4_glue->destroy_cq(cq)); 409 if (!wq) { 410 ret = ENOMEM; 411 msg = "placeholder WQ creation failure"; 412 goto error; 413 } 414 wq_num_check: 415 /* 416 * While guaranteed by the implementation, make sure WQ 417 * numbers are really sequential (as the saying goes, 418 * trust, but verify). 419 */ 420 if (i && wq_num - wq_num_prev != 1) { 421 if (rxq) 422 mlx4_rxq_detach(rxq); 423 ret = ERANGE; 424 msg = "WQ numbers are not sequential"; 425 goto error; 426 } 427 wq_num_prev = wq_num; 428 } 429 priv->rss_init = 1; 430 return 0; 431 error: 432 ERROR("cannot initialize common RSS resources (queue %u): %s: %s", 433 i, msg, strerror(ret)); 434 while (i--) { 435 struct rxq *rxq = priv->dev->data->rx_queues[i]; 436 437 if (rxq) 438 mlx4_rxq_detach(rxq); 439 } 440 rte_errno = ret; 441 return -ret; 442 } 443 444 /** 445 * Release common RSS context resources. 446 * 447 * As the reverse of mlx4_rss_init(), this must be done after removing all 448 * flow rules relying on indirection tables. 449 * 450 * @param priv 451 * Pointer to private structure. 452 */ 453 void 454 mlx4_rss_deinit(struct priv *priv) 455 { 456 unsigned int i; 457 458 if (!priv->rss_init) 459 return; 460 for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { 461 struct rxq *rxq = priv->dev->data->rx_queues[i]; 462 463 if (rxq) { 464 assert(rxq->usecnt == 1); 465 mlx4_rxq_detach(rxq); 466 } 467 } 468 priv->rss_init = 0; 469 } 470 471 /** 472 * Attach a user to a Rx queue. 473 * 474 * Used when the resources of an Rx queue must be instantiated for it to 475 * become in a usable state. 476 * 477 * This function increments the usage count of the Rx queue. 478 * 479 * @param rxq 480 * Pointer to Rx queue structure. 481 * 482 * @return 483 * 0 on success, negative errno value otherwise and rte_errno is set. 484 */ 485 int 486 mlx4_rxq_attach(struct rxq *rxq) 487 { 488 if (rxq->usecnt++) { 489 assert(rxq->cq); 490 assert(rxq->wq); 491 assert(rxq->wqes); 492 assert(rxq->rq_db); 493 return 0; 494 } 495 496 struct priv *priv = rxq->priv; 497 struct rte_eth_dev *dev = priv->dev; 498 const uint32_t elts_n = 1 << rxq->elts_n; 499 const uint32_t sges_n = 1 << rxq->sges_n; 500 struct rte_mbuf *(*elts)[elts_n] = rxq->elts; 501 struct mlx4dv_obj mlxdv; 502 struct mlx4dv_rwq dv_rwq; 503 struct mlx4dv_cq dv_cq = { .comp_mask = MLX4DV_CQ_MASK_UAR, }; 504 const char *msg; 505 struct ibv_cq *cq = NULL; 506 struct ibv_wq *wq = NULL; 507 uint32_t create_flags = 0; 508 uint32_t comp_mask = 0; 509 volatile struct mlx4_wqe_data_seg (*wqes)[]; 510 unsigned int i; 511 int ret; 512 513 assert(rte_is_power_of_2(elts_n)); 514 cq = mlx4_glue->create_cq(priv->ctx, elts_n / sges_n, NULL, 515 rxq->channel, 0); 516 if (!cq) { 517 ret = ENOMEM; 518 msg = "CQ creation failure"; 519 goto error; 520 } 521 /* By default, FCS (CRC) is stripped by hardware. */ 522 if (rxq->crc_present) { 523 create_flags |= IBV_WQ_FLAGS_SCATTER_FCS; 524 comp_mask |= IBV_WQ_INIT_ATTR_FLAGS; 525 } 526 wq = mlx4_glue->create_wq 527 (priv->ctx, 528 &(struct ibv_wq_init_attr){ 529 .wq_type = IBV_WQT_RQ, 530 .max_wr = elts_n / sges_n, 531 .max_sge = sges_n, 532 .pd = priv->pd, 533 .cq = cq, 534 .comp_mask = comp_mask, 535 .create_flags = create_flags, 536 }); 537 if (!wq) { 538 ret = errno ? errno : EINVAL; 539 msg = "WQ creation failure"; 540 goto error; 541 } 542 ret = mlx4_glue->modify_wq 543 (wq, 544 &(struct ibv_wq_attr){ 545 .attr_mask = IBV_WQ_ATTR_STATE, 546 .wq_state = IBV_WQS_RDY, 547 }); 548 if (ret) { 549 msg = "WQ state change to IBV_WQS_RDY failed"; 550 goto error; 551 } 552 /* Retrieve device queue information. */ 553 mlxdv.cq.in = cq; 554 mlxdv.cq.out = &dv_cq; 555 mlxdv.rwq.in = wq; 556 mlxdv.rwq.out = &dv_rwq; 557 ret = mlx4_glue->dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ); 558 if (ret) { 559 msg = "failed to obtain device information from WQ/CQ objects"; 560 goto error; 561 } 562 /* Pre-register Rx mempool. */ 563 DEBUG("port %u Rx queue %u registering mp %s having %u chunks", 564 priv->dev->data->port_id, rxq->stats.idx, 565 rxq->mp->name, rxq->mp->nb_mem_chunks); 566 mlx4_mr_update_mp(dev, &rxq->mr_ctrl, rxq->mp); 567 wqes = (volatile struct mlx4_wqe_data_seg (*)[]) 568 ((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset); 569 for (i = 0; i != RTE_DIM(*elts); ++i) { 570 volatile struct mlx4_wqe_data_seg *scat = &(*wqes)[i]; 571 struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp); 572 573 if (buf == NULL) { 574 while (i--) { 575 rte_pktmbuf_free_seg((*elts)[i]); 576 (*elts)[i] = NULL; 577 } 578 ret = ENOMEM; 579 msg = "cannot allocate mbuf"; 580 goto error; 581 } 582 /* Headroom is reserved by rte_pktmbuf_alloc(). */ 583 assert(buf->data_off == RTE_PKTMBUF_HEADROOM); 584 /* Buffer is supposed to be empty. */ 585 assert(rte_pktmbuf_data_len(buf) == 0); 586 assert(rte_pktmbuf_pkt_len(buf) == 0); 587 /* Only the first segment keeps headroom. */ 588 if (i % sges_n) 589 buf->data_off = 0; 590 buf->port = rxq->port_id; 591 buf->data_len = rte_pktmbuf_tailroom(buf); 592 buf->pkt_len = rte_pktmbuf_tailroom(buf); 593 buf->nb_segs = 1; 594 *scat = (struct mlx4_wqe_data_seg){ 595 .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, 596 uintptr_t)), 597 .byte_count = rte_cpu_to_be_32(buf->data_len), 598 .lkey = mlx4_rx_mb2mr(rxq, buf), 599 }; 600 (*elts)[i] = buf; 601 } 602 DEBUG("%p: allocated and configured %u segments (max %u packets)", 603 (void *)rxq, elts_n, elts_n / sges_n); 604 rxq->cq = cq; 605 rxq->wq = wq; 606 rxq->wqes = wqes; 607 rxq->rq_db = dv_rwq.rdb; 608 rxq->mcq.buf = dv_cq.buf.buf; 609 rxq->mcq.cqe_cnt = dv_cq.cqe_cnt; 610 rxq->mcq.set_ci_db = dv_cq.set_ci_db; 611 rxq->mcq.cqe_64 = (dv_cq.cqe_size & 64) ? 1 : 0; 612 rxq->mcq.arm_db = dv_cq.arm_db; 613 rxq->mcq.arm_sn = dv_cq.arm_sn; 614 rxq->mcq.cqn = dv_cq.cqn; 615 rxq->mcq.cq_uar = dv_cq.cq_uar; 616 rxq->mcq.cq_db_reg = (uint8_t *)dv_cq.cq_uar + MLX4_CQ_DOORBELL; 617 /* Update doorbell counter. */ 618 rxq->rq_ci = elts_n / sges_n; 619 rte_wmb(); 620 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 621 return 0; 622 error: 623 if (wq) 624 claim_zero(mlx4_glue->destroy_wq(wq)); 625 if (cq) 626 claim_zero(mlx4_glue->destroy_cq(cq)); 627 --rxq->usecnt; 628 rte_errno = ret; 629 ERROR("error while attaching Rx queue %p: %s: %s", 630 (void *)rxq, msg, strerror(ret)); 631 return -ret; 632 } 633 634 /** 635 * Detach a user from a Rx queue. 636 * 637 * This function decrements the usage count of the Rx queue and destroys 638 * usage resources after reaching 0. 639 * 640 * @param rxq 641 * Pointer to Rx queue structure. 642 */ 643 void 644 mlx4_rxq_detach(struct rxq *rxq) 645 { 646 unsigned int i; 647 struct rte_mbuf *(*elts)[1 << rxq->elts_n] = rxq->elts; 648 649 if (--rxq->usecnt) 650 return; 651 rxq->rq_ci = 0; 652 memset(&rxq->mcq, 0, sizeof(rxq->mcq)); 653 rxq->rq_db = NULL; 654 rxq->wqes = NULL; 655 claim_zero(mlx4_glue->destroy_wq(rxq->wq)); 656 rxq->wq = NULL; 657 claim_zero(mlx4_glue->destroy_cq(rxq->cq)); 658 rxq->cq = NULL; 659 DEBUG("%p: freeing Rx queue elements", (void *)rxq); 660 for (i = 0; (i != RTE_DIM(*elts)); ++i) { 661 if (!(*elts)[i]) 662 continue; 663 rte_pktmbuf_free_seg((*elts)[i]); 664 (*elts)[i] = NULL; 665 } 666 } 667 668 /** 669 * Returns the per-queue supported offloads. 670 * 671 * @param priv 672 * Pointer to private structure. 673 * 674 * @return 675 * Supported Tx offloads. 676 */ 677 uint64_t 678 mlx4_get_rx_queue_offloads(struct priv *priv) 679 { 680 uint64_t offloads = DEV_RX_OFFLOAD_SCATTER | 681 DEV_RX_OFFLOAD_KEEP_CRC | 682 DEV_RX_OFFLOAD_JUMBO_FRAME; 683 684 if (priv->hw_csum) 685 offloads |= DEV_RX_OFFLOAD_CHECKSUM; 686 return offloads; 687 } 688 689 /** 690 * Returns the per-port supported offloads. 691 * 692 * @param priv 693 * Pointer to private structure. 694 * 695 * @return 696 * Supported Rx offloads. 697 */ 698 uint64_t 699 mlx4_get_rx_port_offloads(struct priv *priv) 700 { 701 uint64_t offloads = DEV_RX_OFFLOAD_VLAN_FILTER; 702 703 (void)priv; 704 return offloads; 705 } 706 707 /** 708 * DPDK callback to configure a Rx queue. 709 * 710 * @param dev 711 * Pointer to Ethernet device structure. 712 * @param idx 713 * Rx queue index. 714 * @param desc 715 * Number of descriptors to configure in queue. 716 * @param socket 717 * NUMA socket on which memory must be allocated. 718 * @param[in] conf 719 * Thresholds parameters. 720 * @param mp 721 * Memory pool for buffer allocations. 722 * 723 * @return 724 * 0 on success, negative errno value otherwise and rte_errno is set. 725 */ 726 int 727 mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 728 unsigned int socket, const struct rte_eth_rxconf *conf, 729 struct rte_mempool *mp) 730 { 731 struct priv *priv = dev->data->dev_private; 732 uint32_t mb_len = rte_pktmbuf_data_room_size(mp); 733 struct rte_mbuf *(*elts)[rte_align32pow2(desc)]; 734 struct rxq *rxq; 735 struct mlx4_malloc_vec vec[] = { 736 { 737 .align = RTE_CACHE_LINE_SIZE, 738 .size = sizeof(*rxq), 739 .addr = (void **)&rxq, 740 }, 741 { 742 .align = RTE_CACHE_LINE_SIZE, 743 .size = sizeof(*elts), 744 .addr = (void **)&elts, 745 }, 746 }; 747 int ret; 748 uint32_t crc_present; 749 uint64_t offloads; 750 751 offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads; 752 753 DEBUG("%p: configuring queue %u for %u descriptors", 754 (void *)dev, idx, desc); 755 756 if (idx >= dev->data->nb_rx_queues) { 757 rte_errno = EOVERFLOW; 758 ERROR("%p: queue index out of range (%u >= %u)", 759 (void *)dev, idx, dev->data->nb_rx_queues); 760 return -rte_errno; 761 } 762 rxq = dev->data->rx_queues[idx]; 763 if (rxq) { 764 rte_errno = EEXIST; 765 ERROR("%p: Rx queue %u already configured, release it first", 766 (void *)dev, idx); 767 return -rte_errno; 768 } 769 if (!desc) { 770 rte_errno = EINVAL; 771 ERROR("%p: invalid number of Rx descriptors", (void *)dev); 772 return -rte_errno; 773 } 774 if (desc != RTE_DIM(*elts)) { 775 desc = RTE_DIM(*elts); 776 WARN("%p: increased number of descriptors in Rx queue %u" 777 " to the next power of two (%u)", 778 (void *)dev, idx, desc); 779 } 780 /* By default, FCS (CRC) is stripped by hardware. */ 781 crc_present = 0; 782 if (offloads & DEV_RX_OFFLOAD_KEEP_CRC) { 783 if (priv->hw_fcs_strip) { 784 crc_present = 1; 785 } else { 786 WARN("%p: CRC stripping has been disabled but will still" 787 " be performed by hardware, make sure MLNX_OFED and" 788 " firmware are up to date", 789 (void *)dev); 790 } 791 } 792 DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from" 793 " incoming frames to hide it", 794 (void *)dev, 795 crc_present ? "disabled" : "enabled", 796 crc_present << 2); 797 /* Allocate and initialize Rx queue. */ 798 mlx4_zmallocv_socket("RXQ", vec, RTE_DIM(vec), socket); 799 if (!rxq) { 800 ERROR("%p: unable to allocate queue index %u", 801 (void *)dev, idx); 802 return -rte_errno; 803 } 804 *rxq = (struct rxq){ 805 .priv = priv, 806 .mp = mp, 807 .port_id = dev->data->port_id, 808 .sges_n = 0, 809 .elts_n = rte_log2_u32(desc), 810 .elts = elts, 811 /* Toggle Rx checksum offload if hardware supports it. */ 812 .csum = priv->hw_csum && 813 (offloads & DEV_RX_OFFLOAD_CHECKSUM), 814 .csum_l2tun = priv->hw_csum_l2tun && 815 (offloads & DEV_RX_OFFLOAD_CHECKSUM), 816 .crc_present = crc_present, 817 .l2tun_offload = priv->hw_csum_l2tun, 818 .stats = { 819 .idx = idx, 820 }, 821 .socket = socket, 822 }; 823 /* Enable scattered packets support for this queue if necessary. */ 824 assert(mb_len >= RTE_PKTMBUF_HEADROOM); 825 if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= 826 (mb_len - RTE_PKTMBUF_HEADROOM)) { 827 ; 828 } else if (offloads & DEV_RX_OFFLOAD_SCATTER) { 829 uint32_t size = 830 RTE_PKTMBUF_HEADROOM + 831 dev->data->dev_conf.rxmode.max_rx_pkt_len; 832 uint32_t sges_n; 833 834 /* 835 * Determine the number of SGEs needed for a full packet 836 * and round it to the next power of two. 837 */ 838 sges_n = rte_log2_u32((size / mb_len) + !!(size % mb_len)); 839 rxq->sges_n = sges_n; 840 /* Make sure sges_n did not overflow. */ 841 size = mb_len * (1 << rxq->sges_n); 842 size -= RTE_PKTMBUF_HEADROOM; 843 if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) { 844 rte_errno = EOVERFLOW; 845 ERROR("%p: too many SGEs (%u) needed to handle" 846 " requested maximum packet size %u", 847 (void *)dev, 848 1 << sges_n, 849 dev->data->dev_conf.rxmode.max_rx_pkt_len); 850 goto error; 851 } 852 } else { 853 WARN("%p: the requested maximum Rx packet size (%u) is" 854 " larger than a single mbuf (%u) and scattered" 855 " mode has not been requested", 856 (void *)dev, 857 dev->data->dev_conf.rxmode.max_rx_pkt_len, 858 mb_len - RTE_PKTMBUF_HEADROOM); 859 } 860 DEBUG("%p: maximum number of segments per packet: %u", 861 (void *)dev, 1 << rxq->sges_n); 862 if (desc % (1 << rxq->sges_n)) { 863 rte_errno = EINVAL; 864 ERROR("%p: number of Rx queue descriptors (%u) is not a" 865 " multiple of maximum segments per packet (%u)", 866 (void *)dev, 867 desc, 868 1 << rxq->sges_n); 869 goto error; 870 } 871 if (mlx4_mr_btree_init(&rxq->mr_ctrl.cache_bh, 872 MLX4_MR_BTREE_CACHE_N, socket)) { 873 /* rte_errno is already set. */ 874 goto error; 875 } 876 if (dev->data->dev_conf.intr_conf.rxq) { 877 rxq->channel = mlx4_glue->create_comp_channel(priv->ctx); 878 if (rxq->channel == NULL) { 879 rte_errno = ENOMEM; 880 ERROR("%p: Rx interrupt completion channel creation" 881 " failure: %s", 882 (void *)dev, strerror(rte_errno)); 883 goto error; 884 } 885 if (mlx4_fd_set_non_blocking(rxq->channel->fd) < 0) { 886 ERROR("%p: unable to make Rx interrupt completion" 887 " channel non-blocking: %s", 888 (void *)dev, strerror(rte_errno)); 889 goto error; 890 } 891 } 892 DEBUG("%p: adding Rx queue %p to list", (void *)dev, (void *)rxq); 893 dev->data->rx_queues[idx] = rxq; 894 return 0; 895 error: 896 dev->data->rx_queues[idx] = NULL; 897 ret = rte_errno; 898 mlx4_rx_queue_release(rxq); 899 rte_errno = ret; 900 assert(rte_errno > 0); 901 return -rte_errno; 902 } 903 904 /** 905 * DPDK callback to release a Rx queue. 906 * 907 * @param dpdk_rxq 908 * Generic Rx queue pointer. 909 */ 910 void 911 mlx4_rx_queue_release(void *dpdk_rxq) 912 { 913 struct rxq *rxq = (struct rxq *)dpdk_rxq; 914 struct priv *priv; 915 unsigned int i; 916 917 if (rxq == NULL) 918 return; 919 priv = rxq->priv; 920 for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) 921 if (priv->dev->data->rx_queues[i] == rxq) { 922 DEBUG("%p: removing Rx queue %p from list", 923 (void *)priv->dev, (void *)rxq); 924 priv->dev->data->rx_queues[i] = NULL; 925 break; 926 } 927 assert(!rxq->cq); 928 assert(!rxq->wq); 929 assert(!rxq->wqes); 930 assert(!rxq->rq_db); 931 if (rxq->channel) 932 claim_zero(mlx4_glue->destroy_comp_channel(rxq->channel)); 933 mlx4_mr_btree_free(&rxq->mr_ctrl.cache_bh); 934 rte_free(rxq); 935 } 936