1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /** 35 * @file 36 * Rx queues configuration for mlx4 driver. 37 */ 38 39 #include <assert.h> 40 #include <errno.h> 41 #include <stddef.h> 42 #include <stdint.h> 43 #include <string.h> 44 45 /* Verbs headers do not support -pedantic. */ 46 #ifdef PEDANTIC 47 #pragma GCC diagnostic ignored "-Wpedantic" 48 #endif 49 #include <infiniband/mlx4dv.h> 50 #include <infiniband/verbs.h> 51 #ifdef PEDANTIC 52 #pragma GCC diagnostic error "-Wpedantic" 53 #endif 54 55 #include <rte_byteorder.h> 56 #include <rte_common.h> 57 #include <rte_errno.h> 58 #include <rte_ethdev.h> 59 #include <rte_flow.h> 60 #include <rte_malloc.h> 61 #include <rte_mbuf.h> 62 #include <rte_mempool.h> 63 64 #include "mlx4.h" 65 #include "mlx4_flow.h" 66 #include "mlx4_rxtx.h" 67 #include "mlx4_utils.h" 68 69 /** 70 * Historical RSS hash key. 71 * 72 * This used to be the default for mlx4 in Linux before v3.19 switched to 73 * generating random hash keys through netdev_rss_key_fill(). 74 * 75 * It is used in this PMD for consistency with past DPDK releases but can 76 * now be overridden through user configuration. 77 * 78 * Note: this is not const to work around API quirks. 79 */ 80 uint8_t 81 mlx4_rss_hash_key_default[MLX4_RSS_HASH_KEY_SIZE] = { 82 0x2c, 0xc6, 0x81, 0xd1, 83 0x5b, 0xdb, 0xf4, 0xf7, 84 0xfc, 0xa2, 0x83, 0x19, 85 0xdb, 0x1a, 0x3e, 0x94, 86 0x6b, 0x9e, 0x38, 0xd9, 87 0x2c, 0x9c, 0x03, 0xd1, 88 0xad, 0x99, 0x44, 0xa7, 89 0xd9, 0x56, 0x3d, 0x59, 90 0x06, 0x3c, 0x25, 0xf3, 91 0xfc, 0x1f, 0xdc, 0x2a, 92 }; 93 94 /** 95 * Obtain a RSS context with specified properties. 96 * 97 * Used when creating a flow rule targeting one or several Rx queues. 98 * 99 * If a matching RSS context already exists, it is returned with its 100 * reference count incremented. 101 * 102 * @param priv 103 * Pointer to private structure. 104 * @param fields 105 * Fields for RSS processing (Verbs format). 106 * @param[in] key 107 * Hash key to use (whose size is exactly MLX4_RSS_HASH_KEY_SIZE). 108 * @param queues 109 * Number of target queues. 110 * @param[in] queue_id 111 * Target queues. 112 * 113 * @return 114 * Pointer to RSS context on success, NULL otherwise and rte_errno is set. 115 */ 116 struct mlx4_rss * 117 mlx4_rss_get(struct priv *priv, uint64_t fields, 118 uint8_t key[MLX4_RSS_HASH_KEY_SIZE], 119 uint16_t queues, const uint16_t queue_id[]) 120 { 121 struct mlx4_rss *rss; 122 size_t queue_id_size = sizeof(queue_id[0]) * queues; 123 124 LIST_FOREACH(rss, &priv->rss, next) 125 if (fields == rss->fields && 126 queues == rss->queues && 127 !memcmp(key, rss->key, MLX4_RSS_HASH_KEY_SIZE) && 128 !memcmp(queue_id, rss->queue_id, queue_id_size)) { 129 ++rss->refcnt; 130 return rss; 131 } 132 rss = rte_malloc(__func__, offsetof(struct mlx4_rss, queue_id) + 133 queue_id_size, 0); 134 if (!rss) 135 goto error; 136 *rss = (struct mlx4_rss){ 137 .priv = priv, 138 .refcnt = 1, 139 .usecnt = 0, 140 .qp = NULL, 141 .ind = NULL, 142 .fields = fields, 143 .queues = queues, 144 }; 145 memcpy(rss->key, key, MLX4_RSS_HASH_KEY_SIZE); 146 memcpy(rss->queue_id, queue_id, queue_id_size); 147 LIST_INSERT_HEAD(&priv->rss, rss, next); 148 return rss; 149 error: 150 rte_errno = ENOMEM; 151 return NULL; 152 } 153 154 /** 155 * Release a RSS context instance. 156 * 157 * Used when destroying a flow rule targeting one or several Rx queues. 158 * 159 * This function decrements the reference count of the context and destroys 160 * it after reaching 0. The context must have no users at this point; all 161 * prior calls to mlx4_rss_attach() must have been followed by matching 162 * calls to mlx4_rss_detach(). 163 * 164 * @param rss 165 * RSS context to release. 166 */ 167 void 168 mlx4_rss_put(struct mlx4_rss *rss) 169 { 170 assert(rss->refcnt); 171 if (--rss->refcnt) 172 return; 173 assert(!rss->usecnt); 174 assert(!rss->qp); 175 assert(!rss->ind); 176 LIST_REMOVE(rss, next); 177 rte_free(rss); 178 } 179 180 /** 181 * Attach a user to a RSS context instance. 182 * 183 * Used when the RSS QP and indirection table objects must be instantiated, 184 * that is, when a flow rule must be enabled. 185 * 186 * This function increments the usage count of the context. 187 * 188 * @param rss 189 * RSS context to attach to. 190 * 191 * @return 192 * 0 on success, a negative errno value otherwise and rte_errno is set. 193 */ 194 int 195 mlx4_rss_attach(struct mlx4_rss *rss) 196 { 197 assert(rss->refcnt); 198 if (rss->usecnt++) { 199 assert(rss->qp); 200 assert(rss->ind); 201 return 0; 202 } 203 204 struct ibv_wq *ind_tbl[rss->queues]; 205 struct priv *priv = rss->priv; 206 const char *msg; 207 unsigned int i = 0; 208 int ret; 209 210 if (!rte_is_power_of_2(RTE_DIM(ind_tbl))) { 211 ret = EINVAL; 212 msg = "number of RSS queues must be a power of two"; 213 goto error; 214 } 215 for (i = 0; i != RTE_DIM(ind_tbl); ++i) { 216 uint16_t id = rss->queue_id[i]; 217 struct rxq *rxq = NULL; 218 219 if (id < priv->dev->data->nb_rx_queues) 220 rxq = priv->dev->data->rx_queues[id]; 221 if (!rxq) { 222 ret = EINVAL; 223 msg = "RSS target queue is not configured"; 224 goto error; 225 } 226 ret = mlx4_rxq_attach(rxq); 227 if (ret) { 228 ret = -ret; 229 msg = "unable to attach RSS target queue"; 230 goto error; 231 } 232 ind_tbl[i] = rxq->wq; 233 } 234 rss->ind = ibv_create_rwq_ind_table 235 (priv->ctx, 236 &(struct ibv_rwq_ind_table_init_attr){ 237 .log_ind_tbl_size = rte_log2_u32(RTE_DIM(ind_tbl)), 238 .ind_tbl = ind_tbl, 239 .comp_mask = 0, 240 }); 241 if (!rss->ind) { 242 ret = errno ? errno : EINVAL; 243 msg = "RSS indirection table creation failure"; 244 goto error; 245 } 246 rss->qp = ibv_create_qp_ex 247 (priv->ctx, 248 &(struct ibv_qp_init_attr_ex){ 249 .comp_mask = (IBV_QP_INIT_ATTR_PD | 250 IBV_QP_INIT_ATTR_RX_HASH | 251 IBV_QP_INIT_ATTR_IND_TABLE), 252 .qp_type = IBV_QPT_RAW_PACKET, 253 .pd = priv->pd, 254 .rwq_ind_tbl = rss->ind, 255 .rx_hash_conf = { 256 .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, 257 .rx_hash_key_len = MLX4_RSS_HASH_KEY_SIZE, 258 .rx_hash_key = rss->key, 259 .rx_hash_fields_mask = rss->fields, 260 }, 261 }); 262 if (!rss->qp) { 263 ret = errno ? errno : EINVAL; 264 msg = "RSS hash QP creation failure"; 265 goto error; 266 } 267 ret = ibv_modify_qp 268 (rss->qp, 269 &(struct ibv_qp_attr){ 270 .qp_state = IBV_QPS_INIT, 271 .port_num = priv->port, 272 }, 273 IBV_QP_STATE | IBV_QP_PORT); 274 if (ret) { 275 msg = "failed to switch RSS hash QP to INIT state"; 276 goto error; 277 } 278 ret = ibv_modify_qp 279 (rss->qp, 280 &(struct ibv_qp_attr){ 281 .qp_state = IBV_QPS_RTR, 282 }, 283 IBV_QP_STATE); 284 if (ret) { 285 msg = "failed to switch RSS hash QP to RTR state"; 286 goto error; 287 } 288 return 0; 289 error: 290 if (rss->qp) { 291 claim_zero(ibv_destroy_qp(rss->qp)); 292 rss->qp = NULL; 293 } 294 if (rss->ind) { 295 claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); 296 rss->ind = NULL; 297 } 298 while (i--) 299 mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); 300 ERROR("mlx4: %s", msg); 301 --rss->usecnt; 302 rte_errno = ret; 303 return -ret; 304 } 305 306 /** 307 * Detach a user from a RSS context instance. 308 * 309 * Used when disabling (not destroying) a flow rule. 310 * 311 * This function decrements the usage count of the context and destroys 312 * usage resources after reaching 0. 313 * 314 * @param rss 315 * RSS context to detach from. 316 */ 317 void 318 mlx4_rss_detach(struct mlx4_rss *rss) 319 { 320 struct priv *priv = rss->priv; 321 unsigned int i; 322 323 assert(rss->refcnt); 324 assert(rss->qp); 325 assert(rss->ind); 326 if (--rss->usecnt) 327 return; 328 claim_zero(ibv_destroy_qp(rss->qp)); 329 rss->qp = NULL; 330 claim_zero(ibv_destroy_rwq_ind_table(rss->ind)); 331 rss->ind = NULL; 332 for (i = 0; i != rss->queues; ++i) 333 mlx4_rxq_detach(priv->dev->data->rx_queues[rss->queue_id[i]]); 334 } 335 336 /** 337 * Initialize common RSS context resources. 338 * 339 * Because ConnectX-3 hardware limitations require a fixed order in the 340 * indirection table, WQs must be allocated sequentially to be part of a 341 * common RSS context. 342 * 343 * Since a newly created WQ cannot be moved to a different context, this 344 * function allocates them all at once, one for each configured Rx queue, 345 * as well as all related resources (CQs and mbufs). 346 * 347 * This must therefore be done before creating any Rx flow rules relying on 348 * indirection tables. 349 * 350 * @param priv 351 * Pointer to private structure. 352 * 353 * @return 354 * 0 on success, a negative errno value otherwise and rte_errno is set. 355 */ 356 int 357 mlx4_rss_init(struct priv *priv) 358 { 359 struct rte_eth_dev *dev = priv->dev; 360 uint8_t log2_range = rte_log2_u32(dev->data->nb_rx_queues); 361 uint32_t wq_num_prev = 0; 362 const char *msg; 363 unsigned int i; 364 int ret; 365 366 if (priv->rss_init) 367 return 0; 368 if (priv->dev->data->nb_rx_queues > priv->hw_rss_max_qps) { 369 ERROR("RSS does not support more than %d queues", 370 priv->hw_rss_max_qps); 371 rte_errno = EINVAL; 372 return -rte_errno; 373 } 374 /* Prepare range for RSS contexts before creating the first WQ. */ 375 ret = mlx4dv_set_context_attr(priv->ctx, 376 MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, 377 &log2_range); 378 if (ret) { 379 ERROR("cannot set up range size for RSS context to %u" 380 " (for %u Rx queues), error: %s", 381 1 << log2_range, dev->data->nb_rx_queues, strerror(ret)); 382 rte_errno = ret; 383 return -ret; 384 } 385 for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { 386 struct rxq *rxq = priv->dev->data->rx_queues[i]; 387 struct ibv_cq *cq; 388 struct ibv_wq *wq; 389 uint32_t wq_num; 390 391 /* Attach the configured Rx queues. */ 392 if (rxq) { 393 assert(!rxq->usecnt); 394 ret = mlx4_rxq_attach(rxq); 395 if (!ret) { 396 wq_num = rxq->wq->wq_num; 397 goto wq_num_check; 398 } 399 ret = -ret; 400 msg = "unable to create Rx queue resources"; 401 goto error; 402 } 403 /* 404 * WQs are temporarily allocated for unconfigured Rx queues 405 * to maintain proper index alignment in indirection table 406 * by skipping unused WQ numbers. 407 * 408 * The reason this works at all even though these WQs are 409 * immediately destroyed is that WQNs are allocated 410 * sequentially and are guaranteed to never be reused in the 411 * same context by the underlying implementation. 412 */ 413 cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); 414 if (!cq) { 415 ret = ENOMEM; 416 msg = "placeholder CQ creation failure"; 417 goto error; 418 } 419 wq = ibv_create_wq 420 (priv->ctx, 421 &(struct ibv_wq_init_attr){ 422 .wq_type = IBV_WQT_RQ, 423 .max_wr = 1, 424 .max_sge = 1, 425 .pd = priv->pd, 426 .cq = cq, 427 }); 428 if (wq) { 429 wq_num = wq->wq_num; 430 claim_zero(ibv_destroy_wq(wq)); 431 } else { 432 wq_num = 0; /* Shut up GCC 4.8 warnings. */ 433 } 434 claim_zero(ibv_destroy_cq(cq)); 435 if (!wq) { 436 ret = ENOMEM; 437 msg = "placeholder WQ creation failure"; 438 goto error; 439 } 440 wq_num_check: 441 /* 442 * While guaranteed by the implementation, make sure WQ 443 * numbers are really sequential (as the saying goes, 444 * trust, but verify). 445 */ 446 if (i && wq_num - wq_num_prev != 1) { 447 if (rxq) 448 mlx4_rxq_detach(rxq); 449 ret = ERANGE; 450 msg = "WQ numbers are not sequential"; 451 goto error; 452 } 453 wq_num_prev = wq_num; 454 } 455 priv->rss_init = 1; 456 return 0; 457 error: 458 ERROR("cannot initialize common RSS resources (queue %u): %s: %s", 459 i, msg, strerror(ret)); 460 while (i--) { 461 struct rxq *rxq = priv->dev->data->rx_queues[i]; 462 463 if (rxq) 464 mlx4_rxq_detach(rxq); 465 } 466 rte_errno = ret; 467 return -ret; 468 } 469 470 /** 471 * Release common RSS context resources. 472 * 473 * As the reverse of mlx4_rss_init(), this must be done after removing all 474 * flow rules relying on indirection tables. 475 * 476 * @param priv 477 * Pointer to private structure. 478 */ 479 void 480 mlx4_rss_deinit(struct priv *priv) 481 { 482 unsigned int i; 483 484 if (!priv->rss_init) 485 return; 486 for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { 487 struct rxq *rxq = priv->dev->data->rx_queues[i]; 488 489 if (rxq) { 490 assert(rxq->usecnt == 1); 491 mlx4_rxq_detach(rxq); 492 } 493 } 494 priv->rss_init = 0; 495 } 496 497 /** 498 * Attach a user to a Rx queue. 499 * 500 * Used when the resources of an Rx queue must be instantiated for it to 501 * become in a usable state. 502 * 503 * This function increments the usage count of the Rx queue. 504 * 505 * @param rxq 506 * Pointer to Rx queue structure. 507 * 508 * @return 509 * 0 on success, negative errno value otherwise and rte_errno is set. 510 */ 511 int 512 mlx4_rxq_attach(struct rxq *rxq) 513 { 514 if (rxq->usecnt++) { 515 assert(rxq->cq); 516 assert(rxq->wq); 517 assert(rxq->wqes); 518 assert(rxq->rq_db); 519 return 0; 520 } 521 522 struct priv *priv = rxq->priv; 523 const uint32_t elts_n = 1 << rxq->elts_n; 524 const uint32_t sges_n = 1 << rxq->sges_n; 525 struct rte_mbuf *(*elts)[elts_n] = rxq->elts; 526 struct mlx4dv_obj mlxdv; 527 struct mlx4dv_rwq dv_rwq; 528 struct mlx4dv_cq dv_cq = { .comp_mask = MLX4DV_CQ_MASK_UAR, }; 529 const char *msg; 530 struct ibv_cq *cq = NULL; 531 struct ibv_wq *wq = NULL; 532 volatile struct mlx4_wqe_data_seg (*wqes)[]; 533 unsigned int i; 534 int ret; 535 536 assert(rte_is_power_of_2(elts_n)); 537 cq = ibv_create_cq(priv->ctx, elts_n / sges_n, NULL, rxq->channel, 0); 538 if (!cq) { 539 ret = ENOMEM; 540 msg = "CQ creation failure"; 541 goto error; 542 } 543 wq = ibv_create_wq 544 (priv->ctx, 545 &(struct ibv_wq_init_attr){ 546 .wq_type = IBV_WQT_RQ, 547 .max_wr = elts_n / sges_n, 548 .max_sge = sges_n, 549 .pd = priv->pd, 550 .cq = cq, 551 }); 552 if (!wq) { 553 ret = errno ? errno : EINVAL; 554 msg = "WQ creation failure"; 555 goto error; 556 } 557 ret = ibv_modify_wq 558 (wq, 559 &(struct ibv_wq_attr){ 560 .attr_mask = IBV_WQ_ATTR_STATE, 561 .wq_state = IBV_WQS_RDY, 562 }); 563 if (ret) { 564 msg = "WQ state change to IBV_WQS_RDY failed"; 565 goto error; 566 } 567 /* Retrieve device queue information. */ 568 mlxdv.cq.in = cq; 569 mlxdv.cq.out = &dv_cq; 570 mlxdv.rwq.in = wq; 571 mlxdv.rwq.out = &dv_rwq; 572 ret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_RWQ | MLX4DV_OBJ_CQ); 573 if (ret) { 574 msg = "failed to obtain device information from WQ/CQ objects"; 575 goto error; 576 } 577 wqes = (volatile struct mlx4_wqe_data_seg (*)[]) 578 ((uintptr_t)dv_rwq.buf.buf + dv_rwq.rq.offset); 579 for (i = 0; i != RTE_DIM(*elts); ++i) { 580 volatile struct mlx4_wqe_data_seg *scat = &(*wqes)[i]; 581 struct rte_mbuf *buf = rte_pktmbuf_alloc(rxq->mp); 582 583 if (buf == NULL) { 584 while (i--) { 585 rte_pktmbuf_free_seg((*elts)[i]); 586 (*elts)[i] = NULL; 587 } 588 ret = ENOMEM; 589 msg = "cannot allocate mbuf"; 590 goto error; 591 } 592 /* Headroom is reserved by rte_pktmbuf_alloc(). */ 593 assert(buf->data_off == RTE_PKTMBUF_HEADROOM); 594 /* Buffer is supposed to be empty. */ 595 assert(rte_pktmbuf_data_len(buf) == 0); 596 assert(rte_pktmbuf_pkt_len(buf) == 0); 597 /* Only the first segment keeps headroom. */ 598 if (i % sges_n) 599 buf->data_off = 0; 600 buf->port = rxq->port_id; 601 buf->data_len = rte_pktmbuf_tailroom(buf); 602 buf->pkt_len = rte_pktmbuf_tailroom(buf); 603 buf->nb_segs = 1; 604 *scat = (struct mlx4_wqe_data_seg){ 605 .addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, 606 uintptr_t)), 607 .byte_count = rte_cpu_to_be_32(buf->data_len), 608 .lkey = rte_cpu_to_be_32(rxq->mr->lkey), 609 }; 610 (*elts)[i] = buf; 611 } 612 DEBUG("%p: allocated and configured %u segments (max %u packets)", 613 (void *)rxq, elts_n, elts_n / sges_n); 614 rxq->cq = cq; 615 rxq->wq = wq; 616 rxq->wqes = wqes; 617 rxq->rq_db = dv_rwq.rdb; 618 rxq->mcq.buf = dv_cq.buf.buf; 619 rxq->mcq.cqe_cnt = dv_cq.cqe_cnt; 620 rxq->mcq.set_ci_db = dv_cq.set_ci_db; 621 rxq->mcq.cqe_64 = (dv_cq.cqe_size & 64) ? 1 : 0; 622 rxq->mcq.arm_db = dv_cq.arm_db; 623 rxq->mcq.arm_sn = dv_cq.arm_sn; 624 rxq->mcq.cqn = dv_cq.cqn; 625 rxq->mcq.cq_uar = dv_cq.cq_uar; 626 rxq->mcq.cq_db_reg = (uint8_t *)dv_cq.cq_uar + MLX4_CQ_DOORBELL; 627 /* Update doorbell counter. */ 628 rxq->rq_ci = elts_n / sges_n; 629 rte_wmb(); 630 *rxq->rq_db = rte_cpu_to_be_32(rxq->rq_ci); 631 return 0; 632 error: 633 if (wq) 634 claim_zero(ibv_destroy_wq(wq)); 635 if (cq) 636 claim_zero(ibv_destroy_cq(cq)); 637 --rxq->usecnt; 638 rte_errno = ret; 639 ERROR("error while attaching Rx queue %p: %s: %s", 640 (void *)rxq, msg, strerror(ret)); 641 return -ret; 642 } 643 644 /** 645 * Detach a user from a Rx queue. 646 * 647 * This function decrements the usage count of the Rx queue and destroys 648 * usage resources after reaching 0. 649 * 650 * @param rxq 651 * Pointer to Rx queue structure. 652 */ 653 void 654 mlx4_rxq_detach(struct rxq *rxq) 655 { 656 unsigned int i; 657 struct rte_mbuf *(*elts)[1 << rxq->elts_n] = rxq->elts; 658 659 if (--rxq->usecnt) 660 return; 661 rxq->rq_ci = 0; 662 memset(&rxq->mcq, 0, sizeof(rxq->mcq)); 663 rxq->rq_db = NULL; 664 rxq->wqes = NULL; 665 claim_zero(ibv_destroy_wq(rxq->wq)); 666 rxq->wq = NULL; 667 claim_zero(ibv_destroy_cq(rxq->cq)); 668 rxq->cq = NULL; 669 DEBUG("%p: freeing Rx queue elements", (void *)rxq); 670 for (i = 0; (i != RTE_DIM(*elts)); ++i) { 671 if (!(*elts)[i]) 672 continue; 673 rte_pktmbuf_free_seg((*elts)[i]); 674 (*elts)[i] = NULL; 675 } 676 } 677 678 /** 679 * DPDK callback to configure a Rx queue. 680 * 681 * @param dev 682 * Pointer to Ethernet device structure. 683 * @param idx 684 * Rx queue index. 685 * @param desc 686 * Number of descriptors to configure in queue. 687 * @param socket 688 * NUMA socket on which memory must be allocated. 689 * @param[in] conf 690 * Thresholds parameters. 691 * @param mp 692 * Memory pool for buffer allocations. 693 * 694 * @return 695 * 0 on success, negative errno value otherwise and rte_errno is set. 696 */ 697 int 698 mlx4_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 699 unsigned int socket, const struct rte_eth_rxconf *conf, 700 struct rte_mempool *mp) 701 { 702 struct priv *priv = dev->data->dev_private; 703 uint32_t mb_len = rte_pktmbuf_data_room_size(mp); 704 struct rte_mbuf *(*elts)[rte_align32pow2(desc)]; 705 struct rxq *rxq; 706 struct mlx4_malloc_vec vec[] = { 707 { 708 .align = RTE_CACHE_LINE_SIZE, 709 .size = sizeof(*rxq), 710 .addr = (void **)&rxq, 711 }, 712 { 713 .align = RTE_CACHE_LINE_SIZE, 714 .size = sizeof(*elts), 715 .addr = (void **)&elts, 716 }, 717 }; 718 int ret; 719 720 (void)conf; /* Thresholds configuration (ignored). */ 721 DEBUG("%p: configuring queue %u for %u descriptors", 722 (void *)dev, idx, desc); 723 if (idx >= dev->data->nb_rx_queues) { 724 rte_errno = EOVERFLOW; 725 ERROR("%p: queue index out of range (%u >= %u)", 726 (void *)dev, idx, dev->data->nb_rx_queues); 727 return -rte_errno; 728 } 729 rxq = dev->data->rx_queues[idx]; 730 if (rxq) { 731 rte_errno = EEXIST; 732 ERROR("%p: Rx queue %u already configured, release it first", 733 (void *)dev, idx); 734 return -rte_errno; 735 } 736 if (!desc) { 737 rte_errno = EINVAL; 738 ERROR("%p: invalid number of Rx descriptors", (void *)dev); 739 return -rte_errno; 740 } 741 if (desc != RTE_DIM(*elts)) { 742 desc = RTE_DIM(*elts); 743 WARN("%p: increased number of descriptors in Rx queue %u" 744 " to the next power of two (%u)", 745 (void *)dev, idx, desc); 746 } 747 /* Allocate and initialize Rx queue. */ 748 mlx4_zmallocv_socket("RXQ", vec, RTE_DIM(vec), socket); 749 if (!rxq) { 750 ERROR("%p: unable to allocate queue index %u", 751 (void *)dev, idx); 752 return -rte_errno; 753 } 754 *rxq = (struct rxq){ 755 .priv = priv, 756 .mp = mp, 757 .port_id = dev->data->port_id, 758 .sges_n = 0, 759 .elts_n = rte_log2_u32(desc), 760 .elts = elts, 761 /* Toggle Rx checksum offload if hardware supports it. */ 762 .csum = (priv->hw_csum && 763 dev->data->dev_conf.rxmode.hw_ip_checksum), 764 .csum_l2tun = (priv->hw_csum_l2tun && 765 dev->data->dev_conf.rxmode.hw_ip_checksum), 766 .l2tun_offload = priv->hw_csum_l2tun, 767 .stats = { 768 .idx = idx, 769 }, 770 .socket = socket, 771 }; 772 /* Enable scattered packets support for this queue if necessary. */ 773 assert(mb_len >= RTE_PKTMBUF_HEADROOM); 774 if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= 775 (mb_len - RTE_PKTMBUF_HEADROOM)) { 776 ; 777 } else if (dev->data->dev_conf.rxmode.enable_scatter) { 778 uint32_t size = 779 RTE_PKTMBUF_HEADROOM + 780 dev->data->dev_conf.rxmode.max_rx_pkt_len; 781 uint32_t sges_n; 782 783 /* 784 * Determine the number of SGEs needed for a full packet 785 * and round it to the next power of two. 786 */ 787 sges_n = rte_log2_u32((size / mb_len) + !!(size % mb_len)); 788 rxq->sges_n = sges_n; 789 /* Make sure sges_n did not overflow. */ 790 size = mb_len * (1 << rxq->sges_n); 791 size -= RTE_PKTMBUF_HEADROOM; 792 if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) { 793 rte_errno = EOVERFLOW; 794 ERROR("%p: too many SGEs (%u) needed to handle" 795 " requested maximum packet size %u", 796 (void *)dev, 797 1 << sges_n, 798 dev->data->dev_conf.rxmode.max_rx_pkt_len); 799 goto error; 800 } 801 } else { 802 WARN("%p: the requested maximum Rx packet size (%u) is" 803 " larger than a single mbuf (%u) and scattered" 804 " mode has not been requested", 805 (void *)dev, 806 dev->data->dev_conf.rxmode.max_rx_pkt_len, 807 mb_len - RTE_PKTMBUF_HEADROOM); 808 } 809 DEBUG("%p: maximum number of segments per packet: %u", 810 (void *)dev, 1 << rxq->sges_n); 811 if (desc % (1 << rxq->sges_n)) { 812 rte_errno = EINVAL; 813 ERROR("%p: number of Rx queue descriptors (%u) is not a" 814 " multiple of maximum segments per packet (%u)", 815 (void *)dev, 816 desc, 817 1 << rxq->sges_n); 818 goto error; 819 } 820 /* Use the entire Rx mempool as the memory region. */ 821 rxq->mr = mlx4_mr_get(priv, mp); 822 if (!rxq->mr) { 823 ERROR("%p: MR creation failure: %s", 824 (void *)dev, strerror(rte_errno)); 825 goto error; 826 } 827 if (dev->data->dev_conf.intr_conf.rxq) { 828 rxq->channel = ibv_create_comp_channel(priv->ctx); 829 if (rxq->channel == NULL) { 830 rte_errno = ENOMEM; 831 ERROR("%p: Rx interrupt completion channel creation" 832 " failure: %s", 833 (void *)dev, strerror(rte_errno)); 834 goto error; 835 } 836 if (mlx4_fd_set_non_blocking(rxq->channel->fd) < 0) { 837 ERROR("%p: unable to make Rx interrupt completion" 838 " channel non-blocking: %s", 839 (void *)dev, strerror(rte_errno)); 840 goto error; 841 } 842 } 843 DEBUG("%p: adding Rx queue %p to list", (void *)dev, (void *)rxq); 844 dev->data->rx_queues[idx] = rxq; 845 return 0; 846 error: 847 dev->data->rx_queues[idx] = NULL; 848 ret = rte_errno; 849 mlx4_rx_queue_release(rxq); 850 rte_errno = ret; 851 assert(rte_errno > 0); 852 return -rte_errno; 853 } 854 855 /** 856 * DPDK callback to release a Rx queue. 857 * 858 * @param dpdk_rxq 859 * Generic Rx queue pointer. 860 */ 861 void 862 mlx4_rx_queue_release(void *dpdk_rxq) 863 { 864 struct rxq *rxq = (struct rxq *)dpdk_rxq; 865 struct priv *priv; 866 unsigned int i; 867 868 if (rxq == NULL) 869 return; 870 priv = rxq->priv; 871 for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) 872 if (priv->dev->data->rx_queues[i] == rxq) { 873 DEBUG("%p: removing Rx queue %p from list", 874 (void *)priv->dev, (void *)rxq); 875 priv->dev->data->rx_queues[i] = NULL; 876 break; 877 } 878 assert(!rxq->cq); 879 assert(!rxq->wq); 880 assert(!rxq->wqes); 881 assert(!rxq->rq_db); 882 if (rxq->channel) 883 claim_zero(ibv_destroy_comp_channel(rxq->channel)); 884 if (rxq->mr) 885 mlx4_mr_put(rxq->mr); 886 rte_free(rxq); 887 } 888