1 /*- 2 * BSD LICENSE 3 * 4 * Copyright 2017 6WIND S.A. 5 * Copyright 2017 Mellanox 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of 6WIND S.A. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /** 35 * @file 36 * Tx queues configuration for mlx4 driver. 37 */ 38 39 #include <assert.h> 40 #include <errno.h> 41 #include <stddef.h> 42 #include <stdint.h> 43 #include <string.h> 44 45 /* Verbs headers do not support -pedantic. */ 46 #ifdef PEDANTIC 47 #pragma GCC diagnostic ignored "-Wpedantic" 48 #endif 49 #include <infiniband/verbs.h> 50 #ifdef PEDANTIC 51 #pragma GCC diagnostic error "-Wpedantic" 52 #endif 53 54 #include <rte_common.h> 55 #include <rte_errno.h> 56 #include <rte_ethdev.h> 57 #include <rte_malloc.h> 58 #include <rte_mbuf.h> 59 #include <rte_mempool.h> 60 61 #include "mlx4.h" 62 #include "mlx4_prm.h" 63 #include "mlx4_rxtx.h" 64 #include "mlx4_utils.h" 65 66 /** 67 * Free Tx queue elements. 68 * 69 * @param txq 70 * Pointer to Tx queue structure. 71 */ 72 static void 73 mlx4_txq_free_elts(struct txq *txq) 74 { 75 unsigned int elts_head = txq->elts_head; 76 unsigned int elts_tail = txq->elts_tail; 77 struct txq_elt (*elts)[txq->elts_n] = txq->elts; 78 79 DEBUG("%p: freeing WRs", (void *)txq); 80 while (elts_tail != elts_head) { 81 struct txq_elt *elt = &(*elts)[elts_tail]; 82 83 assert(elt->buf != NULL); 84 rte_pktmbuf_free(elt->buf); 85 elt->buf = NULL; 86 if (++elts_tail == RTE_DIM(*elts)) 87 elts_tail = 0; 88 } 89 txq->elts_tail = txq->elts_head; 90 } 91 92 struct txq_mp2mr_mbuf_check_data { 93 int ret; 94 }; 95 96 /** 97 * Callback function for rte_mempool_obj_iter() to check whether a given 98 * mempool object looks like a mbuf. 99 * 100 * @param[in] mp 101 * The mempool pointer 102 * @param[in] arg 103 * Context data (struct mlx4_txq_mp2mr_mbuf_check_data). Contains the 104 * return value. 105 * @param[in] obj 106 * Object address. 107 * @param index 108 * Object index, unused. 109 */ 110 static void 111 mlx4_txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj, 112 uint32_t index) 113 { 114 struct txq_mp2mr_mbuf_check_data *data = arg; 115 struct rte_mbuf *buf = obj; 116 117 (void)index; 118 /* 119 * Check whether mbuf structure fits element size and whether mempool 120 * pointer is valid. 121 */ 122 if (sizeof(*buf) > mp->elt_size || buf->pool != mp) 123 data->ret = -1; 124 } 125 126 /** 127 * Iterator function for rte_mempool_walk() to register existing mempools and 128 * fill the MP to MR cache of a Tx queue. 129 * 130 * @param[in] mp 131 * Memory Pool to register. 132 * @param *arg 133 * Pointer to Tx queue structure. 134 */ 135 static void 136 mlx4_txq_mp2mr_iter(struct rte_mempool *mp, void *arg) 137 { 138 struct txq *txq = arg; 139 struct txq_mp2mr_mbuf_check_data data = { 140 .ret = 0, 141 }; 142 143 /* Register mempool only if the first element looks like a mbuf. */ 144 if (rte_mempool_obj_iter(mp, mlx4_txq_mp2mr_mbuf_check, &data) == 0 || 145 data.ret == -1) 146 return; 147 mlx4_txq_mp2mr(txq, mp); 148 } 149 150 /** 151 * Retrieves information needed in order to directly access the Tx queue. 152 * 153 * @param txq 154 * Pointer to Tx queue structure. 155 * @param mlxdv 156 * Pointer to device information for this Tx queue. 157 */ 158 static void 159 mlx4_txq_fill_dv_obj_info(struct txq *txq, struct mlx4dv_obj *mlxdv) 160 { 161 struct mlx4_sq *sq = &txq->msq; 162 struct mlx4_cq *cq = &txq->mcq; 163 struct mlx4dv_qp *dqp = mlxdv->qp.out; 164 struct mlx4dv_cq *dcq = mlxdv->cq.out; 165 uint32_t sq_size = (uint32_t)dqp->rq.offset - (uint32_t)dqp->sq.offset; 166 167 sq->buf = (uint8_t *)dqp->buf.buf + dqp->sq.offset; 168 /* Total length, including headroom and spare WQEs. */ 169 sq->eob = sq->buf + sq_size; 170 sq->head = 0; 171 sq->tail = 0; 172 sq->txbb_cnt = 173 (dqp->sq.wqe_cnt << dqp->sq.wqe_shift) >> MLX4_TXBB_SHIFT; 174 sq->txbb_cnt_mask = sq->txbb_cnt - 1; 175 sq->db = dqp->sdb; 176 sq->doorbell_qpn = dqp->doorbell_qpn; 177 sq->headroom_txbbs = 178 (2048 + (1 << dqp->sq.wqe_shift)) >> MLX4_TXBB_SHIFT; 179 cq->buf = dcq->buf.buf; 180 cq->cqe_cnt = dcq->cqe_cnt; 181 cq->set_ci_db = dcq->set_ci_db; 182 cq->cqe_64 = (dcq->cqe_size & 64) ? 1 : 0; 183 } 184 185 /** 186 * DPDK callback to configure a Tx queue. 187 * 188 * @param dev 189 * Pointer to Ethernet device structure. 190 * @param idx 191 * Tx queue index. 192 * @param desc 193 * Number of descriptors to configure in queue. 194 * @param socket 195 * NUMA socket on which memory must be allocated. 196 * @param[in] conf 197 * Thresholds parameters. 198 * 199 * @return 200 * 0 on success, negative errno value otherwise and rte_errno is set. 201 */ 202 int 203 mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, 204 unsigned int socket, const struct rte_eth_txconf *conf) 205 { 206 struct priv *priv = dev->data->dev_private; 207 struct mlx4dv_obj mlxdv; 208 struct mlx4dv_qp dv_qp; 209 struct mlx4dv_cq dv_cq; 210 struct txq_elt (*elts)[desc]; 211 struct ibv_qp_init_attr qp_init_attr; 212 struct txq *txq; 213 uint8_t *bounce_buf; 214 struct mlx4_malloc_vec vec[] = { 215 { 216 .align = RTE_CACHE_LINE_SIZE, 217 .size = sizeof(*txq), 218 .addr = (void **)&txq, 219 }, 220 { 221 .align = RTE_CACHE_LINE_SIZE, 222 .size = sizeof(*elts), 223 .addr = (void **)&elts, 224 }, 225 { 226 .align = RTE_CACHE_LINE_SIZE, 227 .size = MLX4_MAX_WQE_SIZE, 228 .addr = (void **)&bounce_buf, 229 }, 230 }; 231 int ret; 232 233 (void)conf; /* Thresholds configuration (ignored). */ 234 DEBUG("%p: configuring queue %u for %u descriptors", 235 (void *)dev, idx, desc); 236 if (idx >= dev->data->nb_tx_queues) { 237 rte_errno = EOVERFLOW; 238 ERROR("%p: queue index out of range (%u >= %u)", 239 (void *)dev, idx, dev->data->nb_tx_queues); 240 return -rte_errno; 241 } 242 txq = dev->data->tx_queues[idx]; 243 if (txq) { 244 rte_errno = EEXIST; 245 DEBUG("%p: Tx queue %u already configured, release it first", 246 (void *)dev, idx); 247 return -rte_errno; 248 } 249 if (!desc) { 250 rte_errno = EINVAL; 251 ERROR("%p: invalid number of Tx descriptors", (void *)dev); 252 return -rte_errno; 253 } 254 /* Allocate and initialize Tx queue. */ 255 mlx4_zmallocv_socket("TXQ", vec, RTE_DIM(vec), socket); 256 if (!txq) { 257 ERROR("%p: unable to allocate queue index %u", 258 (void *)dev, idx); 259 return -rte_errno; 260 } 261 *txq = (struct txq){ 262 .priv = priv, 263 .stats = { 264 .idx = idx, 265 }, 266 .socket = socket, 267 .elts_n = desc, 268 .elts = elts, 269 .elts_head = 0, 270 .elts_tail = 0, 271 .elts_comp = 0, 272 /* 273 * Request send completion every MLX4_PMD_TX_PER_COMP_REQ 274 * packets or at least 4 times per ring. 275 */ 276 .elts_comp_cd = 277 RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4), 278 .elts_comp_cd_init = 279 RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4), 280 .csum = priv->hw_csum, 281 .csum_l2tun = priv->hw_csum_l2tun, 282 /* Enable Tx loopback for VF devices. */ 283 .lb = !!priv->vf, 284 .bounce_buf = bounce_buf, 285 }; 286 txq->cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); 287 if (!txq->cq) { 288 rte_errno = ENOMEM; 289 ERROR("%p: CQ creation failure: %s", 290 (void *)dev, strerror(rte_errno)); 291 goto error; 292 } 293 qp_init_attr = (struct ibv_qp_init_attr){ 294 .send_cq = txq->cq, 295 .recv_cq = txq->cq, 296 .cap = { 297 .max_send_wr = 298 RTE_MIN(priv->device_attr.max_qp_wr, desc), 299 .max_send_sge = 1, 300 .max_inline_data = MLX4_PMD_MAX_INLINE, 301 }, 302 .qp_type = IBV_QPT_RAW_PACKET, 303 /* No completion events must occur by default. */ 304 .sq_sig_all = 0, 305 }; 306 txq->qp = ibv_create_qp(priv->pd, &qp_init_attr); 307 if (!txq->qp) { 308 rte_errno = errno ? errno : EINVAL; 309 ERROR("%p: QP creation failure: %s", 310 (void *)dev, strerror(rte_errno)); 311 goto error; 312 } 313 txq->max_inline = qp_init_attr.cap.max_inline_data; 314 ret = ibv_modify_qp 315 (txq->qp, 316 &(struct ibv_qp_attr){ 317 .qp_state = IBV_QPS_INIT, 318 .port_num = priv->port, 319 }, 320 IBV_QP_STATE | IBV_QP_PORT); 321 if (ret) { 322 rte_errno = ret; 323 ERROR("%p: QP state to IBV_QPS_INIT failed: %s", 324 (void *)dev, strerror(rte_errno)); 325 goto error; 326 } 327 ret = ibv_modify_qp 328 (txq->qp, 329 &(struct ibv_qp_attr){ 330 .qp_state = IBV_QPS_RTR, 331 }, 332 IBV_QP_STATE); 333 if (ret) { 334 rte_errno = ret; 335 ERROR("%p: QP state to IBV_QPS_RTR failed: %s", 336 (void *)dev, strerror(rte_errno)); 337 goto error; 338 } 339 ret = ibv_modify_qp 340 (txq->qp, 341 &(struct ibv_qp_attr){ 342 .qp_state = IBV_QPS_RTS, 343 }, 344 IBV_QP_STATE); 345 if (ret) { 346 rte_errno = ret; 347 ERROR("%p: QP state to IBV_QPS_RTS failed: %s", 348 (void *)dev, strerror(rte_errno)); 349 goto error; 350 } 351 /* Retrieve device queue information. */ 352 mlxdv.cq.in = txq->cq; 353 mlxdv.cq.out = &dv_cq; 354 mlxdv.qp.in = txq->qp; 355 mlxdv.qp.out = &dv_qp; 356 ret = mlx4dv_init_obj(&mlxdv, MLX4DV_OBJ_QP | MLX4DV_OBJ_CQ); 357 if (ret) { 358 rte_errno = EINVAL; 359 ERROR("%p: failed to obtain information needed for" 360 " accessing the device queues", (void *)dev); 361 goto error; 362 } 363 mlx4_txq_fill_dv_obj_info(txq, &mlxdv); 364 /* Pre-register known mempools. */ 365 rte_mempool_walk(mlx4_txq_mp2mr_iter, txq); 366 DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq); 367 dev->data->tx_queues[idx] = txq; 368 return 0; 369 error: 370 dev->data->tx_queues[idx] = NULL; 371 ret = rte_errno; 372 mlx4_tx_queue_release(txq); 373 rte_errno = ret; 374 assert(rte_errno > 0); 375 return -rte_errno; 376 } 377 378 /** 379 * DPDK callback to release a Tx queue. 380 * 381 * @param dpdk_txq 382 * Generic Tx queue pointer. 383 */ 384 void 385 mlx4_tx_queue_release(void *dpdk_txq) 386 { 387 struct txq *txq = (struct txq *)dpdk_txq; 388 struct priv *priv; 389 unsigned int i; 390 391 if (txq == NULL) 392 return; 393 priv = txq->priv; 394 for (i = 0; i != priv->dev->data->nb_tx_queues; ++i) 395 if (priv->dev->data->tx_queues[i] == txq) { 396 DEBUG("%p: removing Tx queue %p from list", 397 (void *)priv->dev, (void *)txq); 398 priv->dev->data->tx_queues[i] = NULL; 399 break; 400 } 401 mlx4_txq_free_elts(txq); 402 if (txq->qp) 403 claim_zero(ibv_destroy_qp(txq->qp)); 404 if (txq->cq) 405 claim_zero(ibv_destroy_cq(txq->cq)); 406 for (i = 0; i != RTE_DIM(txq->mp2mr); ++i) { 407 if (!txq->mp2mr[i].mp) 408 break; 409 assert(txq->mp2mr[i].mr); 410 mlx4_mr_put(txq->mp2mr[i].mr); 411 } 412 rte_free(txq); 413 } 414