1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
4 */
5
6 /**
7 * @file
8 * Tx queues configuration for mlx4 driver.
9 */
10
11 #include <errno.h>
12 #include <stddef.h>
13 #include <stdint.h>
14 #include <string.h>
15 #include <sys/mman.h>
16 #include <inttypes.h>
17 #include <unistd.h>
18
19 /* Verbs headers do not support -pedantic. */
20 #ifdef PEDANTIC
21 #pragma GCC diagnostic ignored "-Wpedantic"
22 #endif
23 #include <infiniband/verbs.h>
24 #ifdef PEDANTIC
25 #pragma GCC diagnostic error "-Wpedantic"
26 #endif
27
28 #include <rte_common.h>
29 #include <rte_errno.h>
30 #include <rte_ethdev_driver.h>
31 #include <rte_malloc.h>
32 #include <rte_mbuf.h>
33 #include <rte_mempool.h>
34
35 #include "mlx4.h"
36 #include "mlx4_glue.h"
37 #include "mlx4_prm.h"
38 #include "mlx4_rxtx.h"
39 #include "mlx4_utils.h"
40
41 /**
42 * Initialize Tx UAR registers for primary process.
43 *
44 * @param txq
45 * Pointer to Tx queue structure.
46 */
47 static void
txq_uar_init(struct txq * txq)48 txq_uar_init(struct txq *txq)
49 {
50 struct mlx4_priv *priv = txq->priv;
51 struct mlx4_proc_priv *ppriv = MLX4_PROC_PRIV(PORT_ID(priv));
52
53 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
54 MLX4_ASSERT(ppriv);
55 ppriv->uar_table[txq->stats.idx] = txq->msq.db;
56 }
57
58 #ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET
59 /**
60 * Remap UAR register of a Tx queue for secondary process.
61 *
62 * Remapped address is stored at the table in the process private structure of
63 * the device, indexed by queue index.
64 *
65 * @param txq
66 * Pointer to Tx queue structure.
67 * @param fd
68 * Verbs file descriptor to map UAR pages.
69 *
70 * @return
71 * 0 on success, a negative errno value otherwise and rte_errno is set.
72 */
73 static int
txq_uar_init_secondary(struct txq * txq,int fd)74 txq_uar_init_secondary(struct txq *txq, int fd)
75 {
76 struct mlx4_priv *priv = txq->priv;
77 struct mlx4_proc_priv *ppriv = MLX4_PROC_PRIV(PORT_ID(priv));
78 void *addr;
79 uintptr_t uar_va;
80 uintptr_t offset;
81 const size_t page_size = sysconf(_SC_PAGESIZE);
82
83 MLX4_ASSERT(ppriv);
84 /*
85 * As rdma-core, UARs are mapped in size of OS page
86 * size. Ref to libmlx4 function: mlx4_init_context()
87 */
88 uar_va = (uintptr_t)txq->msq.db;
89 offset = uar_va & (page_size - 1); /* Offset in page. */
90 addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd,
91 txq->msq.uar_mmap_offset);
92 if (addr == MAP_FAILED) {
93 ERROR("port %u mmap failed for BF reg of txq %u",
94 txq->port_id, txq->stats.idx);
95 rte_errno = ENXIO;
96 return -rte_errno;
97 }
98 addr = RTE_PTR_ADD(addr, offset);
99 ppriv->uar_table[txq->stats.idx] = addr;
100 return 0;
101 }
102
103 /**
104 * Unmap UAR register of a Tx queue for secondary process.
105 *
106 * @param txq
107 * Pointer to Tx queue structure.
108 */
109 static void
txq_uar_uninit_secondary(struct txq * txq)110 txq_uar_uninit_secondary(struct txq *txq)
111 {
112 struct mlx4_proc_priv *ppriv = MLX4_PROC_PRIV(PORT_ID(txq->priv));
113 const size_t page_size = sysconf(_SC_PAGESIZE);
114 void *addr;
115
116 addr = ppriv->uar_table[txq->stats.idx];
117 munmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size);
118 }
119
120 /**
121 * Initialize Tx UAR registers for secondary process.
122 *
123 * @param dev
124 * Pointer to Ethernet device.
125 * @param fd
126 * Verbs file descriptor to map UAR pages.
127 *
128 * @return
129 * 0 on success, a negative errno value otherwise and rte_errno is set.
130 */
131 int
mlx4_tx_uar_init_secondary(struct rte_eth_dev * dev,int fd)132 mlx4_tx_uar_init_secondary(struct rte_eth_dev *dev, int fd)
133 {
134 const unsigned int txqs_n = dev->data->nb_tx_queues;
135 struct txq *txq;
136 unsigned int i;
137 int ret;
138
139 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
140 for (i = 0; i != txqs_n; ++i) {
141 txq = dev->data->tx_queues[i];
142 if (!txq)
143 continue;
144 MLX4_ASSERT(txq->stats.idx == (uint16_t)i);
145 ret = txq_uar_init_secondary(txq, fd);
146 if (ret)
147 goto error;
148 }
149 return 0;
150 error:
151 /* Rollback. */
152 do {
153 txq = dev->data->tx_queues[i];
154 if (!txq)
155 continue;
156 txq_uar_uninit_secondary(txq);
157 } while (i--);
158 return -rte_errno;
159 }
160 #else
161 int
mlx4_tx_uar_init_secondary(struct rte_eth_dev * dev __rte_unused,int fd __rte_unused)162 mlx4_tx_uar_init_secondary(struct rte_eth_dev *dev __rte_unused,
163 int fd __rte_unused)
164 {
165 MLX4_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
166 ERROR("UAR remap is not supported");
167 rte_errno = ENOTSUP;
168 return -rte_errno;
169 }
170 #endif
171
172 /**
173 * Free Tx queue elements.
174 *
175 * @param txq
176 * Pointer to Tx queue structure.
177 */
178 static void
mlx4_txq_free_elts(struct txq * txq)179 mlx4_txq_free_elts(struct txq *txq)
180 {
181 unsigned int elts_head = txq->elts_head;
182 unsigned int elts_tail = txq->elts_tail;
183 struct txq_elt (*elts)[txq->elts_n] = txq->elts;
184 unsigned int elts_m = txq->elts_n - 1;
185
186 DEBUG("%p: freeing WRs", (void *)txq);
187 while (elts_tail != elts_head) {
188 struct txq_elt *elt = &(*elts)[elts_tail++ & elts_m];
189
190 MLX4_ASSERT(elt->buf != NULL);
191 rte_pktmbuf_free(elt->buf);
192 elt->buf = NULL;
193 elt->wqe = NULL;
194 }
195 txq->elts_tail = txq->elts_head;
196 }
197
198 /**
199 * Retrieves information needed in order to directly access the Tx queue.
200 *
201 * @param txq
202 * Pointer to Tx queue structure.
203 * @param mlxdv
204 * Pointer to device information for this Tx queue.
205 */
206 static void
mlx4_txq_fill_dv_obj_info(struct txq * txq,struct mlx4dv_obj * mlxdv)207 mlx4_txq_fill_dv_obj_info(struct txq *txq, struct mlx4dv_obj *mlxdv)
208 {
209 struct mlx4_sq *sq = &txq->msq;
210 struct mlx4_cq *cq = &txq->mcq;
211 struct mlx4dv_qp *dqp = mlxdv->qp.out;
212 struct mlx4dv_cq *dcq = mlxdv->cq.out;
213
214 /* Total length, including headroom and spare WQEs. */
215 sq->size = (uint32_t)dqp->rq.offset - (uint32_t)dqp->sq.offset;
216 sq->buf = (uint8_t *)dqp->buf.buf + dqp->sq.offset;
217 sq->eob = sq->buf + sq->size;
218 uint32_t headroom_size = 2048 + (1 << dqp->sq.wqe_shift);
219 /* Continuous headroom size bytes must always stay freed. */
220 sq->remain_size = sq->size - headroom_size;
221 sq->owner_opcode = MLX4_OPCODE_SEND | (0u << MLX4_SQ_OWNER_BIT);
222 sq->stamp = rte_cpu_to_be_32(MLX4_SQ_STAMP_VAL |
223 (0u << MLX4_SQ_OWNER_BIT));
224 #ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET
225 sq->uar_mmap_offset = dqp->uar_mmap_offset;
226 #else
227 sq->uar_mmap_offset = -1; /* Make mmap() fail. */
228 #endif
229 sq->db = dqp->sdb;
230 sq->doorbell_qpn = dqp->doorbell_qpn;
231 cq->buf = dcq->buf.buf;
232 cq->cqe_cnt = dcq->cqe_cnt;
233 cq->set_ci_db = dcq->set_ci_db;
234 cq->cqe_64 = (dcq->cqe_size & 64) ? 1 : 0;
235 }
236
237 /**
238 * Returns the per-port supported offloads.
239 *
240 * @param priv
241 * Pointer to private structure.
242 *
243 * @return
244 * Supported Tx offloads.
245 */
246 uint64_t
mlx4_get_tx_port_offloads(struct mlx4_priv * priv)247 mlx4_get_tx_port_offloads(struct mlx4_priv *priv)
248 {
249 uint64_t offloads = DEV_TX_OFFLOAD_MULTI_SEGS;
250
251 if (priv->hw_csum) {
252 offloads |= (DEV_TX_OFFLOAD_IPV4_CKSUM |
253 DEV_TX_OFFLOAD_UDP_CKSUM |
254 DEV_TX_OFFLOAD_TCP_CKSUM);
255 }
256 if (priv->tso)
257 offloads |= DEV_TX_OFFLOAD_TCP_TSO;
258 if (priv->hw_csum_l2tun) {
259 offloads |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
260 if (priv->tso)
261 offloads |= (DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
262 DEV_TX_OFFLOAD_GRE_TNL_TSO);
263 }
264 return offloads;
265 }
266
267 /**
268 * DPDK callback to configure a Tx queue.
269 *
270 * @param dev
271 * Pointer to Ethernet device structure.
272 * @param idx
273 * Tx queue index.
274 * @param desc
275 * Number of descriptors to configure in queue.
276 * @param socket
277 * NUMA socket on which memory must be allocated.
278 * @param[in] conf
279 * Thresholds parameters.
280 *
281 * @return
282 * 0 on success, negative errno value otherwise and rte_errno is set.
283 */
284 int
mlx4_tx_queue_setup(struct rte_eth_dev * dev,uint16_t idx,uint16_t desc,unsigned int socket,const struct rte_eth_txconf * conf)285 mlx4_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
286 unsigned int socket, const struct rte_eth_txconf *conf)
287 {
288 struct mlx4_priv *priv = dev->data->dev_private;
289 struct mlx4dv_obj mlxdv;
290 struct mlx4dv_qp dv_qp;
291 struct mlx4dv_cq dv_cq;
292 struct txq_elt (*elts)[rte_align32pow2(desc)];
293 struct ibv_qp_init_attr qp_init_attr;
294 struct txq *txq;
295 uint8_t *bounce_buf;
296 struct mlx4_malloc_vec vec[] = {
297 {
298 .align = RTE_CACHE_LINE_SIZE,
299 .size = sizeof(*txq),
300 .addr = (void **)&txq,
301 },
302 {
303 .align = RTE_CACHE_LINE_SIZE,
304 .size = sizeof(*elts),
305 .addr = (void **)&elts,
306 },
307 {
308 .align = RTE_CACHE_LINE_SIZE,
309 .size = MLX4_MAX_WQE_SIZE,
310 .addr = (void **)&bounce_buf,
311 },
312 };
313 int ret;
314 uint64_t offloads;
315
316 offloads = conf->offloads | dev->data->dev_conf.txmode.offloads;
317 DEBUG("%p: configuring queue %u for %u descriptors",
318 (void *)dev, idx, desc);
319 if (idx >= dev->data->nb_tx_queues) {
320 rte_errno = EOVERFLOW;
321 ERROR("%p: queue index out of range (%u >= %u)",
322 (void *)dev, idx, dev->data->nb_tx_queues);
323 return -rte_errno;
324 }
325 txq = dev->data->tx_queues[idx];
326 if (txq) {
327 rte_errno = EEXIST;
328 DEBUG("%p: Tx queue %u already configured, release it first",
329 (void *)dev, idx);
330 return -rte_errno;
331 }
332 if (!desc) {
333 rte_errno = EINVAL;
334 ERROR("%p: invalid number of Tx descriptors", (void *)dev);
335 return -rte_errno;
336 }
337 if (desc != RTE_DIM(*elts)) {
338 desc = RTE_DIM(*elts);
339 WARN("%p: increased number of descriptors in Tx queue %u"
340 " to the next power of two (%u)",
341 (void *)dev, idx, desc);
342 }
343 /* Allocate and initialize Tx queue. */
344 mlx4_zmallocv_socket("TXQ", vec, RTE_DIM(vec), socket);
345 if (!txq) {
346 ERROR("%p: unable to allocate queue index %u",
347 (void *)dev, idx);
348 return -rte_errno;
349 }
350 *txq = (struct txq){
351 .priv = priv,
352 .port_id = dev->data->port_id,
353 .stats = {
354 .idx = idx,
355 },
356 .socket = socket,
357 .elts_n = desc,
358 .elts = elts,
359 .elts_head = 0,
360 .elts_tail = 0,
361 /*
362 * Request send completion every MLX4_PMD_TX_PER_COMP_REQ
363 * packets or at least 4 times per ring.
364 */
365 .elts_comp_cd =
366 RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4),
367 .elts_comp_cd_init =
368 RTE_MIN(MLX4_PMD_TX_PER_COMP_REQ, desc / 4),
369 .csum = priv->hw_csum &&
370 (offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
371 DEV_TX_OFFLOAD_UDP_CKSUM |
372 DEV_TX_OFFLOAD_TCP_CKSUM)),
373 .csum_l2tun = priv->hw_csum_l2tun &&
374 (offloads &
375 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM),
376 /* Enable Tx loopback for VF devices. */
377 .lb = !!priv->vf,
378 .bounce_buf = bounce_buf,
379 };
380 priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_TX_QUEUE;
381 priv->verbs_alloc_ctx.obj = txq;
382 txq->cq = mlx4_glue->create_cq(priv->ctx, desc, NULL, NULL, 0);
383 if (!txq->cq) {
384 rte_errno = ENOMEM;
385 ERROR("%p: CQ creation failure: %s",
386 (void *)dev, strerror(rte_errno));
387 goto error;
388 }
389 qp_init_attr = (struct ibv_qp_init_attr){
390 .send_cq = txq->cq,
391 .recv_cq = txq->cq,
392 .cap = {
393 .max_send_wr =
394 RTE_MIN(priv->device_attr.max_qp_wr, desc),
395 .max_send_sge = 1,
396 .max_inline_data = MLX4_PMD_MAX_INLINE,
397 },
398 .qp_type = IBV_QPT_RAW_PACKET,
399 /* No completion events must occur by default. */
400 .sq_sig_all = 0,
401 };
402 txq->qp = mlx4_glue->create_qp(priv->pd, &qp_init_attr);
403 if (!txq->qp) {
404 rte_errno = errno ? errno : EINVAL;
405 ERROR("%p: QP creation failure: %s",
406 (void *)dev, strerror(rte_errno));
407 goto error;
408 }
409 txq->max_inline = qp_init_attr.cap.max_inline_data;
410 ret = mlx4_glue->modify_qp
411 (txq->qp,
412 &(struct ibv_qp_attr){
413 .qp_state = IBV_QPS_INIT,
414 .port_num = priv->port,
415 },
416 IBV_QP_STATE | IBV_QP_PORT);
417 if (ret) {
418 rte_errno = ret;
419 ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
420 (void *)dev, strerror(rte_errno));
421 goto error;
422 }
423 ret = mlx4_glue->modify_qp
424 (txq->qp,
425 &(struct ibv_qp_attr){
426 .qp_state = IBV_QPS_RTR,
427 },
428 IBV_QP_STATE);
429 if (ret) {
430 rte_errno = ret;
431 ERROR("%p: QP state to IBV_QPS_RTR failed: %s",
432 (void *)dev, strerror(rte_errno));
433 goto error;
434 }
435 ret = mlx4_glue->modify_qp
436 (txq->qp,
437 &(struct ibv_qp_attr){
438 .qp_state = IBV_QPS_RTS,
439 },
440 IBV_QP_STATE);
441 if (ret) {
442 rte_errno = ret;
443 ERROR("%p: QP state to IBV_QPS_RTS failed: %s",
444 (void *)dev, strerror(rte_errno));
445 goto error;
446 }
447 /* Retrieve device queue information. */
448 #ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET
449 dv_qp = (struct mlx4dv_qp){
450 .comp_mask = MLX4DV_QP_MASK_UAR_MMAP_OFFSET,
451 };
452 #endif
453 mlxdv.cq.in = txq->cq;
454 mlxdv.cq.out = &dv_cq;
455 mlxdv.qp.in = txq->qp;
456 mlxdv.qp.out = &dv_qp;
457 ret = mlx4_glue->dv_init_obj(&mlxdv, MLX4DV_OBJ_QP | MLX4DV_OBJ_CQ);
458 if (ret) {
459 rte_errno = EINVAL;
460 ERROR("%p: failed to obtain information needed for"
461 " accessing the device queues", (void *)dev);
462 goto error;
463 }
464 #ifdef HAVE_IBV_MLX4_UAR_MMAP_OFFSET
465 if (!(dv_qp.comp_mask & MLX4DV_QP_MASK_UAR_MMAP_OFFSET)) {
466 WARN("%p: failed to obtain UAR mmap offset", (void *)dev);
467 dv_qp.uar_mmap_offset = -1; /* Make mmap() fail. */
468 }
469 #endif
470 mlx4_txq_fill_dv_obj_info(txq, &mlxdv);
471 txq_uar_init(txq);
472 /* Save first wqe pointer in the first element. */
473 (&(*txq->elts)[0])->wqe =
474 (volatile struct mlx4_wqe_ctrl_seg *)txq->msq.buf;
475 if (mlx4_mr_btree_init(&txq->mr_ctrl.cache_bh,
476 MLX4_MR_BTREE_CACHE_N, socket)) {
477 /* rte_errno is already set. */
478 goto error;
479 }
480 /* Save pointer of global generation number to check memory event. */
481 txq->mr_ctrl.dev_gen_ptr = &priv->mr.dev_gen;
482 DEBUG("%p: adding Tx queue %p to list", (void *)dev, (void *)txq);
483 dev->data->tx_queues[idx] = txq;
484 priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE;
485 return 0;
486 error:
487 dev->data->tx_queues[idx] = NULL;
488 ret = rte_errno;
489 mlx4_tx_queue_release(txq);
490 rte_errno = ret;
491 MLX4_ASSERT(rte_errno > 0);
492 priv->verbs_alloc_ctx.type = MLX4_VERBS_ALLOC_TYPE_NONE;
493 return -rte_errno;
494 }
495
496 /**
497 * DPDK callback to release a Tx queue.
498 *
499 * @param dpdk_txq
500 * Generic Tx queue pointer.
501 */
502 void
mlx4_tx_queue_release(void * dpdk_txq)503 mlx4_tx_queue_release(void *dpdk_txq)
504 {
505 struct txq *txq = (struct txq *)dpdk_txq;
506 struct mlx4_priv *priv;
507 unsigned int i;
508
509 if (txq == NULL)
510 return;
511 priv = txq->priv;
512 for (i = 0; i != ETH_DEV(priv)->data->nb_tx_queues; ++i)
513 if (ETH_DEV(priv)->data->tx_queues[i] == txq) {
514 DEBUG("%p: removing Tx queue %p from list",
515 (void *)ETH_DEV(priv), (void *)txq);
516 ETH_DEV(priv)->data->tx_queues[i] = NULL;
517 break;
518 }
519 mlx4_txq_free_elts(txq);
520 if (txq->qp)
521 claim_zero(mlx4_glue->destroy_qp(txq->qp));
522 if (txq->cq)
523 claim_zero(mlx4_glue->destroy_cq(txq->cq));
524 mlx4_mr_btree_free(&txq->mr_ctrl.cache_bh);
525 rte_free(txq);
526 }
527