1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2015 6WIND S.A.
3 * Copyright 2015 Mellanox Technologies, Ltd
4 */
5
6 #include <unistd.h>
7
8 #include <rte_ether.h>
9 #include <rte_ethdev_driver.h>
10 #include <rte_interrupts.h>
11 #include <rte_alarm.h>
12
13 #include <mlx5_malloc.h>
14
15 #include "mlx5.h"
16 #include "mlx5_mr.h"
17 #include "mlx5_rxtx.h"
18 #include "mlx5_utils.h"
19 #include "rte_pmd_mlx5.h"
20
21 /**
22 * Stop traffic on Tx queues.
23 *
24 * @param dev
25 * Pointer to Ethernet device structure.
26 */
27 static void
mlx5_txq_stop(struct rte_eth_dev * dev)28 mlx5_txq_stop(struct rte_eth_dev *dev)
29 {
30 struct mlx5_priv *priv = dev->data->dev_private;
31 unsigned int i;
32
33 for (i = 0; i != priv->txqs_n; ++i)
34 mlx5_txq_release(dev, i);
35 }
36
37 /**
38 * Start traffic on Tx queues.
39 *
40 * @param dev
41 * Pointer to Ethernet device structure.
42 *
43 * @return
44 * 0 on success, a negative errno value otherwise and rte_errno is set.
45 */
46 static int
mlx5_txq_start(struct rte_eth_dev * dev)47 mlx5_txq_start(struct rte_eth_dev *dev)
48 {
49 struct mlx5_priv *priv = dev->data->dev_private;
50 unsigned int i;
51 int ret;
52
53 for (i = 0; i != priv->txqs_n; ++i) {
54 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
55 struct mlx5_txq_data *txq_data = &txq_ctrl->txq;
56 uint32_t flags = MLX5_MEM_RTE | MLX5_MEM_ZERO;
57
58 if (!txq_ctrl)
59 continue;
60 if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD)
61 txq_alloc_elts(txq_ctrl);
62 MLX5_ASSERT(!txq_ctrl->obj);
63 txq_ctrl->obj = mlx5_malloc(flags, sizeof(struct mlx5_txq_obj),
64 0, txq_ctrl->socket);
65 if (!txq_ctrl->obj) {
66 DRV_LOG(ERR, "Port %u Tx queue %u cannot allocate "
67 "memory resources.", dev->data->port_id,
68 txq_data->idx);
69 rte_errno = ENOMEM;
70 goto error;
71 }
72 ret = priv->obj_ops.txq_obj_new(dev, i);
73 if (ret < 0) {
74 mlx5_free(txq_ctrl->obj);
75 txq_ctrl->obj = NULL;
76 goto error;
77 }
78 if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD) {
79 size_t size = txq_data->cqe_s * sizeof(*txq_data->fcqs);
80
81 txq_data->fcqs = mlx5_malloc(flags, size,
82 RTE_CACHE_LINE_SIZE,
83 txq_ctrl->socket);
84 if (!txq_data->fcqs) {
85 DRV_LOG(ERR, "Port %u Tx queue %u cannot "
86 "allocate memory (FCQ).",
87 dev->data->port_id, i);
88 rte_errno = ENOMEM;
89 goto error;
90 }
91 }
92 DRV_LOG(DEBUG, "Port %u txq %u updated with %p.",
93 dev->data->port_id, i, (void *)&txq_ctrl->obj);
94 LIST_INSERT_HEAD(&priv->txqsobj, txq_ctrl->obj, next);
95 }
96 return 0;
97 error:
98 ret = rte_errno; /* Save rte_errno before cleanup. */
99 do {
100 mlx5_txq_release(dev, i);
101 } while (i-- != 0);
102 rte_errno = ret; /* Restore rte_errno. */
103 return -rte_errno;
104 }
105
106 /**
107 * Stop traffic on Rx queues.
108 *
109 * @param dev
110 * Pointer to Ethernet device structure.
111 */
112 static void
mlx5_rxq_stop(struct rte_eth_dev * dev)113 mlx5_rxq_stop(struct rte_eth_dev *dev)
114 {
115 struct mlx5_priv *priv = dev->data->dev_private;
116 unsigned int i;
117
118 for (i = 0; i != priv->rxqs_n; ++i)
119 mlx5_rxq_release(dev, i);
120 }
121
122 /**
123 * Start traffic on Rx queues.
124 *
125 * @param dev
126 * Pointer to Ethernet device structure.
127 *
128 * @return
129 * 0 on success, a negative errno value otherwise and rte_errno is set.
130 */
131 static int
mlx5_rxq_start(struct rte_eth_dev * dev)132 mlx5_rxq_start(struct rte_eth_dev *dev)
133 {
134 struct mlx5_priv *priv = dev->data->dev_private;
135 unsigned int i;
136 int ret = 0;
137
138 /* Allocate/reuse/resize mempool for Multi-Packet RQ. */
139 if (mlx5_mprq_alloc_mp(dev)) {
140 /* Should not release Rx queues but return immediately. */
141 return -rte_errno;
142 }
143 DRV_LOG(DEBUG, "Port %u device_attr.max_qp_wr is %d.",
144 dev->data->port_id, priv->sh->device_attr.max_qp_wr);
145 DRV_LOG(DEBUG, "Port %u device_attr.max_sge is %d.",
146 dev->data->port_id, priv->sh->device_attr.max_sge);
147 for (i = 0; i != priv->rxqs_n; ++i) {
148 struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
149
150 if (!rxq_ctrl)
151 continue;
152 if (rxq_ctrl->type == MLX5_RXQ_TYPE_STANDARD) {
153 /* Pre-register Rx mempools. */
154 if (mlx5_rxq_mprq_enabled(&rxq_ctrl->rxq)) {
155 mlx5_mr_update_mp(dev, &rxq_ctrl->rxq.mr_ctrl,
156 rxq_ctrl->rxq.mprq_mp);
157 } else {
158 uint32_t s;
159
160 for (s = 0; s < rxq_ctrl->rxq.rxseg_n; s++)
161 mlx5_mr_update_mp
162 (dev, &rxq_ctrl->rxq.mr_ctrl,
163 rxq_ctrl->rxq.rxseg[s].mp);
164 }
165 ret = rxq_alloc_elts(rxq_ctrl);
166 if (ret)
167 goto error;
168 }
169 MLX5_ASSERT(!rxq_ctrl->obj);
170 rxq_ctrl->obj = mlx5_malloc(MLX5_MEM_RTE | MLX5_MEM_ZERO,
171 sizeof(*rxq_ctrl->obj), 0,
172 rxq_ctrl->socket);
173 if (!rxq_ctrl->obj) {
174 DRV_LOG(ERR,
175 "Port %u Rx queue %u can't allocate resources.",
176 dev->data->port_id, (*priv->rxqs)[i]->idx);
177 rte_errno = ENOMEM;
178 goto error;
179 }
180 ret = priv->obj_ops.rxq_obj_new(dev, i);
181 if (ret) {
182 mlx5_free(rxq_ctrl->obj);
183 goto error;
184 }
185 DRV_LOG(DEBUG, "Port %u rxq %u updated with %p.",
186 dev->data->port_id, i, (void *)&rxq_ctrl->obj);
187 LIST_INSERT_HEAD(&priv->rxqsobj, rxq_ctrl->obj, next);
188 }
189 return 0;
190 error:
191 ret = rte_errno; /* Save rte_errno before cleanup. */
192 do {
193 mlx5_rxq_release(dev, i);
194 } while (i-- != 0);
195 rte_errno = ret; /* Restore rte_errno. */
196 return -rte_errno;
197 }
198
199 /**
200 * Binds Tx queues to Rx queues for hairpin.
201 *
202 * Binds Tx queues to the target Rx queues.
203 *
204 * @param dev
205 * Pointer to Ethernet device structure.
206 *
207 * @return
208 * 0 on success, a negative errno value otherwise and rte_errno is set.
209 */
210 static int
mlx5_hairpin_auto_bind(struct rte_eth_dev * dev)211 mlx5_hairpin_auto_bind(struct rte_eth_dev *dev)
212 {
213 struct mlx5_priv *priv = dev->data->dev_private;
214 struct mlx5_devx_modify_sq_attr sq_attr = { 0 };
215 struct mlx5_devx_modify_rq_attr rq_attr = { 0 };
216 struct mlx5_txq_ctrl *txq_ctrl;
217 struct mlx5_rxq_ctrl *rxq_ctrl;
218 struct mlx5_devx_obj *sq;
219 struct mlx5_devx_obj *rq;
220 unsigned int i;
221 int ret = 0;
222 bool need_auto = false;
223 uint16_t self_port = dev->data->port_id;
224
225 for (i = 0; i != priv->txqs_n; ++i) {
226 txq_ctrl = mlx5_txq_get(dev, i);
227 if (!txq_ctrl)
228 continue;
229 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
230 mlx5_txq_release(dev, i);
231 continue;
232 }
233 if (txq_ctrl->hairpin_conf.peers[0].port != self_port)
234 continue;
235 if (txq_ctrl->hairpin_conf.manual_bind) {
236 mlx5_txq_release(dev, i);
237 return 0;
238 }
239 need_auto = true;
240 mlx5_txq_release(dev, i);
241 }
242 if (!need_auto)
243 return 0;
244 for (i = 0; i != priv->txqs_n; ++i) {
245 txq_ctrl = mlx5_txq_get(dev, i);
246 if (!txq_ctrl)
247 continue;
248 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
249 mlx5_txq_release(dev, i);
250 continue;
251 }
252 /* Skip hairpin queues with other peer ports. */
253 if (txq_ctrl->hairpin_conf.peers[0].port != self_port)
254 continue;
255 if (!txq_ctrl->obj) {
256 rte_errno = ENOMEM;
257 DRV_LOG(ERR, "port %u no txq object found: %d",
258 dev->data->port_id, i);
259 mlx5_txq_release(dev, i);
260 return -rte_errno;
261 }
262 sq = txq_ctrl->obj->sq;
263 rxq_ctrl = mlx5_rxq_get(dev,
264 txq_ctrl->hairpin_conf.peers[0].queue);
265 if (!rxq_ctrl) {
266 mlx5_txq_release(dev, i);
267 rte_errno = EINVAL;
268 DRV_LOG(ERR, "port %u no rxq object found: %d",
269 dev->data->port_id,
270 txq_ctrl->hairpin_conf.peers[0].queue);
271 return -rte_errno;
272 }
273 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN ||
274 rxq_ctrl->hairpin_conf.peers[0].queue != i) {
275 rte_errno = ENOMEM;
276 DRV_LOG(ERR, "port %u Tx queue %d can't be binded to "
277 "Rx queue %d", dev->data->port_id,
278 i, txq_ctrl->hairpin_conf.peers[0].queue);
279 goto error;
280 }
281 rq = rxq_ctrl->obj->rq;
282 if (!rq) {
283 rte_errno = ENOMEM;
284 DRV_LOG(ERR, "port %u hairpin no matching rxq: %d",
285 dev->data->port_id,
286 txq_ctrl->hairpin_conf.peers[0].queue);
287 goto error;
288 }
289 sq_attr.state = MLX5_SQC_STATE_RDY;
290 sq_attr.sq_state = MLX5_SQC_STATE_RST;
291 sq_attr.hairpin_peer_rq = rq->id;
292 sq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id;
293 ret = mlx5_devx_cmd_modify_sq(sq, &sq_attr);
294 if (ret)
295 goto error;
296 rq_attr.state = MLX5_SQC_STATE_RDY;
297 rq_attr.rq_state = MLX5_SQC_STATE_RST;
298 rq_attr.hairpin_peer_sq = sq->id;
299 rq_attr.hairpin_peer_vhca = priv->config.hca_attr.vhca_id;
300 ret = mlx5_devx_cmd_modify_rq(rq, &rq_attr);
301 if (ret)
302 goto error;
303 /* Qs with auto-bind will be destroyed directly. */
304 rxq_ctrl->hairpin_status = 1;
305 txq_ctrl->hairpin_status = 1;
306 mlx5_txq_release(dev, i);
307 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue);
308 }
309 return 0;
310 error:
311 mlx5_txq_release(dev, i);
312 mlx5_rxq_release(dev, txq_ctrl->hairpin_conf.peers[0].queue);
313 return -rte_errno;
314 }
315
316 /*
317 * Fetch the peer queue's SW & HW information.
318 *
319 * @param dev
320 * Pointer to Ethernet device structure.
321 * @param peer_queue
322 * Index of the queue to fetch the information.
323 * @param current_info
324 * Pointer to the input peer information, not used currently.
325 * @param peer_info
326 * Pointer to the structure to store the information, output.
327 * @param direction
328 * Positive to get the RxQ information, zero to get the TxQ information.
329 *
330 * @return
331 * 0 on success, a negative errno value otherwise and rte_errno is set.
332 */
333 int
mlx5_hairpin_queue_peer_update(struct rte_eth_dev * dev,uint16_t peer_queue,struct rte_hairpin_peer_info * current_info,struct rte_hairpin_peer_info * peer_info,uint32_t direction)334 mlx5_hairpin_queue_peer_update(struct rte_eth_dev *dev, uint16_t peer_queue,
335 struct rte_hairpin_peer_info *current_info,
336 struct rte_hairpin_peer_info *peer_info,
337 uint32_t direction)
338 {
339 struct mlx5_priv *priv = dev->data->dev_private;
340 RTE_SET_USED(current_info);
341
342 if (dev->data->dev_started == 0) {
343 rte_errno = EBUSY;
344 DRV_LOG(ERR, "peer port %u is not started",
345 dev->data->port_id);
346 return -rte_errno;
347 }
348 /*
349 * Peer port used as egress. In the current design, hairpin Tx queue
350 * will be bound to the peer Rx queue. Indeed, only the information of
351 * peer Rx queue needs to be fetched.
352 */
353 if (direction == 0) {
354 struct mlx5_txq_ctrl *txq_ctrl;
355
356 txq_ctrl = mlx5_txq_get(dev, peer_queue);
357 if (txq_ctrl == NULL) {
358 rte_errno = EINVAL;
359 DRV_LOG(ERR, "Failed to get port %u Tx queue %d",
360 dev->data->port_id, peer_queue);
361 return -rte_errno;
362 }
363 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
364 rte_errno = EINVAL;
365 DRV_LOG(ERR, "port %u queue %d is not a hairpin Txq",
366 dev->data->port_id, peer_queue);
367 mlx5_txq_release(dev, peer_queue);
368 return -rte_errno;
369 }
370 if (txq_ctrl->obj == NULL || txq_ctrl->obj->sq == NULL) {
371 rte_errno = ENOMEM;
372 DRV_LOG(ERR, "port %u no Txq object found: %d",
373 dev->data->port_id, peer_queue);
374 mlx5_txq_release(dev, peer_queue);
375 return -rte_errno;
376 }
377 peer_info->qp_id = txq_ctrl->obj->sq->id;
378 peer_info->vhca_id = priv->config.hca_attr.vhca_id;
379 /* 1-to-1 mapping, only the first one is used. */
380 peer_info->peer_q = txq_ctrl->hairpin_conf.peers[0].queue;
381 peer_info->tx_explicit = txq_ctrl->hairpin_conf.tx_explicit;
382 peer_info->manual_bind = txq_ctrl->hairpin_conf.manual_bind;
383 mlx5_txq_release(dev, peer_queue);
384 } else { /* Peer port used as ingress. */
385 struct mlx5_rxq_ctrl *rxq_ctrl;
386
387 rxq_ctrl = mlx5_rxq_get(dev, peer_queue);
388 if (rxq_ctrl == NULL) {
389 rte_errno = EINVAL;
390 DRV_LOG(ERR, "Failed to get port %u Rx queue %d",
391 dev->data->port_id, peer_queue);
392 return -rte_errno;
393 }
394 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) {
395 rte_errno = EINVAL;
396 DRV_LOG(ERR, "port %u queue %d is not a hairpin Rxq",
397 dev->data->port_id, peer_queue);
398 mlx5_rxq_release(dev, peer_queue);
399 return -rte_errno;
400 }
401 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) {
402 rte_errno = ENOMEM;
403 DRV_LOG(ERR, "port %u no Rxq object found: %d",
404 dev->data->port_id, peer_queue);
405 mlx5_rxq_release(dev, peer_queue);
406 return -rte_errno;
407 }
408 peer_info->qp_id = rxq_ctrl->obj->rq->id;
409 peer_info->vhca_id = priv->config.hca_attr.vhca_id;
410 peer_info->peer_q = rxq_ctrl->hairpin_conf.peers[0].queue;
411 peer_info->tx_explicit = rxq_ctrl->hairpin_conf.tx_explicit;
412 peer_info->manual_bind = rxq_ctrl->hairpin_conf.manual_bind;
413 mlx5_rxq_release(dev, peer_queue);
414 }
415 return 0;
416 }
417
418 /*
419 * Bind the hairpin queue with the peer HW information.
420 * This needs to be called twice both for Tx and Rx queues of a pair.
421 * If the queue is already bound, it is considered successful.
422 *
423 * @param dev
424 * Pointer to Ethernet device structure.
425 * @param cur_queue
426 * Index of the queue to change the HW configuration to bind.
427 * @param peer_info
428 * Pointer to information of the peer queue.
429 * @param direction
430 * Positive to configure the TxQ, zero to configure the RxQ.
431 *
432 * @return
433 * 0 on success, a negative errno value otherwise and rte_errno is set.
434 */
435 int
mlx5_hairpin_queue_peer_bind(struct rte_eth_dev * dev,uint16_t cur_queue,struct rte_hairpin_peer_info * peer_info,uint32_t direction)436 mlx5_hairpin_queue_peer_bind(struct rte_eth_dev *dev, uint16_t cur_queue,
437 struct rte_hairpin_peer_info *peer_info,
438 uint32_t direction)
439 {
440 int ret = 0;
441
442 /*
443 * Consistency checking of the peer queue: opposite direction is used
444 * to get the peer queue info with ethdev port ID, no need to check.
445 */
446 if (peer_info->peer_q != cur_queue) {
447 rte_errno = EINVAL;
448 DRV_LOG(ERR, "port %u queue %d and peer queue %d mismatch",
449 dev->data->port_id, cur_queue, peer_info->peer_q);
450 return -rte_errno;
451 }
452 if (direction != 0) {
453 struct mlx5_txq_ctrl *txq_ctrl;
454 struct mlx5_devx_modify_sq_attr sq_attr = { 0 };
455
456 txq_ctrl = mlx5_txq_get(dev, cur_queue);
457 if (txq_ctrl == NULL) {
458 rte_errno = EINVAL;
459 DRV_LOG(ERR, "Failed to get port %u Tx queue %d",
460 dev->data->port_id, cur_queue);
461 return -rte_errno;
462 }
463 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
464 rte_errno = EINVAL;
465 DRV_LOG(ERR, "port %u queue %d not a hairpin Txq",
466 dev->data->port_id, cur_queue);
467 mlx5_txq_release(dev, cur_queue);
468 return -rte_errno;
469 }
470 if (txq_ctrl->obj == NULL || txq_ctrl->obj->sq == NULL) {
471 rte_errno = ENOMEM;
472 DRV_LOG(ERR, "port %u no Txq object found: %d",
473 dev->data->port_id, cur_queue);
474 mlx5_txq_release(dev, cur_queue);
475 return -rte_errno;
476 }
477 if (txq_ctrl->hairpin_status != 0) {
478 DRV_LOG(DEBUG, "port %u Tx queue %d is already bound",
479 dev->data->port_id, cur_queue);
480 mlx5_txq_release(dev, cur_queue);
481 return 0;
482 }
483 /*
484 * All queues' of one port consistency checking is done in the
485 * bind() function, and that is optional.
486 */
487 if (peer_info->tx_explicit !=
488 txq_ctrl->hairpin_conf.tx_explicit) {
489 rte_errno = EINVAL;
490 DRV_LOG(ERR, "port %u Tx queue %d and peer Tx rule mode"
491 " mismatch", dev->data->port_id, cur_queue);
492 mlx5_txq_release(dev, cur_queue);
493 return -rte_errno;
494 }
495 if (peer_info->manual_bind !=
496 txq_ctrl->hairpin_conf.manual_bind) {
497 rte_errno = EINVAL;
498 DRV_LOG(ERR, "port %u Tx queue %d and peer binding mode"
499 " mismatch", dev->data->port_id, cur_queue);
500 mlx5_txq_release(dev, cur_queue);
501 return -rte_errno;
502 }
503 sq_attr.state = MLX5_SQC_STATE_RDY;
504 sq_attr.sq_state = MLX5_SQC_STATE_RST;
505 sq_attr.hairpin_peer_rq = peer_info->qp_id;
506 sq_attr.hairpin_peer_vhca = peer_info->vhca_id;
507 ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq, &sq_attr);
508 if (ret == 0)
509 txq_ctrl->hairpin_status = 1;
510 mlx5_txq_release(dev, cur_queue);
511 } else {
512 struct mlx5_rxq_ctrl *rxq_ctrl;
513 struct mlx5_devx_modify_rq_attr rq_attr = { 0 };
514
515 rxq_ctrl = mlx5_rxq_get(dev, cur_queue);
516 if (rxq_ctrl == NULL) {
517 rte_errno = EINVAL;
518 DRV_LOG(ERR, "Failed to get port %u Rx queue %d",
519 dev->data->port_id, cur_queue);
520 return -rte_errno;
521 }
522 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) {
523 rte_errno = EINVAL;
524 DRV_LOG(ERR, "port %u queue %d not a hairpin Rxq",
525 dev->data->port_id, cur_queue);
526 mlx5_rxq_release(dev, cur_queue);
527 return -rte_errno;
528 }
529 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) {
530 rte_errno = ENOMEM;
531 DRV_LOG(ERR, "port %u no Rxq object found: %d",
532 dev->data->port_id, cur_queue);
533 mlx5_rxq_release(dev, cur_queue);
534 return -rte_errno;
535 }
536 if (rxq_ctrl->hairpin_status != 0) {
537 DRV_LOG(DEBUG, "port %u Rx queue %d is already bound",
538 dev->data->port_id, cur_queue);
539 mlx5_rxq_release(dev, cur_queue);
540 return 0;
541 }
542 if (peer_info->tx_explicit !=
543 rxq_ctrl->hairpin_conf.tx_explicit) {
544 rte_errno = EINVAL;
545 DRV_LOG(ERR, "port %u Rx queue %d and peer Tx rule mode"
546 " mismatch", dev->data->port_id, cur_queue);
547 mlx5_rxq_release(dev, cur_queue);
548 return -rte_errno;
549 }
550 if (peer_info->manual_bind !=
551 rxq_ctrl->hairpin_conf.manual_bind) {
552 rte_errno = EINVAL;
553 DRV_LOG(ERR, "port %u Rx queue %d and peer binding mode"
554 " mismatch", dev->data->port_id, cur_queue);
555 mlx5_rxq_release(dev, cur_queue);
556 return -rte_errno;
557 }
558 rq_attr.state = MLX5_SQC_STATE_RDY;
559 rq_attr.rq_state = MLX5_SQC_STATE_RST;
560 rq_attr.hairpin_peer_sq = peer_info->qp_id;
561 rq_attr.hairpin_peer_vhca = peer_info->vhca_id;
562 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr);
563 if (ret == 0)
564 rxq_ctrl->hairpin_status = 1;
565 mlx5_rxq_release(dev, cur_queue);
566 }
567 return ret;
568 }
569
570 /*
571 * Unbind the hairpin queue and reset its HW configuration.
572 * This needs to be called twice both for Tx and Rx queues of a pair.
573 * If the queue is already unbound, it is considered successful.
574 *
575 * @param dev
576 * Pointer to Ethernet device structure.
577 * @param cur_queue
578 * Index of the queue to change the HW configuration to unbind.
579 * @param direction
580 * Positive to reset the TxQ, zero to reset the RxQ.
581 *
582 * @return
583 * 0 on success, a negative errno value otherwise and rte_errno is set.
584 */
585 int
mlx5_hairpin_queue_peer_unbind(struct rte_eth_dev * dev,uint16_t cur_queue,uint32_t direction)586 mlx5_hairpin_queue_peer_unbind(struct rte_eth_dev *dev, uint16_t cur_queue,
587 uint32_t direction)
588 {
589 int ret = 0;
590
591 if (direction != 0) {
592 struct mlx5_txq_ctrl *txq_ctrl;
593 struct mlx5_devx_modify_sq_attr sq_attr = { 0 };
594
595 txq_ctrl = mlx5_txq_get(dev, cur_queue);
596 if (txq_ctrl == NULL) {
597 rte_errno = EINVAL;
598 DRV_LOG(ERR, "Failed to get port %u Tx queue %d",
599 dev->data->port_id, cur_queue);
600 return -rte_errno;
601 }
602 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
603 rte_errno = EINVAL;
604 DRV_LOG(ERR, "port %u queue %d not a hairpin Txq",
605 dev->data->port_id, cur_queue);
606 mlx5_txq_release(dev, cur_queue);
607 return -rte_errno;
608 }
609 /* Already unbound, return success before obj checking. */
610 if (txq_ctrl->hairpin_status == 0) {
611 DRV_LOG(DEBUG, "port %u Tx queue %d is already unbound",
612 dev->data->port_id, cur_queue);
613 mlx5_txq_release(dev, cur_queue);
614 return 0;
615 }
616 if (!txq_ctrl->obj || !txq_ctrl->obj->sq) {
617 rte_errno = ENOMEM;
618 DRV_LOG(ERR, "port %u no Txq object found: %d",
619 dev->data->port_id, cur_queue);
620 mlx5_txq_release(dev, cur_queue);
621 return -rte_errno;
622 }
623 sq_attr.state = MLX5_SQC_STATE_RST;
624 sq_attr.sq_state = MLX5_SQC_STATE_RST;
625 ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq, &sq_attr);
626 if (ret == 0)
627 txq_ctrl->hairpin_status = 0;
628 mlx5_txq_release(dev, cur_queue);
629 } else {
630 struct mlx5_rxq_ctrl *rxq_ctrl;
631 struct mlx5_devx_modify_rq_attr rq_attr = { 0 };
632
633 rxq_ctrl = mlx5_rxq_get(dev, cur_queue);
634 if (rxq_ctrl == NULL) {
635 rte_errno = EINVAL;
636 DRV_LOG(ERR, "Failed to get port %u Rx queue %d",
637 dev->data->port_id, cur_queue);
638 return -rte_errno;
639 }
640 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) {
641 rte_errno = EINVAL;
642 DRV_LOG(ERR, "port %u queue %d not a hairpin Rxq",
643 dev->data->port_id, cur_queue);
644 mlx5_rxq_release(dev, cur_queue);
645 return -rte_errno;
646 }
647 if (rxq_ctrl->hairpin_status == 0) {
648 DRV_LOG(DEBUG, "port %u Rx queue %d is already unbound",
649 dev->data->port_id, cur_queue);
650 mlx5_rxq_release(dev, cur_queue);
651 return 0;
652 }
653 if (rxq_ctrl->obj == NULL || rxq_ctrl->obj->rq == NULL) {
654 rte_errno = ENOMEM;
655 DRV_LOG(ERR, "port %u no Rxq object found: %d",
656 dev->data->port_id, cur_queue);
657 mlx5_rxq_release(dev, cur_queue);
658 return -rte_errno;
659 }
660 rq_attr.state = MLX5_SQC_STATE_RST;
661 rq_attr.rq_state = MLX5_SQC_STATE_RST;
662 ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr);
663 if (ret == 0)
664 rxq_ctrl->hairpin_status = 0;
665 mlx5_rxq_release(dev, cur_queue);
666 }
667 return ret;
668 }
669
670 /*
671 * Bind the hairpin port pairs, from the Tx to the peer Rx.
672 * This function only supports to bind the Tx to one Rx.
673 *
674 * @param dev
675 * Pointer to Ethernet device structure.
676 * @param rx_port
677 * Port identifier of the Rx port.
678 *
679 * @return
680 * 0 on success, a negative errno value otherwise and rte_errno is set.
681 */
682 static int
mlx5_hairpin_bind_single_port(struct rte_eth_dev * dev,uint16_t rx_port)683 mlx5_hairpin_bind_single_port(struct rte_eth_dev *dev, uint16_t rx_port)
684 {
685 struct mlx5_priv *priv = dev->data->dev_private;
686 int ret = 0;
687 struct mlx5_txq_ctrl *txq_ctrl;
688 uint32_t i;
689 struct rte_hairpin_peer_info peer = {0xffffff};
690 struct rte_hairpin_peer_info cur;
691 const struct rte_eth_hairpin_conf *conf;
692 uint16_t num_q = 0;
693 uint16_t local_port = priv->dev_data->port_id;
694 uint32_t manual;
695 uint32_t explicit;
696 uint16_t rx_queue;
697
698 if (mlx5_eth_find_next(rx_port, priv->pci_dev) != rx_port) {
699 rte_errno = ENODEV;
700 DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port);
701 return -rte_errno;
702 }
703 /*
704 * Before binding TxQ to peer RxQ, first round loop will be used for
705 * checking the queues' configuration consistency. This would be a
706 * little time consuming but better than doing the rollback.
707 */
708 for (i = 0; i != priv->txqs_n; i++) {
709 txq_ctrl = mlx5_txq_get(dev, i);
710 if (txq_ctrl == NULL)
711 continue;
712 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
713 mlx5_txq_release(dev, i);
714 continue;
715 }
716 /*
717 * All hairpin Tx queues of a single port that connected to the
718 * same peer Rx port should have the same "auto binding" and
719 * "implicit Tx flow" modes.
720 * Peer consistency checking will be done in per queue binding.
721 */
722 conf = &txq_ctrl->hairpin_conf;
723 if (conf->peers[0].port == rx_port) {
724 if (num_q == 0) {
725 manual = conf->manual_bind;
726 explicit = conf->tx_explicit;
727 } else {
728 if (manual != conf->manual_bind ||
729 explicit != conf->tx_explicit) {
730 rte_errno = EINVAL;
731 DRV_LOG(ERR, "port %u queue %d mode"
732 " mismatch: %u %u, %u %u",
733 local_port, i, manual,
734 conf->manual_bind, explicit,
735 conf->tx_explicit);
736 mlx5_txq_release(dev, i);
737 return -rte_errno;
738 }
739 }
740 num_q++;
741 }
742 mlx5_txq_release(dev, i);
743 }
744 /* Once no queue is configured, success is returned directly. */
745 if (num_q == 0)
746 return ret;
747 /* All the hairpin TX queues need to be traversed again. */
748 for (i = 0; i != priv->txqs_n; i++) {
749 txq_ctrl = mlx5_txq_get(dev, i);
750 if (txq_ctrl == NULL)
751 continue;
752 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
753 mlx5_txq_release(dev, i);
754 continue;
755 }
756 if (txq_ctrl->hairpin_conf.peers[0].port != rx_port) {
757 mlx5_txq_release(dev, i);
758 continue;
759 }
760 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue;
761 /*
762 * Fetch peer RxQ's information.
763 * No need to pass the information of the current queue.
764 */
765 ret = rte_eth_hairpin_queue_peer_update(rx_port, rx_queue,
766 NULL, &peer, 1);
767 if (ret != 0) {
768 mlx5_txq_release(dev, i);
769 goto error;
770 }
771 /* Accessing its own device, inside mlx5 PMD. */
772 ret = mlx5_hairpin_queue_peer_bind(dev, i, &peer, 1);
773 if (ret != 0) {
774 mlx5_txq_release(dev, i);
775 goto error;
776 }
777 /* Pass TxQ's information to peer RxQ and try binding. */
778 cur.peer_q = rx_queue;
779 cur.qp_id = txq_ctrl->obj->sq->id;
780 cur.vhca_id = priv->config.hca_attr.vhca_id;
781 cur.tx_explicit = txq_ctrl->hairpin_conf.tx_explicit;
782 cur.manual_bind = txq_ctrl->hairpin_conf.manual_bind;
783 /*
784 * In order to access another device in a proper way, RTE level
785 * private function is needed.
786 */
787 ret = rte_eth_hairpin_queue_peer_bind(rx_port, rx_queue,
788 &cur, 0);
789 if (ret != 0) {
790 mlx5_txq_release(dev, i);
791 goto error;
792 }
793 mlx5_txq_release(dev, i);
794 }
795 return 0;
796 error:
797 /*
798 * Do roll-back process for the queues already bound.
799 * No need to check the return value of the queue unbind function.
800 */
801 do {
802 /* No validation is needed here. */
803 txq_ctrl = mlx5_txq_get(dev, i);
804 if (txq_ctrl == NULL)
805 continue;
806 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue;
807 rte_eth_hairpin_queue_peer_unbind(rx_port, rx_queue, 0);
808 mlx5_hairpin_queue_peer_unbind(dev, i, 1);
809 mlx5_txq_release(dev, i);
810 } while (i--);
811 return ret;
812 }
813
814 /*
815 * Unbind the hairpin port pair, HW configuration of both devices will be clear
816 * and status will be reset for all the queues used between the them.
817 * This function only supports to unbind the Tx from one Rx.
818 *
819 * @param dev
820 * Pointer to Ethernet device structure.
821 * @param rx_port
822 * Port identifier of the Rx port.
823 *
824 * @return
825 * 0 on success, a negative errno value otherwise and rte_errno is set.
826 */
827 static int
mlx5_hairpin_unbind_single_port(struct rte_eth_dev * dev,uint16_t rx_port)828 mlx5_hairpin_unbind_single_port(struct rte_eth_dev *dev, uint16_t rx_port)
829 {
830 struct mlx5_priv *priv = dev->data->dev_private;
831 struct mlx5_txq_ctrl *txq_ctrl;
832 uint32_t i;
833 int ret;
834 uint16_t cur_port = priv->dev_data->port_id;
835
836 if (mlx5_eth_find_next(rx_port, priv->pci_dev) != rx_port) {
837 rte_errno = ENODEV;
838 DRV_LOG(ERR, "Rx port %u does not belong to mlx5", rx_port);
839 return -rte_errno;
840 }
841 for (i = 0; i != priv->txqs_n; i++) {
842 uint16_t rx_queue;
843
844 txq_ctrl = mlx5_txq_get(dev, i);
845 if (txq_ctrl == NULL)
846 continue;
847 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
848 mlx5_txq_release(dev, i);
849 continue;
850 }
851 if (txq_ctrl->hairpin_conf.peers[0].port != rx_port) {
852 mlx5_txq_release(dev, i);
853 continue;
854 }
855 /* Indeed, only the first used queue needs to be checked. */
856 if (txq_ctrl->hairpin_conf.manual_bind == 0) {
857 if (cur_port != rx_port) {
858 rte_errno = EINVAL;
859 DRV_LOG(ERR, "port %u and port %u are in"
860 " auto-bind mode", cur_port, rx_port);
861 mlx5_txq_release(dev, i);
862 return -rte_errno;
863 } else {
864 return 0;
865 }
866 }
867 rx_queue = txq_ctrl->hairpin_conf.peers[0].queue;
868 mlx5_txq_release(dev, i);
869 ret = rte_eth_hairpin_queue_peer_unbind(rx_port, rx_queue, 0);
870 if (ret) {
871 DRV_LOG(ERR, "port %u Rx queue %d unbind - failure",
872 rx_port, rx_queue);
873 return ret;
874 }
875 ret = mlx5_hairpin_queue_peer_unbind(dev, i, 1);
876 if (ret) {
877 DRV_LOG(ERR, "port %u Tx queue %d unbind - failure",
878 cur_port, i);
879 return ret;
880 }
881 }
882 return 0;
883 }
884
885 /*
886 * Bind hairpin ports, Rx could be all ports when using RTE_MAX_ETHPORTS.
887 * @see mlx5_hairpin_bind_single_port()
888 */
889 int
mlx5_hairpin_bind(struct rte_eth_dev * dev,uint16_t rx_port)890 mlx5_hairpin_bind(struct rte_eth_dev *dev, uint16_t rx_port)
891 {
892 int ret = 0;
893 uint16_t p, pp;
894 struct mlx5_priv *priv = dev->data->dev_private;
895
896 /*
897 * If the Rx port has no hairpin configuration with the current port,
898 * the binding will be skipped in the called function of single port.
899 * Device started status will be checked only before the queue
900 * information updating.
901 */
902 if (rx_port == RTE_MAX_ETHPORTS) {
903 MLX5_ETH_FOREACH_DEV(p, priv->pci_dev) {
904 ret = mlx5_hairpin_bind_single_port(dev, p);
905 if (ret != 0)
906 goto unbind;
907 }
908 return ret;
909 } else {
910 return mlx5_hairpin_bind_single_port(dev, rx_port);
911 }
912 unbind:
913 MLX5_ETH_FOREACH_DEV(pp, priv->pci_dev)
914 if (pp < p)
915 mlx5_hairpin_unbind_single_port(dev, pp);
916 return ret;
917 }
918
919 /*
920 * Unbind hairpin ports, Rx could be all ports when using RTE_MAX_ETHPORTS.
921 * @see mlx5_hairpin_unbind_single_port()
922 */
923 int
mlx5_hairpin_unbind(struct rte_eth_dev * dev,uint16_t rx_port)924 mlx5_hairpin_unbind(struct rte_eth_dev *dev, uint16_t rx_port)
925 {
926 int ret = 0;
927 uint16_t p;
928 struct mlx5_priv *priv = dev->data->dev_private;
929
930 if (rx_port == RTE_MAX_ETHPORTS)
931 MLX5_ETH_FOREACH_DEV(p, priv->pci_dev) {
932 ret = mlx5_hairpin_unbind_single_port(dev, p);
933 if (ret != 0)
934 return ret;
935 }
936 else
937 ret = mlx5_hairpin_unbind_single_port(dev, rx_port);
938 return ret;
939 }
940
941 /*
942 * DPDK callback to get the hairpin peer ports list.
943 * This will return the actual number of peer ports and save the identifiers
944 * into the array (sorted, may be different from that when setting up the
945 * hairpin peer queues).
946 * The peer port ID could be the same as the port ID of the current device.
947 *
948 * @param dev
949 * Pointer to Ethernet device structure.
950 * @param peer_ports
951 * Pointer to array to save the port identifiers.
952 * @param len
953 * The length of the array.
954 * @param direction
955 * Current port to peer port direction.
956 * positive - current used as Tx to get all peer Rx ports.
957 * zero - current used as Rx to get all peer Tx ports.
958 *
959 * @return
960 * 0 or positive value on success, actual number of peer ports.
961 * a negative errno value otherwise and rte_errno is set.
962 */
963 int
mlx5_hairpin_get_peer_ports(struct rte_eth_dev * dev,uint16_t * peer_ports,size_t len,uint32_t direction)964 mlx5_hairpin_get_peer_ports(struct rte_eth_dev *dev, uint16_t *peer_ports,
965 size_t len, uint32_t direction)
966 {
967 struct mlx5_priv *priv = dev->data->dev_private;
968 struct mlx5_txq_ctrl *txq_ctrl;
969 struct mlx5_rxq_ctrl *rxq_ctrl;
970 uint32_t i;
971 uint16_t pp;
972 uint32_t bits[(RTE_MAX_ETHPORTS + 31) / 32] = {0};
973 int ret = 0;
974
975 if (direction) {
976 for (i = 0; i < priv->txqs_n; i++) {
977 txq_ctrl = mlx5_txq_get(dev, i);
978 if (!txq_ctrl)
979 continue;
980 if (txq_ctrl->type != MLX5_TXQ_TYPE_HAIRPIN) {
981 mlx5_txq_release(dev, i);
982 continue;
983 }
984 pp = txq_ctrl->hairpin_conf.peers[0].port;
985 if (pp >= RTE_MAX_ETHPORTS) {
986 rte_errno = ERANGE;
987 mlx5_txq_release(dev, i);
988 DRV_LOG(ERR, "port %hu queue %u peer port "
989 "out of range %hu",
990 priv->dev_data->port_id, i, pp);
991 return -rte_errno;
992 }
993 bits[pp / 32] |= 1 << (pp % 32);
994 mlx5_txq_release(dev, i);
995 }
996 } else {
997 for (i = 0; i < priv->rxqs_n; i++) {
998 rxq_ctrl = mlx5_rxq_get(dev, i);
999 if (!rxq_ctrl)
1000 continue;
1001 if (rxq_ctrl->type != MLX5_RXQ_TYPE_HAIRPIN) {
1002 mlx5_rxq_release(dev, i);
1003 continue;
1004 }
1005 pp = rxq_ctrl->hairpin_conf.peers[0].port;
1006 if (pp >= RTE_MAX_ETHPORTS) {
1007 rte_errno = ERANGE;
1008 mlx5_rxq_release(dev, i);
1009 DRV_LOG(ERR, "port %hu queue %u peer port "
1010 "out of range %hu",
1011 priv->dev_data->port_id, i, pp);
1012 return -rte_errno;
1013 }
1014 bits[pp / 32] |= 1 << (pp % 32);
1015 mlx5_rxq_release(dev, i);
1016 }
1017 }
1018 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
1019 if (bits[i / 32] & (1 << (i % 32))) {
1020 if ((size_t)ret >= len) {
1021 rte_errno = E2BIG;
1022 return -rte_errno;
1023 }
1024 peer_ports[ret++] = i;
1025 }
1026 }
1027 return ret;
1028 }
1029
1030 /**
1031 * DPDK callback to start the device.
1032 *
1033 * Simulate device start by attaching all configured flows.
1034 *
1035 * @param dev
1036 * Pointer to Ethernet device structure.
1037 *
1038 * @return
1039 * 0 on success, a negative errno value otherwise and rte_errno is set.
1040 */
1041 int
mlx5_dev_start(struct rte_eth_dev * dev)1042 mlx5_dev_start(struct rte_eth_dev *dev)
1043 {
1044 struct mlx5_priv *priv = dev->data->dev_private;
1045 int ret;
1046 int fine_inline;
1047
1048 DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id);
1049 fine_inline = rte_mbuf_dynflag_lookup
1050 (RTE_PMD_MLX5_FINE_GRANULARITY_INLINE, NULL);
1051 if (fine_inline >= 0)
1052 rte_net_mlx5_dynf_inline_mask = 1UL << fine_inline;
1053 else
1054 rte_net_mlx5_dynf_inline_mask = 0;
1055 if (dev->data->nb_rx_queues > 0) {
1056 ret = mlx5_dev_configure_rss_reta(dev);
1057 if (ret) {
1058 DRV_LOG(ERR, "port %u reta config failed: %s",
1059 dev->data->port_id, strerror(rte_errno));
1060 return -rte_errno;
1061 }
1062 }
1063 ret = mlx5_txpp_start(dev);
1064 if (ret) {
1065 DRV_LOG(ERR, "port %u Tx packet pacing init failed: %s",
1066 dev->data->port_id, strerror(rte_errno));
1067 goto error;
1068 }
1069 ret = mlx5_txq_start(dev);
1070 if (ret) {
1071 DRV_LOG(ERR, "port %u Tx queue allocation failed: %s",
1072 dev->data->port_id, strerror(rte_errno));
1073 goto error;
1074 }
1075 ret = mlx5_rxq_start(dev);
1076 if (ret) {
1077 DRV_LOG(ERR, "port %u Rx queue allocation failed: %s",
1078 dev->data->port_id, strerror(rte_errno));
1079 goto error;
1080 }
1081 /*
1082 * Such step will be skipped if there is no hairpin TX queue configured
1083 * with RX peer queue from the same device.
1084 */
1085 ret = mlx5_hairpin_auto_bind(dev);
1086 if (ret) {
1087 DRV_LOG(ERR, "port %u hairpin auto binding failed: %s",
1088 dev->data->port_id, strerror(rte_errno));
1089 goto error;
1090 }
1091 /* Set started flag here for the following steps like control flow. */
1092 dev->data->dev_started = 1;
1093 ret = mlx5_rx_intr_vec_enable(dev);
1094 if (ret) {
1095 DRV_LOG(ERR, "port %u Rx interrupt vector creation failed",
1096 dev->data->port_id);
1097 goto error;
1098 }
1099 mlx5_os_stats_init(dev);
1100 ret = mlx5_traffic_enable(dev);
1101 if (ret) {
1102 DRV_LOG(ERR, "port %u failed to set defaults flows",
1103 dev->data->port_id);
1104 goto error;
1105 }
1106 /* Set a mask and offset of dynamic metadata flows into Rx queues. */
1107 mlx5_flow_rxq_dynf_metadata_set(dev);
1108 /* Set flags and context to convert Rx timestamps. */
1109 mlx5_rxq_timestamp_set(dev);
1110 /* Set a mask and offset of scheduling on timestamp into Tx queues. */
1111 mlx5_txq_dynf_timestamp_set(dev);
1112 /*
1113 * In non-cached mode, it only needs to start the default mreg copy
1114 * action and no flow created by application exists anymore.
1115 * But it is worth wrapping the interface for further usage.
1116 */
1117 ret = mlx5_flow_start_default(dev);
1118 if (ret) {
1119 DRV_LOG(DEBUG, "port %u failed to start default actions: %s",
1120 dev->data->port_id, strerror(rte_errno));
1121 goto error;
1122 }
1123 rte_wmb();
1124 dev->tx_pkt_burst = mlx5_select_tx_function(dev);
1125 dev->rx_pkt_burst = mlx5_select_rx_function(dev);
1126 /* Enable datapath on secondary process. */
1127 mlx5_mp_os_req_start_rxtx(dev);
1128 if (priv->sh->intr_handle.fd >= 0) {
1129 priv->sh->port[priv->dev_port - 1].ih_port_id =
1130 (uint32_t)dev->data->port_id;
1131 } else {
1132 DRV_LOG(INFO, "port %u starts without LSC and RMV interrupts.",
1133 dev->data->port_id);
1134 dev->data->dev_conf.intr_conf.lsc = 0;
1135 dev->data->dev_conf.intr_conf.rmv = 0;
1136 }
1137 if (priv->sh->intr_handle_devx.fd >= 0)
1138 priv->sh->port[priv->dev_port - 1].devx_ih_port_id =
1139 (uint32_t)dev->data->port_id;
1140 return 0;
1141 error:
1142 ret = rte_errno; /* Save rte_errno before cleanup. */
1143 /* Rollback. */
1144 dev->data->dev_started = 0;
1145 mlx5_flow_stop_default(dev);
1146 mlx5_traffic_disable(dev);
1147 mlx5_txq_stop(dev);
1148 mlx5_rxq_stop(dev);
1149 mlx5_txpp_stop(dev); /* Stop last. */
1150 rte_errno = ret; /* Restore rte_errno. */
1151 return -rte_errno;
1152 }
1153
1154 /**
1155 * DPDK callback to stop the device.
1156 *
1157 * Simulate device stop by detaching all configured flows.
1158 *
1159 * @param dev
1160 * Pointer to Ethernet device structure.
1161 */
1162 int
mlx5_dev_stop(struct rte_eth_dev * dev)1163 mlx5_dev_stop(struct rte_eth_dev *dev)
1164 {
1165 struct mlx5_priv *priv = dev->data->dev_private;
1166
1167 dev->data->dev_started = 0;
1168 /* Prevent crashes when queues are still in use. */
1169 dev->rx_pkt_burst = removed_rx_burst;
1170 dev->tx_pkt_burst = removed_tx_burst;
1171 rte_wmb();
1172 /* Disable datapath on secondary process. */
1173 mlx5_mp_os_req_stop_rxtx(dev);
1174 usleep(1000 * priv->rxqs_n);
1175 DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id);
1176 mlx5_flow_stop_default(dev);
1177 /* Control flows for default traffic can be removed firstly. */
1178 mlx5_traffic_disable(dev);
1179 /* All RX queue flags will be cleared in the flush interface. */
1180 mlx5_flow_list_flush(dev, &priv->flows, true);
1181 mlx5_rx_intr_vec_disable(dev);
1182 priv->sh->port[priv->dev_port - 1].ih_port_id = RTE_MAX_ETHPORTS;
1183 priv->sh->port[priv->dev_port - 1].devx_ih_port_id = RTE_MAX_ETHPORTS;
1184 mlx5_txq_stop(dev);
1185 mlx5_rxq_stop(dev);
1186 mlx5_txpp_stop(dev);
1187
1188 return 0;
1189 }
1190
1191 /**
1192 * Enable traffic flows configured by control plane
1193 *
1194 * @param dev
1195 * Pointer to Ethernet device private data.
1196 * @param dev
1197 * Pointer to Ethernet device structure.
1198 *
1199 * @return
1200 * 0 on success, a negative errno value otherwise and rte_errno is set.
1201 */
1202 int
mlx5_traffic_enable(struct rte_eth_dev * dev)1203 mlx5_traffic_enable(struct rte_eth_dev *dev)
1204 {
1205 struct mlx5_priv *priv = dev->data->dev_private;
1206 struct rte_flow_item_eth bcast = {
1207 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1208 };
1209 struct rte_flow_item_eth ipv6_multi_spec = {
1210 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
1211 };
1212 struct rte_flow_item_eth ipv6_multi_mask = {
1213 .dst.addr_bytes = "\xff\xff\x00\x00\x00\x00",
1214 };
1215 struct rte_flow_item_eth unicast = {
1216 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
1217 };
1218 struct rte_flow_item_eth unicast_mask = {
1219 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
1220 };
1221 const unsigned int vlan_filter_n = priv->vlan_filter_n;
1222 const struct rte_ether_addr cmp = {
1223 .addr_bytes = "\x00\x00\x00\x00\x00\x00",
1224 };
1225 unsigned int i;
1226 unsigned int j;
1227 int ret;
1228
1229 /*
1230 * Hairpin txq default flow should be created no matter if it is
1231 * isolation mode. Or else all the packets to be sent will be sent
1232 * out directly without the TX flow actions, e.g. encapsulation.
1233 */
1234 for (i = 0; i != priv->txqs_n; ++i) {
1235 struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
1236 if (!txq_ctrl)
1237 continue;
1238 /* Only Tx implicit mode requires the default Tx flow. */
1239 if (txq_ctrl->type == MLX5_TXQ_TYPE_HAIRPIN &&
1240 txq_ctrl->hairpin_conf.tx_explicit == 0 &&
1241 txq_ctrl->hairpin_conf.peers[0].port ==
1242 priv->dev_data->port_id) {
1243 ret = mlx5_ctrl_flow_source_queue(dev, i);
1244 if (ret) {
1245 mlx5_txq_release(dev, i);
1246 goto error;
1247 }
1248 }
1249 mlx5_txq_release(dev, i);
1250 }
1251 if (priv->config.dv_esw_en && !priv->config.vf) {
1252 if (mlx5_flow_create_esw_table_zero_flow(dev))
1253 priv->fdb_def_rule = 1;
1254 else
1255 DRV_LOG(INFO, "port %u FDB default rule cannot be"
1256 " configured - only Eswitch group 0 flows are"
1257 " supported.", dev->data->port_id);
1258 }
1259 if (!priv->config.lacp_by_user && priv->pf_bond >= 0) {
1260 ret = mlx5_flow_lacp_miss(dev);
1261 if (ret)
1262 DRV_LOG(INFO, "port %u LACP rule cannot be created - "
1263 "forward LACP to kernel.", dev->data->port_id);
1264 else
1265 DRV_LOG(INFO, "LACP traffic will be missed in port %u."
1266 , dev->data->port_id);
1267 }
1268 if (priv->isolated)
1269 return 0;
1270 if (dev->data->promiscuous) {
1271 struct rte_flow_item_eth promisc = {
1272 .dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
1273 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
1274 .type = 0,
1275 };
1276
1277 ret = mlx5_ctrl_flow(dev, &promisc, &promisc);
1278 if (ret)
1279 goto error;
1280 }
1281 if (dev->data->all_multicast) {
1282 struct rte_flow_item_eth multicast = {
1283 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
1284 .src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
1285 .type = 0,
1286 };
1287
1288 ret = mlx5_ctrl_flow(dev, &multicast, &multicast);
1289 if (ret)
1290 goto error;
1291 } else {
1292 /* Add broadcast/multicast flows. */
1293 for (i = 0; i != vlan_filter_n; ++i) {
1294 uint16_t vlan = priv->vlan_filter[i];
1295
1296 struct rte_flow_item_vlan vlan_spec = {
1297 .tci = rte_cpu_to_be_16(vlan),
1298 };
1299 struct rte_flow_item_vlan vlan_mask =
1300 rte_flow_item_vlan_mask;
1301
1302 ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast,
1303 &vlan_spec, &vlan_mask);
1304 if (ret)
1305 goto error;
1306 ret = mlx5_ctrl_flow_vlan(dev, &ipv6_multi_spec,
1307 &ipv6_multi_mask,
1308 &vlan_spec, &vlan_mask);
1309 if (ret)
1310 goto error;
1311 }
1312 if (!vlan_filter_n) {
1313 ret = mlx5_ctrl_flow(dev, &bcast, &bcast);
1314 if (ret)
1315 goto error;
1316 ret = mlx5_ctrl_flow(dev, &ipv6_multi_spec,
1317 &ipv6_multi_mask);
1318 if (ret)
1319 goto error;
1320 }
1321 }
1322 /* Add MAC address flows. */
1323 for (i = 0; i != MLX5_MAX_MAC_ADDRESSES; ++i) {
1324 struct rte_ether_addr *mac = &dev->data->mac_addrs[i];
1325
1326 if (!memcmp(mac, &cmp, sizeof(*mac)))
1327 continue;
1328 memcpy(&unicast.dst.addr_bytes,
1329 mac->addr_bytes,
1330 RTE_ETHER_ADDR_LEN);
1331 for (j = 0; j != vlan_filter_n; ++j) {
1332 uint16_t vlan = priv->vlan_filter[j];
1333
1334 struct rte_flow_item_vlan vlan_spec = {
1335 .tci = rte_cpu_to_be_16(vlan),
1336 };
1337 struct rte_flow_item_vlan vlan_mask =
1338 rte_flow_item_vlan_mask;
1339
1340 ret = mlx5_ctrl_flow_vlan(dev, &unicast,
1341 &unicast_mask,
1342 &vlan_spec,
1343 &vlan_mask);
1344 if (ret)
1345 goto error;
1346 }
1347 if (!vlan_filter_n) {
1348 ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask);
1349 if (ret)
1350 goto error;
1351 }
1352 }
1353 return 0;
1354 error:
1355 ret = rte_errno; /* Save rte_errno before cleanup. */
1356 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
1357 rte_errno = ret; /* Restore rte_errno. */
1358 return -rte_errno;
1359 }
1360
1361
1362 /**
1363 * Disable traffic flows configured by control plane
1364 *
1365 * @param dev
1366 * Pointer to Ethernet device private data.
1367 */
1368 void
mlx5_traffic_disable(struct rte_eth_dev * dev)1369 mlx5_traffic_disable(struct rte_eth_dev *dev)
1370 {
1371 struct mlx5_priv *priv = dev->data->dev_private;
1372
1373 mlx5_flow_list_flush(dev, &priv->ctrl_flows, false);
1374 }
1375
1376 /**
1377 * Restart traffic flows configured by control plane
1378 *
1379 * @param dev
1380 * Pointer to Ethernet device private data.
1381 *
1382 * @return
1383 * 0 on success, a negative errno value otherwise and rte_errno is set.
1384 */
1385 int
mlx5_traffic_restart(struct rte_eth_dev * dev)1386 mlx5_traffic_restart(struct rte_eth_dev *dev)
1387 {
1388 if (dev->data->dev_started) {
1389 mlx5_traffic_disable(dev);
1390 return mlx5_traffic_enable(dev);
1391 }
1392 return 0;
1393 }
1394