1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0) 2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved. 3 */ 4 5 #include <sys/queue.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <string.h> 9 #include <errno.h> 10 #include <stdint.h> 11 #include <stdarg.h> 12 #include <unistd.h> 13 #include <inttypes.h> 14 15 #include <rte_byteorder.h> 16 #include <rte_common.h> 17 #include <rte_cycles.h> 18 #include <rte_log.h> 19 #include <rte_debug.h> 20 #include <rte_interrupts.h> 21 #include <rte_pci.h> 22 #include <rte_memory.h> 23 #include <rte_memzone.h> 24 #include <rte_launch.h> 25 #include <rte_eal.h> 26 #include <rte_per_lcore.h> 27 #include <rte_lcore.h> 28 #include <rte_atomic.h> 29 #include <rte_branch_prediction.h> 30 #include <rte_mempool.h> 31 #include <rte_malloc.h> 32 #include <rte_mbuf.h> 33 #include <rte_ether.h> 34 #include <ethdev_driver.h> 35 #include <rte_prefetch.h> 36 #include <rte_udp.h> 37 #include <rte_tcp.h> 38 #include <rte_sctp.h> 39 #include <rte_string_fns.h> 40 #include <rte_errno.h> 41 #include <rte_ip.h> 42 #include <rte_net.h> 43 44 #include "ionic_logs.h" 45 #include "ionic_mac_api.h" 46 #include "ionic_ethdev.h" 47 #include "ionic_lif.h" 48 #include "ionic_rxtx.h" 49 50 #define IONIC_RX_RING_DOORBELL_STRIDE (32 - 1) 51 52 /********************************************************************* 53 * 54 * TX functions 55 * 56 **********************************************************************/ 57 58 void 59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 60 struct rte_eth_txq_info *qinfo) 61 { 62 struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id]; 63 struct ionic_queue *q = &txq->qcq.q; 64 65 qinfo->nb_desc = q->num_descs; 66 qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads; 67 qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED; 68 } 69 70 static __rte_always_inline void 71 ionic_tx_flush(struct ionic_tx_qcq *txq) 72 { 73 struct ionic_cq *cq = &txq->qcq.cq; 74 struct ionic_queue *q = &txq->qcq.q; 75 struct rte_mbuf *txm, *next; 76 struct ionic_txq_comp *cq_desc_base = cq->base; 77 struct ionic_txq_comp *cq_desc; 78 void **info; 79 u_int32_t comp_index = (u_int32_t)-1; 80 81 cq_desc = &cq_desc_base[cq->tail_idx]; 82 while (color_match(cq_desc->color, cq->done_color)) { 83 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 84 85 /* Prefetch the next 4 descriptors (not really useful here) */ 86 if ((cq->tail_idx & 0x3) == 0) 87 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 88 89 if (cq->tail_idx == 0) 90 cq->done_color = !cq->done_color; 91 92 comp_index = cq_desc->comp_index; 93 94 cq_desc = &cq_desc_base[cq->tail_idx]; 95 } 96 97 if (comp_index != (u_int32_t)-1) { 98 while (q->tail_idx != comp_index) { 99 info = IONIC_INFO_PTR(q, q->tail_idx); 100 101 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 102 103 /* Prefetch the next 4 descriptors */ 104 if ((q->tail_idx & 0x3) == 0) 105 /* q desc info */ 106 rte_prefetch0(&q->info[q->tail_idx]); 107 108 /* 109 * Note: you can just use rte_pktmbuf_free, 110 * but this loop is faster 111 */ 112 txm = info[0]; 113 while (txm != NULL) { 114 next = txm->next; 115 rte_pktmbuf_free_seg(txm); 116 txm = next; 117 } 118 } 119 } 120 } 121 122 void __rte_cold 123 ionic_dev_tx_queue_release(void *tx_queue) 124 { 125 struct ionic_tx_qcq *txq = tx_queue; 126 127 IONIC_PRINT_CALL(); 128 129 ionic_lif_txq_deinit(txq); 130 131 ionic_qcq_free(&txq->qcq); 132 } 133 134 int __rte_cold 135 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 136 { 137 struct ionic_tx_qcq *txq; 138 139 IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id); 140 141 txq = eth_dev->data->tx_queues[tx_queue_id]; 142 143 eth_dev->data->tx_queue_state[tx_queue_id] = 144 RTE_ETH_QUEUE_STATE_STOPPED; 145 146 /* 147 * Note: we should better post NOP Tx desc and wait for its completion 148 * before disabling Tx queue 149 */ 150 151 ionic_qcq_disable(&txq->qcq); 152 153 ionic_tx_flush(txq); 154 155 return 0; 156 } 157 158 int __rte_cold 159 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id, 160 uint16_t nb_desc, uint32_t socket_id, 161 const struct rte_eth_txconf *tx_conf) 162 { 163 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 164 struct ionic_tx_qcq *txq; 165 uint64_t offloads; 166 int err; 167 168 if (tx_queue_id >= lif->ntxqcqs) { 169 IONIC_PRINT(DEBUG, "Queue index %u not available " 170 "(max %u queues)", 171 tx_queue_id, lif->ntxqcqs); 172 return -EINVAL; 173 } 174 175 offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads; 176 IONIC_PRINT(DEBUG, 177 "Configuring skt %u TX queue %u with %u buffers, offloads %jx", 178 socket_id, tx_queue_id, nb_desc, offloads); 179 180 /* Validate number of receive descriptors */ 181 if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC) 182 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 183 184 /* Free memory prior to re-allocation if needed... */ 185 if (eth_dev->data->tx_queues[tx_queue_id] != NULL) { 186 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id]; 187 ionic_dev_tx_queue_release(tx_queue); 188 eth_dev->data->tx_queues[tx_queue_id] = NULL; 189 } 190 191 eth_dev->data->tx_queue_state[tx_queue_id] = 192 RTE_ETH_QUEUE_STATE_STOPPED; 193 194 err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq); 195 if (err) { 196 IONIC_PRINT(DEBUG, "Queue allocation failure"); 197 return -EINVAL; 198 } 199 200 /* Do not start queue with rte_eth_dev_start() */ 201 if (tx_conf->tx_deferred_start) 202 txq->flags |= IONIC_QCQ_F_DEFERRED; 203 204 /* Convert the offload flags into queue flags */ 205 if (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) 206 txq->flags |= IONIC_QCQ_F_CSUM_L3; 207 if (offloads & DEV_TX_OFFLOAD_TCP_CKSUM) 208 txq->flags |= IONIC_QCQ_F_CSUM_TCP; 209 if (offloads & DEV_TX_OFFLOAD_UDP_CKSUM) 210 txq->flags |= IONIC_QCQ_F_CSUM_UDP; 211 212 eth_dev->data->tx_queues[tx_queue_id] = txq; 213 214 return 0; 215 } 216 217 /* 218 * Start Transmit Units for specified queue. 219 */ 220 int __rte_cold 221 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id) 222 { 223 uint8_t *tx_queue_state = eth_dev->data->tx_queue_state; 224 struct ionic_tx_qcq *txq; 225 int err; 226 227 if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 228 IONIC_PRINT(DEBUG, "TX queue %u already started", 229 tx_queue_id); 230 return 0; 231 } 232 233 txq = eth_dev->data->tx_queues[tx_queue_id]; 234 235 IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs", 236 tx_queue_id, txq->qcq.q.num_descs); 237 238 if (!(txq->flags & IONIC_QCQ_F_INITED)) { 239 err = ionic_lif_txq_init(txq); 240 if (err) 241 return err; 242 } else { 243 ionic_qcq_enable(&txq->qcq); 244 } 245 246 tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 247 248 return 0; 249 } 250 251 static void 252 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm) 253 { 254 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 255 char *l3_hdr = ((char *)eth_hdr) + txm->l2_len; 256 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 257 (l3_hdr + txm->l3_len); 258 259 if (txm->ol_flags & PKT_TX_IP_CKSUM) { 260 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 261 ipv4_hdr->hdr_checksum = 0; 262 tcp_hdr->cksum = 0; 263 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 264 } else { 265 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 266 tcp_hdr->cksum = 0; 267 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 268 } 269 } 270 271 static void 272 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm) 273 { 274 struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *); 275 char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len + 276 txm->outer_l3_len + txm->l2_len; 277 struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *) 278 (l3_hdr + txm->l3_len); 279 280 if (txm->ol_flags & PKT_TX_IPV4) { 281 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr; 282 ipv4_hdr->hdr_checksum = 0; 283 tcp_hdr->cksum = 0; 284 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr); 285 } else { 286 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr; 287 tcp_hdr->cksum = 0; 288 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr); 289 } 290 } 291 292 static void 293 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc, 294 struct rte_mbuf *txm, 295 rte_iova_t addr, uint8_t nsge, uint16_t len, 296 uint32_t hdrlen, uint32_t mss, 297 bool encap, 298 uint16_t vlan_tci, bool has_vlan, 299 bool start, bool done) 300 { 301 uint8_t flags = 0; 302 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 303 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 304 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0; 305 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0; 306 307 desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, 308 flags, nsge, addr); 309 desc->len = len; 310 desc->vlan_tci = vlan_tci; 311 desc->hdr_len = hdrlen; 312 desc->mss = mss; 313 314 ionic_q_post(q, done, done ? txm : NULL); 315 } 316 317 static struct ionic_txq_desc * 318 ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem) 319 { 320 struct ionic_queue *q = &txq->qcq.q; 321 struct ionic_txq_desc *desc_base = q->base; 322 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 323 struct ionic_txq_desc *desc = &desc_base[q->head_idx]; 324 struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx]; 325 326 *elem = sg_desc->elems; 327 return desc; 328 } 329 330 static int 331 ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, 332 bool not_xmit_more) 333 { 334 struct ionic_queue *q = &txq->qcq.q; 335 struct ionic_tx_stats *stats = &txq->stats; 336 struct ionic_txq_desc *desc; 337 struct ionic_txq_sg_elem *elem; 338 struct rte_mbuf *txm_seg; 339 rte_iova_t data_iova; 340 uint64_t desc_addr = 0, next_addr; 341 uint16_t desc_len = 0; 342 uint8_t desc_nsge; 343 uint32_t hdrlen; 344 uint32_t mss = txm->tso_segsz; 345 uint32_t frag_left = 0; 346 uint32_t left; 347 uint32_t seglen; 348 uint32_t len; 349 uint32_t offset = 0; 350 bool start, done; 351 bool encap; 352 bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT); 353 uint16_t vlan_tci = txm->vlan_tci; 354 uint64_t ol_flags = txm->ol_flags; 355 356 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 357 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 358 ((ol_flags & PKT_TX_OUTER_IPV4) || 359 (ol_flags & PKT_TX_OUTER_IPV6)); 360 361 /* Preload inner-most TCP csum field with IP pseudo hdr 362 * calculated with IP length set to zero. HW will later 363 * add in length to each TCP segment resulting from the TSO. 364 */ 365 366 if (encap) { 367 ionic_tx_tcp_inner_pseudo_csum(txm); 368 hdrlen = txm->outer_l2_len + txm->outer_l3_len + 369 txm->l2_len + txm->l3_len + txm->l4_len; 370 } else { 371 ionic_tx_tcp_pseudo_csum(txm); 372 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len; 373 } 374 375 seglen = hdrlen + mss; 376 left = txm->data_len; 377 data_iova = rte_mbuf_data_iova(txm); 378 379 desc = ionic_tx_tso_next(txq, &elem); 380 start = true; 381 382 /* Chop data up into desc segments */ 383 384 while (left > 0) { 385 len = RTE_MIN(seglen, left); 386 frag_left = seglen - len; 387 desc_addr = rte_cpu_to_le_64(data_iova + offset); 388 desc_len = len; 389 desc_nsge = 0; 390 left -= len; 391 offset += len; 392 if (txm->nb_segs > 1 && frag_left > 0) 393 continue; 394 done = (txm->nb_segs == 1 && left == 0); 395 ionic_tx_tso_post(q, desc, txm, 396 desc_addr, desc_nsge, desc_len, 397 hdrlen, mss, 398 encap, 399 vlan_tci, has_vlan, 400 start, done && not_xmit_more); 401 desc = ionic_tx_tso_next(txq, &elem); 402 start = false; 403 seglen = mss; 404 } 405 406 /* Chop frags into desc segments */ 407 408 txm_seg = txm->next; 409 while (txm_seg != NULL) { 410 offset = 0; 411 data_iova = rte_mbuf_data_iova(txm_seg); 412 left = txm_seg->data_len; 413 stats->frags++; 414 415 while (left > 0) { 416 next_addr = rte_cpu_to_le_64(data_iova + offset); 417 if (frag_left > 0) { 418 len = RTE_MIN(frag_left, left); 419 frag_left -= len; 420 elem->addr = next_addr; 421 elem->len = len; 422 elem++; 423 desc_nsge++; 424 } else { 425 len = RTE_MIN(mss, left); 426 frag_left = mss - len; 427 desc_addr = next_addr; 428 desc_len = len; 429 desc_nsge = 0; 430 } 431 left -= len; 432 offset += len; 433 if (txm_seg->next != NULL && frag_left > 0) 434 continue; 435 436 done = (txm_seg->next == NULL && left == 0); 437 ionic_tx_tso_post(q, desc, txm_seg, 438 desc_addr, desc_nsge, desc_len, 439 hdrlen, mss, 440 encap, 441 vlan_tci, has_vlan, 442 start, done && not_xmit_more); 443 desc = ionic_tx_tso_next(txq, &elem); 444 start = false; 445 } 446 447 txm_seg = txm_seg->next; 448 } 449 450 stats->tso++; 451 452 return 0; 453 } 454 455 static __rte_always_inline int 456 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm, 457 bool not_xmit_more) 458 { 459 struct ionic_queue *q = &txq->qcq.q; 460 struct ionic_txq_desc *desc, *desc_base = q->base; 461 struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base; 462 struct ionic_txq_sg_elem *elem; 463 struct ionic_tx_stats *stats = &txq->stats; 464 struct rte_mbuf *txm_seg; 465 bool encap; 466 bool has_vlan; 467 uint64_t ol_flags = txm->ol_flags; 468 uint64_t addr; 469 uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE; 470 uint8_t flags = 0; 471 472 desc = &desc_base[q->head_idx]; 473 474 if ((ol_flags & PKT_TX_IP_CKSUM) && 475 (txq->flags & IONIC_QCQ_F_CSUM_L3)) { 476 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 477 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3; 478 } 479 480 if (((ol_flags & PKT_TX_TCP_CKSUM) && 481 (txq->flags & IONIC_QCQ_F_CSUM_TCP)) || 482 ((ol_flags & PKT_TX_UDP_CKSUM) && 483 (txq->flags & IONIC_QCQ_F_CSUM_UDP))) { 484 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW; 485 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4; 486 } 487 488 if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE) 489 stats->no_csum++; 490 491 has_vlan = (ol_flags & PKT_TX_VLAN_PKT); 492 encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) || 493 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) && 494 ((ol_flags & PKT_TX_OUTER_IPV4) || 495 (ol_flags & PKT_TX_OUTER_IPV6)); 496 497 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0; 498 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0; 499 500 addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm)); 501 502 desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr); 503 desc->len = txm->data_len; 504 desc->vlan_tci = txm->vlan_tci; 505 506 elem = sg_desc_base[q->head_idx].elems; 507 txm_seg = txm->next; 508 while (txm_seg != NULL) { 509 elem->len = txm_seg->data_len; 510 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg)); 511 stats->frags++; 512 elem++; 513 txm_seg = txm_seg->next; 514 } 515 516 ionic_q_post(q, not_xmit_more, txm); 517 518 return 0; 519 } 520 521 uint16_t 522 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 523 uint16_t nb_pkts) 524 { 525 struct ionic_tx_qcq *txq = tx_queue; 526 struct ionic_queue *q = &txq->qcq.q; 527 struct ionic_tx_stats *stats = &txq->stats; 528 uint32_t next_q_head_idx; 529 uint32_t bytes_tx = 0; 530 uint16_t nb_tx = 0; 531 int err; 532 bool last; 533 534 /* Cleaning old buffers */ 535 ionic_tx_flush(txq); 536 537 if (unlikely(ionic_q_space_avail(q) < nb_pkts)) { 538 stats->stop += nb_pkts; 539 return 0; 540 } 541 542 while (nb_tx < nb_pkts) { 543 last = (nb_tx == (nb_pkts - 1)); 544 545 next_q_head_idx = Q_NEXT_TO_POST(q, 1); 546 if ((next_q_head_idx & 0x3) == 0) { 547 struct ionic_txq_desc *desc_base = q->base; 548 rte_prefetch0(&desc_base[next_q_head_idx]); 549 rte_prefetch0(&q->info[next_q_head_idx]); 550 } 551 552 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG) 553 err = ionic_tx_tso(txq, tx_pkts[nb_tx], last); 554 else 555 err = ionic_tx(txq, tx_pkts[nb_tx], last); 556 if (err) { 557 stats->drop += nb_pkts - nb_tx; 558 if (nb_tx > 0) 559 ionic_q_flush(q); 560 break; 561 } 562 563 bytes_tx += tx_pkts[nb_tx]->pkt_len; 564 nb_tx++; 565 } 566 567 stats->packets += nb_tx; 568 stats->bytes += bytes_tx; 569 570 return nb_tx; 571 } 572 573 /********************************************************************* 574 * 575 * TX prep functions 576 * 577 **********************************************************************/ 578 579 #define IONIC_TX_OFFLOAD_MASK ( \ 580 PKT_TX_IPV4 | \ 581 PKT_TX_IPV6 | \ 582 PKT_TX_VLAN | \ 583 PKT_TX_IP_CKSUM | \ 584 PKT_TX_TCP_SEG | \ 585 PKT_TX_L4_MASK) 586 587 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \ 588 (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK) 589 590 uint16_t 591 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts, 592 uint16_t nb_pkts) 593 { 594 struct rte_mbuf *txm; 595 uint64_t offloads; 596 int i = 0; 597 598 for (i = 0; i < nb_pkts; i++) { 599 txm = tx_pkts[i]; 600 601 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS_V1 + 1) { 602 rte_errno = -EINVAL; 603 break; 604 } 605 606 offloads = txm->ol_flags; 607 608 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) { 609 rte_errno = -ENOTSUP; 610 break; 611 } 612 } 613 614 return i; 615 } 616 617 /********************************************************************* 618 * 619 * RX functions 620 * 621 **********************************************************************/ 622 623 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 624 struct rte_mbuf *mbuf); 625 626 void 627 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id, 628 struct rte_eth_rxq_info *qinfo) 629 { 630 struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id]; 631 struct ionic_queue *q = &rxq->qcq.q; 632 633 qinfo->mp = rxq->mb_pool; 634 qinfo->scattered_rx = dev->data->scattered_rx; 635 qinfo->nb_desc = q->num_descs; 636 qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED; 637 qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads; 638 } 639 640 static void __rte_cold 641 ionic_rx_empty(struct ionic_rx_qcq *rxq) 642 { 643 struct ionic_queue *q = &rxq->qcq.q; 644 struct rte_mbuf *mbuf; 645 void **info; 646 647 while (q->tail_idx != q->head_idx) { 648 info = IONIC_INFO_PTR(q, q->tail_idx); 649 mbuf = info[0]; 650 rte_mempool_put(rxq->mb_pool, mbuf); 651 652 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 653 } 654 } 655 656 void __rte_cold 657 ionic_dev_rx_queue_release(void *rx_queue) 658 { 659 struct ionic_rx_qcq *rxq = rx_queue; 660 661 if (!rxq) 662 return; 663 664 IONIC_PRINT_CALL(); 665 666 ionic_rx_empty(rxq); 667 668 ionic_lif_rxq_deinit(rxq); 669 670 ionic_qcq_free(&rxq->qcq); 671 } 672 673 int __rte_cold 674 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, 675 uint16_t rx_queue_id, 676 uint16_t nb_desc, 677 uint32_t socket_id, 678 const struct rte_eth_rxconf *rx_conf, 679 struct rte_mempool *mp) 680 { 681 struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev); 682 struct ionic_rx_qcq *rxq; 683 uint64_t offloads; 684 int err; 685 686 if (rx_queue_id >= lif->nrxqcqs) { 687 IONIC_PRINT(ERR, 688 "Queue index %u not available (max %u queues)", 689 rx_queue_id, lif->nrxqcqs); 690 return -EINVAL; 691 } 692 693 offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads; 694 IONIC_PRINT(DEBUG, 695 "Configuring skt %u RX queue %u with %u buffers, offloads %jx", 696 socket_id, rx_queue_id, nb_desc, offloads); 697 698 if (!rx_conf->rx_drop_en) 699 IONIC_PRINT(WARNING, "No-drop mode is not supported"); 700 701 /* Validate number of receive descriptors */ 702 if (!rte_is_power_of_2(nb_desc) || 703 nb_desc < IONIC_MIN_RING_DESC || 704 nb_desc > IONIC_MAX_RING_DESC) { 705 IONIC_PRINT(ERR, 706 "Bad descriptor count (%u) for queue %u (min: %u)", 707 nb_desc, rx_queue_id, IONIC_MIN_RING_DESC); 708 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */ 709 } 710 711 /* Free memory prior to re-allocation if needed... */ 712 if (eth_dev->data->rx_queues[rx_queue_id] != NULL) { 713 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id]; 714 ionic_dev_rx_queue_release(rx_queue); 715 eth_dev->data->rx_queues[rx_queue_id] = NULL; 716 } 717 718 eth_dev->data->rx_queue_state[rx_queue_id] = 719 RTE_ETH_QUEUE_STATE_STOPPED; 720 721 err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, 722 &rxq); 723 if (err) { 724 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id); 725 return -EINVAL; 726 } 727 728 rxq->mb_pool = mp; 729 730 /* 731 * Note: the interface does not currently support 732 * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN 733 * when the adapter will be able to keep the CRC and subtract 734 * it to the length for all received packets: 735 * if (eth_dev->data->dev_conf.rxmode.offloads & 736 * DEV_RX_OFFLOAD_KEEP_CRC) 737 * rxq->crc_len = ETHER_CRC_LEN; 738 */ 739 740 /* Do not start queue with rte_eth_dev_start() */ 741 if (rx_conf->rx_deferred_start) 742 rxq->flags |= IONIC_QCQ_F_DEFERRED; 743 744 eth_dev->data->rx_queues[rx_queue_id] = rxq; 745 746 return 0; 747 } 748 749 static __rte_always_inline void 750 ionic_rx_clean(struct ionic_rx_qcq *rxq, 751 uint32_t q_desc_index, uint32_t cq_desc_index, 752 void *service_cb_arg) 753 { 754 struct ionic_queue *q = &rxq->qcq.q; 755 struct ionic_cq *cq = &rxq->qcq.cq; 756 struct ionic_rxq_comp *cq_desc_base = cq->base; 757 struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index]; 758 struct rte_mbuf *rxm, *rxm_seg; 759 uint32_t max_frame_size = 760 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 761 uint64_t pkt_flags = 0; 762 uint32_t pkt_type; 763 struct ionic_rx_stats *stats = &rxq->stats; 764 struct ionic_rx_service *recv_args = (struct ionic_rx_service *) 765 service_cb_arg; 766 uint32_t buf_size = (uint16_t) 767 (rte_pktmbuf_data_room_size(rxq->mb_pool) - 768 RTE_PKTMBUF_HEADROOM); 769 uint32_t left; 770 void **info; 771 772 assert(q_desc_index == cq_desc->comp_index); 773 774 info = IONIC_INFO_PTR(q, cq_desc->comp_index); 775 776 rxm = info[0]; 777 778 if (!recv_args) { 779 stats->no_cb_arg++; 780 /* Flush */ 781 rte_pktmbuf_free(rxm); 782 /* 783 * Note: rte_mempool_put is faster with no segs 784 * rte_mempool_put(rxq->mb_pool, rxm); 785 */ 786 return; 787 } 788 789 if (cq_desc->status) { 790 stats->bad_cq_status++; 791 ionic_rx_recycle(q, q_desc_index, rxm); 792 return; 793 } 794 795 if (recv_args->nb_rx >= recv_args->nb_pkts) { 796 stats->no_room++; 797 ionic_rx_recycle(q, q_desc_index, rxm); 798 return; 799 } 800 801 if (cq_desc->len > max_frame_size || 802 cq_desc->len == 0) { 803 stats->bad_len++; 804 ionic_rx_recycle(q, q_desc_index, rxm); 805 return; 806 } 807 808 rxm->data_off = RTE_PKTMBUF_HEADROOM; 809 rte_prefetch1((char *)rxm->buf_addr + rxm->data_off); 810 rxm->nb_segs = 1; /* cq_desc->num_sg_elems */ 811 rxm->pkt_len = cq_desc->len; 812 rxm->port = rxq->qcq.lif->port_id; 813 814 left = cq_desc->len; 815 816 rxm->data_len = RTE_MIN(buf_size, left); 817 left -= rxm->data_len; 818 819 rxm_seg = rxm->next; 820 while (rxm_seg && left) { 821 rxm_seg->data_len = RTE_MIN(buf_size, left); 822 left -= rxm_seg->data_len; 823 824 rxm_seg = rxm_seg->next; 825 rxm->nb_segs++; 826 } 827 828 /* RSS */ 829 pkt_flags |= PKT_RX_RSS_HASH; 830 rxm->hash.rss = cq_desc->rss_hash; 831 832 /* Vlan Strip */ 833 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) { 834 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED; 835 rxm->vlan_tci = cq_desc->vlan_tci; 836 } 837 838 /* Checksum */ 839 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) { 840 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK) 841 pkt_flags |= PKT_RX_IP_CKSUM_GOOD; 842 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD) 843 pkt_flags |= PKT_RX_IP_CKSUM_BAD; 844 845 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) || 846 (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK)) 847 pkt_flags |= PKT_RX_L4_CKSUM_GOOD; 848 else if ((cq_desc->csum_flags & 849 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) || 850 (cq_desc->csum_flags & 851 IONIC_RXQ_COMP_CSUM_F_UDP_BAD)) 852 pkt_flags |= PKT_RX_L4_CKSUM_BAD; 853 } 854 855 rxm->ol_flags = pkt_flags; 856 857 /* Packet Type */ 858 switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) { 859 case IONIC_PKT_TYPE_IPV4: 860 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4; 861 break; 862 case IONIC_PKT_TYPE_IPV6: 863 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6; 864 break; 865 case IONIC_PKT_TYPE_IPV4_TCP: 866 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 867 RTE_PTYPE_L4_TCP; 868 break; 869 case IONIC_PKT_TYPE_IPV6_TCP: 870 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 871 RTE_PTYPE_L4_TCP; 872 break; 873 case IONIC_PKT_TYPE_IPV4_UDP: 874 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 | 875 RTE_PTYPE_L4_UDP; 876 break; 877 case IONIC_PKT_TYPE_IPV6_UDP: 878 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 | 879 RTE_PTYPE_L4_UDP; 880 break; 881 default: 882 { 883 struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm, 884 struct rte_ether_hdr *); 885 uint16_t ether_type = eth_h->ether_type; 886 if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP)) 887 pkt_type = RTE_PTYPE_L2_ETHER_ARP; 888 else 889 pkt_type = RTE_PTYPE_UNKNOWN; 890 break; 891 } 892 } 893 894 rxm->packet_type = pkt_type; 895 896 recv_args->rx_pkts[recv_args->nb_rx] = rxm; 897 recv_args->nb_rx++; 898 899 stats->packets++; 900 stats->bytes += rxm->pkt_len; 901 } 902 903 static void 904 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index, 905 struct rte_mbuf *mbuf) 906 { 907 struct ionic_rxq_desc *desc_base = q->base; 908 struct ionic_rxq_desc *old = &desc_base[q_desc_index]; 909 struct ionic_rxq_desc *new = &desc_base[q->head_idx]; 910 911 new->addr = old->addr; 912 new->len = old->len; 913 914 ionic_q_post(q, true, mbuf); 915 } 916 917 static __rte_always_inline int 918 ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len) 919 { 920 struct ionic_queue *q = &rxq->qcq.q; 921 struct ionic_rxq_desc *desc, *desc_base = q->base; 922 struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base; 923 struct ionic_rxq_sg_elem *elem; 924 rte_iova_t dma_addr; 925 uint32_t i, j, nsegs, buf_size, size; 926 bool ring_doorbell; 927 928 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) - 929 RTE_PKTMBUF_HEADROOM); 930 931 /* Initialize software ring entries */ 932 for (i = ionic_q_space_avail(q); i; i--) { 933 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool); 934 struct rte_mbuf *prev_rxm_seg; 935 936 if (rxm == NULL) { 937 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 938 return -ENOMEM; 939 } 940 941 nsegs = (len + buf_size - 1) / buf_size; 942 943 desc = &desc_base[q->head_idx]; 944 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm)); 945 desc->addr = dma_addr; 946 desc->len = buf_size; 947 size = buf_size; 948 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG : 949 IONIC_RXQ_DESC_OPCODE_SIMPLE; 950 rxm->next = NULL; 951 952 prev_rxm_seg = rxm; 953 sg_desc = &sg_desc_base[q->head_idx]; 954 elem = sg_desc->elems; 955 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) { 956 struct rte_mbuf *rxm_seg; 957 rte_iova_t data_iova; 958 959 rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool); 960 if (rxm_seg == NULL) { 961 IONIC_PRINT(ERR, "RX mbuf alloc failed"); 962 return -ENOMEM; 963 } 964 965 data_iova = rte_mbuf_data_iova(rxm_seg); 966 dma_addr = rte_cpu_to_le_64(data_iova); 967 elem->addr = dma_addr; 968 elem->len = buf_size; 969 size += buf_size; 970 elem++; 971 rxm_seg->next = NULL; 972 prev_rxm_seg->next = rxm_seg; 973 prev_rxm_seg = rxm_seg; 974 } 975 976 if (size < len) 977 IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)", 978 size, len); 979 980 ring_doorbell = ((q->head_idx + 1) & 981 IONIC_RX_RING_DOORBELL_STRIDE) == 0; 982 983 ionic_q_post(q, ring_doorbell, rxm); 984 } 985 986 return 0; 987 } 988 989 /* 990 * Start Receive Units for specified queue. 991 */ 992 int __rte_cold 993 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 994 { 995 uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 996 uint8_t *rx_queue_state = eth_dev->data->rx_queue_state; 997 struct ionic_rx_qcq *rxq; 998 int err; 999 1000 if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) { 1001 IONIC_PRINT(DEBUG, "RX queue %u already started", 1002 rx_queue_id); 1003 return 0; 1004 } 1005 1006 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1007 1008 IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)", 1009 rx_queue_id, rxq->qcq.q.num_descs, frame_size); 1010 1011 if (!(rxq->flags & IONIC_QCQ_F_INITED)) { 1012 err = ionic_lif_rxq_init(rxq); 1013 if (err) 1014 return err; 1015 } else { 1016 ionic_qcq_enable(&rxq->qcq); 1017 } 1018 1019 /* Allocate buffers for descriptor rings */ 1020 if (ionic_rx_fill(rxq, frame_size) != 0) { 1021 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d", 1022 rx_queue_id); 1023 return -1; 1024 } 1025 1026 rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; 1027 1028 return 0; 1029 } 1030 1031 static __rte_always_inline void 1032 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do, 1033 void *service_cb_arg) 1034 { 1035 struct ionic_cq *cq = &rxq->qcq.cq; 1036 struct ionic_queue *q = &rxq->qcq.q; 1037 struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base; 1038 bool more; 1039 uint32_t curr_q_tail_idx, curr_cq_tail_idx; 1040 uint32_t work_done = 0; 1041 1042 if (work_to_do == 0) 1043 return; 1044 1045 cq_desc = &cq_desc_base[cq->tail_idx]; 1046 while (color_match(cq_desc->pkt_type_color, cq->done_color)) { 1047 curr_cq_tail_idx = cq->tail_idx; 1048 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1); 1049 1050 if (cq->tail_idx == 0) 1051 cq->done_color = !cq->done_color; 1052 1053 /* Prefetch the next 4 descriptors */ 1054 if ((cq->tail_idx & 0x3) == 0) 1055 rte_prefetch0(&cq_desc_base[cq->tail_idx]); 1056 1057 do { 1058 more = (q->tail_idx != cq_desc->comp_index); 1059 1060 curr_q_tail_idx = q->tail_idx; 1061 q->tail_idx = Q_NEXT_TO_SRVC(q, 1); 1062 1063 /* Prefetch the next 4 descriptors */ 1064 if ((q->tail_idx & 0x3) == 0) 1065 /* q desc info */ 1066 rte_prefetch0(&q->info[q->tail_idx]); 1067 1068 ionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx, 1069 service_cb_arg); 1070 1071 } while (more); 1072 1073 if (++work_done == work_to_do) 1074 break; 1075 1076 cq_desc = &cq_desc_base[cq->tail_idx]; 1077 } 1078 } 1079 1080 /* 1081 * Stop Receive Units for specified queue. 1082 */ 1083 int __rte_cold 1084 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id) 1085 { 1086 struct ionic_rx_qcq *rxq; 1087 1088 IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id); 1089 1090 rxq = eth_dev->data->rx_queues[rx_queue_id]; 1091 1092 eth_dev->data->rx_queue_state[rx_queue_id] = 1093 RTE_ETH_QUEUE_STATE_STOPPED; 1094 1095 ionic_qcq_disable(&rxq->qcq); 1096 1097 /* Flush */ 1098 ionic_rxq_service(rxq, -1, NULL); 1099 1100 return 0; 1101 } 1102 1103 uint16_t 1104 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, 1105 uint16_t nb_pkts) 1106 { 1107 struct ionic_rx_qcq *rxq = rx_queue; 1108 uint32_t frame_size = 1109 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len; 1110 struct ionic_rx_service service_cb_arg; 1111 1112 service_cb_arg.rx_pkts = rx_pkts; 1113 service_cb_arg.nb_pkts = nb_pkts; 1114 service_cb_arg.nb_rx = 0; 1115 1116 ionic_rxq_service(rxq, nb_pkts, &service_cb_arg); 1117 1118 ionic_rx_fill(rxq, frame_size); 1119 1120 return service_cb_arg.nb_rx; 1121 } 1122